Index: .fossil-settings/ignore-glob ================================================================== --- .fossil-settings/ignore-glob +++ .fossil-settings/ignore-glob @@ -11,11 +11,10 @@ *.so *~ *.vg *.massif *._out -Makefile autosetup/jimsh0 config.log config.make bindings/cpp/Makefile bindings/cpp/test Index: CHANGES.md ================================================================== --- CHANGES.md +++ CHANGES.md @@ -1,5 +1,12 @@ +**fnc 0.5** 2021-11-03 + +- simplify the build system to make-only and remove autosetup (patch from sdk) +- remove invalid fsl_errno_to_rc() call and redundant alloc error check +- reduce noise on app exit by removing redundant error output +- relocate fnc_strlcpy() and fnc_strlcat() routines to the library + **fnc 0.4** 2021-10-31 - resolve database bug in the commit builder logic - prune dead code leftover from the initial single-threaded blame implementation - improve error handling of commands that return no records ADDED LICENSE Index: LICENSE ================================================================== --- LICENSE +++ LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2021 Mark Jamsek + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ADDED Makefile Index: Makefile ================================================================== --- Makefile +++ Makefile @@ -0,0 +1,94 @@ +# +# FNC Makefile +# + +# CONFIGUGRAION +CC ?= cc +PREFIX ?= /usr/local +MANDIR ?= /share/man +VERSION ?= 0.5 + +# USED BELOW FOR PLATFORM SPECIFIC LDFLAGS +UNAME := $(shell uname -s) + +# FLAGS NEEDED TO BUILD SQLITE3 +SQLITE_CFLAGS = ${CFLAGS} -Wall -Werror -Wno-sign-compare -pedantic -std=c99 \ + -DNDEBUG=1 \ + -DSQLITE_DQS=0 \ + -DSQLITE_THREADSAFE=0 \ + -DSQLITE_DEFAULT_MEMSTATUS=0 \ + -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 \ + -DSQLITE_LIKE_DOESNT_MATCH_BLOBS \ + -DSQLITE_OMIT_DECLTYPE \ + -DSQLITE_OMIT_PROGRESS_CALLBACK \ + -DSQLITE_OMIT_SHARED_CACHE \ + -DSQLITE_OMIT_LOAD_EXTENSION \ + -DSQLITE_MAX_EXPR_DEPTH=0 \ + -DSQLITE_USE_ALLOCA \ + -DSQLITE_ENABLE_LOCKING_STYLE=0 \ + -DSQLITE_DEFAULT_FILE_FORMAT=4 \ + -DSQLITE_ENABLE_EXPLAIN_COMMENTS \ + -DSQLITE_ENABLE_FTS4 \ + -DSQLITE_ENABLE_DBSTAT_VTAB \ + -DSQLITE_ENABLE_JSON1 \ + -DSQLITE_ENABLE_FTS5 \ + -DSQLITE_ENABLE_STMTVTAB \ + -DSQLITE_HAVE_ZLIB \ + -DSQLITE_INTROSPECTION_PRAGMAS \ + -DSQLITE_ENABLE_DBPAGE_VTAB \ + -DSQLITE_TRUSTED_SCHEMA=0 + +# This makefile +MAKEFILE := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) + +# FLAGS NEEDED TO BUILD LIBFOSSIL +FOSSIL_CFLAGS = ${CFLAGS} -Wall -Werror -Wsign-compare -pedantic -std=c99 + +# On SOME Linux (e.g., Ubuntu 18.04.6), we have to include wchar curses from +# I/.../ncursesw, but linking to -lncursesw (w/ no special -L path) works fine. +# FLAGS NEEDED TO BUILD FNC +FNC_CFLAGS = ${CFLAGS} -Wall -Werror -Wsign-compare -pedantic -std=c99 \ + -I./lib -I/usr/include/ncursesw -D_XOPEN_SOURCE_EXTENDED \ + -DVERSION=${VERSION} + +FNC_LDFLAGS = ${LDFLAGS} -lm -lutil -lz -lpthread -fPIC + +# OSX has ncursesw, and needs iconv. +ifeq ($(UNAME),Darwin) +# OSX +FNC_LDFLAGS += -lncurses -lpanel -liconv +else +# Linux (tested on Debian), OpenBSD, FreeBSD +FNC_LDFLAGS += -lncursesw -lpanelw +endif + +all: bin + +bin: lib/sqlite3.o lib/libfossil.o src/fnc.o src/fnc + +lib/sqlite3.o: lib/sqlite3.c lib/sqlite3.h $(MAKEFILE) + ${CC} ${SQLITE_CFLAGS} -c $< -o $@ + +lib/libfossil.o: lib/libfossil.c lib/libfossil.h $(MAKEFILE) + ${CC} ${FOSSIL_CFLAGS} -c $< -o $@ + +src/fnc.o: src/fnc.c $(MAKEFILE) + ${CC} ${FNC_CFLAGS} -c $< -o $@ + +src/fnc: src/fnc.o lib/libfossil.o lib/sqlite3.o $(MAKEFILE) + ${CC} -o $@ src/fnc.o lib/libfossil.o lib/sqlite3.o ${FNC_LDFLAGS} + +install: + install -s -m 0755 src/fnc ${PREFIX}/bin/fnc + install -m 0644 src/fnc.1 ${PREFIX}${MANDIR}/man1/fnc.1 + +uninstall: + rm -f ${PREFIX}/bin/fnc ${PREFIX}${MANDIR}/man1/fnc.1 + +clean: + rm -f lib/*.o src/*.o src/fnc + +release: clean + tar czvf ../fnc-${VERSION}.tgz -C .. fnc-${VERSION} + +.PHONY: clean release DELETED Makefile.in Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -1,52 +0,0 @@ -#!/usr/bin/make # help out emacs -# -# Top-level autosetup-filtered Makefile for libfossil. This particular -# build is for Unix platforms with GNU Make 3.81+. -all: -.NOTPARALLEL: # stop subdir makes and reconfigure from launching - # multiple times concurrently. -include config.make -ShakeNMake.CISH_SOURCES := $(wildcard *.c) -# Subdir cleanup rules and deps list must come before shakenmake.make is included -# or they must be set up manually afterwards... -clean-.: clean-fnc clean-src -distclean-.: distclean-fnc distclean-src -include shakenmake.make -MAIN_MAKEFILES := $(PACKAGE.MAKEFILE) $(ShakeNMake.MAKEFILE) @AUTODEPS@ - -SUBDIRS := src fnc -$(eval $(call ShakeNMake.CALL.SUBDIRS,$(SUBDIRS))) -subdir-fnc: subdir-src -all: subdir-src subdir-fnc - -DISTCLEAN_FILES += config.make - -######################################################################## -# Other stuff... - -ifeq ($(MAKE_COMPILATION_DB),yes) -all: compile_commands.json -compile_commands.json: - @$(RM) $@ - sed -e '1s/^/[\'$$'\n''/' -e '$$s/,$$/\'$$'\n'']/' $(compdb_dir)/*.o.json > $@+ - @if test -s $@+; then mv $@+ $@; else $(RM) $@+; fi -endif - -DISTCLEAN_FILES += Makefile config.log autosetup/jimsh0 \ - $(wildcard compile_commands/*) compile_commands.json+ - -# Reconfigure if needed -ifeq ($(findstring clean,$(MAKECMDGOALS)),) -@top_srcdir@/config.make: @AUTODEPS@ @top_srcdir@/config.make.in - @@AUTOREMAKE@ -@top_srcdir@/Makefile: @AUTODEPS@ @top_srcdir@/Makefile.in - chmod +w $@ - @@AUTOREMAKE@ -@top_srcdir@/vars.make: @AUTODEPS@ @top_srcdir@/vars.make.in - @@AUTOREMAKE@ -endif - -# Or on demand -reconfig: - @AUTOREMAKE@ - Index: README.md ================================================================== --- README.md +++ README.md @@ -25,25 +25,21 @@ 1. clone the repository - `fossil clone https://fnc.bsdbox.org` 2. move into the repository checkout - `cd fnc` -3. run the configure script - - `./configure` -4. build fnc +3. build fnc - `make` -5. install the `fnc` binary (*requires privileges*) +4. install the `fnc` binary (*requires privileges*) - `doas make install` -6. move into an open Fossil checkout, and run it: +5. move into an open Fossil checkout, and run it: - `cd ~/museum/repo && fossil open ../repo.fossil && fnc` This will install the `fnc` executable and man page into `/usr/local/bin` and -`/usr/local/share/man/man1`, respectively. To change the install path, use the -`--prefix` option to `configure`; for example, replacing step 3 with -`./configure --prefix=$HOME` will install the executable and man page into -`~/bin` and `~/share/man`, respectively. Alternatively, cryptographically signed -binaries for some of the abovementioned platforms are available to [download][3]. +`/usr/local/share/man/man1`, respectively. Alternatively, cryptographically +signed binaries for some of the abovementioned platforms are available to +[download][3]. # Doc See `fnc --help` for a quick reference, and the [fnc(1)][4] manual page for more comprehensive documentation. In-app help can also be accessed with the `?`, DELETED auto.def Index: auto.def ================================================================== --- auto.def +++ auto.def @@ -1,126 +0,0 @@ -# vim:se syn=tcl: -# -use cc cc-shared cc-lib wh-common - -options { - no-debug=0 => "Disable debug build options." - loud=0 => "Enables 'loud' build mode." - profile=0 => "Enables the -pg (profiling) compile/link flag if CC is gcc." - no-compile-commands=0 => - "Disable compile_commands.json support even if detected (possibly incorrectly)." -} - - -# autosetup interceps 'debug' and 'enable-debug' flags :/ -# prefix:=[get-env HOME /usr/local] -> "Installation prefix." - -#cc-check-c11 - -wh-require-bash -cc-check-sizeof "void *" - -if {![cc-check-includes zlib.h] || - ![cc-check-function-in-lib compress z]} { - user-error "Missing functional zlib" -} -cc-check-function-in-lib iconv iconv - -######################################################################## -# Checks for C99 via (__STDC_VERSION__ >= 199901L). Returns 1 if so, 0 -# 0 if not. -proc cc-check-c99 {} { - msg-checking "Checking for C99 via __STDC_VERSION__... " - if {[cctest -code { - #if !defined(__STDC_VERSION__) || __STDC_VERSION__<199901L - # error "Not C99" - #endif - }]} { - msg-result "got C99" - return 1 - } - return 0 -} - -# The compiler may be able to run in C99 mode without -std=c99, but we -# check for that flag anyway because without it we might inadvertently -# be compiling against a newer C standard for purposes of flags like -# -pedantic -set CC_FLAG_C99 {-std=c99} -if {![cc-check-flags $CC_FLAG_C99]} { - set CC_FLAG_C99 {} -} -if {![cc-check-c99]} { - user-error "As of 2021-02-21, libfossil requires C99." -} - -define CC_FLAG_C99 $CC_FLAG_C99 - -# cc-check-functions getcwd fopen -cc-check-functions opendir stat pipe inet_ntop getaddrinfo -#msg-result [cc-check-functions lstat] - -if {[cc-check-functions lstat]} { - # for lstat() on Linux and FreeBSD: - define _XOPEN_SOURCE 600 - # alternate for lstat() on Linux: - # define _BSD_SOURCE 1 - # ^^^^ causes warning (-Werror breakage) with glibc >=2.20 - define _DEFAULT_SOURCE 1 - define _POSIX_C_SOURCE 200112L -} - -# Find some tools -cc-check-tools ar strip -wh-bin-define install - -set extra_objs {} - -if {[find-an-executable cygpath] ne "" || $::tcl_platform(os)=="Windows NT"} { - set cFlags {} -} else { - set cFlags {-fPIC} -} - -if {![wh-check-ncurses]} { - user-error "Could not find a compatible ncurses/libpanel configuration." -} - -if {[opt-bool no-debug]} { - msg-result "Non-debug build." - set cFlags "$cFlags -O2" -} else { - msg-result "Debug build enabled. Use --no-debug to build in non-debug mode." - set cFlags "$cFlags -g -DDEBUG -O0" -} - -wh-check-compile-commands no-compile-commands - -define CFLAGS $cFlags -if {[wh-opt-bool-01 -v loud BUILD_QUIETLY]} { - puts "Enabling quiet build mode. Use --loud to enable loud mode." -} - -msg-checking "gprof profiling? " -if {[wh-check-profile-flag profile]} { - msg-result "gcc detected: building with profiling option." - # Add CC_PROFILE_FLAG to CFLAGS and LDFLAGS in config.make.in. -} else { - msg-result "no. Use --profile and gcc to enable." -} - -# Each generated Makefile requires an input file with a .in extension: -wh-make-from-dot-in { - config.make - Makefile - src/Makefile - fnc/Makefile -} - -if {0} { - # Achtung: ordering of the -bare/-str options here is important - # because of the mixed use of strings and integers for #defines... - make-config-header include/fossil-scm/autoconfig.h \ - -none {DOXYGEN_*} \ - -bare {HAVE_* FSL_ENABLE_* _DEFAULT_SOURCE _XOPEN_SOURCE} \ - -str {FSL_* PACKAGE_*} -} DELETED autosetup/LICENSE Index: autosetup/LICENSE ================================================================== --- autosetup/LICENSE +++ autosetup/LICENSE @@ -1,35 +0,0 @@ -Unless explicitly stated, all files which form part of autosetup -are released under the following license: - ---------------------------------------------------------------------- -autosetup - A build environment "autoconfigurator" - -Copyright (c) 2010-2011, WorkWare Systems - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials - provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE WORKWARE SYSTEMS ``AS IS'' AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WORKWARE -SYSTEMS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation -are those of the authors and should not be interpreted as representing -official policies, either expressed or implied, of WorkWare Systems. DELETED autosetup/README.autosetup Index: autosetup/README.autosetup ================================================================== --- autosetup/README.autosetup +++ autosetup/README.autosetup @@ -1,11 +0,0 @@ -README.autosetup created by autosetup v0.7.0+ - -This is the autosetup directory for a local install of autosetup. -It contains autosetup, support files and loadable modules. - -*.tcl files in this directory are optional modules which -can be loaded with the 'use' directive. - -*.auto files in this directory are auto-loaded. - -For more information, see http://msteveb.github.io/autosetup/ DELETED autosetup/autosetup Index: autosetup/autosetup ================================================================== --- autosetup/autosetup +++ autosetup/autosetup @@ -1,2478 +0,0 @@ -#!/bin/sh -# Copyright (c) 2006-2011 WorkWare Systems http://www.workware.net.au/ -# All rights reserved -# vim:se syntax=tcl: -# \ -dir=`dirname "$0"`; exec "`$dir/autosetup-find-tclsh`" "$0" "$@" - -# Note that the version has a trailing + on unreleased versions -set autosetup(version) 0.7.0+ - -# Can be set to 1 to debug early-init problems -set autosetup(debug) [expr {"--debug" in $argv}] - -################################################################## -# -# Main flow of control, option handling -# -proc main {argv} { - global autosetup define - - # There are 3 potential directories involved: - # 1. The directory containing autosetup (this script) - # 2. The directory containing auto.def - # 3. The current directory - - # From this we need to determine: - # a. The path to this script (and related support files) - # b. The path to auto.def - # c. The build directory, where output files are created - - # This is also complicated by the fact that autosetup may - # have been run via the configure wrapper ([getenv WRAPPER] is set) - - # Here are the rules. - # a. This script is $::argv0 - # => dir, prog, exe, libdir - # b. auto.def is in the directory containing the configure wrapper, - # otherwise it is in the current directory. - # => srcdir, autodef - # c. The build directory is the current directory - # => builddir, [pwd] - - # 'misc' is needed before we can do anything, so set a temporary libdir - # in case this is the development version - set autosetup(libdir) [file dirname $::argv0]/lib - use misc - - # (a) - set autosetup(dir) [realdir [file dirname [realpath $::argv0]]] - set autosetup(prog) [file join $autosetup(dir) [file tail $::argv0]] - set autosetup(exe) [getenv WRAPPER $autosetup(prog)] - if {$autosetup(installed)} { - set autosetup(libdir) $autosetup(dir) - } else { - set autosetup(libdir) [file join $autosetup(dir) lib] - } - autosetup_add_dep $autosetup(prog) - - # (b) - if {[getenv WRAPPER ""] eq ""} { - # Invoked directly - set autosetup(srcdir) [pwd] - } else { - # Invoked via the configure wrapper - set autosetup(srcdir) [file-normalize [file dirname $autosetup(exe)]] - } - set autosetup(autodef) [relative-path $autosetup(srcdir)/auto.def] - - # (c) - set autosetup(builddir) [pwd] - - set autosetup(argv) $argv - set autosetup(cmdline) {} - # options is a list of known options - set autosetup(options) {} - # optset is a dictionary of option values set by the user based on getopt - set autosetup(optset) {} - # optdefault is a dictionary of default values - set autosetup(optdefault) {} - # options-defaults is a dictionary of overrides for default values for options - set autosetup(options-defaults) {} - set autosetup(optionhelp) {} - set autosetup(showhelp) 0 - - use util - - # Parse options - use getopt - - # At the is point we don't know what is a valid option - # We simply parse anything that looks like an option - set autosetup(getopt) [getopt argv] - - #"=Core Options:" - options-add { - help:=all => "display help and options. Optional: module name, such as --help=system" - licence license => "display the autosetup license" - version => "display the version of autosetup" - ref:=text manual:=text - reference:=text => "display the autosetup command reference. 'text', 'wiki', 'asciidoc' or 'markdown'" - debug => "display debugging output as autosetup runs" - install:=. => "install autosetup to the current or given directory" - } - if {$autosetup(installed)} { - # hidden options so we can produce a nice error - options-add { - sysinstall:path - } - } else { - options-add { - sysinstall:path => "install standalone autosetup to the given directory (e.g.: /usr/local)" - } - } - options-add { - force init:=help => "create initial auto.def, etc. Use --init=help for known types" - # Undocumented options - option-checking=1 - nopager - quiet - timing - conf: - } - - if {[opt-bool version]} { - puts $autosetup(version) - exit 0 - } - - # autosetup --conf=alternate-auto.def - if {[opt-str conf o]} { - set autosetup(autodef) $o - } - - # Debugging output (set this early) - incr autosetup(debug) [opt-bool debug] - incr autosetup(force) [opt-bool force] - incr autosetup(msg-quiet) [opt-bool quiet] - incr autosetup(msg-timing) [opt-bool timing] - - # If the local module exists, source it now to allow for - # project-local customisations - if {[file exists $autosetup(libdir)/local.tcl]} { - use local - } - - # Now any auto-load modules - autosetup_load_auto_modules - - if {[opt-str help o]} { - incr autosetup(showhelp) - use help - autosetup_help $o - } - - if {[opt-bool licence license]} { - use help - autosetup_show_license - exit 0 - } - - if {[opt-str {manual ref reference} o]} { - use help - autosetup_reference $o - } - - # Allow combining --install and --init - set earlyexit 0 - if {[opt-str install o]} { - use install - autosetup_install $o - incr earlyexit - } - - if {[opt-str init o]} { - use init - autosetup_init $o - incr earlyexit - } - - if {$earlyexit} { - exit 0 - } - if {[opt-str sysinstall o]} { - use install - autosetup_install $o 1 - exit 0 - } - - if {![file exists $autosetup(autodef)]} { - # Check for invalid option first - options {} - user-error "No auto.def found in \"$autosetup(srcdir)\" (use [file tail $::autosetup(exe)] --init to create one)" - } - - # Parse extra arguments into autosetup(cmdline) - foreach arg $argv { - if {[regexp {([^=]*)=(.*)} $arg -> n v]} { - dict set autosetup(cmdline) $n $v - define $n $v - } else { - user-error "Unexpected parameter: $arg" - } - } - - autosetup_add_dep $autosetup(autodef) - - # Add $argv to CONFIGURE_OPTS, but ignore duplicates and quote if needed - set configure_opts {} - foreach arg $autosetup(argv) { - set quoted [quote-if-needed $arg] - # O(n^2), but n will be small - if {$quoted ni $configure_opts} { - lappend configure_opts $quoted - } - } - define CONFIGURE_OPTS [join $configure_opts] - define AUTOREMAKE [quote-if-needed $autosetup(exe)] - define-append AUTOREMAKE [get-define CONFIGURE_OPTS] - - - # Log how we were invoked - configlog "Invoked as: [getenv WRAPPER $::argv0] [quote-argv $autosetup(argv)]" - configlog "Tclsh: [info nameofexecutable]" - - # Load auto.def as module "auto.def" - autosetup_load_module auto.def source $autosetup(autodef) - - # Could warn here if options {} was not specified - - show-notices - - if {$autosetup(debug)} { - msg-result "Writing all defines to config.log" - configlog "================ defines ======================" - foreach n [lsort [array names define]] { - configlog "define $n $define($n)" - } - } - - exit 0 -} - -# @opt-bool ?-nodefault? option ... -# -# Check each of the named, boolean options and if any have been explicitly enabled -# or disabled by the user, return 1 or 0 accordingly. -# -# If the option was specified more than once, the last value wins. -# e.g. With '--enable-foo --disable-foo', '[opt-bool foo]' will return 0 -# -# If no value was specified by the user, returns the default value for the -# first option. If '-nodefault' is given, this behaviour changes and -# -1 is returned instead. -# -proc opt-bool {args} { - set nodefault 0 - if {[lindex $args 0] eq "-nodefault"} { - set nodefault 1 - set args [lrange $args 1 end] - } - option-check-names {*}$args - - foreach opt $args { - if {[dict exists $::autosetup(optset) $opt]} { - return [dict get $::autosetup(optset) $opt] - } - } - - if {$nodefault} { - return -1 - } - # Default value is the default for the first option - return [dict get $::autosetup(optdefault) [lindex $args 0]] -} - -# @opt-val optionlist ?default=""? -# -# Returns a list containing all the values given for the non-boolean options in '$optionlist'. -# There will be one entry in the list for each option given by the user, including if the -# same option was used multiple times. -# -# If no options were set, '$default' is returned (exactly, not as a list). -# -# Note: For most use cases, 'opt-str' should be preferred. -# -proc opt-val {names {default ""}} { - option-check-names {*}$names - - foreach opt $names { - if {[dict exists $::autosetup(optset) $opt]} { - lappend result {*}[dict get $::autosetup(optset) $opt] - } - } - if {[info exists result]} { - return $result - } - return $default -} - -# @opt-str optionlist varname ?default? -# -# Sets '$varname' in the callers scope to the value for one of the given options. -# -# For the list of options given in '$optionlist', if any value is set for any option, -# the option value is taken to be the *last* value of the last option (in the order given). -# -# If no option was given, and a default was specified with 'options-defaults', -# that value is used. -# -# If no 'options-defaults' value was given and '$default' was given, it is used. -# -# If none of the above provided a value, no value is set. -# -# The return value depends on whether '$default' was specified. -# If it was, the option value is returned. -# If it was not, 1 is returns if a value was set, or 0 if not. -# -# Typical usage is as follows: -# -## if {[opt-str {myopt altname} o]} { -## do something with $o -## } -# -# Or: -## define myname [opt-str {myopt altname} o "/usr/local"] -# -proc opt-str {names varname args} { - global autosetup - - option-check-names {*}$names - upvar $varname value - - if {[llength $args]} { - # A default was given, so always return the string value of the option - set default [lindex $args 0] - set retopt 1 - } else { - # No default, so return 0 or 1 to indicate if a value was found - set retopt 0 - } - - foreach opt $names { - if {[dict exists $::autosetup(optset) $opt]} { - set result [lindex [dict get $::autosetup(optset) $opt] end] - } - } - - if {![info exists result]} { - # No user-specified value. Has options-defaults been set? - foreach opt $names { - if {[dict exists $::autosetup(optdefault) $opt]} { - set result [dict get $autosetup(optdefault) $opt] - } - } - } - - if {[info exists result]} { - set value $result - if {$retopt} { - return $value - } - return 1 - } - - if {$retopt} { - set value $default - return $value - } - - return 0 -} - -proc option-check-names {args} { - foreach o $args { - if {$o ni $::autosetup(options)} { - autosetup-error "Request for undeclared option --$o" - } - } -} - -# Parse the option definition in $opts and update -# ::autosetup(setoptions) and ::autosetup(optionhelp) appropriately -# -proc options-add {opts} { - global autosetup - - # First weed out comment lines - set realopts {} - foreach line [split $opts \n] { - if {![string match "#*" [string trimleft $line]]} { - append realopts $line \n - } - } - set opts $realopts - - for {set i 0} {$i < [llength $opts]} {incr i} { - set opt [lindex $opts $i] - if {[string match =* $opt]} { - # This is a special heading - lappend autosetup(optionhelp) [list $opt $autosetup(module)] - continue - } - unset -nocomplain defaultvalue equal value - - #puts "i=$i, opt=$opt" - regexp {^([^:=]*)(:)?(=)?(.*)$} $opt -> name colon equal value - if {$name in $autosetup(options)} { - autosetup-error "Option $name already specified" - } - - #puts "$opt => $name $colon $equal $value" - - # Find the corresponding value in the user options - # and set the default if necessary - if {[string match "-*" $opt]} { - # This is a documentation-only option, like "-C " - set opthelp $opt - } elseif {$colon eq ""} { - # Boolean option - lappend autosetup(options) $name - - # Check for override - if {[dict exists $autosetup(options-defaults) $name]} { - # A default was specified with options-defaults, so use it - set value [dict get $autosetup(options-defaults) $name] - } - - if {$value eq "1"} { - set opthelp "--disable-$name" - } else { - set opthelp "--$name" - } - - # Set the default - if {$value eq ""} { - set value 0 - } - set defaultvalue $value - dict set autosetup(optdefault) $name $defaultvalue - - if {[dict exists $autosetup(getopt) $name]} { - # The option was specified by the user. Look at the last value. - lassign [lindex [dict get $autosetup(getopt) $name] end] type setvalue - if {$type eq "str"} { - # Can we convert the value to a boolean? - if {$setvalue in {1 enabled yes}} { - set setvalue 1 - } elseif {$setvalue in {0 disabled no}} { - set setvalue 0 - } else { - user-error "Boolean option $name given as --$name=$setvalue" - } - } - dict set autosetup(optset) $name $setvalue - #puts "Found boolean option --$name=$setvalue" - } - } else { - # String option. - lappend autosetup(options) $name - - if {$equal ne "="} { - # Was the option given as "name:value=default"? - # If so, set $value to the display name and $defaultvalue to the default - # (This is the preferred way to set a default value for a string option) - if {[regexp {^([^=]+)=(.*)$} $value -> value defaultvalue]} { - dict set autosetup(optdefault) $name $defaultvalue - } - } - - # Maybe override the default value - if {[dict exists $autosetup(options-defaults) $name]} { - # A default was specified with options-defaults, so use it - set defaultvalue [dict get $autosetup(options-defaults) $name] - dict set autosetup(optdefault) $name $defaultvalue - } elseif {![info exists defaultvalue]} { - # No default value was given by value=default or options-defaults - # so use the value as the default when the plain option with no - # value is given (.e.g. just --opt instead of --opt=value) - set defaultvalue $value - } - - if {$equal eq "="} { - # String option with optional value - set opthelp "--$name?=$value?" - } else { - # String option with required value - set opthelp "--$name=$value" - } - - # Get the values specified by the user - if {[dict exists $autosetup(getopt) $name]} { - set listvalue {} - - foreach pair [dict get $autosetup(getopt) $name] { - lassign $pair type setvalue - if {$type eq "bool" && $setvalue} { - if {$equal ne "="} { - user-error "Option --$name requires a value" - } - # If given as a boolean, use the default value - set setvalue $defaultvalue - } - lappend listvalue $setvalue - } - - #puts "Found string option --$name=$listvalue" - dict set autosetup(optset) $name $listvalue - } - } - - # Now create the help for this option if appropriate - if {[lindex $opts $i+1] eq "=>"} { - set desc [lindex $opts $i+2] - if {[info exists defaultvalue]} { - set desc [string map [list @default@ $defaultvalue] $desc] - } - # A multi-line description - lappend autosetup(optionhelp) [list $opthelp $autosetup(module) $desc] - incr i 2 - } - } -} - -# @module-options optionlist -# -# Deprecated. Simply use 'options' from within a module. -proc module-options {opts} { - options $opts -} - -proc max {a b} { - expr {$a > $b ? $a : $b} -} - -proc options-wrap-desc {text length firstprefix nextprefix initial} { - set len $initial - set space $firstprefix - foreach word [split $text] { - set word [string trim $word] - if {$word == ""} { - continue - } - if {$len && [string length $space$word] + $len >= $length} { - puts "" - set len 0 - set space $nextprefix - } - incr len [string length $space$word] - puts -nonewline $space$word - set space " " - } - if {$len} { - puts "" - } -} - -# Display options (from $autosetup(optionhelp)) for modules that match -# glob pattern $what -proc options-show {what} { - set local 0 - # Determine the max option width - set max 0 - foreach help $::autosetup(optionhelp) { - lassign $help opt module desc - if {![string match $what $module]} { - continue - } - if {[string match =* $opt] || [string match \n* $desc]} { - continue - } - set max [max $max [string length $opt]] - } - set indent [string repeat " " [expr {$max+4}]] - set cols [getenv COLUMNS 80] - catch { - lassign [exec stty size] rows cols - } - incr cols -1 - # Now output - foreach help $::autosetup(optionhelp) { - lassign $help opt module desc - if {![string match $what $module]} { - continue - } - if {$local == 0 && $module eq "auto.def"} { - puts "Local Options:" - incr local - } - if {[string match =* $opt]} { - # Output a special heading line" - puts [string range $opt 1 end] - continue - } - puts -nonewline " [format %-${max}s $opt]" - if {[string match \n* $desc]} { - # Output a pre-formatted help description as-is - puts $desc - } else { - options-wrap-desc [string trim $desc] $cols " " $indent [expr {$max+2}] - } - } -} - -# @options optionspec -# -# Specifies configuration-time options which may be selected by the user -# and checked with 'opt-str' and 'opt-bool'. '$optionspec' contains a series -# of options specifications separated by newlines, as follows: -# -# A boolean option is of the form: -# -## name[=0|1] => "Description of this boolean option" -# -# The default is 'name=0', meaning that the option is disabled by default. -# If 'name=1' is used to make the option enabled by default, the description should reflect -# that with text like "Disable support for ...". -# -# An argument option (one which takes a parameter) is of one of the following forms: -# -## name:value => "Description of this option" -## name:value=default => "Description of this option with a default value" -## name:=value => "Description of this option with an optional value" -# -# If the 'name:value' form is used, the value must be provided with the option (as '--name=myvalue'). -# If the 'name:value=default' form is used, the option has the given default value even if not -# specified by the user. -# If the 'name:=value' form is used, the value is optional and the given value is used -# if it is not provided. -# -# The description may contain '@default@', in which case it will be replaced with the default -# value for the option (taking into account defaults specified with 'options-defaults'. -# -# Undocumented options are also supported by omitting the '=> description'. -# These options are not displayed with '--help' and can be useful for internal options or as aliases. -# -# For example, '--disable-lfs' is an alias for '--disable=largefile': -# -## lfs=1 largefile=1 => "Disable large file support" -# -proc options {optlist} { - global autosetup - - options-add $optlist - - if {$autosetup(showhelp)} { - # If --help, stop now to show help - return -code break - } - - if {$autosetup(module) eq "auto.def"} { - # Check for invalid options - if {[opt-bool option-checking]} { - foreach o [dict keys $::autosetup(getopt)] { - if {$o ni $::autosetup(options)} { - user-error "Unknown option --$o" - } - } - } - } -} - -# @options-defaults dictionary -# -# Specifies a dictionary of options and a new default value for each of those options. -# Use before any 'use' statements in 'auto.def' to change the defaults for -# subsequently included modules. -proc options-defaults {dict} { - foreach {n v} $dict { - dict set ::autosetup(options-defaults) $n $v - } -} - -proc config_guess {} { - if {[file-isexec $::autosetup(dir)/autosetup-config.guess]} { - if {[catch {exec-with-stderr sh $::autosetup(dir)/autosetup-config.guess} alias]} { - user-error $alias - } - return $alias - } else { - configlog "No autosetup-config.guess, so using uname" - string tolower [exec uname -p]-unknown-[exec uname -s][exec uname -r] - } -} - -proc config_sub {alias} { - if {[file-isexec $::autosetup(dir)/autosetup-config.sub]} { - if {[catch {exec-with-stderr sh $::autosetup(dir)/autosetup-config.sub $alias} alias]} { - user-error $alias - } - } - return $alias -} - -# @define name ?value=1? -# -# Defines the named variable to the given value. -# These (name, value) pairs represent the results of the configuration check -# and are available to be subsequently checked, modified and substituted. -# -proc define {name {value 1}} { - set ::define($name) $value - #dputs "$name <= $value" -} - -# @undefine name -# -# Undefine the named variable. -# -proc undefine {name} { - unset -nocomplain ::define($name) - #dputs "$name <= " -} - -# @define-append name value ... -# -# Appends the given value(s) to the given "defined" variable. -# If the variable is not defined or empty, it is set to '$value'. -# Otherwise the value is appended, separated by a space. -# Any extra values are similarly appended. -# If any value is already contained in the variable (as a substring) it is omitted. -# -proc define-append {name args} { - if {[get-define $name ""] ne ""} { - # Avoid duplicates - foreach arg $args { - if {$arg eq ""} { - continue - } - set found 0 - foreach str [split $::define($name) " "] { - if {$str eq $arg} { - incr found - } - } - if {!$found} { - append ::define($name) " " $arg - } - } - } else { - set ::define($name) [join $args] - } - #dputs "$name += [join $args] => $::define($name)" -} - -# @get-define name ?default=0? -# -# Returns the current value of the "defined" variable, or '$default' -# if not set. -# -proc get-define {name {default 0}} { - if {[info exists ::define($name)]} { - #dputs "$name => $::define($name)" - return $::define($name) - } - #dputs "$name => $default" - return $default -} - -# @is-defined name -# -# Returns 1 if the given variable is defined. -# -proc is-defined {name} { - info exists ::define($name) -} - -# @is-define-set name -# -# Returns 1 if the given variable is defined and is set -# to a value other than "" or 0 -# -proc is-define-set {name} { - if {[get-define $name] in {0 ""}} { - return 0 - } - return 1 -} - -# @all-defines -# -# Returns a dictionary (name, value list) of all defined variables. -# -# This is suitable for use with 'dict', 'array set' or 'foreach' -# and allows for arbitrary processing of the defined variables. -# -proc all-defines {} { - array get ::define -} - - -# @get-env name default -# -# If '$name' was specified on the command line, return it. -# Otherwise if '$name' was set in the environment, return it. -# Otherwise return '$default'. -# -proc get-env {name default} { - if {[dict exists $::autosetup(cmdline) $name]} { - return [dict get $::autosetup(cmdline) $name] - } - getenv $name $default -} - -# @env-is-set name -# -# Returns 1 if '$name' was specified on the command line or in the environment. -# Note that an empty environment variable is not considered to be set. -# -proc env-is-set {name} { - if {[dict exists $::autosetup(cmdline) $name]} { - return 1 - } - if {[getenv $name ""] ne ""} { - return 1 - } - return 0 -} - -# @readfile filename ?default=""? -# -# Return the contents of the file, without the trailing newline. -# If the file doesn't exist or can't be read, returns '$default'. -# -proc readfile {filename {default_value ""}} { - set result $default_value - catch { - set f [open $filename] - set result [read -nonewline $f] - close $f - } - return $result -} - -# @writefile filename value -# -# Creates the given file containing '$value'. -# Does not add an extra newline. -# -proc writefile {filename value} { - set f [open $filename w] - puts -nonewline $f $value - close $f -} - -proc quote-if-needed {str} { - if {[string match {*[\" ]*} $str]} { - return \"[string map [list \" \\" \\ \\\\] $str]\" - } - return $str -} - -proc quote-argv {argv} { - set args {} - foreach arg $argv { - lappend args [quote-if-needed $arg] - } - join $args -} - -# @list-non-empty list -# -# Returns a copy of the given list with empty elements removed -proc list-non-empty {list} { - set result {} - foreach p $list { - if {$p ne ""} { - lappend result $p - } - } - return $result -} - -# @find-executable-path name -# -# Searches the path for an executable with the given name. -# Note that the name may include some parameters, e.g. 'cc -mbig-endian', -# in which case the parameters are ignored. -# The full path to the executable if found, or "" if not found. -# Returns 1 if found, or 0 if not. -# -proc find-executable-path {name} { - # Ignore any parameters - set name [lindex $name 0] - # The empty string is never a valid executable - if {$name ne ""} { - foreach p [split-path] { - dputs "Looking for $name in $p" - set exec [file join $p $name] - if {[file-isexec $exec]} { - dputs "Found $name -> $exec" - return $exec - } - } - } - return {} -} - -# @find-executable name -# -# Searches the path for an executable with the given name. -# Note that the name may include some parameters, e.g. 'cc -mbig-endian', -# in which case the parameters are ignored. -# Returns 1 if found, or 0 if not. -# -proc find-executable {name} { - if {[find-executable-path $name] eq {}} { - return 0 - } - return 1 -} - -# @find-an-executable ?-required? name ... -# -# Given a list of possible executable names, -# searches for one of these on the path. -# -# Returns the name found, or "" if none found. -# If the first parameter is '-required', an error is generated -# if no executable is found. -# -proc find-an-executable {args} { - set required 0 - if {[lindex $args 0] eq "-required"} { - set args [lrange $args 1 end] - incr required - } - foreach name $args { - if {[find-executable $name]} { - return $name - } - } - if {$required} { - if {[llength $args] == 1} { - user-error "failed to find: [join $args]" - } else { - user-error "failed to find one of: [join $args]" - } - } - return "" -} - -# @configlog msg -# -# Writes the given message to the configuration log, 'config.log'. -# -proc configlog {msg} { - if {![info exists ::autosetup(logfh)]} { - set ::autosetup(logfh) [open config.log w] - } - puts $::autosetup(logfh) $msg -} - -# @msg-checking msg -# -# Writes the message with no newline to stdout. -# -proc msg-checking {msg} { - if {$::autosetup(msg-quiet) == 0} { - maybe-show-timestamp - puts -nonewline $msg - set ::autosetup(msg-checking) 1 - } -} - -# @msg-result msg -# -# Writes the message to stdout. -# -proc msg-result {msg} { - if {$::autosetup(msg-quiet) == 0} { - maybe-show-timestamp - puts $msg - set ::autosetup(msg-checking) 0 - show-notices - } -} - -# @msg-quiet command ... -# -# 'msg-quiet' evaluates it's arguments as a command with output -# from 'msg-checking' and 'msg-result' suppressed. -# -# This is useful if a check needs to run a subcheck which isn't -# of interest to the user. -proc msg-quiet {args} { - incr ::autosetup(msg-quiet) - set rc [uplevel 1 $args] - incr ::autosetup(msg-quiet) -1 - return $rc -} - -# Will be overridden by 'use misc' -proc error-stacktrace {msg} { - return $msg -} - -proc error-location {msg} { - return $msg -} - -################################################################## -# -# Debugging output -# -proc dputs {msg} { - if {$::autosetup(debug)} { - puts $msg - } -} - -################################################################## -# -# User and system warnings and errors -# -# Usage errors such as wrong command line options - -# @user-error msg -# -# Indicate incorrect usage to the user, including if required components -# or features are not found. -# 'autosetup' exits with a non-zero return code. -# -proc user-error {msg} { - show-notices - puts stderr "Error: $msg" - puts stderr "Try: '[file tail $::autosetup(exe)] --help' for options" - exit 1 -} - -# @user-notice msg -# -# Output the given message to stderr. -# -proc user-notice {msg} { - lappend ::autosetup(notices) $msg -} - -# Incorrect usage in the auto.def file. Identify the location. -proc autosetup-error {msg} { - autosetup-full-error [error-location $msg] -} - -# Like autosetup-error, except $msg is the full error message. -proc autosetup-full-error {msg} { - show-notices - puts stderr $msg - exit 1 -} - -proc show-notices {} { - if {$::autosetup(msg-checking)} { - puts "" - set ::autosetup(msg-checking) 0 - } - flush stdout - if {[info exists ::autosetup(notices)]} { - puts stderr [join $::autosetup(notices) \n] - unset ::autosetup(notices) - } -} - -proc maybe-show-timestamp {} { - if {$::autosetup(msg-timing) && $::autosetup(msg-checking) == 0} { - puts -nonewline [format {[%6.2f] } [expr {([clock millis] - $::autosetup(start)) % 10000 / 1000.0}]] - } -} - -# @autosetup-require-version required -# -# Checks the current version of 'autosetup' against '$required'. -# A fatal error is generated if the current version is less than that required. -# -proc autosetup-require-version {required} { - if {[compare-versions $::autosetup(version) $required] < 0} { - user-error "autosetup version $required is required, but this is $::autosetup(version)" - } -} - -proc autosetup_version {} { - return "autosetup v$::autosetup(version)" -} - -################################################################## -# -# Directory/path handling -# - -proc realdir {dir} { - set oldpwd [pwd] - cd $dir - set pwd [pwd] - cd $oldpwd - return $pwd -} - -# Follow symlinks until we get to something which is not a symlink -proc realpath {path} { - while {1} { - if {[catch { - set path [file readlink $path] - }]} { - # Not a link - break - } - } - return $path -} - -# Convert absolute path, $path into a path relative -# to the given directory (or the current dir, if not given). -# -proc relative-path {path {pwd {}}} { - set diff 0 - set same 0 - set newf {} - set prefix {} - set path [file-normalize $path] - if {$pwd eq ""} { - set pwd [pwd] - } else { - set pwd [file-normalize $pwd] - } - - if {$path eq $pwd} { - return . - } - - # Try to make the filename relative to the current dir - foreach p [split $pwd /] f [split $path /] { - if {$p ne $f} { - incr diff - } elseif {!$diff} { - incr same - } - if {$diff} { - if {$p ne ""} { - # Add .. for sibling or parent dir - lappend prefix .. - } - if {$f ne ""} { - lappend newf $f - } - } - } - if {$same == 1 || [llength $prefix] > 3} { - return $path - } - - file join [join $prefix /] [join $newf /] -} - -# Add filename as a dependency to rerun autosetup -# The name will be normalised (converted to a full path) -# -proc autosetup_add_dep {filename} { - lappend ::autosetup(deps) [file-normalize $filename] -} - -################################################################## -# -# Library module support -# - -# @use module ... -# -# Load the given library modules. -# e.g. 'use cc cc-shared' -# -# Note that module 'X' is implemented in either 'autosetup/X.tcl' -# or 'autosetup/X/init.tcl' -# -# The latter form is useful for a complex module which requires additional -# support file. In this form, '$::usedir' is set to the module directory -# when it is loaded. -# -proc use {args} { - global autosetup libmodule modsource - - set dirs [list $autosetup(libdir)] - if {[info exists autosetup(srcdir)]} { - lappend dirs $autosetup(srcdir)/autosetup - } - foreach m $args { - if {[info exists libmodule($m)]} { - continue - } - set libmodule($m) 1 - - if {[info exists modsource(${m}.tcl)]} { - autosetup_load_module $m eval $modsource(${m}.tcl) - } else { - set locs [list ${m}.tcl ${m}/init.tcl] - set found 0 - foreach dir $dirs { - foreach loc $locs { - set source $dir/$loc - if {[file exists $source]} { - incr found - break - } - } - if {$found} { - break - } - } - if {$found} { - # For the convenience of the "use" source, point to the directory - # it is being loaded from - set ::usedir [file dirname $source] - autosetup_load_module $m source $source - autosetup_add_dep $source - } else { - autosetup-error "use: No such module: $m" - } - } - } -} - -proc autosetup_load_auto_modules {} { - global autosetup modsource - # First load any embedded auto modules - foreach mod [array names modsource *.auto] { - autosetup_load_module $mod eval $modsource($mod) - } - # Now any external auto modules - foreach file [glob -nocomplain $autosetup(libdir)/*.auto $autosetup(libdir)/*/*.auto] { - autosetup_load_module [file tail $file] source $file - } -} - -# Load module source in the global scope by executing the given command -proc autosetup_load_module {module args} { - global autosetup - set prev $autosetup(module) - set autosetup(module) $module - - if {[catch [list uplevel #0 $args] msg opts] ni {0 2 3}} { - autosetup-full-error [error-dump $msg $opts $::autosetup(debug)] - } - set autosetup(module) $prev -} - -# Initial settings -set autosetup(exe) $::argv0 -set autosetup(istcl) 1 -set autosetup(start) [clock millis] -set autosetup(installed) 0 -set autosetup(sysinstall) 0 -set autosetup(msg-checking) 0 -set autosetup(msg-quiet) 0 -set autosetup(inittypes) {} -set autosetup(module) autosetup - -# Embedded modules are inserted below here -set autosetup(installed) 1 -set autosetup(sysinstall) 0 -# ----- @module asciidoc-formatting.tcl ----- - -set modsource(asciidoc-formatting.tcl) { -# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Module which provides text formatting -# asciidoc format - -use formatting - -proc para {text} { - regsub -all "\[ \t\n\]+" [string trim $text] " " -} -proc title {text} { - underline [para $text] = - nl -} -proc p {text} { - puts [para $text] - nl -} -proc code {text} { - foreach line [parse_code_block $text] { - puts " $line" - } - nl -} -proc codelines {lines} { - foreach line $lines { - puts " $line" - } - nl -} -proc nl {} { - puts "" -} -proc underline {text char} { - regexp "^(\[ \t\]*)(.*)" $text -> indent words - puts $text - puts $indent[string repeat $char [string length $words]] -} -proc section {text} { - underline "[para $text]" - - nl -} -proc subsection {text} { - underline "$text" ~ - nl -} -proc bullet {text} { - puts "* [para $text]" -} -proc indent {text} { - puts " :: " - puts [para $text] -} -proc defn {first args} { - set sep "" - if {$first ne ""} { - puts "${first}::" - } else { - puts " :: " - } - set defn [string trim [join $args \n]] - regsub -all "\n\n" $defn "\n ::\n" defn - puts $defn -} -} - -# ----- @module formatting.tcl ----- - -set modsource(formatting.tcl) { -# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Module which provides common text formatting - -# This is designed for documentation which looks like: -# code {...} -# or -# code { -# ... -# ... -# } -# In the second case, we need to work out the indenting -# and strip it from all lines but preserve the remaining indenting. -# Note that all lines need to be indented with the same initial -# spaces/tabs. -# -# Returns a list of lines with the indenting removed. -# -proc parse_code_block {text} { - # If the text begins with newline, take the following text, - # otherwise just return the original - if {![regexp "^\n(.*)" $text -> text]} { - return [list [string trim $text]] - } - - # And trip spaces off the end - set text [string trimright $text] - - set min 100 - # Examine each line to determine the minimum indent - foreach line [split $text \n] { - if {$line eq ""} { - # Ignore empty lines for the indent calculation - continue - } - regexp "^(\[ \t\]*)" $line -> indent - set len [string length $indent] - if {$len < $min} { - set min $len - } - } - - # Now make a list of lines with this indent removed - set lines {} - foreach line [split $text \n] { - lappend lines [string range $line $min end] - } - - # Return the result - return $lines -} -} - -# ----- @module getopt.tcl ----- - -set modsource(getopt.tcl) { -# Copyright (c) 2006 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Simple getopt module - -# Parse everything out of the argv list which looks like an option -# Everything which doesn't look like an option, or is after --, is left unchanged -# Understands --enable-xxx as a synonym for --xxx to enable the boolean option xxx. -# Understands --disable-xxx to disable the boolean option xxx. -# -# The returned value is a dictionary keyed by option name -# Each value is a list of {type value} ... where type is "bool" or "str". -# The value for a boolean option is 0 or 1. The value of a string option is the value given. -proc getopt {argvname} { - upvar $argvname argv - set nargv {} - - set opts {} - - for {set i 0} {$i < [llength $argv]} {incr i} { - set arg [lindex $argv $i] - - #dputs arg=$arg - - if {$arg eq "--"} { - # End of options - incr i - lappend nargv {*}[lrange $argv $i end] - break - } - - if {[regexp {^--([^=][^=]+)=(.*)$} $arg -> name value]} { - # --name=value - dict lappend opts $name [list str $value] - } elseif {[regexp {^--(enable-|disable-)?([^=]*)$} $arg -> prefix name]} { - if {$prefix in {enable- ""}} { - set value 1 - } else { - set value 0 - } - dict lappend opts $name [list bool $value] - } else { - lappend nargv $arg - } - } - - #puts "getopt: argv=[join $argv] => [join $nargv]" - #array set getopt $opts - #parray getopt - - set argv $nargv - - return $opts -} -} - -# ----- @module help.tcl ----- - -set modsource(help.tcl) { -# Copyright (c) 2010 WorkWare Systems http://workware.net.au/ -# All rights reserved - -# Module which provides usage, help and the command reference - -proc autosetup_help {what} { - use_pager - - puts "Usage: [file tail $::autosetup(exe)] \[options\] \[settings\]\n" - puts "This is [autosetup_version], a build environment \"autoconfigurator\"" - puts "See the documentation online at http://msteveb.github.io/autosetup/\n" - - if {$what in {all local}} { - # Need to load auto.def now - if {[file exists $::autosetup(autodef)]} { - # Load auto.def as module "auto.def" - autosetup_load_module auto.def source $::autosetup(autodef) - } - if {$what eq "all"} { - set what * - } else { - set what auto.def - } - } else { - use $what - puts "Options for module $what:" - } - options-show $what - exit 0 -} - -proc autosetup_show_license {} { - global modsource autosetup - use_pager - - if {[info exists modsource(LICENSE)]} { - puts $modsource(LICENSE) - return - } - foreach dir [list $autosetup(libdir) $autosetup(srcdir)] { - set path [file join $dir LICENSE] - if {[file exists $path]} { - puts [readfile $path] - return - } - } - puts "LICENSE not found" -} - -# If not already paged and stdout is a tty, pipe the output through the pager -# This is done by reinvoking autosetup with --nopager added -proc use_pager {} { - if {![opt-bool nopager] && [getenv PAGER ""] ne "" && [isatty? stdin] && [isatty? stdout]} { - if {[catch { - exec [info nameofexecutable] $::argv0 --nopager {*}$::argv |& {*}[getenv PAGER] >@stdout <@stdin 2>@stderr - } msg opts] == 1} { - if {[dict get $opts -errorcode] eq "NONE"} { - # an internal/exec error - puts stderr $msg - exit 1 - } - } - exit 0 - } -} - -# Outputs the autosetup references in one of several formats -proc autosetup_reference {{type text}} { - - use_pager - - switch -glob -- $type { - wiki {use wiki-formatting} - ascii* {use asciidoc-formatting} - md - markdown {use markdown-formatting} - default {use text-formatting} - } - - title "[autosetup_version] -- Command Reference" - - section {Introduction} - - p { - See http://msteveb.github.com/autosetup/ for the online documentation for 'autosetup' - } - - p { - 'autosetup' provides a number of built-in commands which - are documented below. These may be used from 'auto.def' to test - for features, define variables, create files from templates and - other similar actions. - } - - automf_command_reference - - exit 0 -} - -proc autosetup_output_block {type lines} { - if {[llength $lines]} { - switch $type { - section { - section $lines - } - subsection { - subsection $lines - } - code { - codelines $lines - } - p { - p [join $lines] - } - list { - foreach line $lines { - bullet $line - } - nl - } - } - } -} - -# Generate a command reference from inline documentation -proc automf_command_reference {} { - lappend files $::autosetup(prog) - lappend files {*}[lsort [glob -nocomplain $::autosetup(libdir)/*.tcl]] - - # We want to process all non-module files before module files - # and then modules in alphabetical order. - # So examine all files and extract docs into doc($modulename) and doc(_core_) - # - # Each entry is a list of {type data} where $type is one of: section, subsection, code, list, p - # and $data is a string for section, subsection or a list of text lines for other types. - - # XXX: Should commands be in alphabetical order too? Currently they are in file order. - - set doc(_core_) {} - lappend doc(_core_) [list section "Core Commands"] - - foreach file $files { - set modulename [file rootname [file tail $file]] - set current _core_ - set f [open $file] - while {![eof $f]} { - set line [gets $f] - - # Find embedded module names - if {[regexp {^#.*@module ([^ ]*)} $line -> modulename]} { - continue - } - - # Find lines starting with "# @*" and continuing through the remaining comment lines - if {![regexp {^# @(.*)} $line -> cmd]} { - continue - } - - # Synopsis or command? - if {$cmd eq "synopsis:"} { - set current $modulename - lappend doc($current) [list section "Module: $modulename"] - } else { - lappend doc($current) [list subsection $cmd] - } - - set lines {} - set type p - - # Now the description - while {![eof $f]} { - set line [gets $f] - - if {![regexp {^#(#)? ?(.*)} $line -> hash cmd]} { - break - } - if {$hash eq "#"} { - set t code - } elseif {[regexp {^- (.*)} $cmd -> cmd]} { - set t list - } else { - set t p - } - - #puts "hash=$hash, oldhash=$oldhash, lines=[llength $lines], cmd=$cmd" - - if {$t ne $type || $cmd eq ""} { - # Finish the current block - lappend doc($current) [list $type $lines] - set lines {} - set type $t - } - if {$cmd ne ""} { - lappend lines $cmd - } - } - - lappend doc($current) [list $type $lines] - } - close $f - } - - # Now format and output the results - - # _core_ will sort first - foreach module [lsort [array names doc]] { - foreach item $doc($module) { - autosetup_output_block {*}$item - } - } -} -} - -# ----- @module init.tcl ----- - -set modsource(init.tcl) { -# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Module to help create auto.def and configure - -proc autosetup_init {type} { - set help 0 - if {$type in {? help}} { - incr help - } elseif {![dict exists $::autosetup(inittypes) $type]} { - puts "Unknown type, --init=$type" - incr help - } - if {$help} { - puts "Use one of the following types (e.g. --init=make)\n" - foreach type [lsort [dict keys $::autosetup(inittypes)]] { - lassign [dict get $::autosetup(inittypes) $type] desc - # XXX: Use the options-show code to wrap the description - puts [format "%-10s %s" $type $desc] - } - return - } - lassign [dict get $::autosetup(inittypes) $type] desc script - - puts "Initialising $type: $desc\n" - - # All initialisations happens in the top level srcdir - cd $::autosetup(srcdir) - - uplevel #0 $script -} - -proc autosetup_add_init_type {type desc script} { - dict set ::autosetup(inittypes) $type [list $desc $script] -} - -# This is for in creating build-system init scripts -# -# If the file doesn't exist, create it containing $contents -# If the file does exist, only overwrite if --force is specified. -# -proc autosetup_check_create {filename contents} { - if {[file exists $filename]} { - if {!$::autosetup(force)} { - puts "I see $filename already exists." - return - } else { - puts "I will overwrite the existing $filename because you used --force." - } - } else { - puts "I don't see $filename, so I will create it." - } - writefile $filename $contents -} -} - -# ----- @module install.tcl ----- - -set modsource(install.tcl) { -# Copyright (c) 2006-2010 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Module which can install autosetup - -# autosetup(installed)=1 means that autosetup is not running from source -# autosetup(sysinstall)=1 means that autosetup is running from a sysinstall version -# shared=1 means that we are trying to do a sysinstall. This is only possible from the development source. - -proc autosetup_install {dir {shared 0}} { - global autosetup - if {$shared} { - if {$autosetup(installed) || $autosetup(sysinstall)} { - user-error "Can only --sysinstall from development sources" - } - } elseif {$autosetup(installed) && !$autosetup(sysinstall)} { - user-error "Can't --install from project install" - } - - if {$autosetup(sysinstall)} { - # This is the sysinstall version, so install just uses references - cd $dir - - puts "[autosetup_version] creating configure to use system-installed autosetup" - autosetup_create_configure 1 - puts "Creating autosetup/README.autosetup" - file mkdir autosetup - autosetup_install_readme autosetup/README.autosetup 1 - return - } - - if {[catch { - if {$shared} { - set target $dir/bin/autosetup - set installedas $target - } else { - if {$dir eq "."} { - set installedas autosetup - } else { - set installedas $dir/autosetup - } - cd $dir - file mkdir autosetup - set target autosetup/autosetup - } - set targetdir [file dirname $target] - file mkdir $targetdir - - set f [open $target w] - - set publicmodules {} - - # First the main script, but only up until "CUT HERE" - set in [open $autosetup(dir)/autosetup] - while {[gets $in buf] >= 0} { - if {$buf ne "##-- CUT HERE --##"} { - puts $f $buf - continue - } - - # Insert the static modules here - # i.e. those which don't contain @synopsis: - # All modules are inserted if $shared is set - puts $f "set autosetup(installed) 1" - puts $f "set autosetup(sysinstall) $shared" - foreach file [lsort [glob $autosetup(libdir)/*.{tcl,auto}]] { - set modname [file tail $file] - set ext [file ext $modname] - set buf [readfile $file] - if {!$shared} { - if {$ext eq ".auto" || [string match "*\n# @synopsis:*" $buf]} { - lappend publicmodules $file - continue - } - } - dputs "install: importing lib/[file tail $file]" - puts $f "# ----- @module $modname -----" - puts $f "\nset modsource($modname) \{" - puts $f $buf - puts $f "\}\n" - } - if {$shared} { - foreach {srcname destname} [list $autosetup(libdir)/README.autosetup-lib README.autosetup \ - $autosetup(srcdir)/LICENSE LICENSE] { - dputs "install: importing $srcname as $destname" - puts $f "\nset modsource($destname) \\\n[list [readfile $srcname]\n]\n" - } - } - } - close $in - close $f - catch {exec chmod 755 $target} - - set installfiles {autosetup-config.guess autosetup-config.sub autosetup-test-tclsh} - set removefiles {} - - if {!$shared} { - autosetup_install_readme $targetdir/README.autosetup 0 - - # Install public modules - foreach file $publicmodules { - set tail [file tail $file] - autosetup_install_file $file $targetdir/$tail - } - lappend installfiles jimsh0.c autosetup-find-tclsh LICENSE - lappend removefiles config.guess config.sub test-tclsh find-tclsh - } else { - lappend installfiles {sys-find-tclsh autosetup-find-tclsh} - } - - # Install support files - foreach fileinfo $installfiles { - if {[llength $fileinfo] == 2} { - lassign $fileinfo source dest - } else { - lassign $fileinfo source - set dest $source - } - autosetup_install_file $autosetup(dir)/$source $targetdir/$dest - } - - # Remove obsolete files - foreach file $removefiles { - if {[file exists $targetdir/$file]} { - file delete $targetdir/$file - } - } - } error]} { - user-error "Failed to install autosetup: $error" - } - if {$shared} { - set type "system" - } else { - set type "local" - } - puts "Installed $type [autosetup_version] to $installedas" - - if {!$shared} { - # Now create 'configure' if necessary - autosetup_create_configure 0 - } -} - -proc autosetup_create_configure {shared} { - if {[file exists configure]} { - if {!$::autosetup(force)} { - # Could this be an autosetup configure? - if {![string match "*\nWRAPPER=*" [readfile configure]]} { - puts "I see configure, but not created by autosetup, so I won't overwrite it." - puts "Remove it or use --force to overwrite." - return - } - } else { - puts "I will overwrite the existing configure because you used --force." - } - } else { - puts "I don't see configure, so I will create it." - } - if {$shared} { - writefile configure \ -{#!/bin/sh -WRAPPER="$0"; export WRAPPER; "autosetup" "$@" -} - } else { - writefile configure \ -{#!/bin/sh -dir="`dirname "$0"`/autosetup" -WRAPPER="$0"; export WRAPPER; exec "`"$dir/autosetup-find-tclsh"`" "$dir/autosetup" "$@" -} - } - catch {exec chmod 755 configure} -} - -# Append the contents of $file to filehandle $f -proc autosetup_install_append {f file} { - dputs "install: include $file" - set in [open $file] - puts $f [read $in] - close $in -} - -proc autosetup_install_file {source target} { - dputs "install: $source => $target" - if {![file exists $source]} { - error "Missing installation file '$source'" - } - writefile $target [readfile $source]\n - # If possible, copy the file mode - file stat $source stat - set mode [format %o [expr {$stat(mode) & 0x1ff}]] - catch {exec chmod $mode $target} -} - -proc autosetup_install_readme {target sysinstall} { - set readme "README.autosetup created by [autosetup_version]\n\n" - if {$sysinstall} { - append readme \ -{This is the autosetup directory for a system install of autosetup. -Loadable modules can be added here. -} - } else { - append readme \ -{This is the autosetup directory for a local install of autosetup. -It contains autosetup, support files and loadable modules. -} -} - - append readme { -*.tcl files in this directory are optional modules which -can be loaded with the 'use' directive. - -*.auto files in this directory are auto-loaded. - -For more information, see http://msteveb.github.io/autosetup/ -} - dputs "install: autosetup/README.autosetup" - writefile $target $readme -} -} - -# ----- @module markdown-formatting.tcl ----- - -set modsource(markdown-formatting.tcl) { -# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Module which provides text formatting -# markdown format (kramdown syntax) - -use formatting - -proc para {text} { - regsub -all "\[ \t\n\]+" [string trim $text] " " text - regsub -all {([^a-zA-Z])'([^']*)'} $text {\1**`\2`**} text - regsub -all {^'([^']*)'} $text {**`\1`**} text - regsub -all {(http[^ \t\n]*)} $text {[\1](\1)} text - return $text -} -proc title {text} { - underline [para $text] = - nl -} -proc p {text} { - puts [para $text] - nl -} -proc codelines {lines} { - puts "~~~~~~~~~~~~" - foreach line $lines { - puts $line - } - puts "~~~~~~~~~~~~" - nl -} -proc code {text} { - puts "~~~~~~~~~~~~" - foreach line [parse_code_block $text] { - puts $line - } - puts "~~~~~~~~~~~~" - nl -} -proc nl {} { - puts "" -} -proc underline {text char} { - regexp "^(\[ \t\]*)(.*)" $text -> indent words - puts $text - puts $indent[string repeat $char [string length $words]] -} -proc section {text} { - underline "[para $text]" - - nl -} -proc subsection {text} { - puts "### `$text`" - nl -} -proc bullet {text} { - puts "* [para $text]" -} -proc defn {first args} { - puts "^" - set defn [string trim [join $args \n]] - if {$first ne ""} { - puts "**${first}**" - puts -nonewline ": " - regsub -all "\n\n" $defn "\n: " defn - } - puts "$defn" -} -} - -# ----- @module misc.tcl ----- - -set modsource(misc.tcl) { -# Copyright (c) 2007-2010 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Module containing misc procs useful to modules -# Largely for platform compatibility - -set autosetup(istcl) [info exists ::tcl_library] -set autosetup(iswin) [string equal windows $tcl_platform(platform)] - -if {$autosetup(iswin)} { - # mingw/windows separates $PATH with semicolons - # and doesn't have an executable bit - proc split-path {} { - split [getenv PATH .] {;} - } - proc file-isexec {exec} { - # Basic test for windows. We ignore .bat - if {[file isfile $exec] || [file isfile $exec.exe]} { - return 1 - } - return 0 - } -} else { - # unix separates $PATH with colons and has and executable bit - proc split-path {} { - split [getenv PATH .] : - } - proc file-isexec {exec} { - file executable $exec - } -} - -# Assume that exec can return stdout and stderr -proc exec-with-stderr {args} { - exec {*}$args 2>@1 -} - -if {$autosetup(istcl)} { - # Tcl doesn't have the env command - proc getenv {name args} { - if {[info exists ::env($name)]} { - return $::env($name) - } - if {[llength $args]} { - return [lindex $args 0] - } - return -code error "environment variable \"$name\" does not exist" - } - proc isatty? {channel} { - dict exists [fconfigure $channel] -xchar - } -} else { - if {$autosetup(iswin)} { - # On Windows, backslash convert all environment variables - # (Assume that Tcl does this for us) - proc getenv {name args} { - string map {\\ /} [env $name {*}$args] - } - } else { - # Jim on unix is simple - alias getenv env - } - proc isatty? {channel} { - set tty 0 - catch { - # isatty is a recent addition to Jim Tcl - set tty [$channel isatty] - } - return $tty - } -} - -# In case 'file normalize' doesn't exist -# -proc file-normalize {path} { - if {[catch {file normalize $path} result]} { - if {$path eq ""} { - return "" - } - set oldpwd [pwd] - if {[file isdir $path]} { - cd $path - set result [pwd] - } else { - cd [file dirname $path] - set result [file join [pwd] [file tail $path]] - } - cd $oldpwd - } - return $result -} - -# If everything is working properly, the only errors which occur -# should be generated in user code (e.g. auto.def). -# By default, we only want to show the error location in user code. -# We use [info frame] to achieve this, but it works differently on Tcl and Jim. -# -# This is designed to be called for incorrect usage in auto.def, via autosetup-error -# -proc error-location {msg} { - if {$::autosetup(debug)} { - return -code error $msg - } - # Search back through the stack trace for the first error in a .def file - for {set i 1} {$i < [info level]} {incr i} { - if {$::autosetup(istcl)} { - array set info [info frame -$i] - } else { - lassign [info frame -$i] info(caller) info(file) info(line) - } - if {[string match *.def $info(file)]} { - return "[relative-path $info(file)]:$info(line): Error: $msg" - } - #puts "Skipping $info(file):$info(line)" - } - return $msg -} - -# If everything is working properly, the only errors which occur -# should be generated in user code (e.g. auto.def). -# By default, we only want to show the error location in user code. -# We use [info frame] to achieve this, but it works differently on Tcl and Jim. -# -# This is designed to be called for incorrect usage in auto.def, via autosetup-error -# -proc error-stacktrace {msg} { - if {$::autosetup(debug)} { - return -code error $msg - } - # Search back through the stack trace for the first error in a .def file - for {set i 1} {$i < [info level]} {incr i} { - if {$::autosetup(istcl)} { - array set info [info frame -$i] - } else { - lassign [info frame -$i] info(caller) info(file) info(line) - } - if {[string match *.def $info(file)]} { - return "[relative-path $info(file)]:$info(line): Error: $msg" - } - #puts "Skipping $info(file):$info(line)" - } - return $msg -} - -# Given the return from [catch {...} msg opts], returns an appropriate -# error message. A nice one for Jim and a less-nice one for Tcl. -# If 'fulltrace' is set, a full stack trace is provided. -# Otherwise a simple message is provided. -# -# This is designed for developer errors, e.g. in module code or auto.def code -# -# -proc error-dump {msg opts fulltrace} { - if {$::autosetup(istcl)} { - if {$fulltrace} { - return "Error: [dict get $opts -errorinfo]" - } else { - return "Error: $msg" - } - } else { - lassign $opts(-errorinfo) p f l - if {$f ne ""} { - set result "$f:$l: Error: " - } - append result "$msg\n" - if {$fulltrace} { - append result [stackdump $opts(-errorinfo)] - } - - # Remove the trailing newline - string trim $result - } -} -} - -# ----- @module text-formatting.tcl ----- - -set modsource(text-formatting.tcl) { -# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Module which provides text formatting - -use formatting - -proc wordwrap {text length {firstprefix ""} {nextprefix ""}} { - set len 0 - set space $firstprefix - - foreach word [split $text] { - set word [string trim $word] - if {$word eq ""} { - continue - } - if {[info exists partial]} { - append partial " " $word - if {[string first $quote $word] < 0} { - # Haven't found end of quoted word - continue - } - # Finished quoted word - set word $partial - unset partial - unset quote - } else { - set quote [string index $word 0] - if {$quote in {' *}} { - if {[string first $quote $word 1] < 0} { - # Haven't found end of quoted word - # Not a whole word. - set first [string index $word 0] - # Start of quoted word - set partial $word - continue - } - } - } - - if {$len && [string length $space$word] + $len >= $length} { - puts "" - set len 0 - set space $nextprefix - } - incr len [string length $space$word] - - # Use man-page conventions for highlighting 'quoted' and *quoted* - # single words. - # Use x^Hx for *bold* and _^Hx for 'underline'. - # - # less and more will both understand this. - # Pipe through 'col -b' to remove them. - if {[regexp {^'(.*)'(.*)} $word -> quoted after]} { - set quoted [string map {~ " "} $quoted] - regsub -all . $quoted "&\b&" quoted - set word $quoted$after - } elseif {[regexp {^[*](.*)[*](.*)} $word -> quoted after]} { - set quoted [string map {~ " "} $quoted] - regsub -all . $quoted "_\b&" quoted - set word $quoted$after - } - puts -nonewline $space$word - set space " " - } - if {[info exists partial]} { - # Missing end of quote - puts -nonewline $space$partial - } - if {$len} { - puts "" - } -} -proc title {text} { - underline [string trim $text] = - nl -} -proc p {text} { - wordwrap $text 80 - nl -} -proc codelines {lines} { - foreach line $lines { - puts " $line" - } - nl -} -proc nl {} { - puts "" -} -proc underline {text char} { - regexp "^(\[ \t\]*)(.*)" $text -> indent words - puts $text - puts $indent[string repeat $char [string length $words]] -} -proc section {text} { - underline "[string trim $text]" - - nl -} -proc subsection {text} { - underline "$text" ~ - nl -} -proc bullet {text} { - wordwrap $text 76 " * " " " -} -proc indent {text} { - wordwrap $text 76 " " " " -} -proc defn {first args} { - if {$first ne ""} { - underline " $first" ~ - } - foreach p $args { - if {$p ne ""} { - indent $p - } - } -} -} - -# ----- @module util.tcl ----- - -set modsource(util.tcl) { -# Copyright (c) 2012 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Module which contains miscellaneous utility functions - -# @compare-versions version1 version2 -# -# Versions are of the form 'a.b.c' (may be any number of numeric components) -# -# Compares the two versions and returns: -## -1 if v1 < v2 -## 0 if v1 == v2 -## 1 if v1 > v2 -# -# If one version has fewer components than the other, 0 is substituted to the right. e.g. -## 0.2 < 0.3 -## 0.2.5 > 0.2 -## 1.1 == 1.1.0 -# -proc compare-versions {v1 v2} { - foreach c1 [split $v1 .] c2 [split $v2 .] { - if {$c1 eq ""} { - set c1 0 - } - if {$c2 eq ""} { - set c2 0 - } - if {$c1 < $c2} { - return -1 - } - if {$c1 > $c2} { - return 1 - } - } - return 0 -} - -# @suffix suf list -# -# Takes a list and returns a new list with '$suf' appended -# to each element -# -## suffix .c {a b c} => {a.c b.c c.c} -# -proc suffix {suf list} { - set result {} - foreach p $list { - lappend result $p$suf - } - return $result -} - -# @prefix pre list -# -# Takes a list and returns a new list with '$pre' prepended -# to each element -# -## prefix jim- {a.c b.c} => {jim-a.c jim-b.c} -# -proc prefix {pre list} { - set result {} - foreach p $list { - lappend result $pre$p - } - return $result -} - -# @lpop list -# -# Removes the last entry from the given list and returns it. -proc lpop {listname} { - upvar $listname list - set val [lindex $list end] - set list [lrange $list 0 end-1] - return $val -} -} - -# ----- @module wiki-formatting.tcl ----- - -set modsource(wiki-formatting.tcl) { -# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Module which provides text formatting -# wiki.tcl.tk format output - -use formatting - -proc joinlines {text} { - set lines {} - foreach l [split [string trim $text] \n] { - lappend lines [string trim $l] - } - join $lines -} -proc p {text} { - puts [joinlines $text] - puts "" -} -proc title {text} { - puts "*** [joinlines $text] ***" - puts "" -} -proc codelines {lines} { - puts "======" - foreach line $lines { - puts " $line" - } - puts "======" -} -proc code {text} { - puts "======" - foreach line [parse_code_block $text] { - puts " $line" - } - puts "======" -} -proc nl {} { -} -proc section {text} { - puts "'''$text'''" - puts "" -} -proc subsection {text} { - puts "''$text''" - puts "" -} -proc bullet {text} { - puts " * [joinlines $text]" -} -proc indent {text} { - puts " : [joinlines $text]" -} -proc defn {first args} { - if {$first ne ""} { - indent '''$first''' - } - - foreach p $args { - p $p - } -} -} - - -################################################################## -# -# Entry/Exit -# -if {$autosetup(debug)} { - main $argv -} -if {[catch {main $argv} msg opts] == 1} { - show-notices - autosetup-full-error [error-dump $msg $opts $autosetup(debug)] - if {!$autosetup(debug)} { - puts stderr "Try: '[file tail $autosetup(exe)] --debug' for a full stack trace" - } - exit 1 -} DELETED autosetup/autosetup-config.guess Index: autosetup/autosetup-config.guess ================================================================== --- autosetup/autosetup-config.guess +++ autosetup/autosetup-config.guess @@ -1,1748 +0,0 @@ -#! /bin/sh -# Attempt to guess a canonical system name. -# Copyright 1992-2021 Free Software Foundation, Inc. - -# shellcheck disable=SC2006,SC2268 # see below for rationale - -timestamp='2021-06-03' - -# This file is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that -# program. This Exception is an additional permission under section 7 -# of the GNU General Public License, version 3 ("GPLv3"). -# -# Originally written by Per Bothner; maintained since 2000 by Ben Elliston. -# -# You can get the latest version of this script from: -# https://git.savannah.gnu.org/cgit/config.git/plain/config.guess -# -# Please send patches to . - - -# The "shellcheck disable" line above the timestamp inhibits complaints -# about features and limitations of the classic Bourne shell that were -# superseded or lifted in POSIX. However, this script identifies a wide -# variety of pre-POSIX systems that do not have POSIX shells at all, and -# even some reasonably current systems (Solaris 10 as case-in-point) still -# have a pre-POSIX /bin/sh. - - -me=`echo "$0" | sed -e 's,.*/,,'` - -usage="\ -Usage: $0 [OPTION] - -Output the configuration name of the system \`$me' is run on. - -Options: - -h, --help print this help, then exit - -t, --time-stamp print date of last modification, then exit - -v, --version print version number, then exit - -Report bugs and patches to ." - -version="\ -GNU config.guess ($timestamp) - -Originally written by Per Bothner. -Copyright 1992-2021 Free Software Foundation, Inc. - -This is free software; see the source for copying conditions. There is NO -warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." - -help=" -Try \`$me --help' for more information." - -# Parse command line -while test $# -gt 0 ; do - case $1 in - --time-stamp | --time* | -t ) - echo "$timestamp" ; exit ;; - --version | -v ) - echo "$version" ; exit ;; - --help | --h* | -h ) - echo "$usage"; exit ;; - -- ) # Stop option processing - shift; break ;; - - ) # Use stdin as input. - break ;; - -* ) - echo "$me: invalid option $1$help" >&2 - exit 1 ;; - * ) - break ;; - esac -done - -if test $# != 0; then - echo "$me: too many arguments$help" >&2 - exit 1 -fi - -# Just in case it came from the environment. -GUESS= - -# CC_FOR_BUILD -- compiler used by this script. Note that the use of a -# compiler to aid in system detection is discouraged as it requires -# temporary files to be created and, as you can see below, it is a -# headache to deal with in a portable fashion. - -# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still -# use `HOST_CC' if defined, but it is deprecated. - -# Portable tmp directory creation inspired by the Autoconf team. - -tmp= -# shellcheck disable=SC2172 -trap 'test -z "$tmp" || rm -fr "$tmp"' 0 1 2 13 15 - -set_cc_for_build() { - # prevent multiple calls if $tmp is already set - test "$tmp" && return 0 - : "${TMPDIR=/tmp}" - # shellcheck disable=SC2039,SC3028 - { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || - { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || - { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || - { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } - dummy=$tmp/dummy - case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in - ,,) echo "int x;" > "$dummy.c" - for driver in cc gcc c89 c99 ; do - if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then - CC_FOR_BUILD=$driver - break - fi - done - if test x"$CC_FOR_BUILD" = x ; then - CC_FOR_BUILD=no_compiler_found - fi - ;; - ,,*) CC_FOR_BUILD=$CC ;; - ,*,*) CC_FOR_BUILD=$HOST_CC ;; - esac -} - -# This is needed to find uname on a Pyramid OSx when run in the BSD universe. -# (ghazi@noc.rutgers.edu 1994-08-24) -if test -f /.attbin/uname ; then - PATH=$PATH:/.attbin ; export PATH -fi - -UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown -UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown -UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown -UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown - -case $UNAME_SYSTEM in -Linux|GNU|GNU/*) - LIBC=unknown - - set_cc_for_build - cat <<-EOF > "$dummy.c" - #include - #if defined(__UCLIBC__) - LIBC=uclibc - #elif defined(__dietlibc__) - LIBC=dietlibc - #elif defined(__GLIBC__) - LIBC=gnu - #else - #include - /* First heuristic to detect musl libc. */ - #ifdef __DEFINED_va_list - LIBC=musl - #endif - #endif - EOF - cc_set_libc=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` - eval "$cc_set_libc" - - # Second heuristic to detect musl libc. - if [ "$LIBC" = unknown ] && - command -v ldd >/dev/null && - ldd --version 2>&1 | grep -q ^musl; then - LIBC=musl - fi - - # If the system lacks a compiler, then just pick glibc. - # We could probably try harder. - if [ "$LIBC" = unknown ]; then - LIBC=gnu - fi - ;; -esac - -# Note: order is significant - the case branches are not exclusive. - -case $UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION in - *:NetBSD:*:*) - # NetBSD (nbsd) targets should (where applicable) match one or - # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, - # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently - # switched to ELF, *-*-netbsd* would select the old - # object file format. This provides both forward - # compatibility and a consistent mechanism for selecting the - # object file format. - # - # Note: NetBSD doesn't particularly care about the vendor - # portion of the name. We always set it to "unknown". - UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ - /sbin/sysctl -n hw.machine_arch 2>/dev/null || \ - /usr/sbin/sysctl -n hw.machine_arch 2>/dev/null || \ - echo unknown)` - case $UNAME_MACHINE_ARCH in - aarch64eb) machine=aarch64_be-unknown ;; - armeb) machine=armeb-unknown ;; - arm*) machine=arm-unknown ;; - sh3el) machine=shl-unknown ;; - sh3eb) machine=sh-unknown ;; - sh5el) machine=sh5le-unknown ;; - earmv*) - arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'` - endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'` - machine=${arch}${endian}-unknown - ;; - *) machine=$UNAME_MACHINE_ARCH-unknown ;; - esac - # The Operating System including object format, if it has switched - # to ELF recently (or will in the future) and ABI. - case $UNAME_MACHINE_ARCH in - earm*) - os=netbsdelf - ;; - arm*|i386|m68k|ns32k|sh3*|sparc|vax) - set_cc_for_build - if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ELF__ - then - # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). - # Return netbsd for either. FIX? - os=netbsd - else - os=netbsdelf - fi - ;; - *) - os=netbsd - ;; - esac - # Determine ABI tags. - case $UNAME_MACHINE_ARCH in - earm*) - expr='s/^earmv[0-9]/-eabi/;s/eb$//' - abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"` - ;; - esac - # The OS release - # Debian GNU/NetBSD machines have a different userland, and - # thus, need a distinct triplet. However, they do not need - # kernel version information, so it can be replaced with a - # suitable tag, in the style of linux-gnu. - case $UNAME_VERSION in - Debian*) - release='-gnu' - ;; - *) - release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2` - ;; - esac - # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: - # contains redundant information, the shorter form: - # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - GUESS=$machine-${os}${release}${abi-} - ;; - *:Bitrig:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` - GUESS=$UNAME_MACHINE_ARCH-unknown-bitrig$UNAME_RELEASE - ;; - *:OpenBSD:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` - GUESS=$UNAME_MACHINE_ARCH-unknown-openbsd$UNAME_RELEASE - ;; - *:SecBSD:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/SecBSD.//'` - GUESS=$UNAME_MACHINE_ARCH-unknown-secbsd$UNAME_RELEASE - ;; - *:LibertyBSD:*:*) - UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` - GUESS=$UNAME_MACHINE_ARCH-unknown-libertybsd$UNAME_RELEASE - ;; - *:MidnightBSD:*:*) - GUESS=$UNAME_MACHINE-unknown-midnightbsd$UNAME_RELEASE - ;; - *:ekkoBSD:*:*) - GUESS=$UNAME_MACHINE-unknown-ekkobsd$UNAME_RELEASE - ;; - *:SolidBSD:*:*) - GUESS=$UNAME_MACHINE-unknown-solidbsd$UNAME_RELEASE - ;; - *:OS108:*:*) - GUESS=$UNAME_MACHINE-unknown-os108_$UNAME_RELEASE - ;; - macppc:MirBSD:*:*) - GUESS=powerpc-unknown-mirbsd$UNAME_RELEASE - ;; - *:MirBSD:*:*) - GUESS=$UNAME_MACHINE-unknown-mirbsd$UNAME_RELEASE - ;; - *:Sortix:*:*) - GUESS=$UNAME_MACHINE-unknown-sortix - ;; - *:Twizzler:*:*) - GUESS=$UNAME_MACHINE-unknown-twizzler - ;; - *:Redox:*:*) - GUESS=$UNAME_MACHINE-unknown-redox - ;; - mips:OSF1:*.*) - GUESS=mips-dec-osf1 - ;; - alpha:OSF1:*:*) - # Reset EXIT trap before exiting to avoid spurious non-zero exit code. - trap '' 0 - case $UNAME_RELEASE in - *4.0) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` - ;; - *5.*) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` - ;; - esac - # According to Compaq, /usr/sbin/psrinfo has been available on - # OSF/1 and Tru64 systems produced since 1995. I hope that - # covers most systems running today. This code pipes the CPU - # types through head -n 1, so we only detect the type of CPU 0. - ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` - case $ALPHA_CPU_TYPE in - "EV4 (21064)") - UNAME_MACHINE=alpha ;; - "EV4.5 (21064)") - UNAME_MACHINE=alpha ;; - "LCA4 (21066/21068)") - UNAME_MACHINE=alpha ;; - "EV5 (21164)") - UNAME_MACHINE=alphaev5 ;; - "EV5.6 (21164A)") - UNAME_MACHINE=alphaev56 ;; - "EV5.6 (21164PC)") - UNAME_MACHINE=alphapca56 ;; - "EV5.7 (21164PC)") - UNAME_MACHINE=alphapca57 ;; - "EV6 (21264)") - UNAME_MACHINE=alphaev6 ;; - "EV6.7 (21264A)") - UNAME_MACHINE=alphaev67 ;; - "EV6.8CB (21264C)") - UNAME_MACHINE=alphaev68 ;; - "EV6.8AL (21264B)") - UNAME_MACHINE=alphaev68 ;; - "EV6.8CX (21264D)") - UNAME_MACHINE=alphaev68 ;; - "EV6.9A (21264/EV69A)") - UNAME_MACHINE=alphaev69 ;; - "EV7 (21364)") - UNAME_MACHINE=alphaev7 ;; - "EV7.9 (21364A)") - UNAME_MACHINE=alphaev79 ;; - esac - # A Pn.n version is a patched version. - # A Vn.n version is a released version. - # A Tn.n version is a released field test version. - # A Xn.n version is an unreleased experimental baselevel. - # 1.2 uses "1.2" for uname -r. - OSF_REL=`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` - GUESS=$UNAME_MACHINE-dec-osf$OSF_REL - ;; - Amiga*:UNIX_System_V:4.0:*) - GUESS=m68k-unknown-sysv4 - ;; - *:[Aa]miga[Oo][Ss]:*:*) - GUESS=$UNAME_MACHINE-unknown-amigaos - ;; - *:[Mm]orph[Oo][Ss]:*:*) - GUESS=$UNAME_MACHINE-unknown-morphos - ;; - *:OS/390:*:*) - GUESS=i370-ibm-openedition - ;; - *:z/VM:*:*) - GUESS=s390-ibm-zvmoe - ;; - *:OS400:*:*) - GUESS=powerpc-ibm-os400 - ;; - arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) - GUESS=arm-acorn-riscix$UNAME_RELEASE - ;; - arm*:riscos:*:*|arm*:RISCOS:*:*) - GUESS=arm-unknown-riscos - ;; - SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) - GUESS=hppa1.1-hitachi-hiuxmpp - ;; - Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) - # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. - case `(/bin/universe) 2>/dev/null` in - att) GUESS=pyramid-pyramid-sysv3 ;; - *) GUESS=pyramid-pyramid-bsd ;; - esac - ;; - NILE*:*:*:dcosx) - GUESS=pyramid-pyramid-svr4 - ;; - DRS?6000:unix:4.0:6*) - GUESS=sparc-icl-nx6 - ;; - DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) - case `/usr/bin/uname -p` in - sparc) GUESS=sparc-icl-nx7 ;; - esac - ;; - s390x:SunOS:*:*) - SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` - GUESS=$UNAME_MACHINE-ibm-solaris2$SUN_REL - ;; - sun4H:SunOS:5.*:*) - SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` - GUESS=sparc-hal-solaris2$SUN_REL - ;; - sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` - GUESS=sparc-sun-solaris2$SUN_REL - ;; - i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) - GUESS=i386-pc-auroraux$UNAME_RELEASE - ;; - i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) - set_cc_for_build - SUN_ARCH=i386 - # If there is a compiler, see if it is configured for 64-bit objects. - # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. - # This test works for both compilers. - if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - SUN_ARCH=x86_64 - fi - fi - SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` - GUESS=$SUN_ARCH-pc-solaris2$SUN_REL - ;; - sun4*:SunOS:6*:*) - # According to config.sub, this is the proper way to canonicalize - # SunOS6. Hard to guess exactly what SunOS6 will be like, but - # it's likely to be more like Solaris than SunOS4. - SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` - GUESS=sparc-sun-solaris3$SUN_REL - ;; - sun4*:SunOS:*:*) - case `/usr/bin/arch -k` in - Series*|S4*) - UNAME_RELEASE=`uname -v` - ;; - esac - # Japanese Language versions have a version number like `4.1.3-JL'. - SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/-/_/'` - GUESS=sparc-sun-sunos$SUN_REL - ;; - sun3*:SunOS:*:*) - GUESS=m68k-sun-sunos$UNAME_RELEASE - ;; - sun*:*:4.2BSD:*) - UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` - test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 - case `/bin/arch` in - sun3) - GUESS=m68k-sun-sunos$UNAME_RELEASE - ;; - sun4) - GUESS=sparc-sun-sunos$UNAME_RELEASE - ;; - esac - ;; - aushp:SunOS:*:*) - GUESS=sparc-auspex-sunos$UNAME_RELEASE - ;; - # The situation for MiNT is a little confusing. The machine name - # can be virtually everything (everything which is not - # "atarist" or "atariste" at least should have a processor - # > m68000). The system name ranges from "MiNT" over "FreeMiNT" - # to the lowercase version "mint" (or "freemint"). Finally - # the system name "TOS" denotes a system which is actually not - # MiNT. But MiNT is downward compatible to TOS, so this should - # be no problem. - atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - GUESS=m68k-atari-mint$UNAME_RELEASE - ;; - atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) - GUESS=m68k-atari-mint$UNAME_RELEASE - ;; - *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - GUESS=m68k-atari-mint$UNAME_RELEASE - ;; - milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - GUESS=m68k-milan-mint$UNAME_RELEASE - ;; - hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - GUESS=m68k-hades-mint$UNAME_RELEASE - ;; - *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - GUESS=m68k-unknown-mint$UNAME_RELEASE - ;; - m68k:machten:*:*) - GUESS=m68k-apple-machten$UNAME_RELEASE - ;; - powerpc:machten:*:*) - GUESS=powerpc-apple-machten$UNAME_RELEASE - ;; - RISC*:Mach:*:*) - GUESS=mips-dec-mach_bsd4.3 - ;; - RISC*:ULTRIX:*:*) - GUESS=mips-dec-ultrix$UNAME_RELEASE - ;; - VAX*:ULTRIX*:*:*) - GUESS=vax-dec-ultrix$UNAME_RELEASE - ;; - 2020:CLIX:*:* | 2430:CLIX:*:*) - GUESS=clipper-intergraph-clix$UNAME_RELEASE - ;; - mips:*:*:UMIPS | mips:*:*:RISCos) - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" -#ifdef __cplusplus -#include /* for printf() prototype */ - int main (int argc, char *argv[]) { -#else - int main (argc, argv) int argc; char *argv[]; { -#endif - #if defined (host_mips) && defined (MIPSEB) - #if defined (SYSTYPE_SYSV) - printf ("mips-mips-riscos%ssysv\\n", argv[1]); exit (0); - #endif - #if defined (SYSTYPE_SVR4) - printf ("mips-mips-riscos%ssvr4\\n", argv[1]); exit (0); - #endif - #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) - printf ("mips-mips-riscos%sbsd\\n", argv[1]); exit (0); - #endif - #endif - exit (-1); - } -EOF - $CC_FOR_BUILD -o "$dummy" "$dummy.c" && - dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` && - SYSTEM_NAME=`"$dummy" "$dummyarg"` && - { echo "$SYSTEM_NAME"; exit; } - GUESS=mips-mips-riscos$UNAME_RELEASE - ;; - Motorola:PowerMAX_OS:*:*) - GUESS=powerpc-motorola-powermax - ;; - Motorola:*:4.3:PL8-*) - GUESS=powerpc-harris-powermax - ;; - Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) - GUESS=powerpc-harris-powermax - ;; - Night_Hawk:Power_UNIX:*:*) - GUESS=powerpc-harris-powerunix - ;; - m88k:CX/UX:7*:*) - GUESS=m88k-harris-cxux7 - ;; - m88k:*:4*:R4*) - GUESS=m88k-motorola-sysv4 - ;; - m88k:*:3*:R3*) - GUESS=m88k-motorola-sysv3 - ;; - AViiON:dgux:*:*) - # DG/UX returns AViiON for all architectures - UNAME_PROCESSOR=`/usr/bin/uname -p` - if test "$UNAME_PROCESSOR" = mc88100 || test "$UNAME_PROCESSOR" = mc88110 - then - if test "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx || \ - test "$TARGET_BINARY_INTERFACE"x = x - then - GUESS=m88k-dg-dgux$UNAME_RELEASE - else - GUESS=m88k-dg-dguxbcs$UNAME_RELEASE - fi - else - GUESS=i586-dg-dgux$UNAME_RELEASE - fi - ;; - M88*:DolphinOS:*:*) # DolphinOS (SVR3) - GUESS=m88k-dolphin-sysv3 - ;; - M88*:*:R3*:*) - # Delta 88k system running SVR3 - GUESS=m88k-motorola-sysv3 - ;; - XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) - GUESS=m88k-tektronix-sysv3 - ;; - Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) - GUESS=m68k-tektronix-bsd - ;; - *:IRIX*:*:*) - IRIX_REL=`echo "$UNAME_RELEASE" | sed -e 's/-/_/g'` - GUESS=mips-sgi-irix$IRIX_REL - ;; - ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. - GUESS=romp-ibm-aix # uname -m gives an 8 hex-code CPU id - ;; # Note that: echo "'`uname -s`'" gives 'AIX ' - i*86:AIX:*:*) - GUESS=i386-ibm-aix - ;; - ia64:AIX:*:*) - if test -x /usr/bin/oslevel ; then - IBM_REV=`/usr/bin/oslevel` - else - IBM_REV=$UNAME_VERSION.$UNAME_RELEASE - fi - GUESS=$UNAME_MACHINE-ibm-aix$IBM_REV - ;; - *:AIX:2:3) - if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" - #include - - main() - { - if (!__power_pc()) - exit(1); - puts("powerpc-ibm-aix3.2.5"); - exit(0); - } -EOF - if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` - then - GUESS=$SYSTEM_NAME - else - GUESS=rs6000-ibm-aix3.2.5 - fi - elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then - GUESS=rs6000-ibm-aix3.2.4 - else - GUESS=rs6000-ibm-aix3.2 - fi - ;; - *:AIX:*:[4567]) - IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` - if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then - IBM_ARCH=rs6000 - else - IBM_ARCH=powerpc - fi - if test -x /usr/bin/lslpp ; then - IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | \ - awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` - else - IBM_REV=$UNAME_VERSION.$UNAME_RELEASE - fi - GUESS=$IBM_ARCH-ibm-aix$IBM_REV - ;; - *:AIX:*:*) - GUESS=rs6000-ibm-aix - ;; - ibmrt:4.4BSD:*|romp-ibm:4.4BSD:*) - GUESS=romp-ibm-bsd4.4 - ;; - ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and - GUESS=romp-ibm-bsd$UNAME_RELEASE # 4.3 with uname added to - ;; # report: romp-ibm BSD 4.3 - *:BOSX:*:*) - GUESS=rs6000-bull-bosx - ;; - DPX/2?00:B.O.S.:*:*) - GUESS=m68k-bull-sysv3 - ;; - 9000/[34]??:4.3bsd:1.*:*) - GUESS=m68k-hp-bsd - ;; - hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) - GUESS=m68k-hp-bsd4.4 - ;; - 9000/[34678]??:HP-UX:*:*) - HPUX_REV=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*.[0B]*//'` - case $UNAME_MACHINE in - 9000/31?) HP_ARCH=m68000 ;; - 9000/[34]??) HP_ARCH=m68k ;; - 9000/[678][0-9][0-9]) - if test -x /usr/bin/getconf; then - sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` - sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case $sc_cpu_version in - 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 - 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 - 532) # CPU_PA_RISC2_0 - case $sc_kernel_bits in - 32) HP_ARCH=hppa2.0n ;; - 64) HP_ARCH=hppa2.0w ;; - '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 - esac ;; - esac - fi - if test "$HP_ARCH" = ""; then - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" - - #define _HPUX_SOURCE - #include - #include - - int main () - { - #if defined(_SC_KERNEL_BITS) - long bits = sysconf(_SC_KERNEL_BITS); - #endif - long cpu = sysconf (_SC_CPU_VERSION); - - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1"); break; - case CPU_PA_RISC2_0: - #if defined(_SC_KERNEL_BITS) - switch (bits) - { - case 64: puts ("hppa2.0w"); break; - case 32: puts ("hppa2.0n"); break; - default: puts ("hppa2.0"); break; - } break; - #else /* !defined(_SC_KERNEL_BITS) */ - puts ("hppa2.0"); break; - #endif - default: puts ("hppa1.0"); break; - } - exit (0); - } -EOF - (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"` - test -z "$HP_ARCH" && HP_ARCH=hppa - fi ;; - esac - if test "$HP_ARCH" = hppa2.0w - then - set_cc_for_build - - # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating - # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler - # generating 64-bit code. GNU and HP use different nomenclature: - # - # $ CC_FOR_BUILD=cc ./config.guess - # => hppa2.0w-hp-hpux11.23 - # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess - # => hppa64-hp-hpux11.23 - - if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | - grep -q __LP64__ - then - HP_ARCH=hppa2.0w - else - HP_ARCH=hppa64 - fi - fi - GUESS=$HP_ARCH-hp-hpux$HPUX_REV - ;; - ia64:HP-UX:*:*) - HPUX_REV=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*.[0B]*//'` - GUESS=ia64-hp-hpux$HPUX_REV - ;; - 3050*:HI-UX:*:*) - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" - #include - int - main () - { - long cpu = sysconf (_SC_CPU_VERSION); - /* The order matters, because CPU_IS_HP_MC68K erroneously returns - true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct - results, however. */ - if (CPU_IS_PA_RISC (cpu)) - { - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break; - case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break; - default: puts ("hppa-hitachi-hiuxwe2"); break; - } - } - else if (CPU_IS_HP_MC68K (cpu)) - puts ("m68k-hitachi-hiuxwe2"); - else puts ("unknown-hitachi-hiuxwe2"); - exit (0); - } -EOF - $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` && - { echo "$SYSTEM_NAME"; exit; } - GUESS=unknown-hitachi-hiuxwe2 - ;; - 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:*) - GUESS=hppa1.1-hp-bsd - ;; - 9000/8??:4.3bsd:*:*) - GUESS=hppa1.0-hp-bsd - ;; - *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) - GUESS=hppa1.0-hp-mpeix - ;; - hp7??:OSF1:*:* | hp8?[79]:OSF1:*:*) - GUESS=hppa1.1-hp-osf - ;; - hp8??:OSF1:*:*) - GUESS=hppa1.0-hp-osf - ;; - i*86:OSF1:*:*) - if test -x /usr/sbin/sysversion ; then - GUESS=$UNAME_MACHINE-unknown-osf1mk - else - GUESS=$UNAME_MACHINE-unknown-osf1 - fi - ;; - parisc*:Lites*:*:*) - GUESS=hppa1.1-hp-lites - ;; - C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) - GUESS=c1-convex-bsd - ;; - C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) - if getsysinfo -f scalar_acc - then echo c32-convex-bsd - else echo c2-convex-bsd - fi - exit ;; - C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) - GUESS=c34-convex-bsd - ;; - C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) - GUESS=c38-convex-bsd - ;; - C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) - GUESS=c4-convex-bsd - ;; - CRAY*Y-MP:*:*:*) - CRAY_REL=`echo "$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'` - GUESS=ymp-cray-unicos$CRAY_REL - ;; - CRAY*[A-Z]90:*:*:*) - echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \ - | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ - -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ - -e 's/\.[^.]*$/.X/' - exit ;; - CRAY*TS:*:*:*) - CRAY_REL=`echo "$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'` - GUESS=t90-cray-unicos$CRAY_REL - ;; - CRAY*T3E:*:*:*) - CRAY_REL=`echo "$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'` - GUESS=alphaev5-cray-unicosmk$CRAY_REL - ;; - CRAY*SV1:*:*:*) - CRAY_REL=`echo "$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'` - GUESS=sv1-cray-unicos$CRAY_REL - ;; - *:UNICOS/mp:*:*) - CRAY_REL=`echo "$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/'` - GUESS=craynv-cray-unicosmp$CRAY_REL - ;; - F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` - FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` - FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'` - GUESS=${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL} - ;; - 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` - FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` - GUESS=sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL} - ;; - i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) - GUESS=$UNAME_MACHINE-pc-bsdi$UNAME_RELEASE - ;; - sparc*:BSD/OS:*:*) - GUESS=sparc-unknown-bsdi$UNAME_RELEASE - ;; - *:BSD/OS:*:*) - GUESS=$UNAME_MACHINE-unknown-bsdi$UNAME_RELEASE - ;; - arm:FreeBSD:*:*) - UNAME_PROCESSOR=`uname -p` - set_cc_for_build - if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ARM_PCS_VFP - then - FREEBSD_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` - GUESS=$UNAME_PROCESSOR-unknown-freebsd$FREEBSD_REL-gnueabi - else - FREEBSD_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` - GUESS=$UNAME_PROCESSOR-unknown-freebsd$FREEBSD_REL-gnueabihf - fi - ;; - *:FreeBSD:*:*) - UNAME_PROCESSOR=`/usr/bin/uname -p` - case $UNAME_PROCESSOR in - amd64) - UNAME_PROCESSOR=x86_64 ;; - i386) - UNAME_PROCESSOR=i586 ;; - esac - FREEBSD_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` - GUESS=$UNAME_PROCESSOR-unknown-freebsd$FREEBSD_REL - ;; - i*:CYGWIN*:*) - GUESS=$UNAME_MACHINE-pc-cygwin - ;; - *:MINGW64*:*) - GUESS=$UNAME_MACHINE-pc-mingw64 - ;; - *:MINGW*:*) - GUESS=$UNAME_MACHINE-pc-mingw32 - ;; - *:MSYS*:*) - GUESS=$UNAME_MACHINE-pc-msys - ;; - i*:PW*:*) - GUESS=$UNAME_MACHINE-pc-pw32 - ;; - *:Interix*:*) - case $UNAME_MACHINE in - x86) - GUESS=i586-pc-interix$UNAME_RELEASE - ;; - authenticamd | genuineintel | EM64T) - GUESS=x86_64-unknown-interix$UNAME_RELEASE - ;; - IA64) - GUESS=ia64-unknown-interix$UNAME_RELEASE - ;; - esac ;; - i*:UWIN*:*) - GUESS=$UNAME_MACHINE-pc-uwin - ;; - amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) - GUESS=x86_64-pc-cygwin - ;; - prep*:SunOS:5.*:*) - SUN_REL=`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'` - GUESS=powerpcle-unknown-solaris2$SUN_REL - ;; - *:GNU:*:*) - # the GNU system - GNU_ARCH=`echo "$UNAME_MACHINE" | sed -e 's,[-/].*$,,'` - GNU_REL=`echo "$UNAME_RELEASE" | sed -e 's,/.*$,,'` - GUESS=$GNU_ARCH-unknown-$LIBC$GNU_REL - ;; - *:GNU/*:*:*) - # other systems with GNU libc and userland - GNU_SYS=`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"` - GNU_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` - GUESS=$UNAME_MACHINE-unknown-$GNU_SYS$GNU_REL-$LIBC - ;; - *:Minix:*:*) - GUESS=$UNAME_MACHINE-unknown-minix - ;; - aarch64:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - aarch64_be:Linux:*:*) - UNAME_MACHINE=aarch64_be - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' /proc/cpuinfo 2>/dev/null` in - EV5) UNAME_MACHINE=alphaev5 ;; - EV56) UNAME_MACHINE=alphaev56 ;; - PCA56) UNAME_MACHINE=alphapca56 ;; - PCA57) UNAME_MACHINE=alphapca56 ;; - EV6) UNAME_MACHINE=alphaev6 ;; - EV67) UNAME_MACHINE=alphaev67 ;; - EV68*) UNAME_MACHINE=alphaev68 ;; - esac - objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC=gnulibc1 ; fi - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - arc:Linux:*:* | arceb:Linux:*:* | arc32:Linux:*:* | arc64:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - arm*:Linux:*:*) - set_cc_for_build - if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ARM_EABI__ - then - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - else - if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ARM_PCS_VFP - then - GUESS=$UNAME_MACHINE-unknown-linux-${LIBC}eabi - else - GUESS=$UNAME_MACHINE-unknown-linux-${LIBC}eabihf - fi - fi - ;; - avr32*:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - cris:Linux:*:*) - GUESS=$UNAME_MACHINE-axis-linux-$LIBC - ;; - crisv32:Linux:*:*) - GUESS=$UNAME_MACHINE-axis-linux-$LIBC - ;; - e2k:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - frv:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - hexagon:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - i*86:Linux:*:*) - GUESS=$UNAME_MACHINE-pc-linux-$LIBC - ;; - ia64:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - k1om:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - loongarch32:Linux:*:* | loongarch64:Linux:*:* | loongarchx32:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - m32r*:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - m68*:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - mips:Linux:*:* | mips64:Linux:*:*) - set_cc_for_build - IS_GLIBC=0 - test x"${LIBC}" = xgnu && IS_GLIBC=1 - sed 's/^ //' << EOF > "$dummy.c" - #undef CPU - #undef mips - #undef mipsel - #undef mips64 - #undef mips64el - #if ${IS_GLIBC} && defined(_ABI64) - LIBCABI=gnuabi64 - #else - #if ${IS_GLIBC} && defined(_ABIN32) - LIBCABI=gnuabin32 - #else - LIBCABI=${LIBC} - #endif - #endif - - #if ${IS_GLIBC} && defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 - CPU=mipsisa64r6 - #else - #if ${IS_GLIBC} && !defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 - CPU=mipsisa32r6 - #else - #if defined(__mips64) - CPU=mips64 - #else - CPU=mips - #endif - #endif - #endif - - #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - MIPS_ENDIAN=el - #else - #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - MIPS_ENDIAN= - #else - MIPS_ENDIAN= - #endif - #endif -EOF - cc_set_vars=`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI'` - eval "$cc_set_vars" - test "x$CPU" != x && { echo "$CPU${MIPS_ENDIAN}-unknown-linux-$LIBCABI"; exit; } - ;; - mips64el:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - openrisc*:Linux:*:*) - GUESS=or1k-unknown-linux-$LIBC - ;; - or32:Linux:*:* | or1k*:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - padre:Linux:*:*) - GUESS=sparc-unknown-linux-$LIBC - ;; - parisc64:Linux:*:* | hppa64:Linux:*:*) - GUESS=hppa64-unknown-linux-$LIBC - ;; - parisc:Linux:*:* | hppa:Linux:*:*) - # Look for CPU level - case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in - PA7*) GUESS=hppa1.1-unknown-linux-$LIBC ;; - PA8*) GUESS=hppa2.0-unknown-linux-$LIBC ;; - *) GUESS=hppa-unknown-linux-$LIBC ;; - esac - ;; - ppc64:Linux:*:*) - GUESS=powerpc64-unknown-linux-$LIBC - ;; - ppc:Linux:*:*) - GUESS=powerpc-unknown-linux-$LIBC - ;; - ppc64le:Linux:*:*) - GUESS=powerpc64le-unknown-linux-$LIBC - ;; - ppcle:Linux:*:*) - GUESS=powerpcle-unknown-linux-$LIBC - ;; - riscv32:Linux:*:* | riscv32be:Linux:*:* | riscv64:Linux:*:* | riscv64be:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - s390:Linux:*:* | s390x:Linux:*:*) - GUESS=$UNAME_MACHINE-ibm-linux-$LIBC - ;; - sh64*:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - sh*:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - sparc:Linux:*:* | sparc64:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - tile*:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - vax:Linux:*:*) - GUESS=$UNAME_MACHINE-dec-linux-$LIBC - ;; - x86_64:Linux:*:*) - set_cc_for_build - LIBCABI=$LIBC - if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_X32 >/dev/null - then - LIBCABI=${LIBC}x32 - fi - fi - GUESS=$UNAME_MACHINE-pc-linux-$LIBCABI - ;; - xtensa*:Linux:*:*) - GUESS=$UNAME_MACHINE-unknown-linux-$LIBC - ;; - i*86:DYNIX/ptx:4*:*) - # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. - # earlier versions are messed up and put the nodename in both - # sysname and nodename. - GUESS=i386-sequent-sysv4 - ;; - i*86:UNIX_SV:4.2MP:2.*) - # Unixware is an offshoot of SVR4, but it has its own version - # number series starting with 2... - # I am not positive that other SVR4 systems won't match this, - # I just have to hope. -- rms. - # Use sysv4.2uw... so that sysv4* matches it. - GUESS=$UNAME_MACHINE-pc-sysv4.2uw$UNAME_VERSION - ;; - i*86:OS/2:*:*) - # If we were able to find `uname', then EMX Unix compatibility - # is probably installed. - GUESS=$UNAME_MACHINE-pc-os2-emx - ;; - i*86:XTS-300:*:STOP) - GUESS=$UNAME_MACHINE-unknown-stop - ;; - i*86:atheos:*:*) - GUESS=$UNAME_MACHINE-unknown-atheos - ;; - i*86:syllable:*:*) - GUESS=$UNAME_MACHINE-pc-syllable - ;; - i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) - GUESS=i386-unknown-lynxos$UNAME_RELEASE - ;; - i*86:*DOS:*:*) - GUESS=$UNAME_MACHINE-pc-msdosdjgpp - ;; - i*86:*:4.*:*) - UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'` - if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then - GUESS=$UNAME_MACHINE-univel-sysv$UNAME_REL - else - GUESS=$UNAME_MACHINE-pc-sysv$UNAME_REL - fi - ;; - i*86:*:5:[678]*) - # UnixWare 7.x, OpenUNIX and OpenServer 6. - case `/bin/uname -X | grep "^Machine"` in - *486*) UNAME_MACHINE=i486 ;; - *Pentium) UNAME_MACHINE=i586 ;; - *Pent*|*Celeron) UNAME_MACHINE=i686 ;; - esac - GUESS=$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} - ;; - i*86:*:3.2:*) - if test -f /usr/options/cb.name; then - UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then - UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` - (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 - (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ - && UNAME_MACHINE=i586 - (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \ - && UNAME_MACHINE=i686 - (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ - && UNAME_MACHINE=i686 - GUESS=$UNAME_MACHINE-pc-sco$UNAME_REL - else - GUESS=$UNAME_MACHINE-pc-sysv32 - fi - ;; - pc:*:*:*) - # Left here for compatibility: - # uname -m prints for DJGPP always 'pc', but it prints nothing about - # the processor, so we play safe by assuming i586. - # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configure will decide that - # this is a cross-build. - GUESS=i586-pc-msdosdjgpp - ;; - Intel:Mach:3*:*) - GUESS=i386-pc-mach3 - ;; - paragon:*:*:*) - GUESS=i860-intel-osf1 - ;; - i860:*:4.*:*) # i860-SVR4 - if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then - GUESS=i860-stardent-sysv$UNAME_RELEASE # Stardent Vistra i860-SVR4 - else # Add other i860-SVR4 vendors below as they are discovered. - GUESS=i860-unknown-sysv$UNAME_RELEASE # Unknown i860-SVR4 - fi - ;; - mini*:CTIX:SYS*5:*) - # "miniframe" - GUESS=m68010-convergent-sysv - ;; - mc68k:UNIX:SYSTEM5:3.51m) - GUESS=m68k-convergent-sysv - ;; - M680?0:D-NIX:5.3:*) - GUESS=m68k-diab-dnix - ;; - M68*:*:R3V[5678]*:*) - test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; - 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) - OS_REL='' - test -r /etc/.relid \ - && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } - /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; - 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4; exit; } ;; - NCR*:*:4.2:* | MPRAS*:*:4.2:*) - OS_REL='.3' - test -r /etc/.relid \ - && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } - /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } - /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ - && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; - m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) - GUESS=m68k-unknown-lynxos$UNAME_RELEASE - ;; - mc68030:UNIX_System_V:4.*:*) - GUESS=m68k-atari-sysv4 - ;; - TSUNAMI:LynxOS:2.*:*) - GUESS=sparc-unknown-lynxos$UNAME_RELEASE - ;; - rs6000:LynxOS:2.*:*) - GUESS=rs6000-unknown-lynxos$UNAME_RELEASE - ;; - PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) - GUESS=powerpc-unknown-lynxos$UNAME_RELEASE - ;; - SM[BE]S:UNIX_SV:*:*) - GUESS=mips-dde-sysv$UNAME_RELEASE - ;; - RM*:ReliantUNIX-*:*:*) - GUESS=mips-sni-sysv4 - ;; - RM*:SINIX-*:*:*) - GUESS=mips-sni-sysv4 - ;; - *:SINIX-*:*:*) - if uname -p 2>/dev/null >/dev/null ; then - UNAME_MACHINE=`(uname -p) 2>/dev/null` - GUESS=$UNAME_MACHINE-sni-sysv4 - else - GUESS=ns32k-sni-sysv - fi - ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort - # says - GUESS=i586-unisys-sysv4 - ;; - *:UNIX_System_V:4*:FTX*) - # From Gerald Hewes . - # How about differentiating between stratus architectures? -djm - GUESS=hppa1.1-stratus-sysv4 - ;; - *:*:*:FTX*) - # From seanf@swdc.stratus.com. - GUESS=i860-stratus-sysv4 - ;; - i*86:VOS:*:*) - # From Paul.Green@stratus.com. - GUESS=$UNAME_MACHINE-stratus-vos - ;; - *:VOS:*:*) - # From Paul.Green@stratus.com. - GUESS=hppa1.1-stratus-vos - ;; - mc68*:A/UX:*:*) - GUESS=m68k-apple-aux$UNAME_RELEASE - ;; - news*:NEWS-OS:6*:*) - GUESS=mips-sony-newsos6 - ;; - R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) - if test -d /usr/nec; then - GUESS=mips-nec-sysv$UNAME_RELEASE - else - GUESS=mips-unknown-sysv$UNAME_RELEASE - fi - ;; - BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. - GUESS=powerpc-be-beos - ;; - BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. - GUESS=powerpc-apple-beos - ;; - BePC:BeOS:*:*) # BeOS running on Intel PC compatible. - GUESS=i586-pc-beos - ;; - BePC:Haiku:*:*) # Haiku running on Intel PC compatible. - GUESS=i586-pc-haiku - ;; - x86_64:Haiku:*:*) - GUESS=x86_64-unknown-haiku - ;; - SX-4:SUPER-UX:*:*) - GUESS=sx4-nec-superux$UNAME_RELEASE - ;; - SX-5:SUPER-UX:*:*) - GUESS=sx5-nec-superux$UNAME_RELEASE - ;; - SX-6:SUPER-UX:*:*) - GUESS=sx6-nec-superux$UNAME_RELEASE - ;; - SX-7:SUPER-UX:*:*) - GUESS=sx7-nec-superux$UNAME_RELEASE - ;; - SX-8:SUPER-UX:*:*) - GUESS=sx8-nec-superux$UNAME_RELEASE - ;; - SX-8R:SUPER-UX:*:*) - GUESS=sx8r-nec-superux$UNAME_RELEASE - ;; - SX-ACE:SUPER-UX:*:*) - GUESS=sxace-nec-superux$UNAME_RELEASE - ;; - Power*:Rhapsody:*:*) - GUESS=powerpc-apple-rhapsody$UNAME_RELEASE - ;; - *:Rhapsody:*:*) - GUESS=$UNAME_MACHINE-apple-rhapsody$UNAME_RELEASE - ;; - arm64:Darwin:*:*) - GUESS=aarch64-apple-darwin$UNAME_RELEASE - ;; - *:Darwin:*:*) - UNAME_PROCESSOR=`uname -p` - case $UNAME_PROCESSOR in - unknown) UNAME_PROCESSOR=powerpc ;; - esac - if command -v xcode-select > /dev/null 2> /dev/null && \ - ! xcode-select --print-path > /dev/null 2> /dev/null ; then - # Avoid executing cc if there is no toolchain installed as - # cc will be a stub that puts up a graphical alert - # prompting the user to install developer tools. - CC_FOR_BUILD=no_compiler_found - else - set_cc_for_build - fi - if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - case $UNAME_PROCESSOR in - i386) UNAME_PROCESSOR=x86_64 ;; - powerpc) UNAME_PROCESSOR=powerpc64 ;; - esac - fi - # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc - if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_PPC >/dev/null - then - UNAME_PROCESSOR=powerpc - fi - elif test "$UNAME_PROCESSOR" = i386 ; then - # uname -m returns i386 or x86_64 - UNAME_PROCESSOR=$UNAME_MACHINE - fi - GUESS=$UNAME_PROCESSOR-apple-darwin$UNAME_RELEASE - ;; - *:procnto*:*:* | *:QNX:[0123456789]*:*) - UNAME_PROCESSOR=`uname -p` - if test "$UNAME_PROCESSOR" = x86; then - UNAME_PROCESSOR=i386 - UNAME_MACHINE=pc - fi - GUESS=$UNAME_PROCESSOR-$UNAME_MACHINE-nto-qnx$UNAME_RELEASE - ;; - *:QNX:*:4*) - GUESS=i386-pc-qnx - ;; - NEO-*:NONSTOP_KERNEL:*:*) - GUESS=neo-tandem-nsk$UNAME_RELEASE - ;; - NSE-*:NONSTOP_KERNEL:*:*) - GUESS=nse-tandem-nsk$UNAME_RELEASE - ;; - NSR-*:NONSTOP_KERNEL:*:*) - GUESS=nsr-tandem-nsk$UNAME_RELEASE - ;; - NSV-*:NONSTOP_KERNEL:*:*) - GUESS=nsv-tandem-nsk$UNAME_RELEASE - ;; - NSX-*:NONSTOP_KERNEL:*:*) - GUESS=nsx-tandem-nsk$UNAME_RELEASE - ;; - *:NonStop-UX:*:*) - GUESS=mips-compaq-nonstopux - ;; - BS2000:POSIX*:*:*) - GUESS=bs2000-siemens-sysv - ;; - DS/*:UNIX_System_V:*:*) - GUESS=$UNAME_MACHINE-$UNAME_SYSTEM-$UNAME_RELEASE - ;; - *:Plan9:*:*) - # "uname -m" is not consistent, so use $cputype instead. 386 - # is converted to i386 for consistency with other x86 - # operating systems. - if test "${cputype-}" = 386; then - UNAME_MACHINE=i386 - elif test "x${cputype-}" != x; then - UNAME_MACHINE=$cputype - fi - GUESS=$UNAME_MACHINE-unknown-plan9 - ;; - *:TOPS-10:*:*) - GUESS=pdp10-unknown-tops10 - ;; - *:TENEX:*:*) - GUESS=pdp10-unknown-tenex - ;; - KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) - GUESS=pdp10-dec-tops20 - ;; - XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) - GUESS=pdp10-xkl-tops20 - ;; - *:TOPS-20:*:*) - GUESS=pdp10-unknown-tops20 - ;; - *:ITS:*:*) - GUESS=pdp10-unknown-its - ;; - SEI:*:*:SEIUX) - GUESS=mips-sei-seiux$UNAME_RELEASE - ;; - *:DragonFly:*:*) - DRAGONFLY_REL=`echo "$UNAME_RELEASE" | sed -e 's/[-(].*//'` - GUESS=$UNAME_MACHINE-unknown-dragonfly$DRAGONFLY_REL - ;; - *:*VMS:*:*) - UNAME_MACHINE=`(uname -p) 2>/dev/null` - case $UNAME_MACHINE in - A*) GUESS=alpha-dec-vms ;; - I*) GUESS=ia64-dec-vms ;; - V*) GUESS=vax-dec-vms ;; - esac ;; - *:XENIX:*:SysV) - GUESS=i386-pc-xenix - ;; - i*86:skyos:*:*) - SKYOS_REL=`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'` - GUESS=$UNAME_MACHINE-pc-skyos$SKYOS_REL - ;; - i*86:rdos:*:*) - GUESS=$UNAME_MACHINE-pc-rdos - ;; - *:AROS:*:*) - GUESS=$UNAME_MACHINE-unknown-aros - ;; - x86_64:VMkernel:*:*) - GUESS=$UNAME_MACHINE-unknown-esx - ;; - amd64:Isilon\ OneFS:*:*) - GUESS=x86_64-unknown-onefs - ;; - *:Unleashed:*:*) - GUESS=$UNAME_MACHINE-unknown-unleashed$UNAME_RELEASE - ;; -esac - -# Do we have a guess based on uname results? -if test "x$GUESS" != x; then - echo "$GUESS" - exit -fi - -# No uname command or uname output not recognized. -set_cc_for_build -cat > "$dummy.c" < -#include -#endif -#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) -#if defined (vax) || defined (__vax) || defined (__vax__) || defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) -#include -#if defined(_SIZE_T_) || defined(SIGLOST) -#include -#endif -#endif -#endif -main () -{ -#if defined (sony) -#if defined (MIPSEB) - /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, - I don't know.... */ - printf ("mips-sony-bsd\n"); exit (0); -#else -#include - printf ("m68k-sony-newsos%s\n", -#ifdef NEWSOS4 - "4" -#else - "" -#endif - ); exit (0); -#endif -#endif - -#if defined (NeXT) -#if !defined (__ARCHITECTURE__) -#define __ARCHITECTURE__ "m68k" -#endif - int version; - version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`; - if (version < 4) - printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); - else - printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); - exit (0); -#endif - -#if defined (MULTIMAX) || defined (n16) -#if defined (UMAXV) - printf ("ns32k-encore-sysv\n"); exit (0); -#else -#if defined (CMU) - printf ("ns32k-encore-mach\n"); exit (0); -#else - printf ("ns32k-encore-bsd\n"); exit (0); -#endif -#endif -#endif - -#if defined (__386BSD__) - printf ("i386-pc-bsd\n"); exit (0); -#endif - -#if defined (sequent) -#if defined (i386) - printf ("i386-sequent-dynix\n"); exit (0); -#endif -#if defined (ns32000) - printf ("ns32k-sequent-dynix\n"); exit (0); -#endif -#endif - -#if defined (_SEQUENT_) - struct utsname un; - - uname(&un); - if (strncmp(un.version, "V2", 2) == 0) { - printf ("i386-sequent-ptx2\n"); exit (0); - } - if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ - printf ("i386-sequent-ptx1\n"); exit (0); - } - printf ("i386-sequent-ptx\n"); exit (0); -#endif - -#if defined (vax) -#if !defined (ultrix) -#include -#if defined (BSD) -#if BSD == 43 - printf ("vax-dec-bsd4.3\n"); exit (0); -#else -#if BSD == 199006 - printf ("vax-dec-bsd4.3reno\n"); exit (0); -#else - printf ("vax-dec-bsd\n"); exit (0); -#endif -#endif -#else - printf ("vax-dec-bsd\n"); exit (0); -#endif -#else -#if defined(_SIZE_T_) || defined(SIGLOST) - struct utsname un; - uname (&un); - printf ("vax-dec-ultrix%s\n", un.release); exit (0); -#else - printf ("vax-dec-ultrix\n"); exit (0); -#endif -#endif -#endif -#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) -#if defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) -#if defined(_SIZE_T_) || defined(SIGLOST) - struct utsname *un; - uname (&un); - printf ("mips-dec-ultrix%s\n", un.release); exit (0); -#else - printf ("mips-dec-ultrix\n"); exit (0); -#endif -#endif -#endif - -#if defined (alliant) && defined (i860) - printf ("i860-alliant-bsd\n"); exit (0); -#endif - - exit (1); -} -EOF - -$CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null && SYSTEM_NAME=`"$dummy"` && - { echo "$SYSTEM_NAME"; exit; } - -# Apollos put the system type in the environment. -test -d /usr/apollo && { echo "$ISP-apollo-$SYSTYPE"; exit; } - -echo "$0: unable to guess system type" >&2 - -case $UNAME_MACHINE:$UNAME_SYSTEM in - mips:Linux | mips64:Linux) - # If we got here on MIPS GNU/Linux, output extra information. - cat >&2 <&2 <&2 </dev/null || echo unknown` -uname -r = `(uname -r) 2>/dev/null || echo unknown` -uname -s = `(uname -s) 2>/dev/null || echo unknown` -uname -v = `(uname -v) 2>/dev/null || echo unknown` - -/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` -/bin/uname -X = `(/bin/uname -X) 2>/dev/null` - -hostinfo = `(hostinfo) 2>/dev/null` -/bin/universe = `(/bin/universe) 2>/dev/null` -/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` -/bin/arch = `(/bin/arch) 2>/dev/null` -/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` -/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` - -UNAME_MACHINE = "$UNAME_MACHINE" -UNAME_RELEASE = "$UNAME_RELEASE" -UNAME_SYSTEM = "$UNAME_SYSTEM" -UNAME_VERSION = "$UNAME_VERSION" -EOF -fi - -exit 1 - -# Local variables: -# eval: (add-hook 'before-save-hook 'time-stamp) -# time-stamp-start: "timestamp='" -# time-stamp-format: "%:y-%02m-%02d" -# time-stamp-end: "'" -# End: DELETED autosetup/autosetup-config.sub Index: autosetup/autosetup-config.sub ================================================================== --- autosetup/autosetup-config.sub +++ autosetup/autosetup-config.sub @@ -1,1873 +0,0 @@ -#! /bin/sh -# Configuration validation subroutine script. -# Copyright 1992-2021 Free Software Foundation, Inc. - -# shellcheck disable=SC2006,SC2268 # see below for rationale - -timestamp='2021-07-03' - -# This file is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that -# program. This Exception is an additional permission under section 7 -# of the GNU General Public License, version 3 ("GPLv3"). - - -# Please send patches to . -# -# Configuration subroutine to validate and canonicalize a configuration type. -# Supply the specified configuration type as an argument. -# If it is invalid, we print an error message on stderr and exit with code 1. -# Otherwise, we print the canonical config type on stdout and succeed. - -# You can get the latest version of this script from: -# https://git.savannah.gnu.org/cgit/config.git/plain/config.sub - -# This file is supposed to be the same for all GNU packages -# and recognize all the CPU types, system types and aliases -# that are meaningful with *any* GNU software. -# Each package is responsible for reporting which valid configurations -# it does not support. The user should be able to distinguish -# a failure to support a valid configuration from a meaningless -# configuration. - -# The goal of this file is to map all the various variations of a given -# machine specification into a single specification in the form: -# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM -# or in some cases, the newer four-part form: -# CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM -# It is wrong to echo any other type of specification. - -# The "shellcheck disable" line above the timestamp inhibits complaints -# about features and limitations of the classic Bourne shell that were -# superseded or lifted in POSIX. However, this script identifies a wide -# variety of pre-POSIX systems that do not have POSIX shells at all, and -# even some reasonably current systems (Solaris 10 as case-in-point) still -# have a pre-POSIX /bin/sh. - -me=`echo "$0" | sed -e 's,.*/,,'` - -usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS - -Canonicalize a configuration name. - -Options: - -h, --help print this help, then exit - -t, --time-stamp print date of last modification, then exit - -v, --version print version number, then exit - -Report bugs and patches to ." - -version="\ -GNU config.sub ($timestamp) - -Copyright 1992-2021 Free Software Foundation, Inc. - -This is free software; see the source for copying conditions. There is NO -warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." - -help=" -Try \`$me --help' for more information." - -# Parse command line -while test $# -gt 0 ; do - case $1 in - --time-stamp | --time* | -t ) - echo "$timestamp" ; exit ;; - --version | -v ) - echo "$version" ; exit ;; - --help | --h* | -h ) - echo "$usage"; exit ;; - -- ) # Stop option processing - shift; break ;; - - ) # Use stdin as input. - break ;; - -* ) - echo "$me: invalid option $1$help" >&2 - exit 1 ;; - - *local*) - # First pass through any local machine types. - echo "$1" - exit ;; - - * ) - break ;; - esac -done - -case $# in - 0) echo "$me: missing argument$help" >&2 - exit 1;; - 1) ;; - *) echo "$me: too many arguments$help" >&2 - exit 1;; -esac - -# Split fields of configuration type -# shellcheck disable=SC2162 -IFS="-" read field1 field2 field3 field4 <&2 - exit 1 - ;; - *-*-*-*) - basic_machine=$field1-$field2 - basic_os=$field3-$field4 - ;; - *-*-*) - # Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two - # parts - maybe_os=$field2-$field3 - case $maybe_os in - nto-qnx* | linux-* | uclinux-uclibc* \ - | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ - | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ - | storm-chaos* | os2-emx* | rtmk-nova*) - basic_machine=$field1 - basic_os=$maybe_os - ;; - android-linux) - basic_machine=$field1-unknown - basic_os=linux-android - ;; - *) - basic_machine=$field1-$field2 - basic_os=$field3 - ;; - esac - ;; - *-*) - # A lone config we happen to match not fitting any pattern - case $field1-$field2 in - decstation-3100) - basic_machine=mips-dec - basic_os= - ;; - *-*) - # Second component is usually, but not always the OS - case $field2 in - # Prevent following clause from handling this valid os - sun*os*) - basic_machine=$field1 - basic_os=$field2 - ;; - # Manufacturers - dec* | mips* | sequent* | encore* | pc533* | sgi* | sony* \ - | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ - | unicom* | ibm* | next | hp | isi* | apollo | altos* \ - | convergent* | ncr* | news | 32* | 3600* | 3100* \ - | hitachi* | c[123]* | convex* | sun | crds | omron* | dg \ - | ultra | tti* | harris | dolphin | highlevel | gould \ - | cbm | ns | masscomp | apple | axis | knuth | cray \ - | microblaze* | sim | cisco \ - | oki | wec | wrs | winbond) - basic_machine=$field1-$field2 - basic_os= - ;; - *) - basic_machine=$field1 - basic_os=$field2 - ;; - esac - ;; - esac - ;; - *) - # Convert single-component short-hands not valid as part of - # multi-component configurations. - case $field1 in - 386bsd) - basic_machine=i386-pc - basic_os=bsd - ;; - a29khif) - basic_machine=a29k-amd - basic_os=udi - ;; - adobe68k) - basic_machine=m68010-adobe - basic_os=scout - ;; - alliant) - basic_machine=fx80-alliant - basic_os= - ;; - altos | altos3068) - basic_machine=m68k-altos - basic_os= - ;; - am29k) - basic_machine=a29k-none - basic_os=bsd - ;; - amdahl) - basic_machine=580-amdahl - basic_os=sysv - ;; - amiga) - basic_machine=m68k-unknown - basic_os= - ;; - amigaos | amigados) - basic_machine=m68k-unknown - basic_os=amigaos - ;; - amigaunix | amix) - basic_machine=m68k-unknown - basic_os=sysv4 - ;; - apollo68) - basic_machine=m68k-apollo - basic_os=sysv - ;; - apollo68bsd) - basic_machine=m68k-apollo - basic_os=bsd - ;; - aros) - basic_machine=i386-pc - basic_os=aros - ;; - aux) - basic_machine=m68k-apple - basic_os=aux - ;; - balance) - basic_machine=ns32k-sequent - basic_os=dynix - ;; - blackfin) - basic_machine=bfin-unknown - basic_os=linux - ;; - cegcc) - basic_machine=arm-unknown - basic_os=cegcc - ;; - convex-c1) - basic_machine=c1-convex - basic_os=bsd - ;; - convex-c2) - basic_machine=c2-convex - basic_os=bsd - ;; - convex-c32) - basic_machine=c32-convex - basic_os=bsd - ;; - convex-c34) - basic_machine=c34-convex - basic_os=bsd - ;; - convex-c38) - basic_machine=c38-convex - basic_os=bsd - ;; - cray) - basic_machine=j90-cray - basic_os=unicos - ;; - crds | unos) - basic_machine=m68k-crds - basic_os= - ;; - da30) - basic_machine=m68k-da30 - basic_os= - ;; - decstation | pmax | pmin | dec3100 | decstatn) - basic_machine=mips-dec - basic_os= - ;; - delta88) - basic_machine=m88k-motorola - basic_os=sysv3 - ;; - dicos) - basic_machine=i686-pc - basic_os=dicos - ;; - djgpp) - basic_machine=i586-pc - basic_os=msdosdjgpp - ;; - ebmon29k) - basic_machine=a29k-amd - basic_os=ebmon - ;; - es1800 | OSE68k | ose68k | ose | OSE) - basic_machine=m68k-ericsson - basic_os=ose - ;; - gmicro) - basic_machine=tron-gmicro - basic_os=sysv - ;; - go32) - basic_machine=i386-pc - basic_os=go32 - ;; - h8300hms) - basic_machine=h8300-hitachi - basic_os=hms - ;; - h8300xray) - basic_machine=h8300-hitachi - basic_os=xray - ;; - h8500hms) - basic_machine=h8500-hitachi - basic_os=hms - ;; - harris) - basic_machine=m88k-harris - basic_os=sysv3 - ;; - hp300 | hp300hpux) - basic_machine=m68k-hp - basic_os=hpux - ;; - hp300bsd) - basic_machine=m68k-hp - basic_os=bsd - ;; - hppaosf) - basic_machine=hppa1.1-hp - basic_os=osf - ;; - hppro) - basic_machine=hppa1.1-hp - basic_os=proelf - ;; - i386mach) - basic_machine=i386-mach - basic_os=mach - ;; - isi68 | isi) - basic_machine=m68k-isi - basic_os=sysv - ;; - m68knommu) - basic_machine=m68k-unknown - basic_os=linux - ;; - magnum | m3230) - basic_machine=mips-mips - basic_os=sysv - ;; - merlin) - basic_machine=ns32k-utek - basic_os=sysv - ;; - mingw64) - basic_machine=x86_64-pc - basic_os=mingw64 - ;; - mingw32) - basic_machine=i686-pc - basic_os=mingw32 - ;; - mingw32ce) - basic_machine=arm-unknown - basic_os=mingw32ce - ;; - monitor) - basic_machine=m68k-rom68k - basic_os=coff - ;; - morphos) - basic_machine=powerpc-unknown - basic_os=morphos - ;; - moxiebox) - basic_machine=moxie-unknown - basic_os=moxiebox - ;; - msdos) - basic_machine=i386-pc - basic_os=msdos - ;; - msys) - basic_machine=i686-pc - basic_os=msys - ;; - mvs) - basic_machine=i370-ibm - basic_os=mvs - ;; - nacl) - basic_machine=le32-unknown - basic_os=nacl - ;; - ncr3000) - basic_machine=i486-ncr - basic_os=sysv4 - ;; - netbsd386) - basic_machine=i386-pc - basic_os=netbsd - ;; - netwinder) - basic_machine=armv4l-rebel - basic_os=linux - ;; - news | news700 | news800 | news900) - basic_machine=m68k-sony - basic_os=newsos - ;; - news1000) - basic_machine=m68030-sony - basic_os=newsos - ;; - necv70) - basic_machine=v70-nec - basic_os=sysv - ;; - nh3000) - basic_machine=m68k-harris - basic_os=cxux - ;; - nh[45]000) - basic_machine=m88k-harris - basic_os=cxux - ;; - nindy960) - basic_machine=i960-intel - basic_os=nindy - ;; - mon960) - basic_machine=i960-intel - basic_os=mon960 - ;; - nonstopux) - basic_machine=mips-compaq - basic_os=nonstopux - ;; - os400) - basic_machine=powerpc-ibm - basic_os=os400 - ;; - OSE68000 | ose68000) - basic_machine=m68000-ericsson - basic_os=ose - ;; - os68k) - basic_machine=m68k-none - basic_os=os68k - ;; - paragon) - basic_machine=i860-intel - basic_os=osf - ;; - parisc) - basic_machine=hppa-unknown - basic_os=linux - ;; - psp) - basic_machine=mipsallegrexel-sony - basic_os=psp - ;; - pw32) - basic_machine=i586-unknown - basic_os=pw32 - ;; - rdos | rdos64) - basic_machine=x86_64-pc - basic_os=rdos - ;; - rdos32) - basic_machine=i386-pc - basic_os=rdos - ;; - rom68k) - basic_machine=m68k-rom68k - basic_os=coff - ;; - sa29200) - basic_machine=a29k-amd - basic_os=udi - ;; - sei) - basic_machine=mips-sei - basic_os=seiux - ;; - sequent) - basic_machine=i386-sequent - basic_os= - ;; - sps7) - basic_machine=m68k-bull - basic_os=sysv2 - ;; - st2000) - basic_machine=m68k-tandem - basic_os= - ;; - stratus) - basic_machine=i860-stratus - basic_os=sysv4 - ;; - sun2) - basic_machine=m68000-sun - basic_os= - ;; - sun2os3) - basic_machine=m68000-sun - basic_os=sunos3 - ;; - sun2os4) - basic_machine=m68000-sun - basic_os=sunos4 - ;; - sun3) - basic_machine=m68k-sun - basic_os= - ;; - sun3os3) - basic_machine=m68k-sun - basic_os=sunos3 - ;; - sun3os4) - basic_machine=m68k-sun - basic_os=sunos4 - ;; - sun4) - basic_machine=sparc-sun - basic_os= - ;; - sun4os3) - basic_machine=sparc-sun - basic_os=sunos3 - ;; - sun4os4) - basic_machine=sparc-sun - basic_os=sunos4 - ;; - sun4sol2) - basic_machine=sparc-sun - basic_os=solaris2 - ;; - sun386 | sun386i | roadrunner) - basic_machine=i386-sun - basic_os= - ;; - sv1) - basic_machine=sv1-cray - basic_os=unicos - ;; - symmetry) - basic_machine=i386-sequent - basic_os=dynix - ;; - t3e) - basic_machine=alphaev5-cray - basic_os=unicos - ;; - t90) - basic_machine=t90-cray - basic_os=unicos - ;; - toad1) - basic_machine=pdp10-xkl - basic_os=tops20 - ;; - tpf) - basic_machine=s390x-ibm - basic_os=tpf - ;; - udi29k) - basic_machine=a29k-amd - basic_os=udi - ;; - ultra3) - basic_machine=a29k-nyu - basic_os=sym1 - ;; - v810 | necv810) - basic_machine=v810-nec - basic_os=none - ;; - vaxv) - basic_machine=vax-dec - basic_os=sysv - ;; - vms) - basic_machine=vax-dec - basic_os=vms - ;; - vsta) - basic_machine=i386-pc - basic_os=vsta - ;; - vxworks960) - basic_machine=i960-wrs - basic_os=vxworks - ;; - vxworks68) - basic_machine=m68k-wrs - basic_os=vxworks - ;; - vxworks29k) - basic_machine=a29k-wrs - basic_os=vxworks - ;; - xbox) - basic_machine=i686-pc - basic_os=mingw32 - ;; - ymp) - basic_machine=ymp-cray - basic_os=unicos - ;; - *) - basic_machine=$1 - basic_os= - ;; - esac - ;; -esac - -# Decode 1-component or ad-hoc basic machines -case $basic_machine in - # Here we handle the default manufacturer of certain CPU types. It is in - # some cases the only manufacturer, in others, it is the most popular. - w89k) - cpu=hppa1.1 - vendor=winbond - ;; - op50n) - cpu=hppa1.1 - vendor=oki - ;; - op60c) - cpu=hppa1.1 - vendor=oki - ;; - ibm*) - cpu=i370 - vendor=ibm - ;; - orion105) - cpu=clipper - vendor=highlevel - ;; - mac | mpw | mac-mpw) - cpu=m68k - vendor=apple - ;; - pmac | pmac-mpw) - cpu=powerpc - vendor=apple - ;; - - # Recognize the various machine names and aliases which stand - # for a CPU type and a company and sometimes even an OS. - 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) - cpu=m68000 - vendor=att - ;; - 3b*) - cpu=we32k - vendor=att - ;; - bluegene*) - cpu=powerpc - vendor=ibm - basic_os=cnk - ;; - decsystem10* | dec10*) - cpu=pdp10 - vendor=dec - basic_os=tops10 - ;; - decsystem20* | dec20*) - cpu=pdp10 - vendor=dec - basic_os=tops20 - ;; - delta | 3300 | motorola-3300 | motorola-delta \ - | 3300-motorola | delta-motorola) - cpu=m68k - vendor=motorola - ;; - dpx2*) - cpu=m68k - vendor=bull - basic_os=sysv3 - ;; - encore | umax | mmax) - cpu=ns32k - vendor=encore - ;; - elxsi) - cpu=elxsi - vendor=elxsi - basic_os=${basic_os:-bsd} - ;; - fx2800) - cpu=i860 - vendor=alliant - ;; - genix) - cpu=ns32k - vendor=ns - ;; - h3050r* | hiux*) - cpu=hppa1.1 - vendor=hitachi - basic_os=hiuxwe2 - ;; - hp3k9[0-9][0-9] | hp9[0-9][0-9]) - cpu=hppa1.0 - vendor=hp - ;; - hp9k2[0-9][0-9] | hp9k31[0-9]) - cpu=m68000 - vendor=hp - ;; - hp9k3[2-9][0-9]) - cpu=m68k - vendor=hp - ;; - hp9k6[0-9][0-9] | hp6[0-9][0-9]) - cpu=hppa1.0 - vendor=hp - ;; - hp9k7[0-79][0-9] | hp7[0-79][0-9]) - cpu=hppa1.1 - vendor=hp - ;; - hp9k78[0-9] | hp78[0-9]) - # FIXME: really hppa2.0-hp - cpu=hppa1.1 - vendor=hp - ;; - hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) - # FIXME: really hppa2.0-hp - cpu=hppa1.1 - vendor=hp - ;; - hp9k8[0-9][13679] | hp8[0-9][13679]) - cpu=hppa1.1 - vendor=hp - ;; - hp9k8[0-9][0-9] | hp8[0-9][0-9]) - cpu=hppa1.0 - vendor=hp - ;; - i*86v32) - cpu=`echo "$1" | sed -e 's/86.*/86/'` - vendor=pc - basic_os=sysv32 - ;; - i*86v4*) - cpu=`echo "$1" | sed -e 's/86.*/86/'` - vendor=pc - basic_os=sysv4 - ;; - i*86v) - cpu=`echo "$1" | sed -e 's/86.*/86/'` - vendor=pc - basic_os=sysv - ;; - i*86sol2) - cpu=`echo "$1" | sed -e 's/86.*/86/'` - vendor=pc - basic_os=solaris2 - ;; - j90 | j90-cray) - cpu=j90 - vendor=cray - basic_os=${basic_os:-unicos} - ;; - iris | iris4d) - cpu=mips - vendor=sgi - case $basic_os in - irix*) - ;; - *) - basic_os=irix4 - ;; - esac - ;; - miniframe) - cpu=m68000 - vendor=convergent - ;; - *mint | mint[0-9]* | *MiNT | *MiNT[0-9]*) - cpu=m68k - vendor=atari - basic_os=mint - ;; - news-3600 | risc-news) - cpu=mips - vendor=sony - basic_os=newsos - ;; - next | m*-next) - cpu=m68k - vendor=next - case $basic_os in - openstep*) - ;; - nextstep*) - ;; - ns2*) - basic_os=nextstep2 - ;; - *) - basic_os=nextstep3 - ;; - esac - ;; - np1) - cpu=np1 - vendor=gould - ;; - op50n-* | op60c-*) - cpu=hppa1.1 - vendor=oki - basic_os=proelf - ;; - pa-hitachi) - cpu=hppa1.1 - vendor=hitachi - basic_os=hiuxwe2 - ;; - pbd) - cpu=sparc - vendor=tti - ;; - pbb) - cpu=m68k - vendor=tti - ;; - pc532) - cpu=ns32k - vendor=pc532 - ;; - pn) - cpu=pn - vendor=gould - ;; - power) - cpu=power - vendor=ibm - ;; - ps2) - cpu=i386 - vendor=ibm - ;; - rm[46]00) - cpu=mips - vendor=siemens - ;; - rtpc | rtpc-*) - cpu=romp - vendor=ibm - ;; - sde) - cpu=mipsisa32 - vendor=sde - basic_os=${basic_os:-elf} - ;; - simso-wrs) - cpu=sparclite - vendor=wrs - basic_os=vxworks - ;; - tower | tower-32) - cpu=m68k - vendor=ncr - ;; - vpp*|vx|vx-*) - cpu=f301 - vendor=fujitsu - ;; - w65) - cpu=w65 - vendor=wdc - ;; - w89k-*) - cpu=hppa1.1 - vendor=winbond - basic_os=proelf - ;; - none) - cpu=none - vendor=none - ;; - leon|leon[3-9]) - cpu=sparc - vendor=$basic_machine - ;; - leon-*|leon[3-9]-*) - cpu=sparc - vendor=`echo "$basic_machine" | sed 's/-.*//'` - ;; - - *-*) - # shellcheck disable=SC2162 - IFS="-" read cpu vendor <&2 - exit 1 - ;; - esac - ;; -esac - -# Here we canonicalize certain aliases for manufacturers. -case $vendor in - digital*) - vendor=dec - ;; - commodore*) - vendor=cbm - ;; - *) - ;; -esac - -# Decode manufacturer-specific aliases for certain operating systems. - -if test x$basic_os != x -then - -# First recognize some ad-hoc caes, or perhaps split kernel-os, or else just -# set os. -case $basic_os in - gnu/linux*) - kernel=linux - os=`echo "$basic_os" | sed -e 's|gnu/linux|gnu|'` - ;; - os2-emx) - kernel=os2 - os=`echo "$basic_os" | sed -e 's|os2-emx|emx|'` - ;; - nto-qnx*) - kernel=nto - os=`echo "$basic_os" | sed -e 's|nto-qnx|qnx|'` - ;; - *-*) - # shellcheck disable=SC2162 - IFS="-" read kernel os <&2 - exit 1 - ;; -esac - -# As a final step for OS-related things, validate the OS-kernel combination -# (given a valid OS), if there is a kernel. -case $kernel-$os in - linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* | linux-musl* | linux-uclibc* ) - ;; - uclinux-uclibc* ) - ;; - -dietlibc* | -newlib* | -musl* | -uclibc* ) - # These are just libc implementations, not actual OSes, and thus - # require a kernel. - echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2 - exit 1 - ;; - kfreebsd*-gnu* | kopensolaris*-gnu*) - ;; - vxworks-simlinux | vxworks-simwindows | vxworks-spe) - ;; - nto-qnx*) - ;; - os2-emx) - ;; - *-eabi* | *-gnueabi*) - ;; - -*) - # Blank kernel with real OS is always fine. - ;; - *-*) - echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2 - exit 1 - ;; -esac - -# Here we handle the case where we know the os, and the CPU type, but not the -# manufacturer. We pick the logical manufacturer. -case $vendor in - unknown) - case $cpu-$os in - *-riscix*) - vendor=acorn - ;; - *-sunos*) - vendor=sun - ;; - *-cnk* | *-aix*) - vendor=ibm - ;; - *-beos*) - vendor=be - ;; - *-hpux*) - vendor=hp - ;; - *-mpeix*) - vendor=hp - ;; - *-hiux*) - vendor=hitachi - ;; - *-unos*) - vendor=crds - ;; - *-dgux*) - vendor=dg - ;; - *-luna*) - vendor=omron - ;; - *-genix*) - vendor=ns - ;; - *-clix*) - vendor=intergraph - ;; - *-mvs* | *-opened*) - vendor=ibm - ;; - *-os400*) - vendor=ibm - ;; - s390-* | s390x-*) - vendor=ibm - ;; - *-ptx*) - vendor=sequent - ;; - *-tpf*) - vendor=ibm - ;; - *-vxsim* | *-vxworks* | *-windiss*) - vendor=wrs - ;; - *-aux*) - vendor=apple - ;; - *-hms*) - vendor=hitachi - ;; - *-mpw* | *-macos*) - vendor=apple - ;; - *-*mint | *-mint[0-9]* | *-*MiNT | *-MiNT[0-9]*) - vendor=atari - ;; - *-vos*) - vendor=stratus - ;; - esac - ;; -esac - -echo "$cpu-$vendor-${kernel:+$kernel-}$os" -exit - -# Local variables: -# eval: (add-hook 'before-save-hook 'time-stamp) -# time-stamp-start: "timestamp='" -# time-stamp-format: "%:y-%02m-%02d" -# time-stamp-end: "'" -# End: DELETED autosetup/autosetup-find-tclsh Index: autosetup/autosetup-find-tclsh ================================================================== --- autosetup/autosetup-find-tclsh +++ autosetup/autosetup-find-tclsh @@ -1,15 +0,0 @@ -#!/bin/sh -# Looks for a suitable tclsh or jimsh in the PATH -# If not found, builds a bootstrap jimsh in current dir from source -# Prefer $autosetup_tclsh if is set in the environment (unless ./jimsh0 works) -d="`dirname "$0"`" -for tclsh in ./jimsh0 $autosetup_tclsh jimsh tclsh tclsh8.5 tclsh8.6 tclsh8.7; do - { $tclsh "$d/autosetup-test-tclsh"; } 2>/dev/null && exit 0 -done -echo 1>&2 "No installed jimsh or tclsh, building local bootstrap jimsh0" -for cc in ${CC_FOR_BUILD:-cc} gcc; do - { $cc -o jimsh0 "$d/jimsh0.c"; } 2>/dev/null || continue - ./jimsh0 "$d/autosetup-test-tclsh" && exit 0 -done -echo 1>&2 "No working C compiler found. Tried ${CC_FOR_BUILD:-cc} and gcc." -echo false DELETED autosetup/autosetup-test-tclsh Index: autosetup/autosetup-test-tclsh ================================================================== --- autosetup/autosetup-test-tclsh +++ autosetup/autosetup-test-tclsh @@ -1,20 +0,0 @@ -# A small Tcl script to verify that the chosen -# interpreter works. Sometimes we might e.g. pick up -# an interpreter for a different arch. -# Outputs the full path to the interpreter - -if {[catch {info version} version] == 0} { - # This is Jim Tcl - if {$version >= 0.72} { - # Ensure that regexp works - regexp (a.*?) a - puts [info nameofexecutable] - exit 0 - } -} elseif {[catch {info tclversion} version] == 0} { - if {$version >= 8.5 && ![string match 8.5a* [info patchlevel]]} { - puts [info nameofexecutable] - exit 0 - } -} -exit 1 DELETED autosetup/cc-db.tcl Index: autosetup/cc-db.tcl ================================================================== --- autosetup/cc-db.tcl +++ autosetup/cc-db.tcl @@ -1,15 +0,0 @@ -# Copyright (c) 2011 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# @synopsis: -# -# The 'cc-db' module provides a knowledge-base of system idiosyncrasies. -# In general, this module can always be included. - -use cc - -options {} - -# openbsd needs sys/types.h to detect some system headers -cc-include-needs sys/socket.h sys/types.h -cc-include-needs netinet/in.h sys/types.h DELETED autosetup/cc-lib.tcl Index: autosetup/cc-lib.tcl ================================================================== --- autosetup/cc-lib.tcl +++ autosetup/cc-lib.tcl @@ -1,187 +0,0 @@ -# Copyright (c) 2011 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# @synopsis: -# -# Provides a library of common tests on top of the 'cc' module. - -use cc - -# @cc-check-lfs -# -# The equivalent of the 'AC_SYS_LARGEFILE' macro. -# -# defines 'HAVE_LFS' if LFS is available, -# and defines '_FILE_OFFSET_BITS=64' if necessary -# -# Returns 1 if 'LFS' is available or 0 otherwise -# -proc cc-check-lfs {} { - cc-check-includes sys/types.h - msg-checking "Checking if -D_FILE_OFFSET_BITS=64 is needed..." - set lfs 1 - if {[msg-quiet cc-with {-includes sys/types.h} {cc-check-sizeof off_t}] == 8} { - msg-result no - } elseif {[msg-quiet cc-with {-includes sys/types.h -cflags -D_FILE_OFFSET_BITS=64} {cc-check-sizeof off_t}] == 8} { - define _FILE_OFFSET_BITS 64 - msg-result yes - } else { - set lfs 0 - msg-result none - } - define-feature lfs $lfs - return $lfs -} - -# @cc-check-endian -# -# The equivalent of the 'AC_C_BIGENDIAN' macro. -# -# defines 'HAVE_BIG_ENDIAN' if endian is known to be big, -# or 'HAVE_LITTLE_ENDIAN' if endian is known to be little. -# -# Returns 1 if determined, or 0 if not. -# -proc cc-check-endian {} { - cc-check-includes sys/types.h sys/param.h - set rc 0 - msg-checking "Checking endian..." - cc-with {-includes {sys/types.h sys/param.h}} { - if {[cctest -code { - #if !defined(BIG_ENDIAN) || !defined(BYTE_ORDER) - #error unknown - #elif BYTE_ORDER != BIG_ENDIAN - #error little - #endif - }]} { - define-feature big-endian - msg-result "big" - set rc 1 - } elseif {[cctest -code { - #if !defined(LITTLE_ENDIAN) || !defined(BYTE_ORDER) - #error unknown - #elif BYTE_ORDER != LITTLE_ENDIAN - #error big - #endif - }]} { - define-feature little-endian - msg-result "little" - set rc 1 - } else { - msg-result "unknown" - } - } - return $rc -} - -# @cc-check-flags flag ?...? -# -# Checks whether the given C/C++ compiler flags can be used. Defines feature -# names prefixed with 'HAVE_CFLAG' and 'HAVE_CXXFLAG' respectively, and -# appends working flags to '-cflags' and 'CFLAGS' or 'CXXFLAGS'. -proc cc-check-flags {args} { - set result 1 - array set opts [cc-get-settings] - switch -exact -- $opts(-lang) { - c++ { - set lang C++ - set prefix CXXFLAG - } - c { - set lang C - set prefix CFLAG - } - default { - autosetup-error "cc-check-flags failed with unknown language: $opts(-lang)" - } - } - foreach flag $args { - msg-checking "Checking whether the $lang compiler accepts $flag..." - if {[cctest -cflags $flag]} { - msg-result yes - define-feature $prefix$flag - cc-with [list -cflags [list $flag]] - define-append ${prefix}S $flag - } else { - msg-result no - set result 0 - } - } - return $result -} - -# @cc-check-standards ver ?...? -# -# Checks whether the C/C++ compiler accepts one of the specified '-std=$ver' -# options, and appends the first working one to '-cflags' and 'CFLAGS' or -# 'CXXFLAGS'. -proc cc-check-standards {args} { - array set opts [cc-get-settings] - foreach std $args { - if {[cc-check-flags -std=$std]} { - return $std - } - } - return "" -} - -# Checks whether $keyword is usable as alignof -proc cctest_alignof {keyword} { - msg-checking "Checking for $keyword..." - if {[cctest -code "int x = ${keyword}(char), y = ${keyword}('x');"]} then { - msg-result ok - define-feature $keyword - } else { - msg-result "not found" - } -} - -# @cc-check-c11 -# -# Checks for several C11/C++11 extensions and their alternatives. Currently -# checks for '_Static_assert', '_Alignof', '__alignof__', '__alignof'. -proc cc-check-c11 {} { - msg-checking "Checking for _Static_assert..." - if {[cctest -code { - _Static_assert(1, "static assertions are available"); - }]} then { - msg-result ok - define-feature _Static_assert - } else { - msg-result "not found" - } - - cctest_alignof _Alignof - cctest_alignof __alignof__ - cctest_alignof __alignof -} - -# @cc-check-alloca -# -# The equivalent of the 'AC_FUNC_ALLOCA' macro. -# -# Checks for the existence of 'alloca' -# defines 'HAVE_ALLOCA' and returns 1 if it exists. -proc cc-check-alloca {} { - cc-check-some-feature alloca { - cctest -includes alloca.h -code { alloca (2 * sizeof (int)); } - } -} - -# @cc-signal-return-type -# -# The equivalent of the 'AC_TYPE_SIGNAL' macro. -# -# defines 'RETSIGTYPE' to 'int' or 'void'. -proc cc-signal-return-type {} { - msg-checking "Checking return type of signal handlers..." - cc-with {-includes {sys/types.h signal.h}} { - if {[cctest -code {return *(signal (0, 0)) (0) == 1;}]} { - set type int - } else { - set type void - } - define RETSIGTYPE $type - msg-result $type - } -} DELETED autosetup/cc-shared.tcl Index: autosetup/cc-shared.tcl ================================================================== --- autosetup/cc-shared.tcl +++ autosetup/cc-shared.tcl @@ -1,113 +0,0 @@ -# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# @synopsis: -# -# The 'cc-shared' module provides support for shared libraries and shared objects. -# It defines the following variables: -# -## SH_CFLAGS Flags to use compiling sources destined for a shared library -## SH_LDFLAGS Flags to use linking (creating) a shared library -## SH_SOPREFIX Prefix to use to set the soname when creating a shared library -## SH_SOFULLPATH Set to 1 if the shared library soname should include the full install path -## SH_SOEXT Extension for shared libs -## SH_SOEXTVER Format for versioned shared libs - %s = version -## SHOBJ_CFLAGS Flags to use compiling sources destined for a shared object -## SHOBJ_LDFLAGS Flags to use linking a shared object, undefined symbols allowed -## SHOBJ_LDFLAGS_R - as above, but all symbols must be resolved -## SH_LINKRPATH Format for setting the rpath when linking an executable, %s = path -## SH_LINKFLAGS Flags to use linking an executable which will load shared objects -## LD_LIBRARY_PATH Environment variable which specifies path to shared libraries -## STRIPLIBFLAGS Arguments to strip a dynamic library - -options {} - -# Defaults: gcc on unix -define SHOBJ_CFLAGS -fPIC -define SHOBJ_LDFLAGS -shared -define SH_CFLAGS -fPIC -define SH_LDFLAGS -shared -define SH_LINKFLAGS -rdynamic -define SH_LINKRPATH "-Wl,-rpath -Wl,%s" -define SH_SOEXT .so -define SH_SOEXTVER .so.%s -define SH_SOPREFIX -Wl,-soname, -define LD_LIBRARY_PATH LD_LIBRARY_PATH -define STRIPLIBFLAGS --strip-unneeded - -# Note: This is a helpful reference for identifying the toolchain -# http://sourceforge.net/apps/mediawiki/predef/index.php?title=Compilers - -switch -glob -- [get-define host] { - *-*-darwin* { - define SHOBJ_CFLAGS "-dynamic -fno-common" - define SHOBJ_LDFLAGS "-bundle -undefined dynamic_lookup" - define SHOBJ_LDFLAGS_R -bundle - define SH_CFLAGS -dynamic - define SH_LDFLAGS -dynamiclib - define SH_LINKFLAGS "" - define SH_SOEXT .dylib - define SH_SOEXTVER .%s.dylib - define SH_SOPREFIX -Wl,-install_name, - define SH_SOFULLPATH - define LD_LIBRARY_PATH DYLD_LIBRARY_PATH - define STRIPLIBFLAGS -x - } - *-*-ming* - *-*-cygwin - *-*-msys { - define SHOBJ_CFLAGS "" - define SHOBJ_LDFLAGS -shared - define SH_CFLAGS "" - define SH_LDFLAGS -shared - define SH_LINKRPATH "" - define SH_LINKFLAGS "" - define SH_SOEXT .dll - define SH_SOEXTVER .dll - define SH_SOPREFIX "" - define LD_LIBRARY_PATH PATH - } - sparc* { - if {[msg-quiet cc-check-decls __SUNPRO_C]} { - msg-result "Found sun stdio compiler" - # sun stdio compiler - # XXX: These haven't been fully tested. - define SHOBJ_CFLAGS -KPIC - define SHOBJ_LDFLAGS "-G" - define SH_CFLAGS -KPIC - define SH_LINKFLAGS -Wl,-export-dynamic - define SH_SOPREFIX -Wl,-h, - } - } - *-*-solaris* { - if {[msg-quiet cc-check-decls __SUNPRO_C]} { - msg-result "Found sun stdio compiler" - # sun stdio compiler - # XXX: These haven't been fully tested. - define SHOBJ_CFLAGS -KPIC - define SHOBJ_LDFLAGS "-G" - define SH_CFLAGS -KPIC - define SH_LINKFLAGS -Wl,-export-dynamic - define SH_SOPREFIX -Wl,-h, - } - } - *-*-hpux { - # XXX: These haven't been tested - define SHOBJ_CFLAGS "+O3 +z" - define SHOBJ_LDFLAGS -b - define SH_CFLAGS +z - define SH_LINKFLAGS -Wl,+s - define LD_LIBRARY_PATH SHLIB_PATH - } - *-*-haiku { - define SHOBJ_CFLAGS "" - define SHOBJ_LDFLAGS -shared - define SH_CFLAGS "" - define SH_LDFLAGS -shared - define SH_LINKFLAGS "" - define SH_SOPREFIX "" - define LD_LIBRARY_PATH LIBRARY_PATH - } -} - -if {![is-defined SHOBJ_LDFLAGS_R]} { - define SHOBJ_LDFLAGS_R [get-define SHOBJ_LDFLAGS] -} DELETED autosetup/cc.tcl Index: autosetup/cc.tcl ================================================================== --- autosetup/cc.tcl +++ autosetup/cc.tcl @@ -1,733 +0,0 @@ -# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# @synopsis: -# -# The 'cc' module supports checking various 'features' of the C or C++ -# compiler/linker environment. Common commands are 'cc-check-includes', -# 'cc-check-types', 'cc-check-functions', 'cc-with', 'make-config-header' and 'make-template'. -# -# The following environment variables are used if set: -# -## CC - C compiler -## CXX - C++ compiler -## CPP - C preprocessor -## CCACHE - Set to "none" to disable automatic use of ccache -## CFLAGS - Additional C compiler flags -## CXXFLAGS - Additional C++ compiler flags -## LDFLAGS - Additional compiler flags during linking -## LIBS - Additional libraries to use (for all tests) -## CROSS - Tool prefix for cross compilation -# -# The following variables are defined from the corresponding -# environment variables if set. -# -## CPPFLAGS -## LINKFLAGS -## CC_FOR_BUILD -## LD - -use system - -options {} - -# Checks for the existence of the given function by linking -# -proc cctest_function {function} { - cctest -link 1 -declare "extern void $function\(void);" -code "$function\();" -} - -# Checks for the existence of the given type by compiling -proc cctest_type {type} { - cctest -code "$type _x;" -} - -# Checks for the existence of the given type/structure member. -# e.g. "struct stat.st_mtime" -proc cctest_member {struct_member} { - # split at the first dot - regexp {^([^.]+)[.](.*)$} $struct_member -> struct member - cctest -code "static $struct _s; return sizeof(_s.$member);" -} - -# Checks for the existence of the given define by compiling -# -proc cctest_define {name} { - cctest -code "#ifndef $name\n#error not defined\n#endif" -} - -# Checks for the existence of the given name either as -# a macro (#define) or an rvalue (such as an enum) -# -proc cctest_decl {name} { - cctest -code "#ifndef $name\n(void)$name;\n#endif" -} - -# @cc-check-sizeof type ... -# -# Checks the size of the given types (between 1 and 32, inclusive). -# Defines a variable with the size determined, or 'unknown' otherwise. -# e.g. for type 'long long', defines 'SIZEOF_LONG_LONG'. -# Returns the size of the last type. -# -proc cc-check-sizeof {args} { - foreach type $args { - msg-checking "Checking for sizeof $type..." - set size unknown - # Try the most common sizes first - foreach i {4 8 1 2 16 32} { - if {[cctest -code "static int _x\[sizeof($type) == $i ? 1 : -1\] = { 1 };"]} { - set size $i - break - } - } - msg-result $size - set define [feature-define-name $type SIZEOF_] - define $define $size - } - # Return the last result - get-define $define -} - -# Checks for each feature in $list by using the given script. -# -# When the script is evaluated, $each is set to the feature -# being checked, and $extra is set to any additional cctest args. -# -# Returns 1 if all features were found, or 0 otherwise. -proc cc-check-some-feature {list script} { - set ret 1 - foreach each $list { - if {![check-feature $each $script]} { - set ret 0 - } - } - return $ret -} - -# @cc-check-includes includes ... -# -# Checks that the given include files can be used. -proc cc-check-includes {args} { - cc-check-some-feature $args { - set with {} - if {[dict exists $::autosetup(cc-include-deps) $each]} { - set deps [dict keys [dict get $::autosetup(cc-include-deps) $each]] - msg-quiet cc-check-includes {*}$deps - foreach i $deps { - if {[have-feature $i]} { - lappend with $i - } - } - } - if {[llength $with]} { - cc-with [list -includes $with] { - cctest -includes $each - } - } else { - cctest -includes $each - } - } -} - -# @cc-include-needs include required ... -# -# Ensures that when checking for '$include', a check is first -# made for each '$required' file, and if found, it is included with '#include'. -proc cc-include-needs {file args} { - foreach depfile $args { - dict set ::autosetup(cc-include-deps) $file $depfile 1 - } -} - -# @cc-check-types type ... -# -# Checks that the types exist. -proc cc-check-types {args} { - cc-check-some-feature $args { - cctest_type $each - } -} - -# @cc-check-defines define ... -# -# Checks that the given preprocessor symbols are defined. -proc cc-check-defines {args} { - cc-check-some-feature $args { - cctest_define $each - } -} - -# @cc-check-decls name ... -# -# Checks that each given name is either a preprocessor symbol or rvalue -# such as an enum. Note that the define used is 'HAVE_DECL_xxx' -# rather than 'HAVE_xxx'. -proc cc-check-decls {args} { - set ret 1 - foreach name $args { - msg-checking "Checking for $name..." - set r [cctest_decl $name] - define-feature "decl $name" $r - if {$r} { - msg-result "ok" - } else { - msg-result "not found" - set ret 0 - } - } - return $ret -} - -# @cc-check-functions function ... -# -# Checks that the given functions exist (can be linked). -proc cc-check-functions {args} { - cc-check-some-feature $args { - cctest_function $each - } -} - -# @cc-check-members type.member ... -# -# Checks that the given type/structure members exist. -# A structure member is of the form 'struct stat.st_mtime'. -proc cc-check-members {args} { - cc-check-some-feature $args { - cctest_member $each - } -} - -# @cc-check-function-in-lib function libs ?otherlibs? -# -# Checks that the given function can be found in one of the libs. -# -# First checks for no library required, then checks each of the libraries -# in turn. -# -# If the function is found, the feature is defined and 'lib_$function' is defined -# to '-l$lib' where the function was found, or "" if no library required. -# In addition, '-l$lib' is prepended to the 'LIBS' define. -# -# If additional libraries may be needed for linking, they should be specified -# with '$extralibs' as '-lotherlib1 -lotherlib2'. -# These libraries are not automatically added to 'LIBS'. -# -# Returns 1 if found or 0 if not. -# -proc cc-check-function-in-lib {function libs {otherlibs {}}} { - msg-checking "Checking libs for $function..." - set found 0 - cc-with [list -libs $otherlibs] { - if {[cctest_function $function]} { - msg-result "none needed" - define lib_$function "" - incr found - } else { - foreach lib $libs { - cc-with [list -libs -l$lib] { - if {[cctest_function $function]} { - msg-result -l$lib - define lib_$function -l$lib - # prepend to LIBS - define LIBS "-l$lib [get-define LIBS]" - incr found - break - } - } - } - } - } - define-feature $function $found - if {!$found} { - msg-result "no" - } - return $found -} - -# @cc-check-tools tool ... -# -# Checks for existence of the given compiler tools, taking -# into account any cross compilation prefix. -# -# For example, when checking for 'ar', first 'AR' is checked on the command -# line and then in the environment. If not found, '${host}-ar' or -# simply 'ar' is assumed depending upon whether cross compiling. -# The path is searched for this executable, and if found 'AR' is defined -# to the executable name. -# Note that even when cross compiling, the simple 'ar' is used as a fallback, -# but a warning is generated. This is necessary for some toolchains. -# -# It is an error if the executable is not found. -# -proc cc-check-tools {args} { - foreach tool $args { - set TOOL [string toupper $tool] - set exe [get-env $TOOL [get-define cross]$tool] - if {[find-executable {*}$exe]} { - define $TOOL $exe - continue - } - if {[find-executable {*}$tool]} { - msg-result "Warning: Failed to find $exe, falling back to $tool which may be incorrect" - define $TOOL $tool - continue - } - user-error "Failed to find $exe" - } -} - -# @cc-check-progs prog ... -# -# Checks for existence of the given executables on the path. -# -# For example, when checking for 'grep', the path is searched for -# the executable, 'grep', and if found 'GREP' is defined as 'grep'. -# -# If the executable is not found, the variable is defined as 'false'. -# Returns 1 if all programs were found, or 0 otherwise. -# -proc cc-check-progs {args} { - set failed 0 - foreach prog $args { - set PROG [string toupper $prog] - msg-checking "Checking for $prog..." - if {![find-executable $prog]} { - msg-result no - define $PROG false - incr failed - } else { - msg-result ok - define $PROG $prog - } - } - expr {!$failed} -} - -# @cc-path-progs prog ... -# -# Like cc-check-progs, but sets the define to the full path rather -# than just the program name. -# -proc cc-path-progs {args} { - set failed 0 - foreach prog $args { - set PROG [string toupper $prog] - msg-checking "Checking for $prog..." - set path [find-executable-path $prog] - if {$path eq ""} { - msg-result no - define $PROG false - incr failed - } else { - msg-result $path - define $PROG $path - } - } - expr {!$failed} -} - -# Adds the given settings to $::autosetup(ccsettings) and -# returns the old settings. -# -proc cc-add-settings {settings} { - if {[llength $settings] % 2} { - autosetup-error "settings list is missing a value: $settings" - } - - set prev [cc-get-settings] - # workaround a bug in some versions of jimsh by forcing - # conversion of $prev to a list - llength $prev - - array set new $prev - - foreach {name value} $settings { - switch -exact -- $name { - -cflags - -includes { - # These are given as lists - lappend new($name) {*}[list-non-empty $value] - } - -declare { - lappend new($name) $value - } - -libs { - # Note that new libraries are added before previous libraries - set new($name) [list {*}[list-non-empty $value] {*}$new($name)] - } - -link - -lang - -nooutput { - set new($name) $value - } - -source - -sourcefile - -code { - # XXX: These probably are only valid directly from cctest - set new($name) $value - } - default { - autosetup-error "unknown cctest setting: $name" - } - } - } - - cc-store-settings [array get new] - - return $prev -} - -proc cc-store-settings {new} { - set ::autosetup(ccsettings) $new -} - -proc cc-get-settings {} { - return $::autosetup(ccsettings) -} - -# Similar to cc-add-settings, but each given setting -# simply replaces the existing value. -# -# Returns the previous settings -proc cc-update-settings {args} { - set prev [cc-get-settings] - cc-store-settings [dict merge $prev $args] - return $prev -} - -# @cc-with settings ?{ script }? -# -# Sets the given 'cctest' settings and then runs the tests in '$script'. -# Note that settings such as '-lang' replace the current setting, while -# those such as '-includes' are appended to the existing setting. -# -# If no script is given, the settings become the default for the remainder -# of the 'auto.def' file. -# -## cc-with {-lang c++} { -## # This will check with the C++ compiler -## cc-check-types bool -## cc-with {-includes signal.h} { -## # This will check with the C++ compiler, signal.h and any existing includes. -## ... -## } -## # back to just the C++ compiler -## } -# -# The '-libs' setting is special in that newer values are added *before* earlier ones. -# -## cc-with {-libs {-lc -lm}} { -## cc-with {-libs -ldl} { -## cctest -libs -lsocket ... -## # libs will be in this order: -lsocket -ldl -lc -lm -## } -## } -proc cc-with {settings args} { - if {[llength $args] == 0} { - cc-add-settings $settings - } elseif {[llength $args] > 1} { - autosetup-error "usage: cc-with settings ?script?" - } else { - set save [cc-add-settings $settings] - set rc [catch {uplevel 1 [lindex $args 0]} result info] - cc-store-settings $save - if {$rc != 0} { - return -code [dict get $info -code] $result - } - return $result - } -} - -# @cctest ?settings? -# -# Low level C/C++ compiler checker. Compiles and or links a small C program -# according to the arguments and returns 1 if OK, or 0 if not. -# -# Supported settings are: -# -## -cflags cflags A list of flags to pass to the compiler -## -includes list A list of includes, e.g. {stdlib.h stdio.h} -## -declare code Code to declare before main() -## -link 1 Don't just compile, link too -## -lang c|c++ Use the C (default) or C++ compiler -## -libs liblist List of libraries to link, e.g. {-ldl -lm} -## -code code Code to compile in the body of main() -## -source code Compile a complete program. Ignore -includes, -declare and -code -## -sourcefile file Shorthand for -source [readfile [get-define srcdir]/$file] -## -nooutput 1 Treat any compiler output (e.g. a warning) as an error -# -# Unless '-source' or '-sourcefile' is specified, the C program looks like: -# -## #include /* same for remaining includes in the list */ -## -## declare-code /* any code in -declare, verbatim */ -## -## int main(void) { -## code /* any code in -code, verbatim */ -## return 0; -## } -# -# Any failures are recorded in 'config.log' -# -proc cctest {args} { - set tmp conftest__ - - # Easiest way to merge in the settings - cc-with $args { - array set opts [cc-get-settings] - } - - if {[info exists opts(-sourcefile)]} { - set opts(-source) [readfile [get-define srcdir]/$opts(-sourcefile) "#error can't find $opts(-sourcefile)"] - } - if {[info exists opts(-source)]} { - set lines $opts(-source) - } else { - foreach i $opts(-includes) { - if {$opts(-code) ne "" && ![feature-checked $i]} { - # Compiling real code with an unchecked header file - # Quickly (and silently) check for it now - - # Remove all -includes from settings before checking - set saveopts [cc-update-settings -includes {}] - msg-quiet cc-check-includes $i - cc-store-settings $saveopts - } - if {$opts(-code) eq "" || [have-feature $i]} { - lappend source "#include <$i>" - } - } - lappend source {*}$opts(-declare) - lappend source "int main(void) {" - lappend source $opts(-code) - lappend source "return 0;" - lappend source "}" - - set lines [join $source \n] - } - - # Build the command line - set cmdline {} - lappend cmdline {*}[get-define CCACHE] - switch -exact -- $opts(-lang) { - c++ { - set src conftest__.cpp - lappend cmdline {*}[get-define CXX] {*}[get-define CXXFLAGS] - } - c { - set src conftest__.c - lappend cmdline {*}[get-define CC] {*}[get-define CFLAGS] - } - default { - autosetup-error "cctest called with unknown language: $opts(-lang)" - } - } - - if {$opts(-link)} { - lappend cmdline {*}[get-define LDFLAGS] - } else { - set tmp conftest__.o - lappend cmdline -c - } - lappend cmdline {*}$opts(-cflags) {*}[get-define cc-default-debug ""] - lappend cmdline $src -o $tmp {*}$opts(-libs) - if {$opts(-link)} { - lappend cmdline {*}[get-define LIBS] - } - - # At this point we have the complete command line and the - # complete source to be compiled. Get the result from cache if - # we can - if {[info exists ::cc_cache($cmdline,$lines)]} { - msg-checking "(cached) " - set ok $::cc_cache($cmdline,$lines) - if {$::autosetup(debug)} { - configlog "From cache (ok=$ok): [join $cmdline]" - configlog "============" - configlog $lines - configlog "============" - } - return $ok - } - - writefile $src $lines\n - - set ok 1 - set err [catch {exec-with-stderr {*}$cmdline} result errinfo] - if {$err || ($opts(-nooutput) && [string length $result])} { - configlog "Failed: [join $cmdline]" - configlog $result - configlog "============" - configlog "The failed code was:" - configlog $lines - configlog "============" - set ok 0 - } elseif {$::autosetup(debug)} { - configlog "Compiled OK: [join $cmdline]" - configlog "============" - configlog $lines - configlog "============" - } - file delete $src - file delete $tmp - - # cache it - set ::cc_cache($cmdline,$lines) $ok - - return $ok -} - -# @make-autoconf-h outfile ?auto-patterns=HAVE_*? ?bare-patterns=SIZEOF_*? -# -# Deprecated - see 'make-config-header' -proc make-autoconf-h {file {autopatterns {HAVE_*}} {barepatterns {SIZEOF_* HAVE_DECL_*}}} { - user-notice "*** make-autoconf-h is deprecated -- use make-config-header instead" - make-config-header $file -auto $autopatterns -bare $barepatterns -} - -# @make-config-header outfile ?-auto patternlist? ?-bare patternlist? ?-none patternlist? ?-str patternlist? ... -# -# Examines all defined variables which match the given patterns -# and writes an include file, '$file', which defines each of these. -# Variables which match '-auto' are output as follows: -# - defines which have the value '0' are ignored. -# - defines which have integer values are defined as the integer value. -# - any other value is defined as a string, e.g. '"value"' -# Variables which match '-bare' are defined as-is. -# Variables which match '-str' are defined as a string, e.g. '"value"' -# Variables which match '-none' are omitted. -# -# Note that order is important. The first pattern that matches is selected. -# Default behaviour is: -# -## -bare {SIZEOF_* HAVE_DECL_*} -auto HAVE_* -none * -# -# If the file would be unchanged, it is not written. -proc make-config-header {file args} { - set guard _[string toupper [regsub -all {[^a-zA-Z0-9]} [file tail $file] _]] - file mkdir [file dirname $file] - set lines {} - lappend lines "#ifndef $guard" - lappend lines "#define $guard" - - # Add some defaults - lappend args -bare {SIZEOF_* HAVE_DECL_*} -auto HAVE_* - - foreach n [lsort [dict keys [all-defines]]] { - set value [get-define $n] - set type [calc-define-output-type $n $args] - switch -exact -- $type { - -bare { - # Just output the value unchanged - } - -none { - continue - } - -str { - set value \"[string map [list \\ \\\\ \" \\\"] $value]\" - } - -auto { - # Automatically determine the type - if {$value eq "0"} { - lappend lines "/* #undef $n */" - continue - } - if {![string is integer -strict $value]} { - set value \"[string map [list \\ \\\\ \" \\\"] $value]\" - } - } - "" { - continue - } - default { - autosetup-error "Unknown type in make-config-header: $type" - } - } - lappend lines "#define $n $value" - } - lappend lines "#endif" - set buf [join $lines \n] - write-if-changed $file $buf { - msg-result "Created $file" - } -} - -proc calc-define-output-type {name spec} { - foreach {type patterns} $spec { - foreach pattern $patterns { - if {[string match $pattern $name]} { - return $type - } - } - } - return "" -} - -# Initialise some values from the environment or commandline or default settings -foreach i {LDFLAGS LIBS CPPFLAGS LINKFLAGS {CFLAGS "-g -O2"}} { - lassign $i var default - define $var [get-env $var $default] -} - -if {[env-is-set CC]} { - # Set by the user, so don't try anything else - set try [list [get-env CC ""]] -} else { - # Try some reasonable options - set try [list [get-define cross]cc [get-define cross]gcc] -} -define CC [find-an-executable {*}$try] -if {[get-define CC] eq ""} { - user-error "Could not find a C compiler. Tried: [join $try ", "]" -} - -define CPP [get-env CPP "[get-define CC] -E"] - -# XXX: Could avoid looking for a C++ compiler until requested -# If CXX isn't found, it is set to the empty string. -if {[env-is-set CXX]} { - define CXX [find-an-executable -required [get-env CXX ""]] -} else { - define CXX [find-an-executable [get-define cross]c++ [get-define cross]g++] -} - -# CXXFLAGS default to CFLAGS if not specified -define CXXFLAGS [get-env CXXFLAGS [get-define CFLAGS]] - -# May need a CC_FOR_BUILD, so look for one -define CC_FOR_BUILD [find-an-executable [get-env CC_FOR_BUILD ""] cc gcc false] - -if {[get-define CC] eq ""} { - user-error "Could not find a C compiler. Tried: [join $try ", "]" -} - -define CCACHE [find-an-executable [get-env CCACHE ccache]] - -# If any of these are set in the environment, propagate them to the AUTOREMAKE commandline -foreach i {CC CXX CCACHE CPP CFLAGS CXXFLAGS CXXFLAGS LDFLAGS LIBS CROSS CPPFLAGS LINKFLAGS CC_FOR_BUILD LD} { - if {[env-is-set $i]} { - # Note: If the variable is set on the command line, get-env will return that value - # so the command line will continue to override the environment - define-append AUTOREMAKE [quote-if-needed $i=[get-env $i ""]] - } -} - -# Initial cctest settings -cc-store-settings {-cflags {} -includes {} -declare {} -link 0 -lang c -libs {} -code {} -nooutput 0} -set autosetup(cc-include-deps) {} - -msg-result "C compiler...[get-define CCACHE] [get-define CC] [get-define CFLAGS]" -if {[get-define CXX] ne "false"} { - msg-result "C++ compiler...[get-define CCACHE] [get-define CXX] [get-define CXXFLAGS]" -} -msg-result "Build C compiler...[get-define CC_FOR_BUILD]" - -# On Darwin, we prefer to use -g0 to avoid creating .dSYM directories -# but some compilers may not support it, so test here. -switch -glob -- [get-define host] { - *-*-darwin* { - if {[cctest -cflags {-g0}]} { - define cc-default-debug -g0 - } - } -} - -if {![cc-check-includes stdlib.h]} { - user-error "Compiler does not work. See config.log" -} DELETED autosetup/default.auto Index: autosetup/default.auto ================================================================== --- autosetup/default.auto +++ autosetup/default.auto @@ -1,25 +0,0 @@ -# Copyright (c) 2012 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Auto-load module for 'make' build system integration - -use init - -autosetup_add_init_type make {Simple "make" build system} { - autosetup_check_create auto.def \ -{# Initial auto.def created by 'autosetup --init=make' - -use cc - -# Add any user options here -options { -} - -make-config-header config.h -make-template Makefile.in -} - - if {![file exists Makefile.in]} { - puts "Note: I don't see Makefile.in. You will probably need to create one." - } -} DELETED autosetup/jimsh0.c Index: autosetup/jimsh0.c ================================================================== --- autosetup/jimsh0.c +++ autosetup/jimsh0.c @@ -1,22509 +0,0 @@ -/* This is single source file, bootstrap version of Jim Tcl. See http://jim.tcl.tk/ */ -#define JIM_TCL_COMPAT -#define JIM_ANSIC -#define JIM_REGEXP -#define HAVE_NO_AUTOCONF -#define _JIMAUTOCONF_H -#define TCL_LIBRARY "." -#define jim_ext_bootstrap -#define jim_ext_aio -#define jim_ext_readdir -#define jim_ext_regexp -#define jim_ext_file -#define jim_ext_glob -#define jim_ext_exec -#define jim_ext_clock -#define jim_ext_array -#define jim_ext_stdlib -#define jim_ext_tclcompat -#if defined(_MSC_VER) -#define TCL_PLATFORM_OS "windows" -#define TCL_PLATFORM_PLATFORM "windows" -#define TCL_PLATFORM_PATH_SEPARATOR ";" -#define HAVE_MKDIR_ONE_ARG -#define HAVE_SYSTEM -#elif defined(__MINGW32__) -#define TCL_PLATFORM_OS "mingw" -#define TCL_PLATFORM_PLATFORM "windows" -#define TCL_PLATFORM_PATH_SEPARATOR ";" -#define HAVE_MKDIR_ONE_ARG -#define HAVE_SYSTEM -#define HAVE_SYS_TIME_H -#define HAVE_DIRENT_H -#define HAVE_UNISTD_H -#define HAVE_UMASK -#include -#ifndef S_IRWXG -#define S_IRWXG 0 -#endif -#ifndef S_IRWXO -#define S_IRWXO 0 -#endif -#else -#define TCL_PLATFORM_OS "unknown" -#define TCL_PLATFORM_PLATFORM "unix" -#define TCL_PLATFORM_PATH_SEPARATOR ":" -#ifdef _MINIX -#define vfork fork -#define _POSIX_SOURCE -#else -#define _GNU_SOURCE -#endif -#define HAVE_VFORK -#define HAVE_WAITPID -#define HAVE_ISATTY -#define HAVE_MKSTEMP -#define HAVE_LINK -#define HAVE_SYS_TIME_H -#define HAVE_DIRENT_H -#define HAVE_UNISTD_H -#define HAVE_UMASK -#endif -#define JIM_VERSION 78 -#ifndef JIM_WIN32COMPAT_H -#define JIM_WIN32COMPAT_H - - - -#ifdef __cplusplus -extern "C" { -#endif - - -#if defined(_WIN32) || defined(WIN32) - -#define HAVE_DLOPEN -void *dlopen(const char *path, int mode); -int dlclose(void *handle); -void *dlsym(void *handle, const char *symbol); -char *dlerror(void); - - -#if defined(__MINGW32__) - #define JIM_SPRINTF_DOUBLE_NEEDS_FIX -#endif - -#ifdef _MSC_VER - - -#if _MSC_VER >= 1000 - #pragma warning(disable:4146) -#endif - -#include -#define jim_wide _int64 -#ifndef LLONG_MAX - #define LLONG_MAX 9223372036854775807I64 -#endif -#ifndef LLONG_MIN - #define LLONG_MIN (-LLONG_MAX - 1I64) -#endif -#define JIM_WIDE_MIN LLONG_MIN -#define JIM_WIDE_MAX LLONG_MAX -#define JIM_WIDE_MODIFIER "I64d" -#define strcasecmp _stricmp -#define strtoull _strtoui64 - -#include - -struct timeval { - long tv_sec; - long tv_usec; -}; - -int gettimeofday(struct timeval *tv, void *unused); - -#define HAVE_OPENDIR -struct dirent { - char *d_name; -}; - -typedef struct DIR { - long handle; - struct _finddata_t info; - struct dirent result; - char *name; -} DIR; - -DIR *opendir(const char *name); -int closedir(DIR *dir); -struct dirent *readdir(DIR *dir); - -#endif - -#endif - -#ifdef __cplusplus -} -#endif - -#endif -#ifndef UTF8_UTIL_H -#define UTF8_UTIL_H - -#ifdef __cplusplus -extern "C" { -#endif - - - -#define MAX_UTF8_LEN 4 - -int utf8_fromunicode(char *p, unsigned uc); - -#ifndef JIM_UTF8 -#include - - -#define utf8_strlen(S, B) ((B) < 0 ? (int)strlen(S) : (B)) -#define utf8_strwidth(S, B) utf8_strlen((S), (B)) -#define utf8_tounicode(S, CP) (*(CP) = (unsigned char)*(S), 1) -#define utf8_getchars(CP, C) (*(CP) = (C), 1) -#define utf8_upper(C) toupper(C) -#define utf8_title(C) toupper(C) -#define utf8_lower(C) tolower(C) -#define utf8_index(C, I) (I) -#define utf8_charlen(C) 1 -#define utf8_prev_len(S, L) 1 -#define utf8_width(C) 1 - -#else - -#endif - -#ifdef __cplusplus -} -#endif - -#endif - -#ifndef __JIM__H -#define __JIM__H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include - - -#ifndef HAVE_NO_AUTOCONF -#endif - - - -#ifndef jim_wide -# ifdef HAVE_LONG_LONG -# define jim_wide long long -# ifndef LLONG_MAX -# define LLONG_MAX 9223372036854775807LL -# endif -# ifndef LLONG_MIN -# define LLONG_MIN (-LLONG_MAX - 1LL) -# endif -# define JIM_WIDE_MIN LLONG_MIN -# define JIM_WIDE_MAX LLONG_MAX -# else -# define jim_wide long -# define JIM_WIDE_MIN LONG_MIN -# define JIM_WIDE_MAX LONG_MAX -# endif - - -# ifdef HAVE_LONG_LONG -# define JIM_WIDE_MODIFIER "lld" -# else -# define JIM_WIDE_MODIFIER "ld" -# define strtoull strtoul -# endif -#endif - -#define UCHAR(c) ((unsigned char)(c)) - - -#define JIM_OK 0 -#define JIM_ERR 1 -#define JIM_RETURN 2 -#define JIM_BREAK 3 -#define JIM_CONTINUE 4 -#define JIM_SIGNAL 5 -#define JIM_EXIT 6 - -#define JIM_EVAL 7 - -#define JIM_MAX_CALLFRAME_DEPTH 1000 -#define JIM_MAX_EVAL_DEPTH 2000 - - -#define JIM_PRIV_FLAG_SHIFT 20 - -#define JIM_NONE 0 -#define JIM_ERRMSG 1 -#define JIM_ENUM_ABBREV 2 -#define JIM_UNSHARED 4 -#define JIM_MUSTEXIST 8 - - -#define JIM_SUBST_NOVAR 1 -#define JIM_SUBST_NOCMD 2 -#define JIM_SUBST_NOESC 4 -#define JIM_SUBST_FLAG 128 - - -#define JIM_CASESENS 0 -#define JIM_NOCASE 1 - - -#define JIM_PATH_LEN 1024 - - -#define JIM_NOTUSED(V) ((void) V) - -#define JIM_LIBPATH "auto_path" -#define JIM_INTERACTIVE "tcl_interactive" - - -typedef struct Jim_Stack { - int len; - int maxlen; - void **vector; -} Jim_Stack; - - -typedef struct Jim_HashEntry { - void *key; - union { - void *val; - int intval; - } u; - struct Jim_HashEntry *next; -} Jim_HashEntry; - -typedef struct Jim_HashTableType { - unsigned int (*hashFunction)(const void *key); - void *(*keyDup)(void *privdata, const void *key); - void *(*valDup)(void *privdata, const void *obj); - int (*keyCompare)(void *privdata, const void *key1, const void *key2); - void (*keyDestructor)(void *privdata, void *key); - void (*valDestructor)(void *privdata, void *obj); -} Jim_HashTableType; - -typedef struct Jim_HashTable { - Jim_HashEntry **table; - const Jim_HashTableType *type; - void *privdata; - unsigned int size; - unsigned int sizemask; - unsigned int used; - unsigned int collisions; - unsigned int uniq; -} Jim_HashTable; - -typedef struct Jim_HashTableIterator { - Jim_HashTable *ht; - Jim_HashEntry *entry, *nextEntry; - int index; -} Jim_HashTableIterator; - - -#define JIM_HT_INITIAL_SIZE 16 - - -#define Jim_FreeEntryVal(ht, entry) \ - if ((ht)->type->valDestructor) \ - (ht)->type->valDestructor((ht)->privdata, (entry)->u.val) - -#define Jim_SetHashVal(ht, entry, _val_) do { \ - if ((ht)->type->valDup) \ - (entry)->u.val = (ht)->type->valDup((ht)->privdata, (_val_)); \ - else \ - (entry)->u.val = (_val_); \ -} while(0) - -#define Jim_FreeEntryKey(ht, entry) \ - if ((ht)->type->keyDestructor) \ - (ht)->type->keyDestructor((ht)->privdata, (entry)->key) - -#define Jim_SetHashKey(ht, entry, _key_) do { \ - if ((ht)->type->keyDup) \ - (entry)->key = (ht)->type->keyDup((ht)->privdata, (_key_)); \ - else \ - (entry)->key = (void *)(_key_); \ -} while(0) - -#define Jim_CompareHashKeys(ht, key1, key2) \ - (((ht)->type->keyCompare) ? \ - (ht)->type->keyCompare((ht)->privdata, (key1), (key2)) : \ - (key1) == (key2)) - -#define Jim_HashKey(ht, key) ((ht)->type->hashFunction(key) + (ht)->uniq) - -#define Jim_GetHashEntryKey(he) ((he)->key) -#define Jim_GetHashEntryVal(he) ((he)->u.val) -#define Jim_GetHashTableCollisions(ht) ((ht)->collisions) -#define Jim_GetHashTableSize(ht) ((ht)->size) -#define Jim_GetHashTableUsed(ht) ((ht)->used) - - -typedef struct Jim_Obj { - char *bytes; - const struct Jim_ObjType *typePtr; - int refCount; - int length; - - union { - - jim_wide wideValue; - - int intValue; - - double doubleValue; - - void *ptr; - - struct { - void *ptr1; - void *ptr2; - } twoPtrValue; - - struct { - void *ptr; - int int1; - int int2; - } ptrIntValue; - - struct { - struct Jim_Var *varPtr; - unsigned long callFrameId; - int global; - } varValue; - - struct { - struct Jim_Obj *nsObj; - struct Jim_Cmd *cmdPtr; - unsigned long procEpoch; - } cmdValue; - - struct { - struct Jim_Obj **ele; - int len; - int maxLen; - } listValue; - - struct { - int maxLength; - int charLength; - } strValue; - - struct { - unsigned long id; - struct Jim_Reference *refPtr; - } refValue; - - struct { - struct Jim_Obj *fileNameObj; - int lineNumber; - } sourceValue; - - struct { - struct Jim_Obj *varNameObjPtr; - struct Jim_Obj *indexObjPtr; - } dictSubstValue; - struct { - int line; - int argc; - } scriptLineValue; - } internalRep; - struct Jim_Obj *prevObjPtr; - struct Jim_Obj *nextObjPtr; -} Jim_Obj; - - -#define Jim_IncrRefCount(objPtr) \ - ++(objPtr)->refCount -#define Jim_DecrRefCount(interp, objPtr) \ - if (--(objPtr)->refCount <= 0) Jim_FreeObj(interp, objPtr) -#define Jim_IsShared(objPtr) \ - ((objPtr)->refCount > 1) - -#define Jim_FreeNewObj Jim_FreeObj - - -#define Jim_FreeIntRep(i,o) \ - if ((o)->typePtr && (o)->typePtr->freeIntRepProc) \ - (o)->typePtr->freeIntRepProc(i, o) - - -#define Jim_GetIntRepPtr(o) (o)->internalRep.ptr - - -#define Jim_SetIntRepPtr(o, p) \ - (o)->internalRep.ptr = (p) - - -struct Jim_Interp; - -typedef void (Jim_FreeInternalRepProc)(struct Jim_Interp *interp, - struct Jim_Obj *objPtr); -typedef void (Jim_DupInternalRepProc)(struct Jim_Interp *interp, - struct Jim_Obj *srcPtr, Jim_Obj *dupPtr); -typedef void (Jim_UpdateStringProc)(struct Jim_Obj *objPtr); - -typedef struct Jim_ObjType { - const char *name; - Jim_FreeInternalRepProc *freeIntRepProc; - Jim_DupInternalRepProc *dupIntRepProc; - Jim_UpdateStringProc *updateStringProc; - int flags; -} Jim_ObjType; - - -#define JIM_TYPE_NONE 0 -#define JIM_TYPE_REFERENCES 1 - - - -typedef struct Jim_CallFrame { - unsigned long id; - int level; - struct Jim_HashTable vars; - struct Jim_HashTable *staticVars; - struct Jim_CallFrame *parent; - Jim_Obj *const *argv; - int argc; - Jim_Obj *procArgsObjPtr; - Jim_Obj *procBodyObjPtr; - struct Jim_CallFrame *next; - Jim_Obj *nsObj; - Jim_Obj *fileNameObj; - int line; - Jim_Stack *localCommands; - struct Jim_Obj *tailcallObj; - struct Jim_Cmd *tailcallCmd; -} Jim_CallFrame; - -typedef struct Jim_Var { - Jim_Obj *objPtr; - struct Jim_CallFrame *linkFramePtr; -} Jim_Var; - - -typedef int Jim_CmdProc(struct Jim_Interp *interp, int argc, - Jim_Obj *const *argv); -typedef void Jim_DelCmdProc(struct Jim_Interp *interp, void *privData); - - - -typedef struct Jim_Cmd { - int inUse; - int isproc; - struct Jim_Cmd *prevCmd; - union { - struct { - - Jim_CmdProc *cmdProc; - Jim_DelCmdProc *delProc; - void *privData; - } native; - struct { - - Jim_Obj *argListObjPtr; - Jim_Obj *bodyObjPtr; - Jim_HashTable *staticVars; - int argListLen; - int reqArity; - int optArity; - int argsPos; - int upcall; - struct Jim_ProcArg { - Jim_Obj *nameObjPtr; - Jim_Obj *defaultObjPtr; - } *arglist; - Jim_Obj *nsObj; - } proc; - } u; -} Jim_Cmd; - - -typedef struct Jim_PrngState { - unsigned char sbox[256]; - unsigned int i, j; -} Jim_PrngState; - -typedef struct Jim_Interp { - Jim_Obj *result; - int errorLine; - Jim_Obj *errorFileNameObj; - int addStackTrace; - int maxCallFrameDepth; - int maxEvalDepth; - int evalDepth; - int returnCode; - int returnLevel; - int exitCode; - long id; - int signal_level; - jim_wide sigmask; - int (*signal_set_result)(struct Jim_Interp *interp, jim_wide sigmask); - Jim_CallFrame *framePtr; - Jim_CallFrame *topFramePtr; - struct Jim_HashTable commands; - unsigned long procEpoch; /* Incremented every time the result - of procedures names lookup caching - may no longer be valid. */ - unsigned long callFrameEpoch; /* Incremented every time a new - callframe is created. This id is used for the - 'ID' field contained in the Jim_CallFrame - structure. */ - int local; - Jim_Obj *liveList; - Jim_Obj *freeList; - Jim_Obj *currentScriptObj; - Jim_Obj *nullScriptObj; - Jim_Obj *emptyObj; - Jim_Obj *trueObj; - Jim_Obj *falseObj; - unsigned long referenceNextId; - struct Jim_HashTable references; - unsigned long lastCollectId; /* reference max Id of the last GC - execution. It's set to ~0 while the collection - is running as sentinel to avoid to recursive - calls via the [collect] command inside - finalizers. */ - time_t lastCollectTime; - Jim_Obj *stackTrace; - Jim_Obj *errorProc; - Jim_Obj *unknown; - int unknown_called; - int errorFlag; - void *cmdPrivData; /* Used to pass the private data pointer to - a command. It is set to what the user specified - via Jim_CreateCommand(). */ - - struct Jim_CallFrame *freeFramesList; - struct Jim_HashTable assocData; - Jim_PrngState *prngState; - struct Jim_HashTable packages; - Jim_Stack *loadHandles; -} Jim_Interp; - -#define Jim_InterpIncrProcEpoch(i) (i)->procEpoch++ -#define Jim_SetResultString(i,s,l) Jim_SetResult(i, Jim_NewStringObj(i,s,l)) -#define Jim_SetResultInt(i,intval) Jim_SetResult(i, Jim_NewIntObj(i,intval)) - -#define Jim_SetResultBool(i,b) Jim_SetResultInt(i, b) -#define Jim_SetEmptyResult(i) Jim_SetResult(i, (i)->emptyObj) -#define Jim_GetResult(i) ((i)->result) -#define Jim_CmdPrivData(i) ((i)->cmdPrivData) - -#define Jim_SetResult(i,o) do { \ - Jim_Obj *_resultObjPtr_ = (o); \ - Jim_IncrRefCount(_resultObjPtr_); \ - Jim_DecrRefCount(i,(i)->result); \ - (i)->result = _resultObjPtr_; \ -} while(0) - - -#define Jim_GetId(i) (++(i)->id) - - -#define JIM_REFERENCE_TAGLEN 7 /* The tag is fixed-length, because the reference - string representation must be fixed length. */ -typedef struct Jim_Reference { - Jim_Obj *objPtr; - Jim_Obj *finalizerCmdNamePtr; - char tag[JIM_REFERENCE_TAGLEN+1]; -} Jim_Reference; - - -#define Jim_NewEmptyStringObj(i) Jim_NewStringObj(i, "", 0) -#define Jim_FreeHashTableIterator(iter) Jim_Free(iter) - -#define JIM_EXPORT - - -JIM_EXPORT void *Jim_Alloc (int size); -JIM_EXPORT void *Jim_Realloc(void *ptr, int size); -JIM_EXPORT void Jim_Free (void *ptr); -JIM_EXPORT char * Jim_StrDup (const char *s); -JIM_EXPORT char *Jim_StrDupLen(const char *s, int l); - - -JIM_EXPORT char **Jim_GetEnviron(void); -JIM_EXPORT void Jim_SetEnviron(char **env); -JIM_EXPORT int Jim_MakeTempFile(Jim_Interp *interp, const char *filename_template, int unlink_file); - - -JIM_EXPORT int Jim_Eval(Jim_Interp *interp, const char *script); - - -JIM_EXPORT int Jim_EvalSource(Jim_Interp *interp, const char *filename, int lineno, const char *script); - -#define Jim_Eval_Named(I, S, F, L) Jim_EvalSource((I), (F), (L), (S)) - -JIM_EXPORT int Jim_EvalGlobal(Jim_Interp *interp, const char *script); -JIM_EXPORT int Jim_EvalFile(Jim_Interp *interp, const char *filename); -JIM_EXPORT int Jim_EvalFileGlobal(Jim_Interp *interp, const char *filename); -JIM_EXPORT int Jim_EvalObj (Jim_Interp *interp, Jim_Obj *scriptObjPtr); -JIM_EXPORT int Jim_EvalObjVector (Jim_Interp *interp, int objc, - Jim_Obj *const *objv); -JIM_EXPORT int Jim_EvalObjList(Jim_Interp *interp, Jim_Obj *listObj); -JIM_EXPORT int Jim_EvalObjPrefix(Jim_Interp *interp, Jim_Obj *prefix, - int objc, Jim_Obj *const *objv); -#define Jim_EvalPrefix(i, p, oc, ov) Jim_EvalObjPrefix((i), Jim_NewStringObj((i), (p), -1), (oc), (ov)) -JIM_EXPORT int Jim_EvalNamespace(Jim_Interp *interp, Jim_Obj *scriptObj, Jim_Obj *nsObj); -JIM_EXPORT int Jim_SubstObj (Jim_Interp *interp, Jim_Obj *substObjPtr, - Jim_Obj **resObjPtrPtr, int flags); - - -JIM_EXPORT void Jim_InitStack(Jim_Stack *stack); -JIM_EXPORT void Jim_FreeStack(Jim_Stack *stack); -JIM_EXPORT int Jim_StackLen(Jim_Stack *stack); -JIM_EXPORT void Jim_StackPush(Jim_Stack *stack, void *element); -JIM_EXPORT void * Jim_StackPop(Jim_Stack *stack); -JIM_EXPORT void * Jim_StackPeek(Jim_Stack *stack); -JIM_EXPORT void Jim_FreeStackElements(Jim_Stack *stack, void (*freeFunc)(void *ptr)); - - -JIM_EXPORT int Jim_InitHashTable (Jim_HashTable *ht, - const Jim_HashTableType *type, void *privdata); -JIM_EXPORT void Jim_ExpandHashTable (Jim_HashTable *ht, - unsigned int size); -JIM_EXPORT int Jim_AddHashEntry (Jim_HashTable *ht, const void *key, - void *val); -JIM_EXPORT int Jim_ReplaceHashEntry (Jim_HashTable *ht, - const void *key, void *val); -JIM_EXPORT int Jim_DeleteHashEntry (Jim_HashTable *ht, - const void *key); -JIM_EXPORT int Jim_FreeHashTable (Jim_HashTable *ht); -JIM_EXPORT Jim_HashEntry * Jim_FindHashEntry (Jim_HashTable *ht, - const void *key); -JIM_EXPORT void Jim_ResizeHashTable (Jim_HashTable *ht); -JIM_EXPORT Jim_HashTableIterator *Jim_GetHashTableIterator - (Jim_HashTable *ht); -JIM_EXPORT Jim_HashEntry * Jim_NextHashEntry - (Jim_HashTableIterator *iter); - - -JIM_EXPORT Jim_Obj * Jim_NewObj (Jim_Interp *interp); -JIM_EXPORT void Jim_FreeObj (Jim_Interp *interp, Jim_Obj *objPtr); -JIM_EXPORT void Jim_InvalidateStringRep (Jim_Obj *objPtr); -JIM_EXPORT Jim_Obj * Jim_DuplicateObj (Jim_Interp *interp, - Jim_Obj *objPtr); -JIM_EXPORT const char * Jim_GetString(Jim_Obj *objPtr, - int *lenPtr); -JIM_EXPORT const char *Jim_String(Jim_Obj *objPtr); -JIM_EXPORT int Jim_Length(Jim_Obj *objPtr); - - -JIM_EXPORT Jim_Obj * Jim_NewStringObj (Jim_Interp *interp, - const char *s, int len); -JIM_EXPORT Jim_Obj *Jim_NewStringObjUtf8(Jim_Interp *interp, - const char *s, int charlen); -JIM_EXPORT Jim_Obj * Jim_NewStringObjNoAlloc (Jim_Interp *interp, - char *s, int len); -JIM_EXPORT void Jim_AppendString (Jim_Interp *interp, Jim_Obj *objPtr, - const char *str, int len); -JIM_EXPORT void Jim_AppendObj (Jim_Interp *interp, Jim_Obj *objPtr, - Jim_Obj *appendObjPtr); -JIM_EXPORT void Jim_AppendStrings (Jim_Interp *interp, - Jim_Obj *objPtr, ...); -JIM_EXPORT int Jim_StringEqObj(Jim_Obj *aObjPtr, Jim_Obj *bObjPtr); -JIM_EXPORT int Jim_StringMatchObj (Jim_Interp *interp, Jim_Obj *patternObjPtr, - Jim_Obj *objPtr, int nocase); -JIM_EXPORT Jim_Obj * Jim_StringRangeObj (Jim_Interp *interp, - Jim_Obj *strObjPtr, Jim_Obj *firstObjPtr, - Jim_Obj *lastObjPtr); -JIM_EXPORT Jim_Obj * Jim_FormatString (Jim_Interp *interp, - Jim_Obj *fmtObjPtr, int objc, Jim_Obj *const *objv); -JIM_EXPORT Jim_Obj * Jim_ScanString (Jim_Interp *interp, Jim_Obj *strObjPtr, - Jim_Obj *fmtObjPtr, int flags); -JIM_EXPORT int Jim_CompareStringImmediate (Jim_Interp *interp, - Jim_Obj *objPtr, const char *str); -JIM_EXPORT int Jim_StringCompareObj(Jim_Interp *interp, Jim_Obj *firstObjPtr, - Jim_Obj *secondObjPtr, int nocase); -JIM_EXPORT int Jim_StringCompareLenObj(Jim_Interp *interp, Jim_Obj *firstObjPtr, - Jim_Obj *secondObjPtr, int nocase); -JIM_EXPORT int Jim_Utf8Length(Jim_Interp *interp, Jim_Obj *objPtr); - - -JIM_EXPORT Jim_Obj * Jim_NewReference (Jim_Interp *interp, - Jim_Obj *objPtr, Jim_Obj *tagPtr, Jim_Obj *cmdNamePtr); -JIM_EXPORT Jim_Reference * Jim_GetReference (Jim_Interp *interp, - Jim_Obj *objPtr); -JIM_EXPORT int Jim_SetFinalizer (Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj *cmdNamePtr); -JIM_EXPORT int Jim_GetFinalizer (Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj **cmdNamePtrPtr); - - -JIM_EXPORT Jim_Interp * Jim_CreateInterp (void); -JIM_EXPORT void Jim_FreeInterp (Jim_Interp *i); -JIM_EXPORT int Jim_GetExitCode (Jim_Interp *interp); -JIM_EXPORT const char *Jim_ReturnCode(int code); -JIM_EXPORT void Jim_SetResultFormatted(Jim_Interp *interp, const char *format, ...); - - -JIM_EXPORT void Jim_RegisterCoreCommands (Jim_Interp *interp); -JIM_EXPORT int Jim_CreateCommand (Jim_Interp *interp, - const char *cmdName, Jim_CmdProc *cmdProc, void *privData, - Jim_DelCmdProc *delProc); -JIM_EXPORT int Jim_DeleteCommand (Jim_Interp *interp, - const char *cmdName); -JIM_EXPORT int Jim_RenameCommand (Jim_Interp *interp, - const char *oldName, const char *newName); -JIM_EXPORT Jim_Cmd * Jim_GetCommand (Jim_Interp *interp, - Jim_Obj *objPtr, int flags); -JIM_EXPORT int Jim_SetVariable (Jim_Interp *interp, - Jim_Obj *nameObjPtr, Jim_Obj *valObjPtr); -JIM_EXPORT int Jim_SetVariableStr (Jim_Interp *interp, - const char *name, Jim_Obj *objPtr); -JIM_EXPORT int Jim_SetGlobalVariableStr (Jim_Interp *interp, - const char *name, Jim_Obj *objPtr); -JIM_EXPORT int Jim_SetVariableStrWithStr (Jim_Interp *interp, - const char *name, const char *val); -JIM_EXPORT int Jim_SetVariableLink (Jim_Interp *interp, - Jim_Obj *nameObjPtr, Jim_Obj *targetNameObjPtr, - Jim_CallFrame *targetCallFrame); -JIM_EXPORT Jim_Obj * Jim_MakeGlobalNamespaceName(Jim_Interp *interp, - Jim_Obj *nameObjPtr); -JIM_EXPORT Jim_Obj * Jim_GetVariable (Jim_Interp *interp, - Jim_Obj *nameObjPtr, int flags); -JIM_EXPORT Jim_Obj * Jim_GetGlobalVariable (Jim_Interp *interp, - Jim_Obj *nameObjPtr, int flags); -JIM_EXPORT Jim_Obj * Jim_GetVariableStr (Jim_Interp *interp, - const char *name, int flags); -JIM_EXPORT Jim_Obj * Jim_GetGlobalVariableStr (Jim_Interp *interp, - const char *name, int flags); -JIM_EXPORT int Jim_UnsetVariable (Jim_Interp *interp, - Jim_Obj *nameObjPtr, int flags); - - -JIM_EXPORT Jim_CallFrame *Jim_GetCallFrameByLevel(Jim_Interp *interp, - Jim_Obj *levelObjPtr); - - -JIM_EXPORT int Jim_Collect (Jim_Interp *interp); -JIM_EXPORT void Jim_CollectIfNeeded (Jim_Interp *interp); - - -JIM_EXPORT int Jim_GetIndex (Jim_Interp *interp, Jim_Obj *objPtr, - int *indexPtr); - - -JIM_EXPORT Jim_Obj * Jim_NewListObj (Jim_Interp *interp, - Jim_Obj *const *elements, int len); -JIM_EXPORT void Jim_ListInsertElements (Jim_Interp *interp, - Jim_Obj *listPtr, int listindex, int objc, Jim_Obj *const *objVec); -JIM_EXPORT void Jim_ListAppendElement (Jim_Interp *interp, - Jim_Obj *listPtr, Jim_Obj *objPtr); -JIM_EXPORT void Jim_ListAppendList (Jim_Interp *interp, - Jim_Obj *listPtr, Jim_Obj *appendListPtr); -JIM_EXPORT int Jim_ListLength (Jim_Interp *interp, Jim_Obj *objPtr); -JIM_EXPORT int Jim_ListIndex (Jim_Interp *interp, Jim_Obj *listPrt, - int listindex, Jim_Obj **objPtrPtr, int seterr); -JIM_EXPORT Jim_Obj *Jim_ListGetIndex(Jim_Interp *interp, Jim_Obj *listPtr, int idx); -JIM_EXPORT int Jim_SetListIndex (Jim_Interp *interp, - Jim_Obj *varNamePtr, Jim_Obj *const *indexv, int indexc, - Jim_Obj *newObjPtr); -JIM_EXPORT Jim_Obj * Jim_ConcatObj (Jim_Interp *interp, int objc, - Jim_Obj *const *objv); -JIM_EXPORT Jim_Obj *Jim_ListJoin(Jim_Interp *interp, - Jim_Obj *listObjPtr, const char *joinStr, int joinStrLen); - - -JIM_EXPORT Jim_Obj * Jim_NewDictObj (Jim_Interp *interp, - Jim_Obj *const *elements, int len); -JIM_EXPORT int Jim_DictKey (Jim_Interp *interp, Jim_Obj *dictPtr, - Jim_Obj *keyPtr, Jim_Obj **objPtrPtr, int flags); -JIM_EXPORT int Jim_DictKeysVector (Jim_Interp *interp, - Jim_Obj *dictPtr, Jim_Obj *const *keyv, int keyc, - Jim_Obj **objPtrPtr, int flags); -JIM_EXPORT int Jim_SetDictKeysVector (Jim_Interp *interp, - Jim_Obj *varNamePtr, Jim_Obj *const *keyv, int keyc, - Jim_Obj *newObjPtr, int flags); -JIM_EXPORT int Jim_DictPairs(Jim_Interp *interp, - Jim_Obj *dictPtr, Jim_Obj ***objPtrPtr, int *len); -JIM_EXPORT int Jim_DictAddElement(Jim_Interp *interp, Jim_Obj *objPtr, - Jim_Obj *keyObjPtr, Jim_Obj *valueObjPtr); - -#define JIM_DICTMATCH_KEYS 0x0001 -#define JIM_DICTMATCH_VALUES 0x002 - -JIM_EXPORT int Jim_DictMatchTypes(Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj *patternObj, int match_type, int return_types); -JIM_EXPORT int Jim_DictSize(Jim_Interp *interp, Jim_Obj *objPtr); -JIM_EXPORT int Jim_DictInfo(Jim_Interp *interp, Jim_Obj *objPtr); -JIM_EXPORT Jim_Obj *Jim_DictMerge(Jim_Interp *interp, int objc, Jim_Obj *const *objv); - - -JIM_EXPORT int Jim_GetReturnCode (Jim_Interp *interp, Jim_Obj *objPtr, - int *intPtr); - - -JIM_EXPORT int Jim_EvalExpression (Jim_Interp *interp, - Jim_Obj *exprObjPtr); -JIM_EXPORT int Jim_GetBoolFromExpr (Jim_Interp *interp, - Jim_Obj *exprObjPtr, int *boolPtr); - - -JIM_EXPORT int Jim_GetBoolean(Jim_Interp *interp, Jim_Obj *objPtr, - int *booleanPtr); - - -JIM_EXPORT int Jim_GetWide (Jim_Interp *interp, Jim_Obj *objPtr, - jim_wide *widePtr); -JIM_EXPORT int Jim_GetLong (Jim_Interp *interp, Jim_Obj *objPtr, - long *longPtr); -#define Jim_NewWideObj Jim_NewIntObj -JIM_EXPORT Jim_Obj * Jim_NewIntObj (Jim_Interp *interp, - jim_wide wideValue); - - -JIM_EXPORT int Jim_GetDouble(Jim_Interp *interp, Jim_Obj *objPtr, - double *doublePtr); -JIM_EXPORT void Jim_SetDouble(Jim_Interp *interp, Jim_Obj *objPtr, - double doubleValue); -JIM_EXPORT Jim_Obj * Jim_NewDoubleObj(Jim_Interp *interp, double doubleValue); - - -JIM_EXPORT void Jim_WrongNumArgs (Jim_Interp *interp, int argc, - Jim_Obj *const *argv, const char *msg); -JIM_EXPORT int Jim_GetEnum (Jim_Interp *interp, Jim_Obj *objPtr, - const char * const *tablePtr, int *indexPtr, const char *name, int flags); -JIM_EXPORT int Jim_CheckShowCommands(Jim_Interp *interp, Jim_Obj *objPtr, - const char *const *tablePtr); -JIM_EXPORT int Jim_ScriptIsComplete(Jim_Interp *interp, - Jim_Obj *scriptObj, char *stateCharPtr); - -JIM_EXPORT int Jim_FindByName(const char *name, const char * const array[], size_t len); - - -typedef void (Jim_InterpDeleteProc)(Jim_Interp *interp, void *data); -JIM_EXPORT void * Jim_GetAssocData(Jim_Interp *interp, const char *key); -JIM_EXPORT int Jim_SetAssocData(Jim_Interp *interp, const char *key, - Jim_InterpDeleteProc *delProc, void *data); -JIM_EXPORT int Jim_DeleteAssocData(Jim_Interp *interp, const char *key); - - - -JIM_EXPORT int Jim_PackageProvide (Jim_Interp *interp, - const char *name, const char *ver, int flags); -JIM_EXPORT int Jim_PackageRequire (Jim_Interp *interp, - const char *name, int flags); - - -JIM_EXPORT void Jim_MakeErrorMessage (Jim_Interp *interp); - - -JIM_EXPORT int Jim_InteractivePrompt (Jim_Interp *interp); -JIM_EXPORT void Jim_HistoryLoad(const char *filename); -JIM_EXPORT void Jim_HistorySave(const char *filename); -JIM_EXPORT char *Jim_HistoryGetline(Jim_Interp *interp, const char *prompt); -JIM_EXPORT void Jim_HistorySetCompletion(Jim_Interp *interp, Jim_Obj *commandObj); -JIM_EXPORT void Jim_HistoryAdd(const char *line); -JIM_EXPORT void Jim_HistoryShow(void); - - -JIM_EXPORT int Jim_InitStaticExtensions(Jim_Interp *interp); -JIM_EXPORT int Jim_StringToWide(const char *str, jim_wide *widePtr, int base); -JIM_EXPORT int Jim_IsBigEndian(void); - -#define Jim_CheckSignal(i) ((i)->signal_level && (i)->sigmask) - - -JIM_EXPORT int Jim_LoadLibrary(Jim_Interp *interp, const char *pathName); -JIM_EXPORT void Jim_FreeLoadHandles(Jim_Interp *interp); - - -JIM_EXPORT FILE *Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *command); - - -JIM_EXPORT int Jim_IsDict(Jim_Obj *objPtr); -JIM_EXPORT int Jim_IsList(Jim_Obj *objPtr); - -#ifdef __cplusplus -} -#endif - -#endif - -#ifndef JIM_SUBCMD_H -#define JIM_SUBCMD_H - - -#ifdef __cplusplus -extern "C" { -#endif - - -#define JIM_MODFLAG_HIDDEN 0x0001 -#define JIM_MODFLAG_FULLARGV 0x0002 - - - -typedef int jim_subcmd_function(Jim_Interp *interp, int argc, Jim_Obj *const *argv); - -typedef struct { - const char *cmd; - const char *args; - jim_subcmd_function *function; - short minargs; - short maxargs; - unsigned short flags; -} jim_subcmd_type; - -const jim_subcmd_type * -Jim_ParseSubCmd(Jim_Interp *interp, const jim_subcmd_type *command_table, int argc, Jim_Obj *const *argv); - -int Jim_SubCmdProc(Jim_Interp *interp, int argc, Jim_Obj *const *argv); - -int Jim_CallSubCmd(Jim_Interp *interp, const jim_subcmd_type *ct, int argc, Jim_Obj *const *argv); - -#ifdef __cplusplus -} -#endif - -#endif -#ifndef JIMREGEXP_H -#define JIMREGEXP_H - - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -typedef struct { - int rm_so; - int rm_eo; -} regmatch_t; - - -typedef struct regexp { - - int re_nsub; - - - int cflags; - int err; - int regstart; - int reganch; - int regmust; - int regmlen; - int *program; - - - const char *regparse; - int p; - int proglen; - - - int eflags; - const char *start; - const char *reginput; - const char *regbol; - - - regmatch_t *pmatch; - int nmatch; -} regexp; - -typedef regexp regex_t; - -#define REG_EXTENDED 0 -#define REG_NEWLINE 1 -#define REG_ICASE 2 - -#define REG_NOTBOL 16 - -enum { - REG_NOERROR, - REG_NOMATCH, - REG_BADPAT, - REG_ERR_NULL_ARGUMENT, - REG_ERR_UNKNOWN, - REG_ERR_TOO_BIG, - REG_ERR_NOMEM, - REG_ERR_TOO_MANY_PAREN, - REG_ERR_UNMATCHED_PAREN, - REG_ERR_UNMATCHED_BRACES, - REG_ERR_BAD_COUNT, - REG_ERR_JUNK_ON_END, - REG_ERR_OPERAND_COULD_BE_EMPTY, - REG_ERR_NESTED_COUNT, - REG_ERR_INTERNAL, - REG_ERR_COUNT_FOLLOWS_NOTHING, - REG_ERR_TRAILING_BACKSLASH, - REG_ERR_CORRUPTED, - REG_ERR_NULL_CHAR, - REG_ERR_NUM -}; - -int regcomp(regex_t *preg, const char *regex, int cflags); -int regexec(regex_t *preg, const char *string, size_t nmatch, regmatch_t pmatch[], int eflags); -size_t regerror(int errcode, const regex_t *preg, char *errbuf, size_t errbuf_size); -void regfree(regex_t *preg); - -#ifdef __cplusplus -} -#endif - -#endif -#ifndef JIM_SIGNAL_H -#define JIM_SIGNAL_H - -#ifdef __cplusplus -extern "C" { -#endif - -const char *Jim_SignalId(int sig); - -#ifdef __cplusplus -} -#endif - -#endif -#ifndef JIMIOCOMPAT_H -#define JIMIOCOMPAT_H - - -#include -#include - - -void Jim_SetResultErrno(Jim_Interp *interp, const char *msg); - -int Jim_OpenForWrite(const char *filename, int append); - -int Jim_OpenForRead(const char *filename); - -#if defined(__MINGW32__) - #ifndef STRICT - #define STRICT - #endif - #define WIN32_LEAN_AND_MEAN - #include - #include - #include - #include - - typedef HANDLE pidtype; - #define JIM_BAD_PID INVALID_HANDLE_VALUE - - #define JIM_NO_PID INVALID_HANDLE_VALUE - - - #define WIFEXITED(STATUS) (((STATUS) & 0xff00) == 0) - #define WEXITSTATUS(STATUS) ((STATUS) & 0x00ff) - #define WIFSIGNALED(STATUS) (((STATUS) & 0xff00) != 0) - #define WTERMSIG(STATUS) (((STATUS) >> 8) & 0xff) - #define WNOHANG 1 - - int Jim_Errno(void); - pidtype waitpid(pidtype pid, int *status, int nohang); - - #define HAVE_PIPE - #define pipe(P) _pipe((P), 0, O_NOINHERIT) - -#elif defined(HAVE_UNISTD_H) - #include - #include - #include - #include - - typedef int pidtype; - #define Jim_Errno() errno - #define JIM_BAD_PID -1 - #define JIM_NO_PID 0 - - #ifndef HAVE_EXECVPE - #define execvpe(ARG0, ARGV, ENV) execvp(ARG0, ARGV) - #endif -#endif - -#endif -int Jim_bootstrapInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "bootstrap", "1.0", JIM_ERRMSG)) - return JIM_ERR; - - return Jim_EvalSource(interp, "bootstrap.tcl", 1, -"\n" -"\n" -"proc package {cmd pkg args} {\n" -" if {$cmd eq \"require\"} {\n" -" foreach path $::auto_path {\n" -" set pkgpath $path/$pkg.tcl\n" -" if {$path eq \".\"} {\n" -" set pkgpath $pkg.tcl\n" -" }\n" -" if {[file exists $pkgpath]} {\n" -" uplevel #0 [list source $pkgpath]\n" -" return\n" -" }\n" -" }\n" -" }\n" -"}\n" -); -} -int Jim_initjimshInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "initjimsh", "1.0", JIM_ERRMSG)) - return JIM_ERR; - - return Jim_EvalSource(interp, "initjimsh.tcl", 1, -"\n" -"\n" -"\n" -"proc _jimsh_init {} {\n" -" rename _jimsh_init {}\n" -" global jim::exe jim::argv0 tcl_interactive auto_path tcl_platform\n" -"\n" -"\n" -" if {[exists jim::argv0]} {\n" -" if {[string match \"*/*\" $jim::argv0]} {\n" -" set jim::exe [file join [pwd] $jim::argv0]\n" -" } else {\n" -" foreach path [split [env PATH \"\"] $tcl_platform(pathSeparator)] {\n" -" set exec [file join [pwd] [string map {\\\\ /} $path] $jim::argv0]\n" -" if {[file executable $exec]} {\n" -" set jim::exe $exec\n" -" break\n" -" }\n" -" }\n" -" }\n" -" }\n" -"\n" -"\n" -" lappend p {*}[split [env JIMLIB {}] $tcl_platform(pathSeparator)]\n" -" if {[exists jim::exe]} {\n" -" lappend p [file dirname $jim::exe]\n" -" }\n" -" lappend p {*}$auto_path\n" -" set auto_path $p\n" -"\n" -" if {$tcl_interactive && [env HOME {}] ne \"\"} {\n" -" foreach src {.jimrc jimrc.tcl} {\n" -" if {[file exists [env HOME]/$src]} {\n" -" uplevel #0 source [env HOME]/$src\n" -" break\n" -" }\n" -" }\n" -" }\n" -" return \"\"\n" -"}\n" -"\n" -"if {$tcl_platform(platform) eq \"windows\"} {\n" -" set jim::argv0 [string map {\\\\ /} $jim::argv0]\n" -"}\n" -"\n" -"\n" -"set tcl::autocomplete_commands {info tcl::prefix socket namespace array clock file package string dict signal history}\n" -"\n" -"\n" -"\n" -"proc tcl::autocomplete {prefix} {\n" -" if {[set space [string first \" \" $prefix]] != -1} {\n" -" set cmd [string range $prefix 0 $space-1]\n" -" if {$cmd in $::tcl::autocomplete_commands || [info channel $cmd] ne \"\"} {\n" -" set arg [string range $prefix $space+1 end]\n" -"\n" -" return [lmap p [$cmd -commands] {\n" -" if {![string match \"${arg}*\" $p]} continue\n" -" function \"$cmd $p\"\n" -" }]\n" -" }\n" -" }\n" -"\n" -" if {[string match \"source *\" $prefix]} {\n" -" set path [string range $prefix 7 end]\n" -" return [lmap p [glob -nocomplain \"${path}*\"] {\n" -" function \"source $p\"\n" -" }]\n" -" }\n" -"\n" -" return [lmap p [lsort [info commands $prefix*]] {\n" -" if {[string match \"* *\" $p]} {\n" -" continue\n" -" }\n" -" function $p\n" -" }]\n" -"}\n" -"\n" -"_jimsh_init\n" -); -} -int Jim_globInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "glob", "1.0", JIM_ERRMSG)) - return JIM_ERR; - - return Jim_EvalSource(interp, "glob.tcl", 1, -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"package require readdir\n" -"\n" -"\n" -"proc glob.globdir {dir pattern} {\n" -" if {[file exists $dir/$pattern]} {\n" -"\n" -" return [list $pattern]\n" -" }\n" -"\n" -" set result {}\n" -" set files [readdir $dir]\n" -" lappend files . ..\n" -"\n" -" foreach name $files {\n" -" if {[string match $pattern $name]} {\n" -"\n" -" if {[string index $name 0] eq \".\" && [string index $pattern 0] ne \".\"} {\n" -" continue\n" -" }\n" -" lappend result $name\n" -" }\n" -" }\n" -"\n" -" return $result\n" -"}\n" -"\n" -"\n" -"\n" -"\n" -"proc glob.explode {pattern} {\n" -" set oldexp {}\n" -" set newexp {\"\"}\n" -"\n" -" while 1 {\n" -" set oldexp $newexp\n" -" set newexp {}\n" -" set ob [string first \\{ $pattern]\n" -" set cb [string first \\} $pattern]\n" -"\n" -" if {$ob < $cb && $ob != -1} {\n" -" set mid [string range $pattern 0 $ob-1]\n" -" set subexp [lassign [glob.explode [string range $pattern $ob+1 end]] pattern]\n" -" if {$pattern eq \"\"} {\n" -" error \"unmatched open brace in glob pattern\"\n" -" }\n" -" set pattern [string range $pattern 1 end]\n" -"\n" -" foreach subs $subexp {\n" -" foreach sub [split $subs ,] {\n" -" foreach old $oldexp {\n" -" lappend newexp $old$mid$sub\n" -" }\n" -" }\n" -" }\n" -" } elseif {$cb != -1} {\n" -" set suf [string range $pattern 0 $cb-1]\n" -" set rest [string range $pattern $cb end]\n" -" break\n" -" } else {\n" -" set suf $pattern\n" -" set rest \"\"\n" -" break\n" -" }\n" -" }\n" -"\n" -" foreach old $oldexp {\n" -" lappend newexp $old$suf\n" -" }\n" -" list $rest {*}$newexp\n" -"}\n" -"\n" -"\n" -"\n" -"proc glob.glob {base pattern} {\n" -" set dir [file dirname $pattern]\n" -" if {$pattern eq $dir || $pattern eq \"\"} {\n" -" return [list [file join $base $dir] $pattern]\n" -" } elseif {$pattern eq [file tail $pattern]} {\n" -" set dir \"\"\n" -" }\n" -"\n" -"\n" -" set dirlist [glob.glob $base $dir]\n" -" set pattern [file tail $pattern]\n" -"\n" -"\n" -" set result {}\n" -" foreach {realdir dir} $dirlist {\n" -" if {![file isdir $realdir]} {\n" -" continue\n" -" }\n" -" if {[string index $dir end] ne \"/\" && $dir ne \"\"} {\n" -" append dir /\n" -" }\n" -" foreach name [glob.globdir $realdir $pattern] {\n" -" lappend result [file join $realdir $name] $dir$name\n" -" }\n" -" }\n" -" return $result\n" -"}\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"proc glob {args} {\n" -" set nocomplain 0\n" -" set base \"\"\n" -" set tails 0\n" -"\n" -" set n 0\n" -" foreach arg $args {\n" -" if {[info exists param]} {\n" -" set $param $arg\n" -" unset param\n" -" incr n\n" -" continue\n" -" }\n" -" switch -glob -- $arg {\n" -" -d* {\n" -" set switch $arg\n" -" set param base\n" -" }\n" -" -n* {\n" -" set nocomplain 1\n" -" }\n" -" -ta* {\n" -" set tails 1\n" -" }\n" -" -- {\n" -" incr n\n" -" break\n" -" }\n" -" -* {\n" -" return -code error \"bad option \\\"$arg\\\": must be -directory, -nocomplain, -tails, or --\"\n" -" }\n" -" * {\n" -" break\n" -" }\n" -" }\n" -" incr n\n" -" }\n" -" if {[info exists param]} {\n" -" return -code error \"missing argument to \\\"$switch\\\"\"\n" -" }\n" -" if {[llength $args] <= $n} {\n" -" return -code error \"wrong # args: should be \\\"glob ?options? pattern ?pattern ...?\\\"\"\n" -" }\n" -"\n" -" set args [lrange $args $n end]\n" -"\n" -" set result {}\n" -" foreach pattern $args {\n" -" set escpattern [string map {\n" -" \\\\\\\\ \\x01 \\\\\\{ \\x02 \\\\\\} \\x03 \\\\, \\x04\n" -" } $pattern]\n" -" set patexps [lassign [glob.explode $escpattern] rest]\n" -" if {$rest ne \"\"} {\n" -" return -code error \"unmatched close brace in glob pattern\"\n" -" }\n" -" foreach patexp $patexps {\n" -" set patexp [string map {\n" -" \\x01 \\\\\\\\ \\x02 \\{ \\x03 \\} \\x04 ,\n" -" } $patexp]\n" -" foreach {realname name} [glob.glob $base $patexp] {\n" -" incr n\n" -" if {$tails} {\n" -" lappend result $name\n" -" } else {\n" -" lappend result [file join $base $name]\n" -" }\n" -" }\n" -" }\n" -" }\n" -"\n" -" if {!$nocomplain && [llength $result] == 0} {\n" -" set s $(([llength $args] > 1) ? \"s\" : \"\")\n" -" return -code error \"no files matched glob pattern$s \\\"[join $args]\\\"\"\n" -" }\n" -"\n" -" return $result\n" -"}\n" -); -} -int Jim_stdlibInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "stdlib", "1.0", JIM_ERRMSG)) - return JIM_ERR; - - return Jim_EvalSource(interp, "stdlib.tcl", 1, -"\n" -"\n" -"if {![exists -command ref]} {\n" -"\n" -" proc ref {args} {{count 0}} {\n" -" format %08x [incr count]\n" -" }\n" -"}\n" -"\n" -"\n" -"proc lambda {arglist args} {\n" -" tailcall proc [ref {} function lambda.finalizer] $arglist {*}$args\n" -"}\n" -"\n" -"proc lambda.finalizer {name val} {\n" -" rename $name {}\n" -"}\n" -"\n" -"\n" -"proc curry {args} {\n" -" alias [ref {} function lambda.finalizer] {*}$args\n" -"}\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"proc function {value} {\n" -" return $value\n" -"}\n" -"\n" -"\n" -"\n" -"\n" -"proc stacktrace {{skip 0}} {\n" -" set trace {}\n" -" incr skip\n" -" foreach level [range $skip [info level]] {\n" -" lappend trace {*}[info frame -$level]\n" -" }\n" -" return $trace\n" -"}\n" -"\n" -"\n" -"proc stackdump {stacktrace} {\n" -" set lines {}\n" -" foreach {l f p} [lreverse $stacktrace] {\n" -" set line {}\n" -" if {$p ne \"\"} {\n" -" append line \"in procedure '$p' \"\n" -" if {$f ne \"\"} {\n" -" append line \"called \"\n" -" }\n" -" }\n" -" if {$f ne \"\"} {\n" -" append line \"at file \\\"$f\\\", line $l\"\n" -" }\n" -" if {$line ne \"\"} {\n" -" lappend lines $line\n" -" }\n" -" }\n" -" join $lines \\n\n" -"}\n" -"\n" -"\n" -"\n" -"proc defer {script} {\n" -" upvar jim::defer v\n" -" lappend v $script\n" -"}\n" -"\n" -"\n" -"\n" -"proc errorInfo {msg {stacktrace \"\"}} {\n" -" if {$stacktrace eq \"\"} {\n" -"\n" -" set stacktrace [info stacktrace]\n" -"\n" -" lappend stacktrace {*}[stacktrace 1]\n" -" }\n" -" lassign $stacktrace p f l\n" -" if {$f ne \"\"} {\n" -" set result \"$f:$l: Error: \"\n" -" }\n" -" append result \"$msg\\n\"\n" -" append result [stackdump $stacktrace]\n" -"\n" -"\n" -" string trim $result\n" -"}\n" -"\n" -"\n" -"\n" -"proc {info nameofexecutable} {} {\n" -" if {[exists ::jim::exe]} {\n" -" return $::jim::exe\n" -" }\n" -"}\n" -"\n" -"\n" -"proc {dict update} {&varName args script} {\n" -" set keys {}\n" -" foreach {n v} $args {\n" -" upvar $v var_$v\n" -" if {[dict exists $varName $n]} {\n" -" set var_$v [dict get $varName $n]\n" -" }\n" -" }\n" -" catch {uplevel 1 $script} msg opts\n" -" if {[info exists varName]} {\n" -" foreach {n v} $args {\n" -" if {[info exists var_$v]} {\n" -" dict set varName $n [set var_$v]\n" -" } else {\n" -" dict unset varName $n\n" -" }\n" -" }\n" -" }\n" -" return {*}$opts $msg\n" -"}\n" -"\n" -"proc {dict replace} {dictionary {args {key value}}} {\n" -" if {[llength ${key value}] % 2} {\n" -" tailcall {dict replace}\n" -" }\n" -" tailcall dict merge $dictionary ${key value}\n" -"}\n" -"\n" -"\n" -"proc {dict lappend} {varName key {args value}} {\n" -" upvar $varName dict\n" -" if {[exists dict] && [dict exists $dict $key]} {\n" -" set list [dict get $dict $key]\n" -" }\n" -" lappend list {*}$value\n" -" dict set dict $key $list\n" -"}\n" -"\n" -"\n" -"proc {dict append} {varName key {args value}} {\n" -" upvar $varName dict\n" -" if {[exists dict] && [dict exists $dict $key]} {\n" -" set str [dict get $dict $key]\n" -" }\n" -" append str {*}$value\n" -" dict set dict $key $str\n" -"}\n" -"\n" -"\n" -"proc {dict incr} {varName key {increment 1}} {\n" -" upvar $varName dict\n" -" if {[exists dict] && [dict exists $dict $key]} {\n" -" set value [dict get $dict $key]\n" -" }\n" -" incr value $increment\n" -" dict set dict $key $value\n" -"}\n" -"\n" -"\n" -"proc {dict remove} {dictionary {args key}} {\n" -" foreach k $key {\n" -" dict unset dictionary $k\n" -" }\n" -" return $dictionary\n" -"}\n" -"\n" -"\n" -"proc {dict for} {vars dictionary script} {\n" -" if {[llength $vars] != 2} {\n" -" return -code error \"must have exactly two variable names\"\n" -" }\n" -" dict size $dictionary\n" -" tailcall foreach $vars $dictionary $script\n" -"}\n" -); -} -int Jim_tclcompatInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "tclcompat", "1.0", JIM_ERRMSG)) - return JIM_ERR; - - return Jim_EvalSource(interp, "tclcompat.tcl", 1, -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"set env [env]\n" -"\n" -"\n" -"if {[info commands stdout] ne \"\"} {\n" -"\n" -" foreach p {gets flush close eof seek tell} {\n" -" proc $p {chan args} {p} {\n" -" tailcall $chan $p {*}$args\n" -" }\n" -" }\n" -" unset p\n" -"\n" -"\n" -"\n" -" proc puts {{-nonewline {}} {chan stdout} msg} {\n" -" if {${-nonewline} ni {-nonewline {}}} {\n" -" tailcall ${-nonewline} puts $msg\n" -" }\n" -" tailcall $chan puts {*}${-nonewline} $msg\n" -" }\n" -"\n" -"\n" -"\n" -"\n" -"\n" -" proc read {{-nonewline {}} chan} {\n" -" if {${-nonewline} ni {-nonewline {}}} {\n" -" tailcall ${-nonewline} read {*}${chan}\n" -" }\n" -" tailcall $chan read {*}${-nonewline}\n" -" }\n" -"\n" -" proc fconfigure {f args} {\n" -" foreach {n v} $args {\n" -" switch -glob -- $n {\n" -" -bl* {\n" -" $f ndelay $(!$v)\n" -" }\n" -" -bu* {\n" -" $f buffering $v\n" -" }\n" -" -tr* {\n" -"\n" -" }\n" -" default {\n" -" return -code error \"fconfigure: unknown option $n\"\n" -" }\n" -" }\n" -" }\n" -" }\n" -"}\n" -"\n" -"\n" -"proc fileevent {args} {\n" -" tailcall {*}$args\n" -"}\n" -"\n" -"\n" -"\n" -"proc parray {arrayname {pattern *} {puts puts}} {\n" -" upvar $arrayname a\n" -"\n" -" set max 0\n" -" foreach name [array names a $pattern]] {\n" -" if {[string length $name] > $max} {\n" -" set max [string length $name]\n" -" }\n" -" }\n" -" incr max [string length $arrayname]\n" -" incr max 2\n" -" foreach name [lsort [array names a $pattern]] {\n" -" $puts [format \"%-${max}s = %s\" $arrayname\\($name\\) $a($name)]\n" -" }\n" -"}\n" -"\n" -"\n" -"proc {file copy} {{force {}} source target} {\n" -" try {\n" -" if {$force ni {{} -force}} {\n" -" error \"bad option \\\"$force\\\": should be -force\"\n" -" }\n" -"\n" -" set in [open $source rb]\n" -"\n" -" if {[file exists $target]} {\n" -" if {$force eq \"\"} {\n" -" error \"error copying \\\"$source\\\" to \\\"$target\\\": file already exists\"\n" -" }\n" -"\n" -" if {$source eq $target} {\n" -" return\n" -" }\n" -"\n" -"\n" -" file stat $source ss\n" -" file stat $target ts\n" -" if {$ss(dev) == $ts(dev) && $ss(ino) == $ts(ino) && $ss(ino)} {\n" -" return\n" -" }\n" -" }\n" -" set out [open $target wb]\n" -" $in copyto $out\n" -" $out close\n" -" } on error {msg opts} {\n" -" incr opts(-level)\n" -" return {*}$opts $msg\n" -" } finally {\n" -" catch {$in close}\n" -" }\n" -"}\n" -"\n" -"\n" -"\n" -"proc popen {cmd {mode r}} {\n" -" lassign [pipe] r w\n" -" try {\n" -" if {[string match \"w*\" $mode]} {\n" -" lappend cmd <@$r &\n" -" set pids [exec {*}$cmd]\n" -" $r close\n" -" set f $w\n" -" } else {\n" -" lappend cmd >@$w &\n" -" set pids [exec {*}$cmd]\n" -" $w close\n" -" set f $r\n" -" }\n" -" lambda {cmd args} {f pids} {\n" -" if {$cmd eq \"pid\"} {\n" -" return $pids\n" -" }\n" -" if {$cmd eq \"getfd\"} {\n" -" $f getfd\n" -" }\n" -" if {$cmd eq \"close\"} {\n" -" $f close\n" -"\n" -" set retopts {}\n" -" foreach p $pids {\n" -" lassign [wait $p] status - rc\n" -" if {$status eq \"CHILDSTATUS\"} {\n" -" if {$rc == 0} {\n" -" continue\n" -" }\n" -" set msg \"child process exited abnormally\"\n" -" } else {\n" -" set msg \"child killed: received signal\"\n" -" }\n" -" set retopts [list -code error -errorcode [list $status $p $rc] $msg]\n" -" }\n" -" return {*}$retopts\n" -" }\n" -" tailcall $f $cmd {*}$args\n" -" }\n" -" } on error {error opts} {\n" -" $r close\n" -" $w close\n" -" error $error\n" -" }\n" -"}\n" -"\n" -"\n" -"local proc pid {{channelId {}}} {\n" -" if {$channelId eq \"\"} {\n" -" tailcall upcall pid\n" -" }\n" -" if {[catch {$channelId tell}]} {\n" -" return -code error \"can not find channel named \\\"$channelId\\\"\"\n" -" }\n" -" if {[catch {$channelId pid} pids]} {\n" -" return \"\"\n" -" }\n" -" return $pids\n" -"}\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"\n" -"proc try {args} {\n" -" set catchopts {}\n" -" while {[string match -* [lindex $args 0]]} {\n" -" set args [lassign $args opt]\n" -" if {$opt eq \"--\"} {\n" -" break\n" -" }\n" -" lappend catchopts $opt\n" -" }\n" -" if {[llength $args] == 0} {\n" -" return -code error {wrong # args: should be \"try ?options? script ?argument ...?\"}\n" -" }\n" -" set args [lassign $args script]\n" -" set code [catch -eval {*}$catchopts {uplevel 1 $script} msg opts]\n" -"\n" -" set handled 0\n" -"\n" -" foreach {on codes vars script} $args {\n" -" switch -- $on \\\n" -" on {\n" -" if {!$handled && ($codes eq \"*\" || [info returncode $code] in $codes)} {\n" -" lassign $vars msgvar optsvar\n" -" if {$msgvar ne \"\"} {\n" -" upvar $msgvar hmsg\n" -" set hmsg $msg\n" -" }\n" -" if {$optsvar ne \"\"} {\n" -" upvar $optsvar hopts\n" -" set hopts $opts\n" -" }\n" -"\n" -" set code [catch {uplevel 1 $script} msg opts]\n" -" incr handled\n" -" }\n" -" } \\\n" -" finally {\n" -" set finalcode [catch {uplevel 1 $codes} finalmsg finalopts]\n" -" if {$finalcode} {\n" -"\n" -" set code $finalcode\n" -" set msg $finalmsg\n" -" set opts $finalopts\n" -" }\n" -" break\n" -" } \\\n" -" default {\n" -" return -code error \"try: expected 'on' or 'finally', got '$on'\"\n" -" }\n" -" }\n" -"\n" -" if {$code} {\n" -" incr opts(-level)\n" -" return {*}$opts $msg\n" -" }\n" -" return $msg\n" -"}\n" -"\n" -"\n" -"\n" -"proc throw {code {msg \"\"}} {\n" -" return -code $code $msg\n" -"}\n" -"\n" -"\n" -"proc {file delete force} {path} {\n" -" foreach e [readdir $path] {\n" -" file delete -force $path/$e\n" -" }\n" -" file delete $path\n" -"}\n" -); -} - - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif -#include -#include -#include -#include -#ifdef HAVE_UNISTD_H -#include -#include -#endif - - -#if defined(HAVE_SYS_SOCKET_H) && defined(HAVE_SELECT) && defined(HAVE_NETINET_IN_H) && defined(HAVE_NETDB_H) && defined(HAVE_ARPA_INET_H) -#include -#include -#include -#include -#include -#ifdef HAVE_SYS_UN_H -#include -#endif -#define HAVE_SOCKETS -#elif defined (__MINGW32__) - -#else -#define JIM_ANSIC -#endif - -#if defined(JIM_SSL) -#include -#include -#endif - -#ifdef HAVE_TERMIOS_H -#endif - - -#define AIO_CMD_LEN 32 -#define AIO_BUF_LEN 256 - -#ifndef HAVE_FTELLO - #define ftello ftell -#endif -#ifndef HAVE_FSEEKO - #define fseeko fseek -#endif - -#define AIO_KEEPOPEN 1 - -#if defined(JIM_IPV6) -#define IPV6 1 -#else -#define IPV6 0 -#ifndef PF_INET6 -#define PF_INET6 0 -#endif -#endif - -#ifdef JIM_ANSIC - -#undef HAVE_PIPE -#undef HAVE_SOCKETPAIR -#endif - - -struct AioFile; - -typedef struct { - int (*writer)(struct AioFile *af, const char *buf, int len); - int (*reader)(struct AioFile *af, char *buf, int len); - const char *(*getline)(struct AioFile *af, char *buf, int len); - int (*error)(const struct AioFile *af); - const char *(*strerror)(struct AioFile *af); - int (*verify)(struct AioFile *af); -} JimAioFopsType; - -typedef struct AioFile -{ - FILE *fp; - Jim_Obj *filename; - int type; - int openFlags; - int fd; - Jim_Obj *rEvent; - Jim_Obj *wEvent; - Jim_Obj *eEvent; - int addr_family; - void *ssl; - const JimAioFopsType *fops; -} AioFile; - -static int stdio_writer(struct AioFile *af, const char *buf, int len) -{ - return fwrite(buf, 1, len, af->fp); -} - -static int stdio_reader(struct AioFile *af, char *buf, int len) -{ - return fread(buf, 1, len, af->fp); -} - -static const char *stdio_getline(struct AioFile *af, char *buf, int len) -{ - return fgets(buf, len, af->fp); -} - -static int stdio_error(const AioFile *af) -{ - if (!ferror(af->fp)) { - return JIM_OK; - } - clearerr(af->fp); - - if (feof(af->fp) || errno == EAGAIN || errno == EINTR) { - return JIM_OK; - } -#ifdef ECONNRESET - if (errno == ECONNRESET) { - return JIM_OK; - } -#endif -#ifdef ECONNABORTED - if (errno == ECONNABORTED) { - return JIM_OK; - } -#endif - return JIM_ERR; -} - -static const char *stdio_strerror(struct AioFile *af) -{ - return strerror(errno); -} - -static const JimAioFopsType stdio_fops = { - stdio_writer, - stdio_reader, - stdio_getline, - stdio_error, - stdio_strerror, - NULL -}; - - -static int JimAioSubCmdProc(Jim_Interp *interp, int argc, Jim_Obj *const *argv); -static AioFile *JimMakeChannel(Jim_Interp *interp, FILE *fh, int fd, Jim_Obj *filename, - const char *hdlfmt, int family, const char *mode); - - -static const char *JimAioErrorString(AioFile *af) -{ - if (af && af->fops) - return af->fops->strerror(af); - - return strerror(errno); -} - -static void JimAioSetError(Jim_Interp *interp, Jim_Obj *name) -{ - AioFile *af = Jim_CmdPrivData(interp); - - if (name) { - Jim_SetResultFormatted(interp, "%#s: %s", name, JimAioErrorString(af)); - } - else { - Jim_SetResultString(interp, JimAioErrorString(af), -1); - } -} - -static int JimCheckStreamError(Jim_Interp *interp, AioFile *af) -{ - int ret = af->fops->error(af); - if (ret) { - JimAioSetError(interp, af->filename); - } - return ret; -} - -static void JimAioDelProc(Jim_Interp *interp, void *privData) -{ - AioFile *af = privData; - - JIM_NOTUSED(interp); - - Jim_DecrRefCount(interp, af->filename); - -#ifdef jim_ext_eventloop - - Jim_DeleteFileHandler(interp, af->fd, JIM_EVENT_READABLE | JIM_EVENT_WRITABLE | JIM_EVENT_EXCEPTION); -#endif - -#if defined(JIM_SSL) - if (af->ssl != NULL) { - SSL_free(af->ssl); - } -#endif - if (!(af->openFlags & AIO_KEEPOPEN)) { - fclose(af->fp); - } - - Jim_Free(af); -} - -static int aio_cmd_read(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - char buf[AIO_BUF_LEN]; - Jim_Obj *objPtr; - int nonewline = 0; - jim_wide neededLen = -1; - - if (argc && Jim_CompareStringImmediate(interp, argv[0], "-nonewline")) { - nonewline = 1; - argv++; - argc--; - } - if (argc == 1) { - if (Jim_GetWide(interp, argv[0], &neededLen) != JIM_OK) - return JIM_ERR; - if (neededLen < 0) { - Jim_SetResultString(interp, "invalid parameter: negative len", -1); - return JIM_ERR; - } - } - else if (argc) { - return -1; - } - objPtr = Jim_NewStringObj(interp, NULL, 0); - while (neededLen != 0) { - int retval; - int readlen; - - if (neededLen == -1) { - readlen = AIO_BUF_LEN; - } - else { - readlen = (neededLen > AIO_BUF_LEN ? AIO_BUF_LEN : neededLen); - } - retval = af->fops->reader(af, buf, readlen); - if (retval > 0) { - Jim_AppendString(interp, objPtr, buf, retval); - if (neededLen != -1) { - neededLen -= retval; - } - } - if (retval != readlen) - break; - } - - if (JimCheckStreamError(interp, af)) { - Jim_FreeNewObj(interp, objPtr); - return JIM_ERR; - } - if (nonewline) { - int len; - const char *s = Jim_GetString(objPtr, &len); - - if (len > 0 && s[len - 1] == '\n') { - objPtr->length--; - objPtr->bytes[objPtr->length] = '\0'; - } - } - Jim_SetResult(interp, objPtr); - return JIM_OK; -} - -AioFile *Jim_AioFile(Jim_Interp *interp, Jim_Obj *command) -{ - Jim_Cmd *cmdPtr = Jim_GetCommand(interp, command, JIM_ERRMSG); - - - if (cmdPtr && !cmdPtr->isproc && cmdPtr->u.native.cmdProc == JimAioSubCmdProc) { - return (AioFile *) cmdPtr->u.native.privData; - } - Jim_SetResultFormatted(interp, "Not a filehandle: \"%#s\"", command); - return NULL; -} - -FILE *Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *command) -{ - AioFile *af; - - af = Jim_AioFile(interp, command); - if (af == NULL) { - return NULL; - } - - return af->fp; -} - -static int aio_cmd_getfd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - - fflush(af->fp); - Jim_SetResultInt(interp, fileno(af->fp)); - - return JIM_OK; -} - -static int aio_cmd_copy(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - jim_wide count = 0; - jim_wide maxlen = JIM_WIDE_MAX; - AioFile *outf = Jim_AioFile(interp, argv[0]); - - if (outf == NULL) { - return JIM_ERR; - } - - if (argc == 2) { - if (Jim_GetWide(interp, argv[1], &maxlen) != JIM_OK) { - return JIM_ERR; - } - } - - while (count < maxlen) { - char ch; - - if (af->fops->reader(af, &ch, 1) != 1) { - break; - } - if (outf->fops->writer(outf, &ch, 1) != 1) { - break; - } - count++; - } - - if (JimCheckStreamError(interp, af) || JimCheckStreamError(interp, outf)) { - return JIM_ERR; - } - - Jim_SetResultInt(interp, count); - - return JIM_OK; -} - -static int aio_cmd_gets(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - char buf[AIO_BUF_LEN]; - Jim_Obj *objPtr; - int len; - - errno = 0; - - objPtr = Jim_NewStringObj(interp, NULL, 0); - while (1) { - buf[AIO_BUF_LEN - 1] = '_'; - - if (af->fops->getline(af, buf, AIO_BUF_LEN) == NULL) - break; - - if (buf[AIO_BUF_LEN - 1] == '\0' && buf[AIO_BUF_LEN - 2] != '\n') { - Jim_AppendString(interp, objPtr, buf, AIO_BUF_LEN - 1); - } - else { - len = strlen(buf); - - if (len && (buf[len - 1] == '\n')) { - - len--; - } - - Jim_AppendString(interp, objPtr, buf, len); - break; - } - } - - if (JimCheckStreamError(interp, af)) { - - Jim_FreeNewObj(interp, objPtr); - return JIM_ERR; - } - - if (argc) { - if (Jim_SetVariable(interp, argv[0], objPtr) != JIM_OK) { - Jim_FreeNewObj(interp, objPtr); - return JIM_ERR; - } - - len = Jim_Length(objPtr); - - if (len == 0 && feof(af->fp)) { - - len = -1; - } - Jim_SetResultInt(interp, len); - } - else { - Jim_SetResult(interp, objPtr); - } - return JIM_OK; -} - -static int aio_cmd_puts(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - int wlen; - const char *wdata; - Jim_Obj *strObj; - - if (argc == 2) { - if (!Jim_CompareStringImmediate(interp, argv[0], "-nonewline")) { - return -1; - } - strObj = argv[1]; - } - else { - strObj = argv[0]; - } - - wdata = Jim_GetString(strObj, &wlen); - if (af->fops->writer(af, wdata, wlen) == wlen) { - if (argc == 2 || af->fops->writer(af, "\n", 1) == 1) { - return JIM_OK; - } - } - JimAioSetError(interp, af->filename); - return JIM_ERR; -} - -static int aio_cmd_isatty(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ -#ifdef HAVE_ISATTY - AioFile *af = Jim_CmdPrivData(interp); - Jim_SetResultInt(interp, isatty(fileno(af->fp))); -#else - Jim_SetResultInt(interp, 0); -#endif - - return JIM_OK; -} - - -static int aio_cmd_flush(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - - if (fflush(af->fp) == EOF) { - JimAioSetError(interp, af->filename); - return JIM_ERR; - } - return JIM_OK; -} - -static int aio_cmd_eof(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - - Jim_SetResultInt(interp, feof(af->fp)); - return JIM_OK; -} - -static int aio_cmd_close(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc == 3) { -#if defined(HAVE_SOCKETS) && defined(HAVE_SHUTDOWN) - static const char * const options[] = { "r", "w", NULL }; - enum { OPT_R, OPT_W, }; - int option; - AioFile *af = Jim_CmdPrivData(interp); - - if (Jim_GetEnum(interp, argv[2], options, &option, NULL, JIM_ERRMSG) != JIM_OK) { - return JIM_ERR; - } - if (shutdown(af->fd, option == OPT_R ? SHUT_RD : SHUT_WR) == 0) { - return JIM_OK; - } - JimAioSetError(interp, NULL); -#else - Jim_SetResultString(interp, "async close not supported", -1); -#endif - return JIM_ERR; - } - - return Jim_DeleteCommand(interp, Jim_String(argv[0])); -} - -static int aio_cmd_seek(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - int orig = SEEK_SET; - jim_wide offset; - - if (argc == 2) { - if (Jim_CompareStringImmediate(interp, argv[1], "start")) - orig = SEEK_SET; - else if (Jim_CompareStringImmediate(interp, argv[1], "current")) - orig = SEEK_CUR; - else if (Jim_CompareStringImmediate(interp, argv[1], "end")) - orig = SEEK_END; - else { - return -1; - } - } - if (Jim_GetWide(interp, argv[0], &offset) != JIM_OK) { - return JIM_ERR; - } - if (fseeko(af->fp, offset, orig) == -1) { - JimAioSetError(interp, af->filename); - return JIM_ERR; - } - return JIM_OK; -} - -static int aio_cmd_tell(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - - Jim_SetResultInt(interp, ftello(af->fp)); - return JIM_OK; -} - -static int aio_cmd_filename(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - - Jim_SetResult(interp, af->filename); - return JIM_OK; -} - -#ifdef O_NDELAY -static int aio_cmd_ndelay(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - - int fmode = fcntl(af->fd, F_GETFL); - - if (argc) { - long nb; - - if (Jim_GetLong(interp, argv[0], &nb) != JIM_OK) { - return JIM_ERR; - } - if (nb) { - fmode |= O_NDELAY; - } - else { - fmode &= ~O_NDELAY; - } - (void)fcntl(af->fd, F_SETFL, fmode); - } - Jim_SetResultInt(interp, (fmode & O_NONBLOCK) ? 1 : 0); - return JIM_OK; -} -#endif - - -#ifdef HAVE_FSYNC -static int aio_cmd_sync(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - - fflush(af->fp); - fsync(af->fd); - return JIM_OK; -} -#endif - -static int aio_cmd_buffering(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - - static const char * const options[] = { - "none", - "line", - "full", - NULL - }; - enum - { - OPT_NONE, - OPT_LINE, - OPT_FULL, - }; - int option; - - if (Jim_GetEnum(interp, argv[0], options, &option, NULL, JIM_ERRMSG) != JIM_OK) { - return JIM_ERR; - } - switch (option) { - case OPT_NONE: - setvbuf(af->fp, NULL, _IONBF, 0); - break; - case OPT_LINE: - setvbuf(af->fp, NULL, _IOLBF, BUFSIZ); - break; - case OPT_FULL: - setvbuf(af->fp, NULL, _IOFBF, BUFSIZ); - break; - } - return JIM_OK; -} - -#ifdef jim_ext_eventloop -static void JimAioFileEventFinalizer(Jim_Interp *interp, void *clientData) -{ - Jim_Obj **objPtrPtr = clientData; - - Jim_DecrRefCount(interp, *objPtrPtr); - *objPtrPtr = NULL; -} - -static int JimAioFileEventHandler(Jim_Interp *interp, void *clientData, int mask) -{ - Jim_Obj **objPtrPtr = clientData; - - return Jim_EvalObjBackground(interp, *objPtrPtr); -} - -static int aio_eventinfo(Jim_Interp *interp, AioFile * af, unsigned mask, Jim_Obj **scriptHandlerObj, - int argc, Jim_Obj * const *argv) -{ - if (argc == 0) { - - if (*scriptHandlerObj) { - Jim_SetResult(interp, *scriptHandlerObj); - } - return JIM_OK; - } - - if (*scriptHandlerObj) { - - Jim_DeleteFileHandler(interp, af->fd, mask); - } - - - if (Jim_Length(argv[0]) == 0) { - - return JIM_OK; - } - - - Jim_IncrRefCount(argv[0]); - *scriptHandlerObj = argv[0]; - - Jim_CreateFileHandler(interp, af->fd, mask, - JimAioFileEventHandler, scriptHandlerObj, JimAioFileEventFinalizer); - - return JIM_OK; -} - -static int aio_cmd_readable(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - - return aio_eventinfo(interp, af, JIM_EVENT_READABLE, &af->rEvent, argc, argv); -} - -static int aio_cmd_writable(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - - return aio_eventinfo(interp, af, JIM_EVENT_WRITABLE, &af->wEvent, argc, argv); -} - -static int aio_cmd_onexception(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - AioFile *af = Jim_CmdPrivData(interp); - - return aio_eventinfo(interp, af, JIM_EVENT_EXCEPTION, &af->eEvent, argc, argv); -} -#endif - - - - -static const jim_subcmd_type aio_command_table[] = { - { "read", - "?-nonewline? ?len?", - aio_cmd_read, - 0, - 2, - - }, - { "copyto", - "handle ?size?", - aio_cmd_copy, - 1, - 2, - - }, - { "getfd", - NULL, - aio_cmd_getfd, - 0, - 0, - - }, - { "gets", - "?var?", - aio_cmd_gets, - 0, - 1, - - }, - { "puts", - "?-nonewline? str", - aio_cmd_puts, - 1, - 2, - - }, - { "isatty", - NULL, - aio_cmd_isatty, - 0, - 0, - - }, - { "flush", - NULL, - aio_cmd_flush, - 0, - 0, - - }, - { "eof", - NULL, - aio_cmd_eof, - 0, - 0, - - }, - { "close", - "?r(ead)|w(rite)?", - aio_cmd_close, - 0, - 1, - JIM_MODFLAG_FULLARGV, - - }, - { "seek", - "offset ?start|current|end", - aio_cmd_seek, - 1, - 2, - - }, - { "tell", - NULL, - aio_cmd_tell, - 0, - 0, - - }, - { "filename", - NULL, - aio_cmd_filename, - 0, - 0, - - }, -#ifdef O_NDELAY - { "ndelay", - "?0|1?", - aio_cmd_ndelay, - 0, - 1, - - }, -#endif -#ifdef HAVE_FSYNC - { "sync", - NULL, - aio_cmd_sync, - 0, - 0, - - }, -#endif - { "buffering", - "none|line|full", - aio_cmd_buffering, - 1, - 1, - - }, -#ifdef jim_ext_eventloop - { "readable", - "?readable-script?", - aio_cmd_readable, - 0, - 1, - - }, - { "writable", - "?writable-script?", - aio_cmd_writable, - 0, - 1, - - }, - { "onexception", - "?exception-script?", - aio_cmd_onexception, - 0, - 1, - - }, -#endif - { NULL } -}; - -static int JimAioSubCmdProc(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - return Jim_CallSubCmd(interp, Jim_ParseSubCmd(interp, aio_command_table, argc, argv), argc, argv); -} - -static int JimAioOpenCommand(Jim_Interp *interp, int argc, - Jim_Obj *const *argv) -{ - const char *mode; - - if (argc != 2 && argc != 3) { - Jim_WrongNumArgs(interp, 1, argv, "filename ?mode?"); - return JIM_ERR; - } - - mode = (argc == 3) ? Jim_String(argv[2]) : "r"; - -#ifdef jim_ext_tclcompat - { - const char *filename = Jim_String(argv[1]); - - - if (*filename == '|') { - Jim_Obj *evalObj[3]; - - evalObj[0] = Jim_NewStringObj(interp, "::popen", -1); - evalObj[1] = Jim_NewStringObj(interp, filename + 1, -1); - evalObj[2] = Jim_NewStringObj(interp, mode, -1); - - return Jim_EvalObjVector(interp, 3, evalObj); - } - } -#endif - return JimMakeChannel(interp, NULL, -1, argv[1], "aio.handle%ld", 0, mode) ? JIM_OK : JIM_ERR; -} - - -static AioFile *JimMakeChannel(Jim_Interp *interp, FILE *fh, int fd, Jim_Obj *filename, - const char *hdlfmt, int family, const char *mode) -{ - AioFile *af; - char buf[AIO_CMD_LEN]; - int openFlags = 0; - - snprintf(buf, sizeof(buf), hdlfmt, Jim_GetId(interp)); - - if (fh) { - openFlags = AIO_KEEPOPEN; - } - - snprintf(buf, sizeof(buf), hdlfmt, Jim_GetId(interp)); - if (!filename) { - filename = Jim_NewStringObj(interp, buf, -1); - } - - Jim_IncrRefCount(filename); - - if (fh == NULL) { - if (fd >= 0) { -#ifndef JIM_ANSIC - fh = fdopen(fd, mode); -#endif - } - else - fh = fopen(Jim_String(filename), mode); - - if (fh == NULL) { - JimAioSetError(interp, filename); -#ifndef JIM_ANSIC - if (fd >= 0) { - close(fd); - } -#endif - Jim_DecrRefCount(interp, filename); - return NULL; - } - } - - - af = Jim_Alloc(sizeof(*af)); - memset(af, 0, sizeof(*af)); - af->fp = fh; - af->filename = filename; - af->openFlags = openFlags; -#ifndef JIM_ANSIC - af->fd = fileno(fh); -#ifdef FD_CLOEXEC - if ((openFlags & AIO_KEEPOPEN) == 0) { - (void)fcntl(af->fd, F_SETFD, FD_CLOEXEC); - } -#endif -#endif - af->addr_family = family; - af->fops = &stdio_fops; - af->ssl = NULL; - - Jim_CreateCommand(interp, buf, JimAioSubCmdProc, af, JimAioDelProc); - - Jim_SetResult(interp, Jim_MakeGlobalNamespaceName(interp, Jim_NewStringObj(interp, buf, -1))); - - return af; -} - -#if defined(HAVE_PIPE) || (defined(HAVE_SOCKETPAIR) && defined(HAVE_SYS_UN_H)) -static int JimMakeChannelPair(Jim_Interp *interp, int p[2], Jim_Obj *filename, - const char *hdlfmt, int family, const char *mode[2]) -{ - if (JimMakeChannel(interp, NULL, p[0], filename, hdlfmt, family, mode[0])) { - Jim_Obj *objPtr = Jim_NewListObj(interp, NULL, 0); - Jim_ListAppendElement(interp, objPtr, Jim_GetResult(interp)); - if (JimMakeChannel(interp, NULL, p[1], filename, hdlfmt, family, mode[1])) { - Jim_ListAppendElement(interp, objPtr, Jim_GetResult(interp)); - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - } - - - close(p[0]); - close(p[1]); - JimAioSetError(interp, NULL); - return JIM_ERR; -} -#endif - -#ifdef HAVE_PIPE -static int JimAioPipeCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int p[2]; - static const char *mode[2] = { "r", "w" }; - - if (argc != 1) { - Jim_WrongNumArgs(interp, 1, argv, ""); - return JIM_ERR; - } - - if (pipe(p) != 0) { - JimAioSetError(interp, NULL); - return JIM_ERR; - } - - return JimMakeChannelPair(interp, p, argv[0], "aio.pipe%ld", 0, mode); -} -#endif - - - -int Jim_aioInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "aio", "1.0", JIM_ERRMSG)) - return JIM_ERR; - -#if defined(JIM_SSL) - Jim_CreateCommand(interp, "load_ssl_certs", JimAioLoadSSLCertsCommand, NULL, NULL); -#endif - - Jim_CreateCommand(interp, "open", JimAioOpenCommand, NULL, NULL); -#ifdef HAVE_SOCKETS - Jim_CreateCommand(interp, "socket", JimAioSockCommand, NULL, NULL); -#endif -#ifdef HAVE_PIPE - Jim_CreateCommand(interp, "pipe", JimAioPipeCommand, NULL, NULL); -#endif - - - JimMakeChannel(interp, stdin, -1, NULL, "stdin", 0, "r"); - JimMakeChannel(interp, stdout, -1, NULL, "stdout", 0, "w"); - JimMakeChannel(interp, stderr, -1, NULL, "stderr", 0, "w"); - - return JIM_OK; -} - -#include -#include -#include - - -#ifdef HAVE_DIRENT_H -#include -#endif - -int Jim_ReaddirCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const char *dirPath; - DIR *dirPtr; - struct dirent *entryPtr; - int nocomplain = 0; - - if (argc == 3 && Jim_CompareStringImmediate(interp, argv[1], "-nocomplain")) { - nocomplain = 1; - } - if (argc != 2 && !nocomplain) { - Jim_WrongNumArgs(interp, 1, argv, "?-nocomplain? dirPath"); - return JIM_ERR; - } - - dirPath = Jim_String(argv[1 + nocomplain]); - - dirPtr = opendir(dirPath); - if (dirPtr == NULL) { - if (nocomplain) { - return JIM_OK; - } - Jim_SetResultString(interp, strerror(errno), -1); - return JIM_ERR; - } - else { - Jim_Obj *listObj = Jim_NewListObj(interp, NULL, 0); - - while ((entryPtr = readdir(dirPtr)) != NULL) { - if (entryPtr->d_name[0] == '.') { - if (entryPtr->d_name[1] == '\0') { - continue; - } - if ((entryPtr->d_name[1] == '.') && (entryPtr->d_name[2] == '\0')) - continue; - } - Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, entryPtr->d_name, -1)); - } - closedir(dirPtr); - - Jim_SetResult(interp, listObj); - - return JIM_OK; - } -} - -int Jim_readdirInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "readdir", "1.0", JIM_ERRMSG)) - return JIM_ERR; - - Jim_CreateCommand(interp, "readdir", Jim_ReaddirCmd, NULL, NULL); - return JIM_OK; -} - -#include -#include - -#if defined(JIM_REGEXP) -#else - #include -#endif - -static void FreeRegexpInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) -{ - regfree(objPtr->internalRep.ptrIntValue.ptr); - Jim_Free(objPtr->internalRep.ptrIntValue.ptr); -} - -static const Jim_ObjType regexpObjType = { - "regexp", - FreeRegexpInternalRep, - NULL, - NULL, - JIM_TYPE_NONE -}; - -static regex_t *SetRegexpFromAny(Jim_Interp *interp, Jim_Obj *objPtr, unsigned flags) -{ - regex_t *compre; - const char *pattern; - int ret; - - - if (objPtr->typePtr == ®expObjType && - objPtr->internalRep.ptrIntValue.ptr && objPtr->internalRep.ptrIntValue.int1 == flags) { - - return objPtr->internalRep.ptrIntValue.ptr; - } - - - - - pattern = Jim_String(objPtr); - compre = Jim_Alloc(sizeof(regex_t)); - - if ((ret = regcomp(compre, pattern, REG_EXTENDED | flags)) != 0) { - char buf[100]; - - regerror(ret, compre, buf, sizeof(buf)); - Jim_SetResultFormatted(interp, "couldn't compile regular expression pattern: %s", buf); - regfree(compre); - Jim_Free(compre); - return NULL; - } - - Jim_FreeIntRep(interp, objPtr); - - objPtr->typePtr = ®expObjType; - objPtr->internalRep.ptrIntValue.int1 = flags; - objPtr->internalRep.ptrIntValue.ptr = compre; - - return compre; -} - -int Jim_RegexpCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int opt_indices = 0; - int opt_all = 0; - int opt_inline = 0; - regex_t *regex; - int match, i, j; - int offset = 0; - regmatch_t *pmatch = NULL; - int source_len; - int result = JIM_OK; - const char *pattern; - const char *source_str; - int num_matches = 0; - int num_vars; - Jim_Obj *resultListObj = NULL; - int regcomp_flags = 0; - int eflags = 0; - int option; - enum { - OPT_INDICES, OPT_NOCASE, OPT_LINE, OPT_ALL, OPT_INLINE, OPT_START, OPT_END - }; - static const char * const options[] = { - "-indices", "-nocase", "-line", "-all", "-inline", "-start", "--", NULL - }; - - if (argc < 3) { - wrongNumArgs: - Jim_WrongNumArgs(interp, 1, argv, - "?-switch ...? exp string ?matchVar? ?subMatchVar ...?"); - return JIM_ERR; - } - - for (i = 1; i < argc; i++) { - const char *opt = Jim_String(argv[i]); - - if (*opt != '-') { - break; - } - if (Jim_GetEnum(interp, argv[i], options, &option, "switch", JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { - return JIM_ERR; - } - if (option == OPT_END) { - i++; - break; - } - switch (option) { - case OPT_INDICES: - opt_indices = 1; - break; - - case OPT_NOCASE: - regcomp_flags |= REG_ICASE; - break; - - case OPT_LINE: - regcomp_flags |= REG_NEWLINE; - break; - - case OPT_ALL: - opt_all = 1; - break; - - case OPT_INLINE: - opt_inline = 1; - break; - - case OPT_START: - if (++i == argc) { - goto wrongNumArgs; - } - if (Jim_GetIndex(interp, argv[i], &offset) != JIM_OK) { - return JIM_ERR; - } - break; - } - } - if (argc - i < 2) { - goto wrongNumArgs; - } - - regex = SetRegexpFromAny(interp, argv[i], regcomp_flags); - if (!regex) { - return JIM_ERR; - } - - pattern = Jim_String(argv[i]); - source_str = Jim_GetString(argv[i + 1], &source_len); - - num_vars = argc - i - 2; - - if (opt_inline) { - if (num_vars) { - Jim_SetResultString(interp, "regexp match variables not allowed when using -inline", - -1); - result = JIM_ERR; - goto done; - } - num_vars = regex->re_nsub + 1; - } - - pmatch = Jim_Alloc((num_vars + 1) * sizeof(*pmatch)); - - if (offset) { - if (offset < 0) { - offset += source_len + 1; - } - if (offset > source_len) { - source_str += source_len; - } - else if (offset > 0) { - source_str += offset; - } - eflags |= REG_NOTBOL; - } - - if (opt_inline) { - resultListObj = Jim_NewListObj(interp, NULL, 0); - } - - next_match: - match = regexec(regex, source_str, num_vars + 1, pmatch, eflags); - if (match >= REG_BADPAT) { - char buf[100]; - - regerror(match, regex, buf, sizeof(buf)); - Jim_SetResultFormatted(interp, "error while matching pattern: %s", buf); - result = JIM_ERR; - goto done; - } - - if (match == REG_NOMATCH) { - goto done; - } - - num_matches++; - - if (opt_all && !opt_inline) { - - goto try_next_match; - } - - - j = 0; - for (i += 2; opt_inline ? j < num_vars : i < argc; i++, j++) { - Jim_Obj *resultObj; - - if (opt_indices) { - resultObj = Jim_NewListObj(interp, NULL, 0); - } - else { - resultObj = Jim_NewStringObj(interp, "", 0); - } - - if (pmatch[j].rm_so == -1) { - if (opt_indices) { - Jim_ListAppendElement(interp, resultObj, Jim_NewIntObj(interp, -1)); - Jim_ListAppendElement(interp, resultObj, Jim_NewIntObj(interp, -1)); - } - } - else { - int len = pmatch[j].rm_eo - pmatch[j].rm_so; - - if (opt_indices) { - Jim_ListAppendElement(interp, resultObj, Jim_NewIntObj(interp, - offset + pmatch[j].rm_so)); - Jim_ListAppendElement(interp, resultObj, Jim_NewIntObj(interp, - offset + pmatch[j].rm_so + len - 1)); - } - else { - Jim_AppendString(interp, resultObj, source_str + pmatch[j].rm_so, len); - } - } - - if (opt_inline) { - Jim_ListAppendElement(interp, resultListObj, resultObj); - } - else { - - result = Jim_SetVariable(interp, argv[i], resultObj); - - if (result != JIM_OK) { - Jim_FreeObj(interp, resultObj); - break; - } - } - } - - try_next_match: - if (opt_all && (pattern[0] != '^' || (regcomp_flags & REG_NEWLINE)) && *source_str) { - if (pmatch[0].rm_eo) { - offset += pmatch[0].rm_eo; - source_str += pmatch[0].rm_eo; - } - else { - source_str++; - offset++; - } - if (*source_str) { - eflags = REG_NOTBOL; - goto next_match; - } - } - - done: - if (result == JIM_OK) { - if (opt_inline) { - Jim_SetResult(interp, resultListObj); - } - else { - Jim_SetResultInt(interp, num_matches); - } - } - - Jim_Free(pmatch); - return result; -} - -#define MAX_SUB_MATCHES 50 - -int Jim_RegsubCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int regcomp_flags = 0; - int regexec_flags = 0; - int opt_all = 0; - int offset = 0; - regex_t *regex; - const char *p; - int result; - regmatch_t pmatch[MAX_SUB_MATCHES + 1]; - int num_matches = 0; - - int i, j, n; - Jim_Obj *varname; - Jim_Obj *resultObj; - const char *source_str; - int source_len; - const char *replace_str; - int replace_len; - const char *pattern; - int option; - enum { - OPT_NOCASE, OPT_LINE, OPT_ALL, OPT_START, OPT_END - }; - static const char * const options[] = { - "-nocase", "-line", "-all", "-start", "--", NULL - }; - - if (argc < 4) { - wrongNumArgs: - Jim_WrongNumArgs(interp, 1, argv, - "?-switch ...? exp string subSpec ?varName?"); - return JIM_ERR; - } - - for (i = 1; i < argc; i++) { - const char *opt = Jim_String(argv[i]); - - if (*opt != '-') { - break; - } - if (Jim_GetEnum(interp, argv[i], options, &option, "switch", JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { - return JIM_ERR; - } - if (option == OPT_END) { - i++; - break; - } - switch (option) { - case OPT_NOCASE: - regcomp_flags |= REG_ICASE; - break; - - case OPT_LINE: - regcomp_flags |= REG_NEWLINE; - break; - - case OPT_ALL: - opt_all = 1; - break; - - case OPT_START: - if (++i == argc) { - goto wrongNumArgs; - } - if (Jim_GetIndex(interp, argv[i], &offset) != JIM_OK) { - return JIM_ERR; - } - break; - } - } - if (argc - i != 3 && argc - i != 4) { - goto wrongNumArgs; - } - - regex = SetRegexpFromAny(interp, argv[i], regcomp_flags); - if (!regex) { - return JIM_ERR; - } - pattern = Jim_String(argv[i]); - - source_str = Jim_GetString(argv[i + 1], &source_len); - replace_str = Jim_GetString(argv[i + 2], &replace_len); - varname = argv[i + 3]; - - - resultObj = Jim_NewStringObj(interp, "", 0); - - if (offset) { - if (offset < 0) { - offset += source_len + 1; - } - if (offset > source_len) { - offset = source_len; - } - else if (offset < 0) { - offset = 0; - } - } - - - Jim_AppendString(interp, resultObj, source_str, offset); - - - n = source_len - offset; - p = source_str + offset; - do { - int match = regexec(regex, p, MAX_SUB_MATCHES, pmatch, regexec_flags); - - if (match >= REG_BADPAT) { - char buf[100]; - - regerror(match, regex, buf, sizeof(buf)); - Jim_SetResultFormatted(interp, "error while matching pattern: %s", buf); - return JIM_ERR; - } - if (match == REG_NOMATCH) { - break; - } - - num_matches++; - - Jim_AppendString(interp, resultObj, p, pmatch[0].rm_so); - - - for (j = 0; j < replace_len; j++) { - int idx; - int c = replace_str[j]; - - if (c == '&') { - idx = 0; - } - else if (c == '\\' && j < replace_len) { - c = replace_str[++j]; - if ((c >= '0') && (c <= '9')) { - idx = c - '0'; - } - else if ((c == '\\') || (c == '&')) { - Jim_AppendString(interp, resultObj, replace_str + j, 1); - continue; - } - else { - Jim_AppendString(interp, resultObj, replace_str + j - 1, (j == replace_len) ? 1 : 2); - continue; - } - } - else { - Jim_AppendString(interp, resultObj, replace_str + j, 1); - continue; - } - if ((idx < MAX_SUB_MATCHES) && pmatch[idx].rm_so != -1 && pmatch[idx].rm_eo != -1) { - Jim_AppendString(interp, resultObj, p + pmatch[idx].rm_so, - pmatch[idx].rm_eo - pmatch[idx].rm_so); - } - } - - p += pmatch[0].rm_eo; - n -= pmatch[0].rm_eo; - - - if (!opt_all || n == 0) { - break; - } - - - if ((regcomp_flags & REG_NEWLINE) == 0 && pattern[0] == '^') { - break; - } - - - if (pattern[0] == '\0' && n) { - - Jim_AppendString(interp, resultObj, p, 1); - p++; - n--; - } - - regexec_flags |= REG_NOTBOL; - } while (n); - - Jim_AppendString(interp, resultObj, p, -1); - - - if (argc - i == 4) { - result = Jim_SetVariable(interp, varname, resultObj); - - if (result == JIM_OK) { - Jim_SetResultInt(interp, num_matches); - } - else { - Jim_FreeObj(interp, resultObj); - } - } - else { - Jim_SetResult(interp, resultObj); - result = JIM_OK; - } - - return result; -} - -int Jim_regexpInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "regexp", "1.0", JIM_ERRMSG)) - return JIM_ERR; - - Jim_CreateCommand(interp, "regexp", Jim_RegexpCmd, NULL, NULL); - Jim_CreateCommand(interp, "regsub", Jim_RegsubCmd, NULL, NULL); - return JIM_OK; -} - -#include -#include -#include -#include -#include -#include - - -#ifdef HAVE_UTIMES -#include -#endif -#ifdef HAVE_UNISTD_H -#include -#elif defined(_MSC_VER) -#include -#define F_OK 0 -#define W_OK 2 -#define R_OK 4 -#define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) -#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) -#endif - -# ifndef MAXPATHLEN -# define MAXPATHLEN JIM_PATH_LEN -# endif - -#if defined(__MINGW32__) || defined(__MSYS__) || defined(_MSC_VER) -#define ISWINDOWS 1 -#else -#define ISWINDOWS 0 -#endif - - -#if defined(HAVE_STRUCT_STAT_ST_MTIMESPEC) - #define STAT_MTIME_US(STAT) ((STAT).st_mtimespec.tv_sec * 1000000ll + (STAT).st_mtimespec.tv_nsec / 1000) -#elif defined(HAVE_STRUCT_STAT_ST_MTIM) - #define STAT_MTIME_US(STAT) ((STAT).st_mtim.tv_sec * 1000000ll + (STAT).st_mtim.tv_nsec / 1000) -#endif - - -static const char *JimGetFileType(int mode) -{ - if (S_ISREG(mode)) { - return "file"; - } - else if (S_ISDIR(mode)) { - return "directory"; - } -#ifdef S_ISCHR - else if (S_ISCHR(mode)) { - return "characterSpecial"; - } -#endif -#ifdef S_ISBLK - else if (S_ISBLK(mode)) { - return "blockSpecial"; - } -#endif -#ifdef S_ISFIFO - else if (S_ISFIFO(mode)) { - return "fifo"; - } -#endif -#ifdef S_ISLNK - else if (S_ISLNK(mode)) { - return "link"; - } -#endif -#ifdef S_ISSOCK - else if (S_ISSOCK(mode)) { - return "socket"; - } -#endif - return "unknown"; -} - -static void AppendStatElement(Jim_Interp *interp, Jim_Obj *listObj, const char *key, jim_wide value) -{ - Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, key, -1)); - Jim_ListAppendElement(interp, listObj, Jim_NewIntObj(interp, value)); -} - -static int StoreStatData(Jim_Interp *interp, Jim_Obj *varName, const struct stat *sb) -{ - - Jim_Obj *listObj = Jim_NewListObj(interp, NULL, 0); - - AppendStatElement(interp, listObj, "dev", sb->st_dev); - AppendStatElement(interp, listObj, "ino", sb->st_ino); - AppendStatElement(interp, listObj, "mode", sb->st_mode); - AppendStatElement(interp, listObj, "nlink", sb->st_nlink); - AppendStatElement(interp, listObj, "uid", sb->st_uid); - AppendStatElement(interp, listObj, "gid", sb->st_gid); - AppendStatElement(interp, listObj, "size", sb->st_size); - AppendStatElement(interp, listObj, "atime", sb->st_atime); - AppendStatElement(interp, listObj, "mtime", sb->st_mtime); - AppendStatElement(interp, listObj, "ctime", sb->st_ctime); -#ifdef STAT_MTIME_US - AppendStatElement(interp, listObj, "mtimeus", STAT_MTIME_US(*sb)); -#endif - Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, "type", -1)); - Jim_ListAppendElement(interp, listObj, Jim_NewStringObj(interp, JimGetFileType((int)sb->st_mode), -1)); - - - if (varName) { - Jim_Obj *objPtr; - objPtr = Jim_GetVariable(interp, varName, JIM_NONE); - - if (objPtr) { - Jim_Obj *objv[2]; - - objv[0] = objPtr; - objv[1] = listObj; - - objPtr = Jim_DictMerge(interp, 2, objv); - if (objPtr == NULL) { - - Jim_SetResultFormatted(interp, "can't set \"%#s(dev)\": variable isn't array", varName); - Jim_FreeNewObj(interp, listObj); - return JIM_ERR; - } - - Jim_InvalidateStringRep(objPtr); - - Jim_FreeNewObj(interp, listObj); - listObj = objPtr; - } - Jim_SetVariable(interp, varName, listObj); - } - - - Jim_SetResult(interp, listObj); - - return JIM_OK; -} - -static int file_cmd_dirname(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const char *path = Jim_String(argv[0]); - const char *p = strrchr(path, '/'); - - if (!p && path[0] == '.' && path[1] == '.' && path[2] == '\0') { - Jim_SetResultString(interp, "..", -1); - } else if (!p) { - Jim_SetResultString(interp, ".", -1); - } - else if (p == path) { - Jim_SetResultString(interp, "/", -1); - } - else if (ISWINDOWS && p[-1] == ':') { - - Jim_SetResultString(interp, path, p - path + 1); - } - else { - Jim_SetResultString(interp, path, p - path); - } - return JIM_OK; -} - -static int file_cmd_rootname(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const char *path = Jim_String(argv[0]); - const char *lastSlash = strrchr(path, '/'); - const char *p = strrchr(path, '.'); - - if (p == NULL || (lastSlash != NULL && lastSlash > p)) { - Jim_SetResult(interp, argv[0]); - } - else { - Jim_SetResultString(interp, path, p - path); - } - return JIM_OK; -} - -static int file_cmd_extension(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const char *path = Jim_String(argv[0]); - const char *lastSlash = strrchr(path, '/'); - const char *p = strrchr(path, '.'); - - if (p == NULL || (lastSlash != NULL && lastSlash >= p)) { - p = ""; - } - Jim_SetResultString(interp, p, -1); - return JIM_OK; -} - -static int file_cmd_tail(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const char *path = Jim_String(argv[0]); - const char *lastSlash = strrchr(path, '/'); - - if (lastSlash) { - Jim_SetResultString(interp, lastSlash + 1, -1); - } - else { - Jim_SetResult(interp, argv[0]); - } - return JIM_OK; -} - -static int file_cmd_normalize(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ -#ifdef HAVE_REALPATH - const char *path = Jim_String(argv[0]); - char *newname = Jim_Alloc(MAXPATHLEN + 1); - - if (realpath(path, newname)) { - Jim_SetResult(interp, Jim_NewStringObjNoAlloc(interp, newname, -1)); - return JIM_OK; - } - else { - Jim_Free(newname); - Jim_SetResultFormatted(interp, "can't normalize \"%#s\": %s", argv[0], strerror(errno)); - return JIM_ERR; - } -#else - Jim_SetResultString(interp, "Not implemented", -1); - return JIM_ERR; -#endif -} - -static int file_cmd_join(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int i; - char *newname = Jim_Alloc(MAXPATHLEN + 1); - char *last = newname; - - *newname = 0; - - - for (i = 0; i < argc; i++) { - int len; - const char *part = Jim_GetString(argv[i], &len); - - if (*part == '/') { - - last = newname; - } - else if (ISWINDOWS && strchr(part, ':')) { - - last = newname; - } - else if (part[0] == '.') { - if (part[1] == '/') { - part += 2; - len -= 2; - } - else if (part[1] == 0 && last != newname) { - - continue; - } - } - - - if (last != newname && last[-1] != '/') { - *last++ = '/'; - } - - if (len) { - if (last + len - newname >= MAXPATHLEN) { - Jim_Free(newname); - Jim_SetResultString(interp, "Path too long", -1); - return JIM_ERR; - } - memcpy(last, part, len); - last += len; - } - - - if (last > newname + 1 && last[-1] == '/') { - - if (!ISWINDOWS || !(last > newname + 2 && last[-2] == ':')) { - *--last = 0; - } - } - } - - *last = 0; - - - - Jim_SetResult(interp, Jim_NewStringObjNoAlloc(interp, newname, last - newname)); - - return JIM_OK; -} - -static int file_access(Jim_Interp *interp, Jim_Obj *filename, int mode) -{ - Jim_SetResultBool(interp, access(Jim_String(filename), mode) != -1); - - return JIM_OK; -} - -static int file_cmd_readable(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - return file_access(interp, argv[0], R_OK); -} - -static int file_cmd_writable(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - return file_access(interp, argv[0], W_OK); -} - -static int file_cmd_executable(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ -#ifdef X_OK - return file_access(interp, argv[0], X_OK); -#else - - Jim_SetResultBool(interp, 1); - return JIM_OK; -#endif -} - -static int file_cmd_exists(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - return file_access(interp, argv[0], F_OK); -} - -static int file_cmd_delete(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int force = Jim_CompareStringImmediate(interp, argv[0], "-force"); - - if (force || Jim_CompareStringImmediate(interp, argv[0], "--")) { - argc++; - argv--; - } - - while (argc--) { - const char *path = Jim_String(argv[0]); - - if (unlink(path) == -1 && errno != ENOENT) { - if (rmdir(path) == -1) { - - if (!force || Jim_EvalPrefix(interp, "file delete force", 1, argv) != JIM_OK) { - Jim_SetResultFormatted(interp, "couldn't delete file \"%s\": %s", path, - strerror(errno)); - return JIM_ERR; - } - } - } - argv++; - } - return JIM_OK; -} - -#ifdef HAVE_MKDIR_ONE_ARG -#define MKDIR_DEFAULT(PATHNAME) mkdir(PATHNAME) -#else -#define MKDIR_DEFAULT(PATHNAME) mkdir(PATHNAME, 0755) -#endif - -static int mkdir_all(char *path) -{ - int ok = 1; - - - goto first; - - while (ok--) { - - { - char *slash = strrchr(path, '/'); - - if (slash && slash != path) { - *slash = 0; - if (mkdir_all(path) != 0) { - return -1; - } - *slash = '/'; - } - } - first: - if (MKDIR_DEFAULT(path) == 0) { - return 0; - } - if (errno == ENOENT) { - - continue; - } - - if (errno == EEXIST) { - struct stat sb; - - if (stat(path, &sb) == 0 && S_ISDIR(sb.st_mode)) { - return 0; - } - - errno = EEXIST; - } - - break; - } - return -1; -} - -static int file_cmd_mkdir(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - while (argc--) { - char *path = Jim_StrDup(Jim_String(argv[0])); - int rc = mkdir_all(path); - - Jim_Free(path); - if (rc != 0) { - Jim_SetResultFormatted(interp, "can't create directory \"%#s\": %s", argv[0], - strerror(errno)); - return JIM_ERR; - } - argv++; - } - return JIM_OK; -} - -static int file_cmd_tempfile(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int fd = Jim_MakeTempFile(interp, (argc >= 1) ? Jim_String(argv[0]) : NULL, 0); - - if (fd < 0) { - return JIM_ERR; - } - close(fd); - - return JIM_OK; -} - -static int file_cmd_rename(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const char *source; - const char *dest; - int force = 0; - - if (argc == 3) { - if (!Jim_CompareStringImmediate(interp, argv[0], "-force")) { - return -1; - } - force++; - argv++; - argc--; - } - - source = Jim_String(argv[0]); - dest = Jim_String(argv[1]); - - if (!force && access(dest, F_OK) == 0) { - Jim_SetResultFormatted(interp, "error renaming \"%#s\" to \"%#s\": target exists", argv[0], - argv[1]); - return JIM_ERR; - } - - if (rename(source, dest) != 0) { - Jim_SetResultFormatted(interp, "error renaming \"%#s\" to \"%#s\": %s", argv[0], argv[1], - strerror(errno)); - return JIM_ERR; - } - - return JIM_OK; -} - -#if defined(HAVE_LINK) && defined(HAVE_SYMLINK) -static int file_cmd_link(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int ret; - const char *source; - const char *dest; - static const char * const options[] = { "-hard", "-symbolic", NULL }; - enum { OPT_HARD, OPT_SYMBOLIC, }; - int option = OPT_HARD; - - if (argc == 3) { - if (Jim_GetEnum(interp, argv[0], options, &option, NULL, JIM_ENUM_ABBREV | JIM_ERRMSG) != JIM_OK) { - return JIM_ERR; - } - argv++; - argc--; - } - - dest = Jim_String(argv[0]); - source = Jim_String(argv[1]); - - if (option == OPT_HARD) { - ret = link(source, dest); - } - else { - ret = symlink(source, dest); - } - - if (ret != 0) { - Jim_SetResultFormatted(interp, "error linking \"%#s\" to \"%#s\": %s", argv[0], argv[1], - strerror(errno)); - return JIM_ERR; - } - - return JIM_OK; -} -#endif - -static int file_stat(Jim_Interp *interp, Jim_Obj *filename, struct stat *sb) -{ - const char *path = Jim_String(filename); - - if (stat(path, sb) == -1) { - Jim_SetResultFormatted(interp, "could not read \"%#s\": %s", filename, strerror(errno)); - return JIM_ERR; - } - return JIM_OK; -} - -#ifdef HAVE_LSTAT -static int file_lstat(Jim_Interp *interp, Jim_Obj *filename, struct stat *sb) -{ - const char *path = Jim_String(filename); - - if (lstat(path, sb) == -1) { - Jim_SetResultFormatted(interp, "could not read \"%#s\": %s", filename, strerror(errno)); - return JIM_ERR; - } - return JIM_OK; -} -#else -#define file_lstat file_stat -#endif - -static int file_cmd_atime(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct stat sb; - - if (file_stat(interp, argv[0], &sb) != JIM_OK) { - return JIM_ERR; - } - Jim_SetResultInt(interp, sb.st_atime); - return JIM_OK; -} - -static int JimSetFileTimes(Jim_Interp *interp, const char *filename, jim_wide us) -{ -#ifdef HAVE_UTIMES - struct timeval times[2]; - - times[1].tv_sec = times[0].tv_sec = us / 1000000; - times[1].tv_usec = times[0].tv_usec = us % 1000000; - - if (utimes(filename, times) != 0) { - Jim_SetResultFormatted(interp, "can't set time on \"%s\": %s", filename, strerror(errno)); - return JIM_ERR; - } - return JIM_OK; -#else - Jim_SetResultString(interp, "Not implemented", -1); - return JIM_ERR; -#endif -} - -static int file_cmd_mtime(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct stat sb; - - if (argc == 2) { - jim_wide secs; - if (Jim_GetWide(interp, argv[1], &secs) != JIM_OK) { - return JIM_ERR; - } - return JimSetFileTimes(interp, Jim_String(argv[0]), secs * 1000000); - } - if (file_stat(interp, argv[0], &sb) != JIM_OK) { - return JIM_ERR; - } - Jim_SetResultInt(interp, sb.st_mtime); - return JIM_OK; -} - -#ifdef STAT_MTIME_US -static int file_cmd_mtimeus(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct stat sb; - - if (argc == 2) { - jim_wide us; - if (Jim_GetWide(interp, argv[1], &us) != JIM_OK) { - return JIM_ERR; - } - return JimSetFileTimes(interp, Jim_String(argv[0]), us); - } - if (file_stat(interp, argv[0], &sb) != JIM_OK) { - return JIM_ERR; - } - Jim_SetResultInt(interp, STAT_MTIME_US(sb)); - return JIM_OK; -} -#endif - -static int file_cmd_copy(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - return Jim_EvalPrefix(interp, "file copy", argc, argv); -} - -static int file_cmd_size(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct stat sb; - - if (file_stat(interp, argv[0], &sb) != JIM_OK) { - return JIM_ERR; - } - Jim_SetResultInt(interp, sb.st_size); - return JIM_OK; -} - -static int file_cmd_isdirectory(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct stat sb; - int ret = 0; - - if (file_stat(interp, argv[0], &sb) == JIM_OK) { - ret = S_ISDIR(sb.st_mode); - } - Jim_SetResultInt(interp, ret); - return JIM_OK; -} - -static int file_cmd_isfile(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct stat sb; - int ret = 0; - - if (file_stat(interp, argv[0], &sb) == JIM_OK) { - ret = S_ISREG(sb.st_mode); - } - Jim_SetResultInt(interp, ret); - return JIM_OK; -} - -#ifdef HAVE_GETEUID -static int file_cmd_owned(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct stat sb; - int ret = 0; - - if (file_stat(interp, argv[0], &sb) == JIM_OK) { - ret = (geteuid() == sb.st_uid); - } - Jim_SetResultInt(interp, ret); - return JIM_OK; -} -#endif - -#if defined(HAVE_READLINK) -static int file_cmd_readlink(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const char *path = Jim_String(argv[0]); - char *linkValue = Jim_Alloc(MAXPATHLEN + 1); - - int linkLength = readlink(path, linkValue, MAXPATHLEN); - - if (linkLength == -1) { - Jim_Free(linkValue); - Jim_SetResultFormatted(interp, "couldn't readlink \"%#s\": %s", argv[0], strerror(errno)); - return JIM_ERR; - } - linkValue[linkLength] = 0; - Jim_SetResult(interp, Jim_NewStringObjNoAlloc(interp, linkValue, linkLength)); - return JIM_OK; -} -#endif - -static int file_cmd_type(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct stat sb; - - if (file_lstat(interp, argv[0], &sb) != JIM_OK) { - return JIM_ERR; - } - Jim_SetResultString(interp, JimGetFileType((int)sb.st_mode), -1); - return JIM_OK; -} - -#ifdef HAVE_LSTAT -static int file_cmd_lstat(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct stat sb; - - if (file_lstat(interp, argv[0], &sb) != JIM_OK) { - return JIM_ERR; - } - return StoreStatData(interp, argc == 2 ? argv[1] : NULL, &sb); -} -#else -#define file_cmd_lstat file_cmd_stat -#endif - -static int file_cmd_stat(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct stat sb; - - if (file_stat(interp, argv[0], &sb) != JIM_OK) { - return JIM_ERR; - } - return StoreStatData(interp, argc == 2 ? argv[1] : NULL, &sb); -} - -static const jim_subcmd_type file_command_table[] = { - { "atime", - "name", - file_cmd_atime, - 1, - 1, - - }, - { "mtime", - "name ?time?", - file_cmd_mtime, - 1, - 2, - - }, -#ifdef STAT_MTIME_US - { "mtimeus", - "name ?time?", - file_cmd_mtimeus, - 1, - 2, - - }, -#endif - { "copy", - "?-force? source dest", - file_cmd_copy, - 2, - 3, - - }, - { "dirname", - "name", - file_cmd_dirname, - 1, - 1, - - }, - { "rootname", - "name", - file_cmd_rootname, - 1, - 1, - - }, - { "extension", - "name", - file_cmd_extension, - 1, - 1, - - }, - { "tail", - "name", - file_cmd_tail, - 1, - 1, - - }, - { "normalize", - "name", - file_cmd_normalize, - 1, - 1, - - }, - { "join", - "name ?name ...?", - file_cmd_join, - 1, - -1, - - }, - { "readable", - "name", - file_cmd_readable, - 1, - 1, - - }, - { "writable", - "name", - file_cmd_writable, - 1, - 1, - - }, - { "executable", - "name", - file_cmd_executable, - 1, - 1, - - }, - { "exists", - "name", - file_cmd_exists, - 1, - 1, - - }, - { "delete", - "?-force|--? name ...", - file_cmd_delete, - 1, - -1, - - }, - { "mkdir", - "dir ...", - file_cmd_mkdir, - 1, - -1, - - }, - { "tempfile", - "?template?", - file_cmd_tempfile, - 0, - 1, - - }, - { "rename", - "?-force? source dest", - file_cmd_rename, - 2, - 3, - - }, -#if defined(HAVE_LINK) && defined(HAVE_SYMLINK) - { "link", - "?-symbolic|-hard? newname target", - file_cmd_link, - 2, - 3, - - }, -#endif -#if defined(HAVE_READLINK) - { "readlink", - "name", - file_cmd_readlink, - 1, - 1, - - }, -#endif - { "size", - "name", - file_cmd_size, - 1, - 1, - - }, - { "stat", - "name ?var?", - file_cmd_stat, - 1, - 2, - - }, - { "lstat", - "name ?var?", - file_cmd_lstat, - 1, - 2, - - }, - { "type", - "name", - file_cmd_type, - 1, - 1, - - }, -#ifdef HAVE_GETEUID - { "owned", - "name", - file_cmd_owned, - 1, - 1, - - }, -#endif - { "isdirectory", - "name", - file_cmd_isdirectory, - 1, - 1, - - }, - { "isfile", - "name", - file_cmd_isfile, - 1, - 1, - - }, - { - NULL - } -}; - -static int Jim_CdCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const char *path; - - if (argc != 2) { - Jim_WrongNumArgs(interp, 1, argv, "dirname"); - return JIM_ERR; - } - - path = Jim_String(argv[1]); - - if (chdir(path) != 0) { - Jim_SetResultFormatted(interp, "couldn't change working directory to \"%s\": %s", path, - strerror(errno)); - return JIM_ERR; - } - return JIM_OK; -} - -static int Jim_PwdCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - char *cwd = Jim_Alloc(MAXPATHLEN); - - if (getcwd(cwd, MAXPATHLEN) == NULL) { - Jim_SetResultString(interp, "Failed to get pwd", -1); - Jim_Free(cwd); - return JIM_ERR; - } - else if (ISWINDOWS) { - - char *p = cwd; - while ((p = strchr(p, '\\')) != NULL) { - *p++ = '/'; - } - } - - Jim_SetResultString(interp, cwd, -1); - - Jim_Free(cwd); - return JIM_OK; -} - -int Jim_fileInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "file", "1.0", JIM_ERRMSG)) - return JIM_ERR; - - Jim_CreateCommand(interp, "file", Jim_SubCmdProc, (void *)file_command_table, NULL); - Jim_CreateCommand(interp, "pwd", Jim_PwdCmd, NULL, NULL); - Jim_CreateCommand(interp, "cd", Jim_CdCmd, NULL, NULL); - return JIM_OK; -} - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif -#include -#include - - -#if (!defined(HAVE_VFORK) || !defined(HAVE_WAITPID)) && !defined(__MINGW32__) -static int Jim_ExecCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *cmdlineObj = Jim_NewEmptyStringObj(interp); - int i, j; - int rc; - - - for (i = 1; i < argc; i++) { - int len; - const char *arg = Jim_GetString(argv[i], &len); - - if (i > 1) { - Jim_AppendString(interp, cmdlineObj, " ", 1); - } - if (strpbrk(arg, "\\\" ") == NULL) { - - Jim_AppendString(interp, cmdlineObj, arg, len); - continue; - } - - Jim_AppendString(interp, cmdlineObj, "\"", 1); - for (j = 0; j < len; j++) { - if (arg[j] == '\\' || arg[j] == '"') { - Jim_AppendString(interp, cmdlineObj, "\\", 1); - } - Jim_AppendString(interp, cmdlineObj, &arg[j], 1); - } - Jim_AppendString(interp, cmdlineObj, "\"", 1); - } - rc = system(Jim_String(cmdlineObj)); - - Jim_FreeNewObj(interp, cmdlineObj); - - if (rc) { - Jim_Obj *errorCode = Jim_NewListObj(interp, NULL, 0); - Jim_ListAppendElement(interp, errorCode, Jim_NewStringObj(interp, "CHILDSTATUS", -1)); - Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, 0)); - Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, rc)); - Jim_SetGlobalVariableStr(interp, "errorCode", errorCode); - return JIM_ERR; - } - - return JIM_OK; -} - -int Jim_execInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "exec", "1.0", JIM_ERRMSG)) - return JIM_ERR; - - Jim_CreateCommand(interp, "exec", Jim_ExecCmd, NULL, NULL); - return JIM_OK; -} -#else - - -#include -#include -#include - -struct WaitInfoTable; - -static char **JimOriginalEnviron(void); -static char **JimSaveEnv(char **env); -static void JimRestoreEnv(char **env); -static int JimCreatePipeline(Jim_Interp *interp, int argc, Jim_Obj *const *argv, - pidtype **pidArrayPtr, int *inPipePtr, int *outPipePtr, int *errFilePtr); -static void JimDetachPids(struct WaitInfoTable *table, int numPids, const pidtype *pidPtr); -static int JimCleanupChildren(Jim_Interp *interp, int numPids, pidtype *pidPtr, Jim_Obj *errStrObj); -static int Jim_WaitCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv); - -#if defined(__MINGW32__) -static pidtype JimStartWinProcess(Jim_Interp *interp, char **argv, char **env, int inputId, int outputId, int errorId); -#endif - -static void Jim_RemoveTrailingNewline(Jim_Obj *objPtr) -{ - int len; - const char *s = Jim_GetString(objPtr, &len); - - if (len > 0 && s[len - 1] == '\n') { - objPtr->length--; - objPtr->bytes[objPtr->length] = '\0'; - } -} - -static int JimAppendStreamToString(Jim_Interp *interp, int fd, Jim_Obj *strObj) -{ - char buf[256]; - FILE *fh = fdopen(fd, "r"); - int ret = 0; - - if (fh == NULL) { - return -1; - } - - while (1) { - int retval = fread(buf, 1, sizeof(buf), fh); - if (retval > 0) { - ret = 1; - Jim_AppendString(interp, strObj, buf, retval); - } - if (retval != sizeof(buf)) { - break; - } - } - fclose(fh); - return ret; -} - -static char **JimBuildEnv(Jim_Interp *interp) -{ - int i; - int size; - int num; - int n; - char **envptr; - char *envdata; - - Jim_Obj *objPtr = Jim_GetGlobalVariableStr(interp, "env", JIM_NONE); - - if (!objPtr) { - return JimOriginalEnviron(); - } - - - - num = Jim_ListLength(interp, objPtr); - if (num % 2) { - - num--; - } - size = Jim_Length(objPtr) + 2; - - envptr = Jim_Alloc(sizeof(*envptr) * (num / 2 + 1) + size); - envdata = (char *)&envptr[num / 2 + 1]; - - n = 0; - for (i = 0; i < num; i += 2) { - const char *s1, *s2; - Jim_Obj *elemObj; - - Jim_ListIndex(interp, objPtr, i, &elemObj, JIM_NONE); - s1 = Jim_String(elemObj); - Jim_ListIndex(interp, objPtr, i + 1, &elemObj, JIM_NONE); - s2 = Jim_String(elemObj); - - envptr[n] = envdata; - envdata += sprintf(envdata, "%s=%s", s1, s2); - envdata++; - n++; - } - envptr[n] = NULL; - *envdata = 0; - - return envptr; -} - -static void JimFreeEnv(char **env, char **original_environ) -{ - if (env != original_environ) { - Jim_Free(env); - } -} - -static Jim_Obj *JimMakeErrorCode(Jim_Interp *interp, pidtype pid, int waitStatus, Jim_Obj *errStrObj) -{ - Jim_Obj *errorCode = Jim_NewListObj(interp, NULL, 0); - - if (pid == JIM_BAD_PID || pid == JIM_NO_PID) { - Jim_ListAppendElement(interp, errorCode, Jim_NewStringObj(interp, "NONE", -1)); - Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, (long)pid)); - Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, -1)); - } - else if (WIFEXITED(waitStatus)) { - Jim_ListAppendElement(interp, errorCode, Jim_NewStringObj(interp, "CHILDSTATUS", -1)); - Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, (long)pid)); - Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, WEXITSTATUS(waitStatus))); - } - else { - const char *type; - const char *action; - const char *signame; - - if (WIFSIGNALED(waitStatus)) { - type = "CHILDKILLED"; - action = "killed"; - signame = Jim_SignalId(WTERMSIG(waitStatus)); - } - else { - type = "CHILDSUSP"; - action = "suspended"; - signame = "none"; - } - - Jim_ListAppendElement(interp, errorCode, Jim_NewStringObj(interp, type, -1)); - - if (errStrObj) { - Jim_AppendStrings(interp, errStrObj, "child ", action, " by signal ", Jim_SignalId(WTERMSIG(waitStatus)), "\n", NULL); - } - - Jim_ListAppendElement(interp, errorCode, Jim_NewIntObj(interp, (long)pid)); - Jim_ListAppendElement(interp, errorCode, Jim_NewStringObj(interp, signame, -1)); - } - return errorCode; -} - -static int JimCheckWaitStatus(Jim_Interp *interp, pidtype pid, int waitStatus, Jim_Obj *errStrObj) -{ - if (WIFEXITED(waitStatus) && WEXITSTATUS(waitStatus) == 0) { - return JIM_OK; - } - Jim_SetGlobalVariableStr(interp, "errorCode", JimMakeErrorCode(interp, pid, waitStatus, errStrObj)); - - return JIM_ERR; -} - - -struct WaitInfo -{ - pidtype pid; - int status; - int flags; -}; - - -struct WaitInfoTable { - struct WaitInfo *info; - int size; - int used; - int refcount; -}; - - -#define WI_DETACHED 2 - -#define WAIT_TABLE_GROW_BY 4 - -static void JimFreeWaitInfoTable(struct Jim_Interp *interp, void *privData) -{ - struct WaitInfoTable *table = privData; - - if (--table->refcount == 0) { - Jim_Free(table->info); - Jim_Free(table); - } -} - -static struct WaitInfoTable *JimAllocWaitInfoTable(void) -{ - struct WaitInfoTable *table = Jim_Alloc(sizeof(*table)); - table->info = NULL; - table->size = table->used = 0; - table->refcount = 1; - - return table; -} - -static int JimWaitRemove(struct WaitInfoTable *table, pidtype pid) -{ - int i; - - - for (i = 0; i < table->used; i++) { - if (pid == table->info[i].pid) { - if (i != table->used - 1) { - table->info[i] = table->info[table->used - 1]; - } - table->used--; - return 0; - } - } - return -1; -} - -static int Jim_ExecCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int outputId; - int errorId; - pidtype *pidPtr; - int numPids, result; - int child_siginfo = 1; - Jim_Obj *childErrObj; - Jim_Obj *errStrObj; - struct WaitInfoTable *table = Jim_CmdPrivData(interp); - - if (argc > 1 && Jim_CompareStringImmediate(interp, argv[argc - 1], "&")) { - Jim_Obj *listObj; - int i; - - argc--; - numPids = JimCreatePipeline(interp, argc - 1, argv + 1, &pidPtr, NULL, NULL, NULL); - if (numPids < 0) { - return JIM_ERR; - } - - listObj = Jim_NewListObj(interp, NULL, 0); - for (i = 0; i < numPids; i++) { - Jim_ListAppendElement(interp, listObj, Jim_NewIntObj(interp, (long)pidPtr[i])); - } - Jim_SetResult(interp, listObj); - JimDetachPids(table, numPids, pidPtr); - Jim_Free(pidPtr); - return JIM_OK; - } - - numPids = - JimCreatePipeline(interp, argc - 1, argv + 1, &pidPtr, NULL, &outputId, &errorId); - - if (numPids < 0) { - return JIM_ERR; - } - - result = JIM_OK; - - errStrObj = Jim_NewStringObj(interp, "", 0); - - - if (outputId != -1) { - if (JimAppendStreamToString(interp, outputId, errStrObj) < 0) { - result = JIM_ERR; - Jim_SetResultErrno(interp, "error reading from output pipe"); - } - } - - - childErrObj = Jim_NewStringObj(interp, "", 0); - Jim_IncrRefCount(childErrObj); - - if (JimCleanupChildren(interp, numPids, pidPtr, childErrObj) != JIM_OK) { - result = JIM_ERR; - } - - if (errorId != -1) { - int ret; - lseek(errorId, 0, SEEK_SET); - ret = JimAppendStreamToString(interp, errorId, errStrObj); - if (ret < 0) { - Jim_SetResultErrno(interp, "error reading from error pipe"); - result = JIM_ERR; - } - else if (ret > 0) { - - child_siginfo = 0; - } - } - - if (child_siginfo) { - - Jim_AppendObj(interp, errStrObj, childErrObj); - } - Jim_DecrRefCount(interp, childErrObj); - - - Jim_RemoveTrailingNewline(errStrObj); - - - Jim_SetResult(interp, errStrObj); - - return result; -} - -static pidtype JimWaitForProcess(struct WaitInfoTable *table, pidtype pid, int *statusPtr) -{ - if (JimWaitRemove(table, pid) == 0) { - - waitpid(pid, statusPtr, 0); - return pid; - } - - - return JIM_BAD_PID; -} - -static void JimDetachPids(struct WaitInfoTable *table, int numPids, const pidtype *pidPtr) -{ - int j; - - for (j = 0; j < numPids; j++) { - - int i; - for (i = 0; i < table->used; i++) { - if (pidPtr[j] == table->info[i].pid) { - table->info[i].flags |= WI_DETACHED; - break; - } - } - } -} - -static int JimGetChannelFd(Jim_Interp *interp, const char *name) -{ - Jim_Obj *objv[2]; - - objv[0] = Jim_NewStringObj(interp, name, -1); - objv[1] = Jim_NewStringObj(interp, "getfd", -1); - - if (Jim_EvalObjVector(interp, 2, objv) == JIM_OK) { - jim_wide fd; - if (Jim_GetWide(interp, Jim_GetResult(interp), &fd) == JIM_OK) { - return fd; - } - } - return -1; -} - -static void JimReapDetachedPids(struct WaitInfoTable *table) -{ - struct WaitInfo *waitPtr; - int count; - int dest; - - if (!table) { - return; - } - - waitPtr = table->info; - dest = 0; - for (count = table->used; count > 0; waitPtr++, count--) { - if (waitPtr->flags & WI_DETACHED) { - int status; - pidtype pid = waitpid(waitPtr->pid, &status, WNOHANG); - if (pid == waitPtr->pid) { - - table->used--; - continue; - } - } - if (waitPtr != &table->info[dest]) { - table->info[dest] = *waitPtr; - } - dest++; - } -} - -static int Jim_WaitCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct WaitInfoTable *table = Jim_CmdPrivData(interp); - int nohang = 0; - pidtype pid; - long pidarg; - int status; - Jim_Obj *errCodeObj; - - - if (argc == 1) { - JimReapDetachedPids(table); - return JIM_OK; - } - - if (argc > 1 && Jim_CompareStringImmediate(interp, argv[1], "-nohang")) { - nohang = 1; - } - if (argc != nohang + 2) { - Jim_WrongNumArgs(interp, 1, argv, "?-nohang? ?pid?"); - return JIM_ERR; - } - if (Jim_GetLong(interp, argv[nohang + 1], &pidarg) != JIM_OK) { - return JIM_ERR; - } - - pid = waitpid((pidtype)pidarg, &status, nohang ? WNOHANG : 0); - - errCodeObj = JimMakeErrorCode(interp, pid, status, NULL); - - if (pid != JIM_BAD_PID && (WIFEXITED(status) || WIFSIGNALED(status))) { - - JimWaitRemove(table, pid); - } - Jim_SetResult(interp, errCodeObj); - return JIM_OK; -} - -static int Jim_PidCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc != 1) { - Jim_WrongNumArgs(interp, 1, argv, ""); - return JIM_ERR; - } - - Jim_SetResultInt(interp, (jim_wide)getpid()); - return JIM_OK; -} - -static int -JimCreatePipeline(Jim_Interp *interp, int argc, Jim_Obj *const *argv, pidtype **pidArrayPtr, - int *inPipePtr, int *outPipePtr, int *errFilePtr) -{ - pidtype *pidPtr = NULL; /* Points to malloc-ed array holding all - * the pids of child processes. */ - int numPids = 0; /* Actual number of processes that exist - * at *pidPtr right now. */ - int cmdCount; /* Count of number of distinct commands - * found in argc/argv. */ - const char *input = NULL; /* Describes input for pipeline, depending - * on "inputFile". NULL means take input - * from stdin/pipe. */ - int input_len = 0; - -#define FILE_NAME 0 -#define FILE_APPEND 1 -#define FILE_HANDLE 2 -#define FILE_TEXT 3 - - int inputFile = FILE_NAME; /* 1 means input is name of input file. - * 2 means input is filehandle name. - * 0 means input holds actual - * text to be input to command. */ - - int outputFile = FILE_NAME; /* 0 means output is the name of output file. - * 1 means output is the name of output file, and append. - * 2 means output is filehandle name. - * All this is ignored if output is NULL - */ - int errorFile = FILE_NAME; /* 0 means error is the name of error file. - * 1 means error is the name of error file, and append. - * 2 means error is filehandle name. - * All this is ignored if error is NULL - */ - const char *output = NULL; /* Holds name of output file to pipe to, - * or NULL if output goes to stdout/pipe. */ - const char *error = NULL; /* Holds name of stderr file to pipe to, - * or NULL if stderr goes to stderr/pipe. */ - int inputId = -1; - int outputId = -1; - int errorId = -1; - int lastOutputId = -1; - int pipeIds[2]; - int firstArg, lastArg; /* Indexes of first and last arguments in - * current command. */ - int lastBar; - int i; - pidtype pid; - char **save_environ; -#ifndef __MINGW32__ - char **child_environ; -#endif - struct WaitInfoTable *table = Jim_CmdPrivData(interp); - - - char **arg_array = Jim_Alloc(sizeof(*arg_array) * (argc + 1)); - int arg_count = 0; - - if (inPipePtr != NULL) { - *inPipePtr = -1; - } - if (outPipePtr != NULL) { - *outPipePtr = -1; - } - if (errFilePtr != NULL) { - *errFilePtr = -1; - } - pipeIds[0] = pipeIds[1] = -1; - - cmdCount = 1; - lastBar = -1; - for (i = 0; i < argc; i++) { - const char *arg = Jim_String(argv[i]); - - if (arg[0] == '<') { - inputFile = FILE_NAME; - input = arg + 1; - if (*input == '<') { - inputFile = FILE_TEXT; - input_len = Jim_Length(argv[i]) - 2; - input++; - } - else if (*input == '@') { - inputFile = FILE_HANDLE; - input++; - } - - if (!*input && ++i < argc) { - input = Jim_GetString(argv[i], &input_len); - } - } - else if (arg[0] == '>') { - int dup_error = 0; - - outputFile = FILE_NAME; - - output = arg + 1; - if (*output == '>') { - outputFile = FILE_APPEND; - output++; - } - if (*output == '&') { - - output++; - dup_error = 1; - } - if (*output == '@') { - outputFile = FILE_HANDLE; - output++; - } - if (!*output && ++i < argc) { - output = Jim_String(argv[i]); - } - if (dup_error) { - errorFile = outputFile; - error = output; - } - } - else if (arg[0] == '2' && arg[1] == '>') { - error = arg + 2; - errorFile = FILE_NAME; - - if (*error == '@') { - errorFile = FILE_HANDLE; - error++; - } - else if (*error == '>') { - errorFile = FILE_APPEND; - error++; - } - if (!*error && ++i < argc) { - error = Jim_String(argv[i]); - } - } - else { - if (strcmp(arg, "|") == 0 || strcmp(arg, "|&") == 0) { - if (i == lastBar + 1 || i == argc - 1) { - Jim_SetResultString(interp, "illegal use of | or |& in command", -1); - goto badargs; - } - lastBar = i; - cmdCount++; - } - - arg_array[arg_count++] = (char *)arg; - continue; - } - - if (i >= argc) { - Jim_SetResultFormatted(interp, "can't specify \"%s\" as last word in command", arg); - goto badargs; - } - } - - if (arg_count == 0) { - Jim_SetResultString(interp, "didn't specify command to execute", -1); -badargs: - Jim_Free(arg_array); - return -1; - } - - - save_environ = JimSaveEnv(JimBuildEnv(interp)); - - if (input != NULL) { - if (inputFile == FILE_TEXT) { - inputId = Jim_MakeTempFile(interp, NULL, 1); - if (inputId == -1) { - goto error; - } - if (write(inputId, input, input_len) != input_len) { - Jim_SetResultErrno(interp, "couldn't write temp file"); - close(inputId); - goto error; - } - lseek(inputId, 0L, SEEK_SET); - } - else if (inputFile == FILE_HANDLE) { - int fd = JimGetChannelFd(interp, input); - - if (fd < 0) { - goto error; - } - inputId = dup(fd); - } - else { - inputId = Jim_OpenForRead(input); - if (inputId == -1) { - Jim_SetResultFormatted(interp, "couldn't read file \"%s\": %s", input, strerror(Jim_Errno())); - goto error; - } - } - } - else if (inPipePtr != NULL) { - if (pipe(pipeIds) != 0) { - Jim_SetResultErrno(interp, "couldn't create input pipe for command"); - goto error; - } - inputId = pipeIds[0]; - *inPipePtr = pipeIds[1]; - pipeIds[0] = pipeIds[1] = -1; - } - - if (output != NULL) { - if (outputFile == FILE_HANDLE) { - int fd = JimGetChannelFd(interp, output); - if (fd < 0) { - goto error; - } - lastOutputId = dup(fd); - } - else { - lastOutputId = Jim_OpenForWrite(output, outputFile == FILE_APPEND); - if (lastOutputId == -1) { - Jim_SetResultFormatted(interp, "couldn't write file \"%s\": %s", output, strerror(Jim_Errno())); - goto error; - } - } - } - else if (outPipePtr != NULL) { - if (pipe(pipeIds) != 0) { - Jim_SetResultErrno(interp, "couldn't create output pipe"); - goto error; - } - lastOutputId = pipeIds[1]; - *outPipePtr = pipeIds[0]; - pipeIds[0] = pipeIds[1] = -1; - } - - if (error != NULL) { - if (errorFile == FILE_HANDLE) { - if (strcmp(error, "1") == 0) { - - if (lastOutputId != -1) { - errorId = dup(lastOutputId); - } - else { - - error = "stdout"; - } - } - if (errorId == -1) { - int fd = JimGetChannelFd(interp, error); - if (fd < 0) { - goto error; - } - errorId = dup(fd); - } - } - else { - errorId = Jim_OpenForWrite(error, errorFile == FILE_APPEND); - if (errorId == -1) { - Jim_SetResultFormatted(interp, "couldn't write file \"%s\": %s", error, strerror(Jim_Errno())); - goto error; - } - } - } - else if (errFilePtr != NULL) { - errorId = Jim_MakeTempFile(interp, NULL, 1); - if (errorId == -1) { - goto error; - } - *errFilePtr = dup(errorId); - } - - - pidPtr = Jim_Alloc(cmdCount * sizeof(*pidPtr)); - for (i = 0; i < numPids; i++) { - pidPtr[i] = JIM_BAD_PID; - } - for (firstArg = 0; firstArg < arg_count; numPids++, firstArg = lastArg + 1) { - int pipe_dup_err = 0; - int origErrorId = errorId; - - for (lastArg = firstArg; lastArg < arg_count; lastArg++) { - if (strcmp(arg_array[lastArg], "|") == 0) { - break; - } - if (strcmp(arg_array[lastArg], "|&") == 0) { - pipe_dup_err = 1; - break; - } - } - - if (lastArg == firstArg) { - Jim_SetResultString(interp, "missing command to exec", -1); - goto error; - } - - - arg_array[lastArg] = NULL; - if (lastArg == arg_count) { - outputId = lastOutputId; - lastOutputId = -1; - } - else { - if (pipe(pipeIds) != 0) { - Jim_SetResultErrno(interp, "couldn't create pipe"); - goto error; - } - outputId = pipeIds[1]; - } - - - if (pipe_dup_err) { - errorId = outputId; - } - - - -#ifdef __MINGW32__ - pid = JimStartWinProcess(interp, &arg_array[firstArg], save_environ, inputId, outputId, errorId); - if (pid == JIM_BAD_PID) { - Jim_SetResultFormatted(interp, "couldn't exec \"%s\"", arg_array[firstArg]); - goto error; - } -#else - i = strlen(arg_array[firstArg]); - - child_environ = Jim_GetEnviron(); - pid = vfork(); - if (pid < 0) { - Jim_SetResultErrno(interp, "couldn't fork child process"); - goto error; - } - if (pid == 0) { - - - if (inputId != -1) { - dup2(inputId, fileno(stdin)); - close(inputId); - } - if (outputId != -1) { - dup2(outputId, fileno(stdout)); - if (outputId != errorId) { - close(outputId); - } - } - if (errorId != -1) { - dup2(errorId, fileno(stderr)); - close(errorId); - } - - if (outPipePtr) { - close(*outPipePtr); - } - if (errFilePtr) { - close(*errFilePtr); - } - if (pipeIds[0] != -1) { - close(pipeIds[0]); - } - if (lastOutputId != -1) { - close(lastOutputId); - } - - - (void)signal(SIGPIPE, SIG_DFL); - - execvpe(arg_array[firstArg], &arg_array[firstArg], child_environ); - - if (write(fileno(stderr), "couldn't exec \"", 15) && - write(fileno(stderr), arg_array[firstArg], i) && - write(fileno(stderr), "\"\n", 2)) { - - } -#ifdef JIM_MAINTAINER - { - - static char *const false_argv[2] = {"false", NULL}; - execvp(false_argv[0],false_argv); - } -#endif - _exit(127); - } -#endif - - - - if (table->used == table->size) { - table->size += WAIT_TABLE_GROW_BY; - table->info = Jim_Realloc(table->info, table->size * sizeof(*table->info)); - } - - table->info[table->used].pid = pid; - table->info[table->used].flags = 0; - table->used++; - - pidPtr[numPids] = pid; - - - errorId = origErrorId; - - - if (inputId != -1) { - close(inputId); - } - if (outputId != -1) { - close(outputId); - } - inputId = pipeIds[0]; - pipeIds[0] = pipeIds[1] = -1; - } - *pidArrayPtr = pidPtr; - - - cleanup: - if (inputId != -1) { - close(inputId); - } - if (lastOutputId != -1) { - close(lastOutputId); - } - if (errorId != -1) { - close(errorId); - } - Jim_Free(arg_array); - - JimRestoreEnv(save_environ); - - return numPids; - - - error: - if ((inPipePtr != NULL) && (*inPipePtr != -1)) { - close(*inPipePtr); - *inPipePtr = -1; - } - if ((outPipePtr != NULL) && (*outPipePtr != -1)) { - close(*outPipePtr); - *outPipePtr = -1; - } - if ((errFilePtr != NULL) && (*errFilePtr != -1)) { - close(*errFilePtr); - *errFilePtr = -1; - } - if (pipeIds[0] != -1) { - close(pipeIds[0]); - } - if (pipeIds[1] != -1) { - close(pipeIds[1]); - } - if (pidPtr != NULL) { - for (i = 0; i < numPids; i++) { - if (pidPtr[i] != JIM_BAD_PID) { - JimDetachPids(table, 1, &pidPtr[i]); - } - } - Jim_Free(pidPtr); - } - numPids = -1; - goto cleanup; -} - - -static int JimCleanupChildren(Jim_Interp *interp, int numPids, pidtype *pidPtr, Jim_Obj *errStrObj) -{ - struct WaitInfoTable *table = Jim_CmdPrivData(interp); - int result = JIM_OK; - int i; - - - for (i = 0; i < numPids; i++) { - int waitStatus = 0; - if (JimWaitForProcess(table, pidPtr[i], &waitStatus) != JIM_BAD_PID) { - if (JimCheckWaitStatus(interp, pidPtr[i], waitStatus, errStrObj) != JIM_OK) { - result = JIM_ERR; - } - } - } - Jim_Free(pidPtr); - - return result; -} - -int Jim_execInit(Jim_Interp *interp) -{ - struct WaitInfoTable *waitinfo; - if (Jim_PackageProvide(interp, "exec", "1.0", JIM_ERRMSG)) - return JIM_ERR; - -#ifdef SIGPIPE - (void)signal(SIGPIPE, SIG_IGN); -#endif - - waitinfo = JimAllocWaitInfoTable(); - Jim_CreateCommand(interp, "exec", Jim_ExecCmd, waitinfo, JimFreeWaitInfoTable); - waitinfo->refcount++; - Jim_CreateCommand(interp, "wait", Jim_WaitCommand, waitinfo, JimFreeWaitInfoTable); - Jim_CreateCommand(interp, "pid", Jim_PidCommand, 0, 0); - - return JIM_OK; -} - -#if defined(__MINGW32__) - - -static int -JimWinFindExecutable(const char *originalName, char fullPath[MAX_PATH]) -{ - int i; - static char extensions[][5] = {".exe", "", ".bat"}; - - for (i = 0; i < (int) (sizeof(extensions) / sizeof(extensions[0])); i++) { - snprintf(fullPath, MAX_PATH, "%s%s", originalName, extensions[i]); - - if (SearchPath(NULL, fullPath, NULL, MAX_PATH, fullPath, NULL) == 0) { - continue; - } - if (GetFileAttributes(fullPath) & FILE_ATTRIBUTE_DIRECTORY) { - continue; - } - return 0; - } - - return -1; -} - -static char **JimSaveEnv(char **env) -{ - return env; -} - -static void JimRestoreEnv(char **env) -{ - JimFreeEnv(env, Jim_GetEnviron()); -} - -static char **JimOriginalEnviron(void) -{ - return NULL; -} - -static Jim_Obj * -JimWinBuildCommandLine(Jim_Interp *interp, char **argv) -{ - char *start, *special; - int quote, i; - - Jim_Obj *strObj = Jim_NewStringObj(interp, "", 0); - - for (i = 0; argv[i]; i++) { - if (i > 0) { - Jim_AppendString(interp, strObj, " ", 1); - } - - if (argv[i][0] == '\0') { - quote = 1; - } - else { - quote = 0; - for (start = argv[i]; *start != '\0'; start++) { - if (isspace(UCHAR(*start))) { - quote = 1; - break; - } - } - } - if (quote) { - Jim_AppendString(interp, strObj, "\"" , 1); - } - - start = argv[i]; - for (special = argv[i]; ; ) { - if ((*special == '\\') && (special[1] == '\\' || - special[1] == '"' || (quote && special[1] == '\0'))) { - Jim_AppendString(interp, strObj, start, special - start); - start = special; - while (1) { - special++; - if (*special == '"' || (quote && *special == '\0')) { - - Jim_AppendString(interp, strObj, start, special - start); - break; - } - if (*special != '\\') { - break; - } - } - Jim_AppendString(interp, strObj, start, special - start); - start = special; - } - if (*special == '"') { - if (special == start) { - Jim_AppendString(interp, strObj, "\"", 1); - } - else { - Jim_AppendString(interp, strObj, start, special - start); - } - Jim_AppendString(interp, strObj, "\\\"", 2); - start = special + 1; - } - if (*special == '\0') { - break; - } - special++; - } - Jim_AppendString(interp, strObj, start, special - start); - if (quote) { - Jim_AppendString(interp, strObj, "\"", 1); - } - } - return strObj; -} - -static pidtype -JimStartWinProcess(Jim_Interp *interp, char **argv, char **env, int inputId, int outputId, int errorId) -{ - STARTUPINFO startInfo; - PROCESS_INFORMATION procInfo; - HANDLE hProcess; - char execPath[MAX_PATH]; - pidtype pid = JIM_BAD_PID; - Jim_Obj *cmdLineObj; - char *winenv; - - if (JimWinFindExecutable(argv[0], execPath) < 0) { - return JIM_BAD_PID; - } - argv[0] = execPath; - - hProcess = GetCurrentProcess(); - cmdLineObj = JimWinBuildCommandLine(interp, argv); - - - ZeroMemory(&startInfo, sizeof(startInfo)); - startInfo.cb = sizeof(startInfo); - startInfo.dwFlags = STARTF_USESTDHANDLES; - startInfo.hStdInput = INVALID_HANDLE_VALUE; - startInfo.hStdOutput= INVALID_HANDLE_VALUE; - startInfo.hStdError = INVALID_HANDLE_VALUE; - - if (inputId == -1) { - inputId = _fileno(stdin); - } - DuplicateHandle(hProcess, (HANDLE)_get_osfhandle(inputId), hProcess, &startInfo.hStdInput, - 0, TRUE, DUPLICATE_SAME_ACCESS); - if (startInfo.hStdInput == INVALID_HANDLE_VALUE) { - goto end; - } - - if (outputId == -1) { - outputId = _fileno(stdout); - } - DuplicateHandle(hProcess, (HANDLE)_get_osfhandle(outputId), hProcess, &startInfo.hStdOutput, - 0, TRUE, DUPLICATE_SAME_ACCESS); - if (startInfo.hStdOutput == INVALID_HANDLE_VALUE) { - goto end; - } - - - if (errorId == -1) { - errorId = _fileno(stderr); - } - DuplicateHandle(hProcess, (HANDLE)_get_osfhandle(errorId), hProcess, &startInfo.hStdError, - 0, TRUE, DUPLICATE_SAME_ACCESS); - if (startInfo.hStdError == INVALID_HANDLE_VALUE) { - goto end; - } - - if (env == NULL) { - - winenv = NULL; - } - else if (env[0] == NULL) { - winenv = (char *)"\0"; - } - else { - winenv = env[0]; - } - - if (!CreateProcess(NULL, (char *)Jim_String(cmdLineObj), NULL, NULL, TRUE, - 0, winenv, NULL, &startInfo, &procInfo)) { - goto end; - } - - - WaitForInputIdle(procInfo.hProcess, 5000); - CloseHandle(procInfo.hThread); - - pid = procInfo.hProcess; - - end: - Jim_FreeNewObj(interp, cmdLineObj); - if (startInfo.hStdInput != INVALID_HANDLE_VALUE) { - CloseHandle(startInfo.hStdInput); - } - if (startInfo.hStdOutput != INVALID_HANDLE_VALUE) { - CloseHandle(startInfo.hStdOutput); - } - if (startInfo.hStdError != INVALID_HANDLE_VALUE) { - CloseHandle(startInfo.hStdError); - } - return pid; -} - -#else - -static char **JimOriginalEnviron(void) -{ - return Jim_GetEnviron(); -} - -static char **JimSaveEnv(char **env) -{ - char **saveenv = Jim_GetEnviron(); - Jim_SetEnviron(env); - return saveenv; -} - -static void JimRestoreEnv(char **env) -{ - JimFreeEnv(Jim_GetEnviron(), env); - Jim_SetEnviron(env); -} -#endif -#endif - - - -#ifdef STRPTIME_NEEDS_XOPEN_SOURCE -#ifndef _XOPEN_SOURCE -#define _XOPEN_SOURCE 500 -#endif -#endif - - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif - -#include -#include -#include -#include - - -#ifdef HAVE_SYS_TIME_H -#include -#endif - -struct clock_options { - int gmt; - const char *format; -}; - -static int parse_clock_options(Jim_Interp *interp, int argc, Jim_Obj *const *argv, struct clock_options *opts) -{ - static const char * const options[] = { "-gmt", "-format", NULL }; - enum { OPT_GMT, OPT_FORMAT, }; - int i; - - for (i = 0; i < argc; i += 2) { - int option; - if (Jim_GetEnum(interp, argv[i], options, &option, NULL, JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { - return JIM_ERR; - } - switch (option) { - case OPT_GMT: - if (Jim_GetBoolean(interp, argv[i + 1], &opts->gmt) != JIM_OK) { - return JIM_ERR; - } - break; - case OPT_FORMAT: - opts->format = Jim_String(argv[i + 1]); - break; - } - } - return JIM_OK; -} - -static int clock_cmd_format(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - - char buf[100]; - time_t t; - jim_wide seconds; - struct clock_options options = { 0, "%a %b %d %H:%M:%S %Z %Y" }; - struct tm *tm; - - if (Jim_GetWide(interp, argv[0], &seconds) != JIM_OK) { - return JIM_ERR; - } - if (argc % 2 == 0) { - return -1; - } - if (parse_clock_options(interp, argc - 1, argv + 1, &options) == JIM_ERR) { - return JIM_ERR; - } - - t = seconds; - tm = options.gmt ? gmtime(&t) : localtime(&t); - - if (tm == NULL || strftime(buf, sizeof(buf), options.format, tm) == 0) { - Jim_SetResultString(interp, "format string too long or invalid time", -1); - return JIM_ERR; - } - - Jim_SetResultString(interp, buf, -1); - - return JIM_OK; -} - -#ifdef HAVE_STRPTIME -static time_t jim_timegm(const struct tm *tm) -{ - int m = tm->tm_mon + 1; - int y = 1900 + tm->tm_year - (m <= 2); - int era = (y >= 0 ? y : y - 399) / 400; - unsigned yoe = (unsigned)(y - era * 400); - unsigned doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + tm->tm_mday - 1; - unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; - long days = (era * 146097 + (int)doe - 719468); - int secs = tm->tm_hour * 3600 + tm->tm_min * 60 + tm->tm_sec; - - return days * 24 * 60 * 60 + secs; -} - -static int clock_cmd_scan(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - char *pt; - struct tm tm; - time_t now = time(NULL); - - struct clock_options options = { 0, NULL }; - - if (argc % 2 == 0) { - return -1; - } - - if (parse_clock_options(interp, argc - 1, argv + 1, &options) == JIM_ERR) { - return JIM_ERR; - } - if (options.format == NULL) { - return -1; - } - - localtime_r(&now, &tm); - - pt = strptime(Jim_String(argv[0]), options.format, &tm); - if (pt == 0 || *pt != 0) { - Jim_SetResultString(interp, "Failed to parse time according to format", -1); - return JIM_ERR; - } - - - Jim_SetResultInt(interp, options.gmt ? jim_timegm(&tm) : mktime(&tm)); - - return JIM_OK; -} -#endif - -static int clock_cmd_seconds(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_SetResultInt(interp, time(NULL)); - - return JIM_OK; -} - -static int clock_cmd_micros(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct timeval tv; - - gettimeofday(&tv, NULL); - - Jim_SetResultInt(interp, (jim_wide) tv.tv_sec * 1000000 + tv.tv_usec); - - return JIM_OK; -} - -static int clock_cmd_millis(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - struct timeval tv; - - gettimeofday(&tv, NULL); - - Jim_SetResultInt(interp, (jim_wide) tv.tv_sec * 1000 + tv.tv_usec / 1000); - - return JIM_OK; -} - -static const jim_subcmd_type clock_command_table[] = { - { "clicks", - NULL, - clock_cmd_micros, - 0, - 0, - - }, - { "format", - "seconds ?-format string? ?-gmt boolean?", - clock_cmd_format, - 1, - 5, - - }, - { "microseconds", - NULL, - clock_cmd_micros, - 0, - 0, - - }, - { "milliseconds", - NULL, - clock_cmd_millis, - 0, - 0, - - }, -#ifdef HAVE_STRPTIME - { "scan", - "str -format format ?-gmt boolean?", - clock_cmd_scan, - 3, - 5, - - }, -#endif - { "seconds", - NULL, - clock_cmd_seconds, - 0, - 0, - - }, - { NULL } -}; - -int Jim_clockInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "clock", "1.0", JIM_ERRMSG)) - return JIM_ERR; - - Jim_CreateCommand(interp, "clock", Jim_SubCmdProc, (void *)clock_command_table, NULL); - return JIM_OK; -} - -#include -#include -#include -#include -#include - - -static int array_cmd_exists(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - - Jim_Obj *dictObj = Jim_GetVariable(interp, argv[0], JIM_UNSHARED); - Jim_SetResultInt(interp, dictObj && Jim_DictSize(interp, dictObj) != -1); - return JIM_OK; -} - -static int array_cmd_get(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *objPtr = Jim_GetVariable(interp, argv[0], JIM_NONE); - Jim_Obj *patternObj; - - if (!objPtr) { - return JIM_OK; - } - - patternObj = (argc == 1) ? NULL : argv[1]; - - - if (patternObj == NULL || Jim_CompareStringImmediate(interp, patternObj, "*")) { - if (Jim_IsList(objPtr) && Jim_ListLength(interp, objPtr) % 2 == 0) { - - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - } - - return Jim_DictMatchTypes(interp, objPtr, patternObj, JIM_DICTMATCH_KEYS, JIM_DICTMATCH_KEYS | JIM_DICTMATCH_VALUES); -} - -static int array_cmd_names(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *objPtr = Jim_GetVariable(interp, argv[0], JIM_NONE); - - if (!objPtr) { - return JIM_OK; - } - - return Jim_DictMatchTypes(interp, objPtr, argc == 1 ? NULL : argv[1], JIM_DICTMATCH_KEYS, JIM_DICTMATCH_KEYS); -} - -static int array_cmd_unset(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int i; - int len; - Jim_Obj *resultObj; - Jim_Obj *objPtr; - Jim_Obj **dictValuesObj; - - if (argc == 1 || Jim_CompareStringImmediate(interp, argv[1], "*")) { - - Jim_UnsetVariable(interp, argv[0], JIM_NONE); - return JIM_OK; - } - - objPtr = Jim_GetVariable(interp, argv[0], JIM_NONE); - - if (objPtr == NULL) { - - return JIM_OK; - } - - if (Jim_DictPairs(interp, objPtr, &dictValuesObj, &len) != JIM_OK) { - - Jim_SetResultString(interp, "", -1); - return JIM_OK; - } - - - resultObj = Jim_NewDictObj(interp, NULL, 0); - - for (i = 0; i < len; i += 2) { - if (!Jim_StringMatchObj(interp, argv[1], dictValuesObj[i], 0)) { - Jim_DictAddElement(interp, resultObj, dictValuesObj[i], dictValuesObj[i + 1]); - } - } - Jim_Free(dictValuesObj); - - Jim_SetVariable(interp, argv[0], resultObj); - return JIM_OK; -} - -static int array_cmd_size(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *objPtr; - int len = 0; - - - objPtr = Jim_GetVariable(interp, argv[0], JIM_NONE); - if (objPtr) { - len = Jim_DictSize(interp, objPtr); - if (len < 0) { - - Jim_SetResultInt(interp, 0); - return JIM_OK; - } - } - - Jim_SetResultInt(interp, len); - - return JIM_OK; -} - -static int array_cmd_stat(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *objPtr = Jim_GetVariable(interp, argv[0], JIM_NONE); - if (objPtr) { - return Jim_DictInfo(interp, objPtr); - } - Jim_SetResultFormatted(interp, "\"%#s\" isn't an array", argv[0], NULL); - return JIM_ERR; -} - -static int array_cmd_set(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int i; - int len; - Jim_Obj *listObj = argv[1]; - Jim_Obj *dictObj; - - len = Jim_ListLength(interp, listObj); - if (len % 2) { - Jim_SetResultString(interp, "list must have an even number of elements", -1); - return JIM_ERR; - } - - dictObj = Jim_GetVariable(interp, argv[0], JIM_UNSHARED); - if (!dictObj) { - - return Jim_SetVariable(interp, argv[0], listObj); - } - else if (Jim_DictSize(interp, dictObj) < 0) { - return JIM_ERR; - } - - if (Jim_IsShared(dictObj)) { - dictObj = Jim_DuplicateObj(interp, dictObj); - } - - for (i = 0; i < len; i += 2) { - Jim_Obj *nameObj; - Jim_Obj *valueObj; - - Jim_ListIndex(interp, listObj, i, &nameObj, JIM_NONE); - Jim_ListIndex(interp, listObj, i + 1, &valueObj, JIM_NONE); - - Jim_DictAddElement(interp, dictObj, nameObj, valueObj); - } - return Jim_SetVariable(interp, argv[0], dictObj); -} - -static const jim_subcmd_type array_command_table[] = { - { "exists", - "arrayName", - array_cmd_exists, - 1, - 1, - - }, - { "get", - "arrayName ?pattern?", - array_cmd_get, - 1, - 2, - - }, - { "names", - "arrayName ?pattern?", - array_cmd_names, - 1, - 2, - - }, - { "set", - "arrayName list", - array_cmd_set, - 2, - 2, - - }, - { "size", - "arrayName", - array_cmd_size, - 1, - 1, - - }, - { "stat", - "arrayName", - array_cmd_stat, - 1, - 1, - - }, - { "unset", - "arrayName ?pattern?", - array_cmd_unset, - 1, - 2, - - }, - { NULL - } -}; - -int Jim_arrayInit(Jim_Interp *interp) -{ - if (Jim_PackageProvide(interp, "array", "1.0", JIM_ERRMSG)) - return JIM_ERR; - - Jim_CreateCommand(interp, "array", Jim_SubCmdProc, (void *)array_command_table, NULL); - return JIM_OK; -} -int Jim_InitStaticExtensions(Jim_Interp *interp) -{ -extern int Jim_bootstrapInit(Jim_Interp *); -extern int Jim_aioInit(Jim_Interp *); -extern int Jim_readdirInit(Jim_Interp *); -extern int Jim_regexpInit(Jim_Interp *); -extern int Jim_fileInit(Jim_Interp *); -extern int Jim_globInit(Jim_Interp *); -extern int Jim_execInit(Jim_Interp *); -extern int Jim_clockInit(Jim_Interp *); -extern int Jim_arrayInit(Jim_Interp *); -extern int Jim_stdlibInit(Jim_Interp *); -extern int Jim_tclcompatInit(Jim_Interp *); -Jim_bootstrapInit(interp); -Jim_aioInit(interp); -Jim_readdirInit(interp); -Jim_regexpInit(interp); -Jim_fileInit(interp); -Jim_globInit(interp); -Jim_execInit(interp); -Jim_clockInit(interp); -Jim_arrayInit(interp); -Jim_stdlibInit(interp); -Jim_tclcompatInit(interp); -return JIM_OK; -} -#define JIM_OPTIMIZATION -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - - -#ifdef HAVE_SYS_TIME_H -#include -#endif -#ifdef HAVE_BACKTRACE -#include -#endif -#ifdef HAVE_CRT_EXTERNS_H -#include -#endif - - -#include - - - - - -#ifndef TCL_LIBRARY -#define TCL_LIBRARY "." -#endif -#ifndef TCL_PLATFORM_OS -#define TCL_PLATFORM_OS "unknown" -#endif -#ifndef TCL_PLATFORM_PLATFORM -#define TCL_PLATFORM_PLATFORM "unknown" -#endif -#ifndef TCL_PLATFORM_PATH_SEPARATOR -#define TCL_PLATFORM_PATH_SEPARATOR ":" -#endif - - - - - - - -#ifdef JIM_MAINTAINER -#define JIM_DEBUG_COMMAND -#define JIM_DEBUG_PANIC -#endif - - - -#define JIM_INTEGER_SPACE 24 - -const char *jim_tt_name(int type); - -#ifdef JIM_DEBUG_PANIC -static void JimPanicDump(int fail_condition, const char *fmt, ...); -#define JimPanic(X) JimPanicDump X -#else -#define JimPanic(X) -#endif - -#ifdef JIM_OPTIMIZATION -#define JIM_IF_OPTIM(X) X -#else -#define JIM_IF_OPTIM(X) -#endif - - -static char JimEmptyStringRep[] = ""; - -static void JimFreeCallFrame(Jim_Interp *interp, Jim_CallFrame *cf, int action); -static int ListSetIndex(Jim_Interp *interp, Jim_Obj *listPtr, int listindex, Jim_Obj *newObjPtr, - int flags); -static int JimDeleteLocalProcs(Jim_Interp *interp, Jim_Stack *localCommands); -static Jim_Obj *JimExpandDictSugar(Jim_Interp *interp, Jim_Obj *objPtr); -static void SetDictSubstFromAny(Jim_Interp *interp, Jim_Obj *objPtr); -static Jim_Obj **JimDictPairs(Jim_Obj *dictPtr, int *len); -static void JimSetFailedEnumResult(Jim_Interp *interp, const char *arg, const char *badtype, - const char *prefix, const char *const *tablePtr, const char *name); -static int JimCallProcedure(Jim_Interp *interp, Jim_Cmd *cmd, int argc, Jim_Obj *const *argv); -static int JimGetWideNoErr(Jim_Interp *interp, Jim_Obj *objPtr, jim_wide * widePtr); -static int JimSign(jim_wide w); -static int JimValidName(Jim_Interp *interp, const char *type, Jim_Obj *nameObjPtr); -static void JimPrngSeed(Jim_Interp *interp, unsigned char *seed, int seedLen); -static void JimRandomBytes(Jim_Interp *interp, void *dest, unsigned int len); - - - -#define JimWideValue(objPtr) (objPtr)->internalRep.wideValue - -#define JimObjTypeName(O) ((O)->typePtr ? (O)->typePtr->name : "none") - -static int utf8_tounicode_case(const char *s, int *uc, int upper) -{ - int l = utf8_tounicode(s, uc); - if (upper) { - *uc = utf8_upper(*uc); - } - return l; -} - - -#define JIM_CHARSET_SCAN 2 -#define JIM_CHARSET_GLOB 0 - -static const char *JimCharsetMatch(const char *pattern, int c, int flags) -{ - int not = 0; - int pchar; - int match = 0; - int nocase = 0; - - if (flags & JIM_NOCASE) { - nocase++; - c = utf8_upper(c); - } - - if (flags & JIM_CHARSET_SCAN) { - if (*pattern == '^') { - not++; - pattern++; - } - - - if (*pattern == ']') { - goto first; - } - } - - while (*pattern && *pattern != ']') { - - if (pattern[0] == '\\') { -first: - pattern += utf8_tounicode_case(pattern, &pchar, nocase); - } - else { - - int start; - int end; - - pattern += utf8_tounicode_case(pattern, &start, nocase); - if (pattern[0] == '-' && pattern[1]) { - - pattern++; - pattern += utf8_tounicode_case(pattern, &end, nocase); - - - if ((c >= start && c <= end) || (c >= end && c <= start)) { - match = 1; - } - continue; - } - pchar = start; - } - - if (pchar == c) { - match = 1; - } - } - if (not) { - match = !match; - } - - return match ? pattern : NULL; -} - - - -static int JimGlobMatch(const char *pattern, const char *string, int nocase) -{ - int c; - int pchar; - while (*pattern) { - switch (pattern[0]) { - case '*': - while (pattern[1] == '*') { - pattern++; - } - pattern++; - if (!pattern[0]) { - return 1; - } - while (*string) { - - if (JimGlobMatch(pattern, string, nocase)) - return 1; - string += utf8_tounicode(string, &c); - } - return 0; - - case '?': - string += utf8_tounicode(string, &c); - break; - - case '[': { - string += utf8_tounicode(string, &c); - pattern = JimCharsetMatch(pattern + 1, c, nocase ? JIM_NOCASE : 0); - if (!pattern) { - return 0; - } - if (!*pattern) { - - continue; - } - break; - } - case '\\': - if (pattern[1]) { - pattern++; - } - - default: - string += utf8_tounicode_case(string, &c, nocase); - utf8_tounicode_case(pattern, &pchar, nocase); - if (pchar != c) { - return 0; - } - break; - } - pattern += utf8_tounicode_case(pattern, &pchar, nocase); - if (!*string) { - while (*pattern == '*') { - pattern++; - } - break; - } - } - if (!*pattern && !*string) { - return 1; - } - return 0; -} - -static int JimStringCompare(const char *s1, int l1, const char *s2, int l2) -{ - if (l1 < l2) { - return memcmp(s1, s2, l1) <= 0 ? -1 : 1; - } - else if (l2 < l1) { - return memcmp(s1, s2, l2) >= 0 ? 1 : -1; - } - else { - return JimSign(memcmp(s1, s2, l1)); - } -} - -static int JimStringCompareLen(const char *s1, const char *s2, int maxchars, int nocase) -{ - while (*s1 && *s2 && maxchars) { - int c1, c2; - s1 += utf8_tounicode_case(s1, &c1, nocase); - s2 += utf8_tounicode_case(s2, &c2, nocase); - if (c1 != c2) { - return JimSign(c1 - c2); - } - maxchars--; - } - if (!maxchars) { - return 0; - } - - if (*s1) { - return 1; - } - if (*s2) { - return -1; - } - return 0; -} - -static int JimStringFirst(const char *s1, int l1, const char *s2, int l2, int idx) -{ - int i; - int l1bytelen; - - if (!l1 || !l2 || l1 > l2) { - return -1; - } - if (idx < 0) - idx = 0; - s2 += utf8_index(s2, idx); - - l1bytelen = utf8_index(s1, l1); - - for (i = idx; i <= l2 - l1; i++) { - int c; - if (memcmp(s2, s1, l1bytelen) == 0) { - return i; - } - s2 += utf8_tounicode(s2, &c); - } - return -1; -} - -static int JimStringLast(const char *s1, int l1, const char *s2, int l2) -{ - const char *p; - - if (!l1 || !l2 || l1 > l2) - return -1; - - - for (p = s2 + l2 - 1; p != s2 - 1; p--) { - if (*p == *s1 && memcmp(s1, p, l1) == 0) { - return p - s2; - } - } - return -1; -} - -#ifdef JIM_UTF8 -static int JimStringLastUtf8(const char *s1, int l1, const char *s2, int l2) -{ - int n = JimStringLast(s1, utf8_index(s1, l1), s2, utf8_index(s2, l2)); - if (n > 0) { - n = utf8_strlen(s2, n); - } - return n; -} -#endif - -static int JimCheckConversion(const char *str, const char *endptr) -{ - if (str[0] == '\0' || str == endptr) { - return JIM_ERR; - } - - if (endptr[0] != '\0') { - while (*endptr) { - if (!isspace(UCHAR(*endptr))) { - return JIM_ERR; - } - endptr++; - } - } - return JIM_OK; -} - -static int JimNumberBase(const char *str, int *base, int *sign) -{ - int i = 0; - - *base = 10; - - while (isspace(UCHAR(str[i]))) { - i++; - } - - if (str[i] == '-') { - *sign = -1; - i++; - } - else { - if (str[i] == '+') { - i++; - } - *sign = 1; - } - - if (str[i] != '0') { - - return 0; - } - - - switch (str[i + 1]) { - case 'x': case 'X': *base = 16; break; - case 'o': case 'O': *base = 8; break; - case 'b': case 'B': *base = 2; break; - default: return 0; - } - i += 2; - - if (str[i] != '-' && str[i] != '+' && !isspace(UCHAR(str[i]))) { - - return i; - } - - *base = 10; - return 0; -} - -static long jim_strtol(const char *str, char **endptr) -{ - int sign; - int base; - int i = JimNumberBase(str, &base, &sign); - - if (base != 10) { - long value = strtol(str + i, endptr, base); - if (endptr == NULL || *endptr != str + i) { - return value * sign; - } - } - - - return strtol(str, endptr, 10); -} - - -static jim_wide jim_strtoull(const char *str, char **endptr) -{ -#ifdef HAVE_LONG_LONG - int sign; - int base; - int i = JimNumberBase(str, &base, &sign); - - if (base != 10) { - jim_wide value = strtoull(str + i, endptr, base); - if (endptr == NULL || *endptr != str + i) { - return value * sign; - } - } - - - return strtoull(str, endptr, 10); -#else - return (unsigned long)jim_strtol(str, endptr); -#endif -} - -int Jim_StringToWide(const char *str, jim_wide * widePtr, int base) -{ - char *endptr; - - if (base) { - *widePtr = strtoull(str, &endptr, base); - } - else { - *widePtr = jim_strtoull(str, &endptr); - } - - return JimCheckConversion(str, endptr); -} - -int Jim_StringToDouble(const char *str, double *doublePtr) -{ - char *endptr; - - - errno = 0; - - *doublePtr = strtod(str, &endptr); - - return JimCheckConversion(str, endptr); -} - -static jim_wide JimPowWide(jim_wide b, jim_wide e) -{ - jim_wide res = 1; - - - if (b == 1) { - - return 1; - } - if (e < 0) { - if (b != -1) { - return 0; - } - e = -e; - } - while (e) - { - if (e & 1) { - res *= b; - } - e >>= 1; - b *= b; - } - return res; -} - -#ifdef JIM_DEBUG_PANIC -static void JimPanicDump(int condition, const char *fmt, ...) -{ - va_list ap; - - if (!condition) { - return; - } - - va_start(ap, fmt); - - fprintf(stderr, "\nJIM INTERPRETER PANIC: "); - vfprintf(stderr, fmt, ap); - fprintf(stderr, "\n\n"); - va_end(ap); - -#ifdef HAVE_BACKTRACE - { - void *array[40]; - int size, i; - char **strings; - - size = backtrace(array, 40); - strings = backtrace_symbols(array, size); - for (i = 0; i < size; i++) - fprintf(stderr, "[backtrace] %s\n", strings[i]); - fprintf(stderr, "[backtrace] Include the above lines and the output\n"); - fprintf(stderr, "[backtrace] of 'nm ' in the bug report.\n"); - } -#endif - - exit(1); -} -#endif - - -void *Jim_Alloc(int size) -{ - return size ? malloc(size) : NULL; -} - -void Jim_Free(void *ptr) -{ - free(ptr); -} - -void *Jim_Realloc(void *ptr, int size) -{ - return realloc(ptr, size); -} - -char *Jim_StrDup(const char *s) -{ - return strdup(s); -} - -char *Jim_StrDupLen(const char *s, int l) -{ - char *copy = Jim_Alloc(l + 1); - - memcpy(copy, s, l + 1); - copy[l] = 0; - return copy; -} - - - -static jim_wide JimClock(void) -{ - struct timeval tv; - - gettimeofday(&tv, NULL); - return (jim_wide) tv.tv_sec * 1000000 + tv.tv_usec; -} - - - -static void JimExpandHashTableIfNeeded(Jim_HashTable *ht); -static unsigned int JimHashTableNextPower(unsigned int size); -static Jim_HashEntry *JimInsertHashEntry(Jim_HashTable *ht, const void *key, int replace); - - - - -unsigned int Jim_IntHashFunction(unsigned int key) -{ - key += ~(key << 15); - key ^= (key >> 10); - key += (key << 3); - key ^= (key >> 6); - key += ~(key << 11); - key ^= (key >> 16); - return key; -} - -unsigned int Jim_GenHashFunction(const unsigned char *buf, int len) -{ - unsigned int h = 0; - - while (len--) - h += (h << 3) + *buf++; - return h; -} - - - - -static void JimResetHashTable(Jim_HashTable *ht) -{ - ht->table = NULL; - ht->size = 0; - ht->sizemask = 0; - ht->used = 0; - ht->collisions = 0; -#ifdef JIM_RANDOMISE_HASH - ht->uniq = (rand() ^ time(NULL) ^ clock()); -#else - ht->uniq = 0; -#endif -} - -static void JimInitHashTableIterator(Jim_HashTable *ht, Jim_HashTableIterator *iter) -{ - iter->ht = ht; - iter->index = -1; - iter->entry = NULL; - iter->nextEntry = NULL; -} - - -int Jim_InitHashTable(Jim_HashTable *ht, const Jim_HashTableType *type, void *privDataPtr) -{ - JimResetHashTable(ht); - ht->type = type; - ht->privdata = privDataPtr; - return JIM_OK; -} - -void Jim_ResizeHashTable(Jim_HashTable *ht) -{ - int minimal = ht->used; - - if (minimal < JIM_HT_INITIAL_SIZE) - minimal = JIM_HT_INITIAL_SIZE; - Jim_ExpandHashTable(ht, minimal); -} - - -void Jim_ExpandHashTable(Jim_HashTable *ht, unsigned int size) -{ - Jim_HashTable n; - unsigned int realsize = JimHashTableNextPower(size), i; - - if (size <= ht->used) - return; - - Jim_InitHashTable(&n, ht->type, ht->privdata); - n.size = realsize; - n.sizemask = realsize - 1; - n.table = Jim_Alloc(realsize * sizeof(Jim_HashEntry *)); - - n.uniq = ht->uniq; - - - memset(n.table, 0, realsize * sizeof(Jim_HashEntry *)); - - n.used = ht->used; - for (i = 0; ht->used > 0; i++) { - Jim_HashEntry *he, *nextHe; - - if (ht->table[i] == NULL) - continue; - - - he = ht->table[i]; - while (he) { - unsigned int h; - - nextHe = he->next; - - h = Jim_HashKey(ht, he->key) & n.sizemask; - he->next = n.table[h]; - n.table[h] = he; - ht->used--; - - he = nextHe; - } - } - assert(ht->used == 0); - Jim_Free(ht->table); - - - *ht = n; -} - - -int Jim_AddHashEntry(Jim_HashTable *ht, const void *key, void *val) -{ - Jim_HashEntry *entry; - - entry = JimInsertHashEntry(ht, key, 0); - if (entry == NULL) - return JIM_ERR; - - - Jim_SetHashKey(ht, entry, key); - Jim_SetHashVal(ht, entry, val); - return JIM_OK; -} - - -int Jim_ReplaceHashEntry(Jim_HashTable *ht, const void *key, void *val) -{ - int existed; - Jim_HashEntry *entry; - - entry = JimInsertHashEntry(ht, key, 1); - if (entry->key) { - if (ht->type->valDestructor && ht->type->valDup) { - void *newval = ht->type->valDup(ht->privdata, val); - ht->type->valDestructor(ht->privdata, entry->u.val); - entry->u.val = newval; - } - else { - Jim_FreeEntryVal(ht, entry); - Jim_SetHashVal(ht, entry, val); - } - existed = 1; - } - else { - - Jim_SetHashKey(ht, entry, key); - Jim_SetHashVal(ht, entry, val); - existed = 0; - } - - return existed; -} - - -int Jim_DeleteHashEntry(Jim_HashTable *ht, const void *key) -{ - unsigned int h; - Jim_HashEntry *he, *prevHe; - - if (ht->used == 0) - return JIM_ERR; - h = Jim_HashKey(ht, key) & ht->sizemask; - he = ht->table[h]; - - prevHe = NULL; - while (he) { - if (Jim_CompareHashKeys(ht, key, he->key)) { - - if (prevHe) - prevHe->next = he->next; - else - ht->table[h] = he->next; - Jim_FreeEntryKey(ht, he); - Jim_FreeEntryVal(ht, he); - Jim_Free(he); - ht->used--; - return JIM_OK; - } - prevHe = he; - he = he->next; - } - return JIM_ERR; -} - - -int Jim_FreeHashTable(Jim_HashTable *ht) -{ - unsigned int i; - - - for (i = 0; ht->used > 0; i++) { - Jim_HashEntry *he, *nextHe; - - if ((he = ht->table[i]) == NULL) - continue; - while (he) { - nextHe = he->next; - Jim_FreeEntryKey(ht, he); - Jim_FreeEntryVal(ht, he); - Jim_Free(he); - ht->used--; - he = nextHe; - } - } - - Jim_Free(ht->table); - - JimResetHashTable(ht); - return JIM_OK; -} - -Jim_HashEntry *Jim_FindHashEntry(Jim_HashTable *ht, const void *key) -{ - Jim_HashEntry *he; - unsigned int h; - - if (ht->used == 0) - return NULL; - h = Jim_HashKey(ht, key) & ht->sizemask; - he = ht->table[h]; - while (he) { - if (Jim_CompareHashKeys(ht, key, he->key)) - return he; - he = he->next; - } - return NULL; -} - -Jim_HashTableIterator *Jim_GetHashTableIterator(Jim_HashTable *ht) -{ - Jim_HashTableIterator *iter = Jim_Alloc(sizeof(*iter)); - JimInitHashTableIterator(ht, iter); - return iter; -} - -Jim_HashEntry *Jim_NextHashEntry(Jim_HashTableIterator *iter) -{ - while (1) { - if (iter->entry == NULL) { - iter->index++; - if (iter->index >= (signed)iter->ht->size) - break; - iter->entry = iter->ht->table[iter->index]; - } - else { - iter->entry = iter->nextEntry; - } - if (iter->entry) { - iter->nextEntry = iter->entry->next; - return iter->entry; - } - } - return NULL; -} - - - - -static void JimExpandHashTableIfNeeded(Jim_HashTable *ht) -{ - if (ht->size == 0) - Jim_ExpandHashTable(ht, JIM_HT_INITIAL_SIZE); - if (ht->size == ht->used) - Jim_ExpandHashTable(ht, ht->size * 2); -} - - -static unsigned int JimHashTableNextPower(unsigned int size) -{ - unsigned int i = JIM_HT_INITIAL_SIZE; - - if (size >= 2147483648U) - return 2147483648U; - while (1) { - if (i >= size) - return i; - i *= 2; - } -} - -static Jim_HashEntry *JimInsertHashEntry(Jim_HashTable *ht, const void *key, int replace) -{ - unsigned int h; - Jim_HashEntry *he; - - - JimExpandHashTableIfNeeded(ht); - - - h = Jim_HashKey(ht, key) & ht->sizemask; - - he = ht->table[h]; - while (he) { - if (Jim_CompareHashKeys(ht, key, he->key)) - return replace ? he : NULL; - he = he->next; - } - - - he = Jim_Alloc(sizeof(*he)); - he->next = ht->table[h]; - ht->table[h] = he; - ht->used++; - he->key = NULL; - - return he; -} - - - -static unsigned int JimStringCopyHTHashFunction(const void *key) -{ - return Jim_GenHashFunction(key, strlen(key)); -} - -static void *JimStringCopyHTDup(void *privdata, const void *key) -{ - return Jim_StrDup(key); -} - -static int JimStringCopyHTKeyCompare(void *privdata, const void *key1, const void *key2) -{ - return strcmp(key1, key2) == 0; -} - -static void JimStringCopyHTKeyDestructor(void *privdata, void *key) -{ - Jim_Free(key); -} - -static const Jim_HashTableType JimPackageHashTableType = { - JimStringCopyHTHashFunction, - JimStringCopyHTDup, - NULL, - JimStringCopyHTKeyCompare, - JimStringCopyHTKeyDestructor, - NULL -}; - -typedef struct AssocDataValue -{ - Jim_InterpDeleteProc *delProc; - void *data; -} AssocDataValue; - -static void JimAssocDataHashTableValueDestructor(void *privdata, void *data) -{ - AssocDataValue *assocPtr = (AssocDataValue *) data; - - if (assocPtr->delProc != NULL) - assocPtr->delProc((Jim_Interp *)privdata, assocPtr->data); - Jim_Free(data); -} - -static const Jim_HashTableType JimAssocDataHashTableType = { - JimStringCopyHTHashFunction, - JimStringCopyHTDup, - NULL, - JimStringCopyHTKeyCompare, - JimStringCopyHTKeyDestructor, - JimAssocDataHashTableValueDestructor -}; - -void Jim_InitStack(Jim_Stack *stack) -{ - stack->len = 0; - stack->maxlen = 0; - stack->vector = NULL; -} - -void Jim_FreeStack(Jim_Stack *stack) -{ - Jim_Free(stack->vector); -} - -int Jim_StackLen(Jim_Stack *stack) -{ - return stack->len; -} - -void Jim_StackPush(Jim_Stack *stack, void *element) -{ - int neededLen = stack->len + 1; - - if (neededLen > stack->maxlen) { - stack->maxlen = neededLen < 20 ? 20 : neededLen * 2; - stack->vector = Jim_Realloc(stack->vector, sizeof(void *) * stack->maxlen); - } - stack->vector[stack->len] = element; - stack->len++; -} - -void *Jim_StackPop(Jim_Stack *stack) -{ - if (stack->len == 0) - return NULL; - stack->len--; - return stack->vector[stack->len]; -} - -void *Jim_StackPeek(Jim_Stack *stack) -{ - if (stack->len == 0) - return NULL; - return stack->vector[stack->len - 1]; -} - -void Jim_FreeStackElements(Jim_Stack *stack, void (*freeFunc) (void *ptr)) -{ - int i; - - for (i = 0; i < stack->len; i++) - freeFunc(stack->vector[i]); -} - - - -#define JIM_TT_NONE 0 -#define JIM_TT_STR 1 -#define JIM_TT_ESC 2 -#define JIM_TT_VAR 3 -#define JIM_TT_DICTSUGAR 4 -#define JIM_TT_CMD 5 - -#define JIM_TT_SEP 6 -#define JIM_TT_EOL 7 -#define JIM_TT_EOF 8 - -#define JIM_TT_LINE 9 -#define JIM_TT_WORD 10 - - -#define JIM_TT_SUBEXPR_START 11 -#define JIM_TT_SUBEXPR_END 12 -#define JIM_TT_SUBEXPR_COMMA 13 -#define JIM_TT_EXPR_INT 14 -#define JIM_TT_EXPR_DOUBLE 15 -#define JIM_TT_EXPR_BOOLEAN 16 - -#define JIM_TT_EXPRSUGAR 17 - - -#define JIM_TT_EXPR_OP 20 - -#define TOKEN_IS_SEP(type) (type >= JIM_TT_SEP && type <= JIM_TT_EOF) - -#define TOKEN_IS_EXPR_START(type) (type == JIM_TT_NONE || type == JIM_TT_SUBEXPR_START || type == JIM_TT_SUBEXPR_COMMA) - -#define TOKEN_IS_EXPR_OP(type) (type >= JIM_TT_EXPR_OP) - -struct JimParseMissing { - int ch; - int line; -}; - -struct JimParserCtx -{ - const char *p; - int len; - int linenr; - const char *tstart; - const char *tend; - int tline; - int tt; - int eof; - int inquote; - int comment; - struct JimParseMissing missing; -}; - -static int JimParseScript(struct JimParserCtx *pc); -static int JimParseSep(struct JimParserCtx *pc); -static int JimParseEol(struct JimParserCtx *pc); -static int JimParseCmd(struct JimParserCtx *pc); -static int JimParseQuote(struct JimParserCtx *pc); -static int JimParseVar(struct JimParserCtx *pc); -static int JimParseBrace(struct JimParserCtx *pc); -static int JimParseStr(struct JimParserCtx *pc); -static int JimParseComment(struct JimParserCtx *pc); -static void JimParseSubCmd(struct JimParserCtx *pc); -static int JimParseSubQuote(struct JimParserCtx *pc); -static Jim_Obj *JimParserGetTokenObj(Jim_Interp *interp, struct JimParserCtx *pc); - -static void JimParserInit(struct JimParserCtx *pc, const char *prg, int len, int linenr) -{ - pc->p = prg; - pc->len = len; - pc->tstart = NULL; - pc->tend = NULL; - pc->tline = 0; - pc->tt = JIM_TT_NONE; - pc->eof = 0; - pc->inquote = 0; - pc->linenr = linenr; - pc->comment = 1; - pc->missing.ch = ' '; - pc->missing.line = linenr; -} - -static int JimParseScript(struct JimParserCtx *pc) -{ - while (1) { - if (!pc->len) { - pc->tstart = pc->p; - pc->tend = pc->p - 1; - pc->tline = pc->linenr; - pc->tt = JIM_TT_EOL; - pc->eof = 1; - return JIM_OK; - } - switch (*(pc->p)) { - case '\\': - if (*(pc->p + 1) == '\n' && !pc->inquote) { - return JimParseSep(pc); - } - pc->comment = 0; - return JimParseStr(pc); - case ' ': - case '\t': - case '\r': - case '\f': - if (!pc->inquote) - return JimParseSep(pc); - pc->comment = 0; - return JimParseStr(pc); - case '\n': - case ';': - pc->comment = 1; - if (!pc->inquote) - return JimParseEol(pc); - return JimParseStr(pc); - case '[': - pc->comment = 0; - return JimParseCmd(pc); - case '$': - pc->comment = 0; - if (JimParseVar(pc) == JIM_ERR) { - - pc->tstart = pc->tend = pc->p++; - pc->len--; - pc->tt = JIM_TT_ESC; - } - return JIM_OK; - case '#': - if (pc->comment) { - JimParseComment(pc); - continue; - } - return JimParseStr(pc); - default: - pc->comment = 0; - return JimParseStr(pc); - } - return JIM_OK; - } -} - -static int JimParseSep(struct JimParserCtx *pc) -{ - pc->tstart = pc->p; - pc->tline = pc->linenr; - while (isspace(UCHAR(*pc->p)) || (*pc->p == '\\' && *(pc->p + 1) == '\n')) { - if (*pc->p == '\n') { - break; - } - if (*pc->p == '\\') { - pc->p++; - pc->len--; - pc->linenr++; - } - pc->p++; - pc->len--; - } - pc->tend = pc->p - 1; - pc->tt = JIM_TT_SEP; - return JIM_OK; -} - -static int JimParseEol(struct JimParserCtx *pc) -{ - pc->tstart = pc->p; - pc->tline = pc->linenr; - while (isspace(UCHAR(*pc->p)) || *pc->p == ';') { - if (*pc->p == '\n') - pc->linenr++; - pc->p++; - pc->len--; - } - pc->tend = pc->p - 1; - pc->tt = JIM_TT_EOL; - return JIM_OK; -} - - -static void JimParseSubBrace(struct JimParserCtx *pc) -{ - int level = 1; - - - pc->p++; - pc->len--; - while (pc->len) { - switch (*pc->p) { - case '\\': - if (pc->len > 1) { - if (*++pc->p == '\n') { - pc->linenr++; - } - pc->len--; - } - break; - - case '{': - level++; - break; - - case '}': - if (--level == 0) { - pc->tend = pc->p - 1; - pc->p++; - pc->len--; - return; - } - break; - - case '\n': - pc->linenr++; - break; - } - pc->p++; - pc->len--; - } - pc->missing.ch = '{'; - pc->missing.line = pc->tline; - pc->tend = pc->p - 1; -} - -static int JimParseSubQuote(struct JimParserCtx *pc) -{ - int tt = JIM_TT_STR; - int line = pc->tline; - - - pc->p++; - pc->len--; - while (pc->len) { - switch (*pc->p) { - case '\\': - if (pc->len > 1) { - if (*++pc->p == '\n') { - pc->linenr++; - } - pc->len--; - tt = JIM_TT_ESC; - } - break; - - case '"': - pc->tend = pc->p - 1; - pc->p++; - pc->len--; - return tt; - - case '[': - JimParseSubCmd(pc); - tt = JIM_TT_ESC; - continue; - - case '\n': - pc->linenr++; - break; - - case '$': - tt = JIM_TT_ESC; - break; - } - pc->p++; - pc->len--; - } - pc->missing.ch = '"'; - pc->missing.line = line; - pc->tend = pc->p - 1; - return tt; -} - -static void JimParseSubCmd(struct JimParserCtx *pc) -{ - int level = 1; - int startofword = 1; - int line = pc->tline; - - - pc->p++; - pc->len--; - while (pc->len) { - switch (*pc->p) { - case '\\': - if (pc->len > 1) { - if (*++pc->p == '\n') { - pc->linenr++; - } - pc->len--; - } - break; - - case '[': - level++; - break; - - case ']': - if (--level == 0) { - pc->tend = pc->p - 1; - pc->p++; - pc->len--; - return; - } - break; - - case '"': - if (startofword) { - JimParseSubQuote(pc); - continue; - } - break; - - case '{': - JimParseSubBrace(pc); - startofword = 0; - continue; - - case '\n': - pc->linenr++; - break; - } - startofword = isspace(UCHAR(*pc->p)); - pc->p++; - pc->len--; - } - pc->missing.ch = '['; - pc->missing.line = line; - pc->tend = pc->p - 1; -} - -static int JimParseBrace(struct JimParserCtx *pc) -{ - pc->tstart = pc->p + 1; - pc->tline = pc->linenr; - pc->tt = JIM_TT_STR; - JimParseSubBrace(pc); - return JIM_OK; -} - -static int JimParseCmd(struct JimParserCtx *pc) -{ - pc->tstart = pc->p + 1; - pc->tline = pc->linenr; - pc->tt = JIM_TT_CMD; - JimParseSubCmd(pc); - return JIM_OK; -} - -static int JimParseQuote(struct JimParserCtx *pc) -{ - pc->tstart = pc->p + 1; - pc->tline = pc->linenr; - pc->tt = JimParseSubQuote(pc); - return JIM_OK; -} - -static int JimParseVar(struct JimParserCtx *pc) -{ - - pc->p++; - pc->len--; - -#ifdef EXPRSUGAR_BRACKET - if (*pc->p == '[') { - - JimParseCmd(pc); - pc->tt = JIM_TT_EXPRSUGAR; - return JIM_OK; - } -#endif - - pc->tstart = pc->p; - pc->tt = JIM_TT_VAR; - pc->tline = pc->linenr; - - if (*pc->p == '{') { - pc->tstart = ++pc->p; - pc->len--; - - while (pc->len && *pc->p != '}') { - if (*pc->p == '\n') { - pc->linenr++; - } - pc->p++; - pc->len--; - } - pc->tend = pc->p - 1; - if (pc->len) { - pc->p++; - pc->len--; - } - } - else { - while (1) { - - if (pc->p[0] == ':' && pc->p[1] == ':') { - while (*pc->p == ':') { - pc->p++; - pc->len--; - } - continue; - } - if (isalnum(UCHAR(*pc->p)) || *pc->p == '_' || UCHAR(*pc->p) >= 0x80) { - pc->p++; - pc->len--; - continue; - } - break; - } - - if (*pc->p == '(') { - int count = 1; - const char *paren = NULL; - - pc->tt = JIM_TT_DICTSUGAR; - - while (count && pc->len) { - pc->p++; - pc->len--; - if (*pc->p == '\\' && pc->len >= 1) { - pc->p++; - pc->len--; - } - else if (*pc->p == '(') { - count++; - } - else if (*pc->p == ')') { - paren = pc->p; - count--; - } - } - if (count == 0) { - pc->p++; - pc->len--; - } - else if (paren) { - - paren++; - pc->len += (pc->p - paren); - pc->p = paren; - } -#ifndef EXPRSUGAR_BRACKET - if (*pc->tstart == '(') { - pc->tt = JIM_TT_EXPRSUGAR; - } -#endif - } - pc->tend = pc->p - 1; - } - if (pc->tstart == pc->p) { - pc->p--; - pc->len++; - return JIM_ERR; - } - return JIM_OK; -} - -static int JimParseStr(struct JimParserCtx *pc) -{ - if (pc->tt == JIM_TT_SEP || pc->tt == JIM_TT_EOL || - pc->tt == JIM_TT_NONE || pc->tt == JIM_TT_STR) { - - if (*pc->p == '{') { - return JimParseBrace(pc); - } - if (*pc->p == '"') { - pc->inquote = 1; - pc->p++; - pc->len--; - - pc->missing.line = pc->tline; - } - } - pc->tstart = pc->p; - pc->tline = pc->linenr; - while (1) { - if (pc->len == 0) { - if (pc->inquote) { - pc->missing.ch = '"'; - } - pc->tend = pc->p - 1; - pc->tt = JIM_TT_ESC; - return JIM_OK; - } - switch (*pc->p) { - case '\\': - if (!pc->inquote && *(pc->p + 1) == '\n') { - pc->tend = pc->p - 1; - pc->tt = JIM_TT_ESC; - return JIM_OK; - } - if (pc->len >= 2) { - if (*(pc->p + 1) == '\n') { - pc->linenr++; - } - pc->p++; - pc->len--; - } - else if (pc->len == 1) { - - pc->missing.ch = '\\'; - } - break; - case '(': - - if (pc->len > 1 && pc->p[1] != '$') { - break; - } - - case ')': - - if (*pc->p == '(' || pc->tt == JIM_TT_VAR) { - if (pc->p == pc->tstart) { - - pc->p++; - pc->len--; - } - pc->tend = pc->p - 1; - pc->tt = JIM_TT_ESC; - return JIM_OK; - } - break; - - case '$': - case '[': - pc->tend = pc->p - 1; - pc->tt = JIM_TT_ESC; - return JIM_OK; - case ' ': - case '\t': - case '\n': - case '\r': - case '\f': - case ';': - if (!pc->inquote) { - pc->tend = pc->p - 1; - pc->tt = JIM_TT_ESC; - return JIM_OK; - } - else if (*pc->p == '\n') { - pc->linenr++; - } - break; - case '"': - if (pc->inquote) { - pc->tend = pc->p - 1; - pc->tt = JIM_TT_ESC; - pc->p++; - pc->len--; - pc->inquote = 0; - return JIM_OK; - } - break; - } - pc->p++; - pc->len--; - } - return JIM_OK; -} - -static int JimParseComment(struct JimParserCtx *pc) -{ - while (*pc->p) { - if (*pc->p == '\\') { - pc->p++; - pc->len--; - if (pc->len == 0) { - pc->missing.ch = '\\'; - return JIM_OK; - } - if (*pc->p == '\n') { - pc->linenr++; - } - } - else if (*pc->p == '\n') { - pc->p++; - pc->len--; - pc->linenr++; - break; - } - pc->p++; - pc->len--; - } - return JIM_OK; -} - - -static int xdigitval(int c) -{ - if (c >= '0' && c <= '9') - return c - '0'; - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - if (c >= 'A' && c <= 'F') - return c - 'A' + 10; - return -1; -} - -static int odigitval(int c) -{ - if (c >= '0' && c <= '7') - return c - '0'; - return -1; -} - -static int JimEscape(char *dest, const char *s, int slen) -{ - char *p = dest; - int i, len; - - for (i = 0; i < slen; i++) { - switch (s[i]) { - case '\\': - switch (s[i + 1]) { - case 'a': - *p++ = 0x7; - i++; - break; - case 'b': - *p++ = 0x8; - i++; - break; - case 'f': - *p++ = 0xc; - i++; - break; - case 'n': - *p++ = 0xa; - i++; - break; - case 'r': - *p++ = 0xd; - i++; - break; - case 't': - *p++ = 0x9; - i++; - break; - case 'u': - case 'U': - case 'x': - { - unsigned val = 0; - int k; - int maxchars = 2; - - i++; - - if (s[i] == 'U') { - maxchars = 8; - } - else if (s[i] == 'u') { - if (s[i + 1] == '{') { - maxchars = 6; - i++; - } - else { - maxchars = 4; - } - } - - for (k = 0; k < maxchars; k++) { - int c = xdigitval(s[i + k + 1]); - if (c == -1) { - break; - } - val = (val << 4) | c; - } - - if (s[i] == '{') { - if (k == 0 || val > 0x1fffff || s[i + k + 1] != '}') { - - i--; - k = 0; - } - else { - - k++; - } - } - if (k) { - - if (s[i] == 'x') { - *p++ = val; - } - else { - p += utf8_fromunicode(p, val); - } - i += k; - break; - } - - *p++ = s[i]; - } - break; - case 'v': - *p++ = 0xb; - i++; - break; - case '\0': - *p++ = '\\'; - i++; - break; - case '\n': - - *p++ = ' '; - do { - i++; - } while (s[i + 1] == ' ' || s[i + 1] == '\t'); - break; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - - { - int val = 0; - int c = odigitval(s[i + 1]); - - val = c; - c = odigitval(s[i + 2]); - if (c == -1) { - *p++ = val; - i++; - break; - } - val = (val * 8) + c; - c = odigitval(s[i + 3]); - if (c == -1) { - *p++ = val; - i += 2; - break; - } - val = (val * 8) + c; - *p++ = val; - i += 3; - } - break; - default: - *p++ = s[i + 1]; - i++; - break; - } - break; - default: - *p++ = s[i]; - break; - } - } - len = p - dest; - *p = '\0'; - return len; -} - -static Jim_Obj *JimParserGetTokenObj(Jim_Interp *interp, struct JimParserCtx *pc) -{ - const char *start, *end; - char *token; - int len; - - start = pc->tstart; - end = pc->tend; - len = (end - start) + 1; - if (len < 0) { - len = 0; - } - token = Jim_Alloc(len + 1); - if (pc->tt != JIM_TT_ESC) { - - memcpy(token, start, len); - token[len] = '\0'; - } - else { - - len = JimEscape(token, start, len); - } - - return Jim_NewStringObjNoAlloc(interp, token, len); -} - -static int JimParseListSep(struct JimParserCtx *pc); -static int JimParseListStr(struct JimParserCtx *pc); -static int JimParseListQuote(struct JimParserCtx *pc); - -static int JimParseList(struct JimParserCtx *pc) -{ - if (isspace(UCHAR(*pc->p))) { - return JimParseListSep(pc); - } - switch (*pc->p) { - case '"': - return JimParseListQuote(pc); - - case '{': - return JimParseBrace(pc); - - default: - if (pc->len) { - return JimParseListStr(pc); - } - break; - } - - pc->tstart = pc->tend = pc->p; - pc->tline = pc->linenr; - pc->tt = JIM_TT_EOL; - pc->eof = 1; - return JIM_OK; -} - -static int JimParseListSep(struct JimParserCtx *pc) -{ - pc->tstart = pc->p; - pc->tline = pc->linenr; - while (isspace(UCHAR(*pc->p))) { - if (*pc->p == '\n') { - pc->linenr++; - } - pc->p++; - pc->len--; - } - pc->tend = pc->p - 1; - pc->tt = JIM_TT_SEP; - return JIM_OK; -} - -static int JimParseListQuote(struct JimParserCtx *pc) -{ - pc->p++; - pc->len--; - - pc->tstart = pc->p; - pc->tline = pc->linenr; - pc->tt = JIM_TT_STR; - - while (pc->len) { - switch (*pc->p) { - case '\\': - pc->tt = JIM_TT_ESC; - if (--pc->len == 0) { - - pc->tend = pc->p; - return JIM_OK; - } - pc->p++; - break; - case '\n': - pc->linenr++; - break; - case '"': - pc->tend = pc->p - 1; - pc->p++; - pc->len--; - return JIM_OK; - } - pc->p++; - pc->len--; - } - - pc->tend = pc->p - 1; - return JIM_OK; -} - -static int JimParseListStr(struct JimParserCtx *pc) -{ - pc->tstart = pc->p; - pc->tline = pc->linenr; - pc->tt = JIM_TT_STR; - - while (pc->len) { - if (isspace(UCHAR(*pc->p))) { - pc->tend = pc->p - 1; - return JIM_OK; - } - if (*pc->p == '\\') { - if (--pc->len == 0) { - - pc->tend = pc->p; - return JIM_OK; - } - pc->tt = JIM_TT_ESC; - pc->p++; - } - pc->p++; - pc->len--; - } - pc->tend = pc->p - 1; - return JIM_OK; -} - - - -Jim_Obj *Jim_NewObj(Jim_Interp *interp) -{ - Jim_Obj *objPtr; - - - if (interp->freeList != NULL) { - - objPtr = interp->freeList; - interp->freeList = objPtr->nextObjPtr; - } - else { - - objPtr = Jim_Alloc(sizeof(*objPtr)); - } - - objPtr->refCount = 0; - - - objPtr->prevObjPtr = NULL; - objPtr->nextObjPtr = interp->liveList; - if (interp->liveList) - interp->liveList->prevObjPtr = objPtr; - interp->liveList = objPtr; - - return objPtr; -} - -void Jim_FreeObj(Jim_Interp *interp, Jim_Obj *objPtr) -{ - - JimPanic((objPtr->refCount != 0, "!!!Object %p freed with bad refcount %d, type=%s", objPtr, - objPtr->refCount, objPtr->typePtr ? objPtr->typePtr->name : "")); - - - Jim_FreeIntRep(interp, objPtr); - - if (objPtr->bytes != NULL) { - if (objPtr->bytes != JimEmptyStringRep) - Jim_Free(objPtr->bytes); - } - - if (objPtr->prevObjPtr) - objPtr->prevObjPtr->nextObjPtr = objPtr->nextObjPtr; - if (objPtr->nextObjPtr) - objPtr->nextObjPtr->prevObjPtr = objPtr->prevObjPtr; - if (interp->liveList == objPtr) - interp->liveList = objPtr->nextObjPtr; -#ifdef JIM_DISABLE_OBJECT_POOL - Jim_Free(objPtr); -#else - - objPtr->prevObjPtr = NULL; - objPtr->nextObjPtr = interp->freeList; - if (interp->freeList) - interp->freeList->prevObjPtr = objPtr; - interp->freeList = objPtr; - objPtr->refCount = -1; -#endif -} - - -void Jim_InvalidateStringRep(Jim_Obj *objPtr) -{ - if (objPtr->bytes != NULL) { - if (objPtr->bytes != JimEmptyStringRep) - Jim_Free(objPtr->bytes); - } - objPtr->bytes = NULL; -} - - -Jim_Obj *Jim_DuplicateObj(Jim_Interp *interp, Jim_Obj *objPtr) -{ - Jim_Obj *dupPtr; - - dupPtr = Jim_NewObj(interp); - if (objPtr->bytes == NULL) { - - dupPtr->bytes = NULL; - } - else if (objPtr->length == 0) { - dupPtr->bytes = JimEmptyStringRep; - dupPtr->length = 0; - dupPtr->typePtr = NULL; - return dupPtr; - } - else { - dupPtr->bytes = Jim_Alloc(objPtr->length + 1); - dupPtr->length = objPtr->length; - - memcpy(dupPtr->bytes, objPtr->bytes, objPtr->length + 1); - } - - - dupPtr->typePtr = objPtr->typePtr; - if (objPtr->typePtr != NULL) { - if (objPtr->typePtr->dupIntRepProc == NULL) { - dupPtr->internalRep = objPtr->internalRep; - } - else { - - objPtr->typePtr->dupIntRepProc(interp, objPtr, dupPtr); - } - } - return dupPtr; -} - -const char *Jim_GetString(Jim_Obj *objPtr, int *lenPtr) -{ - if (objPtr->bytes == NULL) { - - JimPanic((objPtr->typePtr->updateStringProc == NULL, "UpdateStringProc called against '%s' type.", objPtr->typePtr->name)); - objPtr->typePtr->updateStringProc(objPtr); - } - if (lenPtr) - *lenPtr = objPtr->length; - return objPtr->bytes; -} - - -int Jim_Length(Jim_Obj *objPtr) -{ - if (objPtr->bytes == NULL) { - - Jim_GetString(objPtr, NULL); - } - return objPtr->length; -} - - -const char *Jim_String(Jim_Obj *objPtr) -{ - if (objPtr->bytes == NULL) { - - Jim_GetString(objPtr, NULL); - } - return objPtr->bytes; -} - -static void JimSetStringBytes(Jim_Obj *objPtr, const char *str) -{ - objPtr->bytes = Jim_StrDup(str); - objPtr->length = strlen(str); -} - -static void FreeDictSubstInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); -static void DupDictSubstInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); - -static const Jim_ObjType dictSubstObjType = { - "dict-substitution", - FreeDictSubstInternalRep, - DupDictSubstInternalRep, - NULL, - JIM_TYPE_NONE, -}; - -static void FreeInterpolatedInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); -static void DupInterpolatedInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); - -static const Jim_ObjType interpolatedObjType = { - "interpolated", - FreeInterpolatedInternalRep, - DupInterpolatedInternalRep, - NULL, - JIM_TYPE_NONE, -}; - -static void FreeInterpolatedInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) -{ - Jim_DecrRefCount(interp, objPtr->internalRep.dictSubstValue.indexObjPtr); -} - -static void DupInterpolatedInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) -{ - - dupPtr->internalRep = srcPtr->internalRep; - - Jim_IncrRefCount(dupPtr->internalRep.dictSubstValue.indexObjPtr); -} - -static void DupStringInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); -static int SetStringFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); - -static const Jim_ObjType stringObjType = { - "string", - NULL, - DupStringInternalRep, - NULL, - JIM_TYPE_REFERENCES, -}; - -static void DupStringInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) -{ - JIM_NOTUSED(interp); - - dupPtr->internalRep.strValue.maxLength = srcPtr->length; - dupPtr->internalRep.strValue.charLength = srcPtr->internalRep.strValue.charLength; -} - -static int SetStringFromAny(Jim_Interp *interp, Jim_Obj *objPtr) -{ - if (objPtr->typePtr != &stringObjType) { - - if (objPtr->bytes == NULL) { - - JimPanic((objPtr->typePtr->updateStringProc == NULL, "UpdateStringProc called against '%s' type.", objPtr->typePtr->name)); - objPtr->typePtr->updateStringProc(objPtr); - } - - Jim_FreeIntRep(interp, objPtr); - - objPtr->typePtr = &stringObjType; - objPtr->internalRep.strValue.maxLength = objPtr->length; - - objPtr->internalRep.strValue.charLength = -1; - } - return JIM_OK; -} - -int Jim_Utf8Length(Jim_Interp *interp, Jim_Obj *objPtr) -{ -#ifdef JIM_UTF8 - SetStringFromAny(interp, objPtr); - - if (objPtr->internalRep.strValue.charLength < 0) { - objPtr->internalRep.strValue.charLength = utf8_strlen(objPtr->bytes, objPtr->length); - } - return objPtr->internalRep.strValue.charLength; -#else - return Jim_Length(objPtr); -#endif -} - - -Jim_Obj *Jim_NewStringObj(Jim_Interp *interp, const char *s, int len) -{ - Jim_Obj *objPtr = Jim_NewObj(interp); - - - if (len == -1) - len = strlen(s); - - if (len == 0) { - objPtr->bytes = JimEmptyStringRep; - } - else { - objPtr->bytes = Jim_StrDupLen(s, len); - } - objPtr->length = len; - - - objPtr->typePtr = NULL; - return objPtr; -} - - -Jim_Obj *Jim_NewStringObjUtf8(Jim_Interp *interp, const char *s, int charlen) -{ -#ifdef JIM_UTF8 - - int bytelen = utf8_index(s, charlen); - - Jim_Obj *objPtr = Jim_NewStringObj(interp, s, bytelen); - - - objPtr->typePtr = &stringObjType; - objPtr->internalRep.strValue.maxLength = bytelen; - objPtr->internalRep.strValue.charLength = charlen; - - return objPtr; -#else - return Jim_NewStringObj(interp, s, charlen); -#endif -} - -Jim_Obj *Jim_NewStringObjNoAlloc(Jim_Interp *interp, char *s, int len) -{ - Jim_Obj *objPtr = Jim_NewObj(interp); - - objPtr->bytes = s; - objPtr->length = (len == -1) ? strlen(s) : len; - objPtr->typePtr = NULL; - return objPtr; -} - -static void StringAppendString(Jim_Obj *objPtr, const char *str, int len) -{ - int needlen; - - if (len == -1) - len = strlen(str); - needlen = objPtr->length + len; - if (objPtr->internalRep.strValue.maxLength < needlen || - objPtr->internalRep.strValue.maxLength == 0) { - needlen *= 2; - - if (needlen < 7) { - needlen = 7; - } - if (objPtr->bytes == JimEmptyStringRep) { - objPtr->bytes = Jim_Alloc(needlen + 1); - } - else { - objPtr->bytes = Jim_Realloc(objPtr->bytes, needlen + 1); - } - objPtr->internalRep.strValue.maxLength = needlen; - } - memcpy(objPtr->bytes + objPtr->length, str, len); - objPtr->bytes[objPtr->length + len] = '\0'; - - if (objPtr->internalRep.strValue.charLength >= 0) { - - objPtr->internalRep.strValue.charLength += utf8_strlen(objPtr->bytes + objPtr->length, len); - } - objPtr->length += len; -} - -void Jim_AppendString(Jim_Interp *interp, Jim_Obj *objPtr, const char *str, int len) -{ - JimPanic((Jim_IsShared(objPtr), "Jim_AppendString called with shared object")); - SetStringFromAny(interp, objPtr); - StringAppendString(objPtr, str, len); -} - -void Jim_AppendObj(Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj *appendObjPtr) -{ - int len; - const char *str = Jim_GetString(appendObjPtr, &len); - Jim_AppendString(interp, objPtr, str, len); -} - -void Jim_AppendStrings(Jim_Interp *interp, Jim_Obj *objPtr, ...) -{ - va_list ap; - - SetStringFromAny(interp, objPtr); - va_start(ap, objPtr); - while (1) { - const char *s = va_arg(ap, const char *); - - if (s == NULL) - break; - Jim_AppendString(interp, objPtr, s, -1); - } - va_end(ap); -} - -int Jim_StringEqObj(Jim_Obj *aObjPtr, Jim_Obj *bObjPtr) -{ - if (aObjPtr == bObjPtr) { - return 1; - } - else { - int Alen, Blen; - const char *sA = Jim_GetString(aObjPtr, &Alen); - const char *sB = Jim_GetString(bObjPtr, &Blen); - - return Alen == Blen && memcmp(sA, sB, Alen) == 0; - } -} - -int Jim_StringMatchObj(Jim_Interp *interp, Jim_Obj *patternObjPtr, Jim_Obj *objPtr, int nocase) -{ - return JimGlobMatch(Jim_String(patternObjPtr), Jim_String(objPtr), nocase); -} - -int Jim_StringCompareObj(Jim_Interp *interp, Jim_Obj *firstObjPtr, Jim_Obj *secondObjPtr, int nocase) -{ - int l1, l2; - const char *s1 = Jim_GetString(firstObjPtr, &l1); - const char *s2 = Jim_GetString(secondObjPtr, &l2); - - if (nocase) { - - return JimStringCompareLen(s1, s2, -1, nocase); - } - return JimStringCompare(s1, l1, s2, l2); -} - -int Jim_StringCompareLenObj(Jim_Interp *interp, Jim_Obj *firstObjPtr, Jim_Obj *secondObjPtr, int nocase) -{ - const char *s1 = Jim_String(firstObjPtr); - const char *s2 = Jim_String(secondObjPtr); - - return JimStringCompareLen(s1, s2, Jim_Utf8Length(interp, firstObjPtr), nocase); -} - -static int JimRelToAbsIndex(int len, int idx) -{ - if (idx < 0) - return len + idx; - return idx; -} - -static void JimRelToAbsRange(int len, int *firstPtr, int *lastPtr, int *rangeLenPtr) -{ - int rangeLen; - - if (*firstPtr > *lastPtr) { - rangeLen = 0; - } - else { - rangeLen = *lastPtr - *firstPtr + 1; - if (rangeLen) { - if (*firstPtr < 0) { - rangeLen += *firstPtr; - *firstPtr = 0; - } - if (*lastPtr >= len) { - rangeLen -= (*lastPtr - (len - 1)); - *lastPtr = len - 1; - } - } - } - if (rangeLen < 0) - rangeLen = 0; - - *rangeLenPtr = rangeLen; -} - -static int JimStringGetRange(Jim_Interp *interp, Jim_Obj *firstObjPtr, Jim_Obj *lastObjPtr, - int len, int *first, int *last, int *range) -{ - if (Jim_GetIndex(interp, firstObjPtr, first) != JIM_OK) { - return JIM_ERR; - } - if (Jim_GetIndex(interp, lastObjPtr, last) != JIM_OK) { - return JIM_ERR; - } - *first = JimRelToAbsIndex(len, *first); - *last = JimRelToAbsIndex(len, *last); - JimRelToAbsRange(len, first, last, range); - return JIM_OK; -} - -Jim_Obj *Jim_StringByteRangeObj(Jim_Interp *interp, - Jim_Obj *strObjPtr, Jim_Obj *firstObjPtr, Jim_Obj *lastObjPtr) -{ - int first, last; - const char *str; - int rangeLen; - int bytelen; - - str = Jim_GetString(strObjPtr, &bytelen); - - if (JimStringGetRange(interp, firstObjPtr, lastObjPtr, bytelen, &first, &last, &rangeLen) != JIM_OK) { - return NULL; - } - - if (first == 0 && rangeLen == bytelen) { - return strObjPtr; - } - return Jim_NewStringObj(interp, str + first, rangeLen); -} - -Jim_Obj *Jim_StringRangeObj(Jim_Interp *interp, - Jim_Obj *strObjPtr, Jim_Obj *firstObjPtr, Jim_Obj *lastObjPtr) -{ -#ifdef JIM_UTF8 - int first, last; - const char *str; - int len, rangeLen; - int bytelen; - - str = Jim_GetString(strObjPtr, &bytelen); - len = Jim_Utf8Length(interp, strObjPtr); - - if (JimStringGetRange(interp, firstObjPtr, lastObjPtr, len, &first, &last, &rangeLen) != JIM_OK) { - return NULL; - } - - if (first == 0 && rangeLen == len) { - return strObjPtr; - } - if (len == bytelen) { - - return Jim_NewStringObj(interp, str + first, rangeLen); - } - return Jim_NewStringObjUtf8(interp, str + utf8_index(str, first), rangeLen); -#else - return Jim_StringByteRangeObj(interp, strObjPtr, firstObjPtr, lastObjPtr); -#endif -} - -Jim_Obj *JimStringReplaceObj(Jim_Interp *interp, - Jim_Obj *strObjPtr, Jim_Obj *firstObjPtr, Jim_Obj *lastObjPtr, Jim_Obj *newStrObj) -{ - int first, last; - const char *str; - int len, rangeLen; - Jim_Obj *objPtr; - - len = Jim_Utf8Length(interp, strObjPtr); - - if (JimStringGetRange(interp, firstObjPtr, lastObjPtr, len, &first, &last, &rangeLen) != JIM_OK) { - return NULL; - } - - if (last < first) { - return strObjPtr; - } - - str = Jim_String(strObjPtr); - - - objPtr = Jim_NewStringObjUtf8(interp, str, first); - - - if (newStrObj) { - Jim_AppendObj(interp, objPtr, newStrObj); - } - - - Jim_AppendString(interp, objPtr, str + utf8_index(str, last + 1), len - last - 1); - - return objPtr; -} - -static void JimStrCopyUpperLower(char *dest, const char *str, int uc) -{ - while (*str) { - int c; - str += utf8_tounicode(str, &c); - dest += utf8_getchars(dest, uc ? utf8_upper(c) : utf8_lower(c)); - } - *dest = 0; -} - -static Jim_Obj *JimStringToLower(Jim_Interp *interp, Jim_Obj *strObjPtr) -{ - char *buf; - int len; - const char *str; - - str = Jim_GetString(strObjPtr, &len); - -#ifdef JIM_UTF8 - len *= 2; -#endif - buf = Jim_Alloc(len + 1); - JimStrCopyUpperLower(buf, str, 0); - return Jim_NewStringObjNoAlloc(interp, buf, -1); -} - -static Jim_Obj *JimStringToUpper(Jim_Interp *interp, Jim_Obj *strObjPtr) -{ - char *buf; - const char *str; - int len; - - str = Jim_GetString(strObjPtr, &len); - -#ifdef JIM_UTF8 - len *= 2; -#endif - buf = Jim_Alloc(len + 1); - JimStrCopyUpperLower(buf, str, 1); - return Jim_NewStringObjNoAlloc(interp, buf, -1); -} - -static Jim_Obj *JimStringToTitle(Jim_Interp *interp, Jim_Obj *strObjPtr) -{ - char *buf, *p; - int len; - int c; - const char *str; - - str = Jim_GetString(strObjPtr, &len); - -#ifdef JIM_UTF8 - len *= 2; -#endif - buf = p = Jim_Alloc(len + 1); - - str += utf8_tounicode(str, &c); - p += utf8_getchars(p, utf8_title(c)); - - JimStrCopyUpperLower(p, str, 0); - - return Jim_NewStringObjNoAlloc(interp, buf, -1); -} - -static const char *utf8_memchr(const char *str, int len, int c) -{ -#ifdef JIM_UTF8 - while (len) { - int sc; - int n = utf8_tounicode(str, &sc); - if (sc == c) { - return str; - } - str += n; - len -= n; - } - return NULL; -#else - return memchr(str, c, len); -#endif -} - -static const char *JimFindTrimLeft(const char *str, int len, const char *trimchars, int trimlen) -{ - while (len) { - int c; - int n = utf8_tounicode(str, &c); - - if (utf8_memchr(trimchars, trimlen, c) == NULL) { - - break; - } - str += n; - len -= n; - } - return str; -} - -static const char *JimFindTrimRight(const char *str, int len, const char *trimchars, int trimlen) -{ - str += len; - - while (len) { - int c; - int n = utf8_prev_len(str, len); - - len -= n; - str -= n; - - n = utf8_tounicode(str, &c); - - if (utf8_memchr(trimchars, trimlen, c) == NULL) { - return str + n; - } - } - - return NULL; -} - -static const char default_trim_chars[] = " \t\n\r"; - -static int default_trim_chars_len = sizeof(default_trim_chars); - -static Jim_Obj *JimStringTrimLeft(Jim_Interp *interp, Jim_Obj *strObjPtr, Jim_Obj *trimcharsObjPtr) -{ - int len; - const char *str = Jim_GetString(strObjPtr, &len); - const char *trimchars = default_trim_chars; - int trimcharslen = default_trim_chars_len; - const char *newstr; - - if (trimcharsObjPtr) { - trimchars = Jim_GetString(trimcharsObjPtr, &trimcharslen); - } - - newstr = JimFindTrimLeft(str, len, trimchars, trimcharslen); - if (newstr == str) { - return strObjPtr; - } - - return Jim_NewStringObj(interp, newstr, len - (newstr - str)); -} - -static Jim_Obj *JimStringTrimRight(Jim_Interp *interp, Jim_Obj *strObjPtr, Jim_Obj *trimcharsObjPtr) -{ - int len; - const char *trimchars = default_trim_chars; - int trimcharslen = default_trim_chars_len; - const char *nontrim; - - if (trimcharsObjPtr) { - trimchars = Jim_GetString(trimcharsObjPtr, &trimcharslen); - } - - SetStringFromAny(interp, strObjPtr); - - len = Jim_Length(strObjPtr); - nontrim = JimFindTrimRight(strObjPtr->bytes, len, trimchars, trimcharslen); - - if (nontrim == NULL) { - - return Jim_NewEmptyStringObj(interp); - } - if (nontrim == strObjPtr->bytes + len) { - - return strObjPtr; - } - - if (Jim_IsShared(strObjPtr)) { - strObjPtr = Jim_NewStringObj(interp, strObjPtr->bytes, (nontrim - strObjPtr->bytes)); - } - else { - - strObjPtr->bytes[nontrim - strObjPtr->bytes] = 0; - strObjPtr->length = (nontrim - strObjPtr->bytes); - } - - return strObjPtr; -} - -static Jim_Obj *JimStringTrim(Jim_Interp *interp, Jim_Obj *strObjPtr, Jim_Obj *trimcharsObjPtr) -{ - - Jim_Obj *objPtr = JimStringTrimLeft(interp, strObjPtr, trimcharsObjPtr); - - - strObjPtr = JimStringTrimRight(interp, objPtr, trimcharsObjPtr); - - - if (objPtr != strObjPtr && objPtr->refCount == 0) { - - Jim_FreeNewObj(interp, objPtr); - } - - return strObjPtr; -} - - -#ifdef HAVE_ISASCII -#define jim_isascii isascii -#else -static int jim_isascii(int c) -{ - return !(c & ~0x7f); -} -#endif - -static int JimStringIs(Jim_Interp *interp, Jim_Obj *strObjPtr, Jim_Obj *strClass, int strict) -{ - static const char * const strclassnames[] = { - "integer", "alpha", "alnum", "ascii", "digit", - "double", "lower", "upper", "space", "xdigit", - "control", "print", "graph", "punct", "boolean", - NULL - }; - enum { - STR_IS_INTEGER, STR_IS_ALPHA, STR_IS_ALNUM, STR_IS_ASCII, STR_IS_DIGIT, - STR_IS_DOUBLE, STR_IS_LOWER, STR_IS_UPPER, STR_IS_SPACE, STR_IS_XDIGIT, - STR_IS_CONTROL, STR_IS_PRINT, STR_IS_GRAPH, STR_IS_PUNCT, STR_IS_BOOLEAN, - }; - int strclass; - int len; - int i; - const char *str; - int (*isclassfunc)(int c) = NULL; - - if (Jim_GetEnum(interp, strClass, strclassnames, &strclass, "class", JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { - return JIM_ERR; - } - - str = Jim_GetString(strObjPtr, &len); - if (len == 0) { - Jim_SetResultBool(interp, !strict); - return JIM_OK; - } - - switch (strclass) { - case STR_IS_INTEGER: - { - jim_wide w; - Jim_SetResultBool(interp, JimGetWideNoErr(interp, strObjPtr, &w) == JIM_OK); - return JIM_OK; - } - - case STR_IS_DOUBLE: - { - double d; - Jim_SetResultBool(interp, Jim_GetDouble(interp, strObjPtr, &d) == JIM_OK && errno != ERANGE); - return JIM_OK; - } - - case STR_IS_BOOLEAN: - { - int b; - Jim_SetResultBool(interp, Jim_GetBoolean(interp, strObjPtr, &b) == JIM_OK); - return JIM_OK; - } - - case STR_IS_ALPHA: isclassfunc = isalpha; break; - case STR_IS_ALNUM: isclassfunc = isalnum; break; - case STR_IS_ASCII: isclassfunc = jim_isascii; break; - case STR_IS_DIGIT: isclassfunc = isdigit; break; - case STR_IS_LOWER: isclassfunc = islower; break; - case STR_IS_UPPER: isclassfunc = isupper; break; - case STR_IS_SPACE: isclassfunc = isspace; break; - case STR_IS_XDIGIT: isclassfunc = isxdigit; break; - case STR_IS_CONTROL: isclassfunc = iscntrl; break; - case STR_IS_PRINT: isclassfunc = isprint; break; - case STR_IS_GRAPH: isclassfunc = isgraph; break; - case STR_IS_PUNCT: isclassfunc = ispunct; break; - default: - return JIM_ERR; - } - - for (i = 0; i < len; i++) { - if (!isclassfunc(UCHAR(str[i]))) { - Jim_SetResultBool(interp, 0); - return JIM_OK; - } - } - Jim_SetResultBool(interp, 1); - return JIM_OK; -} - - - -static const Jim_ObjType comparedStringObjType = { - "compared-string", - NULL, - NULL, - NULL, - JIM_TYPE_REFERENCES, -}; - -int Jim_CompareStringImmediate(Jim_Interp *interp, Jim_Obj *objPtr, const char *str) -{ - if (objPtr->typePtr == &comparedStringObjType && objPtr->internalRep.ptr == str) { - return 1; - } - else { - if (strcmp(str, Jim_String(objPtr)) != 0) - return 0; - - if (objPtr->typePtr != &comparedStringObjType) { - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &comparedStringObjType; - } - objPtr->internalRep.ptr = (char *)str; - return 1; - } -} - -static int qsortCompareStringPointers(const void *a, const void *b) -{ - char *const *sa = (char *const *)a; - char *const *sb = (char *const *)b; - - return strcmp(*sa, *sb); -} - - - -static void FreeSourceInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); -static void DupSourceInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); - -static const Jim_ObjType sourceObjType = { - "source", - FreeSourceInternalRep, - DupSourceInternalRep, - NULL, - JIM_TYPE_REFERENCES, -}; - -void FreeSourceInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) -{ - Jim_DecrRefCount(interp, objPtr->internalRep.sourceValue.fileNameObj); -} - -void DupSourceInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) -{ - dupPtr->internalRep.sourceValue = srcPtr->internalRep.sourceValue; - Jim_IncrRefCount(dupPtr->internalRep.sourceValue.fileNameObj); -} - -static void JimSetSourceInfo(Jim_Interp *interp, Jim_Obj *objPtr, - Jim_Obj *fileNameObj, int lineNumber) -{ - JimPanic((Jim_IsShared(objPtr), "JimSetSourceInfo called with shared object")); - JimPanic((objPtr->typePtr != NULL, "JimSetSourceInfo called with typed object")); - Jim_IncrRefCount(fileNameObj); - objPtr->internalRep.sourceValue.fileNameObj = fileNameObj; - objPtr->internalRep.sourceValue.lineNumber = lineNumber; - objPtr->typePtr = &sourceObjType; -} - -static const Jim_ObjType scriptLineObjType = { - "scriptline", - NULL, - NULL, - NULL, - JIM_NONE, -}; - -static Jim_Obj *JimNewScriptLineObj(Jim_Interp *interp, int argc, int line) -{ - Jim_Obj *objPtr; - -#ifdef DEBUG_SHOW_SCRIPT - char buf[100]; - snprintf(buf, sizeof(buf), "line=%d, argc=%d", line, argc); - objPtr = Jim_NewStringObj(interp, buf, -1); -#else - objPtr = Jim_NewEmptyStringObj(interp); -#endif - objPtr->typePtr = &scriptLineObjType; - objPtr->internalRep.scriptLineValue.argc = argc; - objPtr->internalRep.scriptLineValue.line = line; - - return objPtr; -} - -static void FreeScriptInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); -static void DupScriptInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); - -static const Jim_ObjType scriptObjType = { - "script", - FreeScriptInternalRep, - DupScriptInternalRep, - NULL, - JIM_TYPE_REFERENCES, -}; - -typedef struct ScriptToken -{ - Jim_Obj *objPtr; - int type; -} ScriptToken; - -typedef struct ScriptObj -{ - ScriptToken *token; - Jim_Obj *fileNameObj; - int len; - int substFlags; - int inUse; /* Used to share a ScriptObj. Currently - only used by Jim_EvalObj() as protection against - shimmering of the currently evaluated object. */ - int firstline; - int linenr; - int missing; -} ScriptObj; - -static void JimSetScriptFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); -static int JimParseCheckMissing(Jim_Interp *interp, int ch); -static ScriptObj *JimGetScript(Jim_Interp *interp, Jim_Obj *objPtr); - -void FreeScriptInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) -{ - int i; - struct ScriptObj *script = (void *)objPtr->internalRep.ptr; - - if (--script->inUse != 0) - return; - for (i = 0; i < script->len; i++) { - Jim_DecrRefCount(interp, script->token[i].objPtr); - } - Jim_Free(script->token); - Jim_DecrRefCount(interp, script->fileNameObj); - Jim_Free(script); -} - -void DupScriptInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) -{ - JIM_NOTUSED(interp); - JIM_NOTUSED(srcPtr); - - dupPtr->typePtr = NULL; -} - -typedef struct -{ - const char *token; - int len; - int type; - int line; -} ParseToken; - -typedef struct -{ - - ParseToken *list; - int size; - int count; - ParseToken static_list[20]; -} ParseTokenList; - -static void ScriptTokenListInit(ParseTokenList *tokenlist) -{ - tokenlist->list = tokenlist->static_list; - tokenlist->size = sizeof(tokenlist->static_list) / sizeof(ParseToken); - tokenlist->count = 0; -} - -static void ScriptTokenListFree(ParseTokenList *tokenlist) -{ - if (tokenlist->list != tokenlist->static_list) { - Jim_Free(tokenlist->list); - } -} - -static void ScriptAddToken(ParseTokenList *tokenlist, const char *token, int len, int type, - int line) -{ - ParseToken *t; - - if (tokenlist->count == tokenlist->size) { - - tokenlist->size *= 2; - if (tokenlist->list != tokenlist->static_list) { - tokenlist->list = - Jim_Realloc(tokenlist->list, tokenlist->size * sizeof(*tokenlist->list)); - } - else { - - tokenlist->list = Jim_Alloc(tokenlist->size * sizeof(*tokenlist->list)); - memcpy(tokenlist->list, tokenlist->static_list, - tokenlist->count * sizeof(*tokenlist->list)); - } - } - t = &tokenlist->list[tokenlist->count++]; - t->token = token; - t->len = len; - t->type = type; - t->line = line; -} - -static int JimCountWordTokens(struct ScriptObj *script, ParseToken *t) -{ - int expand = 1; - int count = 0; - - - if (t->type == JIM_TT_STR && !TOKEN_IS_SEP(t[1].type)) { - if ((t->len == 1 && *t->token == '*') || (t->len == 6 && strncmp(t->token, "expand", 6) == 0)) { - - expand = -1; - t++; - } - else { - if (script->missing == ' ') { - - script->missing = '}'; - script->linenr = t[1].line; - } - } - } - - - while (!TOKEN_IS_SEP(t->type)) { - t++; - count++; - } - - return count * expand; -} - -static Jim_Obj *JimMakeScriptObj(Jim_Interp *interp, const ParseToken *t) -{ - Jim_Obj *objPtr; - - if (t->type == JIM_TT_ESC && memchr(t->token, '\\', t->len) != NULL) { - - int len = t->len; - char *str = Jim_Alloc(len + 1); - len = JimEscape(str, t->token, len); - objPtr = Jim_NewStringObjNoAlloc(interp, str, len); - } - else { - objPtr = Jim_NewStringObj(interp, t->token, t->len); - } - return objPtr; -} - -static void ScriptObjAddTokens(Jim_Interp *interp, struct ScriptObj *script, - ParseTokenList *tokenlist) -{ - int i; - struct ScriptToken *token; - - int lineargs = 0; - - ScriptToken *linefirst; - int count; - int linenr; - -#ifdef DEBUG_SHOW_SCRIPT_TOKENS - printf("==== Tokens ====\n"); - for (i = 0; i < tokenlist->count; i++) { - printf("[%2d]@%d %s '%.*s'\n", i, tokenlist->list[i].line, jim_tt_name(tokenlist->list[i].type), - tokenlist->list[i].len, tokenlist->list[i].token); - } -#endif - - - count = tokenlist->count; - for (i = 0; i < tokenlist->count; i++) { - if (tokenlist->list[i].type == JIM_TT_EOL) { - count++; - } - } - linenr = script->firstline = tokenlist->list[0].line; - - token = script->token = Jim_Alloc(sizeof(ScriptToken) * count); - - - linefirst = token++; - - for (i = 0; i < tokenlist->count; ) { - - int wordtokens; - - - while (tokenlist->list[i].type == JIM_TT_SEP) { - i++; - } - - wordtokens = JimCountWordTokens(script, tokenlist->list + i); - - if (wordtokens == 0) { - - if (lineargs) { - linefirst->type = JIM_TT_LINE; - linefirst->objPtr = JimNewScriptLineObj(interp, lineargs, linenr); - Jim_IncrRefCount(linefirst->objPtr); - - - lineargs = 0; - linefirst = token++; - } - i++; - continue; - } - else if (wordtokens != 1) { - - token->type = JIM_TT_WORD; - token->objPtr = Jim_NewIntObj(interp, wordtokens); - Jim_IncrRefCount(token->objPtr); - token++; - if (wordtokens < 0) { - - i++; - wordtokens = -wordtokens - 1; - lineargs--; - } - } - - if (lineargs == 0) { - - linenr = tokenlist->list[i].line; - } - lineargs++; - - - while (wordtokens--) { - const ParseToken *t = &tokenlist->list[i++]; - - token->type = t->type; - token->objPtr = JimMakeScriptObj(interp, t); - Jim_IncrRefCount(token->objPtr); - - JimSetSourceInfo(interp, token->objPtr, script->fileNameObj, t->line); - token++; - } - } - - if (lineargs == 0) { - token--; - } - - script->len = token - script->token; - - JimPanic((script->len >= count, "allocated script array is too short")); - -#ifdef DEBUG_SHOW_SCRIPT - printf("==== Script (%s) ====\n", Jim_String(script->fileNameObj)); - for (i = 0; i < script->len; i++) { - const ScriptToken *t = &script->token[i]; - printf("[%2d] %s %s\n", i, jim_tt_name(t->type), Jim_String(t->objPtr)); - } -#endif - -} - -int Jim_ScriptIsComplete(Jim_Interp *interp, Jim_Obj *scriptObj, char *stateCharPtr) -{ - ScriptObj *script = JimGetScript(interp, scriptObj); - if (stateCharPtr) { - *stateCharPtr = script->missing; - } - return script->missing == ' ' || script->missing == '}'; -} - -static int JimParseCheckMissing(Jim_Interp *interp, int ch) -{ - const char *msg; - - switch (ch) { - case '\\': - case ' ': - return JIM_OK; - - case '[': - msg = "unmatched \"[\""; - break; - case '{': - msg = "missing close-brace"; - break; - case '}': - msg = "extra characters after close-brace"; - break; - case '"': - default: - msg = "missing quote"; - break; - } - - Jim_SetResultString(interp, msg, -1); - return JIM_ERR; -} - -static void SubstObjAddTokens(Jim_Interp *interp, struct ScriptObj *script, - ParseTokenList *tokenlist) -{ - int i; - struct ScriptToken *token; - - token = script->token = Jim_Alloc(sizeof(ScriptToken) * tokenlist->count); - - for (i = 0; i < tokenlist->count; i++) { - const ParseToken *t = &tokenlist->list[i]; - - - token->type = t->type; - token->objPtr = JimMakeScriptObj(interp, t); - Jim_IncrRefCount(token->objPtr); - token++; - } - - script->len = i; -} - -static void JimSetScriptFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr) -{ - int scriptTextLen; - const char *scriptText = Jim_GetString(objPtr, &scriptTextLen); - struct JimParserCtx parser; - struct ScriptObj *script; - ParseTokenList tokenlist; - int line = 1; - - - if (objPtr->typePtr == &sourceObjType) { - line = objPtr->internalRep.sourceValue.lineNumber; - } - - - ScriptTokenListInit(&tokenlist); - - JimParserInit(&parser, scriptText, scriptTextLen, line); - while (!parser.eof) { - JimParseScript(&parser); - ScriptAddToken(&tokenlist, parser.tstart, parser.tend - parser.tstart + 1, parser.tt, - parser.tline); - } - - - ScriptAddToken(&tokenlist, scriptText + scriptTextLen, 0, JIM_TT_EOF, 0); - - - script = Jim_Alloc(sizeof(*script)); - memset(script, 0, sizeof(*script)); - script->inUse = 1; - if (objPtr->typePtr == &sourceObjType) { - script->fileNameObj = objPtr->internalRep.sourceValue.fileNameObj; - } - else { - script->fileNameObj = interp->emptyObj; - } - Jim_IncrRefCount(script->fileNameObj); - script->missing = parser.missing.ch; - script->linenr = parser.missing.line; - - ScriptObjAddTokens(interp, script, &tokenlist); - - - ScriptTokenListFree(&tokenlist); - - - Jim_FreeIntRep(interp, objPtr); - Jim_SetIntRepPtr(objPtr, script); - objPtr->typePtr = &scriptObjType; -} - -static void JimAddErrorToStack(Jim_Interp *interp, ScriptObj *script); - -static ScriptObj *JimGetScript(Jim_Interp *interp, Jim_Obj *objPtr) -{ - if (objPtr == interp->emptyObj) { - - objPtr = interp->nullScriptObj; - } - - if (objPtr->typePtr != &scriptObjType || ((struct ScriptObj *)Jim_GetIntRepPtr(objPtr))->substFlags) { - JimSetScriptFromAny(interp, objPtr); - } - - return (ScriptObj *)Jim_GetIntRepPtr(objPtr); -} - -static int JimScriptValid(Jim_Interp *interp, ScriptObj *script) -{ - if (JimParseCheckMissing(interp, script->missing) == JIM_ERR) { - JimAddErrorToStack(interp, script); - return 0; - } - return 1; -} - - -static void JimIncrCmdRefCount(Jim_Cmd *cmdPtr) -{ - cmdPtr->inUse++; -} - -static void JimDecrCmdRefCount(Jim_Interp *interp, Jim_Cmd *cmdPtr) -{ - if (--cmdPtr->inUse == 0) { - if (cmdPtr->isproc) { - Jim_DecrRefCount(interp, cmdPtr->u.proc.argListObjPtr); - Jim_DecrRefCount(interp, cmdPtr->u.proc.bodyObjPtr); - Jim_DecrRefCount(interp, cmdPtr->u.proc.nsObj); - if (cmdPtr->u.proc.staticVars) { - Jim_FreeHashTable(cmdPtr->u.proc.staticVars); - Jim_Free(cmdPtr->u.proc.staticVars); - } - } - else { - - if (cmdPtr->u.native.delProc) { - cmdPtr->u.native.delProc(interp, cmdPtr->u.native.privData); - } - } - if (cmdPtr->prevCmd) { - - JimDecrCmdRefCount(interp, cmdPtr->prevCmd); - } - Jim_Free(cmdPtr); - } -} - -static void JimVariablesHTValDestructor(void *interp, void *val) -{ - Jim_DecrRefCount(interp, ((Jim_Var *)val)->objPtr); - Jim_Free(val); -} - -static const Jim_HashTableType JimVariablesHashTableType = { - JimStringCopyHTHashFunction, - JimStringCopyHTDup, - NULL, - JimStringCopyHTKeyCompare, - JimStringCopyHTKeyDestructor, - JimVariablesHTValDestructor -}; - -static void JimCommandsHT_ValDestructor(void *interp, void *val) -{ - JimDecrCmdRefCount(interp, val); -} - -static const Jim_HashTableType JimCommandsHashTableType = { - JimStringCopyHTHashFunction, - JimStringCopyHTDup, - NULL, - JimStringCopyHTKeyCompare, - JimStringCopyHTKeyDestructor, - JimCommandsHT_ValDestructor -}; - - - -#ifdef jim_ext_namespace -static Jim_Obj *JimQualifyNameObj(Jim_Interp *interp, Jim_Obj *nsObj) -{ - const char *name = Jim_String(nsObj); - if (name[0] == ':' && name[1] == ':') { - - while (*++name == ':') { - } - nsObj = Jim_NewStringObj(interp, name, -1); - } - else if (Jim_Length(interp->framePtr->nsObj)) { - - nsObj = Jim_DuplicateObj(interp, interp->framePtr->nsObj); - Jim_AppendStrings(interp, nsObj, "::", name, NULL); - } - return nsObj; -} - -Jim_Obj *Jim_MakeGlobalNamespaceName(Jim_Interp *interp, Jim_Obj *nameObjPtr) -{ - Jim_Obj *resultObj; - - const char *name = Jim_String(nameObjPtr); - if (name[0] == ':' && name[1] == ':') { - return nameObjPtr; - } - Jim_IncrRefCount(nameObjPtr); - resultObj = Jim_NewStringObj(interp, "::", -1); - Jim_AppendObj(interp, resultObj, nameObjPtr); - Jim_DecrRefCount(interp, nameObjPtr); - - return resultObj; -} - -static const char *JimQualifyName(Jim_Interp *interp, const char *name, Jim_Obj **objPtrPtr) -{ - Jim_Obj *objPtr = interp->emptyObj; - - if (name[0] == ':' && name[1] == ':') { - - while (*++name == ':') { - } - } - else if (Jim_Length(interp->framePtr->nsObj)) { - - objPtr = Jim_DuplicateObj(interp, interp->framePtr->nsObj); - Jim_AppendStrings(interp, objPtr, "::", name, NULL); - name = Jim_String(objPtr); - } - Jim_IncrRefCount(objPtr); - *objPtrPtr = objPtr; - return name; -} - - #define JimFreeQualifiedName(INTERP, OBJ) Jim_DecrRefCount((INTERP), (OBJ)) - -#else - - #define JimQualifyName(INTERP, NAME, DUMMY) (((NAME)[0] == ':' && (NAME)[1] == ':') ? (NAME) + 2 : (NAME)) - #define JimFreeQualifiedName(INTERP, DUMMY) (void)(DUMMY) - -Jim_Obj *Jim_MakeGlobalNamespaceName(Jim_Interp *interp, Jim_Obj *nameObjPtr) -{ - return nameObjPtr; -} -#endif - -static int JimCreateCommand(Jim_Interp *interp, const char *name, Jim_Cmd *cmd) -{ - Jim_HashEntry *he = Jim_FindHashEntry(&interp->commands, name); - if (he) { - - Jim_InterpIncrProcEpoch(interp); - } - - if (he && interp->local) { - - cmd->prevCmd = Jim_GetHashEntryVal(he); - Jim_SetHashVal(&interp->commands, he, cmd); - } - else { - if (he) { - - Jim_DeleteHashEntry(&interp->commands, name); - } - - Jim_AddHashEntry(&interp->commands, name, cmd); - } - return JIM_OK; -} - - -int Jim_CreateCommand(Jim_Interp *interp, const char *cmdNameStr, - Jim_CmdProc *cmdProc, void *privData, Jim_DelCmdProc *delProc) -{ - Jim_Cmd *cmdPtr = Jim_Alloc(sizeof(*cmdPtr)); - - - memset(cmdPtr, 0, sizeof(*cmdPtr)); - cmdPtr->inUse = 1; - cmdPtr->u.native.delProc = delProc; - cmdPtr->u.native.cmdProc = cmdProc; - cmdPtr->u.native.privData = privData; - - JimCreateCommand(interp, cmdNameStr, cmdPtr); - - return JIM_OK; -} - -static int JimCreateProcedureStatics(Jim_Interp *interp, Jim_Cmd *cmdPtr, Jim_Obj *staticsListObjPtr) -{ - int len, i; - - len = Jim_ListLength(interp, staticsListObjPtr); - if (len == 0) { - return JIM_OK; - } - - cmdPtr->u.proc.staticVars = Jim_Alloc(sizeof(Jim_HashTable)); - Jim_InitHashTable(cmdPtr->u.proc.staticVars, &JimVariablesHashTableType, interp); - for (i = 0; i < len; i++) { - Jim_Obj *objPtr, *initObjPtr, *nameObjPtr; - Jim_Var *varPtr; - int subLen; - - objPtr = Jim_ListGetIndex(interp, staticsListObjPtr, i); - - subLen = Jim_ListLength(interp, objPtr); - if (subLen == 1 || subLen == 2) { - nameObjPtr = Jim_ListGetIndex(interp, objPtr, 0); - if (subLen == 1) { - initObjPtr = Jim_GetVariable(interp, nameObjPtr, JIM_NONE); - if (initObjPtr == NULL) { - Jim_SetResultFormatted(interp, - "variable for initialization of static \"%#s\" not found in the local context", - nameObjPtr); - return JIM_ERR; - } - } - else { - initObjPtr = Jim_ListGetIndex(interp, objPtr, 1); - } - if (JimValidName(interp, "static variable", nameObjPtr) != JIM_OK) { - return JIM_ERR; - } - - varPtr = Jim_Alloc(sizeof(*varPtr)); - varPtr->objPtr = initObjPtr; - Jim_IncrRefCount(initObjPtr); - varPtr->linkFramePtr = NULL; - if (Jim_AddHashEntry(cmdPtr->u.proc.staticVars, - Jim_String(nameObjPtr), varPtr) != JIM_OK) { - Jim_SetResultFormatted(interp, - "static variable name \"%#s\" duplicated in statics list", nameObjPtr); - Jim_DecrRefCount(interp, initObjPtr); - Jim_Free(varPtr); - return JIM_ERR; - } - } - else { - Jim_SetResultFormatted(interp, "too many fields in static specifier \"%#s\"", - objPtr); - return JIM_ERR; - } - } - return JIM_OK; -} - -static void JimUpdateProcNamespace(Jim_Interp *interp, Jim_Cmd *cmdPtr, const char *cmdname) -{ -#ifdef jim_ext_namespace - if (cmdPtr->isproc) { - - const char *pt = strrchr(cmdname, ':'); - if (pt && pt != cmdname && pt[-1] == ':') { - Jim_DecrRefCount(interp, cmdPtr->u.proc.nsObj); - cmdPtr->u.proc.nsObj = Jim_NewStringObj(interp, cmdname, pt - cmdname - 1); - Jim_IncrRefCount(cmdPtr->u.proc.nsObj); - - if (Jim_FindHashEntry(&interp->commands, pt + 1)) { - - Jim_InterpIncrProcEpoch(interp); - } - } - } -#endif -} - -static Jim_Cmd *JimCreateProcedureCmd(Jim_Interp *interp, Jim_Obj *argListObjPtr, - Jim_Obj *staticsListObjPtr, Jim_Obj *bodyObjPtr, Jim_Obj *nsObj) -{ - Jim_Cmd *cmdPtr; - int argListLen; - int i; - - argListLen = Jim_ListLength(interp, argListObjPtr); - - - cmdPtr = Jim_Alloc(sizeof(*cmdPtr) + sizeof(struct Jim_ProcArg) * argListLen); - memset(cmdPtr, 0, sizeof(*cmdPtr)); - cmdPtr->inUse = 1; - cmdPtr->isproc = 1; - cmdPtr->u.proc.argListObjPtr = argListObjPtr; - cmdPtr->u.proc.argListLen = argListLen; - cmdPtr->u.proc.bodyObjPtr = bodyObjPtr; - cmdPtr->u.proc.argsPos = -1; - cmdPtr->u.proc.arglist = (struct Jim_ProcArg *)(cmdPtr + 1); - cmdPtr->u.proc.nsObj = nsObj ? nsObj : interp->emptyObj; - Jim_IncrRefCount(argListObjPtr); - Jim_IncrRefCount(bodyObjPtr); - Jim_IncrRefCount(cmdPtr->u.proc.nsObj); - - - if (staticsListObjPtr && JimCreateProcedureStatics(interp, cmdPtr, staticsListObjPtr) != JIM_OK) { - goto err; - } - - - - for (i = 0; i < argListLen; i++) { - Jim_Obj *argPtr; - Jim_Obj *nameObjPtr; - Jim_Obj *defaultObjPtr; - int len; - - - argPtr = Jim_ListGetIndex(interp, argListObjPtr, i); - len = Jim_ListLength(interp, argPtr); - if (len == 0) { - Jim_SetResultString(interp, "argument with no name", -1); -err: - JimDecrCmdRefCount(interp, cmdPtr); - return NULL; - } - if (len > 2) { - Jim_SetResultFormatted(interp, "too many fields in argument specifier \"%#s\"", argPtr); - goto err; - } - - if (len == 2) { - - nameObjPtr = Jim_ListGetIndex(interp, argPtr, 0); - defaultObjPtr = Jim_ListGetIndex(interp, argPtr, 1); - } - else { - - nameObjPtr = argPtr; - defaultObjPtr = NULL; - } - - - if (Jim_CompareStringImmediate(interp, nameObjPtr, "args")) { - if (cmdPtr->u.proc.argsPos >= 0) { - Jim_SetResultString(interp, "'args' specified more than once", -1); - goto err; - } - cmdPtr->u.proc.argsPos = i; - } - else { - if (len == 2) { - cmdPtr->u.proc.optArity++; - } - else { - cmdPtr->u.proc.reqArity++; - } - } - - cmdPtr->u.proc.arglist[i].nameObjPtr = nameObjPtr; - cmdPtr->u.proc.arglist[i].defaultObjPtr = defaultObjPtr; - } - - return cmdPtr; -} - -int Jim_DeleteCommand(Jim_Interp *interp, const char *name) -{ - int ret = JIM_OK; - Jim_Obj *qualifiedNameObj; - const char *qualname = JimQualifyName(interp, name, &qualifiedNameObj); - - if (Jim_DeleteHashEntry(&interp->commands, qualname) == JIM_ERR) { - Jim_SetResultFormatted(interp, "can't delete \"%s\": command doesn't exist", name); - ret = JIM_ERR; - } - else { - Jim_InterpIncrProcEpoch(interp); - } - - JimFreeQualifiedName(interp, qualifiedNameObj); - - return ret; -} - -int Jim_RenameCommand(Jim_Interp *interp, const char *oldName, const char *newName) -{ - int ret = JIM_ERR; - Jim_HashEntry *he; - Jim_Cmd *cmdPtr; - Jim_Obj *qualifiedOldNameObj; - Jim_Obj *qualifiedNewNameObj; - const char *fqold; - const char *fqnew; - - if (newName[0] == 0) { - return Jim_DeleteCommand(interp, oldName); - } - - fqold = JimQualifyName(interp, oldName, &qualifiedOldNameObj); - fqnew = JimQualifyName(interp, newName, &qualifiedNewNameObj); - - - he = Jim_FindHashEntry(&interp->commands, fqold); - if (he == NULL) { - Jim_SetResultFormatted(interp, "can't rename \"%s\": command doesn't exist", oldName); - } - else if (Jim_FindHashEntry(&interp->commands, fqnew)) { - Jim_SetResultFormatted(interp, "can't rename to \"%s\": command already exists", newName); - } - else { - - cmdPtr = Jim_GetHashEntryVal(he); - JimIncrCmdRefCount(cmdPtr); - JimUpdateProcNamespace(interp, cmdPtr, fqnew); - Jim_AddHashEntry(&interp->commands, fqnew, cmdPtr); - - - Jim_DeleteHashEntry(&interp->commands, fqold); - - - Jim_InterpIncrProcEpoch(interp); - - ret = JIM_OK; - } - - JimFreeQualifiedName(interp, qualifiedOldNameObj); - JimFreeQualifiedName(interp, qualifiedNewNameObj); - - return ret; -} - - -static void FreeCommandInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) -{ - Jim_DecrRefCount(interp, objPtr->internalRep.cmdValue.nsObj); -} - -static void DupCommandInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) -{ - dupPtr->internalRep.cmdValue = srcPtr->internalRep.cmdValue; - dupPtr->typePtr = srcPtr->typePtr; - Jim_IncrRefCount(dupPtr->internalRep.cmdValue.nsObj); -} - -static const Jim_ObjType commandObjType = { - "command", - FreeCommandInternalRep, - DupCommandInternalRep, - NULL, - JIM_TYPE_REFERENCES, -}; - -Jim_Cmd *Jim_GetCommand(Jim_Interp *interp, Jim_Obj *objPtr, int flags) -{ - Jim_Cmd *cmd; - - if (objPtr->typePtr != &commandObjType || - objPtr->internalRep.cmdValue.procEpoch != interp->procEpoch -#ifdef jim_ext_namespace - || !Jim_StringEqObj(objPtr->internalRep.cmdValue.nsObj, interp->framePtr->nsObj) -#endif - ) { - - - - const char *name = Jim_String(objPtr); - Jim_HashEntry *he; - - if (name[0] == ':' && name[1] == ':') { - while (*++name == ':') { - } - } -#ifdef jim_ext_namespace - else if (Jim_Length(interp->framePtr->nsObj)) { - - Jim_Obj *nameObj = Jim_DuplicateObj(interp, interp->framePtr->nsObj); - Jim_AppendStrings(interp, nameObj, "::", name, NULL); - he = Jim_FindHashEntry(&interp->commands, Jim_String(nameObj)); - Jim_FreeNewObj(interp, nameObj); - if (he) { - goto found; - } - } -#endif - - - he = Jim_FindHashEntry(&interp->commands, name); - if (he == NULL) { - if (flags & JIM_ERRMSG) { - Jim_SetResultFormatted(interp, "invalid command name \"%#s\"", objPtr); - } - return NULL; - } -#ifdef jim_ext_namespace -found: -#endif - cmd = Jim_GetHashEntryVal(he); - - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &commandObjType; - objPtr->internalRep.cmdValue.procEpoch = interp->procEpoch; - objPtr->internalRep.cmdValue.cmdPtr = cmd; - objPtr->internalRep.cmdValue.nsObj = interp->framePtr->nsObj; - Jim_IncrRefCount(interp->framePtr->nsObj); - } - else { - cmd = objPtr->internalRep.cmdValue.cmdPtr; - } - while (cmd->u.proc.upcall) { - cmd = cmd->prevCmd; - } - return cmd; -} - - - -#define JIM_DICT_SUGAR 100 - -static int SetVariableFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); - -static const Jim_ObjType variableObjType = { - "variable", - NULL, - NULL, - NULL, - JIM_TYPE_REFERENCES, -}; - -static int JimValidName(Jim_Interp *interp, const char *type, Jim_Obj *nameObjPtr) -{ - - if (nameObjPtr->typePtr != &variableObjType) { - int len; - const char *str = Jim_GetString(nameObjPtr, &len); - if (memchr(str, '\0', len)) { - Jim_SetResultFormatted(interp, "%s name contains embedded null", type); - return JIM_ERR; - } - } - return JIM_OK; -} - -static int SetVariableFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr) -{ - const char *varName; - Jim_CallFrame *framePtr; - Jim_HashEntry *he; - int global; - int len; - - - if (objPtr->typePtr == &variableObjType) { - framePtr = objPtr->internalRep.varValue.global ? interp->topFramePtr : interp->framePtr; - if (objPtr->internalRep.varValue.callFrameId == framePtr->id) { - - return JIM_OK; - } - - } - else if (objPtr->typePtr == &dictSubstObjType) { - return JIM_DICT_SUGAR; - } - else if (JimValidName(interp, "variable", objPtr) != JIM_OK) { - return JIM_ERR; - } - - - varName = Jim_GetString(objPtr, &len); - - - if (len && varName[len - 1] == ')' && strchr(varName, '(') != NULL) { - return JIM_DICT_SUGAR; - } - - if (varName[0] == ':' && varName[1] == ':') { - while (*++varName == ':') { - } - global = 1; - framePtr = interp->topFramePtr; - } - else { - global = 0; - framePtr = interp->framePtr; - } - - - he = Jim_FindHashEntry(&framePtr->vars, varName); - if (he == NULL) { - if (!global && framePtr->staticVars) { - - he = Jim_FindHashEntry(framePtr->staticVars, varName); - } - if (he == NULL) { - return JIM_ERR; - } - } - - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &variableObjType; - objPtr->internalRep.varValue.callFrameId = framePtr->id; - objPtr->internalRep.varValue.varPtr = Jim_GetHashEntryVal(he); - objPtr->internalRep.varValue.global = global; - return JIM_OK; -} - - -static int JimDictSugarSet(Jim_Interp *interp, Jim_Obj *ObjPtr, Jim_Obj *valObjPtr); -static Jim_Obj *JimDictSugarGet(Jim_Interp *interp, Jim_Obj *ObjPtr, int flags); - -static Jim_Var *JimCreateVariable(Jim_Interp *interp, Jim_Obj *nameObjPtr, Jim_Obj *valObjPtr) -{ - const char *name; - Jim_CallFrame *framePtr; - int global; - - - Jim_Var *var = Jim_Alloc(sizeof(*var)); - - var->objPtr = valObjPtr; - Jim_IncrRefCount(valObjPtr); - var->linkFramePtr = NULL; - - name = Jim_String(nameObjPtr); - if (name[0] == ':' && name[1] == ':') { - while (*++name == ':') { - } - framePtr = interp->topFramePtr; - global = 1; - } - else { - framePtr = interp->framePtr; - global = 0; - } - - - Jim_AddHashEntry(&framePtr->vars, name, var); - - - Jim_FreeIntRep(interp, nameObjPtr); - nameObjPtr->typePtr = &variableObjType; - nameObjPtr->internalRep.varValue.callFrameId = framePtr->id; - nameObjPtr->internalRep.varValue.varPtr = var; - nameObjPtr->internalRep.varValue.global = global; - - return var; -} - - -int Jim_SetVariable(Jim_Interp *interp, Jim_Obj *nameObjPtr, Jim_Obj *valObjPtr) -{ - int err; - Jim_Var *var; - - switch (SetVariableFromAny(interp, nameObjPtr)) { - case JIM_DICT_SUGAR: - return JimDictSugarSet(interp, nameObjPtr, valObjPtr); - - case JIM_ERR: - if (JimValidName(interp, "variable", nameObjPtr) != JIM_OK) { - return JIM_ERR; - } - JimCreateVariable(interp, nameObjPtr, valObjPtr); - break; - - case JIM_OK: - var = nameObjPtr->internalRep.varValue.varPtr; - if (var->linkFramePtr == NULL) { - Jim_IncrRefCount(valObjPtr); - Jim_DecrRefCount(interp, var->objPtr); - var->objPtr = valObjPtr; - } - else { - Jim_CallFrame *savedCallFrame; - - savedCallFrame = interp->framePtr; - interp->framePtr = var->linkFramePtr; - err = Jim_SetVariable(interp, var->objPtr, valObjPtr); - interp->framePtr = savedCallFrame; - if (err != JIM_OK) - return err; - } - } - return JIM_OK; -} - -int Jim_SetVariableStr(Jim_Interp *interp, const char *name, Jim_Obj *objPtr) -{ - Jim_Obj *nameObjPtr; - int result; - - nameObjPtr = Jim_NewStringObj(interp, name, -1); - Jim_IncrRefCount(nameObjPtr); - result = Jim_SetVariable(interp, nameObjPtr, objPtr); - Jim_DecrRefCount(interp, nameObjPtr); - return result; -} - -int Jim_SetGlobalVariableStr(Jim_Interp *interp, const char *name, Jim_Obj *objPtr) -{ - Jim_CallFrame *savedFramePtr; - int result; - - savedFramePtr = interp->framePtr; - interp->framePtr = interp->topFramePtr; - result = Jim_SetVariableStr(interp, name, objPtr); - interp->framePtr = savedFramePtr; - return result; -} - -int Jim_SetVariableStrWithStr(Jim_Interp *interp, const char *name, const char *val) -{ - Jim_Obj *valObjPtr; - int result; - - valObjPtr = Jim_NewStringObj(interp, val, -1); - Jim_IncrRefCount(valObjPtr); - result = Jim_SetVariableStr(interp, name, valObjPtr); - Jim_DecrRefCount(interp, valObjPtr); - return result; -} - -int Jim_SetVariableLink(Jim_Interp *interp, Jim_Obj *nameObjPtr, - Jim_Obj *targetNameObjPtr, Jim_CallFrame *targetCallFrame) -{ - const char *varName; - const char *targetName; - Jim_CallFrame *framePtr; - Jim_Var *varPtr; - - - switch (SetVariableFromAny(interp, nameObjPtr)) { - case JIM_DICT_SUGAR: - - Jim_SetResultFormatted(interp, "bad variable name \"%#s\": upvar won't create a scalar variable that looks like an array element", nameObjPtr); - return JIM_ERR; - - case JIM_OK: - varPtr = nameObjPtr->internalRep.varValue.varPtr; - - if (varPtr->linkFramePtr == NULL) { - Jim_SetResultFormatted(interp, "variable \"%#s\" already exists", nameObjPtr); - return JIM_ERR; - } - - - varPtr->linkFramePtr = NULL; - break; - } - - - - varName = Jim_String(nameObjPtr); - - if (varName[0] == ':' && varName[1] == ':') { - while (*++varName == ':') { - } - - framePtr = interp->topFramePtr; - } - else { - framePtr = interp->framePtr; - } - - targetName = Jim_String(targetNameObjPtr); - if (targetName[0] == ':' && targetName[1] == ':') { - while (*++targetName == ':') { - } - targetNameObjPtr = Jim_NewStringObj(interp, targetName, -1); - targetCallFrame = interp->topFramePtr; - } - Jim_IncrRefCount(targetNameObjPtr); - - if (framePtr->level < targetCallFrame->level) { - Jim_SetResultFormatted(interp, - "bad variable name \"%#s\": upvar won't create namespace variable that refers to procedure variable", - nameObjPtr); - Jim_DecrRefCount(interp, targetNameObjPtr); - return JIM_ERR; - } - - - if (framePtr == targetCallFrame) { - Jim_Obj *objPtr = targetNameObjPtr; - - - while (1) { - if (strcmp(Jim_String(objPtr), varName) == 0) { - Jim_SetResultString(interp, "can't upvar from variable to itself", -1); - Jim_DecrRefCount(interp, targetNameObjPtr); - return JIM_ERR; - } - if (SetVariableFromAny(interp, objPtr) != JIM_OK) - break; - varPtr = objPtr->internalRep.varValue.varPtr; - if (varPtr->linkFramePtr != targetCallFrame) - break; - objPtr = varPtr->objPtr; - } - } - - - Jim_SetVariable(interp, nameObjPtr, targetNameObjPtr); - - nameObjPtr->internalRep.varValue.varPtr->linkFramePtr = targetCallFrame; - Jim_DecrRefCount(interp, targetNameObjPtr); - return JIM_OK; -} - -Jim_Obj *Jim_GetVariable(Jim_Interp *interp, Jim_Obj *nameObjPtr, int flags) -{ - switch (SetVariableFromAny(interp, nameObjPtr)) { - case JIM_OK:{ - Jim_Var *varPtr = nameObjPtr->internalRep.varValue.varPtr; - - if (varPtr->linkFramePtr == NULL) { - return varPtr->objPtr; - } - else { - Jim_Obj *objPtr; - - - Jim_CallFrame *savedCallFrame = interp->framePtr; - - interp->framePtr = varPtr->linkFramePtr; - objPtr = Jim_GetVariable(interp, varPtr->objPtr, flags); - interp->framePtr = savedCallFrame; - if (objPtr) { - return objPtr; - } - - } - } - break; - - case JIM_DICT_SUGAR: - - return JimDictSugarGet(interp, nameObjPtr, flags); - } - if (flags & JIM_ERRMSG) { - Jim_SetResultFormatted(interp, "can't read \"%#s\": no such variable", nameObjPtr); - } - return NULL; -} - -Jim_Obj *Jim_GetGlobalVariable(Jim_Interp *interp, Jim_Obj *nameObjPtr, int flags) -{ - Jim_CallFrame *savedFramePtr; - Jim_Obj *objPtr; - - savedFramePtr = interp->framePtr; - interp->framePtr = interp->topFramePtr; - objPtr = Jim_GetVariable(interp, nameObjPtr, flags); - interp->framePtr = savedFramePtr; - - return objPtr; -} - -Jim_Obj *Jim_GetVariableStr(Jim_Interp *interp, const char *name, int flags) -{ - Jim_Obj *nameObjPtr, *varObjPtr; - - nameObjPtr = Jim_NewStringObj(interp, name, -1); - Jim_IncrRefCount(nameObjPtr); - varObjPtr = Jim_GetVariable(interp, nameObjPtr, flags); - Jim_DecrRefCount(interp, nameObjPtr); - return varObjPtr; -} - -Jim_Obj *Jim_GetGlobalVariableStr(Jim_Interp *interp, const char *name, int flags) -{ - Jim_CallFrame *savedFramePtr; - Jim_Obj *objPtr; - - savedFramePtr = interp->framePtr; - interp->framePtr = interp->topFramePtr; - objPtr = Jim_GetVariableStr(interp, name, flags); - interp->framePtr = savedFramePtr; - - return objPtr; -} - -int Jim_UnsetVariable(Jim_Interp *interp, Jim_Obj *nameObjPtr, int flags) -{ - Jim_Var *varPtr; - int retval; - Jim_CallFrame *framePtr; - - retval = SetVariableFromAny(interp, nameObjPtr); - if (retval == JIM_DICT_SUGAR) { - - return JimDictSugarSet(interp, nameObjPtr, NULL); - } - else if (retval == JIM_OK) { - varPtr = nameObjPtr->internalRep.varValue.varPtr; - - - if (varPtr->linkFramePtr) { - framePtr = interp->framePtr; - interp->framePtr = varPtr->linkFramePtr; - retval = Jim_UnsetVariable(interp, varPtr->objPtr, JIM_NONE); - interp->framePtr = framePtr; - } - else { - const char *name = Jim_String(nameObjPtr); - if (nameObjPtr->internalRep.varValue.global) { - name += 2; - framePtr = interp->topFramePtr; - } - else { - framePtr = interp->framePtr; - } - - retval = Jim_DeleteHashEntry(&framePtr->vars, name); - if (retval == JIM_OK) { - - framePtr->id = interp->callFrameEpoch++; - } - } - } - if (retval != JIM_OK && (flags & JIM_ERRMSG)) { - Jim_SetResultFormatted(interp, "can't unset \"%#s\": no such variable", nameObjPtr); - } - return retval; -} - - - -static void JimDictSugarParseVarKey(Jim_Interp *interp, Jim_Obj *objPtr, - Jim_Obj **varPtrPtr, Jim_Obj **keyPtrPtr) -{ - const char *str, *p; - int len, keyLen; - Jim_Obj *varObjPtr, *keyObjPtr; - - str = Jim_GetString(objPtr, &len); - - p = strchr(str, '('); - JimPanic((p == NULL, "JimDictSugarParseVarKey() called for non-dict-sugar (%s)", str)); - - varObjPtr = Jim_NewStringObj(interp, str, p - str); - - p++; - keyLen = (str + len) - p; - if (str[len - 1] == ')') { - keyLen--; - } - - - keyObjPtr = Jim_NewStringObj(interp, p, keyLen); - - Jim_IncrRefCount(varObjPtr); - Jim_IncrRefCount(keyObjPtr); - *varPtrPtr = varObjPtr; - *keyPtrPtr = keyObjPtr; -} - -static int JimDictSugarSet(Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj *valObjPtr) -{ - int err; - - SetDictSubstFromAny(interp, objPtr); - - err = Jim_SetDictKeysVector(interp, objPtr->internalRep.dictSubstValue.varNameObjPtr, - &objPtr->internalRep.dictSubstValue.indexObjPtr, 1, valObjPtr, JIM_MUSTEXIST); - - if (err == JIM_OK) { - - Jim_SetEmptyResult(interp); - } - else { - if (!valObjPtr) { - - if (Jim_GetVariable(interp, objPtr->internalRep.dictSubstValue.varNameObjPtr, JIM_NONE)) { - Jim_SetResultFormatted(interp, "can't unset \"%#s\": no such element in array", - objPtr); - return err; - } - } - - Jim_SetResultFormatted(interp, "can't %s \"%#s\": variable isn't array", - (valObjPtr ? "set" : "unset"), objPtr); - } - return err; -} - -static Jim_Obj *JimDictExpandArrayVariable(Jim_Interp *interp, Jim_Obj *varObjPtr, - Jim_Obj *keyObjPtr, int flags) -{ - Jim_Obj *dictObjPtr; - Jim_Obj *resObjPtr = NULL; - int ret; - - dictObjPtr = Jim_GetVariable(interp, varObjPtr, JIM_ERRMSG); - if (!dictObjPtr) { - return NULL; - } - - ret = Jim_DictKey(interp, dictObjPtr, keyObjPtr, &resObjPtr, JIM_NONE); - if (ret != JIM_OK) { - Jim_SetResultFormatted(interp, - "can't read \"%#s(%#s)\": %s array", varObjPtr, keyObjPtr, - ret < 0 ? "variable isn't" : "no such element in"); - } - else if ((flags & JIM_UNSHARED) && Jim_IsShared(dictObjPtr)) { - - Jim_SetVariable(interp, varObjPtr, Jim_DuplicateObj(interp, dictObjPtr)); - } - - return resObjPtr; -} - - -static Jim_Obj *JimDictSugarGet(Jim_Interp *interp, Jim_Obj *objPtr, int flags) -{ - SetDictSubstFromAny(interp, objPtr); - - return JimDictExpandArrayVariable(interp, - objPtr->internalRep.dictSubstValue.varNameObjPtr, - objPtr->internalRep.dictSubstValue.indexObjPtr, flags); -} - - - -void FreeDictSubstInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) -{ - Jim_DecrRefCount(interp, objPtr->internalRep.dictSubstValue.varNameObjPtr); - Jim_DecrRefCount(interp, objPtr->internalRep.dictSubstValue.indexObjPtr); -} - -static void DupDictSubstInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) -{ - - dupPtr->internalRep = srcPtr->internalRep; - - Jim_IncrRefCount(dupPtr->internalRep.dictSubstValue.varNameObjPtr); - Jim_IncrRefCount(dupPtr->internalRep.dictSubstValue.indexObjPtr); -} - - -static void SetDictSubstFromAny(Jim_Interp *interp, Jim_Obj *objPtr) -{ - if (objPtr->typePtr != &dictSubstObjType) { - Jim_Obj *varObjPtr, *keyObjPtr; - - if (objPtr->typePtr == &interpolatedObjType) { - - - varObjPtr = objPtr->internalRep.dictSubstValue.varNameObjPtr; - keyObjPtr = objPtr->internalRep.dictSubstValue.indexObjPtr; - - Jim_IncrRefCount(varObjPtr); - Jim_IncrRefCount(keyObjPtr); - } - else { - JimDictSugarParseVarKey(interp, objPtr, &varObjPtr, &keyObjPtr); - } - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &dictSubstObjType; - objPtr->internalRep.dictSubstValue.varNameObjPtr = varObjPtr; - objPtr->internalRep.dictSubstValue.indexObjPtr = keyObjPtr; - } -} - -static Jim_Obj *JimExpandDictSugar(Jim_Interp *interp, Jim_Obj *objPtr) -{ - Jim_Obj *resObjPtr = NULL; - Jim_Obj *substKeyObjPtr = NULL; - - SetDictSubstFromAny(interp, objPtr); - - if (Jim_SubstObj(interp, objPtr->internalRep.dictSubstValue.indexObjPtr, - &substKeyObjPtr, JIM_NONE) - != JIM_OK) { - return NULL; - } - Jim_IncrRefCount(substKeyObjPtr); - resObjPtr = - JimDictExpandArrayVariable(interp, objPtr->internalRep.dictSubstValue.varNameObjPtr, - substKeyObjPtr, 0); - Jim_DecrRefCount(interp, substKeyObjPtr); - - return resObjPtr; -} - -static Jim_Obj *JimExpandExprSugar(Jim_Interp *interp, Jim_Obj *objPtr) -{ - if (Jim_EvalExpression(interp, objPtr) == JIM_OK) { - return Jim_GetResult(interp); - } - return NULL; -} - - -static Jim_CallFrame *JimCreateCallFrame(Jim_Interp *interp, Jim_CallFrame *parent, Jim_Obj *nsObj) -{ - Jim_CallFrame *cf; - - if (interp->freeFramesList) { - cf = interp->freeFramesList; - interp->freeFramesList = cf->next; - - cf->argv = NULL; - cf->argc = 0; - cf->procArgsObjPtr = NULL; - cf->procBodyObjPtr = NULL; - cf->next = NULL; - cf->staticVars = NULL; - cf->localCommands = NULL; - cf->tailcallObj = NULL; - cf->tailcallCmd = NULL; - } - else { - cf = Jim_Alloc(sizeof(*cf)); - memset(cf, 0, sizeof(*cf)); - - Jim_InitHashTable(&cf->vars, &JimVariablesHashTableType, interp); - } - - cf->id = interp->callFrameEpoch++; - cf->parent = parent; - cf->level = parent ? parent->level + 1 : 0; - cf->nsObj = nsObj; - Jim_IncrRefCount(nsObj); - - return cf; -} - -static int JimDeleteLocalProcs(Jim_Interp *interp, Jim_Stack *localCommands) -{ - - if (localCommands) { - Jim_Obj *cmdNameObj; - - while ((cmdNameObj = Jim_StackPop(localCommands)) != NULL) { - Jim_HashEntry *he; - Jim_Obj *fqObjName; - Jim_HashTable *ht = &interp->commands; - - const char *fqname = JimQualifyName(interp, Jim_String(cmdNameObj), &fqObjName); - - he = Jim_FindHashEntry(ht, fqname); - - if (he) { - Jim_Cmd *cmd = Jim_GetHashEntryVal(he); - if (cmd->prevCmd) { - Jim_Cmd *prevCmd = cmd->prevCmd; - cmd->prevCmd = NULL; - - - JimDecrCmdRefCount(interp, cmd); - - - Jim_SetHashVal(ht, he, prevCmd); - } - else { - Jim_DeleteHashEntry(ht, fqname); - } - Jim_InterpIncrProcEpoch(interp); - } - Jim_DecrRefCount(interp, cmdNameObj); - JimFreeQualifiedName(interp, fqObjName); - } - Jim_FreeStack(localCommands); - Jim_Free(localCommands); - } - return JIM_OK; -} - -static int JimInvokeDefer(Jim_Interp *interp, int retcode) -{ - Jim_Obj *objPtr; - - - if (Jim_FindHashEntry(&interp->framePtr->vars, "jim::defer") == NULL) { - return retcode; - } - - objPtr = Jim_GetVariableStr(interp, "jim::defer", JIM_NONE); - - if (objPtr) { - int ret = JIM_OK; - int i; - int listLen = Jim_ListLength(interp, objPtr); - Jim_Obj *resultObjPtr; - - Jim_IncrRefCount(objPtr); - - resultObjPtr = Jim_GetResult(interp); - Jim_IncrRefCount(resultObjPtr); - Jim_SetEmptyResult(interp); - - - for (i = listLen; i > 0; i--) { - - Jim_Obj *scriptObjPtr = Jim_ListGetIndex(interp, objPtr, i - 1); - ret = Jim_EvalObj(interp, scriptObjPtr); - if (ret != JIM_OK) { - break; - } - } - - if (ret == JIM_OK || retcode == JIM_ERR) { - - Jim_SetResult(interp, resultObjPtr); - } - else { - retcode = ret; - } - - Jim_DecrRefCount(interp, resultObjPtr); - Jim_DecrRefCount(interp, objPtr); - } - return retcode; -} - -#define JIM_FCF_FULL 0 -#define JIM_FCF_REUSE 1 -static void JimFreeCallFrame(Jim_Interp *interp, Jim_CallFrame *cf, int action) - { - JimDeleteLocalProcs(interp, cf->localCommands); - - if (cf->procArgsObjPtr) - Jim_DecrRefCount(interp, cf->procArgsObjPtr); - if (cf->procBodyObjPtr) - Jim_DecrRefCount(interp, cf->procBodyObjPtr); - Jim_DecrRefCount(interp, cf->nsObj); - if (action == JIM_FCF_FULL || cf->vars.size != JIM_HT_INITIAL_SIZE) - Jim_FreeHashTable(&cf->vars); - else { - int i; - Jim_HashEntry **table = cf->vars.table, *he; - - for (i = 0; i < JIM_HT_INITIAL_SIZE; i++) { - he = table[i]; - while (he != NULL) { - Jim_HashEntry *nextEntry = he->next; - Jim_Var *varPtr = Jim_GetHashEntryVal(he); - - Jim_DecrRefCount(interp, varPtr->objPtr); - Jim_Free(Jim_GetHashEntryKey(he)); - Jim_Free(varPtr); - Jim_Free(he); - table[i] = NULL; - he = nextEntry; - } - } - cf->vars.used = 0; - } - cf->next = interp->freeFramesList; - interp->freeFramesList = cf; -} - - - -int Jim_IsBigEndian(void) -{ - union { - unsigned short s; - unsigned char c[2]; - } uval = {0x0102}; - - return uval.c[0] == 1; -} - - -Jim_Interp *Jim_CreateInterp(void) -{ - Jim_Interp *i = Jim_Alloc(sizeof(*i)); - - memset(i, 0, sizeof(*i)); - - i->maxCallFrameDepth = JIM_MAX_CALLFRAME_DEPTH; - i->maxEvalDepth = JIM_MAX_EVAL_DEPTH; - i->lastCollectTime = time(NULL); - - Jim_InitHashTable(&i->commands, &JimCommandsHashTableType, i); -#ifdef JIM_REFERENCES - Jim_InitHashTable(&i->references, &JimReferencesHashTableType, i); -#endif - Jim_InitHashTable(&i->assocData, &JimAssocDataHashTableType, i); - Jim_InitHashTable(&i->packages, &JimPackageHashTableType, NULL); - i->emptyObj = Jim_NewEmptyStringObj(i); - i->trueObj = Jim_NewIntObj(i, 1); - i->falseObj = Jim_NewIntObj(i, 0); - i->framePtr = i->topFramePtr = JimCreateCallFrame(i, NULL, i->emptyObj); - i->errorFileNameObj = i->emptyObj; - i->result = i->emptyObj; - i->stackTrace = Jim_NewListObj(i, NULL, 0); - i->unknown = Jim_NewStringObj(i, "unknown", -1); - i->errorProc = i->emptyObj; - i->currentScriptObj = Jim_NewEmptyStringObj(i); - i->nullScriptObj = Jim_NewEmptyStringObj(i); - Jim_IncrRefCount(i->emptyObj); - Jim_IncrRefCount(i->errorFileNameObj); - Jim_IncrRefCount(i->result); - Jim_IncrRefCount(i->stackTrace); - Jim_IncrRefCount(i->unknown); - Jim_IncrRefCount(i->currentScriptObj); - Jim_IncrRefCount(i->nullScriptObj); - Jim_IncrRefCount(i->errorProc); - Jim_IncrRefCount(i->trueObj); - Jim_IncrRefCount(i->falseObj); - - - Jim_SetVariableStrWithStr(i, JIM_LIBPATH, TCL_LIBRARY); - Jim_SetVariableStrWithStr(i, JIM_INTERACTIVE, "0"); - - Jim_SetVariableStrWithStr(i, "tcl_platform(engine)", "Jim"); - Jim_SetVariableStrWithStr(i, "tcl_platform(os)", TCL_PLATFORM_OS); - Jim_SetVariableStrWithStr(i, "tcl_platform(platform)", TCL_PLATFORM_PLATFORM); - Jim_SetVariableStrWithStr(i, "tcl_platform(pathSeparator)", TCL_PLATFORM_PATH_SEPARATOR); - Jim_SetVariableStrWithStr(i, "tcl_platform(byteOrder)", Jim_IsBigEndian() ? "bigEndian" : "littleEndian"); - Jim_SetVariableStrWithStr(i, "tcl_platform(threaded)", "0"); - Jim_SetVariableStr(i, "tcl_platform(pointerSize)", Jim_NewIntObj(i, sizeof(void *))); - Jim_SetVariableStr(i, "tcl_platform(wordSize)", Jim_NewIntObj(i, sizeof(jim_wide))); - - return i; -} - -void Jim_FreeInterp(Jim_Interp *i) -{ - Jim_CallFrame *cf, *cfx; - - Jim_Obj *objPtr, *nextObjPtr; - - - for (cf = i->framePtr; cf; cf = cfx) { - - JimInvokeDefer(i, JIM_OK); - cfx = cf->parent; - JimFreeCallFrame(i, cf, JIM_FCF_FULL); - } - - Jim_DecrRefCount(i, i->emptyObj); - Jim_DecrRefCount(i, i->trueObj); - Jim_DecrRefCount(i, i->falseObj); - Jim_DecrRefCount(i, i->result); - Jim_DecrRefCount(i, i->stackTrace); - Jim_DecrRefCount(i, i->errorProc); - Jim_DecrRefCount(i, i->unknown); - Jim_DecrRefCount(i, i->errorFileNameObj); - Jim_DecrRefCount(i, i->currentScriptObj); - Jim_DecrRefCount(i, i->nullScriptObj); - Jim_FreeHashTable(&i->commands); -#ifdef JIM_REFERENCES - Jim_FreeHashTable(&i->references); -#endif - Jim_FreeHashTable(&i->packages); - Jim_Free(i->prngState); - Jim_FreeHashTable(&i->assocData); - -#ifdef JIM_MAINTAINER - if (i->liveList != NULL) { - objPtr = i->liveList; - - printf("\n-------------------------------------\n"); - printf("Objects still in the free list:\n"); - while (objPtr) { - const char *type = objPtr->typePtr ? objPtr->typePtr->name : "string"; - Jim_String(objPtr); - - if (objPtr->bytes && strlen(objPtr->bytes) > 20) { - printf("%p (%d) %-10s: '%.20s...'\n", - (void *)objPtr, objPtr->refCount, type, objPtr->bytes); - } - else { - printf("%p (%d) %-10s: '%s'\n", - (void *)objPtr, objPtr->refCount, type, objPtr->bytes ? objPtr->bytes : "(null)"); - } - if (objPtr->typePtr == &sourceObjType) { - printf("FILE %s LINE %d\n", - Jim_String(objPtr->internalRep.sourceValue.fileNameObj), - objPtr->internalRep.sourceValue.lineNumber); - } - objPtr = objPtr->nextObjPtr; - } - printf("-------------------------------------\n\n"); - JimPanic((1, "Live list non empty freeing the interpreter! Leak?")); - } -#endif - - - objPtr = i->freeList; - while (objPtr) { - nextObjPtr = objPtr->nextObjPtr; - Jim_Free(objPtr); - objPtr = nextObjPtr; - } - - - for (cf = i->freeFramesList; cf; cf = cfx) { - cfx = cf->next; - if (cf->vars.table) - Jim_FreeHashTable(&cf->vars); - Jim_Free(cf); - } - - - Jim_Free(i); -} - -Jim_CallFrame *Jim_GetCallFrameByLevel(Jim_Interp *interp, Jim_Obj *levelObjPtr) -{ - long level; - const char *str; - Jim_CallFrame *framePtr; - - if (levelObjPtr) { - str = Jim_String(levelObjPtr); - if (str[0] == '#') { - char *endptr; - - level = jim_strtol(str + 1, &endptr); - if (str[1] == '\0' || endptr[0] != '\0') { - level = -1; - } - } - else { - if (Jim_GetLong(interp, levelObjPtr, &level) != JIM_OK || level < 0) { - level = -1; - } - else { - - level = interp->framePtr->level - level; - } - } - } - else { - str = "1"; - level = interp->framePtr->level - 1; - } - - if (level == 0) { - return interp->topFramePtr; - } - if (level > 0) { - - for (framePtr = interp->framePtr; framePtr; framePtr = framePtr->parent) { - if (framePtr->level == level) { - return framePtr; - } - } - } - - Jim_SetResultFormatted(interp, "bad level \"%s\"", str); - return NULL; -} - -static Jim_CallFrame *JimGetCallFrameByInteger(Jim_Interp *interp, Jim_Obj *levelObjPtr) -{ - long level; - Jim_CallFrame *framePtr; - - if (Jim_GetLong(interp, levelObjPtr, &level) == JIM_OK) { - if (level <= 0) { - - level = interp->framePtr->level + level; - } - - if (level == 0) { - return interp->topFramePtr; - } - - - for (framePtr = interp->framePtr; framePtr; framePtr = framePtr->parent) { - if (framePtr->level == level) { - return framePtr; - } - } - } - - Jim_SetResultFormatted(interp, "bad level \"%#s\"", levelObjPtr); - return NULL; -} - -static void JimResetStackTrace(Jim_Interp *interp) -{ - Jim_DecrRefCount(interp, interp->stackTrace); - interp->stackTrace = Jim_NewListObj(interp, NULL, 0); - Jim_IncrRefCount(interp->stackTrace); -} - -static void JimSetStackTrace(Jim_Interp *interp, Jim_Obj *stackTraceObj) -{ - int len; - - - Jim_IncrRefCount(stackTraceObj); - Jim_DecrRefCount(interp, interp->stackTrace); - interp->stackTrace = stackTraceObj; - interp->errorFlag = 1; - - len = Jim_ListLength(interp, interp->stackTrace); - if (len >= 3) { - if (Jim_Length(Jim_ListGetIndex(interp, interp->stackTrace, len - 2)) == 0) { - interp->addStackTrace = 1; - } - } -} - -static void JimAppendStackTrace(Jim_Interp *interp, const char *procname, - Jim_Obj *fileNameObj, int linenr) -{ - if (strcmp(procname, "unknown") == 0) { - procname = ""; - } - if (!*procname && !Jim_Length(fileNameObj)) { - - return; - } - - if (Jim_IsShared(interp->stackTrace)) { - Jim_DecrRefCount(interp, interp->stackTrace); - interp->stackTrace = Jim_DuplicateObj(interp, interp->stackTrace); - Jim_IncrRefCount(interp->stackTrace); - } - - - if (!*procname && Jim_Length(fileNameObj)) { - - int len = Jim_ListLength(interp, interp->stackTrace); - - if (len >= 3) { - Jim_Obj *objPtr = Jim_ListGetIndex(interp, interp->stackTrace, len - 3); - if (Jim_Length(objPtr)) { - - objPtr = Jim_ListGetIndex(interp, interp->stackTrace, len - 2); - if (Jim_Length(objPtr) == 0) { - - ListSetIndex(interp, interp->stackTrace, len - 2, fileNameObj, 0); - ListSetIndex(interp, interp->stackTrace, len - 1, Jim_NewIntObj(interp, linenr), 0); - return; - } - } - } - } - - Jim_ListAppendElement(interp, interp->stackTrace, Jim_NewStringObj(interp, procname, -1)); - Jim_ListAppendElement(interp, interp->stackTrace, fileNameObj); - Jim_ListAppendElement(interp, interp->stackTrace, Jim_NewIntObj(interp, linenr)); -} - -int Jim_SetAssocData(Jim_Interp *interp, const char *key, Jim_InterpDeleteProc * delProc, - void *data) -{ - AssocDataValue *assocEntryPtr = (AssocDataValue *) Jim_Alloc(sizeof(AssocDataValue)); - - assocEntryPtr->delProc = delProc; - assocEntryPtr->data = data; - return Jim_AddHashEntry(&interp->assocData, key, assocEntryPtr); -} - -void *Jim_GetAssocData(Jim_Interp *interp, const char *key) -{ - Jim_HashEntry *entryPtr = Jim_FindHashEntry(&interp->assocData, key); - - if (entryPtr != NULL) { - AssocDataValue *assocEntryPtr = Jim_GetHashEntryVal(entryPtr); - return assocEntryPtr->data; - } - return NULL; -} - -int Jim_DeleteAssocData(Jim_Interp *interp, const char *key) -{ - return Jim_DeleteHashEntry(&interp->assocData, key); -} - -int Jim_GetExitCode(Jim_Interp *interp) -{ - return interp->exitCode; -} - -static void UpdateStringOfInt(struct Jim_Obj *objPtr); -static int SetIntFromAny(Jim_Interp *interp, Jim_Obj *objPtr, int flags); - -static const Jim_ObjType intObjType = { - "int", - NULL, - NULL, - UpdateStringOfInt, - JIM_TYPE_NONE, -}; - -static const Jim_ObjType coercedDoubleObjType = { - "coerced-double", - NULL, - NULL, - UpdateStringOfInt, - JIM_TYPE_NONE, -}; - - -static void UpdateStringOfInt(struct Jim_Obj *objPtr) -{ - char buf[JIM_INTEGER_SPACE + 1]; - jim_wide wideValue = JimWideValue(objPtr); - int pos = 0; - - if (wideValue == 0) { - buf[pos++] = '0'; - } - else { - char tmp[JIM_INTEGER_SPACE]; - int num = 0; - int i; - - if (wideValue < 0) { - buf[pos++] = '-'; - i = wideValue % 10; - tmp[num++] = (i > 0) ? (10 - i) : -i; - wideValue /= -10; - } - - while (wideValue) { - tmp[num++] = wideValue % 10; - wideValue /= 10; - } - - for (i = 0; i < num; i++) { - buf[pos++] = '0' + tmp[num - i - 1]; - } - } - buf[pos] = 0; - - JimSetStringBytes(objPtr, buf); -} - -static int SetIntFromAny(Jim_Interp *interp, Jim_Obj *objPtr, int flags) -{ - jim_wide wideValue; - const char *str; - - if (objPtr->typePtr == &coercedDoubleObjType) { - - objPtr->typePtr = &intObjType; - return JIM_OK; - } - - - str = Jim_String(objPtr); - - if (Jim_StringToWide(str, &wideValue, 0) != JIM_OK) { - if (flags & JIM_ERRMSG) { - Jim_SetResultFormatted(interp, "expected integer but got \"%#s\"", objPtr); - } - return JIM_ERR; - } - if ((wideValue == JIM_WIDE_MIN || wideValue == JIM_WIDE_MAX) && errno == ERANGE) { - Jim_SetResultString(interp, "Integer value too big to be represented", -1); - return JIM_ERR; - } - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &intObjType; - objPtr->internalRep.wideValue = wideValue; - return JIM_OK; -} - -#ifdef JIM_OPTIMIZATION -static int JimIsWide(Jim_Obj *objPtr) -{ - return objPtr->typePtr == &intObjType; -} -#endif - -int Jim_GetWide(Jim_Interp *interp, Jim_Obj *objPtr, jim_wide * widePtr) -{ - if (objPtr->typePtr != &intObjType && SetIntFromAny(interp, objPtr, JIM_ERRMSG) == JIM_ERR) - return JIM_ERR; - *widePtr = JimWideValue(objPtr); - return JIM_OK; -} - - -static int JimGetWideNoErr(Jim_Interp *interp, Jim_Obj *objPtr, jim_wide * widePtr) -{ - if (objPtr->typePtr != &intObjType && SetIntFromAny(interp, objPtr, JIM_NONE) == JIM_ERR) - return JIM_ERR; - *widePtr = JimWideValue(objPtr); - return JIM_OK; -} - -int Jim_GetLong(Jim_Interp *interp, Jim_Obj *objPtr, long *longPtr) -{ - jim_wide wideValue; - int retval; - - retval = Jim_GetWide(interp, objPtr, &wideValue); - if (retval == JIM_OK) { - *longPtr = (long)wideValue; - return JIM_OK; - } - return JIM_ERR; -} - -Jim_Obj *Jim_NewIntObj(Jim_Interp *interp, jim_wide wideValue) -{ - Jim_Obj *objPtr; - - objPtr = Jim_NewObj(interp); - objPtr->typePtr = &intObjType; - objPtr->bytes = NULL; - objPtr->internalRep.wideValue = wideValue; - return objPtr; -} - -#define JIM_DOUBLE_SPACE 30 - -static void UpdateStringOfDouble(struct Jim_Obj *objPtr); -static int SetDoubleFromAny(Jim_Interp *interp, Jim_Obj *objPtr); - -static const Jim_ObjType doubleObjType = { - "double", - NULL, - NULL, - UpdateStringOfDouble, - JIM_TYPE_NONE, -}; - -#ifndef HAVE_ISNAN -#undef isnan -#define isnan(X) ((X) != (X)) -#endif -#ifndef HAVE_ISINF -#undef isinf -#define isinf(X) (1.0 / (X) == 0.0) -#endif - -static void UpdateStringOfDouble(struct Jim_Obj *objPtr) -{ - double value = objPtr->internalRep.doubleValue; - - if (isnan(value)) { - JimSetStringBytes(objPtr, "NaN"); - return; - } - if (isinf(value)) { - if (value < 0) { - JimSetStringBytes(objPtr, "-Inf"); - } - else { - JimSetStringBytes(objPtr, "Inf"); - } - return; - } - { - char buf[JIM_DOUBLE_SPACE + 1]; - int i; - int len = sprintf(buf, "%.12g", value); - - - for (i = 0; i < len; i++) { - if (buf[i] == '.' || buf[i] == 'e') { -#if defined(JIM_SPRINTF_DOUBLE_NEEDS_FIX) - char *e = strchr(buf, 'e'); - if (e && (e[1] == '-' || e[1] == '+') && e[2] == '0') { - - e += 2; - memmove(e, e + 1, len - (e - buf)); - } -#endif - break; - } - } - if (buf[i] == '\0') { - buf[i++] = '.'; - buf[i++] = '0'; - buf[i] = '\0'; - } - JimSetStringBytes(objPtr, buf); - } -} - -static int SetDoubleFromAny(Jim_Interp *interp, Jim_Obj *objPtr) -{ - double doubleValue; - jim_wide wideValue; - const char *str; - -#ifdef HAVE_LONG_LONG - -#define MIN_INT_IN_DOUBLE -(1LL << 53) -#define MAX_INT_IN_DOUBLE -(MIN_INT_IN_DOUBLE + 1) - - if (objPtr->typePtr == &intObjType - && JimWideValue(objPtr) >= MIN_INT_IN_DOUBLE - && JimWideValue(objPtr) <= MAX_INT_IN_DOUBLE) { - - - objPtr->typePtr = &coercedDoubleObjType; - return JIM_OK; - } -#endif - str = Jim_String(objPtr); - - if (Jim_StringToWide(str, &wideValue, 10) == JIM_OK) { - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &coercedDoubleObjType; - objPtr->internalRep.wideValue = wideValue; - return JIM_OK; - } - else { - - if (Jim_StringToDouble(str, &doubleValue) != JIM_OK) { - Jim_SetResultFormatted(interp, "expected floating-point number but got \"%#s\"", objPtr); - return JIM_ERR; - } - - Jim_FreeIntRep(interp, objPtr); - } - objPtr->typePtr = &doubleObjType; - objPtr->internalRep.doubleValue = doubleValue; - return JIM_OK; -} - -int Jim_GetDouble(Jim_Interp *interp, Jim_Obj *objPtr, double *doublePtr) -{ - if (objPtr->typePtr == &coercedDoubleObjType) { - *doublePtr = JimWideValue(objPtr); - return JIM_OK; - } - if (objPtr->typePtr != &doubleObjType && SetDoubleFromAny(interp, objPtr) == JIM_ERR) - return JIM_ERR; - - if (objPtr->typePtr == &coercedDoubleObjType) { - *doublePtr = JimWideValue(objPtr); - } - else { - *doublePtr = objPtr->internalRep.doubleValue; - } - return JIM_OK; -} - -Jim_Obj *Jim_NewDoubleObj(Jim_Interp *interp, double doubleValue) -{ - Jim_Obj *objPtr; - - objPtr = Jim_NewObj(interp); - objPtr->typePtr = &doubleObjType; - objPtr->bytes = NULL; - objPtr->internalRep.doubleValue = doubleValue; - return objPtr; -} - -static int SetBooleanFromAny(Jim_Interp *interp, Jim_Obj *objPtr, int flags); - -int Jim_GetBoolean(Jim_Interp *interp, Jim_Obj *objPtr, int * booleanPtr) -{ - if (objPtr->typePtr != &intObjType && SetBooleanFromAny(interp, objPtr, JIM_ERRMSG) == JIM_ERR) - return JIM_ERR; - *booleanPtr = (int) JimWideValue(objPtr); - return JIM_OK; -} - -static int SetBooleanFromAny(Jim_Interp *interp, Jim_Obj *objPtr, int flags) -{ - static const char * const falses[] = { - "0", "false", "no", "off", NULL - }; - static const char * const trues[] = { - "1", "true", "yes", "on", NULL - }; - - int boolean; - - int index; - if (Jim_GetEnum(interp, objPtr, falses, &index, NULL, 0) == JIM_OK) { - boolean = 0; - } else if (Jim_GetEnum(interp, objPtr, trues, &index, NULL, 0) == JIM_OK) { - boolean = 1; - } else { - if (flags & JIM_ERRMSG) { - Jim_SetResultFormatted(interp, "expected boolean but got \"%#s\"", objPtr); - } - return JIM_ERR; - } - - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &intObjType; - objPtr->internalRep.wideValue = boolean; - return JIM_OK; -} - -static void ListInsertElements(Jim_Obj *listPtr, int idx, int elemc, Jim_Obj *const *elemVec); -static void ListAppendElement(Jim_Obj *listPtr, Jim_Obj *objPtr); -static void FreeListInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); -static void DupListInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); -static void UpdateStringOfList(struct Jim_Obj *objPtr); -static int SetListFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); - -static const Jim_ObjType listObjType = { - "list", - FreeListInternalRep, - DupListInternalRep, - UpdateStringOfList, - JIM_TYPE_NONE, -}; - -void FreeListInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) -{ - int i; - - for (i = 0; i < objPtr->internalRep.listValue.len; i++) { - Jim_DecrRefCount(interp, objPtr->internalRep.listValue.ele[i]); - } - Jim_Free(objPtr->internalRep.listValue.ele); -} - -void DupListInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) -{ - int i; - - JIM_NOTUSED(interp); - - dupPtr->internalRep.listValue.len = srcPtr->internalRep.listValue.len; - dupPtr->internalRep.listValue.maxLen = srcPtr->internalRep.listValue.maxLen; - dupPtr->internalRep.listValue.ele = - Jim_Alloc(sizeof(Jim_Obj *) * srcPtr->internalRep.listValue.maxLen); - memcpy(dupPtr->internalRep.listValue.ele, srcPtr->internalRep.listValue.ele, - sizeof(Jim_Obj *) * srcPtr->internalRep.listValue.len); - for (i = 0; i < dupPtr->internalRep.listValue.len; i++) { - Jim_IncrRefCount(dupPtr->internalRep.listValue.ele[i]); - } - dupPtr->typePtr = &listObjType; -} - -#define JIM_ELESTR_SIMPLE 0 -#define JIM_ELESTR_BRACE 1 -#define JIM_ELESTR_QUOTE 2 -static unsigned char ListElementQuotingType(const char *s, int len) -{ - int i, level, blevel, trySimple = 1; - - - if (len == 0) - return JIM_ELESTR_BRACE; - if (s[0] == '"' || s[0] == '{') { - trySimple = 0; - goto testbrace; - } - for (i = 0; i < len; i++) { - switch (s[i]) { - case ' ': - case '$': - case '"': - case '[': - case ']': - case ';': - case '\\': - case '\r': - case '\n': - case '\t': - case '\f': - case '\v': - trySimple = 0; - - case '{': - case '}': - goto testbrace; - } - } - return JIM_ELESTR_SIMPLE; - - testbrace: - - if (s[len - 1] == '\\') - return JIM_ELESTR_QUOTE; - level = 0; - blevel = 0; - for (i = 0; i < len; i++) { - switch (s[i]) { - case '{': - level++; - break; - case '}': - level--; - if (level < 0) - return JIM_ELESTR_QUOTE; - break; - case '[': - blevel++; - break; - case ']': - blevel--; - break; - case '\\': - if (s[i + 1] == '\n') - return JIM_ELESTR_QUOTE; - else if (s[i + 1] != '\0') - i++; - break; - } - } - if (blevel < 0) { - return JIM_ELESTR_QUOTE; - } - - if (level == 0) { - if (!trySimple) - return JIM_ELESTR_BRACE; - for (i = 0; i < len; i++) { - switch (s[i]) { - case ' ': - case '$': - case '"': - case '[': - case ']': - case ';': - case '\\': - case '\r': - case '\n': - case '\t': - case '\f': - case '\v': - return JIM_ELESTR_BRACE; - break; - } - } - return JIM_ELESTR_SIMPLE; - } - return JIM_ELESTR_QUOTE; -} - -static int BackslashQuoteString(const char *s, int len, char *q) -{ - char *p = q; - - while (len--) { - switch (*s) { - case ' ': - case '$': - case '"': - case '[': - case ']': - case '{': - case '}': - case ';': - case '\\': - *p++ = '\\'; - *p++ = *s++; - break; - case '\n': - *p++ = '\\'; - *p++ = 'n'; - s++; - break; - case '\r': - *p++ = '\\'; - *p++ = 'r'; - s++; - break; - case '\t': - *p++ = '\\'; - *p++ = 't'; - s++; - break; - case '\f': - *p++ = '\\'; - *p++ = 'f'; - s++; - break; - case '\v': - *p++ = '\\'; - *p++ = 'v'; - s++; - break; - default: - *p++ = *s++; - break; - } - } - *p = '\0'; - - return p - q; -} - -static void JimMakeListStringRep(Jim_Obj *objPtr, Jim_Obj **objv, int objc) -{ - #define STATIC_QUOTING_LEN 32 - int i, bufLen, realLength; - const char *strRep; - char *p; - unsigned char *quotingType, staticQuoting[STATIC_QUOTING_LEN]; - - - if (objc > STATIC_QUOTING_LEN) { - quotingType = Jim_Alloc(objc); - } - else { - quotingType = staticQuoting; - } - bufLen = 0; - for (i = 0; i < objc; i++) { - int len; - - strRep = Jim_GetString(objv[i], &len); - quotingType[i] = ListElementQuotingType(strRep, len); - switch (quotingType[i]) { - case JIM_ELESTR_SIMPLE: - if (i != 0 || strRep[0] != '#') { - bufLen += len; - break; - } - - quotingType[i] = JIM_ELESTR_BRACE; - - case JIM_ELESTR_BRACE: - bufLen += len + 2; - break; - case JIM_ELESTR_QUOTE: - bufLen += len * 2; - break; - } - bufLen++; - } - bufLen++; - - - p = objPtr->bytes = Jim_Alloc(bufLen + 1); - realLength = 0; - for (i = 0; i < objc; i++) { - int len, qlen; - - strRep = Jim_GetString(objv[i], &len); - - switch (quotingType[i]) { - case JIM_ELESTR_SIMPLE: - memcpy(p, strRep, len); - p += len; - realLength += len; - break; - case JIM_ELESTR_BRACE: - *p++ = '{'; - memcpy(p, strRep, len); - p += len; - *p++ = '}'; - realLength += len + 2; - break; - case JIM_ELESTR_QUOTE: - if (i == 0 && strRep[0] == '#') { - *p++ = '\\'; - realLength++; - } - qlen = BackslashQuoteString(strRep, len, p); - p += qlen; - realLength += qlen; - break; - } - - if (i + 1 != objc) { - *p++ = ' '; - realLength++; - } - } - *p = '\0'; - objPtr->length = realLength; - - if (quotingType != staticQuoting) { - Jim_Free(quotingType); - } -} - -static void UpdateStringOfList(struct Jim_Obj *objPtr) -{ - JimMakeListStringRep(objPtr, objPtr->internalRep.listValue.ele, objPtr->internalRep.listValue.len); -} - -static int SetListFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr) -{ - struct JimParserCtx parser; - const char *str; - int strLen; - Jim_Obj *fileNameObj; - int linenr; - - if (objPtr->typePtr == &listObjType) { - return JIM_OK; - } - - if (Jim_IsDict(objPtr) && objPtr->bytes == NULL) { - Jim_Obj **listObjPtrPtr; - int len; - int i; - - listObjPtrPtr = JimDictPairs(objPtr, &len); - for (i = 0; i < len; i++) { - Jim_IncrRefCount(listObjPtrPtr[i]); - } - - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &listObjType; - objPtr->internalRep.listValue.len = len; - objPtr->internalRep.listValue.maxLen = len; - objPtr->internalRep.listValue.ele = listObjPtrPtr; - - return JIM_OK; - } - - - if (objPtr->typePtr == &sourceObjType) { - fileNameObj = objPtr->internalRep.sourceValue.fileNameObj; - linenr = objPtr->internalRep.sourceValue.lineNumber; - } - else { - fileNameObj = interp->emptyObj; - linenr = 1; - } - Jim_IncrRefCount(fileNameObj); - - - str = Jim_GetString(objPtr, &strLen); - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &listObjType; - objPtr->internalRep.listValue.len = 0; - objPtr->internalRep.listValue.maxLen = 0; - objPtr->internalRep.listValue.ele = NULL; - - - if (strLen) { - JimParserInit(&parser, str, strLen, linenr); - while (!parser.eof) { - Jim_Obj *elementPtr; - - JimParseList(&parser); - if (parser.tt != JIM_TT_STR && parser.tt != JIM_TT_ESC) - continue; - elementPtr = JimParserGetTokenObj(interp, &parser); - JimSetSourceInfo(interp, elementPtr, fileNameObj, parser.tline); - ListAppendElement(objPtr, elementPtr); - } - } - Jim_DecrRefCount(interp, fileNameObj); - return JIM_OK; -} - -Jim_Obj *Jim_NewListObj(Jim_Interp *interp, Jim_Obj *const *elements, int len) -{ - Jim_Obj *objPtr; - - objPtr = Jim_NewObj(interp); - objPtr->typePtr = &listObjType; - objPtr->bytes = NULL; - objPtr->internalRep.listValue.ele = NULL; - objPtr->internalRep.listValue.len = 0; - objPtr->internalRep.listValue.maxLen = 0; - - if (len) { - ListInsertElements(objPtr, 0, len, elements); - } - - return objPtr; -} - -static void JimListGetElements(Jim_Interp *interp, Jim_Obj *listObj, int *listLen, - Jim_Obj ***listVec) -{ - *listLen = Jim_ListLength(interp, listObj); - *listVec = listObj->internalRep.listValue.ele; -} - - -static int JimSign(jim_wide w) -{ - if (w == 0) { - return 0; - } - else if (w < 0) { - return -1; - } - return 1; -} - - -struct lsort_info { - jmp_buf jmpbuf; - Jim_Obj *command; - Jim_Interp *interp; - enum { - JIM_LSORT_ASCII, - JIM_LSORT_NOCASE, - JIM_LSORT_INTEGER, - JIM_LSORT_REAL, - JIM_LSORT_COMMAND - } type; - int order; - int index; - int indexed; - int unique; - int (*subfn)(Jim_Obj **, Jim_Obj **); -}; - -static struct lsort_info *sort_info; - -static int ListSortIndexHelper(Jim_Obj **lhsObj, Jim_Obj **rhsObj) -{ - Jim_Obj *lObj, *rObj; - - if (Jim_ListIndex(sort_info->interp, *lhsObj, sort_info->index, &lObj, JIM_ERRMSG) != JIM_OK || - Jim_ListIndex(sort_info->interp, *rhsObj, sort_info->index, &rObj, JIM_ERRMSG) != JIM_OK) { - longjmp(sort_info->jmpbuf, JIM_ERR); - } - return sort_info->subfn(&lObj, &rObj); -} - - -static int ListSortString(Jim_Obj **lhsObj, Jim_Obj **rhsObj) -{ - return Jim_StringCompareObj(sort_info->interp, *lhsObj, *rhsObj, 0) * sort_info->order; -} - -static int ListSortStringNoCase(Jim_Obj **lhsObj, Jim_Obj **rhsObj) -{ - return Jim_StringCompareObj(sort_info->interp, *lhsObj, *rhsObj, 1) * sort_info->order; -} - -static int ListSortInteger(Jim_Obj **lhsObj, Jim_Obj **rhsObj) -{ - jim_wide lhs = 0, rhs = 0; - - if (Jim_GetWide(sort_info->interp, *lhsObj, &lhs) != JIM_OK || - Jim_GetWide(sort_info->interp, *rhsObj, &rhs) != JIM_OK) { - longjmp(sort_info->jmpbuf, JIM_ERR); - } - - return JimSign(lhs - rhs) * sort_info->order; -} - -static int ListSortReal(Jim_Obj **lhsObj, Jim_Obj **rhsObj) -{ - double lhs = 0, rhs = 0; - - if (Jim_GetDouble(sort_info->interp, *lhsObj, &lhs) != JIM_OK || - Jim_GetDouble(sort_info->interp, *rhsObj, &rhs) != JIM_OK) { - longjmp(sort_info->jmpbuf, JIM_ERR); - } - if (lhs == rhs) { - return 0; - } - if (lhs > rhs) { - return sort_info->order; - } - return -sort_info->order; -} - -static int ListSortCommand(Jim_Obj **lhsObj, Jim_Obj **rhsObj) -{ - Jim_Obj *compare_script; - int rc; - - jim_wide ret = 0; - - - compare_script = Jim_DuplicateObj(sort_info->interp, sort_info->command); - Jim_ListAppendElement(sort_info->interp, compare_script, *lhsObj); - Jim_ListAppendElement(sort_info->interp, compare_script, *rhsObj); - - rc = Jim_EvalObj(sort_info->interp, compare_script); - - if (rc != JIM_OK || Jim_GetWide(sort_info->interp, Jim_GetResult(sort_info->interp), &ret) != JIM_OK) { - longjmp(sort_info->jmpbuf, rc); - } - - return JimSign(ret) * sort_info->order; -} - -static void ListRemoveDuplicates(Jim_Obj *listObjPtr, int (*comp)(Jim_Obj **lhs, Jim_Obj **rhs)) -{ - int src; - int dst = 0; - Jim_Obj **ele = listObjPtr->internalRep.listValue.ele; - - for (src = 1; src < listObjPtr->internalRep.listValue.len; src++) { - if (comp(&ele[dst], &ele[src]) == 0) { - - Jim_DecrRefCount(sort_info->interp, ele[dst]); - } - else { - - dst++; - } - ele[dst] = ele[src]; - } - - - dst++; - if (dst < listObjPtr->internalRep.listValue.len) { - ele[dst] = ele[src]; - } - - - listObjPtr->internalRep.listValue.len = dst; -} - - -static int ListSortElements(Jim_Interp *interp, Jim_Obj *listObjPtr, struct lsort_info *info) -{ - struct lsort_info *prev_info; - - typedef int (qsort_comparator) (const void *, const void *); - int (*fn) (Jim_Obj **, Jim_Obj **); - Jim_Obj **vector; - int len; - int rc; - - JimPanic((Jim_IsShared(listObjPtr), "ListSortElements called with shared object")); - SetListFromAny(interp, listObjPtr); - - - prev_info = sort_info; - sort_info = info; - - vector = listObjPtr->internalRep.listValue.ele; - len = listObjPtr->internalRep.listValue.len; - switch (info->type) { - case JIM_LSORT_ASCII: - fn = ListSortString; - break; - case JIM_LSORT_NOCASE: - fn = ListSortStringNoCase; - break; - case JIM_LSORT_INTEGER: - fn = ListSortInteger; - break; - case JIM_LSORT_REAL: - fn = ListSortReal; - break; - case JIM_LSORT_COMMAND: - fn = ListSortCommand; - break; - default: - fn = NULL; - JimPanic((1, "ListSort called with invalid sort type")); - return -1; - } - - if (info->indexed) { - - info->subfn = fn; - fn = ListSortIndexHelper; - } - - if ((rc = setjmp(info->jmpbuf)) == 0) { - qsort(vector, len, sizeof(Jim_Obj *), (qsort_comparator *) fn); - - if (info->unique && len > 1) { - ListRemoveDuplicates(listObjPtr, fn); - } - - Jim_InvalidateStringRep(listObjPtr); - } - sort_info = prev_info; - - return rc; -} - -static void ListInsertElements(Jim_Obj *listPtr, int idx, int elemc, Jim_Obj *const *elemVec) -{ - int currentLen = listPtr->internalRep.listValue.len; - int requiredLen = currentLen + elemc; - int i; - Jim_Obj **point; - - if (requiredLen > listPtr->internalRep.listValue.maxLen) { - if (requiredLen < 2) { - - requiredLen = 4; - } - else { - requiredLen *= 2; - } - - listPtr->internalRep.listValue.ele = Jim_Realloc(listPtr->internalRep.listValue.ele, - sizeof(Jim_Obj *) * requiredLen); - - listPtr->internalRep.listValue.maxLen = requiredLen; - } - if (idx < 0) { - idx = currentLen; - } - point = listPtr->internalRep.listValue.ele + idx; - memmove(point + elemc, point, (currentLen - idx) * sizeof(Jim_Obj *)); - for (i = 0; i < elemc; ++i) { - point[i] = elemVec[i]; - Jim_IncrRefCount(point[i]); - } - listPtr->internalRep.listValue.len += elemc; -} - -static void ListAppendElement(Jim_Obj *listPtr, Jim_Obj *objPtr) -{ - ListInsertElements(listPtr, -1, 1, &objPtr); -} - -static void ListAppendList(Jim_Obj *listPtr, Jim_Obj *appendListPtr) -{ - ListInsertElements(listPtr, -1, - appendListPtr->internalRep.listValue.len, appendListPtr->internalRep.listValue.ele); -} - -void Jim_ListAppendElement(Jim_Interp *interp, Jim_Obj *listPtr, Jim_Obj *objPtr) -{ - JimPanic((Jim_IsShared(listPtr), "Jim_ListAppendElement called with shared object")); - SetListFromAny(interp, listPtr); - Jim_InvalidateStringRep(listPtr); - ListAppendElement(listPtr, objPtr); -} - -void Jim_ListAppendList(Jim_Interp *interp, Jim_Obj *listPtr, Jim_Obj *appendListPtr) -{ - JimPanic((Jim_IsShared(listPtr), "Jim_ListAppendList called with shared object")); - SetListFromAny(interp, listPtr); - SetListFromAny(interp, appendListPtr); - Jim_InvalidateStringRep(listPtr); - ListAppendList(listPtr, appendListPtr); -} - -int Jim_ListLength(Jim_Interp *interp, Jim_Obj *objPtr) -{ - SetListFromAny(interp, objPtr); - return objPtr->internalRep.listValue.len; -} - -void Jim_ListInsertElements(Jim_Interp *interp, Jim_Obj *listPtr, int idx, - int objc, Jim_Obj *const *objVec) -{ - JimPanic((Jim_IsShared(listPtr), "Jim_ListInsertElement called with shared object")); - SetListFromAny(interp, listPtr); - if (idx >= 0 && idx > listPtr->internalRep.listValue.len) - idx = listPtr->internalRep.listValue.len; - else if (idx < 0) - idx = 0; - Jim_InvalidateStringRep(listPtr); - ListInsertElements(listPtr, idx, objc, objVec); -} - -Jim_Obj *Jim_ListGetIndex(Jim_Interp *interp, Jim_Obj *listPtr, int idx) -{ - SetListFromAny(interp, listPtr); - if ((idx >= 0 && idx >= listPtr->internalRep.listValue.len) || - (idx < 0 && (-idx - 1) >= listPtr->internalRep.listValue.len)) { - return NULL; - } - if (idx < 0) - idx = listPtr->internalRep.listValue.len + idx; - return listPtr->internalRep.listValue.ele[idx]; -} - -int Jim_ListIndex(Jim_Interp *interp, Jim_Obj *listPtr, int idx, Jim_Obj **objPtrPtr, int flags) -{ - *objPtrPtr = Jim_ListGetIndex(interp, listPtr, idx); - if (*objPtrPtr == NULL) { - if (flags & JIM_ERRMSG) { - Jim_SetResultString(interp, "list index out of range", -1); - } - return JIM_ERR; - } - return JIM_OK; -} - -static int ListSetIndex(Jim_Interp *interp, Jim_Obj *listPtr, int idx, - Jim_Obj *newObjPtr, int flags) -{ - SetListFromAny(interp, listPtr); - if ((idx >= 0 && idx >= listPtr->internalRep.listValue.len) || - (idx < 0 && (-idx - 1) >= listPtr->internalRep.listValue.len)) { - if (flags & JIM_ERRMSG) { - Jim_SetResultString(interp, "list index out of range", -1); - } - return JIM_ERR; - } - if (idx < 0) - idx = listPtr->internalRep.listValue.len + idx; - Jim_DecrRefCount(interp, listPtr->internalRep.listValue.ele[idx]); - listPtr->internalRep.listValue.ele[idx] = newObjPtr; - Jim_IncrRefCount(newObjPtr); - return JIM_OK; -} - -int Jim_ListSetIndex(Jim_Interp *interp, Jim_Obj *varNamePtr, - Jim_Obj *const *indexv, int indexc, Jim_Obj *newObjPtr) -{ - Jim_Obj *varObjPtr, *objPtr, *listObjPtr; - int shared, i, idx; - - varObjPtr = objPtr = Jim_GetVariable(interp, varNamePtr, JIM_ERRMSG | JIM_UNSHARED); - if (objPtr == NULL) - return JIM_ERR; - if ((shared = Jim_IsShared(objPtr))) - varObjPtr = objPtr = Jim_DuplicateObj(interp, objPtr); - for (i = 0; i < indexc - 1; i++) { - listObjPtr = objPtr; - if (Jim_GetIndex(interp, indexv[i], &idx) != JIM_OK) - goto err; - if (Jim_ListIndex(interp, listObjPtr, idx, &objPtr, JIM_ERRMSG) != JIM_OK) { - goto err; - } - if (Jim_IsShared(objPtr)) { - objPtr = Jim_DuplicateObj(interp, objPtr); - ListSetIndex(interp, listObjPtr, idx, objPtr, JIM_NONE); - } - Jim_InvalidateStringRep(listObjPtr); - } - if (Jim_GetIndex(interp, indexv[indexc - 1], &idx) != JIM_OK) - goto err; - if (ListSetIndex(interp, objPtr, idx, newObjPtr, JIM_ERRMSG) == JIM_ERR) - goto err; - Jim_InvalidateStringRep(objPtr); - Jim_InvalidateStringRep(varObjPtr); - if (Jim_SetVariable(interp, varNamePtr, varObjPtr) != JIM_OK) - goto err; - Jim_SetResult(interp, varObjPtr); - return JIM_OK; - err: - if (shared) { - Jim_FreeNewObj(interp, varObjPtr); - } - return JIM_ERR; -} - -Jim_Obj *Jim_ListJoin(Jim_Interp *interp, Jim_Obj *listObjPtr, const char *joinStr, int joinStrLen) -{ - int i; - int listLen = Jim_ListLength(interp, listObjPtr); - Jim_Obj *resObjPtr = Jim_NewEmptyStringObj(interp); - - for (i = 0; i < listLen; ) { - Jim_AppendObj(interp, resObjPtr, Jim_ListGetIndex(interp, listObjPtr, i)); - if (++i != listLen) { - Jim_AppendString(interp, resObjPtr, joinStr, joinStrLen); - } - } - return resObjPtr; -} - -Jim_Obj *Jim_ConcatObj(Jim_Interp *interp, int objc, Jim_Obj *const *objv) -{ - int i; - - for (i = 0; i < objc; i++) { - if (!Jim_IsList(objv[i])) - break; - } - if (i == objc) { - Jim_Obj *objPtr = Jim_NewListObj(interp, NULL, 0); - - for (i = 0; i < objc; i++) - ListAppendList(objPtr, objv[i]); - return objPtr; - } - else { - - int len = 0, objLen; - char *bytes, *p; - - - for (i = 0; i < objc; i++) { - len += Jim_Length(objv[i]); - } - if (objc) - len += objc - 1; - - p = bytes = Jim_Alloc(len + 1); - for (i = 0; i < objc; i++) { - const char *s = Jim_GetString(objv[i], &objLen); - - - while (objLen && isspace(UCHAR(*s))) { - s++; - objLen--; - len--; - } - - while (objLen && isspace(UCHAR(s[objLen - 1]))) { - - if (objLen > 1 && s[objLen - 2] == '\\') { - break; - } - objLen--; - len--; - } - memcpy(p, s, objLen); - p += objLen; - if (i + 1 != objc) { - if (objLen) - *p++ = ' '; - else { - len--; - } - } - } - *p = '\0'; - return Jim_NewStringObjNoAlloc(interp, bytes, len); - } -} - -Jim_Obj *Jim_ListRange(Jim_Interp *interp, Jim_Obj *listObjPtr, Jim_Obj *firstObjPtr, - Jim_Obj *lastObjPtr) -{ - int first, last; - int len, rangeLen; - - if (Jim_GetIndex(interp, firstObjPtr, &first) != JIM_OK || - Jim_GetIndex(interp, lastObjPtr, &last) != JIM_OK) - return NULL; - len = Jim_ListLength(interp, listObjPtr); - first = JimRelToAbsIndex(len, first); - last = JimRelToAbsIndex(len, last); - JimRelToAbsRange(len, &first, &last, &rangeLen); - if (first == 0 && last == len) { - return listObjPtr; - } - return Jim_NewListObj(interp, listObjPtr->internalRep.listValue.ele + first, rangeLen); -} - -static void FreeDictInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); -static void DupDictInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); -static void UpdateStringOfDict(struct Jim_Obj *objPtr); -static int SetDictFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); - - -static unsigned int JimObjectHTHashFunction(const void *key) -{ - int len; - const char *str = Jim_GetString((Jim_Obj *)key, &len); - return Jim_GenHashFunction((const unsigned char *)str, len); -} - -static int JimObjectHTKeyCompare(void *privdata, const void *key1, const void *key2) -{ - return Jim_StringEqObj((Jim_Obj *)key1, (Jim_Obj *)key2); -} - -static void *JimObjectHTKeyValDup(void *privdata, const void *val) -{ - Jim_IncrRefCount((Jim_Obj *)val); - return (void *)val; -} - -static void JimObjectHTKeyValDestructor(void *interp, void *val) -{ - Jim_DecrRefCount(interp, (Jim_Obj *)val); -} - -static const Jim_HashTableType JimDictHashTableType = { - JimObjectHTHashFunction, - JimObjectHTKeyValDup, - JimObjectHTKeyValDup, - JimObjectHTKeyCompare, - JimObjectHTKeyValDestructor, - JimObjectHTKeyValDestructor -}; - -static const Jim_ObjType dictObjType = { - "dict", - FreeDictInternalRep, - DupDictInternalRep, - UpdateStringOfDict, - JIM_TYPE_NONE, -}; - -void FreeDictInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) -{ - JIM_NOTUSED(interp); - - Jim_FreeHashTable(objPtr->internalRep.ptr); - Jim_Free(objPtr->internalRep.ptr); -} - -void DupDictInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) -{ - Jim_HashTable *ht, *dupHt; - Jim_HashTableIterator htiter; - Jim_HashEntry *he; - - - ht = srcPtr->internalRep.ptr; - dupHt = Jim_Alloc(sizeof(*dupHt)); - Jim_InitHashTable(dupHt, &JimDictHashTableType, interp); - if (ht->size != 0) - Jim_ExpandHashTable(dupHt, ht->size); - - JimInitHashTableIterator(ht, &htiter); - while ((he = Jim_NextHashEntry(&htiter)) != NULL) { - Jim_AddHashEntry(dupHt, he->key, he->u.val); - } - - dupPtr->internalRep.ptr = dupHt; - dupPtr->typePtr = &dictObjType; -} - -static Jim_Obj **JimDictPairs(Jim_Obj *dictPtr, int *len) -{ - Jim_HashTable *ht; - Jim_HashTableIterator htiter; - Jim_HashEntry *he; - Jim_Obj **objv; - int i; - - ht = dictPtr->internalRep.ptr; - - - objv = Jim_Alloc((ht->used * 2) * sizeof(Jim_Obj *)); - JimInitHashTableIterator(ht, &htiter); - i = 0; - while ((he = Jim_NextHashEntry(&htiter)) != NULL) { - objv[i++] = Jim_GetHashEntryKey(he); - objv[i++] = Jim_GetHashEntryVal(he); - } - *len = i; - return objv; -} - -static void UpdateStringOfDict(struct Jim_Obj *objPtr) -{ - - int len; - Jim_Obj **objv = JimDictPairs(objPtr, &len); - - - JimMakeListStringRep(objPtr, objv, len); - - Jim_Free(objv); -} - -static int SetDictFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr) -{ - int listlen; - - if (objPtr->typePtr == &dictObjType) { - return JIM_OK; - } - - if (Jim_IsList(objPtr) && Jim_IsShared(objPtr)) { - Jim_String(objPtr); - } - - - listlen = Jim_ListLength(interp, objPtr); - if (listlen % 2) { - Jim_SetResultString(interp, "missing value to go with key", -1); - return JIM_ERR; - } - else { - - Jim_HashTable *ht; - int i; - - ht = Jim_Alloc(sizeof(*ht)); - Jim_InitHashTable(ht, &JimDictHashTableType, interp); - - for (i = 0; i < listlen; i += 2) { - Jim_Obj *keyObjPtr = Jim_ListGetIndex(interp, objPtr, i); - Jim_Obj *valObjPtr = Jim_ListGetIndex(interp, objPtr, i + 1); - - Jim_ReplaceHashEntry(ht, keyObjPtr, valObjPtr); - } - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &dictObjType; - objPtr->internalRep.ptr = ht; - - return JIM_OK; - } -} - - - -static int DictAddElement(Jim_Interp *interp, Jim_Obj *objPtr, - Jim_Obj *keyObjPtr, Jim_Obj *valueObjPtr) -{ - Jim_HashTable *ht = objPtr->internalRep.ptr; - - if (valueObjPtr == NULL) { - return Jim_DeleteHashEntry(ht, keyObjPtr); - } - Jim_ReplaceHashEntry(ht, keyObjPtr, valueObjPtr); - return JIM_OK; -} - -int Jim_DictAddElement(Jim_Interp *interp, Jim_Obj *objPtr, - Jim_Obj *keyObjPtr, Jim_Obj *valueObjPtr) -{ - JimPanic((Jim_IsShared(objPtr), "Jim_DictAddElement called with shared object")); - if (SetDictFromAny(interp, objPtr) != JIM_OK) { - return JIM_ERR; - } - Jim_InvalidateStringRep(objPtr); - return DictAddElement(interp, objPtr, keyObjPtr, valueObjPtr); -} - -Jim_Obj *Jim_NewDictObj(Jim_Interp *interp, Jim_Obj *const *elements, int len) -{ - Jim_Obj *objPtr; - int i; - - JimPanic((len % 2, "Jim_NewDictObj() 'len' argument must be even")); - - objPtr = Jim_NewObj(interp); - objPtr->typePtr = &dictObjType; - objPtr->bytes = NULL; - objPtr->internalRep.ptr = Jim_Alloc(sizeof(Jim_HashTable)); - Jim_InitHashTable(objPtr->internalRep.ptr, &JimDictHashTableType, interp); - for (i = 0; i < len; i += 2) - DictAddElement(interp, objPtr, elements[i], elements[i + 1]); - return objPtr; -} - -int Jim_DictKey(Jim_Interp *interp, Jim_Obj *dictPtr, Jim_Obj *keyPtr, - Jim_Obj **objPtrPtr, int flags) -{ - Jim_HashEntry *he; - Jim_HashTable *ht; - - if (SetDictFromAny(interp, dictPtr) != JIM_OK) { - return -1; - } - ht = dictPtr->internalRep.ptr; - if ((he = Jim_FindHashEntry(ht, keyPtr)) == NULL) { - if (flags & JIM_ERRMSG) { - Jim_SetResultFormatted(interp, "key \"%#s\" not known in dictionary", keyPtr); - } - return JIM_ERR; - } - else { - *objPtrPtr = Jim_GetHashEntryVal(he); - return JIM_OK; - } -} - - -int Jim_DictPairs(Jim_Interp *interp, Jim_Obj *dictPtr, Jim_Obj ***objPtrPtr, int *len) -{ - if (SetDictFromAny(interp, dictPtr) != JIM_OK) { - return JIM_ERR; - } - *objPtrPtr = JimDictPairs(dictPtr, len); - - return JIM_OK; -} - - - -int Jim_DictKeysVector(Jim_Interp *interp, Jim_Obj *dictPtr, - Jim_Obj *const *keyv, int keyc, Jim_Obj **objPtrPtr, int flags) -{ - int i; - - if (keyc == 0) { - *objPtrPtr = dictPtr; - return JIM_OK; - } - - for (i = 0; i < keyc; i++) { - Jim_Obj *objPtr; - - int rc = Jim_DictKey(interp, dictPtr, keyv[i], &objPtr, flags); - if (rc != JIM_OK) { - return rc; - } - dictPtr = objPtr; - } - *objPtrPtr = dictPtr; - return JIM_OK; -} - -int Jim_SetDictKeysVector(Jim_Interp *interp, Jim_Obj *varNamePtr, - Jim_Obj *const *keyv, int keyc, Jim_Obj *newObjPtr, int flags) -{ - Jim_Obj *varObjPtr, *objPtr, *dictObjPtr; - int shared, i; - - varObjPtr = objPtr = Jim_GetVariable(interp, varNamePtr, flags); - if (objPtr == NULL) { - if (newObjPtr == NULL && (flags & JIM_MUSTEXIST)) { - - return JIM_ERR; - } - varObjPtr = objPtr = Jim_NewDictObj(interp, NULL, 0); - if (Jim_SetVariable(interp, varNamePtr, objPtr) != JIM_OK) { - Jim_FreeNewObj(interp, varObjPtr); - return JIM_ERR; - } - } - if ((shared = Jim_IsShared(objPtr))) - varObjPtr = objPtr = Jim_DuplicateObj(interp, objPtr); - for (i = 0; i < keyc; i++) { - dictObjPtr = objPtr; - - - if (SetDictFromAny(interp, dictObjPtr) != JIM_OK) { - goto err; - } - - if (i == keyc - 1) { - - if (Jim_DictAddElement(interp, objPtr, keyv[keyc - 1], newObjPtr) != JIM_OK) { - if (newObjPtr || (flags & JIM_MUSTEXIST)) { - goto err; - } - } - break; - } - - - Jim_InvalidateStringRep(dictObjPtr); - if (Jim_DictKey(interp, dictObjPtr, keyv[i], &objPtr, - newObjPtr ? JIM_NONE : JIM_ERRMSG) == JIM_OK) { - if (Jim_IsShared(objPtr)) { - objPtr = Jim_DuplicateObj(interp, objPtr); - DictAddElement(interp, dictObjPtr, keyv[i], objPtr); - } - } - else { - if (newObjPtr == NULL) { - goto err; - } - objPtr = Jim_NewDictObj(interp, NULL, 0); - DictAddElement(interp, dictObjPtr, keyv[i], objPtr); - } - } - - Jim_InvalidateStringRep(objPtr); - Jim_InvalidateStringRep(varObjPtr); - if (Jim_SetVariable(interp, varNamePtr, varObjPtr) != JIM_OK) { - goto err; - } - Jim_SetResult(interp, varObjPtr); - return JIM_OK; - err: - if (shared) { - Jim_FreeNewObj(interp, varObjPtr); - } - return JIM_ERR; -} - -static void UpdateStringOfIndex(struct Jim_Obj *objPtr); -static int SetIndexFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); - -static const Jim_ObjType indexObjType = { - "index", - NULL, - NULL, - UpdateStringOfIndex, - JIM_TYPE_NONE, -}; - -static void UpdateStringOfIndex(struct Jim_Obj *objPtr) -{ - if (objPtr->internalRep.intValue == -1) { - JimSetStringBytes(objPtr, "end"); - } - else { - char buf[JIM_INTEGER_SPACE + 1]; - if (objPtr->internalRep.intValue >= 0) { - sprintf(buf, "%d", objPtr->internalRep.intValue); - } - else { - - sprintf(buf, "end%d", objPtr->internalRep.intValue + 1); - } - JimSetStringBytes(objPtr, buf); - } -} - -static int SetIndexFromAny(Jim_Interp *interp, Jim_Obj *objPtr) -{ - int idx, end = 0; - const char *str; - char *endptr; - - - str = Jim_String(objPtr); - - - if (strncmp(str, "end", 3) == 0) { - end = 1; - str += 3; - idx = 0; - } - else { - idx = jim_strtol(str, &endptr); - - if (endptr == str) { - goto badindex; - } - str = endptr; - } - - - if (*str == '+' || *str == '-') { - int sign = (*str == '+' ? 1 : -1); - - idx += sign * jim_strtol(++str, &endptr); - if (str == endptr || *endptr) { - goto badindex; - } - str = endptr; - } - - while (isspace(UCHAR(*str))) { - str++; - } - if (*str) { - goto badindex; - } - if (end) { - if (idx > 0) { - idx = INT_MAX; - } - else { - - idx--; - } - } - else if (idx < 0) { - idx = -INT_MAX; - } - - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &indexObjType; - objPtr->internalRep.intValue = idx; - return JIM_OK; - - badindex: - Jim_SetResultFormatted(interp, - "bad index \"%#s\": must be integer?[+-]integer? or end?[+-]integer?", objPtr); - return JIM_ERR; -} - -int Jim_GetIndex(Jim_Interp *interp, Jim_Obj *objPtr, int *indexPtr) -{ - - if (objPtr->typePtr == &intObjType) { - jim_wide val = JimWideValue(objPtr); - - if (val < 0) - *indexPtr = -INT_MAX; - else if (val > INT_MAX) - *indexPtr = INT_MAX; - else - *indexPtr = (int)val; - return JIM_OK; - } - if (objPtr->typePtr != &indexObjType && SetIndexFromAny(interp, objPtr) == JIM_ERR) - return JIM_ERR; - *indexPtr = objPtr->internalRep.intValue; - return JIM_OK; -} - - - -static const char * const jimReturnCodes[] = { - "ok", - "error", - "return", - "break", - "continue", - "signal", - "exit", - "eval", - NULL -}; - -#define jimReturnCodesSize (sizeof(jimReturnCodes)/sizeof(*jimReturnCodes) - 1) - -static const Jim_ObjType returnCodeObjType = { - "return-code", - NULL, - NULL, - NULL, - JIM_TYPE_NONE, -}; - -const char *Jim_ReturnCode(int code) -{ - if (code < 0 || code >= (int)jimReturnCodesSize) { - return "?"; - } - else { - return jimReturnCodes[code]; - } -} - -static int SetReturnCodeFromAny(Jim_Interp *interp, Jim_Obj *objPtr) -{ - int returnCode; - jim_wide wideValue; - - - if (JimGetWideNoErr(interp, objPtr, &wideValue) != JIM_ERR) - returnCode = (int)wideValue; - else if (Jim_GetEnum(interp, objPtr, jimReturnCodes, &returnCode, NULL, JIM_NONE) != JIM_OK) { - Jim_SetResultFormatted(interp, "expected return code but got \"%#s\"", objPtr); - return JIM_ERR; - } - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &returnCodeObjType; - objPtr->internalRep.intValue = returnCode; - return JIM_OK; -} - -int Jim_GetReturnCode(Jim_Interp *interp, Jim_Obj *objPtr, int *intPtr) -{ - if (objPtr->typePtr != &returnCodeObjType && SetReturnCodeFromAny(interp, objPtr) == JIM_ERR) - return JIM_ERR; - *intPtr = objPtr->internalRep.intValue; - return JIM_OK; -} - -static int JimParseExprOperator(struct JimParserCtx *pc); -static int JimParseExprNumber(struct JimParserCtx *pc); -static int JimParseExprIrrational(struct JimParserCtx *pc); -static int JimParseExprBoolean(struct JimParserCtx *pc); - - -enum -{ - - - - JIM_EXPROP_MUL = JIM_TT_EXPR_OP, - JIM_EXPROP_DIV, - JIM_EXPROP_MOD, - JIM_EXPROP_SUB, - JIM_EXPROP_ADD, - JIM_EXPROP_LSHIFT, - JIM_EXPROP_RSHIFT, - JIM_EXPROP_ROTL, - JIM_EXPROP_ROTR, - JIM_EXPROP_LT, - JIM_EXPROP_GT, - JIM_EXPROP_LTE, - JIM_EXPROP_GTE, - JIM_EXPROP_NUMEQ, - JIM_EXPROP_NUMNE, - JIM_EXPROP_BITAND, - JIM_EXPROP_BITXOR, - JIM_EXPROP_BITOR, - JIM_EXPROP_LOGICAND, - JIM_EXPROP_LOGICOR, - JIM_EXPROP_TERNARY, - JIM_EXPROP_COLON, - JIM_EXPROP_POW, - - - JIM_EXPROP_STREQ, - JIM_EXPROP_STRNE, - JIM_EXPROP_STRIN, - JIM_EXPROP_STRNI, - - - JIM_EXPROP_NOT, - JIM_EXPROP_BITNOT, - JIM_EXPROP_UNARYMINUS, - JIM_EXPROP_UNARYPLUS, - - - JIM_EXPROP_FUNC_INT, - JIM_EXPROP_FUNC_WIDE, - JIM_EXPROP_FUNC_ABS, - JIM_EXPROP_FUNC_DOUBLE, - JIM_EXPROP_FUNC_ROUND, - JIM_EXPROP_FUNC_RAND, - JIM_EXPROP_FUNC_SRAND, - - - JIM_EXPROP_FUNC_SIN, - JIM_EXPROP_FUNC_COS, - JIM_EXPROP_FUNC_TAN, - JIM_EXPROP_FUNC_ASIN, - JIM_EXPROP_FUNC_ACOS, - JIM_EXPROP_FUNC_ATAN, - JIM_EXPROP_FUNC_ATAN2, - JIM_EXPROP_FUNC_SINH, - JIM_EXPROP_FUNC_COSH, - JIM_EXPROP_FUNC_TANH, - JIM_EXPROP_FUNC_CEIL, - JIM_EXPROP_FUNC_FLOOR, - JIM_EXPROP_FUNC_EXP, - JIM_EXPROP_FUNC_LOG, - JIM_EXPROP_FUNC_LOG10, - JIM_EXPROP_FUNC_SQRT, - JIM_EXPROP_FUNC_POW, - JIM_EXPROP_FUNC_HYPOT, - JIM_EXPROP_FUNC_FMOD, -}; - -struct JimExprNode { - int type; - struct Jim_Obj *objPtr; - - struct JimExprNode *left; - struct JimExprNode *right; - struct JimExprNode *ternary; -}; - - -typedef struct Jim_ExprOperator -{ - const char *name; - int (*funcop) (Jim_Interp *interp, struct JimExprNode *opnode); - unsigned char precedence; - unsigned char arity; - unsigned char attr; - unsigned char namelen; -} Jim_ExprOperator; - -static int JimExprGetTerm(Jim_Interp *interp, struct JimExprNode *node, Jim_Obj **objPtrPtr); -static int JimExprGetTermBoolean(Jim_Interp *interp, struct JimExprNode *node); -static int JimExprEvalTermNode(Jim_Interp *interp, struct JimExprNode *node); - -static int JimExprOpNumUnary(Jim_Interp *interp, struct JimExprNode *node) -{ - int intresult = 1; - int rc; - double dA, dC = 0; - jim_wide wA, wC = 0; - Jim_Obj *A; - - if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { - return rc; - } - - if ((A->typePtr != &doubleObjType || A->bytes) && JimGetWideNoErr(interp, A, &wA) == JIM_OK) { - switch (node->type) { - case JIM_EXPROP_FUNC_INT: - case JIM_EXPROP_FUNC_WIDE: - case JIM_EXPROP_FUNC_ROUND: - case JIM_EXPROP_UNARYPLUS: - wC = wA; - break; - case JIM_EXPROP_FUNC_DOUBLE: - dC = wA; - intresult = 0; - break; - case JIM_EXPROP_FUNC_ABS: - wC = wA >= 0 ? wA : -wA; - break; - case JIM_EXPROP_UNARYMINUS: - wC = -wA; - break; - case JIM_EXPROP_NOT: - wC = !wA; - break; - default: - abort(); - } - } - else if ((rc = Jim_GetDouble(interp, A, &dA)) == JIM_OK) { - switch (node->type) { - case JIM_EXPROP_FUNC_INT: - case JIM_EXPROP_FUNC_WIDE: - wC = dA; - break; - case JIM_EXPROP_FUNC_ROUND: - wC = dA < 0 ? (dA - 0.5) : (dA + 0.5); - break; - case JIM_EXPROP_FUNC_DOUBLE: - case JIM_EXPROP_UNARYPLUS: - dC = dA; - intresult = 0; - break; - case JIM_EXPROP_FUNC_ABS: -#ifdef JIM_MATH_FUNCTIONS - dC = fabs(dA); -#else - dC = dA >= 0 ? dA : -dA; -#endif - intresult = 0; - break; - case JIM_EXPROP_UNARYMINUS: - dC = -dA; - intresult = 0; - break; - case JIM_EXPROP_NOT: - wC = !dA; - break; - default: - abort(); - } - } - - if (rc == JIM_OK) { - if (intresult) { - Jim_SetResultInt(interp, wC); - } - else { - Jim_SetResult(interp, Jim_NewDoubleObj(interp, dC)); - } - } - - Jim_DecrRefCount(interp, A); - - return rc; -} - -static double JimRandDouble(Jim_Interp *interp) -{ - unsigned long x; - JimRandomBytes(interp, &x, sizeof(x)); - - return (double)x / (unsigned long)~0; -} - -static int JimExprOpIntUnary(Jim_Interp *interp, struct JimExprNode *node) -{ - jim_wide wA; - Jim_Obj *A; - int rc; - - if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { - return rc; - } - - rc = Jim_GetWide(interp, A, &wA); - if (rc == JIM_OK) { - switch (node->type) { - case JIM_EXPROP_BITNOT: - Jim_SetResultInt(interp, ~wA); - break; - case JIM_EXPROP_FUNC_SRAND: - JimPrngSeed(interp, (unsigned char *)&wA, sizeof(wA)); - Jim_SetResult(interp, Jim_NewDoubleObj(interp, JimRandDouble(interp))); - break; - default: - abort(); - } - } - - Jim_DecrRefCount(interp, A); - - return rc; -} - -static int JimExprOpNone(Jim_Interp *interp, struct JimExprNode *node) -{ - JimPanic((node->type != JIM_EXPROP_FUNC_RAND, "JimExprOpNone only support rand()")); - - Jim_SetResult(interp, Jim_NewDoubleObj(interp, JimRandDouble(interp))); - - return JIM_OK; -} - -#ifdef JIM_MATH_FUNCTIONS -static int JimExprOpDoubleUnary(Jim_Interp *interp, struct JimExprNode *node) -{ - int rc; - double dA, dC; - Jim_Obj *A; - - if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { - return rc; - } - - rc = Jim_GetDouble(interp, A, &dA); - if (rc == JIM_OK) { - switch (node->type) { - case JIM_EXPROP_FUNC_SIN: - dC = sin(dA); - break; - case JIM_EXPROP_FUNC_COS: - dC = cos(dA); - break; - case JIM_EXPROP_FUNC_TAN: - dC = tan(dA); - break; - case JIM_EXPROP_FUNC_ASIN: - dC = asin(dA); - break; - case JIM_EXPROP_FUNC_ACOS: - dC = acos(dA); - break; - case JIM_EXPROP_FUNC_ATAN: - dC = atan(dA); - break; - case JIM_EXPROP_FUNC_SINH: - dC = sinh(dA); - break; - case JIM_EXPROP_FUNC_COSH: - dC = cosh(dA); - break; - case JIM_EXPROP_FUNC_TANH: - dC = tanh(dA); - break; - case JIM_EXPROP_FUNC_CEIL: - dC = ceil(dA); - break; - case JIM_EXPROP_FUNC_FLOOR: - dC = floor(dA); - break; - case JIM_EXPROP_FUNC_EXP: - dC = exp(dA); - break; - case JIM_EXPROP_FUNC_LOG: - dC = log(dA); - break; - case JIM_EXPROP_FUNC_LOG10: - dC = log10(dA); - break; - case JIM_EXPROP_FUNC_SQRT: - dC = sqrt(dA); - break; - default: - abort(); - } - Jim_SetResult(interp, Jim_NewDoubleObj(interp, dC)); - } - - Jim_DecrRefCount(interp, A); - - return rc; -} -#endif - - -static int JimExprOpIntBin(Jim_Interp *interp, struct JimExprNode *node) -{ - jim_wide wA, wB; - int rc; - Jim_Obj *A, *B; - - if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { - return rc; - } - if ((rc = JimExprGetTerm(interp, node->right, &B)) != JIM_OK) { - Jim_DecrRefCount(interp, A); - return rc; - } - - rc = JIM_ERR; - - if (Jim_GetWide(interp, A, &wA) == JIM_OK && Jim_GetWide(interp, B, &wB) == JIM_OK) { - jim_wide wC; - - rc = JIM_OK; - - switch (node->type) { - case JIM_EXPROP_LSHIFT: - wC = wA << wB; - break; - case JIM_EXPROP_RSHIFT: - wC = wA >> wB; - break; - case JIM_EXPROP_BITAND: - wC = wA & wB; - break; - case JIM_EXPROP_BITXOR: - wC = wA ^ wB; - break; - case JIM_EXPROP_BITOR: - wC = wA | wB; - break; - case JIM_EXPROP_MOD: - if (wB == 0) { - wC = 0; - Jim_SetResultString(interp, "Division by zero", -1); - rc = JIM_ERR; - } - else { - int negative = 0; - - if (wB < 0) { - wB = -wB; - wA = -wA; - negative = 1; - } - wC = wA % wB; - if (wC < 0) { - wC += wB; - } - if (negative) { - wC = -wC; - } - } - break; - case JIM_EXPROP_ROTL: - case JIM_EXPROP_ROTR:{ - - unsigned long uA = (unsigned long)wA; - unsigned long uB = (unsigned long)wB; - const unsigned int S = sizeof(unsigned long) * 8; - - - uB %= S; - - if (node->type == JIM_EXPROP_ROTR) { - uB = S - uB; - } - wC = (unsigned long)(uA << uB) | (uA >> (S - uB)); - break; - } - default: - abort(); - } - Jim_SetResultInt(interp, wC); - } - - Jim_DecrRefCount(interp, A); - Jim_DecrRefCount(interp, B); - - return rc; -} - - - -static int JimExprOpBin(Jim_Interp *interp, struct JimExprNode *node) -{ - int rc = JIM_OK; - double dA, dB, dC = 0; - jim_wide wA, wB, wC = 0; - Jim_Obj *A, *B; - - if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { - return rc; - } - if ((rc = JimExprGetTerm(interp, node->right, &B)) != JIM_OK) { - Jim_DecrRefCount(interp, A); - return rc; - } - - if ((A->typePtr != &doubleObjType || A->bytes) && - (B->typePtr != &doubleObjType || B->bytes) && - JimGetWideNoErr(interp, A, &wA) == JIM_OK && JimGetWideNoErr(interp, B, &wB) == JIM_OK) { - - - - switch (node->type) { - case JIM_EXPROP_POW: - case JIM_EXPROP_FUNC_POW: - if (wA == 0 && wB < 0) { - Jim_SetResultString(interp, "exponentiation of zero by negative power", -1); - rc = JIM_ERR; - goto done; - } - wC = JimPowWide(wA, wB); - goto intresult; - case JIM_EXPROP_ADD: - wC = wA + wB; - goto intresult; - case JIM_EXPROP_SUB: - wC = wA - wB; - goto intresult; - case JIM_EXPROP_MUL: - wC = wA * wB; - goto intresult; - case JIM_EXPROP_DIV: - if (wB == 0) { - Jim_SetResultString(interp, "Division by zero", -1); - rc = JIM_ERR; - goto done; - } - else { - if (wB < 0) { - wB = -wB; - wA = -wA; - } - wC = wA / wB; - if (wA % wB < 0) { - wC--; - } - goto intresult; - } - case JIM_EXPROP_LT: - wC = wA < wB; - goto intresult; - case JIM_EXPROP_GT: - wC = wA > wB; - goto intresult; - case JIM_EXPROP_LTE: - wC = wA <= wB; - goto intresult; - case JIM_EXPROP_GTE: - wC = wA >= wB; - goto intresult; - case JIM_EXPROP_NUMEQ: - wC = wA == wB; - goto intresult; - case JIM_EXPROP_NUMNE: - wC = wA != wB; - goto intresult; - } - } - if (Jim_GetDouble(interp, A, &dA) == JIM_OK && Jim_GetDouble(interp, B, &dB) == JIM_OK) { - switch (node->type) { -#ifndef JIM_MATH_FUNCTIONS - case JIM_EXPROP_POW: - case JIM_EXPROP_FUNC_POW: - case JIM_EXPROP_FUNC_ATAN2: - case JIM_EXPROP_FUNC_HYPOT: - case JIM_EXPROP_FUNC_FMOD: - Jim_SetResultString(interp, "unsupported", -1); - rc = JIM_ERR; - goto done; -#else - case JIM_EXPROP_POW: - case JIM_EXPROP_FUNC_POW: - dC = pow(dA, dB); - goto doubleresult; - case JIM_EXPROP_FUNC_ATAN2: - dC = atan2(dA, dB); - goto doubleresult; - case JIM_EXPROP_FUNC_HYPOT: - dC = hypot(dA, dB); - goto doubleresult; - case JIM_EXPROP_FUNC_FMOD: - dC = fmod(dA, dB); - goto doubleresult; -#endif - case JIM_EXPROP_ADD: - dC = dA + dB; - goto doubleresult; - case JIM_EXPROP_SUB: - dC = dA - dB; - goto doubleresult; - case JIM_EXPROP_MUL: - dC = dA * dB; - goto doubleresult; - case JIM_EXPROP_DIV: - if (dB == 0) { -#ifdef INFINITY - dC = dA < 0 ? -INFINITY : INFINITY; -#else - dC = (dA < 0 ? -1.0 : 1.0) * strtod("Inf", NULL); -#endif - } - else { - dC = dA / dB; - } - goto doubleresult; - case JIM_EXPROP_LT: - wC = dA < dB; - goto intresult; - case JIM_EXPROP_GT: - wC = dA > dB; - goto intresult; - case JIM_EXPROP_LTE: - wC = dA <= dB; - goto intresult; - case JIM_EXPROP_GTE: - wC = dA >= dB; - goto intresult; - case JIM_EXPROP_NUMEQ: - wC = dA == dB; - goto intresult; - case JIM_EXPROP_NUMNE: - wC = dA != dB; - goto intresult; - } - } - else { - - - - int i = Jim_StringCompareObj(interp, A, B, 0); - - switch (node->type) { - case JIM_EXPROP_LT: - wC = i < 0; - goto intresult; - case JIM_EXPROP_GT: - wC = i > 0; - goto intresult; - case JIM_EXPROP_LTE: - wC = i <= 0; - goto intresult; - case JIM_EXPROP_GTE: - wC = i >= 0; - goto intresult; - case JIM_EXPROP_NUMEQ: - wC = i == 0; - goto intresult; - case JIM_EXPROP_NUMNE: - wC = i != 0; - goto intresult; - } - } - - rc = JIM_ERR; -done: - Jim_DecrRefCount(interp, A); - Jim_DecrRefCount(interp, B); - return rc; -intresult: - Jim_SetResultInt(interp, wC); - goto done; -doubleresult: - Jim_SetResult(interp, Jim_NewDoubleObj(interp, dC)); - goto done; -} - -static int JimSearchList(Jim_Interp *interp, Jim_Obj *listObjPtr, Jim_Obj *valObj) -{ - int listlen; - int i; - - listlen = Jim_ListLength(interp, listObjPtr); - for (i = 0; i < listlen; i++) { - if (Jim_StringEqObj(Jim_ListGetIndex(interp, listObjPtr, i), valObj)) { - return 1; - } - } - return 0; -} - - - -static int JimExprOpStrBin(Jim_Interp *interp, struct JimExprNode *node) -{ - Jim_Obj *A, *B; - jim_wide wC; - int rc; - - if ((rc = JimExprGetTerm(interp, node->left, &A)) != JIM_OK) { - return rc; - } - if ((rc = JimExprGetTerm(interp, node->right, &B)) != JIM_OK) { - Jim_DecrRefCount(interp, A); - return rc; - } - - switch (node->type) { - case JIM_EXPROP_STREQ: - case JIM_EXPROP_STRNE: - wC = Jim_StringEqObj(A, B); - if (node->type == JIM_EXPROP_STRNE) { - wC = !wC; - } - break; - case JIM_EXPROP_STRIN: - wC = JimSearchList(interp, B, A); - break; - case JIM_EXPROP_STRNI: - wC = !JimSearchList(interp, B, A); - break; - default: - abort(); - } - Jim_SetResultInt(interp, wC); - - Jim_DecrRefCount(interp, A); - Jim_DecrRefCount(interp, B); - - return rc; -} - -static int ExprBool(Jim_Interp *interp, Jim_Obj *obj) -{ - long l; - double d; - int b; - int ret = -1; - - - Jim_IncrRefCount(obj); - - if (Jim_GetLong(interp, obj, &l) == JIM_OK) { - ret = (l != 0); - } - else if (Jim_GetDouble(interp, obj, &d) == JIM_OK) { - ret = (d != 0); - } - else if (Jim_GetBoolean(interp, obj, &b) == JIM_OK) { - ret = (b != 0); - } - - Jim_DecrRefCount(interp, obj); - return ret; -} - -static int JimExprOpAnd(Jim_Interp *interp, struct JimExprNode *node) -{ - - int result = JimExprGetTermBoolean(interp, node->left); - - if (result == 1) { - - result = JimExprGetTermBoolean(interp, node->right); - } - if (result == -1) { - return JIM_ERR; - } - Jim_SetResultInt(interp, result); - return JIM_OK; -} - -static int JimExprOpOr(Jim_Interp *interp, struct JimExprNode *node) -{ - - int result = JimExprGetTermBoolean(interp, node->left); - - if (result == 0) { - - result = JimExprGetTermBoolean(interp, node->right); - } - if (result == -1) { - return JIM_ERR; - } - Jim_SetResultInt(interp, result); - return JIM_OK; -} - -static int JimExprOpTernary(Jim_Interp *interp, struct JimExprNode *node) -{ - - int result = JimExprGetTermBoolean(interp, node->left); - - if (result == 1) { - - return JimExprEvalTermNode(interp, node->right); - } - else if (result == 0) { - - return JimExprEvalTermNode(interp, node->ternary); - } - - return JIM_ERR; -} - -enum -{ - OP_FUNC = 0x0001, - OP_RIGHT_ASSOC = 0x0002, -}; - -#define OPRINIT_ATTR(N, P, ARITY, F, ATTR) {N, F, P, ARITY, ATTR, sizeof(N) - 1} -#define OPRINIT(N, P, ARITY, F) OPRINIT_ATTR(N, P, ARITY, F, 0) - -static const struct Jim_ExprOperator Jim_ExprOperators[] = { - OPRINIT("*", 110, 2, JimExprOpBin), - OPRINIT("/", 110, 2, JimExprOpBin), - OPRINIT("%", 110, 2, JimExprOpIntBin), - - OPRINIT("-", 100, 2, JimExprOpBin), - OPRINIT("+", 100, 2, JimExprOpBin), - - OPRINIT("<<", 90, 2, JimExprOpIntBin), - OPRINIT(">>", 90, 2, JimExprOpIntBin), - - OPRINIT("<<<", 90, 2, JimExprOpIntBin), - OPRINIT(">>>", 90, 2, JimExprOpIntBin), - - OPRINIT("<", 80, 2, JimExprOpBin), - OPRINIT(">", 80, 2, JimExprOpBin), - OPRINIT("<=", 80, 2, JimExprOpBin), - OPRINIT(">=", 80, 2, JimExprOpBin), - - OPRINIT("==", 70, 2, JimExprOpBin), - OPRINIT("!=", 70, 2, JimExprOpBin), - - OPRINIT("&", 50, 2, JimExprOpIntBin), - OPRINIT("^", 49, 2, JimExprOpIntBin), - OPRINIT("|", 48, 2, JimExprOpIntBin), - - OPRINIT("&&", 10, 2, JimExprOpAnd), - OPRINIT("||", 9, 2, JimExprOpOr), - OPRINIT_ATTR("?", 5, 3, JimExprOpTernary, OP_RIGHT_ASSOC), - OPRINIT_ATTR(":", 5, 3, NULL, OP_RIGHT_ASSOC), - - - OPRINIT_ATTR("**", 120, 2, JimExprOpBin, OP_RIGHT_ASSOC), - - OPRINIT("eq", 60, 2, JimExprOpStrBin), - OPRINIT("ne", 60, 2, JimExprOpStrBin), - - OPRINIT("in", 55, 2, JimExprOpStrBin), - OPRINIT("ni", 55, 2, JimExprOpStrBin), - - OPRINIT_ATTR("!", 150, 1, JimExprOpNumUnary, OP_RIGHT_ASSOC), - OPRINIT_ATTR("~", 150, 1, JimExprOpIntUnary, OP_RIGHT_ASSOC), - OPRINIT_ATTR(" -", 150, 1, JimExprOpNumUnary, OP_RIGHT_ASSOC), - OPRINIT_ATTR(" +", 150, 1, JimExprOpNumUnary, OP_RIGHT_ASSOC), - - - - OPRINIT_ATTR("int", 200, 1, JimExprOpNumUnary, OP_FUNC), - OPRINIT_ATTR("wide", 200, 1, JimExprOpNumUnary, OP_FUNC), - OPRINIT_ATTR("abs", 200, 1, JimExprOpNumUnary, OP_FUNC), - OPRINIT_ATTR("double", 200, 1, JimExprOpNumUnary, OP_FUNC), - OPRINIT_ATTR("round", 200, 1, JimExprOpNumUnary, OP_FUNC), - OPRINIT_ATTR("rand", 200, 0, JimExprOpNone, OP_FUNC), - OPRINIT_ATTR("srand", 200, 1, JimExprOpIntUnary, OP_FUNC), - -#ifdef JIM_MATH_FUNCTIONS - OPRINIT_ATTR("sin", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("cos", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("tan", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("asin", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("acos", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("atan", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("atan2", 200, 2, JimExprOpBin, OP_FUNC), - OPRINIT_ATTR("sinh", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("cosh", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("tanh", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("ceil", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("floor", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("exp", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("log", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("log10", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("sqrt", 200, 1, JimExprOpDoubleUnary, OP_FUNC), - OPRINIT_ATTR("pow", 200, 2, JimExprOpBin, OP_FUNC), - OPRINIT_ATTR("hypot", 200, 2, JimExprOpBin, OP_FUNC), - OPRINIT_ATTR("fmod", 200, 2, JimExprOpBin, OP_FUNC), -#endif -}; -#undef OPRINIT -#undef OPRINIT_ATTR - -#define JIM_EXPR_OPERATORS_NUM \ - (sizeof(Jim_ExprOperators)/sizeof(struct Jim_ExprOperator)) - -static int JimParseExpression(struct JimParserCtx *pc) -{ - - while (isspace(UCHAR(*pc->p)) || (*(pc->p) == '\\' && *(pc->p + 1) == '\n')) { - if (*pc->p == '\n') { - pc->linenr++; - } - pc->p++; - pc->len--; - } - - - pc->tline = pc->linenr; - pc->tstart = pc->p; - - if (pc->len == 0) { - pc->tend = pc->p; - pc->tt = JIM_TT_EOL; - pc->eof = 1; - return JIM_OK; - } - switch (*(pc->p)) { - case '(': - pc->tt = JIM_TT_SUBEXPR_START; - goto singlechar; - case ')': - pc->tt = JIM_TT_SUBEXPR_END; - goto singlechar; - case ',': - pc->tt = JIM_TT_SUBEXPR_COMMA; -singlechar: - pc->tend = pc->p; - pc->p++; - pc->len--; - break; - case '[': - return JimParseCmd(pc); - case '$': - if (JimParseVar(pc) == JIM_ERR) - return JimParseExprOperator(pc); - else { - - if (pc->tt == JIM_TT_EXPRSUGAR) { - return JIM_ERR; - } - return JIM_OK; - } - break; - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '.': - return JimParseExprNumber(pc); - case '"': - return JimParseQuote(pc); - case '{': - return JimParseBrace(pc); - - case 'N': - case 'I': - case 'n': - case 'i': - if (JimParseExprIrrational(pc) == JIM_ERR) - if (JimParseExprBoolean(pc) == JIM_ERR) - return JimParseExprOperator(pc); - break; - case 't': - case 'f': - case 'o': - case 'y': - if (JimParseExprBoolean(pc) == JIM_ERR) - return JimParseExprOperator(pc); - break; - default: - return JimParseExprOperator(pc); - break; - } - return JIM_OK; -} - -static int JimParseExprNumber(struct JimParserCtx *pc) -{ - char *end; - - - pc->tt = JIM_TT_EXPR_INT; - - jim_strtoull(pc->p, (char **)&pc->p); - - if (strchr("eENnIi.", *pc->p) || pc->p == pc->tstart) { - if (strtod(pc->tstart, &end)) { } - if (end == pc->tstart) - return JIM_ERR; - if (end > pc->p) { - - pc->tt = JIM_TT_EXPR_DOUBLE; - pc->p = end; - } - } - pc->tend = pc->p - 1; - pc->len -= (pc->p - pc->tstart); - return JIM_OK; -} - -static int JimParseExprIrrational(struct JimParserCtx *pc) -{ - const char *irrationals[] = { "NaN", "nan", "NAN", "Inf", "inf", "INF", NULL }; - int i; - - for (i = 0; irrationals[i]; i++) { - const char *irr = irrationals[i]; - - if (strncmp(irr, pc->p, 3) == 0) { - pc->p += 3; - pc->len -= 3; - pc->tend = pc->p - 1; - pc->tt = JIM_TT_EXPR_DOUBLE; - return JIM_OK; - } - } - return JIM_ERR; -} - -static int JimParseExprBoolean(struct JimParserCtx *pc) -{ - const char *booleans[] = { "false", "no", "off", "true", "yes", "on", NULL }; - const int lengths[] = { 5, 2, 3, 4, 3, 2, 0 }; - int i; - - for (i = 0; booleans[i]; i++) { - const char *boolean = booleans[i]; - int length = lengths[i]; - - if (strncmp(boolean, pc->p, length) == 0) { - pc->p += length; - pc->len -= length; - pc->tend = pc->p - 1; - pc->tt = JIM_TT_EXPR_BOOLEAN; - return JIM_OK; - } - } - return JIM_ERR; -} - -static const struct Jim_ExprOperator *JimExprOperatorInfoByOpcode(int opcode) -{ - static Jim_ExprOperator dummy_op; - if (opcode < JIM_TT_EXPR_OP) { - return &dummy_op; - } - return &Jim_ExprOperators[opcode - JIM_TT_EXPR_OP]; -} - -static int JimParseExprOperator(struct JimParserCtx *pc) -{ - int i; - const struct Jim_ExprOperator *bestOp = NULL; - int bestLen = 0; - - - for (i = 0; i < (signed)JIM_EXPR_OPERATORS_NUM; i++) { - const struct Jim_ExprOperator *op = &Jim_ExprOperators[i]; - - if (op->name[0] != pc->p[0]) { - continue; - } - - if (op->namelen > bestLen && strncmp(op->name, pc->p, op->namelen) == 0) { - bestOp = op; - bestLen = op->namelen; - } - } - if (bestOp == NULL) { - return JIM_ERR; - } - - - if (bestOp->attr & OP_FUNC) { - const char *p = pc->p + bestLen; - int len = pc->len - bestLen; - - while (len && isspace(UCHAR(*p))) { - len--; - p++; - } - if (*p != '(') { - return JIM_ERR; - } - } - pc->tend = pc->p + bestLen - 1; - pc->p += bestLen; - pc->len -= bestLen; - - pc->tt = (bestOp - Jim_ExprOperators) + JIM_TT_EXPR_OP; - return JIM_OK; -} - -const char *jim_tt_name(int type) -{ - static const char * const tt_names[JIM_TT_EXPR_OP] = - { "NIL", "STR", "ESC", "VAR", "ARY", "CMD", "SEP", "EOL", "EOF", "LIN", "WRD", "(((", ")))", ",,,", "INT", - "DBL", "BOO", "$()" }; - if (type < JIM_TT_EXPR_OP) { - return tt_names[type]; - } - else if (type == JIM_EXPROP_UNARYMINUS) { - return "-VE"; - } - else if (type == JIM_EXPROP_UNARYPLUS) { - return "+VE"; - } - else { - const struct Jim_ExprOperator *op = JimExprOperatorInfoByOpcode(type); - static char buf[20]; - - if (op->name) { - return op->name; - } - sprintf(buf, "(%d)", type); - return buf; - } -} - -static void FreeExprInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); -static void DupExprInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); -static int SetExprFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr); - -static const Jim_ObjType exprObjType = { - "expression", - FreeExprInternalRep, - DupExprInternalRep, - NULL, - JIM_TYPE_REFERENCES, -}; - - -struct ExprTree -{ - struct JimExprNode *expr; - struct JimExprNode *nodes; - int len; - int inUse; -}; - -static void ExprTreeFreeNodes(Jim_Interp *interp, struct JimExprNode *nodes, int num) -{ - int i; - for (i = 0; i < num; i++) { - if (nodes[i].objPtr) { - Jim_DecrRefCount(interp, nodes[i].objPtr); - } - } - Jim_Free(nodes); -} - -static void ExprTreeFree(Jim_Interp *interp, struct ExprTree *expr) -{ - ExprTreeFreeNodes(interp, expr->nodes, expr->len); - Jim_Free(expr); -} - -static void FreeExprInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) -{ - struct ExprTree *expr = (void *)objPtr->internalRep.ptr; - - if (expr) { - if (--expr->inUse != 0) { - return; - } - - ExprTreeFree(interp, expr); - } -} - -static void DupExprInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) -{ - JIM_NOTUSED(interp); - JIM_NOTUSED(srcPtr); - - - dupPtr->typePtr = NULL; -} - -struct ExprBuilder { - int parencount; - int level; - ParseToken *token; - ParseToken *first_token; - Jim_Stack stack; - Jim_Obj *exprObjPtr; - Jim_Obj *fileNameObj; - struct JimExprNode *nodes; - struct JimExprNode *next; -}; - -#ifdef DEBUG_SHOW_EXPR -static void JimShowExprNode(struct JimExprNode *node, int level) -{ - int i; - for (i = 0; i < level; i++) { - printf(" "); - } - if (TOKEN_IS_EXPR_OP(node->type)) { - printf("%s\n", jim_tt_name(node->type)); - if (node->left) { - JimShowExprNode(node->left, level + 1); - } - if (node->right) { - JimShowExprNode(node->right, level + 1); - } - if (node->ternary) { - JimShowExprNode(node->ternary, level + 1); - } - } - else { - printf("[%s] %s\n", jim_tt_name(node->type), Jim_String(node->objPtr)); - } -} -#endif - -#define EXPR_UNTIL_CLOSE 0x0001 -#define EXPR_FUNC_ARGS 0x0002 -#define EXPR_TERNARY 0x0004 - -static int ExprTreeBuildTree(Jim_Interp *interp, struct ExprBuilder *builder, int precedence, int flags, int exp_numterms) -{ - int rc; - struct JimExprNode *node; - - int exp_stacklen = builder->stack.len + exp_numterms; - - if (builder->level++ > 200) { - Jim_SetResultString(interp, "Expression too complex", -1); - return JIM_ERR; - } - - while (builder->token->type != JIM_TT_EOL) { - ParseToken *t = builder->token++; - int prevtt; - - if (t == builder->first_token) { - prevtt = JIM_TT_NONE; - } - else { - prevtt = t[-1].type; - } - - if (t->type == JIM_TT_SUBEXPR_START) { - if (builder->stack.len == exp_stacklen) { - Jim_SetResultFormatted(interp, "unexpected open parenthesis in expression: \"%#s\"", builder->exprObjPtr); - return JIM_ERR; - } - builder->parencount++; - rc = ExprTreeBuildTree(interp, builder, 0, EXPR_UNTIL_CLOSE, 1); - if (rc != JIM_OK) { - return rc; - } - - } - else if (t->type == JIM_TT_SUBEXPR_END) { - if (!(flags & EXPR_UNTIL_CLOSE)) { - if (builder->stack.len == exp_stacklen && builder->level > 1) { - builder->token--; - builder->level--; - return JIM_OK; - } - Jim_SetResultFormatted(interp, "unexpected closing parenthesis in expression: \"%#s\"", builder->exprObjPtr); - return JIM_ERR; - } - builder->parencount--; - if (builder->stack.len == exp_stacklen) { - - break; - } - } - else if (t->type == JIM_TT_SUBEXPR_COMMA) { - if (!(flags & EXPR_FUNC_ARGS)) { - if (builder->stack.len == exp_stacklen) { - - builder->token--; - builder->level--; - return JIM_OK; - } - Jim_SetResultFormatted(interp, "unexpected comma in expression: \"%#s\"", builder->exprObjPtr); - return JIM_ERR; - } - else { - - if (builder->stack.len > exp_stacklen) { - Jim_SetResultFormatted(interp, "too many arguments to math function"); - return JIM_ERR; - } - } - - } - else if (t->type == JIM_EXPROP_COLON) { - if (!(flags & EXPR_TERNARY)) { - if (builder->level != 1) { - - builder->token--; - builder->level--; - return JIM_OK; - } - Jim_SetResultFormatted(interp, ": without ? in expression: \"%#s\"", builder->exprObjPtr); - return JIM_ERR; - } - if (builder->stack.len == exp_stacklen) { - - builder->token--; - builder->level--; - return JIM_OK; - } - - } - else if (TOKEN_IS_EXPR_OP(t->type)) { - const struct Jim_ExprOperator *op; - - - if (TOKEN_IS_EXPR_OP(prevtt) || TOKEN_IS_EXPR_START(prevtt)) { - if (t->type == JIM_EXPROP_SUB) { - t->type = JIM_EXPROP_UNARYMINUS; - } - else if (t->type == JIM_EXPROP_ADD) { - t->type = JIM_EXPROP_UNARYPLUS; - } - } - - op = JimExprOperatorInfoByOpcode(t->type); - - if (op->precedence < precedence || (!(op->attr & OP_RIGHT_ASSOC) && op->precedence == precedence)) { - - builder->token--; - break; - } - - if (op->attr & OP_FUNC) { - if (builder->token->type != JIM_TT_SUBEXPR_START) { - Jim_SetResultString(interp, "missing arguments for math function", -1); - return JIM_ERR; - } - builder->token++; - if (op->arity == 0) { - if (builder->token->type != JIM_TT_SUBEXPR_END) { - Jim_SetResultString(interp, "too many arguments for math function", -1); - return JIM_ERR; - } - builder->token++; - goto noargs; - } - builder->parencount++; - - - rc = ExprTreeBuildTree(interp, builder, 0, EXPR_FUNC_ARGS | EXPR_UNTIL_CLOSE, op->arity); - } - else if (t->type == JIM_EXPROP_TERNARY) { - - rc = ExprTreeBuildTree(interp, builder, op->precedence, EXPR_TERNARY, 2); - } - else { - rc = ExprTreeBuildTree(interp, builder, op->precedence, 0, 1); - } - - if (rc != JIM_OK) { - return rc; - } - -noargs: - node = builder->next++; - node->type = t->type; - - if (op->arity >= 3) { - node->ternary = Jim_StackPop(&builder->stack); - if (node->ternary == NULL) { - goto missingoperand; - } - } - if (op->arity >= 2) { - node->right = Jim_StackPop(&builder->stack); - if (node->right == NULL) { - goto missingoperand; - } - } - if (op->arity >= 1) { - node->left = Jim_StackPop(&builder->stack); - if (node->left == NULL) { -missingoperand: - Jim_SetResultFormatted(interp, "missing operand to %s in expression: \"%#s\"", op->name, builder->exprObjPtr); - builder->next--; - return JIM_ERR; - - } - } - - - Jim_StackPush(&builder->stack, node); - } - else { - Jim_Obj *objPtr = NULL; - - - - - if (!TOKEN_IS_EXPR_START(prevtt) && !TOKEN_IS_EXPR_OP(prevtt)) { - Jim_SetResultFormatted(interp, "missing operator in expression: \"%#s\"", builder->exprObjPtr); - return JIM_ERR; - } - - - if (t->type == JIM_TT_EXPR_INT || t->type == JIM_TT_EXPR_DOUBLE) { - char *endptr; - if (t->type == JIM_TT_EXPR_INT) { - objPtr = Jim_NewIntObj(interp, jim_strtoull(t->token, &endptr)); - } - else { - objPtr = Jim_NewDoubleObj(interp, strtod(t->token, &endptr)); - } - if (endptr != t->token + t->len) { - - Jim_FreeNewObj(interp, objPtr); - objPtr = NULL; - } - } - - if (!objPtr) { - - objPtr = Jim_NewStringObj(interp, t->token, t->len); - if (t->type == JIM_TT_CMD) { - - JimSetSourceInfo(interp, objPtr, builder->fileNameObj, t->line); - } - } - - - node = builder->next++; - node->objPtr = objPtr; - Jim_IncrRefCount(node->objPtr); - node->type = t->type; - Jim_StackPush(&builder->stack, node); - } - } - - if (builder->stack.len == exp_stacklen) { - builder->level--; - return JIM_OK; - } - - if ((flags & EXPR_FUNC_ARGS)) { - Jim_SetResultFormatted(interp, "too %s arguments for math function", (builder->stack.len < exp_stacklen) ? "few" : "many"); - } - else { - if (builder->stack.len < exp_stacklen) { - if (builder->level == 0) { - Jim_SetResultFormatted(interp, "empty expression"); - } - else { - Jim_SetResultFormatted(interp, "syntax error in expression \"%#s\": premature end of expression", builder->exprObjPtr); - } - } - else { - Jim_SetResultFormatted(interp, "extra terms after expression"); - } - } - - return JIM_ERR; -} - -static struct ExprTree *ExprTreeCreateTree(Jim_Interp *interp, const ParseTokenList *tokenlist, Jim_Obj *exprObjPtr, Jim_Obj *fileNameObj) -{ - struct ExprTree *expr; - struct ExprBuilder builder; - int rc; - struct JimExprNode *top = NULL; - - builder.parencount = 0; - builder.level = 0; - builder.token = builder.first_token = tokenlist->list; - builder.exprObjPtr = exprObjPtr; - builder.fileNameObj = fileNameObj; - - builder.nodes = malloc(sizeof(struct JimExprNode) * (tokenlist->count - 1)); - memset(builder.nodes, 0, sizeof(struct JimExprNode) * (tokenlist->count - 1)); - builder.next = builder.nodes; - Jim_InitStack(&builder.stack); - - rc = ExprTreeBuildTree(interp, &builder, 0, 0, 1); - - if (rc == JIM_OK) { - top = Jim_StackPop(&builder.stack); - - if (builder.parencount) { - Jim_SetResultString(interp, "missing close parenthesis", -1); - rc = JIM_ERR; - } - } - - - Jim_FreeStack(&builder.stack); - - if (rc != JIM_OK) { - ExprTreeFreeNodes(interp, builder.nodes, builder.next - builder.nodes); - return NULL; - } - - expr = Jim_Alloc(sizeof(*expr)); - expr->inUse = 1; - expr->expr = top; - expr->nodes = builder.nodes; - expr->len = builder.next - builder.nodes; - - assert(expr->len <= tokenlist->count - 1); - - return expr; -} - -static int SetExprFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr) -{ - int exprTextLen; - const char *exprText; - struct JimParserCtx parser; - struct ExprTree *expr; - ParseTokenList tokenlist; - int line; - Jim_Obj *fileNameObj; - int rc = JIM_ERR; - - - if (objPtr->typePtr == &sourceObjType) { - fileNameObj = objPtr->internalRep.sourceValue.fileNameObj; - line = objPtr->internalRep.sourceValue.lineNumber; - } - else { - fileNameObj = interp->emptyObj; - line = 1; - } - Jim_IncrRefCount(fileNameObj); - - exprText = Jim_GetString(objPtr, &exprTextLen); - - - ScriptTokenListInit(&tokenlist); - - JimParserInit(&parser, exprText, exprTextLen, line); - while (!parser.eof) { - if (JimParseExpression(&parser) != JIM_OK) { - ScriptTokenListFree(&tokenlist); - Jim_SetResultFormatted(interp, "syntax error in expression: \"%#s\"", objPtr); - expr = NULL; - goto err; - } - - ScriptAddToken(&tokenlist, parser.tstart, parser.tend - parser.tstart + 1, parser.tt, - parser.tline); - } - -#ifdef DEBUG_SHOW_EXPR_TOKENS - { - int i; - printf("==== Expr Tokens (%s) ====\n", Jim_String(fileNameObj)); - for (i = 0; i < tokenlist.count; i++) { - printf("[%2d]@%d %s '%.*s'\n", i, tokenlist.list[i].line, jim_tt_name(tokenlist.list[i].type), - tokenlist.list[i].len, tokenlist.list[i].token); - } - } -#endif - - if (JimParseCheckMissing(interp, parser.missing.ch) == JIM_ERR) { - ScriptTokenListFree(&tokenlist); - Jim_DecrRefCount(interp, fileNameObj); - return JIM_ERR; - } - - - expr = ExprTreeCreateTree(interp, &tokenlist, objPtr, fileNameObj); - - - ScriptTokenListFree(&tokenlist); - - if (!expr) { - goto err; - } - -#ifdef DEBUG_SHOW_EXPR - printf("==== Expr ====\n"); - JimShowExprNode(expr->expr, 0); -#endif - - rc = JIM_OK; - - err: - - Jim_DecrRefCount(interp, fileNameObj); - Jim_FreeIntRep(interp, objPtr); - Jim_SetIntRepPtr(objPtr, expr); - objPtr->typePtr = &exprObjType; - return rc; -} - -static struct ExprTree *JimGetExpression(Jim_Interp *interp, Jim_Obj *objPtr) -{ - if (objPtr->typePtr != &exprObjType) { - if (SetExprFromAny(interp, objPtr) != JIM_OK) { - return NULL; - } - } - return (struct ExprTree *) Jim_GetIntRepPtr(objPtr); -} - -#ifdef JIM_OPTIMIZATION -static Jim_Obj *JimExprIntValOrVar(Jim_Interp *interp, struct JimExprNode *node) -{ - if (node->type == JIM_TT_EXPR_INT) - return node->objPtr; - else if (node->type == JIM_TT_VAR) - return Jim_GetVariable(interp, node->objPtr, JIM_NONE); - else if (node->type == JIM_TT_DICTSUGAR) - return JimExpandDictSugar(interp, node->objPtr); - else - return NULL; -} -#endif - - -static int JimExprEvalTermNode(Jim_Interp *interp, struct JimExprNode *node) -{ - if (TOKEN_IS_EXPR_OP(node->type)) { - const struct Jim_ExprOperator *op = JimExprOperatorInfoByOpcode(node->type); - return op->funcop(interp, node); - } - else { - Jim_Obj *objPtr; - - - switch (node->type) { - case JIM_TT_EXPR_INT: - case JIM_TT_EXPR_DOUBLE: - case JIM_TT_EXPR_BOOLEAN: - case JIM_TT_STR: - Jim_SetResult(interp, node->objPtr); - return JIM_OK; - - case JIM_TT_VAR: - objPtr = Jim_GetVariable(interp, node->objPtr, JIM_ERRMSG); - if (objPtr) { - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - return JIM_ERR; - - case JIM_TT_DICTSUGAR: - objPtr = JimExpandDictSugar(interp, node->objPtr); - if (objPtr) { - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - return JIM_ERR; - - case JIM_TT_ESC: - if (Jim_SubstObj(interp, node->objPtr, &objPtr, JIM_NONE) == JIM_OK) { - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - return JIM_ERR; - - case JIM_TT_CMD: - return Jim_EvalObj(interp, node->objPtr); - - default: - - return JIM_ERR; - } - } -} - -static int JimExprGetTerm(Jim_Interp *interp, struct JimExprNode *node, Jim_Obj **objPtrPtr) -{ - int rc = JimExprEvalTermNode(interp, node); - if (rc == JIM_OK) { - *objPtrPtr = Jim_GetResult(interp); - Jim_IncrRefCount(*objPtrPtr); - } - return rc; -} - -static int JimExprGetTermBoolean(Jim_Interp *interp, struct JimExprNode *node) -{ - if (JimExprEvalTermNode(interp, node) == JIM_OK) { - return ExprBool(interp, Jim_GetResult(interp)); - } - return -1; -} - -int Jim_EvalExpression(Jim_Interp *interp, Jim_Obj *exprObjPtr) -{ - struct ExprTree *expr; - int retcode = JIM_OK; - - expr = JimGetExpression(interp, exprObjPtr); - if (!expr) { - return JIM_ERR; - } - -#ifdef JIM_OPTIMIZATION - { - Jim_Obj *objPtr; - - - switch (expr->len) { - case 1: - objPtr = JimExprIntValOrVar(interp, expr->expr); - if (objPtr) { - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - break; - - case 2: - if (expr->expr->type == JIM_EXPROP_NOT) { - objPtr = JimExprIntValOrVar(interp, expr->expr->left); - - if (objPtr && JimIsWide(objPtr)) { - Jim_SetResult(interp, JimWideValue(objPtr) ? interp->falseObj : interp->trueObj); - return JIM_OK; - } - } - break; - - case 3: - objPtr = JimExprIntValOrVar(interp, expr->expr->left); - if (objPtr && JimIsWide(objPtr)) { - Jim_Obj *objPtr2 = JimExprIntValOrVar(interp, expr->expr->right); - if (objPtr2 && JimIsWide(objPtr2)) { - jim_wide wideValueA = JimWideValue(objPtr); - jim_wide wideValueB = JimWideValue(objPtr2); - int cmpRes; - switch (expr->expr->type) { - case JIM_EXPROP_LT: - cmpRes = wideValueA < wideValueB; - break; - case JIM_EXPROP_LTE: - cmpRes = wideValueA <= wideValueB; - break; - case JIM_EXPROP_GT: - cmpRes = wideValueA > wideValueB; - break; - case JIM_EXPROP_GTE: - cmpRes = wideValueA >= wideValueB; - break; - case JIM_EXPROP_NUMEQ: - cmpRes = wideValueA == wideValueB; - break; - case JIM_EXPROP_NUMNE: - cmpRes = wideValueA != wideValueB; - break; - default: - goto noopt; - } - Jim_SetResult(interp, cmpRes ? interp->trueObj : interp->falseObj); - return JIM_OK; - } - } - break; - } - } -noopt: -#endif - - expr->inUse++; - - - retcode = JimExprEvalTermNode(interp, expr->expr); - - expr->inUse--; - - return retcode; -} - -int Jim_GetBoolFromExpr(Jim_Interp *interp, Jim_Obj *exprObjPtr, int *boolPtr) -{ - int retcode = Jim_EvalExpression(interp, exprObjPtr); - - if (retcode == JIM_OK) { - switch (ExprBool(interp, Jim_GetResult(interp))) { - case 0: - *boolPtr = 0; - break; - - case 1: - *boolPtr = 1; - break; - - case -1: - retcode = JIM_ERR; - break; - } - } - return retcode; -} - - - - -typedef struct ScanFmtPartDescr -{ - const char *arg; - const char *prefix; - size_t width; - int pos; - char type; - char modifier; -} ScanFmtPartDescr; - - -typedef struct ScanFmtStringObj -{ - jim_wide size; - char *stringRep; - size_t count; - size_t convCount; - size_t maxPos; - const char *error; - char *scratch; - ScanFmtPartDescr descr[1]; -} ScanFmtStringObj; - - -static void FreeScanFmtInternalRep(Jim_Interp *interp, Jim_Obj *objPtr); -static void DupScanFmtInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr); -static void UpdateStringOfScanFmt(Jim_Obj *objPtr); - -static const Jim_ObjType scanFmtStringObjType = { - "scanformatstring", - FreeScanFmtInternalRep, - DupScanFmtInternalRep, - UpdateStringOfScanFmt, - JIM_TYPE_NONE, -}; - -void FreeScanFmtInternalRep(Jim_Interp *interp, Jim_Obj *objPtr) -{ - JIM_NOTUSED(interp); - Jim_Free((char *)objPtr->internalRep.ptr); - objPtr->internalRep.ptr = 0; -} - -void DupScanFmtInternalRep(Jim_Interp *interp, Jim_Obj *srcPtr, Jim_Obj *dupPtr) -{ - size_t size = (size_t) ((ScanFmtStringObj *) srcPtr->internalRep.ptr)->size; - ScanFmtStringObj *newVec = (ScanFmtStringObj *) Jim_Alloc(size); - - JIM_NOTUSED(interp); - memcpy(newVec, srcPtr->internalRep.ptr, size); - dupPtr->internalRep.ptr = newVec; - dupPtr->typePtr = &scanFmtStringObjType; -} - -static void UpdateStringOfScanFmt(Jim_Obj *objPtr) -{ - JimSetStringBytes(objPtr, ((ScanFmtStringObj *) objPtr->internalRep.ptr)->stringRep); -} - - -static int SetScanFmtFromAny(Jim_Interp *interp, Jim_Obj *objPtr) -{ - ScanFmtStringObj *fmtObj; - char *buffer; - int maxCount, i, approxSize, lastPos = -1; - const char *fmt = Jim_String(objPtr); - int maxFmtLen = Jim_Length(objPtr); - const char *fmtEnd = fmt + maxFmtLen; - int curr; - - Jim_FreeIntRep(interp, objPtr); - - for (i = 0, maxCount = 0; i < maxFmtLen; ++i) - if (fmt[i] == '%') - ++maxCount; - - approxSize = sizeof(ScanFmtStringObj) - +(maxCount + 1) * sizeof(ScanFmtPartDescr) - +maxFmtLen * sizeof(char) + 3 + 1 - + maxFmtLen * sizeof(char) + 1 - + maxFmtLen * sizeof(char) - +(maxCount + 1) * sizeof(char) - +1; - fmtObj = (ScanFmtStringObj *) Jim_Alloc(approxSize); - memset(fmtObj, 0, approxSize); - fmtObj->size = approxSize; - fmtObj->maxPos = 0; - fmtObj->scratch = (char *)&fmtObj->descr[maxCount + 1]; - fmtObj->stringRep = fmtObj->scratch + maxFmtLen + 3 + 1; - memcpy(fmtObj->stringRep, fmt, maxFmtLen); - buffer = fmtObj->stringRep + maxFmtLen + 1; - objPtr->internalRep.ptr = fmtObj; - objPtr->typePtr = &scanFmtStringObjType; - for (i = 0, curr = 0; fmt < fmtEnd; ++fmt) { - int width = 0, skip; - ScanFmtPartDescr *descr = &fmtObj->descr[curr]; - - fmtObj->count++; - descr->width = 0; - - if (*fmt != '%' || fmt[1] == '%') { - descr->type = 0; - descr->prefix = &buffer[i]; - for (; fmt < fmtEnd; ++fmt) { - if (*fmt == '%') { - if (fmt[1] != '%') - break; - ++fmt; - } - buffer[i++] = *fmt; - } - buffer[i++] = 0; - } - - ++fmt; - - if (fmt >= fmtEnd) - goto done; - descr->pos = 0; - if (*fmt == '*') { - descr->pos = -1; - ++fmt; - } - else - fmtObj->convCount++; - - if (sscanf(fmt, "%d%n", &width, &skip) == 1) { - fmt += skip; - - if (descr->pos != -1 && *fmt == '$') { - int prev; - - ++fmt; - descr->pos = width; - width = 0; - - if ((lastPos == 0 && descr->pos > 0) - || (lastPos > 0 && descr->pos == 0)) { - fmtObj->error = "cannot mix \"%\" and \"%n$\" conversion specifiers"; - return JIM_ERR; - } - - for (prev = 0; prev < curr; ++prev) { - if (fmtObj->descr[prev].pos == -1) - continue; - if (fmtObj->descr[prev].pos == descr->pos) { - fmtObj->error = - "variable is assigned by multiple \"%n$\" conversion specifiers"; - return JIM_ERR; - } - } - if (descr->pos < 0) { - fmtObj->error = - "\"%n$\" conversion specifier is negative"; - return JIM_ERR; - } - - if (sscanf(fmt, "%d%n", &width, &skip) == 1) { - descr->width = width; - fmt += skip; - } - if (descr->pos > 0 && (size_t) descr->pos > fmtObj->maxPos) - fmtObj->maxPos = descr->pos; - } - else { - - descr->width = width; - } - } - - if (lastPos == -1) - lastPos = descr->pos; - - if (*fmt == '[') { - int swapped = 1, beg = i, end, j; - - descr->type = '['; - descr->arg = &buffer[i]; - ++fmt; - if (*fmt == '^') - buffer[i++] = *fmt++; - if (*fmt == ']') - buffer[i++] = *fmt++; - while (*fmt && *fmt != ']') - buffer[i++] = *fmt++; - if (*fmt != ']') { - fmtObj->error = "unmatched [ in format string"; - return JIM_ERR; - } - end = i; - buffer[i++] = 0; - - while (swapped) { - swapped = 0; - for (j = beg + 1; j < end - 1; ++j) { - if (buffer[j] == '-' && buffer[j - 1] > buffer[j + 1]) { - char tmp = buffer[j - 1]; - - buffer[j - 1] = buffer[j + 1]; - buffer[j + 1] = tmp; - swapped = 1; - } - } - } - } - else { - - if (fmt < fmtEnd && strchr("hlL", *fmt)) - descr->modifier = tolower((int)*fmt++); - - if (fmt >= fmtEnd) { - fmtObj->error = "missing scan conversion character"; - return JIM_ERR; - } - - descr->type = *fmt; - if (strchr("efgcsndoxui", *fmt) == 0) { - fmtObj->error = "bad scan conversion character"; - return JIM_ERR; - } - else if (*fmt == 'c' && descr->width != 0) { - fmtObj->error = "field width may not be specified in %c " "conversion"; - return JIM_ERR; - } - else if (*fmt == 'u' && descr->modifier == 'l') { - fmtObj->error = "unsigned wide not supported"; - return JIM_ERR; - } - } - curr++; - } - done: - return JIM_OK; -} - - - -#define FormatGetCnvCount(_fo_) \ - ((ScanFmtStringObj*)((_fo_)->internalRep.ptr))->convCount -#define FormatGetMaxPos(_fo_) \ - ((ScanFmtStringObj*)((_fo_)->internalRep.ptr))->maxPos -#define FormatGetError(_fo_) \ - ((ScanFmtStringObj*)((_fo_)->internalRep.ptr))->error - -static Jim_Obj *JimScanAString(Jim_Interp *interp, const char *sdescr, const char *str) -{ - char *buffer = Jim_StrDup(str); - char *p = buffer; - - while (*str) { - int c; - int n; - - if (!sdescr && isspace(UCHAR(*str))) - break; - - n = utf8_tounicode(str, &c); - if (sdescr && !JimCharsetMatch(sdescr, c, JIM_CHARSET_SCAN)) - break; - while (n--) - *p++ = *str++; - } - *p = 0; - return Jim_NewStringObjNoAlloc(interp, buffer, p - buffer); -} - - -static int ScanOneEntry(Jim_Interp *interp, const char *str, int pos, int strLen, - ScanFmtStringObj * fmtObj, long idx, Jim_Obj **valObjPtr) -{ - const char *tok; - const ScanFmtPartDescr *descr = &fmtObj->descr[idx]; - size_t scanned = 0; - size_t anchor = pos; - int i; - Jim_Obj *tmpObj = NULL; - - - *valObjPtr = 0; - if (descr->prefix) { - for (i = 0; pos < strLen && descr->prefix[i]; ++i) { - - if (isspace(UCHAR(descr->prefix[i]))) - while (pos < strLen && isspace(UCHAR(str[pos]))) - ++pos; - else if (descr->prefix[i] != str[pos]) - break; - else - ++pos; - } - if (pos >= strLen) { - return -1; - } - else if (descr->prefix[i] != 0) - return 0; - } - - if (descr->type != 'c' && descr->type != '[' && descr->type != 'n') - while (isspace(UCHAR(str[pos]))) - ++pos; - - scanned = pos - anchor; - - - if (descr->type == 'n') { - - *valObjPtr = Jim_NewIntObj(interp, anchor + scanned); - } - else if (pos >= strLen) { - - return -1; - } - else if (descr->type == 'c') { - int c; - scanned += utf8_tounicode(&str[pos], &c); - *valObjPtr = Jim_NewIntObj(interp, c); - return scanned; - } - else { - - if (descr->width > 0) { - size_t sLen = utf8_strlen(&str[pos], strLen - pos); - size_t tLen = descr->width > sLen ? sLen : descr->width; - - tmpObj = Jim_NewStringObjUtf8(interp, str + pos, tLen); - tok = tmpObj->bytes; - } - else { - - tok = &str[pos]; - } - switch (descr->type) { - case 'd': - case 'o': - case 'x': - case 'u': - case 'i':{ - char *endp; - jim_wide w; - - int base = descr->type == 'o' ? 8 - : descr->type == 'x' ? 16 : descr->type == 'i' ? 0 : 10; - - - if (base == 0) { - w = jim_strtoull(tok, &endp); - } - else { - w = strtoull(tok, &endp, base); - } - - if (endp != tok) { - - *valObjPtr = Jim_NewIntObj(interp, w); - - - scanned += endp - tok; - } - else { - scanned = *tok ? 0 : -1; - } - break; - } - case 's': - case '[':{ - *valObjPtr = JimScanAString(interp, descr->arg, tok); - scanned += Jim_Length(*valObjPtr); - break; - } - case 'e': - case 'f': - case 'g':{ - char *endp; - double value = strtod(tok, &endp); - - if (endp != tok) { - - *valObjPtr = Jim_NewDoubleObj(interp, value); - - scanned += endp - tok; - } - else { - scanned = *tok ? 0 : -1; - } - break; - } - } - if (tmpObj) { - Jim_FreeNewObj(interp, tmpObj); - } - } - return scanned; -} - - -Jim_Obj *Jim_ScanString(Jim_Interp *interp, Jim_Obj *strObjPtr, Jim_Obj *fmtObjPtr, int flags) -{ - size_t i, pos; - int scanned = 1; - const char *str = Jim_String(strObjPtr); - int strLen = Jim_Utf8Length(interp, strObjPtr); - Jim_Obj *resultList = 0; - Jim_Obj **resultVec = 0; - int resultc; - Jim_Obj *emptyStr = 0; - ScanFmtStringObj *fmtObj; - - - JimPanic((fmtObjPtr->typePtr != &scanFmtStringObjType, "Jim_ScanString() for non-scan format")); - - fmtObj = (ScanFmtStringObj *) fmtObjPtr->internalRep.ptr; - - if (fmtObj->error != 0) { - if (flags & JIM_ERRMSG) - Jim_SetResultString(interp, fmtObj->error, -1); - return 0; - } - - emptyStr = Jim_NewEmptyStringObj(interp); - Jim_IncrRefCount(emptyStr); - - resultList = Jim_NewListObj(interp, NULL, 0); - if (fmtObj->maxPos > 0) { - for (i = 0; i < fmtObj->maxPos; ++i) - Jim_ListAppendElement(interp, resultList, emptyStr); - JimListGetElements(interp, resultList, &resultc, &resultVec); - } - - for (i = 0, pos = 0; i < fmtObj->count; ++i) { - ScanFmtPartDescr *descr = &(fmtObj->descr[i]); - Jim_Obj *value = 0; - - - if (descr->type == 0) - continue; - - if (scanned > 0) - scanned = ScanOneEntry(interp, str, pos, strLen, fmtObj, i, &value); - - if (scanned == -1 && i == 0) - goto eof; - - pos += scanned; - - - if (value == 0) - value = Jim_NewEmptyStringObj(interp); - - if (descr->pos == -1) { - Jim_FreeNewObj(interp, value); - } - else if (descr->pos == 0) - - Jim_ListAppendElement(interp, resultList, value); - else if (resultVec[descr->pos - 1] == emptyStr) { - - Jim_DecrRefCount(interp, resultVec[descr->pos - 1]); - Jim_IncrRefCount(value); - resultVec[descr->pos - 1] = value; - } - else { - - Jim_FreeNewObj(interp, value); - goto err; - } - } - Jim_DecrRefCount(interp, emptyStr); - return resultList; - eof: - Jim_DecrRefCount(interp, emptyStr); - Jim_FreeNewObj(interp, resultList); - return (Jim_Obj *)EOF; - err: - Jim_DecrRefCount(interp, emptyStr); - Jim_FreeNewObj(interp, resultList); - return 0; -} - - -static void JimPrngInit(Jim_Interp *interp) -{ -#define PRNG_SEED_SIZE 256 - int i; - unsigned int *seed; - time_t t = time(NULL); - - interp->prngState = Jim_Alloc(sizeof(Jim_PrngState)); - - seed = Jim_Alloc(PRNG_SEED_SIZE * sizeof(*seed)); - for (i = 0; i < PRNG_SEED_SIZE; i++) { - seed[i] = (rand() ^ t ^ clock()); - } - JimPrngSeed(interp, (unsigned char *)seed, PRNG_SEED_SIZE * sizeof(*seed)); - Jim_Free(seed); -} - - -static void JimRandomBytes(Jim_Interp *interp, void *dest, unsigned int len) -{ - Jim_PrngState *prng; - unsigned char *destByte = (unsigned char *)dest; - unsigned int si, sj, x; - - - if (interp->prngState == NULL) - JimPrngInit(interp); - prng = interp->prngState; - - for (x = 0; x < len; x++) { - prng->i = (prng->i + 1) & 0xff; - si = prng->sbox[prng->i]; - prng->j = (prng->j + si) & 0xff; - sj = prng->sbox[prng->j]; - prng->sbox[prng->i] = sj; - prng->sbox[prng->j] = si; - *destByte++ = prng->sbox[(si + sj) & 0xff]; - } -} - - -static void JimPrngSeed(Jim_Interp *interp, unsigned char *seed, int seedLen) -{ - int i; - Jim_PrngState *prng; - - - if (interp->prngState == NULL) - JimPrngInit(interp); - prng = interp->prngState; - - - for (i = 0; i < 256; i++) - prng->sbox[i] = i; - - for (i = 0; i < seedLen; i++) { - unsigned char t; - - t = prng->sbox[i & 0xFF]; - prng->sbox[i & 0xFF] = prng->sbox[seed[i]]; - prng->sbox[seed[i]] = t; - } - prng->i = prng->j = 0; - - for (i = 0; i < 256; i += seedLen) { - JimRandomBytes(interp, seed, seedLen); - } -} - - -static int Jim_IncrCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - jim_wide wideValue, increment = 1; - Jim_Obj *intObjPtr; - - if (argc != 2 && argc != 3) { - Jim_WrongNumArgs(interp, 1, argv, "varName ?increment?"); - return JIM_ERR; - } - if (argc == 3) { - if (Jim_GetWide(interp, argv[2], &increment) != JIM_OK) - return JIM_ERR; - } - intObjPtr = Jim_GetVariable(interp, argv[1], JIM_UNSHARED); - if (!intObjPtr) { - - wideValue = 0; - } - else if (Jim_GetWide(interp, intObjPtr, &wideValue) != JIM_OK) { - return JIM_ERR; - } - if (!intObjPtr || Jim_IsShared(intObjPtr)) { - intObjPtr = Jim_NewIntObj(interp, wideValue + increment); - if (Jim_SetVariable(interp, argv[1], intObjPtr) != JIM_OK) { - Jim_FreeNewObj(interp, intObjPtr); - return JIM_ERR; - } - } - else { - - Jim_InvalidateStringRep(intObjPtr); - JimWideValue(intObjPtr) = wideValue + increment; - - if (argv[1]->typePtr != &variableObjType) { - - Jim_SetVariable(interp, argv[1], intObjPtr); - } - } - Jim_SetResult(interp, intObjPtr); - return JIM_OK; -} - - -#define JIM_EVAL_SARGV_LEN 8 -#define JIM_EVAL_SINTV_LEN 8 - - -static int JimUnknown(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int retcode; - - if (interp->unknown_called > 50) { - return JIM_ERR; - } - - - - if (Jim_GetCommand(interp, interp->unknown, JIM_NONE) == NULL) - return JIM_ERR; - - interp->unknown_called++; - - retcode = Jim_EvalObjPrefix(interp, interp->unknown, argc, argv); - interp->unknown_called--; - - return retcode; -} - -static int JimInvokeCommand(Jim_Interp *interp, int objc, Jim_Obj *const *objv) -{ - int retcode; - Jim_Cmd *cmdPtr; - void *prevPrivData; - -#if 0 - printf("invoke"); - int j; - for (j = 0; j < objc; j++) { - printf(" '%s'", Jim_String(objv[j])); - } - printf("\n"); -#endif - - if (interp->framePtr->tailcallCmd) { - - cmdPtr = interp->framePtr->tailcallCmd; - interp->framePtr->tailcallCmd = NULL; - } - else { - cmdPtr = Jim_GetCommand(interp, objv[0], JIM_ERRMSG); - if (cmdPtr == NULL) { - return JimUnknown(interp, objc, objv); - } - JimIncrCmdRefCount(cmdPtr); - } - - if (interp->evalDepth == interp->maxEvalDepth) { - Jim_SetResultString(interp, "Infinite eval recursion", -1); - retcode = JIM_ERR; - goto out; - } - interp->evalDepth++; - prevPrivData = interp->cmdPrivData; - - - Jim_SetEmptyResult(interp); - if (cmdPtr->isproc) { - retcode = JimCallProcedure(interp, cmdPtr, objc, objv); - } - else { - interp->cmdPrivData = cmdPtr->u.native.privData; - retcode = cmdPtr->u.native.cmdProc(interp, objc, objv); - } - interp->cmdPrivData = prevPrivData; - interp->evalDepth--; - -out: - JimDecrCmdRefCount(interp, cmdPtr); - - return retcode; -} - -int Jim_EvalObjVector(Jim_Interp *interp, int objc, Jim_Obj *const *objv) -{ - int i, retcode; - - - for (i = 0; i < objc; i++) - Jim_IncrRefCount(objv[i]); - - retcode = JimInvokeCommand(interp, objc, objv); - - - for (i = 0; i < objc; i++) - Jim_DecrRefCount(interp, objv[i]); - - return retcode; -} - -int Jim_EvalObjPrefix(Jim_Interp *interp, Jim_Obj *prefix, int objc, Jim_Obj *const *objv) -{ - int ret; - Jim_Obj **nargv = Jim_Alloc((objc + 1) * sizeof(*nargv)); - - nargv[0] = prefix; - memcpy(&nargv[1], &objv[0], sizeof(nargv[0]) * objc); - ret = Jim_EvalObjVector(interp, objc + 1, nargv); - Jim_Free(nargv); - return ret; -} - -static void JimAddErrorToStack(Jim_Interp *interp, ScriptObj *script) -{ - if (!interp->errorFlag) { - - interp->errorFlag = 1; - Jim_IncrRefCount(script->fileNameObj); - Jim_DecrRefCount(interp, interp->errorFileNameObj); - interp->errorFileNameObj = script->fileNameObj; - interp->errorLine = script->linenr; - - JimResetStackTrace(interp); - - interp->addStackTrace++; - } - - - if (interp->addStackTrace > 0) { - - - JimAppendStackTrace(interp, Jim_String(interp->errorProc), script->fileNameObj, script->linenr); - - if (Jim_Length(script->fileNameObj)) { - interp->addStackTrace = 0; - } - - Jim_DecrRefCount(interp, interp->errorProc); - interp->errorProc = interp->emptyObj; - Jim_IncrRefCount(interp->errorProc); - } -} - -static int JimSubstOneToken(Jim_Interp *interp, const ScriptToken *token, Jim_Obj **objPtrPtr) -{ - Jim_Obj *objPtr; - - switch (token->type) { - case JIM_TT_STR: - case JIM_TT_ESC: - objPtr = token->objPtr; - break; - case JIM_TT_VAR: - objPtr = Jim_GetVariable(interp, token->objPtr, JIM_ERRMSG); - break; - case JIM_TT_DICTSUGAR: - objPtr = JimExpandDictSugar(interp, token->objPtr); - break; - case JIM_TT_EXPRSUGAR: - objPtr = JimExpandExprSugar(interp, token->objPtr); - break; - case JIM_TT_CMD: - switch (Jim_EvalObj(interp, token->objPtr)) { - case JIM_OK: - case JIM_RETURN: - objPtr = interp->result; - break; - case JIM_BREAK: - - return JIM_BREAK; - case JIM_CONTINUE: - - return JIM_CONTINUE; - default: - return JIM_ERR; - } - break; - default: - JimPanic((1, - "default token type (%d) reached " "in Jim_SubstObj().", token->type)); - objPtr = NULL; - break; - } - if (objPtr) { - *objPtrPtr = objPtr; - return JIM_OK; - } - return JIM_ERR; -} - -static Jim_Obj *JimInterpolateTokens(Jim_Interp *interp, const ScriptToken * token, int tokens, int flags) -{ - int totlen = 0, i; - Jim_Obj **intv; - Jim_Obj *sintv[JIM_EVAL_SINTV_LEN]; - Jim_Obj *objPtr; - char *s; - - if (tokens <= JIM_EVAL_SINTV_LEN) - intv = sintv; - else - intv = Jim_Alloc(sizeof(Jim_Obj *) * tokens); - - for (i = 0; i < tokens; i++) { - switch (JimSubstOneToken(interp, &token[i], &intv[i])) { - case JIM_OK: - case JIM_RETURN: - break; - case JIM_BREAK: - if (flags & JIM_SUBST_FLAG) { - - tokens = i; - continue; - } - - - case JIM_CONTINUE: - if (flags & JIM_SUBST_FLAG) { - intv[i] = NULL; - continue; - } - - - default: - while (i--) { - Jim_DecrRefCount(interp, intv[i]); - } - if (intv != sintv) { - Jim_Free(intv); - } - return NULL; - } - Jim_IncrRefCount(intv[i]); - Jim_String(intv[i]); - totlen += intv[i]->length; - } - - - if (tokens == 1 && intv[0] && intv == sintv) { - - intv[0]->refCount--; - return intv[0]; - } - - objPtr = Jim_NewStringObjNoAlloc(interp, NULL, 0); - - if (tokens == 4 && token[0].type == JIM_TT_ESC && token[1].type == JIM_TT_ESC - && token[2].type == JIM_TT_VAR) { - - objPtr->typePtr = &interpolatedObjType; - objPtr->internalRep.dictSubstValue.varNameObjPtr = token[0].objPtr; - objPtr->internalRep.dictSubstValue.indexObjPtr = intv[2]; - Jim_IncrRefCount(intv[2]); - } - else if (tokens && intv[0] && intv[0]->typePtr == &sourceObjType) { - - JimSetSourceInfo(interp, objPtr, intv[0]->internalRep.sourceValue.fileNameObj, intv[0]->internalRep.sourceValue.lineNumber); - } - - - s = objPtr->bytes = Jim_Alloc(totlen + 1); - objPtr->length = totlen; - for (i = 0; i < tokens; i++) { - if (intv[i]) { - memcpy(s, intv[i]->bytes, intv[i]->length); - s += intv[i]->length; - Jim_DecrRefCount(interp, intv[i]); - } - } - objPtr->bytes[totlen] = '\0'; - - if (intv != sintv) { - Jim_Free(intv); - } - - return objPtr; -} - - -static int JimEvalObjList(Jim_Interp *interp, Jim_Obj *listPtr) -{ - int retcode = JIM_OK; - - JimPanic((Jim_IsList(listPtr) == 0, "JimEvalObjList() invoked on non-list.")); - - if (listPtr->internalRep.listValue.len) { - Jim_IncrRefCount(listPtr); - retcode = JimInvokeCommand(interp, - listPtr->internalRep.listValue.len, - listPtr->internalRep.listValue.ele); - Jim_DecrRefCount(interp, listPtr); - } - return retcode; -} - -int Jim_EvalObjList(Jim_Interp *interp, Jim_Obj *listPtr) -{ - SetListFromAny(interp, listPtr); - return JimEvalObjList(interp, listPtr); -} - -int Jim_EvalObj(Jim_Interp *interp, Jim_Obj *scriptObjPtr) -{ - int i; - ScriptObj *script; - ScriptToken *token; - int retcode = JIM_OK; - Jim_Obj *sargv[JIM_EVAL_SARGV_LEN], **argv = NULL; - Jim_Obj *prevScriptObj; - - if (Jim_IsList(scriptObjPtr) && scriptObjPtr->bytes == NULL) { - return JimEvalObjList(interp, scriptObjPtr); - } - - Jim_IncrRefCount(scriptObjPtr); - script = JimGetScript(interp, scriptObjPtr); - if (!JimScriptValid(interp, script)) { - Jim_DecrRefCount(interp, scriptObjPtr); - return JIM_ERR; - } - - Jim_SetEmptyResult(interp); - - token = script->token; - -#ifdef JIM_OPTIMIZATION - if (script->len == 0) { - Jim_DecrRefCount(interp, scriptObjPtr); - return JIM_OK; - } - if (script->len == 3 - && token[1].objPtr->typePtr == &commandObjType - && token[1].objPtr->internalRep.cmdValue.cmdPtr->isproc == 0 - && token[1].objPtr->internalRep.cmdValue.cmdPtr->u.native.cmdProc == Jim_IncrCoreCommand - && token[2].objPtr->typePtr == &variableObjType) { - - Jim_Obj *objPtr = Jim_GetVariable(interp, token[2].objPtr, JIM_NONE); - - if (objPtr && !Jim_IsShared(objPtr) && objPtr->typePtr == &intObjType) { - JimWideValue(objPtr)++; - Jim_InvalidateStringRep(objPtr); - Jim_DecrRefCount(interp, scriptObjPtr); - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - } -#endif - - script->inUse++; - - - prevScriptObj = interp->currentScriptObj; - interp->currentScriptObj = scriptObjPtr; - - interp->errorFlag = 0; - argv = sargv; - - for (i = 0; i < script->len && retcode == JIM_OK; ) { - int argc; - int j; - - - argc = token[i].objPtr->internalRep.scriptLineValue.argc; - script->linenr = token[i].objPtr->internalRep.scriptLineValue.line; - - - if (argc > JIM_EVAL_SARGV_LEN) - argv = Jim_Alloc(sizeof(Jim_Obj *) * argc); - - - i++; - - for (j = 0; j < argc; j++) { - long wordtokens = 1; - int expand = 0; - Jim_Obj *wordObjPtr = NULL; - - if (token[i].type == JIM_TT_WORD) { - wordtokens = JimWideValue(token[i++].objPtr); - if (wordtokens < 0) { - expand = 1; - wordtokens = -wordtokens; - } - } - - if (wordtokens == 1) { - - switch (token[i].type) { - case JIM_TT_ESC: - case JIM_TT_STR: - wordObjPtr = token[i].objPtr; - break; - case JIM_TT_VAR: - wordObjPtr = Jim_GetVariable(interp, token[i].objPtr, JIM_ERRMSG); - break; - case JIM_TT_EXPRSUGAR: - wordObjPtr = JimExpandExprSugar(interp, token[i].objPtr); - break; - case JIM_TT_DICTSUGAR: - wordObjPtr = JimExpandDictSugar(interp, token[i].objPtr); - break; - case JIM_TT_CMD: - retcode = Jim_EvalObj(interp, token[i].objPtr); - if (retcode == JIM_OK) { - wordObjPtr = Jim_GetResult(interp); - } - break; - default: - JimPanic((1, "default token type reached " "in Jim_EvalObj().")); - } - } - else { - wordObjPtr = JimInterpolateTokens(interp, token + i, wordtokens, JIM_NONE); - } - - if (!wordObjPtr) { - if (retcode == JIM_OK) { - retcode = JIM_ERR; - } - break; - } - - Jim_IncrRefCount(wordObjPtr); - i += wordtokens; - - if (!expand) { - argv[j] = wordObjPtr; - } - else { - - int len = Jim_ListLength(interp, wordObjPtr); - int newargc = argc + len - 1; - int k; - - if (len > 1) { - if (argv == sargv) { - if (newargc > JIM_EVAL_SARGV_LEN) { - argv = Jim_Alloc(sizeof(*argv) * newargc); - memcpy(argv, sargv, sizeof(*argv) * j); - } - } - else { - - argv = Jim_Realloc(argv, sizeof(*argv) * newargc); - } - } - - - for (k = 0; k < len; k++) { - argv[j++] = wordObjPtr->internalRep.listValue.ele[k]; - Jim_IncrRefCount(wordObjPtr->internalRep.listValue.ele[k]); - } - - Jim_DecrRefCount(interp, wordObjPtr); - - - j--; - argc += len - 1; - } - } - - if (retcode == JIM_OK && argc) { - - retcode = JimInvokeCommand(interp, argc, argv); - - if (Jim_CheckSignal(interp)) { - retcode = JIM_SIGNAL; - } - } - - - while (j-- > 0) { - Jim_DecrRefCount(interp, argv[j]); - } - - if (argv != sargv) { - Jim_Free(argv); - argv = sargv; - } - } - - - if (retcode == JIM_ERR) { - JimAddErrorToStack(interp, script); - } - - else if (retcode != JIM_RETURN || interp->returnCode != JIM_ERR) { - - interp->addStackTrace = 0; - } - - - interp->currentScriptObj = prevScriptObj; - - Jim_FreeIntRep(interp, scriptObjPtr); - scriptObjPtr->typePtr = &scriptObjType; - Jim_SetIntRepPtr(scriptObjPtr, script); - Jim_DecrRefCount(interp, scriptObjPtr); - - return retcode; -} - -static int JimSetProcArg(Jim_Interp *interp, Jim_Obj *argNameObj, Jim_Obj *argValObj) -{ - int retcode; - - const char *varname = Jim_String(argNameObj); - if (*varname == '&') { - - Jim_Obj *objPtr; - Jim_CallFrame *savedCallFrame = interp->framePtr; - - interp->framePtr = interp->framePtr->parent; - objPtr = Jim_GetVariable(interp, argValObj, JIM_ERRMSG); - interp->framePtr = savedCallFrame; - if (!objPtr) { - return JIM_ERR; - } - - - objPtr = Jim_NewStringObj(interp, varname + 1, -1); - Jim_IncrRefCount(objPtr); - retcode = Jim_SetVariableLink(interp, objPtr, argValObj, interp->framePtr->parent); - Jim_DecrRefCount(interp, objPtr); - } - else { - retcode = Jim_SetVariable(interp, argNameObj, argValObj); - } - return retcode; -} - -static void JimSetProcWrongArgs(Jim_Interp *interp, Jim_Obj *procNameObj, Jim_Cmd *cmd) -{ - - Jim_Obj *argmsg = Jim_NewStringObj(interp, "", 0); - int i; - - for (i = 0; i < cmd->u.proc.argListLen; i++) { - Jim_AppendString(interp, argmsg, " ", 1); - - if (i == cmd->u.proc.argsPos) { - if (cmd->u.proc.arglist[i].defaultObjPtr) { - - Jim_AppendString(interp, argmsg, "?", 1); - Jim_AppendObj(interp, argmsg, cmd->u.proc.arglist[i].defaultObjPtr); - Jim_AppendString(interp, argmsg, " ...?", -1); - } - else { - - Jim_AppendString(interp, argmsg, "?arg...?", -1); - } - } - else { - if (cmd->u.proc.arglist[i].defaultObjPtr) { - Jim_AppendString(interp, argmsg, "?", 1); - Jim_AppendObj(interp, argmsg, cmd->u.proc.arglist[i].nameObjPtr); - Jim_AppendString(interp, argmsg, "?", 1); - } - else { - const char *arg = Jim_String(cmd->u.proc.arglist[i].nameObjPtr); - if (*arg == '&') { - arg++; - } - Jim_AppendString(interp, argmsg, arg, -1); - } - } - } - Jim_SetResultFormatted(interp, "wrong # args: should be \"%#s%#s\"", procNameObj, argmsg); -} - -#ifdef jim_ext_namespace -int Jim_EvalNamespace(Jim_Interp *interp, Jim_Obj *scriptObj, Jim_Obj *nsObj) -{ - Jim_CallFrame *callFramePtr; - int retcode; - - - callFramePtr = JimCreateCallFrame(interp, interp->framePtr, nsObj); - callFramePtr->argv = &interp->emptyObj; - callFramePtr->argc = 0; - callFramePtr->procArgsObjPtr = NULL; - callFramePtr->procBodyObjPtr = scriptObj; - callFramePtr->staticVars = NULL; - callFramePtr->fileNameObj = interp->emptyObj; - callFramePtr->line = 0; - Jim_IncrRefCount(scriptObj); - interp->framePtr = callFramePtr; - - - if (interp->framePtr->level == interp->maxCallFrameDepth) { - Jim_SetResultString(interp, "Too many nested calls. Infinite recursion?", -1); - retcode = JIM_ERR; - } - else { - - retcode = Jim_EvalObj(interp, scriptObj); - } - - - interp->framePtr = interp->framePtr->parent; - JimFreeCallFrame(interp, callFramePtr, JIM_FCF_REUSE); - - return retcode; -} -#endif - -static int JimCallProcedure(Jim_Interp *interp, Jim_Cmd *cmd, int argc, Jim_Obj *const *argv) -{ - Jim_CallFrame *callFramePtr; - int i, d, retcode, optargs; - ScriptObj *script; - - - if (argc - 1 < cmd->u.proc.reqArity || - (cmd->u.proc.argsPos < 0 && argc - 1 > cmd->u.proc.reqArity + cmd->u.proc.optArity)) { - JimSetProcWrongArgs(interp, argv[0], cmd); - return JIM_ERR; - } - - if (Jim_Length(cmd->u.proc.bodyObjPtr) == 0) { - - return JIM_OK; - } - - - if (interp->framePtr->level == interp->maxCallFrameDepth) { - Jim_SetResultString(interp, "Too many nested calls. Infinite recursion?", -1); - return JIM_ERR; - } - - - callFramePtr = JimCreateCallFrame(interp, interp->framePtr, cmd->u.proc.nsObj); - callFramePtr->argv = argv; - callFramePtr->argc = argc; - callFramePtr->procArgsObjPtr = cmd->u.proc.argListObjPtr; - callFramePtr->procBodyObjPtr = cmd->u.proc.bodyObjPtr; - callFramePtr->staticVars = cmd->u.proc.staticVars; - - - script = JimGetScript(interp, interp->currentScriptObj); - callFramePtr->fileNameObj = script->fileNameObj; - callFramePtr->line = script->linenr; - - Jim_IncrRefCount(cmd->u.proc.argListObjPtr); - Jim_IncrRefCount(cmd->u.proc.bodyObjPtr); - interp->framePtr = callFramePtr; - - - optargs = (argc - 1 - cmd->u.proc.reqArity); - - - i = 1; - for (d = 0; d < cmd->u.proc.argListLen; d++) { - Jim_Obj *nameObjPtr = cmd->u.proc.arglist[d].nameObjPtr; - if (d == cmd->u.proc.argsPos) { - - Jim_Obj *listObjPtr; - int argsLen = 0; - if (cmd->u.proc.reqArity + cmd->u.proc.optArity < argc - 1) { - argsLen = argc - 1 - (cmd->u.proc.reqArity + cmd->u.proc.optArity); - } - listObjPtr = Jim_NewListObj(interp, &argv[i], argsLen); - - - if (cmd->u.proc.arglist[d].defaultObjPtr) { - nameObjPtr =cmd->u.proc.arglist[d].defaultObjPtr; - } - retcode = Jim_SetVariable(interp, nameObjPtr, listObjPtr); - if (retcode != JIM_OK) { - goto badargset; - } - - i += argsLen; - continue; - } - - - if (cmd->u.proc.arglist[d].defaultObjPtr == NULL || optargs-- > 0) { - retcode = JimSetProcArg(interp, nameObjPtr, argv[i++]); - } - else { - - retcode = Jim_SetVariable(interp, nameObjPtr, cmd->u.proc.arglist[d].defaultObjPtr); - } - if (retcode != JIM_OK) { - goto badargset; - } - } - - - retcode = Jim_EvalObj(interp, cmd->u.proc.bodyObjPtr); - -badargset: - - - retcode = JimInvokeDefer(interp, retcode); - interp->framePtr = interp->framePtr->parent; - JimFreeCallFrame(interp, callFramePtr, JIM_FCF_REUSE); - - - if (interp->framePtr->tailcallObj) { - do { - Jim_Obj *tailcallObj = interp->framePtr->tailcallObj; - - interp->framePtr->tailcallObj = NULL; - - if (retcode == JIM_EVAL) { - retcode = Jim_EvalObjList(interp, tailcallObj); - if (retcode == JIM_RETURN) { - interp->returnLevel++; - } - } - Jim_DecrRefCount(interp, tailcallObj); - } while (interp->framePtr->tailcallObj); - - - if (interp->framePtr->tailcallCmd) { - JimDecrCmdRefCount(interp, interp->framePtr->tailcallCmd); - interp->framePtr->tailcallCmd = NULL; - } - } - - - if (retcode == JIM_RETURN) { - if (--interp->returnLevel <= 0) { - retcode = interp->returnCode; - interp->returnCode = JIM_OK; - interp->returnLevel = 0; - } - } - else if (retcode == JIM_ERR) { - interp->addStackTrace++; - Jim_DecrRefCount(interp, interp->errorProc); - interp->errorProc = argv[0]; - Jim_IncrRefCount(interp->errorProc); - } - - return retcode; -} - -int Jim_EvalSource(Jim_Interp *interp, const char *filename, int lineno, const char *script) -{ - int retval; - Jim_Obj *scriptObjPtr; - - scriptObjPtr = Jim_NewStringObj(interp, script, -1); - Jim_IncrRefCount(scriptObjPtr); - - if (filename) { - Jim_Obj *prevScriptObj; - - JimSetSourceInfo(interp, scriptObjPtr, Jim_NewStringObj(interp, filename, -1), lineno); - - prevScriptObj = interp->currentScriptObj; - interp->currentScriptObj = scriptObjPtr; - - retval = Jim_EvalObj(interp, scriptObjPtr); - - interp->currentScriptObj = prevScriptObj; - } - else { - retval = Jim_EvalObj(interp, scriptObjPtr); - } - Jim_DecrRefCount(interp, scriptObjPtr); - return retval; -} - -int Jim_Eval(Jim_Interp *interp, const char *script) -{ - return Jim_EvalObj(interp, Jim_NewStringObj(interp, script, -1)); -} - - -int Jim_EvalGlobal(Jim_Interp *interp, const char *script) -{ - int retval; - Jim_CallFrame *savedFramePtr = interp->framePtr; - - interp->framePtr = interp->topFramePtr; - retval = Jim_Eval(interp, script); - interp->framePtr = savedFramePtr; - - return retval; -} - -int Jim_EvalFileGlobal(Jim_Interp *interp, const char *filename) -{ - int retval; - Jim_CallFrame *savedFramePtr = interp->framePtr; - - interp->framePtr = interp->topFramePtr; - retval = Jim_EvalFile(interp, filename); - interp->framePtr = savedFramePtr; - - return retval; -} - -#include - -int Jim_EvalFile(Jim_Interp *interp, const char *filename) -{ - FILE *fp; - char *buf; - Jim_Obj *scriptObjPtr; - Jim_Obj *prevScriptObj; - struct stat sb; - int retcode; - int readlen; - - if (stat(filename, &sb) != 0 || (fp = fopen(filename, "rt")) == NULL) { - Jim_SetResultFormatted(interp, "couldn't read file \"%s\": %s", filename, strerror(errno)); - return JIM_ERR; - } - if (sb.st_size == 0) { - fclose(fp); - return JIM_OK; - } - - buf = Jim_Alloc(sb.st_size + 1); - readlen = fread(buf, 1, sb.st_size, fp); - if (ferror(fp)) { - fclose(fp); - Jim_Free(buf); - Jim_SetResultFormatted(interp, "failed to load file \"%s\": %s", filename, strerror(errno)); - return JIM_ERR; - } - fclose(fp); - buf[readlen] = 0; - - scriptObjPtr = Jim_NewStringObjNoAlloc(interp, buf, readlen); - JimSetSourceInfo(interp, scriptObjPtr, Jim_NewStringObj(interp, filename, -1), 1); - Jim_IncrRefCount(scriptObjPtr); - - prevScriptObj = interp->currentScriptObj; - interp->currentScriptObj = scriptObjPtr; - - retcode = Jim_EvalObj(interp, scriptObjPtr); - - - if (retcode == JIM_RETURN) { - if (--interp->returnLevel <= 0) { - retcode = interp->returnCode; - interp->returnCode = JIM_OK; - interp->returnLevel = 0; - } - } - if (retcode == JIM_ERR) { - - interp->addStackTrace++; - } - - interp->currentScriptObj = prevScriptObj; - - Jim_DecrRefCount(interp, scriptObjPtr); - - return retcode; -} - -static void JimParseSubst(struct JimParserCtx *pc, int flags) -{ - pc->tstart = pc->p; - pc->tline = pc->linenr; - - if (pc->len == 0) { - pc->tend = pc->p; - pc->tt = JIM_TT_EOL; - pc->eof = 1; - return; - } - if (*pc->p == '[' && !(flags & JIM_SUBST_NOCMD)) { - JimParseCmd(pc); - return; - } - if (*pc->p == '$' && !(flags & JIM_SUBST_NOVAR)) { - if (JimParseVar(pc) == JIM_OK) { - return; - } - - pc->tstart = pc->p; - flags |= JIM_SUBST_NOVAR; - } - while (pc->len) { - if (*pc->p == '$' && !(flags & JIM_SUBST_NOVAR)) { - break; - } - if (*pc->p == '[' && !(flags & JIM_SUBST_NOCMD)) { - break; - } - if (*pc->p == '\\' && pc->len > 1) { - pc->p++; - pc->len--; - } - pc->p++; - pc->len--; - } - pc->tend = pc->p - 1; - pc->tt = (flags & JIM_SUBST_NOESC) ? JIM_TT_STR : JIM_TT_ESC; -} - - -static int SetSubstFromAny(Jim_Interp *interp, struct Jim_Obj *objPtr, int flags) -{ - int scriptTextLen; - const char *scriptText = Jim_GetString(objPtr, &scriptTextLen); - struct JimParserCtx parser; - struct ScriptObj *script = Jim_Alloc(sizeof(*script)); - ParseTokenList tokenlist; - - - ScriptTokenListInit(&tokenlist); - - JimParserInit(&parser, scriptText, scriptTextLen, 1); - while (1) { - JimParseSubst(&parser, flags); - if (parser.eof) { - - break; - } - ScriptAddToken(&tokenlist, parser.tstart, parser.tend - parser.tstart + 1, parser.tt, - parser.tline); - } - - - script->inUse = 1; - script->substFlags = flags; - script->fileNameObj = interp->emptyObj; - Jim_IncrRefCount(script->fileNameObj); - SubstObjAddTokens(interp, script, &tokenlist); - - - ScriptTokenListFree(&tokenlist); - -#ifdef DEBUG_SHOW_SUBST - { - int i; - - printf("==== Subst ====\n"); - for (i = 0; i < script->len; i++) { - printf("[%2d] %s '%s'\n", i, jim_tt_name(script->token[i].type), - Jim_String(script->token[i].objPtr)); - } - } -#endif - - - Jim_FreeIntRep(interp, objPtr); - Jim_SetIntRepPtr(objPtr, script); - objPtr->typePtr = &scriptObjType; - return JIM_OK; -} - -static ScriptObj *Jim_GetSubst(Jim_Interp *interp, Jim_Obj *objPtr, int flags) -{ - if (objPtr->typePtr != &scriptObjType || ((ScriptObj *)Jim_GetIntRepPtr(objPtr))->substFlags != flags) - SetSubstFromAny(interp, objPtr, flags); - return (ScriptObj *) Jim_GetIntRepPtr(objPtr); -} - -int Jim_SubstObj(Jim_Interp *interp, Jim_Obj *substObjPtr, Jim_Obj **resObjPtrPtr, int flags) -{ - ScriptObj *script = Jim_GetSubst(interp, substObjPtr, flags); - - Jim_IncrRefCount(substObjPtr); - script->inUse++; - - *resObjPtrPtr = JimInterpolateTokens(interp, script->token, script->len, flags); - - script->inUse--; - Jim_DecrRefCount(interp, substObjPtr); - if (*resObjPtrPtr == NULL) { - return JIM_ERR; - } - return JIM_OK; -} - -void Jim_WrongNumArgs(Jim_Interp *interp, int argc, Jim_Obj *const *argv, const char *msg) -{ - Jim_Obj *objPtr; - Jim_Obj *listObjPtr; - - JimPanic((argc == 0, "Jim_WrongNumArgs() called with argc=0")); - - listObjPtr = Jim_NewListObj(interp, argv, argc); - - if (msg && *msg) { - Jim_ListAppendElement(interp, listObjPtr, Jim_NewStringObj(interp, msg, -1)); - } - Jim_IncrRefCount(listObjPtr); - objPtr = Jim_ListJoin(interp, listObjPtr, " ", 1); - Jim_DecrRefCount(interp, listObjPtr); - - Jim_SetResultFormatted(interp, "wrong # args: should be \"%#s\"", objPtr); -} - -typedef void JimHashtableIteratorCallbackType(Jim_Interp *interp, Jim_Obj *listObjPtr, - Jim_HashEntry *he, int type); - -#define JimTrivialMatch(pattern) (strpbrk((pattern), "*[?\\") == NULL) - -static Jim_Obj *JimHashtablePatternMatch(Jim_Interp *interp, Jim_HashTable *ht, Jim_Obj *patternObjPtr, - JimHashtableIteratorCallbackType *callback, int type) -{ - Jim_HashEntry *he; - Jim_Obj *listObjPtr = Jim_NewListObj(interp, NULL, 0); - - - if (patternObjPtr && JimTrivialMatch(Jim_String(patternObjPtr))) { - he = Jim_FindHashEntry(ht, Jim_String(patternObjPtr)); - if (he) { - callback(interp, listObjPtr, he, type); - } - } - else { - Jim_HashTableIterator htiter; - JimInitHashTableIterator(ht, &htiter); - while ((he = Jim_NextHashEntry(&htiter)) != NULL) { - if (patternObjPtr == NULL || JimGlobMatch(Jim_String(patternObjPtr), he->key, 0)) { - callback(interp, listObjPtr, he, type); - } - } - } - return listObjPtr; -} - - -#define JIM_CMDLIST_COMMANDS 0 -#define JIM_CMDLIST_PROCS 1 -#define JIM_CMDLIST_CHANNELS 2 - -static void JimCommandMatch(Jim_Interp *interp, Jim_Obj *listObjPtr, - Jim_HashEntry *he, int type) -{ - Jim_Cmd *cmdPtr = Jim_GetHashEntryVal(he); - Jim_Obj *objPtr; - - if (type == JIM_CMDLIST_PROCS && !cmdPtr->isproc) { - - return; - } - - objPtr = Jim_NewStringObj(interp, he->key, -1); - Jim_IncrRefCount(objPtr); - - if (type != JIM_CMDLIST_CHANNELS || Jim_AioFilehandle(interp, objPtr)) { - Jim_ListAppendElement(interp, listObjPtr, objPtr); - } - Jim_DecrRefCount(interp, objPtr); -} - - -static Jim_Obj *JimCommandsList(Jim_Interp *interp, Jim_Obj *patternObjPtr, int type) -{ - return JimHashtablePatternMatch(interp, &interp->commands, patternObjPtr, JimCommandMatch, type); -} - - -#define JIM_VARLIST_GLOBALS 0 -#define JIM_VARLIST_LOCALS 1 -#define JIM_VARLIST_VARS 2 - -#define JIM_VARLIST_VALUES 0x1000 - -static void JimVariablesMatch(Jim_Interp *interp, Jim_Obj *listObjPtr, - Jim_HashEntry *he, int type) -{ - Jim_Var *varPtr = Jim_GetHashEntryVal(he); - - if (type != JIM_VARLIST_LOCALS || varPtr->linkFramePtr == NULL) { - Jim_ListAppendElement(interp, listObjPtr, Jim_NewStringObj(interp, he->key, -1)); - if (type & JIM_VARLIST_VALUES) { - Jim_ListAppendElement(interp, listObjPtr, varPtr->objPtr); - } - } -} - - -static Jim_Obj *JimVariablesList(Jim_Interp *interp, Jim_Obj *patternObjPtr, int mode) -{ - if (mode == JIM_VARLIST_LOCALS && interp->framePtr == interp->topFramePtr) { - return interp->emptyObj; - } - else { - Jim_CallFrame *framePtr = (mode == JIM_VARLIST_GLOBALS) ? interp->topFramePtr : interp->framePtr; - return JimHashtablePatternMatch(interp, &framePtr->vars, patternObjPtr, JimVariablesMatch, mode); - } -} - -static int JimInfoLevel(Jim_Interp *interp, Jim_Obj *levelObjPtr, - Jim_Obj **objPtrPtr, int info_level_cmd) -{ - Jim_CallFrame *targetCallFrame; - - targetCallFrame = JimGetCallFrameByInteger(interp, levelObjPtr); - if (targetCallFrame == NULL) { - return JIM_ERR; - } - - if (targetCallFrame == interp->topFramePtr) { - Jim_SetResultFormatted(interp, "bad level \"%#s\"", levelObjPtr); - return JIM_ERR; - } - if (info_level_cmd) { - *objPtrPtr = Jim_NewListObj(interp, targetCallFrame->argv, targetCallFrame->argc); - } - else { - Jim_Obj *listObj = Jim_NewListObj(interp, NULL, 0); - - Jim_ListAppendElement(interp, listObj, targetCallFrame->argv[0]); - Jim_ListAppendElement(interp, listObj, targetCallFrame->fileNameObj); - Jim_ListAppendElement(interp, listObj, Jim_NewIntObj(interp, targetCallFrame->line)); - *objPtrPtr = listObj; - } - return JIM_OK; -} - - - -static int Jim_PutsCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc != 2 && argc != 3) { - Jim_WrongNumArgs(interp, 1, argv, "?-nonewline? string"); - return JIM_ERR; - } - if (argc == 3) { - if (!Jim_CompareStringImmediate(interp, argv[1], "-nonewline")) { - Jim_SetResultString(interp, "The second argument must " "be -nonewline", -1); - return JIM_ERR; - } - else { - fputs(Jim_String(argv[2]), stdout); - } - } - else { - puts(Jim_String(argv[1])); - } - return JIM_OK; -} - - -static int JimAddMulHelper(Jim_Interp *interp, int argc, Jim_Obj *const *argv, int op) -{ - jim_wide wideValue, res; - double doubleValue, doubleRes; - int i; - - res = (op == JIM_EXPROP_ADD) ? 0 : 1; - - for (i = 1; i < argc; i++) { - if (Jim_GetWide(interp, argv[i], &wideValue) != JIM_OK) - goto trydouble; - if (op == JIM_EXPROP_ADD) - res += wideValue; - else - res *= wideValue; - } - Jim_SetResultInt(interp, res); - return JIM_OK; - trydouble: - doubleRes = (double)res; - for (; i < argc; i++) { - if (Jim_GetDouble(interp, argv[i], &doubleValue) != JIM_OK) - return JIM_ERR; - if (op == JIM_EXPROP_ADD) - doubleRes += doubleValue; - else - doubleRes *= doubleValue; - } - Jim_SetResult(interp, Jim_NewDoubleObj(interp, doubleRes)); - return JIM_OK; -} - - -static int JimSubDivHelper(Jim_Interp *interp, int argc, Jim_Obj *const *argv, int op) -{ - jim_wide wideValue, res = 0; - double doubleValue, doubleRes = 0; - int i = 2; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "number ?number ... number?"); - return JIM_ERR; - } - else if (argc == 2) { - if (Jim_GetWide(interp, argv[1], &wideValue) != JIM_OK) { - if (Jim_GetDouble(interp, argv[1], &doubleValue) != JIM_OK) { - return JIM_ERR; - } - else { - if (op == JIM_EXPROP_SUB) - doubleRes = -doubleValue; - else - doubleRes = 1.0 / doubleValue; - Jim_SetResult(interp, Jim_NewDoubleObj(interp, doubleRes)); - return JIM_OK; - } - } - if (op == JIM_EXPROP_SUB) { - res = -wideValue; - Jim_SetResultInt(interp, res); - } - else { - doubleRes = 1.0 / wideValue; - Jim_SetResult(interp, Jim_NewDoubleObj(interp, doubleRes)); - } - return JIM_OK; - } - else { - if (Jim_GetWide(interp, argv[1], &res) != JIM_OK) { - if (Jim_GetDouble(interp, argv[1], &doubleRes) - != JIM_OK) { - return JIM_ERR; - } - else { - goto trydouble; - } - } - } - for (i = 2; i < argc; i++) { - if (Jim_GetWide(interp, argv[i], &wideValue) != JIM_OK) { - doubleRes = (double)res; - goto trydouble; - } - if (op == JIM_EXPROP_SUB) - res -= wideValue; - else { - if (wideValue == 0) { - Jim_SetResultString(interp, "Division by zero", -1); - return JIM_ERR; - } - res /= wideValue; - } - } - Jim_SetResultInt(interp, res); - return JIM_OK; - trydouble: - for (; i < argc; i++) { - if (Jim_GetDouble(interp, argv[i], &doubleValue) != JIM_OK) - return JIM_ERR; - if (op == JIM_EXPROP_SUB) - doubleRes -= doubleValue; - else - doubleRes /= doubleValue; - } - Jim_SetResult(interp, Jim_NewDoubleObj(interp, doubleRes)); - return JIM_OK; -} - - - -static int Jim_AddCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - return JimAddMulHelper(interp, argc, argv, JIM_EXPROP_ADD); -} - - -static int Jim_MulCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - return JimAddMulHelper(interp, argc, argv, JIM_EXPROP_MUL); -} - - -static int Jim_SubCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - return JimSubDivHelper(interp, argc, argv, JIM_EXPROP_SUB); -} - - -static int Jim_DivCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - return JimSubDivHelper(interp, argc, argv, JIM_EXPROP_DIV); -} - - -static int Jim_SetCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc != 2 && argc != 3) { - Jim_WrongNumArgs(interp, 1, argv, "varName ?newValue?"); - return JIM_ERR; - } - if (argc == 2) { - Jim_Obj *objPtr; - - objPtr = Jim_GetVariable(interp, argv[1], JIM_ERRMSG); - if (!objPtr) - return JIM_ERR; - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - - if (Jim_SetVariable(interp, argv[1], argv[2]) != JIM_OK) - return JIM_ERR; - Jim_SetResult(interp, argv[2]); - return JIM_OK; -} - -static int Jim_UnsetCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int i = 1; - int complain = 1; - - while (i < argc) { - if (Jim_CompareStringImmediate(interp, argv[i], "--")) { - i++; - break; - } - if (Jim_CompareStringImmediate(interp, argv[i], "-nocomplain")) { - complain = 0; - i++; - continue; - } - break; - } - - while (i < argc) { - if (Jim_UnsetVariable(interp, argv[i], complain ? JIM_ERRMSG : JIM_NONE) != JIM_OK - && complain) { - return JIM_ERR; - } - i++; - } - return JIM_OK; -} - - -static int Jim_WhileCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc != 3) { - Jim_WrongNumArgs(interp, 1, argv, "condition body"); - return JIM_ERR; - } - - - while (1) { - int boolean, retval; - - if ((retval = Jim_GetBoolFromExpr(interp, argv[1], &boolean)) != JIM_OK) - return retval; - if (!boolean) - break; - - if ((retval = Jim_EvalObj(interp, argv[2])) != JIM_OK) { - switch (retval) { - case JIM_BREAK: - goto out; - break; - case JIM_CONTINUE: - continue; - break; - default: - return retval; - } - } - } - out: - Jim_SetEmptyResult(interp); - return JIM_OK; -} - - -static int Jim_ForCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int retval; - int boolean = 1; - Jim_Obj *varNamePtr = NULL; - Jim_Obj *stopVarNamePtr = NULL; - - if (argc != 5) { - Jim_WrongNumArgs(interp, 1, argv, "start test next body"); - return JIM_ERR; - } - - - if ((retval = Jim_EvalObj(interp, argv[1])) != JIM_OK) { - return retval; - } - - retval = Jim_GetBoolFromExpr(interp, argv[2], &boolean); - - -#ifdef JIM_OPTIMIZATION - if (retval == JIM_OK && boolean) { - ScriptObj *incrScript; - struct ExprTree *expr; - jim_wide stop, currentVal; - Jim_Obj *objPtr; - int cmpOffset; - - - expr = JimGetExpression(interp, argv[2]); - incrScript = JimGetScript(interp, argv[3]); - - - if (incrScript == NULL || incrScript->len != 3 || !expr || expr->len != 3) { - goto evalstart; - } - - if (incrScript->token[1].type != JIM_TT_ESC) { - goto evalstart; - } - - if (expr->expr->type == JIM_EXPROP_LT) { - cmpOffset = 0; - } - else if (expr->expr->type == JIM_EXPROP_LTE) { - cmpOffset = 1; - } - else { - goto evalstart; - } - - if (expr->expr->left->type != JIM_TT_VAR) { - goto evalstart; - } - - if (expr->expr->right->type != JIM_TT_VAR && expr->expr->right->type != JIM_TT_EXPR_INT) { - goto evalstart; - } - - - if (!Jim_CompareStringImmediate(interp, incrScript->token[1].objPtr, "incr")) { - goto evalstart; - } - - - if (!Jim_StringEqObj(incrScript->token[2].objPtr, expr->expr->left->objPtr)) { - goto evalstart; - } - - - if (expr->expr->right->type == JIM_TT_EXPR_INT) { - if (Jim_GetWide(interp, expr->expr->right->objPtr, &stop) == JIM_ERR) { - goto evalstart; - } - } - else { - stopVarNamePtr = expr->expr->right->objPtr; - Jim_IncrRefCount(stopVarNamePtr); - - stop = 0; - } - - - varNamePtr = expr->expr->left->objPtr; - Jim_IncrRefCount(varNamePtr); - - objPtr = Jim_GetVariable(interp, varNamePtr, JIM_NONE); - if (objPtr == NULL || Jim_GetWide(interp, objPtr, ¤tVal) != JIM_OK) { - goto testcond; - } - - - while (retval == JIM_OK) { - - - - - if (stopVarNamePtr) { - objPtr = Jim_GetVariable(interp, stopVarNamePtr, JIM_NONE); - if (objPtr == NULL || Jim_GetWide(interp, objPtr, &stop) != JIM_OK) { - goto testcond; - } - } - - if (currentVal >= stop + cmpOffset) { - break; - } - - - retval = Jim_EvalObj(interp, argv[4]); - if (retval == JIM_OK || retval == JIM_CONTINUE) { - retval = JIM_OK; - - objPtr = Jim_GetVariable(interp, varNamePtr, JIM_ERRMSG); - - - if (objPtr == NULL) { - retval = JIM_ERR; - goto out; - } - if (!Jim_IsShared(objPtr) && objPtr->typePtr == &intObjType) { - currentVal = ++JimWideValue(objPtr); - Jim_InvalidateStringRep(objPtr); - } - else { - if (Jim_GetWide(interp, objPtr, ¤tVal) != JIM_OK || - Jim_SetVariable(interp, varNamePtr, Jim_NewIntObj(interp, - ++currentVal)) != JIM_OK) { - goto evalnext; - } - } - } - } - goto out; - } - evalstart: -#endif - - while (boolean && (retval == JIM_OK || retval == JIM_CONTINUE)) { - - retval = Jim_EvalObj(interp, argv[4]); - - if (retval == JIM_OK || retval == JIM_CONTINUE) { - -JIM_IF_OPTIM(evalnext:) - retval = Jim_EvalObj(interp, argv[3]); - if (retval == JIM_OK || retval == JIM_CONTINUE) { - -JIM_IF_OPTIM(testcond:) - retval = Jim_GetBoolFromExpr(interp, argv[2], &boolean); - } - } - } -JIM_IF_OPTIM(out:) - if (stopVarNamePtr) { - Jim_DecrRefCount(interp, stopVarNamePtr); - } - if (varNamePtr) { - Jim_DecrRefCount(interp, varNamePtr); - } - - if (retval == JIM_CONTINUE || retval == JIM_BREAK || retval == JIM_OK) { - Jim_SetEmptyResult(interp); - return JIM_OK; - } - - return retval; -} - - -static int Jim_LoopCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int retval; - jim_wide i; - jim_wide limit; - jim_wide incr = 1; - Jim_Obj *bodyObjPtr; - - if (argc != 5 && argc != 6) { - Jim_WrongNumArgs(interp, 1, argv, "var first limit ?incr? body"); - return JIM_ERR; - } - - if (Jim_GetWide(interp, argv[2], &i) != JIM_OK || - Jim_GetWide(interp, argv[3], &limit) != JIM_OK || - (argc == 6 && Jim_GetWide(interp, argv[4], &incr) != JIM_OK)) { - return JIM_ERR; - } - bodyObjPtr = (argc == 5) ? argv[4] : argv[5]; - - retval = Jim_SetVariable(interp, argv[1], argv[2]); - - while (((i < limit && incr > 0) || (i > limit && incr < 0)) && retval == JIM_OK) { - retval = Jim_EvalObj(interp, bodyObjPtr); - if (retval == JIM_OK || retval == JIM_CONTINUE) { - Jim_Obj *objPtr = Jim_GetVariable(interp, argv[1], JIM_ERRMSG); - - retval = JIM_OK; - - - i += incr; - - if (objPtr && !Jim_IsShared(objPtr) && objPtr->typePtr == &intObjType) { - if (argv[1]->typePtr != &variableObjType) { - if (Jim_SetVariable(interp, argv[1], objPtr) != JIM_OK) { - return JIM_ERR; - } - } - JimWideValue(objPtr) = i; - Jim_InvalidateStringRep(objPtr); - - if (argv[1]->typePtr != &variableObjType) { - if (Jim_SetVariable(interp, argv[1], objPtr) != JIM_OK) { - retval = JIM_ERR; - break; - } - } - } - else { - objPtr = Jim_NewIntObj(interp, i); - retval = Jim_SetVariable(interp, argv[1], objPtr); - if (retval != JIM_OK) { - Jim_FreeNewObj(interp, objPtr); - } - } - } - } - - if (retval == JIM_OK || retval == JIM_CONTINUE || retval == JIM_BREAK) { - Jim_SetEmptyResult(interp); - return JIM_OK; - } - return retval; -} - -typedef struct { - Jim_Obj *objPtr; - int idx; -} Jim_ListIter; - -static void JimListIterInit(Jim_ListIter *iter, Jim_Obj *objPtr) -{ - iter->objPtr = objPtr; - iter->idx = 0; -} - -static Jim_Obj *JimListIterNext(Jim_Interp *interp, Jim_ListIter *iter) -{ - if (iter->idx >= Jim_ListLength(interp, iter->objPtr)) { - return NULL; - } - return iter->objPtr->internalRep.listValue.ele[iter->idx++]; -} - -static int JimListIterDone(Jim_Interp *interp, Jim_ListIter *iter) -{ - return iter->idx >= Jim_ListLength(interp, iter->objPtr); -} - - -static int JimForeachMapHelper(Jim_Interp *interp, int argc, Jim_Obj *const *argv, int doMap) -{ - int result = JIM_OK; - int i, numargs; - Jim_ListIter twoiters[2]; - Jim_ListIter *iters; - Jim_Obj *script; - Jim_Obj *resultObj; - - if (argc < 4 || argc % 2 != 0) { - Jim_WrongNumArgs(interp, 1, argv, "varList list ?varList list ...? script"); - return JIM_ERR; - } - script = argv[argc - 1]; - numargs = (argc - 1 - 1); - - if (numargs == 2) { - iters = twoiters; - } - else { - iters = Jim_Alloc(numargs * sizeof(*iters)); - } - for (i = 0; i < numargs; i++) { - JimListIterInit(&iters[i], argv[i + 1]); - if (i % 2 == 0 && JimListIterDone(interp, &iters[i])) { - result = JIM_ERR; - } - } - if (result != JIM_OK) { - Jim_SetResultString(interp, "foreach varlist is empty", -1); - goto empty_varlist; - } - - if (doMap) { - resultObj = Jim_NewListObj(interp, NULL, 0); - } - else { - resultObj = interp->emptyObj; - } - Jim_IncrRefCount(resultObj); - - while (1) { - - for (i = 0; i < numargs; i += 2) { - if (!JimListIterDone(interp, &iters[i + 1])) { - break; - } - } - if (i == numargs) { - - break; - } - - - for (i = 0; i < numargs; i += 2) { - Jim_Obj *varName; - - - JimListIterInit(&iters[i], argv[i + 1]); - while ((varName = JimListIterNext(interp, &iters[i])) != NULL) { - Jim_Obj *valObj = JimListIterNext(interp, &iters[i + 1]); - if (!valObj) { - - valObj = interp->emptyObj; - } - - Jim_IncrRefCount(valObj); - result = Jim_SetVariable(interp, varName, valObj); - Jim_DecrRefCount(interp, valObj); - if (result != JIM_OK) { - goto err; - } - } - } - switch (result = Jim_EvalObj(interp, script)) { - case JIM_OK: - if (doMap) { - Jim_ListAppendElement(interp, resultObj, interp->result); - } - break; - case JIM_CONTINUE: - break; - case JIM_BREAK: - goto out; - default: - goto err; - } - } - out: - result = JIM_OK; - Jim_SetResult(interp, resultObj); - err: - Jim_DecrRefCount(interp, resultObj); - empty_varlist: - if (numargs > 2) { - Jim_Free(iters); - } - return result; -} - - -static int Jim_ForeachCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - return JimForeachMapHelper(interp, argc, argv, 0); -} - - -static int Jim_LmapCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - return JimForeachMapHelper(interp, argc, argv, 1); -} - - -static int Jim_LassignCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int result = JIM_ERR; - int i; - Jim_ListIter iter; - Jim_Obj *resultObj; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "varList list ?varName ...?"); - return JIM_ERR; - } - - JimListIterInit(&iter, argv[1]); - - for (i = 2; i < argc; i++) { - Jim_Obj *valObj = JimListIterNext(interp, &iter); - result = Jim_SetVariable(interp, argv[i], valObj ? valObj : interp->emptyObj); - if (result != JIM_OK) { - return result; - } - } - - resultObj = Jim_NewListObj(interp, NULL, 0); - while (!JimListIterDone(interp, &iter)) { - Jim_ListAppendElement(interp, resultObj, JimListIterNext(interp, &iter)); - } - - Jim_SetResult(interp, resultObj); - - return JIM_OK; -} - - -static int Jim_IfCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int boolean, retval, current = 1, falsebody = 0; - - if (argc >= 3) { - while (1) { - - if (current >= argc) - goto err; - if ((retval = Jim_GetBoolFromExpr(interp, argv[current++], &boolean)) - != JIM_OK) - return retval; - - if (current >= argc) - goto err; - if (Jim_CompareStringImmediate(interp, argv[current], "then")) - current++; - - if (current >= argc) - goto err; - if (boolean) - return Jim_EvalObj(interp, argv[current]); - - if (++current >= argc) { - Jim_SetResult(interp, Jim_NewEmptyStringObj(interp)); - return JIM_OK; - } - falsebody = current++; - if (Jim_CompareStringImmediate(interp, argv[falsebody], "else")) { - - if (current != argc - 1) - goto err; - return Jim_EvalObj(interp, argv[current]); - } - else if (Jim_CompareStringImmediate(interp, argv[falsebody], "elseif")) - continue; - - else if (falsebody != argc - 1) - goto err; - return Jim_EvalObj(interp, argv[falsebody]); - } - return JIM_OK; - } - err: - Jim_WrongNumArgs(interp, 1, argv, "condition ?then? trueBody ?elseif ...? ?else? falseBody"); - return JIM_ERR; -} - - - -int Jim_CommandMatchObj(Jim_Interp *interp, Jim_Obj *commandObj, Jim_Obj *patternObj, - Jim_Obj *stringObj, int nocase) -{ - Jim_Obj *parms[4]; - int argc = 0; - long eq; - int rc; - - parms[argc++] = commandObj; - if (nocase) { - parms[argc++] = Jim_NewStringObj(interp, "-nocase", -1); - } - parms[argc++] = patternObj; - parms[argc++] = stringObj; - - rc = Jim_EvalObjVector(interp, argc, parms); - - if (rc != JIM_OK || Jim_GetLong(interp, Jim_GetResult(interp), &eq) != JIM_OK) { - eq = -rc; - } - - return eq; -} - - -static int Jim_SwitchCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - enum { SWITCH_EXACT, SWITCH_GLOB, SWITCH_RE, SWITCH_CMD }; - int matchOpt = SWITCH_EXACT, opt = 1, patCount, i; - Jim_Obj *command = NULL, *scriptObj = NULL, *strObj; - Jim_Obj **caseList; - - if (argc < 3) { - wrongnumargs: - Jim_WrongNumArgs(interp, 1, argv, "?options? string " - "pattern body ... ?default body? or " "{pattern body ?pattern body ...?}"); - return JIM_ERR; - } - for (opt = 1; opt < argc; ++opt) { - const char *option = Jim_String(argv[opt]); - - if (*option != '-') - break; - else if (strncmp(option, "--", 2) == 0) { - ++opt; - break; - } - else if (strncmp(option, "-exact", 2) == 0) - matchOpt = SWITCH_EXACT; - else if (strncmp(option, "-glob", 2) == 0) - matchOpt = SWITCH_GLOB; - else if (strncmp(option, "-regexp", 2) == 0) - matchOpt = SWITCH_RE; - else if (strncmp(option, "-command", 2) == 0) { - matchOpt = SWITCH_CMD; - if ((argc - opt) < 2) - goto wrongnumargs; - command = argv[++opt]; - } - else { - Jim_SetResultFormatted(interp, - "bad option \"%#s\": must be -exact, -glob, -regexp, -command procname or --", - argv[opt]); - return JIM_ERR; - } - if ((argc - opt) < 2) - goto wrongnumargs; - } - strObj = argv[opt++]; - patCount = argc - opt; - if (patCount == 1) { - JimListGetElements(interp, argv[opt], &patCount, &caseList); - } - else - caseList = (Jim_Obj **)&argv[opt]; - if (patCount == 0 || patCount % 2 != 0) - goto wrongnumargs; - for (i = 0; scriptObj == NULL && i < patCount; i += 2) { - Jim_Obj *patObj = caseList[i]; - - if (!Jim_CompareStringImmediate(interp, patObj, "default") - || i < (patCount - 2)) { - switch (matchOpt) { - case SWITCH_EXACT: - if (Jim_StringEqObj(strObj, patObj)) - scriptObj = caseList[i + 1]; - break; - case SWITCH_GLOB: - if (Jim_StringMatchObj(interp, patObj, strObj, 0)) - scriptObj = caseList[i + 1]; - break; - case SWITCH_RE: - command = Jim_NewStringObj(interp, "regexp", -1); - - case SWITCH_CMD:{ - int rc = Jim_CommandMatchObj(interp, command, patObj, strObj, 0); - - if (argc - opt == 1) { - JimListGetElements(interp, argv[opt], &patCount, &caseList); - } - - if (rc < 0) { - return -rc; - } - if (rc) - scriptObj = caseList[i + 1]; - break; - } - } - } - else { - scriptObj = caseList[i + 1]; - } - } - for (; i < patCount && Jim_CompareStringImmediate(interp, scriptObj, "-"); i += 2) - scriptObj = caseList[i + 1]; - if (scriptObj && Jim_CompareStringImmediate(interp, scriptObj, "-")) { - Jim_SetResultFormatted(interp, "no body specified for pattern \"%#s\"", caseList[i - 2]); - return JIM_ERR; - } - Jim_SetEmptyResult(interp); - if (scriptObj) { - return Jim_EvalObj(interp, scriptObj); - } - return JIM_OK; -} - - -static int Jim_ListCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *listObjPtr; - - listObjPtr = Jim_NewListObj(interp, argv + 1, argc - 1); - Jim_SetResult(interp, listObjPtr); - return JIM_OK; -} - - -static int Jim_LindexCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *objPtr, *listObjPtr; - int i; - int idx; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "list ?index ...?"); - return JIM_ERR; - } - objPtr = argv[1]; - Jim_IncrRefCount(objPtr); - for (i = 2; i < argc; i++) { - listObjPtr = objPtr; - if (Jim_GetIndex(interp, argv[i], &idx) != JIM_OK) { - Jim_DecrRefCount(interp, listObjPtr); - return JIM_ERR; - } - if (Jim_ListIndex(interp, listObjPtr, idx, &objPtr, JIM_NONE) != JIM_OK) { - Jim_DecrRefCount(interp, listObjPtr); - Jim_SetEmptyResult(interp); - return JIM_OK; - } - Jim_IncrRefCount(objPtr); - Jim_DecrRefCount(interp, listObjPtr); - } - Jim_SetResult(interp, objPtr); - Jim_DecrRefCount(interp, objPtr); - return JIM_OK; -} - - -static int Jim_LlengthCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc != 2) { - Jim_WrongNumArgs(interp, 1, argv, "list"); - return JIM_ERR; - } - Jim_SetResultInt(interp, Jim_ListLength(interp, argv[1])); - return JIM_OK; -} - - -static int Jim_LsearchCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - static const char * const options[] = { - "-bool", "-not", "-nocase", "-exact", "-glob", "-regexp", "-all", "-inline", "-command", - NULL - }; - enum - { OPT_BOOL, OPT_NOT, OPT_NOCASE, OPT_EXACT, OPT_GLOB, OPT_REGEXP, OPT_ALL, OPT_INLINE, - OPT_COMMAND }; - int i; - int opt_bool = 0; - int opt_not = 0; - int opt_nocase = 0; - int opt_all = 0; - int opt_inline = 0; - int opt_match = OPT_EXACT; - int listlen; - int rc = JIM_OK; - Jim_Obj *listObjPtr = NULL; - Jim_Obj *commandObj = NULL; - - if (argc < 3) { - wrongargs: - Jim_WrongNumArgs(interp, 1, argv, - "?-exact|-glob|-regexp|-command 'command'? ?-bool|-inline? ?-not? ?-nocase? ?-all? list value"); - return JIM_ERR; - } - - for (i = 1; i < argc - 2; i++) { - int option; - - if (Jim_GetEnum(interp, argv[i], options, &option, NULL, JIM_ERRMSG) != JIM_OK) { - return JIM_ERR; - } - switch (option) { - case OPT_BOOL: - opt_bool = 1; - opt_inline = 0; - break; - case OPT_NOT: - opt_not = 1; - break; - case OPT_NOCASE: - opt_nocase = 1; - break; - case OPT_INLINE: - opt_inline = 1; - opt_bool = 0; - break; - case OPT_ALL: - opt_all = 1; - break; - case OPT_COMMAND: - if (i >= argc - 2) { - goto wrongargs; - } - commandObj = argv[++i]; - - case OPT_EXACT: - case OPT_GLOB: - case OPT_REGEXP: - opt_match = option; - break; - } - } - - argv += i; - - if (opt_all) { - listObjPtr = Jim_NewListObj(interp, NULL, 0); - } - if (opt_match == OPT_REGEXP) { - commandObj = Jim_NewStringObj(interp, "regexp", -1); - } - if (commandObj) { - Jim_IncrRefCount(commandObj); - } - - listlen = Jim_ListLength(interp, argv[0]); - for (i = 0; i < listlen; i++) { - int eq = 0; - Jim_Obj *objPtr = Jim_ListGetIndex(interp, argv[0], i); - - switch (opt_match) { - case OPT_EXACT: - eq = Jim_StringCompareObj(interp, argv[1], objPtr, opt_nocase) == 0; - break; - - case OPT_GLOB: - eq = Jim_StringMatchObj(interp, argv[1], objPtr, opt_nocase); - break; - - case OPT_REGEXP: - case OPT_COMMAND: - eq = Jim_CommandMatchObj(interp, commandObj, argv[1], objPtr, opt_nocase); - if (eq < 0) { - if (listObjPtr) { - Jim_FreeNewObj(interp, listObjPtr); - } - rc = JIM_ERR; - goto done; - } - break; - } - - - if (!eq && opt_bool && opt_not && !opt_all) { - continue; - } - - if ((!opt_bool && eq == !opt_not) || (opt_bool && (eq || opt_all))) { - - Jim_Obj *resultObj; - - if (opt_bool) { - resultObj = Jim_NewIntObj(interp, eq ^ opt_not); - } - else if (!opt_inline) { - resultObj = Jim_NewIntObj(interp, i); - } - else { - resultObj = objPtr; - } - - if (opt_all) { - Jim_ListAppendElement(interp, listObjPtr, resultObj); - } - else { - Jim_SetResult(interp, resultObj); - goto done; - } - } - } - - if (opt_all) { - Jim_SetResult(interp, listObjPtr); - } - else { - - if (opt_bool) { - Jim_SetResultBool(interp, opt_not); - } - else if (!opt_inline) { - Jim_SetResultInt(interp, -1); - } - } - - done: - if (commandObj) { - Jim_DecrRefCount(interp, commandObj); - } - return rc; -} - - -static int Jim_LappendCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *listObjPtr; - int new_obj = 0; - int i; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "varName ?value value ...?"); - return JIM_ERR; - } - listObjPtr = Jim_GetVariable(interp, argv[1], JIM_UNSHARED); - if (!listObjPtr) { - - listObjPtr = Jim_NewListObj(interp, NULL, 0); - new_obj = 1; - } - else if (Jim_IsShared(listObjPtr)) { - listObjPtr = Jim_DuplicateObj(interp, listObjPtr); - new_obj = 1; - } - for (i = 2; i < argc; i++) - Jim_ListAppendElement(interp, listObjPtr, argv[i]); - if (Jim_SetVariable(interp, argv[1], listObjPtr) != JIM_OK) { - if (new_obj) - Jim_FreeNewObj(interp, listObjPtr); - return JIM_ERR; - } - Jim_SetResult(interp, listObjPtr); - return JIM_OK; -} - - -static int Jim_LinsertCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int idx, len; - Jim_Obj *listPtr; - - if (argc < 3) { - Jim_WrongNumArgs(interp, 1, argv, "list index ?element ...?"); - return JIM_ERR; - } - listPtr = argv[1]; - if (Jim_IsShared(listPtr)) - listPtr = Jim_DuplicateObj(interp, listPtr); - if (Jim_GetIndex(interp, argv[2], &idx) != JIM_OK) - goto err; - len = Jim_ListLength(interp, listPtr); - if (idx >= len) - idx = len; - else if (idx < 0) - idx = len + idx + 1; - Jim_ListInsertElements(interp, listPtr, idx, argc - 3, &argv[3]); - Jim_SetResult(interp, listPtr); - return JIM_OK; - err: - if (listPtr != argv[1]) { - Jim_FreeNewObj(interp, listPtr); - } - return JIM_ERR; -} - - -static int Jim_LreplaceCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int first, last, len, rangeLen; - Jim_Obj *listObj; - Jim_Obj *newListObj; - - if (argc < 4) { - Jim_WrongNumArgs(interp, 1, argv, "list first last ?element ...?"); - return JIM_ERR; - } - if (Jim_GetIndex(interp, argv[2], &first) != JIM_OK || - Jim_GetIndex(interp, argv[3], &last) != JIM_OK) { - return JIM_ERR; - } - - listObj = argv[1]; - len = Jim_ListLength(interp, listObj); - - first = JimRelToAbsIndex(len, first); - last = JimRelToAbsIndex(len, last); - JimRelToAbsRange(len, &first, &last, &rangeLen); - - - if (first > len) { - first = len; - } - - - newListObj = Jim_NewListObj(interp, listObj->internalRep.listValue.ele, first); - - - ListInsertElements(newListObj, -1, argc - 4, argv + 4); - - - ListInsertElements(newListObj, -1, len - first - rangeLen, listObj->internalRep.listValue.ele + first + rangeLen); - - Jim_SetResult(interp, newListObj); - return JIM_OK; -} - - -static int Jim_LsetCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc < 3) { - Jim_WrongNumArgs(interp, 1, argv, "listVar ?index...? newVal"); - return JIM_ERR; - } - else if (argc == 3) { - - if (Jim_SetVariable(interp, argv[1], argv[2]) != JIM_OK) - return JIM_ERR; - Jim_SetResult(interp, argv[2]); - return JIM_OK; - } - return Jim_ListSetIndex(interp, argv[1], argv + 2, argc - 3, argv[argc - 1]); -} - - -static int Jim_LsortCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const argv[]) -{ - static const char * const options[] = { - "-ascii", "-nocase", "-increasing", "-decreasing", "-command", "-integer", "-real", "-index", "-unique", NULL - }; - enum - { OPT_ASCII, OPT_NOCASE, OPT_INCREASING, OPT_DECREASING, OPT_COMMAND, OPT_INTEGER, OPT_REAL, OPT_INDEX, OPT_UNIQUE }; - Jim_Obj *resObj; - int i; - int retCode; - int shared; - - struct lsort_info info; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "?options? list"); - return JIM_ERR; - } - - info.type = JIM_LSORT_ASCII; - info.order = 1; - info.indexed = 0; - info.unique = 0; - info.command = NULL; - info.interp = interp; - - for (i = 1; i < (argc - 1); i++) { - int option; - - if (Jim_GetEnum(interp, argv[i], options, &option, NULL, JIM_ENUM_ABBREV | JIM_ERRMSG) - != JIM_OK) - return JIM_ERR; - switch (option) { - case OPT_ASCII: - info.type = JIM_LSORT_ASCII; - break; - case OPT_NOCASE: - info.type = JIM_LSORT_NOCASE; - break; - case OPT_INTEGER: - info.type = JIM_LSORT_INTEGER; - break; - case OPT_REAL: - info.type = JIM_LSORT_REAL; - break; - case OPT_INCREASING: - info.order = 1; - break; - case OPT_DECREASING: - info.order = -1; - break; - case OPT_UNIQUE: - info.unique = 1; - break; - case OPT_COMMAND: - if (i >= (argc - 2)) { - Jim_SetResultString(interp, "\"-command\" option must be followed by comparison command", -1); - return JIM_ERR; - } - info.type = JIM_LSORT_COMMAND; - info.command = argv[i + 1]; - i++; - break; - case OPT_INDEX: - if (i >= (argc - 2)) { - Jim_SetResultString(interp, "\"-index\" option must be followed by list index", -1); - return JIM_ERR; - } - if (Jim_GetIndex(interp, argv[i + 1], &info.index) != JIM_OK) { - return JIM_ERR; - } - info.indexed = 1; - i++; - break; - } - } - resObj = argv[argc - 1]; - if ((shared = Jim_IsShared(resObj))) - resObj = Jim_DuplicateObj(interp, resObj); - retCode = ListSortElements(interp, resObj, &info); - if (retCode == JIM_OK) { - Jim_SetResult(interp, resObj); - } - else if (shared) { - Jim_FreeNewObj(interp, resObj); - } - return retCode; -} - - -static int Jim_AppendCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *stringObjPtr; - int i; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "varName ?value ...?"); - return JIM_ERR; - } - if (argc == 2) { - stringObjPtr = Jim_GetVariable(interp, argv[1], JIM_ERRMSG); - if (!stringObjPtr) - return JIM_ERR; - } - else { - int new_obj = 0; - stringObjPtr = Jim_GetVariable(interp, argv[1], JIM_UNSHARED); - if (!stringObjPtr) { - - stringObjPtr = Jim_NewEmptyStringObj(interp); - new_obj = 1; - } - else if (Jim_IsShared(stringObjPtr)) { - new_obj = 1; - stringObjPtr = Jim_DuplicateObj(interp, stringObjPtr); - } - for (i = 2; i < argc; i++) { - Jim_AppendObj(interp, stringObjPtr, argv[i]); - } - if (Jim_SetVariable(interp, argv[1], stringObjPtr) != JIM_OK) { - if (new_obj) { - Jim_FreeNewObj(interp, stringObjPtr); - } - return JIM_ERR; - } - } - Jim_SetResult(interp, stringObjPtr); - return JIM_OK; -} - - - -static int Jim_DebugCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ -#if !defined(JIM_DEBUG_COMMAND) - Jim_SetResultString(interp, "unsupported", -1); - return JIM_ERR; -#endif -} - - -static int Jim_EvalCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int rc; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "arg ?arg ...?"); - return JIM_ERR; - } - - if (argc == 2) { - rc = Jim_EvalObj(interp, argv[1]); - } - else { - rc = Jim_EvalObj(interp, Jim_ConcatObj(interp, argc - 1, argv + 1)); - } - - if (rc == JIM_ERR) { - - interp->addStackTrace++; - } - return rc; -} - - -static int Jim_UplevelCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc >= 2) { - int retcode; - Jim_CallFrame *savedCallFrame, *targetCallFrame; - const char *str; - - - savedCallFrame = interp->framePtr; - - - str = Jim_String(argv[1]); - if ((str[0] >= '0' && str[0] <= '9') || str[0] == '#') { - targetCallFrame = Jim_GetCallFrameByLevel(interp, argv[1]); - argc--; - argv++; - } - else { - targetCallFrame = Jim_GetCallFrameByLevel(interp, NULL); - } - if (targetCallFrame == NULL) { - return JIM_ERR; - } - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv - 1, "?level? command ?arg ...?"); - return JIM_ERR; - } - - interp->framePtr = targetCallFrame; - if (argc == 2) { - retcode = Jim_EvalObj(interp, argv[1]); - } - else { - retcode = Jim_EvalObj(interp, Jim_ConcatObj(interp, argc - 1, argv + 1)); - } - interp->framePtr = savedCallFrame; - return retcode; - } - else { - Jim_WrongNumArgs(interp, 1, argv, "?level? command ?arg ...?"); - return JIM_ERR; - } -} - - -static int Jim_ExprCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int retcode; - - if (argc == 2) { - retcode = Jim_EvalExpression(interp, argv[1]); - } - else if (argc > 2) { - Jim_Obj *objPtr; - - objPtr = Jim_ConcatObj(interp, argc - 1, argv + 1); - Jim_IncrRefCount(objPtr); - retcode = Jim_EvalExpression(interp, objPtr); - Jim_DecrRefCount(interp, objPtr); - } - else { - Jim_WrongNumArgs(interp, 1, argv, "expression ?...?"); - return JIM_ERR; - } - if (retcode != JIM_OK) - return retcode; - return JIM_OK; -} - - -static int Jim_BreakCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc != 1) { - Jim_WrongNumArgs(interp, 1, argv, ""); - return JIM_ERR; - } - return JIM_BREAK; -} - - -static int Jim_ContinueCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc != 1) { - Jim_WrongNumArgs(interp, 1, argv, ""); - return JIM_ERR; - } - return JIM_CONTINUE; -} - - -static int Jim_ReturnCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int i; - Jim_Obj *stackTraceObj = NULL; - Jim_Obj *errorCodeObj = NULL; - int returnCode = JIM_OK; - long level = 1; - - for (i = 1; i < argc - 1; i += 2) { - if (Jim_CompareStringImmediate(interp, argv[i], "-code")) { - if (Jim_GetReturnCode(interp, argv[i + 1], &returnCode) == JIM_ERR) { - return JIM_ERR; - } - } - else if (Jim_CompareStringImmediate(interp, argv[i], "-errorinfo")) { - stackTraceObj = argv[i + 1]; - } - else if (Jim_CompareStringImmediate(interp, argv[i], "-errorcode")) { - errorCodeObj = argv[i + 1]; - } - else if (Jim_CompareStringImmediate(interp, argv[i], "-level")) { - if (Jim_GetLong(interp, argv[i + 1], &level) != JIM_OK || level < 0) { - Jim_SetResultFormatted(interp, "bad level \"%#s\"", argv[i + 1]); - return JIM_ERR; - } - } - else { - break; - } - } - - if (i != argc - 1 && i != argc) { - Jim_WrongNumArgs(interp, 1, argv, - "?-code code? ?-errorinfo stacktrace? ?-level level? ?result?"); - } - - - if (stackTraceObj && returnCode == JIM_ERR) { - JimSetStackTrace(interp, stackTraceObj); - } - - if (errorCodeObj && returnCode == JIM_ERR) { - Jim_SetGlobalVariableStr(interp, "errorCode", errorCodeObj); - } - interp->returnCode = returnCode; - interp->returnLevel = level; - - if (i == argc - 1) { - Jim_SetResult(interp, argv[i]); - } - return JIM_RETURN; -} - - -static int Jim_TailcallCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (interp->framePtr->level == 0) { - Jim_SetResultString(interp, "tailcall can only be called from a proc or lambda", -1); - return JIM_ERR; - } - else if (argc >= 2) { - - Jim_CallFrame *cf = interp->framePtr->parent; - - Jim_Cmd *cmdPtr = Jim_GetCommand(interp, argv[1], JIM_ERRMSG); - if (cmdPtr == NULL) { - return JIM_ERR; - } - - JimPanic((cf->tailcallCmd != NULL, "Already have a tailcallCmd")); - - - JimIncrCmdRefCount(cmdPtr); - cf->tailcallCmd = cmdPtr; - - - JimPanic((cf->tailcallObj != NULL, "Already have a tailcallobj")); - - cf->tailcallObj = Jim_NewListObj(interp, argv + 1, argc - 1); - Jim_IncrRefCount(cf->tailcallObj); - - - return JIM_EVAL; - } - return JIM_OK; -} - -static int JimAliasCmd(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *cmdList; - Jim_Obj *prefixListObj = Jim_CmdPrivData(interp); - - - cmdList = Jim_DuplicateObj(interp, prefixListObj); - Jim_ListInsertElements(interp, cmdList, Jim_ListLength(interp, cmdList), argc - 1, argv + 1); - - return JimEvalObjList(interp, cmdList); -} - -static void JimAliasCmdDelete(Jim_Interp *interp, void *privData) -{ - Jim_Obj *prefixListObj = privData; - Jim_DecrRefCount(interp, prefixListObj); -} - -static int Jim_AliasCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *prefixListObj; - const char *newname; - - if (argc < 3) { - Jim_WrongNumArgs(interp, 1, argv, "newname command ?args ...?"); - return JIM_ERR; - } - - prefixListObj = Jim_NewListObj(interp, argv + 2, argc - 2); - Jim_IncrRefCount(prefixListObj); - newname = Jim_String(argv[1]); - if (newname[0] == ':' && newname[1] == ':') { - while (*++newname == ':') { - } - } - - Jim_SetResult(interp, argv[1]); - - return Jim_CreateCommand(interp, newname, JimAliasCmd, prefixListObj, JimAliasCmdDelete); -} - - -static int Jim_ProcCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Cmd *cmd; - - if (argc != 4 && argc != 5) { - Jim_WrongNumArgs(interp, 1, argv, "name arglist ?statics? body"); - return JIM_ERR; - } - - if (JimValidName(interp, "procedure", argv[1]) != JIM_OK) { - return JIM_ERR; - } - - if (argc == 4) { - cmd = JimCreateProcedureCmd(interp, argv[2], NULL, argv[3], NULL); - } - else { - cmd = JimCreateProcedureCmd(interp, argv[2], argv[3], argv[4], NULL); - } - - if (cmd) { - - Jim_Obj *qualifiedCmdNameObj; - const char *cmdname = JimQualifyName(interp, Jim_String(argv[1]), &qualifiedCmdNameObj); - - JimCreateCommand(interp, cmdname, cmd); - - - JimUpdateProcNamespace(interp, cmd, cmdname); - - JimFreeQualifiedName(interp, qualifiedCmdNameObj); - - - Jim_SetResult(interp, argv[1]); - return JIM_OK; - } - return JIM_ERR; -} - - -static int Jim_LocalCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int retcode; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "cmd ?args ...?"); - return JIM_ERR; - } - - - interp->local++; - retcode = Jim_EvalObjVector(interp, argc - 1, argv + 1); - interp->local--; - - - - if (retcode == 0) { - Jim_Obj *cmdNameObj = Jim_GetResult(interp); - - if (Jim_GetCommand(interp, cmdNameObj, JIM_ERRMSG) == NULL) { - return JIM_ERR; - } - if (interp->framePtr->localCommands == NULL) { - interp->framePtr->localCommands = Jim_Alloc(sizeof(*interp->framePtr->localCommands)); - Jim_InitStack(interp->framePtr->localCommands); - } - Jim_IncrRefCount(cmdNameObj); - Jim_StackPush(interp->framePtr->localCommands, cmdNameObj); - } - - return retcode; -} - - -static int Jim_UpcallCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "cmd ?args ...?"); - return JIM_ERR; - } - else { - int retcode; - - Jim_Cmd *cmdPtr = Jim_GetCommand(interp, argv[1], JIM_ERRMSG); - if (cmdPtr == NULL || !cmdPtr->isproc || !cmdPtr->prevCmd) { - Jim_SetResultFormatted(interp, "no previous command: \"%#s\"", argv[1]); - return JIM_ERR; - } - - cmdPtr->u.proc.upcall++; - JimIncrCmdRefCount(cmdPtr); - - - retcode = Jim_EvalObjVector(interp, argc - 1, argv + 1); - - - cmdPtr->u.proc.upcall--; - JimDecrCmdRefCount(interp, cmdPtr); - - return retcode; - } -} - - -static int Jim_ApplyCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "lambdaExpr ?arg ...?"); - return JIM_ERR; - } - else { - int ret; - Jim_Cmd *cmd; - Jim_Obj *argListObjPtr; - Jim_Obj *bodyObjPtr; - Jim_Obj *nsObj = NULL; - Jim_Obj **nargv; - - int len = Jim_ListLength(interp, argv[1]); - if (len != 2 && len != 3) { - Jim_SetResultFormatted(interp, "can't interpret \"%#s\" as a lambda expression", argv[1]); - return JIM_ERR; - } - - if (len == 3) { -#ifdef jim_ext_namespace - - nsObj = JimQualifyNameObj(interp, Jim_ListGetIndex(interp, argv[1], 2)); -#else - Jim_SetResultString(interp, "namespaces not enabled", -1); - return JIM_ERR; -#endif - } - argListObjPtr = Jim_ListGetIndex(interp, argv[1], 0); - bodyObjPtr = Jim_ListGetIndex(interp, argv[1], 1); - - cmd = JimCreateProcedureCmd(interp, argListObjPtr, NULL, bodyObjPtr, nsObj); - - if (cmd) { - - nargv = Jim_Alloc((argc - 2 + 1) * sizeof(*nargv)); - nargv[0] = Jim_NewStringObj(interp, "apply lambdaExpr", -1); - Jim_IncrRefCount(nargv[0]); - memcpy(&nargv[1], argv + 2, (argc - 2) * sizeof(*nargv)); - ret = JimCallProcedure(interp, cmd, argc - 2 + 1, nargv); - Jim_DecrRefCount(interp, nargv[0]); - Jim_Free(nargv); - - JimDecrCmdRefCount(interp, cmd); - return ret; - } - return JIM_ERR; - } -} - - - -static int Jim_ConcatCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_SetResult(interp, Jim_ConcatObj(interp, argc - 1, argv + 1)); - return JIM_OK; -} - - -static int Jim_UpvarCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int i; - Jim_CallFrame *targetCallFrame; - - - if (argc > 3 && (argc % 2 == 0)) { - targetCallFrame = Jim_GetCallFrameByLevel(interp, argv[1]); - argc--; - argv++; - } - else { - targetCallFrame = Jim_GetCallFrameByLevel(interp, NULL); - } - if (targetCallFrame == NULL) { - return JIM_ERR; - } - - - if (argc < 3) { - Jim_WrongNumArgs(interp, 1, argv, "?level? otherVar localVar ?otherVar localVar ...?"); - return JIM_ERR; - } - - - for (i = 1; i < argc; i += 2) { - if (Jim_SetVariableLink(interp, argv[i + 1], argv[i], targetCallFrame) != JIM_OK) - return JIM_ERR; - } - return JIM_OK; -} - - -static int Jim_GlobalCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int i; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "varName ?varName ...?"); - return JIM_ERR; - } - - if (interp->framePtr->level == 0) - return JIM_OK; - for (i = 1; i < argc; i++) { - - const char *name = Jim_String(argv[i]); - if (name[0] != ':' || name[1] != ':') { - if (Jim_SetVariableLink(interp, argv[i], argv[i], interp->topFramePtr) != JIM_OK) - return JIM_ERR; - } - } - return JIM_OK; -} - -static Jim_Obj *JimStringMap(Jim_Interp *interp, Jim_Obj *mapListObjPtr, - Jim_Obj *objPtr, int nocase) -{ - int numMaps; - const char *str, *noMatchStart = NULL; - int strLen, i; - Jim_Obj *resultObjPtr; - - numMaps = Jim_ListLength(interp, mapListObjPtr); - if (numMaps % 2) { - Jim_SetResultString(interp, "list must contain an even number of elements", -1); - return NULL; - } - - str = Jim_String(objPtr); - strLen = Jim_Utf8Length(interp, objPtr); - - - resultObjPtr = Jim_NewStringObj(interp, "", 0); - while (strLen) { - for (i = 0; i < numMaps; i += 2) { - Jim_Obj *eachObjPtr; - const char *k; - int kl; - - eachObjPtr = Jim_ListGetIndex(interp, mapListObjPtr, i); - k = Jim_String(eachObjPtr); - kl = Jim_Utf8Length(interp, eachObjPtr); - - if (strLen >= kl && kl) { - int rc; - rc = JimStringCompareLen(str, k, kl, nocase); - if (rc == 0) { - if (noMatchStart) { - Jim_AppendString(interp, resultObjPtr, noMatchStart, str - noMatchStart); - noMatchStart = NULL; - } - Jim_AppendObj(interp, resultObjPtr, Jim_ListGetIndex(interp, mapListObjPtr, i + 1)); - str += utf8_index(str, kl); - strLen -= kl; - break; - } - } - } - if (i == numMaps) { - int c; - if (noMatchStart == NULL) - noMatchStart = str; - str += utf8_tounicode(str, &c); - strLen--; - } - } - if (noMatchStart) { - Jim_AppendString(interp, resultObjPtr, noMatchStart, str - noMatchStart); - } - return resultObjPtr; -} - - -static int Jim_StringCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int len; - int opt_case = 1; - int option; - static const char * const options[] = { - "bytelength", "length", "compare", "match", "equal", "is", "byterange", "range", "replace", - "map", "repeat", "reverse", "index", "first", "last", "cat", - "trim", "trimleft", "trimright", "tolower", "toupper", "totitle", NULL - }; - enum - { - OPT_BYTELENGTH, OPT_LENGTH, OPT_COMPARE, OPT_MATCH, OPT_EQUAL, OPT_IS, OPT_BYTERANGE, OPT_RANGE, OPT_REPLACE, - OPT_MAP, OPT_REPEAT, OPT_REVERSE, OPT_INDEX, OPT_FIRST, OPT_LAST, OPT_CAT, - OPT_TRIM, OPT_TRIMLEFT, OPT_TRIMRIGHT, OPT_TOLOWER, OPT_TOUPPER, OPT_TOTITLE - }; - static const char * const nocase_options[] = { - "-nocase", NULL - }; - static const char * const nocase_length_options[] = { - "-nocase", "-length", NULL - }; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "option ?arguments ...?"); - return JIM_ERR; - } - if (Jim_GetEnum(interp, argv[1], options, &option, NULL, - JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) - return Jim_CheckShowCommands(interp, argv[1], options); - - switch (option) { - case OPT_LENGTH: - case OPT_BYTELENGTH: - if (argc != 3) { - Jim_WrongNumArgs(interp, 2, argv, "string"); - return JIM_ERR; - } - if (option == OPT_LENGTH) { - len = Jim_Utf8Length(interp, argv[2]); - } - else { - len = Jim_Length(argv[2]); - } - Jim_SetResultInt(interp, len); - return JIM_OK; - - case OPT_CAT:{ - Jim_Obj *objPtr; - if (argc == 3) { - - objPtr = argv[2]; - } - else { - int i; - - objPtr = Jim_NewStringObj(interp, "", 0); - - for (i = 2; i < argc; i++) { - Jim_AppendObj(interp, objPtr, argv[i]); - } - } - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - - case OPT_COMPARE: - case OPT_EQUAL: - { - - long opt_length = -1; - int n = argc - 4; - int i = 2; - while (n > 0) { - int subopt; - if (Jim_GetEnum(interp, argv[i++], nocase_length_options, &subopt, NULL, - JIM_ENUM_ABBREV) != JIM_OK) { -badcompareargs: - Jim_WrongNumArgs(interp, 2, argv, "?-nocase? ?-length int? string1 string2"); - return JIM_ERR; - } - if (subopt == 0) { - - opt_case = 0; - n--; - } - else { - - if (n < 2) { - goto badcompareargs; - } - if (Jim_GetLong(interp, argv[i++], &opt_length) != JIM_OK) { - return JIM_ERR; - } - n -= 2; - } - } - if (n) { - goto badcompareargs; - } - argv += argc - 2; - if (opt_length < 0 && option != OPT_COMPARE && opt_case) { - - Jim_SetResultBool(interp, Jim_StringEqObj(argv[0], argv[1])); - } - else { - if (opt_length >= 0) { - n = JimStringCompareLen(Jim_String(argv[0]), Jim_String(argv[1]), opt_length, !opt_case); - } - else { - n = Jim_StringCompareObj(interp, argv[0], argv[1], !opt_case); - } - Jim_SetResultInt(interp, option == OPT_COMPARE ? n : n == 0); - } - return JIM_OK; - } - - case OPT_MATCH: - if (argc != 4 && - (argc != 5 || - Jim_GetEnum(interp, argv[2], nocase_options, &opt_case, NULL, - JIM_ENUM_ABBREV) != JIM_OK)) { - Jim_WrongNumArgs(interp, 2, argv, "?-nocase? pattern string"); - return JIM_ERR; - } - if (opt_case == 0) { - argv++; - } - Jim_SetResultBool(interp, Jim_StringMatchObj(interp, argv[2], argv[3], !opt_case)); - return JIM_OK; - - case OPT_MAP:{ - Jim_Obj *objPtr; - - if (argc != 4 && - (argc != 5 || - Jim_GetEnum(interp, argv[2], nocase_options, &opt_case, NULL, - JIM_ENUM_ABBREV) != JIM_OK)) { - Jim_WrongNumArgs(interp, 2, argv, "?-nocase? mapList string"); - return JIM_ERR; - } - - if (opt_case == 0) { - argv++; - } - objPtr = JimStringMap(interp, argv[2], argv[3], !opt_case); - if (objPtr == NULL) { - return JIM_ERR; - } - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - - case OPT_RANGE: - case OPT_BYTERANGE:{ - Jim_Obj *objPtr; - - if (argc != 5) { - Jim_WrongNumArgs(interp, 2, argv, "string first last"); - return JIM_ERR; - } - if (option == OPT_RANGE) { - objPtr = Jim_StringRangeObj(interp, argv[2], argv[3], argv[4]); - } - else - { - objPtr = Jim_StringByteRangeObj(interp, argv[2], argv[3], argv[4]); - } - - if (objPtr == NULL) { - return JIM_ERR; - } - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - - case OPT_REPLACE:{ - Jim_Obj *objPtr; - - if (argc != 5 && argc != 6) { - Jim_WrongNumArgs(interp, 2, argv, "string first last ?string?"); - return JIM_ERR; - } - objPtr = JimStringReplaceObj(interp, argv[2], argv[3], argv[4], argc == 6 ? argv[5] : NULL); - if (objPtr == NULL) { - return JIM_ERR; - } - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - - - case OPT_REPEAT:{ - Jim_Obj *objPtr; - jim_wide count; - - if (argc != 4) { - Jim_WrongNumArgs(interp, 2, argv, "string count"); - return JIM_ERR; - } - if (Jim_GetWide(interp, argv[3], &count) != JIM_OK) { - return JIM_ERR; - } - objPtr = Jim_NewStringObj(interp, "", 0); - if (count > 0) { - while (count--) { - Jim_AppendObj(interp, objPtr, argv[2]); - } - } - Jim_SetResult(interp, objPtr); - return JIM_OK; - } - - case OPT_REVERSE:{ - char *buf, *p; - const char *str; - int i; - - if (argc != 3) { - Jim_WrongNumArgs(interp, 2, argv, "string"); - return JIM_ERR; - } - - str = Jim_GetString(argv[2], &len); - buf = Jim_Alloc(len + 1); - p = buf + len; - *p = 0; - for (i = 0; i < len; ) { - int c; - int l = utf8_tounicode(str, &c); - memcpy(p - l, str, l); - p -= l; - i += l; - str += l; - } - Jim_SetResult(interp, Jim_NewStringObjNoAlloc(interp, buf, len)); - return JIM_OK; - } - - case OPT_INDEX:{ - int idx; - const char *str; - - if (argc != 4) { - Jim_WrongNumArgs(interp, 2, argv, "string index"); - return JIM_ERR; - } - if (Jim_GetIndex(interp, argv[3], &idx) != JIM_OK) { - return JIM_ERR; - } - str = Jim_String(argv[2]); - len = Jim_Utf8Length(interp, argv[2]); - if (idx != INT_MIN && idx != INT_MAX) { - idx = JimRelToAbsIndex(len, idx); - } - if (idx < 0 || idx >= len || str == NULL) { - Jim_SetResultString(interp, "", 0); - } - else if (len == Jim_Length(argv[2])) { - - Jim_SetResultString(interp, str + idx, 1); - } - else { - int c; - int i = utf8_index(str, idx); - Jim_SetResultString(interp, str + i, utf8_tounicode(str + i, &c)); - } - return JIM_OK; - } - - case OPT_FIRST: - case OPT_LAST:{ - int idx = 0, l1, l2; - const char *s1, *s2; - - if (argc != 4 && argc != 5) { - Jim_WrongNumArgs(interp, 2, argv, "subString string ?index?"); - return JIM_ERR; - } - s1 = Jim_String(argv[2]); - s2 = Jim_String(argv[3]); - l1 = Jim_Utf8Length(interp, argv[2]); - l2 = Jim_Utf8Length(interp, argv[3]); - if (argc == 5) { - if (Jim_GetIndex(interp, argv[4], &idx) != JIM_OK) { - return JIM_ERR; - } - idx = JimRelToAbsIndex(l2, idx); - } - else if (option == OPT_LAST) { - idx = l2; - } - if (option == OPT_FIRST) { - Jim_SetResultInt(interp, JimStringFirst(s1, l1, s2, l2, idx)); - } - else { -#ifdef JIM_UTF8 - Jim_SetResultInt(interp, JimStringLastUtf8(s1, l1, s2, idx)); -#else - Jim_SetResultInt(interp, JimStringLast(s1, l1, s2, idx)); -#endif - } - return JIM_OK; - } - - case OPT_TRIM: - case OPT_TRIMLEFT: - case OPT_TRIMRIGHT:{ - Jim_Obj *trimchars; - - if (argc != 3 && argc != 4) { - Jim_WrongNumArgs(interp, 2, argv, "string ?trimchars?"); - return JIM_ERR; - } - trimchars = (argc == 4 ? argv[3] : NULL); - if (option == OPT_TRIM) { - Jim_SetResult(interp, JimStringTrim(interp, argv[2], trimchars)); - } - else if (option == OPT_TRIMLEFT) { - Jim_SetResult(interp, JimStringTrimLeft(interp, argv[2], trimchars)); - } - else if (option == OPT_TRIMRIGHT) { - Jim_SetResult(interp, JimStringTrimRight(interp, argv[2], trimchars)); - } - return JIM_OK; - } - - case OPT_TOLOWER: - case OPT_TOUPPER: - case OPT_TOTITLE: - if (argc != 3) { - Jim_WrongNumArgs(interp, 2, argv, "string"); - return JIM_ERR; - } - if (option == OPT_TOLOWER) { - Jim_SetResult(interp, JimStringToLower(interp, argv[2])); - } - else if (option == OPT_TOUPPER) { - Jim_SetResult(interp, JimStringToUpper(interp, argv[2])); - } - else { - Jim_SetResult(interp, JimStringToTitle(interp, argv[2])); - } - return JIM_OK; - - case OPT_IS: - if (argc == 4 || (argc == 5 && Jim_CompareStringImmediate(interp, argv[3], "-strict"))) { - return JimStringIs(interp, argv[argc - 1], argv[2], argc == 5); - } - Jim_WrongNumArgs(interp, 2, argv, "class ?-strict? str"); - return JIM_ERR; - } - return JIM_OK; -} - - -static int Jim_TimeCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - long i, count = 1; - jim_wide start, elapsed; - char buf[60]; - const char *fmt = "%" JIM_WIDE_MODIFIER " microseconds per iteration"; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "script ?count?"); - return JIM_ERR; - } - if (argc == 3) { - if (Jim_GetLong(interp, argv[2], &count) != JIM_OK) - return JIM_ERR; - } - if (count < 0) - return JIM_OK; - i = count; - start = JimClock(); - while (i-- > 0) { - int retval; - - retval = Jim_EvalObj(interp, argv[1]); - if (retval != JIM_OK) { - return retval; - } - } - elapsed = JimClock() - start; - sprintf(buf, fmt, count == 0 ? 0 : elapsed / count); - Jim_SetResultString(interp, buf, -1); - return JIM_OK; -} - - -static int Jim_ExitCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - long exitCode = 0; - - if (argc > 2) { - Jim_WrongNumArgs(interp, 1, argv, "?exitCode?"); - return JIM_ERR; - } - if (argc == 2) { - if (Jim_GetLong(interp, argv[1], &exitCode) != JIM_OK) - return JIM_ERR; - } - interp->exitCode = exitCode; - return JIM_EXIT; -} - - -static int Jim_CatchCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int exitCode = 0; - int i; - int sig = 0; - - - jim_wide ignore_mask = (1 << JIM_EXIT) | (1 << JIM_EVAL) | (1 << JIM_SIGNAL); - static const int max_ignore_code = sizeof(ignore_mask) * 8; - - Jim_SetGlobalVariableStr(interp, "errorCode", Jim_NewStringObj(interp, "NONE", -1)); - - for (i = 1; i < argc - 1; i++) { - const char *arg = Jim_String(argv[i]); - jim_wide option; - int ignore; - - - if (strcmp(arg, "--") == 0) { - i++; - break; - } - if (*arg != '-') { - break; - } - - if (strncmp(arg, "-no", 3) == 0) { - arg += 3; - ignore = 1; - } - else { - arg++; - ignore = 0; - } - - if (Jim_StringToWide(arg, &option, 10) != JIM_OK) { - option = -1; - } - if (option < 0) { - option = Jim_FindByName(arg, jimReturnCodes, jimReturnCodesSize); - } - if (option < 0) { - goto wrongargs; - } - - if (ignore) { - ignore_mask |= ((jim_wide)1 << option); - } - else { - ignore_mask &= (~((jim_wide)1 << option)); - } - } - - argc -= i; - if (argc < 1 || argc > 3) { - wrongargs: - Jim_WrongNumArgs(interp, 1, argv, - "?-?no?code ... --? script ?resultVarName? ?optionVarName?"); - return JIM_ERR; - } - argv += i; - - if ((ignore_mask & (1 << JIM_SIGNAL)) == 0) { - sig++; - } - - interp->signal_level += sig; - if (Jim_CheckSignal(interp)) { - - exitCode = JIM_SIGNAL; - } - else { - exitCode = Jim_EvalObj(interp, argv[0]); - - interp->errorFlag = 0; - } - interp->signal_level -= sig; - - - if (exitCode >= 0 && exitCode < max_ignore_code && (((unsigned jim_wide)1 << exitCode) & ignore_mask)) { - - return exitCode; - } - - if (sig && exitCode == JIM_SIGNAL) { - - if (interp->signal_set_result) { - interp->signal_set_result(interp, interp->sigmask); - } - else { - Jim_SetResultInt(interp, interp->sigmask); - } - interp->sigmask = 0; - } - - if (argc >= 2) { - if (Jim_SetVariable(interp, argv[1], Jim_GetResult(interp)) != JIM_OK) { - return JIM_ERR; - } - if (argc == 3) { - Jim_Obj *optListObj = Jim_NewListObj(interp, NULL, 0); - - Jim_ListAppendElement(interp, optListObj, Jim_NewStringObj(interp, "-code", -1)); - Jim_ListAppendElement(interp, optListObj, - Jim_NewIntObj(interp, exitCode == JIM_RETURN ? interp->returnCode : exitCode)); - Jim_ListAppendElement(interp, optListObj, Jim_NewStringObj(interp, "-level", -1)); - Jim_ListAppendElement(interp, optListObj, Jim_NewIntObj(interp, interp->returnLevel)); - if (exitCode == JIM_ERR) { - Jim_Obj *errorCode; - Jim_ListAppendElement(interp, optListObj, Jim_NewStringObj(interp, "-errorinfo", - -1)); - Jim_ListAppendElement(interp, optListObj, interp->stackTrace); - - errorCode = Jim_GetGlobalVariableStr(interp, "errorCode", JIM_NONE); - if (errorCode) { - Jim_ListAppendElement(interp, optListObj, Jim_NewStringObj(interp, "-errorcode", -1)); - Jim_ListAppendElement(interp, optListObj, errorCode); - } - } - if (Jim_SetVariable(interp, argv[2], optListObj) != JIM_OK) { - return JIM_ERR; - } - } - } - Jim_SetResultInt(interp, exitCode); - return JIM_OK; -} - - - -static int Jim_RenameCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc != 3) { - Jim_WrongNumArgs(interp, 1, argv, "oldName newName"); - return JIM_ERR; - } - - if (JimValidName(interp, "new procedure", argv[2])) { - return JIM_ERR; - } - - return Jim_RenameCommand(interp, Jim_String(argv[1]), Jim_String(argv[2])); -} - -#define JIM_DICTMATCH_KEYS 0x0001 -#define JIM_DICTMATCH_VALUES 0x002 - -int Jim_DictMatchTypes(Jim_Interp *interp, Jim_Obj *objPtr, Jim_Obj *patternObj, int match_type, int return_types) -{ - Jim_HashEntry *he; - Jim_Obj *listObjPtr; - Jim_HashTableIterator htiter; - - if (SetDictFromAny(interp, objPtr) != JIM_OK) { - return JIM_ERR; - } - - listObjPtr = Jim_NewListObj(interp, NULL, 0); - - JimInitHashTableIterator(objPtr->internalRep.ptr, &htiter); - while ((he = Jim_NextHashEntry(&htiter)) != NULL) { - if (patternObj) { - Jim_Obj *matchObj = (match_type == JIM_DICTMATCH_KEYS) ? (Jim_Obj *)he->key : Jim_GetHashEntryVal(he); - if (!JimGlobMatch(Jim_String(patternObj), Jim_String(matchObj), 0)) { - - continue; - } - } - if (return_types & JIM_DICTMATCH_KEYS) { - Jim_ListAppendElement(interp, listObjPtr, (Jim_Obj *)he->key); - } - if (return_types & JIM_DICTMATCH_VALUES) { - Jim_ListAppendElement(interp, listObjPtr, Jim_GetHashEntryVal(he)); - } - } - - Jim_SetResult(interp, listObjPtr); - return JIM_OK; -} - -int Jim_DictSize(Jim_Interp *interp, Jim_Obj *objPtr) -{ - if (SetDictFromAny(interp, objPtr) != JIM_OK) { - return -1; - } - return ((Jim_HashTable *)objPtr->internalRep.ptr)->used; -} - -Jim_Obj *Jim_DictMerge(Jim_Interp *interp, int objc, Jim_Obj *const *objv) -{ - Jim_Obj *objPtr = Jim_NewDictObj(interp, NULL, 0); - int i; - - JimPanic((objc == 0, "Jim_DictMerge called with objc=0")); - - - - for (i = 0; i < objc; i++) { - Jim_HashTable *ht; - Jim_HashTableIterator htiter; - Jim_HashEntry *he; - - if (SetDictFromAny(interp, objv[i]) != JIM_OK) { - Jim_FreeNewObj(interp, objPtr); - return NULL; - } - ht = objv[i]->internalRep.ptr; - JimInitHashTableIterator(ht, &htiter); - while ((he = Jim_NextHashEntry(&htiter)) != NULL) { - Jim_ReplaceHashEntry(objPtr->internalRep.ptr, Jim_GetHashEntryKey(he), Jim_GetHashEntryVal(he)); - } - } - return objPtr; -} - -int Jim_DictInfo(Jim_Interp *interp, Jim_Obj *objPtr) -{ - Jim_HashTable *ht; - unsigned int i; - char buffer[100]; - int sum = 0; - int nonzero_count = 0; - Jim_Obj *output; - int bucket_counts[11] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - - if (SetDictFromAny(interp, objPtr) != JIM_OK) { - return JIM_ERR; - } - - ht = (Jim_HashTable *)objPtr->internalRep.ptr; - - - snprintf(buffer, sizeof(buffer), "%d entries in table, %d buckets\n", ht->used, ht->size); - output = Jim_NewStringObj(interp, buffer, -1); - - for (i = 0; i < ht->size; i++) { - Jim_HashEntry *he = ht->table[i]; - int entries = 0; - while (he) { - entries++; - he = he->next; - } - if (entries > 9) { - bucket_counts[10]++; - } - else { - bucket_counts[entries]++; - } - if (entries) { - sum += entries; - nonzero_count++; - } - } - for (i = 0; i < 10; i++) { - snprintf(buffer, sizeof(buffer), "number of buckets with %d entries: %d\n", i, bucket_counts[i]); - Jim_AppendString(interp, output, buffer, -1); - } - snprintf(buffer, sizeof(buffer), "number of buckets with 10 or more entries: %d\n", bucket_counts[10]); - Jim_AppendString(interp, output, buffer, -1); - snprintf(buffer, sizeof(buffer), "average search distance for entry: %.1f", nonzero_count ? (double)sum / nonzero_count : 0.0); - Jim_AppendString(interp, output, buffer, -1); - Jim_SetResult(interp, output); - return JIM_OK; -} - -static int Jim_EvalEnsemble(Jim_Interp *interp, const char *basecmd, const char *subcmd, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *prefixObj = Jim_NewStringObj(interp, basecmd, -1); - - Jim_AppendString(interp, prefixObj, " ", 1); - Jim_AppendString(interp, prefixObj, subcmd, -1); - - return Jim_EvalObjPrefix(interp, prefixObj, argc, argv); -} - -static int JimDictWith(Jim_Interp *interp, Jim_Obj *dictVarName, Jim_Obj *const *keyv, int keyc, Jim_Obj *scriptObj) -{ - int i; - Jim_Obj *objPtr; - Jim_Obj *dictObj; - Jim_Obj **dictValues; - int len; - int ret = JIM_OK; - - - dictObj = Jim_GetVariable(interp, dictVarName, JIM_ERRMSG); - if (dictObj == NULL || Jim_DictKeysVector(interp, dictObj, keyv, keyc, &objPtr, JIM_ERRMSG) != JIM_OK) { - return JIM_ERR; - } - - if (Jim_DictPairs(interp, objPtr, &dictValues, &len) == JIM_ERR) { - return JIM_ERR; - } - for (i = 0; i < len; i += 2) { - if (Jim_SetVariable(interp, dictValues[i], dictValues[i + 1]) == JIM_ERR) { - Jim_Free(dictValues); - return JIM_ERR; - } - } - - - if (Jim_Length(scriptObj)) { - ret = Jim_EvalObj(interp, scriptObj); - - - if (ret == JIM_OK && Jim_GetVariable(interp, dictVarName, 0) != NULL) { - - Jim_Obj **newkeyv = Jim_Alloc(sizeof(*newkeyv) * (keyc + 1)); - for (i = 0; i < keyc; i++) { - newkeyv[i] = keyv[i]; - } - - for (i = 0; i < len; i += 2) { - - objPtr = Jim_GetVariable(interp, dictValues[i], 0); - newkeyv[keyc] = dictValues[i]; - Jim_SetDictKeysVector(interp, dictVarName, newkeyv, keyc + 1, objPtr, 0); - } - Jim_Free(newkeyv); - } - } - - Jim_Free(dictValues); - - return ret; -} - - -static int Jim_DictCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *objPtr; - int types = JIM_DICTMATCH_KEYS; - int option; - static const char * const options[] = { - "create", "get", "set", "unset", "exists", "keys", "size", "info", - "merge", "with", "append", "lappend", "incr", "remove", "values", "for", - "replace", "update", NULL - }; - enum - { - OPT_CREATE, OPT_GET, OPT_SET, OPT_UNSET, OPT_EXISTS, OPT_KEYS, OPT_SIZE, OPT_INFO, - OPT_MERGE, OPT_WITH, OPT_APPEND, OPT_LAPPEND, OPT_INCR, OPT_REMOVE, OPT_VALUES, OPT_FOR, - OPT_REPLACE, OPT_UPDATE, - }; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "subcommand ?arguments ...?"); - return JIM_ERR; - } - - if (Jim_GetEnum(interp, argv[1], options, &option, "subcommand", JIM_ERRMSG) != JIM_OK) { - return Jim_CheckShowCommands(interp, argv[1], options); - } - - switch (option) { - case OPT_GET: - if (argc < 3) { - Jim_WrongNumArgs(interp, 2, argv, "dictionary ?key ...?"); - return JIM_ERR; - } - if (Jim_DictKeysVector(interp, argv[2], argv + 3, argc - 3, &objPtr, - JIM_ERRMSG) != JIM_OK) { - return JIM_ERR; - } - Jim_SetResult(interp, objPtr); - return JIM_OK; - - case OPT_SET: - if (argc < 5) { - Jim_WrongNumArgs(interp, 2, argv, "varName key ?key ...? value"); - return JIM_ERR; - } - return Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 4, argv[argc - 1], JIM_ERRMSG); - - case OPT_EXISTS: - if (argc < 4) { - Jim_WrongNumArgs(interp, 2, argv, "dictionary key ?key ...?"); - return JIM_ERR; - } - else { - int rc = Jim_DictKeysVector(interp, argv[2], argv + 3, argc - 3, &objPtr, JIM_ERRMSG); - if (rc < 0) { - return JIM_ERR; - } - Jim_SetResultBool(interp, rc == JIM_OK); - return JIM_OK; - } - - case OPT_UNSET: - if (argc < 4) { - Jim_WrongNumArgs(interp, 2, argv, "varName key ?key ...?"); - return JIM_ERR; - } - if (Jim_SetDictKeysVector(interp, argv[2], argv + 3, argc - 3, NULL, 0) != JIM_OK) { - return JIM_ERR; - } - return JIM_OK; - - case OPT_VALUES: - types = JIM_DICTMATCH_VALUES; - - case OPT_KEYS: - if (argc != 3 && argc != 4) { - Jim_WrongNumArgs(interp, 2, argv, "dictionary ?pattern?"); - return JIM_ERR; - } - return Jim_DictMatchTypes(interp, argv[2], argc == 4 ? argv[3] : NULL, types, types); - - case OPT_SIZE: - if (argc != 3) { - Jim_WrongNumArgs(interp, 2, argv, "dictionary"); - return JIM_ERR; - } - else if (Jim_DictSize(interp, argv[2]) < 0) { - return JIM_ERR; - } - Jim_SetResultInt(interp, Jim_DictSize(interp, argv[2])); - return JIM_OK; - - case OPT_MERGE: - if (argc == 2) { - return JIM_OK; - } - objPtr = Jim_DictMerge(interp, argc - 2, argv + 2); - if (objPtr == NULL) { - return JIM_ERR; - } - Jim_SetResult(interp, objPtr); - return JIM_OK; - - case OPT_UPDATE: - if (argc < 6 || argc % 2) { - - argc = 2; - } - break; - - case OPT_CREATE: - if (argc % 2) { - Jim_WrongNumArgs(interp, 2, argv, "?key value ...?"); - return JIM_ERR; - } - objPtr = Jim_NewDictObj(interp, argv + 2, argc - 2); - Jim_SetResult(interp, objPtr); - return JIM_OK; - - case OPT_INFO: - if (argc != 3) { - Jim_WrongNumArgs(interp, 2, argv, "dictionary"); - return JIM_ERR; - } - return Jim_DictInfo(interp, argv[2]); - - case OPT_WITH: - if (argc < 4) { - Jim_WrongNumArgs(interp, 2, argv, "dictVar ?key ...? script"); - return JIM_ERR; - } - return JimDictWith(interp, argv[2], argv + 3, argc - 4, argv[argc - 1]); - } - - return Jim_EvalEnsemble(interp, "dict", options[option], argc - 2, argv + 2); -} - - -static int Jim_SubstCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - static const char * const options[] = { - "-nobackslashes", "-nocommands", "-novariables", NULL - }; - enum - { OPT_NOBACKSLASHES, OPT_NOCOMMANDS, OPT_NOVARIABLES }; - int i; - int flags = JIM_SUBST_FLAG; - Jim_Obj *objPtr; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "?options? string"); - return JIM_ERR; - } - for (i = 1; i < (argc - 1); i++) { - int option; - - if (Jim_GetEnum(interp, argv[i], options, &option, NULL, - JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { - return JIM_ERR; - } - switch (option) { - case OPT_NOBACKSLASHES: - flags |= JIM_SUBST_NOESC; - break; - case OPT_NOCOMMANDS: - flags |= JIM_SUBST_NOCMD; - break; - case OPT_NOVARIABLES: - flags |= JIM_SUBST_NOVAR; - break; - } - } - if (Jim_SubstObj(interp, argv[argc - 1], &objPtr, flags) != JIM_OK) { - return JIM_ERR; - } - Jim_SetResult(interp, objPtr); - return JIM_OK; -} - - -static int Jim_InfoCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int cmd; - Jim_Obj *objPtr; - int mode = 0; - - static const char * const commands[] = { - "body", "statics", "commands", "procs", "channels", "exists", "globals", "level", "frame", "locals", - "vars", "version", "patchlevel", "complete", "args", "hostname", - "script", "source", "stacktrace", "nameofexecutable", "returncodes", - "references", "alias", NULL - }; - enum - { INFO_BODY, INFO_STATICS, INFO_COMMANDS, INFO_PROCS, INFO_CHANNELS, INFO_EXISTS, INFO_GLOBALS, INFO_LEVEL, - INFO_FRAME, INFO_LOCALS, INFO_VARS, INFO_VERSION, INFO_PATCHLEVEL, INFO_COMPLETE, INFO_ARGS, - INFO_HOSTNAME, INFO_SCRIPT, INFO_SOURCE, INFO_STACKTRACE, INFO_NAMEOFEXECUTABLE, - INFO_RETURNCODES, INFO_REFERENCES, INFO_ALIAS, - }; - -#ifdef jim_ext_namespace - int nons = 0; - - if (argc > 2 && Jim_CompareStringImmediate(interp, argv[1], "-nons")) { - - argc--; - argv++; - nons = 1; - } -#endif - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "subcommand ?args ...?"); - return JIM_ERR; - } - if (Jim_GetEnum(interp, argv[1], commands, &cmd, "subcommand", JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { - return Jim_CheckShowCommands(interp, argv[1], commands); - } - - - switch (cmd) { - case INFO_EXISTS: - if (argc != 3) { - Jim_WrongNumArgs(interp, 2, argv, "varName"); - return JIM_ERR; - } - Jim_SetResultBool(interp, Jim_GetVariable(interp, argv[2], 0) != NULL); - break; - - case INFO_ALIAS:{ - Jim_Cmd *cmdPtr; - - if (argc != 3) { - Jim_WrongNumArgs(interp, 2, argv, "command"); - return JIM_ERR; - } - if ((cmdPtr = Jim_GetCommand(interp, argv[2], JIM_ERRMSG)) == NULL) { - return JIM_ERR; - } - if (cmdPtr->isproc || cmdPtr->u.native.cmdProc != JimAliasCmd) { - Jim_SetResultFormatted(interp, "command \"%#s\" is not an alias", argv[2]); - return JIM_ERR; - } - Jim_SetResult(interp, (Jim_Obj *)cmdPtr->u.native.privData); - return JIM_OK; - } - - case INFO_CHANNELS: - mode++; -#ifndef jim_ext_aio - Jim_SetResultString(interp, "aio not enabled", -1); - return JIM_ERR; -#endif - - case INFO_PROCS: - mode++; - - case INFO_COMMANDS: - - if (argc != 2 && argc != 3) { - Jim_WrongNumArgs(interp, 2, argv, "?pattern?"); - return JIM_ERR; - } -#ifdef jim_ext_namespace - if (!nons) { - if (Jim_Length(interp->framePtr->nsObj) || (argc == 3 && JimGlobMatch("::*", Jim_String(argv[2]), 0))) { - return Jim_EvalPrefix(interp, "namespace info", argc - 1, argv + 1); - } - } -#endif - Jim_SetResult(interp, JimCommandsList(interp, (argc == 3) ? argv[2] : NULL, mode)); - break; - - case INFO_VARS: - mode++; - - case INFO_LOCALS: - mode++; - - case INFO_GLOBALS: - - if (argc != 2 && argc != 3) { - Jim_WrongNumArgs(interp, 2, argv, "?pattern?"); - return JIM_ERR; - } -#ifdef jim_ext_namespace - if (!nons) { - if (Jim_Length(interp->framePtr->nsObj) || (argc == 3 && JimGlobMatch("::*", Jim_String(argv[2]), 0))) { - return Jim_EvalPrefix(interp, "namespace info", argc - 1, argv + 1); - } - } -#endif - Jim_SetResult(interp, JimVariablesList(interp, argc == 3 ? argv[2] : NULL, mode)); - break; - - case INFO_SCRIPT: - if (argc != 2) { - Jim_WrongNumArgs(interp, 2, argv, ""); - return JIM_ERR; - } - Jim_SetResult(interp, JimGetScript(interp, interp->currentScriptObj)->fileNameObj); - break; - - case INFO_SOURCE:{ - jim_wide line; - Jim_Obj *resObjPtr; - Jim_Obj *fileNameObj; - - if (argc != 3 && argc != 5) { - Jim_WrongNumArgs(interp, 2, argv, "source ?filename line?"); - return JIM_ERR; - } - if (argc == 5) { - if (Jim_GetWide(interp, argv[4], &line) != JIM_OK) { - return JIM_ERR; - } - resObjPtr = Jim_NewStringObj(interp, Jim_String(argv[2]), Jim_Length(argv[2])); - JimSetSourceInfo(interp, resObjPtr, argv[3], line); - } - else { - if (argv[2]->typePtr == &sourceObjType) { - fileNameObj = argv[2]->internalRep.sourceValue.fileNameObj; - line = argv[2]->internalRep.sourceValue.lineNumber; - } - else if (argv[2]->typePtr == &scriptObjType) { - ScriptObj *script = JimGetScript(interp, argv[2]); - fileNameObj = script->fileNameObj; - line = script->firstline; - } - else { - fileNameObj = interp->emptyObj; - line = 1; - } - resObjPtr = Jim_NewListObj(interp, NULL, 0); - Jim_ListAppendElement(interp, resObjPtr, fileNameObj); - Jim_ListAppendElement(interp, resObjPtr, Jim_NewIntObj(interp, line)); - } - Jim_SetResult(interp, resObjPtr); - break; - } - - case INFO_STACKTRACE: - Jim_SetResult(interp, interp->stackTrace); - break; - - case INFO_LEVEL: - case INFO_FRAME: - switch (argc) { - case 2: - Jim_SetResultInt(interp, interp->framePtr->level); - break; - - case 3: - if (JimInfoLevel(interp, argv[2], &objPtr, cmd == INFO_LEVEL) != JIM_OK) { - return JIM_ERR; - } - Jim_SetResult(interp, objPtr); - break; - - default: - Jim_WrongNumArgs(interp, 2, argv, "?levelNum?"); - return JIM_ERR; - } - break; - - case INFO_BODY: - case INFO_STATICS: - case INFO_ARGS:{ - Jim_Cmd *cmdPtr; - - if (argc != 3) { - Jim_WrongNumArgs(interp, 2, argv, "procname"); - return JIM_ERR; - } - if ((cmdPtr = Jim_GetCommand(interp, argv[2], JIM_ERRMSG)) == NULL) { - return JIM_ERR; - } - if (!cmdPtr->isproc) { - Jim_SetResultFormatted(interp, "command \"%#s\" is not a procedure", argv[2]); - return JIM_ERR; - } - switch (cmd) { - case INFO_BODY: - Jim_SetResult(interp, cmdPtr->u.proc.bodyObjPtr); - break; - case INFO_ARGS: - Jim_SetResult(interp, cmdPtr->u.proc.argListObjPtr); - break; - case INFO_STATICS: - if (cmdPtr->u.proc.staticVars) { - Jim_SetResult(interp, JimHashtablePatternMatch(interp, cmdPtr->u.proc.staticVars, - NULL, JimVariablesMatch, JIM_VARLIST_LOCALS | JIM_VARLIST_VALUES)); - } - break; - } - break; - } - - case INFO_VERSION: - case INFO_PATCHLEVEL:{ - char buf[(JIM_INTEGER_SPACE * 2) + 1]; - - sprintf(buf, "%d.%d", JIM_VERSION / 100, JIM_VERSION % 100); - Jim_SetResultString(interp, buf, -1); - break; - } - - case INFO_COMPLETE: - if (argc != 3 && argc != 4) { - Jim_WrongNumArgs(interp, 2, argv, "script ?missing?"); - return JIM_ERR; - } - else { - char missing; - - Jim_SetResultBool(interp, Jim_ScriptIsComplete(interp, argv[2], &missing)); - if (missing != ' ' && argc == 4) { - Jim_SetVariable(interp, argv[3], Jim_NewStringObj(interp, &missing, 1)); - } - } - break; - - case INFO_HOSTNAME: - - return Jim_Eval(interp, "os.gethostname"); - - case INFO_NAMEOFEXECUTABLE: - - return Jim_Eval(interp, "{info nameofexecutable}"); - - case INFO_RETURNCODES: - if (argc == 2) { - int i; - Jim_Obj *listObjPtr = Jim_NewListObj(interp, NULL, 0); - - for (i = 0; jimReturnCodes[i]; i++) { - Jim_ListAppendElement(interp, listObjPtr, Jim_NewIntObj(interp, i)); - Jim_ListAppendElement(interp, listObjPtr, Jim_NewStringObj(interp, - jimReturnCodes[i], -1)); - } - - Jim_SetResult(interp, listObjPtr); - } - else if (argc == 3) { - long code; - const char *name; - - if (Jim_GetLong(interp, argv[2], &code) != JIM_OK) { - return JIM_ERR; - } - name = Jim_ReturnCode(code); - if (*name == '?') { - Jim_SetResultInt(interp, code); - } - else { - Jim_SetResultString(interp, name, -1); - } - } - else { - Jim_WrongNumArgs(interp, 2, argv, "?code?"); - return JIM_ERR; - } - break; - case INFO_REFERENCES: -#ifdef JIM_REFERENCES - return JimInfoReferences(interp, argc, argv); -#else - Jim_SetResultString(interp, "not supported", -1); - return JIM_ERR; -#endif - } - return JIM_OK; -} - - -static int Jim_ExistsCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *objPtr; - int result = 0; - - static const char * const options[] = { - "-command", "-proc", "-alias", "-var", NULL - }; - enum - { - OPT_COMMAND, OPT_PROC, OPT_ALIAS, OPT_VAR - }; - int option; - - if (argc == 2) { - option = OPT_VAR; - objPtr = argv[1]; - } - else if (argc == 3) { - if (Jim_GetEnum(interp, argv[1], options, &option, NULL, JIM_ERRMSG | JIM_ENUM_ABBREV) != JIM_OK) { - return JIM_ERR; - } - objPtr = argv[2]; - } - else { - Jim_WrongNumArgs(interp, 1, argv, "?option? name"); - return JIM_ERR; - } - - if (option == OPT_VAR) { - result = Jim_GetVariable(interp, objPtr, 0) != NULL; - } - else { - - Jim_Cmd *cmd = Jim_GetCommand(interp, objPtr, JIM_NONE); - - if (cmd) { - switch (option) { - case OPT_COMMAND: - result = 1; - break; - - case OPT_ALIAS: - result = cmd->isproc == 0 && cmd->u.native.cmdProc == JimAliasCmd; - break; - - case OPT_PROC: - result = cmd->isproc; - break; - } - } - } - Jim_SetResultBool(interp, result); - return JIM_OK; -} - - -static int Jim_SplitCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const char *str, *splitChars, *noMatchStart; - int splitLen, strLen; - Jim_Obj *resObjPtr; - int c; - int len; - - if (argc != 2 && argc != 3) { - Jim_WrongNumArgs(interp, 1, argv, "string ?splitChars?"); - return JIM_ERR; - } - - str = Jim_GetString(argv[1], &len); - if (len == 0) { - return JIM_OK; - } - strLen = Jim_Utf8Length(interp, argv[1]); - - - if (argc == 2) { - splitChars = " \n\t\r"; - splitLen = 4; - } - else { - splitChars = Jim_String(argv[2]); - splitLen = Jim_Utf8Length(interp, argv[2]); - } - - noMatchStart = str; - resObjPtr = Jim_NewListObj(interp, NULL, 0); - - - if (splitLen) { - Jim_Obj *objPtr; - while (strLen--) { - const char *sc = splitChars; - int scLen = splitLen; - int sl = utf8_tounicode(str, &c); - while (scLen--) { - int pc; - sc += utf8_tounicode(sc, &pc); - if (c == pc) { - objPtr = Jim_NewStringObj(interp, noMatchStart, (str - noMatchStart)); - Jim_ListAppendElement(interp, resObjPtr, objPtr); - noMatchStart = str + sl; - break; - } - } - str += sl; - } - objPtr = Jim_NewStringObj(interp, noMatchStart, (str - noMatchStart)); - Jim_ListAppendElement(interp, resObjPtr, objPtr); - } - else { - Jim_Obj **commonObj = NULL; -#define NUM_COMMON (128 - 9) - while (strLen--) { - int n = utf8_tounicode(str, &c); -#ifdef JIM_OPTIMIZATION - if (c >= 9 && c < 128) { - - c -= 9; - if (!commonObj) { - commonObj = Jim_Alloc(sizeof(*commonObj) * NUM_COMMON); - memset(commonObj, 0, sizeof(*commonObj) * NUM_COMMON); - } - if (!commonObj[c]) { - commonObj[c] = Jim_NewStringObj(interp, str, 1); - } - Jim_ListAppendElement(interp, resObjPtr, commonObj[c]); - str++; - continue; - } -#endif - Jim_ListAppendElement(interp, resObjPtr, Jim_NewStringObjUtf8(interp, str, 1)); - str += n; - } - Jim_Free(commonObj); - } - - Jim_SetResult(interp, resObjPtr); - return JIM_OK; -} - - -static int Jim_JoinCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const char *joinStr; - int joinStrLen; - - if (argc != 2 && argc != 3) { - Jim_WrongNumArgs(interp, 1, argv, "list ?joinString?"); - return JIM_ERR; - } - - if (argc == 2) { - joinStr = " "; - joinStrLen = 1; - } - else { - joinStr = Jim_GetString(argv[2], &joinStrLen); - } - Jim_SetResult(interp, Jim_ListJoin(interp, argv[1], joinStr, joinStrLen)); - return JIM_OK; -} - - -static int Jim_FormatCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *objPtr; - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "formatString ?arg arg ...?"); - return JIM_ERR; - } - objPtr = Jim_FormatString(interp, argv[1], argc - 2, argv + 2); - if (objPtr == NULL) - return JIM_ERR; - Jim_SetResult(interp, objPtr); - return JIM_OK; -} - - -static int Jim_ScanCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *listPtr, **outVec; - int outc, i; - - if (argc < 3) { - Jim_WrongNumArgs(interp, 1, argv, "string format ?varName varName ...?"); - return JIM_ERR; - } - if (argv[2]->typePtr != &scanFmtStringObjType) - SetScanFmtFromAny(interp, argv[2]); - if (FormatGetError(argv[2]) != 0) { - Jim_SetResultString(interp, FormatGetError(argv[2]), -1); - return JIM_ERR; - } - if (argc > 3) { - int maxPos = FormatGetMaxPos(argv[2]); - int count = FormatGetCnvCount(argv[2]); - - if (maxPos > argc - 3) { - Jim_SetResultString(interp, "\"%n$\" argument index out of range", -1); - return JIM_ERR; - } - else if (count > argc - 3) { - Jim_SetResultString(interp, "different numbers of variable names and " - "field specifiers", -1); - return JIM_ERR; - } - else if (count < argc - 3) { - Jim_SetResultString(interp, "variable is not assigned by any " - "conversion specifiers", -1); - return JIM_ERR; - } - } - listPtr = Jim_ScanString(interp, argv[1], argv[2], JIM_ERRMSG); - if (listPtr == 0) - return JIM_ERR; - if (argc > 3) { - int rc = JIM_OK; - int count = 0; - - if (listPtr != 0 && listPtr != (Jim_Obj *)EOF) { - int len = Jim_ListLength(interp, listPtr); - - if (len != 0) { - JimListGetElements(interp, listPtr, &outc, &outVec); - for (i = 0; i < outc; ++i) { - if (Jim_Length(outVec[i]) > 0) { - ++count; - if (Jim_SetVariable(interp, argv[3 + i], outVec[i]) != JIM_OK) { - rc = JIM_ERR; - } - } - } - } - Jim_FreeNewObj(interp, listPtr); - } - else { - count = -1; - } - if (rc == JIM_OK) { - Jim_SetResultInt(interp, count); - } - return rc; - } - else { - if (listPtr == (Jim_Obj *)EOF) { - Jim_SetResult(interp, Jim_NewListObj(interp, 0, 0)); - return JIM_OK; - } - Jim_SetResult(interp, listPtr); - } - return JIM_OK; -} - - -static int Jim_ErrorCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - if (argc != 2 && argc != 3) { - Jim_WrongNumArgs(interp, 1, argv, "message ?stacktrace?"); - return JIM_ERR; - } - Jim_SetResult(interp, argv[1]); - if (argc == 3) { - JimSetStackTrace(interp, argv[2]); - return JIM_ERR; - } - interp->addStackTrace++; - return JIM_ERR; -} - - -static int Jim_LrangeCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *objPtr; - - if (argc != 4) { - Jim_WrongNumArgs(interp, 1, argv, "list first last"); - return JIM_ERR; - } - if ((objPtr = Jim_ListRange(interp, argv[1], argv[2], argv[3])) == NULL) - return JIM_ERR; - Jim_SetResult(interp, objPtr); - return JIM_OK; -} - - -static int Jim_LrepeatCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *objPtr; - long count; - - if (argc < 2 || Jim_GetLong(interp, argv[1], &count) != JIM_OK || count < 0) { - Jim_WrongNumArgs(interp, 1, argv, "count ?value ...?"); - return JIM_ERR; - } - - if (count == 0 || argc == 2) { - return JIM_OK; - } - - argc -= 2; - argv += 2; - - objPtr = Jim_NewListObj(interp, argv, argc); - while (--count) { - ListInsertElements(objPtr, -1, argc, argv); - } - - Jim_SetResult(interp, objPtr); - return JIM_OK; -} - -char **Jim_GetEnviron(void) -{ -#if defined(HAVE__NSGETENVIRON) - return *_NSGetEnviron(); -#else - #if !defined(NO_ENVIRON_EXTERN) - extern char **environ; - #endif - - return environ; -#endif -} - -void Jim_SetEnviron(char **env) -{ -#if defined(HAVE__NSGETENVIRON) - *_NSGetEnviron() = env; -#else - #if !defined(NO_ENVIRON_EXTERN) - extern char **environ; - #endif - - environ = env; -#endif -} - - -static int Jim_EnvCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const char *key; - const char *val; - - if (argc == 1) { - char **e = Jim_GetEnviron(); - - int i; - Jim_Obj *listObjPtr = Jim_NewListObj(interp, NULL, 0); - - for (i = 0; e[i]; i++) { - const char *equals = strchr(e[i], '='); - - if (equals) { - Jim_ListAppendElement(interp, listObjPtr, Jim_NewStringObj(interp, e[i], - equals - e[i])); - Jim_ListAppendElement(interp, listObjPtr, Jim_NewStringObj(interp, equals + 1, -1)); - } - } - - Jim_SetResult(interp, listObjPtr); - return JIM_OK; - } - - if (argc < 2) { - Jim_WrongNumArgs(interp, 1, argv, "varName ?default?"); - return JIM_ERR; - } - key = Jim_String(argv[1]); - val = getenv(key); - if (val == NULL) { - if (argc < 3) { - Jim_SetResultFormatted(interp, "environment variable \"%#s\" does not exist", argv[1]); - return JIM_ERR; - } - val = Jim_String(argv[2]); - } - Jim_SetResult(interp, Jim_NewStringObj(interp, val, -1)); - return JIM_OK; -} - - -static int Jim_SourceCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - int retval; - - if (argc != 2) { - Jim_WrongNumArgs(interp, 1, argv, "fileName"); - return JIM_ERR; - } - retval = Jim_EvalFile(interp, Jim_String(argv[1])); - if (retval == JIM_RETURN) - return JIM_OK; - return retval; -} - - -static int Jim_LreverseCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - Jim_Obj *revObjPtr, **ele; - int len; - - if (argc != 2) { - Jim_WrongNumArgs(interp, 1, argv, "list"); - return JIM_ERR; - } - JimListGetElements(interp, argv[1], &len, &ele); - len--; - revObjPtr = Jim_NewListObj(interp, NULL, 0); - while (len >= 0) - ListAppendElement(revObjPtr, ele[len--]); - Jim_SetResult(interp, revObjPtr); - return JIM_OK; -} - -static int JimRangeLen(jim_wide start, jim_wide end, jim_wide step) -{ - jim_wide len; - - if (step == 0) - return -1; - if (start == end) - return 0; - else if (step > 0 && start > end) - return -1; - else if (step < 0 && end > start) - return -1; - len = end - start; - if (len < 0) - len = -len; - if (step < 0) - step = -step; - len = 1 + ((len - 1) / step); - if (len > INT_MAX) - len = INT_MAX; - return (int)((len < 0) ? -1 : len); -} - - -static int Jim_RangeCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - jim_wide start = 0, end, step = 1; - int len, i; - Jim_Obj *objPtr; - - if (argc < 2 || argc > 4) { - Jim_WrongNumArgs(interp, 1, argv, "?start? end ?step?"); - return JIM_ERR; - } - if (argc == 2) { - if (Jim_GetWide(interp, argv[1], &end) != JIM_OK) - return JIM_ERR; - } - else { - if (Jim_GetWide(interp, argv[1], &start) != JIM_OK || - Jim_GetWide(interp, argv[2], &end) != JIM_OK) - return JIM_ERR; - if (argc == 4 && Jim_GetWide(interp, argv[3], &step) != JIM_OK) - return JIM_ERR; - } - if ((len = JimRangeLen(start, end, step)) == -1) { - Jim_SetResultString(interp, "Invalid (infinite?) range specified", -1); - return JIM_ERR; - } - objPtr = Jim_NewListObj(interp, NULL, 0); - for (i = 0; i < len; i++) - ListAppendElement(objPtr, Jim_NewIntObj(interp, start + i * step)); - Jim_SetResult(interp, objPtr); - return JIM_OK; -} - - -static int Jim_RandCoreCommand(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - jim_wide min = 0, max = 0, len, maxMul; - - if (argc < 1 || argc > 3) { - Jim_WrongNumArgs(interp, 1, argv, "?min? max"); - return JIM_ERR; - } - if (argc == 1) { - max = JIM_WIDE_MAX; - } else if (argc == 2) { - if (Jim_GetWide(interp, argv[1], &max) != JIM_OK) - return JIM_ERR; - } else if (argc == 3) { - if (Jim_GetWide(interp, argv[1], &min) != JIM_OK || - Jim_GetWide(interp, argv[2], &max) != JIM_OK) - return JIM_ERR; - } - len = max-min; - if (len < 0) { - Jim_SetResultString(interp, "Invalid arguments (max < min)", -1); - return JIM_ERR; - } - maxMul = JIM_WIDE_MAX - (len ? (JIM_WIDE_MAX%len) : 0); - while (1) { - jim_wide r; - - JimRandomBytes(interp, &r, sizeof(jim_wide)); - if (r < 0 || r >= maxMul) continue; - r = (len == 0) ? 0 : r%len; - Jim_SetResultInt(interp, min+r); - return JIM_OK; - } -} - -static const struct { - const char *name; - Jim_CmdProc *cmdProc; -} Jim_CoreCommandsTable[] = { - {"alias", Jim_AliasCoreCommand}, - {"set", Jim_SetCoreCommand}, - {"unset", Jim_UnsetCoreCommand}, - {"puts", Jim_PutsCoreCommand}, - {"+", Jim_AddCoreCommand}, - {"*", Jim_MulCoreCommand}, - {"-", Jim_SubCoreCommand}, - {"/", Jim_DivCoreCommand}, - {"incr", Jim_IncrCoreCommand}, - {"while", Jim_WhileCoreCommand}, - {"loop", Jim_LoopCoreCommand}, - {"for", Jim_ForCoreCommand}, - {"foreach", Jim_ForeachCoreCommand}, - {"lmap", Jim_LmapCoreCommand}, - {"lassign", Jim_LassignCoreCommand}, - {"if", Jim_IfCoreCommand}, - {"switch", Jim_SwitchCoreCommand}, - {"list", Jim_ListCoreCommand}, - {"lindex", Jim_LindexCoreCommand}, - {"lset", Jim_LsetCoreCommand}, - {"lsearch", Jim_LsearchCoreCommand}, - {"llength", Jim_LlengthCoreCommand}, - {"lappend", Jim_LappendCoreCommand}, - {"linsert", Jim_LinsertCoreCommand}, - {"lreplace", Jim_LreplaceCoreCommand}, - {"lsort", Jim_LsortCoreCommand}, - {"append", Jim_AppendCoreCommand}, - {"debug", Jim_DebugCoreCommand}, - {"eval", Jim_EvalCoreCommand}, - {"uplevel", Jim_UplevelCoreCommand}, - {"expr", Jim_ExprCoreCommand}, - {"break", Jim_BreakCoreCommand}, - {"continue", Jim_ContinueCoreCommand}, - {"proc", Jim_ProcCoreCommand}, - {"concat", Jim_ConcatCoreCommand}, - {"return", Jim_ReturnCoreCommand}, - {"upvar", Jim_UpvarCoreCommand}, - {"global", Jim_GlobalCoreCommand}, - {"string", Jim_StringCoreCommand}, - {"time", Jim_TimeCoreCommand}, - {"exit", Jim_ExitCoreCommand}, - {"catch", Jim_CatchCoreCommand}, -#ifdef JIM_REFERENCES - {"ref", Jim_RefCoreCommand}, - {"getref", Jim_GetrefCoreCommand}, - {"setref", Jim_SetrefCoreCommand}, - {"finalize", Jim_FinalizeCoreCommand}, - {"collect", Jim_CollectCoreCommand}, -#endif - {"rename", Jim_RenameCoreCommand}, - {"dict", Jim_DictCoreCommand}, - {"subst", Jim_SubstCoreCommand}, - {"info", Jim_InfoCoreCommand}, - {"exists", Jim_ExistsCoreCommand}, - {"split", Jim_SplitCoreCommand}, - {"join", Jim_JoinCoreCommand}, - {"format", Jim_FormatCoreCommand}, - {"scan", Jim_ScanCoreCommand}, - {"error", Jim_ErrorCoreCommand}, - {"lrange", Jim_LrangeCoreCommand}, - {"lrepeat", Jim_LrepeatCoreCommand}, - {"env", Jim_EnvCoreCommand}, - {"source", Jim_SourceCoreCommand}, - {"lreverse", Jim_LreverseCoreCommand}, - {"range", Jim_RangeCoreCommand}, - {"rand", Jim_RandCoreCommand}, - {"tailcall", Jim_TailcallCoreCommand}, - {"local", Jim_LocalCoreCommand}, - {"upcall", Jim_UpcallCoreCommand}, - {"apply", Jim_ApplyCoreCommand}, - {NULL, NULL}, -}; - -void Jim_RegisterCoreCommands(Jim_Interp *interp) -{ - int i = 0; - - while (Jim_CoreCommandsTable[i].name != NULL) { - Jim_CreateCommand(interp, - Jim_CoreCommandsTable[i].name, Jim_CoreCommandsTable[i].cmdProc, NULL, NULL); - i++; - } -} - -void Jim_MakeErrorMessage(Jim_Interp *interp) -{ - Jim_Obj *argv[2]; - - argv[0] = Jim_NewStringObj(interp, "errorInfo", -1); - argv[1] = interp->result; - - Jim_EvalObjVector(interp, 2, argv); -} - -static char **JimSortStringTable(const char *const *tablePtr) -{ - int count; - char **tablePtrSorted; - - - for (count = 0; tablePtr[count]; count++) { - } - - - tablePtrSorted = Jim_Alloc(sizeof(char *) * (count + 1)); - memcpy(tablePtrSorted, tablePtr, sizeof(char *) * count); - qsort(tablePtrSorted, count, sizeof(char *), qsortCompareStringPointers); - tablePtrSorted[count] = NULL; - - return tablePtrSorted; -} - -static void JimSetFailedEnumResult(Jim_Interp *interp, const char *arg, const char *badtype, - const char *prefix, const char *const *tablePtr, const char *name) -{ - char **tablePtrSorted; - int i; - - if (name == NULL) { - name = "option"; - } - - Jim_SetResultFormatted(interp, "%s%s \"%s\": must be ", badtype, name, arg); - tablePtrSorted = JimSortStringTable(tablePtr); - for (i = 0; tablePtrSorted[i]; i++) { - if (tablePtrSorted[i + 1] == NULL && i > 0) { - Jim_AppendString(interp, Jim_GetResult(interp), "or ", -1); - } - Jim_AppendStrings(interp, Jim_GetResult(interp), prefix, tablePtrSorted[i], NULL); - if (tablePtrSorted[i + 1]) { - Jim_AppendString(interp, Jim_GetResult(interp), ", ", -1); - } - } - Jim_Free(tablePtrSorted); -} - - -int Jim_CheckShowCommands(Jim_Interp *interp, Jim_Obj *objPtr, const char *const *tablePtr) -{ - if (Jim_CompareStringImmediate(interp, objPtr, "-commands")) { - int i; - char **tablePtrSorted = JimSortStringTable(tablePtr); - Jim_SetResult(interp, Jim_NewListObj(interp, NULL, 0)); - for (i = 0; tablePtrSorted[i]; i++) { - Jim_ListAppendElement(interp, Jim_GetResult(interp), Jim_NewStringObj(interp, tablePtrSorted[i], -1)); - } - Jim_Free(tablePtrSorted); - return JIM_OK; - } - return JIM_ERR; -} - -static const Jim_ObjType getEnumObjType = { - "get-enum", - NULL, - NULL, - NULL, - JIM_TYPE_REFERENCES -}; - -int Jim_GetEnum(Jim_Interp *interp, Jim_Obj *objPtr, - const char *const *tablePtr, int *indexPtr, const char *name, int flags) -{ - const char *bad = "bad "; - const char *const *entryPtr = NULL; - int i; - int match = -1; - int arglen; - const char *arg; - - if (objPtr->typePtr == &getEnumObjType) { - if (objPtr->internalRep.ptrIntValue.ptr == tablePtr && objPtr->internalRep.ptrIntValue.int1 == flags) { - *indexPtr = objPtr->internalRep.ptrIntValue.int2; - return JIM_OK; - } - } - - arg = Jim_GetString(objPtr, &arglen); - - *indexPtr = -1; - - for (entryPtr = tablePtr, i = 0; *entryPtr != NULL; entryPtr++, i++) { - if (Jim_CompareStringImmediate(interp, objPtr, *entryPtr)) { - - match = i; - goto found; - } - if (flags & JIM_ENUM_ABBREV) { - if (strncmp(arg, *entryPtr, arglen) == 0) { - if (*arg == '-' && arglen == 1) { - break; - } - if (match >= 0) { - bad = "ambiguous "; - goto ambiguous; - } - match = i; - } - } - } - - - if (match >= 0) { - found: - - Jim_FreeIntRep(interp, objPtr); - objPtr->typePtr = &getEnumObjType; - objPtr->internalRep.ptrIntValue.ptr = (void *)tablePtr; - objPtr->internalRep.ptrIntValue.int1 = flags; - objPtr->internalRep.ptrIntValue.int2 = match; - - *indexPtr = match; - return JIM_OK; - } - - ambiguous: - if (flags & JIM_ERRMSG) { - JimSetFailedEnumResult(interp, arg, bad, "", tablePtr, name); - } - return JIM_ERR; -} - -int Jim_FindByName(const char *name, const char * const array[], size_t len) -{ - int i; - - for (i = 0; i < (int)len; i++) { - if (array[i] && strcmp(array[i], name) == 0) { - return i; - } - } - return -1; -} - -int Jim_IsDict(Jim_Obj *objPtr) -{ - return objPtr->typePtr == &dictObjType; -} - -int Jim_IsList(Jim_Obj *objPtr) -{ - return objPtr->typePtr == &listObjType; -} - -void Jim_SetResultFormatted(Jim_Interp *interp, const char *format, ...) -{ - - int len = strlen(format); - int extra = 0; - int n = 0; - const char *params[5]; - int nobjparam = 0; - Jim_Obj *objparam[5]; - char *buf; - va_list args; - int i; - - va_start(args, format); - - for (i = 0; i < len && n < 5; i++) { - int l; - - if (strncmp(format + i, "%s", 2) == 0) { - params[n] = va_arg(args, char *); - - l = strlen(params[n]); - } - else if (strncmp(format + i, "%#s", 3) == 0) { - Jim_Obj *objPtr = va_arg(args, Jim_Obj *); - - params[n] = Jim_GetString(objPtr, &l); - objparam[nobjparam++] = objPtr; - Jim_IncrRefCount(objPtr); - } - else { - if (format[i] == '%') { - i++; - } - continue; - } - n++; - extra += l; - } - - len += extra; - buf = Jim_Alloc(len + 1); - len = snprintf(buf, len + 1, format, params[0], params[1], params[2], params[3], params[4]); - - va_end(args); - - Jim_SetResult(interp, Jim_NewStringObjNoAlloc(interp, buf, len)); - - for (i = 0; i < nobjparam; i++) { - Jim_DecrRefCount(interp, objparam[i]); - } -} - - -#ifndef jim_ext_package -int Jim_PackageProvide(Jim_Interp *interp, const char *name, const char *ver, int flags) -{ - return JIM_OK; -} -#endif -#ifndef jim_ext_aio -FILE *Jim_AioFilehandle(Jim_Interp *interp, Jim_Obj *fhObj) -{ - Jim_SetResultString(interp, "aio not enabled", -1); - return NULL; -} -#endif - - -#include -#include - - -static int subcmd_null(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - - return JIM_OK; -} - -static const jim_subcmd_type dummy_subcmd = { - "dummy", NULL, subcmd_null, 0, 0, JIM_MODFLAG_HIDDEN -}; - -static void add_commands(Jim_Interp *interp, const jim_subcmd_type * ct, const char *sep) -{ - const char *s = ""; - - for (; ct->cmd; ct++) { - if (!(ct->flags & JIM_MODFLAG_HIDDEN)) { - Jim_AppendStrings(interp, Jim_GetResult(interp), s, ct->cmd, NULL); - s = sep; - } - } -} - -static void bad_subcmd(Jim_Interp *interp, const jim_subcmd_type * command_table, const char *type, - Jim_Obj *cmd, Jim_Obj *subcmd) -{ - Jim_SetResultFormatted(interp, "%#s, %s command \"%#s\": should be ", cmd, type, subcmd); - add_commands(interp, command_table, ", "); -} - -static void show_cmd_usage(Jim_Interp *interp, const jim_subcmd_type * command_table, int argc, - Jim_Obj *const *argv) -{ - Jim_SetResultFormatted(interp, "Usage: \"%#s command ... \", where command is one of: ", argv[0]); - add_commands(interp, command_table, ", "); -} - -static void add_cmd_usage(Jim_Interp *interp, const jim_subcmd_type * ct, Jim_Obj *cmd) -{ - if (cmd) { - Jim_AppendStrings(interp, Jim_GetResult(interp), Jim_String(cmd), " ", NULL); - } - Jim_AppendStrings(interp, Jim_GetResult(interp), ct->cmd, NULL); - if (ct->args && *ct->args) { - Jim_AppendStrings(interp, Jim_GetResult(interp), " ", ct->args, NULL); - } -} - -static void set_wrong_args(Jim_Interp *interp, const jim_subcmd_type * command_table, Jim_Obj *subcmd) -{ - Jim_SetResultString(interp, "wrong # args: should be \"", -1); - add_cmd_usage(interp, command_table, subcmd); - Jim_AppendStrings(interp, Jim_GetResult(interp), "\"", NULL); -} - -static const Jim_ObjType subcmdLookupObjType = { - "subcmd-lookup", - NULL, - NULL, - NULL, - JIM_TYPE_REFERENCES -}; - -const jim_subcmd_type *Jim_ParseSubCmd(Jim_Interp *interp, const jim_subcmd_type * command_table, - int argc, Jim_Obj *const *argv) -{ - const jim_subcmd_type *ct; - const jim_subcmd_type *partial = 0; - int cmdlen; - Jim_Obj *cmd; - const char *cmdstr; - int help = 0; - - if (argc < 2) { - Jim_SetResultFormatted(interp, "wrong # args: should be \"%#s command ...\"\n" - "Use \"%#s -help ?command?\" for help", argv[0], argv[0]); - return 0; - } - - cmd = argv[1]; - - - if (cmd->typePtr == &subcmdLookupObjType) { - if (cmd->internalRep.ptrIntValue.ptr == command_table) { - ct = command_table + cmd->internalRep.ptrIntValue.int1; - goto found; - } - } - - - if (Jim_CompareStringImmediate(interp, cmd, "-help")) { - if (argc == 2) { - - show_cmd_usage(interp, command_table, argc, argv); - return &dummy_subcmd; - } - help = 1; - - - cmd = argv[2]; - } - - - if (Jim_CompareStringImmediate(interp, cmd, "-commands")) { - - Jim_SetResult(interp, Jim_NewEmptyStringObj(interp)); - add_commands(interp, command_table, " "); - return &dummy_subcmd; - } - - cmdstr = Jim_GetString(cmd, &cmdlen); - - for (ct = command_table; ct->cmd; ct++) { - if (Jim_CompareStringImmediate(interp, cmd, ct->cmd)) { - - break; - } - if (strncmp(cmdstr, ct->cmd, cmdlen) == 0) { - if (partial) { - - if (help) { - - show_cmd_usage(interp, command_table, argc, argv); - return &dummy_subcmd; - } - bad_subcmd(interp, command_table, "ambiguous", argv[0], argv[1 + help]); - return 0; - } - partial = ct; - } - continue; - } - - - if (partial && !ct->cmd) { - ct = partial; - } - - if (!ct->cmd) { - - if (help) { - - show_cmd_usage(interp, command_table, argc, argv); - return &dummy_subcmd; - } - bad_subcmd(interp, command_table, "unknown", argv[0], argv[1 + help]); - return 0; - } - - if (help) { - Jim_SetResultString(interp, "Usage: ", -1); - - add_cmd_usage(interp, ct, argv[0]); - return &dummy_subcmd; - } - - - Jim_FreeIntRep(interp, cmd); - cmd->typePtr = &subcmdLookupObjType; - cmd->internalRep.ptrIntValue.ptr = (void *)command_table; - cmd->internalRep.ptrIntValue.int1 = ct - command_table; - -found: - - if (argc - 2 < ct->minargs || (ct->maxargs >= 0 && argc - 2 > ct->maxargs)) { - Jim_SetResultString(interp, "wrong # args: should be \"", -1); - - add_cmd_usage(interp, ct, argv[0]); - Jim_AppendStrings(interp, Jim_GetResult(interp), "\"", NULL); - - return 0; - } - - - return ct; -} - -int Jim_CallSubCmd(Jim_Interp *interp, const jim_subcmd_type * ct, int argc, Jim_Obj *const *argv) -{ - int ret = JIM_ERR; - - if (ct) { - if (ct->flags & JIM_MODFLAG_FULLARGV) { - ret = ct->function(interp, argc, argv); - } - else { - ret = ct->function(interp, argc - 2, argv + 2); - } - if (ret < 0) { - set_wrong_args(interp, ct, argv[0]); - ret = JIM_ERR; - } - } - return ret; -} - -int Jim_SubCmdProc(Jim_Interp *interp, int argc, Jim_Obj *const *argv) -{ - const jim_subcmd_type *ct = - Jim_ParseSubCmd(interp, (const jim_subcmd_type *)Jim_CmdPrivData(interp), argc, argv); - - return Jim_CallSubCmd(interp, ct, argc, argv); -} - -#include -#include -#include -#include -#include - - -int utf8_fromunicode(char *p, unsigned uc) -{ - if (uc <= 0x7f) { - *p = uc; - return 1; - } - else if (uc <= 0x7ff) { - *p++ = 0xc0 | ((uc & 0x7c0) >> 6); - *p = 0x80 | (uc & 0x3f); - return 2; - } - else if (uc <= 0xffff) { - *p++ = 0xe0 | ((uc & 0xf000) >> 12); - *p++ = 0x80 | ((uc & 0xfc0) >> 6); - *p = 0x80 | (uc & 0x3f); - return 3; - } - - else { - *p++ = 0xf0 | ((uc & 0x1c0000) >> 18); - *p++ = 0x80 | ((uc & 0x3f000) >> 12); - *p++ = 0x80 | ((uc & 0xfc0) >> 6); - *p = 0x80 | (uc & 0x3f); - return 4; - } -} - -#include -#include - - -#define JIM_INTEGER_SPACE 24 -#define MAX_FLOAT_WIDTH 320 - -Jim_Obj *Jim_FormatString(Jim_Interp *interp, Jim_Obj *fmtObjPtr, int objc, Jim_Obj *const *objv) -{ - const char *span, *format, *formatEnd, *msg; - int numBytes = 0, objIndex = 0, gotXpg = 0, gotSequential = 0; - static const char * const mixedXPG = - "cannot mix \"%\" and \"%n$\" conversion specifiers"; - static const char * const badIndex[2] = { - "not enough arguments for all format specifiers", - "\"%n$\" argument index out of range" - }; - int formatLen; - Jim_Obj *resultPtr; - - char *num_buffer = NULL; - int num_buffer_size = 0; - - span = format = Jim_GetString(fmtObjPtr, &formatLen); - formatEnd = format + formatLen; - resultPtr = Jim_NewEmptyStringObj(interp); - - while (format != formatEnd) { - char *end; - int gotMinus, sawFlag; - int gotPrecision, useShort; - long width, precision; - int newXpg; - int ch; - int step; - int doubleType; - char pad = ' '; - char spec[2*JIM_INTEGER_SPACE + 12]; - char *p; - - int formatted_chars; - int formatted_bytes; - const char *formatted_buf; - - step = utf8_tounicode(format, &ch); - format += step; - if (ch != '%') { - numBytes += step; - continue; - } - if (numBytes) { - Jim_AppendString(interp, resultPtr, span, numBytes); - numBytes = 0; - } - - - step = utf8_tounicode(format, &ch); - if (ch == '%') { - span = format; - numBytes = step; - format += step; - continue; - } - - - newXpg = 0; - if (isdigit(ch)) { - int position = strtoul(format, &end, 10); - if (*end == '$') { - newXpg = 1; - objIndex = position - 1; - format = end + 1; - step = utf8_tounicode(format, &ch); - } - } - if (newXpg) { - if (gotSequential) { - msg = mixedXPG; - goto errorMsg; - } - gotXpg = 1; - } else { - if (gotXpg) { - msg = mixedXPG; - goto errorMsg; - } - gotSequential = 1; - } - if ((objIndex < 0) || (objIndex >= objc)) { - msg = badIndex[gotXpg]; - goto errorMsg; - } - - p = spec; - *p++ = '%'; - - gotMinus = 0; - sawFlag = 1; - do { - switch (ch) { - case '-': - gotMinus = 1; - break; - case '0': - pad = ch; - break; - case ' ': - case '+': - case '#': - break; - default: - sawFlag = 0; - continue; - } - *p++ = ch; - format += step; - step = utf8_tounicode(format, &ch); - - } while (sawFlag && (p - spec <= 5)); - - - width = 0; - if (isdigit(ch)) { - width = strtoul(format, &end, 10); - format = end; - step = utf8_tounicode(format, &ch); - } else if (ch == '*') { - if (objIndex >= objc - 1) { - msg = badIndex[gotXpg]; - goto errorMsg; - } - if (Jim_GetLong(interp, objv[objIndex], &width) != JIM_OK) { - goto error; - } - if (width < 0) { - width = -width; - if (!gotMinus) { - *p++ = '-'; - gotMinus = 1; - } - } - objIndex++; - format += step; - step = utf8_tounicode(format, &ch); - } - - - gotPrecision = precision = 0; - if (ch == '.') { - gotPrecision = 1; - format += step; - step = utf8_tounicode(format, &ch); - } - if (isdigit(ch)) { - precision = strtoul(format, &end, 10); - format = end; - step = utf8_tounicode(format, &ch); - } else if (ch == '*') { - if (objIndex >= objc - 1) { - msg = badIndex[gotXpg]; - goto errorMsg; - } - if (Jim_GetLong(interp, objv[objIndex], &precision) != JIM_OK) { - goto error; - } - - - if (precision < 0) { - precision = 0; - } - objIndex++; - format += step; - step = utf8_tounicode(format, &ch); - } - - - useShort = 0; - if (ch == 'h') { - useShort = 1; - format += step; - step = utf8_tounicode(format, &ch); - } else if (ch == 'l') { - - format += step; - step = utf8_tounicode(format, &ch); - if (ch == 'l') { - format += step; - step = utf8_tounicode(format, &ch); - } - } - - format += step; - span = format; - - - if (ch == 'i') { - ch = 'd'; - } - - doubleType = 0; - - switch (ch) { - case '\0': - msg = "format string ended in middle of field specifier"; - goto errorMsg; - case 's': { - formatted_buf = Jim_GetString(objv[objIndex], &formatted_bytes); - formatted_chars = Jim_Utf8Length(interp, objv[objIndex]); - if (gotPrecision && (precision < formatted_chars)) { - - formatted_chars = precision; - formatted_bytes = utf8_index(formatted_buf, precision); - } - break; - } - case 'c': { - jim_wide code; - - if (Jim_GetWide(interp, objv[objIndex], &code) != JIM_OK) { - goto error; - } - - formatted_bytes = utf8_getchars(spec, code); - formatted_buf = spec; - formatted_chars = 1; - break; - } - case 'b': { - unsigned jim_wide w; - int length; - int i; - int j; - - if (Jim_GetWide(interp, objv[objIndex], (jim_wide *)&w) != JIM_OK) { - goto error; - } - length = sizeof(w) * 8; - - - - if (num_buffer_size < length + 1) { - num_buffer_size = length + 1; - num_buffer = Jim_Realloc(num_buffer, num_buffer_size); - } - - j = 0; - for (i = length; i > 0; ) { - i--; - if (w & ((unsigned jim_wide)1 << i)) { - num_buffer[j++] = '1'; - } - else if (j || i == 0) { - num_buffer[j++] = '0'; - } - } - num_buffer[j] = 0; - formatted_chars = formatted_bytes = j; - formatted_buf = num_buffer; - break; - } - - case 'e': - case 'E': - case 'f': - case 'g': - case 'G': - doubleType = 1; - - case 'd': - case 'u': - case 'o': - case 'x': - case 'X': { - jim_wide w; - double d; - int length; - - - if (width) { - p += sprintf(p, "%ld", width); - } - if (gotPrecision) { - p += sprintf(p, ".%ld", precision); - } - - - if (doubleType) { - if (Jim_GetDouble(interp, objv[objIndex], &d) != JIM_OK) { - goto error; - } - length = MAX_FLOAT_WIDTH; - } - else { - if (Jim_GetWide(interp, objv[objIndex], &w) != JIM_OK) { - goto error; - } - length = JIM_INTEGER_SPACE; - if (useShort) { - if (ch == 'd') { - w = (short)w; - } - else { - w = (unsigned short)w; - } - } - *p++ = 'l'; -#ifdef HAVE_LONG_LONG - if (sizeof(long long) == sizeof(jim_wide)) { - *p++ = 'l'; - } -#endif - } - - *p++ = (char) ch; - *p = '\0'; - - - if (width > 10000 || length > 10000 || precision > 10000) { - Jim_SetResultString(interp, "format too long", -1); - goto error; - } - - - - if (width > length) { - length = width; - } - if (gotPrecision) { - length += precision; - } - - - if (num_buffer_size < length + 1) { - num_buffer_size = length + 1; - num_buffer = Jim_Realloc(num_buffer, num_buffer_size); - } - - if (doubleType) { - snprintf(num_buffer, length + 1, spec, d); - } - else { - formatted_bytes = snprintf(num_buffer, length + 1, spec, w); - } - formatted_chars = formatted_bytes = strlen(num_buffer); - formatted_buf = num_buffer; - break; - } - - default: { - - spec[0] = ch; - spec[1] = '\0'; - Jim_SetResultFormatted(interp, "bad field specifier \"%s\"", spec); - goto error; - } - } - - if (!gotMinus) { - while (formatted_chars < width) { - Jim_AppendString(interp, resultPtr, &pad, 1); - formatted_chars++; - } - } - - Jim_AppendString(interp, resultPtr, formatted_buf, formatted_bytes); - - while (formatted_chars < width) { - Jim_AppendString(interp, resultPtr, &pad, 1); - formatted_chars++; - } - - objIndex += gotSequential; - } - if (numBytes) { - Jim_AppendString(interp, resultPtr, span, numBytes); - } - - Jim_Free(num_buffer); - return resultPtr; - - errorMsg: - Jim_SetResultString(interp, msg, -1); - error: - Jim_FreeNewObj(interp, resultPtr); - Jim_Free(num_buffer); - return NULL; -} - - -#if defined(JIM_REGEXP) -#include -#include -#include -#include - - - -#define REG_MAX_PAREN 100 - - - -#define END 0 -#define BOL 1 -#define EOL 2 -#define ANY 3 -#define ANYOF 4 -#define ANYBUT 5 -#define BRANCH 6 -#define BACK 7 -#define EXACTLY 8 -#define NOTHING 9 -#define REP 10 -#define REPMIN 11 -#define REPX 12 -#define REPXMIN 13 -#define BOLX 14 -#define EOLX 15 -#define WORDA 16 -#define WORDZ 17 - -#define OPENNC 1000 -#define OPEN 1001 - - - - -#define CLOSENC 2000 -#define CLOSE 2001 -#define CLOSE_END (CLOSE+REG_MAX_PAREN) - -#define REG_MAGIC 0xFADED00D - - -#define OP(preg, p) (preg->program[p]) -#define NEXT(preg, p) (preg->program[p + 1]) -#define OPERAND(p) ((p) + 2) - - - - -#define FAIL(R,M) { (R)->err = (M); return (M); } -#define ISMULT(c) ((c) == '*' || (c) == '+' || (c) == '?' || (c) == '{') -#define META "^$.[()|?{+*" - -#define HASWIDTH 1 -#define SIMPLE 2 -#define SPSTART 4 -#define WORST 0 - -#define MAX_REP_COUNT 1000000 - -static int reg(regex_t *preg, int paren, int *flagp ); -static int regpiece(regex_t *preg, int *flagp ); -static int regbranch(regex_t *preg, int *flagp ); -static int regatom(regex_t *preg, int *flagp ); -static int regnode(regex_t *preg, int op ); -static int regnext(regex_t *preg, int p ); -static void regc(regex_t *preg, int b ); -static int reginsert(regex_t *preg, int op, int size, int opnd ); -static void regtail(regex_t *preg, int p, int val); -static void regoptail(regex_t *preg, int p, int val ); -static int regopsize(regex_t *preg, int p ); - -static int reg_range_find(const int *string, int c); -static const char *str_find(const char *string, int c, int nocase); -static int prefix_cmp(const int *prog, int proglen, const char *string, int nocase); - - -#ifdef DEBUG -static int regnarrate = 0; -static void regdump(regex_t *preg); -static const char *regprop( int op ); -#endif - - -static int str_int_len(const int *seq) -{ - int n = 0; - while (*seq++) { - n++; - } - return n; -} - -int regcomp(regex_t *preg, const char *exp, int cflags) -{ - int scan; - int longest; - unsigned len; - int flags; - -#ifdef DEBUG - fprintf(stderr, "Compiling: '%s'\n", exp); -#endif - memset(preg, 0, sizeof(*preg)); - - if (exp == NULL) - FAIL(preg, REG_ERR_NULL_ARGUMENT); - - - preg->cflags = cflags; - preg->regparse = exp; - - - preg->proglen = (strlen(exp) + 1) * 5; - preg->program = malloc(preg->proglen * sizeof(int)); - if (preg->program == NULL) - FAIL(preg, REG_ERR_NOMEM); - - regc(preg, REG_MAGIC); - if (reg(preg, 0, &flags) == 0) { - return preg->err; - } - - - if (preg->re_nsub >= REG_MAX_PAREN) - FAIL(preg,REG_ERR_TOO_BIG); - - - preg->regstart = 0; - preg->reganch = 0; - preg->regmust = 0; - preg->regmlen = 0; - scan = 1; - if (OP(preg, regnext(preg, scan)) == END) { - scan = OPERAND(scan); - - - if (OP(preg, scan) == EXACTLY) { - preg->regstart = preg->program[OPERAND(scan)]; - } - else if (OP(preg, scan) == BOL) - preg->reganch++; - - if (flags&SPSTART) { - longest = 0; - len = 0; - for (; scan != 0; scan = regnext(preg, scan)) { - if (OP(preg, scan) == EXACTLY) { - int plen = str_int_len(preg->program + OPERAND(scan)); - if (plen >= len) { - longest = OPERAND(scan); - len = plen; - } - } - } - preg->regmust = longest; - preg->regmlen = len; - } - } - -#ifdef DEBUG - regdump(preg); -#endif - - return 0; -} - -static int reg(regex_t *preg, int paren, int *flagp ) -{ - int ret; - int br; - int ender; - int parno = 0; - int flags; - - *flagp = HASWIDTH; - - - if (paren) { - if (preg->regparse[0] == '?' && preg->regparse[1] == ':') { - - preg->regparse += 2; - parno = -1; - } - else { - parno = ++preg->re_nsub; - } - ret = regnode(preg, OPEN+parno); - } else - ret = 0; - - - br = regbranch(preg, &flags); - if (br == 0) - return 0; - if (ret != 0) - regtail(preg, ret, br); - else - ret = br; - if (!(flags&HASWIDTH)) - *flagp &= ~HASWIDTH; - *flagp |= flags&SPSTART; - while (*preg->regparse == '|') { - preg->regparse++; - br = regbranch(preg, &flags); - if (br == 0) - return 0; - regtail(preg, ret, br); - if (!(flags&HASWIDTH)) - *flagp &= ~HASWIDTH; - *flagp |= flags&SPSTART; - } - - - ender = regnode(preg, (paren) ? CLOSE+parno : END); - regtail(preg, ret, ender); - - - for (br = ret; br != 0; br = regnext(preg, br)) - regoptail(preg, br, ender); - - - if (paren && *preg->regparse++ != ')') { - preg->err = REG_ERR_UNMATCHED_PAREN; - return 0; - } else if (!paren && *preg->regparse != '\0') { - if (*preg->regparse == ')') { - preg->err = REG_ERR_UNMATCHED_PAREN; - return 0; - } else { - preg->err = REG_ERR_JUNK_ON_END; - return 0; - } - } - - return(ret); -} - -static int regbranch(regex_t *preg, int *flagp ) -{ - int ret; - int chain; - int latest; - int flags; - - *flagp = WORST; - - ret = regnode(preg, BRANCH); - chain = 0; - while (*preg->regparse != '\0' && *preg->regparse != ')' && - *preg->regparse != '|') { - latest = regpiece(preg, &flags); - if (latest == 0) - return 0; - *flagp |= flags&HASWIDTH; - if (chain == 0) { - *flagp |= flags&SPSTART; - } - else { - regtail(preg, chain, latest); - } - chain = latest; - } - if (chain == 0) - (void) regnode(preg, NOTHING); - - return(ret); -} - -static int regpiece(regex_t *preg, int *flagp) -{ - int ret; - char op; - int next; - int flags; - int min; - int max; - - ret = regatom(preg, &flags); - if (ret == 0) - return 0; - - op = *preg->regparse; - if (!ISMULT(op)) { - *flagp = flags; - return(ret); - } - - if (!(flags&HASWIDTH) && op != '?') { - preg->err = REG_ERR_OPERAND_COULD_BE_EMPTY; - return 0; - } - - - if (op == '{') { - char *end; - - min = strtoul(preg->regparse + 1, &end, 10); - if (end == preg->regparse + 1) { - preg->err = REG_ERR_BAD_COUNT; - return 0; - } - if (*end == '}') { - max = min; - } - else if (*end == '\0') { - preg->err = REG_ERR_UNMATCHED_BRACES; - return 0; - } - else { - preg->regparse = end; - max = strtoul(preg->regparse + 1, &end, 10); - if (*end != '}') { - preg->err = REG_ERR_UNMATCHED_BRACES; - return 0; - } - } - if (end == preg->regparse + 1) { - max = MAX_REP_COUNT; - } - else if (max < min || max >= 100) { - preg->err = REG_ERR_BAD_COUNT; - return 0; - } - if (min >= 100) { - preg->err = REG_ERR_BAD_COUNT; - return 0; - } - - preg->regparse = strchr(preg->regparse, '}'); - } - else { - min = (op == '+'); - max = (op == '?' ? 1 : MAX_REP_COUNT); - } - - if (preg->regparse[1] == '?') { - preg->regparse++; - next = reginsert(preg, flags & SIMPLE ? REPMIN : REPXMIN, 5, ret); - } - else { - next = reginsert(preg, flags & SIMPLE ? REP: REPX, 5, ret); - } - preg->program[ret + 2] = max; - preg->program[ret + 3] = min; - preg->program[ret + 4] = 0; - - *flagp = (min) ? (WORST|HASWIDTH) : (WORST|SPSTART); - - if (!(flags & SIMPLE)) { - int back = regnode(preg, BACK); - regtail(preg, back, ret); - regtail(preg, next, back); - } - - preg->regparse++; - if (ISMULT(*preg->regparse)) { - preg->err = REG_ERR_NESTED_COUNT; - return 0; - } - - return ret; -} - -static void reg_addrange(regex_t *preg, int lower, int upper) -{ - if (lower > upper) { - reg_addrange(preg, upper, lower); - } - - regc(preg, upper - lower + 1); - regc(preg, lower); -} - -static void reg_addrange_str(regex_t *preg, const char *str) -{ - while (*str) { - reg_addrange(preg, *str, *str); - str++; - } -} - -static int reg_utf8_tounicode_case(const char *s, int *uc, int upper) -{ - int l = utf8_tounicode(s, uc); - if (upper) { - *uc = utf8_upper(*uc); - } - return l; -} - -static int hexdigitval(int c) -{ - if (c >= '0' && c <= '9') - return c - '0'; - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - if (c >= 'A' && c <= 'F') - return c - 'A' + 10; - return -1; -} - -static int parse_hex(const char *s, int n, int *uc) -{ - int val = 0; - int k; - - for (k = 0; k < n; k++) { - int c = hexdigitval(*s++); - if (c == -1) { - break; - } - val = (val << 4) | c; - } - if (k) { - *uc = val; - } - return k; -} - -static int reg_decode_escape(const char *s, int *ch) -{ - int n; - const char *s0 = s; - - *ch = *s++; - - switch (*ch) { - case 'b': *ch = '\b'; break; - case 'e': *ch = 27; break; - case 'f': *ch = '\f'; break; - case 'n': *ch = '\n'; break; - case 'r': *ch = '\r'; break; - case 't': *ch = '\t'; break; - case 'v': *ch = '\v'; break; - case 'u': - if (*s == '{') { - - n = parse_hex(s + 1, 6, ch); - if (n > 0 && s[n + 1] == '}' && *ch >= 0 && *ch <= 0x1fffff) { - s += n + 2; - } - else { - - *ch = 'u'; - } - } - else if ((n = parse_hex(s, 4, ch)) > 0) { - s += n; - } - break; - case 'U': - if ((n = parse_hex(s, 8, ch)) > 0) { - s += n; - } - break; - case 'x': - if ((n = parse_hex(s, 2, ch)) > 0) { - s += n; - } - break; - case '\0': - s--; - *ch = '\\'; - break; - } - return s - s0; -} - -static int regatom(regex_t *preg, int *flagp) -{ - int ret; - int flags; - int nocase = (preg->cflags & REG_ICASE); - - int ch; - int n = reg_utf8_tounicode_case(preg->regparse, &ch, nocase); - - *flagp = WORST; - - preg->regparse += n; - switch (ch) { - - case '^': - ret = regnode(preg, BOL); - break; - case '$': - ret = regnode(preg, EOL); - break; - case '.': - ret = regnode(preg, ANY); - *flagp |= HASWIDTH|SIMPLE; - break; - case '[': { - const char *pattern = preg->regparse; - - if (*pattern == '^') { - ret = regnode(preg, ANYBUT); - pattern++; - } else - ret = regnode(preg, ANYOF); - - - if (*pattern == ']' || *pattern == '-') { - reg_addrange(preg, *pattern, *pattern); - pattern++; - } - - while (*pattern && *pattern != ']') { - - int start; - int end; - - enum { - CC_ALPHA, CC_ALNUM, CC_SPACE, CC_BLANK, CC_UPPER, CC_LOWER, - CC_DIGIT, CC_XDIGIT, CC_CNTRL, CC_GRAPH, CC_PRINT, CC_PUNCT, - CC_NUM - }; - int cc; - - pattern += reg_utf8_tounicode_case(pattern, &start, nocase); - if (start == '\\') { - - switch (*pattern) { - case 's': - pattern++; - cc = CC_SPACE; - goto cc_switch; - case 'd': - pattern++; - cc = CC_DIGIT; - goto cc_switch; - case 'w': - pattern++; - reg_addrange(preg, '_', '_'); - cc = CC_ALNUM; - goto cc_switch; - } - pattern += reg_decode_escape(pattern, &start); - if (start == 0) { - preg->err = REG_ERR_NULL_CHAR; - return 0; - } - } - if (pattern[0] == '-' && pattern[1] && pattern[1] != ']') { - - pattern += utf8_tounicode(pattern, &end); - pattern += reg_utf8_tounicode_case(pattern, &end, nocase); - if (end == '\\') { - pattern += reg_decode_escape(pattern, &end); - if (end == 0) { - preg->err = REG_ERR_NULL_CHAR; - return 0; - } - } - - reg_addrange(preg, start, end); - continue; - } - if (start == '[' && pattern[0] == ':') { - static const char *character_class[] = { - ":alpha:", ":alnum:", ":space:", ":blank:", ":upper:", ":lower:", - ":digit:", ":xdigit:", ":cntrl:", ":graph:", ":print:", ":punct:", - }; - - for (cc = 0; cc < CC_NUM; cc++) { - n = strlen(character_class[cc]); - if (strncmp(pattern, character_class[cc], n) == 0) { - - pattern += n + 1; - break; - } - } - if (cc != CC_NUM) { -cc_switch: - switch (cc) { - case CC_ALNUM: - reg_addrange(preg, '0', '9'); - - case CC_ALPHA: - if ((preg->cflags & REG_ICASE) == 0) { - reg_addrange(preg, 'a', 'z'); - } - reg_addrange(preg, 'A', 'Z'); - break; - case CC_SPACE: - reg_addrange_str(preg, " \t\r\n\f\v"); - break; - case CC_BLANK: - reg_addrange_str(preg, " \t"); - break; - case CC_UPPER: - reg_addrange(preg, 'A', 'Z'); - break; - case CC_LOWER: - reg_addrange(preg, 'a', 'z'); - break; - case CC_XDIGIT: - reg_addrange(preg, 'a', 'f'); - reg_addrange(preg, 'A', 'F'); - - case CC_DIGIT: - reg_addrange(preg, '0', '9'); - break; - case CC_CNTRL: - reg_addrange(preg, 0, 31); - reg_addrange(preg, 127, 127); - break; - case CC_PRINT: - reg_addrange(preg, ' ', '~'); - break; - case CC_GRAPH: - reg_addrange(preg, '!', '~'); - break; - case CC_PUNCT: - reg_addrange(preg, '!', '/'); - reg_addrange(preg, ':', '@'); - reg_addrange(preg, '[', '`'); - reg_addrange(preg, '{', '~'); - break; - } - continue; - } - } - - reg_addrange(preg, start, start); - } - regc(preg, '\0'); - - if (*pattern) { - pattern++; - } - preg->regparse = pattern; - - *flagp |= HASWIDTH|SIMPLE; - } - break; - case '(': - ret = reg(preg, 1, &flags); - if (ret == 0) - return 0; - *flagp |= flags&(HASWIDTH|SPSTART); - break; - case '\0': - case '|': - case ')': - preg->err = REG_ERR_INTERNAL; - return 0; - case '?': - case '+': - case '*': - case '{': - preg->err = REG_ERR_COUNT_FOLLOWS_NOTHING; - return 0; - case '\\': - ch = *preg->regparse++; - switch (ch) { - case '\0': - preg->err = REG_ERR_TRAILING_BACKSLASH; - return 0; - case 'A': - ret = regnode(preg, BOLX); - break; - case 'Z': - ret = regnode(preg, EOLX); - break; - case '<': - case 'm': - ret = regnode(preg, WORDA); - break; - case '>': - case 'M': - ret = regnode(preg, WORDZ); - break; - case 'd': - case 'D': - ret = regnode(preg, ch == 'd' ? ANYOF : ANYBUT); - reg_addrange(preg, '0', '9'); - regc(preg, '\0'); - *flagp |= HASWIDTH|SIMPLE; - break; - case 'w': - case 'W': - ret = regnode(preg, ch == 'w' ? ANYOF : ANYBUT); - if ((preg->cflags & REG_ICASE) == 0) { - reg_addrange(preg, 'a', 'z'); - } - reg_addrange(preg, 'A', 'Z'); - reg_addrange(preg, '0', '9'); - reg_addrange(preg, '_', '_'); - regc(preg, '\0'); - *flagp |= HASWIDTH|SIMPLE; - break; - case 's': - case 'S': - ret = regnode(preg, ch == 's' ? ANYOF : ANYBUT); - reg_addrange_str(preg," \t\r\n\f\v"); - regc(preg, '\0'); - *flagp |= HASWIDTH|SIMPLE; - break; - - default: - - - preg->regparse--; - goto de_fault; - } - break; - de_fault: - default: { - int added = 0; - - - preg->regparse -= n; - - ret = regnode(preg, EXACTLY); - - - - while (*preg->regparse && strchr(META, *preg->regparse) == NULL) { - n = reg_utf8_tounicode_case(preg->regparse, &ch, (preg->cflags & REG_ICASE)); - if (ch == '\\' && preg->regparse[n]) { - if (strchr("<>mMwWdDsSAZ", preg->regparse[n])) { - - break; - } - n += reg_decode_escape(preg->regparse + n, &ch); - if (ch == 0) { - preg->err = REG_ERR_NULL_CHAR; - return 0; - } - } - - - if (ISMULT(preg->regparse[n])) { - - if (added) { - - break; - } - - regc(preg, ch); - added++; - preg->regparse += n; - break; - } - - - regc(preg, ch); - added++; - preg->regparse += n; - } - regc(preg, '\0'); - - *flagp |= HASWIDTH; - if (added == 1) - *flagp |= SIMPLE; - break; - } - break; - } - - return(ret); -} - -static void reg_grow(regex_t *preg, int n) -{ - if (preg->p + n >= preg->proglen) { - preg->proglen = (preg->p + n) * 2; - preg->program = realloc(preg->program, preg->proglen * sizeof(int)); - } -} - - -static int regnode(regex_t *preg, int op) -{ - reg_grow(preg, 2); - - - preg->program[preg->p++] = op; - preg->program[preg->p++] = 0; - - - return preg->p - 2; -} - -static void regc(regex_t *preg, int b ) -{ - reg_grow(preg, 1); - preg->program[preg->p++] = b; -} - -static int reginsert(regex_t *preg, int op, int size, int opnd ) -{ - reg_grow(preg, size); - - - memmove(preg->program + opnd + size, preg->program + opnd, sizeof(int) * (preg->p - opnd)); - - memset(preg->program + opnd, 0, sizeof(int) * size); - - preg->program[opnd] = op; - - preg->p += size; - - return opnd + size; -} - -static void regtail(regex_t *preg, int p, int val) -{ - int scan; - int temp; - int offset; - - - scan = p; - for (;;) { - temp = regnext(preg, scan); - if (temp == 0) - break; - scan = temp; - } - - if (OP(preg, scan) == BACK) - offset = scan - val; - else - offset = val - scan; - - preg->program[scan + 1] = offset; -} - - -static void regoptail(regex_t *preg, int p, int val ) -{ - - if (p != 0 && OP(preg, p) == BRANCH) { - regtail(preg, OPERAND(p), val); - } -} - - -static int regtry(regex_t *preg, const char *string ); -static int regmatch(regex_t *preg, int prog); -static int regrepeat(regex_t *preg, int p, int max); - -int regexec(regex_t *preg, const char *string, size_t nmatch, regmatch_t pmatch[], int eflags) -{ - const char *s; - int scan; - - - if (preg == NULL || preg->program == NULL || string == NULL) { - return REG_ERR_NULL_ARGUMENT; - } - - - if (*preg->program != REG_MAGIC) { - return REG_ERR_CORRUPTED; - } - -#ifdef DEBUG - fprintf(stderr, "regexec: %s\n", string); - regdump(preg); -#endif - - preg->eflags = eflags; - preg->pmatch = pmatch; - preg->nmatch = nmatch; - preg->start = string; - - - for (scan = OPERAND(1); scan != 0; scan += regopsize(preg, scan)) { - int op = OP(preg, scan); - if (op == END) - break; - if (op == REPX || op == REPXMIN) - preg->program[scan + 4] = 0; - } - - - if (preg->regmust != 0) { - s = string; - while ((s = str_find(s, preg->program[preg->regmust], preg->cflags & REG_ICASE)) != NULL) { - if (prefix_cmp(preg->program + preg->regmust, preg->regmlen, s, preg->cflags & REG_ICASE) >= 0) { - break; - } - s++; - } - if (s == NULL) - return REG_NOMATCH; - } - - - preg->regbol = string; - - - if (preg->reganch) { - if (eflags & REG_NOTBOL) { - - goto nextline; - } - while (1) { - if (regtry(preg, string)) { - return REG_NOERROR; - } - if (*string) { -nextline: - if (preg->cflags & REG_NEWLINE) { - - string = strchr(string, '\n'); - if (string) { - preg->regbol = ++string; - continue; - } - } - } - return REG_NOMATCH; - } - } - - - s = string; - if (preg->regstart != '\0') { - - while ((s = str_find(s, preg->regstart, preg->cflags & REG_ICASE)) != NULL) { - if (regtry(preg, s)) - return REG_NOERROR; - s++; - } - } - else - - while (1) { - if (regtry(preg, s)) - return REG_NOERROR; - if (*s == '\0') { - break; - } - else { - int c; - s += utf8_tounicode(s, &c); - } - } - - - return REG_NOMATCH; -} - - -static int regtry( regex_t *preg, const char *string ) -{ - int i; - - preg->reginput = string; - - for (i = 0; i < preg->nmatch; i++) { - preg->pmatch[i].rm_so = -1; - preg->pmatch[i].rm_eo = -1; - } - if (regmatch(preg, 1)) { - preg->pmatch[0].rm_so = string - preg->start; - preg->pmatch[0].rm_eo = preg->reginput - preg->start; - return(1); - } else - return(0); -} - -static int prefix_cmp(const int *prog, int proglen, const char *string, int nocase) -{ - const char *s = string; - while (proglen && *s) { - int ch; - int n = reg_utf8_tounicode_case(s, &ch, nocase); - if (ch != *prog) { - return -1; - } - prog++; - s += n; - proglen--; - } - if (proglen == 0) { - return s - string; - } - return -1; -} - -static int reg_range_find(const int *range, int c) -{ - while (*range) { - - if (c >= range[1] && c <= (range[0] + range[1] - 1)) { - return 1; - } - range += 2; - } - return 0; -} - -static const char *str_find(const char *string, int c, int nocase) -{ - if (nocase) { - - c = utf8_upper(c); - } - while (*string) { - int ch; - int n = reg_utf8_tounicode_case(string, &ch, nocase); - if (c == ch) { - return string; - } - string += n; - } - return NULL; -} - -static int reg_iseol(regex_t *preg, int ch) -{ - if (preg->cflags & REG_NEWLINE) { - return ch == '\0' || ch == '\n'; - } - else { - return ch == '\0'; - } -} - -static int regmatchsimplerepeat(regex_t *preg, int scan, int matchmin) -{ - int nextch = '\0'; - const char *save; - int no; - int c; - - int max = preg->program[scan + 2]; - int min = preg->program[scan + 3]; - int next = regnext(preg, scan); - - if (OP(preg, next) == EXACTLY) { - nextch = preg->program[OPERAND(next)]; - } - save = preg->reginput; - no = regrepeat(preg, scan + 5, max); - if (no < min) { - return 0; - } - if (matchmin) { - - max = no; - no = min; - } - - while (1) { - if (matchmin) { - if (no > max) { - break; - } - } - else { - if (no < min) { - break; - } - } - preg->reginput = save + utf8_index(save, no); - reg_utf8_tounicode_case(preg->reginput, &c, (preg->cflags & REG_ICASE)); - - if (reg_iseol(preg, nextch) || c == nextch) { - if (regmatch(preg, next)) { - return(1); - } - } - if (matchmin) { - - no++; - } - else { - - no--; - } - } - return(0); -} - -static int regmatchrepeat(regex_t *preg, int scan, int matchmin) -{ - int *scanpt = preg->program + scan; - - int max = scanpt[2]; - int min = scanpt[3]; - - - if (scanpt[4] < min) { - - scanpt[4]++; - if (regmatch(preg, scan + 5)) { - return 1; - } - scanpt[4]--; - return 0; - } - if (scanpt[4] > max) { - return 0; - } - - if (matchmin) { - - if (regmatch(preg, regnext(preg, scan))) { - return 1; - } - - scanpt[4]++; - if (regmatch(preg, scan + 5)) { - return 1; - } - scanpt[4]--; - return 0; - } - - if (scanpt[4] < max) { - scanpt[4]++; - if (regmatch(preg, scan + 5)) { - return 1; - } - scanpt[4]--; - } - - return regmatch(preg, regnext(preg, scan)); -} - - -static int regmatch(regex_t *preg, int prog) -{ - int scan; - int next; - const char *save; - - scan = prog; - -#ifdef DEBUG - if (scan != 0 && regnarrate) - fprintf(stderr, "%s(\n", regprop(scan)); -#endif - while (scan != 0) { - int n; - int c; -#ifdef DEBUG - if (regnarrate) { - fprintf(stderr, "%3d: %s...\n", scan, regprop(OP(preg, scan))); - } -#endif - next = regnext(preg, scan); - n = reg_utf8_tounicode_case(preg->reginput, &c, (preg->cflags & REG_ICASE)); - - switch (OP(preg, scan)) { - case BOLX: - if ((preg->eflags & REG_NOTBOL)) { - return(0); - } - - case BOL: - if (preg->reginput != preg->regbol) { - return(0); - } - break; - case EOLX: - if (c != 0) { - - return 0; - } - break; - case EOL: - if (!reg_iseol(preg, c)) { - return(0); - } - break; - case WORDA: - - if ((!isalnum(UCHAR(c))) && c != '_') - return(0); - - if (preg->reginput > preg->regbol && - (isalnum(UCHAR(preg->reginput[-1])) || preg->reginput[-1] == '_')) - return(0); - break; - case WORDZ: - - if (preg->reginput > preg->regbol) { - - if (reg_iseol(preg, c) || !isalnum(UCHAR(c)) || c != '_') { - c = preg->reginput[-1]; - - if (isalnum(UCHAR(c)) || c == '_') { - break; - } - } - } - - return(0); - - case ANY: - if (reg_iseol(preg, c)) - return 0; - preg->reginput += n; - break; - case EXACTLY: { - int opnd; - int len; - int slen; - - opnd = OPERAND(scan); - len = str_int_len(preg->program + opnd); - - slen = prefix_cmp(preg->program + opnd, len, preg->reginput, preg->cflags & REG_ICASE); - if (slen < 0) { - return(0); - } - preg->reginput += slen; - } - break; - case ANYOF: - if (reg_iseol(preg, c) || reg_range_find(preg->program + OPERAND(scan), c) == 0) { - return(0); - } - preg->reginput += n; - break; - case ANYBUT: - if (reg_iseol(preg, c) || reg_range_find(preg->program + OPERAND(scan), c) != 0) { - return(0); - } - preg->reginput += n; - break; - case NOTHING: - break; - case BACK: - break; - case BRANCH: - if (OP(preg, next) != BRANCH) - next = OPERAND(scan); - else { - do { - save = preg->reginput; - if (regmatch(preg, OPERAND(scan))) { - return(1); - } - preg->reginput = save; - scan = regnext(preg, scan); - } while (scan != 0 && OP(preg, scan) == BRANCH); - return(0); - - } - break; - case REP: - case REPMIN: - return regmatchsimplerepeat(preg, scan, OP(preg, scan) == REPMIN); - - case REPX: - case REPXMIN: - return regmatchrepeat(preg, scan, OP(preg, scan) == REPXMIN); - - case END: - return 1; - - case OPENNC: - case CLOSENC: - return regmatch(preg, next); - - default: - if (OP(preg, scan) >= OPEN+1 && OP(preg, scan) < CLOSE_END) { - save = preg->reginput; - if (regmatch(preg, next)) { - if (OP(preg, scan) < CLOSE) { - int no = OP(preg, scan) - OPEN; - if (no < preg->nmatch && preg->pmatch[no].rm_so == -1) { - preg->pmatch[no].rm_so = save - preg->start; - } - } - else { - int no = OP(preg, scan) - CLOSE; - if (no < preg->nmatch && preg->pmatch[no].rm_eo == -1) { - preg->pmatch[no].rm_eo = save - preg->start; - } - } - return(1); - } - return(0); - } - return REG_ERR_INTERNAL; - } - - scan = next; - } - - return REG_ERR_INTERNAL; -} - -static int regrepeat(regex_t *preg, int p, int max) -{ - int count = 0; - const char *scan; - int opnd; - int ch; - int n; - - scan = preg->reginput; - opnd = OPERAND(p); - switch (OP(preg, p)) { - case ANY: - - while (!reg_iseol(preg, *scan) && count < max) { - count++; - scan++; - } - break; - case EXACTLY: - while (count < max) { - n = reg_utf8_tounicode_case(scan, &ch, preg->cflags & REG_ICASE); - if (preg->program[opnd] != ch) { - break; - } - count++; - scan += n; - } - break; - case ANYOF: - while (count < max) { - n = reg_utf8_tounicode_case(scan, &ch, preg->cflags & REG_ICASE); - if (reg_iseol(preg, ch) || reg_range_find(preg->program + opnd, ch) == 0) { - break; - } - count++; - scan += n; - } - break; - case ANYBUT: - while (count < max) { - n = reg_utf8_tounicode_case(scan, &ch, preg->cflags & REG_ICASE); - if (reg_iseol(preg, ch) || reg_range_find(preg->program + opnd, ch) != 0) { - break; - } - count++; - scan += n; - } - break; - default: - preg->err = REG_ERR_INTERNAL; - count = 0; - break; - } - preg->reginput = scan; - - return(count); -} - -static int regnext(regex_t *preg, int p ) -{ - int offset; - - offset = NEXT(preg, p); - - if (offset == 0) - return 0; - - if (OP(preg, p) == BACK) - return(p-offset); - else - return(p+offset); -} - -static int regopsize(regex_t *preg, int p ) -{ - - switch (OP(preg, p)) { - case REP: - case REPMIN: - case REPX: - case REPXMIN: - return 5; - - case ANYOF: - case ANYBUT: - case EXACTLY: { - int s = p + 2; - while (preg->program[s++]) { - } - return s - p; - } - } - return 2; -} - - -size_t regerror(int errcode, const regex_t *preg, char *errbuf, size_t errbuf_size) -{ - static const char *error_strings[] = { - "success", - "no match", - "bad pattern", - "null argument", - "unknown error", - "too big", - "out of memory", - "too many ()", - "parentheses () not balanced", - "braces {} not balanced", - "invalid repetition count(s)", - "extra characters", - "*+ of empty atom", - "nested count", - "internal error", - "count follows nothing", - "trailing backslash", - "corrupted program", - "contains null char", - }; - const char *err; - - if (errcode < 0 || errcode >= REG_ERR_NUM) { - err = "Bad error code"; - } - else { - err = error_strings[errcode]; - } - - return snprintf(errbuf, errbuf_size, "%s", err); -} - -void regfree(regex_t *preg) -{ - free(preg->program); -} - -#endif -#include - -void Jim_SetResultErrno(Jim_Interp *interp, const char *msg) -{ - Jim_SetResultFormatted(interp, "%s: %s", msg, strerror(Jim_Errno())); -} - -#if defined(__MINGW32__) -#include - -int Jim_Errno(void) -{ - switch (GetLastError()) { - case ERROR_FILE_NOT_FOUND: return ENOENT; - case ERROR_PATH_NOT_FOUND: return ENOENT; - case ERROR_TOO_MANY_OPEN_FILES: return EMFILE; - case ERROR_ACCESS_DENIED: return EACCES; - case ERROR_INVALID_HANDLE: return EBADF; - case ERROR_BAD_ENVIRONMENT: return E2BIG; - case ERROR_BAD_FORMAT: return ENOEXEC; - case ERROR_INVALID_ACCESS: return EACCES; - case ERROR_INVALID_DRIVE: return ENOENT; - case ERROR_CURRENT_DIRECTORY: return EACCES; - case ERROR_NOT_SAME_DEVICE: return EXDEV; - case ERROR_NO_MORE_FILES: return ENOENT; - case ERROR_WRITE_PROTECT: return EROFS; - case ERROR_BAD_UNIT: return ENXIO; - case ERROR_NOT_READY: return EBUSY; - case ERROR_BAD_COMMAND: return EIO; - case ERROR_CRC: return EIO; - case ERROR_BAD_LENGTH: return EIO; - case ERROR_SEEK: return EIO; - case ERROR_WRITE_FAULT: return EIO; - case ERROR_READ_FAULT: return EIO; - case ERROR_GEN_FAILURE: return EIO; - case ERROR_SHARING_VIOLATION: return EACCES; - case ERROR_LOCK_VIOLATION: return EACCES; - case ERROR_SHARING_BUFFER_EXCEEDED: return ENFILE; - case ERROR_HANDLE_DISK_FULL: return ENOSPC; - case ERROR_NOT_SUPPORTED: return ENODEV; - case ERROR_REM_NOT_LIST: return EBUSY; - case ERROR_DUP_NAME: return EEXIST; - case ERROR_BAD_NETPATH: return ENOENT; - case ERROR_NETWORK_BUSY: return EBUSY; - case ERROR_DEV_NOT_EXIST: return ENODEV; - case ERROR_TOO_MANY_CMDS: return EAGAIN; - case ERROR_ADAP_HDW_ERR: return EIO; - case ERROR_BAD_NET_RESP: return EIO; - case ERROR_UNEXP_NET_ERR: return EIO; - case ERROR_NETNAME_DELETED: return ENOENT; - case ERROR_NETWORK_ACCESS_DENIED: return EACCES; - case ERROR_BAD_DEV_TYPE: return ENODEV; - case ERROR_BAD_NET_NAME: return ENOENT; - case ERROR_TOO_MANY_NAMES: return ENFILE; - case ERROR_TOO_MANY_SESS: return EIO; - case ERROR_SHARING_PAUSED: return EAGAIN; - case ERROR_REDIR_PAUSED: return EAGAIN; - case ERROR_FILE_EXISTS: return EEXIST; - case ERROR_CANNOT_MAKE: return ENOSPC; - case ERROR_OUT_OF_STRUCTURES: return ENFILE; - case ERROR_ALREADY_ASSIGNED: return EEXIST; - case ERROR_INVALID_PASSWORD: return EPERM; - case ERROR_NET_WRITE_FAULT: return EIO; - case ERROR_NO_PROC_SLOTS: return EAGAIN; - case ERROR_DISK_CHANGE: return EXDEV; - case ERROR_BROKEN_PIPE: return EPIPE; - case ERROR_OPEN_FAILED: return ENOENT; - case ERROR_DISK_FULL: return ENOSPC; - case ERROR_NO_MORE_SEARCH_HANDLES: return EMFILE; - case ERROR_INVALID_TARGET_HANDLE: return EBADF; - case ERROR_INVALID_NAME: return ENOENT; - case ERROR_PROC_NOT_FOUND: return ESRCH; - case ERROR_WAIT_NO_CHILDREN: return ECHILD; - case ERROR_CHILD_NOT_COMPLETE: return ECHILD; - case ERROR_DIRECT_ACCESS_HANDLE: return EBADF; - case ERROR_SEEK_ON_DEVICE: return ESPIPE; - case ERROR_BUSY_DRIVE: return EAGAIN; - case ERROR_DIR_NOT_EMPTY: return EEXIST; - case ERROR_NOT_LOCKED: return EACCES; - case ERROR_BAD_PATHNAME: return ENOENT; - case ERROR_LOCK_FAILED: return EACCES; - case ERROR_ALREADY_EXISTS: return EEXIST; - case ERROR_FILENAME_EXCED_RANGE: return ENAMETOOLONG; - case ERROR_BAD_PIPE: return EPIPE; - case ERROR_PIPE_BUSY: return EAGAIN; - case ERROR_PIPE_NOT_CONNECTED: return EPIPE; - case ERROR_DIRECTORY: return ENOTDIR; - } - return EINVAL; -} - -pidtype waitpid(pidtype pid, int *status, int nohang) -{ - DWORD ret = WaitForSingleObject(pid, nohang ? 0 : INFINITE); - if (ret == WAIT_TIMEOUT || ret == WAIT_FAILED) { - - return JIM_BAD_PID; - } - GetExitCodeProcess(pid, &ret); - *status = ret; - CloseHandle(pid); - return pid; -} - -int Jim_MakeTempFile(Jim_Interp *interp, const char *filename_template, int unlink_file) -{ - char name[MAX_PATH]; - HANDLE handle; - - if (!GetTempPath(MAX_PATH, name) || !GetTempFileName(name, filename_template ? filename_template : "JIM", 0, name)) { - return -1; - } - - handle = CreateFile(name, GENERIC_READ | GENERIC_WRITE, 0, NULL, - CREATE_ALWAYS, FILE_ATTRIBUTE_TEMPORARY | (unlink_file ? FILE_FLAG_DELETE_ON_CLOSE : 0), - NULL); - - if (handle == INVALID_HANDLE_VALUE) { - goto error; - } - - Jim_SetResultString(interp, name, -1); - return _open_osfhandle((int)handle, _O_RDWR | _O_TEXT); - - error: - Jim_SetResultErrno(interp, name); - DeleteFile(name); - return -1; -} - -int Jim_OpenForWrite(const char *filename, int append) -{ - if (strcmp(filename, "/dev/null") == 0) { - filename = "nul:"; - } - int fd = _open(filename, _O_WRONLY | _O_CREAT | _O_TEXT | (append ? _O_APPEND : _O_TRUNC), _S_IREAD | _S_IWRITE); - if (fd >= 0 && append) { - - _lseek(fd, 0L, SEEK_END); - } - return fd; -} - -int Jim_OpenForRead(const char *filename) -{ - if (strcmp(filename, "/dev/null") == 0) { - filename = "nul:"; - } - return _open(filename, _O_RDONLY | _O_TEXT, 0); -} - -#elif defined(HAVE_UNISTD_H) - - - -int Jim_MakeTempFile(Jim_Interp *interp, const char *filename_template, int unlink_file) -{ - int fd; - mode_t mask; - Jim_Obj *filenameObj; - - if (filename_template == NULL) { - const char *tmpdir = getenv("TMPDIR"); - if (tmpdir == NULL || *tmpdir == '\0' || access(tmpdir, W_OK) != 0) { - tmpdir = "/tmp/"; - } - filenameObj = Jim_NewStringObj(interp, tmpdir, -1); - if (tmpdir[0] && tmpdir[strlen(tmpdir) - 1] != '/') { - Jim_AppendString(interp, filenameObj, "/", 1); - } - Jim_AppendString(interp, filenameObj, "tcl.tmp.XXXXXX", -1); - } - else { - filenameObj = Jim_NewStringObj(interp, filename_template, -1); - } - - - mask = umask(S_IXUSR | S_IRWXG | S_IRWXO); -#ifdef HAVE_MKSTEMP - fd = mkstemp(filenameObj->bytes); -#else - if (mktemp(filenameObj->bytes) == NULL) { - fd = -1; - } - else { - fd = open(filenameObj->bytes, O_RDWR | O_CREAT | O_TRUNC); - } -#endif - umask(mask); - if (fd < 0) { - Jim_SetResultErrno(interp, Jim_String(filenameObj)); - Jim_FreeNewObj(interp, filenameObj); - return -1; - } - if (unlink_file) { - remove(Jim_String(filenameObj)); - } - - Jim_SetResult(interp, filenameObj); - return fd; -} - -int Jim_OpenForWrite(const char *filename, int append) -{ - return open(filename, O_WRONLY | O_CREAT | (append ? O_APPEND : O_TRUNC), 0666); -} - -int Jim_OpenForRead(const char *filename) -{ - return open(filename, O_RDONLY, 0); -} - -#endif - -#if defined(_WIN32) || defined(WIN32) -#ifndef STRICT -#define STRICT -#endif -#define WIN32_LEAN_AND_MEAN -#include - -#if defined(HAVE_DLOPEN_COMPAT) -void *dlopen(const char *path, int mode) -{ - JIM_NOTUSED(mode); - - return (void *)LoadLibraryA(path); -} - -int dlclose(void *handle) -{ - FreeLibrary((HANDLE)handle); - return 0; -} - -void *dlsym(void *handle, const char *symbol) -{ - return GetProcAddress((HMODULE)handle, symbol); -} - -char *dlerror(void) -{ - static char msg[121]; - FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(), - LANG_NEUTRAL, msg, sizeof(msg) - 1, NULL); - return msg; -} -#endif - -#ifdef _MSC_VER - -#include - - -int gettimeofday(struct timeval *tv, void *unused) -{ - struct _timeb tb; - - _ftime(&tb); - tv->tv_sec = tb.time; - tv->tv_usec = tb.millitm * 1000; - - return 0; -} - - -DIR *opendir(const char *name) -{ - DIR *dir = 0; - - if (name && name[0]) { - size_t base_length = strlen(name); - const char *all = - strchr("/\\", name[base_length - 1]) ? "*" : "/*"; - - if ((dir = (DIR *) Jim_Alloc(sizeof *dir)) != 0 && - (dir->name = (char *)Jim_Alloc(base_length + strlen(all) + 1)) != 0) { - strcat(strcpy(dir->name, name), all); - - if ((dir->handle = (long)_findfirst(dir->name, &dir->info)) != -1) - dir->result.d_name = 0; - else { - Jim_Free(dir->name); - Jim_Free(dir); - dir = 0; - } - } - else { - Jim_Free(dir); - dir = 0; - errno = ENOMEM; - } - } - else { - errno = EINVAL; - } - return dir; -} - -int closedir(DIR * dir) -{ - int result = -1; - - if (dir) { - if (dir->handle != -1) - result = _findclose(dir->handle); - Jim_Free(dir->name); - Jim_Free(dir); - } - if (result == -1) - errno = EBADF; - return result; -} - -struct dirent *readdir(DIR * dir) -{ - struct dirent *result = 0; - - if (dir && dir->handle != -1) { - if (!dir->result.d_name || _findnext(dir->handle, &dir->info) != -1) { - result = &dir->result; - result->d_name = dir->info.name; - } - } - else { - errno = EBADF; - } - return result; -} -#endif -#endif -#include -#include - - - - - - -#ifndef SIGPIPE -#define SIGPIPE 13 -#endif -#ifndef SIGINT -#define SIGINT 2 -#endif - -const char *Jim_SignalId(int sig) -{ - static char buf[10]; - switch (sig) { - case SIGINT: return "SIGINT"; - case SIGPIPE: return "SIGPIPE"; - - } - snprintf(buf, sizeof(buf), "%d", sig); - return buf; -} -#ifndef JIM_BOOTSTRAP_LIB_ONLY -#include -#include - - -#ifdef USE_LINENOISE -#ifdef HAVE_UNISTD_H - #include -#endif -#ifdef HAVE_SYS_STAT_H - #include -#endif -#include "linenoise.h" -#else -#define MAX_LINE_LEN 512 -#endif - -#ifdef USE_LINENOISE -static void JimCompletionCallback(const char *prefix, linenoiseCompletions *comp, void *userdata); -static const char completion_callback_assoc_key[] = "interactive-completion"; -#endif - -char *Jim_HistoryGetline(Jim_Interp *interp, const char *prompt) -{ -#ifdef USE_LINENOISE - struct JimCompletionInfo *compinfo = Jim_GetAssocData(interp, completion_callback_assoc_key); - char *result; - Jim_Obj *objPtr; - long mlmode = 0; - if (compinfo) { - linenoiseSetCompletionCallback(JimCompletionCallback, compinfo); - } - objPtr = Jim_GetVariableStr(interp, "history::multiline", JIM_NONE); - if (objPtr && Jim_GetLong(interp, objPtr, &mlmode) == JIM_NONE) { - linenoiseSetMultiLine(mlmode); - } - - result = linenoise(prompt); - - linenoiseSetCompletionCallback(NULL, NULL); - return result; -#else - int len; - char *line = malloc(MAX_LINE_LEN); - - fputs(prompt, stdout); - fflush(stdout); - - if (fgets(line, MAX_LINE_LEN, stdin) == NULL) { - free(line); - return NULL; - } - len = strlen(line); - if (len && line[len - 1] == '\n') { - line[len - 1] = '\0'; - } - return line; -#endif -} - -void Jim_HistoryLoad(const char *filename) -{ -#ifdef USE_LINENOISE - linenoiseHistoryLoad(filename); -#endif -} - -void Jim_HistoryAdd(const char *line) -{ -#ifdef USE_LINENOISE - linenoiseHistoryAdd(line); -#endif -} - -void Jim_HistorySave(const char *filename) -{ -#ifdef USE_LINENOISE -#ifdef HAVE_UMASK - mode_t mask; - - mask = umask(S_IXUSR | S_IRWXG | S_IRWXO); -#endif - linenoiseHistorySave(filename); -#ifdef HAVE_UMASK - umask(mask); -#endif -#endif -} - -void Jim_HistoryShow(void) -{ -#ifdef USE_LINENOISE - - int i; - int len; - char **history = linenoiseHistory(&len); - for (i = 0; i < len; i++) { - printf("%4d %s\n", i + 1, history[i]); - } -#endif -} - -#ifdef USE_LINENOISE -struct JimCompletionInfo { - Jim_Interp *interp; - Jim_Obj *command; -}; - -static void JimCompletionCallback(const char *prefix, linenoiseCompletions *comp, void *userdata) -{ - struct JimCompletionInfo *info = (struct JimCompletionInfo *)userdata; - Jim_Obj *objv[2]; - int ret; - - objv[0] = info->command; - objv[1] = Jim_NewStringObj(info->interp, prefix, -1); - - ret = Jim_EvalObjVector(info->interp, 2, objv); - - - if (ret == JIM_OK) { - int i; - Jim_Obj *listObj = Jim_GetResult(info->interp); - int len = Jim_ListLength(info->interp, listObj); - for (i = 0; i < len; i++) { - linenoiseAddCompletion(comp, Jim_String(Jim_ListGetIndex(info->interp, listObj, i))); - } - } -} - -static void JimHistoryFreeCompletion(Jim_Interp *interp, void *data) -{ - struct JimCompletionInfo *compinfo = data; - - Jim_DecrRefCount(interp, compinfo->command); - - Jim_Free(compinfo); -} -#endif - -void Jim_HistorySetCompletion(Jim_Interp *interp, Jim_Obj *commandObj) -{ -#ifdef USE_LINENOISE - if (commandObj) { - - Jim_IncrRefCount(commandObj); - } - - Jim_DeleteAssocData(interp, completion_callback_assoc_key); - - if (commandObj) { - struct JimCompletionInfo *compinfo = Jim_Alloc(sizeof(*compinfo)); - compinfo->interp = interp; - compinfo->command = commandObj; - - Jim_SetAssocData(interp, completion_callback_assoc_key, JimHistoryFreeCompletion, compinfo); - } -#endif -} - -int Jim_InteractivePrompt(Jim_Interp *interp) -{ - int retcode = JIM_OK; - char *history_file = NULL; -#ifdef USE_LINENOISE - const char *home; - - home = getenv("HOME"); - if (home && isatty(STDIN_FILENO)) { - int history_len = strlen(home) + sizeof("/.jim_history"); - history_file = Jim_Alloc(history_len); - snprintf(history_file, history_len, "%s/.jim_history", home); - Jim_HistoryLoad(history_file); - } - - Jim_HistorySetCompletion(interp, Jim_NewStringObj(interp, "tcl::autocomplete", -1)); -#endif - - printf("Welcome to Jim version %d.%d\n", - JIM_VERSION / 100, JIM_VERSION % 100); - Jim_SetVariableStrWithStr(interp, JIM_INTERACTIVE, "1"); - - while (1) { - Jim_Obj *scriptObjPtr; - const char *result; - int reslen; - char prompt[20]; - - if (retcode != JIM_OK) { - const char *retcodestr = Jim_ReturnCode(retcode); - - if (*retcodestr == '?') { - snprintf(prompt, sizeof(prompt) - 3, "[%d] . ", retcode); - } - else { - snprintf(prompt, sizeof(prompt) - 3, "[%s] . ", retcodestr); - } - } - else { - strcpy(prompt, ". "); - } - - scriptObjPtr = Jim_NewStringObj(interp, "", 0); - Jim_IncrRefCount(scriptObjPtr); - while (1) { - char state; - char *line; - - line = Jim_HistoryGetline(interp, prompt); - if (line == NULL) { - if (errno == EINTR) { - continue; - } - Jim_DecrRefCount(interp, scriptObjPtr); - retcode = JIM_OK; - goto out; - } - if (Jim_Length(scriptObjPtr) != 0) { - - Jim_AppendString(interp, scriptObjPtr, "\n", 1); - } - Jim_AppendString(interp, scriptObjPtr, line, -1); - free(line); - if (Jim_ScriptIsComplete(interp, scriptObjPtr, &state)) - break; - - snprintf(prompt, sizeof(prompt), "%c> ", state); - } -#ifdef USE_LINENOISE - if (strcmp(Jim_String(scriptObjPtr), "h") == 0) { - - Jim_HistoryShow(); - Jim_DecrRefCount(interp, scriptObjPtr); - continue; - } - - Jim_HistoryAdd(Jim_String(scriptObjPtr)); - if (history_file) { - Jim_HistorySave(history_file); - } -#endif - retcode = Jim_EvalObj(interp, scriptObjPtr); - Jim_DecrRefCount(interp, scriptObjPtr); - - if (retcode == JIM_EXIT) { - break; - } - if (retcode == JIM_ERR) { - Jim_MakeErrorMessage(interp); - } - result = Jim_GetString(Jim_GetResult(interp), &reslen); - if (reslen) { - printf("%s\n", result); - } - } - out: - Jim_Free(history_file); - - return retcode; -} - -#include -#include -#include - - - -extern int Jim_initjimshInit(Jim_Interp *interp); - -static void JimSetArgv(Jim_Interp *interp, int argc, char *const argv[]) -{ - int n; - Jim_Obj *listObj = Jim_NewListObj(interp, NULL, 0); - - - for (n = 0; n < argc; n++) { - Jim_Obj *obj = Jim_NewStringObj(interp, argv[n], -1); - - Jim_ListAppendElement(interp, listObj, obj); - } - - Jim_SetVariableStr(interp, "argv", listObj); - Jim_SetVariableStr(interp, "argc", Jim_NewIntObj(interp, argc)); -} - -static void JimPrintErrorMessage(Jim_Interp *interp) -{ - Jim_MakeErrorMessage(interp); - fprintf(stderr, "%s\n", Jim_String(Jim_GetResult(interp))); -} - -void usage(const char* executable_name) -{ - printf("jimsh version %d.%d\n", JIM_VERSION / 100, JIM_VERSION % 100); - printf("Usage: %s\n", executable_name); - printf("or : %s [options] [filename]\n", executable_name); - printf("\n"); - printf("Without options: Interactive mode\n"); - printf("\n"); - printf("Options:\n"); - printf(" --version : prints the version string\n"); - printf(" --help : prints this text\n"); - printf(" -e CMD : executes command CMD\n"); - printf(" NOTE: all subsequent options will be passed as arguments to the command\n"); - printf(" [filename|-] : executes the script contained in the named file, or from stdin if \"-\"\n"); - printf(" NOTE: all subsequent options will be passed to the script\n\n"); -} - -int main(int argc, char *const argv[]) -{ - int retcode; - Jim_Interp *interp; - char *const orig_argv0 = argv[0]; - - - if (argc > 1 && strcmp(argv[1], "--version") == 0) { - printf("%d.%d\n", JIM_VERSION / 100, JIM_VERSION % 100); - return 0; - } - else if (argc > 1 && strcmp(argv[1], "--help") == 0) { - usage(argv[0]); - return 0; - } - - - interp = Jim_CreateInterp(); - Jim_RegisterCoreCommands(interp); - - - if (Jim_InitStaticExtensions(interp) != JIM_OK) { - JimPrintErrorMessage(interp); - } - - Jim_SetVariableStrWithStr(interp, "jim::argv0", orig_argv0); - Jim_SetVariableStrWithStr(interp, JIM_INTERACTIVE, argc == 1 ? "1" : "0"); - retcode = Jim_initjimshInit(interp); - - if (argc == 1) { - - if (retcode == JIM_ERR) { - JimPrintErrorMessage(interp); - } - if (retcode != JIM_EXIT) { - JimSetArgv(interp, 0, NULL); - retcode = Jim_InteractivePrompt(interp); - } - } - else { - - if (argc > 2 && strcmp(argv[1], "-e") == 0) { - - JimSetArgv(interp, argc - 3, argv + 3); - retcode = Jim_Eval(interp, argv[2]); - if (retcode != JIM_ERR) { - printf("%s\n", Jim_String(Jim_GetResult(interp))); - } - } - else { - Jim_SetVariableStr(interp, "argv0", Jim_NewStringObj(interp, argv[1], -1)); - JimSetArgv(interp, argc - 2, argv + 2); - if (strcmp(argv[1], "-") == 0) { - retcode = Jim_Eval(interp, "eval [info source [stdin read] stdin 1]"); - } else { - retcode = Jim_EvalFile(interp, argv[1]); - } - } - if (retcode == JIM_ERR) { - JimPrintErrorMessage(interp); - } - } - if (retcode == JIM_EXIT) { - retcode = Jim_GetExitCode(interp); - } - else if (retcode == JIM_ERR) { - retcode = 1; - } - else { - retcode = 0; - } - Jim_FreeInterp(interp); - return retcode; -} -#endif DELETED autosetup/local.tcl Index: autosetup/local.tcl ================================================================== --- autosetup/local.tcl +++ autosetup/local.tcl @@ -1,31 +0,0 @@ -# For this project, disable the pager for --help and --ref -# The user can still enable by using --nopager=0 or --disable-nopager -dict set autosetup(optdefault) nopager 1 - -# Searches for a usable Tcl (prefer 8.6, 8.5, 8.4) in the given paths -# Returns a dictionary of the contents of the tclConfig.sh file, or -# empty if not found -proc parse-tclconfig-sh {args} { - foreach p $args { - # Allow pointing directly to the path containing tclConfig.sh - if {[file exists $p/tclConfig.sh]} { - return [parse-tclconfig-sh-file $p/tclConfig.sh] - } - # Some systems allow for multiple versions - foreach libpath {lib/tcl8.6 lib/tcl8.5 lib/tcl8.4 lib/tcl tcl lib} { - if {[file exists $p/$libpath/tclConfig.sh]} { - return [parse-tclconfig-sh-file $p/$libpath/tclConfig.sh] - } - } - } -} - -proc parse-tclconfig-sh-file {filename} { - foreach line [split [readfile $filename] \n] { - if {[regexp {^(TCL_[^=]*)=(.*)$} $line -> name value]} { - set value [regsub -all {\$\{.*\}} $value ""] - set tclconfig($name) [string trim $value '] - } - } - return [array get tclconfig] -} DELETED autosetup/pkg-config.tcl Index: autosetup/pkg-config.tcl ================================================================== --- autosetup/pkg-config.tcl +++ autosetup/pkg-config.tcl @@ -1,168 +0,0 @@ -# Copyright (c) 2016 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# @synopsis: -# -# The 'pkg-config' module allows package information to be found via 'pkg-config'. -# -# If not cross-compiling, the package path should be determined automatically -# by 'pkg-config'. -# If cross-compiling, the default package path is the compiler sysroot. -# If the C compiler doesn't support '-print-sysroot', the path can be supplied -# by the '--sysroot' option or by defining 'SYSROOT'. -# -# 'PKG_CONFIG' may be set to use an alternative to 'pkg-config'. - -use cc - -options { - sysroot:dir => "Override compiler sysroot for pkg-config search path" -} - -# @pkg-config-init ?required? -# -# Initialises the 'pkg-config' system. Unless '$required' is set to 0, -# it is a fatal error if a usable 'pkg-config' is not found . -# -# This command will normally be called automatically as required, -# but it may be invoked explicitly if lack of 'pkg-config' is acceptable. -# -# Returns 1 if ok, or 0 if 'pkg-config' not found/usable (only if '$required' is 0). -# -proc pkg-config-init {{required 1}} { - if {[is-defined HAVE_PKG_CONFIG]} { - return [get-define HAVE_PKG_CONFIG] - } - set found 0 - - define PKG_CONFIG [get-env PKG_CONFIG pkg-config] - msg-checking "Checking for pkg-config..." - - if {[catch {exec [get-define PKG_CONFIG] --version} version]} { - msg-result "[get-define PKG_CONFIG] (not found)" - if {$required} { - user-error "No usable pkg-config" - } - } else { - msg-result $version - define PKG_CONFIG_VERSION $version - - set found 1 - - if {[opt-str sysroot o]} { - define SYSROOT [file-normalize $o] - msg-result "Using specified sysroot [get-define SYSROOT]" - } elseif {[get-define build] ne [get-define host]} { - if {[catch {exec-with-stderr {*}[get-define CC] -print-sysroot} result errinfo] == 0} { - # Use the compiler sysroot, if there is one - define SYSROOT $result - msg-result "Found compiler sysroot $result" - } else { - configlog "[get-define CC] -print-sysroot: $result" - set msg "pkg-config: Cross compiling, but no compiler sysroot and no --sysroot supplied" - if {$required} { - user-error $msg - } else { - msg-result $msg - } - set found 0 - } - } - if {[is-defined SYSROOT]} { - set sysroot [get-define SYSROOT] - - # XXX: It's possible that these should be set only when invoking pkg-config - global env - set env(PKG_CONFIG_DIR) "" - # Supposedly setting PKG_CONFIG_LIBDIR means that PKG_CONFIG_PATH is ignored, - # but it doesn't seem to work that way in practice - set env(PKG_CONFIG_PATH) "" - # Do we need to try /usr/local as well or instead? - set env(PKG_CONFIG_LIBDIR) $sysroot/usr/lib/pkgconfig:$sysroot/usr/share/pkgconfig - set env(PKG_CONFIG_SYSROOT_DIR) $sysroot - } - } - define HAVE_PKG_CONFIG $found - return $found -} - -# @pkg-config module ?requirements? -# -# Use 'pkg-config' to find the given module meeting the given requirements. -# e.g. -# -## pkg-config pango >= 1.37.0 -# -# If found, returns 1 and sets 'HAVE_PKG_PANGO' to 1 along with: -# -## PKG_PANGO_VERSION to the found version -## PKG_PANGO_LIBS to the required libs (--libs-only-l) -## PKG_PANGO_LDFLAGS to the required linker flags (--libs-only-L) -## PKG_PANGO_CFLAGS to the required compiler flags (--cflags) -# -# If not found, returns 0. -# -proc pkg-config {module args} { - set ok [pkg-config-init] - - msg-checking "Checking for $module $args..." - - if {!$ok} { - msg-result "no pkg-config" - return 0 - } - - set pkgconfig [get-define PKG_CONFIG] - - set ret [catch {exec $pkgconfig --modversion "$module $args"} version] - configlog "$pkgconfig --modversion $module $args: $version" - if {$ret} { - msg-result "not found" - return 0 - } - # Sometimes --modversion succeeds but because of dependencies it isn't usable - # This seems to show up with --cflags - set ret [catch {exec $pkgconfig --cflags $module} cflags] - if {$ret} { - msg-result "unusable ($version - see config.log)" - configlog "$pkgconfig --cflags $module" - configlog $cflags - return 0 - } - msg-result $version - set prefix [feature-define-name $module PKG_] - define HAVE_${prefix} - define ${prefix}_VERSION $version - define ${prefix}_CFLAGS $cflags - define ${prefix}_LIBS [exec $pkgconfig --libs-only-l $module] - define ${prefix}_LDFLAGS [exec $pkgconfig --libs-only-L $module] - return 1 -} - -# @pkg-config-get module setting -# -# Convenience access to the results of 'pkg-config'. -# -# For example, '[pkg-config-get pango CFLAGS]' returns -# the value of 'PKG_PANGO_CFLAGS', or '""' if not defined. -proc pkg-config-get {module name} { - set prefix [feature-define-name $module PKG_] - get-define ${prefix}_${name} "" -} - -# @pkg-config-get-var module variable -# -# Return the value of the given variable from the given pkg-config module. -# The module must already have been successfully detected with pkg-config. -# e.g. -# -## if {[pkg-config harfbuzz >= 2.5]} { -## define harfbuzz_libdir [pkg-config-get-var harfbuzz libdir] -## } -# -# Returns the empty string if the variable isn't defined. -proc pkg-config-get-var {module variable} { - set pkgconfig [get-define PKG_CONFIG] - set prefix [feature-define-name $module HAVE_PKG_] - exec $pkgconfig $module --variable $variable -} DELETED autosetup/system.tcl Index: autosetup/system.tcl ================================================================== --- autosetup/system.tcl +++ autosetup/system.tcl @@ -1,407 +0,0 @@ -# Copyright (c) 2010 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# @synopsis: -# -# This module supports common system interrogation and options -# such as '--host', '--build', '--prefix', and setting 'srcdir', 'builddir', and 'EXEEXT'. -# -# It also support the "feature" naming convention, where searching -# for a feature such as 'sys/type.h' defines 'HAVE_SYS_TYPES_H'. -# -# It defines the following variables, based on '--prefix' unless overridden by the user: -# -## datadir -## sysconfdir -## sharedstatedir -## localstatedir -## infodir -## mandir -## includedir -# -# If '--prefix' is not supplied, it defaults to '/usr/local' unless 'defaultprefix' is defined *before* -# including the 'system' module. - -if {[is-defined defaultprefix]} { - user-notice "Note: defaultprefix is deprecated. Use options-defaults to set default options" - options-defaults [list prefix [get-define defaultprefix]] -} - -options { - host:host-alias => {a complete or partial cpu-vendor-opsys for the system where - the application will run (defaults to the same value as --build)} - build:build-alias => {a complete or partial cpu-vendor-opsys for the system - where the application will be built (defaults to the - result of running config.guess)} - prefix:dir=/usr/local => {the target directory for the build (default: '@default@')} - - # These (hidden) options are supported for autoconf/automake compatibility - exec-prefix: - bindir: - sbindir: - includedir: - mandir: - infodir: - libexecdir: - datadir: - libdir: - sysconfdir: - sharedstatedir: - localstatedir: - runstatedir: - maintainer-mode=0 - dependency-tracking=0 - silent-rules=0 -} - -# @check-feature name { script } -# -# defines feature '$name' to the return value of '$script', -# which should be 1 if found or 0 if not found. -# -# e.g. the following will define 'HAVE_CONST' to 0 or 1. -# -## check-feature const { -## cctest -code {const int _x = 0;} -## } -proc check-feature {name code} { - msg-checking "Checking for $name..." - set r [uplevel 1 $code] - define-feature $name $r - if {$r} { - msg-result "ok" - } else { - msg-result "not found" - } - return $r -} - -# @have-feature name ?default=0? -# -# Returns the value of feature '$name' if defined, or '$default' if not. -# -# See 'feature-define-name' for how the "feature" name -# is translated into the "define" name. -# -proc have-feature {name {default 0}} { - get-define [feature-define-name $name] $default -} - -# @define-feature name ?value=1? -# -# Sets the feature 'define' to '$value'. -# -# See 'feature-define-name' for how the "feature" name -# is translated into the "define" name. -# -proc define-feature {name {value 1}} { - define [feature-define-name $name] $value -} - -# @feature-checked name -# -# Returns 1 if feature '$name' has been checked, whether true or not. -# -proc feature-checked {name} { - is-defined [feature-define-name $name] -} - -# @feature-define-name name ?prefix=HAVE_? -# -# Converts a "feature" name to the corresponding "define", -# e.g. 'sys/stat.h' becomes 'HAVE_SYS_STAT_H'. -# -# Converts '*' to 'P' and all non-alphanumeric to underscore. -# -proc feature-define-name {name {prefix HAVE_}} { - string toupper $prefix[regsub -all {[^a-zA-Z0-9]} [regsub -all {[*]} $name p] _] -} - -# @write-if-changed filename contents ?script? -# -# If '$filename' doesn't exist, or it's contents are different to '$contents', -# the file is written and '$script' is evaluated. -# -# Otherwise a "file is unchanged" message is displayed. -proc write-if-changed {file buf {script {}}} { - set old [readfile $file ""] - if {$old eq $buf && [file exists $file]} { - msg-result "$file is unchanged" - } else { - writefile $file $buf\n - uplevel 1 $script - } -} - - -# @include-file infile mapping -# -# The core of make-template, called recursively for each @include -# directive found within that template so that this proc's result -# is the fully-expanded template. -# -# The mapping parameter is how we expand @varname@ within the template. -# We do that inline within this step only for @include directives which -# can have variables in the filename arg. A separate substitution pass -# happens when this recursive function returns, expanding the rest of -# the variables. -# -proc include-file {infile mapping} { - # A stack of true/false conditions, one for each nested conditional - # starting with "true" - set condstack {1} - set result {} - set linenum 0 - foreach line [split [readfile $infile] \n] { - incr linenum - if {[regexp {^@(if|else|endif)(\s*)(.*)} $line -> condtype condspace condargs]} { - if {$condtype eq "if"} { - if {[string length $condspace] == 0} { - autosetup-error "$infile:$linenum: Invalid expression: $line" - } - if {[llength $condargs] == 1} { - # ABC => [get-define ABC] ni {0 ""} - # !ABC => [get-define ABC] in {0 ""} - lassign $condargs condvar - if {[regexp {^!(.*)} $condvar -> condvar]} { - set op in - } else { - set op ni - } - set condexpr "\[[list get-define $condvar]\] $op {0 {}}" - } else { - # Translate alphanumeric ABC into [get-define ABC] and leave the - # rest of the expression untouched - regsub -all {([A-Z][[:alnum:]_]*)} $condargs {[get-define \1]} condexpr - } - if {[catch [list expr $condexpr] condval]} { - dputs $condval - autosetup-error "$infile:$linenum: Invalid expression: $line" - } - dputs "@$condtype: $condexpr => $condval" - } - if {$condtype ne "if"} { - if {[llength $condstack] <= 1} { - autosetup-error "$infile:$linenum: Error: @$condtype missing @if" - } elseif {[string length $condargs] && [string index $condargs 0] ne "#"} { - autosetup-error "$infile:$linenum: Error: Extra arguments after @$condtype" - } - } - switch -exact $condtype { - if { - # push condval - lappend condstack $condval - } - else { - # Toggle the last entry - set condval [lpop condstack] - set condval [expr {!$condval}] - lappend condstack $condval - } - endif { - if {[llength $condstack] == 0} { - user-notice "$infile:$linenum: Error: @endif missing @if" - } - lpop condstack - } - } - continue - } elseif {[regexp {^@include\s+(.*)} $line -> filearg]} { - set incfile [string map $mapping $filearg] - if {[file exists $incfile]} { - lappend ::autosetup(deps) [file-normalize $incfile] - lappend result {*}[include-file $incfile $mapping] - } else { - user-error "$infile:$linenum: Include file $incfile is missing" - } - continue - } elseif {[regexp {^@define\s+(\w+)\s+(.*)} $line -> var val]} { - define $var $val - continue - } - # Only output this line if the stack contains all "true" - if {"0" in $condstack} { - continue - } - lappend result $line - } - return $result -} - - -# @make-template template ?outfile? -# -# Reads the input file '/$template' and writes the output file '$outfile' -# (unless unchanged). -# If '$outfile' is blank/omitted, '$template' should end with '.in' which -# is removed to create the output file name. -# -# Each pattern of the form '@define@' is replaced with the corresponding -# "define", if it exists, or left unchanged if not. -# -# The special value '@srcdir@' is substituted with the relative -# path to the source directory from the directory where the output -# file is created, while the special value '@top_srcdir@' is substituted -# with the relative path to the top level source directory. -# -# Conditional sections may be specified as follows: -## @if NAME eq "value" -## lines -## @else -## lines -## @endif -# -# Where 'NAME' is a defined variable name and '@else' is optional. -# Note that variables names *must* start with an uppercase letter. -# If the expression does not match, all lines through '@endif' are ignored. -# -# The alternative forms may also be used: -## @if NAME (true if the variable is defined, but not empty and not "0") -## @if !NAME (opposite of the form above) -## @if -# -# In the general Tcl expression, any words beginning with an uppercase letter -# are translated into [get-define NAME] -# -# Expressions may be nested -# -proc make-template {template {out {}}} { - set infile [file join $::autosetup(srcdir) $template] - - if {![file exists $infile]} { - user-error "Template $template is missing" - } - - # Define this as late as possible - define AUTODEPS $::autosetup(deps) - - if {$out eq ""} { - if {[file ext $template] ne ".in"} { - autosetup-error "make_template $template has no target file and can't guess" - } - set out [file rootname $template] - } - - set outdir [file dirname $out] - - # Make sure the directory exists - file mkdir $outdir - - # Set up srcdir and top_srcdir to be relative to the target dir - define srcdir [relative-path [file join $::autosetup(srcdir) $outdir] $outdir] - define top_srcdir [relative-path $::autosetup(srcdir) $outdir] - - # Build map from global defines to their values so they can be - # substituted into @include file names. - proc build-define-mapping {} { - set mapping {} - foreach {n v} [array get ::define] { - lappend mapping @$n@ $v - } - return $mapping - } - set mapping [build-define-mapping] - - set result [include-file $infile $mapping] - - # Rebuild the define mapping in case we ran across @define - # directives in the template or a file it @included, then - # apply that mapping to the expanded template. - set mapping [build-define-mapping] - write-if-changed $out [string map $mapping [join $result \n]] { - msg-result "Created [relative-path $out] from [relative-path $template]" - } -} - -# build/host tuples and cross-compilation prefix -opt-str build build "" -define build_alias $build -if {$build eq ""} { - define build [config_guess] -} else { - define build [config_sub $build] -} - -opt-str host host "" -define host_alias $host -if {$host eq ""} { - define host [get-define build] - set cross "" -} else { - define host [config_sub $host] - set cross $host- -} -define cross [get-env CROSS $cross] - -# build/host _cpu, _vendor and _os -foreach type {build host} { - set v [get-define $type] - if {![regexp {^([^-]+)-([^-]+)-(.*)$} $v -> cpu vendor os]} { - user-error "Invalid canonical $type: $v" - } - define ${type}_cpu $cpu - define ${type}_vendor $vendor - define ${type}_os $os -} - -opt-str prefix prefix /usr/local - -# These are for compatibility with autoconf -define target [get-define host] -define prefix $prefix -define builddir $autosetup(builddir) -define srcdir $autosetup(srcdir) -define top_srcdir $autosetup(srcdir) -define abs_top_srcdir [file-normalize $autosetup(srcdir)] -define abs_top_builddir [file-normalize $autosetup(builddir)] - -# autoconf supports all of these -define exec_prefix [opt-str exec-prefix exec_prefix $prefix] -foreach {name defpath} { - bindir /bin - sbindir /sbin - libexecdir /libexec - libdir /lib -} { - define $name [opt-str $name o $exec_prefix$defpath] -} -foreach {name defpath} { - datadir /share - sharedstatedir /com - infodir /share/info - mandir /share/man - includedir /include -} { - define $name [opt-str $name o $prefix$defpath] -} -if {$prefix ne {/usr}} { - opt-str sysconfdir sysconfdir $prefix/etc -} else { - opt-str sysconfdir sysconfdir /etc -} -define sysconfdir $sysconfdir - -define localstatedir [opt-str localstatedir o /var] -define runstatedir [opt-str runstatedir o /run] - -define SHELL [get-env SHELL [find-an-executable sh bash ksh]] - -# These could be used to generate Makefiles following some automake conventions -define AM_SILENT_RULES [opt-bool silent-rules] -define AM_MAINTAINER_MODE [opt-bool maintainer-mode] -define AM_DEPENDENCY_TRACKING [opt-bool dependency-tracking] - -# Windows vs. non-Windows -switch -glob -- [get-define host] { - *-*-ming* - *-*-cygwin - *-*-msys { - define-feature windows - define EXEEXT .exe - } - default { - define EXEEXT "" - } -} - -# Display -msg-result "Host System...[get-define host]" -msg-result "Build System...[get-define build]" DELETED autosetup/tmake.auto Index: autosetup/tmake.auto ================================================================== --- autosetup/tmake.auto +++ autosetup/tmake.auto @@ -1,55 +0,0 @@ -# Copyright (c) 2016 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# Auto-load module for 'tmake' build system integration - -use init - -autosetup_add_init_type tmake "Tcl-based tmake build system" { - autosetup_check_create auto.def \ -{# Initial auto.def created by 'autosetup --init=tmake' -# vim:set syntax=tcl: - -use cc cc-lib cc-db cc-shared -use tmake - -# Add any user options here -# Really want a --configure that takes over the rest of the command line -options { -} - -cc-check-tools ar ranlib - -set objdir [get-env BUILDDIR objdir] - -make-config-header $objdir/include/autoconf.h -make-tmake-settings $objdir/settings.conf {[A-Z]*} *dir lib_* -} - - autosetup_check_create project.spec \ -{# Initial project.spec created by 'autosetup --init=tmake' - -tmake-require-version 0.7.3 - -# vim:set syntax=tcl: -define? DESTDIR _install - -# XXX If configure creates additional/different files than include/autoconf.h -# that should be reflected here -Autosetup include/autoconf.h - -# e.g. for autoconf.h -IncludePaths include - -ifconfig !CONFIGURED { - # Not configured, so don't process subdirs - AutoSubDirs off - # And don't process this file any further - ifconfig false -} -} - - if {![file exists build.spec]} { - puts "Note: I don't see build.spec. Try running: tmake --genie" - } -} DELETED autosetup/tmake.tcl Index: autosetup/tmake.tcl ================================================================== --- autosetup/tmake.tcl +++ autosetup/tmake.tcl @@ -1,52 +0,0 @@ -# Copyright (c) 2011 WorkWare Systems http://www.workware.net.au/ -# All rights reserved - -# @synopsis: -# -# The 'tmake' module makes it easy to support the tmake build system. -# -# The following variables are set: -# -## CONFIGURED - to indicate that the project is configured - -use system - -options {} - -define CONFIGURED - -# @make-tmake-settings outfile patterns ... -# -# Examines all defined variables which match the given patterns (defaults to '*') -# and writes a tmake-compatible .conf file defining those variables. -# For example, if 'ABC' is '"3 monkeys"' and 'ABC' matches a pattern, then the file will include: -# -## define ABC {3 monkeys} -# -# If the file would be unchanged, it is not written. -# -# Typical usage is: -# -## make-tmake-settings [get-env BUILDDIR objdir]/settings.conf {[A-Z]*} -proc make-tmake-settings {file args} { - file mkdir [file dirname $file] - set lines {} - - if {[llength $args] == 0} { - set args * - } - - foreach n [lsort [dict keys [all-defines]]] { - foreach p $args { - if {[string match $p $n]} { - set value [get-define $n] - lappend lines "define $n [list $value]" - break - } - } - } - set buf [join $lines \n] - write-if-changed $file $buf { - msg-result "Created $file" - } -} DELETED autosetup/wh-common.tcl Index: autosetup/wh-common.tcl ================================================================== --- autosetup/wh-common.tcl +++ autosetup/wh-common.tcl @@ -1,411 +0,0 @@ -######################################################################## -# Routines for Steve Bennett's autosetup which are common to trees -# managed under the umbrella of wanderinghorse.net. -# -# In the interest of assisting to keep multiple copies of this file -# up to date:s -# -# The "canonical" version is the one in libfossil: -# -# https://fossil.wanderinghorse.net/r/libfossil/finfo?name=autosetup/wh-common.tcl -######################################################################## - -array set whcache {} ; # used for caching various results. - -######################################################################## -# wh-lshift shifts $count elements from the list named $listVar and -# returns them. -# -# Modified slightly from: https://wiki.tcl-lang.org/page/lshift -# -# On an empty list, returns "". -proc wh-lshift {listVar {count 1}} { - upvar 1 $listVar l - if {![info exists l]} { - # make the error message show the real variable name - error "can't read \"$listVar\": no such variable" - } - if {![llength $l]} { - # error Empty - return "" - } - set r [lrange $l 0 [incr count -1]] - set l [lreplace $l [set l 0] $count] - return $r -} - -######################################################################## -# A proxy for cc-check-function-in-lib which "undoes" any changes that -# routine makes to the LIBS define. Returns the result of -# cc-check-function-in-lib. -proc wh-check-function-in-lib {function libs {otherlibs {}}} { - set _LIBS [get-define LIBS] - set found [cc-check-function-in-lib $function $libs $otherlibs] - define LIBS $_LIBS - return $found -} - -######################################################################## -# Look for binary named $binName and `define`s $defName to that full -# path, or an empty string if not found. Returns the value it defines. -# This caches the result for a given $binName/$defName combination, so -# calls after the first for a given combination will always return the -# same result. -# -# If defName is empty then "BIN_X" is used, where X is the upper-case -# form of $binName with any '-' characters removed. -proc wh-bin-define {binName {defName {}}} { - global whcache - set cacheName "$binName:$defName" - set check {} - if {[info exists whcache($cacheName)]} { - set check $whcache($cacheName) - } - msg-checking "Looking for $binName ... " - if {"" ne $check} { - set lbl $check - if {" _ 0 _ " eq $check} { - set lbl "not found" - set check "" - } - msg-result "(cached) $lbl" - return $check - } - set check [find-executable-path $binName] - if {"" eq $check} { - msg-result "not found" - set whcache($cacheName) " _ 0 _ " - } else { - msg-result $check - set whcache($cacheName) $check - } - if {"" eq $defName} { - set defName "BIN_[string toupper [string map {- {}} $binName]]" - } - define $defName $check - return $check -} - -######################################################################## -# Looks for `bash` binary and dies if not found. On success, defines -# BIN_BASH to the full path to bash and returns that value. We -# _require_ bash because it's the SHELL value used in our makefiles. -proc wh-require-bash {} { - set bash [wh-bin-define bash] - if {"" eq $bash} { - user-error "Our Makefiles require the bash shell." - } - return $bash -} - -######################################################################## -# Internal impl for wh-opt-bool-01 and wh-opt-bool-01-invert. -# -# args = {optName defName invert {descr {}}} -proc wh-opt-bool-01-impl {args} { -} - - -######################################################################## -# Args: [-v] optName defName {descr {}} -# -# Checks [opt-bool $optName] and does [define $defName X] where X is 0 -# for false and 1 for true. descr is an optional [msg-checking] -# argument which defaults to $defName. Returns X. -# -# If args[0] is -v then the boolean semantics are inverted: if -# the option is set, it gets define'd to 0, else 1. Returns the -# define'd value. -proc wh-opt-bool-01 {args} { - set invert 0 - if {[lindex $args 0] eq "-v"} { - set invert 1 - set args [lrange $args 1 end] - } - set optName [wh-lshift args] - set defName [wh-lshift args] - set descr [wh-lshift args] - if {"" eq $descr} { - set descr $defName - } - set rc 0 - msg-checking "$descr ... " - if {[opt-bool $optName]} { - if {0 eq $invert} { - set rc 1 - } else { - set rc 0 - } - } elseif {0 ne $invert} { - set rc 1 - } - msg-result $rc - define $defName $rc - return $rc -} - - -######################################################################## -# Curses! -# -# Jumps through numerous hoops to try to find ncurses libraries and -# appropriate compilation/linker flags. Returns 0 on failure, 1 on -# success, and defines (either way) LIB_CURSES to the various linker -# flags and CFLAGS_CURSES to the various CFLAGS (both empty strings if -# no lib is found). -# -# This impl prefers to use pkg-config to find ncurses and libpanel -# because various platforms either combine, or not, the wide-char -# versions of those libs into the same library (or not). If no -# pkg-config is available, OR the platform looks like Mac, then we -# simply make an educated guess and hope it works. On Mac pkg-config -# is not sufficient because the core system and either brew or -# macports can contain mismatched versions of ncurses and iconv, so on -# that platform we simply guess from the core system level, ignoring -# brew/macports options. -proc wh-check-ncurses {} { - set pcBin [wh-bin-define pkg-config] - msg-checking "Looking for \[n]curses... " - set LIB_CURSES "" - set CFLAGS_CURSES "" - set rc 0 - if {"" ne $pcBin && $::tcl_platform(os)!="Darwin"} { - # Some macOS pkg-config configurations alter library search paths, which make - # the compiler unable to find lib iconv, so don't use pkg-config on macOS. - set np "" - foreach p {ncursesw ncurses} { - if {[catch {exec $pcBin --exists $p}]} { - continue - } - set np $p - msg-result "Using pkg-config curses package \[$p]" - break - } - if {"" ne $np} { - set ppanel "" - if {"ncursesw" eq $np} { - if {![catch {exec $pcBin --exists panelw}]} { - set ppanel panelw - } - } - if {"" eq $ppanel && ![catch {exec $pcBin --exists panel}]} { - set ppanel panel - } - set CFLAGS_CURSES [exec $pcBin --cflags $np] - set LIB_CURSES [exec $pcBin --libs $np] - if {"" eq $ppanel} { - # Apparently Mac brew has pkg-config for ncursesw but not - # panel/panelw, but hard-coding -lpanel seems to work on - # that platform. - append LIB_CURSES " -lpanel" - } else { - append LIB_CURSES " " [exec $pcBin --libs $ppanel] - # append CFLAGS_CURSES " " [exec $pcBin --cflags $ppanel] - # ^^^^ appending the panel cflags will end up duplicating - # at least one -D flag from $np's cflags, leading to - # "already defined" errors at compile-time. Sigh. Note, however, - # that $ppanel's cflags have flags which $np's do not, so we - # may need to include those flags anyway and manually perform - # surgery on the list to remove dupes. Sigh. - } - } - } - - if {"" eq $LIB_CURSES} { - puts "Guessing curses location (will fail for exotic locations)..." - define HAVE_CURSES_H [cc-check-includes curses.h] - if {[get-define HAVE_CURSES_H]} { - # Linux has -lncurses, BSD -lcurses. Both have - msg-result "Found curses.h" - if {[wh-check-function-in-lib waddnwstr ncursesw]} { - msg-result "Found -lncursesw" - set LIB_CURSES "-lncursesw -lpanelw" - } elseif {[wh-check-function-in-lib initscr ncurses]} { - msg-result "Found -lncurses" - set LIB_CURSES "-lncurses -lpanel" - } elseif {[wh-check-function-in-lib initscr curses]} { - msg-result "Found -lcurses" - set LIB_CURSES "-lcurses -lpanel" - } - } - } - if {"" ne $LIB_CURSES} { - set rc 1 - puts {************************************************************ -If your build of fails due to missing ncurses functions -such as waddwstr(), make sure you have the ncursesw (with a -"w") development package installed. Some platforms combine -the "w" and non-w curses builds and some don't. Similarly, -it's easy to get a mismatch between libncursesw and libpanel, -so make sure you have libpanelw (if appropriate for your -platform). - -The package may have a name such as libncursesw5-dev or -some such. -************************************************************} - } - define LIB_CURSES $LIB_CURSES - define CFLAGS_CURSES $CFLAGS_CURSES - return $rc -} -# /wh-check-ncurses -######################################################################## - -######################################################################## -# Check for module-loading APIs (libdl/libltdl)... -# -# Looks for libltdl or dlopen(), the latter either in -ldl or built in -# to libc (as it is on some platforms. Returns 1 if found, else -# 0. Either way, it `define`'s: -# -# - HAVE_LIBLTDL to 1 or 0 if libltdl is found/not found -# - HAVE_LIBDL to 1 or 0 if dlopen() is found/not found -# - LDFLAGS_MODULE_LOADER one of ("-lltdl", "-ldl", or ""), noting that -# the string may legally be empty on some platforms if HAVE_LIBDL is true. -proc wh-check-module-loader {} { - msg-checking "Looking for module-loader APIs... " - if {99 ne [get-define LDFLAGS_MODULE_LOADER]} { - if {1 eq [get-define HAVE_LIBLTDL 0]} { - msg-result "(cached) libltdl" - return 1 - } elseif {1 eq [get-define HAVE_LIBDL 0]} { - msg-result "(cached) libdl" - return 1 - } - # else: wha??? - } - set HAVE_LIBLTDL 0 - set HAVE_LIBDL 0 - set LDFLAGS_MODULE_LOADER "" - set rc 0 - puts "" ;# cosmetic kludge for cc-check-XXX - if {[cc-check-includes ltdl.h] && [cc-check-function-in-lib lt_dlopen ltdl]} { - set HAVE_LIBLTDL 1 - set LDFLAGS_MODULE_LOADER "-lltdl" - puts " - Got libltdl." - set rc 1 - } elseif {[cc-with {-includes dlfcn.h} { - cctest -link 1 -declare "extern char* dlerror(void);" -code "dlerror();"}]} { - puts " - This system can use dlopen() w/o -ldl." - set HAVE_LIBDL 1 - set LDFLAGS_MODULE_LOADER "" - set rc 1 - } elseif {[cc-check-includes dlfcn.h]} { - set HAVE_LIBDL 1 - set rc 1 - if {[cc-check-function-in-lib dlopen dl]} { - puts " - dlopen() needs libdl." - set LDFLAGS_MODULE_LOADER "-ldl" - } else { - puts " - dlopen() not found in libdl. Assuming dlopen() is built-in." - set LDFLAGS_MODULE_LOADER "" - } - } - define HAVE_LIBLTDL $HAVE_LIBLTDL - define HAVE_LIBDL $HAVE_LIBDL - define LDFLAGS_MODULE_LOADER $LDFLAGS_MODULE_LOADER - return $rc -} - - -######################################################################## -# Opens the given file, reads all of its content, and returns it. -proc wh-file-content {fname} { - set fp [open $fname r] - set rc [read $fp] - close $fp - return $rc -} - -######################################################################## -# Returns the contents of the given file as an array, with the EOL -# stripped from each input line. -proc wh-file-content-list {fname} { - set fp [open $fname r] - set rc {} - while { [gets $fp line] >= 0 } { - lappend rc $line - } - return $rc -} - -######################################################################## -# Checks the compiler for compile_commands.json support. If passed an -# argument it is assumed to be the name of an autosetup boolean config -# option to explicitly DISABLE the compile_commands.json support. -# -# Returns 1 if supported, else 0. Defines MAKE_COMPILATION_DB to "yes" -# if supported, "no" if not. -proc wh-check-compile-commands {{configOpt {}}} { - msg-checking "compile_commands.json support... " - if {"" ne $configOpt && [opt-bool $configOpt]} { - msg-result "explicitly disabled" - define MAKE_COMPILATION_DB no - return 0 - } else { - if {[cctest -lang c -cflags {/dev/null -MJ} -source {}]} { - # This test reportedly incorrectly succeeds on one of - # Martin G.'s older systems. - msg-result "compiler supports compile_commands.json" - define MAKE_COMPILATION_DB yes - return 1 - } else { - msg-result "compiler does not support compile_commands.json" - define MAKE_COMPILATION_DB no - return 0 - } - } -} - -######################################################################## -# Uses [make-template] to creates makefile(-like) file $filename from -# $filename.in but explicitly makes the output read-only, to avoid -# inadvertent editing (who, me?). -# -# The second argument is an optional boolean specifying whether to -# `touch` the generates files. This can be used as a workaround for -# cases where (A) autosetup does not update the file because it was -# not really modified and (B) the file *really* needs to be updates to -# the please build process. Pass any non-0 value to enable touching. -# -# The argument may be a list of filenames. -proc wh-make-from-dot-in {filename {touch 0}} { - foreach f $filename { - catch { exec chmod u+w $f } - make-template $f.in $f - if {0 != $touch} { - puts "Touching $f" - catch { exec touch $f } - } - catch { exec chmod u-w $f } - } -} - -######################################################################## -# Checks for the boolean configure option named by $flagname. If set, -# it checks if $CC seems to refer to gcc. If it does (or appears to) -# then it defines CC_PROFILE_FLAG to "-pg" and returns 1, else it -# defines CC_PROFILE_FLAG to "" and returns 0. -# -# Note that the resulting flag must be added to both CFLAGS and -# LDFLAGS in order for binaries to be able to generate "gmon.out". In -# order to avoid potential problems with escaping, space-containing -# tokens, and interfering with autosetup's use of these vars, this -# routine does not directly modify CFLAGS or LDFLAGS. -proc wh-check-profile-flag {{flagname profile}} { - if {[opt-bool $flagname]} { - set CC [get-define CC] - regsub {.*ccache *} $CC "" CC - # ^^^ if CC="ccache gcc" then [exec] treats "ccache gcc" as a - # single binary name and fails. So strip any leading ccache part - # for this purpose. - if { ![catch { exec $CC --version } msg]} { - if {[string first gcc $CC] != -1} { - define CC_PROFILE_FLAG "-pg" - return 1 - } - } - } - define CC_PROFILE_FLAG "" - return 0 -} Index: compile_flags.txt ================================================================== --- compile_flags.txt +++ compile_flags.txt @@ -1,10 +1,10 @@ -std=c89 -I include -I -src +lib -isystem /usr/local/include -isystem /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/12.0.0/include -isystem DELETED config.make.in Index: config.make.in ================================================================== --- config.make.in +++ config.make.in @@ -1,89 +0,0 @@ -# Example of a typical Makefile template for autosetup - -# Tools. CC is standard. The rest are via cc-check-tools -CC = @CC@ -RANLIB = @RANLIB@ -AR = @AR@ -STRIP = @STRIP@ - -# FLAGS/LIBS -CFLAGS += @CFLAGS@ @CC_PROFILE_FLAG@ -LDFLAGS += @LDFLAGS@ @CC_PROFILE_FLAG@ -LDLIBS += @LIBS@ -#LDLIBS += -lsqlite3 -lz -LDFLAGS_MODULE_LOADER += @LDFLAGS_MODULE_LOADER@ - -# ??? HAVE_DLOPEN := @HAVE_DLOPEN@ -HAVE_LIBDL := @HAVE_LIBDL@ -HAVE_LIBLTDL := @HAVE_LIBLTDL@ -#HAVE_LT_DLOPEN := @HAVE_LT_DLOPEN@ - -ENABLE_MODULES ?= @FSL_ENABLE_MODULE_LOADER@ -#ifeq (1,$(HAVE_DLOPEN)) -# ENABLE_MODULES := 1 -#endif -#ifeq (1,$(HAVE_LT_DLOPEN)) -# ENABLE_MODULES := 1 -#endif - -# Install destination -prefix := @prefix@ -exec_prefix = @exec_prefix@ -DESTDIR ?= - -CONFIG.MAKE := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) -#TOP_SRCDIR_ABS := $(dir $(realpath $(CONFIG.MAKE))) -TOP_SRCDIR_REL := $(patsubst %/,%,$(dir $(CONFIG.MAKE))) -TOP_SRCDIR_ABS := $(realpath $(TOP_SRCDIR_REL)) -#$(error TOP_SRCDIR_REL=$(TOP_SRCDIR_REL)) - -SRC.DIR := $(TOP_SRCDIR_REL)/src - -# Project-specific CFLAGS -TOP_INCDIR := $(TOP_SRCDIR_REL)/include -COMPAT_DIR := $(TOP_SRCDIR_REL)/compat -#CPPFLAGS += -D_GNU_SOURCE -Wall -Werror -I. -I$(TOP_INCDIR) -CPPFLAGS += -I. -I$(TOP_INCDIR) -I$(SRC.DIR) -# -I$(COMPAT_DIR)/javavm/export -I$(COMPAT_DIR)/tcl-8.6/generic -ifneq (,$(wildcard $(HOME)/include/*)) - # This is admittedly primarily for my own sake! - CPPFLAGS += -I$(HOME)/include -endif - -ifeq (,$(filter tcc,$(CC))) - # Assume gcc-compatible compilation flags... - CFLAGS += -Wall -Werror -Wsign-compare -pedantic -#??? -fno-strict-aliasing - CFLAGS += @CC_FLAG_C99@ -endif -# /tcc - -# Enable MAKE_COMPILATION_DB = yes to generate a JSON compilation database. For -# each *.c source file being compiled, a corresponding JSON file will be created -# in the './compile_commands' sub-directory, and will be used to generate the -# 'compile_commands.json' compilation database in the repository root. This is -# only available with compilers that support the '-MJ' option (e.g., clang). -MAKE_COMPILATION_DB = @MAKE_COMPILATION_DB@ - - -FSL_ENABLE_READLINE := @FSL_ENABLE_READLINE@ -LDFLAGS_READLINE := @lib_readline@ - -ifeq ($(MAKE_COMPILATION_DB),yes) -compdb_dir = compile_commands -mkdir_compdb = $(TOP_SRCDIR_REL)/$(compdb_dir) -$(mkdir_compdb): - @mkdir -p $@ - -compdb_file = $(TOP_SRCDIR_REL)/$(compdb_dir)/$(subst /,-,$@.json) -compdb_args = -MJ $(compdb_file) -CFLAGS += $(compdb_args) -else -mkdir_compdb = -endif - - -CLEAN_FILES += *.o lib*.a *.so - -ShakeNMake.QUIET ?= @BUILD_QUIETLY@ -ShakeNMake.BIN.INSTALL := @BIN_INSTALL@ DELETED configure Index: configure ================================================================== --- configure +++ configure @@ -1,3 +0,0 @@ -#!/bin/sh -dir="`dirname "$0"`/autosetup" -WRAPPER="$0"; export WRAPPER; exec "`"$dir/autosetup-find-tclsh"`" "$dir/autosetup" "$@" DELETED fnc/Makefile.in Index: fnc/Makefile.in ================================================================== --- fnc/Makefile.in +++ fnc/Makefile.in @@ -1,36 +0,0 @@ -all: -include ../subdir-inc.make -LIB_CURSES := @LIB_CURSES@ -ifeq (,$(LIB_CURSES)) -$(error Expecting LIB_CURSES to be set by the configure process.) -endif -CFLAGS_CURSES := @CFLAGS_CURSES@ -LIBF_DIR := $(TOP_DIR)/src -CPPFLAGS += -I$(LIBF_DIR) $(CFLAGS_CURSES) -DISTCLEAN_FILES += Makefile - -######################################################################## -# To build fnc, we need the curses and pthread libraries. -# -lz is for libfossil -# -liconv is for libfossil on macOS -# -lutil is... ??? -# -lm is for sqlite3 -FNC_LDFLAGS := $(LIBF_DIR)/libfossil.o $(LIBF_DIR)/sqlite3.o -PLATFORM := $(shell sh -c 'uname 2>/dev/null || echo Unknown') -ifneq ($(filter $(PLATFORM),Darwin),) - FNC_LDFLAGS += -liconv -endif -FNC_LDFLAGS += -lm -FNC_LDFLAGS += $(LIB_CURSES) -lutil -lz -lpthread - -# This is needed for Mac builds, else they don't see the -# wide-char curses APIs: -CPPFLAGS += -D_XOPEN_SOURCE_EXTENDED - -fnc.BIN.OBJECTS += fnc.o -fnc.BIN.LDFLAGS := $(FNC_LDFLAGS) -$(eval $(call ShakeNMake.CALL.RULES.BIN,fnc)) -all: $(fnc.BIN) - -ShakeNMake.install.bins := $(fnc.BIN) -ShakeNMake.install.man1 := fnc.1 DELETED fnc/fnc.1 Index: fnc/fnc.1 ================================================================== --- fnc/fnc.1 +++ fnc/fnc.1 @@ -1,708 +0,0 @@ -.\" -.\" Copyright (c) 2021 Mark Jamsek -.\" -.\" Permission to use, copy, modify, and distribute this software for any -.\" purpose with or without fee is hereby granted, provided that the above -.\" copyright notice and this permission notice appear in all copies. -.\" -.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -.\" -.Dd $Mdocdate$ -.Dt FNC 1 -.Os -.Sh NAME -.Nm fnc -.Nd Read-only ncurses-based Fossil repository browser -.Sh SYNOPSIS -.Nm -.Op Ar command -.Op Fl h | -help -.Nm -.Op Fl h | -help -.Op Fl v | -version -.Nm -.Cm timeline -.Op Fl Cz -.Op Fl T Ar tag -.Op Fl b Ar branch -.Op Fl c Ar commit -.Op Fl n Ar number -.Op Fl t Ar type -.Op Fl u Ar user -.Op Ar path -.Nm -.Cm diff -.Op Fl Ciw -.Op Fl x Ar number -.Op Ar artifact1 Op Ar artifact2 -.Op Ar path ... -.Nm -.Cm tree -.Op Fl C -.Op Fl c Ar commit -.Op Ar path -.Nm -.Cm blame -.Op Fl C -.Op Fl c Ar commit Op Fl r -.Op Fl n Ar number -.Ar path -.Nm -.Cm branch -.Op Fl Ccopr -.Op Fl a Ar date | Fl b Ar date -.Op Fl s Ar order -.Op Ar glob -.Nm -.Op Ar path -.Sh DESCRIPTION -.Nm -is an interactive read-only browser for -.Xr fossil 1 -repositories, -and supports multiple views to display repository data: -.Bl -tag -width Ds -.It Timeline view -Display commits from the repository's history in chronologically -descending order. -.Br -If no -.Ar command -or -.Ar arg -are specified, or just a -.Ar path -is passed, -.Nm -will default to displaying this view. -.It Diff view -Display changes introduced in the specified commit, or between two repository -artifacts. -.It Tree view -Display navigable tree reflecting the repository state as at the specified -commit. -.It Blame view -Display and annotate each line in the given file with the hyperlinked historical -version that last modified the line. -.It Branch view -Display navigable list of all repository branches. -.El -.Pp -.Nm -provides both global and command-specific options and in-app key -bindings. Global options are specified absent a command name, and -are as follows: -.Bl -tag -width 6v -.It Fl h , -help -Display program help and usage information then exit. -.It Fl v , -version -Display program version then exit. -.El -.Pp -Note that any global options preceding a command name will be -interpreted as the command-specific variant if such an option exists. -.Pp -Global key bindings are as follows: -.Bl -tag -width Ds -.It Cm H, ?, F1 -Open in-app help. -.It Cm Tab -Switch focus between open views. -.It Cm f -Toggle the active view between fullscreen and splitscreen mode. Note -that -.Nm -will open nested views in splitscreen mode if the terminal window is -equal to or greater than 110 columns wide. -.It Cm Q -Immediatey quit -.Nm . -.It Cm q -Quit the active view. -.El -.Pp -Commands available to -.Nm -are as follows: -.Bl -tag -width 4v -.Tg log -.It Cm timeline Oo Fl C | -no-colour Oc Oo Fl T | -tag Ar tag Oc \ -Oo Fl b | -branch Ar branch Oc Oo Fl c | -commit Ar commit Oc \ -Oo Fl h | -help Oc Oo Fl n | -limit Ar n Oc Oo Fl t | -type Ar type Oc \ -Oo Fl u | -username Ar user Oc Oo Fl z | -utc Oc Op Ar path -.Dl (aliases: Cm log , Cm tl , Cm time , Cm ti ) -Display commit history of a repository. If -.Ar path -is specified, only commits that modified the file(s) at this path will populate -the timeline. The -.Ar path -may be absolute, relative to the current working directory, or relative to the -repository root. This command must be executed from within or below the top -level directory of the repository; that is, -.Nm -assumes a local checkout is open in or above the current working directory. -.Pp -If no command is explicitly specified, this command will be executed by -default. -.Pp -Options for -.Cm fnc timeline -are as follows: -.Bl -tag -width Ds -.It Fl C , -no-colour -Disable colourised timeline, which is enabled by default on supported -terminals. If this option is not used, colour can be toggled with the -.Sy c -timeline view key binding as documented below. -.It Fl T , -tag Ar tag -Only display commits with T cards containing -.Ar tag . -By default, -.Nm -will indiscriminately display all commits irrespective of which T cards -are attached to the commit manifest. -.It Fl b , -branch Ar branch -Display commits that are members of the specified -.Ar branch . -The expected argument is the symbolic name of a branch. By default, -.Nm -will display all commits irrespective of the branch on which they -reside. -.It Fl c , -commit Ar commit -Open the timeline from the check-in identified by -.Ar commit . -The expected argument is either the name of a branch, which will resolve -to the latest commit on the given branch, or (a unique abbreviated -prefix of) a valid commit UUID SHA1 or SHA3 hash. When this option is -not supplied, -.Nm -will open the timeline to the latest leaf on the repository tree. For a -complete list of valid arguments this option accepts, see -.Lk https://fossil-scm.org/home/doc/trunk/www/checkin_names.wiki \ -"Fossil's Check-in Names". -.It Fl h , -help -Display timeline command help and usage information then exit. -.It Fl n , -limit Ar n -Limit timeline to the -latest -.Ar n -commits. -By default, -.Nm -will load the entire history of the repository's local checkout. -Negative values are a no-op. -.It Fl t , -type Ar type -Only display -.Ar type -commits. Valid -.Ar type -values are as follows: -.Bl -column YXZ description -.Sy ci Ta check-in -.Sy w Ta wiki -.Sy t Ta ticket -.Sy e Ta technote -.Sy f Ta forum post -.El -.Pp -By default, when this option is not supplied, -.Nm -will indiscriminately load all commits irrespective of -.Ar type . -Note that this is a repeatable flag (e.g., -.Nm -.Cm timeline -.Cm -t e -t t Ns -). -.It Fl u , -username Ar user -Only display commits authored by -.Ar user . -.It Fl z , -utc -Use Coordinated Universal Time (UTC) rather than local time when -displaying commit dates and timestamps. -.El -.Pp -Key bindings for -.Cm fnc timeline -are as follows: -.Bl -tag -width Ds -.It Cm Arrow-down, j, >, \&. -Move selection cursor down the timeline. -.It Cm Arrow-up, k, <, \&, -Move selection cursor up the timeline. -.It Cm Ctrl+f, Page-down -Move selection cursor one page down the timeline. -.It Cm Ctrl+b, Page-up -Move selection cursor one page up the timeline. -.It Cm G, End -Move selection cursor to the last commit on the timeline (i.e., oldest commit -in the repository). -.It Cm gg, Home -Move selection cursor to the first commit on the timeline (i.e., newest commit -in the repository). -.It Cm Enter, Space -Open a -.Cm diff -view displaying the changeset of the currently selected commit. -.It Cm c -Toggle colourised timeline. On supported terminals, -.Nm -will default to displaying the timeline in colour. -.It Cm t -Display the tree of the repository corresponding to the currently selected -commit. -.It Cm / -Prompt to enter a search term to begin searching for commits matching -the pattern provided. The search term is an extended regular expression, -which is cross-referenced against a commit's comment, the username of -its author, branch, and UUID SHA1 or SHA3 hash. See -.Xr re_format 7 -for regular expression syntax. -.It Cm n -Find the next commit that matches the current search term. The search -will continue until either a match is found or the earliest commit on -the timeline is consumed. -.It Cm N -Find the previous commit that matches the current search term. The -search will continue until either a match is found or the latest commit -on the timeline is consumed. -.El -.Tg di -.It Cm diff Oo Fl C | -no-colour Oc Oo Fl h | -help Oc Oo Fl i | -invert \ -Oc Oo Fl w | -whitespace Oc Oo Fl x | -context Ar n Oc \ -Oo Ar artifact1 Oo Ar artifact2 Oc Oc Op Ar path ... -.Dl (alias: Cm di ) -Display the differences between two repository artifacts, or between the local -changes on disk and a given commit. If neither -.Ar artifact1 -nor -.Ar artifact2 -are specified, -.Nm -will diff the local changes on disk against the version on which the current -checkout is based. If only -.Ar artifact1 -is specified, -.Nm -will diff the current checkout, including any local changes on disk, against -this version. When both arguments are specified, the changes between these two -versions will be displayed. If supplied, -.Nm -will filter diffs between commits by -.Ar path -so that only changes involving the file(s) identified are displayed. Paths may -be absolute, relative to the current working directory, or relative to the -repository root. Both -.Ar artifact1 -and -.Ar artifact2 -must be of the same type, which is expected to be either a symbolic check-in -name, tag, (unique abbreviated prefix of) a commit or blob artifact UUID SHA1 -or SHA3 hash, or an ISO 8601 formatted date. Both artifact arguments must be -supplied when diffing blobs; any following non-option arguments are invalid -and will be ignored. -.Pp -Options for -.Cm fnc diff -are as follows: -.Bl -tag -width Ds -.It Fl C , -no-colour -Disable coloured diff output, which is enabled by default on supported -terminals. If this option is not used, colour can be toggled with the -.Sy c -diff view key binding as documented below. -.It Fl h , -help -Display diff command help and usage information then exit. -.It Fl i , -invert -Invert the difference between artifacts when displaying the diff. -.It Fl w , -whitespace -Ignore whitespace-only changes when displaying the diff. -.It Fl x , -context Ar n -Set -.Ar n -context lines to be shown in the diff. By default, 5 context lines are -shown. Negative values are a no-op. -.El -.Pp -Key bindings for -.Cm fnc diff -are as follows: -.Bl -tag -width Ds -.It Cm c -Toggle coloured diff output. On supported terminals, -.Nm -will default to displaying changes and diff metadata in colour. -.It Cm i -Toggle inversion of diff output. -.It Cm v -Toggle verbosity of diff output. By default, -.Nm -will display the entire content of newly added or deleted files. -.It Cm w -Toggle whether whitespace-only changes are ignored when comparing lines in the -diff. -.It Cm Arrow-down, j -Scroll down one line of diff output. -.It Cm Arrow-up, k -Scroll up one line of diff output. -.It Cm Ctrl+f, Page-down, Space -Scroll down one page of diff output. -.It Cm Ctrl+b, Page-up -Scroll up one page of diff output. -.It Cm G, End -Scroll to the end of the view (i.e., last line of diff output). -.It Cm gg, Home -Scroll to the top of the view (i.e., first line of diff output). -.It Cm \&-, \&_ -Decrease the number of context lines shown in diff output. -.It Cm \&=, \&+ -Increase the number of context lines shown in diff output. -.It Cm Ctrl+k, K, <, \&, -Move up the -.Cm timeline -to the previous (i.e., more recent) commit and display its diff. -.It Cm Ctrl+j, J, >, \&. -Move down the -.Cm timeline -to the next (i.e., earlier) commit and display its diff. -.It Cm / -Prompt to enter a search term to begin searching the diff output for -lines matching the pattern provided. The search term is an extended -regular expression, which is documented in -.Xr re_format 7 . -.It Cm n -Find the next line that matches the current search term. -.It Cm N -Find the previous line that matches the current search term. -.El -.Tg dir -.It Cm tree Oo Fl C | -no-colour Oc Oo Fl c | -commit Ar commit Oc \ -Oo Fl h | -help Oc Op Ar path -.Dl (aliases: Cm dir , Cm tr ) -Display navigable, hierarchical tree of a repository. If a -.Ar path -is specified, display tree nodes of this path. The -.Ar path -may be absolute, relative to the current working directory, or relative to the -repository root. With no options passed, the tree will reflect the state of the -latest commit on trunk. This command must be executed from within or below the -top level directory of the repository; that is, -.Nm -assumes a local checkout is open in or above the current working directory. -.Pp -Tree nodes are lexicographically ordered and may be postfixed with an identifier -corresponding to the mode of the file object on disk as returned by -.Xr lstat 2 : -.Bl -column YXZ description -.It / Ta directory -.It * Ta executable -.It @ Ta symbolic link -.El -.Pp -Nodes representing symbolic links are also annotated with the path of the -source file. -.Pp -Options for -.Cm fnc tree -are as follows: -.Bl -tag -width Ds -.It Fl C , -no-colour -Disable coloured output, which is enabled by default on supported terminals. -If this option is not used, colour can be toggled with the -.Sy c -tree view key binding as documented below. -.It Fl c , -commit Ar commit -The displayed tree will reflect the state of the repository as at the check-in -identified by -.Ar commit . -The expected argument is either the name of a branch, which will resolve -to the latest commit on the given branch, or (a unique abbreviated -prefix of) a valid commit UUID SHA1 or SHA3 hash. For a complete list of valid -arguments this option accepts, see -.Lk https://fossil-scm.org/home/doc/trunk/www/checkin_names.wiki \ -"Fossil's Check-in Names". -.It Fl h , -help -Display tree command help and usage information then exit. -.El -.Pp -Key bindings for -.Cm fnc tree -are as follows: -.Bl -tag -width Ds -.It Cm Enter, Arrow-right, l -Enter the currently selected directory, or open a -.Cm blame -view of the currently selected file. -.It Cm Backspace, Arrow-left, h -Move up a level to the parent directory. This is a no-op when in the root tree. -.It Cm Arrow-down, j -Move selection cursor one node down the tree. -.It Cm Arrow-up, k -Move selection cursor one node up the tree. -.It Cm Page-down, Ctrl+f -Move selection cursor one page down the tree. -.It Cm Page-up, Ctrl+b -Move selection cursor one page up the tree. -.It Cm Home, gg -Move selection cursor to the first node in the tree. -.It Cm End, G -Move selection cursor to the last node in the tree. -.It Cm c -Toggle coloured output. On supported terminals, -.Nm -will default to displaying the tree in colour. -.It Cm t -Open -.Cm timeline -view for the currently selected tree node. This will display the timeline of -all commits that involve the versioned file(s) corresponding to the selected -node. -.It Cm i -Show SHA hash UUID for all file nodes displayed in the tree. -.It Cm / -Prompt to enter a search term to begin searching the tree for nodes matching the -entered pattern. The search term is an extended regular expression, as -documented in -.Xr re_format 7 , -and is matched against the path of each tree node. -.It Cm n -Find the next tree node that matches the current search pattern. -.It Cm N -Find the previous tree node that matches the current search pattern. -.El -.Tg praise -.It Cm blame Oo Fl C | -no-colour Oc \ -Oo Fl c | -commit Ar commit Oo Fl r | -reverse Oc Oc Oo Fl h | -help Oc \ -Oo Fl n | -limit Ar n Oc Ar path -.Dl (aliases: Cm praise , Cm annotate , Cm bl , Cm pr , Cm an ) -Show commit attribution history for each line of the file at the specified -.Ar path , -which may be absolute, relative to the current working directory, or relative to -the repository root. This command must be executed from within or below the top -level directory of the repository; that is, -.Nm -assumes a local checkout is open in or above the current working directory. -.Pp -Options for -.Cm fnc blame -are as follows: -.Bl -tag -width Ds -.It Fl C , -no-colour -Disable coloured output, which is enabled by default on supported terminals. -If this option is not used, colour can be toggled with the -.Sy c -blame view key binding as documented below. -.It Fl c , -commit Ar commit -Start blame of file at the specified -.Ar path -from the check-in identified by -.Ar commit . -The expected argument is either the name of a branch, which will resolve -to the latest commit on the given branch, or (a unique abbreviated -prefix of) a valid commit UUID SHA1 or SHA3 hash. When this option is -not supplied, -.Nm -will blame the version of the file from the current checkout. For a complete -list of valid arguments this option accepts, see -.Lk https://fossil-scm.org/home/doc/trunk/www/checkin_names.wiki \ -"Fossil's Check-in Names". -.It Fl h , -help -Display blame command help and usage information then exit. -.It Fl n , -limit Ar n -Limit depth of blame history to -.Ar n -commits or seconds. The latter is denoted by a postfixed 's' (e.g., 30s). -With this option, -.Nm -will traverse either as many commits as specified, or as possible in the -specified time limit. By default, -.Nm -will traverse the entire historical record of the file, which can be expensive -for large files that span many commits. Use this option for a faster, more -targeted annotation. -.It Fl r , -reverse -Reverse annotate the file starting from a historical commit and move forward in -time. That is, rather than show the most recent change to each line, show the -first time each line was modified by a subsequent commit after the specified -.Ar commit . -(Requires \fB\-c\fP|\fB\-\-commit\fP.) -.El -.Pp -Key bindings for -.Cm fnc blame -are as follows: -.Bl -tag -width Ds -.It Cm Arrow-down, j -Move selection cursor down one line. -.It Cm Arrow-up, k -Move selection cursor up one line. -.It Cm Page-down, Ctrl+f -Move selection cursor down one page. -.It Cm Page-up, Ctrl+b -Move selection cursor up one page. -.It Cm Home, gg -Move selection cursor to the first line in the file. -.It Cm End, G -Move selection cursor to the last line in the file. -.It Cm Enter -Display the -.Cm diff -of the commit corresponding to the currently selected line. -.It Cm b -Blame the version of the file corresponding to the commit in the currently -selected line. -.It Cm p -Blame the version of the file corresponding to the parent of the commit in -the currently selected line. -.It Cm B, Backspace -Reload the previous blamed version of the file. -.It Cm c -Toggle coloured output. On supported terminals, -.Nm -will default to displaying the blamed file in colour. -.It Cm / -Prompt to enter a search term to begin searching the file for tokens matching -the entered pattern. The search term is an extended regular expression, as -documented in -.Xr re_format 7 . -.It Cm n -Find the next token that matches the current search pattern. -.It Cm N -Find the previous token that matches the current search pattern. -.El -.Tg tag -.It Cm branch Oo Fl C | -no-colour Oc Oo Fl -after Ar date | \ -Fl -before Ar date Oc Oo Fl h | -help Oc Oo Fl -open | Fl -closed Oc \ -Oo Fl p | -no-private Oc Oo Fl r | -reverse Oc \ -Oo Fl s | -sort Ar order Oc Op Ar glob -.Dl (aliases: Cm tag , Cm br ) -Display navigable list of repository branches. If -.Ar glob , -is specified, only display branches matching the pattern provided. Pattern -matching comparisons depend on the -.Xr sqlite3 1 -.B -LIKE -operator, which only folds case for the ASCII character set. This -command must be executed from within or below the top level directory of the -repository; that is, -.Nm -assumes a local checkout is open in or above the current working directory. -.Pp -Branches are lexicographically ordered by default, and are prefixed with an -identifier corresponding to the branch state (i.e., open/closed). The -current and private branches are additionally annotated with a postfixed -identifier: -.Bl -column ABCDEFGHIJ description -.It +dev-foo Ta open -.It -rm-bar Ta closed -.It +trunk@ Ta current -.It +wip-baz* Ta private -.El -.Pp -All branches, irrespective of state or privacy, are displayed by default, but -can be filtered based on several characteristics. -.Pp -Options for -.Cm fnc branch -are as follows: -.Bl -tag -width Ds -.It Fl C , -no-colour -Disable coloured output, which is enabled by default on supported terminals. -If this option is not used, colour can be toggled with the -.Sy c -branch view key binding as documented below. -.It Fl a , -after Ar date -Display only those branches with activity after the specified -.Ar date . -.It Fl b , -before Ar date -Display only those branches with activity before the specified -.Ar date . -.It Fl c , -close -Display only closed branches. -.It Fl h , -help -Display branch command help and usage information then exit. -.It Fl o , -open -Display only opened branches. -.It Fl p , -no-private -Do not show private branches, which are included in the list of displayed -branches by default. -.It Fl r , -reverse -Reverse the order in which branches are displayed. -.It Fl s , -sort Ar order -Sort branches by -.Ar order . -Valid -.Ar order -values are as follows: -.Bl -column YXZ description -.Sy mru Ta most recently used -.Sy state Ta open/closed state -.El -.Pp -Branches are sorted in lexicographical order by default. -.El -.Pp -Key bindings for -.Cm fnc branch -are as follows: -.Bl -tag -width Ds -.It Cm Arrow-down, j -Move selection cursor down one branch. -.It Cm Arrow-up, k -Move selection cursor up one branch. -.It Cm Page-down, Ctrl+f -Move selection cursor down one page. -.It Cm Page-up, Ctrl+b -Move selection cursor up one page. -.It Cm Home, gg -Move selection cursor to the first branch in the list. -.It Cm End, G -Move selection cursor to the last branch in the list. -.It Cm Enter, Space -Display the -.Cm timeline -of the currently selected branch. -.It Cm c -Toggle coloured output. On supported terminals, -.Nm -will default to displaying the branch list in colour. -.It Cm d -Toggle display of the date on which the branch last received changes. -.It Cm i -Toggle display of the SHA{1,3} hash that identifies branch, which is the hash -of the commit on the tip of said branch. -.It Cm t -Open the -.Cm tree -view of the currently selected branch. -.It Cm R, Ctrl+l -Reload the view with all repository branches, irrespective of which options -were used in this -.Cm fnc branch -invocation. -.It Cm / -Prompt to enter a search term to begin searching the list for branches matching -the entered pattern. The search term is an extended regular expression, as -documented in -.Xr re_format 7 . -.It Cm n -Find the next branch that matches the current search pattern. -.It Cm N -Find the previous branch that matches the current search pattern. -.El -.El -.Sh EXIT STATUS -.Ex -std fnc -.Sh SEE ALSO -.Xr fossil 1 , -.Xr re_format 7 -.Xr sqlite3 1 -.Sh AUTHOR -.An Mark Jamsek Aq Mt mark@jamsek.com DELETED fnc/fnc.c Index: fnc/fnc.c ================================================================== --- fnc/fnc.c +++ fnc/fnc.c @@ -1,9193 +0,0 @@ -/* - * Copyright (c) 2021 Mark Jamsek - * Copyright (c) 2013-2021 Stephan Beal - * Copyright (c) 2020 Stefan Sperling - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -/* - * This _POSIX_C_SOURCE bit really belongs in a config.h, but in the - * name of expedience... - */ -#if defined __linux__ -# if !defined(_XOPEN_SOURCE) -# define _XOPEN_SOURCE 700 -/* - * _POSIX_C_SOURCE >= 199309L needed for sigaction(), sigemptyset() on Linux, - * but glibc docs claim that _XOPEN_SOURCE>=700 has the same effect, PLUS - * we need _XOPEN_SOURCE>=500 for ncurses wide-char APIs on linux. - */ -# endif -# if !defined(_DEFAULT_SOURCE) -# define _DEFAULT_SOURCE -/* Needed for strsep() on glibc >= 2.19. */ -# endif -#endif - - -#include -#include -#include - -#ifdef _WIN32 -#include -#define ssleep(x) Sleep(x) -#else -#define ssleep(x) usleep((x) * 1000) -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "libfossil.h" - -#define FNC_VERSION 0.4 - -/* Utility macros. */ -#define MIN(a, b) (((a) < (b)) ? (a) : (b)) -#define MAX(a, b) (((a) > (b)) ? (a) : (b)) -#if !defined(CTRL) -#define CTRL(key) ((key) & 037) /* CTRL+ input. */ -#endif -#define nitems(a) (sizeof((a)) / sizeof((a)[0])) -#define STRINGIFYOUT(s) #s -#define STRINGIFY(s) STRINGIFYOUT(s) -#define CONCATOUT(a, b) a ## b -#define CONCAT(a, b) CONCATOUT(a, b) -#define FILE_POSITION __FILE__ ":" STRINGIFY(__LINE__) -#define FLAG_SET(f, b) ((f) |= (b)) -#define FLAG_CHK(f, b) ((f) & (b)) -#define FLAG_TOG(f, b) ((f) ^= (b)) -#define FLAG_CLR(f, b) ((f) &= ~(b)) - -/* Application macros. */ -#define PRINT_VERSION STRINGIFY(FNC_VERSION) -#define DIFF_DEF_CTXT 5 /* Default diff context lines. */ -#define DIFF_MAX_CTXT 64 /* Max diff context lines. */ -#define SPIN_INTERVAL 200 /* Status line progress indicator. */ -#define SPINNER "\\|/-\0" -#define NULL_DEVICE "/dev/null" -#define NULL_DEVICELEN (sizeof(NULL_DEVICE) - 1) -#define KEY_ESCAPE 27 -#if DEBUG -#define RC(r, fmt, ...) fcli_err_set(r, "%s::%s " fmt, \ - __func__, FILE_POSITION, __VA_ARGS__) -#else -#define RC(r, fmt, ...) fcli_err_set(r, fmt, __VA_ARGS__) -#endif /* DEBUG */ - -/* Portability macros. */ -#ifndef __OpenBSD__ -#ifndef HAVE_STRTONUM -# define strtonum(s, min, max, o) strtol(s, (char **)o, 10) -# endif /* HAVE_STRTONUM */ -#endif - -#if !defined(__dead) -#define __dead -#endif - -#ifndef TAILQ_FOREACH_SAFE -/* Rewrite of OpenBSD 6.9 sys/queue.h for Linux builds. */ -#define TAILQ_FOREACH_SAFE(var, head, field, tmp) \ - for ((var) = ((head)->tqh_first); \ - (var) != (NULL) && ((tmp) = TAILQ_NEXT(var, field), 1); \ - (var) = (tmp)) -#endif - -/* - * STAILQ was added to OpenBSD 6.9; fallback to SIMPLEQ for prior versions. - * XXX This is an ugly hack; replace with a better solution. - */ -#ifdef __OpenBSD__ -# ifndef STAILQ_HEAD -# define STAILQ SIMPLEQ -# endif /* STAILQ_HEAD */ -#endif /* OpenBSD */ - -#if defined __linux__ -# ifndef strlcat -# define strlcat(_d, _s, _sz) fnc_strlcat(_d, _s, _sz) -# endif /* strlcat */ -# ifndef strlcpy -# define strlcpy(_d, _s, _sz) fnc_strlcpy(_d, _s, _sz) -# endif /* strlcpy */ -#endif /* __linux__ */ - -__dead static void usage(void); -static void usage_timeline(void); -static void usage_diff(void); -static void usage_tree(void); -static void usage_blame(void); -static void usage_branch(void); -static int fcli_flag_type_arg_cb(fcli_cliflag const *); -static int cmd_timeline(fcli_command const *); -static int cmd_diff(fcli_command const *); -static int cmd_tree(fcli_command const *); -static int cmd_blame(fcli_command const *); -static int cmd_branch(fcli_command const *); - -/* - * Singleton initialising global configuration and state for app startup. - */ -static struct fnc_setup { - /* Global options. */ - const char *cmdarg; /* Retain argv[1] for use/err report. */ - const char *sym; /* Open view from this symbolic name. */ - const char *path; /* Optional path for timeline & tree. */ - int err; /* Indicate fnc error state. */ - bool hflag; /* Flag if --help is requested. */ - bool vflag; /* Flag if --version is requested. */ - bool reverse; /* Reverse branch sort or blame. */ - - /* Timeline options. */ - struct artifact_types { - const char **values; - short nitems; - } *filter_types; /* Only load commits of . */ - union { - const char *zlimit; - int limit; - } nrecords; /* Number of commits to load. */ - const char *filter_tag; /* Only load commits with . */ - const char *filter_branch; /* Only load commits from . */ - const char *filter_user; /* Only load commits from . */ - const char *filter_type; /* Placeholder for repeatable types. */ - bool utc; /* Display UTC sans user local time. */ - - /* Diff options. */ - const char *context; /* Number of context lines. */ - bool ws; /* Ignore whitespace-only changes. */ - bool nocolour; /* Disable colour in diff output. */ - bool quiet; /* Disable verbose diff output. */ - bool invert; /* Toggle inverted diff output. */ - - /* Branch options. */ - const char *before; /* Last branch change before date. */ - const char *after; /* Last branch change after date. */ - const char *sort; /* Lexicographical, MRU, open/closed. */ - bool closed; /* Show only closed branches. */ - bool open; /* Show only open branches */ - bool noprivate; /* Don't show private branches. */ - - /* Command line flags and help. */ - fcli_help_info fnc_help; /* Global help. */ - fcli_cliflag cliflags_global[3]; /* Global options. */ - fcli_command cmd_args[6]; /* App commands. */ - fcli_cliflag cliflags_timeline[11]; /* Timeline options. */ - fcli_cliflag cliflags_diff[7]; /* Diff options. */ - fcli_cliflag cliflags_tree[4]; /* Tree options. */ - fcli_cliflag cliflags_blame[6]; /* Blame options. */ - fcli_cliflag cliflags_branch[10]; /* Branch options. */ -} fnc_init = { - NULL, /* cmdarg copy of argv[1] to aid usage/error report. */ - NULL, /* sym(bolic name) of commit to open defaults to tip. */ - NULL, /* path for tree to open or timeline to find commits. */ - 0, /* err fnc error state. */ - false, /* hflag if --help is requested. */ - false, /* vflag if --version is requested. */ - false, /* reverse branch sort/annotation defaults to off. */ - NULL, /* filter_types defaults to indiscriminate. */ - {0}, /* nrecords defaults to all commits. */ - NULL, /* filter_tag defaults to indiscriminate. */ - NULL, /* filter_branch defaults to indiscriminate. */ - NULL, /* filter_user defaults to indiscriminate. */ - NULL, /* filter_type temporary placeholder. */ - false, /* utc defaults to off (i.e., show user local time). */ - NULL, /* context defaults to five context lines. */ - false, /* ws defaults to acknowledge whitespace. */ - false, /* nocolour defaults to off (i.e., use diff colours). */ - false, /* quiet defaults to off (i.e., verbose diff is on). */ - false, /* invert diff defaults to off. */ - NULL, /* before defaults to any time. */ - NULL, /* after defaults to any time. */ - NULL, /* sort by MRU or open/closed (dflt: lexicographical) */ - false, /* closed only branches is off (defaults to all). */ - false, /* open only branches is off by (defaults to all). */ - false, /* noprivate is off (default to show private branch). */ - - { /* fnc_help global app help details. */ - "A read-only ncurses browser for Fossil repositories in the " - "terminal.", NULL, usage - }, - - { /* cliflags_global global app options. */ - FCLI_FLAG_BOOL("h", "help", &fnc_init.hflag, - "Display program help and usage then exit."), - FCLI_FLAG_BOOL("v", "version", &fnc_init.vflag, - "Display program version number and exit."), - fcli_cliflag_empty_m - }, - - { /* cmd_args available app commands. */ - {"timeline", "tl\0time\0ti\0log\0", - "Show chronologically descending commit history of the repository.", - cmd_timeline, usage_timeline, fnc_init.cliflags_timeline}, - {"diff", "di\0", - "Show changes to versioned files introduced with a given commit.", - cmd_diff, usage_diff, fnc_init.cliflags_diff}, - {"tree", "tr\0dir\0", - "Show repository tree corresponding to a given commit", - cmd_tree, usage_tree, fnc_init.cliflags_tree}, - {"blame", "bl\0praise\0pr\0annotate\0an\0", - "Show commit attribution history for each line of a file.", - cmd_blame, usage_blame, fnc_init.cliflags_blame}, - {"branch", "br\0tag\0", - "Show navigable list of repository branches.", - cmd_branch, usage_branch, fnc_init.cliflags_branch}, - {NULL, NULL, NULL, NULL, NULL} /* Sentinel. */ - }, - - { /* cliflags_timeline timeline command related options. */ - FCLI_FLAG_BOOL("C", "no-colour", &fnc_init.nocolour, - "Disable colourised timeline, which is enabled by default on\n " - "supported terminals. Colour can also be toggled with the 'c' " - "\n key binding in timeline view when this option is not used."), - FCLI_FLAG("T", "tag", "", &fnc_init.filter_tag, - "Only display commits with T cards containing ."), - FCLI_FLAG("b", "branch", "", &fnc_init.filter_branch, - "Only display commits that reside on the given ."), - FCLI_FLAG("c", "commit", "", &fnc_init.sym, - "Open the timeline from . Common symbols are:\n" - "\tSHA{1,3} hash\n" - "\tSHA{1,3} unique prefix\n" - "\tbranch\n" - "\ttag:TAG\n" - "\troot:BRANCH\n" - "\tISO8601 date\n" - "\tISO8601 timestamp\n" - "\t{tip,current,prev,next}\n " - "For a complete list of symbols see Fossil's Check-in Names:\n " - "https://fossil-scm.org/home/doc/trunk/www/checkin_names.wiki"), - FCLI_FLAG_BOOL("h", "help", NULL, - "Display timeline command help and usage."), - FCLI_FLAG("n", "limit", "", &fnc_init.nrecords.zlimit, - "Limit display to latest commits; defaults to entire history " - "of\n current checkout. Negative values are a no-op."), - FCLI_FLAG_X("t", "type", "", &fnc_init.filter_type, - fcli_flag_type_arg_cb, - "Only display commits. Valid types are:\n" - "\tci - check-in\n" - "\tw - wiki\n" - "\tt - ticket\n" - "\te - technote\n" - "\tf - forum post\n" - " n.b. This is a repeatable flag (e.g., -t ci -t w)."), - FCLI_FLAG("u", "username", "", &fnc_init.filter_user, - "Only display commits authored by ."), - FCLI_FLAG_BOOL("z", "utc", &fnc_init.utc, - "Use UTC (instead of local) time."), - fcli_cliflag_empty_m - }, /* End cliflags_timeline. */ - - { /* cliflags_diff diff command related options. */ - FCLI_FLAG_BOOL("C", "no-colour", &fnc_init.nocolour, - "Disable coloured diff output, which is enabled by default on\n " - "supported terminals. Colour can also be toggled with the 'c' " - "\n key binding in diff view when this option is not used."), - FCLI_FLAG_BOOL("h", "help", NULL, - "Display diff command help and usage."), - FCLI_FLAG_BOOL("i", "invert", &fnc_init.invert, - "Invert difference between artifacts. Inversion can also be " - "toggled\n with the 'i' key binding in diff view."), - FCLI_FLAG_BOOL("q", "quiet", &fnc_init.quiet, - "Disable verbose diff output; that is, do not output complete" - " content\n of newly added or deleted files. Verbosity can also" - " be toggled with\n the 'v' key binding in diff view."), - FCLI_FLAG_BOOL("w", "whitespace", &fnc_init.ws, - "Ignore whitespace-only changes when displaying diff. This option " - "can\n also be toggled with the 'w' key binding in diff view."), - FCLI_FLAG("x", "context", "", &fnc_init.context, - "Show context lines when displaying diff; is capped at 64." - "\n Negative values are a no-op."), - fcli_cliflag_empty_m - }, /* End cliflags_diff. */ - - { /* cliflags_tree tree command related options. */ - FCLI_FLAG_BOOL("C", "no-colour", &fnc_init.nocolour, - "Disable coloured output, which is enabled by default on supported" - "\n terminals. Colour can also be toggled with the 'c' key " - "binding when\n this option is not used."), - FCLI_FLAG("c", "commit", "", &fnc_init.sym, - "Display tree that reflects repository state as at .\n" - " Common symbols are:" - "\n\tSHA{1,3} hash\n" - "\tSHA{1,3} unique prefix\n" - "\tbranch\n" - "\ttag:TAG\n" - "\troot:BRANCH\n" - "\tISO8601 date\n" - "\tISO8601 timestamp\n" - "\t{tip,current,prev,next}\n " - "For a complete list of symbols see Fossil's Check-in Names:\n " - "https://fossil-scm.org/home/doc/trunk/www/checkin_names.wiki"), - FCLI_FLAG_BOOL("h", "help", NULL, - "Display tree command help and usage."), - fcli_cliflag_empty_m - }, /* End cliflags_tree. */ - - { /* cliflags_blame blame command related options. */ - FCLI_FLAG_BOOL("C", "no-colour", &fnc_init.nocolour, - "Disable coloured output, which is enabled by default on supported" - "\n terminals. Colour can also be toggled with the 'c' key " - "binding when\n this option is not used."), - FCLI_FLAG("c", "commit", "", &fnc_init.sym, - "Start blame of specified file from . Common symbols are:\n" - "\tSHA{1,3} hash\n" - "\tSHA{1,3} unique prefix\n" - "\tbranch\n" - "\ttag:TAG\n" - "\troot:BRANCH\n" - "\tISO8601 date\n" - "\tISO8601 timestamp\n" - "\t{tip,current,prev,next}\n " - "For a complete list of symbols see Fossil's Check-in Names:\n " - "https://fossil-scm.org/home/doc/trunk/www/checkin_names.wiki"), - FCLI_FLAG_BOOL("h", "help", NULL, - "Display blame command help and usage."), - FCLI_FLAG("n", "limit", "", &fnc_init.nrecords.zlimit, - "Limit depth of blame history to commits or seconds. Denote the" - "\n latter by postfixing 's' (e.g., 30s). Useful for large files" - " with\n extensive history. Persists for the duration of the " - "session."), - FCLI_FLAG_BOOL("r", "reverse", &fnc_init.reverse, - "Reverse annotate the file starting from a historical commit. " - "Rather\n than show the most recent change of each line, show " - "the first time\n each line was modified after the specified " - "commit. Requires -c|--commit."), - fcli_cliflag_empty_m - }, /* End cliflags_blame. */ - - { /* cliflags_branch branch command related options. */ - FCLI_FLAG_BOOL("C", "no-colour", &fnc_init.nocolour, - "Disable coloured output, which is enabled by default on supported" - "\n terminals. Colour can also be toggled with the 'c' key " - "binding when\n this option is not used."), - FCLI_FLAG("a", "after", "", &fnc_init.after, - "Show branches with last activity occuring after , which is\n" - " expected to be either an ISO8601 (e.g., 2020-10-10) or " - "unambiguous\n DD/MM/YYYY or MM/DD/YYYY formatted date."), - FCLI_FLAG("b", "before", "", &fnc_init.before, - "Show branches with last activity occuring before , which is" - "\n expected to be either an ISO8601 (e.g., 2020-10-10) or " - "unambiguous\n DD/MM/YYYY or MM/DD/YYYY formatted date."), - FCLI_FLAG_BOOL("c", "closed", &fnc_init.closed, - "Show closed branches only. Open and closed branches are listed by " - "\n default."), - FCLI_FLAG_BOOL("h", "help", NULL, - "Display branch command help and usage."), - FCLI_FLAG_BOOL("o", "open", &fnc_init.open, - "Show open branches only. Open and closed branches are listed by " - "\n default."), - FCLI_FLAG_BOOL("p", "no-private", &fnc_init.noprivate, - "Do not show private branches, which are otherwise included in the" - "\n list of displayed branches by default."), - FCLI_FLAG_BOOL("r", "reverse", &fnc_init.reverse, - "Reverse the order in which branches are displayed."), - FCLI_FLAG("s", "sort", "", &fnc_init.sort, - "Sort branches by . Available options are:\n" - "\tmru - most recently used\n" - "\tstate - open/closed state\n " - "Branches are sorted in lexicographical order by default."), - fcli_cliflag_empty_m - }, /* End cliflags_blame. */ -}; - -enum fsl_list_object { - FNC_ARTIFACT_OBJ, - FNC_COLOUR_OBJ -}; - -enum date_string { - ISO8601_DATE_ONLY = 10, - ISO8601_TIMESTAMP = 20 -}; - -enum fnc_view_id { - FNC_VIEW_TIMELINE, - FNC_VIEW_DIFF, - FNC_VIEW_TREE, - FNC_VIEW_BLAME, - FNC_VIEW_BRANCH -}; - -enum fnc_search_mvmnt { - SEARCH_DONE, - SEARCH_FORWARD, - SEARCH_REVERSE -}; - -enum fnc_search_state { - SEARCH_WAITING, - SEARCH_CONTINUE, - SEARCH_COMPLETE, - SEARCH_NO_MATCH, - SEARCH_FOR_END -}; - -enum fnc_colours { - FNC_DIFF_META = 1, - FNC_DIFF_MINUS, - FNC_DIFF_PLUS, - FNC_DIFF_CHNK, - FNC_TREE_LINK, - FNC_TREE_DIR, - FNC_TREE_EXEC, - FNC_COMMIT_ID, - FNC_USER_STR, - FNC_DATE_STR, - FNC_TAGS_STR -}; - -enum fnc_diff_type { - FNC_DIFF_CKOUT, - FNC_DIFF_COMMIT, - FNC_DIFF_BLOB, - FNC_DIFF_WIKI -}; - -struct fnc_colour { - regex_t regex; - uint8_t scheme; -}; - -struct fsl_list_state { - enum fsl_list_object obj; -}; - -struct fnc_commit_artifact { - fsl_buffer wiki; - fsl_buffer pwiki; - fsl_list changeset; - fsl_uuid_str uuid; - fsl_uuid_str puuid; - fsl_id_t rid; - fsl_id_t prid; - char *user; - char *timestamp; - char *comment; - char *branch; - char *type; - enum fnc_diff_type diff_type; -}; - -struct fsl_file_artifact { - fsl_card_F *fc; - enum fsl_ckout_change_e change; -}; - -TAILQ_HEAD(commit_tailhead, commit_entry); -struct commit_entry { - TAILQ_ENTRY(commit_entry) entries; - struct fnc_commit_artifact *commit; - int idx; -}; - -struct commit_queue { - struct commit_tailhead head; - int ncommits; -}; - -/* - * The following two structs are used to construct the tree of the entire - * repository; that is, from the root through to all subdirectories and files. - */ -struct fnc_repository_tree { - struct fnc_repo_tree_node *head; /* Head of repository tree */ - struct fnc_repo_tree_node *tail; /* Final node in the tree. */ - struct fnc_repo_tree_node *rootail; /* Final root level node. */ -}; - -struct fnc_repo_tree_node { - struct fnc_repo_tree_node *next; /* Next node in tree. */ - struct fnc_repo_tree_node *prev; /* Prev node in tree. */ - struct fnc_repo_tree_node *parent_dir; /* Dir containing node. */ - struct fnc_repo_tree_node *sibling; /* Next node in same dir */ - struct fnc_repo_tree_node *children; /* List of node children */ - struct fnc_repo_tree_node *lastchild; /* Last child in list. */ - char *basename; /* Final path component. */ - char *path; /* Full pathname of node */ - fsl_uuid_str uuid; /* File artifact hash. */ - mode_t mode; /* File mode. */ - double mtime; /* Mod time of file. */ - uint_fast16_t pathlen; /* Length of path. */ - uint_fast16_t nparents; /* Path components sans- */ - /* -basename. */ -}; - -/* - * The following two structs represent a given subtree within the repository; - * for example, the top level tree and all its elements, or the elements of - * the src/ directory (but not any members of src/ subdirectories). - */ -struct fnc_tree_object { - struct fnc_tree_entry *entries; /* Array of tree entries. */ - int nentries; /* Number of tree entries. */ -}; - -struct fnc_tree_entry { - char *basename; /* Final component of path. */ - char *path; /* Full pathname of tree entry. */ - fsl_uuid_str uuid; /* File artifact hash. */ - mode_t mode; /* File mode. */ - double mtime; /* Modification time of file. */ - int idx; /* Index of this tree entry. */ -}; - -/* - * Each fnc_tree_object that is _not_ the repository root will have a (list of) - * fnc_parent_tree(s) to be tracked. - */ -struct fnc_parent_tree { - TAILQ_ENTRY(fnc_parent_tree) entry; - struct fnc_tree_object *tree; - struct fnc_tree_entry *first_entry_onscreen; - struct fnc_tree_entry *selected_entry; - int selected_idx; -}; - -pthread_mutex_t fnc_mutex = PTHREAD_MUTEX_INITIALIZER; - -struct fnc_tl_thread_cx { - struct commit_queue *commits; - struct commit_entry **first_commit_onscreen; - struct commit_entry **selected_commit; - fsl_db *db; - fsl_stmt *q; - regex_t *regex; - char *path; /* Match commits involving path. */ - enum fnc_search_state *search_status; - enum fnc_search_mvmnt *searching; - int spin_idx; - int ncommits_needed; - /* - * XXX Is there a more elegant solution to retrieving return codes from - * thread functions while pinging between, but before we join, threads? - */ - int rc; - bool tree_open; - bool endjmp; - bool timeline_end; - sig_atomic_t *quit; - pthread_cond_t commit_consumer; - pthread_cond_t commit_producer; -}; - -struct fnc_tl_view_state { - struct fnc_tl_thread_cx thread_cx; - struct commit_queue commits; - struct commit_entry *first_commit_onscreen; - struct commit_entry *last_commit_onscreen; - struct commit_entry *selected_commit; - struct commit_entry *matched_commit; - struct commit_entry *search_commit; - fsl_list colours; - const char *curr_ckout_uuid; - char *path; /* Match commits involving path. */ - int selected_idx; - sig_atomic_t quit; - pthread_t thread_id; - bool colour; -}; - -struct fnc_pathlist_entry { - TAILQ_ENTRY(fnc_pathlist_entry) entry; - const char *path; - size_t pathlen; - void *data; /* XXX May want to save id, mode, etc. */ -}; -TAILQ_HEAD(fnc_pathlist_head, fnc_pathlist_entry); - -struct fnc_diff_view_state { - struct fnc_view *timeline_view; - struct fnc_commit_artifact *selected_commit; - struct fnc_pathlist_head *paths; - fsl_buffer buf; - fsl_list colours; - FILE *f; - fsl_uuid_str id1; - fsl_uuid_str id2; - int first_line_onscreen; - int last_line_onscreen; - int diff_flags; - int context; - int sbs; - int matched_line; - int current_line; - size_t ncols; - size_t nlines; - off_t *line_offsets; - bool eof; - bool colour; - bool showmeta; -}; - -TAILQ_HEAD(fnc_parent_trees, fnc_parent_tree); -struct fnc_tree_view_state { /* Parent trees of the- */ - struct fnc_parent_trees parents; /* -current subtree. */ - struct fnc_repository_tree *repo; /* The repository tree. */ - struct fnc_tree_object *root; /* Top level repo tree. */ - struct fnc_tree_object *tree; /* Currently displayed tree */ - struct fnc_tree_entry *first_entry_onscreen; - struct fnc_tree_entry *last_entry_onscreen; - struct fnc_tree_entry *selected_entry; - struct fnc_tree_entry *matched_entry; - fsl_list colours; - char *tree_label; /* Headline string. */ - fsl_uuid_str commit_id; - fsl_id_t rid; - int ndisplayed; - int selected_idx; - bool colour; - bool show_id; -}; - -struct fnc_blame_line { - fsl_uuid_str id; - bool annotated; -}; - -struct fnc_blame_cb_cx { - struct fnc_view *view; - struct fnc_blame_line *lines; - fsl_uuid_str commit_id; - fsl_uuid_str root_commit; - int nlines; - bool *quit; -}; - -typedef int (*fnc_cancel_cb)(void *); - -struct fnc_blame_thread_cx { - struct fnc_blame_cb_cx *cb_cx; - fsl_annotate_opt blame_opt; - fnc_cancel_cb cancel_cb; - const char *path; - void *cancel_cx; - bool *complete; -}; - -struct fnc_blame { - struct fnc_blame_thread_cx thread_cx; - struct fnc_blame_cb_cx cb_cx; - FILE *f; /* Non-annotated copy of file */ - struct fnc_blame_line *lines; - off_t *line_offsets; - off_t filesz; - fsl_id_t origin; /* Tip rid for reverse blame */ - int nlines; - int nlimit; /* Limit depth traversal. */ - pthread_t thread_id; -}; - -CONCAT(STAILQ, _HEAD)(fnc_commit_id_queue, fnc_commit_qid); -struct fnc_commit_qid { - CONCAT(STAILQ, _ENTRY)(fnc_commit_qid) entry; - fsl_uuid_str id; -}; - -struct fnc_blame_view_state { - struct fnc_blame blame; - struct fnc_commit_id_queue blamed_commits; - struct fnc_commit_qid *blamed_commit; - struct fnc_commit_artifact *selected_commit; - fsl_list colours; - fsl_uuid_str commit_id; - char *path; - int first_line_onscreen; - int last_line_onscreen; - int selected_line; - int matched_line; - int spin_idx; - bool done; - bool blame_complete; - bool eof; - bool colour; -}; - -struct fnc_branch { - char *name; - char *date; - fsl_uuid_str id; - bool private; - bool current; - bool open; -}; - -struct fnc_branchlist_entry { - TAILQ_ENTRY(fnc_branchlist_entry) entries; - struct fnc_branch *branch; - int idx; -}; -TAILQ_HEAD(fnc_branchlist_head, fnc_branchlist_entry); - -struct fnc_branch_view_state { - struct fnc_branchlist_head branches; - struct fnc_branchlist_entry *first_branch_onscreen; - struct fnc_branchlist_entry *last_branch_onscreen; - struct fnc_branchlist_entry *matched_branch; - struct fnc_branchlist_entry *selected_branch; - const char *branch_glob; - double dateline; - int branch_flags; -#define BRANCH_LS_CLOSED_ONLY 0x001 /* Show closed branches only. */ -#define BRANCH_LS_OPEN_ONLY 0x002 /* Show open branches only. */ -#define BRANCH_LS_OPEN_CLOSED 0x003 /* Show open & closed branches (dflt). */ -#define BRANCH_LS_BITMASK 0x003 -#define BRANCH_LS_NO_PRIVATE 0x004 /* Show public branches only. */ -#define BRANCH_SORT_MTIME 0x008 /* Sort by activity. (default: name) */ -#define BRANCH_SORT_STATUS 0x010 /* Sort by open/closed. */ -#define BRANCH_SORT_REVERSE 0x020 /* Reverse sort order. */ - int nbranches; - int ndisplayed; - int selected; - int when; - bool colour; - bool show_date; - bool show_id; -}; - -TAILQ_HEAD(view_tailhead, fnc_view); -struct fnc_view { - TAILQ_ENTRY(fnc_view) entries; - WINDOW *window; - PANEL *panel; - struct fnc_view *parent; - struct fnc_view *child; - union { - struct fnc_diff_view_state diff; - struct fnc_tl_view_state timeline; - struct fnc_tree_view_state tree; - struct fnc_blame_view_state blame; - struct fnc_branch_view_state branch; - } state; - enum fnc_view_id vid; - enum fnc_search_state search_status; - enum fnc_search_mvmnt searching; - int nlines; - int ncols; - int start_ln; - int start_col; - int lines; /* Duplicate curses LINES macro. */ - int cols; /* Duplicate curses COLS macro. */ - bool focus_child; - bool active; /* Only 1 parent or child at a time. */ - bool egress; - bool started_search; - regex_t regex; - regmatch_t regmatch; - - int (*show)(struct fnc_view *); - int (*input)(struct fnc_view **, struct fnc_view *, int); - int (*close)(struct fnc_view *); - int (*search_init)(struct fnc_view *); - int (*search_next)(struct fnc_view *); -}; - -static volatile sig_atomic_t rec_sigwinch; -static volatile sig_atomic_t rec_sigpipe; -static volatile sig_atomic_t rec_sigcont; - -static void fnc_show_version(void); -static int init_curses(void); -static struct fnc_view *view_open(int, int, int, int, enum fnc_view_id); -static int open_timeline_view(struct fnc_view *, fsl_id_t, - const char *); -static int view_loop(struct fnc_view *); -static int show_timeline_view(struct fnc_view *); -static void *tl_producer_thread(void *); -static int block_main_thread_signals(void); -static int build_commits(struct fnc_tl_thread_cx *); -static int commit_builder(struct fnc_commit_artifact **, fsl_id_t, - fsl_stmt *); -static int signal_tl_thread(struct fnc_view *, int); -static int draw_commits(struct fnc_view *); -static void parse_emailaddr_username(char **); -static int formatln(wchar_t **, int *, const char *, int, int); -static int multibyte_to_wchar(const char *, wchar_t **, size_t *); -static int write_commit_line(struct fnc_view *, - struct fnc_commit_artifact *, int); -static int view_input(struct fnc_view **, int *, - struct fnc_view *, struct view_tailhead *); -static int help(struct fnc_view *); -static int padpopup(struct fnc_view *, int, int, FILE *, - const char *); -static void centerprint(WINDOW *, int, int, int, const char *, - chtype); -static int tl_input_handler(struct fnc_view **, struct fnc_view *, - int); -static int timeline_scroll_down(struct fnc_view *, int); -static void timeline_scroll_up(struct fnc_tl_view_state *, int); -static void select_commit(struct fnc_tl_view_state *); -static int view_is_parent(struct fnc_view *); -static int make_splitscreen(struct fnc_view *); -static int make_fullscreen(struct fnc_view *); -static int view_split_start_col(int); -static int view_search_start(struct fnc_view *); -static int tl_search_init(struct fnc_view *); -static int tl_search_next(struct fnc_view *); -static bool find_commit_match(struct fnc_commit_artifact *, - regex_t *); -static int init_diff_commit(struct fnc_view **, int, - struct fnc_commit_artifact *, struct fnc_view *); -static int open_diff_view(struct fnc_view *, - struct fnc_commit_artifact *, int, bool, bool, bool, - struct fnc_view *, bool, - struct fnc_pathlist_head *); -static void show_diff_status(struct fnc_view *); -static int create_diff(struct fnc_diff_view_state *); -static int create_changeset(struct fnc_commit_artifact *); -static int write_commit_meta(struct fnc_diff_view_state *); -static int wrapline(char *, fsl_size_t ncols_avail, - struct fnc_diff_view_state *, off_t *); -static int add_line_offset(off_t **, size_t *, off_t); -static int diff_commit(fsl_buffer *, struct fnc_commit_artifact *, - int, int, int, struct fnc_pathlist_head *); -static int diff_checkout(fsl_buffer *, fsl_id_t, int, int, int, - struct fnc_pathlist_head *); -static int write_diff_meta(fsl_buffer *, const char *, - fsl_uuid_str, const char *, fsl_uuid_str, int, - enum fsl_ckout_change_e); -static int diff_file(fsl_buffer *, fsl_buffer *, const char *, - fsl_uuid_str, const char *, enum fsl_ckout_change_e, - int, int, bool); -static int diff_non_checkin(fsl_buffer *, - struct fnc_commit_artifact *, int, int, int); -static int diff_file_artifact(fsl_buffer *, fsl_id_t, - const fsl_card_F *, fsl_id_t, const fsl_card_F *, - fsl_ckout_change_e, int, int, int, - enum fnc_diff_type); -static int show_diff(struct fnc_view *); -static int write_diff(struct fnc_view *, char *); -static int match_line(const void *, const void *); -static int write_matched_line(int *, const char *, int, int, - WINDOW *, regmatch_t *); -static void draw_vborder(struct fnc_view *); -static int diff_input_handler(struct fnc_view **, - struct fnc_view *, int); -static int set_selected_commit(struct fnc_diff_view_state *, - struct commit_entry *); -static int diff_search_init(struct fnc_view *); -static int diff_search_next(struct fnc_view *); -static int view_close(struct fnc_view *); -static int map_repo_path(char **); -static bool path_is_child(const char *, const char *, size_t); -static int path_skip_common_ancestor(char **, const char *, - size_t, const char *, size_t); -static bool fnc_path_is_root_dir(const char *); -/* static bool fnc_path_is_cwd(const char *); */ -static int fnc_pathlist_insert(struct fnc_pathlist_entry **, - struct fnc_pathlist_head *, const char *, void *); -static int fnc_path_cmp(const char *, const char *, size_t, - size_t); -static void fnc_pathlist_free(struct fnc_pathlist_head *); -static int browse_commit_tree(struct fnc_view **, int, - struct commit_entry *, const char *); -static int open_tree_view(struct fnc_view *, const char *, - fsl_id_t); -static int walk_tree_path(struct fnc_tree_view_state *, - struct fnc_repository_tree *, - struct fnc_tree_object **, const char *); -static int create_repository_tree(struct fnc_repository_tree **, - fsl_uuid_str *, fsl_id_t); -static int tree_builder(struct fnc_repository_tree *, - struct fnc_tree_object **, const char *); -/* static void delete_tree_node(struct fnc_tree_entry **, */ -/* struct fnc_tree_entry *); */ -static int link_tree_node(struct fnc_repository_tree *, - const char *, const char *, fsl_time_t); -static int show_tree_view(struct fnc_view *); -static int tree_input_handler(struct fnc_view **, - struct fnc_view *, int); -static int blame_tree_entry(struct fnc_view **, int, - struct fnc_tree_entry *, struct fnc_parent_trees *, - fsl_uuid_str); -static int tree_search_init(struct fnc_view *); -static int tree_search_next(struct fnc_view *); -static int tree_entry_path(char **, struct fnc_parent_trees *, - struct fnc_tree_entry *); -static int draw_tree(struct fnc_view *, const char *); -static int timeline_tree_entry(struct fnc_view **, int, - struct fnc_tree_view_state *); -static void tree_scroll_up(struct fnc_tree_view_state *, int); -static void tree_scroll_down(struct fnc_tree_view_state *, int); -static int visit_subtree(struct fnc_tree_view_state *, - struct fnc_tree_object *); -static int tree_entry_get_symlink_target(char **, - struct fnc_tree_entry *); -static int match_tree_entry(struct fnc_tree_entry *, regex_t *); -static void fnc_object_tree_close(struct fnc_tree_object *); -static void fnc_close_repository_tree(struct fnc_repository_tree *); -static int open_blame_view(struct fnc_view *, char *, - fsl_uuid_str, fsl_id_t, int); -static int run_blame(struct fnc_view *); -static int fnc_dump_buffer_to_file(off_t *, int *, off_t **, - FILE *, fsl_buffer *); -static int show_blame_view(struct fnc_view *); -static void *blame_thread(void *); -static int blame_cb(void *, fsl_annotate_opt const * const, - fsl_annotate_step const * const); -static int draw_blame(struct fnc_view *); -static int blame_input_handler(struct fnc_view **, - struct fnc_view *, int); -static int blame_search_init(struct fnc_view *); -static int blame_search_next(struct fnc_view *); -static fsl_uuid_cstr get_selected_commit_id(struct fnc_blame_line *, - int, int, int); -static int fnc_commit_qid_alloc(struct fnc_commit_qid **, - fsl_uuid_cstr); -static int close_blame_view(struct fnc_view *); -static int stop_blame(struct fnc_blame *); -static int cancel_blame(void *); -static void fnc_commit_qid_free(struct fnc_commit_qid *); -static int fnc_load_branches(struct fnc_branch_view_state *); -static int create_tmp_branchlist_table(void); -static int alloc_branch(struct fnc_branch **, const char *, - double, bool, bool, bool); -static int fnc_branchlist_insert(struct fnc_branchlist_entry **, - struct fnc_branchlist_head *, struct fnc_branch *); -static int open_branch_view(struct fnc_view *, int, const char *, - double, int); -static int show_branch_view(struct fnc_view *); -static int branch_input_handler(struct fnc_view **, - struct fnc_view *, int); -static int tl_branch_entry(struct fnc_view **, int, - struct fnc_branchlist_entry *); -static int browse_branch_tree(struct fnc_view **, int, - struct fnc_branchlist_entry *); -static void branch_scroll_up(struct fnc_branch_view_state *, int); -static void branch_scroll_down(struct fnc_branch_view_state *, - int); -static int branch_search_next(struct fnc_view *); -static int branch_search_init(struct fnc_view *); -static int match_branchlist_entry(struct fnc_branchlist_entry *, - regex_t *); -static int close_branch_view(struct fnc_view *); -static void fnc_free_branches(struct fnc_branch_view_state *); -static void fnc_branch_close(struct fnc_branch *); -static void view_set_child(struct fnc_view *, struct fnc_view *); -static int view_close_child(struct fnc_view *); -static int close_tree_view(struct fnc_view *); -static int close_timeline_view(struct fnc_view *); -static int close_diff_view(struct fnc_view *); -static int view_resize(struct fnc_view *); -static int screen_is_split(struct fnc_view *); -static bool screen_is_shared(struct fnc_view *); -static void fnc_resizeterm(void); -static int join_tl_thread(struct fnc_tl_view_state *); -static void fnc_free_commits(struct commit_queue *); -static void fnc_commit_artifact_close(struct fnc_commit_artifact*); -static int fsl_list_object_free(void *, void *); -static void sigwinch_handler(int); -static void sigpipe_handler(int); -static void sigcont_handler(int); -static int strtonumcheck(int *, const char *, const int, - const int); -static int fnc_date_to_mtime(double *, const char *, int); -static char *fnc_strsep (char **, const char *); -#ifdef __linux__ -static size_t fnc_strlcat(char *, const char *, size_t); -static size_t fnc_strlcpy(char *, const char *, size_t); -#endif -static int set_colours(fsl_list *, enum fnc_view_id vid); -static int match_colour(const void *, const void *); -static bool fnc_home(struct fnc_view *); -static struct fnc_colour *get_colour(fsl_list *, int); -static struct fnc_tree_entry *get_tree_entry(struct fnc_tree_object *, - int); -static struct fnc_tree_entry *find_tree_entry(struct fnc_tree_object *, - const char *, size_t); - -int -main(int argc, const char **argv) -{ - fcli_command *cmd = NULL; - char *path = NULL; - int rc = 0; - - fnc_init.filter_types = - (struct artifact_types *)fsl_malloc(sizeof(struct artifact_types)); - fnc_init.filter_types->values = fsl_malloc(sizeof(char *)); - fnc_init.filter_types->nitems = 0; - - if (!setlocale(LC_CTYPE, "")) - fsl_fprintf(stderr, "[!] Warning: Can't set locale.\n"); - - fnc_init.cmdarg = argv[1]; /* Which cmd to show usage if needed. */ - fcli.clientFlags.verbose = 2; /* Verbose error reporting. */ - fcli.cliFlags = fnc_init.cliflags_global; - fcli.appHelp = &fnc_init.fnc_help; - rc = fcli_setup(argc, argv); - if (rc) - goto end; - - if (fnc_init.vflag) { - fnc_show_version(); - goto end; - } else if (fnc_init.hflag) - usage(); - /* NOT REACHED */ - - rc = fcli_fingerprint_check(true); - if (rc) - goto end; - - if (argc == 1) - cmd = &fnc_init.cmd_args[FNC_VIEW_TIMELINE]; - else { - rc = fcli_dispatch_commands(fnc_init.cmd_args, false); - if (rc == FSL_RC_NOT_FOUND && argc == 2) { - /* - * Check if user entered fnc path/in/repo; if valid path - * is found, assume fnc timeline path/in/repo was meant. - */ - rc = map_repo_path(&path); - if (rc == FSL_RC_NOT_FOUND || !path) { - rc = RC(rc, - "'%s' is not a valid command or path", - argv[1]); - fnc_init.err = rc; - usage(); - /* NOT REACHED */ - } else if (rc) - goto end; - cmd = &fnc_init.cmd_args[FNC_VIEW_TIMELINE]; - fnc_init.path = path; - fcli_err_reset(); /* cmd_timeline::fcli_process_flags */ - } else if (rc) - goto end; - } - - if ((rc = fcli_has_unused_args(false))) { - fnc_init.err = rc; - usage(); - /* NOT REACHED */ - } - - if (!fsl_cx_db_repo(fcli_cx())) { - rc = RC(FSL_RC_MISUSE, "%s", "repository database required"); - goto end; - } - - if (cmd != NULL) - rc = cmd->f(cmd); -end: - fsl_free(path); - endwin(); - if (rc) { - if (rc == FCLI_RC_HELP) - rc = 0; - else if (rc == FSL_RC_BREAK) { - fsl_cx *f = fcli_cx(); - const char *errstr; - fsl_error_get(&f->error, &errstr, NULL); - fsl_fprintf(stdout, "%s", errstr); - fcli_err_reset(); /* For fcli_end_of_main() */ - rc = 0; - } - else - fsl_fprintf(stderr, "%s: %s %d\n", fcli_progname(), - fsl_rc_cstr(rc), rc); - } - putchar('\n'); - return fcli_end_of_main(rc); -} - -static int -cmd_timeline(fcli_command const *argv) -{ - struct fnc_view *v; - fsl_cx *f = fcli_cx(); - fsl_id_t rid = -1; - char *path = NULL; - int rc = 0; - - rc = fcli_process_flags(argv->flags); - if (rc || (rc = fcli_has_unused_flags(false))) - return rc; - - if (fnc_init.nrecords.zlimit) - if ((rc = strtonumcheck(&fnc_init.nrecords.limit, - fnc_init.nrecords.zlimit, INT_MIN, INT_MAX))) - return rc; - - if (fnc_init.sym != NULL) { - rc = fsl_sym_to_rid(f, fnc_init.sym, FSL_SATYPE_CHECKIN, &rid); - if (rc || rid < 0) - return RC(FSL_RC_TYPE, - "artifact [%s] not resolvable to a commit", - fnc_init.sym); - } - - if (fnc_init.path) - path = fsl_strdup(fnc_init.path); - else - rc = map_repo_path(&path); - if (!rc) - rc = init_curses(); - if (rc) - goto end; - v = view_open(0, 0, 0, 0, FNC_VIEW_TIMELINE); - if (v == NULL) { - RC(FSL_RC_ERROR, "%s", "view_open"); - goto end; - } - rc = open_timeline_view(v, rid, path); - if (!rc) - rc = view_loop(v); -end: - fsl_free(path); - return rc; -} - -/* - * Look for an in-repository path in **argv. If found, canonicalise it as an - * absolute path relative to the repository root (e.g., /ckoutdir/found/path), - * and assign to a dynamically allocated string in *requested_path, which the - * caller must dispose of with fsl_free or free(3). - */ -static int -map_repo_path(char **requested_path) -{ - fsl_cx *const f = fcli_cx(); - fsl_buffer buf = fsl_buffer_empty; - char *canonpath = NULL, *ckoutdir = NULL, *path = NULL; - const char *ckoutdir0 = NULL; - fsl_size_t len; - int rc = 0; - bool root; - - *requested_path = NULL; - - /* If no path argument is supplied, default to repository root. */ - if (!fcli_next_arg(false)) { - *requested_path = fsl_strdup("/"); - if (*requested_path == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - return rc; - } - - canonpath = fsl_strdup(fcli_next_arg(true)); - if (canonpath == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - - /* - * If no checkout (e.g., 'fnc timeline -R') copy the path verbatim to - * check its validity against a deck of F cards in open_timeline_view(). - */ - ckoutdir0 = fsl_cx_ckout_dir_name(f, &len); - if (!ckoutdir0) { - path = fsl_strdup(canonpath); - goto end; - } - - path = realpath(canonpath, NULL); - if (path == NULL && (errno == ENOENT || errno == ENOTDIR)) { - /* Path is not on disk, assume it is relative to repo root. */ - rc = fsl_file_canonical_name2(ckoutdir0, canonpath, &buf, NULL); - if (rc) { - rc = RC(rc, "%s", "fsl_file_canonical_name2"); - goto end; - } - fsl_free(path); - path = realpath(fsl_buffer_cstr(&buf), NULL); - if (path) { - /* Confirmed path is relative to repository root. */ - fsl_free(path); - path = fsl_strdup(canonpath); - if (path == NULL) - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - } else { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_NOT_FOUND), - "'%s' not found in tree", canonpath); - *requested_path = fsl_strdup(canonpath); - } - goto end; - } - fsl_free(path); - /* - * Use the cwd as the virtual root to canonicalise the supplied path if - * it is either: (a) relative; or (b) the root of the current checkout. - * Otherwise, use the root of the current checkout. - */ - rc = fsl_cx_getcwd(f, &buf); - if (rc) - goto end; - ckoutdir = fsl_mprintf("%.*s", len - 1, ckoutdir0); - root = fsl_strcmp(ckoutdir, fsl_buffer_cstr(&buf)) == 0; - fsl_buffer_reuse(&buf); - rc = fsl_ckout_filename_check(f, (canonpath[0] == '.' || !root) ? - true : false, canonpath, &buf); - if (rc) - goto end; - fsl_free(canonpath); - canonpath = fsl_strdup(fsl_buffer_str(&buf)); - - if (canonpath[0] == '\0') { - path = fsl_strdup(canonpath); - if (path == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - } else { - fsl_buffer_reuse(&buf); - rc = fsl_file_canonical_name2(f->ckout.dir, canonpath, &buf, - false); - if (rc) - goto end; - path = fsl_strdup(fsl_buffer_str(&buf)); - if (path == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - if (access(path, F_OK) != 0) { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_ACCESS), - "path does not exist or inaccessible [%s]", path); - goto end; - } - /* - * Now we have an absolute path, check again if it's the ckout - * dir; if so, clear it to signal an open_timeline_view() check. - */ - len = fsl_strlen(path); - if (!fsl_strcmp(path, f->ckout.dir)) { - fsl_free(path); - path = fsl_strdup(""); - if (path == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - } else if (len > f->ckout.dirLen && path_is_child(path, - f->ckout.dir, f->ckout.dirLen)) { - char *child; - /* - * Matched on-disk path within the repository; strip - * common prefix with repository root path. - */ - rc = path_skip_common_ancestor(&child, f->ckout.dir, - f->ckout.dirLen, path, len); - if (rc) - goto end; - fsl_free(path); - path = child; - } else { - /* - * Matched on-disk path outside the repository; treat - * as relative to repo root. (Though this should fail.) - */ - fsl_free(path); - path = canonpath; - canonpath = NULL; - } - } - - /* Trim trailing slash if it exists. */ - if (path[fsl_strlen(path) - 1] == '/') - path[fsl_strlen(path) - 1] = '\0'; - -end: - fsl_buffer_clear(&buf); - fsl_free(canonpath); - fsl_free(ckoutdir); - if (rc) - fsl_free(path); - else { - /* Make path absolute from repository root. */ - if (path[0] != '/' && (path[0] != '.' && path[1] != '/')) { - char *abspath; - if ((abspath = fsl_mprintf("/%s", path)) == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_mprintf"); - goto end; - } - fsl_free(path); - path = abspath; - } - - *requested_path = path; - } - return rc; -} - -static bool -path_is_child(const char *child, const char *parent, size_t parentlen) -{ - if (parentlen == 0 || fnc_path_is_root_dir(parent)) - return true; - - if (fsl_strncmp(parent, child, parentlen) != 0) - return false; - if (child[parentlen - 1 /* Trailing slash */] != '/') - return false; - - return true; -} - -/* - * As a special case, due to fsl_ckout_filename_check() resolving the current - * checkout directory to ".", this function returns true for ".". For this - * reason, when path is intended to be the current working directory for any - * directory other than the repository root, callers must ensure path is either - * absolute or relative to the respository root--not ".". - */ -static bool -fnc_path_is_root_dir(const char *path) -{ - while (*path == '/' || *path == '.') - ++path; - return (*path == '\0'); -} - -static int -path_skip_common_ancestor(char **child, const char *parent_abspath, - size_t parentlen, const char *abspath, size_t len) -{ - size_t bufsz; - int rc = 0; - - *child = NULL; - - if (parentlen >= len) - return RC(FSL_RC_RANGE, "invalid path [%s]", abspath); - if (fsl_strncmp(parent_abspath, abspath, parentlen) != 0) - return RC(FSL_RC_TYPE, "invalid path [%s]", abspath); - if (!fnc_path_is_root_dir(parent_abspath) && - abspath[parentlen - 1 /* Trailing slash */] != '/') - return RC(FSL_RC_TYPE, "invalid path [%s]", abspath); - while (abspath[parentlen] == '/') - ++abspath; - bufsz = len - parentlen + 1; - *child = fsl_malloc(bufsz); - if (*child == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_malloc"); - if (strlcpy(*child, abspath + parentlen, bufsz) >= bufsz) { - rc = RC(FSL_RC_RANGE, "%s", "strlcpy"); - fsl_free(*child); - *child = NULL; - } - return rc; -} - -#if 0 -static bool -fnc_path_is_cwd(const char *path) -{ - return (path[0] == '.' && path[1] == '\0'); -} -#endif - -static int -init_curses(void) -{ - initscr(); - cbreak(); - noecho(); - nonl(); - intrflush(stdscr, FALSE); - keypad(stdscr, TRUE); - curs_set(0); - set_escdelay(0); /* ESC should return immediately. */ -#ifndef __linux__ - typeahead(-1); /* Don't disrupt screen update operations. */ -#endif - - if (!fnc_init.nocolour && has_colors()) { - start_color(); - use_default_colors(); - } - - if (sigaction(SIGPIPE, &(struct sigaction){{sigpipe_handler}}, NULL) - == -1) - return RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), - "%s", "sigaction(SIGPIPE)"); - if (sigaction(SIGWINCH, &(struct sigaction){{sigwinch_handler}}, NULL) - == -1) - return RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), - "%s", "sigaction(SIGWINCH)"); - if (sigaction(SIGCONT, &(struct sigaction){{sigcont_handler}}, NULL) - == -1) - return RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), - "%s", "sigaction(SIGCONT)"); - - return 0; -} - -static struct fnc_view * -view_open(int nlines, int ncols, int start_ln, int start_col, - enum fnc_view_id vid) -{ - struct fnc_view *view = calloc(1, sizeof(*view)); - - if (view == NULL) - return NULL; - - view->vid = vid; - view->lines = LINES; - view->cols = COLS; - view->nlines = nlines ? nlines : LINES - start_ln; - view->ncols = ncols ? ncols : COLS - start_col; - view->start_ln = start_ln; - view->start_col = start_col; - view->window = newwin(nlines, ncols, start_ln, start_col); - if (view->window == NULL) { - view_close(view); - return NULL; - } - view->panel = new_panel(view->window); - if (view->panel == NULL || set_panel_userptr(view->panel, view) != OK) { - view_close(view); - return NULL; - } - - keypad(view->window, TRUE); - return view; -} - -static int -open_timeline_view(struct fnc_view *view, fsl_id_t rid, const char *path) -{ - struct fnc_tl_view_state *s = &view->state.timeline; - fsl_cx *f = fcli_cx(); - fsl_db *db = fsl_cx_db_repo(f); - fsl_buffer sql = fsl_buffer_empty; - char *startdate = NULL; - fsl_id_t idtag = 0; - int idx, rc = 0; - - if (path != s->path) { - fsl_free(s->path); - s->path = fsl_strdup(path); - if (s->path == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - } - - /* - * TODO: See about opening this API. - * If a path has been supplied, create a table of all path's - * ancestors and add "AND blob.rid IN fsl_computed_ancestors" to query. - */ - /* if (path[1]) { */ - /* rc = fsl_compute_ancestors(db, rid, 0, 0); */ - /* if (rc) */ - /* return RC(FSL_RC_DB, "%s", "fsl_compute_ancestors"); */ - /* } */ - s->thread_cx.q = NULL; - /* s->selected_idx = 0; */ /* Unnecessary? */ - - TAILQ_INIT(&s->commits.head); - s->commits.ncommits = 0; - - if (rid != -1) - startdate = fsl_mprintf("(SELECT mtime FROM event " - "WHERE objid=%d)", rid); - else - fsl_ckout_version_info(f, NULL, &s->curr_ckout_uuid); - - /* - * In 'fnc timeline -R repo.fossil path' case, check that path is a - * valid repository path in the repository tree as at either the - * latest check-in or the specified commit. - */ - if (s->curr_ckout_uuid == NULL && path[1]) { - fsl_deck d = fsl_deck_empty; - bool ispath = false; - rc = fsl_deck_load_sym(f, &d, fnc_init.sym ? fnc_init.sym : - "tip", FSL_SATYPE_CHECKIN); - fsl_deck_F_rewind(&d); - if (fsl_deck_F_search(&d, path + 1 /* Slash */) == NULL) { - const fsl_card_F *cf; - fsl_deck_F_next(&d, &cf); - do { - fsl_deck_F_next(&d, &cf); - if (cf && !fsl_strncmp(path + 1 /* Slash */, - cf->name, fsl_strlen(path) - 1)) { - ispath = true; - break; - } - } while (cf); - } else - ispath = true; - fsl_deck_finalize(&d); - if (!ispath) - return RC(FSL_RC_NOT_FOUND, "'%s' invalid path in [%s]", - path + 1, fnc_init.sym ? fnc_init.sym : "tip"); - } - - if ((rc = pthread_cond_init(&s->thread_cx.commit_consumer, NULL))) { - RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_cond_init"); - goto end; - } - if ((rc = pthread_cond_init(&s->thread_cx.commit_producer, NULL))) { - RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_cond_init"); - goto end; - } - - fsl_buffer_appendf(&sql, "SELECT " - /* 0 */"uuid, " - /* 1 */"datetime(event.mtime%s), " - /* 2 */"coalesce(euser, user), " - /* 3 */"rid AS rid, " - /* 4 */"event.type AS eventtype, " - /* 5 */"(SELECT group_concat(substr(tagname,5), ',') " - "FROM tag, tagxref WHERE tagname GLOB 'sym-*' " - "AND tag.tagid=tagxref.tagid AND tagxref.rid=blob.rid " - "AND tagxref.tagtype > 0) as tags, " - /*6*/"coalesce(ecomment, comment) AS comment FROM event JOIN blob " - "WHERE blob.rid=event.objid", fnc_init.utc ? "" : ", 'localtime'"); - - if (fnc_init.filter_types->nitems) { - fsl_buffer_appendf(&sql, " AND ("); - for (idx = 0; idx < fnc_init.filter_types->nitems; ++idx) { - fsl_buffer_appendf(&sql, " eventtype=%Q%s", - fnc_init.filter_types->values[idx], (idx + 1) < - fnc_init.filter_types->nitems ? " OR " : ")"); - /* This produces a double-free? */ - /* fsl_free((char *)fnc_init.filter_types->values[idx]); */ - fnc_init.filter_types->values[idx] = NULL; - } - fsl_free(fnc_init.filter_types->values); - fsl_free(fnc_init.filter_types); - } - - if (fnc_init.filter_branch) { - idtag = fsl_db_g_id(db, 0, - "SELECT tagid FROM tag WHERE tagname='sym-%q'", - fnc_init.filter_branch); - if (idtag > 0) - fsl_buffer_appendf(&sql, - " AND EXISTS(SELECT 1 FROM tagxref" - " WHERE tagid=%"FSL_ID_T_PFMT - " AND tagtype > 0 AND rid=blob.rid)", idtag); - else { - rc = RC(FSL_RC_NOT_FOUND, "Invalid branch name [%s]", - fnc_init.filter_branch); - goto end; - } - } - - if (fnc_init.filter_tag) { - idtag = fsl_db_g_id(db, 0, - "SELECT tagid FROM tag WHERE tagname GLOB 'sym-%q'", - fnc_init.filter_tag); - if (idtag == 0) - idtag = fsl_db_g_id(db, 0, - "SELECT tagid FROM tag WHERE tagname='%q'", - fnc_init.filter_tag); - if (idtag > 0) - fsl_buffer_appendf(&sql, - " AND EXISTS(SELECT 1 FROM tagxref" - " WHERE tagid=%"FSL_ID_T_PFMT - " AND tagtype > 0 AND rid=blob.rid)", idtag); - else { - rc = RC(FSL_RC_NOT_FOUND, "Invalid tag [%s]", - fnc_init.filter_tag); - goto end; - } - } - - if (fnc_init.filter_user) - if ((rc = fsl_buffer_appendf(&sql, - " AND coalesce(euser, user) GLOB lower('*%q*')", - fnc_init.filter_user))) - goto end; - - if (startdate) { - fsl_buffer_appendf(&sql, " AND event.mtime <= %s", startdate); - fsl_free(startdate); - } - - /* - * If path is not root ("/"), a versioned path in the repository has - * been requested, only retrieve commits involving path. - */ - if (path[1]) { - fsl_buffer_appendf(&sql, - " AND EXISTS(SELECT 1 FROM mlink" - " WHERE mlink.mid = event.objid" - " AND mlink.fnid IN "); - if (fsl_cx_is_case_sensitive(f)) { - fsl_buffer_appendf(&sql, - "(SELECT fnid FROM filename" - " WHERE name = %Q OR name GLOB '%q/*')", - path + 1, path + 1); /* Skip prepended slash. */ - } else { - fsl_buffer_appendf(&sql, - "(SELECT fnid FROM filename" - " WHERE name = %Q COLLATE nocase" - " OR lower(name) GLOB lower('%q/*'))", - path + 1, path + 1); /* Skip prepended slash. */ - } - fsl_buffer_append(&sql, ")", 1); - } - - fsl_buffer_appendf(&sql, " ORDER BY event.mtime DESC"); - - if (fnc_init.nrecords.limit > 0) - fsl_buffer_appendf(&sql, " LIMIT %d", fnc_init.nrecords.limit); - - view->show = show_timeline_view; - view->input = tl_input_handler; - view->close = close_timeline_view; - view->search_init = tl_search_init; - view->search_next = tl_search_next; - - s->thread_cx.q = fsl_stmt_malloc(); - rc = fsl_db_prepare(db, s->thread_cx.q, "%b", &sql); - if (rc) { - rc = RC(rc, "%s", "fsl_db_prepare"); - goto end; - } - rc = fsl_stmt_step(s->thread_cx.q); - if (rc) { - switch (rc) { - case FSL_RC_STEP_ROW: - rc = 0; - break; - case FSL_RC_STEP_ERROR: - rc = RC(rc, "%s", "fsl_stmt_step"); - goto end; - case FSL_RC_STEP_DONE: - rc = RC(FSL_RC_BREAK, "%s", "no matching records"); - goto end; - } - } - - s->colour = !fnc_init.nocolour && has_colors(); - s->thread_cx.rc = 0; - s->thread_cx.db = db; - s->thread_cx.spin_idx = 0; - s->thread_cx.ncommits_needed = view->nlines - 1; - s->thread_cx.commits = &s->commits; - s->thread_cx.timeline_end = false; - s->thread_cx.quit = &s->quit; - s->thread_cx.first_commit_onscreen = &s->first_commit_onscreen; - s->thread_cx.selected_commit = &s->selected_commit; - s->thread_cx.searching = &view->searching; - s->thread_cx.search_status = &view->search_status; - s->thread_cx.regex = &view->regex; - s->thread_cx.path = s->path; - - if (s->colour) - set_colours(&s->colours, FNC_VIEW_TIMELINE); -end: - fsl_buffer_clear(&sql); - if (rc) { - close_timeline_view(view); - if (db->error.code) - rc = fsl_cx_uplift_db_error(f, db); - } - return rc; -} - -static int -view_loop(struct fnc_view *view) -{ - struct view_tailhead views; - struct fnc_view *new_view; - int done = 0, err = 0, rc = 0; - - if ((rc = pthread_mutex_lock(&fnc_mutex))) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_lock"); - - TAILQ_INIT(&views); - TAILQ_INSERT_HEAD(&views, view, entries); - - view->active = true; - rc = view->show(view); - if (rc) - return rc; - - while (!TAILQ_EMPTY(&views) && !done && !rec_sigpipe) { - rc = view_input(&new_view, &done, view, &views); - if (rc) - break; - if (view->egress) { - struct fnc_view *v, *prev = NULL; - - if (view_is_parent(view)) - prev = TAILQ_PREV(view, view_tailhead, entries); - else if (view->parent) - prev = view->parent; - - if (view->parent) { - view->parent->child = NULL; - view->parent->focus_child = false; - } else - TAILQ_REMOVE(&views, view, entries); - - rc = view_close(view); - if (rc) - goto end; - - view = NULL; - TAILQ_FOREACH(v, &views, entries) { - if (v->active) - break; - } - if (view == NULL && new_view == NULL) { - /* No view is active; try to pick one. */ - if (prev) - view = prev; - else if (!TAILQ_EMPTY(&views)) - view = TAILQ_LAST(&views, - view_tailhead); - if (view) { - if (view->focus_child) { - view->child->active = true; - view = view->child; - } else - view->active = true; - } - } - } - if (new_view) { - struct fnc_view *v, *t; - /* Allow only one parent view per type. */ - TAILQ_FOREACH_SAFE(v, &views, entries, t) { - if (v->vid != new_view->vid) - continue; - TAILQ_REMOVE(&views, v, entries); - rc = view_close(v); - if (rc) - goto end; - break; - } - TAILQ_INSERT_TAIL(&views, new_view, entries); - view = new_view; - } - if (view) { - if (view_is_parent(view)) { - if (view->child && view->child->active) - view = view->child; - } else { - if (view->parent && view->parent->active) - view = view->parent; - } - show_panel(view->panel); - if (view->child && screen_is_split(view->child)) - show_panel(view->child->panel); - if (view->parent && screen_is_split(view)) { - rc = view->parent->show(view->parent); - if (rc) - goto end; - } - rc = view->show(view); - if (rc) - goto end; - if (view->child) { - rc = view->child->show(view->child); -#ifdef __linux__ - wnoutrefresh(view->child->window); -#endif - if (rc) - goto end; - } -#ifdef __linux__ - wnoutrefresh(view->window); -#else - update_panels(); -#endif - doupdate(); - } - } -end: - while (!TAILQ_EMPTY(&views)) { - view = TAILQ_FIRST(&views); - TAILQ_REMOVE(&views, view, entries); - view_close(view); - } - - if ((err = pthread_mutex_unlock(&fnc_mutex)) && !rc) - rc = RC(fsl_errno_to_rc(err, FSL_RC_ACCESS), - "%s", "pthread_mutex_unlock"); - - return rc; -} - -static int -show_timeline_view(struct fnc_view *view) -{ - struct fnc_tl_view_state *s = &view->state.timeline; - int rc = 0; - - if (!s->thread_id) { - rc = pthread_create(&s->thread_id, NULL, tl_producer_thread, - &s->thread_cx); - if (rc) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_create"); - if (s->thread_cx.ncommits_needed > 0) { - rc = signal_tl_thread(view, 1); - if (rc) - return rc; - } - } - - return draw_commits(view); -} - -static void * -tl_producer_thread(void *state) -{ - struct fnc_tl_thread_cx *cx = state; - int rc; - bool done = false; - - rc = block_main_thread_signals(); - if (rc) - return (void *)(intptr_t)rc; - - while (!done && !rc && !rec_sigpipe) { - switch (rc = build_commits(cx)) { - case FSL_RC_STEP_DONE: - done = true; - /* FALL THROUGH */ - case FSL_RC_STEP_ROW: - rc = 0; - /* FALL THROUGH */ - default: - if (rc) { - cx->rc = rc; - return (void *)(intptr_t)rc; - } - else if (cx->ncommits_needed > 0) - cx->ncommits_needed--; - break; - } - - if ((rc = pthread_mutex_lock(&fnc_mutex))) { - rc = RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_lock"); - break; - } else if (*cx->first_commit_onscreen == NULL) { - *cx->first_commit_onscreen = - TAILQ_FIRST(&cx->commits->head); - *cx->selected_commit = *cx->first_commit_onscreen; - } else if (*cx->quit) - done = true; - - if ((rc = pthread_cond_signal(&cx->commit_producer))) { - rc = RC(fsl_errno_to_rc(rc, FSL_RC_MISUSE), - "%s", "pthread_cond_signal"); - pthread_mutex_unlock(&fnc_mutex); - break; - } - - if (done) - cx->ncommits_needed = 0; - else if (cx->ncommits_needed == 0) { - if ((rc = pthread_cond_wait(&cx->commit_consumer, - &fnc_mutex))) - rc = RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_cond_wait"); - if (*cx->quit) - done = true; - } - - if ((rc = pthread_mutex_unlock(&fnc_mutex))) - rc = RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_unlock"); - } - - cx->timeline_end = true; - return (void *)(intptr_t)rc; -} - -static int -block_main_thread_signals(void) -{ - sigset_t set; - - if (sigemptyset(&set) == -1) - return RC(fsl_errno_to_rc(errno, FSL_RC_MISUSE), "%s", - "sigemptyset"); - - /* Bespoke signal handlers for SIGWINCH and SIGCONT. */ - if (sigaddset(&set, SIGWINCH) == -1) - return RC(fsl_errno_to_rc(errno, FSL_RC_MISUSE), "%s", - "sigaddset"); - if (sigaddset(&set, SIGCONT) == -1) - return RC(fsl_errno_to_rc(errno, FSL_RC_MISUSE), "%s", - "sigaddset"); - - /* ncurses handles SIGTSTP. */ - if (sigaddset(&set, SIGTSTP) == -1) - return RC(fsl_errno_to_rc(errno, FSL_RC_MISUSE), "%s", - "sigaddset"); - - if (pthread_sigmask(SIG_BLOCK, &set, NULL)) - return RC(fsl_errno_to_rc(errno, FSL_RC_MISUSE), "%s", - "pthread_sigmask"); - - return 0; -} - -static int -build_commits(struct fnc_tl_thread_cx *cx) -{ - int rc = 0; - - if (cx->tree_open) { - /* - * XXX If a tree has been opened with the 't' key binding, the - * commit builder statement needs to be reset otherwise one of - * the SQLite3 APIs down the fsl_stmt_step() call stack fails, - * irrespective of whether fsl_db_prepare_cached() is called. - */ - fsl_size_t loaded = cx->q->rowCount; - cx->tree_open = false; - rc = fsl_stmt_reset(cx->q); - if (rc) - return RC(rc, "%s", "fsl_stmt_reset"); - while (loaded--) - if ((rc = fsl_stmt_step(cx->q)) != FSL_RC_STEP_ROW) - return RC(rc, "%s", "fsl_stmt_step"); - } - /* - * Step through the given SQL query, passing each row to the commit - * builder to build commits for the timeline. - */ - do { - struct fnc_commit_artifact *commit = NULL; - struct commit_entry *dup_entry, *entry; - - rc = commit_builder(&commit, 0, cx->q); - if (rc) - return RC(rc, "%s", "commit_builder"); - /* - * TODO: Find out why, without this, fnc reads and displays - * the first (i.e., latest) commit twice. This hack checks to - * see if the current row returned a UUID matching the last - * commit added to the list to avoid adding a duplicate entry. - */ - dup_entry = TAILQ_FIRST(&cx->commits->head); - if (cx->commits->ncommits == 1 && - !fsl_strcmp(dup_entry->commit->uuid, commit->uuid)) { - fnc_commit_artifact_close(commit); - cx->ncommits_needed++; - continue; - } - - entry = fsl_malloc(sizeof(*entry)); - if (entry == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_malloc"); - - entry->commit = commit; - - rc = pthread_mutex_lock(&fnc_mutex); - if (rc) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_lock"); - - entry->idx = cx->commits->ncommits; - TAILQ_INSERT_TAIL(&cx->commits->head, entry, entries); - cx->commits->ncommits++; - - if (!cx->endjmp && *cx->searching == SEARCH_FORWARD && - *cx->search_status == SEARCH_WAITING) { - if (find_commit_match(commit, cx->regex)) - *cx->search_status = SEARCH_CONTINUE; - } - - rc = pthread_mutex_unlock(&fnc_mutex); - if (rc) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_unlock"); - - } while ((rc = fsl_stmt_step(cx->q)) == FSL_RC_STEP_ROW - && *cx->searching == SEARCH_FORWARD - && *cx->search_status == SEARCH_WAITING); - - return rc; -} - -/* - * Given prepared SQL statement q _XOR_ record ID rid, allocate and build the - * corresponding commit artifact from the result set. The commit must - * eventually be disposed of with fnc_commit_artifact_close(). - */ -static int -commit_builder(struct fnc_commit_artifact **ptr, fsl_id_t rid, fsl_stmt *q) -{ - fsl_cx *f = fcli_cx(); - fsl_db *db = fsl_needs_repo(f); - struct fnc_commit_artifact *commit = NULL; - fsl_buffer buf = fsl_buffer_empty; - const char *comment, *prefix, *type; - int rc = 0; - enum fnc_diff_type diff_type = FNC_DIFF_WIKI; - - if (rid) { - rc = fsl_db_prepare(db, q, "SELECT " - /* 0 */"uuid, " - /* 1 */"datetime(event.mtime%s), " - /* 2 */"coalesce(euser, user), " - /* 3 */"rid AS rid, " - /* 4 */"event.type AS eventtype, " - /* 5 */"(SELECT group_concat(substr(tagname,5), ',') " - "FROM tag, tagxref WHERE tagname GLOB 'sym-*' " - "AND tag.tagid=tagxref.tagid AND tagxref.rid=blob.rid " - "AND tagxref.tagtype > 0) as tags, " - /*6*/"coalesce(ecomment, comment) AS comment " - "FROM event JOIN blob WHERE blob.rid=%d AND event.objid=%d", - fnc_init.utc ? "" : ", 'localtime'", rid, rid); - if (rc) - return RC(FSL_RC_DB, "%s", "fsl_db_prepare"); - fsl_stmt_step(q); - } - - type = fsl_stmt_g_text(q, 4, NULL); - comment = fsl_stmt_g_text(q, 6, NULL); - prefix = NULL; - - switch (*type) { - case 'c': - type = "checkin"; - diff_type = FNC_DIFF_COMMIT; - break; - case 'w': - type = "wiki"; - if (comment) { - switch (*comment) { - case '+': - prefix = "Added: "; - ++comment; - break; - case '-': - prefix = "Deleted: "; - ++comment; - break; - case ':': - prefix = "Edited: "; - ++comment; - break; - default: - break; - } - if (prefix) - rc = fsl_buffer_append(&buf, prefix, -1); - } - break; - case 'g': - type = "tag"; - break; - case 'e': - type = "technote"; - break; - case 't': - type = "ticket"; - break; - case 'f': - type = "forum"; - break; - }; - if (!rc && comment) - rc = fsl_buffer_append(&buf, comment, -1); - if (rc) { - rc = RC(rc, "%s", "fsl_buffer_append"); - goto end; - } - - commit = calloc(1, sizeof(*commit)); - if (commit == NULL) { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), "%s", "calloc"); - goto end; - } - - if (!rid && (rc = fsl_stmt_get_id(q, 3, &rid))) { - rc = RC(rc, "%s", "fsl_stmt_get_id"); - goto end; - } - /* Is there a more efficient way to get the parent? */ - commit->puuid = fsl_db_g_text(db, NULL, - "SELECT uuid FROM plink, blob WHERE plink.cid=%d " - "AND blob.rid=plink.pid AND plink.isprim", rid); - commit->prid = fsl_uuid_to_rid(f, commit->puuid); - commit->uuid = fsl_strdup(fsl_stmt_g_text(q, 0, NULL)); - commit->rid = rid; - commit->type = fsl_strdup(type); - commit->diff_type = diff_type; - commit->timestamp = fsl_strdup(fsl_stmt_g_text(q, 1, NULL)); - commit->user = fsl_strdup(fsl_stmt_g_text(q, 2, NULL)); - commit->branch = fsl_strdup(fsl_stmt_g_text(q, 5, NULL)); - commit->comment = fsl_strdup(comment ? fsl_buffer_str(&buf) : ""); - fsl_buffer_clear(&buf); - - *ptr = commit; -end: - return rc; -} - -static int -signal_tl_thread(struct fnc_view *view, int wait) -{ - struct fnc_tl_thread_cx *cx = &view->state.timeline.thread_cx; - int rc = 0; - - while (cx->ncommits_needed > 0) { - if (cx->timeline_end) - break; - - /* Wake timeline thread. */ - if ((rc = pthread_cond_signal(&cx->commit_consumer))) - return RC(fsl_errno_to_rc(rc, FSL_RC_MISUSE), - "%s", "pthread_cond_signal"); - - /* - * Mutex will be released while view_loop().view_input() waits - * in wgetch(), at which point the timeline thread will run. - */ - if (!wait) - break; - - /* Show status update in timeline view. */ - show_timeline_view(view); - update_panels(); - doupdate(); - - /* Wait while the next commit is being loaded. */ - if ((rc = pthread_cond_wait(&cx->commit_producer, &fnc_mutex))) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_cond_wait"); - - /* Show status update in timeline view. */ - show_timeline_view(view); - update_panels(); - doupdate(); - } - - return cx->rc; -} - -static int -draw_commits(struct fnc_view *view) -{ - struct fnc_tl_view_state *s = &view->state.timeline; - struct fnc_tl_thread_cx *tcx = &s->thread_cx; - struct commit_entry *entry = s->selected_commit; - struct fnc_colour *c = NULL; - const char *search_str = NULL; - char *headln = NULL, *idxstr = NULL; - char *branch = NULL, *type = NULL; - char *uuid = NULL; - wchar_t *wcstr; - int ncommits = 0, rc = 0, wstrlen = 0; - int ncols_needed, max_usrlen = -1; - - if (s->selected_commit && !(view->searching != SEARCH_DONE && - view->search_status == SEARCH_WAITING)) { - uuid = fsl_strdup(s->selected_commit->commit->uuid); - branch = fsl_strdup(s->selected_commit->commit->branch); - type = fsl_strdup(s->selected_commit->commit->type); - } - - if (s->thread_cx.ncommits_needed > 0) { - if ((idxstr = fsl_mprintf(" [%d/%d] %s", - entry ? entry->idx + 1 : 0, s->commits.ncommits, - (view->searching && !view->search_status) ? - "searching..." : "loading...")) == NULL) { - rc = RC(FSL_RC_RANGE, "%s", "fsl_mprintf"); - goto end; - } - } else { - if (view->searching) { - if (view->search_status == SEARCH_COMPLETE) - search_str = "no more matches"; - else if (view->search_status == SEARCH_NO_MATCH) - search_str = "no matches found"; - else if (view->search_status == SEARCH_WAITING) - search_str = "searching..."; - } - - if ((idxstr = fsl_mprintf("%s [%d/%d] %s", - !fsl_strcmp(uuid, s->curr_ckout_uuid) ? " [current]" : "", - entry ? entry->idx + 1 : 0, s->commits.ncommits, - search_str ? search_str : (branch ? branch : ""))) - == NULL) { - rc = RC(FSL_RC_RANGE, "%s", "fsl_mprintf"); - goto end; - } - } - /* - * Compute cols needed to fit all components of the headline to truncate - * the hash component if needed. wiki, tag, and ticket artifacts don't - * have a branch component, checkins and some technotes do, so add a col - * for the space separator. Same applies if search_str is being shown. - */ - ncols_needed = fsl_strlen(type) + fsl_strlen(idxstr) + FSL_STRLEN_K256 - + (!search_str && (!fsl_strcmp(type, "wiki") || - !fsl_strcmp(type, "tag") || !fsl_strcmp(type, "ticket") || - (!branch && !fsl_strcmp(type, "technote"))) ? 0 : 1); - /* If a path has been requested, display it in the headline. */ - if (s->path[1]) { - if ((headln = fsl_mprintf("%s%c%.*s %s%s", type ? type : "", - type ? ' ' : SPINNER[tcx->spin_idx], view->ncols < - ncols_needed ? view->ncols - (ncols_needed - - FSL_STRLEN_K256) : FSL_STRLEN_K256, uuid ? uuid : - "........................................", - s->path, idxstr)) == NULL) { - rc = RC(FSL_RC_RANGE, "%s", "fsl_mprintf"); - headln = NULL; - goto end; - } - } else if ((headln = fsl_mprintf("%s%c%.*s%s", type ? type : "", type ? - ' ' : SPINNER[tcx->spin_idx], view->ncols < ncols_needed ? - view->ncols - (ncols_needed - FSL_STRLEN_K256) : FSL_STRLEN_K256, - uuid ? uuid : "........................................", idxstr)) - == NULL) { - rc = RC(FSL_RC_RANGE, "%s", "fsl_mprintf"); - headln = NULL; - goto end; - } - if (SPINNER[++tcx->spin_idx] == '\0') - tcx->spin_idx = 0; - rc = formatln(&wcstr, &wstrlen, headln, view->ncols, 0); - if (rc) - goto end; - - werase(view->window); - - if (screen_is_shared(view)) - wstandout(view->window); - if (s->colour) - c = get_colour(&s->colours, FNC_COMMIT_ID); - if (c) - wattr_on(view->window, COLOR_PAIR(c->scheme), NULL); - waddwstr(view->window, wcstr); - if (c) - wattr_off(view->window, COLOR_PAIR(c->scheme), NULL); - while (wstrlen < view->ncols) { - waddch(view->window, ' '); - ++wstrlen; - } - if (screen_is_shared(view)) - wstandend(view->window); - fsl_free(wcstr); - if (view->nlines <= 1) - goto end; - - /* Parse commits to be written on screen for the longest username. */ - entry = s->first_commit_onscreen; - while (entry) { - wchar_t *usr_wcstr; - char *user; - int usrlen; - if (ncommits >= view->nlines - 1) - break; - user = fsl_strdup(entry->commit->user); - if (user == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - if (strpbrk(user, "<@>") != NULL) - parse_emailaddr_username(&user); - rc = formatln(&usr_wcstr, &usrlen, user, view->ncols, 0); - if (max_usrlen < usrlen) - max_usrlen = usrlen; - fsl_free(usr_wcstr); - fsl_free(user); - ++ncommits; - entry = TAILQ_NEXT(entry, entries); - } - - ncommits = 0; - entry = s->first_commit_onscreen; - s->last_commit_onscreen = s->first_commit_onscreen; - while (entry) { - if (ncommits >= view->nlines - 1) - break; - if (ncommits == s->selected_idx) - wattr_on(view->window, A_REVERSE, NULL); - rc = write_commit_line(view, entry->commit, max_usrlen); - if (ncommits == s->selected_idx) - wattr_off(view->window, A_REVERSE, NULL); - ++ncommits; - s->last_commit_onscreen = entry; - entry = TAILQ_NEXT(entry, entries); - } - draw_vborder(view); - -end: - free(branch); - free(type); - free(uuid); - free(idxstr); - free(headln); - return rc; -} - -static void -parse_emailaddr_username(char **username) -{ - char *lt, *usr; - - lt = strchr(*username, '<'); - if (lt && lt[1] != '\0') { - usr = fsl_strdup(++lt); - fsl_free(*username); - } else - usr = *username; - usr[strcspn(usr, "@>")] = '\0'; - - *username = usr; -} - -static int -formatln(wchar_t **ptr, int *wstrlen, const char *mbstr, int column_limit, - int start_column) -{ - wchar_t *wline = NULL; - size_t i, wlen; - int rc = 0, cols = 0; - - *ptr = NULL; - *wstrlen = 0; - - rc = multibyte_to_wchar(mbstr, &wline, &wlen); - if (rc) - return rc; - - if (wlen > 0 && wline[wlen - 1] == L'\n') { - wline[wlen - 1] = L'\0'; - wlen--; - } - if (wlen > 0 && wline[wlen - 1] == L'\r') { - wline[wlen - 1] = L'\0'; - wlen--; - } - - i = 0; - while (i < wlen) { - int width = wcwidth(wline[i]); - - if (width == 0) { - i++; - continue; - } - - if (width == 1 || width == 2) { - if (cols + width > column_limit) - break; - cols += width; - i++; - } else if (width == -1) { - if (wline[i] == L'\t') { - width = TABSIZE - - ((cols + column_limit) % TABSIZE); - } else { - width = 1; - wline[i] = L'.'; - } - if (cols + width > column_limit) - break; - cols += width; - i++; - } else { - rc = RC(FSL_RC_RANGE, "%s", "wcwidth"); - goto end; - } - } - wline[i] = L'\0'; - if (wstrlen) - *wstrlen = cols; -end: - if (rc) - free(wline); - else - *ptr = wline; - return rc; -} - -static int -multibyte_to_wchar(const char *src, wchar_t **dst, size_t *dstlen) -{ - int rc = 0; - - /* - * mbstowcs POSIX extension specifies that the number of wchar that - * would be written are returned when first arg is a null pointer: - * https://en.cppreference.com/w/cpp/string/multibyte/mbstowcs - */ - *dstlen = mbstowcs(NULL, src, 0); - if (*dstlen == (size_t)-1) { - if (errno == EILSEQ) - return RC(FSL_RC_RANGE, - "invalid multibyte character [%s]", src); - return RC(FSL_RC_MISUSE, "mbstowcs(%s)", src); - } - - - *dst = NULL; - *dst = fsl_malloc(sizeof(wchar_t) * (*dstlen + 1)); - if (*dst == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "malloc"); - goto end; - } - - if (mbstowcs(*dst, src, *dstlen) != *dstlen) - rc = RC(FSL_RC_SIZE_MISMATCH, "mbstowcs(%s)", src); - -end: - if (rc) { - fsl_free(*dst); - *dst = NULL; - *dstlen = 0; - } - - return rc; -} - -/* - * When the terminal is >= 110 columns wide, the commit summary line in the - * timeline view will take the form: - * - * DATE UUID USERNAME COMMIT-COMMENT - * - * Assuming an 8-character username, this scheme provides 80 characters for the - * comment, which should be sufficient considering it's suggested good practice - * to limit commit comment summary lines to a maximum 50 characters, and most - * plaintext-based conventions suggest not exceeding 72-80 characters. - * - * When < 110 columns, the (abbreviated 9-character) UUID will be elided. - */ -static int -write_commit_line(struct fnc_view *view, struct fnc_commit_artifact *commit, - int max_usrlen) -{ - struct fnc_tl_view_state *s = &view->state.timeline; - struct fnc_colour *c = NULL; - wchar_t *usr_wcstr = NULL, *wcomment = NULL; - char *comment0 = NULL, *comment = NULL; - char *date = NULL; - char *eol = NULL, *pad = NULL, *user = NULL; - size_t i = 0; - int col_pos, ncols_avail, usrlen; - int commentlen, rc = 0; - - /* Trim time component from timestamp for the date field. */ - date = fsl_strdup(commit->timestamp); - while (!fsl_isspace(date[i++])) {} - date[i] = '\0'; - col_pos = MIN(view->ncols, ISO8601_DATE_ONLY + 1); - if (s->colour) - c = get_colour(&s->colours, FNC_DATE_STR); - if (c) - wattr_on(view->window, COLOR_PAIR(c->scheme), NULL); - waddnstr(view->window, date, col_pos); - if (c) - wattr_off(view->window, COLOR_PAIR(c->scheme), NULL); - if (col_pos > view->ncols) - goto end; - - /* If enough columns, write abbreviated commit hash. */ - if (view->ncols >= 110) { - if (s->colour) - c = get_colour(&s->colours, FNC_COMMIT_ID); - if (c) - wattr_on(view->window, COLOR_PAIR(c->scheme), NULL); - wprintw(view->window, "%.9s ", commit->uuid); - if (c) - wattr_off(view->window, COLOR_PAIR(c->scheme), NULL); - col_pos += 10; - if (col_pos > view->ncols) - goto end; - } - - /* - * Parse username from emailaddr if needed, and postfix username - * with as much whitespace as needed to fill two spaces beyond - * the longest username on the screen. - */ - user = fsl_strdup(commit->user); - if (user == NULL) - goto end; - if (strpbrk(user, "<@>") != NULL) - parse_emailaddr_username(&user); - rc = formatln(&usr_wcstr, &usrlen, user, view->ncols - col_pos, - col_pos); - if (rc) - goto end; - if (s->colour) - c = get_colour(&s->colours, FNC_USER_STR); - if (c) - wattr_on(view->window, COLOR_PAIR(c->scheme), NULL); - waddwstr(view->window, usr_wcstr); - pad = fsl_mprintf("%*c", max_usrlen - usrlen + 2, ' '); - waddstr(view->window, pad); - if (c) - wattr_off(view->window, COLOR_PAIR(c->scheme), NULL); - col_pos += (max_usrlen + 2); - if (col_pos > view->ncols) - goto end; - - /* Only show comment up to the first newline character. */ - comment0 = fsl_strdup(commit->comment); - comment = comment0; - if (comment == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - while (*comment == '\n') - ++comment; - eol = strchr(comment, '\n'); - if (eol) - *eol = '\0'; - ncols_avail = view->ncols - col_pos; - rc = formatln(&wcomment, &commentlen, comment, ncols_avail, col_pos); - if (rc) - goto end; - waddwstr(view->window, wcomment); - col_pos += commentlen; - while (col_pos < view->ncols) { - waddch(view->window, ' '); - ++col_pos; - } -end: - fsl_free(date); - fsl_free(user); - fsl_free(usr_wcstr); - fsl_free(pad); - fsl_free(comment0); - fsl_free(wcomment); - return rc; -} - -static int -view_input(struct fnc_view **new, int *done, struct fnc_view *view, - struct view_tailhead *views) -{ - struct fnc_view *v; - int ch, rc = 0; - - *new = NULL; - - /* Clear search indicator string. */ - if (view->search_status == SEARCH_COMPLETE || - view->search_status == SEARCH_NO_MATCH) - view->search_status = SEARCH_CONTINUE; - - if (view->searching && view->search_status == SEARCH_WAITING) { - if ((rc = pthread_mutex_unlock(&fnc_mutex))) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_unlock"); - sched_yield(); - if ((rc = pthread_mutex_lock(&fnc_mutex))) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_lock"); - rc = view->search_next(view); - return rc; - } - - nodelay(stdscr, FALSE); - /* Allow thread to make progress while waiting for input. */ - if ((rc = pthread_mutex_unlock(&fnc_mutex))) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_unlock"); - ch = wgetch(view->window); - if ((rc = pthread_mutex_lock(&fnc_mutex))) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_lock"); - - if (rec_sigwinch || rec_sigcont) { - fnc_resizeterm(); - rec_sigwinch = 0; - rec_sigcont = 0; - TAILQ_FOREACH(v, views, entries) { - if ((rc = view_resize(v))) - return rc; - if ((rc = v->input(new, v, KEY_RESIZE))) - return rc; - if (v->child) { - if ((rc = view_resize(v->child))) - return rc; - rc = v->child->input(new, v->child, KEY_RESIZE); - if (rc) - return rc; - } - } - } - - switch (ch) { - case '\t': - if (view->child) { - view->active = false; - view->child->active = true; - view->focus_child = true; - } else if (view->parent) { - view->active = false; - view->parent->active = true; - view->parent->focus_child = false; - } - break; - case KEY_F(1): - case 'H': - case '?': - help(view); - break; - case 'q': - rc = view->input(new, view, ch); - view->egress = true; - break; - case 'f': - if (view_is_parent(view)) { - if (view->child == NULL) - break; - if (screen_is_split(view->child)) { - view->active = false; - view->child->active = true; - rc = make_fullscreen(view->child); - } else - rc = make_splitscreen(view->child); - if (rc) - break; - rc = view->child->input(new, view->child, KEY_RESIZE); - } else { - if (screen_is_split(view)) { - view->parent->active = false; - view->active = true; - rc = make_fullscreen(view); - } else - rc = make_splitscreen(view); - if (rc) - break; - rc = view->input(new, view, KEY_RESIZE); - } - break; - case '/': - if (view->search_init) - view_search_start(view); - else - rc = view->input(new, view, ch); - break; - case 'N': - case 'n': - if (view->started_search && view->search_next) { - view->searching = (ch == 'n' ? - SEARCH_FORWARD : SEARCH_REVERSE); - view->search_status = SEARCH_WAITING; - rc = view->search_next(view); - } else - rc = view->input(new, view, ch); - break; - case KEY_RESIZE: - break; - case ERR: - break; - case 'Q': - *done = 1; - break; - default: - rc = view->input(new, view, ch); - break; - } - - return rc; -} - -static int -help(struct fnc_view *view) -{ - FILE *help = NULL; - char *title = NULL; - static const char *keys[][2] = { - {""}, - {""}, /* Global */ - {" H,?,F1 ", " ❬H❭❬?❭❬F1❭ "}, - {" k, ", " ❬↑❭❬k❭ "}, - {" j, ", " ❬↓❭❬j❭ "}, - {" C-b,PgUp ", " ❬C-b❭❬PgUp❭ "}, - {" C-f,PgDn ", " ❬C-f❭❬PgDn❭ "}, - {" gg,Home ", " ❬gg❭❬Home❭ "}, - {" G,End ", " ❬G❭❬End❭ "}, - {" Tab ", " ❬TAB❭ "}, - {" c ", " ❬c❭ "}, - {" f ", " ❬f❭ "}, - {" / ", " ❬/❭ "}, - {" n ", " ❬n❭ "}, - {" N ", " ❬N❭ "}, - {" q ", " ❬q❭ "}, - {" Q ", " ❬Q❭ "}, - {""}, - {""}, /* Timeline */ - {" <,, ", " ❬<❭❬,❭ "}, - {" >,. ", " ❬>❭❬.❭ "}, - {" Enter,Space ", " ❬Enter❭❬Space❭ "}, - {" t ", " ❬t❭ "}, - {""}, - {""}, /* Diff */ - {" Space ", " ❬Space❭ "}, - {" i ", " ❬i❭ "}, - {" v ", " ❬v❭ "}, - {" w ", " ❬w❭ "}, - {" -,_ ", " ❬-❭❬_❭ "}, - {" +,= ", " ❬+❭❬=❭ "}, - {" C-k,K,<,, ", " ❬C-k❭❬K❭❬<❭❬,❭ "}, - {" C-j,J,>,. ", " ❬C-j❭❬J❭❬>❭❬.❭ "}, - {""}, - {""}, /* Tree */ - {" l,Enter, ", " ❬→❭❬l❭❬Enter❭ "}, - {" h,, ", " ❬←❭❬h❭❬⌫❭ "}, - {" i ", " ❬i❭ "}, - {" t ", " ❬t❭ "}, - {""}, - {""}, /* Blame */ - {" Space ", " ❬Space❭ "}, - {" Enter ", " ❬Enter❭ "}, - {" b ", " ❬b❭ "}, - {" p ", " ❬p❭ "}, - {" B ", " ❬B❭ "}, - {""}, - {""}, /* Branch */ - {" Enter,Space ", " ❬Enter❭❬Space❭ "}, - {" d ", " ❬d❭ "}, - {" i ", " ❬i❭ "}, - {" t ", " ❬t❭ "}, - {" R, ", " ❬R❭❬C-l❭ "}, - {""}, - {""}, - {0} - }; - static const char *desc[] = { - "", - "Global", - "Open in-app help", - "Move selection cursor or page up one line", - "Move selection cursor or page down one line", - "Scroll up one page", - "Scroll down one page", - "Jump to first line or start of the view", - "Jump to last line or end of the view", - "Switch focus between open views", - "Toggle coloured output", - "Toggle fullscreen", - "Open prompt to enter search term (not available in this view)", - "Find next line or token matching the current search term", - "Find previous line or token matching the current search term", - "Quit the active view", - "Quit the program", - "", - "Timeline", - "Move selection cursor up one commit", - "Move selection cursor down one commit", - "Open diff view of the selected commit", - "Display a tree reflecting the state of the selected commit", - "", - "Diff", - "Scroll down one page of diff output", - "Toggle inversion of diff output", - "Toggle verbosity of diff output", - "Toggle ignore whitespace-only changes in diff", - "Decrease the number of context lines", - "Increase the number of context lines", - "Display diff of next (newer) commit in the timeline", - "Display diff of previous (older) commit in the timeline", - "", - "Tree", - "Move into the selected directory", - "Return to the parent directory", - "Toggle display of file artifact SHA hashes", - "Display timeline of all commits modifying the selected entry", - "", - "Blame", - "Scroll down one page", - "Display the diff of the commit corresponding to the selected line", - "Blame the version of the file found in the selected line's commit", - "Blame the version of the file found in the selected line's parent " - "commit", - "Reload the previous blamed version of the file", - "", - "Branch", - "Display the timeline of the currently selected branch", - "Toggle display of the date when the branch last received changes", - "Toggle display of the SHA hash that identifies the branch", - "Open a tree view of the currently selected branch", - "Reload view with all repostory branches and no filters applied", - "", - " See fnc(1) for complete list of options and key bindings." - }; - int cs, ln, width = 0, rc = 0; - - cs = (strcmp(nl_langinfo(CODESET), "UTF-8") == 0) ? 1 : 0; - - title = fsl_mprintf("%s %s Help\n", fcli_progname(), PRINT_VERSION); - if (title == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_mprintf"); - - help = tmpfile(); - if (help == NULL) - return RC(FSL_RC_IO, "%s", "tmpfile"); - - /* - * Format help text, and compute longest line and total number of - * lines in text to be displayed to determine pad dimensions. - */ - width = fsl_strlen(title); - for (ln = 0; keys[ln][0]; ++ln) { - if (keys[ln][1]) { - width = MAX((fsl_size_t)width, - fsl_strlen(keys[ln][cs]) + fsl_strlen(desc[ln])); - } - fsl_fprintf(help, "%s%s%c", keys[ln][cs], desc[ln], - keys[ln + 1] ? '\n' : 0); - } - rewind(help); - - rc = padpopup(view, width, ln, help, title); - if (fclose(help) == EOF) - rc = RC(fsl_errno_to_rc(errno, FSL_RC_IO), "%s", "fclose"); - - fsl_free(title); - return rc; -} - -/* - * Create popup pad in which to write the supplied txt string and optional - * title. The pad is contained within a window that is offset four columns in - * and two lines down from the parent window. - */ -static int -padpopup(struct fnc_view *view, int width, int height, FILE *txt, - const char *title) -{ - WINDOW *win, *content; - char *line = NULL; - ssize_t linelen; - size_t linesz; - int ch, cury, end, wy, wx, x0, y0; - - x0 = 4; /* Number of columns to border window. */ - y0 = 2; /* Number of lines to border window. */ - cury = 0; - wx = getmaxx(view->window) - ((x0 + 1) * 2); /* Width of window. */ - wy = getmaxy(view->window) - ((y0 + 1) * 2); /* Height of window */ - ch = ERR; - - if ((win = newwin(wy, wx, y0, x0)) == 0) - return RC(FSL_RC_ERROR, "%s", "newwin"); - if ((content = newpad(height + 1, width + 1)) == 0) { - delwin(win); - return RC(FSL_RC_ERROR, "%s", "newpad"); - } - - doupdate(); - keypad(content, TRUE); - - /* Write text content to pad. */ - if (title) - centerprint(content, 0, 0, width, title, 0); - while ((linelen = getline(&line, &linesz, txt)) != -1) - waddstr(content, line); - fsl_free(line); - - end = (getcury(content) - (wy - 3)); /* No. lines past end of pad. */ - do { - switch (ch) { - case KEY_UP: - case 'k': - if (cury > 0) - --cury; - break; - case KEY_DOWN: - case 'j': - if (cury < end) - ++cury; - break; - case KEY_PPAGE: - case CTRL('b'): - if (cury > 0) { - cury -= wy / 2; - if (cury < 0) - cury = 0; - } - break; - case KEY_NPAGE: - case CTRL('f'): - case ' ': - if (cury < end) { - cury += wy / 2; - if (cury > end) - cury = end; - } - break; - case 'g': - if (!fnc_home(view)) - break; - /* FALL THROUGH */ - case KEY_HOME: - cury = 0; - break; - case KEY_END: - case 'G': - cury = end; - break; - case ERR: - default: - break; - } - werase(win); - box(win, 0, 0); - wnoutrefresh(win); - pnoutrefresh(content, cury, 0, y0 + 1, x0 + 1, wy, wx); - doupdate(); - } while ((ch = wgetch(content)) != 'q' && ch != KEY_ESCAPE - && ch != ERR); - - /* Destroy window. */ - werase(win); - wrefresh(win); - delwin(win); - delwin(content); - - /* Restore fnc window content. */ - touchwin(view->window); - wnoutrefresh(view->window); - doupdate(); - - return 0; -} - -static void -centerprint(WINDOW *win, int starty, int startx, int cols, const char *str, - chtype colour) -{ - int x, y; - - if (win == NULL) - win = stdscr; - - getyx(win, y, x); - x = startx ? startx : x; - y = starty ? starty : y; - if (!cols) - cols = getmaxx(win); - - x = startx + (cols - fsl_strlen(str)) / 2; - wattron(win, colour ? colour : A_UNDERLINE); - mvwprintw(win, y, x, "%s", str); - wattroff(win, colour ? colour : A_UNDERLINE); - refresh(); -} - -static int -tl_input_handler(struct fnc_view **new_view, struct fnc_view *view, int ch) -{ - struct fnc_tl_view_state *s = &view->state.timeline; - struct fnc_view *diff_view = NULL, *tree_view = NULL; - int rc = 0, start_col = 0; - - switch (ch) { - case KEY_DOWN: - case 'j': - case '.': - case '>': - if (s->first_commit_onscreen == NULL) - break; - if (s->selected_idx < MIN(view->nlines - 2, - s->commits.ncommits - 1)) - ++s->selected_idx; - else if ((rc = timeline_scroll_down(view, 1))) - break; - select_commit(s); - break; - case KEY_NPAGE: - case CTRL('f'): { - struct commit_entry *first; - if ((first = s->first_commit_onscreen) == NULL) - break; - if ((rc = timeline_scroll_down(view, view->nlines - 1))) - break; - if (first == s->first_commit_onscreen && s->selected_idx < - MIN(view->nlines - 2, s->commits.ncommits - 1)) - /* At bottom of timeline. */ - s->selected_idx = MIN(view->nlines - 2, - s->commits.ncommits - 1); - select_commit(s); - break; - } - case KEY_END: - case 'G': - view->search_status = SEARCH_FOR_END; - view_search_start(view); - break; - case 'k': - case KEY_UP: - case '<': - case ',': - if (s->first_commit_onscreen == NULL) - break; - if (s->selected_idx > 0) - --s->selected_idx; - else - timeline_scroll_up(s, 1); - select_commit(s); - break; - case KEY_PPAGE: - case CTRL('b'): - if (s->first_commit_onscreen == NULL) - break; - if (TAILQ_FIRST(&s->commits.head) == s->first_commit_onscreen) - s->selected_idx = 0; - else - timeline_scroll_up(s, view->nlines - 1); - select_commit(s); - break; - case 'g': - if (!fnc_home(view)) - break; - /* FALL THROUGH */ - case KEY_HOME: - if (s->first_commit_onscreen == NULL) - break; - s->selected_idx = 0; - timeline_scroll_up(s, s->commits.ncommits); - select_commit(s); - break; - case KEY_RESIZE: - if (s->selected_idx > view->nlines - 2) - s->selected_idx = view->nlines - 2; - if (s->selected_idx > s->commits.ncommits - 1) - s->selected_idx = s->commits.ncommits - 1; - select_commit(s); - if (s->commits.ncommits < view->nlines - 1 && - !s->thread_cx.timeline_end) { - s->thread_cx.ncommits_needed += (view->nlines - 1) - - s->commits.ncommits; - rc = signal_tl_thread(view, 1); - } - break; - case KEY_ENTER: - case ' ': - case '\r': - if (s->selected_commit == NULL) - break; - if (view_is_parent(view)) - start_col = view_split_start_col(view->start_col); - if ((rc = init_diff_commit(&diff_view, start_col, - s->selected_commit->commit, view))) - break; - view->active = false; - diff_view->active = true; - if (view_is_parent(view)) { - if (view->child != NULL) { - rc = view_close(view->child); - view->child = NULL; - } - if (rc) - return rc; - view->child = diff_view; - diff_view->parent = view; - view->focus_child = true; - } else - *new_view = diff_view; - break; - case 'c': - s->colour = !s->colour; - break; - case 't': - if (s->selected_commit == NULL) - break; - if (!fsl_rid_is_a_checkin(fcli_cx(), - s->selected_commit->commit->rid)) { - wattr_on(view->window, A_BOLD, NULL); - mvwaddstr(view->window, - view->start_ln + view->nlines - 1, 0, - "-- tree requires check-in artifact --"); - wclrtoeol(view->window); - wattr_off(view->window, A_BOLD, NULL); - fcli_err_reset(); - rc = 0; - update_panels(); - doupdate(); - sleep(1); - break; - } - if (view_is_parent(view)) - start_col = view_split_start_col(view->start_col); - rc = browse_commit_tree(&tree_view, start_col, - s->selected_commit, s->path); - if (rc) - break; - s->thread_cx.tree_open = true; - view->active = false; - tree_view->active = true; - if (view_is_parent(view)) { - rc = view_close_child(view); - if (rc) - return rc; - view_set_child(view, tree_view); - view->focus_child = true; - } else - *new_view = tree_view; - break; - case 'q': - s->quit = 1; - break; - default: - break; - } - - return rc; -} - -static int -timeline_scroll_down(struct fnc_view *view, int maxscroll) -{ - struct fnc_tl_view_state *s = &view->state.timeline; - struct commit_entry *pentry; - int rc = 0, nscrolled = 0, ncommits_needed; - - if (s->last_commit_onscreen == NULL) - return 0; - - ncommits_needed = s->last_commit_onscreen->idx + 1 + maxscroll; - if (s->commits.ncommits < ncommits_needed && - !s->thread_cx.timeline_end) { - /* Signal timeline thread for n commits needed. */ - s->thread_cx.ncommits_needed += maxscroll; - rc = signal_tl_thread(view, 1); - if (rc) - return rc; - } - - do { - pentry = TAILQ_NEXT(s->last_commit_onscreen, entries); - if (pentry == NULL) - break; - - s->last_commit_onscreen = pentry; - - pentry = TAILQ_NEXT(s->first_commit_onscreen, entries); - if (pentry == NULL) - break; - s->first_commit_onscreen = pentry; - } while (++nscrolled < maxscroll); - - return rc; -} - -static void -timeline_scroll_up(struct fnc_tl_view_state *s, int maxscroll) -{ - struct commit_entry *entry; - int nscrolled = 0; - - entry = TAILQ_FIRST(&s->commits.head); - if (s->first_commit_onscreen == entry) - return; - - entry = s->first_commit_onscreen; - while (entry && nscrolled < maxscroll) { - entry = TAILQ_PREV(entry, commit_tailhead, entries); - if (entry) { - s->first_commit_onscreen = entry; - ++nscrolled; - } - } -} - -static void -select_commit(struct fnc_tl_view_state *s) -{ - struct commit_entry *entry; - int ncommits = 0; - - entry = s->first_commit_onscreen; - while (entry) { - if (ncommits == s->selected_idx) { - s->selected_commit = entry; - break; - } - entry = TAILQ_NEXT(entry, entries); - ++ncommits; - } -} - -static int -make_splitscreen(struct fnc_view *view) -{ - int rc = 0; - - view->start_ln = 0; - view->start_col = view_split_start_col(0); - view->nlines = LINES; - view->ncols = COLS - view->start_col; - view->lines = LINES; - view->cols = COLS; - if ((rc = view_resize(view))) - return rc; - - if (mvwin(view->window, view->start_ln, view->start_col) == ERR) - return RC(FSL_RC_ERROR, "%s", "mvwin"); - - return rc; -} - -static int -make_fullscreen(struct fnc_view *view) -{ - int rc = 0; - - view->start_col = 0; - view->start_ln = 0; - view->nlines = LINES; - view->ncols = COLS; - view->lines = LINES; - view->cols = COLS; - if ((rc = view_resize(view))) - return rc; - - if (mvwin(view->window, view->start_ln, view->start_col) == ERR) - return RC(FSL_RC_ERROR, "%s", "mvwin"); - - return rc; -} - -/* - * Only open a new view in a splitscreen if the console is >= 120 columns wide. - * Otherwise, open in the current view. If splitting the screen, make the new - * panel the largest of 80 columns or half the current column width. - */ -static int -view_split_start_col(int start_col) -{ - if (start_col > 0 || COLS < 120) - return 0; - return (COLS - MAX(COLS / 2, 80)); -} - -static int -view_search_start(struct fnc_view *view) -{ - char pattern[BUFSIZ]; - int retval; - int rc = 0; - - if (view->started_search) { - regfree(&view->regex); - view->searching = SEARCH_DONE; - memset(&view->regmatch, 0, sizeof(view->regmatch)); - } - view->started_search = false; - - if (view->nlines < 1) - return rc; - - if (view->search_status == SEARCH_FOR_END) { - view->search_init(view); - view->started_search = true; - view->searching = SEARCH_FORWARD; - view->search_status = SEARCH_WAITING; - view->state.timeline.thread_cx.endjmp = true; - rc = view->search_next(view); - - return rc; - } - - mvwaddstr(view->window, view->start_ln + view->nlines - 1, 0, "/"); - wclrtoeol(view->window); - - nocbreak(); - echo(); - retval = wgetnstr(view->window, pattern, sizeof(pattern)); - cbreak(); - noecho(); - if (retval == ERR) - return rc; - - if (regcomp(&view->regex, pattern, REG_EXTENDED | REG_NEWLINE) == 0) { - if ((rc = view->search_init(view))) { - regfree(&view->regex); - return rc; - } - view->started_search = true; - view->searching = SEARCH_FORWARD; - view->search_status = SEARCH_WAITING; - rc = view->search_next(view); - } - - return rc; -} - -static int -tl_search_init(struct fnc_view *view) -{ - struct fnc_tl_view_state *s = &view->state.timeline; - - s->matched_commit = NULL; - s->search_commit = NULL; - return 0; -} - -static int -tl_search_next(struct fnc_view *view) -{ - struct fnc_tl_view_state *s = &view->state.timeline; - struct commit_entry *entry; - int rc = 0; - - if (!s->thread_cx.ncommits_needed && view->started_search) - halfdelay(1); - - /* Show status update in timeline view. */ - show_timeline_view(view); - update_panels(); - doupdate(); - - if (s->search_commit) { - int ch; - if ((rc = pthread_mutex_unlock(&fnc_mutex))) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_unlock"); - ch = wgetch(view->window); - if ((rc = pthread_mutex_lock(&fnc_mutex))) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_lock"); - if (ch == KEY_BACKSPACE) { - view->search_status = SEARCH_CONTINUE; - return rc; - } - if (view->searching == SEARCH_FORWARD) - entry = TAILQ_NEXT(s->search_commit, entries); - else - entry = TAILQ_PREV(s->search_commit, commit_tailhead, - entries); - } else if (s->matched_commit) { - if (view->searching == SEARCH_FORWARD) - entry = TAILQ_NEXT(s->matched_commit, entries); - else - entry = TAILQ_PREV(s->matched_commit, commit_tailhead, - entries); - } else { - if (view->searching == SEARCH_FORWARD) - entry = TAILQ_FIRST(&s->commits.head); - else - entry = TAILQ_LAST(&s->commits.head, commit_tailhead); - } - - while (1) { - if (entry == NULL) { - if (s->thread_cx.timeline_end && s->thread_cx.endjmp) { - s->matched_commit = TAILQ_LAST(&s->commits.head, - commit_tailhead); - view->search_status = SEARCH_COMPLETE; - s->thread_cx.endjmp = false; - break; - } - if (s->thread_cx.timeline_end || view->searching == - SEARCH_REVERSE) { - view->search_status = (s->matched_commit == - NULL ? SEARCH_NO_MATCH : SEARCH_COMPLETE); - s->search_commit = NULL; - cbreak(); - return rc; - } - /* - * Wake the timeline thread to produce more commits. - * Search will resume at s->search_commit upon return. - */ - ++s->thread_cx.ncommits_needed; - return signal_tl_thread(view, 0); - } - - if (!s->thread_cx.endjmp && find_commit_match(entry->commit, - &view->regex)) { - view->search_status = SEARCH_CONTINUE; - s->matched_commit = entry; - break; - } - - s->search_commit = entry; - if (view->searching == SEARCH_FORWARD) - entry = TAILQ_NEXT(entry, entries); - else - entry = TAILQ_PREV(entry, commit_tailhead, entries); - } - - if (s->matched_commit) { - int cur = s->selected_commit->idx; - while (cur < s->matched_commit->idx) { - if ((rc = tl_input_handler(NULL, view, KEY_DOWN))) - return rc; - ++cur; - } - while (cur > s->matched_commit->idx) { - if ((rc = tl_input_handler(NULL, view, KEY_UP))) - return rc; - --cur; - } - } - - s->search_commit = NULL; - cbreak(); - - return rc; -} - -static bool -find_commit_match(struct fnc_commit_artifact *commit, -regex_t *regex) -{ - regmatch_t regmatch; - - if (regexec(regex, commit->user, 1, ®match, 0) == 0 || - regexec(regex, (char *)commit->uuid, 1, ®match, 0) == 0 || - regexec(regex, commit->comment, 1, ®match, 0) == 0 || - (commit->branch && regexec(regex, commit->branch, 1, ®match, 0) - == 0)) - return true; - - return false; -} - -static int -view_close(struct fnc_view *view) -{ - int rc = 0; - - if (view->child) { - view_close(view->child); - view->child = NULL; - } - if (view->close) - rc = view->close(view); - if (view->panel) - del_panel(view->panel); - if (view->window) - delwin(view->window); - free(view); - - return rc; -} - -static int -close_timeline_view(struct fnc_view *view) -{ - struct fnc_tl_view_state *s = &view->state.timeline; - struct fsl_list_state st = { FNC_COLOUR_OBJ }; - int rc = 0; - - rc = join_tl_thread(s); - fsl_stmt_finalize(s->thread_cx.q); - fnc_free_commits(&s->commits); - fsl_list_clear(&s->colours, fsl_list_object_free, &st); - regfree(&view->regex); - fsl_free(s->path); - s->path = NULL; - - return rc; -} - -/* static void */ -/* sspinner(void) */ -/* { */ -/* int idx; */ - -/* while (1) { */ -/* for (idx = 0; idx < 4; ++idx) { */ -/* printf("\b%c", "|/-\\"[idx]); */ -/* fflush(stdout); */ -/* ssleep(SPIN_INTERVAL); */ -/* } */ -/* } */ -/* } */ - -static int -join_tl_thread(struct fnc_tl_view_state *s) -{ - void *err; - int rc = 0; - - if (s->thread_id) { - s->quit = 1; - - if ((rc = pthread_cond_signal(&s->thread_cx.commit_consumer))) - return RC(fsl_errno_to_rc(rc, FSL_RC_MISUSE), - "%s", "pthread_cond_signal"); - if ((rc = pthread_mutex_unlock(&fnc_mutex))) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_unlock"); - if ((rc = pthread_join(s->thread_id, &err)) || - err == PTHREAD_CANCELED) - return RC(fsl_errno_to_rc(rc, FSL_RC_MISUSE), - "%s", "pthread_join"); - if ((rc = pthread_mutex_lock(&fnc_mutex))) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_lock"); - - s->thread_id = 0; - } - - if ((rc = pthread_cond_destroy(&s->thread_cx.commit_consumer))) - RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_cond_destroy"); - - if ((rc = pthread_cond_destroy(&s->thread_cx.commit_producer))) - RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_cond_destroy"); - - return rc; -} - -static void -fnc_free_commits(struct commit_queue *commits) -{ - while (!TAILQ_EMPTY(&commits->head)) { - struct commit_entry *entry; - - entry = TAILQ_FIRST(&commits->head); - TAILQ_REMOVE(&commits->head, entry, entries); - fnc_commit_artifact_close(entry->commit); - free(entry); - --commits->ncommits; - } -} - -static void -fnc_commit_artifact_close(struct fnc_commit_artifact *commit) -{ - struct fsl_list_state st = { FNC_ARTIFACT_OBJ }; - - if (commit->branch) - fsl_free(commit->branch); - if (commit->comment) - fsl_free(commit->comment); - if (commit->timestamp) - fsl_free(commit->timestamp); - if (commit->type) - fsl_free(commit->type); - if (commit->user) - fsl_free(commit->user); - fsl_free(commit->uuid); - fsl_free(commit->puuid); - fsl_list_clear(&commit->changeset, fsl_list_object_free, &st); - fsl_list_reserve(&commit->changeset, 0); - fsl_free(commit); -} - -static int -fsl_list_object_free(void *elem, void *state) -{ - struct fsl_list_state *st = state; - - switch (st->obj) { - case FNC_ARTIFACT_OBJ: { - struct fsl_file_artifact *ffa = elem; - fsl_free(ffa->fc->name); - fsl_free(ffa->fc->uuid); - fsl_free(ffa->fc->priorName); - fsl_free(ffa->fc); - fsl_free(ffa); - break; - } - case FNC_COLOUR_OBJ: { - struct fnc_colour *c = elem; - regfree(&c->regex); - fsl_free(c); - break; - } - default: - return RC(FSL_RC_MISSING_INFO, - "fsl_list_state.obj missing or invalid: %d", st->obj); - } - - return 0; -} - -static int -init_diff_commit(struct fnc_view **new_view, int start_col, - struct fnc_commit_artifact *commit, struct fnc_view *timeline_view) -{ - struct fnc_view *diff_view; - int rc = 0; - - diff_view = view_open(0, 0, 0, start_col, FNC_VIEW_DIFF); - if (diff_view == NULL) - return RC(FSL_RC_ERROR, "%s", "view_open"); - - rc = open_diff_view(diff_view, commit, DIFF_DEF_CTXT, fnc_init.ws, - fnc_init.invert, !fnc_init.quiet, timeline_view, true, NULL); - if (!rc) - *new_view = diff_view; - - return rc; -} - -static int -open_diff_view(struct fnc_view *view, struct fnc_commit_artifact *commit, - int context, bool ignore_ws, bool invert, bool verbosity, - struct fnc_view *timeline_view, bool showmeta, - struct fnc_pathlist_head *paths) -{ - struct fnc_diff_view_state *s = &view->state.diff; - int rc = 0; - - s->paths = paths; - s->selected_commit = commit; - s->first_line_onscreen = 1; - s->last_line_onscreen = view->nlines; - s->current_line = 1; - s->f = NULL; - s->context = context; - s->sbs = 0; - verbosity ? s->diff_flags |= FSL_DIFF_VERBOSE : 0; - ignore_ws ? s->diff_flags |= FSL_DIFF_IGNORE_ALLWS : 0; - invert ? s->diff_flags |= FSL_DIFF_INVERT : 0; - s->timeline_view = timeline_view; - s->colours = fsl_list_empty; - s->colour = !fnc_init.nocolour && has_colors(); - s->showmeta = showmeta; - - if (s->colour) - set_colours(&s->colours, FNC_VIEW_DIFF); - if (timeline_view && screen_is_split(view)) - show_timeline_view(timeline_view); /* draw vborder */ - show_diff_status(view); - - s->line_offsets = NULL; - s->nlines = 0; - s->ncols = view->ncols; - rc = create_diff(s); - if (rc) { - if (s->colour) { - struct fsl_list_state st = { FNC_COLOUR_OBJ }; - fsl_list_clear(&s->colours, fsl_list_object_free, &st); - } - return rc; - } - - view->show = show_diff; - view->input = diff_input_handler; - view->close = close_diff_view; - view->search_init = diff_search_init; - view->search_next = diff_search_next; - - return rc; -} - -static void -show_diff_status(struct fnc_view *view) -{ - mvwaddstr(view->window, 0, 0, "generating diff..."); -#ifdef __linux__ - wnoutrefresh(view->window); -#else - update_panels(); -#endif - doupdate(); -} - -static int -create_diff(struct fnc_diff_view_state *s) -{ - FILE *fout = NULL; - char *line, *st0 = NULL, *st = NULL; - off_t lnoff = 0; - int n, rc = 0; - - free(s->line_offsets); - s->line_offsets = fsl_malloc(sizeof(off_t)); - if (s->line_offsets == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_malloc"); - s->nlines = 0; - - fout = tmpfile(); - if (fout == NULL) { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_IO), "%s", "tmpfile"); - goto end; - } - if (s->f && fclose(s->f) == EOF) { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_IO), "%s", "fclose"); - goto end; - } - s->f = fout; - - /* - * We'll diff artifacts of type "ci" (i.e., "checkin") separately, as - * it's a different process to diff the others (wiki, technote, etc.). - */ - if (s->selected_commit->diff_type == FNC_DIFF_COMMIT) - rc = create_changeset(s->selected_commit); - else if (s->selected_commit->diff_type == FNC_DIFF_BLOB) - rc = diff_file_artifact(&s->buf, s->selected_commit->prid, NULL, - s->selected_commit->rid, NULL, FSL_CKOUT_CHANGE_MOD, - s->diff_flags, s->context, s->sbs, - s->selected_commit->diff_type); - else if (s->selected_commit->diff_type == FNC_DIFF_WIKI) - rc = diff_non_checkin(&s->buf, s->selected_commit, - s->diff_flags, s->context, s->sbs); - if (rc) - goto end; - - /* - * Delay assigning diff headline labels (i.e., diff id1 id2) till now - * because wiki parent commits are obtained in diff_non_checkin(). - */ - if (s->selected_commit->puuid) { - s->id1 = fsl_strdup(s->selected_commit->puuid); - if (s->id1 == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - } else - s->id1 = NULL; /* Initial commit, tag, technote, etc. */ - if (s->selected_commit->uuid) { - s->id2 = fsl_strdup(s->selected_commit->uuid); - if (s->id2 == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - } else - s->id2 = NULL; /* Local work tree. */ - - rc = add_line_offset(&s->line_offsets, &s->nlines, 0); - if (rc) - goto end; - - if (s->showmeta) - write_commit_meta(s); - - /* - * Diff local changes on disk in the current checkout differently to - * checked-in versions: the former compares on disk file content with - * file artifacts; the latter compares file artifact blobs only. - */ - if (s->selected_commit->diff_type == FNC_DIFF_COMMIT) - diff_commit(&s->buf, s->selected_commit, s->diff_flags, - s->context, s->sbs, s->paths); - else if (s->selected_commit->diff_type == FNC_DIFF_CKOUT) - diff_checkout(&s->buf, s->selected_commit->prid, s->diff_flags, - s->context, s->sbs, s->paths); - - /* - * Parse the diff buffer line-by-line to record byte offsets of each - * line for scrolling and searching in diff view. - */ - st0 = fsl_strdup(fsl_buffer_str(&s->buf)); - st = st0; - lnoff = (s->line_offsets)[s->nlines - 1]; - while ((line = fnc_strsep(&st, "\n")) != NULL) { - n = fprintf(s->f, "%s\n", line); - lnoff += n; - rc = add_line_offset(&s->line_offsets, &s->nlines, lnoff); - if (rc) - goto end; - } - - fputc('\n', s->f); - ++lnoff; - rc = add_line_offset(&s->line_offsets, &s->nlines, lnoff); - if (rc) - goto end; - -end: - free(st0); - fsl_buffer_clear(&s->buf); - if (s->f && fflush(s->f) != 0 && rc == 0) - rc = RC(FSL_RC_IO, "%s", "fflush"); - return rc; -} - -static int -create_changeset(struct fnc_commit_artifact *commit) -{ - fsl_cx *f = fcli_cx(); - fsl_stmt *st = NULL; - fsl_list changeset = fsl_list_empty; - int rc = 0; - - st = fsl_stmt_malloc(); - rc = fsl_cx_prepare(f, st, - "SELECT name, mperm, " - "(SELECT uuid FROM blob WHERE rid=mlink.pid), " - "(SELECT uuid FROM blob WHERE rid=mlink.fid), " - "(SELECT name FROM filename WHERE filename.fnid=mlink.pfnid) " - "FROM mlink JOIN filename ON filename.fnid=mlink.fnid " - "WHERE mlink.mid=%d AND NOT mlink.isaux " - "AND (mlink.fid > 0 " - "OR mlink.fnid NOT IN (SELECT pfnid FROM mlink WHERE mid=%d)) " - "ORDER BY name", commit->rid, commit->rid); - if (rc) - return RC(FSL_RC_DB, "%s", "fsl_cx_prepare"); - - while ((rc = fsl_stmt_step(st)) == FSL_RC_STEP_ROW) { - struct fsl_file_artifact *fdiff = NULL; - const char *path, *oldpath, *olduuid, *uuid; - /* TODO: Parse file mode to display in commit changeset. */ - /* int perm; */ - - path = fsl_stmt_g_text(st, 0, NULL); /* Current filename. */ - //perm = fsl_stmt_g_int32(st, 1); /* File permissions. */ - olduuid = fsl_stmt_g_text(st, 2, NULL); /* UUID before change */ - uuid = fsl_stmt_g_text(st, 3, NULL); /* UUID after change. */ - oldpath = fsl_stmt_g_text(st, 4, NULL); /* Old name, if chngd */ - - fdiff = fsl_malloc(sizeof(struct fsl_file_artifact)); - fdiff->fc = fsl_malloc(sizeof(fsl_card_F)); - *fdiff->fc = fsl_card_F_empty; - fdiff->fc->name = fsl_strdup(path); - if (!uuid) { - fdiff->fc->uuid = fsl_strdup(olduuid); - fdiff->change = FSL_CKOUT_CHANGE_REMOVED; - } else if (!olduuid) { - fdiff->fc->uuid = fsl_strdup(uuid); - fdiff->change = FSL_CKOUT_CHANGE_ADDED; - } else if (oldpath) { - fdiff->fc->uuid = fsl_strdup(uuid); - fdiff->fc->priorName = fsl_strdup(oldpath); - fdiff->change = FSL_CKOUT_CHANGE_RENAMED; - } else { - fdiff->fc->uuid = fsl_strdup(uuid); - fdiff->change = FSL_CKOUT_CHANGE_MOD; - } - fsl_list_append(&changeset, fdiff); - } - - commit->changeset = changeset; - fsl_stmt_finalize(st); - - if (rc == FSL_RC_STEP_DONE) - rc = 0; - - return rc; -} - -static int -write_commit_meta(struct fnc_diff_view_state *s) -{ - char *line = NULL, *st0 = NULL, *st = NULL; - fsl_size_t linelen, idx = 0; - off_t lnoff = 0; - int n, rc = 0; - - if ((n = fprintf(s->f,"%s %s\n", s->selected_commit->type, - s->selected_commit->uuid)) < 0) - goto end; - lnoff += n; - if ((rc = add_line_offset(&s->line_offsets, &s->nlines, lnoff))) - goto end; - - if ((n = fprintf(s->f,"user: %s\n", s->selected_commit->user)) < 0) - goto end; - lnoff += n; - if ((rc = add_line_offset(&s->line_offsets, &s->nlines, lnoff))) - goto end; - - if ((n = fprintf(s->f,"tags: %s\n", s->selected_commit->branch ? - s->selected_commit->branch : "/dev/null")) < 0) - goto end; - lnoff += n; - if ((rc = add_line_offset(&s->line_offsets, &s->nlines, lnoff))) - goto end; - - if ((n = fprintf(s->f,"date: %s\n", - s->selected_commit->timestamp)) < 0) - goto end; - lnoff += n; - if ((rc = add_line_offset(&s->line_offsets, &s->nlines, lnoff))) - goto end; - - fputc('\n', s->f); - ++lnoff; - if ((rc = add_line_offset(&s->line_offsets, &s->nlines, lnoff))) - goto end; - - st0 = fsl_strdup(s->selected_commit->comment); - st = st0; - if (st == NULL) { - RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - while ((line = fnc_strsep(&st, "\n")) != NULL) { - linelen = fsl_strlen(line); - if (linelen >= s->ncols) { - rc = wrapline(line, s->ncols, s, &lnoff); - if (rc) - goto end; - } - else { - if ((n = fprintf(s->f, "%s\n", line)) < 0) - goto end; - lnoff += n; - if ((rc = add_line_offset(&s->line_offsets, &s->nlines, - lnoff))) - goto end; - - } - } - - fputc('\n', s->f); - ++lnoff; - if ((rc = add_line_offset(&s->line_offsets, &s->nlines, lnoff))) - goto end; - - for (idx = 0; idx < s->selected_commit->changeset.used; ++idx) { - char *changeline; - struct fsl_file_artifact *file_change; - - file_change = s->selected_commit->changeset.list[idx]; - - switch (file_change->change) { - case FSL_CKOUT_CHANGE_MOD: - changeline = "[~] "; - break; - case FSL_CKOUT_CHANGE_ADDED: - changeline = "[+] "; - break; - case FSL_CKOUT_CHANGE_RENAMED: - changeline = fsl_mprintf("[>] %s -> ", - file_change->fc->priorName); - break; - case FSL_CKOUT_CHANGE_REMOVED: - changeline = "[-] "; - break; - default: - changeline = "[!] "; - break; - } - if ((n = fprintf(s->f, "%s%s\n", changeline, - file_change->fc->name)) < 0) - goto end; - lnoff += n; - if ((rc = add_line_offset(&s->line_offsets, &s->nlines, lnoff))) - goto end; - } - -end: - free(st0); - free(line); - if (rc) { - free(*&s->line_offsets); - s->line_offsets = NULL; - s->nlines = 0; - } - return rc; -} - -/* - * Wrap long lines at the terminal's available column width. The caller - * must ensure the ncols_avail parameter has taken into account whether the - * screen is currently split, and not mistakenly pass in the curses COLS macro - * without deducting the parent panel's width. This function doesn't break - * words, and will wrap at the end of the last word that can wholly fit within - * the ncols_avail limit. - */ -static int -wrapline(char *line, fsl_size_t ncols_avail, struct fnc_diff_view_state *s, - off_t *lnoff) -{ - char *word; - fsl_size_t wordlen, cursor = 0; - int n = 0, rc = 0; - - while ((word = fnc_strsep(&line, " ")) != NULL) { - wordlen = fsl_strlen(word); - if ((cursor + wordlen) >= ncols_avail) { - fputc('\n', s->f); - ++(*lnoff); - rc = add_line_offset(&s->line_offsets, &s->nlines, - *lnoff); - if (rc) - return rc; - cursor = 0; - } - if ((n = fprintf(s->f, "%s ", word)) < 0) - return rc; - *lnoff += n; - cursor += n; - } - fputc('\n', s->f); - ++(*lnoff); - if ((rc = add_line_offset(&s->line_offsets, &s->nlines, *lnoff))) - return rc; - - return 0; -} - -static int -add_line_offset(off_t **line_offsets, size_t *nlines, off_t off) -{ - off_t *p; - - p = fsl_realloc(*line_offsets, (*nlines + 1) * sizeof(off_t)); - if (p == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_realloc"); - *line_offsets = p; - (*line_offsets)[*nlines] = off; - (*nlines)++; - - return 0; -} - -/* - * Fill the buffer with the differences between commit->uuid and commit->puuid. - * commit->rid (to load into deck d2) is the *this* version, and commit->puuid - * (to be loaded into deck d1) is the version we diff against. Step through the - * deck of F(ile) cards from both versions to determine: (1) if we have new - * files added (i.e., no F card counterpart in d1); (2) files deleted (i.e., no - * F card counterpart in d2); (3) or otherwise the same file (i.e., F card - * exists in both d1 and d2). In cases (1) and (2), we call diff_file_artifact() - * to dump the complete content of the added/deleted file if FSL_DIFF_VERBOSE is - * set, otherwise only diff metatadata will be output. In case (3), if the - * hash (UUID) of each F card is the same, there are no changes; if different, - * both artifacts will be passed to diff_file_artifact() to be diffed. - */ -static int -diff_commit(fsl_buffer *buf, struct fnc_commit_artifact *commit, int diff_flags, - int context, int sbs, struct fnc_pathlist_head *paths) -{ - fsl_cx *f = fcli_cx(); - const fsl_card_F *fc1 = NULL; - const fsl_card_F *fc2 = NULL; - fsl_deck d1 = fsl_deck_empty; - fsl_deck d2 = fsl_deck_empty; - fsl_id_t id1; - int different = 0, rc = 0; - - rc = fsl_deck_load_rid(f, &d2, commit->rid, FSL_SATYPE_CHECKIN); - if (rc) - goto end; - rc = fsl_deck_F_rewind(&d2); - if (rc) - goto end; - - /* - * For the one-and-only special case of repositories, such as the - * canonical fnc, that do not have an "initial empty check-in", we - * proceed with no parent version to diff against. - */ - if (commit->puuid) { - rc = fsl_sym_to_rid(f, commit->puuid, FSL_SATYPE_CHECKIN, &id1); - if (rc) - goto end; - rc = fsl_deck_load_rid(f, &d1, id1, FSL_SATYPE_CHECKIN); - if (rc) - goto end; - rc = fsl_deck_F_rewind(&d1); - if (rc) - goto end; - fsl_deck_F_next(&d1, &fc1); - } - - fsl_deck_F_next(&d2, &fc2); - while (fc1 || fc2) { - const fsl_card_F *a = NULL, *b = NULL; - fsl_ckout_change_e change = FSL_CKOUT_CHANGE_NONE; - bool diff = true; - - if (paths != NULL && !TAILQ_EMPTY(paths)) { - struct fnc_pathlist_entry *pe; - diff = false; - TAILQ_FOREACH(pe, paths, entry) - if (!fsl_strcmp(pe->path, fc1->name) || - !fsl_strcmp(pe->path, fc2->name) || - !fsl_strncmp(pe->path, fc1->name, - pe->pathlen) || !fsl_strncmp(pe->path, - fc2->name, pe->pathlen)) { - diff = true; - break; - } - } - - if (!fc1) /* File added. */ - different = 1; - else if (!fc2) /* File deleted. */ - different = -1; - else /* Same filename in both versions. */ - different = fsl_strcmp(fc1->name, fc2->name); - - if (different) { - if (different > 0) { - b = fc2; - change = FSL_CKOUT_CHANGE_ADDED; - fsl_deck_F_next(&d2, &fc2); - } else if (different < 0) { - a = fc1; - change = FSL_CKOUT_CHANGE_REMOVED; - fsl_deck_F_next(&d1, &fc1); - } - if (diff) - rc = diff_file_artifact(buf, id1, a, - commit->rid, b, change, diff_flags, - context, sbs, commit->diff_type); - } else if (!fsl_uuidcmp(fc1->uuid, fc2->uuid)) { /* No change */ - fsl_deck_F_next(&d1, &fc1); - fsl_deck_F_next(&d2, &fc2); - } else { - change = FSL_CKOUT_CHANGE_MOD; - if (diff) - rc = diff_file_artifact(buf, id1, fc1, - commit->rid, fc2, change, diff_flags, - context, sbs, commit->diff_type); - fsl_deck_F_next(&d1, &fc1); - fsl_deck_F_next(&d2, &fc2); - } - if (rc == FSL_RC_RANGE) { - fsl_buffer_append(buf, - "\nDiff has too many changes\n", -1); - rc = 0; - fsl_cx_err_reset(f); - } else if (rc == FSL_RC_DIFF_BINARY) { - fsl_buffer_append(buf, - "\nBinary files cannot be diffed\n", -1); - rc = 0; - fsl_cx_err_reset(f); - } else if (rc) - goto end; - } -end: - fsl_deck_finalize(&d1); - fsl_deck_finalize(&d2); - return rc; -} - -/* - * Diff local changes on disk in the current checkout against either a previous - * commit or, if no version has been supplied, the current checkout. - * buf output buffer in which diff content is appended - * vid repository database record id of the version to diff against - * diff_flags, context, and sbs are the same parameters as diff_file_artifact() - * nb. This routine is only called with 'fnc diff [hash]'; that is, one or - * zero args—not two—supplied to fnc's diff command line interface. - */ -static int -diff_checkout(fsl_buffer *buf, fsl_id_t vid, int diff_flags, int context, - int sbs, struct fnc_pathlist_head *paths) -{ - fsl_cx *f = fcli_cx(); - fsl_stmt *st = NULL; - fsl_buffer sql, abspath, bminus; - fsl_uuid_str xminus = NULL; - fsl_id_t cid; - int rc = 0; - bool allow_symlinks; - - abspath = bminus = sql = fsl_buffer_empty; - fsl_ckout_version_info(f, &cid, NULL); - /* cid = fsl_config_get_id(f, FSL_CONFDB_CKOUT, 0, "checkout"); */ - /* XXX Already done in cmd_diff(): Load vfile table with local state. */ - /* rc = fsl_vfile_changes_scan(f, cid, */ - /* FSL_VFILE_CKSIG_ENOTFILE & FSL_VFILE_CKSIG_KEEP_OTHERS); */ - /* if (rc) */ - /* return RC(rc, "%s", "fsl_vfile_changes_scan"); */ - - /* - * If a previous version is supplied, load its vfile state to query - * changes. Otherwise query the current checkout state for changes. - */ - if (vid != cid) { - /* Keep vfile ckout state; but unload vid when finished. */ - rc = fsl_vfile_load(f, vid, false, NULL); - if (rc) - goto unload; - fsl_buffer_appendf(&sql, "SELECT v2.pathname, v2.deleted, " - "v2.chnged, v2.rid == 0, v1.rid, v1.islink" - " FROM vfile v1, vfile v2" - " WHERE v1.pathname=v2.pathname AND v1.vid=%d AND v2.vid=%d" - " AND (v2.deleted OR v2.chnged OR v1.mrid != v2.rid)" - " UNION " - "SELECT pathname, 1, 0, 0, 0, islink" - " FROM vfile v1" - " WHERE v1.vid = %d" - " AND NOT EXISTS(SELECT 1 FROM vfile v2" - " WHERE v2.vid = %d AND v2.pathname = v1.pathname)" - " UNION " - "SELECT pathname, 0, 0, 1, 0, islink" - " FROM vfile v2" - " WHERE v2.vid = %d" - " AND NOT EXISTS(SELECT 1 FROM vfile v1" - " WHERE v1.vid = %d AND v1.pathname = v2.pathname)" - " ORDER BY 1", vid, cid, vid, cid, cid, vid); - } else { - fsl_buffer_appendf(&sql, "SELECT pathname, deleted, chnged, " - "rid == 0, rid, islink" - " FROM vfile" - " WHERE vid = %d" - " AND (deleted OR chnged OR rid == 0)" - " ORDER BY pathname", cid); - } - st = fsl_stmt_malloc(); - rc = fsl_cx_prepare(f, st, "%b", &sql); - if (rc) { - rc = RC(rc, "%s", "fsl_cx_prepare"); - goto yield; - } - - while ((rc = fsl_stmt_step(st)) == FSL_RC_STEP_ROW) { - const char *path; - int deleted, changed, added, fid, symlink; - enum fsl_ckout_change_e change; - bool diff = true; - - path = fsl_stmt_g_text(st, 0, NULL); - deleted = fsl_stmt_g_int32(st, 1); - changed = fsl_stmt_g_int32(st, 2); - added = fsl_stmt_g_int32(st, 3); - fid = fsl_stmt_g_int32(st, 4); - symlink = fsl_stmt_g_int32(st, 5); - rc = fsl_file_canonical_name2(f->ckout.dir, path, &abspath, - false); - if (rc) - goto yield; - - if (deleted) - change = FSL_CKOUT_CHANGE_REMOVED; - else if (fsl_file_access(fsl_buffer_cstr(&abspath), F_OK)) - change = FSL_CKOUT_CHANGE_MISSING; - else if (added) { - fid = 0; - change = FSL_CKOUT_CHANGE_ADDED; - } else if (changed == 3) { - fid = 0; - change = FSL_CKOUT_CHANGE_MERGE_ADD; - } else if (changed == 5) { - fid = 0; - change = FSL_CKOUT_CHANGE_INTEGRATE_ADD; - } else - change = FSL_CKOUT_CHANGE_MOD; - - /* - * For changed files of which this checkout is already aware, - * grab their hash to make comparisons. For removed files, if - * diffing against a version other than the current checkout, - * load the version's manifest to parse for known versions of - * said files. If we don't, we risk diffing stale or bogus - * content. Known cases include MISSING, DELETED, and RENAMED - * files, which fossil(1) misses in some instances. - */ - if (fid > 0) - xminus = fsl_rid_to_uuid(f, fid); - else if (vid != cid && !added) { - fsl_deck d = fsl_deck_empty; - const fsl_card_F *cf = NULL; - - rc = fsl_deck_load_rid(f, &d, vid, FSL_SATYPE_CHECKIN); - if (!rc) - rc = fsl_deck_F_rewind(&d); - if (rc) - goto yield; - do { - fsl_deck_F_next(&d, &cf); - if (cf && !fsl_strcmp(cf->name, path)) { - xminus = fsl_strdup(cf->uuid); - if (xminus == NULL) { - RC(FSL_RC_ERROR, "%s", - "fsl_strdup"); - goto yield; - } - fid = fsl_uuid_to_rid(f, xminus); - break; - } - } while (cf); - fsl_deck_finalize(&d); - } - if (!xminus) - xminus = fsl_strdup(NULL_DEVICE); - allow_symlinks = fsl_config_get_bool(f, FSL_CONFDB_REPO, false, - "allow-symlinks"); - if (!symlink != !(fsl_is_symlink(fsl_buffer_cstr(&abspath)) && - allow_symlinks)) { - rc = write_diff_meta(buf, path, xminus, path, - NULL_DEVICE, diff_flags, change); - fsl_buffer_append(buf, "\nSymbolic links and regular " - "files cannot be diffed\n", -1); - if (rc) - goto yield; - continue; - } - if (fid > 0 && change != FSL_CKOUT_CHANGE_ADDED) { - rc = fsl_content_get(f, fid, &bminus); - if (rc) - goto yield; - } else - fsl_buffer_clear(&bminus); - if (paths != NULL && !TAILQ_EMPTY(paths)) { - struct fnc_pathlist_entry *pe; - diff = false; - TAILQ_FOREACH(pe, paths, entry) - if (!fsl_strncmp(pe->path, path, pe->pathlen) - || !fsl_strcmp(pe->path, path)) { - diff = true; - break; - } - } - if (diff) - rc = diff_file(buf, &bminus, path, xminus, - fsl_buffer_cstr(&abspath), change, diff_flags, - context, sbs); - fsl_buffer_reuse(&bminus); - fsl_buffer_reuse(&abspath); - fsl_free(xminus); - xminus = NULL; - if (rc == FSL_RC_RANGE) { - fsl_buffer_append(buf, - "\nDiff has too many changes\n", -1); - rc = 0; - fsl_cx_err_reset(f); - } else if (rc == FSL_RC_DIFF_BINARY) { - fsl_buffer_append(buf, - "\nBinary files cannot be diffed\n", -1); - rc = 0; - fsl_cx_err_reset(f); - } else if (rc) - goto yield; - } - -yield: - fsl_stmt_finalize(st); - fsl_free(xminus); -unload: - fsl_vfile_unload_except(f, cid); - fsl_buffer_clear(&abspath); - fsl_buffer_clear(&bminus); - fsl_buffer_clear(&sql); - return rc; -} - -/* - * Write diff index line and file metadata (i.e., file paths and hashes), which - * signify file addition, removal, or modification. - * buf output buffer in which diff output will be appended - * zminus file name of the file being diffed against - * xminus hex hash of file named zminus - * zplus file name of the file being diffed - * xplus hex hash of the file named zplus - * diff_flags bitwise flags to control the diff - * change enum denoting the versioning change of the file - */ -static int -write_diff_meta(fsl_buffer *buf, const char *zminus, fsl_uuid_str xminus, - const char *zplus, fsl_uuid_str xplus, int diff_flags, - enum fsl_ckout_change_e change) -{ - int rc = 0; - const char *index, *plus, *minus; - - index = zplus ? zplus : (zminus ? zminus : NULL_DEVICE); - - switch (change) { - case FSL_CKOUT_CHANGE_MERGE_ADD: - /* FALL THROUGH */ - case FSL_CKOUT_CHANGE_INTEGRATE_ADD: - /* FALL THROUGH */ - case FSL_CKOUT_CHANGE_ADDED: - minus = NULL_DEVICE; - plus = xplus; - zminus = NULL_DEVICE; - break; - case FSL_CKOUT_CHANGE_MISSING: - /* FALL THROUGH */ - case FSL_CKOUT_CHANGE_REMOVED: - minus = xminus; - plus = NULL_DEVICE; - zplus = NULL_DEVICE; - break; - case FSL_CKOUT_CHANGE_RENAMED: - /* FALL THROUGH */ - case FSL_CKOUT_CHANGE_MOD: - /* FALL THROUGH */ - default: - minus = xminus; - plus = xplus; - break; - } - - if (diff_flags & FSL_DIFF_INVERT) { - const char *tmp = minus; - minus = plus; - plus = tmp; - tmp = zminus; - zminus = zplus; - zplus = tmp; - } - - if ((diff_flags & (FSL_DIFF_SIDEBYSIDE | FSL_DIFF_BRIEF)) == 0) { - rc = fsl_buffer_appendf(buf, "\nIndex: %s\n%.71c\n", index, '='); - if (!rc) - rc = fsl_buffer_appendf(buf, "hash - %s\nhash + %s\n", - minus, plus); - } - if (!rc && (diff_flags & FSL_DIFF_BRIEF) == 0) - rc = fsl_buffer_appendf(buf, "--- %s\n+++ %s\n", zminus, zplus); - - return rc; -} - -/* - * The diff_file_artifact() counterpart that diffs actual files on disk rather - * than file artifacts in the Fossil repository's blob table. - * buf output buffer in which diff output will be appended - * bminus blob containing content of the versioned file being diffed against - * zminus filename of bminus - * xminus hex UUID containing the SHA{1,3} hash of the file named zminus - * abspath absolute path to the file on disk being diffed - * change enum denoting the versioning change of the file - * diff_flags, context, and sbs are the same parameters as diff_file_artifact() - */ -static int -diff_file(fsl_buffer *buf, fsl_buffer *bminus, const char *zminus, - fsl_uuid_str xminus, const char *abspath, enum fsl_ckout_change_e change, - int diff_flags, int context, bool sbs) -{ - fsl_cx *f = fcli_cx(); - fsl_buffer bplus = fsl_buffer_empty; - fsl_buffer xplus = fsl_buffer_empty; - const char *zplus = NULL; - int rc = 0; - bool verbose; - - /* - * If it exists, read content of abspath to diff EXCEPT for the content - * of 'fossil rm FILE' files because they will either: (1) have the same - * content as the versioned file's blob in bminus or (2) have changes. - * As a result, the upcoming call to fsl_diff_text_to_buffer() _will_ - * (1) produce an empty diff or (2) show the differences; neither are - * expected behaviour because the SCM has been instructed to remove the - * file; therefore, the diff should display the versioned file content - * as being entirely removed. With this check, fnc now contrasts the - * behaviour of fossil(1), which produces the abovementioned unexpected - * output described in (1) and (2). - */ - if (change != FSL_CKOUT_CHANGE_REMOVED) { - rc = fsl_ckout_file_content(f, false, abspath, &bplus); - if (rc) - goto end; - /* - * To replicate fossil(1)'s behaviour—where a fossil rm'd file - * will either show as an unchanged or edited rather than a - * removed file with 'fossil diff -v' output—remove the above - * 'if (change != FSL_CKOUT_CHANGE_REMOVED)' from the else - * condition and uncomment the following three lines of code. - */ - /* if (change == FSL_CKOUT_CHANGE_REMOVED && */ - /* !fsl_buffer_compare(bminus, &bplus)) */ - /* fsl_buffer_clear(&bplus); */ - zplus = zminus; - } - - switch (fsl_strlen(xminus)) { - case FSL_STRLEN_K256: - rc = fsl_sha3sum_buffer(&bplus, &xplus); - break; - case FSL_STRLEN_SHA1: - rc = fsl_sha1sum_buffer(&bplus, &xplus); - break; - case NULL_DEVICELEN: - switch (fsl_config_get_int32(f, FSL_CONFDB_REPO, - FSL_HPOLICY_AUTO, "hash-policy")) { - case FSL_HPOLICY_SHA1: - rc = fsl_sha1sum_buffer(&bplus, &xplus); - break; - case FSL_HPOLICY_AUTO: - /* FALL THROUGH */ - case FSL_HPOLICY_SHA3: - /* FALL THROUGH */ - case FSL_HPOLICY_SHA3_ONLY: - rc = fsl_sha3sum_buffer(&bplus, &xplus); - break; - } - break; - default: - RC(FSL_RC_SIZE_MISMATCH, "invalid artifact uuid [%s]", xminus); - goto end; - } - if (rc) - goto end; - - rc = write_diff_meta(buf, zminus, xminus, zplus, fsl_buffer_str(&xplus), - diff_flags, change); - if (rc) - goto end; - - verbose = (diff_flags & FSL_DIFF_VERBOSE) != 0 ? true : false; - if (diff_flags & FSL_DIFF_BRIEF) { - rc = fsl_buffer_compare(bminus, &bplus); - if (!rc) - rc = fsl_buffer_appendf(buf, "CHANGED -> %s\n", zminus); - } else if (verbose || (bminus->used && bplus.used)) { - rc = fsl_diff_text_to_buffer(bminus, &bplus, buf, context, - sbs, diff_flags); - } - -end: - fsl_buffer_clear(&bplus); - fsl_buffer_clear(&xplus); - - return rc; -} - -/* - * Parse the deck of non-checkin commits to present a 'fossil ui' equivalent - * of the corresponding artifact when selected from the timeline. - * TODO: Rename this horrible function name. - */ -static int -diff_non_checkin(fsl_buffer *buf, struct fnc_commit_artifact *commit, - int diff_flags, int context, int sbs) -{ - fsl_cx *f = fcli_cx(); - fsl_buffer wiki = fsl_buffer_empty; - fsl_buffer pwiki = fsl_buffer_empty; - fsl_id_t prid = 0; - fsl_size_t idx; - int rc = 0; - - fsl_deck *d = NULL; - d = fsl_deck_malloc(); - if (d == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_deck_malloc"); - - fsl_deck_init(f, d, FSL_SATYPE_ANY); - if ((rc = fsl_deck_load_rid(f, d, commit->rid, FSL_SATYPE_ANY))) - goto end; - - /* - * Present ticket commits as a series of field: value tuples as per - * the Fossil UI /info/UUID view. - */ - if (d->type == FSL_SATYPE_TICKET) { - for (idx = 0; idx < d->J.used; ++idx) { - fsl_card_J *ticket = d->J.list[idx]; - bool icom = !fsl_strncmp(ticket->field, "icom", 4); - fsl_buffer_appendf(buf, "%d. %s:%s%s%c\n", idx + 1, - ticket->field, icom ? "\n\n" : " ", ticket->value, - icom ? '\n' : ' '); - } - goto end; - } - - if (d->type == FSL_SATYPE_CONTROL) { - for (idx = 0; idx < d->T.used; ++idx) { - fsl_card_T *ctl = d->T.list[idx]; - fsl_buffer_appendf(buf, "Tag %d ", idx + 1); - switch (ctl->type) { - case FSL_TAGTYPE_CANCEL: - fsl_buffer_append(buf, "[CANCEL]", -1); - break; - case FSL_TAGTYPE_ADD: - fsl_buffer_append(buf, "[ADD]", -1); - break; - case FSL_TAGTYPE_PROPAGATING: - fsl_buffer_append(buf, "[PROPAGATE]", -1); - break; - default: - break; - } - if (ctl->uuid) - fsl_buffer_appendf(buf, "\ncheckin %s", - ctl->uuid); - fsl_buffer_appendf(buf, "\n%s", ctl->name); - if (!fsl_strcmp(ctl->name, "branch")) - commit->branch = fsl_strdup(ctl->value); - if (ctl->value) - fsl_buffer_appendf(buf, " -> %s", ctl->value); - fsl_buffer_append(buf, "\n\n", 2); - } - goto end; - } - /* - * If neither a ticket nor control artifact, we assume it's a wiki, so - * check if it has a parent commit to diff against. If not, append the - * entire wiki card content. - */ - fsl_buffer_append(&wiki, d->W.mem, d->W.used); - if (commit->puuid == NULL) { - if (d->P.used > 0) - commit->puuid = fsl_strdup(d->P.list[0]); - else { - fsl_buffer_copy(&wiki, buf); - goto end; - } - } - - /* Diff the artifacts if a parent is found. */ - if ((rc = fsl_sym_to_rid(f, commit->puuid, FSL_SATYPE_ANY, &prid))) - goto end; - if ((rc = fsl_deck_load_rid(f, d, prid, FSL_SATYPE_ANY))) - goto end; - fsl_buffer_append(&pwiki, d->W.mem, d->W.used); - - rc = fsl_diff_text_to_buffer(&pwiki, &wiki, buf, context, sbs, - diff_flags); - - /* If a technote, provide the full content after its diff. */ - if (d->type == FSL_SATYPE_TECHNOTE) - fsl_buffer_appendf(buf, "\n---\n\n%s", wiki.mem); - -end: - fsl_buffer_clear(&wiki); - fsl_buffer_clear(&pwiki); - fsl_deck_finalize(d); - return rc; -} - -/* - * Compute the differences between two repository file artifacts to produce the - * set of changes necessary to convert one into the other. - * buf output buffer in which diff output will be appended - * vid1 repo record id of the version from which artifact a belongs - * a file artifact being diffed against - * vid2 repo record id of the version from which artifact b belongs - * b file artifact being diffed - * change enum denoting the versioning change of the file - * diff_flags bitwise flags to control the diff - * context the number of context lines to surround changes - * sbs number of columns in which to display each side-by-side diff - */ -static int -diff_file_artifact(fsl_buffer *buf, fsl_id_t vid1, const fsl_card_F *a, - fsl_id_t vid2, const fsl_card_F *b, enum fsl_ckout_change_e change, - int diff_flags, int context, int sbs, enum fnc_diff_type diff_type) -{ - fsl_cx *f = fcli_cx(); - fsl_stmt stmt = fsl_stmt_empty; - fsl_buffer fbuf1 = fsl_buffer_empty; - fsl_buffer fbuf2 = fsl_buffer_empty; - char *zminus0 = NULL, *zplus0 = NULL; - const char *zplus = NULL, *zminus = NULL; - fsl_uuid_str xplus0 = NULL, xminus0 = NULL; - fsl_uuid_str xplus = NULL, xminus = NULL; - int rc = 0; - bool verbose; - - assert(vid1 != vid2); - assert(vid2 > 0 && - "local checkout should be diffed with diff_checkout()"); - - fbuf2.used = fbuf1.used = 0; - - if (a) { - rc = fsl_card_F_content(f, a, &fbuf1); - if (rc) - goto end; - zminus = a->name; - xminus = a->uuid; - } else if (diff_type == FNC_DIFF_BLOB) { - rc = fsl_cx_prepare(f, &stmt, - "SELECT name FROM filename, mlink " - "WHERE filename.fnid=mlink.fnid AND mlink.fid = %d", vid1); - if (rc) { - rc = RC(FSL_RC_DB, "%s %d", "fsl_cx_prepare", vid1); - goto end; - } - rc = fsl_stmt_step(&stmt); - if (rc == FSL_RC_STEP_ROW) { - rc = 0; - zminus0 = fsl_strdup(fsl_stmt_g_text(&stmt, 0, NULL)); - zminus = zminus0; - } else if (rc == FSL_RC_STEP_DONE) - rc = 0; - else if (rc) { - rc = RC(rc, "%s", "fsl_stmt_step"); - goto end; - } - xminus0 = fsl_rid_to_uuid(f, vid1); - xminus = xminus0; - fsl_stmt_finalize(&stmt); - fsl_content_get(f, vid1, &fbuf1); - } - if (b) { - rc = fsl_card_F_content(f, b, &fbuf2); - if (rc) - goto end; - zplus = b->name; - xplus = b->uuid; - } else if (diff_type == FNC_DIFF_BLOB) { - rc = fsl_cx_prepare(f, &stmt, - "SELECT name FROM filename, mlink " - "WHERE filename.fnid=mlink.fnid AND mlink.fid = %d", vid2); - if (rc) { - rc = RC(FSL_RC_DB, "%s %d", "fsl_cx_prepare", vid2); - goto end; - } - rc = fsl_stmt_step(&stmt); - if (rc == FSL_RC_STEP_ROW) { - rc = 0; - zplus0 = fsl_strdup(fsl_stmt_g_text(&stmt, 0, NULL)); - zplus = zplus0; - } else if (rc == FSL_RC_STEP_DONE) - rc = 0; - else if (rc) { - rc = RC(rc, "%s", "fsl_stmt_step"); - goto end; - } - xplus0 = fsl_rid_to_uuid(f, vid2); - xplus = xplus0; - fsl_stmt_finalize(&stmt); - fsl_content_get(f, vid2, &fbuf2); - } - - rc = write_diff_meta(buf, zminus, xminus, zplus, xplus, diff_flags, - change); - verbose = (diff_flags & FSL_DIFF_VERBOSE) != 0 ? true : false; - if (verbose || (a && b)) - rc = fsl_diff_text_to_buffer(&fbuf1, &fbuf2, buf, context, sbs, - diff_flags); - if (rc) - RC(rc, "%s: fsl_diff_text_to_buffer\n" - " -> %s [%s]\n -> %s [%s]", fsl_rc_cstr(rc), - a ? a->name : NULL_DEVICE, a ? a->uuid : NULL_DEVICE, - b ? b->name : NULL_DEVICE, b ? b->uuid : NULL_DEVICE); -end: - fsl_free(zminus0); - fsl_free(zplus0); - fsl_free(xminus0); - fsl_free(xplus0); - fsl_buffer_clear(&fbuf1); - fsl_buffer_clear(&fbuf2); - return rc; -} - -static int -show_diff(struct fnc_view *view) -{ - struct fnc_diff_view_state *s = &view->state.diff; - char *headln, *id2, *id1 = NULL; - - /* Some diffs (e.g., technote, tag) have no parent hash to display. */ - id1 = fsl_strdup(s->id1 ? s->id1 : "/dev/null"); - if (id1 == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - - /* - * If diffing the work tree, we have no hash to display for it. - * XXX Display "work tree" or "checkout" or "/dev/null" for clarity? - */ - id2 = fsl_strdup(s->id2 ? s->id2 : ""); - if (id2 == NULL) { - fsl_free(id1); - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - } - - if ((headln = fsl_mprintf("diff %.40s %.40s", id1, id2)) == NULL) { - fsl_free(id1); - fsl_free(id2); - return RC(FSL_RC_RANGE, "%s", "fsl_mprintf"); - } - - fsl_free(id1); - fsl_free(id2); - return write_diff(view, headln); -} - -static int -write_diff(struct fnc_view *view, char *headln) -{ - struct fnc_diff_view_state *s = &view->state.diff; - regmatch_t *regmatch = &view->regmatch; - struct fnc_colour *c = NULL; - wchar_t *wcstr; - char *line; - size_t linesz = 0; - ssize_t linelen; - off_t line_offset; - int wstrlen; - int max_lines = view->nlines; - int nlines = s->nlines; - int rc = 0, nprintln = 0; - int match = -1; - - line_offset = s->line_offsets[s->first_line_onscreen - 1]; - if (fseeko(s->f, line_offset, SEEK_SET)) - return RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), "%s", "fseeko"); - - werase(view->window); - - if (headln) { - if ((line = fsl_mprintf("[%d/%d] %s", (s->first_line_onscreen - - 1 + s->current_line), nlines, headln)) == NULL) - return RC(FSL_RC_RANGE, "%s", "fsl_mprintf"); - rc = formatln(&wcstr, &wstrlen, line, view->ncols, 0); - fsl_free(line); - fsl_free(headln); - if (rc) - return rc; - - if (screen_is_shared(view)) - wstandout(view->window); - waddwstr(view->window, wcstr); - fsl_free(wcstr); - wcstr = NULL; - if (screen_is_shared(view)) - wstandend(view->window); - if (wstrlen <= view->ncols - 1) - waddch(view->window, '\n'); - - if (max_lines <= 1) - return rc; - --max_lines; - } - - s->eof = false; - line = NULL; - while (max_lines > 0 && nprintln < max_lines) { - linelen = getline(&line, &linesz, s->f); - if (linelen == -1) { - if (feof(s->f)) { - s->eof = true; - break; - } - fsl_free(line); - RC(ferror(s->f) ? fsl_errno_to_rc(errno, FSL_RC_IO) : - FSL_RC_IO, "%s", "getline"); - return rc; - } - - if (s->colour && (match = fsl_list_index_of(&s->colours, line, - match_line)) != -1) - c = s->colours.list[match]; - if (c) - wattr_on(view->window, COLOR_PAIR(c->scheme), NULL); - if (s->first_line_onscreen + nprintln == s->matched_line && - regmatch->rm_so >= 0 && regmatch->rm_so < regmatch->rm_eo) { - rc = write_matched_line(&wstrlen, line, view->ncols, 0, - view->window, regmatch); - if (rc) { - fsl_free(line); - return rc; - } - } else { - rc = formatln(&wcstr, &wstrlen, line, view->ncols, 0); - if (rc) { - fsl_free(line); - return rc; - } - waddwstr(view->window, wcstr); - fsl_free(wcstr); - wcstr = NULL; - } - if (c) { - wattr_off(view->window, COLOR_PAIR(c->scheme), NULL); - c = NULL; - } - if (wstrlen <= view->ncols - 1) - waddch(view->window, '\n'); - ++nprintln; - } - fsl_free(line); - if (nprintln >= 1) - s->last_line_onscreen = s->first_line_onscreen + (nprintln - 1); - else - s->last_line_onscreen = s->first_line_onscreen; - - draw_vborder(view); - - if (s->eof) { - while (nprintln < view->nlines) { - waddch(view->window, '\n'); - ++nprintln; - } - - wstandout(view->window); - waddstr(view->window, "(END)"); - wstandend(view->window); - } - - return rc; -} - -static int -match_line(const void *ln, const void *key) -{ - struct fnc_colour *c = (struct fnc_colour *)key; - const char *line = ln; - - return regexec(&c->regex, line, 0, NULL, 0); -} - -static bool -screen_is_shared(struct fnc_view *view) -{ - if (view_is_parent(view)) { - if (view->child == NULL || view->child->active || - !screen_is_split(view->child)) - return false; - } else if (!screen_is_split(view)) - return false; - - return view->active; -} - -static int -view_is_parent(struct fnc_view *view) -{ - return view->parent == NULL; -} - -static int -screen_is_split(struct fnc_view *view) -{ - return view->start_col > 0; -} - -static int -write_matched_line(int *col_pos, const char *line, int ncols_avail, - int start_column, WINDOW *window, regmatch_t *regmatch) -{ - wchar_t *wcstr; - char *s; - int wstrlen; - int rc = 0; - - *col_pos = 0; - - /* Copy the line up to the matching substring & write it to screen. */ - s = fsl_strndup(line, regmatch->rm_so); - if (s == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_strndup"); - - rc = formatln(&wcstr, &wstrlen, s, ncols_avail, start_column); - if (rc) { - free(s); - return rc; - } - waddwstr(window, wcstr); - free(wcstr); - free(s); - ncols_avail -= wstrlen; - *col_pos += wstrlen; - - /* If not EOL, copy matching string & write to screen with highlight. */ - if (ncols_avail > 0) { - s = fsl_strndup(line + regmatch->rm_so, - regmatch->rm_eo - regmatch->rm_so); - if (s == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strndup"); - free(s); - return rc; - } - rc = formatln(&wcstr, &wstrlen, s, ncols_avail, start_column); - if (rc) { - free(s); - return rc; - } - wattr_on(window, A_REVERSE, NULL); - waddwstr(window, wcstr); - wattr_off(window, A_REVERSE, NULL); - free(wcstr); - free(s); - ncols_avail -= wstrlen; - *col_pos += wstrlen; - } - - /* Write the rest of the line if not yet at EOL. */ - if (ncols_avail > 0 && fsl_strlen(line) > (fsl_size_t)regmatch->rm_eo) { - rc = formatln(&wcstr, &wstrlen, line + regmatch->rm_eo, - ncols_avail, start_column); - if (rc) - return rc; - waddwstr(window, wcstr); - free(wcstr); - *col_pos += wstrlen; - } - - return rc; -} - -static void -draw_vborder(struct fnc_view *view) -{ - const struct fnc_view *view_above; - char *codeset = nl_langinfo(CODESET); - PANEL *panel; - - if (view->parent) - draw_vborder(view->parent); - - panel = panel_above(view->panel); - if (panel == NULL) - return; - - view_above = panel_userptr(panel); - mvwvline(view->window, view->start_ln, view_above->start_col - 1, - (strcmp(codeset, "UTF-8") == 0) ? ACS_VLINE : '|', view->nlines); -#ifdef __linux__ - wnoutrefresh(view->window); -#endif -} - -static int -diff_input_handler(struct fnc_view **new_view, struct fnc_view *view, int ch) -{ - struct fnc_diff_view_state *s = &view->state.diff; - struct fnc_tl_view_state *tlstate; - struct commit_entry *previous_selection; - char *line = NULL; - ssize_t linelen; - size_t linesz = 0; - int i, rc = 0; - bool tl_down = false; - - switch (ch) { - case KEY_DOWN: - case 'j': - if (!s->eof) - ++s->first_line_onscreen; - break; - case KEY_NPAGE: - case CTRL('f'): - case ' ': - if (s->eof) - break; - i = 0; - while (!s->eof && i++ < view->nlines - 1) { - linelen = getline(&line, &linesz, s->f); - ++s->first_line_onscreen; - if (linelen == -1) { - if (feof(s->f)) - s->eof = true; - else - RC(ferror(s->f) ? - fsl_errno_to_rc(errno, FSL_RC_IO) : - FSL_RC_IO, "%s", "getline"); - break; - } - } - free(line); - break; - case KEY_END: - case 'G': - if (s->eof) - break; - s->first_line_onscreen = (s->nlines - view->nlines) + 2; - s->eof = true; - break; - case KEY_UP: - case 'k': - if (s->first_line_onscreen > 1) - --s->first_line_onscreen; - break; - case KEY_PPAGE: - case CTRL('b'): - if (s->first_line_onscreen == 1) - break; - i = 0; - while (i++ < view->nlines - 1 && s->first_line_onscreen > 1) - --s->first_line_onscreen; - break; - case 'g': - if (!fnc_home(view)) - break; - /* FALL THROUGH */ - case KEY_HOME: - s->first_line_onscreen = 1; - break; - case 'c': - case 'i': - case 'v': - case 'w': - if (ch == 'c') - s->colour = !s->colour; - if (ch == 'i') - s->diff_flags ^= FSL_DIFF_INVERT; - if (ch == 'v') - s->diff_flags ^= FSL_DIFF_VERBOSE; - if (ch == 'w') - s->diff_flags ^= FSL_DIFF_IGNORE_ALLWS; - wclear(view->window); - s->first_line_onscreen = 1; - s->last_line_onscreen = view->nlines; - show_diff_status(view); - rc = create_diff(s); - break; - case '-': - case '_': - if (s->context > 0) { - --s->context; - show_diff_status(view); - rc = create_diff(s); - if (s->first_line_onscreen + view->nlines - 1 > - (int)s->nlines) { - s->first_line_onscreen = 1; - s->last_line_onscreen = view->nlines; - } - } - break; - case '+': - case '=': - if (s->context < DIFF_MAX_CTXT) { - ++s->context; - show_diff_status(view); - rc = create_diff(s); - } - break; - case CTRL('j'): - case '>': - case '.': - case 'J': - tl_down = true; - /* FALL THROUGH */ - case CTRL('k'): - case '<': - case ',': - case 'K': - if (s->timeline_view == NULL) - break; - tlstate = &s->timeline_view->state.timeline; - previous_selection = tlstate->selected_commit; - - if ((rc = tl_input_handler(NULL, s->timeline_view, - tl_down ? KEY_DOWN : KEY_UP))) - break; - - if (previous_selection == tlstate->selected_commit) - break; - - if ((rc = set_selected_commit(s, tlstate->selected_commit))) - break; - - s->first_line_onscreen = 1; - s->last_line_onscreen = view->nlines; - - show_diff_status(view); - rc = create_diff(s); - break; - default: - break; - } - - return rc; -} - -static int -set_selected_commit(struct fnc_diff_view_state *s, struct commit_entry *entry) -{ - fsl_free(s->id2); - s->id2 = fsl_strdup(entry->commit->uuid); - if (s->id2 == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - fsl_free(s->id1); - s->id1 = entry->commit->puuid ? fsl_strdup(entry->commit->puuid) : NULL; - s->selected_commit = entry->commit; - - return 0; -} - -static int -diff_search_init(struct fnc_view *view) -{ - struct fnc_diff_view_state *s = &view->state.diff; - - s->matched_line = 0; - return 0; -} - -static int -diff_search_next(struct fnc_view *view) -{ - struct fnc_diff_view_state *s = &view->state.diff; - char *line = NULL; - ssize_t linelen; - size_t linesz = 0; - int start_ln; - - if (view->searching == SEARCH_DONE) { - view->search_status = SEARCH_CONTINUE; - return 0; - } - - if (s->matched_line) { - if (view->searching == SEARCH_FORWARD) - start_ln = s->matched_line + 1; - else - start_ln = s->matched_line - 1; - } else { - if (view->searching == SEARCH_FORWARD) - start_ln = 1; - else - start_ln = s->nlines; - } - - while (1) { - off_t offset; - - if (start_ln <= 0 || start_ln > (int)s->nlines) { - if (s->matched_line == 0) { - view->search_status = SEARCH_CONTINUE; - break; - } - - if (view->searching == SEARCH_FORWARD) - start_ln = 1; - else - start_ln = s->nlines; - } - - offset = s->line_offsets[start_ln - 1]; - if (fseeko(s->f, offset, SEEK_SET) != 0) { - free(line); - return RC(fsl_errno_to_rc(errno, FSL_RC_IO), - "%s", "fseeko"); - } - linelen = getline(&line, &linesz, s->f); - if (linelen != -1 && regexec(&view->regex, line, 1, - &view->regmatch, 0) == 0) { - view->search_status = SEARCH_CONTINUE; - s->matched_line = start_ln; - break; - } - if (view->searching == SEARCH_FORWARD) - ++start_ln; - else - --start_ln; - } - free(line); - - if (s->matched_line) { - s->first_line_onscreen = s->matched_line; - s->current_line = 1; - } - - return 0; -} - -static int -close_diff_view(struct fnc_view *view) -{ - struct fnc_diff_view_state *s = &view->state.diff; - struct fsl_list_state st = { FNC_COLOUR_OBJ }; - int rc = 0; - - if (s->f && fclose(s->f) == EOF) - rc = RC(fsl_errno_to_rc(errno, FSL_RC_IO), "%s", "fclose"); - fsl_free(s->id1); - s->id1 = NULL; - fsl_free(s->id2); - s->id2 = NULL; - fsl_free(s->line_offsets); - fsl_list_clear(&s->colours, fsl_list_object_free, &st); - s->line_offsets = NULL; - s->nlines = 0; - return rc; -} - -static void -fnc_resizeterm(void) -{ - struct winsize size; - int cols, lines; - - if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &size) < 0) { - cols = 80; - lines = 24; - } else { - cols = size.ws_col; - lines = size.ws_row; - } - resize_term(lines, cols); -} - -static int -view_resize(struct fnc_view *view) -{ - int nlines, ncols; - - if (view->lines > LINES) - nlines = view->nlines - (view->lines - LINES); - else - nlines = view->nlines + (LINES - view->lines); - - if (view->cols > COLS) - ncols = view->ncols - (view->cols - COLS); - else - ncols = view->ncols + (COLS - view->cols); - - if (wresize(view->window, nlines, ncols) == ERR) - return RC(FSL_RC_ERROR, "%s", "wresize"); - if (replace_panel(view->panel, view->window) == ERR) - return RC(FSL_RC_ERROR, "%s", "replace_panel"); - wclear(view->window); - - view->nlines = nlines; - view->ncols = ncols; - view->lines = LINES; - view->cols = COLS; - - if (view->child) { - view->child->start_col = view_split_start_col(view->start_col); - if (view->child->start_col == 0) { - make_fullscreen(view->child); - if (view->child->active) - show_panel(view->child->panel); - else - show_panel(view->panel); - } else { - make_splitscreen(view->child); - show_panel(view->child->panel); - } - } - - return 0; -} - -/* - * Consume repeatable arguments containing artifact type values used in - * constructing the SQL query to generate commit records of the specified - * type for the timeline. TODO: Enhance this to generalise processing of - * various repeatable arguments--paths, usernames, branches, etc.--so we - * can filter on multiples of these values. - */ -static int -fcli_flag_type_arg_cb(fcli_cliflag const *v) -{ - if (fnc_init.filter_types->nitems) - fnc_init.filter_types->values = - fsl_realloc(fnc_init.filter_types->values, - (fnc_init.filter_types->nitems + 1) * sizeof(char *)); - fnc_init.filter_types->values[fnc_init.filter_types->nitems++] = - *((const char **)v->flagValue); - - return FCLI_RC_FLAG_AGAIN; -} - -static void -sigwinch_handler(int sig) -{ - if (sig == SIGWINCH) { - struct winsize winsz; - - ioctl(0, TIOCGWINSZ, &winsz); - rec_sigwinch = 1; - } -} - -static void -sigpipe_handler(int sig) -{ - struct sigaction sact; - int e; - - rec_sigpipe = 1; - memset(&sact, 0, sizeof(sact)); - sact.sa_handler = SIG_IGN; - sact.sa_flags = SA_RESTART; - e = sigaction(SIGPIPE, &sact, NULL); - if (e) - err(1, "SIGPIPE"); -} - -static void -sigcont_handler(int sig) -{ - rec_sigcont = 1; -} - -__dead static void -usage(void) -{ - /* - * It looks like the fsl_cx f member of the ::fcli singleton has - * already been cleaned up by the time this wrapper is called from - * fcli_help() after hijacking the process whenever the '--help' - * argument is passsed on the command line, so we can't use the - * f->output fsl_outputer implementation as we would like. - */ - /* fsl_cx *f = fcli_cx(); */ - /* f->output = fsl_outputer_FILE; */ - /* f->output.state.state = (fnc_init.err == true) ? stderr : stdout; */ - FILE *f = fnc_init.err ? stderr : stdout; - size_t idx = 0; - - endwin(); - - /* If a command was passed on the CLI, output its corresponding help. */ - if (fnc_init.cmdarg) - for (idx = 0; idx < nitems(fnc_init.cmd_args); ++idx) { - fcli_command cmd = fnc_init.cmd_args[idx]; - if (!fsl_strcmp(fnc_init.cmdarg, cmd.name) || - fcli_cmd_aliascmp(&cmd, fnc_init.cmdarg)) { - fcli_command_help(&cmd, true, true); - exit(fcli_end_of_main(fnc_init.err)); - } - } - - /* Otherwise, output help/usage for all commands. */ - fcli_command_help(fnc_init.cmd_args, true, false); - fsl_fprintf(f, " note: %s " - "with no args defaults to the timeline command.\n\n", - fcli_progname()); - - exit(fcli_end_of_main(fnc_init.err)); -} - -static void -usage_timeline(void) -{ - fsl_fprintf(fnc_init.err ? stderr : stdout, - " usage: %s timeline [-C|--no-colour] [-T tag] [-b branch] " - "[-c commit] [-h|--help] [-n n] [-t type] [-u user] [-z|--utc] " - "[path]\n" - " e.g.: %s timeline --type ci -u jimmy src/frobnitz.c\n\n", - fcli_progname(), fcli_progname()); -} - -static void -usage_diff(void) -{ - fsl_fprintf(fnc_init.err ? stderr : stdout, - " usage: %s diff [-C|--no-colour] [-h|--help] [-i|--invert]" - " [-q|--quiet] [-w|--whitespace] [-x|--context n] " - "[artifact1 [artifact2]] [path ...]\n " - "e.g.: %s diff --context 3 d34db33f c0ff33 src/*.c\n\n", - fcli_progname(), fcli_progname()); -} - -static void -usage_tree(void) -{ - fsl_fprintf(fnc_init.err ? stderr : stdout, - " usage: %s tree [-C|--no-colour] [-c commit] [-h|--help] [path]\n" - " e.g.: %s tree -c d34dc0d3\n\n" , - fcli_progname(), fcli_progname()); -} - -static void -usage_blame(void) -{ - fsl_fprintf(fnc_init.err ? stderr : stdout, - " usage: %s blame [-C|--no-colour] [-c commit [-r]] [-h|--help] " - "[-n n] path\n" - " e.g.: %s blame -c d34db33f src/foo.c\n\n" , - fcli_progname(), fcli_progname()); -} - -static void -usage_branch(void) -{ - fsl_fprintf(fnc_init.err ? stderr : stdout, - " usage: %s branch [-C|--no-colour] [-a|--after date] " - "[-b|--before date] [-c|--closed] [-h|--help] [-o|--open] " - "[-p|--no-private] [-r|--reverse] [-s|--sort order] [glob]\n" - " e.g.: %s branch -b 2020-10-10\n\n" , - fcli_progname(), fcli_progname()); -} - -static int -cmd_diff(fcli_command const *argv) -{ - fsl_cx *f = fcli_cx(); - struct fnc_view *view; - struct fnc_commit_artifact *commit = NULL; - struct fnc_pathlist_head paths; - struct fnc_pathlist_entry *pe; - fsl_deck d = fsl_deck_empty; - fsl_stmt *q = NULL; - const char *artifact1 = NULL, *artifact2 = NULL; - char *path0 = NULL; - fsl_id_t prid = -1, rid = -1; - int context = DIFF_DEF_CTXT, rc = 0; - unsigned short blob = 0; - enum fnc_diff_type diff_type; - bool showmeta = false; - - rc = fcli_process_flags(argv->flags); - if (rc || (rc = fcli_has_unused_flags(false))) - return rc; - - TAILQ_INIT(&paths); - - /* - * To provide an intuitive UI, use some magic. First, if there's an arg - * and it's a symbolic checkin name, take as a checkin artifact. Repeat - * for the next arg. If just one is a checkin, diff changes on disk - * against it. If neither are checkins, diff changes on disk against the - * current checkout. If both are checkins, diff against eachother. Treat - * any non-symbol args as paths and try map to a valid repo path or F - * card in the checkin(s) deck(s). It's tricky, but provides a smart UI: - * fnc diff f1 f2 ... -> diff f{1,2,...} on disk against current ckout - * fnc diff sym3 f1 -> diff f1 on disk against f1 found in checkin sym3 - * fnc diff sym1 sym2 f1 f2 -> diff f{1,2} between checkins sym1 & sym2 - */ - if (!fsl_sym_to_rid(f, fcli_next_arg(false), FSL_SATYPE_ANY, &prid)) { - artifact1 = fcli_next_arg(true); - if (!fsl_rid_is_a_checkin(f, prid)) { - if (fsl_rid_to_artifact_uuid(f, prid, FSL_SATYPE_ANY) - != NULL) { - rc = RC(FSL_RC_TYPE, - "artifact [%s] not resolvable to a checkin", - artifact1); - goto end; - } - ++blob; - } - if (!fsl_sym_to_rid(f, fcli_next_arg(false), FSL_SATYPE_ANY, - &rid)) { - artifact2 = fcli_next_arg(true); - diff_type = FNC_DIFF_COMMIT; - if (!fsl_rid_is_a_checkin(f, rid)) { - if (fsl_rid_to_artifact_uuid(f, - prid, FSL_SATYPE_ANY) != NULL) { - rc = RC(FSL_RC_TYPE, "artifact [%s] " - "not resolvable to a checkin", - artifact2); - goto end; - } - ++blob; - } - } - } - if (fcli_error()->code == FSL_RC_NOT_FOUND) { - fcli_err_reset(); /* If args aren't symbols, treat as paths. */ - rc = 0; - } - if (blob == 2) - diff_type = FNC_DIFF_BLOB; - if (!artifact1 && diff_type != FNC_DIFF_BLOB) { - artifact1 = "current"; - rc = fsl_sym_to_rid(f, artifact1, FSL_SATYPE_CHECKIN, &prid); - if (rc || prid < 0) { - rc = RC(rc, "%s", "fsl_sym_to_rid"); - goto end; - } - } - if (!artifact2 && diff_type != FNC_DIFF_BLOB) { - diff_type = FNC_DIFF_CKOUT; - fsl_ckout_version_info(f, &rid, NULL); - if ((rc = fsl_ckout_changes_scan(f))) - return RC(rc, "%s", "fsl_ckout_changes_scan"); - if (!fsl_strcmp(artifact1, "current") && - !fsl_ckout_has_changes(f)) { - fsl_fprintf(stdout, "No local changes.\n"); - return rc; - } - } - while (fcli_next_arg(false) && diff_type != FNC_DIFF_BLOB) { - struct fnc_pathlist_entry *ins; - char *path, *path_to_diff; - rc = map_repo_path(&path0); - path = path0; - while (path[0] == '/') - ++path; - if (rc) { - if (rc != FSL_RC_NOT_FOUND || - (!fsl_strcmp(artifact1, "current") && !artifact2)) - goto end; - rc = 0; - fcli_err_reset(); - /* Path may be valid in tree of specified commit(s). */ - const fsl_card_F *cf = NULL; - rc = fsl_deck_load_sym(f, &d, artifact1, - FSL_SATYPE_CHECKIN); - if (rc) - goto end; - cf = fsl_deck_F_search(&d, path); - if (cf == NULL) { - if (!artifact2) { - rc = RC(FSL_RC_NOT_FOUND, - "'%s' not found in tree [%s]", path, - artifact1); - goto end; - } - fsl_deck_finalize(&d); - rc = fsl_deck_load_sym(f, &d, artifact2, - FSL_SATYPE_CHECKIN); - if (rc) - goto end; - cf = fsl_deck_F_search(&d, path); - if (cf == NULL) { - rc = RC(FSL_RC_NOT_FOUND, - "'%s' not found in trees [%s] [%s]", - path, artifact1, artifact2); - goto end; - } - } - } - path_to_diff = fsl_strdup(path); - if (path_to_diff == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - rc = fnc_pathlist_insert(&ins, &paths, path_to_diff, NULL); - if (rc || ins == NULL /* Duplicate path. */) - fsl_free(path_to_diff); - if (rc) - goto end; - } - - if (diff_type != FNC_DIFF_BLOB && diff_type != FNC_DIFF_CKOUT) { - q = fsl_stmt_malloc(); - rc = commit_builder(&commit, rid, q); - if (rc) - goto end; - if (commit->prid == prid) - showmeta = true; - else { - fsl_free(commit->puuid); - commit->prid = prid; - commit->puuid = fsl_rid_to_uuid(f, prid); - } - } else { - commit = calloc(1, sizeof(*commit)); - if (commit == NULL) { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), - "%s", "calloc"); - goto end; - } - commit->prid = prid; - commit->rid = rid; - commit->puuid = fsl_rid_to_uuid(f, prid); - commit->uuid = fsl_rid_to_uuid(f, rid); - commit->type = fsl_strdup("blob"); - commit->diff_type = diff_type; - } - - rc = init_curses(); - if (rc) - goto end; - - if (fnc_init.context) { - if ((rc = strtonumcheck(&context, fnc_init.context, INT_MIN, - INT_MAX))) - goto end; - context = MIN(DIFF_MAX_CTXT, context); - } - - view = view_open(0, 0, 0, 0, FNC_VIEW_DIFF); - if (view == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "view_open"); - goto end; - } - - rc = open_diff_view(view, commit, context, fnc_init.ws, - fnc_init.invert, !fnc_init.quiet, NULL, showmeta, &paths); - if (!rc) - rc = view_loop(view); -end: - fsl_free(path0); - fsl_deck_finalize(&d); - fsl_stmt_finalize(q); - if (commit) - fnc_commit_artifact_close(commit); - TAILQ_FOREACH(pe, &paths, entry) - free((char *)pe->path); - fnc_pathlist_free(&paths); - return rc; -} - -static int -browse_commit_tree(struct fnc_view **new_view, int start_col, - struct commit_entry *entry, const char *path) -{ - struct fnc_view *tree_view; - int rc = 0; - - tree_view = view_open(0, 0, 0, start_col, FNC_VIEW_TREE); - if (tree_view == NULL) - return RC(FSL_RC_ERROR, "%s", "view_open"); - - rc = open_tree_view(tree_view, path, entry->commit->rid); - if (rc) - return rc; - - *new_view = tree_view; - - return rc; -} - -static int -cmd_tree(fcli_command const *argv) -{ - fsl_cx *f = fcli_cx(); - struct fnc_view *view; - char *path = NULL; - fsl_id_t rid; - int rc = 0; - - rc = fcli_process_flags(argv->flags); - if (rc || (rc = fcli_has_unused_flags(false))) - goto end; - - rc = map_repo_path(&path); - if (rc) - goto end; - if (fnc_init.sym) - rc = fsl_sym_to_rid(f, fnc_init.sym, FSL_SATYPE_ANY, &rid); - else - fsl_ckout_version_info(f, &rid, NULL); - - if (rc) { - switch (rc) { - case FSL_RC_AMBIGUOUS: - RC(rc, "prefix too ambiguous [%s]", - fnc_init.sym); - goto end; - case FSL_RC_NOT_A_REPO: - RC(rc, "%s tree needs a local checkout", - fcli_progname()); - goto end; - case FSL_RC_NOT_FOUND: - RC(rc, "invalid symbolic checkin name [%s]", - fnc_init.sym); - goto end; - case FSL_RC_MISUSE: - /* FALL THROUGH */ - default: - goto end; - } - } - - /* In 'fnc tree -R repo.db [path]' case, use the latest checkin. */ - if (rid == 0) { - rc = fsl_sym_to_rid(f, "tip", FSL_SATYPE_CHECKIN, &rid); - if (rc) - goto end; - } else if (!fsl_rid_is_a_checkin(f, rid)) { - rc = RC(FSL_RC_TYPE, "%s tree requires check-in artifact", - fcli_progname()); - goto end; - } - - rc = init_curses(); - if (rc) - goto end; - - view = view_open(0, 0, 0, 0, FNC_VIEW_TREE); - if (view == NULL) { - RC(FSL_RC_ERROR, "%s", "view_open"); - goto end; - } - - rc = open_tree_view(view, path, rid); - if (!rc) - rc = view_loop(view); -end: - fsl_free(path); - return rc; -} - -static int -open_tree_view(struct fnc_view *view, const char *path, fsl_id_t rid) -{ - fsl_cx *f = fcli_cx(); - struct fnc_tree_view_state *s = &view->state.tree; - int rc = 0; - - TAILQ_INIT(&s->parents); - s->show_id = false; - s->colour = !fnc_init.nocolour && has_colors(); - s->rid = rid; - s->commit_id = fsl_rid_to_uuid(f, rid); - if (s->commit_id == NULL) - return RC(FSL_RC_AMBIGUOUS, "%s", "fsl_rid_to_uuid"); - - /* - * Construct tree of entire repository from which all (sub)tress will - * be derived. This object will be released when this view closes. - */ - rc = create_repository_tree(&s->repo, &s->commit_id, s->rid); - if (rc) - goto end; - - /* - * Open the initial root level of the repository tree now. Subtrees - * opened during traversal are built and destroyed on demand. - */ - rc = tree_builder(s->repo, &s->root, "/"); - if (rc) - goto end; - s->tree = s->root; - /* - * If user has supplied a path arg (i.e., fnc tree path/in/repo), or - * has selected a commit from an 'fnc timeline path/in/repo' command, - * walk the path and open corresponding (sub)tree objects now. - */ - if (!fnc_path_is_root_dir(path)) { - rc = walk_tree_path(s, s->repo, &s->root, path); - if (rc) - goto end; - } - - - if ((s->tree_label = fsl_mprintf("checkin %s", s->commit_id)) == NULL) { - rc = RC(FSL_RC_RANGE, "%s", "fsl_mprintf"); - goto end; - } - - s->first_entry_onscreen = &s->tree->entries[0]; - s->selected_entry = &s->tree->entries[0]; - - if (s->colour) - set_colours(&s->colours, FNC_VIEW_TREE); - - view->show = show_tree_view; - view->input = tree_input_handler; - view->close = close_tree_view; - view->search_init = tree_search_init; - view->search_next = tree_search_next; -end: - if (rc) - close_tree_view(view); - return rc; -} - -/* - * Decompose the supplied path into its constituent components, then build, - * open and visit each subtree segment on the way to the requested entry. - */ -static int -walk_tree_path(struct fnc_tree_view_state *s, struct fnc_repository_tree *repo, - struct fnc_tree_object **root, const char *path) -{ - struct fnc_tree_object *tree = NULL; - const char *p; - char *slash, *subpath = NULL; - int rc = 0; - - /* Find each slash and open preceding directory segment as a tree. */ - p = path; - while (*p) { - struct fnc_tree_entry *te; - char *te_name; - - while (p[0] == '/') - p++; - - slash = strchr(p, '/'); - if (slash == NULL) - te_name = fsl_strdup(p); - else - te_name = fsl_strndup(p, slash - p); - if (te_name == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - break; - } - - te = find_tree_entry(s->tree, te_name, fsl_strlen(te_name)); - if (te == NULL) { - rc = RC(FSL_RC_NOT_FOUND, "find_tree_entry(%s)", - te_name); - fsl_free(te_name); - break; - } - fsl_free(te_name); - - s->first_entry_onscreen = s->selected_entry = te; - if (!S_ISDIR(s->selected_entry->mode)) - break; /* If a file, jump to this entry. */ - - slash = strchr(p, '/'); - if (slash) - subpath = fsl_strndup(path, slash - path); - else - subpath = fsl_strdup(path); - if (subpath == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - break; - } - - rc = tree_builder(repo, &tree, subpath + 1 /* Leading slash */); - if (rc) - break; - rc = visit_subtree(s, tree); - if (rc) { - fnc_object_tree_close(tree); - break; - } - - if (slash == NULL) - break; - fsl_free(subpath); - subpath = NULL; - p = slash; - } - - fsl_free(subpath); - return rc; -} - -/* - * This routine constructs the repository tree, repo, which is a DLL; from this - * tree, all displayed (sub)trees are derived. File paths are extracted from F - * cards of the checkin identified by id referenced in the repo database by rid. - */ -static int -create_repository_tree(struct fnc_repository_tree **repo, fsl_uuid_str *id, - fsl_id_t rid) -{ - fsl_cx *f = fcli_cx(); - struct fnc_repository_tree *ptr; - fsl_deck d = fsl_deck_empty; - const fsl_card_F *cf = NULL; - int rc = 0; - - ptr = fsl_malloc(sizeof(struct fnc_repository_tree)); - if (ptr == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_malloc"); - memset(ptr, 0, sizeof(struct fnc_repository_tree)); - - rc = fsl_deck_load_rid(fcli_cx(), &d, rid, FSL_SATYPE_CHECKIN); - if (rc) - return RC(rc, "fsl_deck_load_rid(%d) [%s]", rid, id); - rc = fsl_deck_F_rewind(&d); - if (rc) - goto end;; - rc = fsl_deck_F_next(&d, &cf); - if (rc) - goto end;; - - while (cf) { - char *filename = NULL, *uuid = NULL; - fsl_time_t mtime; - - filename = fsl_strdup(cf->name); - if (filename == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - uuid = fsl_strdup(cf->uuid); - if (uuid == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - rc = fsl_mtime_of_F_card(f, rid, cf, &mtime); - if (!rc) - rc = link_tree_node(ptr, filename, uuid, mtime); - fsl_free(filename); - fsl_free(uuid); - if (!rc) - rc = fsl_deck_F_next(&d, &cf); - if (rc) - goto end; - } -end: - fsl_deck_finalize(&d); - - *repo = ptr; - return rc; -} - -/* - * This routine constructs the (sub)trees that are displayed. The directory dir - * and its contents form a subtree, which is an array of tree entries copied - * from DLL nodes in repo and stored in tree. This routine is called for each - * directory that is displayed as a tree. - */ -static int -tree_builder(struct fnc_repository_tree *repo, struct fnc_tree_object **tree, - const char *dir) -{ - struct fnc_tree_entry *te = NULL; - struct fnc_repo_tree_node *tn = NULL; - int i = 0; - - *tree = NULL; - *tree = fsl_malloc(sizeof(**tree)); - if (*tree == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_malloc"); - memset(*tree, 0, sizeof(**tree)); - - /* Count how many elements will comprise the tree to be allocated. */ - for(tn = repo->head; tn; tn = tn->next) { - if ((!tn->parent_dir && fsl_strcmp(dir, "/")) || - (tn->parent_dir && fsl_strcmp(dir, tn->parent_dir->path))) - continue; - ++i; - } - (*tree)->entries = calloc(i, sizeof(struct fnc_tree_entry)); - if ((*tree)->entries == NULL) - return RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), "%s", "calloc"); - /* Construct the tree to be displayed. */ - for(tn = repo->head, i = 0; tn; tn = tn->next) { - if ((!tn->parent_dir && fsl_strcmp(dir, "/")) || - (tn->parent_dir && fsl_strcmp(dir, tn->parent_dir->path))) - continue; - te = &(*tree)->entries[i]; - te->mode = tn->mode; - te->mtime = tn->mtime; - te->basename = fsl_strdup(tn->basename); - if (te->basename == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - te->path = fsl_strdup(tn->path); - if (te->path == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - te->uuid = fsl_strdup(tn->uuid); - if (te->uuid == NULL && !S_ISDIR(te->mode)) - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - te->idx = i++; - } - (*tree)->nentries = i; - - return 0; -} - -#if 0 -static void -delete_tree_node(struct fnc_tree_entry **head, struct fnc_tree_entry *del) -{ - struct fnc_tree_entry *temp = *head, *prev; - - if (temp == del) { - *head = temp->next; - fsl_free(temp); - return; - } - - while (temp != NULL && temp != del) { - prev = temp; - temp = temp->next; - } - - if (temp == NULL) - return; - - prev->next = temp->next; - - fsl_free(temp); -} -#endif - -/* - * This routine inserts nodes into the doubly-linked repository tree. Each - * path component of path (i.e., tokens delimited by '/') becomes a node in - * tree. The final path component of each segment is the node's .basename, and - * its full repository relative path its .path. All files in a given directory - * will comprise the directory node's .children list, and each file node's - * .sibling list; said directory will be each file node's .parent_dir. The - * elements of each requested tree will be identified by the node's .parent_dir; - * that is, each node with the same parent_dir will be an entry in the same tree - * tree The repository tree into which nodes are inserted - * path The repository relative pathname of the versioned file - * uuid The SHA hash of the file - * mtime Modification time of the file - * Returns 0 on success, non-zero on error. - */ -static int -link_tree_node(struct fnc_repository_tree *tree, const char *path, - const char *uuid, fsl_time_t mtime) -{ - fsl_cx *f = fcli_cx(); - struct fnc_repo_tree_node *parent_dir; - fsl_buffer buf = fsl_buffer_empty; - struct stat s; - int i, rc = 0; - - parent_dir = tree->tail; - while (parent_dir != 0 && - (strncmp(parent_dir->path, path, parent_dir->pathlen) != 0 || - path[parent_dir->pathlen] != '/')) - parent_dir = parent_dir->parent_dir; - - i = parent_dir ? parent_dir->pathlen + 1 : 0; - - while (path[i]) { - struct fnc_repo_tree_node *tn; - int nodesz, slash = i; - - /* Find slash to demarcate each path component. */ - while (path[i] && path[i] != '/') - i++; - nodesz = sizeof(*tn) + i + 1; - - /* - * If not at end of path string, node is a directory so don't - * allocate space for the hash. - */ - if (uuid != 0 && path[i] == '\0') - nodesz += FSL_STRLEN_K256 + 1; /* NUL */ - tn = fsl_malloc(nodesz); - if (tn == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_malloc"); - memset(tn, 0, sizeof(*tn)); - - tn->path = (char *)&tn[1]; - memcpy(tn->path, path, i); - tn->path[i] = '\0'; - tn->pathlen = i; - - if (uuid != 0 && path[i] == '\0') { - tn->uuid = tn->path + i + 1; - memcpy(tn->uuid, uuid, fsl_strlen(uuid) + 1); - } - - tn->basename = tn->path + slash; - - /* Insert node into DLL or make it the head if first. */ - if (tree->tail) { - tree->tail->next = tn; - tn->prev = tree->tail; - } else - tree->head = tn; - - tree->tail = tn; - tn->parent_dir = parent_dir; - if (parent_dir) { - if (parent_dir->children) - parent_dir->lastchild->sibling = tn; - else - parent_dir->children = tn; - tn->nparents = parent_dir->nparents + 1; - parent_dir->lastchild = tn; - } else { - if (tree->rootail) - tree->rootail->sibling = tn; - tree->rootail = tn; - } - - tn->mtime = mtime; - while (path[i] == '/') /* Consume slashes. */ - ++i; - parent_dir = tn; - - /* Stat path for tree display features. */ - rc = fsl_file_canonical_name2(f->ckout.dir, tn->path, &buf, - false); - if (rc) - goto end; - if (lstat(fsl_buffer_cstr(&buf), &s) == -1) { - if (errno == ENOENT) - tn->mode = (!fsl_strcmp(tn->path, path) && - tn->uuid) ? S_IFREG : S_IFDIR; - else { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_ACCESS), - "lstat(%s)", fsl_buffer_cstr(&buf)); - goto end; - } - } else - tn->mode = s.st_mode; - fsl_buffer_reuse(&buf); - } - - while (parent_dir && parent_dir->parent_dir) { - if (parent_dir->parent_dir->mtime < parent_dir->mtime) - parent_dir->parent_dir->mtime = parent_dir->mtime; - parent_dir = parent_dir->parent_dir; - } -end: - fsl_buffer_clear(&buf); - return rc; -} - -static int -show_tree_view(struct fnc_view *view) -{ - struct fnc_tree_view_state *s = &view->state.tree; - char *treepath; - int rc = 0; - - rc = tree_entry_path(&treepath, &s->parents, NULL); - if (rc) - return rc; - - rc = draw_tree(view, treepath); - fsl_free(treepath); - draw_vborder(view); - - return rc; -} - -/* - * Construct absolute repository path of the currently selected tree entry to - * display in the tree view header, or pass to open_timeline_view() to construct - * a timeline of all commits modifying path. - */ -static int -tree_entry_path(char **path, struct fnc_parent_trees *parents, - struct fnc_tree_entry *te) -{ - struct fnc_parent_tree *pt; - size_t len = 2; /* Leading slash and NUL. */ - int rc = 0; - - TAILQ_FOREACH(pt, parents, entry) - len += strlen(pt->selected_entry->basename) + 1 /* slash */; - if (te) - len += strlen(te->basename); - - *path = calloc(1, len); - if (path == NULL) - return RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), "%s", "calloc"); - - (*path)[0] = '/'; /* Make it absolute from the repository root. */ - pt = TAILQ_LAST(parents, fnc_parent_trees); - while (pt) { - const char *name = pt->selected_entry->basename; - if (strlcat(*path, name, len) >= len) { - rc = RC(FSL_RC_RANGE, "strlcat(%s, %s, %d)", - *path, name, len); - goto end; - } - if (strlcat(*path, "/", len) >= len) { - rc = RC(FSL_RC_RANGE, "strlcat(%s, \"/\", %d)", - *path, len); - goto end; - } - pt = TAILQ_PREV(pt, fnc_parent_trees, entry); - } - if (te) { - if (strlcat(*path, te->basename, len) >= len) { - rc = RC(FSL_RC_RANGE, "strlcat(%s, %s, %d)", - *path, te->basename, len); - goto end; - } - } -end: - if (rc) { - fsl_free(*path); - *path = NULL; - } - return rc; -} - -/* - * Draw the currently visited tree. Headline view with the checkin's SHA hash, - * and write subheader comprised of the tree path. Lexicographically order nodes - * (cf. ls(1)) and postfix with identifier corresponding to the file mode as - * returned by lstat(2) such that the tree takes the following form: - * - * checkin COMMIT-HASH - * /absolute/repository/tree/path/ - * - * .. - * dir/ - * executable* - * regularfile - * symlink@ -> /path/to/source/file - * - * If the 'i' key binding is entered, prefix each versioned file with its - * SHA{1,3} hash. Directories, however, have no such hash UUID to display. - */ -static int -draw_tree(struct fnc_view *view, const char *treepath) -{ - struct fnc_tree_view_state *s = &view->state.tree; - struct fnc_tree_entry *te; - struct fnc_colour *c = NULL; - wchar_t *wcstr; - int match = -1, rc = 0; - int wstrlen, n, idx, nentries; - int limit = view->nlines; - uint_fast8_t hashlen = FSL_UUID_STRLEN_MIN; - - s->ndisplayed = 0; - werase(view->window); - if (limit == 0) - return rc; - - /* Write (highlighted) headline (if view is active in splitscreen). */ - rc = formatln(&wcstr, &wstrlen, s->tree_label, view->ncols, 0); - if (rc) - return rc; - if (screen_is_shared(view)) - wstandout(view->window); - if (s->colour) - c = get_colour(&s->colours, FNC_COMMIT_ID); - if (c) - wattr_on(view->window, COLOR_PAIR(c->scheme), NULL); - waddwstr(view->window, wcstr); - if (c) - wattr_off(view->window, COLOR_PAIR(c->scheme), NULL); - if (screen_is_shared(view)) - wstandend(view->window); - fsl_free(wcstr); - wcstr = NULL; - if (wstrlen < view->ncols - 1) - waddch(view->window, '\n'); - if (--limit <= 0) - return rc; - - /* Write this (sub)tree's absolute repository path subheader. */ - rc = formatln(&wcstr, &wstrlen, treepath, view->ncols, 0); - if (rc) - return rc; - waddwstr(view->window, wcstr); - fsl_free(wcstr); - wcstr = NULL; - if (wstrlen < view->ncols - 1) - waddch(view->window, '\n'); - if (--limit <= 0) - return rc; - waddch(view->window, '\n'); - if (--limit <= 0) - return rc; - - /* Write parent dir entry (i.e., "..") if top of the tree is in view. */ - if (s->first_entry_onscreen == NULL) { - te = &s->tree->entries[0]; - if (s->selected_idx == 0) { - if (view->active) - wattr_on(view->window, A_REVERSE, NULL); - s->selected_entry = NULL; - } - waddstr(view->window, " ..\n"); - if (s->selected_idx == 0 && view->active) - wattr_off(view->window, A_REVERSE, NULL); - ++s->ndisplayed; - if (--limit <= 0) - return rc; - n = 1; - } else { - n = 0; - te = s->first_entry_onscreen; - } - - nentries = s->tree->nentries; - for (idx = 0; idx < nentries; ++idx) /* Find max hash length. */ - if (hashlen < fsl_strlen(s->tree->entries[idx].uuid)) - hashlen = fsl_strlen(s->tree->entries[idx].uuid); - /* Iterate and write tree nodes postfixed with path type identifier. */ - for (idx = te->idx; idx < nentries; ++idx) { - char *line = NULL, *idstr = NULL, *targetlnk = NULL; - const char *modestr = ""; - mode_t mode; - - if (idx < 0 || idx >= s->tree->nentries) - return 0; - te = &s->tree->entries[idx]; - mode = te->mode; - - if (s->show_id) { - idstr = fsl_strdup(te->uuid); - /* Directories don't have UUIDs; pad with "..." dots. */ - if (idstr == NULL && !S_ISDIR(mode)) - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - /* If needed, pad SHA1 hash to align w/ SHA3 hashes. */ - if (idstr == NULL || fsl_strlen(idstr) < hashlen) { - char buf[hashlen], pad = '.'; - idstr = fsl_mprintf("%s%s", idstr ? idstr : "", - (char *)memset(buf, pad, - hashlen - fsl_strlen(idstr))); - if (idstr == NULL) - return RC(FSL_RC_RANGE, - "%s", "fsl_mprintf"); - idstr[hashlen] = '\0'; - /* idstr = fsl_mprintf("%*c", hashlen, ' '); */ - } - } - if (S_ISLNK(mode)) { - fsl_size_t ch; - - rc = tree_entry_get_symlink_target(&targetlnk, te); - if (rc) { - fsl_free(idstr); - return rc; - } - for (ch = 0; ch < fsl_strlen(targetlnk); ++ch) { - if (!isprint((unsigned char)targetlnk[ch])) - targetlnk[ch] = '?'; - } - modestr = "@"; - } - else if (S_ISDIR(mode)) - modestr = "/"; - else if (mode & S_IXUSR) - modestr = "*"; - if ((line = fsl_mprintf("%s %s%s%s%s", idstr ? idstr : "", - te->basename, modestr, targetlnk ? " -> ": "", - targetlnk ? targetlnk : "")) == NULL) { - fsl_free(idstr); - fsl_free(targetlnk); - return RC(FSL_RC_RANGE, "%s", "fsl_mprintf"); - } - fsl_free(idstr); - fsl_free(targetlnk); - rc = formatln(&wcstr, &wstrlen, line, view->ncols, 0); - if (rc) { - fsl_free(line); - break; - } - if (n == s->selected_idx) { - if (view->active) - wattr_on(view->window, A_REVERSE, NULL); - s->selected_entry = te; - } - if (s->colour && (match = fsl_list_index_of(&s->colours, line, - match_line)) != -1) - c = s->colours.list[match]; - else - c = NULL; - if (c) - wattr_on(view->window, COLOR_PAIR(c->scheme), NULL); - waddwstr(view->window, wcstr); - if (c) - wattr_off(view->window, COLOR_PAIR(c->scheme), NULL); - if (wstrlen < view->ncols - 1) - waddch(view->window, '\n'); - if (n == s->selected_idx && view->active) - wattr_off(view->window, A_REVERSE, NULL); - fsl_free(line); - fsl_free(wcstr); - wcstr = NULL; - ++n; - ++s->ndisplayed; - s->last_entry_onscreen = te; - if (--limit <= 0) - break; - } - - return rc; -} - -static int -tree_entry_get_symlink_target(char **targetlnk, struct fnc_tree_entry *te) -{ - struct stat s; - fsl_buffer fb = fsl_buffer_empty; - char *buf = NULL; - ssize_t nbytes, bufsz; - int rc = 0; - - *targetlnk = NULL; - - fsl_file_canonical_name2(fcli_cx()->ckout.dir, te->path, &fb, false); - if (lstat(fsl_buffer_cstr(&fb), &s) == -1) { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_ACCESS), "lstat(%s)", - fsl_buffer_cstr(&fb)); - goto end; - } - - bufsz = s.st_size ? (s.st_size + 1 /* NUL */) : PATH_MAX; - buf = fsl_malloc(bufsz); - if (buf == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_malloc"); - goto end; - } - - nbytes = readlink(fsl_buffer_cstr(&fb), buf, bufsz); - if (nbytes == -1) { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_IO), "readlink(%s)", - fsl_buffer_cstr(&fb)); - goto end; - } - buf[nbytes] = '\0'; /* readlink() does _not_ NUL terminate */ -end: - fsl_buffer_clear(&fb); - if (rc) - fsl_free(buf); - *targetlnk = buf; - return rc; - /* - * XXX Not sure if we should rely on fossil(1) populating symlinks - * with the path of the target source file to obtain the target link. - */ - /* fsl_cx *f = fcli_cx(); */ - /* fsl_buffer blob = fsl_buffer_empty; */ - /* fsl_id_t rid; */ - - /* if (!((te->mode & (S_IFDIR | S_IFLNK)) == S_IFLNK)) */ - /* return RC(FSL_RC_TYPE, "file not symlink [%s]", te->path); */ - /* rc = fsl_sym_to_rid(f, te->uuid, FSL_SATYPE_ANY, &rid); */ - /* if (!rc) */ - /* rc = fsl_content_blob(f, rid, &blob); */ - /* if (rc) */ - /* return rc; */ - - /* *targetlnk = fsl_strdup(fsl_buffer_str(&blob)); */ - /* fsl_buffer_clear(&blob); */ -} - -static int -tree_input_handler(struct fnc_view **new_view, struct fnc_view *view, int ch) -{ - struct fnc_view *timeline_view/*, *branch_view */; - struct fnc_tree_view_state *s = &view->state.tree; - struct fnc_tree_entry *te; - int n, start_col = 0, rc = 0; - - switch (ch) { - case 'c': - s->colour = !s->colour; - break; - case 'i': - s->show_id = !s->show_id; - break; - case 't': - if (!s->selected_entry) - break; - if (view_is_parent(view)) - start_col = view_split_start_col(view->start_col); - rc = timeline_tree_entry(&timeline_view, start_col, s); - view->active = false; - timeline_view->active = true; - if (view_is_parent(view)) { - rc = view_close_child(view); - if (rc) - return rc; - view_set_child(view, timeline_view); - view->focus_child = true; - } else - *new_view = timeline_view; - break; - case 'g': - if (!fnc_home(view)) - break; - /* FALL THROUGH */ - case KEY_HOME: - s->selected_idx = 0; - if (s->tree == s->root) - s->first_entry_onscreen = &s->tree->entries[0]; - else - s->first_entry_onscreen = NULL; - break; - case KEY_END: - case 'G': - s->selected_idx = 0; - te = &s->tree->entries[s->tree->nentries - 1]; - for (n = 0; n < view->nlines - 3; ++n) { - if (te == NULL) { - if(s->tree != s->root) { - s->first_entry_onscreen = NULL; - ++n; - } - break; - } - s->first_entry_onscreen = te; - te = get_tree_entry(s->tree, te->idx - 1); - } - if (n > 0) - s->selected_idx = n - 1; - break; - case KEY_UP: - case 'k': - if (s->selected_idx > 0) { - --s->selected_idx; - break; - } - tree_scroll_up(s, 1); - break; - case KEY_PPAGE: - case CTRL('b'): - if (s->tree == s->root) { - if (&s->tree->entries[0] == s->first_entry_onscreen) - s->selected_idx = 0; - } else { - if (s->first_entry_onscreen == NULL) - s->selected_idx = 0; - } - tree_scroll_up(s, MAX(0, view->nlines - 3)); - break; - case KEY_DOWN: - case 'j': - if (s->selected_idx < s->ndisplayed - 1) { - ++s->selected_idx; - break; - } - if (get_tree_entry(s->tree, s->last_entry_onscreen->idx + 1) - == NULL) - break; /* Reached last entry. */ - tree_scroll_down(s, 1); - break; - case KEY_NPAGE: - case CTRL('f'): - if (get_tree_entry(s->tree, s->last_entry_onscreen->idx + 1) - == NULL) { - /* - * When the last entry on screen is the last node in the - * tree move cursor to it instead of scrolling the view. - */ - if (s->selected_idx < s->ndisplayed - 1) - s->selected_idx = s->ndisplayed - 1; - break; - } - tree_scroll_down(s, view->nlines - 3); - break; - case KEY_BACKSPACE: - case KEY_ENTER: - case KEY_LEFT: - case KEY_RIGHT: - case '\r': - case 'h': - case 'l': - /* - * h/backspace/arrow-left: return to parent dir irrespective - * of selected entry type (unless already at root). - * l/arrow-right: move into selected dir entry. - */ - if (ch != KEY_RIGHT && ch != 'l' && (s->selected_entry == NULL - || ch == 'h' || ch == KEY_BACKSPACE || ch == KEY_LEFT)) { - struct fnc_parent_tree *parent; - /* h/backspace/left-arrow pressed or ".." selected. */ - if (s->tree == s->root) - break; - parent = TAILQ_FIRST(&s->parents); - TAILQ_REMOVE(&s->parents, parent, - entry); - fnc_object_tree_close(s->tree); - s->tree = parent->tree; - s->first_entry_onscreen = parent->first_entry_onscreen; - s->selected_entry = parent->selected_entry; - s->selected_idx = parent->selected_idx; - fsl_free(parent); - } else if (s->selected_entry != NULL && - S_ISDIR(s->selected_entry->mode)) { - struct fnc_tree_object *subtree = NULL; - rc = tree_builder(s->repo, &subtree, - s->selected_entry->path); - if (rc) - break; - rc = visit_subtree(s, subtree); - if (rc) { - fnc_object_tree_close(subtree); - break; - } - } else if (S_ISREG(s->selected_entry->mode)) { - struct fnc_view *blame_view; - int start_col = view_is_parent(view) ? - view_split_start_col(view->start_col) : 0; - - rc = blame_tree_entry(&blame_view, start_col, - s->selected_entry, &s->parents, s->commit_id); - if (rc) - break; - view->active = false; - blame_view->active = true; - if (view_is_parent(view)) { - rc = view_close_child(view); - if (rc) - return rc; - view_set_child(view, blame_view); - view->focus_child = true; - } else - *new_view = blame_view; - } - break; - case KEY_RESIZE: - if (view->nlines >= 4 && s->selected_idx >= view->nlines - 3) - s->selected_idx = view->nlines - 4; - break; - default: - break; - } - - return rc; -} - -static int -timeline_tree_entry(struct fnc_view **new_view, int start_col, - struct fnc_tree_view_state *s) -{ - struct fnc_view *timeline_view; - char *path; - int rc = 0; - - *new_view = NULL; - - timeline_view = view_open(0, 0, 0, start_col, FNC_VIEW_TIMELINE); - if (timeline_view == NULL) - return RC(FSL_RC_ERROR, "%s", "view_open"); - - /* Construct repository relative path for timeline query. */ - rc = tree_entry_path(&path, &s->parents, s->selected_entry); - if (rc) - return rc; - - rc = open_timeline_view(timeline_view, s->rid, path); - if (rc) - view_close(timeline_view); - else - *new_view = timeline_view; - - fsl_free(path); - return rc; -} - -static void -tree_scroll_up(struct fnc_tree_view_state *s, int maxscroll) -{ - struct fnc_tree_entry *te; - int isroot, i = 0; - - isroot = s->tree == s->root; - - if (s->first_entry_onscreen == NULL) - return; - - te = get_tree_entry(s->tree, s->first_entry_onscreen->idx - 1); - while (i++ < maxscroll) { - if (te == NULL) { - if (!isroot) - s->first_entry_onscreen = NULL; - break; - } - s->first_entry_onscreen = te; - te = get_tree_entry(s->tree, te->idx - 1); - } -} - -static void -tree_scroll_down(struct fnc_tree_view_state *s, int maxscroll) -{ - struct fnc_tree_entry *next, *last; - int n = 0; - - if (s->first_entry_onscreen) - next = get_tree_entry(s->tree, - s->first_entry_onscreen->idx + 1); - else - next = &s->tree->entries[0]; - - last = s->last_entry_onscreen; - while (next && last && n++ < maxscroll) { - last = get_tree_entry(s->tree, last->idx + 1); - if (last) { - s->first_entry_onscreen = next; - next = get_tree_entry(s->tree, next->idx + 1); - } - } -} - -static int -visit_subtree(struct fnc_tree_view_state *s, struct fnc_tree_object *subtree) -{ - struct fnc_parent_tree *parent; - - parent = calloc(1, sizeof(*parent)); - if (parent == NULL) - return RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), "%s", "calloc"); - - parent->tree = s->tree; - parent->first_entry_onscreen = s->first_entry_onscreen; - parent->selected_entry = s->selected_entry; - parent->selected_idx = s->selected_idx; - TAILQ_INSERT_HEAD(&s->parents, parent, entry); - s->tree = subtree; - s->selected_idx = 0; - s->first_entry_onscreen = NULL; - - return 0; -} - -static int -blame_tree_entry(struct fnc_view **new_view, int start_col, - struct fnc_tree_entry *te, struct fnc_parent_trees *parents, - fsl_uuid_str commit_id) -{ - struct fnc_view *blame_view; - char *path; - int rc = 0; - - *new_view = NULL; - - rc = tree_entry_path(&path, parents, te); - if (rc) - return rc; - - blame_view = view_open(0, 0, 0, start_col, FNC_VIEW_BLAME); - if (blame_view == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "view_open"); - goto end; - } - - rc = open_blame_view(blame_view, path, commit_id, 0, 0); - if (rc) - view_close(blame_view); - else - *new_view = blame_view; -end: - fsl_free(path); - return rc; -} - -static int -tree_search_init(struct fnc_view *view) -{ - struct fnc_tree_view_state *s = &view->state.tree; - - s->matched_entry = NULL; - return 0; -} - -static int -tree_search_next(struct fnc_view *view) -{ - struct fnc_tree_view_state *s = &view->state.tree; - struct fnc_tree_entry *te = NULL; - int rc = 0; - - if (view->searching == SEARCH_DONE) { - view->search_status = SEARCH_CONTINUE; - return rc; - } - - if (s->matched_entry) { - if (view->searching == SEARCH_FORWARD) { - if (s->selected_entry) - te = &s->tree->entries[s->selected_entry->idx - + 1]; - else - te = &s->tree->entries[0]; - } else { - if (s->selected_entry == NULL) - te = &s->tree->entries[s->tree->nentries - 1]; - else - te = &s->tree->entries[s->selected_entry->idx - - 1]; - } - } else { - if (view->searching == SEARCH_FORWARD) - te = &s->tree->entries[0]; - else - te = &s->tree->entries[s->tree->nentries - 1]; - } - - while (1) { - if (te == NULL) { - if (s->matched_entry == NULL) { - view->search_status = SEARCH_CONTINUE; - return rc; - } - if (view->searching == SEARCH_FORWARD) - te = &s->tree->entries[0]; - else - te = &s->tree->entries[s->tree->nentries - 1]; - } - - if (match_tree_entry(te, &view->regex)) { - view->search_status = SEARCH_CONTINUE; - s->matched_entry = te; - break; - } - - if (view->searching == SEARCH_FORWARD) - te = &s->tree->entries[te->idx + 1]; - else - te = &s->tree->entries[te->idx - 1]; - } - - if (s->matched_entry) { - s->first_entry_onscreen = s->matched_entry; - s->selected_idx = 0; - } - - return rc; -} - -static int -match_tree_entry(struct fnc_tree_entry *te, regex_t *regex) -{ - regmatch_t regmatch; - - return regexec(regex, te->basename, 1, ®match, 0) == 0; -} - -struct fnc_tree_entry * -get_tree_entry(struct fnc_tree_object *tree, int i) -{ - if (i < 0 || i >= tree->nentries) - return NULL; - - return &tree->entries[i]; -} - -/* Find entry in tree with basename name. */ -static struct fnc_tree_entry * -find_tree_entry(struct fnc_tree_object *tree, const char *name, size_t len) -{ - int idx; - - /* Entries are sorted in strcmp() order. */ - for (idx = 0; idx < tree->nentries; ++idx) { - struct fnc_tree_entry *te = &tree->entries[idx]; - int cmp = strncmp(te->basename, name, len); - if (cmp < 0) - continue; - if (cmp > 0) - break; - if (te->basename[len] == '\0') - return te; - } - return NULL; -} - -static int -close_tree_view(struct fnc_view *view) -{ - struct fnc_tree_view_state *s = &view->state.tree; - struct fsl_list_state st = { FNC_COLOUR_OBJ }; - - fsl_list_clear(&s->colours, fsl_list_object_free, &st); - - fsl_free(s->tree_label); - s->tree_label = NULL; - fsl_free(s->commit_id); - s->commit_id = NULL; - - while (!TAILQ_EMPTY(&s->parents)) { - struct fnc_parent_tree *parent; - parent = TAILQ_FIRST(&s->parents); - TAILQ_REMOVE(&s->parents, parent, entry); - if (parent->tree != s->root) - fnc_object_tree_close(parent->tree); - fsl_free(parent); - - } - - if (s->tree != NULL && s->tree != s->root) - fnc_object_tree_close(s->tree); - if (s->root) - fnc_object_tree_close(s->root); - if (s->repo) - fnc_close_repository_tree(s->repo); - - return 0; -} - -static void -fnc_object_tree_close(struct fnc_tree_object *tree) -{ - int idx; - - for (idx = 0; idx < tree->nentries; ++idx) { - fsl_free(tree->entries[idx].basename); - fsl_free(tree->entries[idx].path); - fsl_free(tree->entries[idx].uuid); - } - - fsl_free(tree->entries); - fsl_free(tree); -} - -static void -fnc_close_repository_tree(struct fnc_repository_tree *repo) -{ - struct fnc_repo_tree_node *next, *tn; - - tn = repo->head; - while (tn) { - next = tn->next; - fsl_free(tn); - tn = next; - } - fsl_free(repo); -} - -static int -view_close_child(struct fnc_view *view) -{ - int rc = 0; - - if (view->child == NULL) - return rc; - - rc = view_close(view->child); - view->child = NULL; - - return rc; -} - -static void -view_set_child(struct fnc_view *view, struct fnc_view *child) -{ - view->child = child; - child->parent = view; -} - -static int -set_colours(fsl_list *s, enum fnc_view_id vid) -{ - struct fnc_colour *colour; - const char **regexp = NULL; - const char *regexp_blame[] = {"^"}; - const char *regexp_timeline[] = {"^$", "^$", "^$"}; - const char *regexp_tree[] = {"@$", "/$", "\\*$", "^$"}; - const char *regexp_diff[] = { - "^((checkin|wiki|ticket|technote) " - "[0-9a-f]|hash [+-] |\\[[+~>-]] |" - "[+-]{3} )", "^user:", "^date:", "^tags:", - "^-", "^\\+", "^@@" - }; - int pairs_diff[][2] = { - {FNC_DIFF_META, COLOR_GREEN}, - {FNC_USER_STR, COLOR_CYAN}, - {FNC_DATE_STR, COLOR_YELLOW}, - {FNC_TAGS_STR, COLOR_MAGENTA}, - {FNC_DIFF_MINUS, COLOR_MAGENTA}, - {FNC_DIFF_PLUS, COLOR_CYAN}, - {FNC_DIFF_CHNK, COLOR_YELLOW} - }; - int pairs_tree[][2] = { - {FNC_TREE_LINK, COLOR_MAGENTA}, - {FNC_TREE_DIR, COLOR_CYAN}, - {FNC_TREE_EXEC, COLOR_GREEN}, - {FNC_COMMIT_ID, COLOR_GREEN} - }; - int pairs_timeline[][2] = { - {FNC_COMMIT_ID, COLOR_GREEN}, - {FNC_USER_STR, COLOR_CYAN}, - {FNC_DATE_STR, COLOR_YELLOW} - }; - int pairs_blame[][2] = { - {FNC_COMMIT_ID, COLOR_GREEN} - }; - int (*pairs)[2], rc = 0; - fsl_size_t idx, n; - - switch (vid) { - case FNC_VIEW_DIFF: - n = nitems(regexp_diff); - regexp = regexp_diff; - pairs = pairs_diff; - break; - case FNC_VIEW_TREE: - n = nitems(regexp_tree); - regexp = regexp_tree; - pairs = pairs_tree; - break; - case FNC_VIEW_TIMELINE: - n = nitems(regexp_timeline); - regexp = regexp_timeline; - pairs = pairs_timeline; - break; - case FNC_VIEW_BLAME: - n = nitems(regexp_blame); - regexp = regexp_blame; - pairs = pairs_blame; - break; - default: - return RC(FSL_RC_TYPE, "%s", "invalid fnc_view_id"); - } - - for (idx = 0; idx < n; ++idx) { - colour = fsl_malloc(sizeof(*colour)); - if (colour == NULL) - return RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), - "%s", "fsl_malloc"); - - rc = regcomp(&colour->regex, regexp[idx], - REG_EXTENDED | REG_NEWLINE | REG_NOSUB); - if (rc) { - static char regerr[512]; - regerror(rc, &colour->regex, regerr, sizeof(regerr)); - fsl_free(colour); - return RC(FSL_RC_ERROR, "regcomp(%s) -> %s", - regexp[idx], regerr); - } - - colour->scheme = pairs[idx][0]; - init_pair(colour->scheme, pairs[idx][1], -1); - fsl_list_append(s, colour); - } - - return rc; -} - -struct fnc_colour * -get_colour(fsl_list *colours, int scheme) -{ - struct fnc_colour cs; - int match = -1; - - cs.scheme = scheme; - match = fsl_list_index_of(colours, &cs, match_colour); - - if (match != -1) - return colours->list[match]; - - return NULL; -} - -static int -match_colour(const void *target, const void *key) -{ - struct fnc_colour *c = (struct fnc_colour *)key; - struct fnc_colour *t = (struct fnc_colour *)target; - - return (c->scheme == t->scheme) ? 0 : 1; -} - -/* - * Emulate vim(1) gg: User has 1 sec to follow first 'g' keypress with another. - */ -static bool -fnc_home(struct fnc_view *view) -{ - bool home = true; - - halfdelay(10); /* Block for 1 second, then return ERR. */ - if (wgetch(view->window) != 'g') - home = false; - cbreak(); /* Return to blocking mode on user input. */ - - return home; -} - -static int -cmd_blame(fcli_command const *argv) -{ - fsl_cx *f = fcli_cx(); - struct fnc_view *view; - char *path = NULL; - fsl_uuid_str commit_id = NULL; - fsl_id_t tip = 0, rid = 0; - int nlimit = 0, rc = 0; - - rc = fcli_process_flags(argv->flags); - if (rc || (rc = fcli_has_unused_flags(false))) - goto end; - if (!fcli_next_arg(false)) { - rc = RC(FSL_RC_MISSING_INFO, - "%s blame requires versioned file path", fcli_progname()); - goto end; - } - - if (fnc_init.nrecords.zlimit) { - char *n = (char *)fnc_init.nrecords.zlimit; - bool timed; - if (n[fsl_strlen(n) - 1] == 's') { - n[fsl_strlen(n) - 1] = '\0'; - timed = true; - } - if ((rc = strtonumcheck(&nlimit, n, INT_MIN, INT_MAX))) - goto end; - if (timed) - nlimit *= -1; - } - - if (fnc_init.sym || fnc_init.reverse) { - if (fnc_init.reverse) { - if (!fnc_init.sym) { - rc = RC(FSL_RC_MISSING_INFO, - "%s blame --reverse requires --commit", - fcli_progname()); - goto end; - } - rc = fsl_sym_to_rid(f, "tip", FSL_SATYPE_CHECKIN, &tip); - if (rc) - goto end; - } - rc = fsl_sym_to_rid(f, fnc_init.sym, FSL_SATYPE_CHECKIN, &rid); - if (rc) - goto end; - } else if (!fnc_init.sym) - fsl_ckout_version_info(f, &rid, NULL); - - rc = map_repo_path(&path); - if (rc) { - if (rc != FSL_RC_NOT_FOUND || !fnc_init.sym) - goto end; - /* Path may be valid in repository tree of specified commit. */ - rc = 0; - fcli_err_reset(); - } - - commit_id = fsl_rid_to_uuid(f, rid); - if (rc || (path[0] == '/' && path[1] == '\0')) { - rc = rc ? rc : RC(FSL_RC_MISSING_INFO, - "%s blame requires versioned file path", fcli_progname()); - goto end; - } - - init_curses(); - - view = view_open(0, 0, 0, 0, FNC_VIEW_BLAME); - if (view == NULL) { - rc = RC(FSL_RC_ERROR, "%s", view_open); - goto end; - } - - rc = open_blame_view(view, path, commit_id, tip, nlimit); - if (rc) - goto end; - rc = view_loop(view); -end: - fsl_free(path); - fsl_free(commit_id); - return rc; -} - -static int -open_blame_view(struct fnc_view *view, char *path, fsl_uuid_str commit_id, - fsl_id_t tip, int nlimit) -{ - struct fnc_blame_view_state *s = &view->state.blame; - int rc = 0; - - CONCAT(STAILQ, _INIT)(&s->blamed_commits); - - s->path = fsl_strdup(path); - if (s->path == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - - rc = fnc_commit_qid_alloc(&s->blamed_commit, commit_id); - if (rc) { - fsl_free(s->path); - return rc; - } - - CONCAT(STAILQ, _INSERT_HEAD)(&s->blamed_commits, s->blamed_commit, - entry); - memset(&s->blame, 0, sizeof(s->blame)); - s->first_line_onscreen = 1; - s->last_line_onscreen = view->nlines; - s->selected_line = 1; - s->blame_complete = false; - s->commit_id = commit_id; - s->blame.origin = tip; - s->blame.nlimit = nlimit; - s->spin_idx = 0; - s->colour = !fnc_init.nocolour && has_colors(); - - if (s->colour) { - rc = set_colours(&s->colours, FNC_VIEW_BLAME); - if (rc) - return rc; - } - - view->show = show_blame_view; - view->input = blame_input_handler; - view->close = close_blame_view; - view->search_init = blame_search_init; - view->search_next = blame_search_next; - - return run_blame(view); -} - -static int -run_blame(struct fnc_view *view) -{ - fsl_cx *f = fcli_cx(); - struct fnc_blame_view_state *s = &view->state.blame; - struct fnc_blame *blame = &s->blame; - fsl_deck d = fsl_deck_empty; - fsl_buffer buf = fsl_buffer_empty; - fsl_annotate_opt *opt = NULL; - const fsl_card_F *cf; - char *filepath = NULL; - char *master = NULL, *root = NULL; - int rc = 0; - - /* - * Trim prefixed '/' if path has been processed by map_repo_path(), - * which only occurs when the -c option has not been passed. - * XXX This slash trimming is cumbersome; we should not prefix a slash - * in map_repo_path() as we only want the slash for displaying an - * absolute-repository-relative path, so we should prefix it only then. - */ - filepath = s->path[0] != '/' ? s->path : s->path + 1; - - rc = fsl_deck_load_sym(f, &d, s->blamed_commit->id, FSL_SATYPE_CHECKIN); - if (rc) - goto end; - - cf = fsl_deck_F_search(&d, filepath); - if (cf == NULL) { - rc = RC(FSL_RC_NOT_FOUND, "'%s' not found in tree [%s]", - filepath, s->blamed_commit->id); - goto end; - } - rc = fsl_card_F_content(f, cf, &buf); - if (rc) - goto end; - - /* - * We load f with the actual file content to map line offsets so we - * accurately find tokens when running a search. - */ - blame->f = tmpfile(); - if (blame->f == NULL) { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_IO), "%s", "tmpfile"); - goto end; - } - - opt = &blame->thread_cx.blame_opt; - opt->filename = fsl_strdup(filepath); - fcli_fax((char *)opt->filename); - rc = fsl_sym_to_rid(f, s->blamed_commit->id, FSL_SATYPE_CHECKIN, - &opt->versionRid); - if (rc) - goto end; - opt->originRid = blame->origin; /* tip when -r is passed */ - if (blame->nlimit < 0) - opt->limitMs = abs(blame->nlimit) * 1000; - else - opt->limitVersions = blame->nlimit; - opt->out = blame_cb; - opt->outState = &blame->cb_cx; - - rc = fnc_dump_buffer_to_file(&blame->filesz, &blame->nlines, - &blame->line_offsets, blame->f, &buf); - if (rc) - goto end; - if (blame->nlines == 0) { - s->blame_complete = true; - goto end; - } - - /* Don't include EOF \n in blame line count. */ - if (blame->line_offsets[blame->nlines - 1] == blame->filesz) - --blame->nlines; - - blame->lines = calloc(blame->nlines, sizeof(*blame->lines)); - if (blame->lines == NULL) { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), "%s", "calloc"); - goto end; - } - - master = fsl_config_get_text(f, FSL_CONFDB_REPO, "main-branch", NULL); - if (master == NULL) { - master = fsl_strdup("trunk"); - if (master == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - } - root = fsl_mprintf("root:%s", master); - rc = fsl_sym_to_uuid(f, root, FSL_SATYPE_CHECKIN, - &blame->cb_cx.root_commit, NULL); - if (rc) { - rc = RC(rc, "%s", "fsl_sym_to_uuid"); - goto end; - } - - blame->cb_cx.view = view; - blame->cb_cx.lines = blame->lines; - blame->cb_cx.nlines = blame->nlines; - blame->cb_cx.commit_id = fsl_strdup(s->blamed_commit->id); - if (blame->cb_cx.commit_id == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - goto end; - } - blame->cb_cx.quit = &s->done; - - blame->thread_cx.path = s->path; - blame->thread_cx.cb_cx = &blame->cb_cx; - blame->thread_cx.complete = &s->blame_complete; - blame->thread_cx.cancel_cb = cancel_blame; - blame->thread_cx.cancel_cx = &s->done; - s->blame_complete = false; - - if (s->first_line_onscreen + view->nlines - 1 > blame->nlines) { - s->first_line_onscreen = 1; - s->last_line_onscreen = view->nlines; - s->selected_line = 1; - } - -end: - fsl_free(master); - fsl_free(root); - fsl_deck_finalize(&d); - fsl_buffer_clear(&buf); - if (rc) - stop_blame(blame); - return rc; -} - -/* - * Write file content in buf to out file. Record the number of lines in the file - * in nlines, and total bytes written in filesz. Assign byte offsets of each - * line to the dynamically allocated *line_offsets, which must eventually be - * disposed of by the caller. Flush and rewind out file when done. - */ -static int -fnc_dump_buffer_to_file(off_t *filesz, int *nlines, off_t **line_offsets, - FILE *out, fsl_buffer *buf) -{ - off_t off = 0, total_len = 0; - size_t len, n, i = 0, nalloc = 0; - int rc = 0; - const int alloc_chunksz = MIN(512, BUFSIZ); - - if (line_offsets) - *line_offsets = NULL; - if (filesz) - *filesz = 0; - if (nlines) - *nlines = 0; - - len = buf->used; - if (len == 0) - return RC(FSL_RC_SIZE_MISMATCH, "%s", - "fnc_dump_buffer_to_file"); - if (nlines) { - if (line_offsets && *line_offsets == NULL) { - *nlines = 1; - nalloc = alloc_chunksz; - *line_offsets = calloc(nalloc, sizeof(**line_offsets)); - if (*line_offsets == NULL) - return RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), - "%s", "calloc"); - - /* Consume the first line. */ - while (i < len) { - if (buf->mem[i] == '\n') - break; - ++i; - } - } - /* Scan '\n' offsets. */ - while (i < len) { - if (buf->mem[i] != '\n') { - ++i; - continue; - } - ++(*nlines); - if (line_offsets && nalloc < (size_t)*nlines) { - size_t n, oldsz, newsz; - off_t *new = NULL; - - n = *nlines + alloc_chunksz; - oldsz = nalloc * sizeof(**line_offsets); - newsz = n * sizeof(**line_offsets); - if (newsz <= oldsz) { - size_t b = oldsz - newsz; - if (b < oldsz / 2 && - b < (size_t)getpagesize()) { - memset((char *)*line_offsets - + newsz, 0, b); - goto allocated; - } - } - new = fsl_realloc(*line_offsets, newsz); - if (new == NULL) { - fsl_free(*line_offsets); - *line_offsets = NULL; - return RC(FSL_RC_ERROR, "%s", - "fsl_realloc"); - } - *line_offsets = new; -allocated: - nalloc = n; - } - if (line_offsets) { - off = total_len + i + 1; - (*line_offsets)[*nlines - 1] = off; - } - ++i; - } - } - n = fwrite(buf->mem, 1, len, out); - if (n != len) - return RC(ferror(out) ? fsl_errno_to_rc(errno, FSL_RC_IO) - : FSL_RC_IO, "%s", "fwrite"); - total_len += len; - - if (fflush(out) != 0) - return RC(fsl_errno_to_rc(errno, FSL_RC_IO), "%s", "fflush"); - rewind(out); - - if (filesz) - *filesz = total_len; - - return rc; -} - -static int -show_blame_view(struct fnc_view *view) -{ - struct fnc_blame_view_state *s = &view->state.blame; - int rc = 0; - - if (!s->blame.thread_id && !s->blame_complete) { - rc = pthread_create(&s->blame.thread_id, NULL, blame_thread, - &s->blame.thread_cx); - if (rc) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_create"); - - halfdelay(1); /* Fast refresh while annotating. */ - } - - if (s->blame_complete) - cbreak(); /* Return to blocking mode. */ - - rc = draw_blame(view); - draw_vborder(view); - - return rc; -} - -static void * -blame_thread(void *state) -{ - struct fnc_blame_thread_cx *cx = state; - int rc0, rc; - - rc = block_main_thread_signals(); - if (rc) - return (void *)(intptr_t)rc; - - rc = fsl_annotate(fcli_cx(), &cx->blame_opt); - if (rc && fsl_cx_err_get_e(fcli_cx())->code == FSL_RC_BREAK) { - fcli_err_reset(); - rc = 0; - } - - rc0 = pthread_mutex_lock(&fnc_mutex); - if (rc0) - return (void *)(intptr_t)RC(fsl_errno_to_rc(rc0, FSL_RC_ACCESS), - "%s", "pthread_mutex_lock"); - - *cx->complete = true; - - rc0 = pthread_mutex_unlock(&fnc_mutex); - if (rc0 && !rc) - rc = RC(fsl_errno_to_rc(rc0, FSL_RC_ACCESS), - "%s", "pthread_mutex_unlock"); - - return (void *)(intptr_t)rc; -} - -static int -blame_cb(void *state, fsl_annotate_opt const * const opt, - fsl_annotate_step const * const step) -{ - struct fnc_blame_cb_cx *cx = state; - struct fnc_blame_line *line; - int rc = 0; - - rc = pthread_mutex_lock(&fnc_mutex); - if (rc) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_lock"); - - if (*cx->quit) { - rc = fcli_err_set(FSL_RC_BREAK, "user quit"); - goto end; - } - - line = &cx->lines[step->lineNumber - 1]; - if (line->annotated) - goto end; - - if (step->mtime) { - line->id = fsl_strdup(step->versionHash); - if (line->id == NULL) { - rc = RC(FSL_RC_ERROR, "%s", fsl_strdup); - goto end; - } - line->annotated = true; - } else - line->id = NULL; - - /* -r can return lines with no version, so use root check-in. */ - if (opt->originRid && !line->id) { - line->id = fsl_strdup(cx->root_commit); - line->annotated = true; - } - - ++cx->nlines; -end: - rc = pthread_mutex_unlock(&fnc_mutex); - if (rc) - rc = RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_unlock"); - return rc; -} - -static int -draw_blame(struct fnc_view *view) -{ - struct fnc_blame_view_state *s = &view->state.blame; - struct fnc_blame *blame = &s->blame; - struct fnc_blame_line *blame_line; - regmatch_t *regmatch = &view->regmatch; - struct fnc_colour *c = NULL; - wchar_t *wcstr; - char *line = NULL; - fsl_uuid_str prev_id = NULL, id_str = NULL; - ssize_t linelen; - size_t linesz = 0; - int width, lineno = 0, nprinted = 0; - int rc = 0; - const int idfield = 11; /* Prefix + space. */ - - id_str = fsl_strdup(s->blamed_commit->id); - if (id_str == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - - rewind(blame->f); - werase(view->window); - - if ((line = fsl_mprintf("checkin %s", id_str)) == NULL) { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), - "%s", "fsl_mprintf"); - fsl_free(id_str); - return rc; - } - - rc = formatln(&wcstr, &width, line, view->ncols, 0); - fsl_free(line); - line = NULL; - if (rc) - return rc; - if (screen_is_shared(view)) - wstandout(view->window); - if (s->colour) - c = get_colour(&s->colours, FNC_COMMIT_ID); - if (c) - wattr_on(view->window, COLOR_PAIR(c->scheme), NULL); - waddwstr(view->window, wcstr); - if (c) - wattr_off(view->window, COLOR_PAIR(c->scheme), NULL); - if (screen_is_shared(view)) - wstandend(view->window); - fsl_free(wcstr); - wcstr = NULL; - if (width < view->ncols - 1) - waddch(view->window, '\n'); - - if ((line = fsl_mprintf("[%d/%d] %s%s%s %c", - s->first_line_onscreen - 1 + s->selected_line, blame->nlines, - s->blame_complete ? "" : "annotating... ", - fnc_init.sym ? "/" : "", s->path, - s->blame_complete ? ' ' : SPINNER[s->spin_idx])) == NULL) { - fsl_free(id_str); - return RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), - "%s", "fsl_mprintf"); - } - if (SPINNER[++s->spin_idx] == '\0') - s->spin_idx = 0; - fsl_free(id_str); - rc = formatln(&wcstr, &width, line, view->ncols, 0); - fsl_free(line); - line = NULL; - if (rc) - return rc; - waddwstr(view->window, wcstr); - fsl_free(wcstr); - wcstr = NULL; - if (width < view->ncols - 1) - waddch(view->window, '\n'); - - s->eof = false; - while (nprinted < view->nlines - 2) { - linelen = getline(&line, &linesz, blame->f); - if (linelen == -1) { - if (feof(blame->f)) { - s->eof = true; - break; - } - fsl_free(line); - return RC(ferror(blame->f) ? fsl_errno_to_rc(errno, - FSL_RC_IO) : FSL_RC_IO, "%s", "getline"); - } - if (++lineno < s->first_line_onscreen) - continue; - - if (view->active && nprinted == s->selected_line - 1) - wattr_on(view->window, A_REVERSE, NULL); - - if (blame->nlines > 0) { - blame_line = &blame->lines[lineno - 1]; - if (blame_line->annotated && prev_id && - fsl_uuidcmp(prev_id, blame_line->id) == 0 && - !(view->active && - nprinted == s->selected_line - 1)) { - waddstr(view->window, " "); - } else if (blame_line->annotated) { - char *id_str; - id_str = fsl_strndup(blame_line->id, - idfield - 1); - if (id_str == NULL) { - fsl_free(line); - return RC(FSL_RC_ERROR, "%s", - "fsl_strdup"); - } - if (s->colour) - c = get_colour(&s->colours, - FNC_COMMIT_ID); - if (c) - wattr_on(view->window, - COLOR_PAIR(c->scheme), NULL); - wprintw(view->window, "%.*s", idfield - 1, - id_str); - if (c) - wattr_off(view->window, - COLOR_PAIR(c->scheme), NULL); - fsl_free(id_str); - prev_id = blame_line->id; - } else { - waddstr(view->window, ".........."); - prev_id = NULL; - } - } else { - waddstr(view->window, ".........."); - prev_id = NULL; - } - - if (view->active && nprinted == s->selected_line - 1) - wattr_off(view->window, A_REVERSE, NULL); - waddstr(view->window, " "); - - if (view->ncols <= idfield) { - width = idfield; - wcstr = wcsdup(L""); - if (wcstr == NULL) { - rc = RC(fsl_errno_to_rc(errno, FSL_RC_RANGE), - "%s", "wcsdup"); - fsl_free(line); - return rc; - } - } else if (s->first_line_onscreen + nprinted == s->matched_line - && regmatch->rm_so >= 0 && - regmatch->rm_so < regmatch->rm_eo) { - rc = write_matched_line(&width, line, - view->ncols - idfield, idfield, - view->window, regmatch); - if (rc) { - fsl_free(line); - return rc; - } - width += idfield; - } else { - rc = formatln(&wcstr, &width, line, - view->ncols - idfield, idfield); - waddwstr(view->window, wcstr); - fsl_free(wcstr); - wcstr = NULL; - width += idfield; - } - - if (width <= view->ncols - 1) - waddch(view->window, '\n'); - if (++nprinted == 1) - s->first_line_onscreen = lineno; - } - fsl_free(line); - s->last_line_onscreen = lineno; - - draw_vborder(view); - - return rc; -} - -static int -blame_input_handler(struct fnc_view **new_view, struct fnc_view *view, int ch) -{ - struct fnc_view *diff_view; - struct fnc_blame_view_state *s = &view->state.blame; - int start_col = 0, rc = 0; - - switch (ch) { - case 'q': - s->done = true; - if (s->selected_commit) - fnc_commit_artifact_close(s->selected_commit); - break; - case 'c': - s->colour = !s->colour; - break; - case 'g': - if (!fnc_home(view)) - break; - case KEY_HOME: - s->selected_line = 1; - s->first_line_onscreen = 1; - break; - case KEY_END: - case 'G': - if (s->blame.nlines < view->nlines - 2) { - s->selected_line = s->blame.nlines; - s->first_line_onscreen = 1; - } else { - s->selected_line = view->nlines - 2; - s->first_line_onscreen = s->blame.nlines - - (view->nlines - 3); - } - break; - case KEY_UP: - case 'k': - if (s->selected_line > 1) - --s->selected_line; - else if (s->selected_line == 1 && s->first_line_onscreen > 1) - --s->first_line_onscreen; - break; - case KEY_PPAGE: - case CTRL('b'): - if (s->first_line_onscreen == 1) { - s->selected_line = 1; - break; - } - if (s->first_line_onscreen > view->nlines - 2) - s->first_line_onscreen -= (view->nlines - 2); - else - s->first_line_onscreen = 1; - break; - case KEY_DOWN: - case 'j': - if (s->selected_line < view->nlines - 2 && - s->first_line_onscreen + - s->selected_line <= s->blame.nlines) - ++s->selected_line; - else if (s->last_line_onscreen < s->blame.nlines) - ++s->first_line_onscreen; - break; - case 'b': - case 'p': { - fsl_uuid_cstr id = NULL; - id = get_selected_commit_id(s->blame.lines, s->blame.nlines, - s->first_line_onscreen, s->selected_line); - if (id == NULL) - break; - if (ch == 'p') { - fsl_cx *f = fcli_cx(); - fsl_db *db = fsl_needs_repo(f); - fsl_deck d = fsl_deck_empty; - fsl_id_t rid = fsl_uuid_to_rid(f, id); - fsl_uuid_str pid = fsl_db_g_text(db, NULL, - "SELECT uuid FROM plink, blob WHERE plink.cid=%d " - "AND blob.rid=plink.pid AND plink.isprim", rid); - if (pid == NULL) - break; - /* Check file exists in parent check-in. */ - rc = fsl_deck_load_sym(f, &d, pid, FSL_SATYPE_CHECKIN); - if (rc) { - fsl_deck_finalize(&d); - fsl_free(pid); - return RC(rc, "%s", "fsl_deck_load_sym"); - } - rc = fsl_deck_F_rewind(&d); - if (rc) { - fsl_deck_finalize(&d); - fsl_free(pid); - return RC(rc, "%s", "fsl_deck_F_rewind"); - } - if (fsl_deck_F_search(&d, s->path + - (fnc_init.sym ? 0 : 1)) == NULL) { - char *m = fsl_mprintf("-- %s not in [%.12s] --", - s->path + (fnc_init.sym ? 0 : 1), pid); - if (m == NULL) - rc = RC(FSL_RC_ERROR, "%s", - "fsl_mprintf"); - wattr_on(view->window, A_BOLD, NULL); - mvwaddstr(view->window, - view->start_ln + view->nlines - 1, 0, m); - wclrtoeol(view->window); - wattr_off(view->window, A_BOLD, NULL); - update_panels(); - doupdate(); - sleep(1); - fsl_deck_finalize(&d); - fsl_free(pid); - fsl_free(m); - break; - } - rc = fnc_commit_qid_alloc(&s->blamed_commit, pid); - if (rc) - return rc; - } else { - if (!fsl_uuidcmp(id, s->blamed_commit->id)) - break; - rc = fnc_commit_qid_alloc(&s->blamed_commit, id); - } - if (rc) - break; - s->done = true; - rc = stop_blame(&s->blame); - s->done = false; - if (rc) - break; - CONCAT(STAILQ, _INSERT_HEAD)(&s->blamed_commits, - s->blamed_commit, entry); - rc = run_blame(view); - if (rc) - break; - break; - } - case KEY_BACKSPACE: - case 'B': { - struct fnc_commit_qid *first; - first = CONCAT(STAILQ, _FIRST)(&s->blamed_commits); - if (!fsl_uuidcmp(first->id, s->commit_id)) - break; - s->done = true; - rc = stop_blame(&s->blame); - s->done = false; - if (rc) - break; - CONCAT(STAILQ, _REMOVE_HEAD)(&s->blamed_commits, entry); - fnc_commit_qid_free(s->blamed_commit); - s->blamed_commit = CONCAT(STAILQ, _FIRST)(&s->blamed_commits); - rc = run_blame(view); - if (rc) - break; - break; - } - case KEY_ENTER: - case '\r': { - fsl_cx *f = fcli_cx(); - struct fnc_commit_artifact *commit = NULL; - fsl_stmt *q = NULL; - fsl_uuid_cstr id = NULL; - - id = get_selected_commit_id(s->blame.lines, s->blame.nlines, - s->first_line_onscreen, s->selected_line); - if (id == NULL) - break; - if (s->selected_commit) - fnc_commit_artifact_close(s->selected_commit); - if (rc) - break; - q = fsl_stmt_malloc(); - rc = commit_builder(&commit, fsl_uuid_to_rid(f, id), q); - fsl_stmt_finalize(q); - if (rc) { - fnc_commit_artifact_close(commit); - break; - } - if (view_is_parent(view)) - start_col = view_split_start_col(view->start_col); - diff_view = view_open(0, 0, 0, start_col, FNC_VIEW_DIFF); - if (diff_view == NULL) { - fnc_commit_artifact_close(commit); - rc = RC(FSL_RC_ERROR, "%s", "view_open"); - break; - } - rc = open_diff_view(diff_view, commit, DIFF_DEF_CTXT, - fnc_init.ws, fnc_init.invert, !fnc_init.quiet, NULL, true, - NULL); - s->selected_commit = commit; - if (rc) { - fnc_commit_artifact_close(commit); - view_close(diff_view); - break; - } - view->active = false; - diff_view->active = true; - if (view_is_parent(view)) { - rc = view_close_child(view); - if (rc) - break; - view_set_child(view, diff_view); - view->focus_child = true; - } else - *new_view = diff_view; - if (rc) - break; - break; - } - case KEY_NPAGE: - case CTRL('f'): - case ' ': - if (s->last_line_onscreen >= s->blame.nlines && s->selected_line - >= MIN(s->blame.nlines, view->nlines - 2)) - break; - if (s->last_line_onscreen >= s->blame.nlines && - s->selected_line < view->nlines - 2) { - s->selected_line = MIN(s->blame.nlines, - view->nlines - 2); - break; - } - if (s->last_line_onscreen + view->nlines - 2 <= s->blame.nlines) - s->first_line_onscreen += view->nlines - 2; - else - s->first_line_onscreen = - s->blame.nlines - (view->nlines - 3); - break; - case KEY_RESIZE: - if (s->selected_line > view->nlines - 2) { - s->selected_line = MIN(s->blame.nlines, - view->nlines - 2); - } - break; - default: - break; - } - return rc; -} - -static int -blame_search_init(struct fnc_view *view) -{ - struct fnc_blame_view_state *s = &view->state.blame; - - s->matched_line = 0; - return 0; -} - -static int -blame_search_next(struct fnc_view *view) -{ - struct fnc_blame_view_state *s = &view->state.blame; - char *line = NULL; - ssize_t linelen; - size_t linesz = 0; - int lineno; - - if (view->searching == SEARCH_DONE) { - view->search_status = SEARCH_CONTINUE; - return 0; - } - - if (s->matched_line) { - if (view->searching == SEARCH_FORWARD) - lineno = s->matched_line + 1; - else - lineno = s->matched_line - 1; - } else { - if (view->searching == SEARCH_FORWARD) - lineno = 1; - else - lineno = s->blame.nlines; - } - - while (1) { - off_t offset; - - if (lineno <= 0 || lineno > s->blame.nlines) { - if (s->matched_line == 0) { - view->search_status = SEARCH_CONTINUE; - break; - } - - if (view->searching == SEARCH_FORWARD) - lineno = 1; - else - lineno = s->blame.nlines; - } - - offset = s->blame.line_offsets[lineno - 1]; - if (fseeko(s->blame.f, offset, SEEK_SET) != 0) { - fsl_free(line); - return RC(fsl_errno_to_rc(errno, FSL_RC_IO), - "%s", "fseeko"); - } - linelen = getline(&line, &linesz, s->blame.f); - if (linelen != -1 && regexec(&view->regex, line, 1, - &view->regmatch, 0) == 0) { - view->search_status = SEARCH_CONTINUE; - s->matched_line = lineno; - break; - } - if (view->searching == SEARCH_FORWARD) - ++lineno; - else - --lineno; - } - fsl_free(line); - - if (s->matched_line) { - s->first_line_onscreen = s->matched_line; - s->selected_line = 1; - } - - return 0; -} - -static fsl_uuid_cstr -get_selected_commit_id(struct fnc_blame_line *lines, int nlines, - int first_line_onscreen, int selected_line) -{ - struct fnc_blame_line *line; - - if (nlines <= 0) - return NULL; - - line = &lines[first_line_onscreen - 1 + selected_line - 1]; - - return line->id; -} - -static int -fnc_commit_qid_alloc(struct fnc_commit_qid **qid, fsl_uuid_cstr id) -{ - int rc = 0; - - *qid = calloc(1, sizeof(**qid)); - if (*qid == NULL) - return RC(fsl_errno_to_rc(errno, FSL_RC_ERROR), "%s", "calloc"); - - (*qid)->id = fsl_strdup(id); - if ((*qid)->id == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - fnc_commit_qid_free(*qid); - *qid = NULL; - } - - return rc; -} - -static int -close_blame_view(struct fnc_view *view) -{ - struct fnc_blame_view_state *s = &view->state.blame; - struct fsl_list_state st = { FNC_COLOUR_OBJ }; - int rc = 0; - - rc = stop_blame(&s->blame); - - while (!CONCAT(STAILQ, _EMPTY)(&s->blamed_commits)) { - struct fnc_commit_qid *blamed_commit; - blamed_commit = CONCAT(STAILQ, _FIRST)(&s->blamed_commits); - CONCAT(STAILQ, _REMOVE_HEAD)(&s->blamed_commits, entry); - fnc_commit_qid_free(blamed_commit); - } - - fsl_free(s->path); - fsl_list_clear(&s->colours, fsl_list_object_free, &st); - - return rc; -} - -static int -stop_blame(struct fnc_blame *blame) -{ - int idx, rc = 0; - - if (blame->thread_id) { - int retval; - rc = pthread_mutex_unlock(&fnc_mutex); - if (rc) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_unlock"); - rc = pthread_join(blame->thread_id, (void **)&retval); - if (rc) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_join"); - rc = pthread_mutex_lock(&fnc_mutex); - if (rc) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_lock"); - if (!rc && fsl_cx_err_get_e(fcli_cx())->code == FSL_RC_BREAK) { - rc = 0; - fcli_err_reset(); - } - blame->thread_id = 0; - } - if (blame->f) { - if (fclose(blame->f) == EOF && rc == 0) - rc = RC(fsl_errno_to_rc(errno, FSL_RC_IO), "%s", - fclose); - blame->f = NULL; - } - if (blame->lines) { - for (idx = 0; idx < blame->nlines; ++idx) - fsl_free(blame->lines[idx].id); - fsl_free(blame->lines); - blame->lines = NULL; - } - - fsl_free(blame->cb_cx.root_commit); - blame->cb_cx.root_commit = NULL; - fsl_free(blame->cb_cx.commit_id); - blame->cb_cx.commit_id = NULL; - fsl_free(blame->line_offsets); - - return rc; -} - -static int -cancel_blame(void *state) -{ - int *done = state; - int rc = 0; - - rc = pthread_mutex_lock(&fnc_mutex); - if (rc) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_unlock"); - - if (*done) - rc = fcli_err_set(FSL_RC_BREAK, "user quit"); - - rc = pthread_mutex_unlock(&fnc_mutex); - if (rc) - return RC(fsl_errno_to_rc(rc, FSL_RC_ACCESS), - "%s", "pthread_mutex_lock"); - - return rc; -} - -static void -fnc_commit_qid_free(struct fnc_commit_qid *qid) -{ - fsl_free(qid->id); - fsl_free(qid); -} - -static int -cmd_branch(fcli_command const *argv) -{ - struct fnc_view *view; - char *glob = NULL; - double dateline; - int branch_flags, rc = 0, when = 0; - - rc = fcli_process_flags(argv->flags); - if (rc || (rc = fcli_has_unused_flags(false))) - return rc; - - branch_flags = BRANCH_LS_OPEN_CLOSED; - if (fnc_init.open && fnc_init.closed) - return RC(FSL_RC_MISUSE, "%s", - "--open and --close are mutually exclusive options"); - else if (fnc_init.open) - branch_flags = BRANCH_LS_OPEN_ONLY; - else if (fnc_init.closed) - branch_flags = BRANCH_LS_CLOSED_ONLY; - - if (fnc_init.sort) { - if (!fsl_strcmp(fnc_init.sort, "mru")) - FLAG_SET(branch_flags, BRANCH_SORT_MTIME); - else if (!fsl_strcmp(fnc_init.sort, "state")) - FLAG_SET(branch_flags, BRANCH_SORT_STATUS); - else - return RC(FSL_RC_MISUSE, "invalid sort order: %s", - fnc_init.sort); - } - if (fnc_init.noprivate) - FLAG_SET(branch_flags, BRANCH_LS_NO_PRIVATE); - if (fnc_init.reverse) - FLAG_SET(branch_flags, BRANCH_SORT_REVERSE); - - if (fnc_init.after && fnc_init.before) { - return RC(FSL_RC_MISUSE, "%s", - "--before and --after are mutually exclusive options"); - } else if (fnc_init.after || fnc_init.before) { - const char *d = NULL; - d = fnc_init.after ? fnc_init.after : fnc_init.before; - when = fnc_init.after ? 1 : -1; - rc = fnc_date_to_mtime(&dateline, d, when); - if (rc) - return rc; - } - glob = fsl_strdup(fcli_next_arg(true)); - - init_curses(); - - view = view_open(0, 0, 0, 0, FNC_VIEW_BRANCH); - if (view == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "view_open"); - goto end; - } - - rc = open_branch_view(view, branch_flags, glob, dateline, when); - if (rc) - goto end; - - rc = view_loop(view); -end: - fnc_free_branches(&view->state.branch); - fsl_free(glob); - return rc; -} - -static int -open_branch_view(struct fnc_view *view, int branch_flags, const char *glob, - double dateline, int when) -{ - struct fnc_branch_view_state *s = &view->state.branch; - int rc = 0; - - s->selected_branch = 0; - s->colour = !fnc_init.nocolour && has_colors(); - s->branch_flags = branch_flags; - s->branch_glob = glob; - s->dateline = dateline; - s->when = when; - - rc = fnc_load_branches(s); - if (rc) - return rc; - - if (s->colour) - init_pair(FNC_COMMIT_ID, COLOR_GREEN, -1); - - view->show = show_branch_view; - view->input = branch_input_handler; - view->close = close_branch_view; - view->search_init = branch_search_init; - view->search_next = branch_search_next; - - return 0; -} - -static int -fnc_load_branches(struct fnc_branch_view_state *s) -{ - fsl_cx *const f = fcli_cx(); - fsl_buffer sql = fsl_buffer_empty; - fsl_stmt *stmt = NULL; - char *curr_branch = NULL; - fsl_id_t ckoutrid; - int rc = 0; - - rc = create_tmp_branchlist_table(); - if (rc) - goto end; - - TAILQ_INIT(&s->branches); - s->nbranches = 0; - - switch (FLAG_CHK(s->branch_flags, BRANCH_LS_BITMASK)) { - case BRANCH_LS_OPEN_CLOSED: - rc = fsl_buffer_append(&sql, - "SELECT name, isprivate, isclosed, mtime" - " FROM tmp_brlist WHERE 1", -1); - break; - case BRANCH_LS_OPEN_ONLY: - rc = fsl_buffer_append(&sql, - "SELECT name, isprivate, isclosed, mtime" - " FROM tmp_brlist WHERE NOT isclosed", -1); - break; - case BRANCH_LS_CLOSED_ONLY: - rc = fsl_buffer_append(&sql, - "SELECT name, isprivate, isclosed, mtime" - " FROM tmp_brlist WHERE isclosed", -1); - break; - } - if (rc) - goto end; - - if (s->branch_glob) { - char *like = fsl_mprintf("%%%%%s%%%%", s->branch_glob); - rc = fsl_buffer_appendf(&sql, " AND name LIKE %Q", like); - fsl_free(like); - } - if (FLAG_CHK(s->branch_flags, BRANCH_LS_NO_PRIVATE)) - rc = fsl_buffer_append(&sql, " AND NOT isprivate", -1); - if (FLAG_CHK(s->branch_flags, BRANCH_SORT_MTIME)) - rc = fsl_buffer_append(&sql, " ORDER BY -mtime", -1); - else if (FLAG_CHK(s->branch_flags, BRANCH_SORT_STATUS)) - rc = fsl_buffer_append(&sql, " ORDER BY isclosed", -1); - else - rc = fsl_buffer_append(&sql, " ORDER BY name COLLATE nocase", - -1); - - if (!rc && (FLAG_CHK(s->branch_flags, BRANCH_SORT_REVERSE))) - rc = fsl_buffer_append(&sql," DESC", -1); - - stmt = fsl_stmt_malloc(); - if (stmt == NULL) - goto end; - if (!rc) - rc = fsl_cx_prepare(f, stmt, fsl_buffer_cstr(&sql)); - - fsl_ckout_version_info(f, &ckoutrid, NULL); - curr_branch = fsl_db_g_text(fsl_needs_repo(f), NULL, - "SELECT value FROM tagxref WHERE rid=%d AND tagid=%d", ckoutrid, 8); - - while (fsl_stmt_step(stmt) == FSL_RC_STEP_ROW) { - struct fnc_branch *new_branch; - struct fnc_branchlist_entry *be; - const char *brname = fsl_stmt_g_text(stmt, 0, NULL); - bool private = (curr_branch && fsl_stmt_g_int32(stmt, 1) == 1); - bool open = fsl_stmt_g_int32(stmt, 2) == 0; - double mtime = fsl_stmt_g_int64(stmt, 3); - bool curr = curr_branch && !fsl_strcmp(curr_branch, brname); - if ((s->when > 0 && mtime < s->dateline) || - (s->when < 0 && mtime > s->dateline)) - continue; - alloc_branch(&new_branch, brname, mtime, open, private, curr); - fnc_branchlist_insert(&be, &s->branches, new_branch); - if (be) - be->idx = s->nbranches++; - } - s->first_branch_onscreen = TAILQ_FIRST(&s->branches); - - if (!stmt->rowCount) - rc = RC(FSL_RC_BREAK, "no matching records: %s", - s->branch_glob); -end: - fsl_stmt_finalize(stmt); - fsl_free(curr_branch); - fsl_buffer_clear(&sql); - return rc; -} - -static int -create_tmp_branchlist_table(void) -{ - fsl_cx *const f = fcli_cx(); - fsl_db *db = fsl_needs_ckout(f); - static const char tmp_branchlist_table[] = - "CREATE TEMP TABLE IF NOT EXISTS tmp_brlist AS " - "SELECT tagxref.value AS name," - " max(event.mtime) AS mtime," - " EXISTS(SELECT 1 FROM tagxref AS tx WHERE tx.rid=tagxref.rid" - " AND tx.tagid=(SELECT tagid FROM tag WHERE tagname='closed')" - " AND tx.tagtype > 0) AS isclosed," - " (SELECT tagxref.value FROM plink CROSS JOIN tagxref" - " WHERE plink.pid=event.objid" - " AND tagxref.rid=plink.cid" - " AND tagxref.tagid=(SELECT tagid FROM tag WHERE tagname='branch')" - " AND tagtype>0) AS mergeto," - " count(*) AS nckin," - " (SELECT uuid FROM blob WHERE rid=tagxref.rid) AS ckin," - " event.bgcolor AS bgclr," - " EXISTS(SELECT 1 FROM private WHERE rid=tagxref.rid) AS isprivate " - "FROM tagxref, tag, event " - "WHERE tagxref.tagid=tag.tagid" - " AND tagxref.tagtype>0" - " AND tag.tagname='branch'" - " AND event.objid=tagxref.rid " - "GROUP BY 1;"; - int rc = 0; - - if (!db) - return RC(FSL_RC_NOT_A_CKOUT, "%s", "fsl_needs_ckout"); - rc = fsl_db_exec(db, tmp_branchlist_table); - - return rc ? RC(fsl_cx_uplift_db_error2(f, db, rc), "%s", "fsl_db_exec") - : rc; -} - -static int -alloc_branch(struct fnc_branch **branch, const char *name, double mtime, - bool open, bool priv, bool curr) -{ - fsl_uuid_str id = NULL; - char iso8601[ISO8601_TIMESTAMP], *date = NULL; - int rc = 0; - - *branch = calloc(1, sizeof(**branch)); - if (*branch == NULL) - return RC(FSL_RC_ERROR, "%s", "calloc"); - - rc = fsl_sym_to_uuid(fcli_cx(), name, FSL_SATYPE_ANY, &id, NULL); - if (rc || id == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_sym_to_uuid"); - fnc_branch_close(*branch); - *branch = NULL; - return rc; - } - - fsl_julian_to_iso8601(mtime, iso8601, false); - date = fsl_mprintf("%.*s", ISO8601_DATE_ONLY, iso8601); - - (*branch)->id = id; - (*branch)->name = fsl_strdup(name); - (*branch)->date = date; - (*branch)->open = open; - (*branch)->private = priv; - (*branch)->current = curr; - if ((*branch)->name == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "fsl_strdup"); - fnc_branch_close(*branch); - *branch = NULL; - } - - return rc; -} - -static int -fnc_branchlist_insert(struct fnc_branchlist_entry **newp, - struct fnc_branchlist_head *branches, struct fnc_branch *branch) -{ - struct fnc_branchlist_entry *new, *be; - - *newp = NULL; - - new = fsl_malloc(sizeof(*new)); - if (new == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_malloc"); - new->branch = branch; - *newp = new; - - be = TAILQ_LAST(branches, fnc_branchlist_head); - if (!be) { - /* Empty list; add first branch. */ - TAILQ_INSERT_HEAD(branches, new, entries); - return 0; - } - - /* - * Deduplicate (extremely unlikely or impossible?) entries on insert. - * Don't force lexicographical order; we already retrieved the branch - * names from the database using a query to obtain (a) lexicographical - * or (b) user-specified sorted results (i.e., MRU or LRU). - */ - while (be) { - if (!fsl_strcmp(be->branch->name, new->branch->name)) { - /* Duplicate entry. */ - fsl_free(new); - *newp = NULL; - return 0; - } - be = TAILQ_PREV(be, fnc_branchlist_head, entries); - } - - /* No duplicates; add to end of list. */ - TAILQ_INSERT_TAIL(branches, new, entries); - return 0; -} - -static int -show_branch_view(struct fnc_view *view) -{ - struct fnc_branch_view_state *s = &view->state.branch; - struct fnc_branchlist_entry *be; - char *line = NULL; - wchar_t *wline; - int limit, n, width, rc = 0; - - werase(view->window); - s->ndisplayed = 0; - - limit = view->nlines; - if (limit == 0) - return rc; - - be = s->first_branch_onscreen; - - if ((line = fsl_mprintf("branches [%d/%d]", be->idx + s->selected + 1, - s->nbranches)) == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_mprintf"); - - rc = formatln(&wline, &width, line, view->ncols, 0); - if (rc) { - fsl_free(line); - return rc; - } - if (screen_is_shared(view)) - wstandout(view->window); - waddwstr(view->window, wline); - if (screen_is_shared(view)) - wstandend(view->window); - fsl_free(wline); - wline = NULL; - fsl_free(line); - line = NULL; - if (width < view->ncols - 1) - waddch(view->window, '\n'); - if (--limit <= 0) - return rc; - - n = 0; - while (be && limit > 0) { - char *line = NULL; - - line = fsl_mprintf(" %c%s%s%s%s%s%s%c %s", - be->branch->open ? '+' : '-', be->branch->name, - be->branch->private ? "*" : "", - be->branch->current ? "@" : "", s->show_id ? " [" : "", - s->show_id ? be->branch->id : "", s->show_id ? "]" : "", - s->show_date ? ':' : 0, - s->show_date ? be->branch->date : 0); - if (line == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_mprintf"); - - rc = formatln(&wline, &width, line, view->ncols, 0); - if (rc) { - fsl_free(line); - return rc; - } - - if (n == s->selected) { - if (view->active) - wattr_on(view->window, A_REVERSE, NULL); - s->selected_branch = be; - } - if (s->colour) - wattr_on(view->window, COLOR_PAIR(FNC_COMMIT_ID), NULL); - waddwstr(view->window, wline); - if (s->colour) - wattr_off(view->window, COLOR_PAIR(FNC_COMMIT_ID), NULL); - if (width < view->ncols - 1) - waddch(view->window, '\n'); - if (n == s->selected && view->active) - wattr_off(view->window, A_REVERSE, NULL); - - fsl_free(line); - fsl_free(wline); - wline = NULL; - ++n; - ++s->ndisplayed; - --limit; - s->last_branch_onscreen = be; - be = TAILQ_NEXT(be, entries); - } - - draw_vborder(view); - return rc; -} - -static int -branch_input_handler(struct fnc_view **new_view, struct fnc_view *view, int ch) -{ - struct fnc_branch_view_state *s = &view->state.branch; - struct fnc_view *timeline_view, *tree_view; - struct fnc_branchlist_entry *be; - int start_col = 0, n, rc = 0; - - switch (ch) { - case 'c': - s->colour = !s->colour; - break; - case 'd': - s->show_date = !s->show_date; - break; - case 'i': - s->show_id = !s->show_id; - break; - case KEY_ENTER: - case '\r': - case ' ': - if (!s->selected_branch) - break; - if (view_is_parent(view)) - start_col = view_split_start_col(view->start_col); - rc = tl_branch_entry(&timeline_view, start_col, - s->selected_branch); - view->active = false; - timeline_view->active = true; - if (view_is_parent(view)) { - rc = view_close_child(view); - if (rc) - return rc; - view_set_child(view, timeline_view); - view->focus_child = true; - } else - *new_view = timeline_view; - break; - case 't': - if (!s->selected_branch) - break; - if (view_is_parent(view)) - start_col = view_split_start_col(view->start_col); - rc = browse_branch_tree(&tree_view, start_col, - s->selected_branch); - if (rc || tree_view == NULL) - break; - view->active = false; - tree_view->active = true; - if (view_is_parent(view)) { - rc = view_close_child(view); - if (rc) - return rc; - view_set_child(view, tree_view); - view->focus_child = true; - } else - *new_view = tree_view; - break; - case 'g': - if (!fnc_home(view)) - break; - /* FALL THROUGH */ - case KEY_HOME: - s->selected = 0; - s->first_branch_onscreen = TAILQ_FIRST(&s->branches); - break; - case KEY_END: - case 'G': - s->selected = 0; - be = TAILQ_LAST(&s->branches, fnc_branchlist_head); - for (n = 0; n < view->nlines - 1; ++n) { - if (be == NULL) - break; - s->first_branch_onscreen = be; - be = TAILQ_PREV(be, fnc_branchlist_head, entries); - } - if (n > 0) - s->selected = n - 1; - break; - case KEY_UP: - case 'k': - if (s->selected > 0) { - --s->selected; - break; - } - branch_scroll_up(s, 1); - break; - case KEY_DOWN: - case 'j': - if (s->selected < s->ndisplayed - 1) { - ++s->selected; - break; - } - if (TAILQ_NEXT(s->last_branch_onscreen, entries) == NULL) - /* Reached last entry. */ - break; - branch_scroll_down(s, 1); - break; - case KEY_PPAGE: - case CTRL('b'): - if (s->first_branch_onscreen == TAILQ_FIRST(&s->branches)) - s->selected = 0; - branch_scroll_up(s, MAX(0, view->nlines - 1)); - break; - case KEY_NPAGE: - case CTRL('f'): - if (TAILQ_NEXT(s->last_branch_onscreen, entries) == NULL) { - /* No more entries off-page; move cursor down. */ - if (s->selected < s->ndisplayed - 1) - s->selected = s->ndisplayed - 1; - break; - } - branch_scroll_down(s, view->nlines - 1); - break; - case CTRL('l'): - case 'R': - fnc_free_branches(s); - s->branch_glob = NULL; /* Shared pointer. */ - s->when = 0; - s->branch_flags = BRANCH_LS_OPEN_CLOSED; - rc = fnc_load_branches(s); - break; - case KEY_RESIZE: - if (view->nlines >= 2 && s->selected >= view->nlines - 1) - s->selected = view->nlines - 2; - break; - default: - break; - } - - return rc; -} - -static int -tl_branch_entry(struct fnc_view **new_view, int start_col, - struct fnc_branchlist_entry *be) -{ - struct fnc_view *timeline_view; - fsl_id_t rid; - int rc = 0; - - *new_view = NULL; - - rid = fsl_uuid_to_rid(fcli_cx(), be->branch->id); - if (rid < 0) - return RC(rc, "%s", "fsl_uuid_to_rid"); - - timeline_view = view_open(0, 0, 0, start_col, FNC_VIEW_TIMELINE); - if (timeline_view == NULL) { - rc = RC(FSL_RC_ERROR, "%s", "view_open"); - goto end; - } - - rc = open_timeline_view(timeline_view, rid, "/"); -end: - if (rc) - view_close(timeline_view); - else - *new_view = timeline_view; - return rc; -} - -static int -browse_branch_tree(struct fnc_view **new_view, int start_col, - struct fnc_branchlist_entry *be) -{ - struct fnc_view *tree_view; - fsl_id_t rid; - int rc = 0; - - *new_view = NULL; - - rid = fsl_uuid_to_rid(fcli_cx(), be->branch->id); - if (rid < 0) - return RC(rc, "%s", "fsl_uuid_to_rid"); - - tree_view = view_open(0, 0, 0, start_col, FNC_VIEW_TREE); - if (tree_view == NULL) - return RC(FSL_RC_ERROR, "%s", "view_open"); - - rc = open_tree_view(tree_view, "/", rid); - if (!rc) - *new_view = tree_view; - return rc; -} - -static void -branch_scroll_up(struct fnc_branch_view_state *s, int maxscroll) -{ - struct fnc_branchlist_entry *be; - int idx = 0; - - if (s->first_branch_onscreen == TAILQ_FIRST(&s->branches)) - return; - - be = TAILQ_PREV(s->first_branch_onscreen, fnc_branchlist_head, entries); - while (idx++ < maxscroll) { - if (be == NULL) - break; - s->first_branch_onscreen = be; - be = TAILQ_PREV(be, fnc_branchlist_head, entries); - } -} - -static void -branch_scroll_down(struct fnc_branch_view_state *s, int maxscroll) -{ - struct fnc_branchlist_entry *next, *last; - int idx = 0; - - if (s->first_branch_onscreen) - next = TAILQ_NEXT(s->first_branch_onscreen, entries); - else - next = TAILQ_FIRST(&s->branches); - - last = s->last_branch_onscreen; - while (next && last && idx++ < maxscroll) { - last = TAILQ_NEXT(last, entries); - if (last) { - s->first_branch_onscreen = next; - next = TAILQ_NEXT(next, entries); - } - } -} - -static int -branch_search_init(struct fnc_view *view) -{ - struct fnc_branch_view_state *s = &view->state.branch; - - s->matched_branch = NULL; - return 0; -} - -static int -branch_search_next(struct fnc_view *view) -{ - struct fnc_branch_view_state *s = &view->state.branch; - struct fnc_branchlist_entry *be = NULL; - - if (view->searching == SEARCH_DONE) { - view->search_status = SEARCH_CONTINUE; - return 0; - } - - if (s->matched_branch) { - if (view->searching == SEARCH_FORWARD) { - if (s->selected_branch) - be = TAILQ_NEXT(s->selected_branch, entries); - else - be = TAILQ_PREV(s->selected_branch, - fnc_branchlist_head, entries); - } else { - if (s->selected_branch == NULL) - be = TAILQ_LAST(&s->branches, - fnc_branchlist_head); - else - be = TAILQ_PREV(s->selected_branch, - fnc_branchlist_head, entries); - } - } else { - if (view->searching == SEARCH_FORWARD) - be = TAILQ_FIRST(&s->branches); - else - be = TAILQ_LAST(&s->branches, fnc_branchlist_head); - } - - while (1) { - if (be == NULL) { - if (s->matched_branch == NULL) { - view->search_status = SEARCH_CONTINUE; - return 0; - } - if (view->searching == SEARCH_FORWARD) - be = TAILQ_FIRST(&s->branches); - else - be = TAILQ_LAST(&s->branches, - fnc_branchlist_head); - } - - if (match_branchlist_entry(be, &view->regex)) { - view->search_status = SEARCH_CONTINUE; - s->matched_branch = be; - break; - } - - if (view->searching == SEARCH_FORWARD) - be = TAILQ_NEXT(be, entries); - else - be = TAILQ_PREV(be, fnc_branchlist_head, entries); - } - - if (s->matched_branch) { - s->first_branch_onscreen = s->matched_branch; - s->selected = 0; - } - - return 0; -} - -static int -match_branchlist_entry(struct fnc_branchlist_entry *be, regex_t *regex) -{ - regmatch_t regmatch; - - return regexec(regex, be->branch->name, 1, ®match, 0) == 0; -} - -static int -close_branch_view(struct fnc_view *view) -{ - struct fnc_branch_view_state *s = &view->state.branch; - - fnc_free_branches(s); - - return 0; -} - -static void -fnc_free_branches(struct fnc_branch_view_state *s) -{ - struct fnc_branchlist_entry *be; - - while (!TAILQ_EMPTY(&s->branches)) { - be = TAILQ_FIRST(&s->branches); - TAILQ_REMOVE(&s->branches, be, entries); - fnc_branch_close(be->branch); - fsl_free(be); - } -} - -static void -fnc_branch_close(struct fnc_branch *branch) -{ - fsl_free(branch->name); - fsl_free(branch->date); - fsl_free(branch->id); - fsl_free(branch); -} - -/* - * Assign path to **inserted->path, with optional ->data assignment, and insert - * in lexicographically sorted order into the doubly-linked list rooted at - * *pathlist. If path is not unique, return without adding a duplicate entry. - */ -static int -fnc_pathlist_insert(struct fnc_pathlist_entry **inserted, - struct fnc_pathlist_head *pathlist, const char *path, void *data) -{ - struct fnc_pathlist_entry *new, *pe; - int rc = 0; - - if (inserted) - *inserted = NULL; - - new = fsl_malloc(sizeof(*new)); - if (new == NULL) - return RC(FSL_RC_ERROR, "%s", "fsl_malloc"); - new->path = path; - new->pathlen = fsl_strlen(path); - new->data = data; - - /* - * Most likely, supplied paths will be sorted (e.g., fnc diff *.c), so - * post-order traversal will be more efficient when inserting entries. - */ - pe = TAILQ_LAST(pathlist, fnc_pathlist_head); - while (pe) { - int cmp = fnc_path_cmp(pe->path, new->path, pe->pathlen, - new->pathlen); - if (cmp == 0) { - fsl_free(new); /* Duplicate path; don't insert. */ - return rc; - } else if (cmp < 0) { - TAILQ_INSERT_AFTER(pathlist, pe, new, entry); - if (inserted) - *inserted = new; - return rc; - } - pe = TAILQ_PREV(pe, fnc_pathlist_head, entry); - } - - TAILQ_INSERT_HEAD(pathlist, new, entry); - if (inserted) - *inserted = new; - return rc; -} - -static int -fnc_path_cmp(const char *path1, const char *path2, size_t len1, size_t len2) -{ - size_t minlen; - size_t idx = 0; - - /* Trim any leading path separators. */ - while (path1[0] == '/') { - ++path1; - --len1; - } - while (path2[0] == '/') { - ++path2; - --len2; - } - minlen = MIN(len1, len2); - - /* Skip common prefix. */ - while (idx < minlen && path1[idx] == path2[idx]) - ++idx; - - /* Are path lengths exactly equal (exluding path separators)? */ - if (len1 == len2 && idx >= minlen) - return 0; - - /* Trim any redundant trailing path seperators. */ - while (path1[idx] == '/' && path1[idx + 1] == '/') - ++path1; - while (path2[idx] == '/' && path2[idx + 1] == '/') - ++path2; - - /* Ignore trailing path separators. */ - if (path1[idx] == '/' && path1[idx + 1] == '\0' && path2[idx] == '\0') - return 0; - if (path2[idx] == '/' && path2[idx + 1] == '\0' && path1[idx] == '\0') - return 0; - - /* Order children in subdirectories directly after their parents. */ - if (path1[idx] == '/' && path2[idx] == '\0') - return 1; - if (path2[idx] == '/' && path1[idx] == '\0') - return -1; - if (path1[idx] == '/' && path2[idx] != '\0') - return -1; - if (path2[idx] == '/' && path1[idx] != '\0') - return 1; - - /* Character immediately after the common prefix determines order. */ - return (unsigned char)path1[idx] < (unsigned char)path2[idx] ? -1 : 1; -} - -static void -fnc_pathlist_free(struct fnc_pathlist_head *pathlist) -{ - struct fnc_pathlist_entry *pe; - - while ((pe = TAILQ_FIRST(pathlist)) != NULL) { - TAILQ_REMOVE(pathlist, pe, entry); - free(pe); - } -} - -static void -fnc_show_version(void) -{ - printf("%s %s\n", fcli_progname(), PRINT_VERSION); -} - -static int -strtonumcheck(int *ret, const char *nstr, const int min, const int max) -{ - const char *ptr; - int n; - - ptr = NULL; - errno = 0; - - n = strtonum(nstr, min, max, &ptr); - if (errno == ERANGE) - return RC(FSL_RC_RANGE, " out of range: -n|--limit=%s [%s]", - nstr, ptr); - else if (errno != 0 || errno == EINVAL) - return RC(FSL_RC_MISUSE, " not a number: -n|--limit=%s [%s]", - nstr, ptr); - else if (ptr && *ptr != '\0') - return RC(FSL_RC_MISUSE, - "invalid char in : -n|--limit=%s [%s]", nstr, ptr); - - *ret = n; - return 0; -} - -/* - * Attempt to parse string d, which must resemble either an ISO8601 formatted - * date (e.g., 2021-10-10, 2020-01-01T10:10:10), disgregarding any trailing - * garbage or space characters such that "2021-10-10x" or "2020-01-01 10:10:10" - * will pass, or an _unambiguous_ DD/MM/YYYY or MM/DD/YYYY formatted date. Upon - * success, use when to determine which time component to add to the date (i.e., - * 1 sec before or after midnight), and convert to an mtime suitable for - * comparisons with repository mtime fields and assign to *ret. Upon failure, - * the error state will be updated with an appropriate error message and code. - */ -static int -fnc_date_to_mtime(double *ret, const char *d, int when) -{ - struct tm t = {0, 0, 0, 0, 0, 0}; - char iso8601[ISO8601_TIMESTAMP]; - - /* Fill the tm structure. */ - if (strptime(d, "%Y-%m-%d", &t) == NULL) { - /* If not YYYY-MM-DD, try MM/DD/YYYY and DD/MM/YYYY. */ - if (strptime(d, "%D", &t) != NULL) { - /* If MM/DD/YYYY, check if it could be DD/MM/YYYY too */ - if (strptime(d, "%d/%m/%Y", &t) != NULL) - return RC(FSL_RC_AMBIGUOUS, - "ambiguous date [%s]", d); - } else if (strptime(d, "%d/%m/%Y", &t) != NULL) { - /* If DD/MM/YYYY, check if it could be MM/DD/YYYY too */ - if (strptime(d, "%D", &t) != NULL) - return RC(FSL_RC_AMBIGUOUS, - "ambiguous date [%s]", d); - } else - return RC(FSL_RC_TYPE, "unable to parse date: %s", d); - } - - /* Format tm into ISO8601 string then convert to mtime. */ - if (when > 0) /* After date d. */ - strftime(iso8601, ISO8601_TIMESTAMP, "%FT23:59:59", &t); - else /* Before date d. */ - strftime(iso8601, ISO8601_TIMESTAMP, "%FT00:00:01", &t); - if (!fsl_iso8601_to_julian(iso8601, ret)) - return RC(FSL_RC_ERROR, "fsl_iso8601_to_julian(%s)", iso8601); - - return 0; -} - -static char * -fnc_strsep(char **ptr, const char *sep) -{ - char *s, *token; - - if ((s = *ptr) == NULL) - return NULL; - - if (*(token = s + strcspn(s, sep)) != '\0') { - *token++ = '\0'; - *ptr = token; - } else - *ptr = NULL; - - return s; -} - -#ifdef __linux__ -static size_t -fnc_strlcat(char *restrict dst, const char *restrict src, size_t dstsz) -{ - size_t offset; - int dstlen, srclen, idx = 0; - - offset = dstlen = fsl_strlen(dst); - srclen = fsl_strlen(src); - - while ((*(dst + offset++) = *(src + idx++)) != '\0') - if (offset == dstsz - 1) - break; - - *(dst + offset) = '\0'; - - return dstlen + srclen; -} - -static size_t -fnc_strlcpy(char *restrict dst, const char *restrict src, size_t dstsz) -{ - size_t offset = 0; - - if (dstsz < 1) - goto end; - - while ((*(dst + offset) = *(src + offset)) != '\0') - if (++offset == dstsz) { - --offset; - break; - } - -end: - *(dst + offset) = '\0'; - while (*(src + offset) != '\0') - ++offset; /* Return src length. */ - - return offset; -} -#endif - ADDED lib/libfossil-config.h Index: lib/libfossil-config.h ================================================================== --- lib/libfossil-config.h +++ lib/libfossil-config.h @@ -0,0 +1,142 @@ +#if !defined(_NET_FOSSIL_SCM_FSL_AMALGAMATION_CONFIG_H_INCLUDED_) +#define _NET_FOSSIL_SCM_FSL_AMALGAMATION_CONFIG_H_INCLUDED_ 1 +#define FSL_AUX_SCHEMA "2015-01-24" +#define FSL_CONTENT_SCHEMA "2" +#define FSL_PACKAGE_NAME "libfossil" +#define FSL_LIBRARY_VERSION "0.0.1-alphabeta" +/* Tweak the following for your system... */ +#if !defined(HAVE_COMPRESS) +# define HAVE_COMPRESS 1 +#endif +#if !defined(HAVE_DLFCN_H) +# define HAVE_DLFCN_H 0 +#endif +#if !defined(HAVE_DLOPEN) +# define HAVE_DLOPEN 0 +#endif +#if !defined(HAVE_GETADDRINFO) +# define HAVE_GETADDRINFO 0 +#endif +#if !defined(HAVE_INET_NTOP) +# define HAVE_INET_NTOP 0 +#endif +#if !defined(HAVE_INTTYPES_H) +# define HAVE_INTTYPES_H 0 +#endif +#if !defined(HAVE_LIBDL) +# define HAVE_LIBDL 0 +#endif +#if !defined(HAVE_LIBLTDL) +# define HAVE_LIBLTDL 0 +#endif +#if !defined(_WIN32) +#if !defined(HAVE_LSTAT) +# define HAVE_LSTAT 1 +#endif +#if !defined(HAVE_LTDL_H) +# define HAVE_LTDL_H 0 +#endif +#if !defined(HAVE_LT_DLOPEN) +# define HAVE_LT_DLOPEN 0 +#endif +#if !defined(HAVE_OPENDIR) +# define HAVE_OPENDIR 1 +#endif +#if !defined(HAVE_PIPE) +# define HAVE_PIPE 1 +#endif +#if !defined(HAVE_STAT) +# define HAVE_STAT 1 +#endif +#if !defined(HAVE_STDINT_H) +# define HAVE_STDINT_H 1 +#endif +#if !defined(_DEFAULT_SOURCE) +# define _DEFAULT_SOURCE 1 +#endif +#if !defined(_XOPEN_SOURCE) +# define _XOPEN_SOURCE 500 +#endif +#else +#if !defined(HAVE_LSTAT) +# define HAVE_LSTAT 0 +#endif +#if !defined(HAVE_LTDL_H) +# define HAVE_LTDL_H 0 +#endif +#if !defined(HAVE_LT_DLOPEN) +# define HAVE_LT_DLOPEN 0 +#endif +#if !defined(HAVE_OPENDIR) +# define HAVE_OPENDIR 1 +#endif +#if !defined(HAVE_PIPE) +# define HAVE_PIPE 0 +#endif +#if !defined(HAVE_STAT) +# define HAVE_STAT 0 +#endif +#if !defined(HAVE_STDINT_H) +# define HAVE_STDINT_H 1 +#endif +#endif +/* _WIN32 */ + + +#define FSL_LIB_VERSION_HASH "acbd39275f0d573a74d996caea7918afafff2ff7" +#define FSL_LIB_VERSION_TIMESTAMP "2021-10-08 03:15:51.855 UTC" +#define FSL_LIB_CONFIG_TIME "2021-10-08 04:07 GMT" +#if defined(_MSC_VER) +#define FSL_PLATFORM_OS "windows" +#define FSL_PLATFORM_IS_WINDOWS 1 +#define FSL_PLATFORM_IS_UNIX 0 +#define FSL_PLATFORM_PLATFORM "windows" +#define FSL_PLATFORM_PATH_SEPARATOR ";" +#define FSL_CHECKOUTDB_NAME "./_FOSSIL_" +/* define a __func__ compatibility macro */ +#if _MSC_VER < 1500 /* (vc9.0; dev studio 2008) */ +/* sorry; cant do much better than nothing at all on those earlier ones */ +#define __func__ "(func)" +#else +#define __func__ __FUNCTION__ +#endif +/* for the time being at least, don't complain about there being secure crt alternatives: */ +#ifndef _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_WARNINGS +#endif +/* for the time being at least, don't complain about using POSIX names instead of ISO C++: */ +#pragma warning ( disable : 4996 ) +/* for the time being at least, suppresss some int conversion warnings */ +#pragma warning ( disable : 4244 ) /*'fsl_size_t' to 'int'; this masks other problems that should be fixed*/ +#pragma warning ( disable : 4761 ) /*'integral size mismatch in argument'; more size_t problems*/ +#pragma warning ( disable : 4267 ) /*'size_t' to 'int'; crops up especially in 64-bit builds*/ +/* these were extracted from fossil's unistd.h */ +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +#include +#elif defined(__MINGW32__) +#define FSL_PLATFORM_OS "mingw" +#define FSL_PLATFORM_IS_WINDOWS 1 +#define FSL_PLATFORM_IS_UNIX 0 +#define FSL_PLATFORM_PLATFORM "windows" +#define FSL_PLATFORM_PATH_SEPARATOR ";" +#define FSL_CHECKOUTDB_NAME "./.fslckout" +#elif defined(__CYGWIN__) +#define FSL_PLATFORM_OS "cygwin" +#define FSL_PLATFORM_IS_WINDOWS 0 +#define FSL_PLATFORM_IS_UNIX 1 +#define FSL_PLATFORM_PLATFORM "unix" +#define FSL_PLATFORM_PATH_SEPARATOR ":" +#define FSL_CHECKOUTDB_NAME "./_FOSSIL_" +#else +#define FSL_PLATFORM_OS "unknown" +#define FSL_PLATFORM_IS_WINDOWS 0 +#define FSL_PLATFORM_IS_UNIX 1 +#define FSL_PLATFORM_PLATFORM "unix" +#define FSL_PLATFORM_PATH_SEPARATOR ":" +#define FSL_CHECKOUTDB_NAME "./.fslckout" +#endif + + +#endif +/* _NET_FOSSIL_SCM_FSL_AMALGAMATION_CONFIG_H_INCLUDED_ */ + ADDED lib/libfossil.c Index: lib/libfossil.c ================================================================== --- lib/libfossil.c +++ lib/libfossil.c @@ -0,0 +1,41135 @@ +#include "libfossil.h" +/* start of file fsl.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/***************************************************************************** + This file houses some context-independent API routines as well as + some of the generic helper functions and types. +*/ + +#include +#include /* malloc() and friends, qsort() */ +#include /* memset() */ +#include /* strftime() and gmtime() */ + +#if defined(_WIN32) || defined(WIN32) +# include +#define isatty(h) _isatty(h) +#else +# include /* isatty() */ +#endif +/* extern int isatty(int); */ + +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +/* + Please keep all fsl_XXX_empty initializers in one place (here) + and lexically sorted. +*/ +const fsl_acache fsl_acache_empty = fsl_acache_empty_m; +const fsl_branch_opt fsl_branch_opt_empty = fsl_branch_opt_empty_m; +const fsl_buffer fsl_buffer_empty = fsl_buffer_empty_m; +const fsl_card_F fsl_card_F_empty = fsl_card_F_empty_m; +const fsl_card_F_list fsl_card_F_list_empty = fsl_card_F_list_empty_m; +const fsl_card_J fsl_card_J_empty = fsl_card_J_empty_m; +const fsl_card_Q fsl_card_Q_empty = fsl_card_Q_empty_m; +const fsl_card_T fsl_card_T_empty = fsl_card_T_empty_m; +const fsl_checkin_opt fsl_checkin_opt_empty = fsl_checkin_opt_empty_m; +const fsl_ckout_manage_opt fsl_ckout_manage_opt_empty = + fsl_ckout_manage_opt_empty_m; +const fsl_ckout_unmanage_opt fsl_ckout_unmanage_opt_empty = + fsl_ckout_unmanage_opt_empty_m; +const fsl_ckup_opt fsl_ckup_opt_empty = fsl_ckup_opt_m; +const fsl_confirmer fsl_confirmer_empty = fsl_confirmer_empty_m; +const fsl_cx fsl_cx_empty = fsl_cx_empty_m; +const fsl_cx_config fsl_cx_config_empty = fsl_cx_config_empty_m; +const fsl_cx_init_opt fsl_cx_init_opt_default = fsl_cx_init_opt_default_m; +const fsl_cx_init_opt fsl_cx_init_opt_empty = fsl_cx_init_opt_empty_m; +const fsl_db fsl_db_empty = fsl_db_empty_m; +const fsl_deck fsl_deck_empty = fsl_deck_empty_m; +const fsl_confirm_detail fsl_confirm_detail_empty = + fsl_confirm_detail_empty_m; +const fsl_confirm_response fsl_confirm_response_empty = + fsl_confirm_response_empty_m; +const fsl_error fsl_error_empty = fsl_error_empty_m; +const fsl_checkin_queue_opt fsl_checkin_queue_opt_empty = + fsl_checkin_queue_opt_empty_m; +const fsl_fstat fsl_fstat_empty = fsl_fstat_empty_m; +const fsl_list fsl_list_empty = fsl_list_empty_m; +const fsl_mcache fsl_mcache_empty = fsl_mcache_empty_m; +const fsl_merge_opt fsl_merge_opt_empty = fsl_merge_opt_empty_m; +const fsl_outputer fsl_outputer_FILE = fsl_outputer_FILE_m; +const fsl_outputer fsl_outputer_empty = fsl_outputer_empty_m; +const fsl_pathfinder fsl_pathfinder_empty = fsl_pathfinder_empty_m; +const fsl_pq fsl_pq_empty = fsl_pq_empty_m; +const fsl_repo_create_opt fsl_repo_create_opt_empty = + fsl_repo_create_opt_empty_m; +const fsl_repo_extract_opt fsl_repo_extract_opt_empty = + fsl_repo_extract_opt_empty_m; +const fsl_repo_extract_state fsl_repo_extract_state_empty = + fsl_repo_extract_state_empty_m; +const fsl_repo_open_ckout_opt fsl_repo_open_ckout_opt_empty = + fsl_repo_open_ckout_opt_m; +const fsl_ckout_revert_opt fsl_ckout_revert_opt_empty = + fsl_ckout_revert_opt_empty_m; +const fsl_sha1_cx fsl_sha1_cx_empty = fsl_sha1_cx_empty_m; +const fsl_state fsl_state_empty = fsl_state_empty_m; +const fsl_stmt fsl_stmt_empty = fsl_stmt_empty_m; +const fsl_timer_state fsl_timer_state_empty = fsl_timer_state_empty_m; +const fsl_xlinker fsl_xlinker_empty = fsl_xlinker_empty_m; +const fsl_xlinker_list fsl_xlinker_list_empty = fsl_xlinker_list_empty_m; +const fsl_zip_writer fsl_zip_writer_empty = fsl_zip_writer_empty_m; + +const fsl_allocator fsl_allocator_stdalloc = { +fsl_realloc_f_stdalloc, +NULL +}; + +fsl_lib_configurable_t fsl_lib_configurable = { + {/*allocator*/ fsl_realloc_f_stdalloc, NULL} +}; + +void * fsl_malloc( fsl_size_t n ){ + return n + ? fsl_realloc(NULL, n) + : NULL; +} + +void fsl_free( void * mem ){ + if(mem) fsl_realloc(mem, 0); +} + +void * fsl_realloc( void * mem, fsl_size_t n ){ +#define FLCA fsl_lib_configurable.allocator + if(!mem){ + /* malloc() */ + return n + ? FLCA.f(FLCA.state, NULL, n) + : NULL; + }else if(!n){ + /* free() */ + FLCA.f(FLCA.state, mem, 0); + return NULL; + }else{ + /* realloc() */ + return FLCA.f(FLCA.state, mem, n); + } +#undef FLCA +} + +void * fsl_realloc_f_stdalloc(void * state, void * mem, fsl_size_t n){ + if(!mem){ + return malloc(n); + }else if(!n){ + free(mem); + return NULL; + }else{ + return realloc(mem, n); + } +} + +int fsl_is_uuid(char const * str){ + fsl_size_t const len = fsl_strlen(str); + if(FSL_STRLEN_SHA1==len){ + return fsl_validate16(str, FSL_STRLEN_SHA1) ? FSL_STRLEN_SHA1 : 0; + }else if(FSL_STRLEN_K256==len){ + return fsl_validate16(str, FSL_STRLEN_K256) ? FSL_STRLEN_K256 : 0; + }else{ + return 0; + } +} +int fsl_is_uuid_len(int x){ + switch(x){ + case FSL_STRLEN_SHA1: + case FSL_STRLEN_K256: + return x; + default: + return 0; + } +} +void fsl_error_clear( fsl_error * const err ){ + if(err){ + fsl_buffer_clear(&err->msg); + *err = fsl_error_empty; + } +} + +void fsl_error_reset( fsl_error * const err ){ + if(err){ + err->code = 0; + err->msg.used = err->msg.cursor = 0; + if(err->msg.mem) err->msg.mem[0] = 0; + } +} + + +int fsl_error_copy( fsl_error const * const src, fsl_error * const dest ){ + if(!src || !dest || (src==dest)) return FSL_RC_MISUSE; + else { + int rc = 0; + dest->msg.used = dest->msg.cursor = 0; + dest->code = src->code; + if(FSL_RC_OOM!=src->code){ + rc = fsl_buffer_append( &dest->msg, src->msg.mem, src->msg.used ); + } + return rc; + } +} + +void fsl_error_move( fsl_error * const lower, fsl_error * const higher ){ + fsl_error const err = *lower; + *lower = *higher; + lower->code = 0; + lower->msg.used = lower->msg.cursor = 0; + *higher = err; +} + +int fsl_error_setv( fsl_error * const err, int code, char const * fmt, + va_list args ){ + if(!err) return FSL_RC_MISUSE; + else if(!code){ /* clear error state */ + err->code = 0; + err->msg.used = err->msg.cursor = 0; + if(err->msg.mem){ + err->msg.mem[0] = 0; + } + return 0; + }else{ + int rc = 0; + err->msg.used = err->msg.cursor = 0; + err->code = code; + if(FSL_RC_OOM!=code){ + rc = fmt + ? fsl_buffer_appendfv(&err->msg, fmt, args) + : fsl_buffer_append(&err->msg, fsl_rc_cstr(code), -1); + if(rc) err->code = rc; + } + return rc ? rc : code; + } +} + +int fsl_error_set( fsl_error * const err, int code, char const * fmt, + ... ){ + int rc; + va_list args; + va_start(args,fmt); + rc = fsl_error_setv(err, code, fmt, args); + va_end(args); + return rc; +} + + +int fsl_error_get( fsl_error const * const err, char const ** str, + fsl_size_t * const len ){ + if(!err) return FSL_RC_MISUSE; + else{ + if(str) *str = err->msg.used + ? (char const *)err->msg.mem + : NULL; + if(len) *len = err->msg.used; + return err->code; + } +} + + +char const * fsl_rc_cstr(int rc){ + fsl_rc_e const RC = (fsl_rc_e)rc + /* we do this so that gcc will warn if the switch() below is + missing any fsl_rc_e entries. */ + ; + switch(RC){ +#define STR(T) case FSL_RC_##T: return "FSL_RC_" #T + STR(ACCESS); + STR(ALREADY_EXISTS); + STR(AMBIGUOUS); + STR(BREAK); + STR(SYNTAX); + STR(CHECKSUM_MISMATCH); + STR(CONFLICT); + STR(CONSISTENCY); + STR(DB); + STR(DELTA_INVALID_OPERATOR); + STR(DELTA_INVALID_SEPARATOR); + STR(DELTA_INVALID_SIZE); + STR(DELTA_INVALID_TERMINATOR); + STR(DIFF_BINARY); + STR(DIFF_WS_ONLY); + STR(end); + STR(ERROR); + STR(IO); + STR(MISSING_INFO); + STR(MISUSE); + STR(NOOP); + STR(NOT_A_CKOUT); + STR(NOT_A_REPO); + STR(NOT_FOUND); + STR(NYI); + STR(OK); + STR(OOM); + STR(PHANTOM); + STR(RANGE); + STR(REPO_MISMATCH); + STR(REPO_NEEDS_REBUILD); + STR(REPO_VERSION); + STR(SIZE_MISMATCH); + STR(STEP_DONE); + STR(STEP_ERROR); + STR(STEP_ROW); + STR(TYPE); + STR(UNKNOWN_RESOURCE); + STR(UNSUPPORTED); +#undef STR + } + return "Unknown result code"; +} + +char const * fsl_library_version(){ + return FSL_LIBRARY_VERSION; +} + +bool fsl_library_version_matches(char const * yourLibVersion){ + return 0 == fsl_strcmp(FSL_LIBRARY_VERSION, yourLibVersion); +} + +double fsl_unix_to_julian( fsl_time_t unix_ ){ + return (unix_ * 1.0 / 86400.0 ) + 2440587.5; +} + +double fsl_julian_now(){ + return fsl_unix_to_julian( time(0) ); +} + + +int fsl_strcmp(const char *zA, const char *zB){ + if( zA==0 ) return zB ? -1 : 0; + else if( zB==0 ) return 1; + else{ + int a, b; + do{ + a = *zA++; + b = *zB++; + }while( a==b && a!=0 ); + return ((unsigned char)a) - (unsigned char)b; + } +} + + +int fsl_strcmp_cmp( void const * lhs, void const * rhs ){ + return fsl_strcmp((char const *)lhs, (char const *)rhs); +} + +int fsl_strncmp(const char *zA, const char *zB, fsl_size_t nByte){ + if( !zA ) return zB ? -1 : 0; + else if( !zB ) return +1; + else if(!nByte) return 0; + else{ + int a, b; + do{ + a = *zA++; + b = *zB++; + }while( a==b && a!=0 && (--nByte)>0 ); + return (nByte>0) ? (((unsigned char)a) - (unsigned char)b) : 0; + } +} + + +int fsl_uuidcmp( fsl_uuid_cstr lhs, fsl_uuid_cstr rhs ){ + if(!lhs) return rhs ? -1 : 0; + else if(!rhs) return 1; + else if(lhs[FSL_STRLEN_SHA1] && rhs[FSL_STRLEN_SHA1]){ + return fsl_strncmp( lhs, rhs, FSL_STRLEN_K256); + }else if(!lhs[FSL_STRLEN_SHA1] && !rhs[FSL_STRLEN_SHA1]){ + return fsl_strncmp( lhs, rhs, FSL_STRLEN_SHA1 ); + }else{ + return fsl_strcmp(lhs, rhs); + } +} + +int fsl_strnicmp(const char *zA, const char *zB, fsl_int_t nByte){ + if( zA==0 ){ + if( zB==0 ) return 0; + return -1; + }else if( zB==0 ){ + return +1; + } + if( nByte<0 ) nByte = (fsl_int_t)fsl_strlen(zB); + return sqlite3_strnicmp(zA, zB, nByte); +} + +int fsl_stricmp(const char *zA, const char *zB){ + if( zA==0 ) return zB ? -1 : 0; + else if( zB==0 ) return 1; + else{ + fsl_int_t nByte; + int rc; + nByte = (fsl_int_t)fsl_strlen(zB); + rc = sqlite3_strnicmp(zA, zB, nByte); + return ( rc==0 && zA[nByte] ) ? 1 : rc; + } +} + +int fsl_stricmp_cmp( void const * lhs, void const * rhs ){ + return fsl_stricmp((char const *)lhs, (char const *)rhs); +} + +fsl_size_t fsl_strlen( char const * src ){ + fsl_size_t i = 0; + if(src) for( ; *src; ++i, ++src ){} + return i; +} + +char * fsl_strndup( char const * src, fsl_int_t len ){ + if(!src) return NULL; + else{ + fsl_buffer b = fsl_buffer_empty; + if(len<0) len = (fsl_int_t)fsl_strlen(src); + fsl_buffer_append( &b, src, len ); + return (char*)b.mem; + } +} + +char * fsl_strdup( char const * src ){ + return fsl_strndup(src, -1); +} + +size_t fsl_strlcpy(char *restrict dst, const char *restrict src, size_t dstsz){ + size_t offset = 0; + + if(dstsz<1){ + goto end; + } + while((*(dst+offset) = *(src+offset))!='\0'){ + if(++offset == dstsz){ + --offset; + break; + } + } +end: + *(dst+offset) = '\0'; + while(*(src+offset)!='\0'){ + ++offset; /* Return src length. */ + } + return offset; +} + +size_t fsl_strlcat(char *restrict dst, const char *restrict src, size_t dstsz){ + size_t offset; + int dstlen, srclen, idx = 0; + + offset = dstlen = fsl_strlen(dst); + srclen = fsl_strlen(src); + + while((*(dst+offset++) = *(src+idx++))!='\0'){ + if(offset==dstsz-1){ + break; + } + } + *(dst+offset)='\0'; + return dstlen+srclen; +} + +/* + Return TRUE if the string begins with something that looks roughly + like an ISO date/time string. The SQLite date/time functions will + have the final say-so about whether or not the date/time string is + well-formed. +*/ +char fsl_str_is_date(const char *z){ + if(!z || !*z) return 0; + else if( !fsl_isdigit(z[0]) ) return 0; + else if( !fsl_isdigit(z[1]) ) return 0; + else if( !fsl_isdigit(z[2]) ) return 0; + else if( !fsl_isdigit(z[3]) ) return 0; + else if( z[4]!='-') return 0; + else if( !fsl_isdigit(z[5]) ) return 0; + else if( !fsl_isdigit(z[6]) ) return 0; + else if( z[7]!='-') return 0; + else if( !fsl_isdigit(z[8]) ) return 0; + else if( !fsl_isdigit(z[9]) ) return 0; + else return 1; +} + +int fsl_str_is_date2(const char *z){ + int rc = -1; + int pos = 0; + if(!z || !*z) return 0; + else if( !fsl_isdigit(z[pos++]) ) return 0; + else if( !fsl_isdigit(z[pos++]) ) return 0; + else if( !fsl_isdigit(z[pos++]) ) return 0; + else if( !fsl_isdigit(z[pos++]) ) return 0; + else if( z[pos]=='-') ++pos; + else{ + if(fsl_isdigit(z[pos++]) && '-'==z[pos++]){ + rc = 1; + }else{ + return 0; + } + } + if( !fsl_isdigit(z[pos++]) ) return 0; + else if( !fsl_isdigit(z[pos++]) ) return 0; + else if( z[pos++]!='-') return 0; + else if( !fsl_isdigit(z[pos++]) ) return 0; + else if( !fsl_isdigit(z[pos++]) ) return 0; + assert(10==pos || 11==pos); + return rc; +} + +bool fsl_str_bool( char const * s ){ + switch(s ? *s : 0){ + case 0: case '0': + case 'f': case 'F': // "false" + case 'n': case 'N': // "no" + return false; + case '1': + case 't': case 'T': // "true" + case 'y': case 'Y': // "yes" + return true; + default: { + char buf[5] = {0,0,0,0,0}; + int i; + for( i = 0; (i<5) && *s; ++i, ++s ){ + buf[i] = fsl_tolower(*s); + } + if(0==fsl_strncmp(buf, "off", 3)) return false; + return true; + } + } +} + +char * fsl_guess_user_name(){ + char const ** e; + static char const * list[] = { + "FOSSIL_USER", +#if defined(_WIN32) + "USERNAME", +#else + "USER", + "LOGNAME", +#endif + NULL /* sentinel */ + }; + char * rv = NULL; + for( e = list; *e; ++e ){ + rv = fsl_getenv(*e); + if(rv){ + /* + Because fsl_getenv() has the odd requirement of needing + fsl_filename_free(), and we want strings returned from this + function to be safe for passing to fsl_free(), we have to dupe + the string. We "could" block this off to happen only on the + platforms for which fsl_getenv() requires an extra encoding + step, but that would likely eventually lead to a bug. + */ + char * kludge = fsl_strdup(rv); + fsl_filename_free(rv); + rv = kludge; + break; + } + } + return rv; +} + +void fsl_fatal( int code, char const * fmt, ... ){ + static bool inFatal = false; + if(inFatal){ + /* This can only happen if the fsl_appendv() bits + call this AND trigger it via fsl_fprintf() below, + neither of which is currently the case. + */ + assert(!"fsl_fatal() called recursively."); + abort(); + }else{ + va_list args; + inFatal = true; + fsl_fprintf(stderr, "FATAL ERROR: code=%d (%s)\n", + code, fsl_rc_cstr(code)); + if(fmt){ + va_start(args,fmt); + fsl_fprintfv(stderr, fmt, args); + va_end(args); + fwrite("\n", 1, 1, stderr); + } + exit(EXIT_FAILURE); + } +} + +#if 0 +char * fsl_unix_to_iso8601( fsl_time_t u ){ + enum { BufSize = 20 }; + char buf[BufSize]= {0,}; + time_t const tt = (time_t)u; + fsl_strftime( buf, BufSize, "%Y-%m-%dT%H:%M:%S", gmtime(&tt) ); + return fsl_strdup(buf); +} +#endif + + +bool fsl_iso8601_to_julian( char const * zDate, double * out ){ + /* Adapted from this article: + + https://quasar.as.utexas.edu/BillInfo/JulianDatesG.html + */ + char const * p = zDate; + int y = 0, m = 0, d = 0; + int h = 0, mi = 0, s = 0, f = 0; + double j = 0; + if(!zDate || !*zDate){ + return 0; + } +#define DIG(NUM) if(!fsl_isdigit(*p)) return 0; \ + NUM=(NUM*10)+(*(p++)-'0') + + DIG(y);DIG(y);DIG(y);DIG(y); + if('-'!=*p++) return 0; + DIG(m);DIG(m); + if('-'!=*p++) return 0; + DIG(d);DIG(d); + if('T' != *p++) return 0; + DIG(h);DIG(h); + if(':'!=*p++) return 0; + DIG(mi);DIG(mi); + if(':'!=*p++) return 0; + DIG(s);DIG(s); + if('.'==*p++){ + DIG(f);DIG(f);DIG(f); + } + if(out){ + typedef int64_t TI; + TI A, B, C, E, F; + if(m<3){ + --y; + m += 12; + } + A = y/100; + B = A/4; + C = 2-A+B; + E = (TI)(365.25*(y+4716)); + F = (TI)(30.6001*(m+1)); + j = C + d + E + F - 1524.5; + j += ((1.0*h)/24) + ((1.0*mi)/1440) + ((1.0*s)/86400); + if(0 != f){ + j += (1.0*f)/86400000; + } + *out = j; + } + return 1; +#undef DIG +} + +fsl_time_t fsl_julian_to_unix( double JD ){ + return (fsl_time_t) ((JD - 2440587.5) * 86400); +} + +bool fsl_julian_to_iso8601( double J, char * out, bool addMs ){ + /* Adapted from this article: + + https://quasar.as.utexas.edu/BillInfo/JulianDatesG.html + */ + typedef int64_t TI; + int Y, M, D, H, MI, S, F; + TI ms; + char * z = out; + if(!out || (J<=0)) return 0; + else{ + double Z; + TI W, X; + TI A, B; + TI C, DD, E, F; + + Z = J + 0.5; + W = (TI)((Z-1867216.25)/36524.25); + X = W/4; + A = (TI)(Z+1+W-X); + B = A+1524; + C = (TI)((B-122.1)/365.25); + DD = (TI)(365.25 * C); + E = (TI)((B-DD)/30.6001); + F = (TI)(30.6001 * E); + D = (int)(B - DD - F); + M = (E<=13) ? (E-1) : (E-13); + Y = (M<3) ? (C-4715) : (C-4716); + } + + if(Y<0 || Y>9999) return 0; + else if(M<1 || M>12) return 0; + else if(D<1 || D>31) return 0; + + ms = (TI)((J-(TI)J) * 86400001.0) + /* number of milliseconds in the fraction part of the JDay. The + non-0 at the end works around a problem where SS.000 converts + to (SS-1).999. This will only hide the bug for the cases i've + seen it, and might introduce other inaccuracies + elsewhere. Testing it against the current libfossil event table + produces good results - at most a 1ms round-trip fidelity loss + for the (currently ~1157) records being checked. The suffix of + 1.0 was found to be a decent value via much testing with the + libfossil and fossil(1) source repos. + */; + + if( (H = ms / 3600000) ){ + ms -= H * 3600000; + H = (H + 12) % 24; + }else{ + H = 12 /* astronomers start their day at noon. */; + } + if( (MI = ms / 60000) ) ms -= MI * 60000; + if( (S = ms / 1000) ) ms -= S * 1000; + assert(ms<1000); + F = (int)(ms); + + assert(H>=0 && H<24); + assert(MI>=0 && MI<60); + assert(S>=0 && S<60); + assert(F>=0 && F<1000); + + if(H<0 || H>23) return 0; + else if(MI<0 || MI>59) return 0; + else if(S<0 || S>59) return 0; + else if(F<0 || F>999) return 0; +#define UGLY_999_KLUDGE 1 + /* The fossil(1) repo has 27 of 10041 records which exhibit the + SS.999 behaviour commented on above. With this kludge, that + number drops to 0. But it's still an ugly, ugly kludge. + OTOH, the chance of the .999 being correct is 1 in 1000, + whereas we see "correct" behaviour more often (2.7 in 1000) + with this workaround. + */ +#if UGLY_999_KLUDGE + if(999==F){ + char oflow = 0; + int s2 = S, mi2 = MI, h2 = H; + if(++s2 == 60){ /* Overflow minute */ + s2 = 0; + if(++mi2 == 60){ /* Overflow hour */ + mi2 = 0; + if(++h2 == 24){ /* Overflow day */ + /* leave this corner-corner case in place */ + oflow = 1; + } + } + } + /* MARKER(("UGLY 999 KLUDGE (A): H=%d MI=%d S=%d F=%d\n", H, MI, S, F)); */ + if(!oflow){ + F = 0; + S = s2; + MI = mi2; + H = h2; + /* MARKER(("UGLY 999 KLUDGE (B): H=%d MI=%d S=%d F=%d\n", H, MI, S, F)); */ + } + } +#endif +#undef UGLY_999_KLUDGE + *(z++) = '0'+(Y/1000); + *(z++) = '0'+(Y%1000/100); + *(z++) = '0'+(Y%100/10); + *(z++) = '0'+(Y%10); + *(z++) = '-'; + *(z++) = '0'+(M/10); + *(z++) = '0'+(M%10); + *(z++) = '-'; + *(z++) = '0'+(D/10); + *(z++) = '0'+(D%10); + *(z++) = 'T'; + *(z++) = '0'+(H/10); + *(z++) = '0'+(H%10); + *(z++) = ':'; + *(z++) = '0'+(MI/10); + *(z++) = '0'+(MI%10); + *(z++) = ':'; + *(z++) = '0'+(S/10); + *(z++) = '0'+(S%10); + if(addMs){ + *(z++) = '.'; + *(z++) = '0'+(F%1000/100); + *(z++) = '0'+(F%100/10); + *(z++) = '0'+(F%10); + } + *z = 0; + return 1; +} + +#if FSL_CONFIG_ENABLE_TIMER +/** + For the fsl_timer_xxx() family of functions... +*/ +#ifdef _WIN32 +# include +#else +# include +# include +# include +# include +# include +#endif +#endif +/* FSL_CONFIG_ENABLE_TIMER */ + +/** + Get user and kernel times in microseconds. +*/ +static void fsl_cpu_times(uint64_t *piUser, uint64_t *piKernel){ +#if !FSL_CONFIG_ENABLE_TIMER + if(piUser) *piUser = 0U; + if(piKernel) *piKernel = 0U; +#else +#ifdef _WIN32 + FILETIME not_used; + FILETIME kernel_time; + FILETIME user_time; + GetProcessTimes(GetCurrentProcess(), ¬_used, ¬_used, + &kernel_time, &user_time); + if( piUser ){ + *piUser = ((((uint64_t)user_time.dwHighDateTime)<<32) + + (uint64_t)user_time.dwLowDateTime + 5)/10; + } + if( piKernel ){ + *piKernel = ((((uint64_t)kernel_time.dwHighDateTime)<<32) + + (uint64_t)kernel_time.dwLowDateTime + 5)/10; + } +#else + struct rusage s; + getrusage(RUSAGE_SELF, &s); + if( piUser ){ + *piUser = ((uint64_t)s.ru_utime.tv_sec)*1000000 + s.ru_utime.tv_usec; + } + if( piKernel ){ + *piKernel = + ((uint64_t)s.ru_stime.tv_sec)*1000000 + s.ru_stime.tv_usec; + } +#endif +#endif +/* FSL_CONFIG_ENABLE_TIMER */ +} + + +void fsl_timer_start(fsl_timer_state * const ft){ + fsl_cpu_times( &ft->user, &ft->system ); +} + +uint64_t fsl_timer_fetch(fsl_timer_state const * const t){ + uint64_t eu = 0, es = 0; + fsl_cpu_times( &eu, &es ); + return (eu - t->user) + (es - t->system); +} + +uint64_t fsl_timer_reset(fsl_timer_state * const t){ + uint64_t const rc = fsl_timer_fetch(t); + fsl_cpu_times( &t->user, &t->system ); + return rc; +} + +uint64_t fsl_timer_stop(fsl_timer_state * const t){ + uint64_t const rc = fsl_timer_fetch(t); + *t = fsl_timer_state_empty; + return rc; +} + +unsigned int fsl_rgb_encode( int r, int g, int b ){ + return (unsigned int)(((r&0xFF)<<16) + ((g&0xFF)<<8) + (b&0xFF)); +} + +void fsl_rgb_decode( unsigned int src, int *r, int *g, int *b ){ + if(r) *r = (src&0xFF0000)>>16; + if(g) *g = (src&0xFF00)>>8; + if(b) *b = src&0xFF; +} + +unsigned fsl_gradient_color(unsigned c1, unsigned c2, unsigned int n, unsigned int i){ + unsigned c; /* Result color */ + unsigned x1, x2; + if( i==0 || n==0 ) return c1; + else if(i>=n) return c2; + x1 = (c1>>16)&0xff; + x2 = (c2>>16)&0xff; + c = (x1*(n-i) + x2*i)/n<<16 & 0xff0000; + x1 = (c1>>8)&0xff; + x2 = (c2>>8)&0xff; + c |= (x1*(n-i) + x2*i)/n<<8 & 0xff00; + x1 = c1&0xff; + x2 = c2&0xff; + c |= (x1*(n-i) + x2*i)/n & 0xff; + return c; +} + + +fsl_size_t fsl_simplify_sql( char * sql, fsl_int_t len ){ + char * wat = sql /* write pos */; + char * rat = sql /* read pos */; + char const * end /* one-past-the-end */; + char inStr = 0 /* in an SQL string? */; + char prev = 0 /* previous character. Sometimes. */; + if(!sql || !*sql) return 0; + else if(len < 0) len = fsl_strlen(sql); + if(!len) return 0; + end = sql + len; + while( *rat && (rat < end) ){ + switch(*rat){ + case 0: break; + case '\r': + case '\n': + /* Bug: we don't handle \r\n pairs. Because nobody + should never have to :/. */ + if(inStr || (prev!=*rat)){ + /* Keep them as-is */ + prev = *wat++ = *rat++; + }else{ + /* Collapse multiples into one. */ + ++rat; + } + continue; + case ' ': + case '\t': + case '\v': + case '\f': + if(inStr){ + /* Keep them as-is */ + prev = *wat++ = *rat++; + }else{ + /* Reduce to a single space. */ + /* f_out("prev=[%c] rat=[%c]\n", prev, *rat); */ + if(prev != *rat){ + *wat++ = ' '; + prev = *rat; + } + ++rat; + } + continue; + case '\'': /* SQL strings */ + prev = *wat++ = *rat++; + if(!inStr){ + inStr = 1; + }else if('\'' == *rat){ + /* Escaped quote */ + *wat++ = *rat++; + }else{ + /* End of '...' string. */ + inStr = 0; + } + continue; + default: + prev = *wat++ = *rat++; + continue; + } + } + *wat = 0; + return (fsl_size_t)(wat - sql); +} + +/** + Convenience form of fsl_simplify_sql() which assumes b holds an SQL + string. It gets processed by fsl_simplify_sql() and its 'used' + length potentially gets adjusted to match the adjusted SQL string. +*/ +fsl_size_t fsl_simplify_sql_buffer( fsl_buffer * b ){ + return b->used = fsl_simplify_sql( (char *)b->mem, (fsl_int_t)b->used ); +} + +char const *fsl_preferred_ckout_db_name(){ +#if FSL_PLATFORM_IS_WINDOWS + return "_FOSSIL_"; +#else + return ".fslckout"; +#endif +} + +char fsl_isatty(int fd){ + return isatty(fd) ? 1 : 0; +} + +bool fsl_is_reserved_fn_windows(const char *zPath, fsl_int_t nameLen){ + static const char *const azRes[] = { + "CON", "PRN", "AUX", "NUL", "COM", "LPT" + }; + unsigned int i; + char const * zEnd; + if(nameLen<0) nameLen = (fsl_int_t)fsl_strlen(zPath); + zEnd = zPath + nameLen; + while( zPath < zEnd ){ + for(i=0; i=4 && fsl_isdigit(zPath[3]) + && (zPath[4]=='/' || zPath[4]=='.' || zPath[4]==0)) + || (i<4 && (zPath[3]=='/' || zPath[3]=='.' || zPath[3]==0))) + ){ + return true; + } + } + while( zPath=0 + ? (fsl_size_t)nameLen : fsl_strlen(zFilename); + char const * zEnd; + int gotSuffix = 0; + assert( zFilename && "API misuse" ); +#if FSL_PLATFORM_IS_WINDOWS // || 1 + if(nFilename>2 && fsl_is_reserved_fn_windows(zFilename, nameLen)){ + return true; + } +#endif + if( nFilename<8 ) return false; /* strlen("_FOSSIL_") */ + zEnd = zFilename + nFilename; + if( nFilename>=12 ){ /* strlen("_FOSSIL_-(shm|wal)") */ + /* Check for (-wal, -shm, -journal) suffixes, with an eye towards + ** runtime speed. */ + if( zEnd[-4]=='-' ){ + if( fsl_strnicmp("wal", &zEnd[-3], 3) + && fsl_strnicmp("shm", &zEnd[-3], 3) ){ + return false; + } + gotSuffix = 4; + }else if( nFilename>=16 && zEnd[-8]=='-' ){ /*strlen(_FOSSIL_-journal) */ + if( fsl_strnicmp("journal", &zEnd[-7], 7) ) return false; + gotSuffix = 8; + } + if( gotSuffix ){ + assert( 4==gotSuffix || 8==gotSuffix ); + zEnd -= gotSuffix; + nFilename -= gotSuffix; + gotSuffix = 1; + } + assert( nFilename>=8 && "strlen(_FOSSIL_)" ); + assert( gotSuffix==0 || gotSuffix==1 ); + } + switch( zEnd[-1] ){ + case '_':{ + if( fsl_strnicmp("_FOSSIL_", &zEnd[-8], 8) ) return false; + if( 8==nFilename ) return true; + return zEnd[-9]=='/' ? true : !!gotSuffix; + } + case 'T': + case 't':{ + if( nFilename<9 || zEnd[-9]!='.' + || fsl_strnicmp(".fslckout", &zEnd[-9], 9) ){ + return false; + } + if( 9==nFilename ) return true; + return zEnd[-10]=='/' ? true : !!gotSuffix; + } + default:{ + return false; + } + } +} + +#undef MARKER +#if defined(_WIN32) || defined(WIN32) +#undef isatty +#endif +/* end of file fsl.c */ +/* start of file annotate.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************ + This file implements the annoate/blame/praise-related APIs. +*/ +#include +#include /*memset()*/ + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +const fsl_annotate_opt fsl_annotate_opt_empty = fsl_annotate_opt_empty_m; + +/* +** The status of an annotation operation is recorded by an instance +** of the following structure. +*/ +typedef struct Annotator Annotator; +struct Annotator { + fsl_diff_cx c; /* The diff-engine context */ + fsl_buffer headVersion;/*starting version of the content*/ + struct AnnLine { /* Lines of the original files... */ + const char *z; /* The text of the line. Points into + this->headVersion. */ + short int n; /* Number of bytes (omitting trailing \n) */ + short int iVers; /* Level at which tag was set */ + } *aOrig; + unsigned int nOrig;/* Number of elements in aOrig[] */ + unsigned int nVers;/* Number of versions analyzed */ + bool bMoreToDo; /* True if the limit was reached */ + fsl_id_t origId; /* RID for the zOrigin version */ + fsl_id_t showId; /* RID for the version being analyzed */ + struct AnnVers { + char *zFUuid; /* File being analyzed */ + char *zMUuid; /* Check-in containing the file */ + char *zUser; /* Name of user who did the check-in */ + double mtime; /* [event].[mtime] db entry */ + } *aVers; /* For each check-in analyzed */ + unsigned int naVers; /* # of entries allocated in this->aVers */ + fsl_timer_state timer; +}; + +static const Annotator Annotator_empty = { +fsl_diff_cx_empty_m, +fsl_buffer_empty_m/*headVersion*/, +NULL/*aOrig*/, +0U/*nOrig*/, 0U/*nVers*/, +false/*bMoreToDo*/, +0/*origId*/, +0/*showId*/, +NULL/*aVers*/, +0U/*naVerse*/, +fsl_timer_state_empty_m +}; + +static void fsl__annotator_clean(Annotator * const a){ + unsigned i; + fsl__diff_cx_clean(&a->c); + for(i = 0; i < a->nVers; ++i){ + fsl_free(a->aVers[i].zFUuid); + fsl_free(a->aVers[i].zMUuid); + fsl_free(a->aVers[i].zUser); + } + fsl_free(a->aVers); + fsl_free(a->aOrig); + fsl_buffer_clear(&a->headVersion); +} + +static uint64_t fsl__annotate_opt_difflags(fsl_annotate_opt const * const opt){ + uint64_t diffFlags = FSL_DIFF2_STRIP_EOLCR; + if(opt->spacePolicy>0) diffFlags |= FSL_DIFF2_IGNORE_ALLWS; + else if(opt->spacePolicy<0) diffFlags |= FSL_DIFF2_IGNORE_EOLWS; + return diffFlags; +} + +/** + Initializes the annocation process by populating `a` from + a->toAnnote, which must have been previously populated. `a` must + have already been cleanly initialized via copying from + Annotator_empty and a->headVersion populated. Returns 0 on success, + else: + + - FSL_RC_RANGE if pInput is empty. + - FSL_RC_OOM on OOM. + + Regardless of success or failure, `a` must eventually be passed + to fsl__annotator_clean() to free up any resources. +*/ +static int fsl__annotation_start(Annotator * const a, + fsl_annotate_opt const * const opt){ + int rc; + uint64_t const diffFlags = fsl__annotate_opt_difflags(opt); + if(opt->spacePolicy>0){ + a->c.cmpLine = fsl_dline_cmp_ignore_ws; + }else{ + assert(fsl_dline_cmp == a->c.cmpLine); + } + rc = fsl_break_into_dlines(fsl_buffer_cstr(&a->headVersion), + (fsl_int_t)a->headVersion.used, + (uint32_t*)&a->c.nTo, &a->c.aTo, diffFlags); + if(rc) goto end; + if(!a->c.nTo){ + rc = FSL_RC_RANGE; + goto end; + } + a->aOrig = fsl_malloc( (fsl_size_t)(sizeof(a->aOrig[0]) * a->c.nTo) ); + if(!a->aOrig){ + rc = FSL_RC_OOM; + goto end; + } + for(int i = 0; i < a->c.nTo; ++i){ + a->aOrig[i].z = a->c.aTo[i].z; + a->aOrig[i].n = a->c.aTo[i].n; + a->aOrig[i].iVers = -1; + } + a->nOrig = (unsigned)a->c.nTo; + end: + return rc; +} + +/** + The input pParent is the next most recent ancestor of the file + being annotated. Do another step of the annotation. On success + return 0 and, if additional annotation is required, assign *doMore + (if not NULL) to true. +*/ +static int fsl__annotation_step( + Annotator * const a, + fsl_buffer const *pParent, + int iVers, + fsl_annotate_opt const * const opt +){ + int i, j, rc; + int lnTo; + uint64_t const diffFlags = fsl__annotate_opt_difflags(opt); + + /* Prepare the parent file to be diffed */ + rc = fsl_break_into_dlines(fsl_buffer_cstr(pParent), + (fsl_int_t)pParent->used, + (uint32_t*)&a->c.nFrom, &a->c.aFrom, + diffFlags); + if(rc) goto end; + else if( a->c.aFrom==0 ){ + return 0; + } + //MARKER(("Line #1: %.*s\n", (int)a->c.aFrom[0].n, a->c.aFrom[0].z)); + /* Compute the differences going from pParent to the file being + ** annotated. */ + rc = fsl__diff_all(&a->c); + if(rc) goto end; + + /* Where new lines are inserted on this difference, record the + ** iVers as the source of the new line. + */ + for(i=lnTo=0; ic.nEdit; i+=3){ + int const nCopy = a->c.aEdit[i]; + int const nIns = a->c.aEdit[i+2]; + lnTo += nCopy; + for(j=0; jaOrig[lnTo].iVers<0 ){ + a->aOrig[lnTo].iVers = iVers; + } + } + } + + /* Clear out the diff results except for c.aTo, as that's pointed to + by a->aOrig.*/ + fsl_free(a->c.aEdit); + a->c.aEdit = 0; + a->c.nEdit = 0; + a->c.nEditAlloc = 0; + + /* Clear out the from file */ + fsl_free(a->c.aFrom); + a->c.aFrom = 0; + a->c.nFrom = 0; + end: + return rc; +} + +/* MISSING(?) fossil(1) converts the diff inputs into utf8 with no + BOM. Whether we really want to do that here or rely on the caller + to is up for debate. If we do it here, we have to make the inputs + non-const, which seems "wrong" for a library API. */ +#define blob_to_utf8_no_bom(A,B) (void)0 + +static int fsl__annotate_file(fsl_cx * const f, + Annotator * const a, + fsl_annotate_opt const * const opt){ + int rc = FSL_RC_NYI; + fsl_buffer step = fsl_buffer_empty /*previous revision*/; + fsl_id_t cid = 0, fnid = 0; // , rid = 0; + fsl_stmt q = fsl_stmt_empty; + bool openedTransaction = false; + fsl_db * const db = fsl_needs_repo(f); + if(!db) return FSL_RC_NOT_A_REPO; + rc = fsl_cx_transaction_begin(f); + if(rc) goto dberr; + openedTransaction = true; + + fnid = fsl_db_g_id(db, 0, + "SELECT fnid FROM filename WHERE name=%Q %s", + opt->filename, fsl_cx_filename_collation(f)); + if(0==fnid){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "File not found in repository: %s", + opt->filename); + goto end; + } + if(opt->versionRid>0){ + cid = opt->versionRid; + }else{ + fsl_ckout_version_info(f, &cid, NULL); + if(cid<=0){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_A_CKOUT, + "Cannot determine version RID to " + "annotate from."); + goto end; + } + } + if(opt->originRid>0){ + rc = fsl_vpath_shortest_store_in_ancestor(f, cid, opt->originRid, NULL); + }else{ + rc = fsl_compute_direct_ancestors(f, cid); + } + if(rc) goto end; + + rc = fsl_db_prepare(db, &q, + "SELECT DISTINCT" + " (SELECT uuid FROM blob WHERE rid=mlink.fid)," + " (SELECT uuid FROM blob WHERE rid=mlink.mid)," + " coalesce(event.euser,event.user)," + " mlink.fid, event.mtime" + " FROM mlink, event, ancestor" + " WHERE mlink.fnid=%" FSL_ID_T_PFMT + " AND ancestor.rid=mlink.mid" + " AND event.objid=mlink.mid" + " AND mlink.mid!=mlink.pid" + " ORDER BY ancestor.generation;", + fnid + ); + if(rc) goto dberr; + + while(FSL_RC_STEP_ROW==fsl_stmt_step(&q)){ + if(a->nVers>=3){ + /* Process at least 3 rows before imposing any limit. That is + historical behaviour inherited from fossil(1). */ + if(opt->limitMs>0 && + fsl_timer_fetch(&a->timer)/1000 >= opt->limitMs){ + a->bMoreToDo = true; + break; + }else if(opt->limitVersions>0 && a->nVers>=opt->limitVersions){ + a->bMoreToDo = true; + break; + } + } + char * zTmp = 0; + char const * zCol = 0; + fsl_size_t nCol = 0; + fsl_id_t const rid = fsl_stmt_g_id(&q, 3); + double const mtime = fsl_stmt_g_double(&q, 4); + if(0==a->nVers){ + rc = fsl_content_get(f, rid, &a->headVersion); + if(rc) goto end; + blob_to_utf8_no_bom(&a->headVersion,0); + rc = fsl__annotation_start(a, opt); + if(rc) goto end; + a->bMoreToDo = opt->originRid>0; + a->origId = opt->originRid; + a->showId = cid; + assert(0==a->nVers); + assert(NULL==a->aVers); + } + if(a->naVers==a->nVers){ + unsigned int const n = a->naVers ? a->naVers*3/2 : 10; + void * const x = fsl_realloc(a->aVers, n*sizeof(a->aVers[0])); + if(NULL==x){ + rc = FSL_RC_OOM; + goto end; + } + a->aVers = x; + a->naVers = n; + } +#define AnnStr(COL,FLD) zCol = fsl_stmt_g_text(&q, COL, &nCol); \ + zTmp = fsl_strndup(zCol, nCol); \ + if(!zTmp){ rc = FSL_RC_OOM; goto end; } \ + a->aVers[a->nVers].FLD = zTmp + AnnStr(0,zFUuid); + AnnStr(1,zMUuid); + AnnStr(2,zUser); +#undef AnnStr + a->aVers[a->nVers].mtime = mtime; + if( a->nVers>0 ){ + rc = fsl_content_get(f, rid, &step); + if(!rc){ + rc = fsl__annotation_step(a, &step, a->nVers-1, opt); + } + fsl_buffer_reuse(&step); + if(rc) goto end; + } + ++a->nVers; + } + + assert(0==rc); + if(0==a->nVers){ + if(opt->versionRid>0){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "File [%s] does not exist " + "in checkin RID %" FSL_ID_T_PFMT, + opt->filename, opt->versionRid); + }else{ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "No history found for file: %s", + opt->filename); + } + } + + end: + fsl_buffer_clear(&step); + fsl_stmt_finalize(&q); + if(openedTransaction) fsl_cx_transaction_end(f, rc!=0); + return rc; + dberr: + assert(openedTransaction); + assert(rc!=0); + fsl_stmt_finalize(&q); + fsl_buffer_clear(&step); + rc = fsl_cx_uplift_db_error2(f, db, rc); + if(openedTransaction) fsl_cx_transaction_end(f, rc!=0); + return rc; +} + +int fsl_annotate_step_f_fossilesque(void * state, + fsl_annotate_opt const * const opt, + fsl_annotate_step const * const step){ + static const int szHash = 10; + fsl_outputer const * fout = (fsl_outputer*)state; + int rc = 0; + char ymd[24]; + if(step->mtime>0){ + fsl_julian_to_iso8601(step->mtime, &ymd[0], false); + ymd[10] = 0; + } + switch(step->stepType){ + case FSL_ANNOTATE_STEP_VERSION: + rc = fsl_appendf(fout->out, fout->state, + "version %3d: %s %.*s file %.*s\n", + step->stepNumber+1, ymd, szHash, + step->versionHash, szHash, step->fileHash); + break; + case FSL_ANNOTATE_STEP_FULL: + if(opt->praise){ + rc = fsl_appendf(fout->out, fout->state, + "%.*s %s %13.13s: %.*s\n", + szHash, + opt->fileVersions ? step->fileHash : step->versionHash, + ymd, step->username, + (int)step->lineLength, step->line); + }else{ + rc = fsl_appendf(fout->out, fout->state, + "%.*s %s %5d: %.*s\n", + szHash, opt->fileVersions ? step->fileHash : step->versionHash, + ymd, step->lineNumber, + (int)step->lineLength, step->line); + } + break; + case FSL_ANNOTATE_STEP_LIMITED: + if(opt->praise){ + rc = fsl_appendf(fout->out, fout->state, + "%*s %.*s\n", szHash+26, "", + (int)step->lineLength, step->line); + }else{ + rc = fsl_appendf(fout->out, fout->state, + "%*s %5" PRIu32 ": %.*s\n", + szHash+11, "", step->lineNumber, + (int)step->lineLength, step->line); + } + break; + } + return rc; +} + + +int fsl_annotate( fsl_cx * const f, fsl_annotate_opt const * const opt ){ + int rc; + Annotator ann = Annotator_empty; + unsigned int i; + fsl_buffer * const scratch = fsl_cx_scratchpad(f); + fsl_annotate_step aStep; + assert(opt->out); + + if(opt->limitMs>0) fsl_timer_start(&ann.timer); + rc = fsl__annotate_file(f, &ann, opt); + if(rc) goto end; + memset(&aStep,0,sizeof(fsl_annotate_step)); + + if(opt->dumpVersions){ + struct AnnVers *av; + for(av = ann.aVers, i = 0; + 0==rc && i < ann.nVers; ++i, ++av){ + aStep.fileHash = av->zFUuid; + aStep.versionHash = av->zMUuid; + aStep.mtime = av->mtime; + aStep.stepNumber = i; + aStep.stepType = FSL_ANNOTATE_STEP_VERSION; + rc = opt->out(opt->outState, opt, &aStep); + } + if(rc) goto end; + } + + for(i = 0; 0==rc && iused; + + if(iVers>=0){ + struct AnnVers * const av = &ann.aVers[iVers]; + aStep.fileHash = av->zFUuid; + aStep.versionHash = av->zMUuid; + aStep.mtime = av->mtime; + aStep.username = av->zUser; + aStep.stepType = FSL_ANNOTATE_STEP_FULL; + }else{ + aStep.fileHash = aStep.versionHash = + aStep.username = NULL; + aStep.stepType = FSL_ANNOTATE_STEP_LIMITED; + aStep.mtime = 0.0; + } + rc = opt->out(opt->outState, opt, &aStep); + } + + end: + fsl_cx_scratchpad_yield(f, scratch); + fsl__annotator_clean(&ann); + return rc; +} + +#undef MARKER +#undef blob_to_utf8_no_bom +/* end of file annotate.c */ +/* start of file appendf.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/************************************************************************ +The printf-like implementation in this file is based on the one found +in the sqlite3 distribution is in the Public Domain. + +This copy was forked for use with the clob API in Feb 2008 by Stephan +Beal (https://wanderinghorse.net/home/stephan/) and modified to send +its output to arbitrary targets via a callback mechanism. Also +refactored the %X specifier handlers a bit to make adding/removing +specific handlers easier. + +All code in this file is released into the Public Domain. + +The printf implementation (fsl_appendfv()) is pretty easy to extend +(e.g. adding or removing %-specifiers for fsl_appendfv()) if you're +willing to poke around a bit and see how the specifiers are declared +and dispatched. For an example, grep for 'etSTRING' and follow it +through the process of declaration to implementation. + +See below for several FSLPRINTF_OMIT_xxx macros which can be set to +remove certain features/extensions. + +LICENSE: + + This program is free software; you can redistribute it and/or + modify it under the terms of the Simplified BSD License (also + known as the "2-Clause License" or "FreeBSD License".) + + This program is distributed in the hope that it will be useful, + but without any warranty; without even the implied warranty of + merchantability or fitness for a particular purpose. +**********************************************************************/ + +#include /* strlen() */ +#include +#include + +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +/* FIXME: determine this type at compile time via configuration + options. OTOH, it compiles everywhere as-is so far. + */ +typedef long double LONGDOUBLE_TYPE; + +/* + If FSLPRINTF_OMIT_FLOATING_POINT is defined to a true value, then + floating point conversions are disabled. +*/ +#ifndef FSLPRINTF_OMIT_FLOATING_POINT +# define FSLPRINTF_OMIT_FLOATING_POINT 0 +#endif + +/* + If FSLPRINTF_OMIT_SQL is defined to a true value, then + the %q, %Q, and %B specifiers are disabled. +*/ +#ifndef FSLPRINTF_OMIT_SQL +# define FSLPRINTF_OMIT_SQL 0 +#endif + +/* + If FSLPRINTF_OMIT_HTML is defined to a true value then the %h (HTML + escape), %t (URL escape), and %T (URL unescape) specifiers are + disabled. +*/ +#ifndef FSLPRINTF_OMIT_HTML +# define FSLPRINTF_OMIT_HTML 0 +#endif + +/** + If true, the %j (JSON string) format is enabled. +*/ +#define FSLPRINTF_ENABLE_JSON 1 + +/* + Most C compilers handle variable-sized arrays, so we enable + that by default. Some (e.g. tcc) do not, so we provide a way + to disable it: set FSLPRINTF_HAVE_VARARRAY to 0 + + One approach would be to look at: + + defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) + + but some compilers support variable-sized arrays even when not + explicitly running in c99 mode. +*/ +#if !defined(FSLPRINTF_HAVE_VARARRAY) +# if defined(__TINYC__) +# define FSLPRINTF_HAVE_VARARRAY 0 +# else +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) +# define FSLPRINTF_HAVE_VARARRAY 1 /*use 1 in C99 mode */ +# else +# define FSLPRINTF_HAVE_VARARRAY 0 +# endif +# endif +#endif + +/* + Conversion types fall into various categories as defined by the + following enumeration. +*/ +enum PrintfCategory {etRADIX = 1, /* Integer types. %d, %x, %o, and so forth */ + etFLOAT = 2, /* Floating point. %f */ + etEXP = 3, /* Exponentional notation. %e and %E */ + etGENERIC = 4, /* Floating or exponential, depending on exponent. %g */ + /* 5 can be reused. Formerly etSIZE (%n) */ + etSTRING = 6, /* Strings. %s */ + etDYNSTRING = 7, /* Dynamically allocated strings. %z */ + etPERCENT = 8, /* Percent symbol. %% */ + etCHARX = 9, /* Characters. %c */ + /* The rest are extensions, not normally found in printf() */ + etCHARLIT = 10, /* Literal characters. %' */ +#if !FSLPRINTF_OMIT_SQL + etSQLESCAPE = 11, /* Strings with '\'' doubled. %q */ + etSQLESCAPE2 = 12, /* Strings with '\'' doubled and enclosed in '', + NULL pointers replaced by SQL NULL. %Q */ + etSQLESCAPE3 = 14, /* %!Q -> identifiers wrapped in \" + with inner '\"' doubled */ + etBLOBSQL = 13, /* %B -> Works like %Q, + but requires a (fsl_buffer*) argument. */ +#endif /* !FSLPRINTF_OMIT_SQL */ + etPOINTER = 15, /* The %p conversion */ + etORDINAL = 17, /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#if ! FSLPRINTF_OMIT_HTML + etHTML = 18, /* %h -> basic HTML escaping. */ + etURLENCODE = 19, /* %t -> URL encoding. */ + etURLDECODE = 20, /* %T -> URL decoding. */ +#endif + etPATH = 21, /* %/ -> replace '\\' with '/' in path-like strings. */ + etBLOB = 22, /* Works like %s, but requires a (fsl_buffer*) argument. */ + etFOSSILIZE = 23, /* %F => like %s, but fossilizes it. */ + etSTRINGID = 24, /* String with length limit for a UUID prefix: %S */ +#if FSLPRINTF_ENABLE_JSON + etJSONSTR = 25, +#endif + etPLACEHOLDER = 100 + }; + +/* + An "etByte" is an 8-bit unsigned value. +*/ +typedef unsigned char etByte; + +/* + Each builtin conversion character (ex: the 'd' in "%d") is described + by an instance of the following structure +*/ +typedef struct et_info { /* Information about each format field */ + char fmttype; /* The format field code letter */ + etByte base; /* The base for radix conversion */ + etByte flags; /* One or more of FLAG_ constants below */ + etByte type; /* Conversion paradigm */ + etByte charset; /* Offset into aDigits[] of the digits string */ + etByte prefix; /* Offset into aPrefix[] of the prefix string */ +} et_info; + +/* + Allowed values for et_info.flags +*/ +enum et_info_flags { FLAG_SIGNED = 1, /* True if the value to convert is signed */ + FLAG_EXTENDED = 2, /* True if for internal/extended use only. */ + FLAG_STRING = 4 /* Allow infinity precision */ +}; + +/* + Historically, the following table was searched linearly, so the most + common conversions were kept at the front. + + Change 2008 Oct 31 by Stephan Beal: we reserve an array of ordered + entries for all chars in the range [32..126]. Format character + checks can now be done in constant time by addressing that array + directly. This takes more static memory, but reduces the time and + per-call overhead costs of fsl_appendfv(). +*/ +static const char aDigits[] = "0123456789ABCDEF0123456789abcdef"; +static const char aPrefix[] = "-x0\000X0"; +static const et_info fmtinfo[] = { +/* + These entries MUST stay in ASCII order, sorted + on their fmttype member! They MUST start with + fmttype==32 and end at fmttype==126. +*/ +{' '/*32*/, 0, 0, 0, 0, 0 }, +{'!'/*33*/, 0, 0, 0, 0, 0 }, +{'"'/*34*/, 0, 0, 0, 0, 0 }, +{'#'/*35*/, 0, 0, 0, 0, 0 }, +{'$'/*36*/, 0, 0, 0, 0, 0 }, +{'%'/*37*/, 0, 0, etPERCENT, 0, 0 }, +{'&'/*38*/, 0, 0, 0, 0, 0 }, +{'\''/*39*/, 0, 0, 0, 0, 0 }, +{'('/*40*/, 0, 0, 0, 0, 0 }, +{')'/*41*/, 0, 0, 0, 0, 0 }, +{'*'/*42*/, 0, 0, 0, 0, 0 }, +{'+'/*43*/, 0, 0, 0, 0, 0 }, +{','/*44*/, 0, 0, 0, 0, 0 }, +{'-'/*45*/, 0, 0, 0, 0, 0 }, +{'.'/*46*/, 0, 0, 0, 0, 0 }, +{'/'/*47*/, 0, 0, etPATH, 0, 0 }, +{'0'/*48*/, 0, 0, 0, 0, 0 }, +{'1'/*49*/, 0, 0, 0, 0, 0 }, +{'2'/*50*/, 0, 0, 0, 0, 0 }, +{'3'/*51*/, 0, 0, 0, 0, 0 }, +{'4'/*52*/, 0, 0, 0, 0, 0 }, +{'5'/*53*/, 0, 0, 0, 0, 0 }, +{'6'/*54*/, 0, 0, 0, 0, 0 }, +{'7'/*55*/, 0, 0, 0, 0, 0 }, +{'8'/*56*/, 0, 0, 0, 0, 0 }, +{'9'/*57*/, 0, 0, 0, 0, 0 }, +{':'/*58*/, 0, 0, 0, 0, 0 }, +{';'/*59*/, 0, 0, 0, 0, 0 }, +{'<'/*60*/, 0, 0, 0, 0, 0 }, +{'='/*61*/, 0, 0, 0, 0, 0 }, +{'>'/*62*/, 0, 0, 0, 0, 0 }, +{'?'/*63*/, 0, 0, 0, 0, 0 }, +{'@'/*64*/, 0, 0, 0, 0, 0 }, +{'A'/*65*/, 0, 0, 0, 0, 0 }, +#if FSLPRINTF_OMIT_SQL +{'B'/*66*/, 0, 0, 0, 0, 0 }, +#else +{'B'/*66*/, 0, 2, etBLOBSQL, 0, 0 }, +#endif +{'C'/*67*/, 0, 0, 0, 0, 0 }, +{'D'/*68*/, 0, 0, 0, 0, 0 }, +{'E'/*69*/, 0, FLAG_SIGNED, etEXP, 14, 0 }, +{'F'/*70*/, 0, 4, etFOSSILIZE, 0, 0 }, +{'G'/*71*/, 0, FLAG_SIGNED, etGENERIC, 14, 0 }, +{'H'/*72*/, 0, 0, 0, 0, 0 }, +{'I'/*73*/, 0, 0, 0, 0, 0 }, +{'J'/*74*/, 0, 0, 0, 0, 0 }, +{'K'/*75*/, 0, 0, 0, 0, 0 }, +{'L'/*76*/, 0, 0, 0, 0, 0 }, +{'M'/*77*/, 0, 0, 0, 0, 0 }, +{'N'/*78*/, 0, 0, 0, 0, 0 }, +{'O'/*79*/, 0, 0, 0, 0, 0 }, +{'P'/*80*/, 0, 0, 0, 0, 0 }, +#if FSLPRINTF_OMIT_SQL +{'Q'/*81*/, 0, 0, 0, 0, 0 }, +#else +{'Q'/*81*/, 0, FLAG_STRING, etSQLESCAPE2, 0, 0 }, +#endif +{'R'/*82*/, 0, 0, 0, 0, 0 }, +{'S'/*83*/, 0, FLAG_STRING, etSTRINGID, 0, 0 }, +{'T'/*84*/, 0, FLAG_STRING, etURLDECODE, 0, 0 }, +{'U'/*85*/, 0, 0, 0, 0, 0 }, +{'V'/*86*/, 0, 0, 0, 0, 0 }, +{'W'/*87*/, 0, 0, 0, 0, 0 }, +{'X'/*88*/, 16, 0, etRADIX, 0, 4 }, +{'Y'/*89*/, 0, 0, 0, 0, 0 }, +{'Z'/*90*/, 0, 0, 0, 0, 0 }, +{'['/*91*/, 0, 0, 0, 0, 0 }, +{'\\'/*92*/, 0, 0, 0, 0, 0 }, +{']'/*93*/, 0, 0, 0, 0, 0 }, +{'^'/*94*/, 0, 0, 0, 0, 0 }, +{'_'/*95*/, 0, 0, 0, 0, 0 }, +{'`'/*96*/, 0, 0, 0, 0, 0 }, +{'a'/*97*/, 0, 0, 0, 0, 0 }, +{'b'/*98*/, 0, 2, etBLOB, 0, 0 }, +{'c'/*99*/, 0, 0, etCHARX, 0, 0 }, +{'d'/*100*/, 10, FLAG_SIGNED, etRADIX, 0, 0 }, +{'e'/*101*/, 0, FLAG_SIGNED, etEXP, 30, 0 }, +{'f'/*102*/, 0, FLAG_SIGNED, etFLOAT, 0, 0}, +{'g'/*103*/, 0, FLAG_SIGNED, etGENERIC, 30, 0 }, +{'h'/*104*/, 0, FLAG_STRING, etHTML, 0, 0 }, +{'i'/*105*/, 10, FLAG_SIGNED, etRADIX, 0, 0}, +#if FSLPRINTF_ENABLE_JSON +{'j'/*106*/, 0, 0, etJSONSTR, 0, 0 }, +#else +{'j'/*106*/, 0, 0, 0, 0, 0 }, +#endif +{'k'/*107*/, 0, 0, 0, 0, 0 }, +{'l'/*108*/, 0, 0, 0, 0, 0 }, +{'m'/*109*/, 0, 0, 0, 0, 0 }, +{'n'/*110*/, 0, 0, 0, 0, 0 }, +{'o'/*111*/, 8, 0, etRADIX, 0, 2 }, +{'p'/*112*/, 16, 0, etPOINTER, 0, 1 }, +#if FSLPRINTF_OMIT_SQL +{'q'/*113*/, 0, 0, 0, 0, 0 }, +#else +{'q'/*113*/, 0, FLAG_STRING, etSQLESCAPE, 0, 0 }, +#endif +{'r'/*114*/, 10, (FLAG_EXTENDED|FLAG_SIGNED), etORDINAL, 0, 0}, +{'s'/*115*/, 0, FLAG_STRING, etSTRING, 0, 0 }, +{'t'/*116*/, 0, FLAG_STRING, etURLENCODE, 0, 0 }, +{'u'/*117*/, 10, 0, etRADIX, 0, 0 }, +{'v'/*118*/, 0, 0, 0, 0, 0 }, +#if 1 || FSLPRINTF_OMIT_SQL +{'w'/*119*/, 0, 0, 0, 0, 0 }, +#else +/* This role is filled by %!Q. %w is not currently used/documented. */ +{'w'/*119*/, 0, FLAG_STRING, etSQLESCAPE3, 0, 0 }, +#endif +{'x'/*120*/, 16, 0, etRADIX, 16, 1 }, +{'y'/*121*/, 0, 0, 0, 0, 0 }, +{'z'/*122*/, 0, FLAG_STRING, etDYNSTRING, 0, 0}, +{'{'/*123*/, 0, 0, 0, 0, 0 }, +{'|'/*124*/, 0, 0, 0, 0, 0 }, +{'}'/*125*/, 0, 0, 0, 0, 0 }, +{'~'/*126*/, 0, 0, 0, 0, 0 } +}; +#define etNINFO (sizeof(fmtinfo)/sizeof(fmtinfo[0])) + +#if ! FSLPRINTF_OMIT_FLOATING_POINT +/* + "*val" is a double such that 0.1 <= *val < 10.0 + Return the ascii code for the leading digit of *val, then + multiply "*val" by 10.0 to renormalize. + + Example: + input: *val = 3.14159 + output: *val = 1.4159 function return = '3' + + The counter *cnt is incremented each time. After counter exceeds + 16 (the number of significant digits in a 64-bit float) '0' is + always returned. +*/ +static int et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ + int digit; + LONGDOUBLE_TYPE d; + if( (*cnt)++ >= 16 ) return '0'; + digit = (int)*val; + d = digit; + digit += '0'; + *val = (*val - d)*10.0; + return digit; +} +#endif /* !FSLPRINTF_OMIT_FLOATING_POINT */ + +/* + On machines with a small(?) stack size, you can redefine the + FSLPRINTF_BUF_SIZE to be less than 350. But beware - for smaller + values some %f conversions may go into an infinite loop. +*/ +#ifndef FSLPRINTF_BUF_SIZE +# define FSLPRINTF_BUF_SIZE 350 /* Size of the output buffer for numeric conversions */ +#endif + +#if defined(FSL_INT_T_PFMT) +/* int64_t is already defined. */ +#else +#if ! defined(__STDC__) && !defined(__TINYC__) +#ifdef FSLPRINTF_INT64_TYPE +typedef FSLPRINTF_INT64_TYPE int64_t; +typedef unsigned FSLPRINTF_INT64_TYPE uint64_t; +#elif defined(_MSC_VER) || defined(__BORLANDC__) +typedef __int64 int64_t; +typedef unsigned __int64 uint64_t; +#else +typedef long long int int64_t; +typedef unsigned long long int uint64_t; +#endif +#endif +#endif +/* Set up of int64 type */ + +#if 0 +/ Not yet used. */ +enum PrintfArgTypes { +TypeInt = 0, +TypeIntP = 1, +TypeFloat = 2, +TypeFloatP = 3, +TypeCString = 4 +}; +#endif + + +#if 0 +/ Not yet used. */ +typedef struct fsl_appendf_spec_handler_def +{ + char letter; / e.g. %s */ + int xtype; /* reference to the etXXXX values, or fmtinfo[*].type. */ + int ntype; /* reference to PrintfArgTypes enum. */ +} spec_handler; +#endif + +/** + fsl_appendf_spec_handler is an almost-generic interface for farming + work out of fsl_appendfv()'s code into external functions. It doesn't + actually save much (if any) overall code, but it makes the fsl_appendfv() + code more manageable. + + + REQUIREMENTS of implementations: + + - Expects an implementation-specific vargp pointer. + fsl_appendfv() passes a pointer to the converted value of + an entry from the format va_list. If it passes a type + other than the expected one, undefined results. + + - If it calls pf it must do: pf( pfArg, D, N ), where D is + the data to export and N is the number of bytes to export. + It may call pf() an arbitrary number of times + + - If pf() successfully is called, the return value must be the + accumulated totals of its return value(s), plus (possibly, but + unlikely) an implementation-specific amount. + + - If it does not call pf() then it must return 0 (success) + or a negative number (an error) or do all of the export + processing itself and return the number of bytes exported. + + SIGNIFICANT LIMITATIONS: + + - Has no way of iterating over the format string, so handling + precisions and such here can't work too well. (Nevermind: + precision/justification is handled in fsl_appendfv().) +*/ +typedef int (*fsl_appendf_spec_handler)( fsl_output_f pf, + void * pfArg, + unsigned int pfLen, + void * vargp ); + + +/** + fsl_appendf_spec_handler for etSTRING types. It assumes that varg + is a NUL-terminated (char [const] *) +*/ +static int spech_string( fsl_output_f pf, void * pfArg, + unsigned int pfLen, void * varg ){ + char const * ch = (char const *) varg; + return ch ? pf( pfArg, ch, pfLen ) : 0; +} + +/** + fsl_appendf_spec_handler for etDYNSTRING types. It assumes that + varg is a non-const (char *). It behaves identically to + spec_string() and then calls fsl_free() on that (char *). +*/ +static int spech_dynstring( fsl_output_f pf, void * pfArg, + unsigned int pfLen, void * varg ){ + int const rc = spech_string( pf, pfArg, pfLen, varg ); + fsl_free( varg ); + return rc; +} + +#if !FSLPRINTF_OMIT_HTML +static int spech_string_to_html( fsl_output_f pf, void * pfArg, + unsigned int pfLen, void * varg ){ + char const * ch = (char const *) varg; + unsigned int i; + int rc = 0; + if( ! ch ) return 0; + rc = 0; + for( i = 0; 0==rc && (i= 32 && c <=47) + || ( c>=58 && c<=64) + || ( c>=91 && c<=96) + || ( c>=123 && c<=126) + || ( c<32 || c>=127) + ); +} + +/** + The handler for the etURLENCODE specifier. + + It expects varg to be a string value, which it will preceed to + encode using an URL encoding algothrim (certain characters are + converted to %XX, where XX is their hex value) and passes the + encoded string to pf(). It returns the total length of the output + string. +*/ +static int spech_urlencode( fsl_output_f pf, void * pfArg, + unsigned int pfLen, void * varg ){ + char const * str = (char const *) varg; + int rc = 0; + char ch = 0; + char const * hex = "0123456789ABCDEF"; +#define xbufsz 10 + char xbuf[xbufsz]; + int slen = 0; + if( ! str ) return 0; + memset( xbuf, 0, xbufsz ); + ch = *str; +#define xbufsz 10 + slen = 0; + for( ; 0==rc && ch; ch = *(++str) ){ + if( ! httpurl_needs_escape( ch ) ){ + rc = pf( pfArg, str, 1 ); + continue; + }else{ + xbuf[0] = '%'; + xbuf[1] = hex[((ch>>4)&0xf)]; + xbuf[2] = hex[(ch&0xf)]; + xbuf[3] = 0; + slen = 3; + rc = pf( pfArg, xbuf, slen ); + } + } +#undef xbufsz + return rc; +} + +/* + hexchar_to_int(): + + For 'a'-'f', 'A'-'F' and '0'-'9', returns the appropriate decimal + number. For any other character it returns -1. +*/ +static int hexchar_to_int( int ch ){ + if( (ch>='0' && ch<='9') ) return ch-'0'; + else if( (ch>='a' && ch<='f') ) return ch-'a'+10; + else if( (ch>='A' && ch<='F') ) return ch-'A'+10; + else return -1; +} + +/** + The handler for the etURLDECODE specifier. + + It expects varg to be a ([const] char *), possibly encoded + with URL encoding. It decodes the string using a URL decode + algorithm and passes the decoded string to + pf(). It returns the total length of the output string. + If the input string contains malformed %XX codes then this + function will return prematurely. +*/ +static int spech_urldecode( fsl_output_f pf, void * pfArg, + unsigned int pfLen, void * varg ){ + char const * str = (char const *) varg; + int rc = 0; + char ch = 0; + char ch2 = 0; + char xbuf[4]; + int decoded; + if( ! str ) return 0; + ch = *str; + while( 0==rc && ch ){ + if( ch == '%' ){ + ch = *(++str); + ch2 = *(++str); + if( isxdigit((int)ch) && + isxdigit((int)ch2) ) + { + decoded = (hexchar_to_int( ch ) * 16) + + hexchar_to_int( ch2 ); + xbuf[0] = (char)decoded; + xbuf[1] = 0; + rc = pf( pfArg, xbuf, 1 ); + ch = *(++str); + continue; + }else{ + xbuf[0] = '%'; + xbuf[1] = ch; + xbuf[2] = ch2; + xbuf[3] = 0; + rc = pf( pfArg, xbuf, 3 ); + ch = *(++str); + continue; + } + }else if( ch == '+' ){ + xbuf[0] = ' '; + xbuf[1] = 0; + rc = pf( pfArg, xbuf, 1 ); + ch = *(++str); + continue; + } + xbuf[0] = ch; + xbuf[1] = 0; + rc = pf( pfArg, xbuf, 1 ); + ch = *(++str); + } + return rc; +} + +#endif /* !FSLPRINTF_OMIT_HTML */ + + +#if !FSLPRINTF_OMIT_SQL +/** + Quotes the (char *) varg as an SQL string 'should' be quoted. The + exact type of the conversion is specified by xtype, which must be + one of etSQLESCAPE, etSQLESCAPE2, etSQLESCAPE3. + + Search this file for those constants to find the associated + documentation. +*/ +static int spech_sqlstring( int xtype, fsl_output_f pf, + void * pfArg, unsigned int pfLen, + void * varg ){ + enum { BufLen = 512 }; + char buf[BufLen]; + unsigned int i = 0, j = 0; + int ch; + char const q = xtype==etSQLESCAPE3 ?'"':'\''; /* Quote character */ + char const * escarg = (char const *) varg; + bool const isnull = escarg==0; + bool const needQuote = + !isnull && (xtype==etSQLESCAPE2 || xtype==etSQLESCAPE3); + if( isnull ){ + escarg = (xtype==etSQLESCAPE2||xtype==etSQLESCAPE3) + ? "NULL" : "(NULL)"; + } + if( needQuote ) buf[j++] = q; + for(i=0; (ch=escarg[i])!=0 && i=BufLen){ + int const rc = pf( pfArg, &buf[0], j ); + if(rc) return rc; + j = 0; + } + } + if( needQuote ) buf[j++] = q; + buf[j] = 0; + return j>0 ? pf( pfArg, &buf[0], j ) : 0; +} + +#endif /* !FSLPRINTF_OMIT_SQL */ + +#if FSLPRINTF_ENABLE_JSON +/* TODO? Move these UTF8 bits into the public API? */ +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. +** +** Taken from sqlite3: +** https://www.sqlite.org/src/artifact?ln=48-61&name=810fbfebe12359f1 +*/ +static const unsigned char fsl_utfTrans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00 +}; +unsigned int fsl_utf8_read_char( + const unsigned char *zIn, /* First byte of UTF-8 character */ + const unsigned char *zTerm, /* Pretend this byte is 0x00 */ + const unsigned char **pzNext /* Write first byte past UTF-8 char here */ +){ + /* + Adapted from sqlite3: + https://www.sqlite.org/src/artifact?ln=155-165&name=810fbfebe12359f1 + */ + unsigned c; + if(zIn>=zTerm){ + *pzNext = zTerm; + c = 0; + }else{ + c = (unsigned int)*(zIn++); + if( c>=0xc0 ){ + c = fsl_utfTrans1[c-0xc0]; + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ) + c = (c<<6) + (0x3f & *(zIn++)); + if( c<0x80 + || (c&0xFFFFF800)==0xD800 + || (c&0xFFFFFFFE)==0xFFFE ) c = 0xFFFD; + } + *pzNext = zIn; + } + return c; +} + +static int fsl_utf8_char_to_cstr(unsigned int c, unsigned char *output, unsigned char length){ + /* Stolen from the internet, adapted from several variations which + all _seem_ to have derived from librdf. */ + unsigned char size=0; + + /* check for illegal code positions: + * U+D800 to U+DFFF (UTF-16 surrogates) + * U+FFFE and U+FFFF + */ + if((c > 0xD7FF && c < 0xE000) + || c == 0xFFFE || c == 0xFFFF) return -1; + + /* Unicode 3.2 only defines U+0000 to U+10FFFF and UTF-8 encodings of it */ + if(c > 0x10ffff) return -1; + + if (c < 0x00000080) size = 1; + else if (c < 0x00000800) size = 2; + else if (c < 0x00010000) size = 3; + else size = 4; + if(!output) return (int)size; + else if(size > length) return -1; + else switch(size) { + case 0: + assert(!"can't happen anymore"); + output[0] = 0; + return 0; + case 4: + output[3] = 0x80 | (c & 0x3F); + c = c >> 6; + c |= 0x10000; + /* Fall through */ + case 3: + output[2] = 0x80 | (c & 0x3F); + c = c >> 6; + c |= 0x800; + /* Fall through */ + case 2: + output[1] = 0x80 | (c & 0x3F); + c = c >> 6; + c |= 0xc0; + /* Fall through */ + case 1: + output[0] = (unsigned char)c; + /* Fall through */ + default: + return (int)size; + } +} + +struct SpechJson { + char const * z; + bool addQuotes; + bool escapeSmallUtf8; +}; + +/** + fsl_appendf_spec_handler for etJSONSTR. It assumes that varg is a + SpechJson struct instance. +*/ +static int spech_json( fsl_output_f pf, void * pfArg, + unsigned int pfLen, void * varg ){ + struct SpechJson const * state = (struct SpechJson *)varg; + int pfRc = 0; + const unsigned char *z = (const unsigned char *)state->z; + const unsigned char *zEnd = z + pfLen; + const unsigned char * zNext = 0; + unsigned int c; + unsigned char c1; + +#define out(X,N) pfRc=pf(pfArg, (char const *)(X), N); \ + if(0!=pfRc) return pfRc +#define outc c1 = (unsigned char)c; out(&c1,1) + if(!z){ + out("null",4); + return pfRc; + } + if(state->addQuotes){ + out("\"", 1); + } + for( ; 0==pfRc && (z < zEnd) + && (c=fsl_utf8_read_char(z, zEnd, &zNext)); + z = zNext ){ + if( c=='\\' || c=='"' ){ + out("\\", 1); + outc; + }else if( c<' ' ){ + out("\\",1); + if( c=='\n' ){ + out("n",1); + }else if( c=='\r' ){ + out("r",1); + }else{ + unsigned char ubuf[5] = {'u',0,0,0,0}; + int i; + for(i = 4; i>0; --i){ + ubuf[i] = "0123456789abcdef"[c&0xf]; + c >>= 4; + } + out(ubuf,5); + } + }else if(c<128){ + outc; + }/* At this point we know that c is part of a multi-byte + character. We're assuming legal UTF8 input, which means + emitting a surrogate pair if the value is > 0xffff. */ + else if(c<0xFFFF){ + unsigned char ubuf[6]; + if(state->escapeSmallUtf8){ + /* Output char in \u#### form. */ + fsl_snprintf((char *)ubuf, 6, "\\u%04x", c); + out(ubuf, 6); + }else{ + /* Output character literal. */ + int const n = fsl_utf8_char_to_cstr(c, ubuf, 4); + if(n<0){ + out("?",1); + }else{ + assert(n>0); + out(ubuf, n); + } + } + }else{ + /* Surrogate pair. */ + unsigned char ubuf[12]; + c -= 0x10000; + fsl_snprintf((char *)ubuf, 12, "\\u%04x\\u%04x", + (0xd800 | (c>>10)), + (0xdc00 | (c & 0x3ff))); + out(ubuf, 12); + } + } + if(state->addQuotes){ + out("\"",1); + } + return pfRc; +#undef out +#undef outc +} +#endif /* FSLPRINTF_ENABLE_JSON */ + +/* + Find the length of a string as long as that length does not + exceed N bytes. If no zero terminator is seen in the first + N bytes then return N. If N is negative, then this routine + is an alias for strlen(). +*/ +static int StrNLen32(const char *z, int N){ + int n = 0; + while( (N-- != 0) && *(z++)!=0 ){ n++; } + return n; +} + +#if 0 +/** + Given the first byte of an assumed-to-be well-formed UTF8 + character, returns the length of that character. Returns 0 if the + character appears to be an invalid UTF8 character, else returns its + length, in bytes (1-4). Note that a NUL byte is a valid length-1 + character. +*/ +static int utf8__char_length( unsigned char const * const c ){ + switch(0xF0 & *c) { + case 0xF0: return (c[1]&0x80 && c[2]&0x80 && c[3]&0x80) ? 4 : 0; + case 0xE0: return (c[1]&0x80 && c[2]&0x80) ? 3 : 0; + case 0xC0: return (c[1]&0x80) ? 2 : 0; + case 0x80: return 0; + default: return 1; + /* See also: https://stackoverflow.com/questions/4884656/utf-8-encoding-size */ + } +} +#endif + +/** + Internal helper for %#W.Ps format. +*/ +static void appendf__utf8_altform(char const * z, int * pLength, + int * pPrecision, int * pWidth){ + /* Treat %#W.Ps as a width/precision limit of W resp. P UTF8 + characters instead of bytes. */ + int pC = 0/*precision, chars*/, pB = 0/*precision, bytes*/, + wC = 0/*width, chars*/, wB = 0/*width, bytes*/; + char const * const zEnd = z + *pLength; + int lc; + while( z < zEnd ){ + switch(0xF0 & *z) { + case 0xF0: lc = (z[1]&0x80 && z[2]&0x80 && z[3]&0x80) ? 4 : 0; break; + case 0xE0: lc = (z[1]&0x80 && z[2]&0x80) ? 3 : 0; break; + case 0xC0: lc = (z[1]&0x80) ? 2 : 0; break; + case 0x80: lc = 0; break; + default: lc = 1; break; + } + if(!lc) break; + else if(wC<*pWidth && (*pPrecision<=0 || pC<*pPrecision)){ ++wC; wB+=lc;} + if(pC<*pPrecision){ ++pC; pB+=lc;} + z+=lc; + } + if(*pPrecision>0) *pLength = pB; + if(*pWidth>0) *pWidth = *pWidth - wC + wB; +} + +/* + The root printf program. All variations call this core. It + implements most of the common printf behaviours plus (optionally) + some extended ones. + + INPUTS: + + pfAppend : The is a fsl_output_f function which is responsible for + accumulating the output. If pfAppend returns non-0 then processing + stops immediately. + + pfAppendArg : is ignored by this function but passed as the first + argument to pfAppend. pfAppend will presumably use it as a data + store for accumulating its string. + + fmt : This is the format string, as in the usual printf(). + + ap : This is a pointer to a list of arguments. Same as in + vprintf() and friends. + + OUTPUTS: + + The return value is 0 on success, non-0 on error. Historically it + returned the total number bytes reported appended by pfAppend, but + those semantics (A) are only very, very rarely useful and (B) they + make sensibly reporting errors via the generic callback interface + next to impossible. e.g. the callback may encounter I/O or allocation + errors. + + Much of this code dates back to the early 1980's, supposedly. + + Known change history (most historic info has been lost): + + 10 Feb 2008 by Stephan Beal: refactored to remove the 'useExtended' + flag (which is now always on). Added the fsl_output_f typedef to + make this function generic enough to drop into other source trees + without much work. + + 31 Oct 2008 by Stephan Beal: refactored the et_info lookup to be + constant-time instead of linear. +*/ +int fsl_appendfv(fsl_output_f pfAppend, /* Accumulate results here */ + void * pfAppendArg, /* Passed as first arg to pfAppend. */ + const char *fmt, /* Format string */ + va_list ap /* arguments */ + ){ + /** + HISTORIC NOTE (author and year unknown): + + Note that the order in which automatic variables are declared below + seems to make a big difference in determining how fast this beast + will run. + */ + int pfrc = 0; /* result from calling pfAppend */ + int c; /* Next character in the format string */ + char *bufpt = 0; /* Pointer to the conversion buffer */ + int precision = 0; /* Precision of the current field */ + int length; /* Length of the field */ + int idx; /* A general purpose loop counter */ + int width; /* Width of the current field */ + etByte flag_leftjustify; /* True if "-" flag is present */ + etByte flag_plussign; /* True if "+" flag is present */ + etByte flag_blanksign; /* True if " " flag is present */ + etByte flag_alternateform; /* True if "#" flag is present */ + etByte flag_altform2; /* True if "!" flag is present */ + etByte flag_zeropad; /* True if field width constant starts with zero */ + etByte flag_long; /* True if "l" flag is present */ + etByte flag_longlong; /* True if the "ll" flag is present */ + etByte done; /* Loop termination flag */ + etByte cThousand /* Thousands separator for %d and %u */ + /* ported in from https://fossil-scm.org/home/info/2cdbdbb1c9b7ad2b */; + uint64_t longvalue; /* Value for integer types */ + LONGDOUBLE_TYPE realvalue; /* Value for real types */ + const et_info *infop = 0; /* Pointer to the appropriate info structure */ + char buf[FSLPRINTF_BUF_SIZE]; /* Conversion buffer */ + char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ + etByte xtype = 0; /* Conversion paradigm */ + char * zExtra = 0; /* Extra memory used for etTCLESCAPE conversions */ +#if ! FSLPRINTF_OMIT_FLOATING_POINT + int exp, e2; /* exponent of real numbers */ + double rounder; /* Used for rounding floating point values */ + etByte flag_dp; /* True if decimal point should be shown */ + etByte flag_rtz; /* True if trailing zeros should be removed */ + etByte flag_exp; /* True to force display of the exponent */ + int nsd; /* Number of significant digits returned */ +#endif + + +/** + FSLPRINTF_CHARARRAY is a helper to allocate variable-sized arrays. + This exists mainly so this code can compile with the tcc compiler. +*/ +#if FSLPRINTF_HAVE_VARARRAY +# define FSLPRINTF_CHARARRAY(V,N) char V[N+1]; memset(V,0,N+1) +# define FSLPRINTF_CHARARRAY_FREE(V) +#else +# define FSLPRINTF_CHARARRAY_STACK(V) +# define FSLPRINTF_CHARARRAY(V,N) char V##2[256]; \ + char * V; \ + if((int)(N)<((int)sizeof(V##2))){ \ + V = V##2; \ + }else{ \ + V = (char *)fsl_malloc(N+1); \ + if(!V) {FSLPRINTF_RETURN;} \ + } +# define FSLPRINTF_CHARARRAY_FREE(V) if(V!=V##2) fsl_free(V) +#endif + + /* FSLPRINTF_RETURN, FSLPRINTF_CHECKERR, and FSLPRINTF_SPACES + are internal helpers. + */ +#define FSLPRINTF_RETURN if( zExtra ) fsl_free(zExtra); return pfrc +#define FSLPRINTF_CHECKERR if( 0!=pfrc ) { FSLPRINTF_RETURN; } (void)0 +#define FSLPRINTF_SPACES(N) \ + { \ + FSLPRINTF_CHARARRAY(zSpaces,N); \ + memset( zSpaces,' ',N); \ + pfrc = pfAppend(pfAppendArg, zSpaces, N); \ + FSLPRINTF_CHARARRAY_FREE(zSpaces); \ + FSLPRINTF_CHECKERR; \ + } (void)0 + + length = 0; + bufpt = 0; + for(; (c=(*fmt))!=0; ++fmt){ + assert(0==pfrc); + if( c!='%' ){ + int amt; + bufpt = (char *)fmt; + amt = 1; + while( (c=(*++fmt))!='%' && c!=0 ) amt++; + pfrc = pfAppend( pfAppendArg, bufpt, amt); + FSLPRINTF_CHECKERR; + if( c==0 ) break; + } + if( (c=(*++fmt))==0 ){ + pfrc = pfAppend( pfAppendArg, "%", 1); + FSLPRINTF_CHECKERR; + break; + } + /* Find out what flags are present */ + flag_leftjustify = flag_plussign = flag_blanksign = cThousand = + flag_alternateform = flag_altform2 = flag_zeropad = 0; + done = 0; + do{ + switch( c ){ + case '-': flag_leftjustify = 1; break; + case '+': flag_plussign = 1; break; + case ' ': flag_blanksign = 1; break; + case '#': flag_alternateform = 1; break; + case '!': flag_altform2 = 1; break; + case '0': flag_zeropad = 1; break; + case ',': cThousand = ','; break; + default: done = 1; break; + } + }while( !done && (c=(*++fmt))!=0 ); + /* Get the field width */ + width = 0; + if( c=='*' ){ + width = va_arg(ap,int); + if( width<0 ){ + flag_leftjustify = 1; + width = width >= -2147483647 ? -width : 0; + } + c = *++fmt; + }else{ + unsigned wx = 0; + while( c>='0' && c<='9' ){ + wx = wx * 10 + c - '0'; + width = width*10 + c - '0'; + c = *++fmt; + } + width = wx & 0x7fffffff; + } + if( width > FSLPRINTF_BUF_SIZE-10 ){ + width = FSLPRINTF_BUF_SIZE-10; + } + /* Get the precision */ + if( c=='.' ){ + precision = 0; + c = *++fmt; + if( c=='*' ){ + precision = va_arg(ap,int); + c = *++fmt; + if( precision<0 ){ + precision = precision >= -2147483647 ? -precision : -1; + } + }else{ + unsigned px = 0; + while( c>='0' && c<='9' ){ + px = px*10 + c - '0'; + c = *++fmt; + } + precision = px & 0x7fffffff; + } + }else{ + precision = -1; + } + /* Get the conversion type modifier */ + if( c=='l' ){ + flag_long = 1; + c = *++fmt; + if( c=='l' ){ + flag_longlong = 1; + c = *++fmt; + }else{ + flag_longlong = 0; + } + }else{ + flag_long = flag_longlong = 0; + } + /* Fetch the info entry for the field */ + infop = 0; +#define FMTNDX(N) (N - fmtinfo[0].fmttype) +#define FMTINFO(N) (fmtinfo[ FMTNDX(N) ]) + infop = ((c>=(fmtinfo[0].fmttype)) && (cfmttype,infop->type);*/ + if( infop ) xtype = infop->type; +#undef FMTINFO +#undef FMTNDX + zExtra = 0; + if( (!infop) || (!infop->type) ){ + FSLPRINTF_RETURN; + } + + /* Limit the precision to prevent overflowing buf[] during conversion */ + if( precision>FSLPRINTF_BUF_SIZE-40 && (infop->flags & FLAG_STRING)==0 ){ + precision = FSLPRINTF_BUF_SIZE-40; + } + + /* + At this point, variables are initialized as follows: + + flag_alternateform TRUE if a '#' is present. + flag_altform2 TRUE if a '!' is present. + flag_plussign TRUE if a '+' is present. + flag_leftjustify TRUE if a '-' is present or if the + field width was negative. + flag_zeropad TRUE if the width began with 0. + flag_long TRUE if the letter 'l' (ell) prefixed + the conversion character. + flag_longlong TRUE if the letter 'll' (ell ell) prefixed + the conversion character. + flag_blanksign TRUE if a ' ' is present. + width The specified field width. This is + always non-negative. Default is 0. + precision The specified precision. The default + is -1. + xtype The class of the conversion. + infop Pointer to the appropriate info struct. + */ + switch( xtype ){ + case etPOINTER: + flag_longlong = sizeof(char*)==sizeof(int64_t); + flag_long = sizeof(char*)==sizeof(long int); + /* Fall through into the next case */ + case etORDINAL: + case etRADIX: + if( infop->flags & FLAG_SIGNED ){ + int64_t v; + if( flag_longlong ) v = va_arg(ap,int64_t); + else if( flag_long ) v = va_arg(ap,long int); + else v = va_arg(ap,int); + if( v<0 ){ + longvalue = -v; + prefix = '-'; + }else{ + longvalue = v; + if( flag_plussign ) prefix = '+'; + else if( flag_blanksign ) prefix = ' '; + else prefix = 0; + } + }else{ + if( flag_longlong ) longvalue = va_arg(ap,uint64_t); + else if( flag_long ) longvalue = va_arg(ap,unsigned long int); + else longvalue = va_arg(ap,unsigned int); + prefix = 0; + } + if( longvalue==0 ) flag_alternateform = 0; + if( flag_zeropad && precision=4 || (longvalue/10)%10==1 ){ + x = 0; + } + buf[FSLPRINTF_BUF_SIZE-3] = zOrd[x*2]; + buf[FSLPRINTF_BUF_SIZE-2] = zOrd[x*2+1]; + bufpt -= 2; + } + { + int const base = infop->base; + const char *cset = &aDigits[infop->charset]; + do{ /* Convert to ascii */ + *(--bufpt) = cset[longvalue%base]; + longvalue = longvalue/base; + }while( longvalue>0 ); + } + length = &buf[FSLPRINTF_BUF_SIZE-1]-bufpt; + while( precision>length ){ + *(--bufpt) = '0'; /* Zero pad */ + ++length; + } + if( cThousand ){ + int nn = (length - 1)/3; /* Number of "," to insert */ + int ix = (length - 1)%3 + 1; + bufpt -= nn; + for(idx=0; nn>0; idx++){ + bufpt[idx] = bufpt[idx+nn]; + ix--; + if( ix==0 ){ + bufpt[++idx] = cThousand; + nn--; + ix = 3; + } + } + } + if( prefix ) *(--bufpt) = prefix; /* Add sign */ + if( flag_alternateform && infop->prefix ){ /* Add "0" or "0x" */ + const char *pre; + char x; + pre = &aPrefix[infop->prefix]; + if( *bufpt!=pre[0] ){ + for(; (x=(*pre))!=0; pre++) *(--bufpt) = x; + } + } + length = &buf[FSLPRINTF_BUF_SIZE-1]-bufpt; + break; + case etFLOAT: + case etEXP: + case etGENERIC: + realvalue = va_arg(ap,double); +#if ! FSLPRINTF_OMIT_FLOATING_POINT + if( precision<0 ) precision = 6; /* Set default precision */ + if( precision>FSLPRINTF_BUF_SIZE/2-10 ) precision = FSLPRINTF_BUF_SIZE/2-10; + if( realvalue<0.0 ){ + realvalue = -realvalue; + prefix = '-'; + }else{ + if( flag_plussign ) prefix = '+'; + else if( flag_blanksign ) prefix = ' '; + else prefix = 0; + } + if( xtype==etGENERIC && precision>0 ) precision--; +#if 0 + /* Rounding works like BSD when the constant 0.4999 is used. Wierd! */ + for(idx=precision & 0xfff, rounder=0.4999; idx>0; idx--, rounder*=0.1); +#else + /* It makes more sense to use 0.5 */ + for(idx=precision & 0xfff, rounder=0.5; idx>0; idx--, rounder*=0.1){} +#endif + if( xtype==etFLOAT ) realvalue += rounder; + /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ + exp = 0; +#if 1 + if( (realvalue)!=(realvalue) ){ + /* from sqlite3: #define sqlite3_isnan(X) ((X)!=(X)) */ + /* This weird array thing is to avoid constness violations + when assinging, e.g. "NaN" to bufpt. + */ + static char NaN[4] = {'N','a','N','\0'}; + bufpt = NaN; + length = 3; + break; + } +#endif + if( realvalue>0.0 ){ + while( realvalue>=1e32 && exp<=350 ){ realvalue *= 1e-32; exp+=32; } + while( realvalue>=1e8 && exp<=350 ){ realvalue *= 1e-8; exp+=8; } + while( realvalue>=10.0 && exp<=350 ){ realvalue *= 0.1; exp++; } + while( realvalue<1e-8 && exp>=-350 ){ realvalue *= 1e8; exp-=8; } + while( realvalue<1.0 && exp>=-350 ){ realvalue *= 10.0; exp--; } + if( exp>350 || exp<-350 ){ + if( prefix=='-' ){ + static char Inf[5] = {'-','I','n','f','\0'}; + bufpt = Inf; + }else if( prefix=='+' ){ + static char Inf[5] = {'+','I','n','f','\0'}; + bufpt = Inf; + }else{ + static char Inf[4] = {'I','n','f','\0'}; + bufpt = Inf; + } + length = (int)strlen(bufpt); + break; + } + } + bufpt = buf; + /* + If the field type is etGENERIC, then convert to either etEXP + or etFLOAT, as appropriate. + */ + flag_exp = xtype==etEXP; + if( xtype!=etFLOAT ){ + realvalue += rounder; + if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } + } + if( xtype==etGENERIC ){ + flag_rtz = !flag_alternateform; + if( exp<-4 || exp>precision ){ + xtype = etEXP; + }else{ + precision = precision - exp; + xtype = etFLOAT; + } + }else{ + flag_rtz = 0; + } + if( xtype==etEXP ){ + e2 = 0; + }else{ + e2 = exp; + } + nsd = 0; + flag_dp = (precision>0) | flag_alternateform | flag_altform2; + /* The sign in front of the number */ + if( prefix ){ + *(bufpt++) = prefix; + } + /* Digits prior to the decimal point */ + if( e2<0 ){ + *(bufpt++) = '0'; + }else{ + for(; e2>=0; e2--){ + *(bufpt++) = et_getdigit(&realvalue,&nsd); + } + } + /* The decimal point */ + if( flag_dp ){ + *(bufpt++) = '.'; + } + /* "0" digits after the decimal point but before the first + significant digit of the number */ + for(e2++; e2<0 && precision>0; precision--, e2++){ + *(bufpt++) = '0'; + } + /* Significant digits after the decimal point */ + while( (precision--)>0 ){ + *(bufpt++) = et_getdigit(&realvalue,&nsd); + } + /* Remove trailing zeros and the "." if no digits follow the "." */ + if( flag_rtz && flag_dp ){ + while( bufpt[-1]=='0' ) *(--bufpt) = 0; + /* assert( bufpt>buf ); */ + if( bufpt[-1]=='.' ){ + if( flag_altform2 ){ + *(bufpt++) = '0'; + }else{ + *(--bufpt) = 0; + } + } + } + /* Add the "eNNN" suffix */ + if( flag_exp || (xtype==etEXP && exp) ){ + *(bufpt++) = aDigits[infop->charset]; + if( exp<0 ){ + *(bufpt++) = '-'; exp = -exp; + }else{ + *(bufpt++) = '+'; + } + if( exp>=100 ){ + *(bufpt++) = (exp/100)+'0'; /* 100's digit */ + exp %= 100; + } + *(bufpt++) = exp/10+'0'; /* 10's digit */ + *(bufpt++) = exp%10+'0'; /* 1's digit */ + } + *bufpt = 0; + + /* The converted number is in buf[] and zero terminated. Output it. + Note that the number is in the usual order, not reversed as with + integer conversions. */ + length = bufpt-buf; + bufpt = buf; + + /* Special case: Add leading zeros if the flag_zeropad flag is + set and we are not left justified */ + if( flag_zeropad && !flag_leftjustify && length < width){ + int i; + int nPad = width - length; + for(i=width; i>=nPad; i--){ + bufpt[i] = bufpt[i-nPad]; + } + i = prefix!=0; + while( nPad-- ) bufpt[i++] = '0'; + length = width; + } +#endif /* !FSLPRINTF_OMIT_FLOATING_POINT */ + break; + case etPERCENT: + buf[0] = '%'; + bufpt = buf; + length = 1; + break; + case etCHARLIT: + case etCHARX: + c = buf[0] = (xtype==etCHARX ? va_arg(ap,int) : *++fmt); + if( precision>=0 ){ + for(idx=1; idx0 && flag_alternateform) + ? precision*4/*max bytes per char*/ + : (precision ? precision : -1)) + : (int)0; + if(flag_alternateform && length && (precision>0 || width>0)){ + appendf__utf8_altform(bufpt, &length, &precision, &width); + }else if( length && precision>=0 && precision0 && flag_alternateform) + ? precision*4/*max bytes per char*/ + : (precision ? precision : -1)) + : (int)0; + if(flag_alternateform && length && (precision>0 || width>0)){ + appendf__utf8_altform(bufpt, &length, &precision, &width); + }else if( length && precision>=0 && precision=0 && precision=0 && limit=0) && (length>limit)) length = limit; + check = fsl_bytes_fossilize((unsigned char const *)bufpt, length, &fb); + if(check){ + fsl_buffer_reserve(&fb,0); + FSLPRINTF_RETURN; + } + zExtra = bufpt = (char*)fb.mem + /*transfer ownership*/; + length = (int)fb.used; + if( precision>=0 && precision=0 ? precision : (int)fsl_strlen(bufpt)) + : 0; + state.z = bufpt; + state.addQuotes = flag_altform2 ? true : false; + state.escapeSmallUtf8 = flag_alternateform ? true : false; + pfrc = spech_json( pfAppend, pfAppendArg, (unsigned)length, &state ); + bufpt = NULL; + FSLPRINTF_CHECKERR; + length = 0; + break; + } +#endif +#if ! FSLPRINTF_OMIT_HTML + case etHTML:{ + bufpt = va_arg(ap,char*); + length = bufpt ? (int)fsl_strlen(bufpt) : 0; + pfrc = spech_string_to_html( pfAppend, pfAppendArg, + (precision0 ){ + FSLPRINTF_SPACES(nspace); + } + } + if( length>0 ){ + pfrc = pfAppend( pfAppendArg, bufpt, length); + FSLPRINTF_CHECKERR; + } + if( flag_leftjustify ){ + int nspace; + nspace = width-length; + if( nspace>0 ){ + FSLPRINTF_SPACES(nspace); + } + } + if( zExtra ){ + fsl_free(zExtra); + zExtra = 0; + } + }/* End for loop over the format string */ + FSLPRINTF_RETURN; +} /* End of function */ + + +#undef FSLPRINTF_CHARARRAY_STACK +#undef FSLPRINTF_CHARARRAY +#undef FSLPRINTF_CHARARRAY_FREE +#undef FSLPRINTF_SPACES +#undef FSLPRINTF_CHECKERR +#undef FSLPRINTF_RETURN +#undef FSLPRINTF_OMIT_FLOATING_POINT +#undef FSLPRINTF_OMIT_SIZE +#undef FSLPRINTF_OMIT_SQL +#undef FSLPRINTF_BUF_SIZE +#undef FSLPRINTF_OMIT_HTML + +int fsl_appendf(fsl_output_f pfAppend, void * pfAppendArg, + const char *fmt, ... ){ + int ret; + va_list vargs; + va_start( vargs, fmt ); + ret = fsl_appendfv( pfAppend, pfAppendArg, fmt, vargs ); + va_end(vargs); + return ret; +} + +int fsl_fprintfv( FILE * fp, char const * fmt, va_list args ){ + return (fp && fmt) + ? fsl_appendfv( fsl_output_f_FILE, fp, fmt, args ) + : -1; +} + +int fsl_fprintf( FILE * fp, char const * fmt, ... ){ + int ret; + va_list vargs; + va_start( vargs, fmt ); + ret = fsl_appendfv( fsl_output_f_FILE, fp, fmt, vargs ); + va_end(vargs); + return ret; +} + +char * fsl_mprintfv( char const * fmt, va_list vargs ){ + if( !fmt ) return 0; + else if(!*fmt) return fsl_strndup("",0); + else{ + fsl_buffer buf = fsl_buffer_empty; + int const rc = fsl_buffer_appendfv( &buf, fmt, vargs ); + if(rc){ + fsl_buffer_reserve(&buf, 0); + assert(0==buf.mem); + } + return (char*)buf.mem /*transfer ownership*/; + } +} + +char * fsl_mprintf( char const * fmt, ... ){ + char * ret; + va_list vargs; + va_start( vargs, fmt ); + ret = fsl_mprintfv( fmt, vargs ); + va_end( vargs ); + return ret; +} + + +/** + Internal state for fsl_snprintfv(). + */ +struct fsl_snp_state { + /** Destination memory */ + char * dest; + /** Current output position in this->dest. */ + fsl_size_t pos; + /** Length of this->dest. */ + fsl_size_t len; +}; +typedef struct fsl_snp_state fsl_snp_state; + +static int fsl_output_f_snprintf( void * arg, + void const * data_, + fsl_size_t n ){ + char const * data = (char const *)data_; + fsl_snp_state * st = (fsl_snp_state*) arg; + assert(n>=0); + if(n==0 || (st->pos >= st->len)) return 0; + else if((n + st->pos) > st->len){ + n = st->len - st->pos; + } + memcpy(st->dest + st->pos, data, n); + st->pos += n; + assert(st->pos <= st->len); + return 0; +} + +int fsl_snprintfv( char * dest, fsl_size_t n, + char const * fmt, va_list args){ + fsl_snp_state st = {NULL,0,0}; + int rc = 0; + if(!dest || !fmt) return FSL_RC_MISUSE; + else if(!n || !*fmt){ + if(dest) *dest = 0; + return 0; + } + st.len = n; + st.dest = dest; + rc = fsl_appendfv( fsl_output_f_snprintf, &st, fmt, args ); + if(st.pos < st.len){ + dest[st.pos] = 0; + } + return rc; +} + +int fsl_snprintf( char * dest, fsl_size_t n, char const * fmt, ... ){ + int rc = 0; + va_list vargs; + va_start( vargs, fmt ); + rc = fsl_snprintfv( dest, n, fmt, vargs ); + va_end( vargs ); + return rc; +} + +#undef MARKER +/* end of file appendf.c */ +/* start of file auth.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ + +/*************************************************************************** + This file contains routines related to working with user authentication. +*/ + + +FSL_EXPORT char * fsl_sha1_shared_secret( fsl_cx * f, char const * zLoginName, + char const * zPw ){ + if(!f || !zPw || !zLoginName) return 0; + else{ + fsl_sha1_cx hash = fsl_sha1_cx_empty; + unsigned char zResult[20]; + char zDigest[41]; + if(!f->cache.projectCode){ + f->cache.projectCode = fsl_config_get_text(f, FSL_CONFDB_REPO, + "project-code", 0); + /* + fossil(1) returns a copy of zPw here if !f->cache.projectCode, + with the following comment: + */ + /* On the first xfer request of a clone, the project-code is not yet + ** known. Use the cleartext password, since that is all we have. + */ + if(!f->cache.projectCode) return 0; + } + fsl_sha1_update(&hash, f->cache.projectCode, + fsl_strlen(f->cache.projectCode)); + fsl_sha1_update(&hash, "/", 1); + fsl_sha1_update(&hash, zLoginName, fsl_strlen(zLoginName)); + fsl_sha1_update(&hash, "/", 1); + fsl_sha1_update(&hash, zPw, fsl_strlen(zPw)); + fsl_sha1_final(&hash, zResult); + fsl_sha1_digest_to_base16(zResult, zDigest); + return fsl_strndup( zDigest, FSL_STRLEN_SHA1 ); + } +} + +FSL_EXPORT char * fsl_repo_login_group_name(fsl_cx * f){ + return f + ? fsl_config_get_text(f, FSL_CONFDB_REPO, + "login-group-name", 0) + : 0; +} + +FSL_EXPORT char * fsl_repo_login_cookie_name(fsl_cx * f){ + fsl_db * db; + if(!f || !(db = fsl_cx_db_repo(f))) return 0; + else{ + char const * sql = + "SELECT 'fossil-' || substr(value,1,16)" + " FROM config" + " WHERE name IN ('project-code','login-group-code')" + " ORDER BY name /*sort*/"; + return fsl_db_g_text(db, 0, sql); + } +} + +FSL_EXPORT int fsl_repo_login_search_uid(fsl_cx * f, char const * zUsername, + char const * zPasswd, + fsl_id_t * userId){ + int rc; + char * zSecret; + fsl_db * db; + if(!f || !userId + || !zUsername || !*zUsername + || !zPasswd /*??? || !*zPasswd*/){ + return FSL_RC_MISUSE; + } + else if(!(db = fsl_needs_repo(f))){ + return FSL_RC_NOT_A_REPO; + } + *userId = 0; + zSecret = fsl_sha1_shared_secret(f, zUsername, zPasswd ); + if(!zSecret) return FSL_RC_OOM; + rc = fsl_db_get_id(db, userId, + "SELECT uid FROM user" + " WHERE login=%Q" + " AND length(cap)>0 AND length(pw)>0" + " AND login NOT IN ('anonymous','nobody','developer','reader')" + " AND (pw=%Q OR (length(pw)<>40 AND pw=%Q))", + zUsername, zSecret, zPasswd); + fsl_free(zSecret); + return rc; +} + +FSL_EXPORT int fsl_repo_login_clear( fsl_cx * f, fsl_id_t userId ){ + fsl_db * db; + if(!f) return FSL_RC_MISUSE; + else if(!(db = fsl_needs_repo(f))) return FSL_RC_NOT_A_REPO; + else{ + int const rc = fsl_db_exec(db, + "UPDATE user SET cookie=NULL, ipaddr=NULL, " + " cexpire=0 WHERE " + " CASE WHEN %"FSL_ID_T_PFMT">=0 THEN uid=%"FSL_ID_T_PFMT"" + " ELSE uid>0 END" + " AND login NOT IN('anonymous','nobody'," + " 'developer','reader')", + (fsl_id_t)userId, (fsl_id_t)userId); + if(rc){ + fsl_cx_uplift_db_error(f, db); + } + return rc; + } +} + +#undef MARKER +/* end of file auth.c */ +/* start of file bag.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/*************************************************************************** + This file houses the code for the fsl_id_bag class. +*/ +#include +#include /* memset() */ + +const fsl_id_bag fsl_id_bag_empty = fsl_id_bag_empty_m; + +void fsl_id_bag_clear(fsl_id_bag * const p){ + fsl_free(p->list); + *p = fsl_id_bag_empty; +} + +/* + The hash function +*/ +#define fsl_id_bag_hash(i) (i*101) + +/* + Change the size of the hash table on a bag so that + it contains N slots + + Completely reconstruct the hash table from scratch. Deleted + entries (indicated by a -1) are removed. When finished, + p->entryCount==p->used and p->capacity==newSize. + + Returns on on success, FSL_RC_OOM on allocation error. +*/ +static int fsl_id_bag_resize(fsl_id_bag * const p, fsl_size_t newSize){ + fsl_size_t i; + fsl_id_bag old; + fsl_size_t nDel = 0; /* Number of deleted entries */ + fsl_size_t nLive = 0; /* Number of live entries */ + fsl_id_t * newList; + assert( newSize > p->entryCount ); + newList = (fsl_id_t*)fsl_malloc( sizeof(p->list[0])*newSize ); + if(!newList) return FSL_RC_OOM; + old = *p; + p->list = newList; + p->capacity = newSize; + memset(p->list, 0, sizeof(p->list[0])*newSize ); + for(i=0; i0 ){ + unsigned h = fsl_id_bag_hash(e)%newSize; + while( p->list[h] ){ + h++; + if( h==newSize ) h = 0; + } + p->list[h] = e; + nLive++; + }else if( e<0 ){ + nDel++; + } + } + assert( p->entryCount == nLive ); + assert( p->used == nLive+nDel ); + p->used = p->entryCount; + fsl_id_bag_clear(&old); + return 0; +} + +void fsl_id_bag_reset(fsl_id_bag * const p){ + p->entryCount = p->used = 0; + if(p->list){ + memset(p->list, 0, sizeof(p->list[0])*p->capacity); + } +} + + +int fsl_id_bag_insert(fsl_id_bag * const p, fsl_id_t e){ + fsl_size_t h; + int rc = 0; + assert( e>0 ); + if( p->used+1 >= p->capacity/2 ){ + fsl_size_t const n = p->capacity ? p->capacity*2 : 30; + rc = fsl_id_bag_resize(p, n ); + if(rc) return rc; + } + h = fsl_id_bag_hash(e)%p->capacity; + while( p->list[h]>0 && p->list[h]!=e ){ + h++; + if( h>=p->capacity ) h = 0; + } + if( p->list[h]<=0 ){ + if( p->list[h]==0 ) ++p->used; + p->list[h] = e; + ++p->entryCount; + rc = 0; + } + return rc; +} + +bool fsl_id_bag_contains(fsl_id_bag const * const p, fsl_id_t e){ + fsl_size_t h; + assert( e>0 ); + if( p->capacity==0 || 0==p->used ){ + return false; + } + assert(p->list); + h = fsl_id_bag_hash(e)%p->capacity; + while( p->list[h] && p->list[h]!=e ){ + h++; + if( h>=p->capacity ) h = 0 + /*loop around to the start*/ + ; + } + return p->list[h]==e; +} + +bool fsl_id_bag_remove(fsl_id_bag * const p, fsl_id_t e){ + fsl_size_t h; + bool rv = false; + assert( e>0 ); + if( !p->capacity || !p->used ) return rv; + assert(p->list); + h = fsl_id_bag_hash(e)%p->capacity; + while( p->list[h] && p->list[h]!=e ){ + ++h; + if( h>=p->capacity ) h = 0; + } + rv = p->list[h]==e; + if( p->list[h] ){ + fsl_size_t nx = h+1; + if( nx>=p->capacity ) nx = 0; + if( p->list[nx]==0 ){ + p->list[h] = 0; + --p->used; + }else{ + p->list[h] = -1; + } + --p->entryCount; + if( p->entryCount==0 ){ + memset(p->list, 0, p->capacity*sizeof(p->list[0])); + p->used = 0; + }else if( p->capacity>40 && p->entryCountcapacity/8 ){ + fsl_id_bag_resize(p, p->capacity/2) + /* ignore realloc error and keep the old size. */; + } + } + return rv; +} + +fsl_id_t fsl_id_bag_first(fsl_id_bag const * const p){ + if( p->capacity==0 || 0==p->used ){ + return 0; + }else{ + fsl_size_t i; + for(i=0; icapacity && p->list[i]<=0; ++i){} + if( icapacity ){ + return p->list[i]; + }else{ + return 0; + } + } +} + +fsl_id_t fsl_id_bag_next(fsl_id_bag const * const p, fsl_id_t e){ + fsl_size_t h; + assert( p->capacity>0 ); + assert( e>0 ); + assert(p->list); + h = fsl_id_bag_hash(e)%p->capacity; + while( p->list[h] && p->list[h]!=e ){ + ++h; + if( h>=p->capacity ) h = 0; + } + assert( p->list[h] ); + h++; + while( hcapacity && p->list[h]<=0 ){ + h++; + } + return hcapacity ? p->list[h] : 0; +} + +fsl_size_t fsl_id_bag_count(fsl_id_bag const * const p){ + return p->entryCount; +} + +void fsl_id_bag_swap(fsl_id_bag * const lhs, fsl_id_bag * const rhs){ + fsl_id_bag x = *lhs; + *lhs = *rhs; + *rhs = x; +} + +#undef fsl_id_bag_hash +/* end of file bag.c */ +/* start of file buffer.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ + +#include +#include /* strlen() */ +#include /* NULL on linux */ +#include + +#include + + +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + + +fsl_buffer * fsl_buffer_reuse( fsl_buffer * b ){ + if(b->capacity){ + assert(b->mem); + b->mem[0] = 0; + } + b->used = b->cursor = 0; + return b; +} + +void fsl_buffer_clear( fsl_buffer * buf ){ + if(buf){ + if(buf->mem) fsl_free(buf->mem); + *buf = fsl_buffer_empty; + } +} + +int fsl_buffer_reserve( fsl_buffer * buf, fsl_size_t n ){ + if( ! buf ) return FSL_RC_MISUSE; + else if( 0 == n ){ + fsl_free(buf->mem); + *buf = fsl_buffer_empty; + return 0; + }else if( buf->capacity >= n ){ + return 0; + }else{ + unsigned char * x; + assert((buf->used < n) && "Buffer in-use greater than capacity!"); + x = (unsigned char *)fsl_realloc( buf->mem, n ); + if( ! x ) return FSL_RC_OOM; + memset( x + buf->used, 0, n - buf->used ); + buf->mem = x; + buf->capacity = n; + return 0; + } +} + +int fsl_buffer_resize( fsl_buffer * buf, fsl_size_t n ){ + if( !buf ) return FSL_RC_MISUSE; + else if(n && (buf->capacity == n+1)){ + buf->used = n; + buf->mem[n] = 0; + return 0; + }else{ + unsigned char * x = (unsigned char *)fsl_realloc( buf->mem, + n+1/*NUL*/ ); + if( ! x ) return FSL_RC_OOM; + if(n > buf->capacity){ + /* zero-fill new parts */ + memset( x + buf->capacity, 0, n - buf->capacity +1/*NUL*/ ); + } + buf->capacity = n + 1 /*NUL*/; + buf->used = n; + buf->mem = x; + buf->mem[buf->used] = 0; + return 0; + } +} + +int fsl_buffer_compare(fsl_buffer const * lhs, fsl_buffer const * rhs){ + fsl_size_t const szL = lhs->used; + fsl_size_t const szR = rhs->used; + fsl_size_t const sz = (szLmem, rhs->mem, sz); + if(0 == rc){ + rc = (szL==szR) + ? 0 + : ((szLused; + fsl_size_t const szR = rhs->used; + fsl_size_t i; + unsigned char const *buf1; + unsigned char const *buf2; + unsigned char rc = 0; + if( szL!=szR || szL==0 ) return 1; + buf1 = lhs->mem; + buf2 = rhs->mem; + for( i=0; iused; + int rc = 0; + assert(b->capacity ? !!b->mem : !b->mem); + assert(b->used <= b->capacity); + if(len<0){ + len = (fsl_int_t)fsl_strlen((char const *)data); + } + sz += len + 1/*NUL*/; + rc = b->capacitycapacity >= sz); + if(len>0) memcpy(b->mem + b->used, data, (size_t)len); + b->used += len; + b->mem[b->used] = 0; + } + return rc; + } +} + +int fsl_buffer_appendfv( fsl_buffer * const b, char const * fmt, + va_list args){ + return fsl_appendfv( fsl_output_f_buffer, b, fmt, args ); +} + + +int fsl_buffer_appendf( fsl_buffer * const b, + char const * fmt, ... ){ + if(!b || !fmt) return FSL_RC_MISUSE; + else{ + int rc; + va_list args; + va_start(args,fmt); + rc = fsl_buffer_appendfv( b, fmt, args ); + va_end(args); + return rc; + } +} + +char const * fsl_buffer_cstr(fsl_buffer const *b){ + return b ? (char const *)b->mem : NULL; +} + +char const * fsl_buffer_cstr2(fsl_buffer const *b, fsl_size_t * len){ + char const * rc = NULL; + if(b){ + rc = (char const *)b->mem; + if(len) *len = b->used; + } + return rc; +} + +char * fsl_buffer_str(fsl_buffer const *b){ + return b ? (char *)b->mem : NULL; +} + + +fsl_size_t fsl_buffer_size(fsl_buffer const * b){ + return b ? b->used : 0U; +} + +fsl_size_t fsl_buffer_capacity(fsl_buffer const * b){ + return b ? b->capacity : 0; +} + +bool fsl_data_is_compressed(unsigned char const * mem, fsl_size_t len){ + if(!mem || (len<6)) return 0; +#if 0 + else return ('x'==mem[4]) + && (0234==mem[5]); + /* + This check fails for one particular artifact in the tcl core. + Notes gathered while debugging... + + https://core.tcl.tk/tcl/ + + Delta manifest #5f37dcc3 while processing file #687 + (1-based): + + FSL_RC_RANGE: "Delta: copy extends past end of input" + + To reproduce from tcl repo: + + f-acat 5f37dcc3 | f-mfparse -r + + More details: + + Filename: library/encoding/gb2312-raw.enc + Content: dba09c670f24d47b95d12d4bb9704391b81dda9a + + That artifact is a delta of bccc899015b688d5c426bc791c2fcde3a03a3eb5, + which is actually two files: + + library/encoding/euc-cn.enc + library/encoding/gb2312.enc + + When we go to apply the delta, the contents of bccc8 appear to + be badly compressed data. They have the 'x' at byte offset + 4 but not the 0234 at byte offset 5. + + Turns out it is the fsl_buffer_is_compressed() impl which fails + for that one. + */ +#else + else{ + /** + Adapted from: + + https://blog.2of1.org/2011/03/03/decompressing-zlib-images/ + + Remember that fossil-compressed data has a 4-byte big-endian + header holding the uncompressed size of the data, so we skip + those first 4 bytes. + + See also: + + https://tools.ietf.org/html/rfc6713 + + search for "magic number". + */ + int16_t const head = (((int16_t)mem[4]) << 8) | mem[5]; + /* MARKER(("isCompressed header=%04x\n", head)); */ + switch(head){ + case 0x083c: case 0x087a: case 0x08b8: case 0x08f6: + case 0x1838: case 0x1876: case 0x18b4: case 0x1872: + case 0x2834: case 0x2872: case 0x28b0: case 0x28ee: + case 0x3830: case 0x386e: case 0x38ac: case 0x38ea: + case 0x482c: case 0x486a: case 0x48a8: case 0x48e6: + case 0x5828: case 0x5866: case 0x58a4: case 0x58e2: + case 0x6824: case 0x6862: case 0x68bf: case 0x68fd: + case 0x7801: case 0x785e: case 0x789c: case 0x78da: + return true; + default: + return false; + } + } +#endif +} + +bool fsl_buffer_is_compressed(fsl_buffer const *buf){ + return fsl_data_is_compressed( buf->mem, buf->used ); +} + +fsl_int_t fsl_data_uncompressed_size(unsigned char const *mem, + fsl_size_t len){ + return fsl_data_is_compressed(mem,len) + ? ((mem[0]<<24) + (mem[1]<<16) + (mem[2]<<8) + mem[3]) + : -1; +} + +fsl_int_t fsl_buffer_uncompressed_size(fsl_buffer const * b){ + return fsl_data_uncompressed_size(b->mem, b->used); +} + +int fsl_buffer_compress(fsl_buffer const *pIn, fsl_buffer *pOut){ + unsigned int nIn = pIn->used; + unsigned int nOut = 13 + nIn + (nIn+999)/1000; + fsl_buffer temp = fsl_buffer_empty; + int rc = fsl_buffer_resize(&temp, nOut+4); + if(rc) return rc; + else{ + unsigned long int nOut2; + unsigned char *outBuf; + unsigned long int outSize; + outBuf = temp.mem; + outBuf[0] = nIn>>24 & 0xff; + outBuf[1] = nIn>>16 & 0xff; + outBuf[2] = nIn>>8 & 0xff; + outBuf[3] = nIn & 0xff; + nOut2 = (long int)nOut; + rc = compress(&outBuf[4], &nOut2, + pIn->mem, pIn->used); + if(rc){ + fsl_buffer_clear(&temp); + return FSL_RC_ERROR; + } + outSize = nOut2+4; + rc = fsl_buffer_resize(&temp, outSize); + if(rc){ + fsl_buffer_clear(&temp); + }else{ + fsl_buffer_swap_free(&temp, pOut, -1); + assert(0==temp.used); + assert(outSize==pOut->used); + } + return rc; + } +} + +int fsl_buffer_compress2(fsl_buffer const *pIn1, + fsl_buffer const *pIn2, fsl_buffer *pOut){ + unsigned int nIn = pIn1->used + pIn2->used; + unsigned int nOut = 13 + nIn + (nIn+999)/1000; + fsl_buffer temp = fsl_buffer_empty; + int rc; + rc = fsl_buffer_resize(&temp, nOut+4); + if(rc) return rc; + else{ + unsigned char *outBuf; + z_stream stream; + outBuf = temp.mem; + outBuf[0] = nIn>>24 & 0xff; + outBuf[1] = nIn>>16 & 0xff; + outBuf[2] = nIn>>8 & 0xff; + outBuf[3] = nIn & 0xff; + stream.zalloc = (alloc_func)0; + stream.zfree = (free_func)0; + stream.opaque = 0; + stream.avail_out = nOut; + stream.next_out = &outBuf[4]; + deflateInit(&stream, 9); + stream.avail_in = pIn1->used; + stream.next_in = pIn1->mem; + deflate(&stream, 0); + stream.avail_in = pIn2->used; + stream.next_in = pIn2->mem; + deflate(&stream, 0); + deflate(&stream, Z_FINISH); + rc = fsl_buffer_resize(&temp, stream.total_out + 4); + deflateEnd(&stream); + if(!rc){ + temp.used = stream.total_out + 4; + if( pOut==pIn1 ) fsl_buffer_reserve(pOut, 0); + else if( pOut==pIn2 ) fsl_buffer_reserve(pOut, 0); + assert(!pOut->mem); + *pOut = temp; + }else{ + fsl_buffer_reserve(&temp, 0); + } + return rc; + } +} + +int fsl_buffer_uncompress(fsl_buffer const *pIn, fsl_buffer *pOut){ + unsigned int nOut; + unsigned char *inBuf; + unsigned int nIn = pIn->used; + fsl_buffer temp = fsl_buffer_empty; + int rc; + unsigned long int nOut2; + if( nIn<=4 ){ + return FSL_RC_RANGE; + } + inBuf = pIn->mem; + nOut = (inBuf[0]<<24) + (inBuf[1]<<16) + (inBuf[2]<<8) + inBuf[3]; + /* MARKER(("decompress size: %u\n", nOut)); */ + rc = fsl_buffer_reserve(&temp, nOut+1); + if(rc) return rc; + nOut2 = (long int)nOut; + rc = uncompress(temp.mem, &nOut2, + &inBuf[4], nIn - 4) + /* valgrind says there's an uninitialized memory access + somewhere under uncompress(), _presumably_ for one of + these arguments, but i can't find it. fsl_buffer_reserve() + always memsets() new bytes to 0. + + Turns out it's a known problem: + + https://www.zlib.net/zlib_faq.html#faq36 + */; + if( rc!=Z_OK ){ + fsl_buffer_reserve(&temp, 0); + return FSL_RC_ERROR; + } + rc = fsl_buffer_resize(&temp, nOut2); + if(!rc){ + temp.used = (fsl_size_t)nOut2; + if( pOut==pIn ){ + fsl_buffer_reserve(pOut, 0); + } + assert(!pOut->mem); + *pOut = temp; + }else{ + fsl_buffer_reserve(&temp, 0); + } + return rc; +} + + +int fsl_buffer_fill_from( fsl_buffer * const dest, fsl_input_f src, + void * const state ) +{ + int rc; + enum { BufSize = 512 * 8 }; + char rbuf[BufSize]; + fsl_size_t total = 0; + fsl_size_t rlen = 0; + if( !dest || ! src ) return FSL_RC_MISUSE; + fsl_buffer_reuse(dest); + while(1){ + rlen = BufSize; + rc = src( state, rbuf, &rlen ); + if( rc ) break; + total += rlen; + if(totalcapacity < (total+1) ){ + rc = fsl_buffer_reserve( dest, + total + ((rlenmem + dest->used, rbuf, rlen ); + dest->used += rlen; + if( rlen < BufSize ) break; + } + if( !rc && dest->used ){ + assert( dest->used < dest->capacity ); + dest->mem[dest->used] = 0; + } + return rc; +} + +int fsl_buffer_fill_from_FILE( fsl_buffer * const dest, + FILE * const src ){ + return fsl_buffer_fill_from( dest, fsl_input_f_FILE, src ); +} + + +int fsl_buffer_fill_from_filename( fsl_buffer * const dest, + char const * filename ){ + int rc; + FILE * src; + fsl_fstat st = fsl_fstat_empty; + /* This stat() is only an optimization to reserve all needed + memory up front. + */ + rc = fsl_stat( filename, &st, 1 ); + if(!rc && st.size>0){ + rc = fsl_buffer_reserve(dest, st.size +1/*NUL terminator*/); + if(rc) return rc; + } /* Else it might not be a real file, e.g. "-", so we'll try anyway... */ + src = fsl_fopen(filename,"rb"); + if(!src) rc = fsl_errno_to_rc(errno, FSL_RC_IO); + else { + rc = fsl_buffer_fill_from( dest, fsl_input_f_FILE, src ); + fsl_fclose(src); + } + return rc; +} + +void fsl_buffer_swap( fsl_buffer * left, fsl_buffer * right ){ + fsl_buffer const tmp = *left; + *left = *right; + *right = tmp; +} + +void fsl_buffer_swap_free( fsl_buffer * left, fsl_buffer * right, int clearWhich ){ + fsl_buffer_swap(left, right); + if(0 != clearWhich) fsl_buffer_reserve((clearWhich<0) ? left : right, 0); +} + +int fsl_buffer_copy( fsl_buffer const * src, fsl_buffer * dest ){ + fsl_buffer_reuse(dest); + return src->used + ? fsl_buffer_append( dest, src->mem, src->used ) + : 0; +} + +int fsl_buffer_delta_apply2( fsl_buffer const * orig, + fsl_buffer const * pDelta, + fsl_buffer * pTarget, + fsl_error * pErr){ + int rc; + fsl_size_t n = 0; + fsl_buffer out = fsl_buffer_empty; + rc = fsl_delta_applied_size( pDelta->mem, pDelta->used, &n); + if(rc){ + if(pErr){ + fsl_error_set(pErr, rc, "fsl_delta_applied_size() failed."); + } + return rc; + } + rc = fsl_buffer_resize( &out, n ); + if(rc) return rc; + rc = fsl_delta_apply2( orig->mem, orig->used, + pDelta->mem, pDelta->used, + out.mem, pErr); + if(rc){ + fsl_buffer_clear(&out); + }else{ + fsl_buffer_clear(pTarget); + *pTarget = out; + } + return rc; +} + +int fsl_buffer_delta_apply( fsl_buffer const * orig, + fsl_buffer const * pDelta, + fsl_buffer * pTarget){ + return fsl_buffer_delta_apply2(orig, pDelta, pTarget, NULL); +} + +void fsl_buffer_defossilize( fsl_buffer * b ){ + if(b){ + fsl_bytes_defossilize( b->mem, &b->used ); + } +} + +int fsl_buffer_to_filename( fsl_buffer const * b, char const * fname ){ + FILE * f; + int rc = 0; + if(!b || !fname) return FSL_RC_MISUSE; + f = fsl_fopen(fname, "wb"); + if(!f) rc = fsl_errno_to_rc(errno, FSL_RC_IO); + else{ + if(b->used) { + size_t const frc = fwrite(b->mem, b->used, 1, f); + rc = (1==frc) ? 0 : FSL_RC_IO; + } + fsl_fclose(f); + } + return rc; +} + +int fsl_buffer_delta_create( fsl_buffer const * src, + fsl_buffer const * newVers, + fsl_buffer * delta){ + if(!src || !newVers || !delta) return FSL_RC_MISUSE; + else if((src == newVers) + || (src==delta) + || (newVers==delta)) return FSL_RC_MISUSE; + else{ + int rc = fsl_buffer_reserve( delta, newVers->used + 60 ); + if(!rc){ + delta->used = 0; + rc = fsl_delta_create( src->mem, src->used, + newVers->mem, newVers->used, + delta->mem, &delta->used ); + if(!rc){ + rc = fsl_buffer_resize( delta, delta->used ); + } + } + return rc; + } +} + + +int fsl_output_f_buffer( void * state, + void const * src, fsl_size_t n ){ + return (!state || !src) + ? FSL_RC_MISUSE + : fsl_buffer_append((fsl_buffer*)state, src, n); +} + +int fsl_finalizer_f_buffer( void * state, void * mem ){ + fsl_buffer * b = (fsl_buffer*)mem; + fsl_buffer_reserve(b, 0); + *b = fsl_buffer_empty; + return 0; +} + +int fsl_buffer_strftime(fsl_buffer * const b, char const * format, + const struct tm *timeptr){ + if(!b || !format || !*format || !timeptr) return FSL_RC_MISUSE; + else{ + enum {BufSize = 128}; + char buf[BufSize]; + fsl_size_t len = fsl_strftime(buf, BufSize, format, timeptr); + if(!len) return FSL_RC_RANGE; + return fsl_buffer_append(b, buf, (fsl_int_t)len); + } +} + +int fsl_buffer_stream_lines(fsl_output_f fTo, void * const toState, + fsl_buffer * const pFrom, fsl_size_t N){ + char *z = (char *)pFrom->mem; + fsl_size_t i = pFrom->cursor; + fsl_size_t n = pFrom->used; + fsl_size_t cnt = 0; + int rc = 0; + if( N==0 ) return 0; + while( imem[pFrom->cursor], i - pFrom->cursor); + } + if(!rc){ + pFrom->cursor = i; + } + return rc; +} + + +int fsl_buffer_copy_lines(fsl_buffer * const pTo, + fsl_buffer * const pFrom, + fsl_size_t N){ +#if 1 + return fsl_buffer_stream_lines( pTo ? fsl_output_f_buffer : NULL, pTo, + pFrom, N ); +#else + char *z = (char *)pFrom->mem; + fsl_size_t i = pFrom->cursor; + fsl_size_t n = pFrom->used; + fsl_size_t cnt = 0; + int rc = 0; + if( N==0 ) return 0; + while( imem[pFrom->cursor], i - pFrom->cursor); + } + if(!rc){ + pFrom->cursor = i; + } + return rc; +#endif +} + +int fsl_input_f_buffer( void * state, void * dest, fsl_size_t * n ){ + fsl_buffer * b = (fsl_buffer*)state; + fsl_size_t const from = b->cursor; + fsl_size_t to; + fsl_size_t c; + if(from >= b->used){ + *n = 0; + return 0; + } + to = from + *n; + if(to>b->used) to = b->used; + c = to - from; + if(c){ + memcpy(dest, b->mem+from, c); + b->cursor += c; + } + *n = c; + return 0; +} + +int fsl_buffer_compare_file( fsl_buffer const * b, char const * zFile ){ + int rc; + fsl_fstat fst = fsl_fstat_empty; + rc = fsl_stat(zFile, &fst, 1); + if(rc || (FSL_FSTAT_TYPE_FILE != fst.type)) return -1; + else if(b->used < fst.size) return -1; + else if(b->used > fst.size) return 1; + else{ +#if 1 + FILE * f; + f = fsl_fopen(zFile,"r"); + if(!f) rc = -1; + else{ + fsl_buffer fc = *b /* so fsl_input_f_buffer() can manipulate its + cursor */; + rc = fsl_stream_compare(fsl_input_f_buffer, &fc, + fsl_input_f_FILE, f); + assert(fc.mem==b->mem); + fsl_fclose(f); + } + +#else + fsl_buffer fc = fsl_buffer_empty; + rc = fsl_buffer_fill_from_filename(&fc, zFile); + if(rc){ + rc = -1; + }else{ + rc = fsl_buffer_compare(b, &fc); + } + fsl_buffer_clear(&fc); +#endif + return rc; + } +} + +char * fsl_buffer_take(fsl_buffer * const b){ + char * z = (char *)b->mem; + *b = fsl_buffer_empty; + return z; +} + +fsl_size_t fsl_buffer_seek(fsl_buffer * const b, fsl_int_t offset, + fsl_buffer_seek_e whence){ + int64_t c = (int64_t)b->cursor; + switch(whence){ + case FSL_BUFFER_SEEK_SET: c = offset; + case FSL_BUFFER_SEEK_CUR: c = (int64_t)b->cursor + offset; break; + case FSL_BUFFER_SEEK_END: + c = (int64_t)b->used + offset; + /* ^^^^^ fossil(1) uses (used + offset - 1) but + + That seems somewhat arguable because (used + 0 - 1) is at the + last-written byte (or 1 before the begining), not the + one-past-the-end point (which corresponds to the + "end-of-file" described by the fseek() man page). It then + goes on, in other algos, to operate on that final byte using + that position, e.g. blob_read() after a seek-to-end would + read that last byte, rather than treating the buffer as being + at the end. + + So... i'm going to naively remove that -1 bit. + */ + break; + } + if(!b->used || c<0) b->cursor = 0; + else if((fsl_size_t)c > b->used) b->cursor = b->used; + else b->cursor = (fsl_size_t)c; + return b->cursor; +} + +fsl_size_t fsl_buffer_tell(fsl_buffer const * const b){ + return b->cursor; +} + +void fsl_buffer_rewind(fsl_buffer * const b){ + b->cursor = 0; +} + +int fsl_id_bag_to_buffer(fsl_id_bag const * bag, fsl_buffer * b, + char const * separator){ + int i = 0; + fsl_int_t const sepLen = (fsl_id_t)fsl_strlen(separator); + int rc = fsl_buffer_reserve(b, b->used + (bag->entryCount * 7) + + (bag->entryCount * sepLen)); + for(fsl_id_t e = fsl_id_bag_first(bag); + !rc && e; e = fsl_id_bag_next(bag, e)){ + if(i++) rc = fsl_buffer_append(b, separator, sepLen); + if(!rc) rc = fsl_buffer_appendf(b, "%" FSL_ID_T_PFMT, e); + } + return rc; +} + +int fsl_buffer_append_tcl_literal(fsl_buffer * const b, + char const * z, fsl_int_t n){ + int rc; + if(n<0) n = fsl_strlen(z); + rc = fsl_buffer_append(b, "\"", 1); + for(fsl_int_t i=0; 0==rc && i + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +bool fsl_acache_expire_oldest(fsl_acache * c){ + static uint16_t const sentinel = 0xFFFF; + uint16_t i; + fsl_int_t mnAge = c->nextAge; + uint16_t mn = sentinel; + for(i=0; iused; i++){ + if( c->list[i].agelist[i].age; + mn = i; + } + } + if( mninCache, c->list[mn].rid); + c->szTotal -= (uint32_t)c->list[mn].content.capacity; + fsl_buffer_clear(&c->list[mn].content); + --c->used; + c->list[mn] = c->list[c->used]; + } + return sentinel!=mn; +} + +int fsl_acache_insert(fsl_acache * c, fsl_id_t rid, fsl_buffer *pBlob){ + fsl_acache_line *p; + if( c->used>c->usedLimit || c->szTotal>c->szLimit ){ + fsl_size_t szBefore; + do{ + szBefore = c->szTotal; + fsl_acache_expire_oldest(c); + }while( c->szTotal>c->szLimit && c->szTotalusedLimit || !c->szLimit) + || (c->used+1 >= c->usedLimit)){ + fsl_buffer_clear(pBlob); + return 0; + } + if( c->used>=c->capacity ){ + uint16_t const cap = c->capacity ? (c->capacity*2) : 10; + void * remem = c->list + ? fsl_realloc(c->list, cap*sizeof(c->list[0])) + : fsl_malloc( cap*sizeof(c->list[0]) ); + assert((c->capacity && capcapacity) ? !"Numeric overflow" : 1); + if(c->capacity && capcapacity){ + fsl_fatal(FSL_RC_RANGE,"Numeric overflow. Bump " + "fsl_acache::capacity to a larger int type."); + } + if(!remem){ + fsl_buffer_clear(pBlob) /* for consistency */; + return FSL_RC_OOM; + } + c->capacity = cap; + c->list = (fsl_acache_line*)remem; + } + p = &c->list[c->used++]; + p->rid = rid; + p->age = c->nextAge++; + c->szTotal += pBlob->capacity; + p->content = *pBlob /* Transfer ownership */; + *pBlob = fsl_buffer_empty; + return fsl_id_bag_insert(&c->inCache, rid); +} + + +void fsl_acache_clear(fsl_acache * c){ +#if 0 + while(fsl_acache_expire_oldest(c)){} +#else + fsl_size_t i; + for(i=0; iused; i++){ + fsl_buffer_clear(&c->list[i].content); + } +#endif + fsl_free(c->list); + fsl_id_bag_clear(&c->missing); + fsl_id_bag_clear(&c->available); + fsl_id_bag_clear(&c->inCache); + *c = fsl_acache_empty; +} + + +int fsl_acache_check_available(fsl_cx * f, fsl_id_t rid){ + fsl_id_t srcid; + int depth = 0; /* Limit to recursion depth */ + static const int limit = 10000000 /* historical value */; + int rc; + fsl_acache * c = &f->cache.arty; + assert(f); + assert(c); + assert(rid>0); + assert(fsl_cx_db_repo(f)); + while( depth++ < limit ){ + fsl_int_t cSize = -1; + if( fsl_id_bag_contains(&c->missing, rid) ){ + return FSL_RC_NOT_FOUND; + } + else if( fsl_id_bag_contains(&c->available, rid) ){ + return 0; + } + else if( (cSize=fsl_content_size(f, rid)) <0){ + rc = fsl_id_bag_insert(&c->missing, rid); + return rc ? rc : FSL_RC_NOT_FOUND; + } + srcid = 0; + rc = fsl_delta_src_id(f, rid, &srcid); + if(rc) return rc; + else if( srcid==0 ){ + rc = fsl_id_bag_insert(&c->available, rid); + return rc ? rc : 0; + } + rid = srcid; + } + assert(!"delta-loop in repository"); + return fsl_cx_err_set(f, FSL_RC_CONSISTENCY, + "Serious problem: delta-loop in repository"); +} + +#undef MARKER +/* end of file cache.c */ +/* start of file checkin.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ + +/***************************************************************************** + This file houses the code for checkin-level APIS. +*/ +#include + + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + + +/** + Expects f to have an opened checkout. Assumes zRelName is a + checkout-relative simple path. It loads the file's contents and + stores them into the blob table. If rid is not NULL, *rid is + assigned the blob.rid (possibly new, possilbly re-used!). If uuid + is not NULL then *uuid is assigned to the content's UUID. The *uuid + bytes are owned by the caller, who must eventually fsl_free() + them. If content with the same UUID already exists, it does not get + re-imported but rid/uuid will (if not NULL) contain the old values. + + If parentRid is >0 then it must refer to the previous version of + zRelName's content. The parent version gets deltified vs the new + one. Note that deltification is a suggestion which the library will + ignore if (e.g.) the parent content is already a delta of something + else. + + The wise caller will have a transaction in place when calling this. + + Returns 0 on success. On error rid and uuid are not modified. +*/ +static int fsl_checkin_import_file( fsl_cx * f, char const * zRelName, + fsl_id_t parentRid, + bool allowMergeConflict, + fsl_id_t *rid, fsl_uuid_str * uuid){ + fsl_buffer * nbuf = fsl_cx_scratchpad(f); + fsl_size_t const oldSize = nbuf->used; + fsl_buffer * fbuf = &f->fileContent; + char const * fn; + int rc; + fsl_id_t fnid = 0; + fsl_id_t rcRid = 0; + assert(!fbuf->used && "Misuse of f->fileContent"); + assert(f->ckout.dir); + rc = fsl_repo_filename_fnid2(f, zRelName, &fnid, 1); + if(rc) goto end; + assert(fnid>0); + + rc = fsl_buffer_appendf(nbuf, "%s%s", f->ckout.dir, zRelName); + nbuf->used = oldSize; + if(rc) goto end; + fn = fsl_buffer_cstr(nbuf) + oldSize; + rc = fsl_buffer_fill_from_filename( fbuf, fn ); + if(rc){ + fsl_cx_err_set(f, rc, "Error %s importing file: %s", + fsl_rc_cstr(rc), fn); + goto end; + }else if(!allowMergeConflict && + fsl_buffer_contains_merge_marker(fbuf)){ + rc = fsl_cx_err_set(f, FSL_RC_CONFLICT, + "File contains a merge conflict marker: %s", + zRelName); + goto end; + } + + rc = fsl_content_put( f, fbuf, &rcRid ); + if(!rc){ + assert(rcRid > 0); + if(parentRid>0){ + rc = fsl_content_deltify(f, parentRid, rcRid, 0); + } + if(!rc){ + if(rid) *rid = rcRid; + if(uuid){ + *uuid = fsl_rid_to_uuid(f, rcRid); + if(!*uuid) rc = (f->error.code ? f->error.code : FSL_RC_OOM); + } + } + } + end: + fsl_cx_scratchpad_yield(f, nbuf); + fsl_cx_content_buffer_yield(f); + assert(0==fbuf->used); + return rc; +} + +int fsl_filename_to_vfile_ids( fsl_cx * f, fsl_id_t vid, + fsl_id_bag * dest, char const * zName, + bool changedOnly){ + fsl_stmt st = fsl_stmt_empty; + fsl_db * const db = fsl_needs_ckout(f); + int rc; + fsl_buffer * sql = 0; + if(!db) return FSL_RC_NOT_A_CKOUT; + sql = fsl_cx_scratchpad(f); + if(0>=vid) vid = f->ckout.rid; + if(zName && *zName + && !('.'==*zName && !zName[1])){ + rc = fsl_buffer_appendf(sql, + "SELECT id FROM vfile WHERE vid=%" + FSL_ID_T_PFMT + " AND fsl_match_vfile_or_dir(pathname,%Q)", + vid, zName); + }else{ + rc = fsl_buffer_appendf(sql, + "SELECT id FROM vfile WHERE vid=%" FSL_ID_T_PFMT, + vid); + } + if(rc) goto end; + else if(changedOnly){ + rc = fsl_buffer_append(sql, " AND (chnged OR deleted OR rid=0 " + "OR (origname IS NOT NULL AND " + " origname<>pathname))", -1); + if(rc) goto end; + } + rc = fsl_buffer_appendf(sql, " /* %s() */", __func__); + if(rc) goto end; + rc = fsl_db_prepare(db, &st, "%b", sql); + while(!rc && (FSL_RC_STEP_ROW == (rc=fsl_stmt_step(&st)))){ + rc = fsl_id_bag_insert( dest, fsl_stmt_g_id(&st, 0) ); + } + if(FSL_RC_STEP_DONE==rc) rc = 0; + end: + fsl_cx_scratchpad_yield(f, sql); + fsl_stmt_finalize(&st); + if(rc && !f->error.code && db->error.code){ + fsl_cx_uplift_db_error(f, db); + } + return rc; +} + +int fsl_filename_to_vfile_id( fsl_cx * f, fsl_id_t vid, char const * zName, fsl_id_t * vfid ){ + fsl_db * db = fsl_needs_ckout(f); + int rc; + fsl_stmt st = fsl_stmt_empty; + assert(db); + if(!db) return FSL_RC_NOT_A_CKOUT; + else if(!zName || !fsl_is_simple_pathname(zName, true)){ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "Filename is not a \"simple\" path: %s", + zName); + } + if(0>=vid) vid = f->ckout.rid; + rc = fsl_db_prepare(db, &st, + "SELECT id FROM vfile WHERE vid=%" FSL_ID_T_PFMT + " AND pathname=%Q %s /*%s()*/", + vid, zName, + fsl_cx_filename_collation(f), + __func__); + if(!rc){ + rc = fsl_stmt_step(&st); + switch(rc){ + case FSL_RC_STEP_ROW: + rc = 0; + *vfid = fsl_stmt_g_id(&st, 0); + break; + case FSL_RC_STEP_DONE: + rc = 0; + /* fall through */ + default: + *vfid = 0; + } + fsl_stmt_finalize(&st); + } + if(rc){ + rc = fsl_cx_uplift_db_error2(f, db, rc); + } + return rc; +} + +/** + Internal helper for fsl_checkin_enqueue() and + fsl_checkin_dequeue(). Prepares, if needed, st with a query to + fetch a vfile entry where vfile.id=vfid, then passes that name on + to opt->callback(). Returns 0 on success. +*/ +static int fsl_xqueue_callback(fsl_cx * f, fsl_db * db, fsl_stmt * st, + fsl_id_t vfid, + fsl_checkin_queue_opt const * opt){ + + int rc; + assert(opt->callback); + if(!st->stmt){ + rc = fsl_db_prepare(db, st, + "SELECT pathname FROM vfile " + "WHERE id=?1"); + if(rc) return fsl_cx_uplift_db_error2(f, db, rc); + } + fsl_stmt_bind_id(st, 1, vfid); + rc = fsl_stmt_step(st); + switch(rc){ + case FSL_RC_STEP_ROW:{ + char const * zName = fsl_stmt_g_text(st, 0, NULL); + rc = opt->callback(zName, opt->callbackState); + break; + } + case FSL_RC_STEP_DONE: + rc = fsl_cx_err_set(f, rc, "Very unexpectedly did not find " + "vfile.id which we just found."); + break; + default: + rc = fsl_cx_uplift_db_error2(f, db, rc); + break; + } + fsl_stmt_reset(st); + return rc; +} + +int fsl_checkin_enqueue(fsl_cx * f, fsl_checkin_queue_opt const * opt){ + fsl_db * const db = fsl_needs_ckout(f); + if(!db) return FSL_RC_NOT_A_CKOUT; + fsl_buffer * const canon = opt->vfileIds ? 0 : fsl_cx_scratchpad(f); + fsl_stmt qName = fsl_stmt_empty; + fsl_id_bag _vfileIds = fsl_id_bag_empty; + fsl_id_bag const * const vfileIds = + opt->vfileIds ? opt->vfileIds : &_vfileIds; + int rc = fsl_db_transaction_begin(db); + if(rc) return fsl_cx_uplift_db_error2(f, db, rc); + if(opt->vfileIds){ + if(!fsl_id_bag_count(opt->vfileIds)){ + rc = fsl_cx_err_set(f, FSL_RC_MISUSE, + "fsl_checkin_queue_opt::vfileIds " + "may not be empty."); + goto end; + } + }else{ + rc = fsl_ckout_filename_check(f, opt->relativeToCwd, + opt->filename, canon); + if(rc) goto end; + fsl_buffer_strip_slashes(canon); + } + if(opt->scanForChanges){ + rc = fsl_vfile_changes_scan(f, -1, 0); + if(rc) goto end; + } + if(opt->vfileIds){ + assert(vfileIds == opt->vfileIds); + }else{ + assert(vfileIds == &_vfileIds); + rc = fsl_filename_to_vfile_ids(f, 0, &_vfileIds, + fsl_buffer_cstr(canon), + opt->onlyModifiedFiles); + } + if(rc) goto end; + /* Walk through each found ID and queue up any which are not already + enqueued. */ + for(fsl_id_t vfid = fsl_id_bag_first(vfileIds); + !rc && vfid; vfid = fsl_id_bag_next(vfileIds, vfid)){ + fsl_size_t const entryCount = f->ckin.selectedIds.entryCount; + rc = fsl_id_bag_insert(&f->ckin.selectedIds, vfid); + if(!rc + && entryCount < f->ckin.selectedIds.entryCount + /* Was enqueued */ + && opt->callback){ + rc = fsl_xqueue_callback(f, db, &qName, vfid, opt); + } + } + end: + if(opt->vfileIds){ + assert(!canon); + assert(!_vfileIds.list); + }else{ + assert(canon); + fsl_cx_scratchpad_yield(f, canon); + fsl_id_bag_clear(&_vfileIds); + } + fsl_stmt_finalize(&qName); + if(rc) fsl_db_transaction_rollback(db); + else{ + rc = fsl_cx_uplift_db_error2(f, db, fsl_db_transaction_commit(db)); + } + return rc; +} + +int fsl_checkin_dequeue(fsl_cx * f, fsl_checkin_queue_opt const * opt){ + fsl_db * const db = fsl_needs_ckout(f); + if(!db) return FSL_RC_NOT_A_CKOUT; + int rc = fsl_db_transaction_begin(db); + if(rc) return fsl_cx_uplift_db_error2(f, db, rc); + fsl_id_bag list = fsl_id_bag_empty; + fsl_buffer * canon = 0; + char const * fn; + fsl_stmt qName = fsl_stmt_empty; + if(opt->filename && *opt->filename){ + canon = fsl_cx_scratchpad(f); + rc = fsl_ckout_filename_check(f, opt->relativeToCwd, + opt->filename, canon); + if(rc) goto end; + else fsl_buffer_strip_slashes(canon); + } + fn = canon ? fsl_buffer_cstr(canon) : opt->filename; + rc = fsl_filename_to_vfile_ids(f, 0, &list, fn, false); + if(!rc && list.entryCount){ + /* Walk through each found ID and dequeue up any which are + enqueued. */ + for( fsl_id_t nid = fsl_id_bag_first(&list); + !rc && nid; + nid = fsl_id_bag_next(&list, nid)){ + if(fsl_id_bag_remove(&f->ckin.selectedIds, nid) + && opt->callback){ + rc = fsl_xqueue_callback(f, db, &qName, nid, opt); + } + } + } + end: + if(canon) fsl_cx_scratchpad_yield(f, canon); + fsl_stmt_finalize(&qName); + fsl_id_bag_clear(&list); + if(rc) fsl_db_transaction_rollback(db); + else{ + rc = fsl_cx_uplift_db_error2(f, db, fsl_db_transaction_commit(db)); + } + return rc; +} + +bool fsl_checkin_is_enqueued(fsl_cx * f, char const * zName, + bool relativeToCwd){ + fsl_db * db; + if(!f || !zName || !*zName) return 0; + else if(!(db = fsl_needs_ckout(f))) return 0; + else if(!f->ckin.selectedIds.entryCount){ + /* Behave like fsl_is_enqueued() SQL function. */ + return true; + } + else { + bool rv = false; + fsl_buffer * const canon = fsl_cx_scratchpad(f); + int rc = fsl_ckout_filename_check(f, relativeToCwd, zName, canon); + if(!rc){ + fsl_id_t vfid = 0; + rc = fsl_filename_to_vfile_id(f, 0, fsl_buffer_cstr(canon), + &vfid); + rv = (rc && (vfid>0)) + ? false + : ((vfid>0) + ? fsl_id_bag_contains(&f->ckin.selectedIds, vfid) + /* ^^^^ asserts that arg2!=0*/ + : false); + } + fsl_cx_scratchpad_yield(f, canon); + return rv; + } +} + + +void fsl_checkin_discard(fsl_cx * f){ + if(f){ + fsl_id_bag_clear(&f->ckin.selectedIds); + fsl_deck_finalize(&f->ckin.mf); + } +} + +/** + Adds the given rid to the "unsent" db list, Returns 0 on success, + updates f's error state on error. +*/ +static int fsl_checkin_add_unsent(fsl_cx * f, fsl_id_t rid){ + fsl_db * const r = fsl_cx_db_repo(f); + int rc; + assert(r); + rc = fsl_db_exec(r,"INSERT OR IGNORE INTO unsent " + "VALUES(%" FSL_ID_T_PFMT ")", rid); + if(rc){ + fsl_cx_uplift_db_error(f, r); + } + return rc; +} + +/** + Calculates the F-cards for deck d based on the commit file + selection list and the contents of the vfile table (where vid==the + vid parameter). vid is the version to check against, and this code + assumes that the vfile table has been populated with that version + and its state represents a recent scan (with no filesystem-level + changes made since the scan). + + If pBaseline is not NULL then d is calculated as being a delta + from pBaseline, but d->B is not modified by this routine. + + On success, d->F.list will contain "all the F-cards it needs." + + If changeCount is not NULL, then on success it is set to the number + of F-cards added to d due to changes queued via the checkin process + (as opposed to those added solely for delta inheritance reasons). +*/ +static +int fsl_checkin_calc_F_cards2( fsl_cx * f, fsl_deck * d, + fsl_deck * pBaseline, fsl_id_t vid, + fsl_size_t * changeCount, + fsl_checkin_opt const * ciOpt){ + int rc = 0; + fsl_db * dbR = fsl_needs_repo(f); + fsl_db * dbC = fsl_needs_ckout(f); + fsl_stmt stUpdateFileRid = fsl_stmt_empty; + fsl_stmt stmt = fsl_stmt_empty; + fsl_stmt * q = &stmt; + char * fUuid = NULL; + fsl_card_F const * pFile = NULL; + fsl_size_t changeCounter = 0; + if(!f) return FSL_RC_MISUSE; + else if(!dbR) return FSL_RC_NOT_A_REPO; + else if(!dbC) return FSL_RC_NOT_A_CKOUT; + assert( (!pBaseline || !pBaseline->B.uuid) && "Baselines must not have a baseline." ); + assert( d->B.baseline ? (!pBaseline || pBaseline==d->B.baseline) : 1 ); + assert(vid>=0); +#define RC if(rc) goto end + + if(pBaseline){ + assert(!d->B.baseline); + assert(0!=vid); + rc = fsl_deck_F_rewind(pBaseline); + RC; + fsl_deck_F_next( pBaseline, &pFile ); + } + + rc = fsl_db_prepare(dbC, &stUpdateFileRid, + "UPDATE vfile SET mrid=?1, rid=?1, " + "mhash=NULL WHERE id=?2"); + RC; + + rc = fsl_db_prepare( dbC, q, + "SELECT " + /*0*/"fsl_is_enqueued(vf.id) as isSel, " + /*1*/"vf.id," + /*2*/"vf.vid," + /*3*/"vf.chnged," + /*4*/"vf.deleted," + /*5*/"vf.isexe," + /*6*/"vf.islink," + /*7*/"vf.rid," + /*8*/"mrid," + /*9*/"pathname," + /*10*/"origname, " + /*11*/"b.rid, " + /*12*/"b.uuid " + "FROM vfile vf LEFT JOIN blob b ON vf.mrid=b.rid " + "WHERE" + " vf.vid=%"FSL_ID_T_PFMT" AND" +#if 0 + /* Historical (fossil(1)). This introduces an interesting + corner case which i would like to avoid here because + it causes a "no files changed" error in the checkin + op. The behaviour is actually correct (and the deletion + is picked up) but fsl_checkin_commit() has no mechanism + for catching this particular case. So we'll try a + slightly different approach... + */ + " (NOT deleted OR NOT isSel)" +#else + " ((NOT deleted OR NOT isSel)" + " OR (deleted AND isSel))" /* workaround to allow + us to count deletions via + changeCounter. */ +#endif + " ORDER BY fsl_if_enqueued(vf.id, pathname, origname)", + (fsl_id_t)vid); + RC; + /* MARKER(("SQL:\n%s\n", (char const *)q->sql.mem)); */ + while( FSL_RC_STEP_ROW==fsl_stmt_step(q) ){ + int const isSel = fsl_stmt_g_int32(q,0); + fsl_id_t const id = fsl_stmt_g_id(q,1); +#if 0 + fsl_id_t const vid = fsl_stmt_g_id(q,2); +#endif + int const changed = fsl_stmt_g_int32(q,3); + int const deleted = fsl_stmt_g_int32(q,4); + int const isExe = fsl_stmt_g_int32(q,5); + int const isLink = fsl_stmt_g_int32(q,6); + fsl_id_t const rid = fsl_stmt_g_id(q,7); + fsl_id_t const mergeRid = fsl_stmt_g_id(q,8); + char const * zName = fsl_stmt_g_text(q, 9, NULL); + char const * zOrig = fsl_stmt_g_text(q, 10, NULL); + fsl_id_t const frid = fsl_stmt_g_id(q,11); + char const * zUuid = fsl_stmt_g_text(q, 12, NULL); + fsl_fileperm_e perm = FSL_FILE_PERM_REGULAR; + int cmp; + fsl_id_t fileBlobRid = rid; + int const renamed = (zOrig && *zOrig) ? fsl_strcmp(zName,zOrig) : 0 + /* For some as-yet-unknown reason, some fossil(1) code + sets (origname=pathname WHERE origname=NULL). e.g. + the 'mv' command does that. + */; + if(zOrig && !renamed) zOrig = NULL; + fUuid = NULL; + if(!isSel && !zUuid){ + assert(!rid); + assert(!mergeRid); + /* An unselected ADDed file. Skip it. */ + continue; + } + + if(isExe) perm = FSL_FILE_PERM_EXE; + else if(isLink){ + fsl_fatal(FSL_RC_NYI, "This code does not yet deal " + "with symlinks. file: %s", zName) + /* does not return */; + perm = FSL_FILE_PERM_LINK; + } + /* + TODO: symlinks + */ + + if(!f->cache.markPrivate){ + rc = fsl_content_make_public(f, frid); + if(rc) break; + } + +#if 0 + if(mergeRid && (mergeRid != rid)){ + fsl_fatal(FSL_RC_NYI, "This code does not yet deal " + "with merges. file: %s", zName) + /* does not return */; + } +#endif + while(pFile && fsl_strcmp(pFile->name, zName)<0){ + /* Baseline has files with lexically smaller names. + Interesting corner case: + + f-rm th1ish/makefile.gnu + f-checkin ... th1ish/makefile.gnu + + makefile.gnu does not get picked up by the historical query + but gets picked up here. We really need to ++changeCounter in + that case, but we don't know we're in that case because we're + now traversing a filename which is not in the result set. + The end result (because we don't increment changeCounter) is + that fsl_checkin_commit() thinks we have no made any changes + and errors out. If we ++changeCounter for all deletions we + have a different corner case, where a no-change commit is not + seen as such because we've counted deletions from (other) + versions between the baseline and the checkout. + */ + rc = fsl_deck_F_add(d, pFile->name, NULL, pFile->perm, NULL); + if(rc) break; + fsl_deck_F_next(pBaseline, &pFile); + } + if(rc) goto end; + else if(isSel && (changed || deleted || renamed)){ + /* MARKER(("isSel && (changed||deleted||renamed): %s\n", zName)); */ + ++changeCounter; + if(deleted){ + zOrig = NULL; + }else if(changed){ + rc = fsl_checkin_import_file(f, zName, rid, + ciOpt->allowMergeConflict, + &fileBlobRid, &fUuid); + if(!rc) rc = fsl_checkin_add_unsent(f, fileBlobRid); + RC; + /* MARKER(("New content: %d / %s / %s\n", (int)fileBlobRid, fUuid, zName)); */ + if(0 != fsl_uuidcmp(zUuid, fUuid)){ + zUuid = fUuid; + } + fsl_stmt_reset(&stUpdateFileRid); + fsl_stmt_bind_id(&stUpdateFileRid, 1, fileBlobRid); + fsl_stmt_bind_id(&stUpdateFileRid, 2, id); + if(FSL_RC_STEP_DONE!=fsl_stmt_step(&stUpdateFileRid)){ + rc = fsl_cx_uplift_db_error(f, stUpdateFileRid.db); + assert(rc); + goto end; + } + }else{ + assert(renamed); + assert(zOrig); + } + } + assert(!rc); + cmp = 1; + if(!pFile + || (cmp = fsl_strcmp(pFile->name,zName))!=0 + /* ^^^^ the cmp assignment must come right after (!pFile)! */ + || deleted + || (perm != pFile->perm)/* permissions change */ + || fsl_strcmp(pFile->uuid, zUuid)!=0 + /* ^^^^^ file changed somewhere between baseline and delta */ + ){ + if(isSel && deleted){ + if(pBaseline /* d is-a delta */){ + /* Deltas mark deletions with F-cards having only + a file name (no UUID or permission). + */ + rc = fsl_deck_F_add(d, zName, NULL, perm, NULL); + }/*else elide F-card to mark a deletion in a baseline.*/ + }else{ + if(zOrig && !isSel){ + /* File is renamed in vfile but is not being committed, so + make sure we use the original name for the F-card. + */ + zName = zOrig; + zOrig = NULL; + } + assert(zUuid); + assert(fileBlobRid); + if( !zOrig || !renamed ){ + rc = fsl_deck_F_add(d, zName, zUuid, perm, NULL); + }else{ + /* Rename this file */ + rc = fsl_deck_F_add(d, zName, zUuid, perm, zOrig); + } + } + } + fsl_free(fUuid); + fUuid = NULL; + RC; + if( 0 == cmp ){ + fsl_deck_F_next(pBaseline, &pFile); + } + }/*while step()*/ + + while( !rc && pFile ){ + /* Baseline has remaining files with lexically larger names. Let's import them. */ + rc = fsl_deck_F_add(d, pFile->name, NULL, pFile->perm, NULL); + if(!rc) fsl_deck_F_next(pBaseline, &pFile); + } + + end: +#undef RC + fsl_free(fUuid); + fsl_stmt_finalize(q); + fsl_stmt_finalize(&stUpdateFileRid); + if(!rc && changeCount) *changeCount = changeCounter; + return rc; +} + +/** + Cancels all symbolic tags (branches) on the given version by + adding one T-card to d for each active branch tag set on vid. + When creating a branch, d would represent the branch and vid + would be the version being branched from. + + Returns 0 on success. +*/ +static int fsl_cancel_sym_tags( fsl_deck * d, fsl_id_t vid ){ + int rc; + fsl_stmt q = fsl_stmt_empty; + fsl_db * db = fsl_needs_repo(d->f); + assert(db); + rc = fsl_db_prepare(db, &q, + "SELECT tagname FROM tagxref, tag" + " WHERE tagxref.rid=%"FSL_ID_T_PFMT + " AND tagxref.tagid=tag.tagid" + " AND tagtype>0 AND tagname GLOB 'sym-*'" + " ORDER BY tagname", + (fsl_id_t)vid); + while( !rc && (FSL_RC_STEP_ROW==fsl_stmt_step(&q)) ){ + const char *zTag = fsl_stmt_g_text(&q, 0, NULL); + rc = fsl_deck_T_add(d, FSL_TAGTYPE_CANCEL, + NULL, zTag, "Cancelled by branch."); + } + fsl_stmt_finalize(&q); + return rc; +} + +#if 0 +static int fsl_leaf_set( fsl_cx * f, fsl_id_t rid, char isLeaf ){ + int rc; + fsl_stmt * st = NULL; + fsl_db * db = fsl_needs_repo(f); + assert(db); + rc = fsl_db_prepare_cached(db, &st, isLeaf + ? "INSERT OR IGNORE INTO leaf(rid) VALUES(?)" + : "DELETE FROM leaf WHERE rid=?"); + if(!rc){ + fsl_stmt_bind_id(st, 1, rid); + fsl_stmt_step(st); + fsl_stmt_cached_yield(st); + } + if(rc){ + fsl_cx_uplift_db_error(f, db); + } + return rc; +} +#endif + +/** + Checks vfile for any files (where chnged in (2,3,4,5)), i.e. + having something to do with a merge. If either all of those + changes are enqueued for checkin, or none of them are, then + this function returns 0, otherwise it sets f's error + state and returns non-0. +*/ +static int fsl_check_for_partial_merge(fsl_cx * f){ + if(!f->ckin.selectedIds.entryCount){ + /* All files are considered enqueued. */ + return 0; + }else{ + fsl_db * db = fsl_cx_db_ckout(f); + int32_t counter = 0; + int rc = + fsl_db_get_int32(db, &counter, + "SELECT COUNT(*) FROM (" +#if 1 + "SELECT DISTINCT fsl_is_enqueued(id)" + " FROM vfile WHERE chnged IN (2,3,4,5)" +#else + "SELECT fsl_is_enqueued(id) isSel " + "FROM vfile WHERE chnged IN (2,3,4,5) " + "GROUP BY isSel" +#endif + ")" + ); + /** + Result is 0 if no merged files are in vfile, 1 row if isSel is + the same for all merge-modified files, and 2 if there is a mix + of selected/unselected merge-modified files. + */ + if(!rc && (counter>1)){ + assert(2==counter); + rc = fsl_cx_err_set(f, FSL_RC_MISUSE, + "Only Chuck Norris can commit " + "a partial merge. Commit either all " + "or none of it."); + } + return rc; + } +} + +/** + Populates d with the contents for a FSL_SATYPE_CHECKIN manifest + based on repository version basedOnVid. + + d is the deck to populate. + + basedOnVid must currently be f->ckout.rid OR the vfile table must + be current for basedOnVid (see fsl_vfile_changes_scan() and + fsl_vfile_load()). It "should" work with basedOnVid==0 but + that's untested so far. + + opt is the options object passed to fsl_checkin_commit(). +*/ +static int fsl_checkin_calc_manifest( fsl_cx * f, fsl_deck * d, + fsl_id_t basedOnVid, + fsl_checkin_opt const * opt ){ + int rc; + fsl_db * dbR = fsl_cx_db_repo(f); + fsl_db * dbC = fsl_cx_db_ckout(f); + fsl_stmt q = fsl_stmt_empty; + fsl_deck dBase = fsl_deck_empty; + char const * zColor; + int deltaPolicy = opt->deltaPolicy; + assert(d->f == f); + assert(FSL_SATYPE_CHECKIN==d->type); +#define RC if(rc) goto end + /* assert(basedOnVid>0); */ + rc = (opt->message && *opt->message) + ? fsl_deck_C_set( d, opt->message, -1 ) + : fsl_cx_err_set(f, FSL_RC_MISSING_INFO, + "Cowardly refusing to commit with " + "empty checkin comment."); + RC; + + if(deltaPolicy!=0 && fsl_repo_forbids_delta_manifests(f)){ + deltaPolicy = 0; + }else if(deltaPolicy<0 && f->cache.seenDeltaManifest<=0){ + deltaPolicy = 0; + } + { + char const * zUser = opt->user ? opt->user : fsl_cx_user_get(f); + rc = (zUser && *zUser) + ? fsl_deck_U_set( d, zUser ) + : fsl_cx_err_set(f, FSL_RC_MISSING_INFO, + "Cowardly refusing to commit without " + "a user name."); + RC; + } + + rc = fsl_check_for_partial_merge(f); + RC; + + rc = fsl_deck_D_set( d, (opt->julianTime>0) + ? opt->julianTime + : fsl_db_julian_now(dbR) ); + RC; + + if(opt->messageMimeType && *opt->messageMimeType){ + rc = fsl_deck_N_set( d, opt->messageMimeType, -1 ); + RC; + } + + + { /* F-cards */ + static char const * errNoFilesMsg = + "No files have changed. Cowardly refusing to commit."; + static int const errNoFilesRc = FSL_RC_NOOP; + fsl_deck * pBase = NULL /* baseline for delta generation purposes */; + fsl_size_t szD = 0, szB = 0 /* see commentary below */; + if(basedOnVid && deltaPolicy!=0){ + /* Figure out a baseline for a delta manifest... */ + fsl_uuid_str bUuid = NULL /* UUID for d's B-card */; + rc = fsl_deck_load_rid(f, &dBase, basedOnVid, FSL_SATYPE_CHECKIN); + RC; + if(dBase.B.uuid){ + /* dBase is a delta. Let's use its baseline for manifest + generation. */ + fsl_id_t const baseRid = fsl_uuid_to_rid(f, dBase.B.uuid); + fsl_deck_finalize(&dBase); + if(baseRid>0){ + rc = fsl_deck_load_rid(f, &dBase, baseRid, + FSL_SATYPE_CHECKIN); + }else{ + rc = fsl_cx_err_get(f, NULL, NULL); + assert(0!=rc); + } + RC; + }else{ + /* dBase version is a suitable baseline. */ + bUuid = fsl_rid_to_uuid(f, basedOnVid); + if(!bUuid){ + assert(f->error.code); + rc = f->error.code; + RC; + } + } + /* MARKER(("Baseline = %d / %s\n", (int)pBase->rid, pBase->uuid)); */ + assert(dBase.B.uuid || bUuid); + rc = fsl_deck_B_set(d, dBase.B.uuid ? dBase.B.uuid : bUuid); + fsl_free(bUuid); + RC; + pBase = &dBase; + } + rc = fsl_checkin_calc_F_cards2(f, d, pBase, basedOnVid, + &szD, opt); + /*MARKER(("szD=%d\n", (int)szD));*/ + RC; + if(basedOnVid && !szD){ + rc = fsl_cx_err_set(f, errNoFilesRc, errNoFilesMsg); + goto end; + } + szB = pBase ? pBase->F.used : 0; + /* The following text was copied verbatim from fossil(1). It does + not apply 100% here (because we use a slightly different + manifest generation approach) but it clearly describes what's + going on after the comment block.... + */ + /* + ** At this point, two manifests have been constructed, either of + ** which would work for this checkin. The first manifest (held + ** in the "manifest" variable) is a baseline manifest and the second + ** (held in variable named "delta") is a delta manifest. The + ** question now is: which manifest should we use? + ** + ** Let B be the number of F-cards in the baseline manifest and + ** let D be the number of F-cards in the delta manifest, plus one for + ** the B-card. (B is held in the szB variable and D is held in the + ** szD variable.) Assume that all delta manifests adds X new F-cards. + ** Then to minimize the total number of F- and B-cards in the repository, + ** we should use the delta manifest if and only if: + ** + ** D*D < B*X - X*X + ** + ** X is an unknown here, but for most repositories, we will not be + ** far wrong if we assume X=3. + */ + ++szD /* account for the d->B card */; + if(pBase){ + /* For this calculation, i believe the correct approach is to + simply count the F-cards, including those changed between the + baseline and the delta, as opposed to only those changed in + the delta itself. + */ + szD = 1 + d->F.used; + } + /* MARKER(("szB=%d szD=%d\n", (int)szB, (int)szD)); */ + if(pBase && (deltaPolicy<0/*non-force-mode*/ + && !(((int)(szD*szD)) < (((int)szB*3)-9)) + /* ^^^ see comments above */ + ) + ){ + /* Too small of a delta to be worth it. Re-calculate + F-cards with no baseline. + + Maintenance reminder: i initially wanted to update vfile's + status incrementally as F-cards are calculated, but this + discard/retry breaks on the retry because vfile's state has + been modified. Thus instead of updating vfile incrementally, + we re-scan it after the checkin completes. + */ + fsl_deck tmp = fsl_deck_empty; + /* Free up d->F using a kludge... */ + tmp.F = d->F; + d->F = fsl_deck_empty.F; + fsl_deck_finalize(&tmp); + fsl_deck_B_set(d, NULL); + /* MARKER(("Delta is too big - re-calculating F-cards for a baseline.\n")); */ + szD = 0; + rc = fsl_checkin_calc_F_cards2(f, d, NULL, basedOnVid, + &szD, opt); + RC; + if(basedOnVid && !szD){ + rc = fsl_cx_err_set(f, errNoFilesRc, errNoFilesMsg); + goto end; + } + } + }/* F-cards */ + + /* parents... */ + if( basedOnVid ){ + char * zParentUuid = fsl_rid_to_artifact_uuid(f, basedOnVid, FSL_SATYPE_CHECKIN); + if(!zParentUuid){ + assert(f->error.code); + rc = f->error.code + ? f->error.code + : fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Could not find checkin UUID " + "for RID %"FSL_ID_T_PFMT".", + basedOnVid); + goto end; + } + rc = fsl_deck_P_add(d, zParentUuid) + /* pedantic side-note: we could alternately transfer ownership + of zParentUuid by fsl_list_append()ing it to d->P, but that + would bypass e.g. any checking that routine chooses to apply. + */; + fsl_free(zParentUuid); + /* if(!rc) rc = fsl_leaf_set(f, basedOnVid, 0); */ + /* TODO: + if( p->verifyDate ) checkin_verify_younger(vid, zParentUuid, zDate); */ + RC; + rc = fsl_db_prepare(dbC, &q, "SELECT merge FROM vmerge WHERE id=0 OR id<-2"); + RC; + while( FSL_RC_STEP_ROW == fsl_stmt_step(&q) ){ + char *zMergeUuid; + fsl_id_t const mid = fsl_stmt_g_id(&q, 0); + //MARKER(("merging? %d\n", (int)mid)); + if( (mid == basedOnVid) + || (!f->cache.markPrivate && fsl_content_is_private(f,mid))){ + continue; + } + zMergeUuid = fsl_rid_to_uuid(f, mid) + /* FIXME? Adjust the query to join on blob and return the UUID? */ + ; + //MARKER(("merging %d %s\n", (int)mid, zMergeUuid)); + if(zMergeUuid){ + rc = fsl_deck_P_add(d, zMergeUuid); + fsl_free(zMergeUuid); + } + RC; + /* TODO: + if( p->verifyDate ) checkin_verify_younger(mid, zMergeUuid, zDate); */ + } + fsl_stmt_finalize(&q); + } + + { /* Q-cards... */ + rc = fsl_db_prepare(dbR, &q, + "SELECT " + "CASE vmerge.id WHEN -1 THEN '+' ELSE '-' END || mhash," + " merge" + " FROM vmerge" + " WHERE (vmerge.id=-1 OR vmerge.id=-2)" + " ORDER BY 1"); + while( !rc && (FSL_RC_STEP_ROW==fsl_stmt_step(&q)) ){ + fsl_id_t const mid = fsl_stmt_g_id(&q, 1); + if( mid != basedOnVid ){ + const char *zCherrypickUuid = fsl_stmt_g_text(&q, 0, NULL); + int const qType = '+'==*(zCherrypickUuid++) ? 1 : -1; + rc = fsl_deck_Q_add( d, qType, zCherrypickUuid, NULL ); + } + } + fsl_stmt_finalize(&q); + RC; + } + + zColor = opt->bgColor; + if(opt->branch && *opt->branch){ + char * sym = fsl_mprintf("sym-%s", opt->branch); + if(!sym){ + rc = FSL_RC_OOM; + goto end; + } + rc = fsl_deck_T_add( d, FSL_TAGTYPE_PROPAGATING, + NULL, sym, NULL ); + fsl_free(sym); + RC; + if(opt->bgColor && *opt->bgColor){ + zColor = NULL; + rc = fsl_deck_T_add( d, FSL_TAGTYPE_PROPAGATING, + NULL, "bgcolor", opt->bgColor); + RC; + } + rc = fsl_deck_T_add( d, FSL_TAGTYPE_PROPAGATING, + NULL, "branch", opt->branch ); + RC; + if(basedOnVid){ + rc = fsl_cancel_sym_tags(d, basedOnVid); + } + } + if(zColor && *zColor){ + /* One-shot background color */ + rc = fsl_deck_T_add( d, FSL_TAGTYPE_ADD, + NULL, "bgcolor", opt->bgColor); + RC; + } + + if(opt->closeBranch){ + rc = fsl_deck_T_add( d, FSL_TAGTYPE_ADD, + NULL, "closed", + *opt->closeBranch + ? opt->closeBranch + : NULL); + RC; + } + + { + /* + Close any INTEGRATE merges if !op->integrate, or type-0 and + integrate merges if opt->integrate. + */ + rc = fsl_db_prepare(dbC, &q, + "SELECT mhash, merge FROM vmerge " + " WHERE id %s ORDER BY 1", + opt->integrate ? "IN(0,-4)" : "=(-4)"); + while( !rc && (FSL_RC_STEP_ROW==fsl_stmt_step(&q)) ){ + fsl_id_t const rid = fsl_stmt_g_id(&q, 1); + //MARKER(("Integrating %d? opt->integrate=%d\n",(int)rid, opt->integrate)); + if( fsl_rid_is_leaf(f, rid) + && !fsl_db_exists(dbR, /* Is not closed already... */ + "SELECT 1 FROM tagxref " + "WHERE tagid=%d AND rid=%"FSL_ID_T_PFMT + " AND tagtype>0", + FSL_TAGID_CLOSED, rid)){ + const char *zIntegrateUuid = fsl_stmt_g_text(&q, 0, NULL); + //MARKER(("Integrating %d %s\n",(int)rid, zIntegrateUuid)); + rc = fsl_deck_T_add( d, FSL_TAGTYPE_ADD, zIntegrateUuid, + "closed", "Closed by integrate-merge." ); + } + } + fsl_stmt_finalize(&q); + RC; + } + + end: +#undef RC + fsl_stmt_finalize(&q); + fsl_deck_finalize(&dBase); + assert(NULL==d->B.baseline || &dBase==d->B.baseline); + d->B.baseline = NULL /* if it was set, it was &dBase */; + if(rc && !f->error.code){ + if(dbR->error.code) fsl_cx_uplift_db_error(f, dbR); + else if(dbC->error.code) fsl_cx_uplift_db_error(f, dbC); + else if(f->dbMain->error.code) fsl_cx_uplift_db_error(f, f->dbMain); + } + return rc; +} + +int fsl_checkin_T_add2( fsl_cx * f, fsl_card_T * t){ + return fsl_deck_T_add2( &f->ckin.mf, t ); +} + +int fsl_checkin_T_add( fsl_cx * f, fsl_tagtype_e tagType, + fsl_uuid_cstr uuid, char const * name, + char const * value){ + return fsl_deck_T_add( &f->ckin.mf, tagType, uuid, name, value ); +} + +/** + Returns true if the given blob RID is has a "closed" tag. This is + generally intended only to be passed the RID of the current + checkout, before attempting to perform a commit against it. +*/ +static bool fsl_leaf_is_closed(fsl_cx * f, fsl_id_t rid){ + fsl_db * const dbR = fsl_needs_repo(f); + return dbR + ? fsl_db_exists(dbR, "SELECT 1 FROM tagxref" + " WHERE tagid=%d " + " AND rid=%"FSL_ID_T_PFMT" AND tagtype>0", + FSL_TAGID_CLOSED, rid) + : false; +} + +/** + Returns true if the given name is the current branch + for the given checkin version. + */ +static bool fsl_is_current_branch(fsl_db * dbR, fsl_id_t vid, + char const * name){ + return fsl_db_exists(dbR, + "SELECT 1 FROM tagxref" + " WHERE tagid=%d AND rid=%"FSL_ID_T_PFMT + " AND tagtype>0" + " AND value=%Q", + FSL_TAGID_BRANCH, vid, name); +} + +int fsl_checkin_commit(fsl_cx * f, fsl_checkin_opt const * opt, + fsl_id_t * newRid, fsl_uuid_str * newUuid ){ + int rc; + fsl_deck deck = fsl_deck_empty; + fsl_deck *d = &deck; + fsl_db * dbC; + fsl_db * dbR; + char inTrans = 0; + char oldPrivate; + int const oldFlags = f ? f->flags : 0; + fsl_id_t const vid = f ? f->ckout.rid : 0; + if(!f || !opt) return FSL_RC_MISUSE; + else if(!(dbC = fsl_needs_ckout(f))) return FSL_RC_NOT_A_CKOUT; + else if(!(dbR = fsl_needs_repo(f))) return FSL_RC_NOT_A_REPO; + assert(vid>=0); + /** + Do not permit a checkin to a closed leaf unless opt->branch would + switch us to a new branch. + */ + if( fsl_leaf_is_closed(f, vid) + && (!opt->branch || !*opt->branch + || fsl_is_current_branch(dbR, vid, opt->branch))){ + return fsl_cx_err_set(f, FSL_RC_ACCESS, + "Only Chuck Norris can commit to " + "a closed leaf."); + } + + if(vid && opt->scanForChanges){ + /* We need to ensure this state is current in order to determine + whether a given file is locally modified vis-a-vis the + commit-time vfile state. */ + rc = fsl_vfile_changes_scan(f, vid, 0); + if(rc) return rc; + } + + fsl_cx_err_reset(f) /* avoid propagating an older error by accident. + Did that in test code. */; + + oldPrivate = f->cache.markPrivate; + if(opt->isPrivate || fsl_content_is_private(f, vid)){ + f->cache.markPrivate = 1; + } + +#define RC if(rc) goto end + fsl_deck_init(f, d, FSL_SATYPE_CHECKIN); + + rc = fsl_db_transaction_begin(dbR); + RC; + inTrans = 1; + if(f->ckin.mf.T.used){ + /* Transfer accumulated tags. */ + assert(!f->ckin.mf.content.used); + d->T = f->ckin.mf.T; + f->ckin.mf.T = fsl_deck_empty.T; + } + rc = fsl_checkin_calc_manifest(f, d, vid, opt); + RC; + if(!d->F.used){ + rc = fsl_cx_err_set(f, FSL_RC_NOOP, + "Cowardly refusing to generate an empty commit."); + RC; + } + + if(opt->calcRCard) f->flags |= FSL_CX_F_CALC_R_CARD; + else f->flags &= ~FSL_CX_F_CALC_R_CARD; + rc = fsl_deck_save( d, opt->isPrivate ); + RC; + assert(d->rid>0); + /* Now get vfile back into shape. We do not do a vfile scan + because that loses state like add/rm-queued files. */ + rc = fsl_db_exec_multi(dbC, + "DELETE FROM vfile WHERE vid<>" + "%" FSL_ID_T_PFMT ";" + "UPDATE vfile SET vid=%" FSL_ID_T_PFMT ";" + "DELETE FROM vfile WHERE deleted AND " + "fsl_is_enqueued(id); " + "UPDATE vfile SET rid=mrid, mhash=NULL, " + "chnged=0, deleted=0, origname=NULL " + "WHERE fsl_is_enqueued(id)", + vid, d->rid); + if(!rc) rc = fsl_ckout_version_write(f, d->rid, NULL); + RC; + assert(d->f == f); + rc = fsl_checkin_add_unsent(f, d->rid); + RC; + rc = fsl_ckout_clear_merge_state(f); + RC; + /* + todo(?) from fossil(1) follows. Most of this seems to be what the + vfile handling does (above). + + db_multi_exec("PRAGMA %s.application_id=252006673;", db_name("repository")); + db_multi_exec("PRAGMA %s.application_id=252006674;", db_name("localdb")); + + // Update the vfile and vmerge tables + db_multi_exec( + "DELETE FROM vfile WHERE (vid!=%d OR deleted) AND is_selected(id);" + "DELETE FROM vmerge;" + "UPDATE vfile SET vid=%d;" + "UPDATE vfile SET rid=mrid, chnged=0, deleted=0, origname=NULL" + " WHERE is_selected(id);" + , vid, nvid + ); + db_lset_int("checkout", nvid); + + + // Update the isexe and islink columns of the vfile table + db_prepare(&q, + "UPDATE vfile SET isexe=:exec, islink=:link" + " WHERE vid=:vid AND pathname=:path AND (isexe!=:exec OR islink!=:link)" + ); + db_bind_int(&q, ":vid", nvid); + pManifest = manifest_get(nvid, CFTYPE_MANIFEST, 0); + manifest_file_rewind(pManifest); + while( (pFile = manifest_file_next(pManifest, 0)) ){ + db_bind_int(&q, ":exec", pFile->zPerm && strstr(pFile->zPerm, "x")); + db_bind_int(&q, ":link", pFile->zPerm && strstr(pFile->zPerm, "l")); + db_bind_text(&q, ":path", pFile->zName); + db_step(&q); + db_reset(&q); + } + db_finalize(&q); + */ + + if(opt->dumpManifestFile){ + FILE * out; + /* MARKER(("Dumping generated manifest to file [%s]:\n", opt->dumpManifestFile)); */ + out = fsl_fopen(opt->dumpManifestFile, "w"); + if(out){ + rc = fsl_deck_output( d, fsl_output_f_FILE, out ); + fsl_fclose(out); + }else{ + rc = fsl_cx_err_set(f, FSL_RC_IO, "Could not open output " + "file for writing: %s", opt->dumpManifestFile); + } + RC; + } + + if(d->P.used){ + /* deltify the parent manifest */ + char const * p0 = (char const *)d->P.list[0]; + fsl_id_t const prid = fsl_uuid_to_rid(f, p0); + /* MARKER(("Deltifying parent manifest #%d...\n", (int)prid)); */ + assert(p0); + assert(prid>0); + rc = fsl_content_deltify(f, prid, d->rid, 0); + RC; + } + + end: + f->flags = oldFlags; +#undef RC + f->cache.markPrivate = oldPrivate; + /* fsl_buffer_reuse(&f->fileContent); */ + if(inTrans){ + if(rc) fsl_db_transaction_rollback(dbR); + else{ + rc = fsl_db_transaction_commit(dbR); + if(!rc){ + if(newRid) *newRid = d->rid; + if(newUuid){ + if(NULL==(*newUuid = fsl_rid_to_uuid(f, d->rid))){ + rc = FSL_RC_OOM; + } + } + } + } + } + if(rc && !f->error.code){ + if(f->dbMain->error.code) fsl_cx_uplift_db_error(f, f->dbMain); + else f->error.code = rc; + } + fsl_checkin_discard(f); + fsl_deck_finalize(d); + return rc; +} + + +#undef MARKER +/* end of file checkin.c */ +/* start of file checkout.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/* + ***************************************************************************** + This file houses the code for checkout-level APIS. +*/ +#include + +#include /* memcmp() */ + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + + +/** + Kludge for type-safe strncmp/strnicmp inconsistency. +*/ +static int fsl_strnicmp_int(char const *zA, char const * zB, fsl_size_t nByte){ + return fsl_strnicmp( zA, zB, (fsl_int_t)nByte); +} + +int fsl_ckout_filename_check( fsl_cx * f, bool relativeToCwd, + char const * zOrigName, fsl_buffer * pOut ){ + int rc; + if(!zOrigName || !*zOrigName) return FSL_RC_MISUSE; + else if(!fsl_needs_ckout(f)/* will update f's error state*/){ + return FSL_RC_NOT_A_CKOUT; + } +#if 0 + /* Is this sane? */ + else if(fsl_is_simple_pathname(zOrigName,1)){ + rc = 0; + if(pOut){ + rc = fsl_buffer_append(pOut, zOrigName, fsl_strlen(zOrigName)); + } + } +#endif + else{ + char const * zLocalRoot; + char const * zFull; + fsl_size_t nLocalRoot; + fsl_size_t nFull; + fsl_buffer * full = fsl_cx_scratchpad(f); + int (*xCmp)(char const *, char const *,fsl_size_t); + bool endsWithSlash; + assert(f->ckout.dir); + zLocalRoot = f->ckout.dir; + assert(zLocalRoot); + assert(*zLocalRoot); + nLocalRoot = f->ckout.dirLen; + assert(nLocalRoot); + assert('/' == zLocalRoot[nLocalRoot-1]); + rc = fsl_file_canonical_name2(relativeToCwd ? NULL : zLocalRoot, + zOrigName, full, 1); +#if 0 + MARKER(("canon2: %p (%s) %s ==> %s\n", (void const *)full->mem, + relativeToCwd ? "cwd" : "ckout", zOrigName, fsl_buffer_cstr(full))); +#endif + if(rc){ + if(FSL_RC_OOM != rc){ + rc = fsl_cx_err_set(f, rc, "Error #%d (%s) canonicalizing " + "file name: %s\n", + rc, fsl_rc_cstr(rc), + zOrigName); + } + goto end; + } + zFull = fsl_buffer_cstr2(full, &nFull); + xCmp = fsl_cx_is_case_sensitive(f) + ? fsl_strncmp + : fsl_strnicmp_int; + assert(zFull); + assert(nFull>0); + endsWithSlash = '/' == zFull[nFull-1]; + if( ((nFull==nLocalRoot-1 || (nFull==nLocalRoot && endsWithSlash)) + && xCmp(zLocalRoot, zFull, nFull)==0) + || (nFull==1 && zFull[0]=='/' && nLocalRoot==1 && zLocalRoot[0]=='/') ){ + /* Special case. zOrigName refers to zLocalRoot directory. + + Outputing "." instead of nothing is a historical decision + which may be worth re-evaluating. Currently fsl_cx_stat() relies + on it. + */ + if(pOut){ + char const * zOut; + fsl_size_t nOut; + if(endsWithSlash){ /* retain trailing slash */ + zOut = "./"; + nOut = 2; + }else{ + zOut = "."; + nOut = 1; + }; + rc = fsl_buffer_append(pOut, zOut, nOut); + }else{ + rc = 0; + } + goto end; + } + + if( nFull<=nLocalRoot || xCmp(zLocalRoot, zFull, nLocalRoot) ){ + rc = fsl_cx_err_set(f, FSL_RC_RANGE, + "File is outside of checkout tree: %s", + zOrigName); + goto end; + } + + if(pOut){ + rc = fsl_buffer_append(pOut, zFull + nLocalRoot, nFull - nLocalRoot); + } + + end: + fsl_cx_scratchpad_yield(f, full); + } + return rc; +} + + +/** + Returns a fsl_ckout_change_e value for the given + fsl_vfile_change_e value. + + Why are these not consolidated into one enum? 2021-03-13: because + there are more checkout-level change codes than vfile-level + changes. We could still consolidate them, giving the vfile changes + their hard-coded values and leaving room in the enum for upward + growth of that set. +*/ +static fsl_ckout_change_e fsl_vfile_to_ckout_change(int vChange){ + switch((fsl_vfile_change_e)vChange){ +#define EE(X) case FSL_VFILE_CHANGE_##X: return FSL_CKOUT_CHANGE_##X + EE(NONE); + EE(MOD); + EE(MERGE_MOD); + EE(MERGE_ADD); + EE(INTEGRATE_MOD); + EE(INTEGRATE_ADD); + EE(IS_EXEC); + EE(BECAME_SYMLINK); + EE(NOT_EXEC); + EE(NOT_SYMLINK); +#undef EE + default: + assert(!"Unhandled fsl_vfile_change_e value!"); + return FSL_CKOUT_CHANGE_NONE; + } +} + +int fsl_ckout_changes_visit( fsl_cx * f, fsl_id_t vid, + bool doScan, + fsl_ckout_changes_f visitor, + void * state ){ + int rc; + fsl_db * db; + fsl_stmt st = fsl_stmt_empty; + int count = 0; + fsl_ckout_change_e coChange; + fsl_fstat fstat; + if(!f || !visitor) return FSL_RC_MISUSE; + db = fsl_needs_ckout(f); + if(!db) return FSL_RC_NOT_A_CKOUT; + if(vid<0){ + vid = f->ckout.rid; + assert(vid>=0); + } + if(doScan){ + rc = fsl_vfile_changes_scan(f, vid, 0); + if(rc) goto end; + } + rc = fsl_db_prepare(db, &st, + "SELECT chnged, deleted, rid, " + "pathname, origname " + "FROM vfile WHERE vid=%" FSL_ID_T_PFMT + " /*%s()*/", + vid,__func__); + assert(!rc); + while( FSL_RC_STEP_ROW == fsl_stmt_step(&st) ){ + int const changed = fsl_stmt_g_int32(&st, 0); + int const deleted = fsl_stmt_g_int32(&st,1); + fsl_id_t const vrid = fsl_stmt_g_id(&st,2); + char const * name; + char const * oname = NULL; + name = fsl_stmt_g_text(&st, 3, NULL); + oname = fsl_stmt_g_text(&st,4,NULL); + if(oname && (0==fsl_strcmp(name, oname))){ + /* Work around a fossil oddity which sets origname=pathname + during a 'mv' operation. + */ + oname = NULL; + } + coChange = FSL_CKOUT_CHANGE_NONE; + if(deleted){ + coChange = FSL_CKOUT_CHANGE_REMOVED; + }else if(0==vrid){ + coChange = FSL_CKOUT_CHANGE_ADDED; + }else if(!changed && NULL != oname){ + /* In fossil ^^, the "changed" state trumps the "renamed" state + for status view purposes, so we'll do that here. */ + coChange = FSL_CKOUT_CHANGE_RENAMED; + }else{ + fstat = fsl_fstat_empty; + if( fsl_cx_stat(f, false, name, &fstat ) ){ + coChange = FSL_CKOUT_CHANGE_MISSING; + fsl_cx_err_reset(f) /* keep FSL_RC_NOT_FOUND from bubbling + up to the client! */; + }else if(!changed){ + continue; + }else{ + coChange = fsl_vfile_to_ckout_change(changed); + } + } + if(!coChange){ + MARKER(("INTERNAL ERROR: unhandled vfile.chnged " + "value %d for file [%s]\n", + changed, name)); + continue; + } + ++count; + rc = visitor(state, coChange, name, oname); + if(rc){ + if(FSL_RC_BREAK==rc){ + rc = 0; + break; + }else if(!f->error.code && (FSL_RC_OOM!=rc)){ + fsl_cx_err_set(f, rc, "Error %s returned from changes callback.", + fsl_rc_cstr(rc)); + } + break; + } + } + end: + fsl_stmt_finalize(&st); + if(rc && db->error.code && !f->error.code){ + fsl_cx_uplift_db_error(f, db); + } + + return rc; +} + +static bool fsl_co_is_in_vfile(fsl_cx *f, + char const *zFilename){ + return fsl_db_exists(fsl_cx_db_ckout(f), + "SELECT 1 FROM vfile" + " WHERE vid=%"FSL_ID_T_PFMT + " AND pathname=%Q %s", + f->ckout.rid, zFilename, + fsl_cx_filename_collation(f)); +} +/** + Internal machinery for fsl_ckout_manage(). zFilename MUST + be a checkout-relative file which is known to exist. fst MUST + be an object populated by fsl_stat()'ing zFilename. isInVFile + MUST be the result of having passed zFilename to fsl_co_is_in_vfile(). + */ +static int fsl_ckout_manage_impl( fsl_cx * f, char const *zFilename, + fsl_fstat const *fst, + bool isInVFile){ + int rc = 0; + fsl_db * const db = fsl_needs_ckout(f); + assert(fsl_is_simple_pathname(zFilename, true)); + if( isInVFile ){ + rc = fsl_db_exec(db, "UPDATE vfile SET deleted=0," + " mtime=%"PRIi64 + " WHERE vid=%"FSL_ID_T_PFMT + " AND pathname=%Q %s", + (int64_t)fst->mtime, + f->ckout.rid, zFilename, + fsl_cx_filename_collation(f)); + }else{ + int const chnged = FSL_VFILE_CHANGE_MOD + /* fossil(1) sets chnged=0 on 'add'ed vfile records, but then the 'status' + command updates the field to 1. To avoid down-stream inconsistencies + (such as the ones which lead me here), we'll go ahead and set it to + 1 here. + */; + rc = fsl_db_exec(db, + "INSERT INTO " + "vfile(vid,chnged,deleted,rid,mrid,pathname,isexe,islink,mtime)" + "VALUES(%"FSL_ID_T_PFMT",%d,0,0,0,%Q,%d,%d,%"PRIi64")", + f->ckout.rid, chnged, zFilename, + (FSL_FSTAT_PERM_EXE==fst->perm) ? 1 : 0, + (FSL_FSTAT_TYPE_LINK==fst->type) ? 1 : 0, + (int64_t)fst->mtime + ); + } + if(rc) rc = fsl_cx_uplift_db_error2(f, db, rc); + return rc; +} + +/** + Internal state for the recursive file-add process. +*/ +struct CoAddState { + fsl_cx * f; + fsl_ckout_manage_opt * opt; + fsl_buffer * absBuf; // absolute path of file to check + fsl_buffer * coRelBuf; // checkout-relative path of absBuf + fsl_fstat fst; // fsl_stat() state of absBuf's file +}; +typedef struct CoAddState CoAddState; +static const CoAddState CoAddState_empty = + {NULL, NULL, NULL, NULL, fsl_fstat_empty_m}; + +/** + fsl_dircrawl_f() impl for recursively adding files to a + repo. state must be a (CoAddState*)/ +*/ +static int fsl_dircrawl_f_add(fsl_dircrawl_state const *); + +/** + Attempts to add file or directory (recursively) cas->absBuf to the + current repository. isCrawling must be true if this is a + fsl_dircrawl()-invoked call, else false. +*/ +static int co_add_one(CoAddState * cas, bool isCrawling){ + int rc = 0; + fsl_buffer_reuse(cas->coRelBuf); + rc = fsl_cx_stat2(cas->f, cas->opt->relativeToCwd, + fsl_buffer_cstr(cas->absBuf), &cas->fst, + fsl_buffer_reuse(cas->coRelBuf), false) + /* Reminder: will fail if file is outside of the checkout tree */; + if(rc) return rc; + switch(cas->fst.type){ + case FSL_FSTAT_TYPE_FILE:{ + bool skipped = false; + char const * zCoRel = fsl_buffer_cstr(cas->coRelBuf); + bool const isInVFile = fsl_co_is_in_vfile(cas->f, zCoRel); + if(!isInVFile){ + if(fsl_reserved_fn_check(cas->f, zCoRel,-1,false)){ + /* ^^^ we need to use fsl_reserved_fn_check(), instead of + fsl_is_reserved_fn(), so that we will inherit any + new checks which require a context object. If that + check fails, though, it updates cas->f with an error + message which we need to suppress here to avoid it + accidentally propagating and causing downstream + confusion. */ + fsl_cx_err_reset(cas->f); + skipped = true; + }else if(cas->opt->checkIgnoreGlobs){ + char const * m = + fsl_cx_glob_matches(cas->f, FSL_GLOBS_IGNORE, zCoRel); + if(m) skipped = true; + } + if(!skipped && cas->opt->callback){ + bool yes = false; + rc = cas->opt->callback( zCoRel, &yes, + cas->opt->callbackState ); + if(rc) goto end; + else if(!yes) skipped = true; + } + } + if(skipped){ + ++cas->opt->counts.skipped; + }else{ + rc = fsl_ckout_manage_impl(cas->f, zCoRel, &cas->fst, + isInVFile); + if(!rc){ + if(isInVFile) ++cas->opt->counts.updated; + else ++cas->opt->counts.added; + } + } + break; + } + case FSL_FSTAT_TYPE_DIR: + if(!isCrawling){ + /* Reminder to self: fsl_dircrawl() copies its first argument + for canonicalizing it, so this is safe even though + cas->absBuf may be reallocated during the recursive + call. We're done with these particular contents of + cas->absBuf at this point. */ + rc = fsl_dircrawl(fsl_buffer_cstr(cas->absBuf), + fsl_dircrawl_f_add, cas); + if(rc && !cas->f->error.code){ + rc = fsl_cx_err_set(cas->f, rc, "fsl_dircrawl() returned %s.", + fsl_rc_cstr(rc)); + } + }else{ + assert(!"Cannot happen - caught higher up"); + fsl_fatal(FSL_RC_ERROR, "Internal API misuse in/around %s().", + __func__); + } + break; + default: + rc = fsl_cx_err_set(cas->f, FSL_RC_TYPE, + "Unhandled filesystem entry type: " + "fsl_fstat_type_e #%d", cas->fst.type); + break; + } + end: + return rc; +} + +static int fsl_dircrawl_f_add(fsl_dircrawl_state const *dst){ + if(FSL_FSTAT_TYPE_FILE!=dst->entryType) return 0; + CoAddState * cas = (CoAddState*)dst->callbackState; + int rc = fsl_buffer_appendf(fsl_buffer_reuse(cas->absBuf), + "%s/%s", dst->absoluteDir, dst->entryName); + if(!rc) rc = co_add_one(cas, true); + return rc; +} + +int fsl_ckout_manage( fsl_cx * const f, fsl_ckout_manage_opt * const opt_ ){ + int rc = 0; + CoAddState cas = CoAddState_empty; + fsl_ckout_manage_opt opt; + if(!f) return FSL_RC_MISUSE; + else if(!fsl_needs_ckout(f)) return FSL_RC_NOT_A_CKOUT; + assert(f->ckout.rid>=0); + opt = *opt_ + /*use a copy in case the user manages to modify + opt_ from a callback. */; + cas.absBuf = fsl_cx_scratchpad(f); + cas.coRelBuf = fsl_cx_scratchpad(f); + rc = fsl_file_canonical_name(opt.filename, cas.absBuf, false); + if(!rc){ + cas.f = f; + cas.opt = &opt; + rc = co_add_one(&cas, false); + opt_->counts = opt.counts; + } + fsl_cx_scratchpad_yield(f, cas.absBuf); + fsl_cx_scratchpad_yield(f, cas.coRelBuf); + return rc; +} + +/** + Creates, if needed, a TEMP TABLE named [tableName] with a single + [id] field and populates it with all ids from the given bag. + + Returns 0 on success, any number of non-0 codes on error. +*/ +static int fsl_ckout_bag_to_ids(fsl_cx *f, fsl_db * db, + char const * tableName, + fsl_id_bag const * bag){ + fsl_stmt insId = fsl_stmt_empty; + int rc = fsl_db_exec_multi(db, + "CREATE TEMP TABLE IF NOT EXISTS " + "[%s](id); " + "DELETE FROM [%s] /* %s() */;", + tableName, tableName, __func__); + if(rc) goto dberr; + rc = fsl_db_prepare(db, &insId, + "INSERT INTO [%s](id) values(?1) " + "/* %s() */", tableName, __func__); + if(rc) goto dberr; + for(fsl_id_t e = fsl_id_bag_first(bag); + e; e = fsl_id_bag_next(bag, e)){ + fsl_stmt_bind_id(&insId, 1, e); + rc = fsl_stmt_step(&insId); + switch(rc){ + case FSL_RC_STEP_DONE: + rc = 0; + break; + default: + fsl_stmt_finalize(&insId); + goto dberr; + } + fsl_stmt_reset(&insId); + } + assert(!rc); + end: + fsl_stmt_finalize(&insId); + return rc; + dberr: + assert(rc); + rc = fsl_cx_uplift_db_error2(f, db, rc); + goto end; +} + +int fsl_ckout_unmanage(fsl_cx * f, fsl_ckout_unmanage_opt const * opt){ + int rc; + fsl_db * const db = fsl_needs_ckout(f); + fsl_buffer * fname = 0; + fsl_id_t const vid = f->ckout.rid; + fsl_stmt q = fsl_stmt_empty; + bool inTrans = false; + if(!db) return FSL_RC_NOT_A_CKOUT; + else if((!opt->filename || !*opt->filename) + && !opt->vfileIds){ + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "Empty file set is not legal for %s()", + __func__); + } + assert(vid>=0); + rc = fsl_db_transaction_begin(db); + if(rc) goto dberr; + inTrans = true; + if(opt->vfileIds){ + rc = fsl_ckout_bag_to_ids(f, db, "fx_unmanage_id", opt->vfileIds); + if(rc) goto end; + rc = fsl_db_exec(db, + "UPDATE vfile SET deleted=1 " + "WHERE vid=%" FSL_ID_T_PFMT " " + "AND NOT deleted " + "AND id IN fx_unmanage_id /* %s() */", + vid, __func__); + if(rc) goto dberr; + if(opt->callback){ + rc = fsl_db_prepare(db,&q, + "SELECT pathname FROM vfile " + "WHERE vid=%" FSL_ID_T_PFMT " " + "AND deleted " + "AND id IN fx_unmanage_id " + "/* %s() */", + vid, __func__); + if(rc) goto dberr; + } + }else{// Process opt->filename + fname = fsl_cx_scratchpad(f); + rc = fsl_ckout_filename_check(f, opt->relativeToCwd, + opt->filename, fname); + if(rc) goto end; + char const * zNorm = fsl_buffer_cstr(fname); + /* MARKER(("fsl_ckout_unmanage(%d, %s) ==> %s\n", relativeToCwd, zFilename, zNorm)); */ + assert(zNorm); + if(fname->used){ + fsl_buffer_strip_slashes(fname); + if(1==fname->used && '.'==*zNorm){ + /* Special case: handle "." from ckout root intuitively */ + fsl_buffer_reuse(fname); + assert(0==*zNorm); + } + } + rc = fsl_db_exec(db, + "UPDATE vfile SET deleted=1 " + "WHERE vid=%" FSL_ID_T_PFMT " " + "AND NOT deleted " + "AND CASE WHEN %Q='' THEN 1 " + "ELSE fsl_match_vfile_or_dir(pathname,%Q) " + "END /*%s()*/", + vid, zNorm, zNorm, __func__); + if(rc) goto dberr; + if(opt->callback){ + rc = fsl_db_prepare(db,&q, + "SELECT pathname FROM vfile " + "WHERE vid=%" FSL_ID_T_PFMT " " + "AND deleted " + "AND CASE WHEN %Q='' THEN 1 " + "ELSE fsl_match_vfile_or_dir(pathname,%Q) " + "END " + "UNION " + "SELECT pathname FROM vfile " + "WHERE vid=%" FSL_ID_T_PFMT " " + "AND rid=0 AND deleted " + "/*%s()*/", + vid, zNorm, zNorm, vid, __func__); + if(rc) goto dberr; + } + }/*opt->filename*/ + + if(q.stmt){ + while(FSL_RC_STEP_ROW==fsl_stmt_step(&q)){ + char const * fn = fsl_stmt_g_text(&q, 0, NULL); + rc = opt->callback(fn, opt->callbackState); + if(rc) goto end; + } + fsl_stmt_finalize(&q); + } + /* Remove rm'd ADDed-but-not-yet-committed entries... */ + rc = fsl_db_exec(db, + "DELETE FROM vfile WHERE vid=%" FSL_ID_T_PFMT + " AND rid=0 AND deleted", + vid); + if(rc) goto dberr; + end: + if(fname) fsl_cx_scratchpad_yield(f, fname); + fsl_stmt_finalize(&q); + if(opt->vfileIds){ + fsl_db_exec(db, "DROP TABLE IF EXISTS fx_unmanage_id /* %s() */", + __func__) + /* Ignoring result code */; + } + if(inTrans){ + int const rc2 = fsl_db_transaction_end(db, !!rc); + if(!rc) rc = rc2; + } + return rc; + dberr: + assert(rc); + rc = fsl_cx_uplift_db_error2(f, db, rc); + goto end; + +} + +int fsl_ckout_changes_scan(fsl_cx * f){ + return fsl_vfile_changes_scan(f, -1, 0); +} + +int fsl_ckout_install_schema(fsl_cx *f, bool dropIfExists){ + char const * tNames[] = { + "vvar", "vfile", "vmerge", 0 + }; + int rc; + fsl_db * const db = fsl_needs_ckout(f); + if(!db) return f->error.code; + if(dropIfExists){ + char const * t; + int i; + char const * dbName = fsl_db_role_label(FSL_DBROLE_CKOUT); + for(i=0; 0!=(t = tNames[i]); ++i){ + rc = fsl_db_exec(db, "DROP TABLE IF EXISTS %s.%s /*%s()*/", + dbName, t, __func__); + if(rc) break; + } + if(!rc){ + rc = fsl_db_exec(db, "DROP TRIGGER IF EXISTS " + "%s.vmerge_ck1 /*%s()*/", + dbName, __func__); + } + }else{ + if(fsl_db_table_exists(db, FSL_DBROLE_CKOUT, + tNames[0])){ + return 0; + } + } + rc = fsl_db_exec_multi(db, "%s", fsl_schema_ckout()); + return fsl_cx_uplift_db_error2(f, db, rc); +} + +bool fsl_ckout_has_changes(fsl_cx *f){ + fsl_db * const db = fsl_cx_db_ckout(f); + if(!db) return false; + return fsl_db_exists(db, + "SELECT 1 FROM vfile WHERE chnged " + "OR coalesce(origname != pathname, 0) " + "/*%s()*/", __func__) + || fsl_db_exists(db,"SELECT 1 FROM vmerge /*%s()*/", __func__); +} + +int fsl_ckout_clear_merge_state( fsl_cx *f ){ + fsl_db * const d = fsl_needs_ckout(f); + int rc; + if(d){ + rc = fsl_db_exec(d,"DELETE FROM vmerge /*%s()*/", __func__); + rc = fsl_cx_uplift_db_error2(f, d, rc); + }else{ + rc = FSL_RC_NOT_A_CKOUT; + } + return rc; +} + +int fsl_ckout_clear_db(fsl_cx *f){ + fsl_db * const db = fsl_needs_ckout(f); + if(!db) return f->error.code; + return fsl_db_exec_multi(db, + "DELETE FROM vfile;" + "DELETE FROM vmerge;" + "DELETE FROM vvar WHERE name IN" + "('checkout','checkout-hash') " + "/*%s()*/", __func__); +} + +fsl_db * fsl_cx_db_for_role(fsl_cx *, fsl_dbrole_e) + /* defined in cx.c */; + +/** + Updates f->ckout.dir and dirLen based on the current state of + f->ckout.db. Returns 0 on success, FSL_RC_OOM on allocation error, + some other code if canonicalization of the name fails + (e.g. filesystem error or cwd cannot be resolved). +*/ +static int fsl_update_ckout_dir(fsl_cx *f){ + int rc; + fsl_buffer ckDir = fsl_buffer_empty; + fsl_db * dbC = fsl_cx_db_for_role(f, FSL_DBROLE_CKOUT); + assert(dbC->filename); + assert(*dbC->filename); + rc = fsl_file_canonical_name(dbC->filename, &ckDir, false); + if(rc) return rc; + char * zCanon = fsl_buffer_take(&ckDir); + //MARKER(("dbC->filename=%s\n", dbC->filename)); + //MARKER(("zCanon=%s\n", zCanon)); + rc = fsl_file_dirpart(zCanon, -1, &ckDir, true); + fsl_free(zCanon); + if(rc){ + fsl_buffer_clear(&ckDir); + }else{ + fsl_free(f->ckout.dir); + f->ckout.dirLen = ckDir.used; + f->ckout.dir = fsl_buffer_take(&ckDir); + assert('/'==f->ckout.dir[f->ckout.dirLen-1]); + /*MARKER(("Updated ckout.dir: %d %s\n", + (int)f->ckout.dirLen, f->ckout.dir));*/ + } + return rc; +} + + +int fsl_repo_open_ckout(fsl_cx *f, const fsl_repo_open_ckout_opt *opt){ + fsl_db *dbC = 0; + fsl_buffer *cwd = 0; + int rc = 0; + + if(!opt) return FSL_RC_MISUSE; + else if(!fsl_needs_repo(f)){ + return f->error.code; + }else if(fsl_cx_db_ckout(f)){ + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "A checkout is already attached."); + } + if(opt->targetDir && *opt->targetDir){ + if(fsl_chdir(opt->targetDir)){ + return fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Directory not found or inaccessible: %s", + opt->targetDir); + } + } + cwd = fsl_cx_scratchpad(f); + assert(!cwd->used); + if((rc = fsl_cx_getcwd(f, cwd))){ + assert(!cwd->used); + fsl_cx_scratchpad_yield(f, cwd); + return fsl_cx_err_set(f, rc, "Error %d [%s]: unable to " + "determine current directory.", + rc, fsl_rc_cstr(rc)); + } + /** + AS OF HERE: do not use 'return'. Use goto end so that we can + chdir() back to our original cwd! + */ + if(!fsl_dir_is_empty("."/*we've already chdir'd if + we were going to*/)) { + switch(opt->fileOverwritePolicy){ + case FSL_OVERWRITE_ALWAYS: + case FSL_OVERWRITE_NEVER: break; + default: + assert(FSL_OVERWRITE_ERROR==opt->fileOverwritePolicy); + rc = fsl_cx_err_set(f, FSL_RC_ACCESS, + "Directory is not empty and " + "fileOverwritePolicy is " + "FSL_OVERWRITE_ERROR: " + "%b", cwd); + goto end; + } + } + if(opt->checkForOpenedCkout){ + /* Check target and parent dirs for a checkout and bail out if we + find one. If opt->checkForOpenedCkout is false then we will use + the dbOverwritePolicy to determine what to do if we find a + checkout db in cwd (as opposed to a parent). */ + fsl_buffer * const foundAt = fsl_cx_scratchpad(f); + if (!fsl_ckout_db_search(fsl_buffer_cstr(cwd), true, foundAt)) { + rc = fsl_cx_err_set(f, FSL_RC_ALREADY_EXISTS, + "There is already a checkout db at %b", + foundAt); + } + fsl_cx_scratchpad_yield(f, foundAt); + if(rc) goto end; + } + + /** + Create and attach ckout db... + */ + assert(!fsl_cx_db_ckout(f)); + const char * dbName = opt->ckoutDbFile + ? opt->ckoutDbFile : fsl_preferred_ckout_db_name(); + fsl_cx_err_reset(f); + int fsl_cx_attach_role(fsl_cx * const , const char *, fsl_dbrole_e) + /* defined in cx.c */; + rc = fsl_cx_attach_role(f, dbName, FSL_DBROLE_CKOUT); + if(rc) goto end; + fsl_db * const theDbC = fsl_cx_db_ckout(f); + dbC = fsl_cx_db_for_role(f, FSL_DBROLE_CKOUT); + assert(theDbC != dbC && "Not anymore."); + assert(theDbC == f->dbMain); + assert(!f->error.code); + assert(dbC->name); + assert(dbC->filename); + rc = fsl_ckout_install_schema(f, opt->dbOverwritePolicy); + if(!rc){ + rc = fsl_db_exec(theDbC,"INSERT OR IGNORE INTO " + "%s.vvar (name,value) " + "VALUES('checkout',0)," + "('checkout-hash',null)", + dbC->name); + } + if(rc) rc = fsl_cx_uplift_db_error(f, theDbC); + end: + if(opt->targetDir && *opt->targetDir && cwd->used){ + fsl_chdir(fsl_buffer_cstr(cwd)) + /* Ignoring error because we have no recovery strategy! */; + } + fsl_cx_scratchpad_yield(f, cwd); + if(!rc){ + fsl_db * const dbR = fsl_cx_db_for_role(f, FSL_DBROLE_REPO); + assert(dbR); + assert(dbR->filename && *dbR->filename); + rc = fsl_config_set_text(f, FSL_CONFDB_CKOUT, "repository", + dbR->filename); + } + if(!rc) rc = fsl_update_ckout_dir(f); + fsl_buffer_clear(cwd); + return rc; +} + +int fsl_is_locally_modified(fsl_cx * f, const char * zFilename, + fsl_size_t origSize, + const char * zOrigHash, + fsl_int_t zOrigHashLen, + fsl_fileperm_e origPerm, + int * isModified){ + int rc = 0; + int const hashLen = zOrigHashLen>=0 + ? zOrigHashLen : fsl_is_uuid(zOrigHash); + fsl_buffer * hash = 0; + fsl_buffer * fname = fsl_cx_scratchpad(f); + fsl_fstat * const fst = &f->cache.fstat; + int mod = 0; + if(!fsl_is_uuid_len(hashLen)){ + return fsl_cx_err_set(f, FSL_RC_RANGE, "%s(): invalid hash length " + "%d for file: %s", __func__, hashLen, zFilename); + }else if(!f->ckout.dir){ + return fsl_cx_err_set(f, FSL_RC_NOT_A_CKOUT, + "%s() requires a checkout.", __func__); + } + if(!fsl_is_absolute_path(zFilename)){ + rc = fsl_file_canonical_name2(f->ckout.dir, zFilename, fname, false); + if(rc) goto end; + zFilename = fsl_buffer_cstr(fname); + } + rc = fsl_stat(zFilename, fst, false); + if(0==rc){ + if(origSize!=fst->size){ + mod |= 0x02; + } + if((FSL_FILE_PERM_EXE==origPerm && + FSL_FSTAT_PERM_EXE!=fst->perm) + || (FSL_FILE_PERM_EXE!=origPerm && + FSL_FSTAT_PERM_EXE==fst->perm)){ + mod |= 0x01; + }else if((FSL_FILE_PERM_LINK==origPerm && + FSL_FSTAT_TYPE_LINK!=fst->type) + || (FSL_FILE_PERM_LINK!=origPerm && + FSL_FSTAT_TYPE_LINK==fst->type)){ + mod |= 0x04; + } + if(mod & 0x06) goto end; + /* ^^^^^^^^^^ else we unfortunately need, for behavioral + consistency, to fall through and determine whether the file + contents differ. */ + }else{ + if(FSL_RC_NOT_FOUND==rc){ + rc = 0; + mod = 0x10; + }else{ + rc = fsl_cx_err_set(f, rc, "%s(): stat() failed for file: %s", + __func__, zFilename); + } + goto end; + } + hash = fsl_cx_scratchpad(f); + switch(hashLen){ + case FSL_STRLEN_SHA1: + rc = fsl_sha1sum_filename(zFilename, hash); + break; + case FSL_STRLEN_K256: + rc = fsl_sha3sum_filename(zFilename, hash); + break; + default: + fsl_fatal(FSL_RC_UNSUPPORTED, "This cannot happen. %s()", + __func__); + } + if(rc){ + rc = fsl_cx_err_set(f, rc, "%s: error hashing file: %s", + __func__, zFilename); + }else{ + assert(hashLen==(int)hash->used); + mod |= memcmp(hash->mem, zOrigHash, (size_t)hashLen) + ? 0x02 : 0; + /*MARKER(("%d: %s %s %s\n", *isModified, zOrigHash, + (char const *)hash.mem, zFilename));*/ + } + end: + if(!rc && isModified) *isModified = mod; + fsl_cx_scratchpad_yield(f, fname); + if(hash) fsl_cx_scratchpad_yield(f, hash); + return rc; +} + +/** + Infrastructure for fsl_repo_ckout(), + fsl_ckout_update(), and fsl_ckout_merge(). +*/ +typedef struct { + /** The pre-checkout vfile.vid. 0 if no version was + checked out. */ + fsl_id_t originRid; + fsl_repo_extract_opt const * eOpt; + fsl_ckup_opt const * cOpt; + /* Checkout root. We re-use this when internally converting to + absolute paths. */ + fsl_buffer * tgtDir; + /* Initial length of this->tgtDir, including trailing slash */ + fsl_size_t tgtDirLen; + /* Number of files we've written out so far. Used for adapting + some error reporting. */ + fsl_size_t fileWriteCount; + /* Stores the most recent fsl_cx_confirm() answer for questions + about overwriting/removing modified files. (Exactly which answer + it represents depends on the current phase of processing.) + */ + fsl_confirm_response confirmAnswer; + /* Is-changed vis-a-vis vfile query. */ + fsl_stmt stChanged; + /* Is-same-filename-and-rid-in-vfile query. */ + fsl_stmt stIsInVfile; + /* blob.size for vfile.rid query. */ + fsl_stmt stRidSize; +} RepoExtractCkup; + +static const RepoExtractCkup RepoExtractCkup_empty = { +0/*originRid*/,NULL/*eOpt*/, NULL/*cOpt*/, +NULL/*tgtDir*/, 0/*tgtDirLen*/, +0/*fileWriteCount*/, +fsl_confirm_response_empty_m/*confirmAnswer*/, +fsl_stmt_empty_m/*stChanged*/, +fsl_stmt_empty_m/*stIsInVfile*/, +fsl_stmt_empty_m/*stRidSize*/ +}; + +static const fsl_ckup_state fsl_ckup_state_empty = { +NULL/*xState*/, NULL/*callbackState*/, +FSL_CKUP_FCHANGE_INVALID/*fileChangeType*/, +FSL_CKUP_RM_NOT/*fileRmInfo*/, +0/*mtime*/,0/*size*/, +false/*dryRun*/ +}; + +/** + File modification types reported by + fsl_reco_is_file_modified(). + */ +typedef enum { +// Sentinel value +FSL_RECO_MOD_UNKNOWN, +// Not modified +FSL_RECO_MOD_NO, +// Modified +FSL_RECO_MOD_YES, +// "Unmanaged replaced by managed" +FSL_RECO_MOD_UnReMa +} fsl_ckup_localmod_e; + +/** + Determines whether the file referred to by the given + checkout-root-relative file name, which is assumed to be known to + exist, has been modified. It simply looks to the vfile state, + rather than doing its own filesystem-level comparison. Returns 0 on + success and stores its answer in *modType. Errors must be + considered unrecoverable. +*/ +static int fsl_reco_is_file_modified(fsl_cx *f, fsl_stmt * st, + char const *zName, + fsl_ckup_localmod_e * modType){ + int rc = 0; + if(!st->stmt){ // no prior version + *modType = FSL_RECO_MOD_NO; + return 0; + } + fsl_stmt_reset(st); + rc = fsl_stmt_bind_text(st, 1, zName, -1, false); + if(rc){ + return fsl_cx_uplift_db_error2(f, st->db, rc); + } + rc = fsl_stmt_step(st); + switch(rc){ + case FSL_RC_STEP_DONE: + /* This can happen when navigating from a version in which a + file was SCM-removed/unmanaged, but on disk, to a version + where that file was in SCM. For now we'll mark these as + modified but we need a better way of handling this case, and + maybe a new FSL_CEVENT_xxx ID. */ + *modType = FSL_RECO_MOD_UnReMa; + rc = 0; + break; + case FSL_RC_STEP_ROW: + *modType = fsl_stmt_g_int32(st,0)>0 + ? FSL_RECO_MOD_YES : FSL_RECO_MOD_NO; + rc = 0; + break; + default: + rc = fsl_cx_uplift_db_error2(f, st->db, rc); + break; + } + return rc; +} + +/** + Sets *isInVfile to true if the given combination of filename and + file content RID are in the vfile table, as per + RepoExtractCkup::stIsInVfile, else false. Returns non-0 on + catastrophic failure. +*/ +static int fsl_repo_co_is_in_vfile(fsl_stmt * st, + char const *zFilename, + fsl_id_t fileRid, + bool *isInVfile){ + int rc = 0; + if(st->stmt){ + fsl_stmt_reset(st); + rc = fsl_stmt_bind_text(st, 1, zFilename, -1, false); + if(!rc) rc = fsl_stmt_bind_id(st, 2, fileRid); + if(!rc) *isInVfile = (FSL_RC_STEP_ROW==fsl_stmt_step(st)); + }else{ // no prior version + *isInVfile = false; + } + return rc; +} + +/** + Infrastructure for fsl_repo_ckout(). This is the fsl_repo_extract_f + impl which fsl_repo_extract() calls to give us the pieces we want to + check out. + + When this is run (once for each row of the new checkout version), + the vfile table still holds the state for the previous version, and + we use that to determine whether a file is changed or new. +*/ +static int fsl_repo_extract_f_ckout( fsl_repo_extract_state const * xs ){ + int rc = 0; + fsl_cx * const f = xs->f; + RepoExtractCkup * const rec = (RepoExtractCkup *)xs->callbackState; + const char * zFilename; + fsl_ckup_state coState = fsl_ckup_state_empty; + fsl_time_t mtime = 0; + fsl_fstat fst = fsl_fstat_empty; + fsl_ckup_localmod_e modType = FSL_RECO_MOD_UNKNOWN; + bool loadedContent = false; + fsl_buffer * content = &f->fileContent; + assert(0==content->used + && "Internal Misuse of fsl_cx::fileContent buffer."); + //assert(xs->content); + assert(xs->fCard->uuid && "We shouldn't be getting deletions " + "via delta manifests."); + rc = fsl_buffer_append(rec->tgtDir, xs->fCard->name, -1); + if(rc) return rc; + fsl_buffer_reuse(content); + coState.dryRun = rec->cOpt->dryRun; + coState.fileRmInfo = FSL_CKUP_RM_NOT; + coState.fileChangeType = FSL_CKUP_FCHANGE_INVALID; + zFilename = fsl_buffer_cstr(rec->tgtDir); + rc = fsl_stat(zFilename, &fst, 0); + switch(rc){ + case 0: + /* File exists. If it is modified, as reported by vfile, get + confirmation before overwriting it, otherwise just overwrite + it (or keep it - that's much more efficient). */ + mtime = fst.mtime; + if(rec->confirmAnswer.response!=FSL_CRESPONSE_ALWAYS){ + rc = fsl_reco_is_file_modified(f, &rec->stChanged, + xs->fCard->name, &modType); + if(rc) goto end; + switch(modType){ + case FSL_RECO_MOD_YES: + case FSL_RECO_MOD_UnReMa: + if(rec->confirmAnswer.response!=FSL_CRESPONSE_NEVER){ + fsl_confirm_detail detail = fsl_confirm_detail_empty; + detail.eventId = FSL_RECO_MOD_YES==modType + ? FSL_CEVENT_OVERWRITE_MOD_FILE + : FSL_CEVENT_OVERWRITE_UNMGD_FILE; + detail.filename = xs->fCard->name; + rec->confirmAnswer.response = FSL_CRESPONSE_INVALID; + rc = fsl_cx_confirm(f, &detail, &rec->confirmAnswer); + if(rc) goto end; + } + break; + case FSL_RECO_MOD_NO:{ + /** If vfile says that the content of this exact + combination of filename and file RID is unchanged, we + already have this content. If so, skip rewriting + it. */ + bool isSameFile = false; + rc = fsl_repo_co_is_in_vfile(&rec->stIsInVfile, xs->fCard->name, + xs->fileRid, &isSameFile); + if(rc) goto end; + rec->confirmAnswer.response = isSameFile + ? FSL_CRESPONSE_NO // We already have this content + : FSL_CRESPONSE_YES; // Overwrite it + coState.fileChangeType = isSameFile + ? FSL_CKUP_FCHANGE_NONE + : FSL_CKUP_FCHANGE_UPDATED; + break; + } + default: + fsl_fatal(FSL_RC_UNSUPPORTED,"Internal error: invalid " + "fsl_reco_is_file_modified() response."); + } + } + switch(rec->confirmAnswer.response){ + case FSL_CRESPONSE_NO: + case FSL_CRESPONSE_NEVER: + // Keep existing. + coState.fileChangeType = FSL_CKUP_FCHANGE_NONE; + goto do_callback; + case FSL_CRESPONSE_YES: + case FSL_CRESPONSE_ALWAYS: + // Overwrite it. + coState.fileChangeType = FSL_CKUP_FCHANGE_UPDATED; + break; + case FSL_CRESPONSE_CANCEL: + rc = fsl_cx_err_set(f, FSL_RC_BREAK, + "Checkout operation cancelled by " + "confirmation callback.%s", + rec->fileWriteCount + ? " Filesystem contents may now be " + "in an inconsistent state!" + : ""); + goto end; + default: + rc = fsl_cx_err_set(f, FSL_RC_MISUSE, + "Invalid response from confirmation " + "callback."); + goto end; + } + break; + case FSL_RC_NOT_FOUND: + rc = 0; + coState.fileChangeType = FSL_CKUP_FCHANGE_UPDATED; + // Write it + break; + default: + rc = fsl_cx_err_set(f, rc, "Error %s stat()'ing file: %s", + fsl_rc_cstr(rc), zFilename); + goto end; + } + assert(FSL_CKUP_FCHANGE_INVALID != coState.fileChangeType); + if(coState.dryRun){ + mtime = time(0); + }else{ + if((rc=fsl_mkdir_for_file(zFilename, true))){ + rc = fsl_cx_err_set(f, rc, "mkdir() failed for file: %s", zFilename); + goto end; + } + assert(!xs->content); + rc = fsl_card_F_content(f, xs->fCard, content); + if(rc) goto end; + else if((rc=fsl_buffer_to_filename(content, zFilename))){ + rc = fsl_cx_err_set(f, rc, "Error %s writing to file: %s", + fsl_rc_cstr(rc), zFilename); + goto end; + }else{ + loadedContent = true; + ++rec->fileWriteCount; + mtime = time(0); + } + rc = fsl_file_exec_set(zFilename, + FSL_FILE_PERM_EXE == xs->fCard->perm); + if(rc){ + rc = fsl_cx_err_set(f, rc, "Error %s changing file permissions: %s", + fsl_rc_cstr(rc), xs->fCard->name); + goto end; + } + } + if(rec->cOpt->setMtime){ + rc = fsl_mtime_of_manifest_file(xs->f, xs->checkinRid, + xs->fileRid, &mtime); + if(rc) goto end; + if(!coState.dryRun){ + rc = fsl_file_mtime_set(zFilename, mtime); + if(rc){ + rc = fsl_cx_err_set(f, rc, "Error %s setting mtime of file: %s", + fsl_rc_cstr(rc), zFilename); + goto end; + } + } + } + do_callback: + assert(0==rc); + if(rec->cOpt->callback){ + assert(mtime); + coState.mtime = mtime; + coState.extractState = xs; + coState.callbackState = rec->cOpt->callbackState; + if(loadedContent){ + coState.size = content->used; + }else{ + fsl_stmt_reset(&rec->stRidSize); + fsl_stmt_bind_id(&rec->stRidSize, 1, xs->fileRid); + coState.size = + (FSL_RC_STEP_ROW==fsl_stmt_step(&rec->stRidSize)) + ? (fsl_int_t)fsl_stmt_g_int64(&rec->stRidSize, 0) + : -1; + } + rc = rec->cOpt->callback( &coState ); + } + end: + fsl_buffer_reuse(content); + rec->tgtDir->used = rec->tgtDirLen; + rec->tgtDir->mem[rec->tgtDirLen] = 0; + return rc; +} + +/** + For each file in vfile(vid=rec->originRid) which is not in the + current vfile(vid=rec->cOpt->checkinRid), remove it from disk (or + not, depending on confirmer response). Afterwards, try to remove + any dangling directories left by that removal. + + Returns 0 on success. Ignores any filesystem-level errors during + removal because, frankly, we have no recovery strategy for that + case. + + TODO: do not remove dirs from the 'empty-dirs' config setting. +*/ +static int fsl_repo_ckout_rm_list_fini(fsl_cx * f, + RepoExtractCkup * rec){ + int rc; + fsl_db * db = fsl_cx_db_ckout(f); + fsl_stmt q = fsl_stmt_empty; + fsl_buffer * absPath = fsl_cx_scratchpad(f); + fsl_size_t const ckdirLen = f->ckout.dirLen; + char const *zAbs; + int rmCounter = 0; + fsl_ckup_opt const * cOpt = rec->cOpt; + fsl_ckup_state cuState = fsl_ckup_state_empty; + fsl_repo_extract_state rxState = fsl_repo_extract_state_empty; + fsl_card_F fCard = fsl_card_F_empty; + + assert(db); + rc = fsl_buffer_append(absPath, f->ckout.dir, + (fsl_int_t)f->ckout.dirLen); + if(rc) goto end; + /* Select files which were in the previous version + (rec->originRid) but are not in the newly co'd version + (cOpt->checkinRid). */ + rc = fsl_db_prepare(db, &q, + "SELECT " + /*0*/"v.rid frid," + /*1*/"v.pathname fn," + /*2*/"b.uuid," + /*3*/"v.isexe," + /*4*/"v.islink," + /*5*/"v.chnged, " + /*6*/"b.size " + "FROM vfile v, blob b " + "WHERE v.vid=%" FSL_ID_T_PFMT " " + "AND v.rid=b.rid " + "AND fn NOT IN " + "(SELECT pathname FROM vfile " + " WHERE vid=%" FSL_ID_T_PFMT + ") " + "ORDER BY fn %s /*%s()*/", + rec->originRid, + cOpt->checkinRid + /*new checkout version resp. update target + version*/, + fsl_cx_filename_collation(f), + __func__); + if(rc) goto end; + + rec->confirmAnswer.response = FSL_CRESPONSE_INVALID; + cuState.mtime = 0; + cuState.size = -1; + cuState.callbackState = cOpt->callbackState; + cuState.extractState = &rxState; + cuState.dryRun = cOpt->dryRun; + cuState.fileChangeType = FSL_CKUP_FCHANGE_RM; + rxState.f = f; + rxState.fCard = &fCard; + rxState.checkinRid = cOpt->checkinRid; + while(FSL_RC_STEP_ROW==(rc = fsl_stmt_step(&q))){ + /** + Each row is one file listed in vfile (the old checkout + version) which is not in vfile (the new checkout). + */ + fsl_size_t nFn = 0; + fsl_size_t hashLen = 0; + char const * fn = fsl_stmt_g_text(&q, 1, &nFn); + char const * hash = fsl_stmt_g_text(&q, 2, &hashLen); + bool const isChanged = fsl_stmt_g_int32(&q, 5)!=0; + int64_t const fSize = fsl_stmt_g_int64(&q, 6); + if(FSL_CRESPONSE_ALWAYS!=rec->confirmAnswer.response){ + /** + If the user has previously responded to + FSL_CEVENT_RM_MOD_UNMGD_FILE, keep that response, else + ask again if the file was flagged as changed in the + vfile table before all of this started. + */ + if(isChanged){ + // Modified: ask user unless they've already answered NEVER. + if(FSL_CRESPONSE_NEVER!=rec->confirmAnswer.response){ + fsl_confirm_detail detail = fsl_confirm_detail_empty; + detail.eventId = FSL_CEVENT_RM_MOD_UNMGD_FILE; + detail.filename = fn; + rec->confirmAnswer.response = FSL_CRESPONSE_INVALID; + rc = fsl_cx_confirm(f, &detail, &rec->confirmAnswer); + if(rc) goto end; + } + }else{ + // Not modified. Nuke it. + rec->confirmAnswer.response = FSL_CRESPONSE_YES; + } + } + absPath->used = ckdirLen; + rc = fsl_buffer_append(absPath, fn, nFn); + if(rc) break; + zAbs = fsl_buffer_cstr(absPath); + /* Ignore deletion errors. We cannot roll back previous deletions, + so failing here, which would roll back the transaction, could + leave the checkout in a weird state, potentially with some + files missing and others not. */ + switch(rec->confirmAnswer.response){ + case FSL_CRESPONSE_YES: + case FSL_CRESPONSE_ALWAYS: + //MARKER(("Unlinking: %s\n",zAbs)); + if(!cOpt->dryRun && 0==fsl_file_unlink(zAbs)){ + ++rmCounter; + } + cuState.fileRmInfo = FSL_CKUP_RM; + break; + case FSL_CRESPONSE_NO: + case FSL_CRESPONSE_NEVER: + //assert(FSL_RECO_MOD_YES==modType); + //MARKER(("NOT removing locally-modified file: %s\n", zN)); + cuState.fileRmInfo = FSL_CKUP_RM_KEPT; + break; + case FSL_CRESPONSE_CANCEL: + rc = fsl_cx_err_set(f, FSL_RC_BREAK, + "Checkout operation cancelled by " + "confirmation callback. " + "Filesystem contents may now be " + "in an inconsistent state!"); + goto end; + default: + fsl_fatal(FSL_RC_UNSUPPORTED,"Internal error: invalid " + "fsl_cx_confirm() response #%d.", + rec->confirmAnswer.response); + break; + } + if(!cOpt->callback) continue; + /* Now report the deletion to the callback... */ + fsl_id_t const frid = fsl_stmt_g_id(&q, 0); + const bool isExe = 0!=fsl_stmt_g_int32(&q, 3); + const bool isLink = 0!=fsl_stmt_g_int32(&q, 4); + cuState.size = (FSL_CKUP_RM==cuState.fileRmInfo) ? -1 : fSize; + rxState.fileRid = frid; + fCard = fsl_card_F_empty; + fCard.name = (char *)fn; + fCard.uuid = (char *)hash; + fCard.perm = isExe ? FSL_FILE_PERM_EXE : + (isLink ? FSL_FILE_PERM_LINK : FSL_FILE_PERM_REGULAR); + rc = cOpt->callback( &cuState ); + if(rc) goto end; + } + if(FSL_RC_STEP_DONE==rc) rc = 0; + else goto end; + if(rmCounter>0){ + /* Clean up any empty directories left over by removal of + files... */ + assert(!cOpt->dryRun); + fsl_stmt_finalize(&q); + /* Select dirs which were in the previous version + (rec->originRid) but are not in the newly co'd version + (cOpt->checkinRid). Any of these may _potentially_ + be empty now. This query could be improved to filter + out more in advance. */ + rc = fsl_db_prepare(db, &q, + "SELECT DISTINCT(fsl_dirpart(pathname,0)) dir " + "FROM vfile " + "WHERE vid=%" FSL_ID_T_PFMT " " + "AND pathname NOT IN " + "(SELECT pathname FROM vfile " + "WHERE vid=%" FSL_ID_T_PFMT ") " + "AND dir IS NOT NULL " + "ORDER BY length(dir) DESC /*%s()*/", + /*get deepest dirs first*/ + rec->originRid, cOpt->checkinRid, + __func__); + if(rc) goto end; + while(FSL_RC_STEP_ROW==(rc = fsl_stmt_step(&q))){ + fsl_size_t nFn = 0; + char const * fn = fsl_stmt_g_text(&q, 0, &nFn); + absPath->used = ckdirLen; + rc = fsl_buffer_append(absPath, fn, nFn); + if(rc) break; + fsl_ckout_rm_empty_dirs(f, absPath) + /* To see this in action, use (f-co tip) to check out the tip of + a repo, then use (f-co rid:1) to back up to the initial empty + checkin. It "should" leave you with a directory devoid of + anything but .fslckout and any non-SCM'd content. + */; + } + if(FSL_RC_STEP_DONE==rc) rc = 0; + } + end: + fsl_stmt_finalize(&q); + fsl_cx_scratchpad_yield(f, absPath); + return fsl_cx_uplift_db_error2(f, db, rc); +} + +int fsl_repo_ckout(fsl_cx * f, fsl_ckup_opt const * cOpt){ + int rc = 0; + fsl_id_t const prevRid = f->ckout.rid; + fsl_db * const dbR = fsl_needs_repo(f); + RepoExtractCkup rec = RepoExtractCkup_empty; + fsl_confirmer oldConfirm = fsl_confirmer_empty; + if(!dbR) return f->error.code; + else if(!fsl_needs_ckout(f)) return f->error.code; + rc = fsl_cx_transaction_begin(f); + if(rc) return rc; + rec.tgtDir = fsl_cx_scratchpad(f); + if(cOpt->confirmer.callback){ + fsl_cx_confirmer(f, &cOpt->confirmer, &oldConfirm); + } + //MARKER(("ckout.rid=%d\n",(int)prevRid)); + if(prevRid>=0 && cOpt->scanForChanges){ + /* We need to ensure this state is current in order to determine + whether a given file is locally modified vis-a-vis the + pre-extract checkout state. */ + rc = fsl_vfile_changes_scan(f, prevRid, 0); + if(rc) goto end; + } + if(0){ + fsl_db_each(dbR,fsl_stmt_each_f_dump, NULL, + "SELECT * FROM vfile ORDER BY pathname"); + } + assert(f->ckout.dirLen); + fsl_repo_extract_opt eOpt = fsl_repo_extract_opt_empty; + rc = fsl_buffer_append(rec.tgtDir, f->ckout.dir, + (fsl_int_t)f->ckout.dirLen); + if(rc) goto end; + if(prevRid){ + rc = fsl_db_prepare(dbR, &rec.stChanged, + "SELECT chnged FROM vfile " + "WHERE vid=%" FSL_ID_T_PFMT + " AND pathname=? %s", + prevRid, + fsl_cx_filename_collation(f)); + } + if(!rc && prevRid){ + /* Optimization: before we load content for a blob and write it to + a file, check this query for whether we already have the same + name/rid combination in vfile, and skip loading/writing the + content if we do. */ + rc = fsl_db_prepare(dbR, &rec.stIsInVfile, + "SELECT 1 FROM vfile " + "WHERE vid=%" FSL_ID_T_PFMT + " AND pathname=? AND rid=? %s", + prevRid, fsl_cx_filename_collation(f)); + } + if(!rc){ + /* Files for which we don't load content (see rec.stIsInVfile) + still have a size we need to report via fsl_ckup_state, + and we fetch that with this query. */ + rc = fsl_db_prepare(dbR, &rec.stRidSize, + "SELECT size FROM blob WHERE rid=?"); + } + if(rc){ + rc = fsl_cx_uplift_db_error2(f, dbR, rc); + goto end; + } + rec.originRid = prevRid; + rec.tgtDirLen = f->ckout.dirLen; + eOpt.checkinRid = cOpt->checkinRid; + eOpt.extractContent = false; + eOpt.callbackState = &rec; + eOpt.callback = fsl_repo_extract_f_ckout; + rec.eOpt = &eOpt; + rec.cOpt = cOpt; + rc = fsl_repo_extract(f, &eOpt); + if(!rc){ + /* + We need to call fsl_vfile_load(f, cOpt->vid) to + populate vfile but we also need to call + fsl_vfile_changes_scan(f, cOpt->vid, 0) to set the vfile.mtime + fields. The latter calls the former, so... + */ + rc = fsl_vfile_changes_scan(f, cOpt->checkinRid, + FSL_VFILE_CKSIG_WRITE_CKOUT_VERSION + | + (prevRid==0 + ? 0 : FSL_VFILE_CKSIG_KEEP_OTHERS) + | + (cOpt->setMtime + ? 0 : FSL_VFILE_CKSIG_SETMTIME) + /* Note that mtimes were set during + extraction if cOpt->setMtime is + true. */); + if(rc) goto end; + assert(f->ckout.rid==cOpt->checkinRid); + assert(f->ckout.rid ? !!f->ckout.uuid : 1); + } + if(!rc && prevRid!=0){ + rc = fsl_repo_ckout_rm_list_fini(f, &rec); + if(rc) goto end; + } + rc = fsl_ckout_manifest_write(f, -1, -1, -1, NULL); + + end: + if(!rc){ + rc = fsl_vfile_unload_except(f, cOpt->checkinRid); + if(!rc) rc = fsl_ckout_clear_merge_state(f); + } + /* + TODO: if "repo-cksum" config db setting is set, confirm R-card of + cOpt->checkinRid against on-disk contents. + */ + if(cOpt->confirmer.callback){ + fsl_cx_confirmer(f, &oldConfirm, NULL); + } + fsl_stmt_finalize(&rec.stChanged); + fsl_stmt_finalize(&rec.stIsInVfile); + fsl_stmt_finalize(&rec.stRidSize); + fsl_cx_scratchpad_yield(f, rec.tgtDir); + int const rc2 = fsl_cx_transaction_end(f, rc || cOpt->dryRun); + return rc ? rc : rc2; +} + +int fsl_ckout_update(fsl_cx * f, fsl_ckup_opt const *cuOpt){ + fsl_db * const dbR = fsl_needs_repo(f); + fsl_db * const dbC = dbR ? fsl_needs_ckout(f) : 0; + if(!dbR) return FSL_RC_NOT_A_REPO; + else if(!dbC) return FSL_RC_NOT_A_CKOUT; + int rc = 0, rc2 = 0; + char const * collation = fsl_cx_filename_collation(f); + fsl_id_t const ckRid = f->ckout.rid /* current version */; + fsl_id_t const tid = cuOpt->checkinRid /* target version */; + fsl_stmt q = fsl_stmt_empty; + fsl_stmt mtimeXfer = fsl_stmt_empty; + fsl_stmt mtimeGet = fsl_stmt_empty; + fsl_stmt mtimeSet = fsl_stmt_empty; + fsl_buffer * bFullPath = 0; + fsl_buffer * bFullNewPath = 0; + fsl_buffer * bFileUuid = 0; + fsl_repo_extract_opt eOpt = fsl_repo_extract_opt_empty + /* We won't actually use fsl_repo_extract() here because it's a + poor fit for the update selection algorithm, but in order to + consolidate some code between the ckout/update cases we need to + behave as if we were using it. */; + fsl_repo_extract_state xState = fsl_repo_extract_state_empty; + fsl_card_F fCard = fsl_card_F_empty; + fsl_ckup_state uState = fsl_ckup_state_empty; + RepoExtractCkup rec = RepoExtractCkup_empty; + enum { MergeBufCount = 4 }; + fsl_buffer bufMerge[MergeBufCount] = { + fsl_buffer_empty_m/* pivot: ridv */, + fsl_buffer_empty_m/* local file to merge into */, + fsl_buffer_empty_m/* update-to: ridt */, + fsl_buffer_empty_m/* merged copy */ + }; + + rc = fsl_db_transaction_begin(dbC); + if(rc) return fsl_cx_uplift_db_error2(f, dbC, rc); + if(cuOpt->scanForChanges){ + rc = fsl_vfile_changes_scan(f, ckRid, FSL_VFILE_CKSIG_ENOTFILE); + if(rc) goto end; + } + if(tid != ckRid){ + uint32_t missingCount = 0; + rc = fsl_vfile_load(f, tid, false, + &missingCount); + if(rc) goto end; + else if(missingCount/* && !forceMissing*/){ + rc = fsl_cx_err_set(f, FSL_RC_PHANTOM, + "Unable to update due to missing content in " + "%"PRIu32" blob(s).", missingCount); + goto end; + } + } + /* + ** The record.fn field is used to match files against each other. The + ** FV table contains one row for each each unique filename in + ** in the current checkout, the pivot, and the version being merged. + */ + rc = fsl_db_exec_multi(dbC, + "DROP TABLE IF EXISTS fv;" + "CREATE TEMP TABLE fv(" + " fn TEXT %s PRIMARY KEY," /* The filename relative to root */ + " idv INTEGER," /* VFILE entry for current version */ + " idt INTEGER," /* VFILE entry for target version */ + " chnged BOOLEAN," /* True if current version has been edited */ + " islinkv BOOLEAN," /* True if current file is a link */ + " islinkt BOOLEAN," /* True if target file is a link */ + " ridv INTEGER," /* Record ID for current version */ + " ridt INTEGER," /* Record ID for target */ + " isexe BOOLEAN," /* Does target have execute permission? */ + " deleted BOOLEAN DEFAULT 0,"/* File marked by "rm" to become unmanaged */ + " fnt TEXT %s" /* Filename of same file on target version */ + ") /*%s()*/;", + collation, collation, __func__ ); + if(rc) goto dberr; + /* Add files found in the current version + */ + rc = fsl_db_exec_multi(dbC, + "INSERT OR IGNORE INTO fv(" + "fn,fnt,idv,idt,ridv," + "ridt,isexe,chnged,deleted" + ") SELECT pathname, pathname, id, 0, rid, 0, " + "isexe, chnged, deleted " + "FROM vfile WHERE vid=%" FSL_ID_T_PFMT + "/*%s()*/", + ckRid, __func__ + ); + if(rc) goto dberr; + + /* Compute file name changes on V->T. Record name changes in files that + ** have changed locally. + */ + if( ckRid ){ + uint32_t nChng = 0; + fsl_id_t * aChng = 0; + rc = fsl__find_filename_changes(f, ckRid, tid, + true, &nChng, &aChng); + if(rc){ + assert(!aChng); + assert(!nChng); + goto end; + } + if( nChng ){ + for(uint32_t i=0; icallback){ + /* Queries we need only if we need to collect info for a + callback... */ + rc = fsl_db_prepare(dbC, &mtimeGet, + "SELECT mtime FROM vfile WHERE id=?1"/*idt*/); + if(rc) goto dberr; + rc = fsl_db_prepare(dbC, &mtimeSet, + "UPDATE vfile SET mtime=?2 WHERE id=?1"/*idt*/); + if(rc) goto dberr; + /* Files for which we don't load content still have a size we need + to report via fsl_ckup_state, and we fetch that with this + query. */ + rc = fsl_db_prepare(dbR, &rec.stRidSize, + "SELECT size FROM blob WHERE rid=?"); + if(rc) goto dberr; + } + + xState.f = f; + xState.fCard = &fCard; + xState.checkinRid = eOpt.checkinRid = tid; + xState.count.fileCount = + (uint32_t)fsl_db_g_int32(dbC, 0, "SELECT COUNT(*) FROM vfile " + "WHERE vid=%" FSL_ID_T_PFMT, + tid); + uState.extractState = &xState; + uState.callbackState = cuOpt->callbackState; + uState.dryRun = cuOpt->dryRun; + uState.fileRmInfo = FSL_CKUP_RM_NOT; + rec.originRid = ckRid; + rec.eOpt = &eOpt; + rec.cOpt = cuOpt; + rec.tgtDir = fsl_cx_scratchpad(f); + rec.tgtDirLen = f->ckout.dirLen; + rc = fsl_buffer_append(rec.tgtDir, f->ckout.dir, + (fsl_int_t)f->ckout.dirLen); + if(rc) goto end; + + /** + Missing features from fossil we still need for this include, + but are not limited to: + + - file_unsafe_in_tree_path() (done, untested) + - file_nondir_objects_on_path() (done, untested) + - symlink_create() (done, untested) + - ... + */ + bFullPath = fsl_cx_scratchpad(f); + bFullNewPath = fsl_cx_scratchpad(f); + bFileUuid = fsl_cx_scratchpad(f); + rc = fsl_buffer_append(bFullPath, f->ckout.dir, + (fsl_int_t)f->ckout.dirLen); + if(rc) goto end; + rc = fsl_buffer_append(bFullNewPath, f->ckout.dir, + (fsl_int_t)f->ckout.dirLen); + if(rc) goto end; + unsigned int nConflict = 0; + while( FSL_RC_STEP_ROW==fsl_stmt_step(&q) ){ + const char *zName = fsl_stmt_g_text(&q, 0, NULL) + /* The filename from root */; + fsl_id_t const idv = fsl_stmt_g_id(&q, 1) + /* VFILE entry for current */; + fsl_id_t const ridv = fsl_stmt_g_id(&q, 2) + /* RecordID for current */; + fsl_id_t const idt = fsl_stmt_g_id(&q, 3) + /* VFILE entry for target */; + fsl_id_t const ridt = fsl_stmt_g_id(&q, 4) + /* RecordID for target */; + int const chnged = fsl_stmt_g_int32(&q, 5) + /* Current is edited */; + const char *zNewName = fsl_stmt_g_text(&q,6, NULL) + /* New filename */; + int const isexe = fsl_stmt_g_int32(&q, 7) + /* EXE perm for new file */; + int const islinkv = fsl_stmt_g_int32(&q, 8) + /* Is current file is a link */; + int const islinkt = fsl_stmt_g_int32(&q, 9) + /* Is target file is a link */; + int const deleted = fsl_stmt_g_int32(&q, 10) + /* Marked for deletion */; + char const *zFullPath /* Full pathname of the file */; + char const *zFullNewPath /* Full pathname of dest */; + bool const nameChng = !!fsl_strcmp(zName, zNewName) + /* True if the name changed */; + int wasWritten = 0 + /* 1=perms written to disk, 2=content written */; + fsl_fstat fst = fsl_fstat_empty; + if(chnged || isexe || islinkv || islinkt){/*unused*/} + + bFullPath->used = bFullNewPath->used = f->ckout.dirLen; + rc = fsl_buffer_appendf(bFullPath, zName, -1); + if(!rc) rc = fsl_buffer_appendf(bFullNewPath, zNewName, -1); + if(rc) goto end; + zFullPath = fsl_buffer_cstr(bFullPath); + zFullNewPath = fsl_buffer_cstr(bFullNewPath); + uState.mtime = 0; + uState.fileChangeType = FSL_CKUP_FCHANGE_INVALID; + uState.fileRmInfo = FSL_CKUP_RM_NOT; + ++xState.count.fileNumber; + //MARKER(("#%03u/%03d %s\n", xState.count.fileNumber, xState.count.fileCount, zName)); + if( deleted ){ + /* Carry over pending file deletions from the current version + into the target version. If the file was already deleted in + the target version, that will be picked up by the file-deletion + loop later on. */ + uState.fileChangeType = FSL_CKUP_FCHANGE_RM_PROPAGATED; + rc = fsl_db_exec(dbC, "UPDATE vfile SET deleted=1 " + "WHERE id=%" FSL_ID_T_PFMT" /*%s()*/", + idt, __func__); + if(rc) goto dberr; + } + if( idv>0 && ridv==0 && idt>0 && ridt>0 ){ + /* Conflict. This file has been added to the current checkout + ** but also exists in the target checkout. Use the current version. + */ + uState.fileChangeType = FSL_CKUP_FCHANGE_CONFLICT_ADDED; + //fossil_print("CONFLICT %s\n", zName); + nConflict++; + }else if( idt>0 && idv==0 ){ + /* File added in the target. */ + if( fsl_is_file_or_link(zFullPath) ){ + //fossil_print("ADD %s - overwrites an unmanaged file\n", zName); + uState.fileChangeType = + FSL_CKUP_FCHANGE_CONFLICT_ADDED_UNMANAGED; + //nOverwrite++; + /* TODO/FIXME: if the files have the same content, treat this + as FSL_CKUP_FCHANGE_ADDED. If they don't, use confirmer to + ask the user what to do. */ + }else{ + //fsl_outputf(f, "ADD %s\n", zName); + uState.fileChangeType = FSL_CKUP_FCHANGE_ADDED; + } + //if( !dryRunFlag && !internalUpdate ) undo_save(zName); + if( !cuOpt->dryRun ){ + rc = fsl_vfile_to_ckout(f, idt, &wasWritten); + if(rc) goto end; + } + }else if( idt>0 && idv>0 && ridt!=ridv && (chnged==0 || deleted) ){ + /* The file is unedited. Change it to the target version */ + if( deleted ){ + //fossil_print("UPDATE %s - change to unmanaged file\n", zName); + uState.fileChangeType = FSL_CKUP_FCHANGE_RM; + }else{ + //fossil_print("UPDATE %s\n", zName); + uState.fileChangeType = FSL_CKUP_FCHANGE_UPDATED; + } + if( !cuOpt->dryRun ){ + rc = fsl_vfile_to_ckout(f, idt, &wasWritten); + if(rc) goto end; + } + }else if( idt>0 && idv>0 && !deleted && + 0!=fsl_stat(zFullPath, NULL, false) ){ + /* The file is missing from the local check-out. Restore it to + ** the version that appears in the target. */ + uState.fileChangeType = FSL_CKUP_FCHANGE_UPDATED; + if( !cuOpt->dryRun ){ + rc = fsl_vfile_to_ckout(f, idt, &wasWritten); + if(rc) goto end; + } + }else if( idt==0 && idv>0 ){ + /* Is in the current version but not in the target. */ + if( ridv==0 ){ + /* Added in current checkout. Continue to hold the file as + ** as an addition */ + uState.fileChangeType = FSL_CKUP_FCHANGE_ADD_PROPAGATED; + rc = fsl_db_exec(dbC, "UPDATE vfile SET vid=%" FSL_ID_T_PFMT + " WHERE id=%" FSL_ID_T_PFMT " /*%s()*/", + tid, idv, __func__); + if(rc) goto dberr; + }else if( chnged ){ + /* Edited locally but deleted from the target. Do not track the + ** file but keep the edited version around. */ + uState.fileChangeType = FSL_CKUP_FCHANGE_CONFLICT_RM; + ++nConflict; + uState.fileRmInfo = FSL_CKUP_RM_KEPT; + /* Delete idv from vfile so that the post-processing rm + loop will not delete this file. */ + rc = fsl_db_exec(dbC, "DELETE FROM vfile WHERE id=%" + FSL_ID_T_PFMT " /*%s()*/", + idv, __func__); + if(rc) goto dberr; + + }else{ + uState.fileChangeType = FSL_CKUP_FCHANGE_RM; + if( !cuOpt->dryRun ){ + fsl_file_unlink(zFullPath)/*ignore errors*/; + /* At this point fossil(1) adds each directory to the + dir_to_delete table. We can probably use the same + infrastructure which ckout uses, though. One + hiccup there is that our infrastructure does not + handle the locally-modified-removed case from the + block above this one. */ + } + } + }else if( idt>0 && idv>0 && ridt!=ridv && chnged ){ + /* Merge the changes in the current tree into the target version */ + if( islinkv || islinkt ){ + uState.fileChangeType = FSL_CKUP_FCHANGE_CONFLICT_SYMLINK; + ++nConflict; + }else{ + unsigned int conflictCount = 0; + for(int i = 0; i < MergeBufCount; ++i){ + fsl_buffer_reuse(&bufMerge[i]); + } + rc = fsl_content_get(f, ridv, &bufMerge[0]); + if(!rc) rc = fsl_content_get(f, ridt, &bufMerge[2]); + if(!rc){ + rc = fsl_buffer_fill_from_filename(&bufMerge[1], zFullPath); + } + if(rc) goto end; + rc = fsl_buffer_merge3(&bufMerge[0], &bufMerge[1], + &bufMerge[2], &bufMerge[3], + &conflictCount); + if(FSL_RC_TYPE==rc){ + /* Binary content: we can't merge this, so use target + version. */ + rc = 0; + uState.fileChangeType = FSL_CKUP_FCHANGE_UPDATED_BINARY; + if( !cuOpt->dryRun ){ + rc = fsl_buffer_to_filename(&bufMerge[2], zFullNewPath); + if(!rc) fsl_file_exec_set(zFullNewPath, !!isexe); + } + }else if(!rc){ + if( !cuOpt->dryRun ){ + rc = fsl_buffer_to_filename(&bufMerge[3], zFullNewPath); + if(!rc) fsl_file_exec_set(zFullNewPath, !!isexe); + } + uState.fileChangeType = conflictCount + ? FSL_CKUP_FCHANGE_CONFLICT_MERGED + : FSL_CKUP_FCHANGE_MERGED; + if(conflictCount) ++nConflict; + } + if(rc) goto end; + } + if( nameChng && !cuOpt->dryRun ){ + fsl_file_unlink(zFullPath); + } + }else{ + if( chnged ){ + if( !deleted ){ + uState.fileChangeType = FSL_CKUP_FCHANGE_EDITED; + }else{ + assert(FSL_CKUP_FCHANGE_RM_PROPAGATED==uState.fileChangeType); + } + }else{ + uState.fileChangeType = FSL_CKUP_FCHANGE_NONE; + rc = fsl_stmt_bind_step(&mtimeXfer, "RR", idv, idt); + if(rc) goto dberr; + } + } + if(wasWritten && cuOpt->setMtime){ + if(0==fsl_mtime_of_manifest_file(f, tid, ridt, &uState.mtime)){ + fsl_file_mtime_set(zFullNewPath, uState.mtime); + rc = fsl_stmt_bind_step(&mtimeSet, "RI", idt, uState.mtime); + if(rc) goto dberr; + } + } + assert(FSL_CKUP_FCHANGE_INVALID != uState.fileChangeType); + assert(!rc); + if(cuOpt->callback + && (FSL_CKUP_FCHANGE_RM != uState.fileChangeType) + /* removals are reported separately in the file + deletion phase */){ + if(FSL_CKUP_FCHANGE_ADD_PROPAGATED==uState.fileChangeType){ + /* This file is not yet in SCM, so its size is not in + the db. */ + if(0==fsl_stat(zFullNewPath, &fst, false)){ + uState.size = (fsl_int_t)fst.size; + uState.mtime = fst.mtime; + }else{ + uState.size = -1; + } + }else{ + /* If we have the record's size in the db, use that. */ + fsl_stmt_bind_id(&rec.stRidSize, 1, ridt); + if(FSL_RC_STEP_ROW==fsl_stmt_step(&rec.stRidSize)){ + uState.size = fsl_stmt_g_int32(&rec.stRidSize, 0); + }else{ + uState.size = -1; + } + fsl_stmt_reset(&rec.stRidSize); + } + if(!uState.mtime){ + fsl_stmt_bind_id(&mtimeGet, 1, idt); + if(FSL_RC_STEP_ROW==fsl_stmt_step(&mtimeGet)){ + uState.mtime = fsl_stmt_g_id(&mtimeGet, 0); + } + if(0==uState.mtime && 0==fsl_stat(zFullNewPath, &fst, false)){ + uState.mtime = fst.mtime; + } + fsl_stmt_reset(&mtimeGet); + } + xState.fileRid = ridt; + fCard.name = (char *)zNewName; + fCard.priorName = (char *)(nameChng ? zName : NULL); + fCard.perm = islinkt ? FSL_FILE_PERM_LINK + : (isexe ? FSL_FILE_PERM_EXE : FSL_FILE_PERM_REGULAR); + if(ridt){ + rc = fsl_rid_to_uuid2(f, ridt, bFileUuid); + if(rc) goto end; + fCard.uuid = fsl_buffer_str(bFileUuid); + }else{ + //MARKER(("ridt=%d uState.fileChangeType=%d name=%s\n", + // ridt, uState.fileChangeType, fCard.name)); + assert(FSL_CKUP_FCHANGE_CONFLICT_RM==uState.fileChangeType + || FSL_CKUP_FCHANGE_ADD_PROPAGATED==uState.fileChangeType + || FSL_CKUP_FCHANGE_EDITED==uState.fileChangeType + ); + fCard.uuid = 0; + } + rc = cuOpt->callback( &uState ); + if(rc) goto end; + uState.mtime = 0; + } + }/*fsl_stmt_step(&q)*/ + fsl_stmt_finalize(&q); + if(nConflict){/*unused*/} + /* + At this point, fossil(1) does: + + ensure_empty_dirs_created(1); + checkout_set_all_exe(); + */ + assert(!rc); + rc = fsl_repo_ckout_rm_list_fini(f, &rec); + if(!rc){ + rc = fsl_vfile_unload_except(f, tid); + } + if(!rc){ + rc = fsl_ckout_version_write(f, tid, 0); + } + + end: + /* clang bug? If we declare rc2 here, it says "expression expected". + Moving the decl to the top resolves it. Wha? */ + if(rec.tgtDir) fsl_cx_scratchpad_yield(f, rec.tgtDir); + if(bFullPath) fsl_cx_scratchpad_yield(f, bFullPath); + if(bFullNewPath) fsl_cx_scratchpad_yield(f, bFullNewPath); + if(bFileUuid) fsl_cx_scratchpad_yield(f, bFileUuid); + for(int i = 0; i < MergeBufCount; ++i){ + fsl_buffer_clear(&bufMerge[i]); + } + fsl_stmt_finalize(&rec.stRidSize); + fsl_stmt_finalize(&rec.stChanged); + fsl_stmt_finalize(&mtimeGet); + fsl_stmt_finalize(&mtimeSet); + fsl_stmt_finalize(&q); + fsl_stmt_finalize(&mtimeXfer); + fsl_db_exec(dbC, "DROP TABLE fv /*%s()*/", __func__); + rc2 = fsl_db_transaction_end(dbC, !!rc); + return rc ? rc : rc2; + dberr: + assert(rc); + rc = fsl_cx_uplift_db_error2(f, dbC, rc); + goto end; +} + + +/** Helper for generating a list of ambiguous leaf UUIDs. */ +struct AmbiguousLeavesOutput { + int count; + int rc; + fsl_buffer * buffer; +}; +typedef struct AmbiguousLeavesOutput AmbiguousLeavesOutput; +static const AmbiguousLeavesOutput AmbiguousLeavesOutput_empty = + {0, 0, NULL}; + +static int fsl_stmt_each_f_ambiguous_leaves( fsl_stmt * stmt, void * state ){ + AmbiguousLeavesOutput * alo = (AmbiguousLeavesOutput*)state; + if(alo->count++){ + alo->rc = fsl_buffer_append(alo->buffer, ", ", 2); + } + if(!alo->rc){ + fsl_size_t n = 0; + char const * uuid = fsl_stmt_g_text(stmt, 0, &n); + assert(n==FSL_STRLEN_SHA1 || n==FSL_STRLEN_K256); + alo->rc = fsl_buffer_append(alo->buffer, uuid, 16); + } + return alo->rc; +} + +int fsl_ckout_calc_update_version(fsl_cx * f, fsl_id_t * outRid){ + fsl_db * const dbRepo = fsl_needs_repo(f); + if(!dbRepo) return FSL_RC_NOT_A_REPO; + else if(!fsl_needs_ckout(f)) return FSL_RC_NOT_A_CKOUT; + int rc = 0; + fsl_id_t tgtRid = 0; + fsl_leaves_compute_e leafMode = FSL_LEAVES_COMPUTE_OPEN; + fsl_id_t const ckRid = f->ckout.rid; + rc = fsl_leaves_compute(f, ckRid, leafMode); + if(rc) goto end; + if( !fsl_leaves_computed_has(f) ){ + leafMode = FSL_LEAVES_COMPUTE_ALL; + rc = fsl_leaves_compute(f, ckRid, leafMode); + if(rc) goto end; + } + /* Delete [leaves] entries from any branches other than + ckRid's... */ + rc = fsl_db_exec_multi(dbRepo, + "DELETE FROM leaves WHERE rid NOT IN" + " (SELECT leaves.rid FROM leaves, tagxref" + " WHERE leaves.rid=tagxref.rid AND tagxref.tagid=%d" + " AND tagxref.value==(SELECT value FROM tagxref" + " WHERE tagid=%d AND rid=%" + FSL_ID_T_PFMT "))", + FSL_TAGID_BRANCH, FSL_TAGID_BRANCH, ckRid + ); + if(rc) goto end; + else if( fsl_leaves_computed_count(f)>1 ){ + AmbiguousLeavesOutput alo = AmbiguousLeavesOutput_empty; + alo.buffer = fsl_cx_scratchpad(f); + rc = fsl_buffer_append(alo.buffer, + "Multiple viable descendants found: ", -1); + if(!rc){ + fsl_stmt q = fsl_stmt_empty; + rc = fsl_db_prepare(dbRepo, &q, "SELECT uuid FROM blob " + "WHERE rid IN leaves ORDER BY uuid"); + if(!rc){ + rc = fsl_stmt_each(&q, fsl_stmt_each_f_ambiguous_leaves, &alo); + } + fsl_stmt_finalize(&q); + } + if(!rc){ + rc = fsl_cx_err_set(f, FSL_RC_AMBIGUOUS, "%b", alo.buffer); + } + fsl_cx_scratchpad_yield(f, alo.buffer); + } + end: + if(!rc){ + tgtRid = fsl_leaves_computed_latest(f); + *outRid = tgtRid; + fsl_leaves_computed_cleanup(f) + /* We might want to keep [leaves] around for the case where we + return FSL_RC_AMBIGUOUS, to give the client a way to access + that list in its raw form. Higher-level code could join that + with the event table to give the user more context. */; + } + return rc; +} + +void fsl_ckout_manifest_setting(fsl_cx *f, int *m){ + if(!m){ + f->cache.manifestSetting = -1; + return; + }else if(f->cache.manifestSetting>=0){ + *m = f->cache.manifestSetting; + return; + } + char * str = fsl_config_get_text(f, FSL_CONFDB_VERSIONABLE, + "manifest", NULL); + if(!str){ + str = fsl_config_get_text(f, FSL_CONFDB_REPO, + "manifest", NULL); + } + *m = 0; + if(str){ + char const * z = str; + if('1'==*z || 0==fsl_strncmp(z,"on",2) + || 0==fsl_strncmp(z,"true",4)){ + z = "ru"/*historical default*/; + }else if(!fsl_str_bool(z)){ + z = ""; + } + for(;*z;++z){ + switch(*z){ + case 'r': *m |= 0x001; break; + case 'u': *m |= 0x010; break; + case 't': *m |= 0x100; break; + default: break; + } + } + fsl_free(str); + } + f->cache.manifestSetting = (short)*m; +} + +int fsl_ckout_manifest_write(fsl_cx *f, int manifest, int manifestUuid, + int manifestTags, + int * wrote){ + fsl_db * const db = fsl_needs_ckout(f); + if(!db) return FSL_RC_NOT_A_CKOUT; + else if(!f->ckout.rid){ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "Checkout RID is 0, so it has no manifest."); + } + int W = 0; + int rc = 0; + fsl_buffer * b = fsl_cx_scratchpad(f); + fsl_buffer * content = &f->fileContent; + char * str = 0; + fsl_time_t const mtime = f->ckout.mtime>0 + ? fsl_julian_to_unix(f->ckout.mtime) + : 0; + fsl_buffer_reuse(content); + if(manifest<0 || manifestUuid<0 || manifestTags<0){ + int setting = 0; + fsl_ckout_manifest_setting(f, &setting); + if(manifest<0 && setting & FSL_MANIFEST_MAIN) manifest=1; + if(manifestUuid<0 && setting & FSL_MANIFEST_UUID) manifestUuid=1; + if(manifestTags<0 && setting & FSL_MANIFEST_TAGS) manifestTags=1; + } + if(manifest || manifestUuid || manifestTags){ + rc = fsl_buffer_append(b, f->ckout.dir, (fsl_int_t)f->ckout.dirLen); + if(rc) goto end; + } + if(manifest>0){ + rc = fsl_buffer_append(b, "manifest", 8); + if(rc) goto end; + rc = fsl_content_get(f, f->ckout.rid, content); + if(rc) goto end; + rc = fsl_buffer_to_filename(content, fsl_buffer_cstr(b)); + if(rc){ + rc = fsl_cx_err_set(f, rc, "Error writing file: %b", b); + goto end; + } + if(mtime) fsl_file_mtime_set(fsl_buffer_cstr(b), mtime); + W |= FSL_MANIFEST_MAIN; + }else if(!fsl_db_exists(db, + "SELECT 1 FROM vfile WHERE " + "pathname='manifest' /*%s()*/", + __func__)){ + b->used = f->ckout.dirLen; + rc = fsl_buffer_append(b, "manifest", 8); + if(rc) goto end; + fsl_file_unlink(fsl_buffer_cstr(b)); + } + + if(manifestUuid>0){ + b->used = f->ckout.dirLen; + fsl_buffer_reuse(content); + rc = fsl_buffer_append(b, "manifest.uuid", 13); + if(rc) goto end; + assert(f->ckout.uuid); + rc = fsl_buffer_append(content, f->ckout.uuid, -1); + if(!rc) rc = fsl_buffer_append(content, "\n", 1); + if(rc) goto end; + rc = fsl_buffer_to_filename(content, fsl_buffer_cstr(b)); + if(rc){ + rc = fsl_cx_err_set(f, rc, "Error writing file: %b", b); + goto end; + } + if(mtime) fsl_file_mtime_set(fsl_buffer_cstr(b), mtime); + W |= FSL_MANIFEST_UUID; + }else if(!fsl_db_exists(db, + "SELECT 1 FROM vfile WHERE " + "pathname='manifest.uuid' /*%s()*/", + __func__)){ + b->used = f->ckout.dirLen; + rc = fsl_buffer_append(b, "manifest.uuid", 13); + if(rc) goto end; + fsl_file_unlink(fsl_buffer_cstr(b)); + } + + if(manifestTags>0){ + fsl_stmt q = fsl_stmt_empty; + fsl_db * const db = fsl_cx_db_repo(f); + assert(db && "We can't have a checkout w/o a repo."); + b->used = f->ckout.dirLen; + fsl_buffer_reuse(content); + rc = fsl_buffer_append(b, "manifest.tags", 13); + if(rc) goto end; + str = fsl_db_g_text(db, NULL, "SELECT VALUE FROM tagxref " + "WHERE rid=%" FSL_ID_T_PFMT + " AND tagid=%d /*%s()*/", + f->ckout.rid, FSL_TAGID_BRANCH, __func__); + rc = fsl_buffer_appendf(content, "branch %z\n", str); + str = 0; + if(rc) goto end; + rc = fsl_db_prepare(db, &q, + "SELECT substr(tagname, 5)" + " FROM tagxref, tag" + " WHERE tagxref.rid=%" FSL_ID_T_PFMT + " AND tagxref.tagtype>0" + " AND tag.tagid=tagxref.tagid" + " AND tag.tagname GLOB 'sym-*'" + " /*%s()*/", + f->ckout.rid, __func__); + if(rc) goto end; + while( FSL_RC_STEP_ROW==fsl_stmt_step(&q) ){ + const char *zName = fsl_stmt_g_text(&q, 0, NULL); + rc = fsl_buffer_appendf(content, "tag %s\n", zName); + if(rc) break; + } + fsl_stmt_finalize(&q); + if(!rc){ + rc = fsl_buffer_to_filename(content, fsl_buffer_cstr(b)); + if(rc){ + rc = fsl_cx_err_set(f, rc, "Error writing file: %b", b); + } + } + if(mtime) fsl_file_mtime_set(fsl_buffer_cstr(b), mtime); + W |= FSL_MANIFEST_TAGS; + }else if(!fsl_db_exists(db, + "SELECT 1 FROM vfile WHERE " + "pathname='manifest.tags' /*%s()*/", + __func__)){ + b->used = f->ckout.dirLen; + rc = fsl_buffer_append(b, "manifest.tags", 13); + if(rc) goto end; + fsl_file_unlink(fsl_buffer_cstr(b)); + } + + end: + if(wrote) *wrote = W; + fsl_cx_scratchpad_yield(f, b); + fsl_buffer_reuse(content); + return rc; +} + +/** + Check every sub-directory of f's current checkout dir along the + path to zFilename. If any sub-directory part is really an ordinary file + or a symbolic link, set *errLen to the length of the prefix of zFilename + which is the name of that object. + + Returns 0 except on allocation error, in which case it returned FSL_RC_OOM. + If it finds nothing untowards about the path, *errLen will be set to 0. + + Example: Given inputs + + ckout = /home/alice/project1 + zFilename = /home/alice/project1/main/src/js/fileA.js + + Look for objects in the following order: + + /home/alice/project/main + /home/alice/project/main/src + /home/alice/project/main/src/js + + If any of those objects exist and are something other than a + directory then *errLen will be the length of the name of the first + non-directory object seen. + + If a given element of the path does not exist in the filesystem, + traversal stops without an error. +*/ +static int fsl_ckout_nondir_file_check(fsl_cx *f, char const * zFilename, + fsl_size_t * errLen); + +int fsl_ckout_nondir_file_check(fsl_cx *f, char const * zFilename, + fsl_size_t * errLen){ + if(!fsl_needs_ckout(f)) return FSL_RC_NOT_A_CKOUT; + int rc = 0; + int frc; + fsl_buffer * const fn = fsl_cx_scratchpad(f); + if(!fsl_is_rooted_in_ckout(f, zFilename)){ + assert(!"Misuse of this API. This condition should never fail."); + rc = fsl_cx_err_set(f, FSL_RC_MISUSE, "Path is not rooted at the " + "current checkout directory: %s", zFilename); + goto end; + } + rc = fsl_buffer_append(fn, zFilename, -1); + if(rc) goto end; + char * z = fsl_buffer_str(fn); + fsl_size_t i = f->ckout.dirLen; + fsl_size_t j; + fsl_fstat fst = fsl_fstat_empty; + char const * const zRoot = f->ckout.dir; + if(i && '/'==zRoot[i-1]) --i; + *errLen = 0; + while( z[i]=='/' ){ + for(j=i+1; z[j] && z[j]!='/'; ++j){} + if( z[j]!='/' ) break; + z[j] = 0; + frc = fsl_stat(z, &fst, false); + if(frc){ + /* A not[-yet]-existing path element is okay */ + break; + } + if(FSL_FSTAT_TYPE_DIR!=fst.type){ + *errLen = j; + break; + } + z[j] = '/'; + i = j; + } + end: + fsl_cx_scratchpad_yield(f, fn); + return rc; +} + +int fsl_ckout_safe_file_check(fsl_cx *f, char const * zFilename){ + if(!fsl_needs_ckout(f)) return FSL_RC_NOT_A_CKOUT; + int rc = 0; + fsl_buffer * const fn = fsl_cx_scratchpad(f); + if(!fsl_is_absolute_path(zFilename)){ + rc = fsl_file_canonical_name2(f->ckout.dir, zFilename, fn, false); + if(rc) goto end; + zFilename = fsl_buffer_cstr(fn); + }else if(!fsl_is_rooted_in_ckout(f, zFilename)){ + rc = fsl_cx_err_set(f, FSL_RC_MISUSE, "Path is not rooted at the " + "current checkout directory: %s", zFilename); + goto end; + } + + fsl_size_t errLen = 0; + rc = fsl_ckout_nondir_file_check(f, zFilename, &errLen); + if(rc) goto end /* OOM */; + else if(errLen){ + rc = fsl_cx_err_set(f, FSL_RC_TYPE, "Directory part of path refers " + "to a non-directory: %.*s", + (int)errLen, zFilename); + } + end: + fsl_cx_scratchpad_yield(f, fn); + return rc; +} + +bool fsl_is_rooted_in_ckout(fsl_cx *f, char const *zAbsPath){ + return f->ckout.dir + ? 0==fsl_strncmp(zAbsPath, f->ckout.dir, f->ckout.dirLen) + /* ^^^ fossil(1) uses stricmp() there, but that's a bug. However, + NOT using stricmp() on case-insensitive filesystems is arguably + also a bug. */ + : false; +} + +int fsl_is_rooted_in_ckout2(fsl_cx *f, char const *zAbsPath){ + int rc = 0; + if(!fsl_is_rooted_in_ckout(f, zAbsPath)){ + rc = fsl_cx_err_set(f, FSL_RC_RANGE, "Path is not rooted " + "in the current checkout: %s", + zAbsPath); + } + return rc; +} + +int fsl_ckout_symlink_create(fsl_cx * f, char const *zTgtFile, + char const * zLinkFile){ + if(!fsl_needs_ckout(f)) return FSL_RC_NOT_A_CKOUT; + int rc = 0; + fsl_buffer * const fn = fsl_cx_scratchpad(f); + if(!fsl_is_absolute_path(zLinkFile)){ + rc = fsl_file_canonical_name2(f->ckout.dir, zLinkFile, fn, false); + if(rc) goto end; + zLinkFile = fsl_buffer_cstr(fn); + }else if(0!=(rc = fsl_is_rooted_in_ckout2(f, zLinkFile))){ + goto end; + } + fsl_buffer * const b = fsl_cx_scratchpad(f); + rc = fsl_buffer_append(b, zTgtFile, -1); + if(!rc){ + rc = fsl_buffer_to_filename(b, fsl_buffer_cstr(fn)); + } + fsl_cx_scratchpad_yield(f, b); + end: + fsl_cx_scratchpad_yield(f, fn); + return rc; +} + +/** + Queues the directory part of the given filename into temp table + fx_revert_rmdir for an eventual rmdir() attempt on it in + fsl_revert_rmdir_fini(). +*/ +static int fsl_revert_rmdir_queue(fsl_cx * f, fsl_db * db, fsl_stmt * st, + char const * zFilename){ + int rc = 0; + if( !st->stmt ){ + rc = fsl_db_exec(db, "CREATE TEMP TABLE IF NOT EXISTS " + "fx_revert_rmdir(n TEXT PRIMARY KEY) " + "WITHOUT ROWID /* %s() */", __func__); + if(rc) goto dberr; + rc = fsl_db_prepare(db, st, "INSERT OR IGNORE INTO " + "fx_revert_rmdir(n) " + "VALUES(fsl_dirpart(?,0)) /* %s() */", + __func__); + if(rc) goto dberr; + } + rc = fsl_stmt_bind_step(st, "s", zFilename); + if(rc) goto dberr; + end: + return rc; + dberr: + rc = fsl_cx_uplift_db_error2(f, db, rc); + goto end; +} + +/** + Attempts to rmdir all dirs queued by fsl_revert_rmdir_queue(). Silently + ignores rmdir failure but will return non-0 for db errors. +*/ +static int fsl_revert_rmdir_fini(fsl_cx * f, fsl_db * db){ + int rc; + fsl_stmt st = fsl_stmt_empty; + fsl_buffer * const b = fsl_cx_scratchpad(f); + rc = fsl_db_prepare(db, &st, + "SELECT fsl_ckout_dir()||n " + "FROM fx_revert_rmdir " + "ORDER BY length(n) DESC /* %s() */", + __func__); + if(rc) goto dberr; + while(FSL_RC_STEP_ROW == fsl_stmt_step(&st)){ + fsl_size_t nDir = 0; + char const * zDir = fsl_stmt_g_text(&st, 0, &nDir); + fsl_buffer_reuse(b); + rc = fsl_buffer_append(b, zDir, (fsl_int_t)nDir); + if(rc) break; + fsl_ckout_rm_empty_dirs(f, b); + } + end: + fsl_cx_scratchpad_yield(f, b); + fsl_stmt_finalize(&st); + return rc; + dberr: + rc = fsl_cx_uplift_db_error2(f, db, rc); + goto end; +} + +int fsl_ckout_revert( fsl_cx * f, fsl_ckout_revert_opt const * opt ){ + /** + Reminder to whoever works on this code: the initial + implementation was done almost entirely without the benefit of + looking at fossil's implementation, thus this code is notably + different from fossil's. If any significant misbehaviors are + found here, vis a vis fossil, it might be worth reverting (as it + were) to that implementation. + */ + int rc; + fsl_db * const db = fsl_needs_ckout(f); + fsl_buffer * fname = 0; + char const * zNorm = 0; + fsl_id_t const vid = f->ckout.rid; + bool inTrans = false; + fsl_stmt q = fsl_stmt_empty; + fsl_stmt vfUpdate = fsl_stmt_empty; + fsl_stmt qRmdir = fsl_stmt_empty; + fsl_buffer * sql = 0; + if(!db) return FSL_RC_NOT_A_CKOUT; + assert(vid>=0); + if(!opt->vfileIds && opt->filename && *opt->filename){ + fname = fsl_cx_scratchpad(f); + rc = fsl_ckout_filename_check(f, opt->relativeToCwd, + opt->filename, fname); + if(rc){ + fsl_cx_scratchpad_yield(f, fname); + return rc; + } + zNorm = fsl_buffer_cstr(fname); + /* MARKER(("fsl_ckout_unmanage(%d, %s) ==> %s\n", opt->relativeToCwd, opt->filename, zNorm)); */ + assert(zNorm); + if(fname->used) fsl_buffer_strip_slashes(fname); + if(1==fname->used && '.'==*zNorm){ + /* Special case: handle "." from ckout root intuitively */ + fsl_buffer_reuse(fname); + assert(0==*zNorm); + } + } + rc = fsl_db_transaction_begin(db); + if(rc) goto dberr; + inTrans = true; + if(opt->scanForChanges){ + rc = fsl_vfile_changes_scan(f, 0, 0); + if(rc) goto end; + } + sql = fsl_cx_scratchpad(f); + rc = fsl_buffer_appendf(sql, + "SELECT id, rid, deleted, " + "fsl_ckout_dir()||pathname, " + "fsl_ckout_dir()||origname " + "FROM vfile WHERE vid=%" FSL_ID_T_PFMT " ", + vid); + if(rc) goto end; + if(zNorm && *zNorm){ + rc = fsl_buffer_appendf(sql, + "AND CASE WHEN %Q='' THEN 1 " + "ELSE (" + " fsl_match_vfile_or_dir(pathname,%Q) " + " OR fsl_match_vfile_or_dir(origname,%Q)" + ") END", + zNorm, zNorm, zNorm); + if(rc) goto end; + }else if(opt->vfileIds){ + rc = fsl_ckout_bag_to_ids(f, db, "fx_revert_id", opt->vfileIds); + if(rc) goto end; + rc = fsl_buffer_append(sql, "AND id IN fx_revert_id", -1); + if(rc) goto end; + }else{ + rc = fsl_buffer_append(sql, + "AND (" + " chnged<>0" + " OR deleted<>0" + " OR rid=0" + " OR coalesce(origname,pathname)" + " <>pathname" + ")", -1); + } + assert(!rc); + rc = fsl_db_prepare(db, &q, "%b /* %s() */", sql, __func__); + fsl_cx_scratchpad_yield(f, sql); + sql = 0; + if(rc) goto dberr; + if((!zNorm || !*zNorm) && !opt->vfileIds){ + rc = fsl_ckout_clear_merge_state(f); + if(rc) goto end; + } + while((FSL_RC_STEP_ROW==fsl_stmt_step(&q))){ + fsl_id_t const id = fsl_stmt_g_id(&q, 0); + fsl_id_t const rid = fsl_stmt_g_id(&q, 1); + int32_t const deleted = fsl_stmt_g_int32(&q, 2); + char const * const zName = fsl_stmt_g_text(&q, 3, NULL); + char const * const zNameOrig = fsl_stmt_g_text(&q, 4, NULL); + bool const renamed = + zNameOrig ? !!fsl_strcmp(zName, zNameOrig) : false; + fsl_ckout_revert_e changeType = FSL_REVERT_NONE; + if(!rid){ // Added but not yet checked in. + rc = fsl_db_exec(db, "DELETE FROM vfile WHERE id=%" FSL_ID_T_PFMT, + id); + if(rc) goto dberr; + changeType = FSL_REVERT_UNMANAGE; + }else{ + int wasWritten = 0; + if(renamed){ + if((rc=fsl_mkdir_for_file(zNameOrig, true))){ + rc = fsl_cx_err_set(f, rc, "mkdir() failed for file: %s", + zNameOrig); + break; + } + /* Move, if possible, the new name back over the original + name. This will possibly allow fsl_vfile_to_ckout() to + avoid having to load that file's contents and overwrite + it. */ + int mvCheck = fsl_stat(zName, NULL, false); + if(0==mvCheck || FSL_RC_NOT_FOUND==mvCheck){ + mvCheck = fsl_file_unlink(zNameOrig); + if(0==mvCheck || FSL_RC_NOT_FOUND==mvCheck){ + if(0==fsl_file_rename(zName, zNameOrig)){ + rc = fsl_revert_rmdir_queue(f, db, &qRmdir, zName); + if(rc) break; + } + } + } + /* Ignore any errors: this operation is an optimization, + not a requirement. Worse case, the entry with the old + name is left in the filesystem. */ + } + if(!vfUpdate.stmt){ + rc = fsl_db_prepare(db, &vfUpdate, + "UPDATE vfile SET chnged=0, deleted=0, " + "pathname=coalesce(origname,pathname), " + "origname=NULL " + "WHERE id=?1 /*%s()*/", __func__); + if(rc) goto dberr; + } + rc = fsl_stmt_bind_step(&vfUpdate, "R", id) + /* Has to be done before fsl_vfile_to_ckout() because that + function writes to vfile.pathname. */; + if(rc) goto dberr; + rc = fsl_vfile_to_ckout(f, id, &wasWritten); + if(rc) break; + if(opt->callback){ + if(renamed){ + changeType = FSL_REVERT_RENAME; + }else if(wasWritten){ + changeType = (2==wasWritten) + ? FSL_REVERT_CONTENTS + : FSL_REVERT_PERMISSIONS; + }else if(deleted){ + changeType = FSL_REVERT_REMOVE; + } + } + } + if(opt->callback && FSL_REVERT_NONE!=changeType){ + char const * name = renamed ? zNameOrig : zName; + rc = opt->callback(&name[f->ckout.dirLen], + changeType, opt->callbackState); + if(rc) break; + } + }/*step() loop*/ + end: + if(fname) fsl_cx_scratchpad_yield(f, fname); + if(sql) fsl_cx_scratchpad_yield(f, sql); + fsl_stmt_finalize(&q); + fsl_stmt_finalize(&vfUpdate); + if(qRmdir.stmt){ + fsl_stmt_finalize(&qRmdir); + if(!rc) rc = fsl_revert_rmdir_fini(f, db); + fsl_db_exec(db, "DROP TABLE IF EXISTS fx_revert_rmdir /* %s() */", + __func__); + } + if(opt->vfileIds){ + fsl_db_exec_multi(db, "DROP TABLE IF EXISTS fx_revert_id " + "/* %s() */", __func__) + /* Ignoring result code */; + } + if(inTrans){ + int const rc2 = fsl_db_transaction_end(db, !!rc); + if(!rc) rc = rc2; + } + return rc; + dberr: + assert(rc); + rc = fsl_cx_uplift_db_error2(f, db, rc); + goto end; +} + +int fsl_ckout_vfile_ids( fsl_cx * f, fsl_id_t vid, + fsl_id_bag * dest, char const * zName, + bool relativeToCwd, bool changedOnly ) { + if(!fsl_needs_ckout(f)) return FSL_RC_NOT_A_CKOUT; + fsl_buffer * const canon = fsl_cx_scratchpad(f); + int rc = fsl_ckout_filename_check(f, relativeToCwd, zName, canon); + if(!rc){ + fsl_buffer_strip_slashes(canon); + rc = fsl_filename_to_vfile_ids(f, vid, dest, + fsl_buffer_cstr(canon), + changedOnly); + } + fsl_cx_scratchpad_yield(f, canon); + return rc; +} + +int fsl_ckout_file_content(fsl_cx * const f, bool relativeToCwd, char const * zName, + fsl_buffer * const dest ){ + int rc; + fsl_buffer * fname; + if(!fsl_needs_ckout(f)) return FSL_RC_NOT_A_CKOUT; + fname = fsl_cx_scratchpad(f); + rc = fsl_file_canonical_name2( relativeToCwd + ? NULL + : fsl_cx_ckout_dir_name(f, NULL), + zName, fname, 1 ); + if(!rc){ + assert(fname->used); + if('/'==fname->mem[fname->used-1]){ + rc = fsl_cx_err_set(f, FSL_RC_MISUSE,"Filename may not have a trailing slash."); + /* If we don't do this, we might end up reading a directory entry in raw form. + Well, we still might. */ + }else{ + fsl_fstat fstat = fsl_fstat_empty; + const char * zCanon = fsl_buffer_cstr(fname); + rc = fsl_stat(zCanon, &fstat, true); + if(rc){ + rc = fsl_cx_err_set(f, rc, "Cannot stat file; %b", fname); + }else if(FSL_FSTAT_TYPE_FILE!=fstat.type){ + rc = fsl_cx_err_set(f, FSL_RC_TYPE, + "Not a regular file file; %b", fname); + }else{ + dest->used =0; + rc = fsl_buffer_fill_from_filename(dest, fsl_buffer_cstr(fname)); + if(rc){ + rc = fsl_cx_err_set(f, rc, "%s error reading file; %b", + fsl_rc_cstr(rc), fname); + } + } + } + } + fsl_cx_scratchpad_yield(f, fname); + return rc; +} + +int fsl_card_F_ckout_mtime(fsl_cx * const f, + fsl_id_t vid, + fsl_card_F const * fc, + fsl_time_t * repoMtime, + fsl_time_t * localMtime){ + + int rc = 0; + fsl_id_t fid = 0; + fsl_fstat fst = fsl_fstat_empty; + if(!fsl_needs_ckout(f)) return FSL_RC_NOT_A_CKOUT; + if(0<=vid){ + fsl_ckout_version_info(f, &vid, NULL); + } + fid = fsl_repo_filename_fnid(f, fc->name); + if(fid<=0){ + rc = fsl_cx_err_get(f, NULL, NULL); + return rc ? rc : fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Could not resolve filename: %s", + fc->name); + } + else if(!fid){ + return fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Could not resolve filename: %s", + fc->name); + } + if(localMtime){ + rc = fsl_cx_stat(f, 0, fc->name, &fst); + if(rc){ + return fsl_cx_err_set(f, rc, "Could not stat() file: %s", + fc->name); + } + *localMtime = fst.mtime; + } + if(repoMtime){ + rc = fsl_mtime_of_manifest_file(f, vid, fid, repoMtime); + } + return rc; +} + + +#undef MARKER +/* end of file checkout.c */ +/* start of file cli.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ + +#include /* for strchr() */ + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +/** Convenience form of FCLI_VN for level-3 verbosity. */ +#define FCLI_V3(pfexp) FCLI_VN(3,pfexp) +#define fcli_empty_m { \ + NULL/*appHelp*/, \ + NULL/*cliFlags*/, \ + NULL/*f*/, \ + NULL/*argv*/, \ + 0/*argc*/, \ + NULL/*appName*/, \ + {/*clientFlags*/ \ + "."/*checkoutDir*/, \ + 0/*verbose*/ \ + }, \ + {/*transient*/ \ + NULL/*repoDb*/, \ + NULL/*userArg*/, \ + 0/*helpRequested*/, \ + false/*versionRequested*/\ + }, \ + {/*config*/ \ + -1/*traceSql*/, \ + fsl_outputer_empty_m \ + }, \ + fsl_error_empty_m/*err*/ \ +} + +const fcli_t fcli_empty = fcli_empty_m; +fcli_t fcli = fcli_empty_m; +const fcli_cliflag fcli_cliflag_empty = fcli_cliflag_empty_m; +static fsl_timer_state fcliTimer = fsl_timer_state_empty_m; + +void fcli_printf(char const * fmt, ...){ + va_list args; + va_start(args,fmt); + if(fcli.f){ + fsl_outputfv(fcli.f, fmt, args); + }else{ + fsl_fprintfv(stdout, fmt, args); + } + va_end(args); +} + +/** + Outputs app-level help. How it does this depends on the state of + the fcli object, namely fcli.cliFlags and the verbosity + level. Normally this is triggered automatically by the CLI flag + handling in fcli_setup(). +*/ +static void fcli_help(void); + +unsigned short fcli_is_verbose(void){ + return fcli.clientFlags.verbose; +} + +fsl_cx * fcli_cx(void){ + return fcli.f; +} + +static int fcli_open(void){ + int rc = 0; + fsl_cx * f = fcli.f; + assert(f); + if(fcli.transient.repoDbArg){ + FCLI_V3(("Trying to open repo db file [%s]...\n", fcli.transient.repoDbArg)); + rc = fsl_repo_open( f, fcli.transient.repoDbArg ); + } + else if(fcli.clientFlags.checkoutDir){ + fsl_buffer dir = fsl_buffer_empty; + char const * dirName; + rc = fsl_file_canonical_name(fcli.clientFlags.checkoutDir, + &dir, 0); + assert(!rc); + dirName = (char const *)fsl_buffer_cstr(&dir); + FCLI_V3(("Trying to open checkout from [%s]...\n", + dirName)); + rc = fsl_ckout_open_dir(f, dirName, true); + FCLI_V3(("checkout open rc=%s\n", fsl_rc_cstr(rc))); + + /* if(FSL_RC_NOT_FOUND==rc) rc = FSL_RC_NOT_A_CKOUT; */ + if(rc){ + if(!fsl_cx_err_get(f,NULL,NULL)){ + rc = fsl_cx_err_set(f, rc, "Opening of checkout under " + "[%s] failed with code %d (%s).", + dirName, rc, fsl_rc_cstr(rc)); + } + } + fsl_buffer_reserve(&dir, 0); + if(rc) return rc; + } + if(!rc){ + if(fcli.clientFlags.verbose>1){ + fsl_db * dbC = fsl_cx_db_ckout(f); + fsl_db * dbR = fsl_cx_db_repo(f); + if(dbC){ + FCLI_V3(("Checkout DB name: %s\n", f->ckout.db.filename)); + } + if(dbR){ + FCLI_V3(("Opened repo db: %s\n", f->repo.db.filename)); + FCLI_V3(("Repo user name: %s\n", f->repo.user)); + } + } +#if 0 + /* + Only(?) here for testing purposes. + + We don't really need/want to update the repo db on each + open of the checkout db, do we? Or do we? + */ + fsl_repo_record_filename(f) /* ignore rc - not critical */; +#endif + } + return rc; +} + + +#define fcli__error (fcli.f ? &fcli.f->error : &fcli.err) +fsl_error * fcli_error(void){ + return fcli__error; +} + +void fcli_err_reset(void){ + fsl_error_reset(fcli__error); +} + + +static struct TempFlags { + bool traceSql; + bool doTimer; +} TempFlags = { +false, +false +}; + +static struct { + fsl_list list; +} FCliFree = { +fsl_list_empty_m +}; + +static void fcli_shutdown(void){ + fsl_cx * f = fcli.f; + int rc = 0; + + fsl_error_clear(&fcli.err); + fsl_free(fcli.argv)/*contents are in the FCliFree list*/; + + if(f){ + while(fsl_cx_transaction_level(f)){ + MARKER(("WARNING: open db transaction at shutdown-time. " + "Rolling back.\n")); + fsl_cx_transaction_end(f, true); + } + if(1 && + fsl_cx_db_ckout(f)){ + /* For testing/demo only: this is implicit + when we call fsl_cx_finalize(). + */ + rc = fsl_ckout_close(f); + FCLI_V3(("Closed checkout/repo db(s). rc=%s\n", fsl_rc_cstr(rc))); + //assert(0==rc); + } + } + fsl_list_clear(&FCliFree.list, fsl_list_v_fsl_free, 0); + fsl_list_reserve(&FCliFree.list, 0); + if(f){ + FCLI_V3(("Finalizing fsl_cx @%p\n", (void const *)f)); + fsl_cx_finalize( f ); + } + fcli = fcli_empty; + if(TempFlags.doTimer){ + double const runTime = + ((int64_t)fsl_timer_stop(&fcliTimer)) / 1000.0; + f_out("Total fcli run time: %f seconds of CPU time\n", + runTime/1000); + } +} + +static struct { + fcli_cliflag const * flags; +} FCliHelpState = { +NULL +}; + + +static int fcli_flag_f_nocheckoutDir(fcli_cliflag const *f){ + if(f){/*unused*/} + fcli.clientFlags.checkoutDir = 0; + return 0; +} +static int fcli_flag_f_verbose(fcli_cliflag const *f){ + if(f){/*unused*/} + ++fcli.clientFlags.verbose; + return FCLI_RC_FLAG_AGAIN; +} +static int fcli_flag_f_help(fcli_cliflag const *f){ + if(f){/*unused*/} + ++fcli.transient.helpRequested; + return FCLI_RC_FLAG_AGAIN; +} + +static const fcli_cliflag FCliFlagsGlobal[] = { + FCLI_FLAG_BOOL_X("?","help",NULL, + fcli_flag_f_help, + "Show app help. Also triggered if the first non-flag is \"help\"."), + FCLI_FLAG_BOOL(0,"lib-version", &fcli.transient.versionRequested, + "Show app version number."), + FCLI_FLAG("R","repo","REPO-FILE",&fcli.transient.repoDbArg, + "Selects a specific repository database, ignoring the one " + "used by the current directory's checkout (if any)."), + FCLI_FLAG(NULL,"user","username",&fcli.transient.userArg, + "Sets the name of the fossil user name for this session."), + FCLI_FLAG_BOOL_X(NULL, "no-checkout",NULL,fcli_flag_f_nocheckoutDir, + "Disable automatic attempt to open checkout."), + FCLI_FLAG(NULL,"checkout-dir","DIRECTORY", &fcli.clientFlags.checkoutDir, + "Open the given directory as a checkout, instead of the current dir."), + FCLI_FLAG_BOOL_X("V","verbose",NULL,fcli_flag_f_verbose, + "Increases the verbosity level by 1. May be used multiple times."), + FCLI_FLAG_BOOL(NULL,"trace-sql",&TempFlags.traceSql, + "Enable SQL tracing."), + FCLI_FLAG_BOOL(NULL,"timer",&TempFlags.doTimer, + "At the end of successful app execution, output how long it took " + "from the call to fcli_setup() until the end of main()."), + fcli_cliflag_empty_m +}; + +void fcli_cliflag_help(fcli_cliflag const *defs){ + fcli_cliflag const * f; + const char * tab = " "; + for( f = defs; f->flagShort || f->flagLong; ++f ){ + const char * s = f->flagShort; + const char * l = f->flagLong; + const char * fvl = f->flagValueLabel; + const char * valLbl = 0; + switch(f->flagType){ + case FCLI_FLAG_TYPE_BOOL: + case FCLI_FLAG_TYPE_BOOL_INVERT: break; + case FCLI_FLAG_TYPE_INT32: valLbl = fvl ? fvl : "int32"; break; + case FCLI_FLAG_TYPE_INT64: valLbl = fvl ? fvl : "int64"; break; + case FCLI_FLAG_TYPE_ID: valLbl = fvl ? fvl : "db-record-id"; break; + case FCLI_FLAG_TYPE_DOUBLE: valLbl = fvl ? fvl : "double"; break; + case FCLI_FLAG_TYPE_CSTR: valLbl = fvl ? fvl : "string"; break; + default: + break; + } + f_out("%s%s%s%s%s%s%s%s", + tab, + s?"-":"", s?s:"", (s&&l)?"|":"", + l?"--":"",l?l:"", + valLbl ? "=" : "", valLbl); + if(f->helpText){ + f_out("\n%s%s%s", tab, tab, f->helpText); + } + f_out("\n\n"); + } +} + +void fcli_help(void){ + if(fcli.appHelp){ + if(fcli.appHelp->briefUsage){ + f_out("Usage: %s [options] %s\n", fcli.appName, fcli.appHelp->briefUsage); + } + if(fcli.appHelp->briefDescription){ + f_out("\n%s\n", fcli.appHelp->briefDescription); + } + }else{ + f_out("Help for %s:\n", fcli.appName); + } + const int helpCount = fcli.transient.helpRequested + + fcli.clientFlags.verbose; + bool const showGlobal = helpCount>1; + bool const showApp = (2!=helpCount); + if(showGlobal){ + f_out("\nFCli global flags:\n\n"); + fcli_cliflag_help(FCliFlagsGlobal); + }else{ + f_out("\n"); + } + if(showApp){ + if(FCliHelpState.flags + && (FCliHelpState.flags[0].flagShort || FCliHelpState.flags[0].flagLong)){ + f_out("App-specific flags:\n\n"); + fcli_cliflag_help(FCliHelpState.flags); + //f_out("\n"); + } + if(fcli.appHelp && fcli.appHelp->callback){ + fcli.appHelp->callback(); + f_out("\n"); + } + } + if(showGlobal){ + if(!showApp){ + f_out("Invoke --help three times to list " + "both the framework- and app-level options.\n"); + }else{ + f_out("Invoke --help once to list only the " + "app-level flags.\n"); + } + }else{ + f_out("Invoke --help twice to list the framework-level " + "options. Use --help three times to list both " + "framework- and app-level options.\n"); + } + f_out("\nFlags which require values may be passed as " + "--flag=value or --flag value.\n\n"); +} + +int fcli_process_flags( fcli_cliflag const * defs ) { + fcli_cliflag const * f; + int rc = 0; + /** + TODO/FIXME/NICE-TO-HAVE: we "really should" process the CLI flags + in the order they are provided on the CLI, as opposed to the + order they're defined in the defs array. The current approach is + much simpler to process but keeps us from being able to support + certain useful flag-handling options, e.g.: + + f-tag -a artifact-id-1 --tag x=y --tag y=z -a artifact-id-2 --tag a=b... + + The current approach consumes the -a flags first, leaving us + unable to match the --tag flags to their corresponding + (left-hand) -a flag. + + Processing them the other way around, however, requires that we + keep track of which flags we've already seen so that we can + reject, where appropriate, duplicate invocations. + + We could, instead of looping on the defs array, loop over the + head of fcli.argv. If it's a non-flag, move it out of the way + temporarily (into a new list), else look over the defs array + looking for a flag match. We don't know, until finding such a + match, whether the current flag requires a value. If it does, we + then have to check the current fcli.argv entry to see if it has a + value (--x=y) or whether the next argv entry is its value (--x + y). If the current tip has no matching defs entry, we have no + choice but to skip over it in the hopes that the user can use + fcli_flag() and friends to consume it, but we cannot know, from + here, whether such a stray flag requires a value, which means we + cannot know, for sure, how to process the _next_ argument. The + best we could do is have a heuristic like "if it starts with a + dash, assume it's a flag, otherwise assume it's a value for the + previous flag and skip over it," but whether or not that's sane + enough for daily use is as yet undetermined. + + If we change the CLI interface to require --flag=value for all + flags, as opposed to optionally allowing (--flag value), the + above becomes simpler, but CLI usage suffers. Hmmm. e.g.: + + f-ci -m="message" ... + + simply doesn't fit the age-old muscle memory of: + + svn ci -m ... + cvs ci -m ... + fossil ci -m ... + */ + for( f = defs; f->flagShort || f->flagLong; ++f ){ + if(!f->flagValue && !f->callback){ + /* We accept these for purposes of generating the --help text, + but we can't otherwise do anything sensible with them and + assume the app will handle such flags downstream or ignore + them altogether.*/ + continue; + } + char const * v = NULL; + const char ** passV = f->flagValue ? &v : NULL; + switch(f->flagType){ + case FCLI_FLAG_TYPE_BOOL: + case FCLI_FLAG_TYPE_BOOL_INVERT: + passV = NULL; + break; + default: break; + }; + bool const gotIt = fcli_flag2(f->flagShort, f->flagLong, passV); + if(fcli__error->code){ + /** + Corner case. Consider: + + FCLI_FLAG("x","y","xy", &foo, "blah"); + + And: my-app -x + + That will cause fcli_flag2() to return false, but it will + also populate fcli__error for us. + */ + rc = fcli__error->code; + break; + } + //MARKER(("Got?=%d flag: %s/%s %s\n",gotIt, f->flagShort, f->flagLong, v ? v : "")); + if(!gotIt){ + continue; + } + assert(f->flagValue || f->callback); + if(f->flagValue) switch(f->flagType){ + case FCLI_FLAG_TYPE_BOOL: + *((bool*)f->flagValue) = true; + break; + case FCLI_FLAG_TYPE_BOOL_INVERT: + *((bool*)f->flagValue) = false; + break; + case FCLI_FLAG_TYPE_CSTR: + if(!v) goto missing_val; + *((char const **)f->flagValue) = v; + break; + case FCLI_FLAG_TYPE_INT32: + if(!v) goto missing_val; + *((int32_t*)f->flagValue) = atoi(v); + break; + case FCLI_FLAG_TYPE_INT64: + if(!v) goto missing_val; + *((int64_t*)f->flagValue) = atoll(v); + break; + case FCLI_FLAG_TYPE_ID: + if(!v) goto missing_val; + if(sizeof(fsl_id_t)>32){ + *((fsl_id_t*)f->flagValue) = (fsl_id_t)atoll(v); + }else{ + *((fsl_id_t*)f->flagValue) = (fsl_id_t)atol(v); + } + break; + case FCLI_FLAG_TYPE_DOUBLE: + if(!v) goto missing_val; + *((double*)f->flagValue) = strtod(v, NULL); + break; + default: + MARKER(("As-yet-unhandled flag type for flag %s%s%s.", + f->flagShort ? f->flagShort : "", + (f->flagShort && f->flagLong) ? "|" : "", + f->flagLong ? f->flagLong : "")); + rc = FSL_RC_MISUSE; + break; + } + if(rc) break; + else if(f->callback){ + rc = f->callback(f); + if(rc==FCLI_RC_FLAG_AGAIN){ + rc = 0; + --f; + }else if(rc){ + break; + } + } + } + //MARKER(("fcli__error->code==%s\n", fsl_rc_cstr(fcli__error->code))); + return rc; + missing_val: + rc = fcli_err_set(FSL_RC_MISUSE,"Missing value for flag %s%s%s.", + f->flagShort ? f->flagShort : "", + (f->flagShort && f->flagLong) ? "|" : "", + f->flagLong ? f->flagLong : ""); + return rc; +} + +/** + oldMode must be true if fcli.cliFlags is NULL, else false. +*/ +static int fcli_process_argv( bool oldMode, int argc, char const * const * argv ){ + int i; + int rc = 0; + char * cp; + fcli.appName = argv[0]; + fcli.argc = 0; + fcli.argv = (char **)fsl_malloc( (argc + 1) * sizeof(char*)); + fcli.argv[argc] = NULL; + for( i = 1; i < argc; ++i ){ + char const * arg = argv[i]; + if('-'==*arg){ + char const * flag = arg+1; + while('-'==*flag) ++flag; +#define FLAG(F) if(0==fsl_strcmp(F,flag)) + if(oldMode){ + FLAG("help") { + ++fcli.transient.helpRequested; + continue; + } + FLAG("?") { + ++fcli.transient.helpRequested; + continue; + } + FLAG("V") { + fcli.clientFlags.verbose += 1; + continue; + } + FLAG("VV") { + fcli.clientFlags.verbose += 2; + continue; + } + FLAG("VVV") { + fcli.clientFlags.verbose += 3; + continue; + } + FLAG("verbose") { + fcli.clientFlags.verbose += 1; + continue; + } + } +#undef FLAG + /* else fall through */ + } + cp = fsl_strdup(arg); + if(!cp) return FSL_RC_OOM; + fcli.argv[fcli.argc++] = cp; + fcli_fax(cp); + } + if(!rc && !oldMode){ + rc = fcli_process_flags(FCliFlagsGlobal); + } + return rc; +} + +bool fcli_flag(char const * opt, const char ** value){ + int i = 0; + int remove = 0 /* number of items to remove from argv */; + bool rc = false /* true if found, else 0 */; + fsl_size_t optLen = fsl_strlen(opt); + for( ; i < fcli.argc; ++i ){ + char const * arg = fcli.argv[i]; + char const * x; + char const * vp = NULL; + if(!arg || ('-' != *arg)) continue; + rc = false; + x = arg+1; + if('-' == *x) { ++x;} + if(0 != fsl_strncmp(x, opt, optLen)) continue; + if(!value){ + if(x[optLen]) continue /* not exact match */; + /* Treat this as a boolean. */ + rc = true; + ++remove; + break; + }else{ + /* -FLAG VALUE or -FLAG=VALUE */ + if(x[optLen] == '='){ + rc = true; + vp = x+optLen+1; + ++remove; + } + else if(x[optLen]) continue /* not an exact match */; + else if(i<(fcli.argc-1)){ /* -FLAG VALUE */ + vp = fcli.argv[i+1]; + if('-'==*vp && vp[1]/*allow "-" by itself!*/){ + // VALUE looks like a flag. + fcli_err_set(FSL_RC_MISUSE, "Missing value for flag [%s].", + opt); + rc = false; + assert(!remove); + break; + } + rc = true; + remove += 2; + } + else{ + /* + --FLAG is expecting VALUE but we're at end of argv. Leave + --FLAG in the args and report this as "not found." + */ + rc = false; + assert(!remove); + fcli_err_set(FSL_RC_MISUSE, + "Missing value for flag [%s].", + opt); + assert(fcli__error->code); + //MARKER(("Missing flag value for [%s]\n",opt)); + break; + } + if(rc){ + *value = vp; + } + break; + } + } + if(remove>0){ + int x; + for( x = 0; x < remove; ++x ){ + fcli.argv[i+x] = NULL/*memory ownership==>FCliFree*/; + } + for( ; i < fcli.argc; ++i ){ + fcli.argv[i] = fcli.argv[i+remove]; + } + fcli.argc -= remove; + fcli.argv[i] = NULL; + } + //MARKER(("flag %s check rc=%s\n",opt,fsl_rc_cstr(fcli__error->code))); + return rc; +} + +bool fcli_flag2(char const * shortOpt, + char const * longOpt, + const char ** value){ + bool rc = 0; + if(shortOpt) rc = fcli_flag(shortOpt, value); + if(!rc && longOpt && !fcli__error->code) rc = fcli_flag(longOpt, value); + //MARKER(("flag %s check rc=%s\n",shortOpt,fsl_rc_cstr(fcli__error->code))); + return rc; +} + +bool fcli_flag_or_arg(char const * shortOpt, + char const * longOpt, + const char ** value){ + bool rc = fcli_flag(shortOpt, value); + if(!rc && !fcli__error->code){ + rc = fcli_flag(longOpt, value); + if(!rc && value){ + const char * arg = fcli_next_arg(1); + if(arg){ + rc = true; + *value = arg; + } + } + } + return rc; +} + + +/** + We copy fsl_lib_configurable.allocator as a base allocator. + */ +static fsl_allocator fslAllocOrig; + +/** + Proxies fslAllocOrig.f() and abort()s on OOM conditions. +*/ +static void * fsl_realloc_f_failing(void * state, void * mem, fsl_size_t n){ + void * rv = fslAllocOrig.f(fslAllocOrig.state, mem, n); + if(n && !rv){ + fsl_fatal(FSL_RC_OOM, NULL)/*does not return*/; + } + return rv; +} + +/** + Replacement for fsl_memory_allocator() which abort()s on OOM. + Why? Because fossil(1) has shown how much that can simplify error + checking in an allocates-often API. + */ +static const fsl_allocator fcli_allocator = { +fsl_realloc_f_failing, +NULL/*state*/ +}; + +void fcli_pre_setup(void){ + static int run = 0; + if(run++) return; + fslAllocOrig = fsl_lib_configurable.allocator; + fsl_lib_configurable.allocator = fcli_allocator + /* This MUST be done BEFORE the fsl API allocates + ANY memory! */; + atexit(fcli_shutdown); +} +/** + oldMode must be true if fcli.cliFlags is NULL, else false. +*/ +static int fcli_setup_common1(bool oldMode, int argc, char const * const *argv){ + static char once = 0; + int rc = 0; + if(once++){ + fprintf(stderr,"MISUSE: fcli_setup() must " + "not be called more than once."); + return FSL_RC_MISUSE; + } + fsl_timer_start(&fcliTimer); + fcli_pre_setup(); + rc = fcli_process_argv(oldMode, argc, argv); + if(!rc && fcli.argc && 0==fsl_strcmp("help",fcli.argv[0])){ + fcli_next_arg(1) /* strip argument */; + ++fcli.transient.helpRequested; + } + return rc; +} + +static int fcli_setup_common2(void){ + int rc = 0; + fsl_cx_init_opt init = fsl_cx_init_opt_empty; + fsl_cx * f = 0; + + init.config.sqlPrint = 1; + if(fcli.config.outputer.out){ + init.output = fcli.config.outputer; + fcli.config.outputer = fsl_outputer_empty + /* To avoid any confusion about ownership */; + }else{ + init.output = fsl_outputer_FILE; + init.output.state = stdout; + } + if(fcli.config.traceSql>0 || TempFlags.traceSql){ + init.config.traceSql = fcli.config.traceSql; + } + + rc = fsl_cx_init( &f, &init ); + fcli.f = f; +#if 0 + /* Just for testing cache size effects... */ + f->cache.arty.szLimit = 1024 * 1024 * 20; + f->cache.arty.usedLimit = 300; +#endif + fsl_error_clear(&fcli.err); + FCLI_V3(("Initialized fsl_cx @0x%p. rc=%s\n", + (void const *)f, fsl_rc_cstr(rc))); + if(!rc){ +#if 0 + if(fcli.transient.gmtTime){ + fsl_cx_flag_set(f, FSL_CX_F_LOCALTIME_GMT, 1); + } +#endif + if(fcli.clientFlags.checkoutDir || fcli.transient.repoDbArg){ + rc = fcli_open(); + FCLI_V3(("fcli_open() rc=%s\n", fsl_rc_cstr(rc))); + if(!fcli.transient.repoDbArg && fcli.clientFlags.checkoutDir + && (FSL_RC_NOT_FOUND == rc)){ + /* If [it looks like] we tried an implicit checkout-open but + didn't find one, suppress the error. */ + rc = 0; + fcli_err_reset(); + } + } + } + if(!rc){ + char const * userName = fcli.transient.userArg; + if(userName){ + fsl_cx_user_set(f, userName); + }else if(!fsl_cx_user_get(f)){ + char * u = fsl_guess_user_name(); + fsl_cx_user_set(f, u); + fsl_free(u); + } + } + return rc; +} + +static int fcli_setup2(int argc, char const * const * argv, + const fcli_cliflag * flags){ + int rc; + FCliHelpState.flags = flags; + rc = fcli_setup_common1(false, argc, argv); + if(rc) return rc; + assert(!fcli__error->code); + if(fcli.transient.helpRequested){ + /* Do this last so that we can get the default user name and such + for display in the help text. */ + fcli_help(); + rc = FCLI_RC_HELP; + }else if(fcli.transient.versionRequested){ + f_out("libfossil version: %s\nCheckin: %s\nCheckin timestamp: %s\n", + fsl_library_version(), + FSL_LIB_VERSION_HASH, + FSL_LIB_VERSION_TIMESTAMP); + rc = FCLI_RC_HELP; + }else{ + rc = fcli_process_flags(flags); + if(rc) assert(fcli__error->msg.used); + if(!rc){ + rc = fcli_setup_common2(); + } + } + return rc; +} + +int fcli_setup_v2(int argc, char const * const * argv, + fcli_cliflag const * const cliFlags, + fcli_help_info const * const helpInfo ){ + if(NULL!=cliFlags) fcli.cliFlags = cliFlags; + if(NULL!=helpInfo) fcli.appHelp = helpInfo; + if(fcli.cliFlags){ + return fcli_setup2(argc, argv, fcli.cliFlags); + } + int rc = fcli_setup_common1(true, argc, argv); + if(!rc){ + //f_out("fcli.transient.helpRequested=%d\n",fcli.transient.helpRequested); + if(fcli.transient.helpRequested){ + /* Do this last so that we can get the default user name and such + for display in the help text. */ + fcli_help(); + rc = FCLI_RC_HELP; + }else{ + if( fcli_flag2(NULL, "no-checkout", NULL) ){ + fcli.clientFlags.checkoutDir = NULL; + } + fcli_flag2(NULL,"user", &fcli.transient.userArg); + fcli.config.traceSql = fcli_flag2(NULL,"trace-sql", NULL); + fcli_flag2("R", "repo", &fcli.transient.repoDbArg); + rc = fcli_setup_common2(); + } + } + return rc; +} + +int fcli_setup(int argc, char const * const * argv ){ + return fcli_setup_v2(argc, argv, NULL, NULL); +} + +int fcli_err_report2(bool clear, char const * file, int line){ + int errRc = 0; + char const * msg = NULL; + errRc = fsl_error_get( fcli__error, &msg, NULL ); + if(FCLI_RC_HELP==errRc){ + errRc = 0; + }else if(errRc || msg){ + if(fcli.clientFlags.verbose>0){ + fcli_printf("%s %s:%d: ERROR #%d (%s): %s\n", + fcli.appName, + file, line, errRc, fsl_rc_cstr(errRc), msg); + }else{ + fcli_printf("%s: ERROR #%d (%s): %s\n", + fcli.appName, errRc, fsl_rc_cstr(errRc), msg); + } + } + if(clear) fcli_err_reset(); + return errRc; +} + + +const char * fcli_next_arg(bool remove){ + const char * rc = (fcli.argc>0) ? fcli.argv[0] : NULL; + if(rc && remove){ + int i; + --fcli.argc; + for(i = 0; i < fcli.argc; ++i){ + fcli.argv[i] = fcli.argv[i+1]; + } + fcli.argv[fcli.argc] = NULL/*owned by FCliFree*/; + } + return rc; +} + +int fcli_has_unused_args(bool outputError){ + int rc = 0; + if(fcli.argc){ + rc = fsl_cx_err_set(fcli.f, FSL_RC_MISUSE, + "Unhandled extra argument: %s", + fcli.argv[0]); + if(outputError){ + fcli_err_report(false); + } + } + return rc; +} +int fcli_has_unused_flags(bool outputError){ + int i; + for( i = 0; i < fcli.argc; ++i ){ + char const * arg = fcli.argv[i]; + if('-'==*arg){ + int rc = fsl_cx_err_set(fcli.f, FSL_RC_MISUSE, + "Unhandled/unknown flag or missing value: %s", + arg); + if(outputError){ + fcli_err_report(false); + } + return rc; + } + } + return 0; +} + +int fcli_err_set(int code, char const * fmt, ...){ + int rc; + va_list va; + va_start(va, fmt); + rc = fsl_error_setv(fcli__error, code, fmt, va); + va_end(va); + return rc; +} + +int fcli_end_of_main(int mainRc){ + if(FCLI_RC_HELP==mainRc){ + mainRc = 0; + } + if(fcli_err_report(true)){ + return EXIT_FAILURE; + }else if(mainRc){ + fcli_err_set(mainRc,"Ending with unadorned end-of-app " + "error code %d/%s.", + mainRc, fsl_rc_cstr(mainRc)); + fcli_err_report(true); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} + +int fcli_dispatch_commands( fcli_command const * cmd, + bool reportErrors){ + int rc = 0; + const char * arg = fcli_next_arg(0); + fcli_command const * orig = cmd; + fcli_command const * helpPos = 0; + int helpState = 0; + if(!arg){ + return fcli_err_set(FSL_RC_MISUSE, + "Missing command argument. Try --help."); + } + assert(fcli.f); + for(; arg && cmd->name; ++cmd ){ + if(cmd==orig && 0==fsl_strcmp(arg,"help")){ + /* Accept either (help command) or (command help) as help. */ + /* Except that it turns out that fcli_setup() will trump the + former and doesn't have the fcli_command state, so can't do + this. Maybe we can change that somehow. */ + helpState = 1; + helpPos = orig; + arg = fcli_next_arg(1); // consume it + }else if(0==fsl_strcmp(arg,cmd->name) || fcli_cmd_aliascmp(cmd,arg)){ + if(!cmd->f){ + rc = fcli_err_set(FSL_RC_NYI, + "Command [%s] has no " + "callback function."); + }else{ + fcli_next_arg(1)/*consume it*/; + if(helpState){ + assert(1==helpState); + helpState = 2; + helpPos = cmd; + break; + } + const char * helpCheck = fcli_next_arg(false); + if(helpCheck && 0==fsl_strcmp("help",helpCheck)){ + helpState = 3; + helpPos = cmd; + break; + }else{ + rc = cmd->f(cmd); + } + } + break; + } + } + if(helpState){ + f_out("\n"); + fcli_command_help(helpPos, true, helpState>1); + }else if(!cmd->name){ + fsl_buffer msg = fsl_buffer_empty; + int rc2; + if(!arg){ + rc2 = FSL_RC_MISUSE; + fsl_buffer_appendf(&msg, "No command provided."); + }else{ + rc2 = FSL_RC_NOT_FOUND; + fsl_buffer_appendf(&msg, "Command not found: %s.",arg); + } + fsl_buffer_appendf(&msg, " Available commands: "); + cmd = orig; + for( ; cmd && cmd->name; ++cmd ){ + fsl_buffer_appendf( &msg, "%s%s", + (cmd==orig) ? "" : ", ", + cmd->name); + } + rc = fcli_err_set(rc2, "%b", &msg); + fsl_buffer_clear(&msg); + } + if(rc && reportErrors){ + fcli_err_report(0); + } + return rc; +} + +bool fcli_cmd_aliascmp(fcli_command const * cmd, char const * arg){ + char const * alias = cmd->aliases; + while ( alias && *alias!=0 ){ + if( 0==fsl_strcmp(alias, arg) ){ + return true; + } + alias = strchr(alias, 0) + 1; + } + return false; +} + +void fcli_command_help(fcli_command const * cmd, bool showUsage, bool onlyOne){ + fcli_command const * c = cmd; + for( ; c->name; ++c ){ + f_out("[%s] command:\n\n", c->name); + if(c->briefDescription){ + f_out(" %s\n", c->briefDescription); + } + if(c->aliases){ + fcli_help_show_aliases(c->aliases); + }else{ + f_out("\n"); + } + if(c->flags){ + f_out("\n"); + fcli_cliflag_help(c->flags); + } + if(showUsage && c->usage){ + c->usage(); + } + if(onlyOne) break; + } +} + +void fcli_help_show_aliases(char const * aliases){ + char const * alias = aliases; + f_out(" (aliases: "); + while ( *alias!=0 ){ + f_out("%s%s", alias, *(strchr(alias, 0) + 1) ? ", " : ")\n"); + alias = strchr(alias, 0) + 1; + } +} + +void fcli_fax(void * mem){ + if(mem){ + fsl_list_append( &FCliFree.list, mem ); + } +} + +int fcli_ckout_show_info(bool useUtc){ + fsl_cx * const f = fcli_cx(); + int rc = 0; + fsl_stmt st = fsl_stmt_empty; + fsl_db * const dbR = fsl_cx_db_repo(f); + fsl_db * const dbC = fsl_cx_db_ckout(f); + int lblWidth = -20; + if(!fsl_needs_ckout(f)){ + return FSL_RC_NOT_A_CKOUT; + } + assert(dbR); + assert(dbC); + + fsl_id_t rid = 0; + fsl_uuid_cstr uuid = NULL; + fsl_ckout_version_info(f, &rid, &uuid); + assert((uuid && (rid>0)) || (!uuid && (0==rid))); + + f_out("%*s %s\n", lblWidth, "repository-db:", + fsl_cx_db_file_repo(f, NULL)); + f_out("%*s %s\n", lblWidth, "checkout-root:", + fsl_cx_ckout_dir_name(f, NULL)); + + rc = fsl_db_prepare(dbR, &st, "SELECT " + /*0*/"datetime(event.mtime%s) AS timestampString, " + /*1*/"coalesce(euser, user) AS user, " + /*2*/"(SELECT group_concat(substr(tagname,5), ', ') FROM tag, tagxref " + "WHERE tagname GLOB 'sym-*' AND tag.tagid=tagxref.tagid " + "AND tagxref.rid=blob.rid AND tagxref.tagtype>0) as tags, " + /*3*/"coalesce(ecomment, comment) AS comment, " + /*4*/"uuid AS uuid " + "FROM event JOIN blob " + "WHERE " + "event.type='ci' " + "AND blob.rid=%"FSL_ID_T_PFMT" " + "AND blob.rid=event.objid " + "ORDER BY event.mtime DESC", + useUtc ? "" : ", 'localtime'", + rid); + if(rc) goto dberr; + if( FSL_RC_STEP_ROW != fsl_stmt_step(&st)){ + /* fcli_err_set(FSL_RC_ERROR, "Event data for checkout not found."); */ + f_out("\nNo 'event' data found. This is only normal for an empty repo.\n"); + goto end; + + } + + f_out("%*s %s %s %s (RID %"FSL_ID_T_PFMT")\n", + lblWidth, "checkout-version:", + fsl_stmt_g_text(&st, 4, NULL), + fsl_stmt_g_text(&st, 0, NULL), + useUtc ? "UTC" : "local", + rid ); + + { + /* list parent(s) */ + fsl_stmt stP = fsl_stmt_empty; + rc = fsl_db_prepare(dbR, &stP, "SELECT " + "uuid, pid, isprim " + "FROM plink JOIN blob ON pid=rid " + "WHERE cid=%"FSL_ID_T_PFMT" " + "ORDER BY isprim DESC, mtime DESC /*sort*/", + rid); + if(rc) goto dberr; + while( FSL_RC_STEP_ROW == fsl_stmt_step(&stP) ){ + char const * zLabel = fsl_stmt_g_int32(&stP,2) + ? "parent:" : "merged-from:"; + f_out("%*s %s\n", lblWidth, zLabel, + fsl_stmt_g_text(&stP, 0, NULL)); + + } + fsl_stmt_finalize(&stP); + } + { + /* list children */ + fsl_stmt stC = fsl_stmt_empty; + rc = fsl_db_prepare(dbR, &stC, "SELECT " + "uuid, cid, isprim " + "FROM plink JOIN blob ON cid=rid " + "WHERE pid=%"FSL_ID_T_PFMT" " + "ORDER BY isprim DESC, mtime DESC /*sort*/", + rid); + if(rc) goto dberr; + while( FSL_RC_STEP_ROW == fsl_stmt_step(&stC) ){ + char const * zLabel = fsl_stmt_g_int32(&stC,2) + ? "child:" : "merged-into:"; + f_out("%*s %s\n", lblWidth, zLabel, + fsl_stmt_g_text(&stC, 0, NULL)); + + } + fsl_stmt_finalize(&stC); + } + + f_out("%*s %s\n", lblWidth, "user:", + fsl_stmt_g_text(&st, 1, NULL)); + + f_out("%*s %s\n", lblWidth, "tags:", + fsl_stmt_g_text(&st, 2, NULL)); + + f_out("%*s %s\n", lblWidth, "comment:", + fsl_stmt_g_text(&st, 3, NULL)); + + dberr: + if(rc){ + fsl_cx_uplift_db_error(f, dbR); + } + end: + fsl_stmt_finalize(&st); + + return rc; +} + +static int fsl_stmt_each_f_ambiguous( fsl_stmt * stmt, void * state ){ + int rc; + if(1==stmt->rowCount) stmt->rowCount=0 + /* HORRIBLE KLUDGE to elide header. */; + rc = fsl_stmt_each_f_dump(stmt, state); + if(0==stmt->rowCount) stmt->rowCount = 1; + return rc; +} + +void fcli_list_ambiguous_artifacts(char const * label, + char const *prefix){ + fsl_db * const db = fsl_cx_db_repo(fcli.f); + assert(db); + if(!label){ + f_out("Artifacts matching ambiguous prefix: %s\n",prefix); + }else if(*label){ + f_out("%s\n", label); + } + /* Possible fixme? Do we only want to list checkins + here? */ + int rc = fsl_db_each(db, fsl_stmt_each_f_ambiguous, 0, + "SELECT uuid, CASE " + "WHEN type='ci' THEN 'Checkin' " + "WHEN type='w' THEN 'Wiki' " + "WHEN type='g' THEN 'Control' " + "WHEN type='e' THEN 'Technote' " + "WHEN type='t' THEN 'Ticket' " + "WHEN type='f' THEN 'Forum' " + "ELSE '?'||'?'||'?' END " /* '???' ==> trigraph! */ + "FROM blob b, event e WHERE uuid LIKE %Q||'%%' " + "AND b.rid=e.objid " + "ORDER BY uuid", + prefix); + if(rc){ + fsl_cx_uplift_db_error(fcli.f, db); + fcli_err_report(false); + } +} + +fsl_db * fcli_db_ckout(void){ + return fcli.f ? fsl_cx_db_ckout(fcli.f) : NULL; +} + +fsl_db * fcli_db_repo(void){ + return fcli.f ? fsl_cx_db_repo(fcli.f) : NULL; +} + +fsl_db * fcli_needs_ckout(void){ + if(fcli.f) return fsl_needs_ckout(fcli.f); + fcli_err_set(FSL_RC_NOT_A_CKOUT, + "No checkout db is opened."); + return NULL; +} + +fsl_db * fcli_needs_repo(void){ + if(fcli.f) return fsl_needs_repo(fcli.f); + fcli_err_set(FSL_RC_NOT_A_REPO, + "No repository db is opened."); + return NULL; +} + +int fcli_args_to_vfile_ids(fsl_id_bag *tgt, fsl_id_t vid, + bool relativeToCwd, + bool changedFilesOnly){ + if(!fcli.argc){ + return fcli_err_set(FSL_RC_MISUSE, + "No file/dir name arguments provided."); + } + int rc = 0; + char const * zName; + while( !rc && (zName = fcli_next_arg(true))){ + FCLI_V3(("Collecting vfile ID(s) for: %s\n", zName)); + rc = fsl_ckout_vfile_ids(fcli.f, vid, tgt, zName, + relativeToCwd, changedFilesOnly); + } + return rc; +} + +int fcli_fingerprint_check(bool reportImmediately){ + int rc = fsl_ckout_fingerprint_check(fcli.f); + if(rc && reportImmediately){ + f_out("ERROR: repo/checkout fingerprint mismatch detected. " + "To recover from this, (fossil close) the current checkout, " + "then re-open it. Be sure to store any modified files somewhere " + "safe and restore them after re-opening the repository.\n"); + } + return rc; +} + +char const * fcli_progname(){ + if(!fcli.appName || !*fcli.appName) return NULL; + char const * z = fcli.appName; + char const * zEnd = z + fsl_strlen(z) - 1; + for( ; zEnd > z; --zEnd ){ + switch((int)*zEnd){ + case (int)'/': + case (int)'\\': + return zEnd+1; + default: break; + } + } + return zEnd; +} + +void fcli_diff_colors(fsl_diff_opt * const tgt, fcli_diff_colors_e theme){ + char const * zIns = 0; + char const * zEdit = 0; + char const * zDel = 0; + char const * zReset = 0; + switch(theme){ + case FCLI_DIFF_COLORS_RG: + zIns = "\x1b[32m"; + zEdit = "\x1b[36m"; + zDel = "\x1b[31m"; + zReset = "\x1b[0m"; + break; + case FCLI_DIFF_COLORS_NONE: + default: break; + } + tgt->ansiColor.insertion = zIns; + tgt->ansiColor.edit = zEdit; + tgt->ansiColor.deletion = zDel; + tgt->ansiColor.reset = zReset; +} + +void fcli_dump_stmt_cache(bool forceVerbose){ + int i = 0; + fsl_stmt * st; + fsl_db * const db = fsl_cx_db(fcli_cx()); + assert(db); + for( st = db->cacheHead; st; st = st->next ) ++i; + f_out("%s(): Cached fsl_stmt count: %d\n", __func__, i); + if(i>0 && (forceVerbose || fcli_is_verbose()>1)){ + for( i = 1, st = db->cacheHead; st; ++i, st = st->next ){ + f_out("CACHED fsl_stmt #%d (%d hit(s)): %b\n", i, + (int)st->cachedHits, &st->sql); + } + } +} + +#undef FCLI_V3 +#undef fcli_empty_m +#undef fcli__error +#undef MARKER +/* end of file cli.c */ +/* start of file content.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************** + This file houses the code for the fsl_content_xxx() APIS. +*/ +#include +#include /* memcmp() */ + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + + +fsl_int_t fsl_content_size( fsl_cx * f, fsl_id_t blobRid ){ + fsl_db * dbR = f ? fsl_cx_db_repo(f) : NULL; + if(!f) return -3; + else if(blobRid<=0) return -4; + else if(!dbR) return -5; + else{ + int rc; + fsl_int_t rv = -2; + fsl_stmt * q = NULL; + rc = fsl_db_prepare_cached(dbR, &q, + "SELECT size FROM blob " + "WHERE rid=? " + "/*%s()*/",__func__); + if(!rc){ + rc = fsl_stmt_bind_id(q, 1, blobRid); + if(!rc){ + if(FSL_RC_STEP_ROW==fsl_stmt_step(q)){ + rv = (fsl_int_t)fsl_stmt_g_int64(q, 0); + } + } + fsl_stmt_cached_yield(q); + } + return rv; + } +} + +bool fsl_content_is_available(fsl_cx * f, fsl_id_t rid){ + fsl_id_t srcid = 0; + int rc = 0, depth = 0 /* Limit delta recursion depth */; + while( depth++ < 100000 ){ + if( fsl_id_bag_contains(&f->cache.arty.missing, rid) ){ + return false; + }else if( fsl_id_bag_contains(&f->cache.arty.available, rid) ){ + return true; + }else if( fsl_content_size(f, rid)<0 ){ + fsl_id_bag_insert(&f->cache.arty.missing, rid) + /* ignore possible OOM error */; + return false; + } + rc = fsl_delta_src_id(f, rid, &srcid); + if(rc) break; + else if( 0==srcid ){ + fsl_id_bag_insert(&f->cache.arty.available, rid); + return true; + } + rid = srcid; + } + if(0==rc){ + /* This "cannot happen" (never has historically, and would be + indicative of what amounts to corruption in the repo). */ + fsl_fatal(FSL_RC_RANGE,"delta-loop in repository"); + } + return false; +} + + + +int fsl_content_blob( fsl_cx * f, fsl_id_t blobRid, fsl_buffer * tgt ){ + fsl_db * const dbR = fsl_cx_db_repo(f); + if(blobRid<=0) return FSL_RC_RANGE; + else if(!dbR) return FSL_RC_NOT_A_REPO; + else{ + int rc; + fsl_stmt * q = NULL; + rc = fsl_db_prepare_cached( dbR, &q, + "SELECT content, size FROM blob " + "WHERE rid=?" + "/*%s()*/",__func__); + if(!rc){ + rc = fsl_stmt_bind_id(q, 1, blobRid); + if(!rc && (FSL_RC_STEP_ROW==(rc=fsl_stmt_step(q)))){ + void const * mem = NULL; + fsl_size_t memLen = 0; + if(fsl_stmt_g_int64(q, 1)<0){ + rc = fsl_cx_err_set(f, FSL_RC_PHANTOM, + "Cannot fetch content for phantom " + "blob #%"FSL_ID_T_PFMT".", + blobRid); + }else{ + tgt->used = 0; + fsl_stmt_get_blob(q, 0, &mem, &memLen); + if(mem && memLen){ + rc = fsl_buffer_append(tgt, mem, memLen); + if(!rc && fsl_buffer_is_compressed(tgt)){ + rc = fsl_buffer_uncompress(tgt, tgt); + } + } + } + }else if(FSL_RC_STEP_DONE==rc){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "No blob found for rid %"FSL_ID_T_PFMT".", + blobRid); + } + fsl_stmt_cached_yield(q); + } + if(rc && !f->error.code && dbR->error.code){ + fsl_cx_uplift_db_error(f, dbR); + } + return rc; + } +} + + +bool fsl_content_is_private(fsl_cx * const f, fsl_id_t rid){ + fsl_stmt * s1 = NULL; + fsl_db * db = fsl_cx_db_repo(f); + int rc = db + ? fsl_db_prepare_cached(db, &s1, + "SELECT 1 FROM private " + "WHERE rid=?" + "/*%s()*/",__func__) + : FSL_RC_MISUSE; + if(!rc){ + rc = fsl_stmt_bind_id(s1, 1, rid); + if(!rc) rc = fsl_stmt_step(s1); + fsl_stmt_cached_yield(s1); + } + return rc==FSL_RC_STEP_ROW ? true : false; +} + + +int fsl_content_get( fsl_cx * const f, fsl_id_t rid, + fsl_buffer * const tgt ){ + fsl_db * const db = fsl_cx_db_repo(f); + if(!tgt) return FSL_RC_MISUSE; + else if(rid<=0){ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "RID %"FSL_ID_T_PFMT" is out of range.", + rid); + } + else if(!db){ + return fsl_cx_err_set(f, FSL_RC_NOT_A_REPO, + "Fossil has no repo opened."); + } + else{ + int rc; + bool gotIt = 0; + fsl_id_t nextRid; + fsl_acache * const ac = &f->cache.arty; + fsl_buffer_reuse(tgt); + if(fsl_id_bag_contains(&ac->missing, rid)){ + /* Early out if we know the content is not available */ + return FSL_RC_NOT_FOUND; + } + + /* Look for the artifact in the cache first */ + if( fsl_id_bag_contains(&ac->inCache, rid) ){ + fsl_size_t i; + fsl_acache_line * line; + for(i=0; iused; ++i){ + line = &ac->list[i]; + if( line->rid==rid ){ + rc = fsl_buffer_copy(&line->content, tgt); + line->age = ac->nextAge++; + return rc; + } + } + } + + nextRid = 0; + rc = fsl_delta_src_id(f, rid, &nextRid); + /* MARKER(("rc=%d, nextRid=%"FSL_ID_T_PFMT"\n", rc, nextRid)); */ + if(rc) return rc; + if( nextRid == 0 ){ + /* This is not a delta, so get its raw content. */ + rc = fsl_content_blob(f, rid, tgt); + gotIt = 0==rc; + }else{ + /* Looks like a delta, so let's expand it... */ + fsl_int_t n /* number of used entries in 'a' */; + fsl_int_t nAlloc = 10 /* number it items allocated in 'a' */; + fsl_id_t * a = NULL /* array of rids we expand */; + fsl_int_t mx; + fsl_buffer delta = fsl_buffer_empty; + fsl_buffer next = fsl_buffer_empty /* delta-applied content */ ; + assert(nextRid>0); + a = fsl_malloc( sizeof(a[0]) * nAlloc ); + if(!a) return FSL_RC_OOM; + a[0] = rid; + a[1] = nextRid; + n = 1; + while( !fsl_id_bag_contains(&ac->inCache, nextRid) + && !fsl_delta_src_id(f, nextRid, &nextRid) + && (nextRid>0)){ + /* Figure out how big n needs to be... */ + ++n; + if( n >= nAlloc ){ + /* Expand 'a' */ + void * remem; + if( n > fsl_db_g_int64(db, 0, + "SELECT max(rid) FROM blob")){ + rc = fsl_cx_err_set(f, FSL_RC_RANGE, + "Infinite loop in delta table."); + goto end_delta; + } + nAlloc = nAlloc * 2; + remem = fsl_realloc(a, nAlloc*sizeof(a[0])); + if(!remem){ + rc = FSL_RC_OOM; + goto end_delta; + } + a = (fsl_id_t*)remem; + } + a[n] = nextRid; + } + /** + Recursively expand deltas to get the content... + */ + mx = n; + rc = fsl_content_get( f, a[n], tgt ); + /* MARKER(("Getting content for rid #%"FSL_ID_T_PFMT", rc=%d\n", a[n], rc)); */ + --n; + for( ; !rc && (n>=0); --n){ + rc = fsl_content_blob(f, a[n], &delta); + /* MARKER(("Getting/applying delta rid #%"FSL_ID_T_PFMT", rc=%d\n", a[n], rc)); */ + if(rc) goto end_delta; + if(!delta.used){ + assert(!"Is this possible? The fossil tree has a similar " + "condition but i naively don't believe it's necessary."); + continue; + } + next = fsl_buffer_empty; + rc = fsl_buffer_delta_apply2(tgt, &delta, &next, &f->error); + if(rc) goto end_delta; +#if 1 + /* + In my (very simple) tests this cache costs us more than it + saves. TODO: re-test this once we can do a 'rebuild', or + something more intensive than processing a single + manifest's R-card. At that point we can set a f->flags bit + to enable or disable this block for per-use-case + optimization purposes. + + We also probably want to cache fsl_deck instances instead + of Manifest blobs (fsl_buffer) like fossil(1) does, + otherwise this cache really doesn't save us much + work/memory. + + 2021-03-24: in a debug build, running: + + f-parseparty -t c -c -q + + (i.e.: parse and crosslink all checkin artifacts) + + on the libfossil repo with 2003 checkins takes: + + 10.5s without this cache + 5.2s with this cache + + We shave another 0.5s if we always cache instead of using + this mysterious (mx-n)%8 heuristic. + */ + //MARKER(("mx=%d, n=%d, (mx-n)%%8=%d\n", + //(int)mx, (int)n, (int)(mx-n)%8)); + //MARKER(("nAlloc=%d\n", (int)nAlloc)); + if( (mx-n)%8==0 ){ + //MARKER(("Caching artifact %d\n", (int)a[n+1])); + rc = fsl_acache_insert( ac, a[n+1], tgt ); + if(rc){ + fsl_buffer_clear(&next); + goto end_delta; + } + assert(!tgt->mem && "Passed to artifact cache."); + }else{ + fsl_buffer_clear(tgt); + } +#else + if(mx){/*unused var*/} + fsl_buffer_clear(tgt); +#endif + *tgt = next; + } + end_delta: + fsl_buffer_clear(&delta); + fsl_free(a); + gotIt = 0==rc; + } + + if(!rc){ + rc = fsl_id_bag_insert(gotIt + ? &f->cache.arty.available + : &f->cache.arty.missing, + rid); + } + return rc; + } +} + +int fsl_content_get_sym( fsl_cx * const f, char const * sym, + fsl_buffer * const tgt ){ + int rc; + fsl_db * db = f ? fsl_needs_repo(f) : NULL; + fsl_id_t rid = 0; + if(!f || !sym || !tgt) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + rc = fsl_sym_to_rid(f, sym, FSL_SATYPE_ANY, &rid); + return rc ? rc : fsl_content_get(f, rid, tgt); +} + +/** + Mark artifact rid as being available now. Update f's cache to show + that everything that was formerly unavailable because rid was + missing is now available. Returns 0 on success. f must have + an opened repo and rid must be valid. + */ +static int fsl_content_mark_available(fsl_cx * f, fsl_id_t rid){ + fsl_id_bag pending = fsl_id_bag_empty; + int rc; + fsl_stmt * st = NULL; + fsl_db * db = fsl_cx_db_repo(f); + assert(f); + assert(db); + assert(rid>0); + if( fsl_id_bag_contains(&f->cache.arty.available, rid) ) return 0; + rc = fsl_id_bag_insert(&pending, rid); + if(rc) goto end; + while( (rid = fsl_id_bag_first(&pending))!=0 ){ + fsl_id_bag_remove(&pending, rid); + rc = fsl_id_bag_insert(&f->cache.arty.available, rid); + if(rc) goto end; + fsl_id_bag_remove(&f->cache.arty.missing, rid); + if(!st){ + rc = fsl_db_prepare_cached(db, &st, + "SELECT rid FROM delta " + "WHERE srcid=?" + "/*%s()*/",__func__); + if(rc) goto end; + } + rc = fsl_stmt_bind_id(st, 1, rid); + while( !rc && (FSL_RC_STEP_ROW==fsl_stmt_step(st)) ){ + fsl_id_t const nx = fsl_stmt_g_id(st,0); + assert(nx>0); + rc = fsl_id_bag_insert(&pending, nx); + } + + } + end: + if(st) fsl_stmt_cached_yield(st); + fsl_id_bag_clear(&pending); + return rc; +} + +/** + When a record is converted from a phantom to a real record, if that + record has other records that are derived by delta, then call + fsl_deck_crosslink() on those other records. + + If the formerly phantom record or any of the other records derived + by delta from the former phantom are a baseline manifest, then also + invoke fsl_deck_crosslink() on the delta-manifests associated with + that baseline. + + Tail recursion is used to minimize stack depth. + + Returns 0 on success, any number of non-0 results on error. + + The 3rd argument must always be false except in recursive calls to + this function. +*/ +static int fsl_after_dephantomize(fsl_cx * f, fsl_id_t rid, bool doCrosslink){ + int rc = 0; + unsigned nChildAlloc = 0; + fsl_id_t * aChild = 0; + fsl_buffer bufChild = fsl_buffer_empty; + fsl_db * const db = fsl_cx_db_repo(f); + fsl_stmt q = fsl_stmt_empty; + + MARKER(("WARNING: fsl_after_dephantomization() is UNTESTED.\n")); + if(f->cache.ignoreDephantomizations) return 0; + while(rid){ + unsigned nChildUsed = 0; + unsigned i = 0; + + /* Parse the object rid itself */ + if(doCrosslink){ + fsl_deck deck = fsl_deck_empty; + rc = fsl_deck_load_rid(f, &deck, rid, FSL_SATYPE_ANY); + if(!rc){ + assert(aChild[i]==deck.rid); + rc = fsl_deck_crosslink(&deck); + } + fsl_deck_finalize(&deck); + if(rc) break; + } + /* Parse all delta-manifests that depend on baseline-manifest rid */ + rc = fsl_db_prepare(db, &q, + "SELECT rid FROM orphan WHERE baseline=%"FSL_ID_T_PFMT, + rid); + if(rc) break; + while(FSL_RC_STEP_ROW==fsl_stmt_step(&q)){ + fsl_id_t const child = fsl_stmt_g_id(&q, 0); + if(nChildUsed>=nChildAlloc){ + nChildAlloc = nChildAlloc ? nChildAlloc*2 : 10; + rc = fsl_buffer_reserve(&bufChild, sizeof(fsl_id_t)*nChildAlloc); + if(rc) goto end; + aChild = (fsl_id_t*)bufChild.mem; + } + aChild[nChildUsed++] = child; + } + fsl_stmt_finalize(&q); + for(i=0; i=nChildAlloc){ + nChildAlloc = nChildAlloc ? nChildAlloc*2 : 10; + rc = fsl_buffer_reserve(&bufChild, sizeof(fsl_id_t)*nChildAlloc); + if(rc) goto end; + aChild = (fsl_id_t*)bufChild.mem; + } + aChild[nChildUsed++] = child; + } + fsl_stmt_finalize(&q); + for(i=1; i0 ? aChild[0] : 0; + doCrosslink = true; + } + end: + fsl_stmt_finalize(&q); + fsl_buffer_clear(&bufChild); + return rc; +} + +int fsl_content_put_ex( fsl_cx * const f, + fsl_buffer const * pBlob, + fsl_uuid_cstr zUuid, + fsl_id_t srcId, + fsl_size_t uncompSize, + bool isPrivate, + fsl_id_t * outRid){ + fsl_size_t size; + fsl_id_t rid; + fsl_stmt * s1 = NULL; + fsl_buffer cmpr = fsl_buffer_empty; + fsl_buffer hash = fsl_buffer_empty; + bool markAsUnclustered = false; + bool markAsUnsent = true; + bool isDephantomize = false; + fsl_db * dbR = fsl_cx_db_repo(f); + int const zUuidLen = zUuid ? fsl_is_uuid(zUuid) : 0; + int rc = 0; + bool inTrans = false; + assert(f); + assert(dbR); + assert(pBlob); + assert(srcId==0 || zUuid!=NULL); + assert(!zUuid || zUuidLen); + if(!dbR) return FSL_RC_NOT_A_REPO; + static const fsl_size_t MaxSize = 0x70000000; + if(pBlob->used>=MaxSize || uncompSize>=MaxSize){ + /* fossil(1) uses int for all blob sizes, and therefore has a + hard-coded limit of 2GB max size per blob. That property of the + API is well-entrenched, and correcting it properly, including + all algorithms which access blobs using integer indexes, would + require a large coding effort with a non-trivial risk of + lingering, difficult-to-trace bugs. + + For compatibility, we limit ourselves to 2GB, but to ensure a + bit of leeway, we set our limit slightly less than 2GB. + */ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "For compatibility with fossil(1), " + "blobs may not exceed %d bytes in size.", + (int)MaxSize); + } + if(!zUuid){ + assert(0==uncompSize); + /* "auxiliary hash" bits from: + https://fossil-scm.org/fossil/file?ci=c965636958eb58aa&name=src%2Fcontent.c&ln=527-537 + */ + /* First check the auxiliary hash to see if there is already an artifact + ** that uses the auxiliary hash name */ + /* 2021-04-13: we can now use fsl_repo_blob_lookup() to do this, + but the following code is known to work, so touching it is a + low priority. */ + rc = fsl_cx_hash_buffer(f, true, pBlob, &hash); + if(FSL_RC_UNSUPPORTED==rc) rc = 0; + else if(rc) goto end; + assert(hash.used==0 || hash.used>=FSL_STRLEN_SHA1); + rid = hash.used ? fsl_uuid_to_rid(f, fsl_buffer_cstr(&hash)) : 0; + assert(rid>=0 && "Cannot have malformed/ambiguous UUID at this point."); + if(!rid){ + /* No existing artifact with the auxiliary hash name. Therefore, use + ** the primary hash name. */ + hash.used = 0; + rc = fsl_cx_hash_buffer(f, false, pBlob, &hash); + if(rc) goto end; + assert(hash.used>=FSL_STRLEN_SHA1); + } + }else{ + rc = fsl_buffer_append(&hash, zUuid, zUuidLen); + if(rc) goto end; + } + assert(!rc); + if(uncompSize){ + /* pBlob is assumed to be compressed. */ + assert(fsl_buffer_is_compressed(pBlob)); + size = uncompSize; + }else{ + size = pBlob->used; + if(srcId>0){ + rc = fsl_delta_applied_size(pBlob->mem, pBlob->used, &size); + if(rc) goto end; + } + } + rc = fsl_db_transaction_begin(dbR); + if(rc) goto end; + inTrans = true; + if( f->cxConfig.hashPolicy==FSL_HPOLICY_AUTO && hash.used>FSL_STRLEN_SHA1 ){ + fsl_cx_err_reset(f); + fsl_cx_hash_policy_set(f, FSL_HPOLICY_SHA3); + if((rc = f->error.code)){ + goto end; + } + } + /* Check to see if the entry already exists and if it does whether + or not the entry is a phantom. */ + rc = fsl_db_prepare_cached(dbR, &s1, + "SELECT rid, size FROM blob " + "WHERE uuid=?" + "/*%s()*/",__func__); + if(rc) goto end; + rc = fsl_stmt_bind_step( s1, "b", &hash); + switch(rc){ + case FSL_RC_STEP_ROW: + rc = 0; + rid = fsl_stmt_g_id(s1, 0); + if( fsl_stmt_g_int64(s1, 1)>=0 ){ + /* The entry is not a phantom. There is nothing for us to do + other than return the RID. + */ + /* + Reminder: the do-nothing-for-empty-phantom behaviour is + arguable (but historical). There is a corner case there + involving an empty file. So far, so good, though. After + all... all empty files have the same hash. + */ + fsl_stmt_cached_yield(s1); + assert(inTrans); + fsl_db_transaction_end(dbR,0); + if(outRid) *outRid = rid; + fsl_buffer_clear(&hash); + return 0; + } + break; + case 0: + /* No entry with the same UUID currently exists */ + rid = 0; + markAsUnclustered = true; + break; + default: + goto end; + } + if(s1){ + fsl_stmt_cached_yield(s1); + s1 = NULL; + } + if(rc) goto end; + +#if 0 + /* Requires app-level data. We might need a client hook mechanism or + other metadata here. + */ + /* Construct a received-from ID if we do not already have one */ + if( f->cache.rcvid <= 0 ){ + /* FIXME: use cached statement. */ + rc = fsl_db_exec(dbR, + "INSERT INTO rcvfrom(uid, mtime, nonce, ipaddr)" + "VALUES(%d, julianday('now'), %Q, %Q)", + g.userUid, g.zNonce, g.zIpAddr + ); + f->cache.rcvid = fsl_db_last_insert_id(dbR); + } +#endif + + if( uncompSize ){ + cmpr = *pBlob; + }else{ + rc = fsl_buffer_compress(pBlob, &cmpr); + if(rc) goto end; + } + + if( rid>0 ){ +#if 0 + assert(!"NYI: adding data to phantom. Requires some missing pieces."); + rc = fsl_cx_err_set(f, FSL_RC_NYI, + "NYI: adding data to phantom. " + "Requires missing rcvId pieces."); + goto end; +#else + /* We are just adding data to a phantom */ + rc = fsl_db_prepare_cached(dbR, &s1, + "UPDATE blob SET " + "rcvid=?, size=?, content=? " + "WHERE rid=?" + "/*%s()*/",__func__); + if(rc) goto end; + rc = fsl_stmt_bind_step(s1, "RIBR", f->cache.rcvId, (int64_t)size, + &cmpr, rid); + if(!rc){ + rc = fsl_db_exec(dbR, "DELETE FROM phantom " + "WHERE rid=%"FSL_ID_T_PFMT, rid + /* FIXME? use cached statement? */); + if( !rc && (srcId==0 || + 0==fsl_acache_check_available(f, srcId)) ){ + isDephantomize = true; + rc = fsl_content_mark_available(f, rid); + } + } + fsl_stmt_cached_yield(s1); + s1 = NULL; + if(rc) goto end; +#endif + }else{ + /* We are creating a new entry */ + rc = fsl_db_prepare_cached(dbR, &s1, + "INSERT INTO blob " + "(rcvid,size,uuid,content) " + "VALUES(?,?,?,?)" + "/*%s()*/",__func__); + if(rc) goto end; + rc = fsl_stmt_bind_step(s1, "RIbB", f->cache.rcvId, (int64_t)size, + &hash, &cmpr); + if(!rc){ + rid = fsl_db_last_insert_id(dbR); + if(!pBlob ){ + rc = fsl_db_exec_multi(dbR,/* FIXME? use cached statement? */ + "INSERT OR IGNORE INTO phantom " + "VALUES(%"FSL_ID_T_PFMT")", + rid); + markAsUnsent = false; + } + if( !rc && (f->cache.markPrivate || isPrivate) ){ + rc = fsl_db_exec_multi(dbR,/* FIXME? use cached statement? */ + "INSERT INTO private " + "VALUES(%"FSL_ID_T_PFMT")", + rid); + markAsUnclustered = false; + markAsUnsent = false; + } + } + if(rc) rc = fsl_cx_uplift_db_error2(f, dbR, rc); + fsl_stmt_cached_yield(s1); + s1 = NULL; + if(rc) goto end; + } + + /* If the srcId is specified, then the data we just added is + really a delta. Record this fact in the delta table. + */ + if( srcId ){ + rc = fsl_db_prepare_cached(dbR, &s1, + "REPLACE INTO delta(rid,srcid) " + "VALUES(?,?)" + "/*%s()*/",__func__); + if(!rc){ + rc = fsl_stmt_bind_step(s1, "RR", rid, srcId); + if(rc) rc = fsl_cx_uplift_db_error2(f, dbR, rc); + fsl_stmt_cached_yield(s1); + s1 = NULL; + } + if(rc) goto end; + } + if( !isDephantomize + && fsl_id_bag_contains(&f->cache.arty.missing, rid) && + (srcId==0 || (0==fsl_acache_check_available(f,srcId)))){ + /* + TODO: document what this is for. + TODO: figure out what that is. + */ + rc = fsl_content_mark_available(f, rid); + if(rc) goto end; + } + if( isDephantomize ){ + rc = fsl_after_dephantomize(f, rid, false); + if(rc) goto end; + } + + /* Add the element to the unclustered table if has never been + previously seen. + */ + if( markAsUnclustered ){ + /* FIXME: use a cached statement. */ + rc = fsl_db_exec_multi(dbR, + "INSERT OR IGNORE INTO unclustered VALUES" + "(%"FSL_ID_T_PFMT")", rid); + if(rc) goto end; + } + + if( markAsUnsent ){ + /* FIXME: use a cached statement. */ + rc = fsl_db_exec(dbR, "INSERT OR IGNORE INTO unsent " + "VALUES(%"FSL_ID_T_PFMT")", rid); + if(rc) goto end; + } + + rc = fsl_repo_verify_before_commit(f, rid); + if(rc) goto end /* FSL_RC_OOM is basically the "only possible" failure + after this point. */; + /* Code after end: relies on the following 2 lines: */ + rc = fsl_db_transaction_end(dbR, false); + inTrans = false; + if(!rc){ + if(outRid) *outRid = rid; + } + end: + if(inTrans){ + assert(0!=rc); + fsl_db_transaction_end(dbR,true); + } + fsl_buffer_clear(&hash); + if(!uncompSize){ + fsl_buffer_clear(&cmpr); + }/* else cmpr.mem (if any) belongs to pBlob */ + return rc; +} + +int fsl_content_put( fsl_cx * const f, fsl_buffer const * pBlob, fsl_id_t * newRid){ + return fsl_content_put_ex(f, pBlob, NULL, 0, 0, 0, newRid); +} + +int fsl_uuid_is_shunned(fsl_cx * const f, fsl_uuid_cstr zUuid){ + fsl_db * db = fsl_cx_db_repo(f); + if( !db || zUuid==0 || zUuid[0]==0 ) return 0; + else if(FSL_HPOLICY_SHUN_SHA1==f->cxConfig.hashPolicy + && FSL_STRLEN_SHA1==fsl_is_uuid(zUuid)){ + return 1; + } + /* TODO? cached query */ + return 1==fsl_db_g_int32( db, 0, + "SELECT 1 FROM shun WHERE uuid=%Q", + zUuid); +} + +int fsl_content_new( fsl_cx * f, fsl_uuid_cstr uuid, bool isPrivate, + fsl_id_t * newId ){ + fsl_id_t rid = 0; + int rc; + fsl_db * db = fsl_cx_db_repo(f); + fsl_stmt * s1 = NULL, * s2 = NULL; + int const uuidLen = uuid ? fsl_is_uuid(uuid) : 0; + if(!f || !uuid) return FSL_RC_MISUSE; + else if(!uuidLen) return FSL_RC_RANGE; + if(!db) return FSL_RC_NOT_A_REPO; + if( fsl_uuid_is_shunned(f, uuid) ){ + return fsl_cx_err_set(f, FSL_RC_ACCESS, + "UUID is shunned: %s", uuid) + /* need new error code? */; + } + rc = fsl_db_transaction_begin(db); + if(rc) return rc; + + rc = fsl_db_prepare_cached(db, &s1, + "INSERT INTO blob(rcvid,size,uuid,content)" + "VALUES(0,-1,?,NULL)" + "/*%s()*/",__func__); + if(rc) goto end; + rc = fsl_stmt_bind_text(s1, 1, uuid, uuidLen, 0); + if(!rc) rc = fsl_stmt_step(s1); + fsl_stmt_cached_yield(s1); + if(FSL_RC_STEP_DONE!=rc) goto end; + else rc = 0; + rid = fsl_db_last_insert_id(db); + assert(rid>0); + rc = fsl_db_prepare_cached(db, &s2, + "INSERT INTO phantom VALUES (?)" + "/*%s()*/",__func__); + if(rc) goto end; + rc = fsl_stmt_bind_id(s2, 1, rid); + if(!rc) rc = fsl_stmt_step(s2); + fsl_stmt_cached_yield(s2); + if(FSL_RC_STEP_DONE!=rc) goto end; + else rc = 0; + + if( f->cache.markPrivate || isPrivate ){ + /* Should be seldom enough that we don't need to cache + this statement. */ + rc = fsl_db_exec(db, + "INSERT INTO private VALUES(%"FSL_ID_T_PFMT")", + (fsl_id_t)rid); + }else{ + fsl_stmt * s3 = NULL; + rc = fsl_db_prepare_cached(db, &s3, + "INSERT INTO unclustered VALUES(?)"); + if(!rc){ + rc = fsl_stmt_bind_id(s3, 1, rid); + if(!rc) rc = fsl_stmt_step(s3); + fsl_stmt_cached_yield(s3); + if(FSL_RC_STEP_DONE!=rc) goto end; + else rc = 0; + } + } + + if(!rc) rc = fsl_id_bag_insert(&f->cache.arty.missing, rid); + + end: + if(rc){ + if(db->error.code && !f->error.code){ + fsl_cx_uplift_db_error(f, db); + } + fsl_db_transaction_rollback(db); + } + else{ + rc = fsl_db_transaction_commit(db); + if(!rc && newId) *newId = rid; + else if(rc && !f->error.code){ + fsl_cx_uplift_db_error(f, db); + } + } + return rc; +} + +int fsl_content_undeltify(fsl_cx * const f, fsl_id_t rid){ + int rc; + fsl_db * db = f ? fsl_cx_db_repo(f) : NULL; + fsl_id_t srcid = 0; + fsl_buffer x = fsl_buffer_empty; + fsl_stmt s = fsl_stmt_empty; + if(!f) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + else if(rid<=0) return FSL_RC_RANGE; + rc = fsl_db_transaction_begin(db); + if(rc) return fsl_cx_uplift_db_error2(f, db, rc); + /* Reminder: the original impl does not do this in a + transaction, _possibly_ because it's only done from places + where a transaction is active (that's unconfirmed). + Nested transactions are very cheap, though. + */ + rc = fsl_delta_src_id( f, rid, &srcid ); + if(rc || srcid<=0) goto end; + rc = fsl_content_get(f, rid, &x); + if( rc || !x.used ) goto end; + /* TODO? use cached statements */ + rc = fsl_db_prepare(db, &s, + "UPDATE blob SET content=?," + " size=%" FSL_SIZE_T_PFMT + " WHERE rid=%" FSL_ID_T_PFMT, + x.used, rid); + if(rc) goto dberr; + rc = fsl_buffer_compress(&x, &x); + if(rc) goto end; + rc = fsl_stmt_bind_blob(&s, 1, x.mem, + (fsl_int_t)x.used, 0); + if(rc) goto dberr; + rc = fsl_stmt_step(&s); + if(FSL_RC_STEP_DONE==rc) rc = 0; + else goto dberr; + rc = fsl_db_exec(db, "DELETE FROM delta " + "WHERE rid=%"FSL_ID_T_PFMT, + (fsl_id_t)rid); + if(rc) goto dberr; +#if 0 + /* + fossil does not do this, but that seems like an inconsistency. + + On that topic Richard says: + + "When you undelta an artifact, however, it is then stored as + plain text. (Actually, as zlib compressed plain text.) There + is no possibility of delta loops or bugs in the delta encoder or + missing source artifacts. And so there is much less of a chance + of losing content. Hence, I didn't see the need to verify the + content of artifacts that are undelta-ed." + + Potential TODO: f->flags FSL_CX_F_PEDANTIC_VERIFICATION, which + enables the R-card and this check, and any similarly superfluous + ones. + */ + if(!rc) fsl_repo_verify_before_commit(f, rid); +#endif + end: + fsl_buffer_clear(&x); + fsl_stmt_finalize(&s); + if(rc) fsl_db_transaction_rollback(db); + else rc = fsl_db_transaction_commit(db); + return rc; + dberr: + assert(rc); + rc = fsl_cx_uplift_db_error2(f, db, rc); + goto end; +} + +int fsl_content_deltify(fsl_cx * f, fsl_id_t rid, + fsl_id_t srcid, bool force){ + fsl_id_t s; + fsl_buffer data = fsl_buffer_empty; + fsl_buffer src = fsl_buffer_empty; + fsl_buffer delta = fsl_buffer_empty; + fsl_db * db = f ? fsl_cx_db_repo(f) : NULL; + int rc = 0; + enum { MinSizeThreshold = 50 }; + if(!f) return FSL_RC_MISUSE; + else if(rid<=0 || srcid<=0) return FSL_RC_RANGE; + else if(!db) return FSL_RC_NOT_A_REPO; + else if( srcid==rid ) return 0; + else if(!fsl_content_is_available(f, rid)){ + return 0; + } + if(!force){ + fsl_id_t tmpRid = 0; + rc = fsl_delta_src_id(f, rid, &tmpRid); + if(tmpRid>0){ + /* + We already have a delta, it seems. Nothing left to do + :-D. Should we return FSL_RC_ALREADY_EXISTS here? + */ + return 0; + } + else if(rc) return rc; + } + + if( fsl_content_is_private(f, srcid) + && !fsl_content_is_private(f, rid) ){ + /* + See API doc comments about crossing the private/public + boundaries. Do we want to report okay here or + FSL_RC_ACCESS? Not yet sure how this routine is used. + + Since delitifying is an internal optimization/implementation + detail, it seems best to return 0 for this case. + */ + return 0; + } + /** + Undeltify srcid if needed... + */ + s = srcid; + while( (0==(rc=fsl_delta_src_id(f, s, &s))) + && (s>0) ){ + if( s==rid ){ + rc = fsl_content_undeltify(f, srcid); + break; + } + } + if(rc) return rc; + /* As of here, don't return on error. Use (goto end) instead, or be + really careful, b/c buffers might need cleaning. */ + rc = fsl_content_get(f, srcid, &src); + if(rc + || (src.used < MinSizeThreshold) + /* See API doc comments about minimum size to delta/undelta. */ + ) goto end; + rc = fsl_content_get(f, rid, &data); + if(rc || (data.used < MinSizeThreshold)) goto end; + rc = fsl_buffer_delta_create(&src, &data, &delta); + if( !rc && (delta.used <= (data.used * 3 / 4 /* 75% */))){ + fsl_stmt * s1 = NULL; + fsl_stmt * s2 = NULL; + rc = fsl_buffer_compress(&delta, &delta); + if(rc) goto end; + rc = fsl_db_prepare_cached(db, &s1, + "UPDATE blob SET content=? " + "WHERE rid=?/*%s()*/",__func__); + if(!rc){ + fsl_stmt_bind_id(s1, 2, rid); + rc = fsl_stmt_bind_blob(s1, 1, delta.mem, delta.used, 0); + if(!rc){ + rc = fsl_db_prepare_cached(db, &s2, + "REPLACE INTO delta(rid,srcid) " + "VALUES(?,?)/*%s()*/",__func__); + if(!rc){ + fsl_stmt_bind_id(s2, 1, rid); + fsl_stmt_bind_id(s2, 2, srcid); + rc = fsl_db_transaction_begin(db); + if(!rc){ + rc = fsl_stmt_step(s1); + if(FSL_RC_STEP_DONE==rc){ + rc = fsl_stmt_step(s2); + if(FSL_RC_STEP_DONE==rc) rc = 0; + } + if(!rc) rc = fsl_db_transaction_end(db, 0); + else fsl_db_transaction_end(db, 1) /* keep rc intact */; + } + } + } + } + fsl_stmt_cached_yield(s1); + fsl_stmt_cached_yield(s2); + if(!rc) fsl_repo_verify_before_commit(f, rid); + } + end: + if(rc && db->error.code && !f->error.code){ + fsl_cx_uplift_db_error(f,db); + } + fsl_buffer_clear(&src); + fsl_buffer_clear(&data); + fsl_buffer_clear(&delta); + return rc; +} + +/** + Removes all entries from the repo's blob table which are listed in + the shun table. + */ +int fsl_repo_shun_artifacts(fsl_cx * f){ + fsl_stmt q = fsl_stmt_empty; + int rc; + fsl_db * db = f ? fsl_cx_db_repo(f) : NULL; + if(!f) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + rc = fsl_db_transaction_begin(db); + if(rc) return rc; + rc = fsl_db_exec_multi(db, + "CREATE TEMP TABLE IF NOT EXISTS " + "toshun(rid INTEGER PRIMARY KEY);" + "INSERT INTO toshun SELECT rid FROM blob, shun " + "WHERE blob.uuid=shun.uuid;" + ); + if(rc) goto end; + /* Ensure that deltas generated from the to-be-shunned data + are unpacked into non-delta form... + */ + rc = fsl_db_prepare(db, &q, + "SELECT rid FROM delta WHERE srcid IN toshun" + ); + if(rc) goto end; + while( !rc && (FSL_RC_STEP_ROW==fsl_stmt_step(&q)) ){ + fsl_id_t const srcid = fsl_stmt_g_id(&q, 0); + rc = fsl_content_undeltify(f, srcid); + } + fsl_stmt_finalize(&q); + if(!rc){ + rc = fsl_db_exec_multi(db, + "DELETE FROM delta WHERE rid IN toshun;" + "DELETE FROM blob WHERE rid IN toshun;" + "DROP TABLE toshun;" + "DELETE FROM private " + "WHERE NOT EXISTS " + "(SELECT 1 FROM blob WHERE rid=private.rid);" + ); + } + end: + if(!rc) rc = fsl_db_transaction_commit(db); + else fsl_db_transaction_rollback(db); + if(rc && db->error.code && !f->error.code){ + rc = fsl_cx_uplift_db_error(f, db); + } + return rc; +} + +int fsl_content_make_public(fsl_cx * const f, fsl_id_t rid){ + int rc; + fsl_db * db = f ? fsl_cx_db_repo(f) : NULL; + if(!f) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + rc = fsl_db_exec(db, "DELETE FROM private " + "WHERE rid=%" FSL_ID_T_PFMT, rid); + return rc ? fsl_cx_uplift_db_error(f, db) : 0; +} + +/** + Load the record ID rid and up to N-1 closest ancestors into + the "fsl_computed_ancestors" table. + */ +static int fsl_compute_ancestors( fsl_db * db, fsl_id_t rid, + int N, char directOnly ){ + fsl_stmt st = fsl_stmt_empty; + int rc = fsl_db_prepare(db, &st, + "WITH RECURSIVE " + " ancestor(rid, mtime) AS (" + " SELECT ?, mtime " + " FROM event WHERE objid=? " + " UNION " + " SELECT plink.pid, event.mtime" + " FROM ancestor, plink, event" + " WHERE plink.cid=ancestor.rid" + " AND event.objid=plink.pid %s" + " ORDER BY mtime DESC LIMIT ?" + " )" + "INSERT INTO fsl_computed_ancestors" + " SELECT rid FROM ancestor;", + directOnly ? "AND plink.isPrim" : "" + ); + if(!rc){ + fsl_stmt_bind_id(&st, 1, rid); + fsl_stmt_bind_id(&st, 2, rid); + fsl_stmt_bind_int32(&st, 3, (int32_t)N); + rc = fsl_stmt_step(&st); + if(FSL_RC_STEP_DONE==rc){ + rc = 0; + } + } + fsl_stmt_finalize(&st); + return rc; +} + +int fsl_mtime_of_F_card(fsl_cx * f, fsl_id_t vid, fsl_card_F const * fc, fsl_time_t *pMTime){ + if(!f || !fc) return FSL_RC_MISUSE; + else if(vid<=0) return FSL_RC_RANGE; + else if(!fc->uuid){ + if(pMTime) *pMTime = 0; + return 0; + }else{ + fsl_id_t fid = fsl_uuid_to_rid(f, fc->uuid); + if(fid<=0){ + assert(f->error.code); + return f->error.code; + }else{ + return fsl_mtime_of_manifest_file(f, vid, fid, pMTime); + } + } +} + +int fsl_mtime_of_manifest_file(fsl_cx * f, fsl_id_t vid, fsl_id_t fid, fsl_time_t *pMTime){ + fsl_db * db = fsl_needs_repo(f); + fsl_stmt * q = NULL; + int rc; + if(!db) return FSL_RC_NOT_A_REPO; + + if(fid<=0){ + /* Only fetch the checkin time... */ + int64_t i = -1; + rc = fsl_db_get_int64(db, &i, + "SELECT (mtime-2440587.5)*86400 " + "FROM event WHERE objid=%"FSL_ID_T_PFMT + " AND type='ci'", + (fsl_id_t)vid); + if(!rc){ + if(i<0) rc = FSL_RC_NOT_FOUND; + else if(pMTime) *pMTime = (fsl_time_t)i; + } + return rc; + } + + if( f->cache.mtimeManifest != vid ){ + /* + Computing (and keeping) ancestors is relatively costly, so we + keep only the copy associated with f->cache.mtimeManifest + around. For the general case, we will be feeding this function + files from the same manifest. + */ + f->cache.mtimeManifest = vid; + rc = fsl_db_exec_multi(db,"DROP TABLE IF EXISTS temp.fsl_computed_ancestors;" + "CREATE TEMP TABLE fsl_computed_ancestors" + "(x INTEGER PRIMARY KEY);"); + if(!rc){ + rc = fsl_compute_ancestors(db, vid, 1000000, 1); + } + if(rc){ + fsl_cx_uplift_db_error(f, db); + return rc; + } + } + rc = fsl_db_prepare_cached(db, &q, + "SELECT (max(event.mtime)-2440587.5)*86400 FROM mlink, event" + " WHERE mlink.mid=event.objid" + " AND mlink.fid=?" + " AND +mlink.mid IN fsl_computed_ancestors" + ); + if(!rc){ + fsl_stmt_bind_id(q, 1, fid); + rc = fsl_stmt_step(q); + if( FSL_RC_STEP_ROW==rc ){ + rc = 0; + if(pMTime) *pMTime = (fsl_time_t)fsl_stmt_g_int64(q, 0); + }else{ + assert(rc); + if(FSL_RC_STEP_DONE==rc) rc = FSL_RC_NOT_FOUND; + } + fsl_stmt_cached_yield(q); + } + return rc; +} + +int fsl_card_F_content( fsl_cx * f, fsl_card_F const * fc, + fsl_buffer * dest ){ + if(!f || !fc || !dest) return FSL_RC_MISUSE; + else if(!fc->uuid){ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "Cannot fetch content of a deleted file " + "because it has no UUID."); + } + else if(!fsl_needs_repo(f)) return FSL_RC_NOT_A_REPO; + else{ + fsl_id_t const rid = fsl_uuid_to_rid(f, fc->uuid); + if(!rid) return fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "UUID not found: %s", + fc->uuid); + else if(rid<0){ + assert(f->error.code); + return f->error.code; + }else{ + return fsl_content_get(f, rid, dest); + } + } +} + + +/** + UNTESTED (but closely derived from known-working code). + + Expects f to have an opened checkout. Assumes zName is resolvable + (via fsl_ckout_filename_check() - see that function for the + meaning of the relativeToCwd argument) to a path under the current + checkout root. It loads the file's contents and stores them into + the blob table. If rid is not NULL, *rid is assigned the blob.rid + (possibly new, possilbly re-used!). If uuid is not NULL then *uuid + is assigned to the content's UUID. The *uuid bytes are owned by the + caller, who must eventually fsl_free() them. If content with the + same UUID already exists, it does not get re-imported but rid/uuid + will (if not NULL) contain the values of any previous content + with the same hash. + + ACHTUNG: this function DOES NOT CARE whether or not the file is + actually part of a checkout or not, nor whether it is actually + referenced by any checkins, or such, other than that it must + resolve to something under the checkout root (to avoid breaking any + internal assumptions in fossil about filenames). It will add new + repo.filename entries as needed for this function. Thus is can be + used to import "shadow files" either not known about by fossil or + not _yet_ known about by fossil. + + If parentRid is >0 then it must refer to the previous version of + zName's content. The parent version gets deltified vs the new one, + but deltification is a suggestion which the library will ignore if + (e.g.) the parent content is already a delta of something else. + + This function does its DB-side work in a transaction, so, e.g. if + saving succeeds but deltification of the parent version fails for + some reason, the whole save operation is rolled back. + + Returns 0 on success. On error rid and uuid are not modified. +*/ +int fsl_import_file( fsl_cx * f, char relativeToCwd, + char const * zName, + fsl_id_t parentRid, + fsl_id_t *rid, fsl_uuid_str * uuid ){ + fsl_buffer * canon = 0; // canonicalized filename + fsl_buffer * nbuf = 0; // filename buffer + fsl_buffer * fbuf = &f->fileContent; // file content buffer + char const * fn; + int rc; + fsl_id_t fnid = 0; + fsl_id_t rcRid = 0; + fsl_db * db = f ? fsl_needs_repo(f) : NULL; + char inTrans = 0; + if(!zName || !*zName) return FSL_RC_MISUSE; + else if(!f->ckout.dir) return FSL_RC_NOT_A_CKOUT; + else if(!db) return FSL_RC_NOT_A_REPO; + canon = fsl_cx_scratchpad(f); + nbuf = fsl_cx_scratchpad(f); + + assert(!fbuf->used && "Misuse of f->fileContent"); + assert(f->ckout.dir); + + /* Normalize the name... i often regret having + fsl_ckout_filename_check() return checkout-relative paths. + */ + rc = fsl_ckout_filename_check(f, relativeToCwd, zName, canon); + if(rc) goto end; + + /* Find or create a repo.filename entry... */ + fn = fsl_buffer_cstr(canon); + + rc = fsl_db_transaction_begin(db); + if(rc) goto end; + inTrans = 1; + + rc = fsl_repo_filename_fnid2(f, fn, &fnid, 1); + if(rc) goto end; + + /* Import the file... */ + assert(fnid>0); + rc = fsl_buffer_appendf(nbuf, "%s%s", f->ckout.dir, fn); + if(rc) goto end; + fn = fsl_buffer_cstr(nbuf); + rc = fsl_buffer_fill_from_filename( fbuf, fn ); + if(rc){ + fsl_cx_err_set(f, rc, "Error %s importing file: %s", + fsl_rc_cstr(rc), fn); + goto end; + } + fn = NULL; + rc = fsl_content_put( f, fbuf, &rcRid ); + if(!rc){ + assert(rcRid > 0); + if(parentRid>0){ + /* Make parent version a delta of this one, if possible... */ + rc = fsl_content_deltify(f, parentRid, rcRid, 0); + } + if(!rc){ + if(rid) *rid = rcRid; + if(uuid){ + fsl_cx_err_reset(f); + *uuid = fsl_rid_to_uuid(f, rcRid); + if(!*uuid) rc = (f->error.code ? f->error.code : FSL_RC_OOM); + } + } + } + + if(!rc){ + assert(inTrans); + inTrans = 0; + rc = fsl_db_transaction_commit(db); + } + + end: + fsl_cx_content_buffer_yield(f); + assert(0==fbuf->used); + fsl_cx_scratchpad_yield(f, canon); + fsl_cx_scratchpad_yield(f, nbuf); + if(inTrans) fsl_db_transaction_rollback(db); + return rc; +} + +fsl_hash_types_e fsl_validate_hash(const char *zHash, int nHash){ + /* fossil(1) counterpart: hname_validate() */ + fsl_hash_types_e rc; + switch(nHash){ + case FSL_STRLEN_SHA1: rc = FSL_HTYPE_SHA1; break; + case FSL_STRLEN_K256: rc = FSL_HTYPE_K256; break; + default: return FSL_HTYPE_ERROR; + } + return fsl_validate16(zHash, (fsl_size_t)nHash) ? rc : FSL_HTYPE_ERROR; +} + +const char * fsl_hash_type_name(fsl_hash_types_e h, const char *zUnknown){ + /* fossil(1) counterpart: hname_alg() */ + switch(h){ + case FSL_HTYPE_SHA1: return "SHA1"; + case FSL_HTYPE_K256: return "SHA3-256"; + default: return zUnknown; + } +} + +fsl_hash_types_e fsl_verify_blob_hash(fsl_buffer const * pIn, + const char *zHash, int nHash){ + fsl_hash_types_e id = FSL_HTYPE_ERROR; + switch(nHash){ + case FSL_STRLEN_SHA1:{ + fsl_sha1_cx cx; + char hex[FSL_STRLEN_SHA1+1] = {0}; + fsl_sha1_init(&cx); + fsl_sha1_update(&cx, pIn->mem, (unsigned)pIn->used); + fsl_sha1_final_hex(&cx, hex); + if(0==memcmp(hex, zHash, FSL_STRLEN_SHA1)){ + id = FSL_HTYPE_SHA1; + } + break; + } + case FSL_STRLEN_K256:{ + fsl_sha3_cx cx; + unsigned char const * hex; + fsl_sha3_init(&cx); + fsl_sha3_update(&cx, pIn->mem, (unsigned)pIn->used); + hex = fsl_sha3_end(&cx); + if(0==memcmp(hex, zHash, FSL_STRLEN_K256)){ + id = FSL_HTYPE_K256; + } + break; + } + default: + break; + } + return id; +} + + +int fsl__shunned_remove(fsl_cx * const f){ + fsl_stmt q = fsl_stmt_empty; + int rc; + assert(fsl_cx_db_repo(f)); + rc = fsl_cx_exec_multi(f, + "CREATE TEMP TABLE toshun(rid INTEGER PRIMARY KEY);" + "INSERT INTO toshun SELECT rid FROM blob, shun WHERE blob.uuid=shun.uuid;" + ); + if(rc) goto end; + rc = fsl_cx_prepare(f, &q, + "SELECT rid FROM delta WHERE srcid IN toshun" + ); + while( 0==rc && FSL_RC_STEP_ROW==fsl_stmt_step(&q) ){ + rc = fsl_content_undeltify(f, fsl_stmt_g_id(&q, 0)); + } + fsl_stmt_finalize(&q); + if(rc) goto end; + rc = fsl_cx_exec_multi(f, + "DELETE FROM delta WHERE rid IN toshun;" + "DELETE FROM blob WHERE rid IN toshun;" + "DROP TABLE toshun;" + "DELETE FROM private " + " WHERE NOT EXISTS (SELECT 1 FROM blob WHERE rid=private.rid);" + ); + end: + fsl_stmt_finalize(&q); + return rc; +} + +#undef MARKER +/* end of file content.c */ +/* start of file config.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************** + This file implements (most of) the fsl_xxx APIs related to handling + configuration data from the db(s). +*/ +#include +#include /* bsearch() */ +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +/** + File-local macro used to ensure that all cached statements used by + the fsl_config_get_xxx() APIs use an equivalent string (so that + they use the same underlying cache fsl_stmt handle). The first %s + represents one of the config table names (config, vvfar, + global_config). Normally we wouldn't use %s in a cached statement, + but we're only expecting 3 values for table here and each one will + only be cached once. The 2nd %s must be __FILE__. +*/ +#define SELECT_FROM_CONFIG "SELECT value FROM %s WHERE name=?/*%s*/" + +char const * fsl_config_table_for_role(fsl_confdb_e mode){ + switch(mode){ + case FSL_CONFDB_REPO: return "config"; + case FSL_CONFDB_CKOUT: return "vvar"; + case FSL_CONFDB_GLOBAL: return "global_config"; + case FSL_CONFDB_VERSIONABLE: return NULL; + default: + assert(!"Invalid fsl_confdb_e value"); + return NULL; + } +} + +fsl_db * fsl_config_for_role(fsl_cx * f, fsl_confdb_e mode){ + switch(mode){ + case FSL_CONFDB_REPO: return fsl_cx_db_repo(f); + case FSL_CONFDB_CKOUT: return fsl_cx_db_ckout(f); + case FSL_CONFDB_GLOBAL: return fsl_cx_db_config(f); + case FSL_CONFDB_VERSIONABLE: return fsl_cx_db(f); + default: + assert(!"Invalid fsl_confdb_e value"); + return NULL; + } +} + +int fsl_config_versionable_filename(fsl_cx *f, char const * key, + fsl_buffer *b){ + if(!f || !fsl_needs_ckout(f)) return FSL_RC_NOT_A_CKOUT; + else if(!key || !*key || !fsl_is_simple_pathname(key, true)){ + return FSL_RC_MISUSE; + } + fsl_buffer_reuse(b); + return fsl_buffer_appendf(b, "%s.fossil-settings/%s", + f->ckout.dir, key); +} + + +int fsl_config_unset( fsl_cx * f, fsl_confdb_e mode, char const * key ){ + fsl_db * db = fsl_config_for_role(f, mode); + if(!db || !key || !*key) return FSL_RC_MISUSE; + else if(mode==FSL_CONFDB_VERSIONABLE) return FSL_RC_UNSUPPORTED; + else{ + char const * table = fsl_config_table_for_role(mode); + assert(table); + return fsl_db_exec(db, "DELETE FROM %s WHERE name=%Q", table, key); + } +} + +int32_t fsl_config_get_int32( fsl_cx * f, fsl_confdb_e mode, + int32_t dflt, char const * key ){ + int32_t rv = dflt; + switch(mode){ + case FSL_CONFDB_VERSIONABLE:{ + char * val = fsl_config_get_text(f, mode, key, NULL); + if(val){ + rv = (int32_t)atoi(val); + fsl_free(val); + } + break; + } + default: { + fsl_db * db = fsl_config_for_role(f, mode); + char const * table = fsl_config_table_for_role(mode); + assert(table); + if(db){ + fsl_stmt * st = NULL; + fsl_db_prepare_cached(db, &st, SELECT_FROM_CONFIG, + table, __FILE__); + if(st){ + fsl_stmt_bind_text(st, 1, key, -1, 0); + if(FSL_RC_STEP_ROW==fsl_stmt_step(st)){ + rv = fsl_stmt_g_int32(st, 0); + } + fsl_stmt_cached_yield(st); + } + } + break; + } + } + return rv; +} + +int64_t fsl_config_get_int64( fsl_cx * f, fsl_confdb_e mode, + int64_t dflt, char const * key ){ + int64_t rv = dflt; + switch(mode){ + case FSL_CONFDB_VERSIONABLE:{ + char * val = fsl_config_get_text(f, mode, key, NULL); + if(val){ + rv = (int64_t)strtoll(val, NULL, 10); + fsl_free(val); + } + break; + } + default: { + fsl_db * db = fsl_config_for_role(f, mode); + char const * table = fsl_config_table_for_role(mode); + assert(table); + if(db){ + fsl_stmt * st = NULL; + fsl_db_prepare_cached(db, &st, SELECT_FROM_CONFIG, + table, __FILE__); + if(st){ + fsl_stmt_bind_text(st, 1, key, -1, 0); + if(FSL_RC_STEP_ROW==fsl_stmt_step(st)){ + rv = fsl_stmt_g_int64(st, 0); + } + fsl_stmt_cached_yield(st); + } + } + break; + } + } + return rv; +} + +fsl_id_t fsl_config_get_id( fsl_cx * f, fsl_confdb_e mode, + fsl_id_t dflt, char const * key ){ + return (sizeof(fsl_id_t)==sizeof(int32_t)) + ? (fsl_id_t)fsl_config_get_int32(f, mode, dflt, key) + : (fsl_id_t)fsl_config_get_int64(f, mode, dflt, key); +} + +double fsl_config_get_double( fsl_cx * f, fsl_confdb_e mode, + double dflt, char const * key ){ + double rv = dflt; + switch(mode){ + case FSL_CONFDB_VERSIONABLE:{ + char * val = fsl_config_get_text(f, mode, key, NULL); + if(val){ + rv = strtod(val, NULL); + fsl_free(val); + } + break; + } + default: { + fsl_db * db = fsl_config_for_role(f, mode); + if(!db) break/*e.g. global config is not opened*/; + fsl_stmt * st = NULL; + char const * table = fsl_config_table_for_role(mode); + assert(table); + fsl_db_prepare_cached(db, &st, SELECT_FROM_CONFIG, + table, __FILE__); + if(st){ + fsl_stmt_bind_text(st, 1, key, -1, 0); + if(FSL_RC_STEP_ROW==fsl_stmt_step(st)){ + rv = fsl_stmt_g_double(st, 0); + } + fsl_stmt_cached_yield(st); + } + break; + } + } + return rv; +} + +char * fsl_config_get_text( fsl_cx * f, fsl_confdb_e mode, + char const * key, fsl_size_t * len ){ + char * rv = NULL; + fsl_buffer val = fsl_buffer_empty; + if(fsl_config_get_buffer(f, mode, key, &val)){ + fsl_cx_err_reset(f); + if(len) *len = 0; + fsl_buffer_clear(&val)/*in case of partial read failure*/; + }else{ + if(len) *len = val.used; + rv = fsl_buffer_take(&val); + } + return rv; +} + +int fsl_config_get_buffer( fsl_cx * f, fsl_confdb_e mode, + char const * key, fsl_buffer * b ){ + int rc = FSL_RC_NOT_FOUND; + switch(mode){ + case FSL_CONFDB_VERSIONABLE:{ + if(!fsl_needs_ckout(f)){ + rc = FSL_RC_NOT_A_CKOUT; + break; + } + fsl_buffer * fname = fsl_cx_scratchpad(f); + rc = fsl_config_versionable_filename(f, key, fname); + if(!rc){ + char const * zFile = fsl_buffer_cstr(fname); + rc = fsl_stat(zFile, 0, false); + if(rc){ + rc = fsl_cx_err_set(f, rc, "Could not stat file: %s", + zFile); + }else{ + rc = fsl_buffer_fill_from_filename(b, zFile); + } + } + fsl_cx_scratchpad_yield(f,fname); + break; + } + default: { + char const * table = fsl_config_table_for_role(mode); + assert(table); + fsl_db * const db = fsl_config_for_role(f, mode); + if(!db) break; + fsl_stmt * st = NULL; + rc = fsl_db_prepare_cached(db, &st, SELECT_FROM_CONFIG, + table, __FILE__); + if(rc){ + rc = fsl_cx_uplift_db_error2(f, db, rc); + break; + } + fsl_stmt_bind_text(st, 1, key, -1, 0); + if(FSL_RC_STEP_ROW==fsl_stmt_step(st)){ + fsl_size_t len = 0; + char const * s = fsl_stmt_g_text(st, 0, &len); + rc = s ? fsl_buffer_append(b, s, len) : 0; + }else{ + rc = FSL_RC_NOT_FOUND; + } + fsl_stmt_cached_yield(st); + break; + } + } + return rc; +} + +bool fsl_config_get_bool( fsl_cx * f, fsl_confdb_e mode, + bool dflt, char const * key ){ + bool rv = dflt; + switch(mode){ + case FSL_CONFDB_VERSIONABLE:{ + char * val = fsl_config_get_text(f, mode, key, NULL); + if(val){ + rv = fsl_str_bool(val); + fsl_free(val); + } + break; + } + default:{ + int rc; + fsl_stmt * st = NULL; + char const * table = fsl_config_table_for_role(mode); + fsl_db * db; + if(!f || !key || !*key) break; + db = fsl_config_for_role(f, mode); + if(!db) break; + assert(table); + rc = fsl_db_prepare_cached(db, &st, SELECT_FROM_CONFIG, + table, __FILE__); + if(!rc){ + fsl_stmt_bind_text(st, 1, key, -1, 0); + if(FSL_RC_STEP_ROW==fsl_stmt_step(st)){ + char const * col = fsl_stmt_g_text(st, 0, NULL); + rv = col ? fsl_str_bool(col) : dflt /* 0? */; + } + fsl_stmt_cached_yield(st); + } + break; + } + } + return rv; +} + +/** + Sets up a REPLACE statement for the given config db and key. On + success 0 is returned and *st holds the cached statement. The caller + must bind() parameter #2 and step() the statement, then + fsl_stmt_cached_yield() it. + + Returns non-0 on error. +*/ +static int fsl_config_set_prepare( fsl_cx * f, fsl_stmt **st, + fsl_confdb_e mode, char const * key ){ + char const * table = fsl_config_table_for_role(mode); + fsl_db * db = fsl_config_for_role(f,mode); + assert(table); + if(!db || !key) return FSL_RC_MISUSE; + else if(!*key) return FSL_RC_RANGE; + else{ + const char * sql = FSL_CONFDB_REPO==mode + ? "REPLACE INTO %s(name,value,mtime) VALUES(?,?,now())/*%s()*/" + : "REPLACE INTO %s(name,value) VALUES(?,?)/*%s()*/"; + int rc = fsl_db_prepare_cached(db, st, sql, table, __func__); + if(!rc) rc = fsl_stmt_bind_text(*st, 1, key, -1, 1); + if(rc && !f->error.code){ + fsl_cx_uplift_db_error(f, db); + } + return rc; + } +} + + +/* + TODO/FIXME: the fsl_config_set_xxx() routines all use the same basic + structure, differing only in the concrete bind() op they call. They + should be consolidated somehow. +*/ + +/** + Writes valLen bytes of val to a versioned-setting file. Returns 0 + on success. Requires a checkout db. +*/ +static int fsl_config_set_versionable( fsl_cx * f, char const * key, + char const * val, + fsl_size_t valLen){ + assert(key && *key); + if(!fsl_needs_ckout(f)){ + return FSL_RC_NOT_A_CKOUT; + } + fsl_buffer * fName = fsl_cx_scratchpad(f); + int rc = fsl_config_versionable_filename(f, key, fName); + if(!rc){ + fsl_buffer fake = fsl_buffer_empty; + fake.mem = (void*)val; + fake.capacity = fake.used = valLen; + rc = fsl_buffer_to_filename(&fake, fsl_buffer_cstr(fName)); + } + fsl_cx_scratchpad_yield(f, fName); + return rc; +} + + +int fsl_config_set_text( fsl_cx * f, fsl_confdb_e mode, + char const * key, char const * val ){ + if(!key) return FSL_RC_MISUSE; + else if(!*key) return FSL_RC_RANGE; + else if(FSL_CONFDB_VERSIONABLE==mode){ + return fsl_config_set_versionable(f, key, val, + val ? fsl_strlen(val) : 0); + } + fsl_db * db = fsl_config_for_role(f,mode); + if(!db) return FSL_RC_MISUSE; + fsl_stmt * st = NULL; + int rc = fsl_config_set_prepare(f, &st, mode, key); + //MARKER(("config-set mode=%d k=%s v=%s\n", + // mode, key, val)); + if(!rc){ + if(val){ + rc = fsl_stmt_bind_text(st, 2, val, -1, 0); + }else{ + rc = fsl_stmt_bind_null(st, 2); + } + if(!rc){ + rc = fsl_stmt_step(st); + } + fsl_stmt_cached_yield(st); + if(FSL_RC_STEP_DONE==rc) rc = 0; + } + if(rc && !f->error.code) fsl_cx_uplift_db_error(f, db); + return rc; +} + +int fsl_config_set_blob( fsl_cx * f, fsl_confdb_e mode, char const * key, + void const * val, fsl_int_t len ){ + if(!key) return FSL_RC_MISUSE; + else if(!*key) return FSL_RC_RANGE; + else if(FSL_CONFDB_VERSIONABLE==mode){ + return fsl_config_set_versionable(f, key, val, + (val && len<0) + ? fsl_strlen((char const *)val) + : (fsl_size_t)len); + } + fsl_db * db = fsl_config_for_role(f,mode); + if(!db) return FSL_RC_MISUSE; + fsl_stmt * st = NULL; + int rc = fsl_config_set_prepare(f, &st, mode, key); + if(!rc){ + if(val){ + if(len<0) len = fsl_strlen((char const *)val); + rc = fsl_stmt_bind_blob(st, 2, val, len, 0); + }else{ + rc = fsl_stmt_bind_null(st, 2); + } + if(!rc){ + rc = fsl_stmt_step(st); + } + fsl_stmt_cached_yield(st); + if(FSL_RC_STEP_DONE==rc) rc = 0; + } + if(rc && !f->error.code) fsl_cx_uplift_db_error(f, db); + return rc; +} + +int fsl_config_set_int32( fsl_cx * f, fsl_confdb_e mode, + char const * key, int32_t val ){ + if(!key) return FSL_RC_MISUSE; + else if(!*key) return FSL_RC_RANGE; + else if(FSL_CONFDB_VERSIONABLE==mode){ + char buf[64] = {0}; + fsl_snprintf(buf, sizeof(buf), "%" PRIi32 "\n", val); + return fsl_config_set_versionable(f, key, buf, + fsl_strlen(buf)); + } + fsl_db * db = fsl_config_for_role(f,mode); + if(!db) return FSL_RC_MISUSE; + fsl_stmt * st = NULL; + int rc = fsl_config_set_prepare(f, &st, mode, key); + if(!rc){ + rc = fsl_stmt_bind_int32(st, 2, val); + if(!rc){ + rc = fsl_stmt_step(st); + } + fsl_stmt_cached_yield(st); + if(FSL_RC_STEP_DONE==rc) rc = 0; + } + if(rc && !f->error.code) fsl_cx_uplift_db_error(f, db); + return rc; +} + +int fsl_config_set_int64( fsl_cx * f, fsl_confdb_e mode, + char const * key, int64_t val ){ + if(!key) return FSL_RC_MISUSE; + else if(!*key) return FSL_RC_RANGE; + else if(FSL_CONFDB_VERSIONABLE==mode){ + char buf[64] = {0}; + fsl_snprintf(buf, sizeof(buf), "%" PRIi64 "\n", val); + return fsl_config_set_versionable(f, key, buf, + fsl_strlen(buf)); + } + fsl_db * db = fsl_config_for_role(f,mode); + if(!db) return FSL_RC_MISUSE; + fsl_stmt * st = NULL; + int rc = fsl_config_set_prepare(f, &st, mode, key); + if(!rc){ + rc = fsl_stmt_bind_int64(st, 2, val); + if(!rc){ + rc = fsl_stmt_step(st); + } + fsl_stmt_cached_yield(st); + if(FSL_RC_STEP_DONE==rc) rc = 0; + } + if(rc && !f->error.code) fsl_cx_uplift_db_error(f, db); + return rc; +} + +int fsl_config_set_id( fsl_cx * f, fsl_confdb_e mode, + char const * key, fsl_id_t val ){ + if(!key) return FSL_RC_MISUSE; + else if(!*key) return FSL_RC_RANGE; + else if(FSL_CONFDB_VERSIONABLE==mode){ + char buf[64] = {0}; + fsl_snprintf(buf, sizeof(buf), "%" FSL_ID_T_PFMT "\n", val); + return fsl_config_set_versionable(f, key, buf, + fsl_strlen(buf)); + } + fsl_db * db = fsl_config_for_role(f,mode); + if(!db) return FSL_RC_MISUSE; + fsl_stmt * st = NULL; + int rc = fsl_config_set_prepare(f, &st, mode, key); + if(!rc){ + rc = fsl_stmt_bind_id(st, 2, val); + if(!rc){ + rc = fsl_stmt_step(st); + } + fsl_stmt_cached_yield(st); + if(FSL_RC_STEP_DONE==rc) rc = 0; + } + if(rc && !f->error.code) fsl_cx_uplift_db_error(f, db); + return rc; +} + +int fsl_config_set_double( fsl_cx * f, fsl_confdb_e mode, + char const * key, double val ){ + if(!key) return FSL_RC_MISUSE; + else if(!*key) return FSL_RC_RANGE; + else if(FSL_CONFDB_VERSIONABLE==mode){ + char buf[128] = {0}; + fsl_snprintf(buf, sizeof(buf), "%f\n", val); + return fsl_config_set_versionable(f, key, buf, + fsl_strlen(buf)); + } + fsl_db * db = fsl_config_for_role(f,mode); + if(!db) return FSL_RC_MISUSE; + fsl_stmt * st = NULL; + int rc = fsl_config_set_prepare(f, &st, mode, key); + if(!rc){ + rc = fsl_stmt_bind_double(st, 2, val); + if(!rc){ + rc = fsl_stmt_step(st); + } + fsl_stmt_cached_yield(st); + if(FSL_RC_STEP_DONE==rc) rc = 0; + } + if(rc && !f->error.code) fsl_cx_uplift_db_error(f, db); + return rc; +} + +int fsl_config_set_bool( fsl_cx * f, fsl_confdb_e mode, + char const * key, bool val ){ + if(!key) return FSL_RC_MISUSE; + else if(!*key) return FSL_RC_RANGE; + char buf[4] = {'o','n','\n','\n'}; + if(!val){ + buf[1] = buf[2] = 'f'; + } + if(FSL_CONFDB_VERSIONABLE==mode){ + return fsl_config_set_versionable(f, key, buf, + val ? 3 : 4); + } + fsl_db * db = fsl_config_for_role(f,mode); + if(!db) return FSL_RC_MISUSE; + fsl_stmt * st = NULL; + int rc = fsl_config_set_prepare(f, &st, mode, key); + if(!rc){ + rc = fsl_stmt_bind_text(st, 2, buf, val ? 2 : 3, false); + if(!rc){ + rc = fsl_stmt_step(st); + } + fsl_stmt_cached_yield(st); + if(FSL_RC_STEP_DONE==rc) rc = 0; + } + if(rc && !f->error.code) fsl_cx_uplift_db_error(f, db); + return rc; +} + +int fsl_config_transaction_begin(fsl_cx * f, fsl_confdb_e mode){ + fsl_db * db = fsl_config_for_role(f,mode); + if(!db) return FSL_RC_MISUSE; + else{ + int const rc = fsl_db_transaction_begin(db); + if(rc) fsl_cx_uplift_db_error(f, db); + return rc; + } +} + +int fsl_config_transaction_end(fsl_cx * f, fsl_confdb_e mode, char rollback){ + fsl_db * db = fsl_config_for_role(f,mode); + if(!db) return FSL_RC_MISUSE; + else{ + int const rc = fsl_db_transaction_end(db, rollback); + if(rc) fsl_cx_uplift_db_error(f, db); + return rc; + } +} + +int fsl_config_globs_load(fsl_cx * f, fsl_list * li, char const * key){ + int rc = 0; + char * val = NULL; + if(!f || !li || !key || !*key) return FSL_RC_MISUSE; + else if(fsl_cx_db_ckout(f)){ + /* Try versionable settings... */ + fsl_buffer buf = fsl_buffer_empty; + rc = fsl_config_get_buffer(f, FSL_CONFDB_VERSIONABLE, key, &buf); + if(rc){ + switch(rc){ + case FSL_RC_NOT_FOUND: + fsl_cx_err_reset(f); + rc = 0; + break; + default: + fsl_buffer_clear(&buf); + return rc; + } + /* Fall through and try the next option. */ + }else{ + if(buf.mem){ + val = fsl_buffer_take(&buf); + }else{ + /* Empty but existing list, so it trumps the + repo/global settings. */; + fsl_buffer_clear(&buf); + } + goto gotone; + } + } + if(fsl_cx_db_repo(f)){ + /* See if the repo can serve us... */ + val = fsl_config_get_text(f, FSL_CONFDB_REPO, key, NULL); + if(val) goto gotone; + /* Else fall through and try global config... */ + } + if(fsl_cx_db_config(f)){ + /*FIXME?: we arguably should open the global config for this if it + is not already opened.*/ + val = fsl_config_get_text(f, FSL_CONFDB_GLOBAL, key, NULL); + if(val) goto gotone; + } + gotone: + if(val){ + rc = fsl_glob_list_parse( li, val ); + fsl_free(val); + val = 0; + return rc; + } + return rc; +} + +/* + TODO???: the infrastructure from fossil's configure.c and db.c which + deals with the config db and the list of known/allowed settings. + + ================== + +static const struct { + const char *zName; / * Name of the configuration set * / + int groupMask; / * Mask for that configuration set * / + const char *zHelp; / * What it does * / + +} fslConfigGroups[] = { + { "/email", FSL_CONFIGSET_ADDR, "Concealed email addresses in tickets" }, + { "/project", FSL_CONFIGSET_PROJ, "Project name and description" }, + { "/skin", FSL_CONFIGSET_SKIN | FSL_CONFIGSET_CSS, + "Web interface appearance settings" }, + { "/css", FSL_CONFIGSET_CSS, "Style sheet" }, + { "/shun", FSL_CONFIGSET_SHUN, "List of shunned artifacts" }, + { "/ticket", FSL_CONFIGSET_TKT, "Ticket setup", }, + { "/user", FSL_CONFIGSET_USER, "Users and privilege settings" }, + { "/xfer", FSL_CONFIGSET_XFER, "Transfer setup", }, + { "/all", FSL_CONFIGSET_ALL, "All of the above" }, + {NULL, 0, NULL} +}; + */ + +/* + The following is a list of settings that we are willing to + transfer. + + Setting names that begin with an alphabetic characters refer to + single entries in the CONFIG table. Setting names that begin with + "@" are for special processing. +*/ +static struct { + const char *zName; /* Name of the configuration parameter */ + int groupMask; /* Which config groups is it part of */ +} fslConfigXfer[] = { + { "css", FSL_CONFIGSET_CSS }, + + { "header", FSL_CONFIGSET_SKIN }, + { "footer", FSL_CONFIGSET_SKIN }, + { "logo-mimetype", FSL_CONFIGSET_SKIN }, + { "logo-image", FSL_CONFIGSET_SKIN }, + { "background-mimetype", FSL_CONFIGSET_SKIN }, + { "background-image", FSL_CONFIGSET_SKIN }, + { "index-page", FSL_CONFIGSET_SKIN }, + { "timeline-block-markup", FSL_CONFIGSET_SKIN }, + { "timeline-max-comment", FSL_CONFIGSET_SKIN }, + { "timeline-plaintext", FSL_CONFIGSET_SKIN }, + { "adunit", FSL_CONFIGSET_SKIN }, + { "adunit-omit-if-admin", FSL_CONFIGSET_SKIN }, + { "adunit-omit-if-user", FSL_CONFIGSET_SKIN }, + + { "th1-setup", FSL_CONFIGSET_ALL }, + + { "tcl", FSL_CONFIGSET_SKIN|FSL_CONFIGSET_TKT|FSL_CONFIGSET_XFER }, + { "tcl-setup", FSL_CONFIGSET_SKIN|FSL_CONFIGSET_TKT|FSL_CONFIGSET_XFER }, + + { "project-name", FSL_CONFIGSET_PROJ }, + { "project-description", FSL_CONFIGSET_PROJ }, + { "manifest", FSL_CONFIGSET_PROJ }, + { "binary-glob", FSL_CONFIGSET_PROJ }, + { "clean-glob", FSL_CONFIGSET_PROJ }, + { "ignore-glob", FSL_CONFIGSET_PROJ }, + { "keep-glob", FSL_CONFIGSET_PROJ }, + { "crnl-glob", FSL_CONFIGSET_PROJ }, + { "encoding-glob", FSL_CONFIGSET_PROJ }, + { "empty-dirs", FSL_CONFIGSET_PROJ }, + { "allow-symlinks", FSL_CONFIGSET_PROJ }, + + { "ticket-table", FSL_CONFIGSET_TKT }, + { "ticket-common", FSL_CONFIGSET_TKT }, + { "ticket-change", FSL_CONFIGSET_TKT }, + { "ticket-newpage", FSL_CONFIGSET_TKT }, + { "ticket-viewpage", FSL_CONFIGSET_TKT }, + { "ticket-editpage", FSL_CONFIGSET_TKT }, + { "ticket-reportlist", FSL_CONFIGSET_TKT }, + { "ticket-report-template", FSL_CONFIGSET_TKT }, + { "ticket-key-template", FSL_CONFIGSET_TKT }, + { "ticket-title-expr", FSL_CONFIGSET_TKT }, + { "ticket-closed-expr", FSL_CONFIGSET_TKT }, + { "@reportfmt", FSL_CONFIGSET_TKT }, + + { "@user", FSL_CONFIGSET_USER }, + + { "@concealed", FSL_CONFIGSET_ADDR }, + + { "@shun", FSL_CONFIGSET_SHUN }, + + { "xfer-common-script", FSL_CONFIGSET_XFER }, + { "xfer-push-script", FSL_CONFIGSET_XFER }, +}; + +#define ARRAYLEN(X) (sizeof(X)/sizeof(X[0])) +/* + Return a pointer to a string that contains the RHS of an IN + operator that will select CONFIG table names that are part of the + configuration that matches iMatch. The returned string must + eventually be fsl_free()'d. +*/ +char *fsl_config_inop_rhs(int iMask){ + fsl_buffer x = fsl_buffer_empty; + const char *zSep = ""; + const int n = (int)ARRAYLEN(fslConfigXfer); + int i; + int rc = fsl_buffer_append(&x, "(", 1); + for(i=0; !rc && (iname; ++ct){ + rc = fsl_buffer_appendf(&x, "%s%Q", zSep, ct->name); + zSep = ","; + } + if(!rc){ + rc = fsl_buffer_append(&x, ")", 1); + } + if(rc){ + fsl_buffer_clear(&x); + assert(!x.mem); + }else{ + fsl_buffer_resize(&x, x.used); + } + return (char *)x.mem; +} + +static int fsl_config_ctrl_cmp(const void *lhs, const void *rhs){ + fsl_config_ctrl const * l = (fsl_config_ctrl const *)lhs; + fsl_config_ctrl const * r = (fsl_config_ctrl const *)rhs; + if(!l) return r ? -1 : 0; + else if(!r) return 1; + else return fsl_strcmp(l->name, r->name); +} + +fsl_config_ctrl const * fsl_config_ctrl_get(char const * key){ + fsl_config_ctrl const * fcc; + fsl_config_ctrl bogo = {0,0,0,0,0}; + bogo.name = key; + fcc = (fsl_config_ctrl const *) + bsearch( &bogo, fslConfigCtrl, + ARRAYLEN(fslConfigCtrl) -1 /* for empty tail entry */, + sizeof(fsl_config_ctrl), + fsl_config_ctrl_cmp ); + return (fcc && fcc->name) ? fcc : NULL; +} + +bool fsl_config_key_is_fossil(char const * key){ + fsl_config_ctrl const * fcc = fsl_config_ctrl_get(key); + return (fcc && fcc->name) ? 1 : 0; +} + +bool fsl_config_key_is_versionable(char const * key){ + fsl_config_ctrl const * fcc = fsl_config_ctrl_get(key); + return (fcc && fcc->versionable) ? 1 : 0; +} + +char const * fsl_config_key_default_value(char const * key){ + fsl_config_ctrl const * fcc = fsl_config_ctrl_get(key); + return (fcc && fcc->name) ? fcc->defaultValue : NULL; +} + +bool fsl_config_has_versionable( fsl_cx * f, char const * key ){ + if(!f || !key || !*key || !f->ckout.dir) return 0; + else if(!fsl_config_key_is_fossil(key)) return 0; + else{ + fsl_buffer * fn = fsl_cx_scratchpad(f); + int rc = fsl_config_versionable_filename(f, key, fn); + if(!rc) rc = fsl_stat(fsl_buffer_cstr(fn), NULL, 0); + fsl_cx_scratchpad_yield(f, fn); + return 0==rc; + } +} + + +#undef SELECT_FROM_CONFIG +#undef MARKER +#undef ARRAYLEN +/* end of file config.c */ +/* start of file cx.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************* + This file houses most of the context-related APIs. +*/ +#if !defined(FSL_ENABLE_SQLITE_REGEXP) +# define FSL_ENABLE_SQLITE_REGEXP 0 +#endif +#if FSL_ENABLE_SQLITE_REGEXP +#endif +#include "sqlite3.h" +#include + +#if defined(_WIN32) +# include +# define F_OK 0 +# define W_OK 2 +#else +# include /* F_OK */ +#endif + +#include +#include + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +/** Number of fsl_cx::scratchpads buffers. */ +#define FSL_CX_NSCRATCH \ + ((int)(sizeof(fsl_cx_empty.scratchpads.buf) \ + /sizeof(fsl_cx_empty.scratchpads.buf[0]))) +const int StaticAssert_scratchpadsCounts[ + (FSL_CX_NSCRATCH== + ((int)(sizeof(fsl_cx_empty.scratchpads.used) + /sizeof(fsl_cx_empty.scratchpads.used[0])))) + ? 1 : -1 +]; + +#if FSL_ENABLE_SQLITE_REGEXP +/** + Used for setup and teardown of sqlite3_auto_extension(). +*/ +static volatile long sg_autoregctr = 0; +#endif + + +/** + Clears (most) dynamic state in f, but does not free f and does + not free "static" state (that set up by the init process). If + closeDatabases is true then any databases managed by f are + closed, else they are kept open. + + Client code will not normally need this - it is intended for a + particular potential memory optimization case. If (and only if) + closeDatabases is true then after calling this, f may be legally + re-used as a target for fsl_cx_init(). + + This function does not trigger any finializers set for f's client + state or output channel. + + Results are undefined if !f or f's memory has not been properly + initialized. +*/ +static void fsl_cx_reset( fsl_cx * const f, bool closeDatabases ); + + +int fsl_cx_init( fsl_cx ** tgt, fsl_cx_init_opt const * param ){ + static fsl_cx_init_opt paramDefaults = fsl_cx_init_opt_default_m; + int rc = 0; + fsl_cx * f; + extern int fsl_cx_install_timeline_crosslinkers(fsl_cx * const f) + /*in deck.c*/; + if(!tgt) return FSL_RC_MISUSE; + else if(!param){ + if(!paramDefaults.output.state){ + paramDefaults.output.state = stdout; + } + param = ¶mDefaults; + } + if(*tgt){ + void const * allocStamp = (*tgt)->allocStamp; + fsl_cx_reset(*tgt, true) /* just to be safe */; + f = *tgt; + *f = fsl_cx_empty; + f->allocStamp = allocStamp; + }else{ + f = fsl_cx_malloc(); + if(!f) return FSL_RC_OOM; + + *tgt = f; + } + memset(&f->cache.mcache, 0, sizeof(f->cache.mcache)); + f->output = param->output; + f->cxConfig = param->config; + + enum { + /* Because testing shows a lot of re-allocs via some of the + lower-level stat()-related bits, we pre-allocate this many + bytes into f->scratchpads.buf[]. Curiously, there is almost no + difference in (re)allocation behaviour until this size goes + above about 200. + + We ignore allocation errors here, as they're not critical (but + upcoming ops will fail when _they_ run out of memory). + */ + InitialScratchCapacity = 256 + }; + assert(FSL_CX_NSCRATCH + == (sizeof(f->scratchpads.used)/sizeof(f->scratchpads.used[0]))); + for(int i = 0; i < FSL_CX_NSCRATCH; ++i){ + f->scratchpads.buf[i] = fsl_buffer_empty; + f->scratchpads.used[i] = false; + fsl_buffer_reserve(&f->scratchpads.buf[i], InitialScratchCapacity); + } + /* We update f->error.msg often, so go ahead and pre-allocate that, too, + also ignoring any OOM error at this point. */ + fsl_buffer_reserve(&f->error.msg, InitialScratchCapacity); + +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 + sqlite3_initialize(); /*the SQLITE_MUTEX_STATIC_MASTER will not cause autoinit of sqlite for some reason*/ + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); +#endif +#if FSL_ENABLE_SQLITE_REGEXP + if ( 1 == ++sg_autoregctr ){ + /*register our statically linked extensions to be auto-init'ed at the appropriate time*/ + sqlite3_auto_extension((void(*)(void))(sqlite3_regexp_init)); /*sqlite regexp extension*/ + atexit(sqlite3_reset_auto_extension) + /* Clean up pseudo-leak valgrind complains about: + https://www.sqlite.org/c3ref/auto_extension.html */; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); +#endif + f->dbMem.f = f /* so that dbMem gets fsl_xxx() SQL funcs installed. */; + rc = fsl_db_open( &f->dbMem, "", 0 ); + if(!rc){ + /* Attempt to ensure that the TEMP tables/indexes use FILE storage. */ + rc = fsl_db_exec(&f->dbMem, "PRAGMA temp_store=FILE;"); + } + if(!rc){ + rc = fsl_cx_install_timeline_crosslinkers(f); + } + if(rc){ + if(f->dbMem.error.code){ + fsl_cx_uplift_db_error(f, &f->dbMem); + } + }else{ + f->dbMain = &f->dbMem; + f->dbMem.role = FSL_DBROLE_MAIN; + } + return rc; +} + +static void fsl_cx_mcache_clear(fsl_cx *f){ + const unsigned cacheLen = + (unsigned)(sizeof(fsl_mcache_empty.aAge) + /sizeof(fsl_mcache_empty.aAge[0])); + for(unsigned i = 0; i < cacheLen; ++i){ + fsl_deck_finalize(&f->cache.mcache.decks[i]); + } + f->cache.mcache = fsl_mcache_empty; +} + +static void fsl_cx_reset(fsl_cx * const f, bool closeDatabases){ + fsl_checkin_discard(f); +#define SFREE(X) fsl_free(X); X = NULL + if(closeDatabases){ + fsl_cx_close_dbs(f); + /* + Reminder: f->dbMem is NOT closed here: it's an internal detail, + not public state. We could arguably close and reopen it here, + but then we introduce a potenital error case (OOM) where we + currently have none (thus the void return). + */ + SFREE(f->ckout.dir); + f->ckout.dirLen = 0; + /* assert(NULL==f->dbMain); */ + assert(!f->repo.db.dbh); + assert(!f->ckout.db.dbh); + assert(!f->config.db.dbh); + assert(!f->repo.db.filename); + assert(!f->ckout.db.filename); + assert(!f->config.db.filename); + } + SFREE(f->repo.user); + SFREE(f->ckout.uuid); + SFREE(f->cache.projectCode); +#undef SFREE + fsl_error_clear(&f->error); + fsl_card_J_list_free(&f->ticket.customFields, 1); + fsl_buffer_clear(&f->fileContent); + for(int i = 0; i < FSL_CX_NSCRATCH; ++i){ + fsl_buffer_clear(&f->scratchpads.buf[i]); + f->scratchpads.used[i] = false; + } + fsl_acache_clear(&f->cache.arty); + fsl_id_bag_clear(&f->cache.leafCheck); + fsl_id_bag_clear(&f->cache.toVerify); + fsl_cx_clear_mf_seen(f); + if(f->xlinkers.list){ + fsl_free(f->xlinkers.list); + f->xlinkers = fsl_xlinker_list_empty; + } +#define SLIST(L) fsl_list_visit_free(L, 1) +#define GLOBL(X) SLIST(&f->cache.globs.X) + GLOBL(ignore); + GLOBL(binary); + GLOBL(crnl); +#undef GLOBL +#undef SLIST + fsl_cx_mcache_clear(f); + f->cache = fsl_cx_empty.cache; +} + +void fsl_cx_clear_mf_seen(fsl_cx * f){ + fsl_id_bag_clear(&f->cache.mfSeen); +} + +void fsl_cx_finalize( fsl_cx * f ){ + void const * allocStamp = f ? f->allocStamp : NULL; + if(!f) return; + + if(f->clientState.finalize.f){ + f->clientState.finalize.f( f->clientState.finalize.state, + f->clientState.state ); + } + f->clientState = fsl_state_empty; + f->output = fsl_outputer_empty; + fsl_cx_reset(f, true); + fsl_db_close(&f->dbMem); + *f = fsl_cx_empty; + if(&fsl_cx_empty == allocStamp){ + fsl_free(f); + }else{ + f->allocStamp = allocStamp; + } + + /* clean up the auto extension; not strictly necessary, but pleases debug malloc's */ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 + sqlite3_mutex_enter(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); +#endif +#if FSL_ENABLE_SQLITE_REGEXP + if ( 0 == --sg_autoregctr ){ + /*register our statically linked extensions to be auto-init'ed at the appropriate time*/ + sqlite3_cancel_auto_extension((void(*)(void))(sqlite3_regexp_init)); /*sqlite regexp extension*/ + } + assert(sg_autoregctr>=0); +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 + sqlite3_mutex_leave(sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER)); +#endif +} + +void fsl_cx_err_reset(fsl_cx * f){ + if(f){ + fsl_error_reset(&f->error); + fsl_db_err_reset(&f->dbMem); + fsl_db_err_reset(&f->repo.db); + fsl_db_err_reset(&f->config.db); + fsl_db_err_reset(&f->ckout.db); + } +} + +int fsl_cx_err_set_e( fsl_cx * f, fsl_error * err ){ + if(!f) return FSL_RC_MISUSE; + else if(!err){ + return fsl_cx_err_set(f, 0, NULL); + }else{ + fsl_error_move(err, &f->error); + fsl_error_clear(err); + return f->error.code; + } +} + +int fsl_cx_err_setv( fsl_cx * f, int code, char const * fmt, + va_list args ){ + return f + ? fsl_error_setv( &f->error, code, fmt, args ) + : FSL_RC_MISUSE; +} + +int fsl_cx_err_set( fsl_cx * f, int code, char const * fmt, + ... ){ + if(!f) return FSL_RC_MISUSE; + else{ + int rc; + va_list args; + va_start(args,fmt); + rc = fsl_error_setv( &f->error, code, fmt, args ); + va_end(args); + return rc; + } +} + +int fsl_cx_err_get( fsl_cx * f, char const ** str, fsl_size_t * len ){ + return f + ? fsl_error_get( &f->error, str, len ) + : FSL_RC_MISUSE; +} + +fsl_id_t fsl_cx_last_insert_id(fsl_cx * const f){ + return (f && f->dbMain && f->dbMain->dbh) + ? fsl_db_last_insert_id(f->dbMain) + : -1; +} + +fsl_cx * fsl_cx_malloc(){ + fsl_cx * rc = (fsl_cx *)fsl_malloc(sizeof(fsl_cx)); + if(rc) { + *rc = fsl_cx_empty; + rc->allocStamp = &fsl_cx_empty; + } + return rc; +} + +int fsl_cx_err_report( fsl_cx * const f, bool addNewline ){ + if(!f) return FSL_RC_MISUSE; + else if(f->error.code){ + char const * msg = f->error.msg.used + ? (char const *)f->error.msg.mem + : fsl_rc_cstr(f->error.code) + ; + return fsl_outputf(f, "Error #%d: %s%s", + f->error.code, msg, + addNewline ? "\n" : ""); + } + else return 0; +} + +int fsl_cx_uplift_db_error( fsl_cx * const f, fsl_db * db ){ + assert(f); + if(!f) return FSL_RC_MISUSE; + if(!db){ + db = f->dbMain; + assert(db && "misuse: no DB handle to uplift error from!"); + if(!db) return FSL_RC_MISUSE; + } + fsl_error_move( &db->error, &f->error ); + return f->error.code; +} + +int fsl_cx_uplift_db_error2(fsl_cx * const f, fsl_db * db, int rc){ + if(!db) db = f->dbMain; + assert(db); + if(rc && FSL_RC_OOM!=rc && !f->error.code && db->error.code){ + rc = fsl_cx_uplift_db_error(f, db); + } + return rc; +} + +fsl_db * fsl_cx_db_config( fsl_cx * const f ){ + if(!f) return NULL; + else if(f->config.db.dbh) return &f->config.db; + else if(f->dbMain && (FSL_DBROLE_CONFIG & f->dbMain->role)) return f->dbMain; + else return NULL; +} + +fsl_db * fsl_cx_db_repo( fsl_cx * const f ){ + if(!f) return NULL; + else if(f->repo.db.dbh) return &f->repo.db; + else if(f->dbMain && (FSL_DBROLE_REPO & f->dbMain->role)) return f->dbMain; + else return NULL; +} + +fsl_db * fsl_needs_repo(fsl_cx * const f){ + fsl_db * const db = fsl_cx_db_repo(f); + if(!db){ + fsl_cx_err_set(f, FSL_RC_NOT_A_REPO, + "Fossil context has no opened repository db."); + } + return db; +} + +fsl_db * fsl_needs_ckout(fsl_cx * const f){ + fsl_db * const db = fsl_cx_db_ckout(f); + if(!db){ + fsl_cx_err_set(f, FSL_RC_NOT_A_CKOUT, + "Fossil context has no opened checkout db."); + } + return db; +} + +fsl_db * fsl_cx_db_ckout( fsl_cx * const f ){ + if(!f) return NULL; + else if(f->ckout.db.dbh) return &f->ckout.db; + else if(f->dbMain && (FSL_DBROLE_CKOUT & f->dbMain->role)) return f->dbMain; + else return NULL; +} + +fsl_db * fsl_cx_db( fsl_cx * const f ){ + return f ? f->dbMain : NULL; +} +/** @internal + + Returns one of f->db{Config,Repo,Ckout,Mem} + or NULL. + + ACHTUNG and REMINDER TO SELF: the current (2021-03) design means + that none of these handles except for FSL_DBROLE_MAIN actually has + an sqlite3 db handle assigned to it. This returns a handle to the + "level of abstraction" we need to keep track of each db's name and + db-specific other state. + + e.g. passing a role of FSL_DBROLE_CKOUT this does NOT return + the same thing as fsl_cx_db_ckout(). +*/ +fsl_db * fsl_cx_db_for_role(fsl_cx * const f, fsl_dbrole_e r){ + switch(r){ + case FSL_DBROLE_CONFIG: + return &f->config.db; + case FSL_DBROLE_REPO: + return &f->repo.db; + case FSL_DBROLE_CKOUT: + return &f->ckout.db; + case FSL_DBROLE_MAIN: + return &f->dbMem; + case FSL_DBROLE_NONE: + default: + return NULL; + } +} + +/** + Detaches the given db role from f->dbMain and removes the role + from f->dbMain->role. +*/ +static int fsl_cx_detach_role(fsl_cx * f, fsl_dbrole_e r){ + if(!f || !f->dbMain) return FSL_RC_MISUSE; + else if(!(r & f->dbMain->role)){ + assert(!"Misuse: cannot detach unattached role."); + return FSL_RC_NOT_FOUND; + } + else{ + fsl_db * db = fsl_cx_db_for_role(f,r); + int rc; + if(!db) return FSL_RC_RANGE; + assert(f->dbMain != db); + f->dbMain->role &= ~r; + rc = fsl_db_detach( f->dbMain, fsl_db_role_label(r) ); + //MARKER(("rc=%s %s %s\n", fsl_rc_cstr(rc), fsl_db_role_label(r), + // fsl_buffer_cstr(&f->dbMain->error.msg))); + fsl_free(db->filename); + fsl_free(db->name); + db->filename = NULL; + db->name = NULL; + return rc; + } +} + + +/** @internal + + Attaches the given db file to f with the given role. This function "should" + be static but we need it in fsl_repo.c when creating a new repository. +*/ +int fsl_cx_attach_role(fsl_cx * const f, const char *zDbName, + fsl_dbrole_e r){ + char const * label = fsl_db_role_label(r); + fsl_db * db = fsl_cx_db_for_role(f, r); + char ** nameDest = NULL; + int rc; + if(!f->dbMain){ + fsl_fatal(FSL_RC_MISUSE,"Internal API misuse: f->dbMain has " + "not been set, so cannot attach role."); + return FSL_RC_MISUSE; + } + else if(r & f->dbMain->role){ + assert(!"Misuse: role is already attached."); + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "Db role %s is already attached.", + label); + } +#if 0 + MARKER(("r=%s db=%p, ckout.db=%p\n", label, + (void*)db, (void*)&f->ckout.db)); + MARKER(("r=%s db=%p, repo.db=%p\n", label, + (void*)db, (void*)&f->repo.db)); + MARKER(("r=%s db=%p, dbMain=%p\n", label, + (void*)db, (void*)f->dbMain)); +#endif + assert(db); + assert(label); + assert(f->dbMain != db); + assert(!db->filename); + assert(!db->name); + nameDest = &db->filename; + switch(r){ + case FSL_DBROLE_CONFIG: + case FSL_DBROLE_REPO: + case FSL_DBROLE_CKOUT: + break; + case FSL_DBROLE_MAIN: + case FSL_DBROLE_NONE: + default: + assert(!"cannot happen/not legal"); + return FSL_RC_RANGE; + } + *nameDest = fsl_strdup(zDbName); + db->name = *nameDest ? fsl_strdup(label) : NULL; + if(!db->name){ + rc = FSL_RC_OOM; + /* Design note: we do the strdup() before the ATTACH because if + the attach succeeds and strdup fails, detaching the db will + almost certainly fail because it must allocate for its prepared + statement and other internals. We would end up having to leave + the db attached and returning a failure, which could lead to a + memory leak (or worse) downstream. + */ + }else{ + /*MARKER(("Attached %p role %d %s %s\n", + (void const *)db, r, db->name, db->filename));*/ + rc = fsl_db_attach(f->dbMain, zDbName, label); + if(rc){ + fsl_cx_uplift_db_error(f, f->dbMain); + }else{ + //MARKER(("Attached db %p %s from %s\n", + // (void*)db, label, db->filename)); + f->dbMain->role |= r; + } + } + return rc; +} + +int fsl_config_close( fsl_cx * const f ){ + if(!f) return FSL_RC_MISUSE; + else{ + int rc; + fsl_db * const db = &f->config.db; + if(f->dbMain && (FSL_DBROLE_CONFIG & f->dbMain->role)){ + /* Config db is ATTACHed. */ + rc = fsl_cx_detach_role(f, FSL_DBROLE_CONFIG); + } + else rc = FSL_RC_NOT_FOUND; + assert(!db->dbh); + fsl_db_clear_strings(db, 1); + return rc; + } +} + +int fsl_repo_close( fsl_cx * const f ){ + if(fsl_cx_transaction_level(f)){ + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "Cannot close repo with opened transaction."); + }else{ + int rc; + fsl_db * const db = &f->repo.db; + if(f->dbMain && (FSL_DBROLE_REPO & f->dbMain->role)){ + /* Repo db is ATTACHed. */ + if(FSL_DBROLE_CKOUT & f->dbMain->role){ + rc = fsl_cx_err_set(f, FSL_RC_MISUSE, + "Cannot close repo while checkout is " + "opened."); + }else{ + assert(f->dbMain!=db); + rc = fsl_cx_detach_role(f, FSL_DBROLE_REPO); + } + } + else rc = FSL_RC_NOT_FOUND; + assert(!db->dbh); + fsl_db_clear_strings(db, true); + return rc; + } +} + +int fsl_ckout_close( fsl_cx * const f ){ + if(fsl_cx_transaction_level(f)){ + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "Cannot close checkout with opened transaction."); + }else{ + int rc; + fsl_db * const db = &f->ckout.db; + if(f->dbMain && (FSL_DBROLE_CKOUT & f->dbMain->role)){ + /* Checkout db is ATTACHed. */ + rc = fsl_cx_detach_role(f, FSL_DBROLE_CKOUT); + fsl_repo_close(f) /* + Because the repo is implicitly opened, we + "should" implicitly close it. This is + debatable but "probably almost always" + desired. i can't currently envisage a + reasonable use-case which requires closing + the checkout but keeping the repo opened. + The repo can always be re-opened by itself. + */; + }else{ + rc = FSL_RC_NOT_FOUND; + } + fsl_free(f->ckout.uuid); + f->ckout.uuid = NULL; + f->ckout.rid = 0; + assert(!db->dbh); + fsl_db_clear_strings(db, true); + return rc; + } +} + +/** + If zDbName is a valid checkout database file, open it and return 0. + If it is not a valid local database file, return a non-0 code. +*/ +static int fsl_cx_ckout_open_db(fsl_cx * f, const char *zDbName){ + /* char *zVFileDef; */ + int rc; + fsl_int_t const lsize = fsl_file_size(zDbName); + if( -1 == lsize ){ + return FSL_RC_NOT_FOUND /* might be FSL_RC_ACCESS? */; + } + if( lsize%1024!=0 || lsize<4096 ){ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "File's size is not correct for a " + "checkout db: %s", + zDbName); + } + rc = fsl_cx_attach_role(f, zDbName, FSL_DBROLE_CKOUT); + return rc; +} + + +int fsl_cx_execv( fsl_cx * const f, char const * sql, va_list args ){ + int const rc = (f->dbMain && sql) + ? fsl_db_execv(f->dbMain, sql, args) + : FSL_RC_MISUSE; + return rc ? fsl_cx_uplift_db_error2(f, f->dbMain, rc) : 0; +} + +int fsl_cx_exec( fsl_cx * const f, char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_cx_execv( f, sql, args ); + va_end(args); + return rc; +} + +int fsl_cx_exec_multiv( fsl_cx * const f, char const * sql, va_list args ){ + int const rc = (f->dbMain && sql) + ? fsl_db_exec_multiv(f->dbMain, sql, args) + : FSL_RC_MISUSE; + return rc ? fsl_cx_uplift_db_error2(f, f->dbMain, rc) : 0; +} + +int fsl_cx_exec_multi( fsl_cx * const f, char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_cx_exec_multiv( f, sql, args ); + va_end(args); + return rc; +} + +int fsl_cx_preparev( fsl_cx * const f, fsl_stmt * const tgt, char const * sql, + va_list args ){ + int const rc = (f->dbMain && tgt) + ? fsl_db_preparev(f->dbMain, tgt, sql, args) + : FSL_RC_MISUSE; + return rc ? fsl_cx_uplift_db_error2(f, f->dbMain, rc) : 0; +} + +int fsl_cx_prepare( fsl_cx * const f, fsl_stmt * const tgt, char const * sql, + ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_cx_preparev( f, tgt, sql, args ); + va_end(args); + return rc; +} + +/** + Passes the fsl_schema_config() SQL code through a new/truncated + file named dbName. If the file exists before this call, it is + unlink()ed and fails if that operation fails. + + FIXME: this was broken by the addition of "cfg." prefix on the + schema's tables. + */ +static int fsl_config_file_reset(fsl_cx * f, char const * dbName){ + fsl_db DB = fsl_db_empty; + fsl_db * db = &DB; + int rc = 0; + bool isAttached = false; + const char * zPrefix = fsl_db_role_label(FSL_DBROLE_CONFIG); + if(-1 != fsl_file_size(dbName)){ + rc = fsl_file_unlink(dbName); + if(rc){ + return fsl_cx_err_set(f, rc, + "Error %s while removing old config file (%s)", + fsl_rc_cstr(rc), dbName); + } + } + /** + Hoop-jumping: because the schema file has a cfg. prefix for the + table(s), and we cannot assign an arbitrary name to an open()'d + db, we first open the db (making the the "main" db), then + ATTACH it to itself to provide the fsl_db_role_label() alias. + */ + rc = fsl_db_open(db, dbName, FSL_OPEN_F_RWC); + if(rc) goto end; + rc = fsl_db_attach(db, dbName, zPrefix); + if(rc) goto end; + isAttached = true; + rc = fsl_db_exec_multi(db, "%s", fsl_schema_config()); + end: + rc = fsl_cx_uplift_db_error2(f, db, rc); + if(isAttached) fsl_db_detach(db, zPrefix); + fsl_db_close(db); + return rc; +} + +int fsl_config_global_preferred_name(char ** zOut){ + char * zEnv = 0; + char * zRc = 0; + int rc = 0; + fsl_buffer buf = fsl_buffer_empty; + +#if FSL_PLATFORM_IS_WINDOWS +# error "TODO: port in fossil(1) db.c:db_configdb_name() Windows bits" +#else + +#endif + + /* Option 1: $FOSSIL_HOME/.fossil */ + zEnv = fsl_getenv("FOSSIL_HOME"); + if(zEnv){ + zRc = fsl_mprintf("%s/.fossil", zEnv); + if(!zRc) rc = FSL_RC_OOM; + goto end; + } + /* Option 2: if $HOME/.fossil exists, use that */ + rc = fsl_find_home_dir(&buf, 0); + if(rc) goto end; + rc = fsl_buffer_append(&buf, "/.fossil", 8); + if(rc) goto end; + if(fsl_file_size(fsl_buffer_cstr(&buf))>1024*3){ + zRc = fsl_buffer_take(&buf); + goto end; + } + /* Option 3: $XDG_CONFIG_HOME/fossil.db */ + fsl_filename_free(zEnv); + zEnv = fsl_getenv("XDG_CONFIG_HOME"); + if(zEnv){ + zRc = fsl_mprintf("%s/fossil.db", zEnv); + if(!zRc) rc = FSL_RC_OOM; + goto end; + } + /* Option 4: If $HOME/.config is a directory, + use $HOME/.config/fossil.db */ + buf.used -= 8 /* "/.fossil" */; + buf.mem[buf.used] = 0; + rc = fsl_buffer_append(&buf, "/.config", 8); + if(rc) goto end; + if(fsl_dir_check(fsl_buffer_cstr(&buf))>0){ + zRc = fsl_mprintf("%b/fossil.db", &buf); + if(!zRc) rc = FSL_RC_OOM; + goto end; + } + /* Option 5: fall back to $HOME/.fossil */ + buf.used -= 8 /* "/.config" */; + buf.mem[buf.used] = 0; + rc = fsl_buffer_append(&buf, "/.fossil", 8); + if(!rc) zRc = fsl_buffer_take(&buf); + + end: + if(zEnv) fsl_filename_free(zEnv); + if(!rc){ + assert(zRc); + *zOut = zRc; + } + fsl_buffer_clear(&buf); + return rc; +} + +int fsl_config_open( fsl_cx * const f, char const * openDbName ){ + int rc = 0; + const char * zDbName = 0; + char * zPrefName = 0; + if(!f) return FSL_RC_MISUSE; + else if(f->config.db.dbh){ + fsl_config_close(f); + } + if(openDbName && *openDbName){ + zDbName = openDbName; + }else{ + rc = fsl_config_global_preferred_name(&zPrefName); + if(rc) goto end; + zDbName = zPrefName; + } + { + fsl_int_t const fsize = fsl_file_size(zDbName); + if( -1==fsize || (fsize<1024*3) ){ + rc = fsl_config_file_reset(f, zDbName); + if(rc) goto end; + } + } +#if defined(_WIN32) || defined(__CYGWIN__) + /* TODO: Jan made some changes in this area in fossil(1) in + January(?) 2014, such that only the config file needs to be + writable, not the directory. Port that in. + */ + if( fsl_file_access(zDbName, W_OK) ){ + rc = fsl_cx_err_set(f, FSL_RC_ACCESS, + "Configuration database [%s] " + "must be writeable.", zDbName); + goto end; + } +#endif + rc = fsl_cx_attach_role(f, zDbName, FSL_DBROLE_CONFIG); + end: + fsl_free(zPrefName); + return rc; +} + +static void fsl_cx_username_from_repo(fsl_cx * f){ + fsl_db * dbR = fsl_cx_db_repo(f); + char * u; + assert(dbR); + u = fsl_db_g_text(fsl_cx_db_repo(f), NULL, + "SELECT login FROM user WHERE uid=1"); + if(u){ + fsl_free(f->repo.user); + f->repo.user = u; + } +} + +static int fsl_cx_load_glob_lists(fsl_cx * f){ + int rc; + rc = fsl_config_globs_load(f, &f->cache.globs.ignore, "ignore-glob"); + if(!rc) rc = fsl_config_globs_load(f, &f->cache.globs.binary, "binary-glob"); + if(!rc) rc = fsl_config_globs_load(f, &f->cache.globs.crnl, "crnl-glob"); + return rc; +} + +int fsl_cx_glob_list( fsl_cx * const f, + fsl_glob_category_e gtype, + fsl_list **tgt, + bool reload ){ + fsl_list * li = NULL; + char const * reloadKey = NULL; + switch(gtype){ + case FSL_GLOBS_IGNORE: li = &f->cache.globs.ignore; + reloadKey = "ignore-glob"; break; + case FSL_GLOBS_CRNL: li = &f->cache.globs.crnl; + reloadKey = "crnl-glob"; break; + case FSL_GLOBS_BINARY: li = &f->cache.globs.binary; + reloadKey = "binary-glob"; break; + default: + return FSL_RC_RANGE; + } + int rc = 0; + if(reload){ + assert(reloadKey); + fsl_glob_list_clear(li); + rc = fsl_config_globs_load(f, li, reloadKey); + } + if(0==rc) *tgt = li; + return rc; +} + +fsl_glob_category_e fsl_glob_name_to_category(char const * str){ + if(str){ +#define CHECK(PRE,E) \ + if(*str==PRE[0] && \ + (0==fsl_strcmp(PRE "-glob",str) \ + || 0==fsl_strcmp(PRE,str))) return E; + CHECK("ignore", FSL_GLOBS_IGNORE); + CHECK("binary", FSL_GLOBS_BINARY); + CHECK("crnl", FSL_GLOBS_CRNL); +#undef CHECK + } + return FSL_GLOBS_INVALID; +} + + +/** + To be called after a repo or checkout/repo combination has been + opened. This updates some internal cached info based on the + checkout and/or repo. +*/ +static int fsl_cx_after_open(fsl_cx * f){ + int rc = fsl_ckout_version_fetch(f); + if(!rc) rc = fsl_cx_load_glob_lists(f); + return rc; +} + + +static void fsl_cx_fetch_hash_policy(fsl_cx * f){ + int const iPol = + fsl_config_get_int32( f, FSL_CONFDB_REPO, + FSL_HPOLICY_AUTO, "hash-policy"); + fsl_hashpolicy_e p; + switch(iPol){ + case FSL_HPOLICY_SHA3: p = FSL_HPOLICY_SHA3; break; + case FSL_HPOLICY_SHA3_ONLY: p = FSL_HPOLICY_SHA3_ONLY; break; + case FSL_HPOLICY_SHA1: p = FSL_HPOLICY_SHA1; break; + case FSL_HPOLICY_SHUN_SHA1: p = FSL_HPOLICY_SHUN_SHA1; break; + default: p = FSL_HPOLICY_AUTO; break; + } + f->cxConfig.hashPolicy = p; +} + +int fsl_repo_open( fsl_cx * const f, char const * repoDbFile + /* , bool readOnlyCurrentlyIgnored */ ){ + if(fsl_cx_db_repo(f)){ + return fsl_cx_err_set(f, FSL_RC_ACCESS, + "Context already has an opened repository."); + }else{ + int rc; + if(0!=fsl_file_access( repoDbFile, F_OK )){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Repository db [%s] not found or cannot be read.", + repoDbFile); + }else{ + rc = fsl_cx_attach_role(f, repoDbFile, FSL_DBROLE_REPO); + if(!rc && !(FSL_CX_F_IS_OPENING_CKOUT & f->flags)){ + rc = fsl_cx_after_open(f); + } + if(!rc){ + fsl_db * const db = fsl_cx_db_repo(f); + fsl_cx_username_from_repo(f); + f->cache.allowSymlinks = + fsl_config_get_bool(f, FSL_CONFDB_REPO, + false, + "allow-symlinks"); + f->cache.seenDeltaManifest = + fsl_config_get_int32(f, FSL_CONFDB_REPO, -1, + "seen-delta-manifest"); + fsl_cx_fetch_hash_policy(f); + if(f->cxConfig.hashPolicy==FSL_HPOLICY_AUTO){ + if(fsl_db_exists(db, "SELECT 1 FROM blob WHERE length(uuid)>40") + || !fsl_db_exists(db, "SELECT 1 FROM blob WHERE length(uuid)==40")){ + f->cxConfig.hashPolicy = FSL_HPOLICY_SHA3; + } + } + } + } + return rc; + } +} + +/** + Tries to open the repository from which the current checkout + derives. Returns 0 on success. +*/ +static int fsl_repo_open_for_ckout(fsl_cx * f){ + char * repoDb = NULL; + int rc; + fsl_buffer nameBuf = fsl_buffer_empty; + fsl_db * db = fsl_cx_db_ckout(f); + assert(f); + assert(f->ckout.dir); + assert(db); + rc = fsl_db_get_text(db, &repoDb, NULL, + "SELECT value FROM vvar " + "WHERE name='repository'"); + if(rc) fsl_cx_uplift_db_error( f, db ); + else if(repoDb){ + if(!fsl_is_absolute_path(repoDb)){ + /* Make it relative to the checkout db dir */ + rc = fsl_buffer_appendf(&nameBuf, "%s/%s", f->ckout.dir, repoDb); + fsl_free(repoDb); + if(rc) { + fsl_buffer_clear(&nameBuf); + return rc; + } + repoDb = (char*)nameBuf.mem /* transfer ownership */; + nameBuf = fsl_buffer_empty; + } + rc = fsl_file_canonical_name(repoDb, &nameBuf, 0); + fsl_free(repoDb); + if(!rc){ + repoDb = fsl_buffer_str(&nameBuf); + assert(repoDb); + rc = fsl_repo_open(f, repoDb); + } + fsl_buffer_reserve(&nameBuf, 0); + }else{ + /* This can only happen if we are not using a proper + checkout db or someone has removed the repo link. + */ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Could not determine this checkout's " + "repository db file."); + } + return rc; +} + +static void fsl_ckout_mtime_set(fsl_cx * const f){ + f->ckout.mtime = f->ckout.rid>0 + ? fsl_db_g_double(fsl_cx_db_repo(f), 0.0, + "SELECT mtime FROM event " + "WHERE objid=%" FSL_ID_T_PFMT, + f->ckout.rid) + : 0.0; +} + +int fsl_ckout_version_fetch( fsl_cx *f ){ + fsl_id_t rid = 0; + int rc = 0; + fsl_db * dbC = fsl_cx_db_ckout(f); + fsl_db * dbR = dbC ? fsl_needs_repo(f) : NULL; + assert(!dbC || (dbC && dbR)); + fsl_free(f->ckout.uuid); + f->ckout.rid = -1; + f->ckout.uuid = NULL; + f->ckout.mtime = 0.0; + if(!dbC){ + return 0; + } + fsl_cx_err_reset(f); + rid = fsl_config_get_id(f, FSL_CONFDB_CKOUT, -1, "checkout"); + //MARKER(("rc=%s rid=%d\n",fsl_rc_cstr(f->error.code), (int)rid)); + if(rid>0){ + f->ckout.uuid = fsl_rid_to_uuid(f, rid); + if(!f->ckout.uuid){ + assert(f->error.code); + if(!f->error.code){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Could not load UUID for RID %"FSL_ID_T_PFMT, + (fsl_id_t)rid); + } + }else{ + assert(fsl_is_uuid(f->ckout.uuid)); + } + f->ckout.rid = rid; + fsl_ckout_mtime_set(f); + }else if(rid==0){ + /* This is a legal case not possible before libfossil (and only + afterwards possible in fossil(1)) - an empty repo without an + active checkin. [Much later:] that capability has since been + removed from fossil. + */ + f->ckout.rid = 0; + }else{ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Cannot determine checkout version."); + } + return rc; +} + +/** @internal + + Sets f->ckout.rid to the given rid (which must be 0 or a valid + RID) and f->ckout.uuid to a copy of the given uuid. If uuid is + NULL and rid is not 0 then the uuid is fetched using + fsl_rid_to_uuid(), else if uuid is not NULL then it is assumed to + be the UUID for the given RID and is copies to f->ckout.uuid. + + Returns 0 on success, FSL_RC_OOM if copying uuid fails, or some + error from fsl_rid_to_uuid() if that fails. + + Does not write the changes to disk. Use fsl_ckout_version_write() + for that. That routine also calls this one, so there's no need to + call both. +*/ +static int fsl_cx_ckout_version_set(fsl_cx *f, fsl_id_t rid, + fsl_uuid_cstr uuid){ + char * u = 0; + assert(rid>=0); + u = uuid + ? fsl_strdup(uuid) + : (rid ? fsl_rid_to_uuid(f, rid) : NULL); + if(rid && !u) return FSL_RC_OOM; + f->ckout.rid = rid; + fsl_free(f->ckout.uuid); + f->ckout.uuid = u; + fsl_ckout_mtime_set(f); + return 0; +} + +int fsl_ckout_version_write( fsl_cx *f, fsl_id_t vid, + fsl_uuid_cstr hash ){ + int rc = 0; + if(!fsl_needs_ckout(f)) return FSL_RC_NOT_A_CKOUT; + else if(vid<0){ + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "Invalid vid for fsl_ckout_version_write()"); + } + if(f->ckout.rid!=vid){ + rc = fsl_cx_ckout_version_set(f, vid, hash); + } + if(!rc){ + rc = fsl_config_set_id(f, FSL_CONFDB_CKOUT, + "checkout", f->ckout.rid); + if(!rc){ + rc = fsl_config_set_text(f, FSL_CONFDB_CKOUT, + "checkout-hash", f->ckout.uuid); + } + } + if(!rc){ + char * zFingerprint = 0; + rc = fsl_repo_fingerprint_search(f, 0, &zFingerprint); + if(!rc){ + rc = fsl_config_set_text(f, FSL_CONFDB_CKOUT, + "fingerprint", zFingerprint); + fsl_free(zFingerprint); + } + } + if(!rc){ + int const mode = vid ? -1 : 0; + rc = fsl_ckout_manifest_write(f, mode, mode, mode, 0); + } + return rc; +} + +void fsl_ckout_version_info(fsl_cx * const f, fsl_id_t * const rid, + fsl_uuid_cstr * const uuid ){ + if(uuid) *uuid = f->ckout.uuid; + if(rid) *rid = f->ckout.rid>=0 ? f->ckout.rid : 0; +} + +int fsl_ckout_db_search( char const * dirName, bool checkParentDirs, + fsl_buffer * const pOut ){ + int rc; + fsl_int_t dLen = 0, i; + enum { DbCount = 2 }; + const char aDbName[DbCount][10] = { "_FOSSIL_", ".fslckout" }; + fsl_buffer Buf = fsl_buffer_empty; + fsl_buffer * buf = &Buf; + buf->used = 0; + if(dirName){ + dLen = fsl_strlen(dirName); + if(0==dLen) return FSL_RC_RANGE; + rc = fsl_buffer_reserve( buf, (fsl_size_t)(dLen + 10) ); + if(!rc) rc = fsl_buffer_append( buf, dirName, dLen ); + if(rc){ + fsl_buffer_clear(buf); + return rc; + } + }else{ + char zPwd[4000]; + fsl_size_t pwdLen = 0; + rc = fsl_getcwd( zPwd, sizeof(zPwd)/sizeof(zPwd[0]), &pwdLen ); + if(rc){ + fsl_buffer_clear(buf); +#if 0 + return fsl_cx_err_set(f, rc, + "Could not determine current directory. " + "Error code %d (%s).", + rc, fsl_rc_cstr(rc)); +#else + return rc; +#endif + } + if(1 == pwdLen && '/'==*zPwd) *zPwd = '.' + /* When in the root directory (or chroot) then change dir name + name to something we can use. + */; + rc = fsl_buffer_append(buf, zPwd, pwdLen); + if(rc){ + fsl_buffer_clear(buf); + return rc; + } + dLen = (fsl_int_t)pwdLen; + } + if(rc){ + fsl_buffer_clear(buf); + return rc; + } + assert(buf->capacity>=buf->used); + assert((buf->used == (fsl_size_t)dLen) || (1==buf->used && (int)'.'==(int)buf->mem[0])); + assert(0==buf->mem[buf->used]); + + while(dLen>0){ + /* + Loop over the list in aDbName, appending each one to + the dir name in the search for something we can use. + */ + fsl_int_t lenMarker = dLen /* position to re-set to on each + sub-iteration. */ ; + /* trim trailing slashes on this part, so that we don't end up + with multiples between the dir and file in the final output. */ + while( dLen && ((int)'/'==(int)buf->mem[dLen-1])) --dLen; + for( i = 0; i < DbCount; ++i ){ + char const * zName; + buf->used = (fsl_size_t)lenMarker; + dLen = lenMarker; + rc = fsl_buffer_appendf( buf, "/%s", aDbName[i]); + if(rc){ + fsl_buffer_clear(buf); + return rc; + } + zName = fsl_buffer_cstr(buf); + if(0==fsl_file_access(zName, 0)){ + if(pOut) rc = fsl_buffer_append( pOut, buf->mem, buf->used ); + fsl_buffer_clear(buf); + return rc; + } + if(!checkParentDirs){ + dLen = 0; + break; + }else{ + /* Traverse up one dir and try again. */ + --dLen; + while( dLen>0 && (int)buf->mem[dLen]!=(int)'/' ){ --dLen; } + while( dLen>0 && (int)buf->mem[dLen-1]==(int)'/' ){ --dLen; } + if(dLen>lenMarker){ + buf->mem[dLen] = 0; + } + } + } + } + fsl_buffer_clear(buf); + return FSL_RC_NOT_FOUND; +} + +int fsl_cx_getcwd(fsl_cx * f, fsl_buffer * pOut){ + char cwd[FILENAME_MAX] = {0}; + fsl_size_t cwdLen = 0; + int rc = fsl_getcwd(cwd, (fsl_size_t)sizeof(cwd), &cwdLen); + if(rc){ + return fsl_cx_err_set(f, rc, + "Could not get current working directory!"); + } + rc = fsl_buffer_append(pOut, cwd, cwdLen); + return rc + ? fsl_cx_err_set(f, rc/*must be an OOM*/, NULL) + : 0; +} + +int fsl_ckout_open_dir( fsl_cx * f, char const * dirName, + bool checkParentDirs ){ + int rc; + fsl_buffer Buf = fsl_buffer_empty; + fsl_buffer * buf = &Buf; + char const * zName; + if(fsl_cx_db_ckout(f)){ + return fsl_cx_err_set( f, FSL_RC_ACCESS, + "A checkout is already opened. " + "Close it before opening another."); + } + rc = fsl_ckout_db_search(dirName, checkParentDirs, buf); + if(rc){ + if(FSL_RC_NOT_FOUND==rc){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Could not find checkout under [%s].", + dirName ? dirName : "."); + } + fsl_buffer_clear(buf); + return rc; + } + assert(buf->used>1 /* "/" */); + zName = fsl_buffer_cstr(buf); + rc = fsl_cx_ckout_open_db(f, zName); + if(rc){ + fsl_buffer_clear(buf); + return rc; + }else{ + /* Checkout db is now opened. Fiddle some internal + bits... + */ + unsigned char * end = buf->mem+buf->used-1; + /* Find dir part */ + while(end>buf->mem && (unsigned char)'/'!=*end) --end; + assert('/' == (char)*end && "fsl_ckout_db_search() appends '/'"); + fsl_free(f->ckout.dir); + f->ckout.dirLen = end - buf->mem +1 /* for trailing '/' */ ; + *(end+1) = 0; /* Rather than strdup'ing, we'll just lop off the + filename part. Keep the '/' for historical + conventions purposes - it simplifies path + manipulation later on. */ + f->ckout.dir = fsl_buffer_take(buf); + assert(!f->ckout.dir[f->ckout.dirLen]); + assert('/' == f->ckout.dir[f->ckout.dirLen-1]); + f->flags |= FSL_CX_F_IS_OPENING_CKOUT; + rc = fsl_repo_open_for_ckout(f); + f->flags &= ~FSL_CX_F_IS_OPENING_CKOUT; + if(!rc) rc = fsl_cx_after_open(f); + if(rc){ + /* Is this sane? Is not doing it sane? */ + fsl_ckout_close(f); + } + return rc; + } +} + + +char const * fsl_cx_db_file_for_role(fsl_cx const * f, + fsl_dbrole_e r, + fsl_size_t * len){ + fsl_db const * db = fsl_cx_db_for_role((fsl_cx*)f, r); + char const * rc = db ? db->filename : NULL; + if(len) *len = fsl_strlen(rc); + return rc; +} + +char const * fsl_cx_db_name_for_role(fsl_cx const * f, + fsl_dbrole_e r, + fsl_size_t * len){ + if(FSL_DBROLE_MAIN == r){ + /* special case to be removed when f->dbMem bits are + finished. */ + if(len) *len=4; + return "main"; + }else{ + fsl_db const * db = fsl_cx_db_for_role((fsl_cx*)f, r); + char const * rc = db ? db->name : NULL; + if(len) *len = rc ? fsl_strlen(rc) : 0; + return rc; + } +} + +char const * fsl_cx_db_file_config(fsl_cx const * f, + fsl_size_t * len){ + char const * rc = NULL; + if(f && f->config.db.filename){ + rc = f->config.db.filename; + if(len) *len = fsl_strlen(rc); + } + return rc; +} + +char const * fsl_cx_db_file_repo(fsl_cx const * f, + fsl_size_t * len){ + char const * rc = NULL; + if(f && f->repo.db.filename){ + rc = f->repo.db.filename; + if(len) *len = fsl_strlen(rc); + } + return rc; +} + +char const * fsl_cx_db_file_ckout(fsl_cx const * f, + fsl_size_t * len){ + char const * rc = NULL; + if(f && f->ckout.db.filename){ + rc = f->ckout.db.filename; + if(len) *len = fsl_strlen(rc); + } + return rc; +} + +char const * fsl_cx_ckout_dir_name(fsl_cx const * f, + fsl_size_t * len){ + char const * rc = NULL; + if(f && f->ckout.dir){ + rc = f->ckout.dir; + if(len) *len = f->ckout.dirLen; + } + return rc; +} + +int fsl_cx_flags_get( fsl_cx * f ){ + return f->flags; +} + +int fsl_cx_flag_set( fsl_cx * f, int flags, bool enable ){ + int const oldFlags = f->flags; + if(enable) f->flags |= flags; + else f->flags &= ~flags; + return oldFlags; +} + + +fsl_xlinker * fsl_xlinker_by_name( fsl_cx * f, char const * name ){ + + fsl_xlinker * rv = NULL; + fsl_size_t i; + for( i = 0; i < f->xlinkers.used; ++i ){ + rv = f->xlinkers.list + i; + if(0==fsl_strcmp(rv->name, name)) return rv; + } + return NULL; +} + +int fsl_xlink_listener( fsl_cx * const f, char const * name, + fsl_deck_xlink_f cb, void * cbState ){ + fsl_xlinker * x; + if(!*name) return FSL_RC_MISUSE; + x = fsl_xlinker_by_name(f, name); + if(x){ + /* Replace existing entry */ + x->f = cb; + x->state = cbState; + return 0; + }else if(f->xlinkers.used <= f->xlinkers.capacity){ + /* Expand the array */ + fsl_size_t const n = f->xlinkers.used ? f->xlinkers.used * 2 : 5; + fsl_xlinker * re = + (fsl_xlinker *)fsl_realloc(f->xlinkers.list, + n * sizeof(fsl_xlinker)); + if(!re) return FSL_RC_OOM; + f->xlinkers.list = re; + f->xlinkers.capacity = n; + } + x = f->xlinkers.list + f->xlinkers.used++; + *x = fsl_xlinker_empty; + x->f = cb; + x->state = cbState; + x->name = name; + return 0; +} + +int fsl_cx_user_set( fsl_cx * f, char const * userName ){ + if(!f) return FSL_RC_MISUSE; + else if(!userName || !*userName){ + fsl_free(f->repo.user); + f->repo.user = NULL; + return 0; + }else{ + char * u = fsl_strdup(userName); + if(!u) return FSL_RC_OOM; + else{ + fsl_free(f->repo.user); + f->repo.user = u; + return 0; + } + } +} + +char const * fsl_cx_user_get( fsl_cx const * f ){ + return f ? f->repo.user : NULL; +} + +int fsl_cx_schema_ticket(fsl_cx * f, fsl_buffer * pOut){ + fsl_db * db = f ? fsl_needs_repo(f) : NULL; + if(!f || !pOut) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + else{ + fsl_size_t const oldUsed = pOut->used; + int rc = fsl_config_get_buffer(f, FSL_CONFDB_REPO, + "ticket-table", pOut); + if((FSL_RC_NOT_FOUND==rc) + || (oldUsed == pOut->used/*found but it was empty*/) + ){ + rc = fsl_buffer_append(pOut, fsl_schema_ticket(), -1); + } + return rc; + } +} + + +int fsl_cx_stat2( fsl_cx * const f, bool relativeToCwd, + char const * zName, fsl_fstat * const tgt, + fsl_buffer * const nameOut, bool fullPath){ + int rc; + fsl_buffer * b = 0; + fsl_buffer * bufRel = 0; + fsl_size_t n; + assert(f); + if(!zName || !*zName) return FSL_RC_MISUSE; + else if(!fsl_needs_ckout(f)) return FSL_RC_NOT_A_CKOUT; + b = fsl_cx_scratchpad(f); + bufRel = fsl_cx_scratchpad(f); +#if 1 + rc = fsl_ckout_filename_check(f, relativeToCwd, zName, bufRel); + if(rc) goto end; + zName = fsl_buffer_cstr2( bufRel, &n ); +#else + if(!fsl_is_simple_pathname(zName, 1)){ + rc = fsl_ckout_filename_check(f, relativeToCwd, zName, bufRel); + if(rc) goto end; + zName = fsl_buffer_cstr2( bufRel, &n ); + /* MARKER(("bufRel=%s\n",zName)); */ + }else{ + n = fsl_strlen(zName); + } +#endif + assert(n>0 && + "Will fail if fsl_ckout_filename_check() changes " + "to return nothing if zName==checkout root"); + if(!n + /* i don't like the "." resp "./" result when zName==checkout root */ + || (1==n && '.'==bufRel->mem[0]) + || (2==n && '.'==bufRel->mem[0] && '/'==bufRel->mem[1])){ + rc = fsl_buffer_appendf(b, "%s%s", f->ckout.dir, + (2==n) ? "/" : ""); + }else{ + rc = fsl_buffer_appendf(b, "%s%s", f->ckout.dir, zName); + } + if(!rc){ + rc = fsl_stat( fsl_buffer_cstr(b), tgt, false ); + if(rc){ + fsl_cx_err_set(f, rc, "Error %s from fsl_stat(\"%b\")", + fsl_rc_cstr(rc), b); + }else if(nameOut){ + rc = fullPath + ? fsl_buffer_append(nameOut, b->mem, b->used) + : fsl_buffer_append(nameOut, zName, n); + } + } + end: + fsl_cx_scratchpad_yield(f, b); + fsl_cx_scratchpad_yield(f, bufRel); + return rc; +} + +int fsl_cx_stat(fsl_cx * const f, bool relativeToCwd, + char const * zName, fsl_fstat * const tgt){ + return fsl_cx_stat2(f, relativeToCwd, zName, tgt, NULL, false); +} + + +void fsl_cx_case_sensitive_set(fsl_cx * f, bool caseSensitive){ + f->cache.caseInsensitive = caseSensitive; +} + +bool fsl_cx_is_case_sensitive(fsl_cx const * f){ + return !f->cache.caseInsensitive; +} + +char const * fsl_cx_filename_collation(fsl_cx const * f){ + return f->cache.caseInsensitive ? "COLLATE nocase" : ""; +} + + +void fsl_cx_content_buffer_yield(fsl_cx * const f){ + enum { MaxSize = 1024 * 1024 * 2 }; + assert(f); + if(f->fileContent.capacity>MaxSize){ + fsl_buffer_resize(&f->fileContent, MaxSize); + assert(f->fileContent.capacity<=MaxSize+1); + } + fsl_buffer_reuse(&f->fileContent); +} + +fsl_error const * fsl_cx_err_get_e(fsl_cx const * f){ + return f ? &f->error : NULL; +} + +int fsl_cx_close_dbs( fsl_cx * const f ){ + if(fsl_cx_transaction_level(f)){ + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "Cannot close the databases when a " + "transaction is pending."); + } + int rc = 0, rc1; + rc1 = fsl_ckout_close(f); + if(rc1) rc = rc1; + rc1 = fsl_repo_close(f); + if(rc1) rc = rc1; + rc1 = fsl_config_close(f); + if(rc1) rc = rc1; + assert(!f->repo.db.dbh); + assert(!f->ckout.db.dbh); + assert(!f->config.db.dbh); + return rc; +} + +char const * fsl_cx_glob_matches( fsl_cx * const f, int gtype, + char const * str ){ + int i, count = 0; + char const * rv = NULL; + fsl_list const * lists[] = {0,0,0}; + if(!f || !str || !*str) return NULL; + if(gtype & FSL_GLOBS_IGNORE) lists[count++] = &f->cache.globs.ignore; + if(gtype & FSL_GLOBS_CRNL) lists[count++] = &f->cache.globs.crnl; + /*CRNL/BINARY together makes little sense, but why strictly prohibit + it?*/ + if(gtype & FSL_GLOBS_BINARY) lists[count++] = &f->cache.globs.binary; + for( i = 0; i < count; ++i ){ + if( (rv = fsl_glob_list_matches( lists[i], str )) ) break; + } + return rv; +} + +int fsl_output_f_fsl_cx(void * state, void const * src, fsl_size_t n ){ + return (state && src && n) + ? fsl_output((fsl_cx*)state, src, n) + : (n ? FSL_RC_MISUSE : 0); +} + +int fsl_cx_hash_buffer( fsl_cx const * f, bool useAlternate, + fsl_buffer const * pIn, fsl_buffer * pOut){ + /* fossil(1) counterpart: hname_hash() */ + if(useAlternate){ + switch(f->cxConfig.hashPolicy){ + case FSL_HPOLICY_AUTO: + case FSL_HPOLICY_SHA1: + return fsl_sha3sum_buffer(pIn, pOut); + case FSL_HPOLICY_SHA3: + return fsl_sha1sum_buffer(pIn, pOut); + default: return FSL_RC_UNSUPPORTED; + } + }else{ + switch(f->cxConfig.hashPolicy){ + case FSL_HPOLICY_SHA1: + case FSL_HPOLICY_AUTO: + return fsl_sha1sum_buffer(pIn, pOut); + case FSL_HPOLICY_SHA3: + case FSL_HPOLICY_SHA3_ONLY: + case FSL_HPOLICY_SHUN_SHA1: + return fsl_sha3sum_buffer(pIn, pOut); + } + } + assert(!"not reached"); + return FSL_RC_RANGE; +} + +int fsl_cx_hash_filename( fsl_cx * f, bool useAlternate, + const char * zFilename, fsl_buffer * pOut){ + /* FIXME: reimplement this to stream the content in bite-sized + chunks. That requires duplicating most of fsl_buffer_fill_from() + and fsl_cx_hash_buffer(). */ + fsl_buffer * content = &f->fileContent; + int rc; + assert(!content->used && "Internal recursive misuse of fsl_cx::fileContent"); + fsl_buffer_reuse(content); + rc = fsl_buffer_fill_from_filename(content, zFilename); + if(!rc){ + rc = fsl_cx_hash_buffer(f, useAlternate, content, pOut); + } + fsl_buffer_reuse(content); + return rc; +} + +char const * fsl_hash_policy_name(fsl_hashpolicy_e p){ + switch(p){ + case FSL_HPOLICY_SHUN_SHA1: return "shun-sha1"; + case FSL_HPOLICY_SHA3: return "sha3"; + case FSL_HPOLICY_SHA3_ONLY: return "sha3-only"; + case FSL_HPOLICY_SHA1: return "sha1"; + case FSL_HPOLICY_AUTO: return "auto"; + default: return NULL; + } +} + +fsl_hashpolicy_e fsl_cx_hash_policy_set(fsl_cx *f, fsl_hashpolicy_e p){ + fsl_hashpolicy_e const old = f->cxConfig.hashPolicy; + fsl_db * const dbR = fsl_cx_db_repo(f); + if(dbR){ + /* Write it regardless of whether it's the same as the old policy + so that we're sure the db knows the policy. */ + if(FSL_HPOLICY_AUTO==p && + fsl_db_exists(dbR,"SELECT 1 FROM blob WHERE length(uuid)>40")){ + p = FSL_HPOLICY_SHA3; + } + fsl_config_set_int32(f, FSL_CONFDB_REPO, "hash-policy", p); + } + f->cxConfig.hashPolicy = p; + return old; +} + +fsl_hashpolicy_e fsl_cx_hash_policy_get(fsl_cx const*f){ + return f->cxConfig.hashPolicy; +} + +int fsl_cx_transaction_level(fsl_cx * const f){ + return f->dbMain + ? fsl_db_transaction_level(f->dbMain) + : 0; +} + +int fsl_cx_transaction_begin(fsl_cx * const f){ + int const rc = fsl_db_transaction_begin(f->dbMain); + return rc ? fsl_cx_uplift_db_error2(f, f->dbMain, rc) : 0; +} + +int fsl_cx_transaction_end(fsl_cx * const f, bool doRollback){ + int const rc = fsl_db_transaction_end(f->dbMain, doRollback); + return rc ? fsl_cx_uplift_db_error2(f, f->dbMain, rc) : 0; +} + +void fsl_cx_confirmer(fsl_cx * f, + fsl_confirmer const * newConfirmer, + fsl_confirmer * prevConfirmer){ + if(prevConfirmer) *prevConfirmer = f->confirmer; + f->confirmer = newConfirmer ? *newConfirmer : fsl_confirmer_empty; +} + +void fsl_cx_confirmer_get(fsl_cx const * f, fsl_confirmer * dest){ + *dest = f->confirmer; +} + +int fsl_cx_confirm(fsl_cx *f, fsl_confirm_detail const * detail, + fsl_confirm_response *outAnswer){ + if(f->confirmer.callback){ + return f->confirmer.callback(detail, outAnswer, + f->confirmer.callbackState); + } + /* Default answers... */ + switch(detail->eventId){ + case FSL_CEVENT_OVERWRITE_MOD_FILE: + case FSL_CEVENT_OVERWRITE_UNMGD_FILE: + outAnswer->response = FSL_CRESPONSE_NEVER; + break; + case FSL_CEVENT_RM_MOD_UNMGD_FILE: + outAnswer->response = FSL_CRESPONSE_NEVER; + break; + case FSL_CEVENT_MULTIPLE_VERSIONS: + outAnswer->response = FSL_CRESPONSE_CANCEL; + break; + default: + assert(!"Unhandled fsl_confirm_event_e value"); + fsl_fatal(FSL_RC_UNSUPPORTED, + "Unhandled fsl_confirm_event_e value: %d", + detail->eventId)/*does not return*/; + } + return 0; +} + +int fsl_cx_update_seen_delta_mf(fsl_cx *f){ + int rc = 0; + fsl_db * const d = fsl_cx_db_repo(f); + if(d && f->cache.seenDeltaManifest <= 0){ + f->cache.seenDeltaManifest = 1; + rc = fsl_config_set_bool(f, FSL_CONFDB_REPO, + "seen-delta-manifest", 1); + } + return rc; +} + +int fsl_reserved_fn_check(fsl_cx *f, const char *zPath, + fsl_int_t nPath, bool relativeToCwd){ + static const int errRc = FSL_RC_RANGE; + int rc = 0; + char const * z1 = 0; + if(nPath<0) nPath = (fsl_int_t)fsl_strlen(zPath); + if(fsl_is_reserved_fn(zPath, nPath)){ + return fsl_cx_err_set(f, errRc, + "Filename is reserved, not legal " + "for adding to a repository: %.*s", + (int)nPath, zPath); + } + if(!(f->flags & FSL_CX_F_ALLOW_WINDOWS_RESERVED_NAMES) + && fsl_is_reserved_fn_windows(zPath, nPath)){ + return fsl_cx_err_set(f, errRc, + "Filename is a Windows reserved name: %.*s", + (int)nPath, zPath); + } + if((z1 = fsl_cx_db_file_for_role(f, FSL_DBROLE_REPO, NULL))){ + fsl_buffer * c1 = fsl_cx_scratchpad(f); + fsl_buffer * c2 = fsl_cx_scratchpad(f); + rc = fsl_file_canonical_name2(relativeToCwd ? NULL : f->ckout.dir/*NULL is okay*/, + z1, c1, false); + if(!rc) rc = fsl_file_canonical_name2(relativeToCwd ? NULL : f->ckout.dir, + zPath, c2, false); + //MARKER(("\nzPath=%s\nc1=%s\nc2=%s\n", zPath, + //fsl_buffer_cstr(c1), fsl_buffer_cstr(c2))); + if(!rc && c1->used == c2->used && + 0==fsl_stricmp(fsl_buffer_cstr(c1), fsl_buffer_cstr(c2))){ + rc = fsl_cx_err_set(f, errRc, "File is the repository database: %.*s", + (int)nPath, zPath); + } + fsl_cx_scratchpad_yield(f, c1); + fsl_cx_scratchpad_yield(f, c2); + if(rc) return rc; + } + assert(!rc); + while(f->ckout.dir || relativeToCwd){ + int manifestSetting = 0; + fsl_ckout_manifest_setting(f, &manifestSetting); + if(!manifestSetting) break; + typedef struct { + short flag; + char const * fn; + } MSetting; + const MSetting M[] = { + {FSL_MANIFEST_MAIN, "manifest"}, + {FSL_MANIFEST_UUID, "manifest.uuid"}, + {FSL_MANIFEST_TAGS, "manifest.tags"}, + {0,0} + }; + fsl_buffer * c1 = fsl_cx_scratchpad(f); + if(f->ckout.dir){ + rc = fsl_ckout_filename_check(f, relativeToCwd, zPath, c1); + }else{ + rc = fsl_file_canonical_name2("", zPath, c1, false); + } + if(rc) goto yield; + char const * const z = fsl_buffer_cstr(c1); + //MARKER(("Checking file against manifest setting 0x%03x: %s\n", + //manifestSetting, z)); + for( MSetting const * m = &M[0]; m->fn; ++m ){ + if((m->flag & manifestSetting) + && 0==fsl_strcmp(z, m->fn)){ + rc = fsl_cx_err_set(f, errRc, + "Filename is reserved due to the " + "'manifest' setting: %s", + m->fn); + break; + } + } + yield: + fsl_cx_scratchpad_yield(f, c1); + break; + } + return rc; +} + +fsl_buffer * fsl_cx_scratchpad(fsl_cx *f){ + fsl_buffer * rc = 0; + int i = (f->scratchpads.nextscratchpads.next : 0; + for(; i < FSL_CX_NSCRATCH; ++i){ + if(!f->scratchpads.used[i]){ + rc = &f->scratchpads.buf[i]; + f->scratchpads.used[i] = true; + ++f->scratchpads.next; + //MARKER(("Doling out scratchpad[%d] w/ capacity=%d next=%d\n", + // i, (int)rc->capacity, f->scratchpads.next)); + break; + } + } + if(!rc){ + assert(!"Fatal fsl_cx::scratchpads misuse."); + fsl_fatal(FSL_RC_MISUSE, + "Fatal internal fsl_cx::scratchpads misuse: " + "too many unyielded buffer requests."); + }else if(0!=rc->used){ + assert(!"Fatal fsl_cx::scratchpads misuse."); + fsl_fatal(FSL_RC_MISUSE, + "Fatal internal fsl_cx::scratchpads misuse: " + "used buffer after yielding it."); + } + return rc; +} + +void fsl_cx_scratchpad_yield(fsl_cx *f, fsl_buffer * b){ + int i; + assert(b); + for(i = 0; i < FSL_CX_NSCRATCH; ++i){ + if(b == &f->scratchpads.buf[i]){ + assert(f->scratchpads.next != i); + assert(f->scratchpads.used[i] && "Scratchpad misuse."); + f->scratchpads.used[i] = false; + fsl_buffer_reuse(b); + if(f->scratchpads.next>i) f->scratchpads.next = i; + //MARKER(("Yielded scratchpad[%d] w/ capacity=%d, next=%d\n", + // i, (int)b->capacity, f->scratchpads.next)); + return; + } + } + fsl_fatal(FSL_RC_MISUSE, + "Fatal internal fsl_cx::scratchpads misuse: " + "passed a non-scratchpad buffer."); +} + + +/** @internal + + Don't use this. Use fsl_cx_rm_empty_dirs() instead. + + Attempts to remove empty directories from under a checkout, + starting with tgtDir and working upwards until it either cannot + remove one or it reaches the top of the checkout dir. + + The first argument must be the canonicalized absolute path to the + checkout root. The second is the length of coRoot - if it's + negative then fsl_strlen() is used to calculate it. The third must + be the canonicalized absolute path to some directory under the + checkout root. The contents of the buffer may, for efficiency's + sake, be modified by this routine as it traverses the directory + tree. It will never grow the buffer but may mutate its memory's + contents. + + Returns the number of directories it is able to remove. + + Results are undefined if tgtDir is not an absolute path or does not + have coRoot as its initial prefix. + + There are any number of valid reasons removal of a directory might + fail, and this routine stops at the first one which does. +*/ +static unsigned fsl_rm_empty_dirs(char const *coRoot, fsl_int_t rootLen, + fsl_buffer * tgtDir){ + if(rootLen<0) rootLen = fsl_strlen(coRoot); + char const * zAbs = fsl_buffer_cstr(tgtDir); + char const * zCoDirPart = zAbs + rootLen; + char * zEnd = fsl_buffer_str(tgtDir) + tgtDir->used - 1; + unsigned rc = 0; + assert(coRoot); + if(0!=memcmp(coRoot, zAbs, (size_t)rootLen)){ + assert(!"Misuse of fsl_rm_empty_dirs()"); + return 0; + } + if(fsl_rmdir(zAbs)) return rc; + ++rc; + /** Now walk up each dir in the path and try to remove each, + stopping when removal of one fails or we reach coRoot. */ + while(zEnd>zCoDirPart){ + for( ; zEnd>zCoDirPart && '/'!=*zEnd; --zEnd ){} + if(zEnd==zCoDirPart) break; + else if('/'==*zEnd){ + *zEnd = 0; + assert(zEnd>zCoDirPart); + if(fsl_rmdir(zAbs)) break; + ++rc; + } + } + return rc; +} + +unsigned int fsl_ckout_rm_empty_dirs(fsl_cx * f, fsl_buffer * tgtDir){ + int rc = f->ckout.dir ? 0 : FSL_RC_NOT_A_CKOUT; + if(!rc){ + rc = fsl_rm_empty_dirs(f->ckout.dir, f->ckout.dirLen, tgtDir); + } + return rc; +} + +int fsl_ckout_rm_empty_dirs_for_file(fsl_cx * f, char const *zAbsPath){ + if(!fsl_is_rooted_in_ckout(f, zAbsPath)){ + assert(!"Internal API misuse!"); + return FSL_RC_MISUSE; + }else{ + fsl_buffer * const p = fsl_cx_scratchpad(f); + fsl_int_t const nAbs = (fsl_int_t)fsl_strlen(zAbsPath); + int const rc = fsl_file_dirpart(zAbsPath, nAbs, p, false); + if(!rc) fsl_rm_empty_dirs(f->ckout.dir, f->ckout.dirLen, p); + fsl_cx_scratchpad_yield(f,p); + return rc; + } +} + +bool fsl_repo_forbids_delta_manifests(fsl_cx * f){ + return fsl_config_get_bool(f, FSL_CONFDB_REPO, false, + "forbid-delta-manifests"); +} + +int fsl_ckout_fingerprint_check(fsl_cx * f){ + fsl_db * const db = fsl_cx_db_ckout(f); + if(!db) return 0; + int rc = 0; + char const * zCkout = 0; + char * zRepo = 0; + fsl_id_t rcvCkout = 0; + fsl_buffer * const buf = fsl_cx_scratchpad(f); + rc = fsl_config_get_buffer(f, FSL_CONFDB_CKOUT, "fingerprint", buf); + if(FSL_RC_NOT_FOUND==rc){ + /* Older checkout with no fingerprint. Assume it's okay. */ + rc = 0; + goto end; + }else if(rc){ + goto end; + } + zCkout = fsl_buffer_cstr(buf); +#if 0 + /* Inject a bogus byte for testing purposes */ + buf->mem[6] = 'x'; +#endif + rcvCkout = (fsl_id_t)atoi(zCkout); + rc = fsl_repo_fingerprint_search(f, rcvCkout, &zRepo); + switch(rc){ + case FSL_RC_NOT_FOUND: goto mismatch; + case 0: + assert(zRepo); + if(fsl_strcmp(zRepo,zCkout)){ + goto mismatch; + } + break; + default: + break; + } + end: + fsl_cx_scratchpad_yield(f, buf); + fsl_free(zRepo); + return rc; + mismatch: + rc = fsl_cx_err_set(f, FSL_RC_REPO_MISMATCH, + "Mismatch found between repo/checkout " + "fingerprints."); + goto end; +} + +bool fsl_cx_has_ckout(fsl_cx const * const f ){ + return f->ckout.dir ? true : false; +} + + +#if 0 +struct tm * fsl_cx_localtime( fsl_cx const * f, const time_t * clock ){ + if(!clock) return NULL; + else if(!f) return localtime(clock); + else return (f->flags & FSL_CX_F_LOCALTIME_GMT) + ? gmtime(clock) + : localtime(clock) + ; +} + +struct tm * fsl_localtime( const time_t * clock ){ + return fsl_cx_localtime(NULL, clock); +} + +time_t fsl_cx_time_adj(fsl_cx const * f, time_t clock){ + struct tm * tm = fsl_cx_localtime(f, &clock); + return tm ? mktime(tm) : 0; +} +#endif + +#undef MARKER +#undef FSL_CX_NSCRATCH +/* end of file cx.c */ +/* start of file db.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). + + ***************************************************************************** + This file contains the fsl_db_xxx() and fsl_stmt_xxx() parts of the + API. + + Maintenance reminders: + + When returning dynamically allocated memory to the client, it needs + to come from fsl_malloc(), as opposed to sqlite3_malloc(), so that + it is legal to pass to fsl_free(). +*/ +#include +#include /* NULL on linux */ +#include /* time() and friends */ + +/* Only for debugging */ +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +#if 0 +/** + fsl_list_visitor_f() impl which requires that obj be NULL or + a (fsl_stmt*), which it passed to fsl_stmt_finalize(). + */ +static int fsl_list_v_fsl_stmt_finalize(void * obj, void * visitorState ){ + if(obj) fsl_stmt_finalize( (fsl_stmt*)obj ); + return 0; +} +#endif + +void fsl_db_clear_strings(fsl_db * const db, bool alsoErrorState ){ + fsl_free(db->filename); + db->filename = NULL; + fsl_free(db->name); + db->name = NULL; + if(alsoErrorState) fsl_error_clear(&db->error); +} + +int fsl_db_err_get( fsl_db const * const db, char const ** msg, fsl_size_t * len ){ + return db + ? fsl_error_get(&db->error, msg, len) + : FSL_RC_MISUSE; +} + +fsl_db * fsl_stmt_db( fsl_stmt * const stmt ){ + return stmt ? stmt->db : NULL; +} + +/** + Resets db->error state based on the given code and the current + error string from the db driver. Returns FSL_RC_DB on success, + some other non-0 value on error (most likely FSL_RC_OOM while + allocating the error string - that's the only other error case as + long as db is opened). Results are undefined if !db or db is not + opened. + */ +static int fsl_err_from_db( fsl_db * const db, int dbCode ){ + assert(db && db->dbh); + db->error.msg.used =0 ; + return fsl_error_set(&db->error, FSL_RC_DB, + "Db error #%d: %s", + dbCode, sqlite3_errmsg(db->dbh)); +} + +char const * fsl_stmt_sql( fsl_stmt * const stmt, fsl_size_t * const len ){ + return stmt + ? fsl_buffer_cstr2(&stmt->sql, len) + : NULL; +} + +char const * fsl_db_filename(fsl_db const * db, fsl_size_t * len){ + if(len && db->filename) *len = fsl_strlen(db->filename); + return db->filename; +} + +fsl_id_t fsl_db_last_insert_id(fsl_db * const db){ + return (db && db->dbh) + ? (fsl_id_t)sqlite3_last_insert_rowid(db->dbh) + : -1; +} + +/** + Cleans up db->beforeCommit and its contents. + */ +static void fsl_db_cleanup_beforeCommit( fsl_db * const db ){ + fsl_list_visit( &db->beforeCommit, -1, fsl_list_v_fsl_free, NULL ); + fsl_list_reserve(&db->beforeCommit, 0); +} + + +/** + Immediately cleans up all cached statements (if any) and returns + the number of statements cleaned up. It is illegal to call this + while any of the cached statements are actively being used (have + not been fsl_stmt_cached_yield()ed), and doing so will lead to + undefined results if the statement(s) in question are used after + this function completes. + + @see fsl_db_prepare_cached() + @see fsl_stmt_cached_yield() +*/ +static fsl_size_t fsl_db_stmt_cache_clear(fsl_db * const db){ + fsl_size_t rc = 0; + if(db && db->cacheHead){ + fsl_stmt * st; + fsl_stmt * next = 0; + for( st = db->cacheHead; st; st = next, ++rc ){ + next = st->next; + st->next = 0; + fsl_stmt_finalize( st ); + } + db->cacheHead = 0; + } + return rc; +} + +void fsl_db_close( fsl_db * const db ){ + if(!db) return; + void const * allocStamp = db->allocStamp; + fsl_cx * const f = db->f; + fsl_db_stmt_cache_clear(db); + if(db->f && db->f->dbMain==db){ + /* + Horrible, horrible dependency, and only necessary if the + fsl_cx API gets sloppy or repo/checkout/config DBs are + otherwised closed improperly (i.e. not via the fsl_cx API). + */ + assert(0 != db->role); + f->dbMain = NULL; + } + while(db->beginCount>0){ + fsl_db_transaction_end(db, 1); + } + if(0!=db->openStatementCount){ + MARKER(("WARNING: %d open statement(s) left on db [%s].\n", + (int)db->openStatementCount, db->filename)); + } + if(db->dbh){ + sqlite3_close(db->dbh); + /* ignoring results in the style of "destructors may not + throw". */ + } + fsl_db_clear_strings(db, 1); + fsl_db_cleanup_beforeCommit(db); + fsl_buffer_clear(&db->cachePrepBuf); + *db = fsl_db_empty; + if(&fsl_db_empty == allocStamp){ + fsl_free( db ); + }else{ + db->allocStamp = allocStamp; + db->f = f; + } + return; +} + +void fsl_db_err_reset( fsl_db * const db ){ + if(db && (db->error.code||db->error.msg.used)){ + fsl_error_reset(&db->error); + } +} + + +int fsl_db_attach(fsl_db * const db, const char *zDbName, const char *zLabel){ + return (db && db->dbh && zDbName && *zDbName && zLabel && *zLabel) + ? fsl_db_exec(db, "ATTACH DATABASE %Q AS %s", zDbName, zLabel) + : FSL_RC_MISUSE; +} +int fsl_db_detach(fsl_db * const db, const char *zLabel){ + return (db && db->dbh && zLabel && *zLabel) + ? fsl_db_exec(db, "DETACH DATABASE %s /*%s()*/", zLabel, __func__) + : FSL_RC_MISUSE; +} + +char const * fsl_db_name(fsl_db const * const db){ + return db ? db->name : NULL; +} + +/** + Returns the db name for the given role. + */ +const char * fsl_db_role_label(fsl_dbrole_e r){ + switch(r){ + case FSL_DBROLE_CONFIG: + return "cfg"; + case FSL_DBROLE_REPO: + return "repo"; + case FSL_DBROLE_CKOUT: + return "ckout"; + case FSL_DBROLE_MAIN: + return "main"; + case FSL_DBROLE_TEMP: + return "temp"; + case FSL_DBROLE_NONE: + default: + assert(!"cannot happen/not legal"); + return NULL; + } +} + +char * fsl_db_julian_to_iso8601( fsl_db * const db, double j, + bool msPrecision, + bool localTime){ + char * s = NULL; + fsl_stmt * st = NULL; + if(db && db->dbh && (j>=0.0)){ + char const * sql; + if(msPrecision){ + sql = localTime + ? "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%f',?, 'localtime')" + : "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%f',?)"; + }else{ + sql = localTime + ? "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S',?, 'localtime')" + : "SELECT strftime('%%Y-%%m-%%dT%%H:%%M:%%S',?)"; + } + fsl_db_prepare_cached(db, &st, sql); + if(st){ + fsl_stmt_bind_double( st, 1, j ); + if( FSL_RC_STEP_ROW==fsl_stmt_step(st) ){ + s = fsl_strdup(fsl_stmt_g_text(st, 0, NULL)); + } + fsl_stmt_cached_yield(st); + } + } + return s; +} + +char * fsl_db_unix_to_iso8601( fsl_db * const db, fsl_time_t t, bool localTime ){ + char * s = NULL; + fsl_stmt st = fsl_stmt_empty; + if(db && db->dbh && (t>=0)){ + char const * sql = localTime + ? "SELECT datetime(?, 'unixepoch', 'localtime')/*%s()*/" + : "SELECT datetime(?, 'unixepoch')/*%s()*/" + ; + int const rc = fsl_db_prepare(db, &st, sql,__func__); + if(!rc){ + fsl_stmt_bind_int64( &st, 1, t ); + if( FSL_RC_STEP_ROW==fsl_stmt_step(&st) ){ + fsl_size_t n = 0; + char const * v = fsl_stmt_g_text(&st, 0, &n); + s = (v&&n) ? fsl_strndup(v, (fsl_int_t)n) : NULL; + } + fsl_stmt_finalize(&st); + } + } + return s; +} + +enum fsl_stmt_flags_e { +/** + fsl_stmt::flags bit indicating that fsl_db_preparev_cache() has + doled out this statement, effectively locking it until + fsl_stmt_cached_yield() is called to release it. + */ +FSL_STMT_F_CACHE_HELD = 0x01, + +/** + Propagates our intent to "statically" prepare a given statement + through various internal API calls. +*/ +FSL_STMT_F_PREP_CACHE = 0x10 +}; + +int fsl_db_preparev( fsl_db * const db, fsl_stmt * const tgt, char const * sql, va_list args ){ + if(!db || !tgt || !sql) return FSL_RC_MISUSE; + else if(!db->dbh){ + return fsl_error_set(&db->error, FSL_RC_NOT_FOUND, "Db is not opened."); + }else if(!*sql){ + return fsl_error_set(&db->error, FSL_RC_RANGE, "SQL is empty."); + }else if(tgt->stmt){ + return fsl_error_set(&db->error, FSL_RC_ALREADY_EXISTS, + "Error: attempt to re-prepare " + "active statement."); + } + else{ + int rc; + fsl_buffer buf = fsl_buffer_empty; + fsl_stmt_t * liteStmt = NULL; + rc = fsl_buffer_appendfv( &buf, sql, args ); + if(!rc){ +#if 0 + /* Arguably improves readability of some queries. + And breaks some caching uses. */ + fsl_simplify_sql_buffer(&buf); +#endif + sql = fsl_buffer_cstr(&buf); + if(!sql || !*sql){ + rc = fsl_error_set(&db->error, FSL_RC_RANGE, + "Input SQL is empty."); + }else{ + /* + Achtung: if sql==NULL here, or evaluates to a no-op + (e.g. only comments or spaces), prepare_v2 succeeds but has + a NULL liteStmt, which is why we handle the empty-SQL case + specially. We don't want that specific behaviour leaking up + through the API. Though doing so would arguably more correct + in a generic API, for this particular API we have no reason + to be able to handle empty SQL. Were we do let through + through we'd have to add a flag to fsl_stmt to tell us + whether it's really prepared or not, since checking of + st->stmt would no longer be useful. + */ + rc = sqlite3_prepare_v3(db->dbh, sql, (int)buf.used, + (FSL_STMT_F_PREP_CACHE & tgt->flags) + ? SQLITE_PREPARE_PERSISTENT + : 0, + &liteStmt, NULL); + if(rc){ + rc = fsl_error_set(&db->error, FSL_RC_DB, + "Db statement preparation failed. " + "Error #%d: %s. SQL: %.*s", + rc, sqlite3_errmsg(db->dbh), + (int)buf.used, (char const *)buf.mem); + }else if(!liteStmt){ + /* SQL was empty. In sqlite this is allowed, but this API will + disallow this because it leads to headaches downstream. + */ + rc = fsl_error_set(&db->error, FSL_RC_RANGE, + "Input SQL is empty."); + } + } + } + if(!rc){ + assert(liteStmt); + ++db->openStatementCount; + tgt->stmt = liteStmt; + tgt->db = db; + tgt->sql = buf /*transfer ownership*/; + tgt->colCount = sqlite3_column_count(tgt->stmt); + tgt->paramCount = sqlite3_bind_parameter_count(tgt->stmt); + }else{ + assert(!liteStmt); + fsl_buffer_clear(&buf); + /* TODO: consider _copying_ the error state to db->f->error if + db->f is not NULL. OTOH, we don't _always_ want to propagate + a db error to the parent fossil context, and doing so here + could lead to a "stale" error laying around downstream and + getting evaluated later on (incorrectly) in a success + context. + */ + } + return rc; + } +} + +int fsl_db_prepare( fsl_db * const db, fsl_stmt * const tgt, char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_preparev( db, tgt, sql, args ); + va_end(args); + return rc; +} + + +int fsl_db_preparev_cached( fsl_db * const db, fsl_stmt ** rv, + char const * sql, va_list args ){ + int rc = 0; + fsl_buffer * const buf = &db->cachePrepBuf; + fsl_stmt * st = NULL; + fsl_stmt * cs = NULL; + if(!db || !rv || !sql) return FSL_RC_MISUSE; + else if(!*sql) return FSL_RC_RANGE; + if(!buf->capacity && fsl_buffer_reserve(buf, 1024*2)){ + return FSL_RC_OOM; + } + fsl_buffer_reuse(buf); + rc = fsl_buffer_appendfv(buf, sql, args); + if(rc) goto end; + /** + Hash buf's contents using a very primitive algo and stores the + hash in buf->cursor. This is a blatant abuse of that member but + we're otherwise not using it on these buffer instances. We use + this to slightly speed up lookup of previously-cached entries and + reduce the otherwise tremendous number of calls to + fsl_buffer_compare() libfossil makes. + */ + for(fsl_size_t i = 0; i < buf->used; ++i){ + //buf->cursor = (buf->cursor<<3) ^ buf->cursor ^ buf->mem[i]; + buf->cursor = 31 * buf->cursor + (buf->mem[i] * 307); + } + for( cs = db->cacheHead; cs; cs = cs->next ){ + if(cs->sql.cursor==buf->cursor/*hash value!*/ + && buf->used==cs->sql.used + && 0==fsl_buffer_compare(buf, &cs->sql)){ + if(cs->flags & FSL_STMT_F_CACHE_HELD){ + rc = fsl_error_set(&db->error, FSL_RC_ACCESS, + "Cached statement is already in use. " + "Do not use cached statements if recursion " + "involving the statement is possible, and use " + "fsl_stmt_cached_yield() to release them " + "for further (re)use. SQL: %b", + &cs->sql); + goto end; + } + cs->flags |= FSL_STMT_F_CACHE_HELD; + ++cs->cachedHits; + *rv = cs; + goto end; + } + } + st = fsl_stmt_malloc(); + if(!st){ + rc = FSL_RC_OOM; + goto end; + } + st->flags |= FSL_STMT_F_PREP_CACHE; + rc = fsl_db_prepare( db, st, "%b", buf ); + if(rc){ + fsl_free(st); + st = 0; + }else{ + st->sql.cursor = buf->cursor/*hash value!*/; + st->next = db->cacheHead; + db->cacheHead = st; + st->flags = FSL_STMT_F_CACHE_HELD; + *rv = st; + } + end: + return rc; +} + +int fsl_db_prepare_cached( fsl_db * db, fsl_stmt ** st, char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_preparev_cached( db, st, sql, args ); + va_end(args); + return rc; +} + +int fsl_stmt_cached_yield( fsl_stmt * const st ){ + if(!st || !st->db || !st->stmt) return FSL_RC_MISUSE; + else if(!(st->flags & FSL_STMT_F_CACHE_HELD)) { + return fsl_error_set(&st->db->error, FSL_RC_MISUSE, + "fsl_stmt_cached_yield() was passed a " + "statement which is not marked as cached. " + "SQL: %b", + &st->sql); + }else{ + fsl_stmt_reset(st); + st->flags &= ~FSL_STMT_F_CACHE_HELD; + return 0; + } +} + +int fsl_db_before_commitv( fsl_db * const db, char const * const sql, + va_list args ){ + int rc = 0; + char * cp = NULL; + if(!db || !sql) return FSL_RC_MISUSE; + else if(!*sql) return FSL_RC_RANGE; + cp = fsl_mprintfv(sql, args); + if(cp){ + rc = fsl_list_append(&db->beforeCommit, cp); + if(rc) fsl_free(cp); + }else{ + rc = FSL_RC_OOM; + } + return rc; +} + +int fsl_db_before_commit( fsl_db * const db, char const * const sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_before_commitv( db, sql, args ); + va_end(args); + return rc; +} + +int fsl_stmt_finalize( fsl_stmt * const stmt ){ + if(!stmt) return FSL_RC_MISUSE; + else{ + void const * allocStamp = stmt->allocStamp; + fsl_db * db = stmt->db; + if(db){ + if(stmt->sql.mem){ + /* ^^^ b/c that buffer is set at the same time + that openStatementCount is incremented. + */ + --stmt->db->openStatementCount; + } + if(allocStamp && db->cacheHead){ + /* It _might_ be cached - let's remove it. + We use allocStamp as a check here only + because most statements allocated on the + heap currently come from caching. + */ + fsl_stmt * s; + fsl_stmt * prev = 0; + for( s = db->cacheHead; s; prev = s, s = s->next ){ + if(s == stmt){ + if(prev){ + assert(prev->next == s); + prev->next = s->next; + }else{ + assert(s == db->cacheHead); + db->cacheHead = s->next; + } + s->next = 0; + break; + } + } + } + } + fsl_buffer_clear(&stmt->sql); + if(stmt->stmt){ + sqlite3_finalize( stmt->stmt ); + } + *stmt = fsl_stmt_empty; + if(&fsl_stmt_empty==allocStamp){ + fsl_free(stmt); + }else{ + stmt->allocStamp = allocStamp; + } + return 0; + } +} + +int fsl_stmt_step( fsl_stmt * const stmt ){ + if(!stmt || !stmt->stmt) return FSL_RC_MISUSE; + else{ + int const rc = sqlite3_step(stmt->stmt); + assert(stmt->db); + switch( rc ){ + case SQLITE_ROW: + ++stmt->rowCount; + return FSL_RC_STEP_ROW; + case SQLITE_DONE: + return FSL_RC_STEP_DONE; + default: + return fsl_error_set(&stmt->db->error, FSL_RC_STEP_ERROR, + "sqlite3_step(): sqlite error #%d: %s", + rc, sqlite3_errmsg(stmt->db->dbh)); + } + } +} + +int fsl_db_eachv( fsl_db * db, fsl_stmt_each_f callback, + void * callbackState, char const * sql, va_list args ){ + if(!db || !db->dbh || !callback || !sql) return FSL_RC_MISUSE; + else if(!*sql) return FSL_RC_RANGE; + else{ + fsl_stmt st = fsl_stmt_empty; + int rc; + rc = fsl_db_preparev( db, &st, sql, args ); + if(!rc){ + rc = fsl_stmt_each( &st, callback, callbackState ); + fsl_stmt_finalize( &st ); + } + return rc; + } +} + +int fsl_db_each( fsl_db * db, fsl_stmt_each_f callback, + void * callbackState, char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_eachv( db, callback, callbackState, sql, args ); + va_end(args); + return rc; +} + +int fsl_stmt_each( fsl_stmt * stmt, fsl_stmt_each_f callback, + void * callbackState ){ + if(!stmt || !callback) return FSL_RC_MISUSE; + else{ + int strc; + int rc = 0; + char doBreak = 0; + while( !doBreak && (FSL_RC_STEP_ROW == (strc=fsl_stmt_step(stmt)))){ + rc = callback( stmt, callbackState ); + switch(rc){ + case 0: continue; + case FSL_RC_BREAK: + rc = 0; + /* fall through */ + default: + doBreak = 1; + break; + } + } + return rc + ? rc + : ((FSL_RC_STEP_ERROR==strc) + ? FSL_RC_DB + : 0); + } +} + +int fsl_stmt_reset2( fsl_stmt * const stmt, bool resetRowCounter ){ + if(!stmt->stmt || !stmt->db) return FSL_RC_MISUSE; + else{ + int const rc = sqlite3_reset(stmt->stmt); + if(resetRowCounter) stmt->rowCount = 0; + assert(stmt->db); + return rc + ? fsl_err_from_db(stmt->db, rc) + : 0; + } +} + +int fsl_stmt_reset( fsl_stmt * const stmt ){ + return fsl_stmt_reset2(stmt, 0); +} + +int fsl_stmt_col_count( fsl_stmt const * const stmt ){ + return (!stmt || !stmt->stmt) + ? -1 + : stmt->colCount + ; +} + +char const * fsl_stmt_col_name(fsl_stmt * const stmt, int index){ + return (stmt && stmt->stmt && (index>=0 && indexcolCount)) + ? sqlite3_column_name(stmt->stmt, index) + : NULL; +} + +int fsl_stmt_param_count( fsl_stmt const * const stmt ){ + return (!stmt || !stmt->stmt) + ? -1 + : stmt->paramCount; +} + +int fsl_stmt_bind_fmtv( fsl_stmt * st, char const * fmt, va_list args ){ + int rc = 0, ndx; + char const * pos = fmt; + if(!fmt || + !(st && st->stmt && st->db && st->db->dbh)) return FSL_RC_MISUSE; + else if(!*fmt) return FSL_RC_RANGE; + for( ndx = 1; !rc && *pos; ++pos, ++ndx ){ + if(' '==*pos){ + --ndx; + continue; + } + if(ndx > st->paramCount){ + rc = fsl_error_set(&st->db->error, FSL_RC_RANGE, + "Column index %d is out of bounds.", ndx); + break; + } + switch(*pos){ + case '-': + va_arg(args,void const *) /* skip arg */; + rc = fsl_stmt_bind_null(st, ndx); + break; + case 'i': + rc = fsl_stmt_bind_int32(st, ndx, va_arg(args,int32_t)); + break; + case 'I': + rc = fsl_stmt_bind_int64(st, ndx, va_arg(args,int64_t)); + break; + case 'R': + rc = fsl_stmt_bind_id(st, ndx, va_arg(args,fsl_id_t)); + break; + case 'f': + rc = fsl_stmt_bind_double(st, ndx, va_arg(args,double)); + break; + case 's':{/* C-string as TEXT or NULL */ + char const * s = va_arg(args,char const *); + rc = s + ? fsl_stmt_bind_text(st, ndx, s, -1, false) + : fsl_stmt_bind_null(st, ndx); + break; + } + case 'S':{ /* C-string as BLOB or NULL */ + char const * s = va_arg(args,char const *); + rc = s + ? fsl_stmt_bind_blob(st, ndx, s, fsl_strlen(s), false) + : fsl_stmt_bind_null(st, ndx); + break; + } + case 'b':{ /* fsl_buffer as TEXT or NULL */ + fsl_buffer const * b = va_arg(args,fsl_buffer const *); + rc = (b && b->mem) + ? fsl_stmt_bind_text(st, ndx, (char const *)b->mem, + (fsl_int_t)b->used, false) + : fsl_stmt_bind_null(st, ndx); + break; + } + case 'B':{ /* fsl_buffer as BLOB or NULL */ + fsl_buffer const * b = va_arg(args,fsl_buffer const *); + rc = (b && b->mem) + ? fsl_stmt_bind_blob(st, ndx, b->mem, b->used, false) + : fsl_stmt_bind_null(st, ndx); + break; + } + default: + rc = fsl_error_set(&st->db->error, FSL_RC_RANGE, + "Invalid format character: '%c'", *pos); + break; + } + } + return rc; +} + +/** + The elipsis counterpart of fsl_stmt_bind_fmtv(). +*/ +int fsl_stmt_bind_fmt( fsl_stmt * const st, char const * fmt, ... ){ + int rc; + va_list args; + va_start(args,fmt); + rc = fsl_stmt_bind_fmtv(st, fmt, args); + va_end(args); + return rc; +} + +int fsl_stmt_bind_stepv( fsl_stmt * const st, char const * fmt, + va_list args ){ + int rc; + fsl_stmt_reset(st); + rc = fsl_stmt_bind_fmtv(st, fmt, args); + if(!rc){ + rc = fsl_stmt_step(st); + switch(rc){ + case FSL_RC_STEP_DONE: + rc = 0; + fsl_stmt_reset(st); + break; + case FSL_RC_STEP_ROW: + /* Don't reset() for ROW b/c that clears the column + data! */ + break; + default: + rc = fsl_error_set(&st->db->error, rc, + "Error stepping statement."); + break; + } + } + return rc; +} + +int fsl_stmt_bind_step( fsl_stmt * st, char const * fmt, ... ){ + int rc; + va_list args; + va_start(args,fmt); + rc = fsl_stmt_bind_stepv(st, fmt, args); + va_end(args); + return rc; +} + + +#define BIND_PARAM_CHECK \ + if(!(stmt && stmt->stmt && stmt->db && stmt->db->dbh)) return FSL_RC_MISUSE; else +#define BIND_PARAM_CHECK2 BIND_PARAM_CHECK \ + if(ndx<1 || ndx>stmt->paramCount) return FSL_RC_RANGE; else +int fsl_stmt_bind_null( fsl_stmt * const stmt, int ndx ){ + BIND_PARAM_CHECK2 { + int const rc = sqlite3_bind_null( stmt->stmt, ndx ); + return rc ? fsl_err_from_db(stmt->db, rc) : 0; + } +} + +int fsl_stmt_bind_int32( fsl_stmt * const stmt, int ndx, int32_t v ){ + BIND_PARAM_CHECK2 { + int const rc = sqlite3_bind_int( stmt->stmt, ndx, (int)v ); + return rc ? fsl_err_from_db(stmt->db, rc) : 0; + } +} + +int fsl_stmt_bind_int64( fsl_stmt * const stmt, int ndx, int64_t v ){ + BIND_PARAM_CHECK2 { + int const rc = sqlite3_bind_int64( stmt->stmt, ndx, (sqlite3_int64)v ); + return rc ? fsl_err_from_db(stmt->db, rc) : 0; + } +} + +int fsl_stmt_bind_id( fsl_stmt * const stmt, int ndx, fsl_id_t v ){ + BIND_PARAM_CHECK2 { + int const rc = sqlite3_bind_int64( stmt->stmt, ndx, (sqlite3_int64)v ); + return rc ? fsl_err_from_db(stmt->db, rc) : 0; + } +} + +int fsl_stmt_bind_double( fsl_stmt * const stmt, int ndx, double v ){ + BIND_PARAM_CHECK2 { + int const rc = sqlite3_bind_double( stmt->stmt, ndx, (double)v ); + return rc ? fsl_err_from_db(stmt->db, rc) : 0; + } +} + +int fsl_stmt_bind_blob( fsl_stmt * const stmt, int ndx, void const * src, + fsl_size_t len, bool makeCopy ){ + BIND_PARAM_CHECK2 { + int rc; + rc = sqlite3_bind_blob( stmt->stmt, ndx, src, (int)len, + makeCopy ? SQLITE_TRANSIENT : SQLITE_STATIC ); + return rc ? fsl_err_from_db(stmt->db, rc) : 0; + } +} + +int fsl_stmt_bind_text( fsl_stmt * const stmt, int ndx, char const * src, + fsl_int_t len, bool makeCopy ){ + BIND_PARAM_CHECK { + int rc; + if(len<0) len = fsl_strlen((char const *)src); + rc = sqlite3_bind_text( stmt->stmt, ndx, src, len, + makeCopy ? SQLITE_TRANSIENT : SQLITE_STATIC ); + return rc ? fsl_err_from_db(stmt->db, rc) : 0; + } +} + +int fsl_stmt_bind_null_name( fsl_stmt * const stmt, char const * param ){ + BIND_PARAM_CHECK{ + return fsl_stmt_bind_null( stmt, + sqlite3_bind_parameter_index( stmt->stmt, + param) ); + } +} + +int fsl_stmt_bind_int32_name( fsl_stmt * const stmt, char const * param, int32_t v ){ + BIND_PARAM_CHECK { + return fsl_stmt_bind_int32( stmt, + sqlite3_bind_parameter_index( stmt->stmt, + param), + v); + } +} + +int fsl_stmt_bind_int64_name( fsl_stmt * const stmt, char const * param, int64_t v ){ + BIND_PARAM_CHECK { + return fsl_stmt_bind_int64( stmt, + sqlite3_bind_parameter_index( stmt->stmt, + param), + v); + } +} + +int fsl_stmt_bind_id_name( fsl_stmt * const stmt, char const * param, fsl_id_t v ){ + BIND_PARAM_CHECK { + return fsl_stmt_bind_id( stmt, + sqlite3_bind_parameter_index( stmt->stmt, + param), + v); + } +} + +int fsl_stmt_bind_double_name( fsl_stmt * const stmt, char const * param, double v ){ + BIND_PARAM_CHECK { + return fsl_stmt_bind_double( stmt, + sqlite3_bind_parameter_index( stmt->stmt, + param), + v); + } +} + +int fsl_stmt_bind_text_name( fsl_stmt * const stmt, char const * param, + char const * v, fsl_int_t n, + bool makeCopy ){ + BIND_PARAM_CHECK { + return fsl_stmt_bind_text(stmt, + sqlite3_bind_parameter_index( stmt->stmt, + param), + v, n, makeCopy); + } +} + +int fsl_stmt_bind_blob_name( fsl_stmt * const stmt, char const * param, + void const * v, fsl_int_t len, + bool makeCopy ){ + BIND_PARAM_CHECK { + return fsl_stmt_bind_blob(stmt, + sqlite3_bind_parameter_index( stmt->stmt, + param), + v, len, makeCopy); + } +} + +int fsl_stmt_param_index( fsl_stmt * const stmt, char const * const param){ + return (stmt && stmt->stmt) + ? sqlite3_bind_parameter_index( stmt->stmt, param) + : -1; +} + +#undef BIND_PARAM_CHECK +#undef BIND_PARAM_CHECK2 + +#define GET_CHECK if(!stmt->colCount) return FSL_RC_MISUSE; \ + else if((ndx<0) || (ndx>=stmt->colCount)) return FSL_RC_RANGE; else + +int fsl_stmt_get_int32( fsl_stmt * const stmt, int ndx, int32_t * v ){ + GET_CHECK { + if(v) *v = (int32_t)sqlite3_column_int(stmt->stmt, ndx); + return 0; + } +} +int fsl_stmt_get_int64( fsl_stmt * const stmt, int ndx, int64_t * v ){ + GET_CHECK { + if(v) *v = (int64_t)sqlite3_column_int64(stmt->stmt, ndx); + return 0; + } +} + +int fsl_stmt_get_double( fsl_stmt * const stmt, int ndx, double * v ){ + GET_CHECK { + if(v) *v = (double)sqlite3_column_double(stmt->stmt, ndx); + return 0; + } +} + +int fsl_stmt_get_id( fsl_stmt * const stmt, int ndx, fsl_id_t * v ){ + GET_CHECK { + if(v) *v = (4==sizeof(fsl_id_t)) + ? (fsl_id_t)sqlite3_column_int(stmt->stmt, ndx) + : (fsl_id_t)sqlite3_column_int64(stmt->stmt, ndx); + return 0; + } +} + +int fsl_stmt_get_text( fsl_stmt * const stmt, int ndx, char const **out, + fsl_size_t * outLen ){ + GET_CHECK { + unsigned char const * t = (out || outLen) + ? sqlite3_column_text(stmt->stmt, ndx) + : NULL; + if(out) *out = (char const *)t; + if(outLen){ + int const x = sqlite3_column_bytes(stmt->stmt, ndx); + *outLen = (x>0) ? (fsl_size_t)x : 0; + } + return 0; + } +} + +int fsl_stmt_get_blob( fsl_stmt * const stmt, int ndx, void const **out, + fsl_size_t * outLen ){ + GET_CHECK { + void const * t = (out || outLen) + ? sqlite3_column_blob(stmt->stmt, ndx) + : NULL; + if(out) *out = t; + if(outLen){ + if(!t) *outLen = 0; + else{ + int sz = sqlite3_column_bytes(stmt->stmt, ndx); + *outLen = (sz>=0) ? (fsl_size_t)sz : 0; + } + } + return 0; + } +} + +#undef GET_CHECK + +fsl_id_t fsl_stmt_g_id( fsl_stmt * const stmt, int index ){ + fsl_id_t rv = -1; + fsl_stmt_get_id(stmt, index, &rv); + return rv; +} +int32_t fsl_stmt_g_int32( fsl_stmt * const stmt, int index ){ + int32_t rv = 0; + fsl_stmt_get_int32(stmt, index, &rv); + return rv; +} +int64_t fsl_stmt_g_int64( fsl_stmt * const stmt, int index ){ + int64_t rv = 0; + fsl_stmt_get_int64(stmt, index, &rv); + return rv; +} +double fsl_stmt_g_double( fsl_stmt * const stmt, int index ){ + double rv = 0; + fsl_stmt_get_double(stmt, index, &rv); + return rv; +} + +char const * fsl_stmt_g_text( fsl_stmt * const stmt, int index, + fsl_size_t * outLen ){ + char const * rv = NULL; + fsl_stmt_get_text(stmt, index, &rv, outLen); + return rv; +} + + +/** + This function outputs tracing info using fsl_fprintf((FILE*)zFILE,...). + Defaults to stdout if zFILE is 0. + */ +static void fsl_db_sql_trace(void *zFILE, const char *zSql){ + int const n = fsl_strlen(zSql); + static int counter = 0; + /* FIXME: in v1 this uses fossil_trace(), but don't have + that functionality here yet. */ + fsl_fprintf(zFILE ? (FILE*)zFILE : stdout, + "SQL TRACE #%d: %s%s\n", ++counter, + zSql, (n>0 && zSql[n-1]==';') ? "" : ";"); +} + +/* + SQL function for debugging. + + The print() function writes its arguments to fsl_output() + if the bound fsl_cx->cxConfig.sqlPrint flag is true. +*/ +static void fsl_db_sql_print( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fsl_cx * f = (fsl_cx*)sqlite3_user_data(context); + assert(f); + if( f->cxConfig.sqlPrint ){ + int i; + for(i=0; ickout.dir && f->ckout.dirLen){ + sqlite3_result_text(context, f->ckout.dir, + (int)f->ckout.dirLen + - (includeSlash ? 0 : 1), + SQLITE_TRANSIENT); + }else{ + sqlite3_result_null(context); + } +} + + +/** + SQL Function to return the check-in time for a file. + Requires (vid,fid) RID arguments, as described for + fsl_mtime_of_manifest_file(). + */ +static void fsl_db_checkin_mtime_udf( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fsl_cx * f = (fsl_cx*)sqlite3_user_data(context); + fsl_time_t mtime = 0; + int rc; + fsl_id_t vid, fid; + assert(f); + vid = (fsl_id_t)sqlite3_value_int(argv[0]); + fid = (fsl_id_t)sqlite3_value_int(argv[1]); + rc = fsl_mtime_of_manifest_file(f, vid, fid, &mtime); + if( rc==0 ){ + sqlite3_result_int64(context, mtime); + }else{ + sqlite3_result_error(context, "fsl_mtime_of_manifest_file() failed", -1); + } +} + + +static void fsl_db_content_udf( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fsl_cx * f = (fsl_cx*)sqlite3_user_data(context); + fsl_id_t rid = 0; + char const * arg; + int rc; + fsl_buffer b = fsl_buffer_empty; + assert(f); + if(1 != argc){ + sqlite3_result_error(context, "Expecting one argument", -1); + return; + } + if(SQLITE_INTEGER==sqlite3_value_type(argv[0])){ + rid = (fsl_id_t)sqlite3_value_int64(argv[0]); + arg = NULL; + }else{ + arg = (const char*)sqlite3_value_text(argv[0]); + if(!arg){ + sqlite3_result_error(context, "Invalid argument", -1); + return; + } + rc = fsl_sym_to_rid(f, arg, FSL_SATYPE_CHECKIN, &rid); + if(rc) goto cx_err; + else if(!rid){ + sqlite3_result_error(context, "No blob found", -1); + return; + } + } + rc = fsl_content_get(f, rid, &b); + if(rc) goto cx_err; + /* Curiously, i'm seeing no difference in allocation counts here... */ + sqlite3_result_blob(context, b.mem, (int)b.used, fsl_free); + b = fsl_buffer_empty; + return; + cx_err: + fsl_buffer_clear(&b); + assert(f->error.msg.used); + if(FSL_RC_OOM==rc){ + sqlite3_result_error_nomem(context); + }else{ + assert(f->error.msg.used); + sqlite3_result_error(context, (char const *)f->error.msg.mem, + (int)f->error.msg.used); + } +} + +static void fsl_db_sym2rid_udf( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fsl_cx * f = (fsl_cx*)sqlite3_user_data(context); + char const * arg; + assert(f); + if(1 != argc){ + sqlite3_result_error(context, "Expecting one argument", -1); + return; + } + arg = (const char*)sqlite3_value_text(argv[0]); + if(!arg){ + sqlite3_result_error(context, "Expecting a STRING argument", -1); + }else{ + fsl_id_t rid = 0; + int const rc = fsl_sym_to_rid(f, arg, FSL_SATYPE_CHECKIN, &rid); + if(rc){ + if(FSL_RC_OOM==rc){ + sqlite3_result_error_nomem(context); + }else{ + assert(f->error.msg.used); + sqlite3_result_error(context, (char const *)f->error.msg.mem, + (int)f->error.msg.used); + } + fsl_cx_err_reset(f) + /* This is arguable but keeps this error from poluting + down-stream code (seen it happen in unit tests). The irony + is, it's very possible/likely that the error will propagate + back up into f->error at some point. + */; + }else{ + assert(rid>0); + sqlite3_result_int64(context, rid); + } + } +} + +static void fsl_db_dirpart_udf( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + char const * arg; + int rc; + fsl_buffer b = fsl_buffer_empty; + int fSlash = 0; + if(argc<1 || argc>2){ + sqlite3_result_error(context, + "Expecting (string) or (string,bool) arguments", + -1); + return; + } + arg = (const char*)sqlite3_value_text(argv[0]); + if(!arg){ + sqlite3_result_error(context, "Invalid argument", -1); + return; + } + if(argc>1){ + fSlash = sqlite3_value_int(argv[1]); + } + rc = fsl_file_dirpart(arg, -1, &b, fSlash ? 1 : 0); + if(!rc){ + if(b.used && *b.mem){ +#if 0 + sqlite3_result_text(context, (char const *)b.mem, + (int)b.used, SQLITE_TRANSIENT); +#else + sqlite3_result_text(context, (char const *)b.mem, + (int)b.used, fsl_free); + b = fsl_buffer_empty /* we passed ^^^^^ on ownership of b.mem */; +#endif + }else{ + sqlite3_result_null(context); + } + }else{ + if(FSL_RC_OOM==rc){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_error(context, "fsl_dirpart() failed!", -1); + } + } + fsl_buffer_clear(&b); +} + + +/* + Implement the user() SQL function. user() takes no arguments and + returns the user ID of the current user. +*/ +static void fsl_db_user_udf( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fsl_cx * f = (fsl_cx*)sqlite3_user_data(context); + assert(f); + if(f->repo.user){ + sqlite3_result_text(context, f->repo.user, -1, SQLITE_STATIC); + }else{ + sqlite3_result_null(context); + } +} + +/** + SQL function: + + fsl_is_enqueued(vfile.id) + fsl_if_enqueued(vfile.id, X, Y) + + On the commit command, when filenames are specified (in order to do + a partial commit) the vfile.id values for the named files are + loaded into the fsl_cx state. This function looks at that state to + see if a file is named in that list. + + In the first form (1 argument) return TRUE if either no files are + named (meaning that all changes are to be committed) or if id is + found in the list. + + In the second form (3 arguments) return argument X if true and Y if + false unless Y is NULL, in which case always return X. +*/ +static void fsl_db_selected_for_checkin_udf( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int rc = 0; + fsl_cx * f = (fsl_cx*)sqlite3_user_data(context); + fsl_id_bag * bag = &f->ckin.selectedIds; + assert(argc==1 || argc==3); + if( bag->entryCount ){ + fsl_id_t const iId = (fsl_id_t)sqlite3_value_int64(argv[0]); + rc = iId ? (fsl_id_bag_contains(bag, iId) ? 1 : 0) : 0; + }else{ + rc = 1; + } + if(1==argc){ + sqlite3_result_int(context, rc); + }else{ + assert(3 == argc); + assert( rc==0 || rc==1 ); + if( sqlite3_value_type(argv[2-rc])==SQLITE_NULL ) rc = 1-rc; + sqlite3_result_value(context, argv[2-rc]); + } +} + +/** + fsl_match_vfile_or_dir(p1,p2) + + A helper for resolving expressions like: + + WHERE pathname='X' C OR + (pathname>'X/' C AND pathname<'X0' C) + + i.e. is 'X' a match for the LHS or is it a directory prefix of + LHS? + + C = empty or COLLATE NOCASE, depending on the case-sensitivity + setting of the fsl_cx instance associated with + sqlite3_user_data(context). p1 is typically vfile.pathname or + vfile.origname, and p2 is the string being compared against that. + + Resolves to NULL if either argument is NULL, 0 if the comparison + shown above is false, 1 if the comparison is an exact match, or 2 + if p2 is a directory prefix part of p1. +*/ +static void fsl_db_match_vfile_or_dir( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fsl_cx * f = (fsl_cx*)sqlite3_user_data(context); + char const * p1; + char const * p2; + fsl_buffer * b = 0; + int rc = 0; + assert(f); + if(2 != argc){ + sqlite3_result_error(context, "Expecting two arguments", -1); + return; + } + p1 = (const char*)sqlite3_value_text(argv[0]); + p2 = (const char*)sqlite3_value_text(argv[1]); + if(!p1 || !p2){ + sqlite3_result_null(context); + return; + } + int (*cmp)(char const *, char const *) = + f->cache.caseInsensitive ? fsl_stricmp : fsl_strcmp; + if(0==cmp(p1, p2)){ + sqlite3_result_int(context, 1); + return; + } + b = fsl_cx_scratchpad(f); + rc = fsl_buffer_appendf(b, "%s/", p2); + if(rc) goto oom; + else if(cmp(p1, fsl_buffer_cstr(b))>0){ + b->mem[b->used-1] = '0'; + if(cmp(p1, fsl_buffer_cstr(b))<0) + rc = 2; + } + assert(0==rc || 2==rc); + sqlite3_result_int(context, rc); + end: + fsl_cx_scratchpad_yield(f, b); + return; + oom: + sqlite3_result_error_nomem(context); + goto end; +} + +/** + F(glob-list-name, filename) + + Returns 1 if the 2nd argument matches any glob in the fossil glob + list named by the first argument. The first argument must be a name + resolvable via fsl_glob_name_to_category() or an error is + triggered. The second value is intended to be a string, but NULL is + accepted (but never matches anything). + + If no match is found, 0 is returned. An empty glob list never matches + anything. +*/ +static void fsl_db_cx_glob_udf( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fsl_cx * const f = (fsl_cx*)sqlite3_user_data(context); + fsl_list * li = NULL; + fsl_glob_category_e globType; + char const * p1; + char const * p2; + p2 = (const char*)sqlite3_value_text(argv[1])/*value to check*/; + if(NULL==p2 || 0==p2[0]){ + sqlite3_result_int(context, 0); + return; + } + p1 = (const char*)sqlite3_value_text(argv[0])/*glob set name*/; + globType = fsl_glob_name_to_category(p1); + if(FSL_GLOBS_INVALID==globType){ + char buf[100] = {0}; + buf[sizeof(buf)-1] = 0; + fsl_snprintf(buf, (fsl_size_t)sizeof(buf)-1, + "Unknown glob pattern name: %#.*s", + 50, p1 ? p1 : "NULL"); + sqlite3_result_error(context, buf, -1); + return; + } + fsl_cx_glob_list(f, globType, &li, false); + assert(li); + sqlite3_result_int(context, fsl_glob_list_matches(li, p2) ? 1 : 0); +} + + +fsl_db * fsl_db_malloc(){ + fsl_db * rc = (fsl_db *)fsl_malloc(sizeof(fsl_db)); + if(rc){ + *rc = fsl_db_empty; + rc->allocStamp = &fsl_db_empty; + } + return rc; +} + +fsl_stmt * fsl_stmt_malloc(){ + fsl_stmt * rc = (fsl_stmt *)fsl_malloc(sizeof(fsl_stmt)); + if(rc){ + *rc = fsl_stmt_empty; + rc->allocStamp = &fsl_stmt_empty; + } + return rc; +} + + +/** + Return true if the schema is out-of-date. db must be an opened + repo db. + */ +static char fsl_db_repo_schema_is_outofdate(fsl_db *db){ + return fsl_db_exists(db, "SELECT 1 FROM config " + "WHERE name='aux-schema' " + "AND value<>'%s'", + FSL_AUX_SCHEMA); +} + +/* + Returns 0 if db appears to have a current repository schema, 1 if + it appears to have an out of date schema, and -1 if it appears to + not be a repository. +*/ +int fsl_db_repo_verify_schema(fsl_db * db){ + if(fsl_db_repo_schema_is_outofdate(db)) return 1; + else return fsl_db_exists(db, + "SELECT 1 FROM config " + "WHERE name='project-code'") + ? 0 : -1; +} + +/** + Callback for use with sqlite3_commit_hook(). The argument must be a + (fsl_db*). This function returns 0 only if it surmises that + fsl_db_transaction_end() triggered the COMMIT. On error it might + assert() or abort() the application, so this really is just a + sanity check for something which "must not happen." +*/ +static int fsl_db_verify_begin_was_not_called(void * db_fsl){ +#if 0 + return 0; +#else + /* i cannot explain why, but the ptr i'm getting here is most definately + not a proper fsl_db. */ + fsl_db * db = (fsl_db *)db_fsl; + assert(db && "What else could it be?"); + assert(db->dbh && "Else we can't have been called by sqlite3, could we have?"); + if(db->beginCount>0){ + fsl_fatal(FSL_RC_MISUSE,"SQL: COMMIT was called from " + "outside of fsl_db_transaction_end() while a " + "fsl_db_transaction_begin()-started transaction " + "is pending."); + return 2; + } + /* we have no context: sqlite3_result_error(context, "fsl_mtime_of_manifest_file() failed", -1); */ + else return 0; +#endif +} + +int fsl_db_open( fsl_db * db, char const * dbFile, + int openFlags ){ + int rc; + fsl_dbh_t * dbh = NULL; + int isMem = 0; + if(!db || !dbFile) return FSL_RC_MISUSE; + else if(db->dbh) return FSL_RC_MISUSE; + else if(!(isMem = (!*dbFile || 0==fsl_strcmp(":memory:", dbFile))) + && !(FSL_OPEN_F_CREATE & openFlags) + && fsl_file_access(dbFile, 0)){ + return fsl_error_set(&db->error, FSL_RC_NOT_FOUND, + "DB file not found: %s", dbFile); + } + else{ + int sOpenFlags = 0; + if(isMem){ + sOpenFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + }else{ + if(FSL_OPEN_F_RO & openFlags){ + sOpenFlags |= SQLITE_OPEN_READONLY; + }else{ + if(FSL_OPEN_F_RW & openFlags){ + sOpenFlags |= SQLITE_OPEN_READWRITE; + } + if(FSL_OPEN_F_CREATE & openFlags){ + sOpenFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + } + if(!sOpenFlags) sOpenFlags = SQLITE_OPEN_READONLY; + } + } + rc = sqlite3_open_v2( dbFile, &dbh, sOpenFlags, NULL ); + if(rc){ + if(dbh){ + /* By some complete coincidence, FSL_RC_DB==SQLITE_CANTOPEN. */ + rc = fsl_error_set(&db->error, FSL_RC_DB, + "Opening db file [%s] failed with " + "sqlite code #%d: %s", + dbFile, rc, sqlite3_errmsg(dbh)); + }else{ + rc = fsl_error_set(&db->error, FSL_RC_DB, + "Opening db file [%s] failed with " + "sqlite code #%d", + dbFile, rc); + } + /* MARKER(("Error msg: %s\n", (char const *)db->error.msg.mem)); */ + goto end; + }else{ + assert(!db->filename); + if(!*dbFile || ':'==*dbFile){ + /* assume "" or ":memory:" or some such: don't canonicalize it, + but copy it nonetheless for consistency. */ + db->filename = fsl_strdup(dbFile); + }else{ + fsl_buffer tmp = fsl_buffer_empty; + rc = fsl_file_canonical_name(dbFile, &tmp, 0); + if(!rc){ + db->filename = (char *)tmp.mem + /* transfering ownership */; + }else if(tmp.mem){ + fsl_buffer_clear(&tmp); + } + } + if(rc){ + goto end; + }else if(!db->filename){ + rc = FSL_RC_OOM; + goto end; + } + } + db->dbh = dbh; + if(FSL_OPEN_F_SCHEMA_VALIDATE & openFlags){ + int check; + check = fsl_db_repo_verify_schema(db); + if(0 != check){ + rc = (check<0) + ? fsl_error_set(&db->error, FSL_RC_NOT_A_REPO, + "DB file [%s] does not appear to be " + "a repository.", dbFile) + : fsl_error_set(&db->error, FSL_RC_REPO_NEEDS_REBUILD, + "DB file [%s] appears to be a fossil " + "repsitory, but is out-of-date and needs " + "a rebuild.", + dbFile) + ; + assert(rc == db->error.code); + goto end; + } + } + + if( (openFlags & FSL_OPEN_F_TRACE_SQL) + || (db->f && db->f->cxConfig.traceSql) ){ + fsl_db_sqltrace_enable(db, stdout); + } + + if(db->f){ + /* + Plug in fsl_cx-specific functionality to this one. + + TODO: move this into the fsl_cx code. Its placement here is + largely historical. + */ + fsl_cx * f = db->f; + /* This all comes from db.c:db_open()... */ + /* FIXME: check result codes here. */ + sqlite3_commit_hook(dbh, fsl_db_verify_begin_was_not_called, db); + sqlite3_busy_timeout(dbh, 5000 /* historical value */); + sqlite3_wal_autocheckpoint(dbh, 1); /* Set to checkpoint frequently */ + sqlite3_exec(dbh, "PRAGMA foreign_keys=OFF;", 0, 0, 0); + sqlite3_create_function(dbh, "now", 0, SQLITE_ANY, 0, + fsl_db_now_udf, 0, 0); + sqlite3_create_function(dbh, "fsl_ci_mtime", 2, + SQLITE_ANY | SQLITE_DETERMINISTIC, f, + fsl_db_checkin_mtime_udf, 0, 0); + sqlite3_create_function(dbh, "fsl_user", 0, + SQLITE_ANY | SQLITE_DETERMINISTIC, f, + fsl_db_user_udf, 0, 0); + sqlite3_create_function(dbh, "fsl_print", -1, + SQLITE_UTF8 + /* not strictly SQLITE_DETERMINISTIC + because it produces output */, + f, fsl_db_sql_print,0,0); + sqlite3_create_function(dbh, "fsl_content", 1, + SQLITE_ANY | SQLITE_DETERMINISTIC, f, + fsl_db_content_udf, 0, 0); + sqlite3_create_function(dbh, "fsl_sym2rid", 1, + SQLITE_ANY | SQLITE_DETERMINISTIC, f, + fsl_db_sym2rid_udf, 0, 0); + sqlite3_create_function(dbh, "fsl_dirpart", 1, + SQLITE_ANY | SQLITE_DETERMINISTIC, NULL, + fsl_db_dirpart_udf, 0, 0); + sqlite3_create_function(dbh, "fsl_dirpart", 2, + SQLITE_ANY | SQLITE_DETERMINISTIC, NULL, + fsl_db_dirpart_udf, 0, 0); + sqlite3_create_function(dbh, "fsl_j2u", 1, + SQLITE_ANY | SQLITE_DETERMINISTIC, NULL, + fsl_db_j2u_udf, 0, 0); + /* + fsl_i[sf]_selected() both require access to the f's list of + files being considered for commit. + */ + sqlite3_create_function(dbh, "fsl_is_enqueued", 1, SQLITE_UTF8, f, + fsl_db_selected_for_checkin_udf,0,0 ); + sqlite3_create_function(dbh, "fsl_if_enqueued", 3, SQLITE_UTF8, f, + fsl_db_selected_for_checkin_udf,0,0 ); + + sqlite3_create_function(dbh, "fsl_ckout_dir", -1, + SQLITE_ANY /* | SQLITE_DETERMINISTIC ? */, + f, fsl_db_cx_chkout_dir_udf,0,0 ); + sqlite3_create_function(dbh, "fsl_match_vfile_or_dir", 2, + SQLITE_ANY | SQLITE_DETERMINISTIC, + f, fsl_db_match_vfile_or_dir,0,0 ); + sqlite3_create_function(dbh, "fsl_glob", 2, + SQLITE_UTF8 | SQLITE_DETERMINISTIC, + /* noting that ^^^^^ it's only deterministic + for a given statement execution IF no SQL + triggers an effect which forces the globs to + reload. That "shouldn't ever happen." */ + f, fsl_db_cx_glob_udf, 0, 0 ); + +#if 0 + /* functions registered in v1 by db.c:db_open(). */ + /* porting cgi() requires access to the HTTP/CGI + layer. i.e. this belongs downstream. */ + sqlite3_create_function(dbh, "cgi", 1, SQLITE_ANY, 0, db_sql_cgi, 0, 0); + sqlite3_create_function(dbh, "cgi", 2, SQLITE_ANY, 0, db_sql_cgi, 0, 0); + re_add_sql_func(db) /* Requires the regex bits. */; +#endif + }/*if(db->f)*/ + } + end: + if(rc){ +#if 1 + /* This is arguable... */ + if(db->f && db->error.code && !db->f->error.code){ + /* COPY db's error state as f's. */ + fsl_error_copy( &db->error, &db->f->error ); + } +#endif + if(dbh){ + sqlite3_close(dbh); + db->dbh = NULL; + } + }else{ + assert(db->dbh); + } + return rc; +} + +int fsl_db_exec_multiv( fsl_db * const db, const char * sql, va_list args){ + if(!db || !db->dbh || !sql) return FSL_RC_MISUSE; + else{ + fsl_buffer buf = fsl_buffer_empty; + int rc = 0; + char const * z; + char const * zEnd = NULL; + rc = fsl_buffer_appendfv( &buf, sql, args ); + if(rc){ + fsl_buffer_clear(&buf); + return rc; + } + z = fsl_buffer_cstr(&buf); + while( (SQLITE_OK==rc) && *z ){ + fsl_stmt_t * pStmt = NULL; + rc = sqlite3_prepare_v2(db->dbh, z, buf.used, &pStmt, &zEnd); + if( SQLITE_OK != rc ){ + rc = fsl_err_from_db(db, rc); + break; + } + if(pStmt){ + while( SQLITE_ROW == sqlite3_step(pStmt) ){} + rc = sqlite3_finalize(pStmt); + if(rc) rc = fsl_err_from_db(db, rc); + } + buf.used -= (zEnd-z); + z = zEnd; + } + fsl_buffer_reserve(&buf, 0); + return rc; + } +} + +int fsl_db_exec_multi( fsl_db * const db, const char * sql, ...){ + if(!db || !db->dbh || !sql) return FSL_RC_MISUSE; + else{ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_exec_multiv( db, sql, args ); + va_end(args); + return rc; + } +} + +int fsl_db_execv( fsl_db * const db, const char * sql, va_list args){ + if(!db || !db->dbh || !sql) return FSL_RC_MISUSE; + else{ + fsl_stmt st = fsl_stmt_empty; + int rc = 0; + rc = fsl_db_preparev( db, &st, sql, args ); + if(rc) return rc; + /* rc = fsl_stmt_step( &st ); */ + while(FSL_RC_STEP_ROW == (rc=fsl_stmt_step(&st))){} + fsl_stmt_finalize(&st); + return (FSL_RC_STEP_ERROR==rc) + ? FSL_RC_DB + : 0; + } +} + +int fsl_db_exec( fsl_db * const db, const char * sql, ...){ + if(!db || !db->dbh || !sql) return FSL_RC_MISUSE; + else{ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_execv( db, sql, args ); + va_end(args); + return rc; + } +} + +int fsl_db_changes_recent(fsl_db * const db){ + return (db && db->dbh) + ? sqlite3_changes(db->dbh) + : 0; +} + +int fsl_db_changes_total(fsl_db * const db){ + return (db && db->dbh) + ? sqlite3_total_changes(db->dbh) + : 0; +} + +/** + Sets db->priorChanges to sqlite3_total_changes(db->dbh). +*/ +static void fsl_db_reset_change_count(fsl_db * const db){ + db->priorChanges = sqlite3_total_changes(db->dbh); +} + +int fsl_db_transaction_begin(fsl_db * const db){ + if(!db || !db->dbh) return FSL_RC_MISUSE; + else { + int rc = (0==db->beginCount) + ? fsl_db_exec(db,"BEGIN TRANSACTION") + : 0; + if(!rc){ + if(1 == ++db->beginCount){ + fsl_db_reset_change_count(db); + } + } + return rc; + } +} + +int fsl_db_transaction_level(fsl_db * const db){ + return db->doRollback ? -db->beginCount : db->beginCount; +} + +int fsl_db_transaction_commit(fsl_db * const db){ + return (db && db->dbh) + ? fsl_db_transaction_end(db, 0) + : FSL_RC_MISUSE; +} + +int fsl_db_transaction_rollback(fsl_db * const db){ + return (db && db->dbh) + ? fsl_db_transaction_end(db, 1) + : FSL_RC_MISUSE; +} + +int fsl_db_rollback_force( fsl_db * const db ){ + if(!db || !db->dbh) return FSL_RC_MISUSE; + else{ + int rc; + db->beginCount = 0; + fsl_db_cleanup_beforeCommit(db); + rc = fsl_db_exec(db, "ROLLBACK"); + fsl_db_reset_change_count(db); + return rc; + } +} + +int fsl_db_transaction_end(fsl_db * const db, bool doRollback){ + int rc = 0; + if(!db || !db->dbh) return FSL_RC_MISUSE; + else if (db->beginCount<=0){ + return fsl_error_set(&db->error, FSL_RC_RANGE, + "No transaction is active."); + } + if(doRollback) ++db->doRollback + /* ACHTUNG: note that db->dbRollback is set before + continuing so that if we return due to a non-0 beginCount + that the rollback flag propagates through the + transaction's stack. + */ + ; + if(--db->beginCount > 0) return 0; + assert(0==db->beginCount && "The commit-hook check relies on this."); + assert(db->doRollback>=0); + if((0==db->doRollback) + && (db->priorChanges < sqlite3_total_changes(db->dbh))){ + /* Execute before-commit hooks and leaf checks */ + fsl_size_t x = 0; + for( ; !rc && (x < db->beforeCommit.used); ++x ){ + char const * sql = (char const *)db->beforeCommit.list[x]; + /* MARKER(("Running before-commit code: [%s]\n", sql)); */ + if(sql) rc = fsl_db_exec_multi( db, "%s", sql ); + } + if(!rc && db->f && (FSL_DBROLE_REPO & db->role)){ + /* + i don't like this one bit - this is low-level SCM + functionality in an otherwise generic routine. Maybe we need + fsl_cx_transaction_begin/end() instead. + + Much later: we have that routine now but will need to replace + all relevant calls to fsl_db_transaction_begin()/end() with + those routines before we can consider moving this there. + */ + rc = fsl_repo_leaf_do_pending_checks(db->f); + if(!rc && db->f->cache.toVerify.used){ + rc = fsl_repo_verify_at_commit(db->f); + }else{ + fsl_repo_verify_cancel(db->f); + } + } + db->doRollback = rc ? 1 : 0; + } + fsl_db_cleanup_beforeCommit(db); + fsl_db_reset_change_count(db); + rc = fsl_db_exec(db, db->doRollback ? "ROLLBACK" : "COMMIT"); + db->doRollback = 0; + return rc; +#if 0 + /* original impl, for reference purposes during testing */ + if( g.db==0 ) return; + if( db.nBegin<=0 ) return; + if( rollbackFlag ) db.doRollback = 1; + db.nBegin--; + if( db.nBegin==0 ){ + int i; + if( db.doRollback==0 && db.nPriorChangesdbh || !rv || !sql || !*sql) return FSL_RC_MISUSE; + else{ + fsl_stmt st = fsl_stmt_empty; + int rc = 0; + rc = fsl_db_preparev( db, &st, sql, args ); + if(rc) return rc; + rc = fsl_stmt_step( &st ); + switch(rc){ + case FSL_RC_STEP_ROW: + *rv = sqlite3_column_int(st.stmt, 0); + /* Fall through */ + case FSL_RC_STEP_DONE: + rc = 0; + break; + default: + assert(FSL_RC_STEP_ERROR==rc); + break; + } + fsl_stmt_finalize(&st); + return rc; + } +} + +int fsl_db_get_int32( fsl_db * const db, int32_t * rv, + char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_get_int32v(db, rv, sql, args); + va_end(args); + return rc; +} + +int fsl_db_get_int64v( fsl_db * db, int64_t * rv, + char const * sql, va_list args){ + if(!db || !db->dbh || !rv || !sql || !*sql) return FSL_RC_MISUSE; + else{ + fsl_stmt st = fsl_stmt_empty; + int rc = 0; + rc = fsl_db_preparev( db, &st, sql, args ); + if(rc) return rc; + rc = fsl_stmt_step( &st ); + switch(rc){ + case FSL_RC_STEP_ROW: + *rv = sqlite3_column_int64(st.stmt, 0); + /* Fall through */ + case FSL_RC_STEP_DONE: + rc = 0; + break; + default: + assert(FSL_RC_STEP_ERROR==rc); + break; + } + fsl_stmt_finalize(&st); + return rc; + } +} + +int fsl_db_get_int64( fsl_db * db, int64_t * rv, + char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_get_int64v(db, rv, sql, args); + va_end(args); + return rc; +} + + +int fsl_db_get_idv( fsl_db * db, fsl_id_t * rv, + char const * sql, va_list args){ + if(!db || !db->dbh || !rv || !sql || !*sql) return FSL_RC_MISUSE; + else{ + fsl_stmt st = fsl_stmt_empty; + int rc = 0; + rc = fsl_db_preparev( db, &st, sql, args ); + if(rc) return rc; + rc = fsl_stmt_step( &st ); + switch(rc){ + case FSL_RC_STEP_ROW: + *rv = (fsl_id_t)sqlite3_column_int64(st.stmt, 0); + /* Fall through */ + case FSL_RC_STEP_DONE: + rc = 0; + break; + default: + assert(FSL_RC_STEP_ERROR==rc); + break; + } + fsl_stmt_finalize(&st); + return rc; + } +} + +int fsl_db_get_id( fsl_db * db, fsl_id_t * rv, + char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_get_idv(db, rv, sql, args); + va_end(args); + return rc; +} + + +int fsl_db_get_sizev( fsl_db * db, fsl_size_t * rv, + char const * sql, va_list args){ + if(!db || !db->dbh || !rv || !sql || !*sql) return FSL_RC_MISUSE; + else{ + fsl_stmt st = fsl_stmt_empty; + int rc = 0; + rc = fsl_db_preparev( db, &st, sql, args ); + if(rc) return rc; + rc = fsl_stmt_step( &st ); + switch(rc){ + case FSL_RC_STEP_ROW:{ + sqlite3_int64 const i = sqlite3_column_int64(st.stmt, 0); + if(i<0){ + rc = FSL_RC_RANGE; + break; + } + *rv = (fsl_size_t)i; + rc = 0; + break; + } + case FSL_RC_STEP_DONE: + rc = 0; + break; + default: + assert(FSL_RC_STEP_ERROR==rc); + break; + } + fsl_stmt_finalize(&st); + return rc; + } +} + +int fsl_db_get_size( fsl_db * db, fsl_size_t * rv, + char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_get_sizev(db, rv, sql, args); + va_end(args); + return rc; +} + + +int fsl_db_get_doublev( fsl_db * db, double * rv, + char const * sql, va_list args){ + if(!db || !db->dbh || !rv || !sql || !*sql) return FSL_RC_MISUSE; + else{ + fsl_stmt st = fsl_stmt_empty; + int rc = 0; + rc = fsl_db_preparev( db, &st, sql, args ); + if(rc) return rc; + rc = fsl_stmt_step( &st ); + switch(rc){ + case FSL_RC_STEP_ROW: + *rv = sqlite3_column_double(st.stmt, 0); + /* Fall through */ + case FSL_RC_STEP_DONE: + rc = 0; + break; + default: + assert(FSL_RC_STEP_ERROR==rc); + break; + } + fsl_stmt_finalize(&st); + return rc; + } +} + +int fsl_db_get_double( fsl_db * db, double * rv, + char const * sql, + ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_get_doublev(db, rv, sql, args); + va_end(args); + return rc; +} + + +int fsl_db_get_textv( fsl_db * db, char ** rv, + fsl_size_t *rvLen, + char const * sql, va_list args){ + if(!db || !db->dbh || !rv || !sql || !*sql) return FSL_RC_MISUSE; + else{ + fsl_stmt st = fsl_stmt_empty; + int rc = 0; + rc = fsl_db_preparev( db, &st, sql, args ); + if(rc) return rc; + rc = fsl_stmt_step( &st ); + switch(rc){ + case FSL_RC_STEP_ROW:{ + char const * str = (char const *)sqlite3_column_text(st.stmt, 0); + int const len = sqlite3_column_bytes(st.stmt,0); + if(!str){ + *rv = NULL; + if(rvLen) *rvLen = 0; + }else{ + char * x = fsl_strndup(str, len); + if(!x){ + rc = FSL_RC_OOM; + }else{ + *rv = x; + if(rvLen) *rvLen = (fsl_size_t)len; + rc = 0; + } + } + break; + } + case FSL_RC_STEP_DONE: + *rv = NULL; + if(rvLen) *rvLen = 0; + rc = 0; + break; + default: + assert(FSL_RC_STEP_ERROR==rc); + break; + } + fsl_stmt_finalize(&st); + return rc; + } +} + +int fsl_db_get_text( fsl_db * db, char ** rv, + fsl_size_t * rvLen, + char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_get_textv(db, rv, rvLen, sql, args); + va_end(args); + return rc; +} + +int fsl_db_get_blobv( fsl_db * db, void ** rv, + fsl_size_t *rvLen, + char const * sql, va_list args){ + if(!db || !db->dbh || !rv || !sql || !*sql) return FSL_RC_MISUSE; + else{ + fsl_stmt st = fsl_stmt_empty; + int rc = 0; + rc = fsl_db_preparev( db, &st, sql, args ); + if(rc) return rc; + rc = fsl_stmt_step( &st ); + switch(rc){ + case FSL_RC_STEP_ROW:{ + fsl_buffer buf = fsl_buffer_empty; + void const * str = sqlite3_column_blob(st.stmt, 0); + int const len = sqlite3_column_bytes(st.stmt,0); + if(!str){ + *rv = NULL; + if(rvLen) *rvLen = 0; + }else{ + rc = fsl_buffer_append(&buf, str, len); + if(!rc){ + *rv = buf.mem; + if(rvLen) *rvLen = buf.used; + } + } + break; + } + case FSL_RC_STEP_DONE: + *rv = NULL; + if(rvLen) *rvLen = 0; + rc = 0; + break; + default: + assert(FSL_RC_STEP_ERROR==rc); + break; + } + fsl_stmt_finalize(&st); + return rc; + } +} + +int fsl_db_get_blob( fsl_db * db, void ** rv, + fsl_size_t * rvLen, + char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_get_blobv(db, rv, rvLen, sql, args); + va_end(args); + return rc; +} + +int fsl_db_get_bufferv( fsl_db * db, fsl_buffer * b, + char asBlob, char const * sql, + va_list args){ + if(!db || !db->dbh || !b || !sql || !*sql) return FSL_RC_MISUSE; + else{ + fsl_stmt st = fsl_stmt_empty; + int rc = 0; + rc = fsl_db_preparev( db, &st, sql, args ); + if(rc) return rc; + rc = fsl_stmt_step( &st ); + switch(rc){ + case FSL_RC_STEP_ROW:{ + void const * str = asBlob + ? sqlite3_column_blob(st.stmt, 0) + : (void const *)sqlite3_column_text(st.stmt, 0); + int const len = sqlite3_column_bytes(st.stmt,0); + rc = 0; + b->used = 0; + rc = fsl_buffer_append( b, str, len ); + break; + } + case FSL_RC_STEP_DONE: + rc = 0; + break; + default: + assert(FSL_RC_STEP_ERROR==rc); + break; + } + fsl_stmt_finalize(&st); + return rc; + } +} + +int fsl_db_get_buffer( fsl_db * db, fsl_buffer * b, + char asBlob, + char const * sql, ... ){ + int rc; + va_list args; + va_start(args,sql); + rc = fsl_db_get_bufferv(db, b, asBlob, sql, args); + va_end(args); + return rc; +} + +int32_t fsl_db_g_int32( fsl_db * const db, int32_t dflt, + char const * sql, ... ){ + int32_t rv = dflt; + va_list args; + va_start(args,sql); + fsl_db_get_int32v(db, &rv, sql, args); + va_end(args); + return rv; +} + +int64_t fsl_db_g_int64( fsl_db * db, int64_t dflt, + char const * sql, + ... ){ + int64_t rv = dflt; + va_list args; + va_start(args,sql); + fsl_db_get_int64v(db, &rv, sql, args); + va_end(args); + return rv; +} + +fsl_id_t fsl_db_g_id( fsl_db * db, fsl_id_t dflt, + char const * sql, + ... ){ + fsl_id_t rv = dflt; + va_list args; + va_start(args,sql); + fsl_db_get_idv(db, &rv, sql, args); + va_end(args); + return rv; +} + +fsl_size_t fsl_db_g_size( fsl_db * db, fsl_size_t dflt, + char const * sql, + ... ){ + fsl_size_t rv = dflt; + va_list args; + va_start(args,sql); + fsl_db_get_sizev(db, &rv, sql, args); + va_end(args); + return rv; +} + +double fsl_db_g_double( fsl_db * db, double dflt, + char const * sql, + ... ){ + double rv = dflt; + va_list args; + va_start(args,sql); + fsl_db_get_doublev(db, &rv, sql, args); + va_end(args); + return rv; +} + +char * fsl_db_g_text( fsl_db * db, fsl_size_t * len, + char const * sql, + ... ){ + char * rv = NULL; + va_list args; + va_start(args,sql); + fsl_db_get_textv(db, &rv, len, sql, args); + va_end(args); + return rv; +} + +void * fsl_db_g_blob( fsl_db * db, fsl_size_t * len, + char const * sql, + ... ){ + void * rv = NULL; + va_list args; + va_start(args,sql); + fsl_db_get_blob(db, &rv, len, sql, args); + va_end(args); + return rv; +} + +double fsl_db_julian_now(fsl_db * db){ + double rc = -1.0; + if(db && db->dbh){ + /* TODO? use cached statement? So far not used often enough to + justify it. */ + fsl_db_get_double( db, &rc, "SELECT julianday('now')"); + } + return rc; +} + +double fsl_db_string_to_julian(fsl_db * db, char const * str){ + double rc = -1.0; + if(db && db->dbh){ + /* TODO? use cached statement? So far not used often enough to + justify it. */ + fsl_db_get_double( db, &rc, "SELECT julianday(%Q)",str); + } + return rc; +} + +bool fsl_db_existsv(fsl_db * const db, char const * sql, va_list args ){ + if(!db || !db->dbh || !sql) return 0; + else if(!*sql) return 0; + else{ + fsl_stmt st = fsl_stmt_empty; + bool rv = false; + if(!fsl_db_preparev(db, &st, sql, args)){ + rv = FSL_RC_STEP_ROW==fsl_stmt_step(&st) ? true : false; + } + fsl_stmt_finalize(&st); + return rv; + } + +} + +bool fsl_db_exists(fsl_db * const db, char const * sql, ... ){ + bool rc; + va_list args; + va_start(args,sql); + rc = fsl_db_existsv(db, sql, args); + va_end(args); + return rc; +} + +bool fsl_db_table_exists(fsl_db * const db, + fsl_dbrole_e whichDb, + const char *zTable +){ + const char *zDb = fsl_db_role_label( whichDb ); + int rc = db->dbh + ? sqlite3_table_column_metadata(db->dbh, zDb, zTable, 0, + 0, 0, 0, 0, 0) + : !SQLITE_OK; + return rc==SQLITE_OK ? true : false; +} + +bool fsl_db_table_has_column( fsl_db * const db, char const *zTableName, char const *zColName ){ + fsl_stmt q = fsl_stmt_empty; + int rc = 0; + bool rv = 0; + if(!zTableName || !*zTableName || !zColName || !*zColName) return false; + rc = fsl_db_prepare(db, &q, "PRAGMA table_info(%Q)", zTableName ); + if(!rc) while(FSL_RC_STEP_ROW==fsl_stmt_step(&q)){ + /* Columns: (cid, name, type, notnull, dflt_value, pk) */ + fsl_size_t colLen = 0; + char const * zCol = fsl_stmt_g_text(&q, 1, &colLen); + if(0==fsl_strncmp(zColName, zCol, colLen)){ + rv = true; + break; + } + } + fsl_stmt_finalize(&q); + return rv; +} + +char * fsl_db_random_hex(fsl_db * db, fsl_size_t n){ + if(!db || !n) return NULL; + else{ + fsl_size_t rvLen = 0; + char * rv = fsl_db_g_text(db, &rvLen, + "SELECT lower(hex(" + "randomblob(%"FSL_SIZE_T_PFMT")))", + (fsl_size_t)(n/2+1)); + if(rv){ + assert(rvLen>=n); + rv[n]=0; + } + return rv; + } +} + + +int fsl_db_select_slistv( fsl_db * db, fsl_list * tgt, + char const * fmt, va_list args ){ + if(!db || !tgt || !fmt) return FSL_RC_MISUSE; + else if(!*fmt) return FSL_RC_RANGE; + else{ + int rc; + fsl_stmt st = fsl_stmt_empty; + fsl_size_t nlen; + char const * n; + char * cp; + rc = fsl_db_preparev(db, &st, fmt, args); + while( !rc && (FSL_RC_STEP_ROW==fsl_stmt_step(&st)) ){ + nlen = 0; + n = fsl_stmt_g_text(&st, 0, &nlen); + cp = n ? fsl_strndup(n, (fsl_int_t)nlen) : NULL; + if(n && !cp) rc = FSL_RC_OOM; + else{ + rc = fsl_list_append(tgt, cp); + if(rc && cp) fsl_free(cp); + } + } + fsl_stmt_finalize(&st); + return rc; + } +} + +int fsl_db_select_slist( fsl_db * const db, fsl_list * tgt, + char const * fmt, ... ){ + int rc; + va_list va; + va_start (va,fmt); + rc = fsl_db_select_slistv(db, tgt, fmt, va); + va_end(va); + return rc; +} + +void fsl_db_sqltrace_enable( fsl_db * const db, FILE * outStream ){ + if(db && db->dbh){ + sqlite3_trace(db->dbh, fsl_db_sql_trace, outStream); + } +} + +int fsl_db_init( fsl_error * err, + char const * zFilename, + char const * zSchema, + ... ){ + fsl_db DB = fsl_db_empty; + fsl_db * db = &DB; + char const * zSql; + int rc; + char inTrans = 0; + va_list ap; + rc = fsl_db_open(db, zFilename, 0); + if(rc) goto end; + rc = fsl_db_exec(db, "BEGIN EXCLUSIVE"); + if(rc) goto end; + inTrans = 1; + rc = fsl_db_exec_multi(db, "%s", zSchema); + if(rc) goto end; + va_start(ap, zSchema); + while( !rc && (zSql = va_arg(ap, const char*))!=NULL ){ + rc = fsl_db_exec_multi(db, "%s", zSql); + } + va_end(ap); + end: + if(rc){ + if(inTrans) fsl_db_exec(db, "ROLLBACK"); + }else{ + rc = fsl_db_exec(db, "COMMIT"); + } + if(err){ + if(db->error.code){ + fsl_error_move(&db->error, err); + }else if(rc){ + err->code = rc; + err->msg.used = 0; + } + } + fsl_db_close(db); + return rc; +} + +int fsl_stmt_each_f_dump( fsl_stmt * const stmt, void * state ){ + int i; + fsl_cx * f = (stmt && stmt->db) ? stmt->db->f : NULL; + char const * sep = "\t"; + if(!f) return FSL_RC_MISUSE; + if(state){/*unused arg*/} + if(1==stmt->rowCount){ + for( i = 0; i < stmt->colCount; ++i ){ + fsl_outputf(f, "%s%s", fsl_stmt_col_name(stmt, i), + (i==stmt->colCount-1) ? "" : sep); + } + fsl_output(f, "\n", 1); + } + for( i = 0; i < stmt->colCount; ++i ){ + char const * val = fsl_stmt_g_text(stmt, i, NULL); + fsl_outputf(f, "%s%s", val ? val : "NULL", + (i==stmt->colCount-1) ? "" : sep); + } + fsl_output(f, "\n", 1); + return 0; +} + + +#undef MARKER +/* end of file db.c */ +/* start of file deck.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 Stephan Beal (https://wanderinghorse.net). + + + + This program is free software; you can redistribute it and/or + modify it under the terms of the Simplified BSD License (also + known as the "2-Clause License" or "FreeBSD License".) + + This program is distributed in the hope that it will be useful, + but without any warranty; without even the implied warranty of + merchantability or fitness for a particular purpose. + + ***************************************************************************** + This file houses the manifest/control-artifact-related APIs. +*/ +#include +#include /* qsort() */ +#include /* memcmp() */ + +/* Only for debugging */ +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +typedef int StaticAssertMCacheArraySizes[ + ((sizeof(fsl_mcache_empty.aAge) + /sizeof(fsl_mcache_empty.aAge[0])) + == (sizeof(fsl_mcache_empty.decks) + /sizeof(fsl_mcache_empty.decks[0]))) + ? 1 : -1 +]; + +enum fsl_card_F_list_flags_e { +FSL_CARD_F_LIST_NEEDS_SORT = 0x01 +}; + +/** + Transfers the contents of d into f->cache.mcache. If d is + dynamically allocated then it is also freed. In any case, after + calling this the caller must behave as if the deck had been passed + to fsl_deck_finalize(). + + If manifest caching is disabled for f, d is immediately finalized. +*/ +static void fsl_cx_mcache_insert(fsl_cx *f, fsl_deck * d){ + if(!(f->flags & FSL_CX_F_MANIFEST_CACHE)){ + fsl_deck_finalize(d); + return; + } + static const unsigned cacheLen = + (unsigned)(sizeof(fsl_mcache_empty.aAge) + /sizeof(fsl_mcache_empty.aAge[0])); + fsl_mcache * const mc = &f->cache.mcache; + while( d ){ + unsigned i; + fsl_deck *pBaseline = d->B.baseline; + d->B.baseline = 0; + for(i=0; idecks[i].rid ) break; + } + if( i>=cacheLen ){ + unsigned oldest = 0; + unsigned oldestAge = mc->aAge[0]; + for(i=1; iaAge[i]aAge[i]; + } + } + fsl_deck_finalize(&mc->decks[oldest]); + i = oldest; + } + mc->aAge[i] = ++mc->nextAge; + mc->decks[i] = *d; + *d = fsl_deck_empty; + if(&fsl_deck_empty == mc->decks[i].allocStamp){ + /* d was fsl_deck_malloc()'d so we need to free it, but cannot + send it through fsl_deck_finalize() because that would try to + clean up the memory we just transferred ownership of to + mc->decks[i]. So... */ + mc->decks[i].allocStamp = 0; + fsl_free(d); + } + d = pBaseline; + } +} + +/** + Searches f->cache.mcache for a deck with the given RID. If found, + it is bitwise copied over tgt, that entry is removed from the + cache, and true is returned. If no match is found, tgt is not + modified and false is returned. + + If manifest caching is disabled for f, false is immediately + returned without causing side effects. +*/ +static bool fsl_cx_mcache_search(fsl_cx * f, fsl_id_t rid, fsl_deck * tgt){ + if(!(f->flags & FSL_CX_F_MANIFEST_CACHE)) return false; + static const unsigned cacheLen = + (int)(sizeof(fsl_mcache_empty.aAge) + /sizeof(fsl_mcache_empty.aAge[0])); + unsigned i; + assert(cacheLen == + (unsigned)(sizeof(fsl_mcache_empty.decks) + /sizeof(fsl_mcache_empty.decks[0]))); + for(i=0; icache.mcache.decks[i].rid==rid ){ + *tgt = f->cache.mcache.decks[i]; + f->cache.mcache.decks[i] = fsl_deck_empty; + ++f->cache.mcache.hits; + return true; + } + } + ++f->cache.mcache.misses; + return false; +} + +/** + If mem is NULL or inside d->content.mem then this function does + nothing, else it passes mem to fsl_free(). Intended to be used to + clean up d->XXX string members (or sub-members) which have been + optimized away via d->content. +*/ +static void fsl_deck_free_string(fsl_deck * d, char * mem){ + assert(d); + if(mem + && (!d->content.used + || !(((unsigned char const *)mem >=d->content.mem) + && + ((unsigned char const *)mem < (d->content.mem+d->content.capacity))) + )){ + fsl_free(mem); + }/* else do nothing - the memory is NULL or owned by d->content. */ +} + +/** + fsl_list_visitor_f() impl which frees fsl_list-of-(char*) card entries + in ((fsl_deck*)visitorState). +*/ +static int fsl_list_v_card_string_free(void * mCard, void * visitorState ){ + fsl_deck_free_string( (fsl_deck*)visitorState, mCard ); + return 0; +} + +/** Evals to a pointer to the F-card at the given index + in the given fsl_card_F_list pointer. Each arg is + evaluated only once. */ +#define F_at(LISTP,NDX) (&(LISTP)->list[NDX]) + +static int fsl_card_F_list_reserve2( fsl_card_F_list * li ){ + return (li->usedcapacity) + ? 0 + : fsl_card_F_list_reserve(li, li->capacity + ? li->capacity*4/3+1 + : 50); +} + +static void fsl_card_F_clean( fsl_card_F * f ){ + if(!f->deckOwnsStrings){ + fsl_free(f->name); + fsl_free(f->uuid); + fsl_free(f->priorName); + } + *f = fsl_card_F_empty; +} + +/** + Cleans up the F-card at li->list[ndx] and shifts all F-cards to its + right one entry to the left. +*/ +static void fsl_card_F_list_remove(fsl_card_F_list * li, + uint32_t ndx){ + uint32_t i; + assert(li->used); + assert(ndxused); + fsl_card_F_clean(F_at(li,ndx)); + for( i = ndx; i < li->used - 1; ++i ){ + li->list[i] = li->list[i+1]; + } + li->list[li->used] = fsl_card_F_empty; + --li->used; +} + +void fsl_card_F_list_finalize( fsl_card_F_list * li ){ + uint32_t i; + for(i=0; i < li->used; ++i){ + fsl_card_F_clean(F_at(li,i)); + } + li->used = li->capacity = 0; + fsl_free(li->list); + *li = fsl_card_F_list_empty; +} + +int fsl_card_F_list_reserve( fsl_card_F_list * li, uint32_t n ){ + if(li->capacity>=n) return 0; + else if(n==0){ + fsl_card_F_list_finalize(li); + return 0; + }else{ + fsl_card_F * re = fsl_realloc(li->list, n * sizeof(fsl_card_F)); + if(re){ + li->list = re; + li->capacity = n; + } + return re ? 0 : FSL_RC_OOM; + } +} + +/** + Adjusts the end of the give list by +1, reserving more space if + needed, and returns the next available F-card in a cleanly-wiped + state. Returns NULL on alloc error. +*/ +static fsl_card_F * fsl_card_F_list_push( fsl_card_F_list * li ){ + if(li->used==li->capacity && fsl_card_F_list_reserve2(li)) return NULL; + li->list[li->used] = fsl_card_F_empty; + if(li->used){ + li->flags |= FSL_CARD_F_LIST_NEEDS_SORT/*pessimistic assumption*/; + } + return &li->list[li->used++]; +} +/** + Chops the last entry off of the given list, freeing any resources + owned by that entry. Decrements li->used. Asserts that li->used is + positive. +*/ +static void fsl_card_F_list_pop( fsl_card_F_list * li ){ + assert(li->used); + if(li->used) fsl_card_F_clean(F_at(li, --li->used)); +} + +fsl_card_Q * fsl_card_Q_malloc(fsl_cherrypick_type_e type, + fsl_uuid_cstr target, + fsl_uuid_cstr baseline){ + int const targetLen = target ? fsl_is_uuid(target) : 0; + int const baselineLen = baseline ? fsl_is_uuid(baseline) : 0; + if((type!=FSL_CHERRYPICK_ADD && type!=FSL_CHERRYPICK_BACKOUT) + || !target || !targetLen + || (baseline && !baselineLen)) return NULL; + else{ + fsl_card_Q * c = + (fsl_card_Q*)fsl_malloc(sizeof(fsl_card_Q)); + if(c){ + int rc = 0; + *c = fsl_card_Q_empty; + c->type = type; + c->target = fsl_strndup(target, targetLen); + if(!c->target) rc = FSL_RC_OOM; + else if(baseline){ + c->baseline = fsl_strndup(baseline, baselineLen); + if(!c->baseline) rc = FSL_RC_OOM; + } + if(rc){ + fsl_card_Q_free(c); + c = NULL; + } + } + return c; + } +} + +void fsl_card_Q_free( fsl_card_Q * cp ){ + if(cp){ + fsl_free(cp->target); + fsl_free(cp->baseline); + *cp = fsl_card_Q_empty; + fsl_free(cp); + } +} + +fsl_card_J * fsl_card_J_malloc(bool isAppend, + char const * field, + char const * value){ + if(!field || !*field) return NULL; + else{ + fsl_card_J * c = + (fsl_card_J*)fsl_malloc(sizeof(fsl_card_J)); + if(c){ + int rc = 0; + fsl_size_t const lF = fsl_strlen(field); + fsl_size_t const lV = value ? fsl_strlen(value) : 0; + *c = fsl_card_J_empty; + c->append = isAppend ? 1 : 0; + c->field = fsl_strndup(field, (fsl_int_t)lF); + if(!c->field) rc = FSL_RC_OOM; + else if(value && *value){ + c->value = fsl_strndup(value, (fsl_int_t)lV); + if(!c->value) rc = FSL_RC_OOM; + } + if(rc){ + fsl_card_J_free(c); + c = NULL; + } + } + return c; + } +} + +void fsl_card_J_free( fsl_card_J * cp ){ + if(cp){ + fsl_free(cp->field); + fsl_free(cp->value); + *cp = fsl_card_J_empty; + fsl_free(cp); + } +} + +/** + fsl_list_visitor_f() impl which requires that obj be-a (fsl_card_T*), + which this function passes to fsl_card_T_free(). +*/ +static int fsl_list_v_card_T_free(void * obj, void * visitorState ){ + if(obj) fsl_card_T_free( (fsl_card_T*)obj ); + return 0; +} + +static int fsl_list_v_card_Q_free(void * obj, void * visitorState ){ + if(obj) fsl_card_Q_free( (fsl_card_Q*)obj ); + return 0; +} + +static int fsl_list_v_card_J_free(void * obj, void * visitorState ){ + if(obj) fsl_card_J_free( (fsl_card_J*)obj ); + return 0; +} + +fsl_deck * fsl_deck_malloc(){ + fsl_deck * rc = (fsl_deck *)fsl_malloc(sizeof(fsl_deck)); + if(rc){ + *rc = fsl_deck_empty; + rc->allocStamp = &fsl_deck_empty; + } + return rc; +} + +void fsl_deck_init( fsl_cx * f, fsl_deck * cards, fsl_satype_e type ){ + void const * allocStamp = cards->allocStamp; + *cards = fsl_deck_empty; + cards->allocStamp = allocStamp; + cards->f = f; + cards->type = type; +} + +void fsl_card_J_list_free(fsl_list * li, bool alsoListMem){ + if(li->used) fsl_list_visit(li, 0, fsl_list_v_card_J_free, NULL); + if(alsoListMem) fsl_list_reserve(li, 0); + else li->used = 0; +} + +/* fsl_deck cleanup helpers... */ +#define SFREE(X) fsl_deck_clean_string(m, &m->X) +#define SLIST(X) fsl_list_clear(&m->X, fsl_list_v_card_string_free, m) +#define CBUF(X) fsl_buffer_clear(&m->X) +static void fsl_deck_clean_string(fsl_deck *m, char **member){ + fsl_deck_free_string(m, *member); + *member = 0; +} +static void fsl_deck_clean_version(fsl_deck *const m){ + m->rid = 0; +} +static void fsl_deck_clean_A(fsl_deck *const m){ + SFREE(A.name); + SFREE(A.tgt); + SFREE(A.src); +} +static void fsl_deck_clean_B(fsl_deck * const m){ + if(m->B.baseline){ + assert(!m->B.baseline->B.uuid && "Baselines cannot have a B-card. API misuse?"); + fsl_deck_finalize(m->B.baseline); + m->B.baseline = NULL; + } + SFREE(B.uuid); +} +static void fsl_deck_clean_C(fsl_deck * const m){ + fsl_deck_clean_string(m, &m->C); +} +static void fsl_deck_clean_E(fsl_deck * const m){ + fsl_deck_clean_string(m, &m->E.uuid); + m->E = fsl_deck_empty.E; +} +static void fsl_deck_clean_F(fsl_deck * const m){ + if(m->F.list){ + fsl_card_F_list_finalize(&m->F); + m->F = fsl_deck_empty.F; + } +} +static void fsl_deck_clean_G(fsl_deck * const m){ + fsl_deck_clean_string(m, &m->G); +} +static void fsl_deck_clean_H(fsl_deck * const m){ + fsl_deck_clean_string(m, &m->H); +} +static void fsl_deck_clean_I(fsl_deck * const m){ + fsl_deck_clean_string(m, &m->I); +} +static void fsl_deck_clean_J(fsl_deck * const m, bool alsoListMem){ + fsl_card_J_list_free(&m->J, alsoListMem); +} +static void fsl_deck_clean_K(fsl_deck * const m){ + fsl_deck_clean_string(m, &m->K); +} +static void fsl_deck_clean_L(fsl_deck * const m){ + fsl_deck_clean_string(m, &m->L); +} +static void fsl_deck_clean_M(fsl_deck * const m){ + SLIST(M); +} +static void fsl_deck_clean_N(fsl_deck * const m){ + fsl_deck_clean_string(m, &m->N); +} +static void fsl_deck_clean_P(fsl_deck * const m){ + fsl_list_clear(&m->P, fsl_list_v_card_string_free, m); +} +static void fsl_deck_clean_Q(fsl_deck * const m){ + fsl_list_clear(&m->Q, fsl_list_v_card_Q_free, NULL); +} +static void fsl_deck_clean_R(fsl_deck * const m){ + fsl_deck_clean_string(m, &m->R); +} +static void fsl_deck_clean_T(fsl_deck * const m){ + fsl_list_clear(&m->T, fsl_list_v_card_T_free, NULL); +} +static void fsl_deck_clean_U(fsl_deck * const m){ + fsl_deck_clean_string(m, &m->U); +} +static void fsl_deck_clean_W(fsl_deck * const m){ + CBUF(W); +} + +void fsl_deck_clean2(fsl_deck * const m, fsl_buffer * const xferBuf){ + if(!m) return; + fsl_deck_clean_version(m); + fsl_deck_clean_A(m); + fsl_deck_clean_B(m); + fsl_deck_clean_C(m); + m->D = 0.0; + fsl_deck_clean_E(m); + fsl_deck_clean_F(m); + fsl_deck_clean_G(m); + fsl_deck_clean_H(m); + fsl_deck_clean_I(m); + fsl_deck_clean_J(m,true); + fsl_deck_clean_K(m); + fsl_deck_clean_L(m); + fsl_deck_clean_M(m); + fsl_deck_clean_N(m); + fsl_deck_clean_P(m); + fsl_deck_clean_Q(m); + fsl_deck_clean_R(m); + fsl_deck_clean_T(m); + fsl_deck_clean_U(m); + fsl_deck_clean_W(m); + if(xferBuf){ + fsl_buffer_swap(&m->content, xferBuf); + fsl_buffer_reuse(xferBuf); + } + CBUF(content) /* content must be after all cards because some point + back into it and we need this memory intact in + order to know that! + */; + { + void const * const allocStampKludge = m->allocStamp; + fsl_cx * const f = m->f; + *m = fsl_deck_empty; + m->allocStamp = allocStampKludge; + m->f = f; + } +} +#undef CBUF +#undef SFREE +#undef SLIST + +void fsl_deck_clean(fsl_deck * const m){ + fsl_deck_clean2(m, NULL); +} + +void fsl_deck_finalize(fsl_deck * const m){ + void const * allocStamp; + if(!m) return; + allocStamp = m->allocStamp; + fsl_deck_clean(m); + if(allocStamp == &fsl_deck_empty){ + fsl_free(m); + }else{ + m->allocStamp = allocStamp; + } +} + +int fsl_card_is_legal( fsl_satype_e t, char card ){ + /* + Implements this table: + + https://fossil-scm.org/index.html/doc/trunk/www/fileformat.wiki#summary + */ + if('Z'==card) return 1; + else switch(t){ + case FSL_SATYPE_ANY: + switch(card){ + case 'A': case 'B': case 'C': case 'D': + case 'E': case 'F': case 'J': case 'K': + case 'L': case 'M': case 'N': case 'P': + case 'Q': case 'R': case 'T': case 'U': + case 'W': + return -1; + default: + return 0; + } + case FSL_SATYPE_ATTACHMENT: + switch(card){ + case 'A': case 'D': + return 1; + case 'C': case 'N': case 'U': + return -1; + default: + return 0; + }; + case FSL_SATYPE_CLUSTER: + return 'M'==card ? 1 : 0; + case FSL_SATYPE_CONTROL: + switch(card){ + case 'D': case 'T': case 'U': + return 1; + default: + return 0; + }; + case FSL_SATYPE_EVENT: + switch(card){ + case 'D': case 'E': + case 'W': + return 1; + case 'C': case 'N': + case 'P': case 'T': + case 'U': + return -1; + default: + return 0; + }; + case FSL_SATYPE_CHECKIN: + switch(card){ + case 'C': case 'D': + case 'U': + return 1; + case 'B': case 'F': + case 'N': case 'P': + case 'Q': case 'R': + case 'T': + return -1; + default: + return 0; + }; + case FSL_SATYPE_TICKET: + switch(card){ + case 'D': case 'J': + case 'K': case 'U': + return 1; + default: + return 0; + }; + case FSL_SATYPE_WIKI: + switch(card){ + case 'D': case 'L': + case 'U': case 'W': + return 1; + case 'C': + case 'N': case 'P': + return -1; + default: + return 0; + }; + case FSL_SATYPE_FORUMPOST: + switch(card){ + case 'D': case 'U': case 'W': + return 1; + case 'G': case 'H': case 'I': + case 'N': case 'P': + return -1; + default: + return 0; + }; + default: + MARKER(("invalid fsl_satype_e value: %d, card=%c\n", t, card)); + assert(!"Invalid fsl_satype_e."); + return 0; + }; +} + +bool fsl_deck_has_required_cards( fsl_deck const * d ){ + if(!d) return 0; + switch(d->type){ + case FSL_SATYPE_ANY: + return 0; +#define NEED(CARD,COND) \ + if(!(COND)) { \ + fsl_cx_err_set(d->f, FSL_RC_SYNTAX, \ + "Required %c-card is missing or invalid.", \ + *#CARD); \ + return false; \ + } (void)0 + case FSL_SATYPE_ATTACHMENT: + NEED(A,d->A.name); + NEED(A,d->A.tgt); + NEED(D,d->D > 0); + return 1; + case FSL_SATYPE_CLUSTER: + NEED(M,d->M.used); + return 1; + case FSL_SATYPE_CONTROL: + NEED(D,d->D > 0); + NEED(U,d->U); + NEED(T,d->T.used>0); + return 1; + case FSL_SATYPE_EVENT: + NEED(D,d->D > 0); + NEED(E,d->E.julian>0); + NEED(E,d->E.uuid); + NEED(W,d->W.used); + return 1; + case FSL_SATYPE_CHECKIN: + /* + Historically we need both or neither of F- and R-cards, but + the R-card has become optional because it's so expensive to + calculate and verify. + + Manifest #1 has an empty file list and an R-card with a + constant (repo/manifest-independent) hash + (d41d8cd98f00b204e9800998ecf8427e, the initial MD5 hash + state). + + R-card calculation is runtime-configurable option. + */ + NEED(D,d->D > 0); + NEED(C,d->C); + NEED(U,d->U); +#if 0 + /* It turns out that because the R-card is optional, + we can have a legal manifest with no F-cards. */ + NEED(F,d->F.used || d->R/*with initial-state md5 hash!*/); +#endif + if(!d->R + && (FSL_CX_F_CALC_R_CARD & d->f->flags)){ + fsl_cx_err_set(d->f, FSL_RC_SYNTAX, + "%s deck is missing an R-card, " + "yet R-card calculation is enabled.", + fsl_satype_cstr(d->type)); + return 0; + }else if(d->R + && !d->F.used + && 0!=fsl_strcmp(d->R, FSL_MD5_INITIAL_HASH) + ){ + fsl_cx_err_set(d->f, FSL_RC_SYNTAX, + "Deck has no F-cards, so we expect its " + "R-card is to have the initial-state MD5 " + "hash (%.12s...). Instead we got: %s", + FSL_MD5_INITIAL_HASH, d->R); + return 0; + } + return 1; + case FSL_SATYPE_TICKET: + NEED(D,d->D > 0); + NEED(K,d->K); + NEED(U,d->U); + NEED(J,d->J.used) + /* Is a J strictly required? Spec is not clear but DRH + confirms the current fossil(1) code expects a J card. */; + return 1; + case FSL_SATYPE_WIKI: + NEED(D,d->D > 0); + NEED(L,d->L); + NEED(U,d->U); + /*NEED(W,d->W.used);*/ + return 1; + case FSL_SATYPE_FORUMPOST: + NEED(D,d->D > 0); + NEED(U,d->U); + /*NEED(W,d->W.used);*/ + return 1; + case FSL_SATYPE_INVALID: + default: + assert(!"Invalid fsl_satype_e."); + return 0; + } +#undef NEED +} + +char const * fsl_satype_cstr(fsl_satype_e t){ + switch(t){ +#define C(X) case FSL_SATYPE_##X: return #X + C(ANY); + C(CHECKIN); + C(CLUSTER); + C(CONTROL); + C(WIKI); + C(TICKET); + C(ATTACHMENT); + C(EVENT); + C(FORUMPOST); + C(INVALID); + C(BRANCH_START); + default: + assert(!"UNHANDLED fsl_satype_e"); + return "!UNKNOWN!"; + } +} + +char const * fsl_satype_event_cstr(fsl_satype_e t){ + switch(t){ + case FSL_SATYPE_ANY: return "*"; + case FSL_SATYPE_BRANCH_START: + case FSL_SATYPE_CHECKIN: return "ci"; + case FSL_SATYPE_EVENT: return "e"; + case FSL_SATYPE_CONTROL: return "g"; + case FSL_SATYPE_TICKET: return "t"; + case FSL_SATYPE_WIKI: return "w"; + case FSL_SATYPE_FORUMPOST: return "f"; + default: + return NULL; + } +} + + +/** + If fsl_card_is_legal(d->type, card), returns true, else updates + d->f->error with a description of the constraint violation and + returns 0. + */ +static bool fsl_deck_check_type( fsl_deck * d, char card ){ + if(fsl_card_is_legal(d->type, card)) return true; + else{ + fsl_cx_err_set(d->f, FSL_RC_TYPE, + "Card type '%c' is not allowed " + "in artifacts of type %s.", + card, fsl_satype_cstr(d->type)); + return false; + } +} + +/** + If the first n bytes of the given string contain any values <=32, + returns FSL_RC_SYNTAX, else returns 0. mf->f's error state is + updated no error. n<0 means to use fsl_strlen() to count the + length. +*/ +static int fsl_deck_strcheck_ctrl_chars(fsl_deck * const mf, char cardName, char const * v, fsl_int_t n){ + const char * z = v; + int rc = 0; + if(v && n<0) n = fsl_strlen(v); + for( ; v && z < v+n; ++z ){ + if(*z <= 32){ + rc = fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "Invalid character in %c-card.", cardName); + break; + } + } + return rc; +} + +/* + Implements fsl_deck_LETTER_set() for certain letters: those + implemented as a fsl_uuid_str or an md5, holding a single hex string + value. + + The function returns FSL_RC_SYNTAX if + (valLen!=ASSERTLEN). ASSERTLEN is assumed to be either an SHA1, + SHA3, or MD5 hash value and it is validated against + fsl_validate16(value,valLen), returning FSL_RC_SYNTAX if that + check fails. In debug builds, the expected ranges are assert()ed. + + If value is NULL then it is removed from the card instead + (semantically freed), *mfMember is set to NULL, and 0 is returned. +*/ +static int fsl_deck_sethex_impl( fsl_deck * const mf, fsl_uuid_cstr value, + char letter, + fsl_size_t assertLen, + char ** mfMember ){ + assert(mf); + assert( value ? (assertLen==FSL_STRLEN_SHA1 + || assertLen==FSL_STRLEN_K256 + || assertLen==FSL_STRLEN_MD5) + : 0==assertLen ); + if(value && !fsl_deck_check_type(mf,letter)) return mf->f->error.code; + else if(!value){ + fsl_deck_free_string(mf, *mfMember); + *mfMember = NULL; + return 0; + }else if(fsl_strlen(value) != assertLen){ + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "Invalid length for %c-card: expecting %d.", + letter, (int)assertLen); + }else if(!fsl_validate16(value, assertLen)) { + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "Invalid hexadecimal value for %c-card.", letter); + }else{ + fsl_deck_free_string(mf, *mfMember); + *mfMember = fsl_strndup(value, assertLen); + return *mfMember ? 0 : FSL_RC_OOM; + } +} + +/** + Implements fsl_set_set_XXX() where XXX is a fsl_buffer member of fsl_deck. + */ +static int fsl_deck_b_setuffer_impl( fsl_deck * const mf, char const * value, + fsl_int_t valLen, + char letter, fsl_buffer * buf){ + assert(mf); + if(!fsl_deck_check_type(mf,letter)) return mf->f->error.code; + else if(valLen<0) valLen = (fsl_int_t)fsl_strlen(value); + buf->used = 0; + if(value && (valLen>0)){ + return fsl_buffer_append( buf, value, valLen ); + }else{ + if(buf->mem) buf->mem[0] = 0; + return 0; + } +} + +int fsl_deck_B_set( fsl_deck * const mf, fsl_uuid_cstr uuidBaseline){ + if(!mf) return FSL_RC_MISUSE; + else{ + int const bLen = uuidBaseline ? fsl_is_uuid(uuidBaseline) : 0; + if(uuidBaseline && !bLen){ + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "Invalid B-card value: %s", uuidBaseline); + } + if(mf->B.baseline){ + fsl_deck_finalize(mf->B.baseline); + mf->B.baseline = NULL; + } + return fsl_deck_sethex_impl(mf, uuidBaseline, 'B', + bLen, &mf->B.uuid); + } +} + +/** + Internal impl for card setters which consist of a simple (char *) + member. Replaces and frees any prior value. Passing NULL for the + 4th argument unsets the given card (assigns NULL to it). + */ +static int fsl_deck_set_string( fsl_deck * const mf, char letter, char ** member, char const * v, fsl_int_t n ){ + if(!fsl_deck_check_type(mf, letter)) return mf->f->error.code; + fsl_deck_free_string(mf, *member); + *member = v ? fsl_strndup(v, n) : NULL; + if(v && !*member) return FSL_RC_OOM; + else return 0; +} + +int fsl_deck_C_set( fsl_deck * const mf, char const * v, fsl_int_t n){ + return fsl_deck_set_string( mf, 'C', &mf->C, v, n ); +} + +int fsl_deck_G_set( fsl_deck * const mf, fsl_uuid_cstr uuid){ + int const uLen = fsl_is_uuid(uuid); + return uLen + ? fsl_deck_sethex_impl(mf, uuid, 'G', uLen, &mf->G) + : FSL_RC_SYNTAX; +} + +int fsl_deck_H_set( fsl_deck * const mf, char const * v, fsl_int_t n){ + if(v && mf->I) return FSL_RC_SYNTAX; + return fsl_deck_set_string( mf, 'H', &mf->H, v, n ); +} + +int fsl_deck_I_set( fsl_deck * const mf, fsl_uuid_cstr uuid){ + if(uuid && mf->H) return FSL_RC_SYNTAX; + int const uLen = uuid ? fsl_is_uuid(uuid) : 0; + return fsl_deck_sethex_impl(mf, uuid, 'I', uLen, &mf->I); +} + +int fsl_deck_J_add( fsl_deck * const mf, char isAppend, + char const * field, char const * value){ + if(!field) return FSL_RC_MISUSE; + else if(!*field) return FSL_RC_SYNTAX; + else if(!fsl_deck_check_type(mf,'J')) return mf->f->error.code; + else{ + int rc; + fsl_card_J * cp = fsl_card_J_malloc(isAppend, field, value); + if(!cp) rc = FSL_RC_OOM; + else if( 0 != (rc = fsl_list_append(&mf->J, cp))){ + fsl_card_J_free(cp); + } + return rc; + } +} + + +int fsl_deck_K_set( fsl_deck * const mf, fsl_uuid_cstr uuid){ + int const uLen = fsl_is_uuid(uuid); + return uLen + ? fsl_deck_sethex_impl(mf, uuid, 'K', uLen, &mf->K) + : FSL_RC_SYNTAX; +} + +int fsl_deck_L_set( fsl_deck * const mf, char const * v, fsl_int_t n){ + return mf + ? fsl_deck_set_string(mf, 'L', &mf->L, v, n) + : FSL_RC_SYNTAX; +} + +int fsl_deck_M_add( fsl_deck * const mf, char const *uuid){ + int rc; + char * dupe; + int const uLen = uuid ? fsl_is_uuid(uuid) : 0; + if(!uuid) return FSL_RC_MISUSE; + else if(!fsl_deck_check_type(mf, 'M')) return mf->f->error.code; + else if(!uLen) return FSL_RC_SYNTAX; + dupe = fsl_strndup(uuid, uLen); + if(!dupe) rc = FSL_RC_OOM; + else{ + rc = fsl_list_append( &mf->M, dupe ); + if(rc){ + fsl_free(dupe); + } + } + return rc; +} + +int fsl_deck_N_set( fsl_deck * const mf, char const * v, fsl_int_t n){ + int rc = 0; + if(v && n!=0){ + if(n<0) n = fsl_strlen(v); + rc = fsl_deck_strcheck_ctrl_chars(mf, 'N', v, n); + } + return rc ? rc : fsl_deck_set_string( mf, 'N', &mf->N, v, n ); +} + + +/** + Adds either parentUuid or takeParentUuid to mf->P. ONE + of those must e non-NULL and the other must be NULL. If + takeParentUuid is not NULL then ownership of it is transfered + to this function regardless of success or failure. +*/ +static int fsl__deck_P_add_impl( fsl_deck * const mf, + fsl_uuid_cstr parentUuid, + fsl_uuid_str takeParentUuid){ + if(!fsl_deck_check_type(mf, 'P')){ + fsl_free(takeParentUuid); + return mf->f->error.code; + } + int rc; + char * dupe; + fsl_uuid_cstr z = parentUuid ? parentUuid : takeParentUuid; + assert(parentUuid ? !takeParentUuid : !!takeParentUuid); + int const uLen = fsl_is_uuid(z); + if(!uLen){ + fsl_free(takeParentUuid); + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "Invalid UUID for P-card."); + } + dupe = takeParentUuid + ? takeParentUuid + : fsl_strndup(parentUuid, uLen); + if(!dupe) rc = FSL_RC_OOM; + else{ + rc = fsl_list_append( &mf->P, dupe ); + if(rc){ + fsl_free(dupe); + } + } + return rc; +} + + +int fsl_deck_P_add(fsl_deck * const mf, char const *parentUuid){ + return fsl__deck_P_add_impl(mf, parentUuid, NULL); +} + +int fsl_deck_P_add_rid( fsl_deck * const mf, fsl_id_t rid ){ + fsl_uuid_str pU = fsl_rid_to_uuid(mf->f, rid); + return pU ? fsl__deck_P_add_impl(mf, NULL, pU) : mf->f->error.code; +} + + +fsl_id_t fsl_deck_P_get_id(fsl_deck * const d, int index){ + if(!d->f) return -1; + else if(index>(int)d->P.used) return 0; + else return fsl_uuid_to_rid(d->f, (char const *)d->P.list[index]); +} + + +int fsl_deck_Q_add( fsl_deck * const mf, int type, + fsl_uuid_cstr target, + fsl_uuid_cstr baseline ){ + if(!target) return FSL_RC_MISUSE; + else if(!fsl_deck_check_type(mf,'Q')) return mf->f->error.code; + else if(!type || !fsl_is_uuid(target) + || (baseline && !fsl_is_uuid(baseline))) return FSL_RC_SYNTAX; + else{ + int rc; + fsl_card_Q * cp = fsl_card_Q_malloc(type, target, baseline); + if(!cp) rc = FSL_RC_OOM; + else if( 0 != (rc = fsl_list_append(&mf->Q, cp))){ + fsl_card_Q_free(cp); + } + return rc; + } +} + +/** + A comparison routine for qsort(3) which compares fsl_card_F + instances in a lexical manner based on their names. The order is + important for card ordering in generated manifests. + + It expects that each argument is a (fsl_card_F const *). +*/ +static int fsl_card_F_cmp( void const * lhs, void const * rhs ){ + fsl_card_F const * const l = (fsl_card_F const *)lhs; + fsl_card_F const * const r = (fsl_card_F const *)rhs; + /* Compare NULL as larger so that NULLs move to the right. That said, + we aren't expecting any NULLs. */ + assert(l); + assert(r); + if(!l) return r ? 1 : 0; + else if(!r) return -1; + else return fsl_strcmp(l->name, r->name); +} + +static void fsl_card_F_list_sort(fsl_card_F_list * li){ + if(FSL_CARD_F_LIST_NEEDS_SORT & li->flags){ + qsort(li->list, li->used, sizeof(fsl_card_F), + fsl_card_F_cmp ); + li->flags &= ~FSL_CARD_F_LIST_NEEDS_SORT; + } +} + +static void fsl_deck_F_sort(fsl_deck * const mf){ + fsl_card_F_list_sort(&mf->F); +} + +int fsl_card_F_compare_name( fsl_card_F const * const lhs, + fsl_card_F const * const rhs){ + return (lhs == rhs) ? 0 : fsl_card_F_cmp( lhs, rhs ); +} + +int fsl_deck_R_set( fsl_deck * const mf, fsl_uuid_cstr md5){ + return mf + ? fsl_deck_sethex_impl(mf, md5, 'R', md5 ? FSL_STRLEN_MD5 : 0, &mf->R) + : FSL_RC_MISUSE; +} + +int fsl_deck_R_calc2(fsl_deck * const mf, char ** tgt){ + fsl_cx * const f = mf->f; + char const * theHash = 0; + char hex[FSL_STRLEN_MD5+1]; + if(!f) return FSL_RC_MISUSE; + else if(!fsl_needs_repo(f)){ + return FSL_RC_NOT_A_REPO; + }else if(!fsl_deck_check_type(mf,'R')) { + assert(mf->f->error.code); + return mf->f->error.code; + }else if(!mf->F.used){ + theHash = FSL_MD5_INITIAL_HASH; + /* fall through and set hash */ + }else{ + int rc = 0; + fsl_card_F const * fc; + fsl_id_t fileRid; + fsl_buffer * const buf = &f->fileContent; + unsigned char digest[16]; + fsl_md5_cx md5 = fsl_md5_cx_empty; + enum { NumBufSize = 40 }; + char numBuf[NumBufSize] = {0}; + assert(!buf->used && "Misuse of f->fileContent buffer."); + rc = fsl_deck_F_rewind(mf); + if(rc) goto end; + fsl_deck_F_sort(mf); + /* + TODO: + + Missing functionality: + + - The "wd" (working directory) family of functions, needed for + symlink handling. + */ + while(1){ + rc = fsl_deck_F_next(mf, &fc); + /* MARKER(("R rc=%s file #%d: %s %s\n", fsl_rc_cstr(rc), ++i, fc ? fc->name : "", fc ? fc->uuid : NULL)); */ + if(rc || !fc) break; + assert(fc->uuid && "We no longer iterate over deleted entries."); + if(FSL_FILE_PERM_LINK==fc->perm){ + rc = fsl_cx_err_set(f, FSL_RC_UNSUPPORTED, + "This code does not yet properly handle " + "F-cards of symlinks."); + goto end; + } + fileRid = fsl_uuid_to_rid( f, fc->uuid ); + if(0==fileRid){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Cannot resolve RID for F-card UUID [%s].", + fc->uuid); + goto end; + }else if(fileRid<0){ + assert(f->error.code); + rc = f->error.code + ? f->error.code + : fsl_cx_err_set(f, FSL_RC_ERROR, + "Error resolving RID for F-card UUID [%s].", + fc->uuid); + goto end; + } + fsl_md5_update_cstr(&md5, fc->name, -1); + rc = fsl_content_get(f, fileRid, buf); + if(rc){ + goto end; + } + numBuf[0] = 0; + fsl_snprintf(numBuf, NumBufSize, + " %"FSL_SIZE_T_PFMT"\n", + buf->used); + fsl_md5_update_cstr(&md5, numBuf, -1); + fsl_md5_update_buffer(&md5, buf); + } + if(!rc){ + fsl_md5_final(&md5, digest); + fsl_md5_digest_to_base16(digest, hex); + } + end: + fsl_cx_content_buffer_yield(f); + assert(0==buf->used); + if(rc) return rc; + fsl_deck_F_rewind(mf); + theHash = hex; + } + assert(theHash); + if(*tgt){ + memcpy(*tgt, theHash, FSL_STRLEN_MD5); + (*tgt)[FSL_STRLEN_MD5] = 0; + return 0; + }else{ + char * x = fsl_strdup(theHash); + if(x) *tgt = x; + return x ? 0 : FSL_RC_OOM; + } +} + +int fsl_deck_R_calc(fsl_deck * const mf){ + char R[FSL_STRLEN_MD5+1] = {0}; + char * r = R; + const int rc = fsl_deck_R_calc2(mf, &r); + return rc ? rc : fsl_deck_R_set(mf, r); +} + +int fsl_deck_T_add2( fsl_deck * const mf, fsl_card_T * t){ + if(!t) return FSL_RC_MISUSE; + else if(!fsl_deck_check_type(mf, 'T')){ + return mf->f->error.code; + }else if(FSL_SATYPE_CONTROL==mf->type && NULL==t->uuid){ + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "CONTROL artifacts may not have " + "self-referential tags."); + }else if(FSL_SATYPE_TECHNOTE==mf->type){ + if(NULL!=t->uuid){ + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "TECHNOTE artifacts may not have " + "tags which refer to other objects."); + }else if(FSL_TAGTYPE_ADD!=t->type){ + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "TECHNOTE artifacts may only have " + "ADD-type tags."); + } + } + if(!t->name || !*t->name){ + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "Tag name may not be empty."); + }else if(fsl_validate16(t->name, fsl_strlen(t->name))){ + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "Tag name may not be hexadecimal."); + }else if(t->uuid && !fsl_is_uuid(t->uuid)){ + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "Invalid UUID in tag."); + } + return fsl_list_append(&mf->T, t); +} + +int fsl_deck_T_add( fsl_deck * const mf, fsl_tagtype_e tagType, + char const * uuid, char const * name, + char const * value){ + if(!name) return FSL_RC_MISUSE; + else if(!fsl_deck_check_type(mf, 'T')) return mf->f->error.code; + else if(!*name || (uuid &&!fsl_is_uuid(uuid))) return FSL_RC_SYNTAX; + else switch(tagType){ + case FSL_TAGTYPE_CANCEL: + case FSL_TAGTYPE_ADD: + case FSL_TAGTYPE_PROPAGATING:{ + int rc; + fsl_card_T * t; + t = fsl_card_T_malloc(tagType, uuid, name, value); + if(!t) return FSL_RC_OOM; + rc = fsl_deck_T_add2( mf, t ); + if(rc) fsl_card_T_free(t); + return rc; + } + default: + assert(!"Invalid tagType value"); + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "Invalid tag-type value: %d", + (int)tagType); + } +} + +/** + Returns true if the NUL-terminated string contains only + "reasonable" branch name character, with the native assumption that + anything <=32d is "unreasonable" and anything >=128 is part of a + multibyte UTF8 character. +*/ +static bool fsl_is_valid_branchname(char const * z_){ + unsigned char const * z = (unsigned char const*)z_; + unsigned len = 0; + for(; z[len]; ++len){ + if(z[len] <= 32) return false; + } + return len>0; +} + +int fsl_deck_branch_set( fsl_deck * d, char const * branchName ){ + if(!fsl_is_valid_branchname(branchName)){ + return fsl_cx_err_set(d->f, FSL_RC_RANGE, "Branch name contains " + "invalid characters."); + } + int rc= fsl_deck_T_add(d, FSL_TAGTYPE_PROPAGATING, NULL, + "branch", branchName); + if(!rc){ + char * sym = fsl_mprintf("sym-%s", branchName); + if(sym){ + rc = fsl_deck_T_add(d, FSL_TAGTYPE_PROPAGATING, NULL, + sym, NULL); + fsl_free(sym); + }else{ + rc = FSL_RC_OOM; + } + } + return rc; +} + +int fsl_deck_U_set( fsl_deck * const mf, char const * v){ + return fsl_deck_set_string( mf, 'U', &mf->U, v, -1 ); +} + +int fsl_deck_W_set( fsl_deck * const mf, char const * v, fsl_int_t n){ + return fsl_deck_b_setuffer_impl(mf, v, n, 'W', &mf->W); +} + +int fsl_deck_A_set( fsl_deck * const mf, char const * name, + char const * tgt, + char const * uuidSrc ){ + int const uLen = (uuidSrc && *uuidSrc) ? fsl_is_uuid(uuidSrc) : 0; + if(!name || !tgt) return FSL_RC_MISUSE; + else if(!fsl_deck_check_type(mf, 'A')) return mf->f->error.code; + else if(!*tgt){ + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "Invalid target name in A card."); + } + /* TODO: validate tgt based on mf->type and require UUID + for types EVENT/TICKET. + */ + else if(uuidSrc && *uuidSrc && !uLen){ + return fsl_cx_err_set(mf->f, FSL_RC_SYNTAX, + "Invalid source UUID in A card."); + } + else{ + int rc = 0; + fsl_deck_free_string(mf, mf->A.tgt); + fsl_deck_free_string(mf, mf->A.src); + fsl_deck_free_string(mf, mf->A.name); + mf->A.name = mf->A.src = NULL; + if(! (mf->A.tgt = fsl_strdup(tgt))) rc = FSL_RC_OOM; + else if( !(mf->A.name = fsl_strdup(name))) rc = FSL_RC_OOM; + else if(uLen){ + mf->A.src = fsl_strndup(uuidSrc,uLen); + if(!mf->A.src) rc = FSL_RC_OOM + /* Leave mf->A.tgt/name for downstream cleanup. */; + } + return rc; + } +} + + +int fsl_deck_D_set( fsl_deck * const mf, double date){ + if(date<0) return FSL_RC_RANGE; + else if(date>0 && !fsl_deck_check_type(mf, 'D')){ + return mf->f->error.code; + }else{ + mf->D = date; + return 0; + } +} + +int fsl_deck_E_set( fsl_deck * const mf, double date, char const * uuid){ + int const uLen = uuid ? fsl_is_uuid(uuid) : 0; + if(!mf || !uLen) return FSL_RC_MISUSE; + else if(date<=0){ + return fsl_cx_err_set(mf->f, FSL_RC_RANGE, + "Invalid date value for E card."); + }else if(!uLen){ + return fsl_cx_err_set(mf->f, FSL_RC_RANGE, + "Invalid UUID for E card."); + } + else{ + mf->E.julian = date; + fsl_deck_free_string(mf, mf->E.uuid); + mf->E.uuid = fsl_strndup(uuid, uLen); + return mf->E.uuid ? 0 : FSL_RC_OOM; + } +} + +int fsl_deck_F_add( fsl_deck * const mf, char const * name, + char const * uuid, + fsl_fileperm_e perms, + char const * oldName){ + int const uLen = uuid ? fsl_is_uuid(uuid) : 0; + if(!mf || !name) return FSL_RC_MISUSE; + else if(!uuid && !mf->B.uuid){ + return fsl_cx_err_set(mf->f, FSL_RC_MISUSE, + "NULL UUID is not valid for baseline " + "manifests."); + } + else if(!fsl_deck_check_type(mf, 'F')) return mf->f->error.code; + else if(!*name){ + return fsl_cx_err_set(mf->f, FSL_RC_RANGE, + "F-card name may not be empty."); + } + else if(!fsl_is_simple_pathname(name, 1) + || (oldName && !fsl_is_simple_pathname(oldName, 1))){ + return fsl_cx_err_set(mf->f, FSL_RC_RANGE, + "Invalid filename for F-card (simple form required): " + "name=[%s], oldName=[%s].", name, oldName); + } + else if(uuid && !uLen){ + return fsl_cx_err_set(mf->f, FSL_RC_RANGE, + "Invalid UUID for F-card."); + } + else { + int rc = 0; + fsl_card_F * t; + switch(perms){ + case FSL_FILE_PERM_EXE: + case FSL_FILE_PERM_LINK: + case FSL_FILE_PERM_REGULAR: + break; + default: + assert(!"Invalid fsl_fileperm_e value"); + return fsl_cx_err_set(mf->f, FSL_RC_RANGE, + "Invalid fsl_fileperm_e value " + "(%d) for file [%s].", + perms, name); + } + t = fsl_card_F_list_push(&mf->F); + if(!t) return FSL_RC_OOM; + assert(mf->F.used>1 + ? (FSL_CARD_F_LIST_NEEDS_SORT & mf->F.flags) + : 1); + assert(!t->name); + assert(!t->uuid); + assert(!t->priorName); + assert(!t->deckOwnsStrings); + t->perm = perms; + if(0==(t->name = fsl_strdup(name))){ + rc = FSL_RC_OOM; + }else if(uuid && 0==(t->uuid = fsl_strdup(uuid))){ + rc = FSL_RC_OOM; + }else if(oldName && 0==(t->priorName = fsl_strdup(oldName))){ + rc = FSL_RC_OOM; + } + if(rc){ + fsl_card_F_list_pop(&mf->F); + } + return rc; + } +} + +int fsl_deck_F_foreach( fsl_deck * d, fsl_card_F_visitor_f cb, void * visitorState ){ + if(!cb) return FSL_RC_MISUSE; + else{ + fsl_card_F const * fc; + int rc = fsl_deck_F_rewind(d); + while( !rc && !(rc=fsl_deck_F_next(d, &fc)) && fc) { + rc = cb( fc, visitorState ); + } + return (FSL_RC_BREAK==rc) ? 0 : rc; + } +} + + +/** + Output state for fsl_output_f_mf() and friends. Used for managing + the output of a fsl_deck. + */ +struct fsl_deck_out_state { + /** + The set of cards being output. We use this to delegate certain + output bits. + */ + fsl_deck const * d; + /** + Output routine to send manifest to. + */ + fsl_output_f out; + /** + State to pass as the first arg of this->out(). + */ + void * outState; + + /** + The previously-visited card, for confirming that all cards are in + proper lexical order. + */ + fsl_card_F const * prevCard; + /** + f() result code, so that we can feed the code back through the + fsl_appendf() layer. If this is non-0, processing must stop. We + "could" use this->error.code instead, but this is simple. + */ + int rc; + /** + Counter for list-visiting routines. Must be re-set before each + visit loop if the visitor makes use of this (most do not). + */ + fsl_int_t counter; + + /** + Incrementally-calculated MD5 sum of all output sent via + fsl_output_f_mf(). + */ + fsl_md5_cx md5; + + /* Holds error state for propagating back to the client. */ + fsl_error error; + + /** + Scratch buffer for fossilizing bytes and other temporary work. + This value comes from fsl_cx_scratchpad(). + */ + fsl_buffer * scratch; +}; +typedef struct fsl_deck_out_state fsl_deck_out_state; +static const fsl_deck_out_state fsl_deck_out_state_empty = { +NULL/*d*/, +NULL/*out*/, +NULL/*outState*/, +NULL/*prevCard*/, +0/*rc*/, +0/*counter*/, +fsl_md5_cx_empty_m/*md5*/, +fsl_error_empty_m/*error*/, +NULL/*scratch*/ +}; + +/** + fsl_output_f() impl which forwards its data to arg->out(). arg + must be a (fsl_deck_out_state *). Updates arg->rc to the result of + calling arg->out(fp->fState, data, n). If arg->out() succeeds then + arg->md5 is updated to reflect the given data. i.e. this is where + the Z-card gets calculated incrementally during output of a deck. +*/ +static int fsl_output_f_mf( void * arg, void const * data, + fsl_size_t n ){ + fsl_deck_out_state * const os = (fsl_deck_out_state *)arg; + if((n>0) + && !(os->rc = os->out(os->outState, data, (fsl_size_t)n)) + && (os->md5.isInit)){ + fsl_md5_update( &os->md5, data, (fsl_size_t)n ); + } + return os->rc; +} + +/** + Internal helper for fsl_deck_output(). Appends formatted output to + os->out() via fsl_output_f_mf(). Returns os->rc (0 on success). + */ +static int fsl_deck_append( fsl_deck_out_state * os, + char const * fmt, ... ){ + fsl_int_t rc; + va_list args; + assert(os); + assert(fmt && *fmt); + va_start(args,fmt); + rc = fsl_appendfv( fsl_output_f_mf, os, fmt, args); + va_end(args); + if(rc<0 && !os->rc) os->rc = FSL_RC_IO; + return os->rc; +} + +/** + Fossilizes (inp, inp+len] bytes to os->scratch, + overwriting any existing contents. + Updates and returns os->rc. + */ +static int fsl_deck_fossilize( fsl_deck_out_state * os, + unsigned char const * inp, + fsl_int_t len){ + fsl_buffer_reuse(os->scratch); + return os->rc = len + ? fsl_bytes_fossilize(inp, len, os->scratch) + : 0; +} + +/** Confirms that the given card letter is valid for od->d->type, and + updates os->rc and os->error if it's not. Returns true if it's + valid. +*/ +static bool fsl_deck_out_tcheck(fsl_deck_out_state * os, char letter){ + if(!fsl_card_is_legal(os->d->type, letter)){ + os->rc = fsl_error_set(&os->error, FSL_RC_TYPE, + "%c-card is not valid for deck type %s.", + letter, fsl_satype_cstr(os->d->type)); + } + return os->rc ? false : true; +} + +/* Appends a UUID-valued card to os from os->d->{{card}} if the given + UUID is not NULL, else this is a no-op. */ +static int fsl_deck_out_uuid( fsl_deck_out_state * os, char card, fsl_uuid_str uuid ){ + if(uuid && fsl_deck_out_tcheck(os, card)){ + if(!fsl_is_uuid(uuid)){ + os->rc = fsl_error_set(&os->error, FSL_RC_RANGE, + "Malformed UUID in %c card.", card); + }else{ + fsl_deck_append(os, "%c %s\n", card, uuid); + } + } + return os->rc; +} + +/* Appends the B card to os from os->d->B. */ +static int fsl_deck_out_B( fsl_deck_out_state * os ){ + return fsl_deck_out_uuid(os, 'B', os->d->B.uuid); +} + + +/* Appends the A card to os from os->d->A. */ +static int fsl_deck_out_A( fsl_deck_out_state * os ){ + if(os->d->A.name && fsl_deck_out_tcheck(os, 'A')){ + if(!os->d->A.name || !*os->d->A.name){ + os->rc = fsl_error_set(&os->error, FSL_RC_SYNTAX, + "A-card is missing its name property"); + + }else if(!os->d->A.tgt || !*os->d->A.tgt){ + os->rc = fsl_error_set(&os->error, FSL_RC_SYNTAX, + "A-card is missing its tgt property: %s", + os->d->A.name); + + }else if(os->d->A.src && !fsl_is_uuid(os->d->A.src)){ + os->rc = fsl_error_set(&os->error, FSL_RC_TYPE, + "Invalid src UUID in A-card: name=%s, " + "invalid uuid=%s", + os->d->A.name, os->d->A.src); + }else{ + fsl_deck_append(os, "A %F %F", + os->d->A.name, os->d->A.tgt); + if(!os->rc){ + if(os->d->A.src){ + fsl_deck_append(os, " %s", os->d->A.src); + } + if(!os->rc) fsl_deck_append(os, "\n"); + } + } + } + return os->rc; +} + +/** + Internal helper for outputing cards which are simple strings. + str is the card to output (NULL values are ignored), letter is + the card letter being output. If doFossilize is true then + the output gets fossilize-formatted. + */ +static int fsl_deck_out_letter_str( fsl_deck_out_state * os, char letter, + char const * str, char doFossilize ){ + if(str && fsl_deck_out_tcheck(os, letter)){ + if(doFossilize){ + fsl_deck_fossilize(os, (unsigned char const *)str, -1); + if(!os->rc){ + fsl_deck_append(os, "%c %b\n", letter, os->scratch); + } + }else{ + fsl_deck_append(os, "%c %s\n", letter, str); + } + } + return os->rc; +} + +/* Appends the C card to os from os->d->C. */ +static int fsl_deck_out_C( fsl_deck_out_state * os ){ + return fsl_deck_out_letter_str( os, 'C', os->d->C, 1 ); +} + + +/* Appends the D card to os from os->d->D. */ +static int fsl_deck_out_D( fsl_deck_out_state * os ){ + if((os->d->D > 0.0) && fsl_deck_out_tcheck(os, 'D')){ + char ds[24]; + if(!fsl_julian_to_iso8601(os->d->D, ds, 1)){ + os->rc = fsl_error_set(&os->error, FSL_RC_RANGE, + "D-card contains invalid " + "Julian Day value."); + }else{ + fsl_deck_append(os, "D %s\n", ds); + } + } + return os->rc; +} + +/* Appends the E card to os from os->d->E. */ +static int fsl_deck_out_E( fsl_deck_out_state * os ){ + if(os->d->E.uuid && fsl_deck_out_tcheck(os, 'E')){ + char ds[24]; + char msPrecision = FSL_SATYPE_EVENT!=os->d->type + /* The timestamps on Events historically have seconds precision, + not ms. + */; + if(!fsl_is_uuid(os->d->E.uuid)){ + os->rc = fsl_error_set(&os->error, FSL_RC_TYPE, + "Invalid UUID in E-card: %s", + os->d->E.uuid); + } + else if(!fsl_julian_to_iso8601(os->d->E.julian, ds, msPrecision)){ + os->rc = fsl_error_set(&os->error, FSL_RC_TYPE, + "Invalid Julian Day value in E-card."); + } + else{ + fsl_deck_append(os, "E %s %s\n", ds, os->d->E.uuid); + } + } + return os->rc; +} + +/* Appends the G card to os from os->d->G. */ +static int fsl_deck_out_G( fsl_deck_out_state * os ){ + return fsl_deck_out_uuid(os, 'G', os->d->G); +} + +/* Appends the H card to os from os->d->H. */ +static int fsl_deck_out_H( fsl_deck_out_state * os ){ + if(os->d->H && os->d->I){ + return os->rc = fsl_error_set(&os->error, FSL_RC_SYNTAX, + "Forum post may not have both H- and I-cards."); + } + return fsl_deck_out_letter_str( os, 'H', os->d->H, 1 ); +} + +/* Appends the I card to os from os->d->I. */ +static int fsl_deck_out_I( fsl_deck_out_state * os ){ + if(os->d->I && os->d->H){ + return os->rc = fsl_error_set(&os->error, FSL_RC_SYNTAX, + "Forum post may not have both H- and I-cards."); + } + return fsl_deck_out_uuid(os, 'I', os->d->I); +} + + +static int fsl_deck_out_F_one(fsl_deck_out_state *os, + fsl_card_F const * f){ + int rc; + char hasOldName; + char const * zPerm; + assert(f); + if(os->prevCard){ + int const cmp = fsl_strcmp(os->prevCard->name, f->name); + if(0==cmp){ + return fsl_error_set(&os->error, FSL_RC_RANGE, + "Duplicate F-card name: %s", + f->name); + }else if(cmp>0){ + return fsl_error_set(&os->error, FSL_RC_RANGE, + "Out-of-order F-card names: %s before %s", + os->prevCard->name, f->name); + } + } + if(!fsl_is_simple_pathname(f->name, true)){ + return fsl_error_set(&os->error, FSL_RC_RANGE, + "Filename is invalid as F-card: %s", + f->name); + } + if(!f->uuid && !os->d->B.uuid){ + return fsl_error_set(&os->error, FSL_RC_MISUSE, + "Baseline manifests may not have F-cards " + "without UUIDs (file deletion entries). To " + "delete files, simply do not inject an F-card " + "for them. Delta manifests, however, require " + "NULL UUIDs for deletion entries! File: %s", + f->name); + } + + rc = fsl_deck_fossilize(os, (unsigned char const *)f->name, -1); + if(!rc) rc = fsl_deck_append(os, "F %b", os->scratch); + if(!rc && f->uuid){ + assert(fsl_is_uuid(f->uuid)); + rc = fsl_deck_append( os, " %s", f->uuid); + if(rc) return rc; + } + if(f->uuid){ + hasOldName = f->priorName && (0!=fsl_strcmp(f->name,f->priorName)); + switch(f->perm){ + case FSL_FILE_PERM_EXE: zPerm = " x"; break; + case FSL_FILE_PERM_LINK: zPerm = " l"; break; + default: + /* When hasOldName, we have to inject an otherwise optional + 'w' to avoid an ambiguity. Or at least that's what the + fossil F-card-generating code does. + */ + zPerm = hasOldName ? " w" : ""; break; + } + if(*zPerm) rc = fsl_deck_append( os, "%s", zPerm); + if(!rc && hasOldName){ + assert(*zPerm); + rc = fsl_deck_fossilize(os, (unsigned char const *)f->priorName, -1); + if(!rc) rc = fsl_deck_append( os, " %b", os->scratch); + } + } + if(!rc) fsl_output_f_mf(os, "\n", 1); + return os->rc; +} + +static int fsl_deck_out_list_obj( fsl_deck_out_state * os, + char letter, + fsl_list const * li, + fsl_list_visitor_f visitor){ + if(li->used && fsl_deck_out_tcheck(os, letter)){ + os->rc = fsl_list_visit( li, 0, visitor, os ); + } + return os->rc; +} + +static int fsl_deck_out_F( fsl_deck_out_state * os ){ + if(os->d->F.used && fsl_deck_out_tcheck(os, 'F')){ + uint32_t i; + for(i=0; !os->rc && i d->F.used; ++i){ + os->rc = fsl_deck_out_F_one(os, F_at(&os->d->F, i)); + } + } + return os->rc; +} + + +/** + A comparison routine for qsort(3) which compares fsl_card_J + instances in a lexical manner based on their names. The order is + important for card ordering in generated manifests. + */ +int fsl_qsort_cmp_J_cards( void const * lhs, void const * rhs ){ + fsl_card_J const * l = *((fsl_card_J const **)lhs); + fsl_card_J const * r = *((fsl_card_J const **)rhs); + /* Compare NULL as larger so that NULLs move to the right. That said, + we aren't expecting any NULLs. */ + assert(l); + assert(r); + if(!l) return r ? 1 : 0; + else if(!r) return -1; + else{ + /* The '+' sorts before any legal field name bits (letters). */ + if(l->append != r->append) return r->append - l->append + /* Footnote: that will break if, e.g. l->isAppend==2 and + r->isAppend=1, or some such. Shame C89 doesn't have a true + boolean. + */; + else return fsl_strcmp(l->field, r->field); + } +} + +/** + fsl_list_visitor_f() impl for outputing J cards. obj must + be a (fsl_card_J *). + */ +static int fsl_list_v_mf_output_card_J(void * obj, void * visitorState ){ + fsl_deck_out_state * os = (fsl_deck_out_state *)visitorState; + fsl_card_J const * c = (fsl_card_J const *)obj; + fsl_deck_fossilize( os, (unsigned char const *)c->field, -1 ); + if(!os->rc){ + fsl_deck_append(os, "J %s%b", c->append ? "+" : "", os->scratch); + if(!os->rc){ + if(c->value && *c->value){ + fsl_deck_fossilize( os, (unsigned char const *)c->value, -1 ); + if(!os->rc){ + fsl_deck_append(os, " %b\n", os->scratch); + } + }else{ + fsl_deck_append(os, "\n"); + } + } + } + return os->rc; +} + +static int fsl_deck_out_J( fsl_deck_out_state * os ){ + return fsl_deck_out_list_obj(os, 'J', &os->d->J, + fsl_list_v_mf_output_card_J); +} + +/* Appends the K card to os from os->d->K. */ +static int fsl_deck_out_K( fsl_deck_out_state * os ){ + if(os->d->K && fsl_deck_out_tcheck(os, 'K')){ + if(!fsl_is_uuid(os->d->K)){ + os->rc = fsl_error_set(&os->error, FSL_RC_RANGE, + "Invalid UUID in K card."); + } + else{ + fsl_deck_append(os, "K %s\n", os->d->K); + } + } + return os->rc; +} + + +/* Appends the L card to os from os->d->L. */ +static int fsl_deck_out_L( fsl_deck_out_state * os ){ + return fsl_deck_out_letter_str(os, 'L', os->d->L, 1); +} + +/* Appends the N card to os from os->d->N. */ +static int fsl_deck_out_N( fsl_deck_out_state * os ){ + return fsl_deck_out_letter_str( os, 'N', os->d->N, 1 ); +} + +/** + fsl_list_visitor_f() impl for outputing P cards. obj must + be a (fsl_deck_out_state *) and obj->counter must be + set to 0 before running the visit iteration. + */ +static int fsl_list_v_mf_output_card_P(void * obj, void * visitorState ){ + fsl_deck_out_state * os = (fsl_deck_out_state *)visitorState; + char const * uuid = (char const *)obj; + int const uLen = uuid ? fsl_is_uuid(uuid) : 0; + if(!uLen){ + os->rc = fsl_error_set(&os->error, FSL_RC_RANGE, + "Invalid UUID in P card."); + } + else if(!os->counter++) fsl_output_f_mf(os, "P ", 2); + else fsl_output_f_mf(os, " ", 1); + /* Reminder: fsl_appendf_f_mf() updates os->rc. */ + if(!os->rc){ + fsl_output_f_mf(os, uuid, (fsl_size_t)uLen); + } + return os->rc; +} + + +static int fsl_deck_out_P( fsl_deck_out_state * os ){ + if(!fsl_deck_out_tcheck(os, 'P')) return os->rc; + else if(os->d->P.used){ + os->counter = 0; + os->rc = fsl_list_visit( &os->d->P, 0, fsl_list_v_mf_output_card_P, os ); + assert(os->counter); + if(!os->rc) fsl_output_f_mf(os, "\n", 1); + } +#if 1 + /* Arguable: empty P-cards are harmless but cosmetically unsightly. */ + else if(FSL_SATYPE_CHECKIN==os->d->type){ + /* + Evil ugly hack, primarily for round-trip compatibility with + manifest #1, which has an empty P card. + + fossil(1) ignores empty P-cards in all cases, and must continue + to do so for backwards compatibility with rid #1 in all repos. + + Pedantic note: there must be no space between the 'P' and the + newline. + */ + fsl_deck_append(os, "P\n"); + } +#endif + return os->rc; +} + +/** + A comparison routine for qsort(3) which compares fsl_card_Q + instances in a lexical manner. The order is important for card + ordering in generated manifests. +*/ +static int qsort_cmp_Q_cards( void const * lhs, void const * rhs ){ + fsl_card_Q const * l = *((fsl_card_Q const **)lhs); + fsl_card_Q const * r = *((fsl_card_Q const **)rhs); + /* Compare NULL as larger so that NULLs move to the right. That said, + we aren't expecting any NULLs. */ + assert(l); + assert(r); + if(!l) return r ? 1 : 0; + else if(!r) return -1; + else{ + /* Lexical sorting must account for the +/- characters, and a '+' + sorts before '-', which is why this next part may seem + backwards at first. + */ + assert(l->type); + assert(r->type); + if(l->type<0 && r->type>0) return 1; + else if(l->type>0 && r->type<0) return -1; + else return fsl_strcmp(l->target, r->target); + } +} + +/** + fsl_list_visitor_f() impl for outputing Q cards. obj must + be a (fsl_deck_out_state *). +*/ +static int fsl_list_v_mf_output_card_Q(void * obj, void * visitorState ){ + fsl_deck_out_state * os = (fsl_deck_out_state *)visitorState; + fsl_card_Q const * cp = (fsl_card_Q const *)obj; + char const prefix = (cp->type==FSL_CHERRYPICK_ADD) + ? '+' : '-'; + assert(cp->type); + assert(cp->target); + if(cp->type != FSL_CHERRYPICK_ADD && + cp->type != FSL_CHERRYPICK_BACKOUT){ + os->rc = fsl_error_set(&os->error, FSL_RC_RANGE, + "Invalid type value in Q-card."); + }else if(!fsl_card_is_legal(os->d->type, 'Q')){ + os->rc = fsl_error_set(&os->error, FSL_RC_TYPE, + "Q-card is not valid for deck type %s", + fsl_satype_cstr(os->d->type)); + }else if(!fsl_is_uuid(cp->target)){ + os->rc = fsl_error_set(&os->error, FSL_RC_RANGE, + "Invalid target UUID in Q-card: %s", + cp->target); + }else if(cp->baseline){ + if(!fsl_is_uuid(cp->baseline)){ + os->rc = fsl_error_set(&os->error, FSL_RC_RANGE, + "Invalid baseline UUID in Q-card: %s", + cp->baseline); + }else{ + fsl_deck_append(os, "Q %c%s %s\n", prefix, cp->target, cp->baseline); + } + }else{ + fsl_deck_append(os, "Q %c%s\n", prefix, cp->target); + } + return os->rc; +} + +static int fsl_deck_out_Q( fsl_deck_out_state * os ){ + return fsl_deck_out_list_obj(os, 'Q', &os->d->Q, + fsl_list_v_mf_output_card_Q); +} + +/** + Appends the R card from os->d->R to os. + */ +static int fsl_deck_out_R( fsl_deck_out_state * os ){ + if(os->d->R && fsl_deck_out_tcheck(os, 'R')){ + if((FSL_STRLEN_MD5!=fsl_strlen(os->d->R)) + || !fsl_validate16(os->d->R, FSL_STRLEN_MD5)){ + os->rc = fsl_error_set(&os->error, FSL_RC_RANGE, + "Malformed MD5 in R-card."); + } + else{ + fsl_deck_append(os, "R %s\n", os->d->R); + } + } + return os->rc; +} + +/** + fsl_list_visitor_f() impl for outputing T cards. obj must + be a (fsl_deck_out_state *). + */ +static int fsl_list_v_mf_output_card_T(void * obj, void * visitorState ){ + fsl_deck_out_state * os = (fsl_deck_out_state *)visitorState; + fsl_card_T * t = (fsl_card_T *)obj; + char prefix = 0; + switch(os->d->type){ + case FSL_SATYPE_TECHNOTE: + if( t->uuid ){ + return os->rc = fsl_error_set(&os->error, FSL_RC_SYNTAX, + "Non-self-referential T-card is not " + "permitted in a technote."); + }else if(FSL_TAGTYPE_ADD!=t->type){ + return os->rc = fsl_error_set(&os->error, FSL_RC_SYNTAX, + "Non-ADD T-card is not permitted " + "in a technote."); + } + break; + case FSL_SATYPE_CONTROL: + if( !t->uuid ){ + return os->rc = fsl_error_set(&os->error, FSL_RC_SYNTAX, + "Self-referential T-card is not " + "permitted in a control artifact."); + } + break; + default: + break; + } + /* Determine the prefix character... */ + switch(t->type){ + case FSL_TAGTYPE_CANCEL: prefix = '-'; break; + case FSL_TAGTYPE_ADD: prefix = '+'; break; + case FSL_TAGTYPE_PROPAGATING: prefix = '*'; break; + default: + return os->rc = fsl_error_set(&os->error, FSL_RC_TYPE, + "Invalid tag type #%d in T-card.", + t->type); + } + if(!t->name || !*t->name){ + return os->rc = fsl_error_set(&os->error, FSL_RC_SYNTAX, + "T-card name may not be empty."); + }else if(fsl_validate16(t->name, fsl_strlen(t->name))){ + return os->rc = fsl_error_set(&os->error, FSL_RC_SYNTAX, + "T-card name may not be hexadecimal."); + }else if(t->uuid && !fsl_is_uuid(t->uuid)){ + return os->rc = fsl_error_set(&os->error, FSL_RC_SYNTAX, + "Malformed UUID in T-card: %s", + t->uuid); + } + /* + Fossilize and output the prefix, name, and uuid, or a '*' if no + uuid is set (which is only legal when tagging the current + artifact, as '*' is a placeholder for the current artifact's + UUID, which is not yet known). + */ + fsl_buffer_reuse(os->scratch); + fsl_deck_fossilize(os, (unsigned const char *)t->name, -1); + if(os->rc) return os->rc; + os->rc = fsl_deck_append(os, "T %c%s %s", prefix, + (char const*)os->scratch->mem, + t->uuid ? t->uuid : "*"); + if(os->rc) return os->rc; + if(/*(t->type != FSL_TAGTYPE_CANCEL) &&*/t->value && *t->value){ + /* CANCEL tags historically don't store a value but + the spec doesn't disallow it and they are harmless + for (aren't used by) fossil(1). */ + fsl_deck_fossilize(os, (unsigned char const *)t->value, -1); + if(!os->rc) fsl_output_f_mf(os, " ", 1); + if(!os->rc) fsl_output_f_mf(os, (char const*)os->scratch->mem, + (fsl_int_t)os->scratch->used); + } + if(!os->rc){ + fsl_output_f_mf(os, "\n", 1); + } + return os->rc; +} + + +char fsl_tag_prefix_char( fsl_tagtype_e t ){ + switch(t){ + case FSL_TAGTYPE_CANCEL: return '-'; + case FSL_TAGTYPE_ADD: return '+'; + case FSL_TAGTYPE_PROPAGATING: return '*'; + default: + return 0; + } +} + +/** + A comparison routine for qsort(3) which compares fsl_card_T + instances in a lexical manner based on (type, name, uuid, value). + The order of those is important for card ordering in generated + manifests. Interestingly, CANCEL tags (with a '-' prefix) sort + last, meaning it is possible to cancel a tag set in the same + manifest because crosslinking processes them in the order given + (which will be lexical order for all legal manifests). + + Reminder: lhs and rhs must be (fsl_card_T**), as we use this to + qsort() such lists. When using it to compare two tags, make sure + to pass ptr-to-ptr. +*/ +static int fsl_card_T_cmp( void const * lhs, void const * rhs ){ + fsl_card_T const * l = *((fsl_card_T const **)lhs); + fsl_card_T const * r = *((fsl_card_T const **)rhs); + /* Compare NULL as larger so that NULLs move to the right. That said, + we aren't expecting any NULLs. */ + assert(l); + assert(r); + if(!l) return r ? 1 : 0; + else if(!r) return -1; + else if(l->type != r->type){ + char const lc = fsl_tag_prefix_char(l->type); + char const rc = fsl_tag_prefix_char(r->type); + return (lcname, r->name); + if(rc) return rc; + else { + rc = fsl_uuidcmp(l->uuid, r->uuid); + return rc + ? rc + : fsl_strcmp(l->value, r->value); + } + } +} + +/** + Confirms that any T-cards in d are properly sorted. If not, + returns non-0. If err is not NULL, it is updated with a + description of the problem. + + Possibly fixme one day: this code permits that the same tag/target + combination may be added or removed, or added as a normal and + propagating tag, in the same deck. Though that's not technically + disallowed, we "should" disallow it. That requires a more thorough + scan of the cards, though. +*/ +static int fsl_deck_T_verify_order( fsl_deck const * d, fsl_error * err ){ + if(d->T.used<2) return 0; + else{ + fsl_size_t i = 0, j; + int rc = 0; + fsl_card_T const * tag; + fsl_card_T const * prev = NULL; + for( i = 0; i < d->T.used; ++i, prev = tag, rc = 0){ + tag = (fsl_card_T const *)d->T.list[i]; + if(prev){ + if( (rc = fsl_card_T_cmp(&prev, &tag)) >= 0 ){ + if(!err) rc = FSL_RC_SYNTAX; + else{ + rc = rc + ? fsl_error_set(err, FSL_RC_SYNTAX, + "Invalid T-card order: " + "[%c%s] must precede [%c%s]", + fsl_tag_prefix_char(prev->type), + prev->name, + fsl_tag_prefix_char(tag->type), + tag->name) + : fsl_error_set(err, FSL_RC_SYNTAX, + "Duplicate T-card: %c%s", + fsl_tag_prefix_char(prev->type), + prev->name) + ; + } + break; + } + } + } + /** + And now, for bonus points: disallow the same tag name/artifact + combination appearing twice in the deck. Though that's not + explicitly disallowed by the fossil specs, we "should" disallow + it. That requires a more thorough scan of the cards, though. + + This logic is NOT in fossil, and though we don't have any such + tags in the fossil repo, we may have to disable this for + compatibility's sake. OTOH, we only check this when outputing + manifests, and we never (aside from testing) have to output + manifests which were generated by fossil. Thus... this only + triggers (except for some tests) on manifests generated by + libfossil, so we can justify having it. + */ + for( i=0; !rc && i < d->T.used; ++i ){ + fsl_card_T const * t1 = (fsl_card_T const *)d->T.list[i]; + for( j = 0; j < d->T.used; ++j ){ + if(i==j) continue; + fsl_card_T const * t2 = (fsl_card_T const *)d->T.list[j]; + if(0==fsl_strcmp(t1->name, t2->name) + && ((!t1->uuid && !t2->uuid) + || 0==fsl_strcmp(t1->uuid, t2->uuid))){ + rc = fsl_error_set(err, FSL_RC_SYNTAX, + "An artifact may not contain the same " + "T-card name and target artifact " + "multiple times: " + "name=%s target=%s", + t1->name, t1->uuid ? t1->uuid : "*"); + break; + } + } + } + return rc; + } +} + +/* Appends the T cards to os from os->d->T. */ +static int fsl_deck_out_T( fsl_deck_out_state * os ){ + os->rc = fsl_deck_T_verify_order( os->d, &os->error); + return os->rc + ? os->rc + : fsl_deck_out_list_obj(os, 'T', &os->d->T, + fsl_list_v_mf_output_card_T); +} + +/* Appends the U card to os from os->d->U. */ +static int fsl_deck_out_U( fsl_deck_out_state * os ){ + return fsl_deck_out_letter_str(os, 'U', os->d->U, 1); +} + +/* Appends the W card to os from os->d->W. */ +static int fsl_deck_out_W( fsl_deck_out_state * os ){ + if(os->d->W.used && fsl_deck_out_tcheck(os, 'W')){ + fsl_deck_append(os, "W %"FSL_SIZE_T_PFMT"\n%b\n", + (fsl_size_t)os->d->W.used, + &os->d->W ); + } + return os->rc; +} + + +/* Appends the Z card to os from os' accummulated md5 hash. */ +static int fsl_deck_out_Z( fsl_deck_out_state * os ){ + unsigned char digest[16]; + char md5[FSL_STRLEN_MD5+1]; + fsl_md5_final(&os->md5, digest); + fsl_md5_digest_to_base16(digest, md5); + assert(!md5[32]); + os->md5.isInit = 0 /* Keep further output from updating the MD5 */; + return fsl_deck_append(os, "Z %.*s\n", FSL_STRLEN_MD5, md5); +} + +static int qsort_cmp_strings( void const * lhs, void const * rhs ){ + char const * l = *((char const **)lhs); + char const * r = *((char const **)rhs); + return fsl_strcmp(l,r); +} + +static int fsl_list_v_mf_output_card_M(void * obj, void * visitorState ){ + fsl_deck_out_state * os = (fsl_deck_out_state *)visitorState; + char const * m = (char const *)obj; + return fsl_deck_append(os, "M %s\n", m); +} + +static int fsl_deck_output_cluster( fsl_deck_out_state * os ){ + if(!os->d->M.used){ + os->rc = fsl_error_set(&os->error, FSL_RC_RANGE, + "M-card list may not be empty."); + }else{ + fsl_deck_out_list_obj(os, 'M', &os->d->M, + fsl_list_v_mf_output_card_M); + } + return os->rc; +} + + +/* Helper for fsl_deck_output_CATYPE() */ +#define DOUT(LETTER) rc = fsl_deck_out_##LETTER(os); \ + if(rc || os->rc) return os->rc ? os->rc : rc + +static int fsl_deck_output_control( fsl_deck_out_state * os ){ + int rc; + /* Reminder: cards must be output in strict lexical order. */ + DOUT(D); + DOUT(T); + DOUT(U); + return os->rc; +} + +static int fsl_deck_output_event( fsl_deck_out_state * os ){ + int rc = 0; + /* Reminder: cards must be output in strict lexical order. */ + DOUT(C); + DOUT(D); + DOUT(E); + DOUT(N); + DOUT(P); + DOUT(T); + DOUT(U); + DOUT(W); + return os->rc; +} + +static int fsl_deck_output_mf( fsl_deck_out_state * os ){ + int rc = 0; + /* Reminder: cards must be output in strict lexical order. */ + DOUT(B); + DOUT(C); + DOUT(D); + DOUT(F); + DOUT(K); + DOUT(L); + DOUT(N); + DOUT(P); + DOUT(Q); + DOUT(R); + DOUT(T); + DOUT(U); + DOUT(W); + return os->rc; +} + + +static int fsl_deck_output_ticket( fsl_deck_out_state * os ){ + int rc; + /* Reminder: cards must be output in strict lexical order. */ + DOUT(D); + DOUT(J); + DOUT(K); + DOUT(U); + return os->rc; +} + +static int fsl_deck_output_wiki( fsl_deck_out_state * os ){ + int rc; + /* Reminder: cards must be output in strict lexical order. */ + DOUT(C); + DOUT(D); + DOUT(L); + DOUT(N); + DOUT(P); + DOUT(U); + DOUT(W); + return os->rc; +} + +static int fsl_deck_output_attachment( fsl_deck_out_state * os ){ + int rc = 0; + /* Reminder: cards must be output in strict lexical order. */ + DOUT(A); + DOUT(C); + DOUT(D); + DOUT(N); + DOUT(U); + return os->rc; +} + +static int fsl_deck_output_forumpost( fsl_deck_out_state * os ){ + int rc; + /* Reminder: cards must be output in strict lexical order. */ + DOUT(D); + DOUT(G); + DOUT(H); + DOUT(I); + DOUT(N); + DOUT(P); + DOUT(U); + DOUT(W); + return os->rc; +} + +/** + Only for testing/debugging purposes, as it allows constructs which + are not semantically legal and are CERTAINLY not legal to stuff in + the database. + */ +static int fsl_deck_output_any( fsl_deck_out_state * os ){ + int rc = 0; + /* Reminder: cards must be output in strict lexical order. */ + DOUT(B); + DOUT(C); + DOUT(D); + DOUT(E); + DOUT(F); + DOUT(J); + DOUT(K); + DOUT(L); + DOUT(N); + DOUT(P); + DOUT(Q); + DOUT(R); + DOUT(T); + DOUT(U); + DOUT(W); + return os->rc; +} + +#undef DOUT + + +int fsl_deck_unshuffle( fsl_deck * d, bool calculateRCard ){ + fsl_list * li; + int rc = 0; + if(!d || !d->f) return FSL_RC_MISUSE; + fsl_cx_err_reset(d->f); +#define SORT(CARD,CMP) li = &d->CARD; fsl_list_sort(li, CMP) + SORT(J,fsl_qsort_cmp_J_cards); + SORT(M,qsort_cmp_strings); + SORT(Q,qsort_cmp_Q_cards); + SORT(T,fsl_card_T_cmp); +#undef SORT + if(FSL_SATYPE_CHECKIN!=d->type){ + assert(!fsl_card_is_legal(d->type,'R')); + assert(!fsl_card_is_legal(d->type,'F')); + }else{ + assert(fsl_card_is_legal(d->type, 'R') && "in-lib unit testing"); + if(calculateRCard){ + rc = fsl_deck_R_calc(d) /* F-card list is sorted there */; + }else{ + fsl_deck_F_sort(d); + if(!d->R){ + rc = fsl_deck_R_set(d, + (d->F.used || d->B.uuid || d->P.used) + ? NULL + : FSL_MD5_INITIAL_HASH) + /* Special case: for manifests with no (B,F,P)-cards we inject + the initial-state R-card, analog to the initial checkin + (RID 1). We need one of (B,F,P,R) to unambiguously identify + a MANIFEST from a CONTROL, but RID 1 has an empty P-card, + no F-cards, and no B-card, so it _needs_ an R-card in order + to be unambiguously a Manifest. That said, that ambiguity + is/would be harmless in practice because CONTROLs go + through most of the same crosslinking processes as + MANIFESTs (the ones which are important for this purpose, + anyway). + */; + } + } + } + return rc; +} + +int fsl_deck_output( fsl_deck * d, fsl_output_f out, void * outputState ){ + static const bool allowTypeAny = false + /* Only enable for debugging/testing. Allows outputing decks of + type FSL_SATYPE_ANY, which bypasses some validation checks and + may trigger other validation assertions. And may allow you to + inject garbage into the repo. So be careful. + */; + + fsl_deck_out_state OS = fsl_deck_out_state_empty; + fsl_deck_out_state * const os = &OS; + fsl_cx * const f = d->f; + int rc = 0; + if(!f || !out) return FSL_RC_MISUSE; + else if(FSL_SATYPE_ANY==d->type){ + if(!allowTypeAny){ + return fsl_cx_err_set(d->f, FSL_RC_TYPE, + "Artifact type ANY cannot be" + "output unless it is enabled in this " + "code (it's dangerous)."); + } + /* fall through ... */ + } + rc = fsl_deck_unshuffle(d, + (FSL_CX_F_CALC_R_CARD & f->flags) + ? ((d->F.used && !d->R) ? 1 : 0) + : 0); + /* ^^^^ unshuffling might install an R-card, so we have to + do that before checking whether all required cards are + set... */ + if(rc) return rc; + else if(!fsl_deck_has_required_cards(d)){ + return FSL_RC_SYNTAX; + } + + os->d = d; + os->out = out; + os->outState = outputState; + os->scratch = fsl_cx_scratchpad(f); + switch(d->type){ + case FSL_SATYPE_CLUSTER: + rc = fsl_deck_output_cluster(os); + break; + case FSL_SATYPE_CONTROL: + rc = fsl_deck_output_control(os); + break; + case FSL_SATYPE_EVENT: + rc = fsl_deck_output_event(os); + break; + case FSL_SATYPE_CHECKIN: + rc = fsl_deck_output_mf(os); + break; + case FSL_SATYPE_TICKET: + rc = fsl_deck_output_ticket(os); + break; + case FSL_SATYPE_WIKI: + rc = fsl_deck_output_wiki(os); + break; + case FSL_SATYPE_ANY: + assert(allowTypeAny); + rc = fsl_deck_output_any(os); + break; + case FSL_SATYPE_ATTACHMENT: + rc = fsl_deck_output_attachment(os); + break; + case FSL_SATYPE_FORUMPOST: + rc = fsl_deck_output_forumpost(os); + break; + default: + rc = fsl_cx_err_set(f, FSL_RC_TYPE, + "Invalid/unhandled deck type (#%d).", + d->type); + goto end; + } + if(!rc){ + rc = fsl_deck_out_Z( os ); + } + end: + fsl_cx_scratchpad_yield(f, os->scratch); + if(os->rc && os->error.code){ + fsl_error_move(&os->error, &f->error); + } + fsl_error_clear(&os->error); + return os->rc ? os->rc : rc; +} + +/* Timestamps might be adjusted slightly to ensure that checkins appear + on the timeline in chronological order. This is the maximum amount + of the adjustment window, in days. +*/ +#define AGE_FUDGE_WINDOW (2.0/86400.0) /* 2 seconds */ + +/* This is increment (in days) by which timestamps are adjusted for + use on the timeline. +*/ +#define AGE_ADJUST_INCREMENT (25.0/86400000.0) /* 25 milliseconds */ + +/** + Adds a record in the pending_xlink temp table, to be processed + when crosslinking is completed. Returns 0 on success, non-0 for + db error. +*/ +static int fsl_deck_crosslink_add_pending(fsl_cx * f, char cType, fsl_uuid_cstr uuid){ + int rc = 0; + assert(f->cache.isCrosslinking); + rc = fsl_db_exec(f->dbMain, + "INSERT OR IGNORE INTO pending_xlink VALUES('%c%q')", + cType, uuid); + return fsl_cx_uplift_db_error2(f, 0, rc); +} + + +/** @internal + + Add a single entry to the mlink table. Also add the filename to + the filename table if it is not there already. + + Parameters: + + pmid: Record for parent manifest. Use 0 to indicate no parent. + + zFromUuid: UUID for the content in parent (the new ==mlink.pid). 0 + or "" to add file. + + mid: The record ID of the manifest + + zToUuid:UUID for the mlink.fid. "" to delete + + zFilename: Filename + + zPrior: Previous filename. NULL if unchanged + + isPublic:True if mid is not a private manifest + + isPrimary: true if pmid is the primary parent of mid. + + mperm: permissions + */ +static +int fsl_mlink_add_one( fsl_cx * f, + fsl_id_t pmid, fsl_uuid_cstr zFromUuid, + fsl_id_t mid, fsl_uuid_cstr zToUuid, + char const * zFilename, + char const * zPrior, + bool isPublic, + bool isPrimary, + fsl_fileperm_e mperm){ + fsl_id_t fnid, pfnid, pid, fid; + fsl_db * db = fsl_cx_db_repo(f); + fsl_stmt * s1 = NULL; + int rc; + bool doInsert = false; + assert(f); + assert(db); + assert(db->beginCount>0); + //MARKER(("%s() pmid=%d mid=%d\n", __func__, (int)pmid, (int)mid)); + + rc = fsl_repo_filename_fnid2(f, zFilename, &fnid, 1); + if(rc) return rc; + if( zPrior && *zPrior ){ + rc = fsl_repo_filename_fnid2(f, zPrior, &pfnid, 1); + if(rc) return rc; + }else{ + pfnid = 0; + } + if( zFromUuid && *zFromUuid ){ + pid = fsl_uuid_to_rid2(f, zFromUuid, FSL_PHANTOM_PUBLIC); + if(pid<0){ + assert(f->error.code); + return f->error.code; + } + assert(pid>0); + }else{ + pid = 0; + } + + if( zToUuid && *zToUuid ){ + fid = fsl_uuid_to_rid2(f, zToUuid, FSL_PHANTOM_PUBLIC); + if(fid<0){ + assert(f->error.code); + return f->error.code; + }else if( isPublic ){ + rc = fsl_content_make_public(f, fid); + if(rc) return rc; + } + }else{ + fid = 0; + } + + if(isPrimary){ + doInsert = true; + }else{ + fsl_stmt * sInsCheck = 0; + rc = fsl_db_prepare_cached(db, &sInsCheck, + "SELECT 1 FROM mlink WHERE " + "mid=? AND fnid=? AND NOT isaux" + "/*%s()*/",__func__); + if(rc){ + rc = fsl_cx_uplift_db_error(f, db); + goto end; + } + fsl_stmt_bind_id(sInsCheck, 1, mid); + fsl_stmt_bind_id(sInsCheck, 2, fnid); + rc = fsl_stmt_step(sInsCheck); + fsl_stmt_cached_yield(sInsCheck); + doInsert = (FSL_RC_STEP_ROW==rc) ? true : false; + rc = 0; + } + if(doInsert){ + rc = fsl_db_prepare_cached(db, &s1, + "INSERT INTO mlink(" + "mid,fid,pmid,pid," + "fnid,pfnid,mperm,isaux" + ")VALUES(" + ":m,:f,:pm,:p,:n,:pfn,:mp,:isaux" + ")" + "/*%s()*/",__func__); + if(!rc){ + fsl_stmt_bind_id_name(s1, ":m", mid); + fsl_stmt_bind_id_name(s1, ":f", fid); + fsl_stmt_bind_id_name(s1, ":pm", pmid); + fsl_stmt_bind_id_name(s1, ":p", pid); + fsl_stmt_bind_id_name(s1, ":n", fnid); + fsl_stmt_bind_id_name(s1, ":pfn", pfnid); + fsl_stmt_bind_id_name(s1, ":mp", mperm); + fsl_stmt_bind_int32_name(s1, ":isaux", isPrimary ? 0 : 1); + rc = fsl_stmt_step(s1); + fsl_stmt_cached_yield(s1); + if(FSL_RC_STEP_DONE==rc){ + rc = 0; + }else{ + fsl_cx_uplift_db_error(f, db); + } + } + } + if(!rc && pid>0 && fid){ + /* Reminder to self: this costs almost 1ms per checkin in very + basic tests with 2003 checkins on my NUC unit. */ + rc = fsl_content_deltify(f, pid, fid, 0); + } + end: + return rc; +} + +/** + Do a binary search to find a file in d->F.list. + + As an optimization, guess that the file we seek is at index + d->F.cursor. That will usually be the case. If it is not found + there, then do the actual binary search. + + Update d->F.cursor to be the index of the file that is found. + + If d->f is NULL then this perform a case-sensitive search, + otherwise it uses case-sensitive or case-insensitive, + depending on f->cache.caseInsensitive. + + If the 3rd argument is not NULL and non-NULL is returned then + *atNdx gets set to the d->F.list index of the resulting object. + If NULL is returned, *atNdx is not modified. + + Reminder to self: if this requires a non-const deck (and it does + right now) then the whole downstream chain will require a + non-const instance or they'll have to make local copies to make + the manipulation of d->F.cursor legal (but that would break + following of baselines without yet more trickery). + + Reminder to self: + + Fossil(1) added another parameter to this since it was ported, + indicating whether only an exact match or the "closest match" is + acceptable, but currently (2021-03-10) only the fusefs module uses + the closest-match option. It's a trivial code change but currently + looks like YAGNI. +*/ +static fsl_card_F * fsl_deck_F_seek_base(fsl_deck * d, + char const * zName, + uint32_t * atNdx ){ + /* Maintenance reminder: this algo relies on the various + counters being signed. */ + fsl_int_t lwr, upr; + int c; + fsl_int_t i; + assert(d); + assert(zName && *zName); + if(!d->F.used) return NULL; + else if(FSL_CARD_F_LIST_NEEDS_SORT & d->F.flags){ + fsl_card_F_list_sort(&d->F); + } +#define FCARD(NDX) F_at(&d->F, (NDX)) + lwr = 0; + upr = d->F.used-1; + if( d->F.cursor>=lwr && d->F.cursorf && d->f->cache.caseInsensitive) + ? fsl_stricmp(FCARD(d->F.cursor+1)->name, zName) + : fsl_strcmp(FCARD(d->F.cursor+1)->name, zName); + if( c==0 ){ + if(atNdx) *atNdx = (uint32_t)d->F.cursor+1; + return FCARD(++d->F.cursor); + }else if( c>0 ){ + upr = d->F.cursor; + }else{ + lwr = d->F.cursor+1; + } + } + while( lwr<=upr ){ + i = (lwr+upr)/2; + c = (d->f && d->f->cache.caseInsensitive) + ? fsl_stricmp(FCARD(i)->name, zName) + : fsl_strcmp(FCARD(i)->name, zName); + if( c<0 ){ + lwr = i+1; + }else if( c>0 ){ + upr = i-1; + }else{ + d->F.cursor = i; + if(atNdx) *atNdx = (uint32_t)i; + return FCARD(i); + } + } + return NULL; +#undef FCARD +} + +fsl_card_F * fsl_deck_F_seek(fsl_deck * const d, const char *zName){ + fsl_card_F *pFile; + assert(d); + assert(zName && *zName); + if(!d || (FSL_SATYPE_CHECKIN!=d->type) || !zName || !*zName + || !d->F.used) return NULL; + pFile = fsl_deck_F_seek_base(d, zName, NULL); + if( !pFile && + (d->B.baseline /* we have a baseline or... */ + || (d->f && d->B.uuid) /* we can load the baseline */ + )){ + /* Check baseline manifest... + + Sidebar: while the delta manifest model outwardly appears + to support recursive delta manifests, fossil(1) does not + use them and there would seem to be little practical use + for them (no notable size benefit for the majority of + cases), so we're not recursing here. + */ + int const rc = d->B.baseline ? 0 : fsl_deck_baseline_fetch(d); + if(rc){ + assert(d->f->error.code); + }else if( d->B.baseline ){ + assert(d->B.baseline->f && "How can this happen?"); + assert((d->B.baseline->f == d->f) && + "Universal laws are out of balance."); + pFile = fsl_deck_F_seek_base(d->B.baseline, zName, NULL); + if(pFile){ + assert(pFile->uuid && + "Per fossil-dev thread with DRH on 20140422, " + "baselines never have removed files."); + } + } + } + return pFile; +} + +fsl_card_F const * fsl_deck_F_search(fsl_deck *d, const char *zName){ + assert(d); + return fsl_deck_F_seek(d, zName); +} + +int fsl_deck_F_set( fsl_deck * d, char const * zName, + char const * uuid, + fsl_fileperm_e perms, + char const * priorName){ + uint32_t fcNdx = 0; + fsl_card_F * fc = 0; + if(d->rid>0){ + return fsl_cx_err_set(d->f, FSL_RC_MISUSE, + "%s() cannot be applied to a saved deck.", + __func__); + }else if(!fsl_deck_check_type(d, 'F')){ + return d->f->error.code; + } + fc = fsl_deck_F_seek_base(d, zName, &fcNdx); + if(!uuid){ + if(fc){ + fsl_card_F_list_remove(&d->F, fcNdx); + return 0; + }else{ + return FSL_RC_NOT_FOUND; + } + }else if(!fsl_is_uuid(uuid)){ + return fsl_cx_err_set(d->f, FSL_RC_RANGE, + "Invalid UUID for F-card."); + } + if(fc){ + /* Got a match. Replace its contents. */ + char * n = 0; + if(!fc->deckOwnsStrings){ + /* We can keep fc->name but need a tiny bit of hoop-jumping + to do so. */ + n = fc->name; + fc->name = 0; + } + fsl_card_F_clean(fc); + assert(!fc->deckOwnsStrings); + if(!(fc->name = n ? n : fsl_strdup(zName))) return FSL_RC_OOM; + if(!(fc->uuid = fsl_strdup(uuid))) return FSL_RC_OOM; + if(priorName && *priorName){ + if(!fsl_is_simple_pathname(priorName, 1)){ + return fsl_cx_err_set(d->f, FSL_RC_RANGE, + "Invalid priorName for F-card " + "(simple form required): %s", priorName); + }else if(!(fc->priorName = fsl_strdup(priorName))){ + return FSL_RC_OOM; + } + } + fc->perm = perms; + return 0; + }else{ + return fsl_deck_F_add(d, zName, uuid, perms, priorName); + } +} + +int fsl_deck_F_set_content( fsl_deck * d, char const * zName, + fsl_buffer const * src, + fsl_fileperm_e perm, + char const * priorName){ + fsl_uuid_str zHash = 0; + fsl_id_t rid = 0; + fsl_id_t prevRid = 0; + int rc = 0; + if(d->rid>0){ + return fsl_cx_err_set(d->f, FSL_RC_MISUSE, + "%s() cannot be applied to a saved deck.", + __func__); + }else if(!fsl_cx_transaction_level(d->f)){ + return fsl_cx_err_set(d->f, FSL_RC_MISUSE, + "%s() requires that a transaction is active.", + __func__); + } + rc = fsl_repo_blob_lookup(d->f, src, &rid, &zHash); + if(rc && FSL_RC_NOT_FOUND!=rc) goto end; + assert(zHash); + if(!rid){ + fsl_card_F const * fc; + /* This is new content. Save it, then see if we have a previous version + to delta against this one. */ + rc = fsl_content_put_ex(d->f, src, zHash, 0, 0, false, &rid); + if(rc) goto end; + fc = fsl_deck_F_seek(d, zName); + if(fc){ + prevRid = fsl_uuid_to_rid(d->f, fc->uuid); + if(prevRid<0) goto end; + else if(!prevRid){ + assert(!"cannot happen"); + rc = fsl_cx_err_set(d->f, FSL_RC_NOT_FOUND, + "Cannot find RID of file content %s [%s]\n", + fc->name, fc->uuid); + goto end; + } + rc = fsl_content_deltify(d->f, prevRid, rid, false); + if(rc) goto end; + } + } + rc = fsl_deck_F_set(d, zName, zHash, perm, priorName); + end: + fsl_free(zHash); + return rc; +} + +void fsl_deck_clean_cards(fsl_deck * d, char const * letters){ + char const * c = letters + ? letters + : "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + for( ; *c; ++c ){ + switch(*c){ + case 'A': fsl_deck_clean_A(d); break; + case 'B': fsl_deck_clean_B(d); break; + case 'C': fsl_deck_clean_C(d); break; + case 'D': d->D = 0.0; break; + case 'E': fsl_deck_clean_E(d); break; + case 'F': fsl_deck_clean_F(d); break; + case 'G': fsl_deck_clean_G(d); break; + case 'H': fsl_deck_clean_H(d); break; + case 'I': fsl_deck_clean_I(d); break; + case 'J': fsl_deck_clean_J(d,true); break; + case 'K': fsl_deck_clean_K(d); break; + case 'L': fsl_deck_clean_L(d); break; + case 'M': fsl_deck_clean_M(d); break; + case 'N': fsl_deck_clean_N(d); break; + case 'P': fsl_deck_clean_P(d); break; + case 'Q': fsl_deck_clean_Q(d); break; + case 'R': fsl_deck_clean_R(d); break; + case 'T': fsl_deck_clean_T(d); break; + case 'U': fsl_deck_clean_U(d); break; + case 'W': fsl_deck_clean_W(d); break; + default: break; + } + } +} + +int fsl_deck_derive(fsl_deck * const d){ + int rc = 0; + if(d->rid<=0) return FSL_RC_MISUSE; + assert(d->f); + if(FSL_SATYPE_CHECKIN!=d->type) return FSL_RC_TYPE; + fsl_deck_clean_P(d); + { + fsl_uuid_str pUuid = fsl_rid_to_uuid(d->f, d->rid); + if(pUuid){ + rc = fsl_list_append(&d->P, pUuid); + if(rc){ + assert(NULL==d->P.list); + fsl_free(pUuid); + } + }else{ + assert(d->f->error.code); + rc = d->f->error.code; + } + if(rc) return rc; + } + d->rid = 0; + fsl_deck_clean_cards(d, "ACDEGHIJKLMNQRTUW"); + while(d->B.uuid){ + /* This is a delta manifest. Convert this deck into a baseline by + build a new, complete F-card list. */ + fsl_card_F const * fc; + fsl_card_F_list flist = fsl_card_F_list_empty; + uint32_t fCount = 0; + rc = fsl_deck_F_rewind(d); + if(rc) return rc; + while( 0==(rc=fsl_deck_F_next(d, &fc)) && fc ){ + ++fCount; + } + rc = fsl_deck_F_rewind(d); + assert(0==rc + && "fsl_deck_F_rewind() cannot fail after initial call."); + assert(0==d->F.cursor); + assert(0==d->B.baseline->F.cursor); + rc = fsl_card_F_list_reserve(&flist, fCount); + if(rc) break; + while( 1 ){ + rc = fsl_deck_F_next(d, &fc); + if(rc || !fc) break; + fsl_card_F * const fNew = fsl_card_F_list_push(&flist); + assert(fc->uuid); + assert(fc->name); + /* We must copy these strings because their ownership is + otherwise unmanageable. e.g. they might live in d->content + or d->B.baseline->content. */ + if(!(fNew->name = fsl_strdup(fc->name)) + || !(fNew->uuid = fsl_strdup(fc->uuid))){ + /* Reminder: we do not want/need to copy fc->priorName. Those + renames were already applied in the parent checkin. */ + rc = FSL_RC_OOM; + break; + } + fNew->perm = fc->perm; + } + fsl_deck_clean_B(d); + fsl_deck_clean_F(d); + if(rc) fsl_card_F_list_finalize(&flist); + else d->F = flist/*transfer ownership*/; + break; + } + return rc; +} + +/** + Returns true if repo contains an mlink entry where mid=rid, else + false. +*/ +static bool fsl_repo_has_mlink_mid( fsl_db * repo, fsl_id_t rid ){ +#if 0 + return fsl_db_exists(repo, + "SELECT 1 FROM mlink WHERE mid=%"FSL_ID_T_PFMT, + rid); +#else + fsl_stmt * st = NULL; + bool gotone = false; + int rc = fsl_db_prepare_cached(repo, &st, + "SELECT 1 FROM mlink WHERE mid=?" + "/*%s()*/",__func__); + + if(!rc){ + fsl_stmt_bind_id(st, 1, rid); + rc = fsl_stmt_step(st); + fsl_stmt_cached_yield(st); + gotone = rc==FSL_RC_STEP_ROW; + } + return gotone; +#endif +} + +static bool fsl_repo_has_mlink_pmid_mid( fsl_db * repo, fsl_id_t pmid, fsl_id_t mid ){ + fsl_stmt * st = NULL; + int rc = fsl_db_prepare_cached(repo, &st, + "SELECT 1 FROM mlink WHERE mid=? " + "AND pmid=?" + "/*%s()*/",__func__); + if(!rc){ + fsl_stmt_bind_id(st, 1, mid); + fsl_stmt_bind_id(st, 2, pmid); + rc = fsl_stmt_step(st); + fsl_stmt_cached_yield(st); + if( rc==FSL_RC_STEP_ROW ) rc = 0; + } + /* MARKER(("fsl_repo_has_mlink_mid(%d) rc=%d\n", (int)rid, rc)); */ + return rc ? false : true; +} + +/** + Add mlink table entries associated with manifest cid, pChild. The + parent manifest is pid, pParent. One of either pChild or pParent + will be NULL and it will be computed based on cid/pid. + + A single mlink entry is added for every file that changed content, + name, and/or permissions going from pid to cid. + + Deleted files have mlink.fid=0. + + Added files have mlink.pid=0. + + File added by merge have mlink.pid=-1. + + Edited files have both mlink.pid!=0 and mlink.fid!=0 + + Comments from the original implementation: + + Many mlink entries for merge parents will only be added if another + mlink entry already exists for the same file from the primary + parent. Therefore, to ensure that all merge-parent mlink entries + are properly created: + + (1) Make this routine a no-op if pParent is a merge parent and the + primary parent is a phantom. + + (2) Invoke this routine recursively for merge-parents if pParent + is the primary parent. +*/ +static int fsl_mlink_add( fsl_cx * const f, + fsl_id_t pmid, fsl_deck /*const*/ * pParent, + fsl_id_t cid, fsl_deck /*const*/ * pChild, + bool isPrimary){ + fsl_buffer otherContent = fsl_buffer_empty; + fsl_id_t otherRid; + fsl_size_t i = 0; + int rc = 0; + fsl_card_F const * pChildFile = NULL; + fsl_card_F const * pParentFile = NULL; + fsl_deck dOther = fsl_deck_empty; + fsl_db * const db = fsl_cx_db_repo(f); + bool isPublic; + assert(db); + assert(db->beginCount>0); + /* If mlink table entires are already set for pmid/cid, then abort + early doing no work. + */ + //MARKER(("%s() pmid=%d cid=%d\n", __func__, (int)pmid, (int)cid)); + if(fsl_repo_has_mlink_pmid_mid(db, pmid, cid)) return 0; + /* Compute the value of the missing pParent or pChild parameter. + Fetch the baseline checkins for both. + */ + assert( pParent==0 || pChild==0 ); + if( pParent ){ + assert(!pChild); + pChild = &dOther; + otherRid = cid; + }else{ + pParent = &dOther; + otherRid = pmid; + } + + if(otherRid && !fsl_cx_mcache_search(f, otherRid, &dOther)){ + rc = fsl_content_get(f, otherRid, &otherContent); + if(rc){ + /* fossil(1) simply ignores errors here and returns. We'll ignore + the phantom case because (1) erroring out here would be bad and + (2) fossil does so. The exact implications of doing so are + unclear, though. */ + if(FSL_RC_PHANTOM==rc){ + rc = 0; + }else if(!f->error.msg.used && FSL_RC_OOM!=rc){ + rc = fsl_cx_err_set(f, rc, + "Fetching content of rid %"FSL_ID_T_PFMT" failed: %s", + otherRid, fsl_rc_cstr(rc)); + } + goto end; + } + if( !otherContent.used ){ + /* ??? fossil(1) ignores this case and returns. */ + fsl_buffer_clear(&otherContent)/*for empty file case*/; + rc = 0; + goto end; + } + dOther.f = f; + rc = fsl_deck_parse2(&dOther, &otherContent, otherRid); + assert(dOther.f); + if(rc) goto end; + } + if( (pParent->f && (rc=fsl_deck_baseline_fetch(pParent))) + || (pChild->f && (rc=fsl_deck_baseline_fetch(pChild)))){ + goto end; + } + isPublic = !fsl_content_is_private(f, cid); + + /* If pParent is not the primary parent of pChild, and the primary + ** parent of pChild is a phantom, then abort this routine without + ** doing any work. The mlink entries will be computed when the + ** primary parent dephantomizes. + */ + if( !isPrimary && otherRid==cid ){ + assert(pChild->P.used); + if(!fsl_db_exists(db,"SELECT 1 FROM blob WHERE uuid=%Q AND size>0", + (char const *)pChild->P.list[0])){ + rc = 0; + fsl_cx_mcache_insert(f, &dOther); + goto end; + } + } + + if(pmid>0){ + /* Try to make the parent manifest a delta from the child, if that + is an appropriate thing to do. For a new baseline, make the + previous baseline a delta from the current baseline. + */ + if( (pParent->B.uuid==0)==(pChild->B.uuid==0) ){ + rc = fsl_content_deltify(f, pmid, cid, 0); + }else if( pChild->B.uuid==NULL && pParent->B.uuid!=NULL ){ + rc = fsl_content_deltify(f, pParent->B.baseline->rid, cid, 0); + } + if(rc) goto end; + } + + /* Remember all children less than a few seconds younger than their parent, + as we might want to fudge the times for those children. + */ + if( f->cache.isCrosslinking && + (pChild->D < pParent->D+AGE_FUDGE_WINDOW) + ){ + rc = fsl_db_exec(db, "INSERT OR REPLACE INTO time_fudge VALUES" + "(%"FSL_ID_T_PFMT", %"FSL_JULIAN_T_PFMT + ", %"FSL_ID_T_PFMT", %"FSL_JULIAN_T_PFMT");", + pParent->rid, pParent->D, + pChild->rid, pChild->D); + if(rc) goto end; + } + + /* First look at all files in pChild, ignoring its baseline. This + is where most of the changes will be found. + */ +#define FCARD(DECK,NDX) \ + ((((NDX)<(DECK)->F.used)) \ + ? F_at(&(DECK)->F,NDX) \ + : NULL) + for(i=0, pChildFile=FCARD(pChild,0); + iF.used; + ++i, pChildFile=FCARD(pChild,i)){ + fsl_fileperm_e const mperm = pChildFile->perm; + if( pChildFile->priorName ){ + pParentFile = pmid + ? fsl_deck_F_seek(pParent, pChildFile->priorName) + : 0; + if( pParentFile ){ + /* File with name change */ + /* + libfossil checkin 8625a31eff708dea93b16582e4ec5d583794d1af + contains these two interesting F-cards: + +F src/net/wanderinghorse/libfossil/FossilCheckout.java +F src/org/fossil_scm/libfossil/Checkout.java 6e58a47089d3f4911c9386c25bac36c8e98d4d21 w src/net/wanderinghorse/libfossil/FossilCheckout.java + + Note the placement of FossilCheckout.java (twice). + + Up until then, i thought a delete/rename combination was not possible. + */ + rc = fsl_mlink_add_one(f, pmid, pParentFile->uuid, + cid, pChildFile->uuid, pChildFile->name, + pChildFile->priorName, isPublic, + isPrimary, mperm); + }else{ + /* File name changed, but the old name is not found in the parent! + Treat this like a new file. */ + rc = fsl_mlink_add_one(f, pmid, 0, cid, pChildFile->uuid, + pChildFile->name, 0, + isPublic, isPrimary, mperm); + } + }else if(pmid){ + pParentFile = fsl_deck_F_seek(pParent, pChildFile->name); + if(!pParentFile || !pParentFile->uuid){ + /* Parent does not have it or it was removed in parent. */ + if( pChildFile->uuid ){ + /* A new or re-added file */ + rc = fsl_mlink_add_one(f, pmid, 0, cid, pChildFile->uuid, + pChildFile->name, 0, + isPublic, isPrimary, mperm); + } + } + else if( fsl_strcmp(pChildFile->uuid, pParentFile->uuid)!=0 + || (pParentFile->perm!=mperm) ){ + /* Changes in file content or permissions */ + rc = fsl_mlink_add_one(f, pmid, pParentFile->uuid, + cid, pChildFile->uuid, + pChildFile->name, 0, + isPublic, isPrimary, mperm); + } + } + } /* end pChild card list loop */ + if(rc) goto end; + else if( pParent->B.uuid && pChild->B.uuid ){ + /* Both parent and child are delta manifests. Look for files that + are deleted or modified in the parent but which reappear or revert + to baseline in the child and show such files as being added or changed + in the child. */ + for(i=0, pParentFile=FCARD(pParent,0); + iF.used; + ++i, pParentFile = FCARD(pParent,i)){ + if( pParentFile->uuid ){ + pChildFile = fsl_deck_F_seek_base(pChild, pParentFile->name, NULL); + if( !pChildFile || !pChildFile->uuid){ + /* The child file reverts to baseline or is deleted. + Show this as a change. */ + if(!pChildFile){ + pChildFile = fsl_deck_F_seek(pChild, pParentFile->name); + } + if( pChildFile && pChildFile->uuid ){ + rc = fsl_mlink_add_one(f, pmid, pParentFile->uuid, cid, + pChildFile->uuid, pChildFile->name, + 0, isPublic, isPrimary, + pChildFile->perm); + } + } + }else{ + /* Was deleted in the parent. */ + pChildFile = fsl_deck_F_seek(pChild, pParentFile->name); + if( pChildFile && pChildFile->uuid ){ + /* File resurrected in the child after having been deleted in + the parent. Show this as an added file. */ + rc = fsl_mlink_add_one(f, pmid, 0, cid, pChildFile->uuid, + pChildFile->name, 0, isPublic, + isPrimary, pChildFile->perm); + } + } + if(rc) goto end; + } + assert(0==rc); + }else if( pmid && !pChild->B.uuid ){ + /* pChild is a baseline with a parent. Look for files that are + present in pParent but are missing from pChild and mark them as + having been deleted. */ + fsl_card_F const * cfc = NULL; + fsl_deck_F_rewind(pParent); + while( (0==(rc=fsl_deck_F_next(pParent,&cfc))) && cfc){ + pParentFile = cfc; + pChildFile = fsl_deck_F_seek(pChild, pParentFile->name); + if( (!pChildFile || !pChildFile->uuid) && pParentFile->uuid ){ + rc = fsl_mlink_add_one(f, pmid, pParentFile->uuid, cid, 0, + pParentFile->name, 0, isPublic, + isPrimary, pParentFile->perm); + } + } + if(rc) goto end; + } + + fsl_cx_mcache_insert(f, &dOther); + + /* If pParent is the primary parent of pChild, also run this analysis + ** for all merge parents of pChild */ + if( pmid && isPrimary ){ + for(i=1; iP.used; i++){ + pmid = fsl_uuid_to_rid(f, (char const*)pChild->P.list[i]); + if( pmid<=0 ) continue; + rc = fsl_mlink_add(f, pmid, 0, cid, pChild, false); + if(rc) goto end; + } + for(i=0; iQ.used; i++){ + fsl_card_Q const * q = (fsl_card_Q const *)pChild->Q.list[i]; + if( q->type>0 && (pmid = fsl_uuid_to_rid(f, q->target))>0 ){ + rc = fsl_mlink_add(f, pmid, 0, cid, pChild, false); + if(rc) goto end; + } + } + } + + end: + fsl_deck_finalize(&dOther); + fsl_buffer_clear(&otherContent); + if(rc && !f->error.code && db->error.code){ + rc = fsl_cx_uplift_db_error(f, db); + } + return rc; +#undef FCARD +} + +/** + Apply all tags defined in deck d. If parentId is >0 then any + propagating tags from that parent are well and duly propagated. + Returns 0 on success. Potential TODO: if parentId<=0 and + d->P.used>0 then use d->P.list[0] in place of parentId. +*/ +static int fsl_deck_crosslink_apply_tags(fsl_cx * f, fsl_deck *d, + fsl_db * db, fsl_id_t rid, + fsl_id_t parentId){ + int rc = 0; + fsl_size_t i; + fsl_list const * li = &d->T; + double tagTime = d->D; + if(li->used && tagTime<=0){ + tagTime = fsl_db_julian_now(db); + if(tagTime<=0){ + rc = FSL_RC_DB; + goto end; + } + } + for( i = 0; !rc && (i < li->used); ++i){ + fsl_id_t tid; + fsl_card_T const * tag = (fsl_card_T const *)li->list[i]; + assert(tag); + if(!tag->uuid){ + tid = rid; + }else{ + tid = fsl_uuid_to_rid( f, tag->uuid); + } + if(tid<0){ + assert(f->error.code); + rc = f->error.code; + break; + }else if(0==tid){ + rc = fsl_cx_err_set(f, FSL_RC_RANGE, + "Could not get RID for [%.12s].", + tag->uuid); + break; + } + rc = fsl_tag_insert(f, tag->type, + tag->name, tag->value, + rid, tagTime, tid, NULL); + } + if( !rc && (parentId>0) ){ + rc = fsl_tag_propagate_all(f, parentId); + } + end: + return rc; +} + +/** + Part of the checkin crosslink machinery: create all appropriate + plink and mlink table entries for d->P. + + If parentId is not NULL, *parentId gets assigned to the rid of the + first parent, or 0 if d->P is empty. +*/ +static int fsl_deck_add_checkin_linkages(fsl_deck *d, fsl_id_t * parentId){ + int rc = 0; + fsl_size_t nLink = 0; + char zBaseId[30] = {0}/*RID of baseline or "NULL" if no baseline */; + fsl_size_t i; + fsl_stmt q = fsl_stmt_empty; + fsl_id_t _parentId = 0; + fsl_cx * const f = d->f; + fsl_db * const db = fsl_cx_db_repo(f); + assert(f && db); + if(!parentId) parentId = &_parentId; + if(d->B.uuid){ + fsl_id_t const baseid = d->B.baseline + ? d->B.baseline->rid + : fsl_uuid_to_rid(d->f, d->B.uuid); + if(baseid<0){ + rc = d->f->error.code; + assert(0 != rc); + goto end; + } + assert(baseid>0); + fsl_snprintf( zBaseId, sizeof(zBaseId), + "%"FSL_ID_T_PFMT, + baseid ); + + }else{ + fsl_snprintf( zBaseId, sizeof(zBaseId), "NULL" ); + } + *parentId = 0; + for(i=0; iP.used; ++i){ + char const * parentUuid = (char const *)d->P.list[i]; + fsl_id_t const pid = fsl_uuid_to_rid2(f, parentUuid, FSL_PHANTOM_PUBLIC); + if(pid<0){ + assert(f->error.code); + rc = f->error.code; + goto end; + } + rc = fsl_db_exec(db, "INSERT OR IGNORE " + "INTO plink(pid, cid, isprim, mtime, baseid) " + "VALUES(%"FSL_ID_T_PFMT", %"FSL_ID_T_PFMT + ", %d, %"FSL_JULIAN_T_PFMT", %s)", + pid, d->rid, + ((i==0) ? 1 : 0), d->D, zBaseId); + if(rc) goto end; + if(0==i) *parentId = pid; + } + rc = fsl_mlink_add(f, *parentId, NULL, d->rid, d, true); + if(rc) goto end; + nLink = d->P.used; + for(i=0; iQ.used; ++i){ + fsl_card_Q const * q = (fsl_card_Q const *)d->Q.list[i]; + if(q->type>0) ++nLink; + } + if(nLink>1){ + /* https://www.fossil-scm.org/index.html/info/8e44cf6f4df4f9f0 */ + /* Change MLINK.PID from 0 to -1 for files that are added by merge. */ + rc = fsl_db_exec(db, + "UPDATE mlink SET pid=-1" + " WHERE mid=%"FSL_ID_T_PFMT + " AND pid=0" + " AND fnid IN " + " (SELECT fnid FROM mlink WHERE mid=%"FSL_ID_T_PFMT + " GROUP BY fnid" + " HAVING count(*)<%d)", + d->rid, d->rid, (int)nLink + ); + if(rc) goto end; + } + rc = fsl_db_prepare(db, &q, + "SELECT cid, isprim FROM plink " + "WHERE pid=%"FSL_ID_T_PFMT, + d->rid); + while( !rc && (FSL_RC_STEP_ROW==(rc=fsl_stmt_step(&q))) ){ + fsl_id_t const cid = fsl_stmt_g_id(&q, 0); + int const isPrim = fsl_stmt_g_int32(&q, 1); + /* This block is only hit a couple of times during a fresh rebuild (empty mlink/plink + tables), but many times on a rebuilds if those tables are not emptied in advance? */ + assert(cid>0); + rc = fsl_mlink_add(f, d->rid, d, cid, NULL, isPrim ? true : false); + } + if(FSL_RC_STEP_DONE==rc) rc = 0; + fsl_stmt_finalize(&q); + if(rc) goto end; + if( !d->P.used ){ + /* For root files (files without parents) add mlink entries + showing all content as new. + + Historically, fossil has been unable to create such checkins + because the initial checkin has no files. + */ + int const isPublic = !fsl_content_is_private(f, d->rid); + for(i=0; !rc && (iF.used); ++i){ + fsl_card_F const * fc = F_at(&d->F, i); + rc = fsl_mlink_add_one(f, 0, 0, d->rid, fc->uuid, fc->name, 0, + isPublic, 1, fc->perm); + } + } + end: + return rc; +} + +/** + Applies the value of a "parent" tag (reparent) to the given + artifact id. zTagVal must be the value of a parent tag (a list of + full UUIDs). This is only to be run as part of fsl_crosslink_end(). + + Returns 0 on success. + + POTENTIAL fixme: perhaps return without side effects if rid is not + found (like fossil(1) does). That said, this step is only run after + crosslinking and would only result in a not-found if the tagxref + table contents is out of date. + + POTENTIAL fixme: fail without error if the tag value is malformed, + under the assumption that the tag was intended for some purpose + other than reparenting. +*/ +static int fsl_crosslink_reparent(fsl_cx * f, fsl_id_t rid, char const *zTagVal){ + int rc = 0; + char * zDup = 0; + char * zPos; + fsl_size_t maxP, nP = 0; + fsl_deck d = fsl_deck_empty; + fsl_list fakeP = fsl_list_empty + /* fake P-card for purposes of passing the reparented deck through + fsl_deck_add_checkin_linkages() */; + maxP = (fsl_strlen(zTagVal)+1) / (FSL_STRLEN_SHA1+1); + if(!maxP) return FSL_RC_RANGE; + rc = fsl_list_reserve(&fakeP, maxP); + if(rc) return rc; + zDup = fsl_strdup(zTagVal); + if(!zDup){ + rc = FSL_RC_OOM; + goto end; + } + /* Split zTagVal into list of parent IDs... */ + for( nP = 0, zPos = zDup; *zPos; ){ + char const * zBegin = zPos; + for( ; *zPos && ' '!=*zPos; ++zPos){} + if(' '==*zPos){ + *zPos = 0; + ++zPos; + } + if(!fsl_is_uuid(zBegin)){ + rc = fsl_cx_err_set(f, FSL_RC_RANGE, + "Invalid value [%s] in reparent tag value " + "[%s] for rid %"FSL_ID_T_PFMT".", + zBegin, zTagVal, rid); + goto end; + } + fakeP.list[nP++] = (void *)zBegin; + } + assert(!rc); + fakeP.used = nP; + rc = fsl_deck_load_rid(f, &d, rid, FSL_SATYPE_ANY); + if(rc) goto end; + switch(d.type){ + case FSL_SATYPE_CHECKIN: + case FSL_SATYPE_TECHNOTE: + case FSL_SATYPE_WIKI: + case FSL_SATYPE_FORUMPOST: + break; + default: + rc = fsl_cx_err_set(f, FSL_RC_TYPE, "Invalid deck type (%s) " + "for use with the 'parent' tag.", + fsl_satype_cstr(d.type)); + goto end; + } + assert(d.rid==rid); + assert(d.f); + fsl_db * const db = fsl_cx_db_repo(f); + rc = fsl_db_exec_multi(db, + "DELETE FROM plink WHERE cid=%"FSL_ID_T_PFMT";" + "DELETE FROM mlink WHERE mid=%"FSL_ID_T_PFMT";", + rid, rid); + if(rc) goto end; + fsl_list const origP = d.P; + d.P = fakeP; + rc = fsl_deck_add_checkin_linkages(&d, NULL); + d.P = origP; + fsl_deck_finalize(&d); + + end: + fsl_list_reserve(&fakeP, 0); + fsl_free(zDup); + return rc; +} + +/** + Inserts plink entries for FORUM, WIKI, and TECHNOTE manifests. May + assert for other manifest types. If a parent entry exists, it also + propagates any tags for that parent. This is a no-op if + the deck has no parents. +*/ +static int fsl_deck_crosslink_fwt_plink(fsl_deck * d){ + int i; + fsl_id_t parentId = 0; + fsl_db * db; + int rc = 0; + assert(d->type==FSL_SATYPE_WIKI || + d->type==FSL_SATYPE_FORUMPOST || + d->type==FSL_SATYPE_TECHNOTE); + assert(d->f); + assert(d->rid>0); + if(!d->P.used) return rc; + db = fsl_cx_db_repo(d->f); + fsl_phantom_e const fantomMode = fsl_content_is_private(d->f, d->rid) + ? FSL_PHANTOM_PRIVATE : FSL_PHANTOM_PUBLIC; + for(i=0; 0==rc && i<(int)d->P.used; ++i){ + fsl_id_t const pid = fsl_uuid_to_rid2(d->f, (char const *)d->P.list[i], + fantomMode); + if(0==i) parentId = pid; + rc = fsl_db_exec_multi(db, + "INSERT OR IGNORE INTO plink" + "(pid, cid, isprim, mtime, baseid)" + "VALUES(%"FSL_ID_T_PFMT", %"FSL_ID_T_PFMT", " + "%d, %"FSL_JULIAN_T_PFMT", NULL)", + pid, d->rid, i==0, d->D); + } + if(!rc && parentId){ + rc = fsl_tag_propagate_all(d->f, parentId); + } + return rc; +} + + +/** + Overrideable crosslink listener which updates the timeline for + attachment records. +*/ +static int fsl_deck_xlink_f_attachment(fsl_deck * d, void * state){ + if(FSL_SATYPE_ATTACHMENT!=d->type) return 0; + int rc; + fsl_db * db; + fsl_buffer comment = fsl_buffer_empty; + const char isAdd = (d->A.src && *d->A.src) ? 1 : 0; + char attachToType = 'w' + /* Assume wiki until we know otherwise, keeping in mind that the + d->A.tgt might not yet be in the blob table, in which case + we are unable to know, for certain, what the target is. + That only affects the timeline (event table), though, not + the crosslinking of the attachment itself. */; + db = fsl_cx_db_repo(d->f); + assert(db); + if(fsl_is_uuid(d->A.tgt)){ + if( fsl_db_exists(db, "SELECT 1 FROM tag WHERE tagname='tkt-%q'", + d->A.tgt)){ + attachToType = 't' /* attach to a known ticket */; + }else if( fsl_db_exists(db, "SELECT 1 FROM tag WHERE tagname='event-%q'", + d->A.tgt)){ + attachToType = 'e' /* attach to a known technote (event) */; + } + } + if('w'==attachToType){ + /* Attachment applies to a wiki page */ + if(isAdd){ + rc = fsl_buffer_appendf(&comment, + "Add attachment \"%h\" " + "to wiki page [%h]", + d->A.name, d->A.tgt); + }else{ + rc = fsl_buffer_appendf(&comment, + "Delete attachment \"%h\" " + "from wiki page [%h]", + d->A.name, d->A.tgt); + } + }else if('e' == attachToType){/*technote*/ + if(isAdd){ + rc = fsl_buffer_appendf(&comment, + "Add attachment [/artifact/%!S|%h] to " + "tech note [/technote/%!S|%S]", + d->A.src, d->A.name, d->A.tgt, d->A.tgt); + }else{ + rc = fsl_buffer_appendf(&comment, + "Delete attachment \"/artifact/%!S|%h\" " + "from tech note [/technote/%!S|%S]", + d->A.name, d->A.name, d->A.tgt, + d->A.tgt); + } + }else{ + /* Attachment applies to a ticket */ + if(isAdd){ + rc = fsl_buffer_appendf(&comment, + "Add attachment [/artifact/%!S|%h] " + "to ticket [%!S|%S]", + d->A.src, d->A.name, d->A.tgt, d->A.tgt); + }else{ + rc = fsl_buffer_appendf(&comment, + "Delete attachment \"%h\" " + "from ticket [%!S|%S]", + d->A.name, d->A.tgt, d->A.tgt); + } + } + if(!rc){ + rc = fsl_db_exec(db, + "REPLACE INTO event(type,mtime,objid,user,comment)" + "VALUES(" + "'%c',%"FSL_JULIAN_T_PFMT",%"FSL_ID_T_PFMT"," + "%Q,%B)", + attachToType, d->D, d->rid, d->U, &comment); + } + fsl_buffer_clear(&comment); + return rc; +} + +/** + Overrideable crosslink listener which updates the timeline for + checkin records. +*/ +static int fsl_deck_xlink_f_checkin(fsl_deck * d, void * state){ + if(FSL_SATYPE_CHECKIN!=d->type) return 0; + int rc; + fsl_db * db; + db = fsl_cx_db_repo(d->f); + assert(db); + rc = fsl_db_exec(db, + "REPLACE INTO event(type,mtime,objid,user,comment," + "bgcolor,euser,ecomment,omtime)" + "VALUES('ci'," + " coalesce(" /*mtime*/ + " (SELECT julianday(value) FROM tagxref " + " WHERE tagid=%d AND rid=%"FSL_ID_T_PFMT + " )," + " %"FSL_JULIAN_T_PFMT"" + " )," + " %"FSL_ID_T_PFMT","/*objid*/ + " %Q," /*user*/ +#if 1 + " %Q," /*comment. No, the comment _field_. */ +#else + /* just for testing... */ + " 'xlink: %q'," /*comment. No, the comment _field_. */ +#endif + " (SELECT value FROM tagxref " /*bgcolor*/ + " WHERE tagid=%d AND rid=%"FSL_ID_T_PFMT + " AND tagtype>0" + " )," + " (SELECT value FROM tagxref " /*euser*/ + " WHERE tagid=%d AND rid=%"FSL_ID_T_PFMT + " )," + " (SELECT value FROM tagxref " /*ecomment*/ + " WHERE tagid=%d AND rid=%"FSL_ID_T_PFMT + " )," + " %"FSL_JULIAN_T_PFMT/*omtime*/ + /* RETURNING coalesce(ecomment,comment) + see comments below about zCom */ + ")", + /* The casts here are to please the va_list. */ + (int)FSL_TAGID_DATE, d->rid, d->D, + d->rid, d->U, d->C, + (int)FSL_TAGID_BGCOLOR, d->rid, + (int)FSL_TAGID_USER, d->rid, + (int)FSL_TAGID_COMMENT, d->rid, d->D + ); + return fsl_cx_uplift_db_error2(d->f, db, rc); +} + +static int fsl_deck_xlink_f_control(fsl_deck * d, void * state){ + if(FSL_SATYPE_CONTROL!=d->type) return 0; + /* + Create timeline event entry for all tags in this control + construct. Note that we are using a lot of historical code which + hard-codes english-lanuage text and links which only work in + fossil(1). i would prefer to farm this out to a crosslink + callback, and provide a default implementation which more or + less mimics fossil(1). + */ + int rc = 0; + fsl_buffer comment = fsl_buffer_empty; + fsl_size_t i; + const char *zName; + const char *zValue; + const char *zUuid; + int branchMove = 0; + int const uuidLen = 8; + fsl_card_T const * tag = NULL; + fsl_card_T const * prevTag = NULL; + fsl_list const * li = &d->T; + fsl_db * const db = fsl_cx_db_repo(d->f); + double mtime = (d->D>0) + ? d->D + : fsl_db_julian_now(db); + assert(db); + /** + Reminder to self: fossil(1) has a comment here: + + // Next loop expects tags to be sorted on UUID, so sort it. + qsort(p->aTag, p->nTag, sizeof(p->aTag[0]), tag_compare); + + That sort plays a role in hook code execution and is needed to + avoid duplicate hook execution in some cases. libfossil + outsources that type of thing to crosslink callbacks, though, + so we won't concern ourselves with it here. We also don't + really want to modify the deck during crosslinking. The only + reason the deck is not const in this routine is because of the + fsl_deck::F::cursor bits inherited from fossil(1), largely + worth its cost except that many routines can no longer be + const. Shame C doesn't have C++'s "mutable" keyword. + + That said, sorting by UUID would have a nice side-effect on the + output of grouping tags by the UUID they tag. So far + (201404) such groups of tags have not appeared in the wild + because fossil(1) has no mechanism for creating them. + */ + for( i = 0; !rc && (i < li->used); ++i, prevTag = tag){ + char isProp = 0, isAdd = 0, isCancel = 0; + tag = (fsl_card_T const *)li->list[i]; + zUuid = tag->uuid; + if(!zUuid /*tag on self*/) continue; + if( i==0 || 0!=fsl_uuidcmp(tag->uuid, prevTag->uuid)){ + rc = fsl_buffer_appendf(&comment, + " Edit [%.*s]:", uuidLen, zUuid); + branchMove = 0; + } + if(rc) goto end; + isProp = FSL_TAGTYPE_PROPAGATING==tag->type; + isAdd = FSL_TAGTYPE_ADD==tag->type; + isCancel = FSL_TAGTYPE_CANCEL==tag->type; + assert(isProp || isAdd || isCancel); + zName = tag->name; + zValue = tag->value; + if( isProp && 0==fsl_strcmp(zName, "branch")){ + rc = fsl_buffer_appendf(&comment, + " Move to branch %s" + "[/timeline?r=%h&nd&dp=%.*s | %h].", + zValue, zValue, uuidLen, zUuid, zValue); + branchMove = 1; + }else if( isProp && fsl_strcmp(zName, "bgcolor")==0 ){ + rc = fsl_buffer_appendf(&comment, + " Change branch background color to \"%h\".", zValue); + }else if( isAdd && fsl_strcmp(zName, "bgcolor")==0 ){ + rc = fsl_buffer_appendf(&comment, + " Change background color to \"%h\".", zValue); + }else if( isCancel && fsl_strcmp(zName, "bgcolor")==0 ){ + rc = fsl_buffer_appendf(&comment, " Cancel background color."); + }else if( isAdd && fsl_strcmp(zName, "comment")==0 ){ + rc = fsl_buffer_appendf(&comment, " Edit check-in comment."); + }else if( isAdd && fsl_strcmp(zName, "user")==0 ){ + rc = fsl_buffer_appendf(&comment, " Change user to \"%h\".", zValue); + }else if( isAdd && fsl_strcmp(zName, "date")==0 ){ + rc = fsl_buffer_appendf(&comment, " Timestamp %h.", zValue); + }else if( isCancel && memcmp(zName, "sym-",4)==0 ){ + if( !branchMove ){ + rc = fsl_buffer_appendf(&comment, " Cancel tag %h.", zName+4); + } + }else if( isProp && memcmp(zName, "sym-",4)==0 ){ + if( !branchMove ){ + rc = fsl_buffer_appendf(&comment, " Add propagating tag \"%h\".", zName+4); + } + }else if( isAdd && memcmp(zName, "sym-",4)==0 ){ + rc = fsl_buffer_appendf(&comment, " Add tag \"%h\".", zName+4); + }else if( isCancel && memcmp(zName, "sym-",4)==0 ){ + rc = fsl_buffer_appendf(&comment, " Cancel tag \"%h\".", zName+4); + }else if( isAdd && fsl_strcmp(zName, "closed")==0 ){ + rc = fsl_buffer_append(&comment, " Marked \"Closed\"", -1); + if( !rc && zValue && *zValue ){ + rc = fsl_buffer_appendf(&comment, " with note \"%h\"", zValue); + } + if(!rc) rc = fsl_buffer_append(&comment, ".", 1); + }else if( isCancel && fsl_strcmp(zName, "closed")==0 ){ + rc = fsl_buffer_append(&comment, " Removed the \"Closed\" mark", -1); + if( !rc && zValue && *zValue ){ + rc = fsl_buffer_appendf(&comment, " with note \"%h\"", zValue); + } + if(!rc) rc = fsl_buffer_append(&comment, ".", 1); + }else { + if( isCancel ){ + rc = fsl_buffer_appendf(&comment, " Cancel \"%h\"", zName); + }else if( isAdd ){ + rc = fsl_buffer_appendf(&comment, " Add \"%h\"", zName); + }else{ + assert(isProp); + rc = fsl_buffer_appendf(&comment, " Add propagating \"%h\"", zName); + } + if(rc) goto end; + if( zValue && zValue[0] ){ + rc = fsl_buffer_appendf(&comment, " with value \"%h\".", zValue); + }else{ + rc = fsl_buffer_append(&comment, ".", 1); + } + } + } /* foreach tag loop */ + if(!rc){ + /* TODO: cached statement */ + rc = fsl_db_exec(db, + "REPLACE INTO event" + "(type,mtime,objid,user,comment) " + "VALUES('g'," + "%"FSL_JULIAN_T_PFMT"," + "%"FSL_ID_T_PFMT"," + "%Q,%Q)", + mtime, d->rid, d->U, + (comment.used>1) + ? (fsl_buffer_cstr(&comment) + +1/*leading space on all entries*/) + : NULL); + } + + end: + fsl_buffer_clear(&comment); + return rc; + +} + +static int fsl_deck_xlink_f_forum(fsl_deck * d, void * state){ + if(FSL_SATYPE_FORUMPOST!=d->type) return 0; + int rc = 0; + fsl_db * const db = fsl_cx_db_repo(d->f); + assert(db); + fsl_cx * const f = d->f; + fsl_id_t const froot = d->G ? fsl_uuid_to_rid(f, d->G) : d->rid; + fsl_id_t const fprev = d->P.used ? fsl_uuid_to_rid(f, (char const *)d->P.list[0]): 0; + fsl_id_t const firt = d->I ? fsl_uuid_to_rid(f, d->I) : 0; + if( 0==firt ){ + /* This is the start of a new thread, either the initial entry + ** or an edit of the initial entry. */ + const char * zTitle = d->H; + const char * zFType; + if(!zTitle || !*zTitle){ + zTitle = "(Deleted)"; + } + zFType = fprev ? "Edit" : "Post"; + /* FSL-MISSING: + assert( manifest_event_triggers_are_enabled ); */ + rc = fsl_db_exec_multi(db, + "REPLACE INTO event(type,mtime,objid,user,comment)" + "VALUES('f',%"FSL_JULIAN_T_PFMT",%" FSL_ID_T_PFMT + ",%Q,'%q: %q')", + d->D, d->rid, d->U, zFType, zTitle); + if(rc) goto dberr; + /* + ** If this edit is the most recent, then make it the title for + ** all other entries for the same thread + */ + if( !fsl_db_exists(db,"SELECT 1 FROM forumpost " + "WHERE froot=%" FSL_ID_T_PFMT " AND firt=0" + " AND fpid!=%" FSL_ID_T_PFMT + " AND fmtime>%"FSL_JULIAN_T_PFMT, + froot, d->rid, d->D)){ + /* This entry establishes a new title for all entries on the thread */ + rc = fsl_db_exec_multi(db, + "UPDATE event" + " SET comment=substr(comment,1,instr(comment,':')) || ' %q'" + " WHERE objid IN (SELECT fpid FROM forumpost WHERE froot=% " FSL_ID_T_PFMT ")", + zTitle, froot); + if(rc) goto dberr; + } + }else{ + /* This is a reply to a prior post. Take the title from the root. */ + char const * zFType = 0; + char * zTitle = fsl_db_g_text( + db, 0, "SELECT substr(comment,instr(comment,':')+2)" + " FROM event WHERE objid=%"FSL_ID_T_PFMT, froot); + if( zTitle==0 ){ + zTitle = fsl_strdup("Unknown"); + if(!zTitle){ + rc = FSL_RC_OOM; + goto end; + } + } + if( !d->W.used ){ + zFType = "Delete reply"; + }else if( fprev ){ + zFType = "Edit reply"; + }else{ + zFType = "Reply"; + } + /* FSL-MISSING: + assert( manifest_event_triggers_are_enabled ); */ + rc = fsl_db_exec_multi(db, + "REPLACE INTO event(type,mtime,objid,user,comment)" + "VALUES('f',%"FSL_JULIAN_T_PFMT + ",%"FSL_ID_T_PFMT",%Q,'%q: %q')", + d->D, d->rid, d->U, zFType, zTitle); + fsl_free(zTitle); + if(rc) goto end; + if( d->W.used ){ + /* FSL-MISSING: + backlink_extract(&d->W, d->N, d->rid, BKLNK_FORUM, d->D, 1); */ + } + } + end: + return rc; + dberr: + assert(rc); + assert(db->error.code); + return fsl_cx_uplift_db_error(f, db); +} + + +static int fsl_deck_xlink_f_technote(fsl_deck * d, void * state){ + if(FSL_SATYPE_TECHNOTE!=d->type) return 0; + char buf[FSL_STRLEN_K256 + 7 /* event-UUID\0 */] = {0}; + fsl_id_t tagid; + char const * zTag; + int rc = 0; + fsl_cx * const f = d->f; + fsl_db * const db = fsl_cx_db_repo(d->f); + fsl_snprintf(buf, sizeof(buf), "event-%s", d->E.uuid); + zTag = buf; + tagid = fsl_tag_id( f, zTag, 1 ); + if(tagid<=0){ + return f->error.code ? f->error.code : + fsl_cx_err_set(f, FSL_RC_RANGE, + "Got unexpected RID (%"FSL_ID_T_PFMT") " + "for tag [%s].", + tagid, zTag); + } + fsl_id_t const subsequent + = fsl_db_g_id(db, 0, + "SELECT rid FROM tagxref" + " WHERE tagid=%"FSL_ID_T_PFMT + " AND mtime>=%"FSL_JULIAN_T_PFMT + " AND rid!=%"FSL_ID_T_PFMT + " ORDER BY mtime", + tagid, d->D, d->rid); + if(subsequent<0){ + rc = fsl_cx_uplift_db_error(d->f, db); + }else{ + rc = fsl_db_exec(db, + "REPLACE INTO event(" + "type,mtime," + "objid,tagid," + "user,comment,bgcolor" + ")VALUES(" + "'e',%"FSL_JULIAN_T_PFMT"," + "%"FSL_ID_T_PFMT",%"FSL_ID_T_PFMT"," + "%Q,%Q," + " (SELECT value FROM tagxref WHERE " + " tagid=%d" + " AND rid=%"FSL_ID_T_PFMT")" + ");", + d->E.julian, d->rid, tagid, + d->U, d->C, + (int)FSL_TAGID_BGCOLOR, d->rid); + } + return rc; +} + +static int fsl_deck_xlink_f_wiki(fsl_deck * d, void * state){ + if(FSL_SATYPE_WIKI!=d->type) return 0; + int rc; + char const * zWiki; + fsl_size_t nWiki = 0; + char cPrefix = 0; + char * zTag = fsl_mprintf("wiki-%s", d->L); + if(!zTag) return FSL_RC_OOM; + fsl_id_t const tagid = fsl_tag_id( d->f, zTag, 1 ); + if(tagid<=0){ + rc = fsl_cx_err_set(d->f, FSL_RC_ERROR, + "Tag [%s] must have been added by main wiki crosslink step.", + zTag); + goto end; + } + /* Some of this is duplicated in the main wiki crosslinking code :/. */ + zWiki = d->W.used ? fsl_buffer_cstr(&d->W) : ""; + while( *zWiki && fsl_isspace(*zWiki) ){ + ++zWiki; + /* Historical behaviour: strip leading spaces. */ + } + /* As of late 2020, fossil changed the conventions for how wiki + entries are to be added to the timeline. They requrie a prefix + character which tells the timeline display and email notification + generator code what type of change this is: create/update/delete */ + nWiki = fsl_strlen(zWiki); + if(!nWiki) cPrefix = '-'; + else if( !d->P.used ) cPrefix = '+'; + else cPrefix = ':'; + fsl_db * const db = fsl_cx_db_repo(d->f); + rc = fsl_db_exec(db, + "REPLACE INTO event(type,mtime,objid,user,comment) " + "VALUES('w',%"FSL_JULIAN_T_PFMT + ",%"FSL_ID_T_PFMT",%Q,'%c%q%q%q');", + d->D, d->rid, d->U, cPrefix, d->L, + ((d->C && *d->C) ? ": " : ""), + ((d->C && *d->C) ? d->C : "")); + /* Note that wiki pages optionally support d->C (change comment), + but it's historically unused because it was a late addition to + the artifact format and is not supported by older fossil + versions. */ + rc = fsl_cx_uplift_db_error2(d->f, db, rc); + end: + fsl_free(zTag); + return rc; +} + + +/** @internal + + Installs the core overridable crosslink listeners. "The plan" is + to do all updates to the event (timeline) table via these + crosslinkers and perform the core, UI-agnostic, crosslinking bits + in the internal fsl_deck_crosslink_XXX() functions. That should + allow clients to override how the timeline is updated without + requiring them to understand the rest of the required schema + updates. +*/ +int fsl_cx_install_timeline_crosslinkers(fsl_cx * const f){ + int rc; + assert(!f->xlinkers.used); + assert(!f->xlinkers.list); + rc = fsl_xlink_listener(f, "fsl/attachment/timeline", + fsl_deck_xlink_f_attachment, 0); + if(!rc) rc = fsl_xlink_listener(f, "fsl/checkin/timeline", + fsl_deck_xlink_f_checkin, 0); + if(!rc) rc = fsl_xlink_listener(f, "fsl/control/timeline", + fsl_deck_xlink_f_control, 0); + if(!rc) rc = fsl_xlink_listener(f, "fsl/forumpost/timeline", + fsl_deck_xlink_f_forum, 0); + if(!rc) rc = fsl_xlink_listener(f, "fsl/technote/timeline", + fsl_deck_xlink_f_technote, 0); + if(!rc) rc = fsl_xlink_listener(f, "fsl/wiki/timeline", + fsl_deck_xlink_f_wiki, 0); + return rc; +} + + +static int fsl_deck_crosslink_checkin(fsl_deck * const d, + fsl_id_t *parentid ){ + int rc = 0; + fsl_cx * const f = d->f; + fsl_db * const db = fsl_cx_db_repo(f); + + /* TODO: convert these queries to cached statements, for + the sake of rebuild and friends. And bind() doubles + instead of %FSL_JULIAN_T_PFMT'ing them. + */ + if(d->Q.used && fsl_db_table_exists(db, FSL_DBROLE_REPO, + "cherrypick")){ + fsl_size_t i; + for(i=0; i < d->Q.used; ++i){ + fsl_card_Q const * q = (fsl_card_Q const *)d->Q.list[i]; + rc = fsl_db_exec(db, + "REPLACE INTO cherrypick(parentid,childid,isExclude)" + " SELECT rid, %"FSL_ID_T_PFMT", %d" + " FROM blob WHERE uuid=%Q", + d->rid, q->type<0 ? 1 : 0, q->target + ); + if(rc) goto end; + } + } + if(!fsl_repo_has_mlink_mid(db, d->rid)){ + rc = fsl_deck_add_checkin_linkages(d, parentid); + if(rc) goto end; + /* FSL-MISSING: + assert( manifest_event_triggers_are_enabled ); */ + rc = fsl_search_doc_touch(f, d->type, d->rid, 0); + if(rc) goto end; + /* If this is a delta-manifest, record the fact that this repository + contains delta manifests, to free the "commit" logic to generate + new delta manifests. */ + if(d->B.uuid){ + rc = fsl_cx_update_seen_delta_mf(f); + if(rc) goto end; + } + assert(!rc); + }/*!exists mlink*/ + end: + if(rc && !f->error.code && db->error.code){ + fsl_cx_uplift_db_error(f, db); + } + return rc; +} + +static int fsl_deck_crosslink_wiki(fsl_deck *d){ + char zLength[40] = {0}; + fsl_id_t prior = 0; + char const * zWiki; + fsl_size_t nWiki = 0; + int rc; + char * zTag = fsl_mprintf("wiki-%s", d->L); + fsl_cx * const f = d->f; + fsl_db * const db = fsl_cx_db_repo(f); + if(!zTag){ + return FSL_RC_OOM; + } + assert(f && db); + zWiki = d->W.used ? fsl_buffer_cstr(&d->W) : ""; + while( *zWiki && fsl_isspace(*zWiki) ){ + ++zWiki; + /* Historical behaviour: strip leading spaces. */ + } + nWiki = fsl_strlen(zWiki) + /* Reminder: use strlen instead of d->W.used just in case that + one contains embedded NULs in the content. "Shouldn't + happen," but the API doesn't explicitly prohibit it. + */; + fsl_snprintf(zLength, sizeof(zLength), "%"FSL_SIZE_T_PFMT, + (fsl_size_t)nWiki); + rc = fsl_tag_insert(f, FSL_TAGTYPE_ADD, zTag, zLength, + d->rid, d->D, d->rid, NULL ); + if(rc) goto end; + if(d->P.used){ + prior = fsl_uuid_to_rid(f, (const char *)d->P.list[0]); + } + if(prior>0){ + rc = fsl_content_deltify(f, prior, d->rid, 0); + if(rc) goto end; + } + rc = fsl_search_doc_touch(f, d->type, d->rid, d->L); + if(rc) goto end; + if( f->cache.isCrosslinking ){ + rc = fsl_deck_crosslink_add_pending(f, 'w',d->L); + if(rc) goto end; + }else{ + /* FSL-MISSING: + backlink_wiki_refresh(d->L); */ + } + assert(0==rc); + rc = fsl_deck_crosslink_fwt_plink(d); + end: + fsl_free(zTag); + return rc; +} + +static int fsl_deck_crosslink_attachment(fsl_deck * const d){ + int rc; + fsl_cx * const f = d->f; + fsl_db * const db = fsl_cx_db_repo(f); + + rc = fsl_db_exec(db, + /* REMINDER: fossil(1) uses INSERT here, but that + breaks libfossil crosslinking tests due to a + unique constraint violation on attachid. */ + "REPLACE INTO attachment(attachid, mtime, src, target," + "filename, comment, user) VALUES(" + "%"FSL_ID_T_PFMT",%"FSL_JULIAN_T_PFMT"," + "%Q,%Q,%Q," + "%Q,%Q);", + d->rid, d->D, + d->A.src, d->A.tgt, d->A.name, + (d->C ? d->C : ""), d->U); + if(!rc){ + rc = fsl_db_exec(db, + "UPDATE attachment SET isLatest = (mtime==" + "(SELECT max(mtime) FROM attachment" + " WHERE target=%Q AND filename=%Q))" + " WHERE target=%Q AND filename=%Q", + d->A.tgt, d->A.name, + d->A.tgt, d->A.name); + } + return rc; +} + +static int fsl_deck_crosslink_cluster(fsl_deck * const d){ + /* Clean up the unclustered table... */ + fsl_size_t i; + fsl_stmt * st = NULL; + int rc; + fsl_cx * const f = d->f; + fsl_db * const db = fsl_cx_db_repo(f); + + rc = fsl_db_prepare_cached(db, &st, + "DELETE FROM unclustered WHERE rid=?" + "/*%s()*/",__func__); + if(rc) return fsl_cx_uplift_db_error(f, db); + assert(st); + for( i = 0; i < d->M.used; ++i ){ + fsl_id_t mid; + char const * uuid = (char const *)d->M.list[i]; + mid = fsl_uuid_to_rid(f, uuid); + if(mid>0){ + fsl_stmt_bind_id(st, 1, mid); + if(FSL_RC_STEP_DONE!=fsl_stmt_step(st)){ + rc = fsl_cx_uplift_db_error(f, db); + break; + } + fsl_stmt_reset(st); + } + } + fsl_stmt_cached_yield(st); + return rc; +} + +#if 0 +static int fsl_deck_crosslink_control(fsl_deck *const d){ +} +#endif + +static int fsl_deck_crosslink_forum(fsl_deck * const d){ + int rc = 0; + fsl_cx * const f = d->f; + rc = fsl_repo_install_schema_forum(f); + if(rc) return rc; + fsl_db * const db = fsl_cx_db_repo(f); + fsl_id_t const froot = d->G ? fsl_uuid_to_rid(f, d->G) : d->rid; + fsl_id_t const fprev = d->P.used ? fsl_uuid_to_rid(f, (char const *)d->P.list[0]): 0; + fsl_id_t const firt = d->I ? fsl_uuid_to_rid(f, d->I) : 0; + assert(f && db); + rc = fsl_db_exec_multi(db, + "REPLACE INTO forumpost(fpid,froot,fprev,firt,fmtime)" + "VALUES(%" FSL_ID_T_PFMT ",%" FSL_ID_T_PFMT "," + "nullif(%" FSL_ID_T_PFMT ",0)," + "nullif(%" FSL_ID_T_PFMT ",0),%"FSL_JULIAN_T_PFMT")", + d->rid, froot, fprev, firt, d->D + ); + rc = fsl_cx_uplift_db_error2(f, db, rc); + if(!rc){ + rc = fsl_search_doc_touch(f, d->type, d->rid, 0); + } + if(!rc){ + rc = fsl_deck_crosslink_fwt_plink(d); + } + return rc; +} + +static int fsl_deck_crosslink_technote(fsl_deck * const d){ + char buf[FSL_STRLEN_K256 + 7 /* event-UUID\0 */] = {0}; + char zLength[40] = {0}; + fsl_id_t tagid; + fsl_id_t prior = 0, subsequent; + char const * zWiki; + char const * zTag; + fsl_size_t nWiki = 0; + int rc; + fsl_cx * const f = d->f; + fsl_db * const db = fsl_cx_db_repo(f); + fsl_snprintf(buf, sizeof(buf), "event-%s", d->E.uuid); + zTag = buf; + tagid = fsl_tag_id( f, zTag, 1 ); + if(tagid<=0){ + rc = f->error.code ? f->error.code : + fsl_cx_err_set(f, FSL_RC_RANGE, + "Got unexpected RID (%"FSL_ID_T_PFMT") " + "for tag [%s].", + tagid, zTag); + goto end; + } + zWiki = d->W.used ? fsl_buffer_cstr(&d->W) : ""; + while( *zWiki && fsl_isspace(*zWiki) ){ + ++zWiki; + /* Historical behaviour: strip leading spaces. */ + } + nWiki = fsl_strlen(zWiki); + fsl_snprintf( zLength, sizeof(zLength), "%"FSL_SIZE_T_PFMT, + (fsl_size_t)nWiki); + rc = fsl_tag_insert(f, FSL_TAGTYPE_ADD, zTag, zLength, + d->rid, d->D, d->rid, NULL ); + if(rc) goto end; + if(d->P.used){ + prior = fsl_uuid_to_rid(f, (const char *)d->P.list[0]); + if(prior<0){ + assert(f->error.code); + rc = f->error.code; + goto end; + } + } + subsequent = fsl_db_g_id(db, 0, + /* BUG: see: + https://fossil-scm.org/forum/forumpost/c58fd8de53 */ + "SELECT rid FROM tagxref" + " WHERE tagid=%"FSL_ID_T_PFMT + " AND mtime>=%"FSL_JULIAN_T_PFMT + " AND rid!=%"FSL_ID_T_PFMT + " ORDER BY mtime", + tagid, d->D, d->rid); + if(subsequent<0){ + assert(db->error.code); + rc = fsl_cx_uplift_db_error(f, db); + goto end; + } + else if( prior > 0 ){ + rc = fsl_content_deltify(f, prior, d->rid, 0); + if( !rc && !subsequent ){ + rc = fsl_db_exec(db, + "DELETE FROM event" + " WHERE type='e'" + " AND tagid=%"FSL_ID_T_PFMT + " AND objid IN" + " (SELECT rid FROM tagxref " + " WHERE tagid=%"FSL_ID_T_PFMT")", + tagid, tagid); + } + } + if(rc) goto end; + if( subsequent>0 ){ + rc = fsl_content_deltify(f, d->rid, subsequent, 0); + }else{ + /* timeline update is deferred to another crosslink + handler */ + rc = fsl_search_doc_touch(f, d->type, d->rid, 0); + /* FSL-MISSING: + assert( manifest_event_triggers_are_enabled ); */ + } + if(!rc){ + rc = fsl_deck_crosslink_fwt_plink(d); + } + end: + return rc; +} + +static int fsl_deck_crosslink_ticket(fsl_deck * const d){ + int rc; + fsl_cx * const f = d->f; +#if 0 + fsl_db * const db = fsl_cx_db_repo(f); +#endif + /* + TODO: huge block from manifest_crosslink(). A full port + requires other infrastructure for collapsing relatively close + time values into the same time for timeline purposes. i'd prefer + to farm this out to a crosslink callback. Even then, the future + of tickets in libfossil is uncertain, but we should crosslink + them so that repos stay compatible with fossil(1) without + requiring a rebuild using fossil(1). + */ + if(f->flags & FSL_CX_F_SKIP_UNKNOWN_CROSSLINKS){ + return 0; + } + rc = fsl_cx_err_set(f, FSL_RC_NYI, + "MISSING: a huge block of TICKET stuff from " + "manifest_crosslink(). It requires infrastructure " + "libfossil does not yet have."); + return rc; +} + +int fsl_deck_crosslink_one( fsl_deck * const d ){ + int rc; + if(!d->f) return FSL_RC_MISUSE; + rc = fsl_crosslink_begin(d->f); + if(rc) return rc; + rc = fsl_deck_crosslink(d); + if(rc){ + fsl_db_transaction_rollback(fsl_cx_db_repo(d->f)) + /* Ignore result - keep existing error state */; + d->f->cache.isCrosslinking = false; + }else{ + assert(fsl_db_transaction_level(fsl_cx_db_repo(d->f))); + rc = fsl_crosslink_end(d->f); + } + return rc; +} + +int fsl_deck_crosslink( fsl_deck /* const */ * const d ){ + int rc = 0; + fsl_cx * f = d->f; + fsl_db * db = f ? fsl_needs_repo(f) : NULL; + fsl_id_t parentid = 0; + fsl_int_t const rid = d->rid; + if(!f) return FSL_RC_MISUSE; + else if(rid<=0){ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "Invalid RID for crosslink: %"FSL_ID_T_PFMT, + rid); + } + else if(!db) return FSL_RC_NOT_A_REPO; + else if(!fsl_deck_has_required_cards(d)){ + assert(d->f->error.code); + return d->f->error.code; + }else if(f->cache.xlinkClustersOnly && (FSL_SATYPE_CLUSTER!=d->type)){ + /* is it okay to bypass the registered xlink listeners here? The + use case called for by this is not yet implemented in + libfossil. */ + return 0; + } + rc = fsl_db_transaction_begin(db); + if(rc) goto end; + if(FSL_SATYPE_CHECKIN==d->type + && d->B.uuid && !d->B.baseline){ + rc = fsl_deck_baseline_fetch(d); + if(rc) goto end; + assert(d->B.baseline); + } + switch(d->type){ + case FSL_SATYPE_CHECKIN: + rc = fsl_deck_crosslink_checkin(d, &parentid); + break; + case FSL_SATYPE_CLUSTER: + rc = fsl_deck_crosslink_cluster(d); + break; + default: + break; + } + if(rc) goto end; + switch(d->type){ + case FSL_SATYPE_CONTROL: + case FSL_SATYPE_CHECKIN: + case FSL_SATYPE_TECHNOTE: + rc = fsl_deck_crosslink_apply_tags(f, d, db, rid, parentid); + break; + default: + break; + } + if(rc) goto end; + switch(d->type){ + case FSL_SATYPE_WIKI: + rc = fsl_deck_crosslink_wiki(d); + break; + case FSL_SATYPE_FORUMPOST: + rc = fsl_deck_crosslink_forum(d); + break; + case FSL_SATYPE_TECHNOTE: + rc = fsl_deck_crosslink_technote(d); + break; + case FSL_SATYPE_TICKET: + rc = fsl_deck_crosslink_ticket(d); + break; + case FSL_SATYPE_ATTACHMENT: + rc = fsl_deck_crosslink_attachment(d); + break; + /* FSL_SATYPE_CONTROL is handled above except for the timeline + update, which is handled by a callback below */ + default: + break; + } + if(rc) goto end; + + /* Call any crosslink callbacks... */ + if(f->xlinkers.list){ + fsl_size_t i; + fsl_xlinker * xl = NULL; + for( i = 0; !rc && (i < f->xlinkers.used); ++i ){ + xl = f->xlinkers.list+i; + rc = xl->f( d, xl->state ); + } + if(rc){ + assert(xl); + if(!f->error.code){ + fsl_cx_err_set(f, rc, "Crosslink callback handler " + "'%s' failed with code %d (%s) for " + "artifact RID #%" FSL_ID_T_PFMT ".", + xl->name, rc, fsl_rc_cstr(rc), + d->rid); + } + } + }/*end crosslink callbacks*/ + end: + if(!rc){ + rc = fsl_db_transaction_end(db, false); + }else{ + if(db->error.code && !f->error.code){ + fsl_cx_uplift_db_error(f,db); + } + fsl_db_transaction_end(db, true); + } + return rc; +}/*end fsl_deck_crosslink()*/ + + + +/** + Return true if z points to the first character after a blank line. + Tolerate either \r\n or \n line endings. As this looks backwards + in z, z must point to at least 3 characters past the beginning of + a legal string. + */ +static bool fsl_after_blank_line(const char *z){ + if( z[-1]!='\n' ) return false; + if( z[-2]=='\n' ) return true; + if( z[-2]=='\r' && z[-3]=='\n' ) return true; + return false; +} + +/** + Verifies that ca points to at least 35 bytes of memory + which hold (at the end) a Z card and its hash value. + + Returns 0 if the string does not contain a Z card, + a positive value if it can validate the Z card's hash, + and a negative value on hash mismatch. +*/ +static int fsl_deck_verify_Z_card(unsigned char const * ca, fsl_size_t n){ + if( n<35 ) return 0; + if( ca[n-35]!='Z' || ca[n-34]!=' ' ) return 0; + else{ + unsigned char digest[16]; + char hex[FSL_STRLEN_MD5+1]; + unsigned char const * zHash = ca+n-FSL_STRLEN_MD5-1; + fsl_md5_cx md5 = fsl_md5_cx_empty; + unsigned char const * zHashEnd = + ca + n - + 2 /* 'Z ' */ + - FSL_STRLEN_MD5 + - 1 /* \n */; + assert( 'Z' == (char)*zHashEnd ); + fsl_md5_update(&md5, ca, zHashEnd-ca); + fsl_md5_final(&md5, digest); + fsl_md5_digest_to_base16(digest, hex); + return (0==memcmp(zHash, hex, FSL_STRLEN_MD5)) + ? 1 + : -1; + } +} + +void fsl_remove_pgp_signature(unsigned char const **pz, fsl_size_t *pn){ + unsigned char const *z = *pz; + fsl_int_t n = (fsl_int_t)*pn; + fsl_int_t i; + if( n<59 || memcmp(z, "-----BEGIN PGP SIGNED MESSAGE-----", 34)!=0 ) return; + for(i=34; i=n ) return; + z += i; + n -= i; + *pz = z; + for(i=n-1; i>=0; i--){ + if( z[i]=='\n' && memcmp(&z[i],"\n-----BEGIN PGP SIGNATURE-", 25)==0 ){ + n = i+1; + break; + } + } + *pn = (fsl_size_t)n; + return; +} + + +/** + Internal helper for parsing manifests. Holds a source file (memory + range) and gets updated by fsl_deck_next_token() and friends. +*/ +struct fsl_src { + /** + First char of the next token. + */ + unsigned char * z; + /** + One-past-the-end of the manifest. + */ + unsigned char * zEnd; + /** + True if z points to the start of a new line. + */ + char atEol; +}; +typedef struct fsl_src fsl_src; +static const fsl_src fsl_src_empty = {NULL,NULL,0}; + +/** + Return a pointer to the next token. The token is zero-terminated. + Return NULL if there are no more tokens on the current line. If + pLen is not NULL and this function returns non-NULL then *pLen is + set to the byte length of the new token. +*/ +static unsigned char *fsl_deck_next_token(fsl_src *p, fsl_size_t *pLen){ + unsigned char *z; + unsigned char *zStart; + int c; + if( p->atEol ) return NULL; + zStart = z = p->z; + while( (c=(*z))!=' ' && c!='\n' ){ ++z; } + *z = 0; + p->z = &z[1]; + p->atEol = c=='\n'; + if( pLen ) *pLen = z - zStart; + return zStart; +} + +/** + Return the card-type for the next card. Return 0 if there are no + more cards or if we are not at the end of the current card. + */ +static unsigned char mf_next_card(fsl_src *p){ + unsigned char c; + if( !p->atEol || p->z>=p->zEnd ) return 0; + c = p->z[0]; + if( p->z[1]==' ' ){ + p->z += 2; + p->atEol = 0; + }else if( p->z[1]=='\n' ){ + p->z += 2; + p->atEol = 1; + }else{ + c = 0; + } + return c; +} + +/** + Internal helper for fsl_deck_parse(). Expects l to be an array of + 26 entries, representing the letters of the alphabet (A-Z), with a + value of 0 if the card was not seen during parsing and a value >0 + if it was. Returns the deduced artifact type. Returns + FSL_SATYPE_ANY if the result is ambiguous. + + Note that we cannot reliably guess until we've seen at least 3 + cards. 2 cards is enough for most cases but can lead to + FSL_SATYPE_CHECKIN being prematurely selected in one case. + + It should guess right for any legal manifests, but it does not go + out of its way to detect incomplete/invalid ones. + */ +static fsl_satype_e fsl_deck_guess_type( const int * l ){ +#if 0 + /* For parser testing only... */ + int i; + assert(!l[26]); + MARKER(("Cards seen during parse:\n")); + for( i = 0; i < 26; ++i ){ + if(l[i]) putchar('A'+i); + } + putchar('\n'); +#endif + /* + Now look for combinations of cards which will uniquely + identify any syntactical legal combination of cards. + + A larger brain than mine could probably come up with a hash of + l[] which could determine this in O(1). But please don't + reimplement this as such unless mere mortals can maintain it - + any performance gain is insignificant in the context of the + underlying SCM/db operations. + + Note that the order of these checks is sometimes significant! + */ +#define L(X) l[X-'A'] + if(L('M')) return FSL_SATYPE_CLUSTER; + else if(L('E')) return FSL_SATYPE_EVENT; + else if(L('G') || L('H') || L('I')) return FSL_SATYPE_FORUMPOST; + else if(L('L') || L('W')) return FSL_SATYPE_WIKI; + else if(L('J') || L('K')) return FSL_SATYPE_TICKET; + else if(L('A')) return FSL_SATYPE_ATTACHMENT; + else if(L('B') || L('C') || L('F') + || L('P') || L('Q') || L('R')) return FSL_SATYPE_CHECKIN; + else if(L('D') && L('T') && L('U')) return FSL_SATYPE_CONTROL; +#undef L + return FSL_SATYPE_ANY; +} + +bool fsl_might_be_artifact(fsl_buffer const * src){ + unsigned const char * z = src->mem; + fsl_size_t n = src->used; + if(n<36) return 0; + fsl_remove_pgp_signature(&z, &n); + if(n<36) return 0; + else if(z[0]<'A' || z[0]>'Z' || z[1]!=' ' + || z[n-35]!='Z' + || z[n-34]!=' ' + || !fsl_validate16((const char *)z+n-33, FSL_STRLEN_MD5)){ + return 0; + } + return 1; +} + +int fsl_deck_parse2(fsl_deck * const d, fsl_buffer * const src, fsl_id_t rid){ +#ifdef ERROR +# undef ERROR +#endif +#define ERROR(RC,MSG) do{ rc = (RC); zMsg = (MSG); goto bailout; } while(0) +#define SYNTAX(MSG) ERROR(rc ? rc : FSL_RC_SYNTAX,MSG) + bool isRepeat = 0/* , hasSelfRefTag = 0 */; + int rc = 0; + fsl_src x = fsl_src_empty; + char const * zMsg = NULL; + fsl_id_bag * seen; + char cType = 0, cPrevType = 0; + unsigned char * z = src ? src->mem : NULL; + fsl_size_t tokLen = 0; + unsigned char * token; + fsl_size_t n = z ? src->used : 0; + unsigned char * uuid; + double ts; + int cardCount = 0; + fsl_cx * f; + fsl_error * err; + int stealBuf = 0 /* gets incremented if we need to steal src->mem. */; + unsigned nSelfTag = 0 /* number of T cards which refer to '*' (this artifact). */; + unsigned nSimpleTag = 0 /* number of T cards with "+" prefix */; + /* + lettersSeen keeps track of the card letters we have seen so that + we can then relatively quickly figure out what type of manifest we + have parsed without having to inspect the card contents. Each + index records a count of how many of that card we've seen. + */ + int lettersSeen[27] = {0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0}; + if(!d->f || !z) return FSL_RC_MISUSE; + /* Every control artifact ends with a '\n' character. Exit early + if that is not the case for this artifact. */ + f = d->f; + err = &f->error; + if(!*z || !n || ( '\n' != z[n-1]) ){ + return fsl_error_set(err, FSL_RC_SYNTAX, "%s.", + n ? "Not terminated with \\n" + : "Zero-length input"); + } + else if(rid<0){ + return fsl_error_set(err, FSL_RC_RANGE, + "Invalid (negative) RID %"FSL_ID_T_PFMT + " for fsl_deck_parse()", rid); + } + seen = &f->cache.mfSeen; + if((0==rid) || fsl_id_bag_contains(seen,rid)){ + isRepeat = 1; + }else{ + isRepeat = 0; + rc = fsl_id_bag_insert(seen, rid); + if(rc){ + assert(FSL_RC_OOM==rc); + return rc; + } + } + fsl_deck_clean(d); + fsl_deck_init(f, d, FSL_SATYPE_ANY); + + /* + Verify that the first few characters of the artifact look like a + control artifact. + */ + if( !fsl_might_be_artifact(src) ){ + ERROR(FSL_RC_SYNTAX, "Content does not look like " + "a structural artifact"); + } + + /* + Strip off the PGP signature if there is one. Example of signed + manifest: + + https://fossil-scm.org/index.html/artifact/28987096ac + */ + { + unsigned char const * zz = z; + fsl_remove_pgp_signature(&zz, &n); + z = (unsigned char *)zz; + } + + /* Verify the Z card */ + if( fsl_deck_verify_Z_card(z, n) < 0 ){ + ERROR(FSL_RC_CONSISTENCY, "Z-card checksum mismatch"); + } + + /* legacy: not yet clear if we need this: + if( !isRepeat ) g.parseCnt[0]++; */ + + /* + Reminder: parsing modifies the input (to simplify the + tokenization/parsing). + + As of mid-201403, we recycle as much as possible from the source + buffer and take over ownership _if_ we do so. + */ + /* Now parse, card by card... */ + x.z = z; + x.zEnd = z+n; + x.atEol= 1; + + /* Parsing helpers... */ +#define TOKEN(DEFOS) tokLen=0; token = fsl_deck_next_token(&x,&tokLen); \ + if(token && tokLen && (DEFOS)) fsl_bytes_defossilize(token, &tokLen) +#define TOKEN_EXISTS(MSG_IF_NOT) if(!token){ SYNTAX(MSG_IF_NOT); }(void)0 +#define TOKEN_CHECKHEX(MSG) if(token && (int)tokLen!=fsl_is_uuid((char const *)token))\ + { SYNTAX(MSG); } +#define TOKEN_UUID(CARD) TOKEN_CHECKHEX("Malformed UUID in " #CARD "-card") +#define TOKEN_MD5(ERRMSG) if(!token || FSL_STRLEN_MD5!=(int)tokLen) \ + {SYNTAX(ERRMSG);} + /** + Reminder: we do not know the type of the manifest at this point, + so all of the fsl_deck_add/set() bits below can't do their + validation. We have to determine at parse-time (or afterwards) + which type of deck it is based on the cards we've seen. We guess + the type as early as possible to enable during-parse validation, + and do a post-parse check for the legality of cards added before + validation became possible. + */ + +#define SEEN(CARD) lettersSeen[*#CARD - 'A'] + for( cPrevType=1; !rc && (0 < (cType = mf_next_card(&x))); + cPrevType = cType ){ + ++cardCount; + if(cTypeE.uuid && 'N'==cType && 'P'==cPrevType){ + /* Workaround for a pair of historical fossil bugs + which synergized to allow malformed technotes to + be saved: + https://fossil-scm.org/home/info/023fddeec4029306 */ + }else{ + SYNTAX("Cards are not in strict lexical order"); + } + } + assert(cType>='A' && cType<='Z'); + if(cType>='A' && cType<='Z'){ + ++lettersSeen[cType-'A']; + }else{ + SYNTAX("Invalid card name"); + } + switch(cType){ + /* + A ?? + + Identifies an attachment to either a wiki page, a ticket, or + a technote. is the artifact that is the attachment. + is omitted to delete an attachment. is the + name of a wiki page, technote, or ticket to which that + attachment is connected. + */ + case 'A':{ + unsigned char * name, * src; + if(1A.name = (char *)name; + d->A.tgt = (char *)uuid; + d->A.src = (char *)src; + ++stealBuf; + /*rc = fsl_deck_A_set(d, (char const *)name, + (char const *)uuid, (char const *)src);*/ + d->type = FSL_SATYPE_ATTACHMENT; + break; + } + /* + B + + A B-line gives the UUID for the baseline of a delta-manifest. + */ + case 'B':{ + if(d->B.uuid){ + SYNTAX("Multiple B-cards"); + } + TOKEN(0); + TOKEN_UUID(B); + d->B.uuid = (char *)token; + ++stealBuf; + d->type = FSL_SATYPE_CHECKIN; + /* rc = fsl_deck_B_set(d, (char const *)token); */ + break; + } + /* + C + + Comment text is fossil-encoded. There may be no more than + one C line. C lines are required for manifests, are optional + for Events and Attachments, and are disallowed on all other + control files. + */ + case 'C':{ + if( d->C ){ + SYNTAX("more than one C-card"); + } + TOKEN(1); + TOKEN_EXISTS("Missing comment text for C-card"); + /* rc = fsl_deck_C_set(d, (char const *)token, (fsl_int_t)tokLen); */ + d->C = (char *)token; + ++stealBuf; + break; + } + /* + D + + The timestamp should be ISO 8601. YYYY-MM-DDtHH:MM:SS + There can be no more than 1 D line. D lines are required + for all control files except for clusters. + */ + case 'D':{ +#define TOKEN_DATETIME(LETTER,MEMBER) \ + if( d->MEMBER>0.0 ) { SYNTAX("More than one "#LETTER"-card"); } \ + TOKEN(0); \ + TOKEN_EXISTS("Missing date part of "#LETTER"-card"); \ + if(!fsl_str_is_date((char const *)token)){\ + SYNTAX("Malformed date part of "#LETTER"-card"); \ + } \ + if(!fsl_iso8601_to_julian((char const *)token, &ts)){ \ + SYNTAX("Cannot parse date from "#LETTER"-card"); \ + } (void)0 + + TOKEN_DATETIME(D,D); + rc = fsl_deck_D_set(d, ts); + break; + } + /* + E + + An "event" card that contains the timestamp of the event in the + format YYYY-MM-DDtHH:MM:SS and a unique identifier for the event. + The event timestamp is distinct from the D timestamp. The D + timestamp is when the artifact was created whereas the E timestamp + is when the specific event is said to occur. + */ + case 'E':{ + TOKEN_DATETIME(E,E.julian); + TOKEN(0); + TOKEN_EXISTS("Missing UUID part of E-card"); + TOKEN_UUID(E); + d->E.julian = ts; + d->E.uuid = (char *)token; + ++stealBuf; + d->type = FSL_SATYPE_EVENT; + break; + } + /* + F ?? ?? ?? + + Identifies a file in a manifest. Multiple F lines are + allowed in a manifest. F lines are not allowed in any other + control file. The filename and old-name are fossil-encoded. + + In delta manifests, deleted files are denoted by the 1-arg + form. In baseline manifests, deleted files simply are not in + the manifest. + */ + case 'F':{ + char * name; + char * perms = NULL; + char * priorName = NULL; + fsl_fileperm_e perm = FSL_FILE_PERM_REGULAR; + fsl_card_F * fc = NULL; + /** + Basic tests with various repos have shown that the + approximate number of F-cards in a manifest is rougly the + manifest size/75. We'll use that as an initial alloc size. + */ + rc = 0; + if(!d->F.capacity){ + rc = fsl_card_F_list_reserve(&d->F, src->used/75+10); + } + TOKEN(0); + TOKEN_EXISTS("Missing name for F-card"); + name = (char *)token; + TOKEN(0); + TOKEN_UUID(F); + uuid = token; + TOKEN(0); + if(token){ + perms = (char *)token; + switch(*perms){ + case 0: + /* Some (maybe only 1) ancient fossil(1) artifact(s) have a trailing + space which triggers this. e.g. + + https://fossil-scm.org/home/info/32b480faa3465591b8549bdfd889d62d7a8d16a8 + */ + break; + case 'w': perm = FSL_FILE_PERM_REGULAR; break; + case 'x': perm = FSL_FILE_PERM_EXE; break; + case 'l': perm = FSL_FILE_PERM_LINK; break; + default: + /*MARKER(("Unmatched perms string character: %d / %c !", (int)*perms, *perms));*/ + assert(!"Unmatched perms string character!"); + ERROR(FSL_RC_ERROR,"Internal error: unmatched perms string character"); + } + TOKEN(0); + if(token) priorName = (char *)token; + } + fsl_bytes_defossilize( (unsigned char *)name, 0 ); + if(priorName) fsl_bytes_defossilize( (unsigned char *)priorName, 0 ); + if(fsl_is_reserved_fn(name, -1)){ + /* Some historical (pre-late-2020) manifests contain files + they really shouldn't, like _FOSSIL_ and .fslckout. + Since late 2020, fossil simply skips over these when + parsing manifests, so we'll do the same. */ + break; + } + fc = rc ? 0 : fsl_card_F_list_push(&d->F); + if(!fc){ + zMsg = "OOM"; + goto bailout; + } + ++stealBuf; + assert(d->F.used>1 + ? (FSL_CARD_F_LIST_NEEDS_SORT & d->F.flags) + : 1); + fc->deckOwnsStrings = true; + fc->name = name; + fc->priorName = priorName; + fc->perm = perm; + fc->uuid = (fsl_uuid_str)uuid; + d->type = FSL_SATYPE_CHECKIN; + break; + } + /* + G + + A G-line gives the UUID for the thread root of a forum post. + */ + case 'G':{ + if(d->G){ + SYNTAX("Multiple G-cards"); + } + TOKEN(0); + TOKEN_EXISTS("Missing UUID in G-card"); + TOKEN_UUID(G); + d->G = (char*)token; + ++stealBuf; + d->type = FSL_SATYPE_FORUMPOST; + break; + } + /* + H + + H text is fossil-encoded. There may be no more than one H + line. H lines are optional for forum posts and are + disallowed on all other control files. + */ + case 'H':{ + if( d->H ){ + SYNTAX("more than one H-card"); + } + TOKEN(1); + TOKEN_EXISTS("Missing text for H-card"); + d->H = (char *)token; + ++stealBuf; + d->type = FSL_SATYPE_FORUMPOST; + break; + } + /* + I + + A I-line gives the UUID for the in-response-to UUID for + a forum post. + */ + case 'I':{ + if(d->I){ + SYNTAX("Multiple I-cards"); + } + TOKEN(0); + TOKEN_EXISTS("Missing UUID in I-card"); + TOKEN_UUID(I); + d->I = (char*)token; + ++stealBuf; + d->type = FSL_SATYPE_FORUMPOST; + break; + } + /* + J ?? + + Specifies a name value pair for ticket. If the first character + of is "+" then the is appended to any preexisting + value. If is omitted then it is understood to be an + empty string. + */ + case 'J':{ + char const * field; + bool isAppend = 0; + TOKEN(1); + TOKEN_EXISTS("Missing field name for J-card"); + field = (char const *)token; + if('+'==*field){ + isAppend = 1; + ++field; + } + TOKEN(1); + rc = fsl_deck_J_add(d, isAppend, field, + (char const *)token); + d->type = FSL_SATYPE_TICKET; + break; + } + /* + K + + A K-line gives the UUID for the ticket which this control file + is amending. + */ + case 'K':{ + if(d->K){ + SYNTAX("Multiple K-cards"); + } + TOKEN(0); + TOKEN_EXISTS("Missing UUID in K-card"); + TOKEN_UUID(K); + d->K = (char*)token; + ++stealBuf; + d->type = FSL_SATYPE_TICKET; + break; + } + /* + L + + The wiki page title is fossil-encoded. There may be no more than + one L line. + */ + case 'L':{ + if(d->L){ + SYNTAX("Multiple L-cards"); + } + TOKEN(1); + TOKEN_EXISTS("Missing text for L-card"); + d->L = (char*)token; + ++stealBuf; + d->type = FSL_SATYPE_WIKI; + break; + } + /* + M + + An M-line identifies another artifact by its UUID. M-lines + occur in clusters only. + */ + case 'M':{ + TOKEN(0); + TOKEN_EXISTS("Missing UUID for M-card"); + TOKEN_UUID(M); + ++stealBuf; + d->type = FSL_SATYPE_CLUSTER; + rc = fsl_list_append(&d->M, token); + if( !rc && d->M.used>1 && + fsl_strcmp((char const *)d->M.list[d->M.used-2], + (char const *)token)>=0 ){ + SYNTAX("M-card in the wrong order"); + } + break; + } + /* + N + + An N-line identifies the mimetype of wiki or comment text. + */ + case 'N':{ + if(1N = (char *)token; + break; + } + + /* + P ... + + Specify one or more other artifacts which are the parents of + this artifact. The first parent is the primary parent. All + others are parents by merge. + */ + case 'P':{ + if(1P, token); + if(!rc){ + TOKEN(0); + } + } + break; + } + /* + Q (+|-) ?? + + Specify one or a range of checkins that are cherrypicked into + this checkin ("+") or backed out of this checkin ("-"). + */ + case 'Q':{ + fsl_cherrypick_type_e qType = FSL_CHERRYPICK_INVALID; + TOKEN(0); + TOKEN_EXISTS("Missing target UUID for Q-card"); + switch((char)*token){ + case '-': qType = FSL_CHERRYPICK_BACKOUT; break; + case '+': qType = FSL_CHERRYPICK_ADD; break; + default: + SYNTAX("Malformed target UUID in Q-card"); + } + assert(qType); + uuid = ++token; --tokLen; + TOKEN_UUID(Q); + TOKEN(0); + if(token){ + TOKEN_UUID(Q); + } + d->type = FSL_SATYPE_CHECKIN; + rc = fsl_deck_Q_add(d, qType, (char const *)uuid, + (char const *)token); + break; + } + /* + R + + Specify the MD5 checksum over the name and content of all files + in the manifest. + */ + case 'R':{ + if(1R = (char *)token; + ++stealBuf; + d->type = FSL_SATYPE_CHECKIN; + break; + } + /* + T (+|*|-) ?? + + Create or cancel a tag or property. The tagname is fossil-encoded. + The first character of the name must be either "+" to create a + singleton tag, "*" to create a propagating tag, or "-" to create + anti-tag that undoes a prior "+" or blocks propagation of of + a "*". + + The tag is applied to . If is "*" then the tag is + applied to the current manifest. If is provided then + the tag is really a property with the given value. + + Tags are not allowed in clusters. Multiple T lines are allowed. + */ + case 'T':{ + unsigned char * name, * value; + fsl_tagtype_e tagType = FSL_TAGTYPE_INVALID; + TOKEN(1); + TOKEN_EXISTS("Missing name for T-card"); + name = token; + if( fsl_validate16((char const *)&name[1], + fsl_strlen((char const *)&name[1])) ){ + /* Do not allow tags whose names look like a hash */ + SYNTAX("T-card name looks like a hexadecimal hash"); + } + TOKEN(0); + TOKEN_EXISTS("Missing UUID on T-card"); + if(fsl_is_uuid_len((int)tokLen)){ + TOKEN_UUID(T); + uuid = token; + }else if( 1==tokLen && '*'==(char)*token ){ + /* tag for the current artifact */ + ++nSelfTag; + uuid = NULL; + }else{ + SYNTAX("Malformed UUID in T-card"); + } + TOKEN(1); + value = token; + switch(*name){ + case '*': tagType = FSL_TAGTYPE_PROPAGATING; break; + case '+': tagType = FSL_TAGTYPE_ADD; + ++nSimpleTag; + break; + case '-': tagType = FSL_TAGTYPE_CANCEL; break; + default: SYNTAX("Malformed tag name"); + } + ++name /* skip type marker byte */; + /* Potential todo: add the order check from this commit: + + https://fossil-scm.org/index.html/info/55cacfcace + */ + rc = fsl_deck_T_add(d, tagType, (fsl_uuid_cstr)uuid, + (char const *)name, + (char const *)value); + break; + } + /* + U ?? + + Identify the user who created this control file by their + login. Only one U line is allowed. Prohibited in clusters. + If the user name is omitted, take that to be "anonymous". + */ + case 'U':{ + if(d->U) SYNTAX("More than one U-card"); + TOKEN(1); + if(token){ + /* rc = fsl_deck_U_set( d, (char const *)token, (fsl_int_t)tokLen ); */ + ++stealBuf; + d->U = (char *)token; + }else{ + rc = fsl_deck_U_set( d, "anonymous" ); + } + break; + } + /* + W + + The next bytes of the file contain the text of the wiki + page. There is always an extra \n before the start of the next + record. + */ + case 'W':{ + fsl_size_t wlen; + if(d->W.used){ + SYNTAX("More than one W-card"); + } + TOKEN(0); + TOKEN_EXISTS("Missing size token for W-card"); + wlen = fsl_str_to_size((char const *)token); + if((fsl_size_t)-1==wlen){ + ERROR(FSL_RC_RANGE,"Wiki size token is invalid"); + } + if( (&x.z[wlen+1]) > x.zEnd){ + SYNTAX("Not enough content after W-card"); + } + rc = fsl_buffer_append(&d->W, x.z, wlen); + if(rc) goto bailout; + x.z += wlen; + if( '\n' != x.z[0] ){ + SYNTAX("W-card content not \\n terminated"); + } + x.z[0] = 0; + ++x.z; + break; + } + /* + Z + + MD5 checksum on this control file. The checksum is over all + lines (other than PGP-signature lines) prior to the current + line. This must be the last record. + + This card is required for all control file types except for + Manifest. It is not required for manifest only for historical + compatibility reasons. + */ + case 'Z':{ + /* We validated the Z card first. We cannot compare against + the original blob now because we've modified it. + */ + goto end; + } + default: + rc = fsl_cx_err_set(f, FSL_RC_SYNTAX, + "Unknown card '%c' in manifest", + cType); + goto bailout; + }/*switch(cType)*/ + if(rc) goto bailout; + }/* for-each-card */ + +#if 1 + /* Remove these when we are done porting + resp. we can avoid these unused-var warnings. */ + if(isRepeat){} +#endif + + end: + assert(0==rc); + if(cardCount>2 && FSL_SATYPE_ANY==d->type){ + /* See if we need to guess the type now. + We need(?) at least two card to ensure that this is + free of ambiguities. */ + d->type = fsl_deck_guess_type(lettersSeen); + if(FSL_SATYPE_ANY!=d->type){ + assert(FSL_SATYPE_INVALID!=d->type); +#if 0 + MARKER(("Guessed manifest type with %d cards: %s\n", + cardCount, fsl_satype_cstr(d->type))); +#endif + } + } + /* Make sure all of the cards we put in it belong to that deck + type. */ + if( !fsl_deck_check_type(d, cType) ){ + rc = d->f->error.code; + goto bailout; + } + + if(FSL_SATYPE_ANY==d->type){ + rc = fsl_cx_err_set(f, FSL_RC_ERROR, + "Internal error: could not determine type of " + "control artifact we just (successfully!) " + "parsed."); + goto bailout; + }else { + /* + Make sure we didn't pick up any cards which were picked up + before d->type was guessed and are invalid for the post-guessed + type. + */ + int i = 0; + for( ; i < 27; ++i ){ + if((lettersSeen[i]>0) && !fsl_card_is_legal(d->type, 'A'+i )){ + rc = fsl_cx_err_set(f, FSL_RC_SYNTAX, + "Determined during post-parse processing that " + "the parsed deck (type %s) contains an illegal " + "card type (%c).", fsl_satype_cstr(d->type), + 'A'+i); + goto bailout; + } + } + } + assert(FSL_SATYPE_CHECKIN==d->type || + FSL_SATYPE_CLUSTER==d->type || + FSL_SATYPE_CONTROL==d->type || + FSL_SATYPE_WIKI==d->type || + FSL_SATYPE_TICKET==d->type || + FSL_SATYPE_ATTACHMENT==d->type || + FSL_SATYPE_TECHNOTE==d->type || + FSL_SATYPE_FORUMPOST==d->type); + assert(0==rc); + + /* Additional checks based on artifact type */ + switch( d->type ){ + case FSL_SATYPE_CONTROL: { + if( nSelfTag ){ + SYNTAX("self-referential T-card in control artifact"); + } + break; + } + case FSL_SATYPE_TECHNOTE: { + if( d->T.used!=nSelfTag ){ + SYNTAX("non-self-referential T-card in technote"); + }else if( d->T.used!=nSimpleTag ){ + SYNTAX("T-card with '*' or '-' in technote"); + } + break; + } + case FSL_SATYPE_FORUMPOST: { + if( d->H && d->I ){ + SYNTAX("cannot have I-card and H-card in a forum post"); + }else if( d->P.used>1 ){ + SYNTAX("too many arguments to P-card"); + } + break; + } + default: break; + } + + assert(!d->content.mem); + if(stealBuf>0){ + /* We stashed something which points to src->mem, so we need to + steal that memory. + */ + d->content = *src; + *src = fsl_buffer_empty; + } + d->rid = rid; + d->F.flags &= ~FSL_CARD_F_LIST_NEEDS_SORT/*we know all cards were read in order*/; + return 0; + + bailout: + if(stealBuf>0){ + d->content = *src; + *src = fsl_buffer_empty; + } + assert(0 != rc); + if(zMsg){ + fsl_error_set(err, rc, "%s", zMsg); + } + return rc; +#undef SEEN +#undef TOKEN_DATETIME +#undef SYNTAX +#undef TOKEN_CHECKHEX +#undef TOKEN_EXISTS +#undef TOKEN_UUID +#undef TOKEN_MD5 +#undef TOKEN +#undef ERROR +} + +int fsl_deck_parse(fsl_deck * const d, fsl_buffer * const src){ + return fsl_deck_parse2(d, src, 0); +} + +int fsl_deck_load_rid( fsl_cx * const f, fsl_deck * const d, + fsl_id_t rid, fsl_satype_e type ){ + fsl_buffer buf = fsl_buffer_empty; + int rc = 0; + if(0==rid) rid = f->ckout.rid; + if(rid<0){ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "Invalid RID for fsl_deck_load_rid(): " + "%"FSL_ID_T_PFMT, rid); + } + fsl_deck_clean(d); + d->f = f; + if(fsl_cx_mcache_search(f, rid, d)){ + assert(d->f); + if(type!=FSL_SATYPE_ANY && type!=d->type){ + rc = fsl_cx_err_set(f, FSL_RC_TYPE, + "Unexpected match of RID #%" FSL_ID_T_PFMT " " + "to a different artifact type (%d) " + "than requested (%d).", + d->type, type); + fsl_cx_mcache_insert(f, d); + assert(!d->f); + }else{ + //MARKER(("Got cached deck: %s\n", d->uuid)); + } + return rc; + } + rc = fsl_content_get(f, rid, &buf); + if(rc) goto end; +#if 0 + MARKER(("fsl_content_get(%d) len=%d =\n%.*s\n", + (int)rid, (int)buf.used, (int)buf.used, (char const*)buf.mem)); +#endif + fsl_deck_init(f, d, FSL_SATYPE_ANY); +#if 0 + /* + If we set d->type=type, the parser can fail more + quickly. However, that failure will bypass our more specific + reporting of the problem (see below). As the type mismatch case + is expected to be fairly rare, we'll leave this out for now, but + it might be worth considering as a small optimization later on. + */ + d->type = type /* may help parsing fail more quickly if + it's not the type we want.*/; +#endif + rc = fsl_deck_parse(d, &buf); + if(!rc){ + if( type!=FSL_SATYPE_ANY && d->type!=type ){ + rc = fsl_cx_err_set(f, FSL_RC_TYPE, + "RID %"FSL_ID_T_PFMT" is of type %s, " + "but the caller requested type %s.", + rid, + fsl_satype_cstr(d->type), + fsl_satype_cstr(type)); + }else if(d->B.uuid ){ + rc = fsl_cx_update_seen_delta_mf(f); + } + } + end: + if(0==rc) d->rid = rid; + fsl_buffer_clear(&buf); + return rc; +} + +int fsl_deck_load_sym( fsl_cx * const f, fsl_deck * const d, + char const * symbolicName, fsl_satype_e type ){ + if(!symbolicName || !d) return FSL_RC_MISUSE; + else{ + fsl_id_t vid = 0; + int rc = fsl_sym_to_rid(f, symbolicName, type, &vid); + if(!rc){ + assert(vid>0); + rc = fsl_deck_load_rid(f, d, vid, type); + } + return rc; + } +} + + +static int fsl_deck_baseline_load( fsl_deck * d ){ + int rc = 0; + fsl_deck bl = fsl_deck_empty; + fsl_id_t rid; + fsl_cx * f = d ? d->f : NULL; + fsl_db * db = f ? fsl_needs_repo(f) : NULL; + assert(d->f); + assert(d); + if(!d->f) return FSL_RC_MISUSE; + else if(d->B.baseline || !d->B.uuid) return 0 /* nothing to do! */; + else if(!db) return FSL_RC_NOT_A_REPO; +#if 0 + else if(d->rid<=0){ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "fsl_deck_baseline_load(): " + "fsl_deck::rid is not set."); + } +#endif + rid = fsl_uuid_to_rid(f, d->B.uuid); + if(rid<0){ + assert(f->error.code); + return f->error.code; + } + else if(!rid){ + if(d->rid>0){ + fsl_db_exec(db, + "INSERT OR IGNORE INTO orphan(rid, baseline) " + "VALUES(%"FSL_ID_T_PFMT",%"FSL_ID_T_PFMT")", + d->rid, rid); + } + rc = fsl_cx_err_set(f, FSL_RC_RANGE, + "Could not find/load baseline manifest [%s], " + "parent of manifest rid #%"FSL_ID_T_PFMT".", + d->B.uuid, d->rid); + }else{ + rc = fsl_deck_load_rid(f, &bl, rid, FSL_SATYPE_CHECKIN); + if(!rc){ + d->B.baseline = fsl_deck_malloc(); + if(!d->B.baseline){ + fsl_deck_clean(&bl); + rc = FSL_RC_OOM; + }else{ + void const * allocStampKludge = d->B.baseline->allocStamp; + *d->B.baseline = bl /* Transfer ownership */; + d->B.baseline->allocStamp = allocStampKludge /* But we need this intact + for deallocation to work */; + assert(f==d->B.baseline->f); + } + }else{ + /* bl might be partially populated */ + fsl_deck_finalize(&bl); + } + } + return rc; +} + +int fsl_deck_baseline_fetch( fsl_deck * d ){ + return (d->B.baseline || !d->B.uuid) + ? 0 + : fsl_deck_baseline_load(d); +} + +int fsl_deck_F_rewind( fsl_deck * d ){ + int rc = 0; + d->F.cursor = 0; + assert(d->f); + if(d->B.uuid){ + rc = fsl_deck_baseline_fetch(d); + if(!rc){ + assert(d->B.baseline); + d->B.baseline->F.cursor = 0; + } + } + return rc; +} + +int fsl_deck_F_next( fsl_deck * d, fsl_card_F const ** rv ){ + assert(d); + assert(d->f); + assert(rv); +#define FCARD(DECK,NDX) F_at(&(DECK)->F, NDX) + *rv = NULL; + if(!d->B.baseline){ + /* Manifest d is a baseline-manifest. Just scan down the list + of files. */ + if(d->B.uuid){ + return fsl_cx_err_set(d->f, FSL_RC_MISUSE, + "Deck has a B-card (%s) but no baseline " + "loaded. Load the baseline before calling " + "%s().", + d->B.uuid, __func__) + /* We "could" just load the baseline from here. */; + } + if( d->F.cursor < (int32_t)d->F.used ){ + *rv = FCARD(d, d->F.cursor++); + assert(*rv); + assert((*rv)->uuid && "Baseline manifest has deleted F-card entry!"); + } + return 0; + }else{ + /* Manifest d is a delta-manifest. Scan the baseline but amend the + file list in the baseline with changes described by d. + */ + fsl_deck * const pB = d->B.baseline; + int cmp; + while(1){ + if( pB->F.cursor >= (fsl_int_t)pB->F.used ){ + /* We have used all entries out of the baseline. Return the next + entry from the delta. */ + if( d->F.cursor < (fsl_int_t)d->F.used ) *rv = FCARD(d, d->F.cursor++); + break; + }else if( d->F.cursor >= (fsl_int_t)d->F.used ){ + /* We have used all entries from the delta. Return the next + entry from the baseline. */ + if( pB->F.cursor < (fsl_int_t)pB->F.used ) *rv = FCARD(pB, pB->F.cursor++); + break; + }else if( (cmp = fsl_strcmp(FCARD(pB,pB->F.cursor)->name, + FCARD(d, d->F.cursor)->name)) < 0){ + /* The next baseline entry comes before the next delta entry. + So return the baseline entry. */ + *rv = FCARD(pB, pB->F.cursor++); + break; + }else if( cmp>0 ){ + /* The next delta entry comes before the next baseline + entry so return the delta entry */ + *rv = FCARD(d, d->F.cursor++); + break; + }else if( FCARD(d, d->F.cursor)->uuid ){ + /* The next delta entry is a replacement for the next baseline + entry. Skip the baseline entry and return the delta entry */ + pB->F.cursor++; + *rv = FCARD(d, d->F.cursor++); + break; + }else{ + assert(0==cmp); + /* + The next delta entry is a delete of the next baseline entry. + */ + /* Skip them both. Repeat the loop to find the next + non-delete entry. */ + pB->F.cursor++; + d->F.cursor++; + continue; + } + } + return 0; + } +#undef FCARD +} + +int fsl_deck_save( fsl_deck * const d, bool isPrivate ){ + int rc; + fsl_cx * const f = d->f; + fsl_db * const db = fsl_needs_repo(f); + fsl_buffer * const buf = &d->f->fileContent; + fsl_id_t newRid = 0; + bool const oldPrivate = f->cache.markPrivate; + if(!f || !d ) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + if(d->rid>0){ +#if 1 + return 0; +#else + return fsl_cx_err_set(f, FSL_RC_ALREADY_EXISTS, + "Cannot re-save an existing deck, as that could " + "lead to inconsistent data."); +#endif + } + if(d->B.uuid && fsl_repo_forbids_delta_manifests(f)){ + return fsl_cx_err_set(f, FSL_RC_ACCESS, + "This deck is a delta manifest, but this " + "repository has disallowed those via the " + "forbid-delta-manifests config option."); + } + + fsl_cx_err_reset(f); + fsl_size_t const reserveSize = + (1024 * 10) + + (50 * d->T.used) + + (50 * d->M.used) + + (120 * d->F.used) + + d->W.used; + fsl_buffer_reuse(buf); + rc = fsl_buffer_reserve(buf, reserveSize); + if(0==rc) rc = fsl_deck_output(d, fsl_output_f_buffer, buf); + if(rc){ + fsl_buffer_reuse(buf); + return rc; + } + + rc = fsl_db_transaction_begin(db); + if(rc){ + fsl_buffer_reuse(buf); + return rc; + } + if(0){ + MARKER(("Saving deck:\n%s\n", fsl_buffer_cstr(buf))); + } + + /* Starting here, don't return, use (goto end) instead. */ + + f->cache.markPrivate = isPrivate; + { + rc = fsl_content_put_ex(f, buf, NULL, 0, + 0U, isPrivate, &newRid); + if(rc) goto end; + assert(newRid>0); + } + + /* We need d->rid for crosslinking purposes, but will unset it on + error because its value will no longer be in the db after + rollback... + */ + d->rid = newRid; + +#if 0 + /* Something to consider: has a parent, deltify the parent. The + branch operation does this, but it is not yet clear whether that + is a general pattern for manifests. + */ + if(d->P.used){ + fsl_id_t pid; + assert(FSL_SATYPE_CHECKIN == d->type); + pid = fsl_uuid_to_rid(f, (char const *)d->P.list[0]); + if(pid>0){ + rc = fsl_content_deltify(f, pid, d->rid, 0); + if(rc) goto end; + } + } +#endif + + if(FSL_SATYPE_WIKI==d->type){ + /* Analog to fossil's wiki.c:wiki_put(): */ + /* + MISSING: + fossil's wiki.c:wiki_put() handles the moderation bits. + */ + if(d->P.used){ + fsl_id_t const pid = fsl_deck_P_get_id(d, 0); + assert(pid>0); + if(pid<0){ + assert(f->error.code); + rc = f->error.code; + goto end; + }else if(!pid){ + if(!f->error.code){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Did not find matching RID " + "for P-card[0] (%s).", + (char const *)d->P.list[0]); + } + goto end; + } + rc = fsl_content_deltify(f, pid, d->rid, 0); + if(rc) goto end; + } + rc = fsl_db_exec_multi(db, + "INSERT OR IGNORE INTO unsent " + "VALUES(%"FSL_ID_T_PFMT");" + "INSERT OR IGNORE INTO unclustered " + "VALUES(%"FSL_ID_T_PFMT");", + d->rid, d->rid); + if(rc){ + fsl_cx_uplift_db_error(f, db); + goto end; + } + } + + rc = f->cache.isCrosslinking + ? fsl_deck_crosslink(d) + : fsl_deck_crosslink_one(d); + + end: + f->cache.markPrivate = oldPrivate; + if(!rc) rc = fsl_db_transaction_end(db, 0); + else fsl_db_transaction_end(db, 1); + if(rc){ + d->rid = 0 /* this blob.rid will be lost after rollback */; + if(!f->error.code && db->error.code){ + rc = fsl_cx_uplift_db_error(f, db); + } + } + fsl_buffer_reuse(buf); + return rc; +} + +int fsl_crosslink_end(fsl_cx * f){ + int rc = 0; + fsl_db * db = fsl_cx_db_repo(f); + fsl_stmt q = fsl_stmt_empty; + fsl_stmt u = fsl_stmt_empty; + int i; + assert(f); + assert(db); + assert(f->cache.isCrosslinking); + if(!f->cache.isCrosslinking){ + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "Crosslink is not running."); + } + f->cache.isCrosslinking = false; + assert(db->beginCount > 0); + + /* Handle any reparenting via tags... */ + rc = fsl_db_prepare(db, &q, + "SELECT rid, value FROM tagxref" + " WHERE tagid=%d AND tagtype=%d", + (int)FSL_TAGID_PARENT, (int)FSL_TAGTYPE_ADD); + if(rc) goto end; + while(FSL_RC_STEP_ROW==fsl_stmt_step(&q)){ + fsl_id_t const rid = fsl_stmt_g_id(&q, 0); + const char *zTagVal = fsl_stmt_g_text(&q, 1, 0); + rc = fsl_crosslink_reparent(f,rid, zTagVal); + if(rc) break; + } + fsl_stmt_finalize(&q); + if(rc) goto end; + + /* Process entries from pending_xlink temp table... */ + rc = fsl_db_prepare(db, &q, "SELECT id FROM pending_xlink"); + if(rc) goto end; + while( FSL_RC_STEP_ROW==fsl_stmt_step(&q) ){ + const char *zId = fsl_stmt_g_text(&q, 0, NULL); + char cType; + if(!zId || !*zId) continue; + cType = zId[0]; + ++zId; + if('t'==cType){ + /* FSL-MISSING: + ticket_rebuild_entry(zId) */ + continue; + }else if('w'==cType){ + /* FSL-MISSING: + backlink_wiki_refresh(zId) */ + continue; + } + } + fsl_stmt_finalize(&q); + rc = fsl_db_exec(db, "DROP TABLE pending_xlink"); + if(rc) goto end; + /* If multiple check-ins happen close together in time, adjust their + times by a few milliseconds to make sure they appear in chronological + order. + */ + rc = fsl_db_prepare(db, &q, + "UPDATE time_fudge SET m1=m2-:incr " + "WHERE m1>=m2 AND m1cache.isCrosslinking); + if(f->cache.isCrosslinking){ + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "Crosslink is already running."); + } + rc = fsl_db_transaction_begin(db); + if(rc) return fsl_cx_uplift_db_error(f, db); + rc = fsl_db_exec_multi(db, + "CREATE TEMP TABLE pending_xlink(id TEXT PRIMARY KEY)WITHOUT ROWID;" + "CREATE TEMP TABLE time_fudge(" + " mid INTEGER PRIMARY KEY," /* The rid of a manifest */ + " m1 REAL," /* The timestamp on mid */ + " cid INTEGER," /* A child or mid */ + " m2 REAL" /* Timestamp on the child */ + ");"); + if(!rc){ + f->cache.isCrosslinking = 1; + return 0; + }else{ + rc = fsl_cx_uplift_db_error2(f, db, rc); + fsl_db_transaction_rollback(db); + return rc; + } +} + +#undef MARKER +#undef AGE_FUDGE_WINDOW +#undef AGE_ADJUST_INCREMENT +#undef F_at +/* end of file deck.c */ +/* start of file delta.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************** + This file houses Fossil's delta generation and application + routines. This code is functionally independent of the rest of the + library, relying only on fsl_malloc(), fsl_free(), and the integer + typedefs defined by the configuration process. i.e. it can easily + be pulled out and used in arbitrary projects. +*/ +#include +#include + +/** + 2021-03-10: The delta checksum self-test is a significant run-time + sink when processing many deltas. Fossil does not enable this + feature by default so we'll leave it off by default, too. +*/ +#if !defined(FSL_OMIT_DELTA_CKSUM_TEST) +# define FSL_OMIT_DELTA_CKSUM_TEST +#endif + +/* + Macros for turning debugging printfs on and off +*/ +#if 0 +# define DEBUG1(X) X +#else +# define DEBUG1(X) +#endif +#if 0 +#define DEBUG2(X) X +/* + For debugging: + Print 16 characters of text from zBuf +*/ +static const char *print16(const char *z){ + int i; + static char zBuf[20]; + for(i=0; i<16; i++){ + if( z[i]>=0x20 && z[i]<=0x7e ){ + zBuf[i] = z[i]; + }else{ + zBuf[i] = '.'; + } + } + zBuf[i] = 0; + return zBuf; +} +#else +# define DEBUG2(X) +#endif + +/* + The width of a hash window in bytes. The algorithm only works if this + is a power of 2. +*/ +#define NHASH 16 + +/* + The current state of the rolling hash. + + z[] holds the values that have been hashed. z[] is a circular buffer. + z[i] is the first entry and z[(i+NHASH-1)%NHASH] is the last entry of + the window. + + Hash.a is the sum of all elements of hash.z[]. Hash.b is a weighted + sum. Hash.b is z[i]*NHASH + z[i+1]*(NHASH-1) + ... + z[i+NHASH-1]*1. + (Each index for z[] should be module NHASH, of course. The %NHASH operator + is omitted in the prior expression for brevity.) +*/ +typedef struct fsl_delta_hash fsl_delta_hash; +struct fsl_delta_hash { + uint16_t a, b; /* Hash values */ + uint16_t i; /* Start of the hash window */ + unsigned char z[NHASH]; /* The values that have been hashed */ +}; + +/* + Initialize the rolling hash using the first NHASH characters of z[] +*/ +static void fsl_delta_hash_init(fsl_delta_hash *pHash, + unsigned char const *z){ + uint16_t a, b, i; + a = b = 0; + for(i=0; iz, z, NHASH); + pHash->a = a & 0xffff; + pHash->b = b & 0xffff; + pHash->i = 0; +} + +/* + Advance the rolling hash by a single character "c" +*/ +static void fsl_delta_hash_next(fsl_delta_hash *pHash, int c){ + uint16_t old = pHash->z[pHash->i]; + pHash->z[pHash->i] = c; + pHash->i = (pHash->i+1)&(NHASH-1); + pHash->a = pHash->a - old + c; + pHash->b = pHash->b - NHASH*old + pHash->a; +} + +/* + Return a 32-bit hash value +*/ +static uint32_t fsl_delta_hash_32bit(fsl_delta_hash *pHash){ + return (pHash->a & 0xffff) | (((uint32_t)(pHash->b & 0xffff))<<16); +} + +/** + Compute a hash on NHASH bytes. + + This routine is intended to be equivalent to: + fsl_delta_hash h; + fsl_delta_hash_init(&h, zInput); + return fsl_delta_hash_32bit(&h); +*/ +static uint32_t fsl_delta_hash_once(unsigned const char *z){ + uint16_t a = 0, b = 0, i = 0; + for(i=0; i0; ++rc, ++i, v>>=6){ + zBuf[i] = zDigits[v&0x3f]; + } + zBuf[i]=0; + for(j=i-1; j>=0; j--){ + *(*pz)++ = zBuf[j]; + } + } + return rc; +} + +/* + Read bytes from *pz and convert them into a positive integer. When + finished, leave *pz pointing to the first character past the end of + the integer. The *pLen parameter holds the length of the string + in *pz and is decremented once for each character in the integer. +*/ +static fsl_size_t fsl_delta_int_get(unsigned char const **pz, fsl_int_t *pLen){ + static const signed char zValue[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, + }; + fsl_size_t v = 0; + fsl_int_t c; + unsigned char const *z = (unsigned char const*)*pz; + unsigned char const *zStart = z; + while( (c = zValue[0x7f&*(z++)])>=0 ){ + v = (v<<6) + c; + } + z--; + *pLen -= z - zStart; + *pz = z; + return v; +} + +/* + Return the number digits in the base-64 representation of a positive integer +*/ +static int fsl_delta_digit_count(fsl_int_t v){ + unsigned int x; + int i; + for(i=1, x=64; v>=(fsl_int_t)x; i++, x <<= 6){} + return i; +} + +/* + Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int fsl_delta_checksum(void const *zIn, fsl_size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; + } + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; + } + return sum3; +} + +int fsl_delta_create2( unsigned char const *zSrc, fsl_size_t lenSrc, + unsigned char const *zOut, fsl_size_t lenOut, + fsl_output_f out, void * outState){ + enum { IntegerBufSize = 50 /* buffer size for integer conversions. */}; + unsigned int i, base; + unsigned int nHash; /* Number of hash table entries */ + unsigned int *landmark; /* Primary hash table */ + unsigned int *collide = NULL; /* Collision chain */ + int lastRead = -1; /* Last byte of zSrc read by a COPY command */ + int rc; /* generic return code checker. */ + unsigned int olen = 0; /* current output length. */ + unsigned int total = 0; /* total byte count. */ + fsl_delta_hash h; + unsigned char theBuf[IntegerBufSize] = {0,}; + unsigned char * intBuf = theBuf; + if(!zSrc || !zOut || !out) return FSL_RC_MISUSE; + /* Add the target file size to the beginning of the delta + */ +#ifdef OUT +#undef OUT +#endif +#define OUT(BLOB,LEN) rc=out(outState, BLOB, LEN); if(0 != rc) {fsl_free(collide); return rc;} else total += LEN +#define OUTCH(CHAR) OUT(CHAR,1) +#define PINT(I) intBuf = theBuf; olen=fsl_delta_int_put(I, &intBuf); OUT(theBuf,olen) + PINT(lenOut); + OUTCH("\n"); + + /* If the source file is very small, it means that we have no + chance of ever doing a copy command. Just output a single + literal segment for the entire target and exit. + */ + if( lenSrc<=NHASH ){ + PINT(lenOut); + OUTCH(":"); + OUT(zOut,lenOut); + PINT((fsl_delta_checksum(zOut, lenOut))); + OUTCH(";"); + return 0; + } + + /* Compute the hash table used to locate matching sections in the + source file. + */ + nHash = lenSrc/NHASH; + collide = (unsigned int *)malloc( nHash*2*sizeof(int) ); + if(!collide){ + return FSL_RC_OOM; + } + landmark = &collide[nHash]; + memset(landmark, -1, nHash*sizeof(int)); + memset(collide, -1, nHash*sizeof(int)); + for(i=0; i=0 && (limit--)>0 ){ + /* + The hash window has identified a potential match against + landmark block iBlock. But we need to investigate further. + + Look for a region in zOut that matches zSrc. Anchor the search + at zSrc[iSrc] and zOut[base+i]. Do not include anything prior to + zOut[base] or after zOut[outLen] nor anything after zSrc[srcLen]. + + Set cnt equal to the length of the match and set ofst so that + zSrc[ofst] is the first element of the match. litsz is the number + of characters between zOut[base] and the beginning of the match. + sz will be the overhead (in bytes) needed to encode the copy + command. Only generate copy command if the overhead of the + copy command is less than the amount of literal text to be copied. + */ + fsl_int_t cnt, ofst, litsz; + fsl_int_t j, k, x, y; + fsl_int_t sz; + fsl_int_t limitX; + + /* Beginning at iSrc, match forwards as far as we can. j counts + the number of characters that match */ + iSrc = iBlock*NHASH; + y = base + i; + limitX = ( lenSrc-iSrc <= lenOut-y ) ? lenSrc : iSrc + lenOut - y; + for(x=iSrc; x=sz && cnt>bestCnt ){ + /* Remember this match only if it is the best so far and it + does not increase the file size */ + bestCnt = cnt; + bestOfst = iSrc-k; + bestLitsz = litsz; + DEBUG2( printf("... BEST SO FAR\n"); ) + } + + /* Check the next matching block */ + iBlock = collide[iBlock]; + } + + /* We have a copy command that does not cause the delta to be larger + than a literal insert. So add the copy command to the delta. + */ + if( bestCnt>0 ){ + if( bestLitsz>0 ){ + /* Add an insert command before the copy */ + PINT(bestLitsz); + OUTCH(":"); + OUT(zOut+base, bestLitsz); + base += bestLitsz; + DEBUG2( printf("insert %d\n", bestLitsz); ) + } + base += bestCnt; + PINT(bestCnt); + OUTCH("@"); + PINT(bestOfst); + DEBUG2( printf("copy %d bytes from %d\n", bestCnt, bestOfst); ) + OUTCH(","); + if( bestOfst + bestCnt -1 > lastRead ){ + lastRead = bestOfst + bestCnt - 1; + DEBUG2( printf("lastRead becomes %d\n", lastRead); ) + } + bestCnt = 0; + break; + } + + /* If we reach this point, it means no match is found so far */ + if( base+i+NHASH>=lenOut ){ + /* We have reached the end of the input and have not found any + matches. Do an "insert" for everything that does not match */ + PINT(lenOut-base); + OUTCH(":"); + OUT(zOut+base, lenOut-base); + base = lenOut; + break; + } + + /* Advance the hash by one character. Keep looking for a match */ + fsl_delta_hash_next(&h, zOut[base+i+NHASH]); + i++; + } + } + fsl_free(collide); + /* Output a final "insert" record to get all the text at the end of + the file that does not match anything in the source file. + */ + if( basemem, increments state->cursor by n, and returns 0. + */ +static int fsl_output_f_ostring( void * state, void const * src, + fsl_size_t n ){ + DeltaOutputString * os = (DeltaOutputString*)state; + memcpy( os->mem + os->cursor, src, n ); + os->cursor += n; + return 0; +} + +int fsl_delta_create( unsigned char const *zSrc, fsl_size_t lenSrc, + unsigned char const *zOut, fsl_size_t lenOut, + unsigned char *zDelta, fsl_size_t * deltaSize){ + int rc; + DeltaOutputString os; + if(!zSrc || !zOut || !zDelta || !deltaSize) return FSL_RC_MISUSE; + os.mem = (unsigned char *)zDelta; + os.cursor = 0; + rc = fsl_delta_create2( zSrc, lenSrc, zOut, lenOut, + fsl_output_f_ostring, &os ); + if(!rc){ + os.mem[os.cursor] = 0; + if(deltaSize) *deltaSize = os.cursor; + } + return rc; +} + +/* + Calculates the size (in bytes) of the output from applying a + delta. On success 0 is returned and *deltaSize will be updated with + the amount of memory required for applying the delta. + + This routine is provided so that an procedure that is able + to call fsl_delta_apply() can learn how much space is required + for the output and hence allocate nor more space that is really + needed. +*/ +int fsl_delta_applied_size(unsigned char const *zDelta, fsl_size_t lenDelta_, + fsl_size_t * deltaSize){ + if(!zDelta || (lenDelta_<2) || !deltaSize) return FSL_RC_MISUSE; + else{ + fsl_size_t size; + fsl_int_t lenDelta = (fsl_int_t)lenDelta_; + size = fsl_delta_int_get(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return FSL_RC_DELTA_INVALID_TERMINATOR; + } + *deltaSize = size; + return 0; + } +} + + +int fsl_delta_apply2( + unsigned char const *zSrc, /* The source or pattern file */ + fsl_size_t lenSrc_, /* Length of the source file */ + unsigned char const *zDelta, /* Delta to apply to the pattern */ + fsl_size_t lenDelta_, /* Length of the delta */ + unsigned char *zOut, /* Write the output into this preallocated buffer */ + fsl_error * pErr +){ + fsl_size_t limit; + fsl_size_t total = 0; +#if !defined(FSL_OMIT_DELTA_CKSUM_TEST) + unsigned char *zOrigOut = zOut; +#endif + /* lenSrc/lenDelta are cast to ints to avoid any potential side-effects + caused by changing the function signature from signed to unsigned + int types when porting from v1. + */ + fsl_int_t lenSrc = (fsl_int_t)lenSrc_; + fsl_int_t lenDelta = (fsl_int_t)lenDelta_; + if(!zSrc || !zDelta || !zOut) return FSL_RC_MISUSE; + else if(lenSrc<0 || lenDelta<0) return FSL_RC_RANGE; + limit = fsl_delta_int_get(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + if(pErr){ + fsl_error_set(pErr, + FSL_RC_DELTA_INVALID_TERMINATOR, + "Delta: size integer not terminated by \\n"); + } + return FSL_RC_DELTA_INVALID_TERMINATOR; + } + zDelta++; lenDelta--; + while( *zDelta && lenDelta>0 ){ + fsl_int_t cnt, ofst; + cnt = fsl_delta_int_get(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = fsl_delta_int_get(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + if(pErr){ + fsl_error_set(pErr, + FSL_RC_DELTA_INVALID_TERMINATOR, + "Delta: copy command not terminated by ','"); + } + return FSL_RC_DELTA_INVALID_TERMINATOR; + } + zDelta++; lenDelta--; + DEBUG1( printf("COPY %d from %d\n", cnt, ofst); ) + total += cnt; + if( total>limit ){ + if(pErr){ + fsl_error_set(pErr, FSL_RC_RANGE, + "Delta: copy exceeds output file size"); + } + return FSL_RC_RANGE; + } + if( ofst+cnt > lenSrc ){ + if(pErr){ + fsl_error_set(pErr, FSL_RC_RANGE, + "Delta: copy extends past end of input"); + } + return FSL_RC_RANGE; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; + } + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + if(pErr){ + fsl_error_set(pErr, FSL_RC_RANGE, + "Delta: insert command gives an output " + "larger than predicted"); + } + return FSL_RC_RANGE; + } + DEBUG1( printf("INSERT %d\n", cnt); ) + if( cnt>lenDelta ){ + if(pErr){ + fsl_error_set(pErr, FSL_RC_RANGE, + "Delta: insert count exceeds size of delta"); + } + return FSL_RC_RANGE; + } + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; + } + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#if !defined(FSL_OMIT_DELTA_CKSUM_TEST) + if( cnt!=fsl_delta_checksum(zOrigOut, total) ){ + if(pErr){ + fsl_error_set(pErr, FSL_RC_CHECKSUM_MISMATCH, + "Delta: bad checksum"); + } + return FSL_RC_CHECKSUM_MISMATCH; + } +#endif + if( total!=limit ){ + if(pErr){ + fsl_error_set(pErr, FSL_RC_SIZE_MISMATCH, + "Delta: generated size does not match " + "predicted size"); + } + return FSL_RC_SIZE_MISMATCH; + } + return 0; + } + default: { + if(pErr){ + fsl_error_set(pErr, FSL_RC_DELTA_INVALID_OPERATOR, + "Delta: unknown delta operator"); + } + return FSL_RC_DELTA_INVALID_OPERATOR; + } + } + } + /* ERROR: unterminated delta */ + if(pErr){ + fsl_error_set(pErr, FSL_RC_DELTA_INVALID_TERMINATOR, + "Delta: unterminated delta"); + } + return FSL_RC_DELTA_INVALID_TERMINATOR; +} +int fsl_delta_apply( + unsigned char const *zSrc, /* The source or pattern file */ + fsl_size_t lenSrc_, /* Length of the source file */ + unsigned char const *zDelta, /* Delta to apply to the pattern */ + fsl_size_t lenDelta_, /* Length of the delta */ + unsigned char *zOut /* Write the output into this preallocated buffer */ +){ + return fsl_delta_apply2(zSrc, lenSrc_, zDelta, lenDelta_, zOut, NULL); +} + +#undef NHASH +#undef DEBUG1 +#undef DEBUG2 +/* end of file delta.c */ +/* start of file diff.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************ + This file houses Fossil's diff-generation routines (as opposed to + the delta-generation). +*/ +#include +#include +#include +#include /* for memmove()/strlen() */ + +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +typedef uint64_t u64; +typedef void ReCompiled /* porting crutch. i would strongly prefer to + replace the regex support with a stateful + predicate callback. + */; + +#define DIFF_CONTEXT_MASK ((u64)0x0000ffff) /* Lines of context. Default if 0 */ +#define DIFF_WIDTH_MASK ((u64)0x00ff0000) /* side-by-side column width */ +#define DIFF_IGNORE_EOLWS ((u64)0x01000000) /* Ignore end-of-line whitespace */ +#define DIFF_IGNORE_ALLWS ((u64)0x03000000) /* Ignore all whitespace */ +#define DIFF_SIDEBYSIDE ((u64)0x04000000) /* Generate a side-by-side diff */ +#define DIFF_VERBOSE ((u64)0x08000000) /* Missing shown as empty files */ +#define DIFF_BRIEF ((u64)0x10000000) /* Show filenames only */ +#define DIFF_HTML ((u64)0x20000000) /* Render for HTML */ +#define DIFF_LINENO ((u64)0x40000000) /* Show line numbers */ +#define DIFF_NOOPT (((u64)0x01)<<32) /* Suppress optimizations (debug) */ +#define DIFF_INVERT (((u64)0x02)<<32) /* Invert the diff (debug) */ +#define DIFF_CONTEXT_EX (((u64)0x04)<<32) /* Use context even if zero */ +#define DIFF_NOTTOOBIG (((u64)0x08)<<32) /* Only display if not too big */ +#define DIFF_STRIP_EOLCR (((u64)0x10)<<32) /* Strip trailing CR */ + +/* Annotation flags (any DIFF flag can be used as Annotation flag as well) */ +#define ANN_FILE_VERS (((u64)0x20)<<32) /* Show file vers rather than commit vers */ +#define ANN_FILE_ANCEST (((u64)0x40)<<32) /* Prefer check-ins in the ANCESTOR table */ + + +/* + ANSI escape codes: https://en.wikipedia.org/wiki/ANSI_escape_code +*/ +#define ANSI_COLOR_BLACK(BOLD) ((BOLD) ? "\x1b[30m" : "\x1b[30m") +#define ANSI_COLOR_RED(BOLD) ((BOLD) ? "\x1b[31;1m" : "\x1b[31m") +#define ANSI_COLOR_GREEN(BOLD) ((BOLD) ? "\x1b[32;1m" : "\x1b[32m") +#define ANSI_COLOR_YELLOW(BOLD) ((BOLD) ? "\x1b[33;1m" : "\x1b[33m") +#define ANSI_COLOR_BLUE(BOLD) ((BOLD) ? "\x1b[34;1m" : "\x1b[34m") +#define ANSI_COLOR_MAGENTA(BOLD) ((BOLD) ? "\x1b[35;1m" : "\x1b[35m") +#define ANSI_COLOR_CYAN(BOLD) ((BOLD) ? "\x1b[36;1m" : "\x1b[36m") +#define ANSI_COLOR_WHITE(BOLD) ((BOLD) ? "\x1b[37;1m" : "\x1b[37m") +#define ANSI_DIFF_ADD(BOLD) ANSI_COLOR_GREEN(BOLD) +#define ANSI_DIFF_RM(BOLD) ANSI_COLOR_RED(BOLD) +#define ANSI_DIFF_MOD(BOLD) ANSI_COLOR_BLUE(BOLD) +#define ANSI_BG_BLACK(BOLD) ((BOLD) ? "\x1b[40;1m" : "\x1b[40m") +#define ANSI_BG_RED(BOLD) ((BOLD) ? "\x1b[41;1m" : "\x1b[41m") +#define ANSI_BG_GREEN(BOLD) ((BOLD) ? "\x1b[42;1m" : "\x1b[42m") +#define ANSI_BG_YELLOW(BOLD) ((BOLD) ? "\x1b[43;1m" : "\x1b[43m") +#define ANSI_BG_BLUE(BOLD) ((BOLD) ? "\x1b[44;1m" : "\x1b[44m") +#define ANSI_BG_MAGENTA(BOLD) ((BOLD) ? "\x1b[45;1m" : "\x1b[45m") +#define ANSI_BG_CYAN(BOLD) ((BOLD) ? "\x1b[46;1m" : "\x1b[46m") +#define ANSI_BG_WHITE(BOLD) ((BOLD) ? "\x1b[47;1m" : "\x1b[47m") +#define ANSI_RESET_COLOR "\x1b[39;49m" +#define ANSI_RESET_ALL "\x1b[0m" +#define ANSI_RESET ANSI_RESET_ALL +/*#define ANSI_BOLD ";1m"*/ + +/** + Extract the number of lines of context from diffFlags. + */ +static int diff_context_lines(uint64_t diffFlags){ + int n = diffFlags & DIFF_CONTEXT_MASK; + if( n==0 && (diffFlags & DIFF_CONTEXT_EX)==0 ) n = 5; + return n; +} + +/* + Extract the width of columns for side-by-side diff. Supply an + appropriate default if no width is given. +*/ +static int diff_width(uint64_t diffFlags){ + int w = (diffFlags & DIFF_WIDTH_MASK)/(DIFF_CONTEXT_MASK+1); + if( w==0 ) w = 80; + return w; +} + +/** + Converts mask of public fsl_diff_flag_t (32-bit) values to the + Fossil-internal 64-bit bitmask used by the DIFF_xxx macros. Why? + (A) fossil(1) uses the macro approach and a low-level encoding of + data in the bitmask (e.g. the context lines count). The public API + hides the lower-level flags and allows the internal API to take + care of the encoding. +*/ +static uint64_t fsl_diff_flags_convert( int mask ){ + uint64_t rc = 0U; +#define DO(F) if( (mask & F)==F ) rc |= (((u64)F) << 24) + DO(FSL_DIFF_IGNORE_EOLWS); + DO(FSL_DIFF_IGNORE_ALLWS); + DO(FSL_DIFF_SIDEBYSIDE); + DO(FSL_DIFF_VERBOSE); + DO(FSL_DIFF_BRIEF); + DO(FSL_DIFF_HTML); + DO(FSL_DIFF_LINENO); + DO(FSL_DIFF_NOOPT); + DO(FSL_DIFF_INVERT); + DO(FSL_DIFF_NOTTOOBIG); + DO(FSL_DIFF_STRIP_EOLCR); +#undef DO + return rc; +} + +/* + Length of a dline +*/ +#define LENGTH(X) ((X)->n) + +/** + Holds output state for diff generation. + */ +struct DiffOutState { + /** Output callback. */ + fsl_output_f out; + /** State for this->out(). */ + void * oState; + /** For propagating output errors. */ + int rc; + char ansiColor; +}; +typedef struct DiffOutState DiffOutState; +static const DiffOutState DiffOutState_empty = { +NULL/*out*/, +NULL/*oState*/, +0/*rc*/, +0/*useAnsiColor*/ +}; + +/** + Internal helper. Sends src to o->out(). If n is negative, fsl_strlen() + is used to determine the length. + */ +static int diff_out( DiffOutState * const o, void const * src, fsl_int_t n ){ + return o->rc = n + ? o->out(o->oState, src, n<0 ? fsl_strlen((char const *)src) : (fsl_size_t)n) + : 0; +} + +/** + fsl_output_f() impl for use with diff_outf(). state must be a + (DiffOutState*). + */ +static int fsl_output_f_diff_out( void * state, void const * src, + fsl_size_t n ){ + DiffOutState * const os = (DiffOutState *)state; + return os->rc = os->out(os->oState, src, n); +} + +static int diff_outf( DiffOutState * o, char const * fmt, ... ){ + va_list va; + va_start(va,fmt); + fsl_appendfv(fsl_output_f_diff_out, o, fmt, va); + va_end(va); + return o->rc; +} + +/* + Append a single line of context-diff output to pOut. +*/ +static int appendDiffLine( + DiffOutState *const pOut, /* Where to write the line of output */ + char cPrefix, /* One of " ", "+", or "-" */ + fsl_dline *pLine, /* The line to be output */ + int html, /* True if generating HTML. False for plain text */ + ReCompiled *pRe /* Colorize only if line matches this Regex */ +){ + int rc = 0; + char const * ansiPrefix = + !pOut->ansiColor + ? NULL + : (('+'==cPrefix) + ? ANSI_DIFF_ADD(0) + : (('-'==cPrefix) ? ANSI_DIFF_RM(0) : NULL)) + ; + if(ansiPrefix) rc = diff_out(pOut, ansiPrefix, -1 ); + if(!rc) rc = diff_out(pOut, &cPrefix, 1); + if(rc) return rc; + else if( html ){ +#if 0 + if( pRe /*MISSING: && re_dline_match(pRe, pLine, 1)==0 */ ){ + cPrefix = ' '; + }else +#endif + if( cPrefix=='+' ){ + rc = diff_out(pOut, "", -1); + }else if( cPrefix=='-' ){ + rc = diff_out(pOut, "", -1); + } + if(!rc){ + /* unsigned short n = pLine->n; */ + /* while( n>0 && (pLine->z[n-1]=='\n' || pLine->z[n-1]=='\r') ) n--; */ + rc = pOut->rc = fsl_htmlize(pOut->out, pOut->oState, + pLine->z, pLine->n); + if( !rc && cPrefix!=' ' ){ + rc = diff_out(pOut, "", -1); + } + } + }else{ + rc = diff_out(pOut, pLine->z, pLine->n); + } + if(!rc){ + if(ansiPrefix){ + rc = diff_out(pOut, ANSI_RESET, -1 ); + } + if(!rc) rc = diff_out(pOut, "\n", 1); + } + return rc; +} + + +/* + Add two line numbers to the beginning of an output line for a context + diff. One or the other of the two numbers might be zero, which means + to leave that number field blank. The "html" parameter means to format + the output for HTML. +*/ +static int appendDiffLineno(DiffOutState *pOut, int lnA, + int lnB, int html){ + int rc = 0; + if( html ){ + rc = diff_out(pOut, "", -1); + } + if(!rc){ + if( lnA>0 ){ + rc = diff_outf(pOut, "%6d ", lnA); + }else{ + rc = diff_out(pOut, " ", 7); + } + } + if(!rc){ + if( lnB>0 ){ + rc = diff_outf(pOut, "%6d ", lnB); + }else{ + rc = diff_out(pOut, " ", 8); + } + if( !rc && html ){ + rc = diff_out(pOut, "", -1); + } + } + return rc; +} + + +/* + Minimum of two values +*/ +static int minInt(int a, int b){ return aaFrom[] */ + int iS2, int iE2, /* Range of lines in p->aTo[] */ + int *piSX, int *piEX, /* Write p->aFrom[] common segment here */ + int *piSY, int *piEY /* Write p->aTo[] common segment here */ +){ + int mxLength = 0; /* Length of longest common subsequence */ + int i, j; /* Loop counters */ + int k; /* Length of a candidate subsequence */ + int iSXb = iS1; /* Best match so far */ + int iSYb = iS2; /* Best match so far */ + + for(i=iS1; icmpLine(&p->aFrom[i], &p->aTo[j]) ) continue; + if( mxLength && p->cmpLine(&p->aFrom[i+mxLength], &p->aTo[j+mxLength]) ){ + continue; + } + k = 1; + while( i+kcmpLine(&p->aFrom[i+k],&p->aTo[j+k])==0 ){ + k++; + } + if( k>mxLength ){ + iSXb = i; + iSYb = j; + mxLength = k; + } + } + } + *piSX = iSXb; + *piEX = iSXb + mxLength; + *piSY = iSYb; + *piEY = iSYb + mxLength; +} + +/** + Compare two blocks of text on lines iS1 through iE1-1 of the aFrom[] + file and lines iS2 through iE2-1 of the aTo[] file. Locate a sequence + of lines in these two blocks that are exactly the same. Return + the bounds of the matching sequence. + + If there are two or more possible answers of the same length, the + returned sequence should be the one closest to the center of the + input range. + + Ideally, the common sequence should be the longest possible common + sequence. However, an exact computation of LCS is O(N*N) which is + way too slow for larger files. So this routine uses an O(N) + heuristic approximation based on hashing that usually works about + as well. But if the O(N) algorithm doesn't get a good solution + and N is not too large, we fall back to an exact solution by + calling fsl__diff_optimal_lcs(). +*/ +static void fsl__diff_lcs( + fsl_diff_cx * const p, /* Two files being compared */ + int iS1, int iE1, /* Range of lines in p->aFrom[] */ + int iS2, int iE2, /* Range of lines in p->aTo[] */ + int *piSX, int *piEX, /* Write p->aFrom[] common segment here */ + int *piSY, int *piEY /* Write p->aTo[] common segment here */ +){ + int i, j, k; /* Loop counters */ + int n; /* Loop limit */ + fsl_dline *pA, *pB; /* Pointers to lines */ + int iSX, iSY, iEX, iEY; /* Current match */ + int skew = 0; /* How lopsided is the match */ + int dist = 0; /* Distance of match from center */ + int mid; /* Center of the chng */ + int iSXb, iSYb, iEXb, iEYb; /* Best match so far */ + int iSXp, iSYp, iEXp, iEYp; /* Previous match */ + sqlite3_int64 bestScore; /* Best score so far */ + sqlite3_int64 score; /* Score for current candidate LCS */ + int span; /* combined width of the input sequences */ + + span = (iE1 - iS1) + (iE2 - iS2); + bestScore = -10000; + score = 0; + iSXb = iSXp = iS1; + iEXb = iEXp = iS1; + iSYb = iSYp = iS2; + iEYb = iEYp = iS2; + mid = (iE1 + iS1)/2; + for(i=iS1; iaTo[p->aFrom[i].h % p->nTo].iHash; + while( j>0 + && (j-1=iE2 || p->cmpLine(&p->aFrom[i], &p->aTo[j-1])) + ){ + if( limit++ > 10 ){ + j = 0; + break; + } + j = p->aTo[j-1].iNext; + } + if( j==0 ) continue; + assert( i>=iSXb && i>=iSXp ); + if( i=iSYb && j=iSYp && jaFrom[iSX-1]; + pB = &p->aTo[iSY-1]; + n = minInt(iSX-iS1, iSY-iS2); + for(k=0; kcmpLine(pA,pB)==0; k++, pA--, pB--){} + iSX -= k; + iSY -= k; + iEX = i+1; + iEY = j; + pA = &p->aFrom[iEX]; + pB = &p->aTo[iEY]; + n = minInt(iE1-iEX, iE2-iEY); + for(k=0; kcmpLine(pA,pB)==0; k++, pA++, pB++){} + iEX += k; + iEY += k; + skew = (iSX-iS1) - (iSY-iS2); + if( skew<0 ) skew = -skew; + dist = (iSX+iEX)/2 - mid; + if( dist<0 ) dist = -dist; + score = (iEX - iSX)*(sqlite3_int64)span - (skew + dist); + if( score>bestScore ){ + bestScore = score; + iSXb = iSX; + iSYb = iSY; + iEXb = iEX; + iEYb = iEY; + }else if( iEX>iEXp ){ + iSXp = iSX; + iSYp = iSY; + iEXp = iEX; + iEYp = iEY; + } + } + if( +#if 0 + 1 + /* CANNOT EXPLAIN why we get different diff results than fossil + unless we use fsl_diff_optimal_lcs() despite using, insofar as + i can tell, the same inputs. There is some aspect i'm + overlooking. */ +#else + iSXb==iEXb && (sqlite3_int64)(iE1-iS1)*(iE2-iS2)<400 +#endif + ){ + /* If no common sequence is found using the hashing heuristic and + ** the input is not too big, use the expensive exact solution */ + fsl__diff_optimal_lcs(p, iS1, iE1, iS2, iE2, piSX, piEX, piSY, piEY); + }else{ + *piSX = iSXb; + *piSY = iSYb; + *piEX = iEXb; + *piEY = iEYb; + } +} + + +void fsl__dump_triples(fsl_diff_cx const * const p, + char const * zFile, int ln ){ + // Compare this with (fossil xdiff --raw) on the same inputs + fprintf(stderr,"%s:%d: Compare this with (fossil xdiff --raw) on the same inputs:\n", + zFile, ln); + for(int i = 0; p->aEdit[i] || p->aEdit[i+1] || p->aEdit[i+2]; i+=3){ + printf(" copy %6d delete %6d insert %6d\n", + p->aEdit[i], p->aEdit[i+1], p->aEdit[i+2]); + } +} + +/** @internal + Expand the size of p->aEdit array to hold at least nEdit elements. + */ +static int fsl__diff_expand_edit(fsl_diff_cx * const p, int nEdit){ + void * re = fsl_realloc(p->aEdit, nEdit*sizeof(int)); + if(!re) return FSL_RC_OOM; + else{ + p->aEdit = (int*)re; + p->nEditAlloc = nEdit; + return 0; + } +} + +/** + Append a new COPY/DELETE/INSERT triple. + + Returns 0 on success, FSL_RC_OOM on OOM. + */ +static int appendTriple(fsl_diff_cx *p, int nCopy, int nDel, int nIns){ + /* printf("APPEND %d/%d/%d\n", nCopy, nDel, nIns); */ + if( p->nEdit>=3 ){ + if( p->aEdit[p->nEdit-1]==0 ){ + if( p->aEdit[p->nEdit-2]==0 ){ + p->aEdit[p->nEdit-3] += nCopy; + p->aEdit[p->nEdit-2] += nDel; + p->aEdit[p->nEdit-1] += nIns; + return 0; + } + if( nCopy==0 ){ + p->aEdit[p->nEdit-2] += nDel; + p->aEdit[p->nEdit-1] += nIns; + return 0; + } + } + if( nCopy==0 && nDel==0 ){ + p->aEdit[p->nEdit-1] += nIns; + return 0; + } + } + if( p->nEdit+3>p->nEditAlloc ){ + int const rc = fsl__diff_expand_edit(p, p->nEdit*2 + 15); + if(rc) return rc; + else if( p->aEdit==0 ) return 0; + } + p->aEdit[p->nEdit++] = nCopy; + p->aEdit[p->nEdit++] = nDel; + p->aEdit[p->nEdit++] = nIns; + return 0; +} + +/** + Do a single step in the difference. Compute a sequence of + copy/delete/insert steps that will convert lines iS1 through iE1-1 + of the input into lines iS2 through iE2-1 of the output and write + that sequence into the difference context. + + The algorithm is to find a block of common text near the middle of + the two segments being diffed. Then recursively compute + differences on the blocks before and after that common segment. + Special cases apply if either input segment is empty or if the two + segments have no text in common. + */ +static int diff_step(fsl_diff_cx *p, int iS1, int iE1, int iS2, int iE2){ + int iSX, iEX, iSY, iEY; + int rc = 0; + if( iE1<=iS1 ){ + /* The first segment is empty */ + if( iE2>iS2 ){ + rc = appendTriple(p, 0, 0, iE2-iS2); + } + return rc; + } + if( iE2<=iS2 ){ + /* The second segment is empty */ + return appendTriple(p, 0, iE1-iS1, 0); + } + + /* Find the longest matching segment between the two sequences */ + fsl__diff_lcs(p, iS1, iE1, iS2, iE2, &iSX, &iEX, &iSY, &iEY); + if( iEX>iSX ){ + /* A common segment has been found. + Recursively diff either side of the matching segment */ + rc = diff_step(p, iS1, iSX, iS2, iSY); + if(!rc){ + if(iEX>iSX){ + rc = appendTriple(p, iEX - iSX, 0, 0); + } + if(!rc) rc = diff_step(p, iEX, iE1, iEY, iE2); + } + }else{ + /* The two segments have nothing in common. Delete the first then + insert the second. */ + rc = appendTriple(p, 0, iE1-iS1, iE2-iS2); + } + return rc; +} + +int fsl__diff_all(fsl_diff_cx * const p){ + int mnE, iS, iE1, iE2; + int rc = 0; + /* Carve off the common header and footer */ + iE1 = p->nFrom; + iE2 = p->nTo; + while( iE1>0 && iE2>0 && p->cmpLine(&p->aFrom[iE1-1], &p->aTo[iE2-1])==0 ){ + iE1--; + iE2--; + } + mnE = iE1cmpLine(&p->aFrom[iS],&p->aTo[iS])==0; iS++){} + + /* do the difference */ + if( iS>0 ){ + rc = appendTriple(p, iS, 0, 0); + if(rc) return rc; + } + rc = diff_step(p, iS, iE1, iS, iE2); + //fsl__dump_triples(p, __FILE__, __LINE__); + if(rc) return rc; + else if( iE1nFrom ){ + rc = appendTriple(p, p->nFrom - iE1, 0, 0); + if(rc) return rc; + } + /* Terminate the COPY/DELETE/INSERT triples with three zeros */ + rc = fsl__diff_expand_edit(p, p->nEdit+3); + if(0==rc){ + if(p->aEdit ){ + p->aEdit[p->nEdit++] = 0; + p->aEdit[p->nEdit++] = 0; + p->aEdit[p->nEdit++] = 0; + //fsl__dump_triples(p, __FILE__, __LINE__); + } + } + return rc; +} + + +/* + Attempt to shift insertion or deletion blocks so that they begin and + end on lines that are pure whitespace. In other words, try to transform + this: + + int func1(int x){ + return x*10; + +} + + + +int func2(int x){ + + return x*20; + } + + int func3(int x){ + return x/5; + } + + Into one of these: + + int func1(int x){ int func1(int x){ + return x*10; return x*10; + } } + + + +int func2(int x){ +int func2(int x){ + + return x*20; + return x*20; + +} +} + + + int func3(int x){ int func3(int x){ + return x/5; return x/5; + } } +*/ +void fsl__diff_optimize(fsl_diff_cx * const p){ + int r; /* Index of current triple */ + int lnFrom; /* Line number in p->aFrom */ + int lnTo; /* Line number in p->aTo */ + int cpy, del, ins; + + //fsl__dump_triples(p, __FILE__, __LINE__); + lnFrom = lnTo = 0; + for(r=0; rnEdit; r += 3){ + cpy = p->aEdit[r]; + del = p->aEdit[r+1]; + ins = p->aEdit[r+2]; + lnFrom += cpy; + lnTo += cpy; + + /* Shift insertions toward the beginning of the file */ + while( cpy>0 && del==0 && ins>0 ){ + fsl_dline *pTop = &p->aFrom[lnFrom-1]; /* Line before start of insert */ + fsl_dline *pBtm = &p->aTo[lnTo+ins-1]; /* Last line inserted */ + if( p->cmpLine(pTop, pBtm) ) break; + if( LENGTH(pTop+1)+LENGTH(pBtm)<=LENGTH(pTop)+LENGTH(pBtm-1) ) break; + lnFrom--; + lnTo--; + p->aEdit[r]--; + p->aEdit[r+3]++; + cpy--; + } + + /* Shift insertions toward the end of the file */ + while( r+3nEdit && p->aEdit[r+3]>0 && del==0 && ins>0 ){ + fsl_dline *pTop = &p->aTo[lnTo]; /* First line inserted */ + fsl_dline *pBtm = &p->aTo[lnTo+ins]; /* First line past end of insert */ + if( p->cmpLine(pTop, pBtm) ) break; + if( LENGTH(pTop)+LENGTH(pBtm-1)<=LENGTH(pTop+1)+LENGTH(pBtm) ) break; + lnFrom++; + lnTo++; + p->aEdit[r]++; + p->aEdit[r+3]--; + cpy++; + } + + /* Shift deletions toward the beginning of the file */ + while( cpy>0 && del>0 && ins==0 ){ + fsl_dline *pTop = &p->aFrom[lnFrom-1]; /* Line before start of delete */ + fsl_dline *pBtm = &p->aFrom[lnFrom+del-1]; /* Last line deleted */ + if( p->cmpLine(pTop, pBtm) ) break; + if( LENGTH(pTop+1)+LENGTH(pBtm)<=LENGTH(pTop)+LENGTH(pBtm-1) ) break; + lnFrom--; + lnTo--; + p->aEdit[r]--; + p->aEdit[r+3]++; + cpy--; + } + + /* Shift deletions toward the end of the file */ + while( r+3nEdit && p->aEdit[r+3]>0 && del>0 && ins==0 ){ + fsl_dline *pTop = &p->aFrom[lnFrom]; /* First line deleted */ + fsl_dline *pBtm = &p->aFrom[lnFrom+del]; /* First line past end of delete */ + if( p->cmpLine(pTop, pBtm) ) break; + if( LENGTH(pTop)+LENGTH(pBtm-1)<=LENGTH(pTop)+LENGTH(pBtm) ) break; + lnFrom++; + lnTo++; + p->aEdit[r]++; + p->aEdit[r+3]--; + cpy++; + } + + lnFrom += del; + lnTo += ins; + } + //fsl__dump_triples(p, __FILE__, __LINE__); +} + + +/* + Given a raw diff p[] in which the p->aEdit[] array has been filled + in, compute a context diff into pOut. +*/ +static int contextDiff( + fsl_diff_cx *p, /* The difference */ + DiffOutState *pOut, /* Output a context diff to here */ + ReCompiled *pRe, /* Only show changes that match this regex */ + u64 diffFlags /* Flags controlling the diff format */ +){ + fsl_dline *A; /* Left side of the diff */ + fsl_dline *B; /* Right side of the diff */ + int a = 0; /* Index of next line in A[] */ + int b = 0; /* Index of next line in B[] */ + int *R; /* Array of COPY/DELETE/INSERT triples */ + int r; /* Index into R[] */ + int nr; /* Number of COPY/DELETE/INSERT triples to process */ + int mxr; /* Maximum value for r */ + int na, nb; /* Number of lines shown from A and B */ + int i, j; /* Loop counters */ + int m; /* Number of lines to output */ + int skip; /* Number of lines to skip */ + static int nChunk = 0; /* Number of diff chunks seen so far */ + int nContext; /* Number of lines of context */ + int showLn; /* Show line numbers */ + int html; /* Render as HTML */ + int showDivider = 0; /* True to show the divider between diff blocks */ + int rc = 0; + nContext = diff_context_lines(diffFlags); + showLn = (diffFlags & DIFF_LINENO)!=0; + html = (diffFlags & DIFF_HTML)!=0; + A = p->aFrom; + B = p->aTo; + R = p->aEdit; + mxr = p->nEdit; + while( mxr>2 && R[mxr-1]==0 && R[mxr-2]==0 ){ mxr -= 3; } + for(r=0; r0 && R[r+nr*3]nContext ){ + na = nb = nContext; + skip = R[r] - nContext; + }else{ + na = nb = R[r]; + skip = 0; + } + for(i=0; inContext ){ + na += nContext; + nb += nContext; + }else{ + na += R[r+nr*3]; + nb += R[r+nr*3]; + } + for(i=1; i%.80c\n", '.'); + }else{ + rc = diff_outf(pOut, "%.80c\n", '.'); + } + if( !rc && html ){ + rc = diff_outf(pOut, "", nChunk); + } + }else{ + char const * ansi1 = ""; + char const * ansi2 = ""; + char const * ansi3 = ""; + if( html ) rc = diff_outf(pOut, ""); + else if(0 && pOut->ansiColor){ + /* Turns out this just confuses the output */ + ansi1 = ANSI_DIFF_RM(0); + ansi2 = ANSI_DIFF_ADD(0); + ansi3 = ANSI_RESET; + } + /* + * If the patch changes an empty file or results in an empty file, + * the block header must use 0,0 as position indicator and not 1,0. + * Otherwise, patch would be confused and may reject the diff. + */ + if(!rc) rc = diff_outf(pOut,"@@ %s-%d,%d %s+%d,%d%s @@", + ansi1, na ? a+skip+1 : 0, na, + ansi2, nb ? b+skip+1 : 0, nb, + ansi3); + if( !rc ){ + if( html ) rc = diff_outf(pOut, ""); + if(!rc) rc = diff_out(pOut, "\n", 1); + } + } + if(rc) return rc; + + /* Show the initial common area */ + a += skip; + b += skip; + m = R[r] - skip; + for(j=0; !rc && jnContext ) m = nContext; + for(j=0; !rc && j tag */ + int iEnd; /* Write prior to character iEnd */ + int iStart2; /* Write zStart2 prior to character iStart2 */ + const char *zStart2; /* A tag */ + int iEnd2; /* Write prior to character iEnd2 */ + ReCompiled *pRe; /* Only colorize matching lines, if not NULL */ + DiffOutState * pOut; +}; + +/* + Column indices for SbsLine.apCols[] +*/ +#define SBS_LNA 0 /* Left line number */ +#define SBS_TXTA 1 /* Left text */ +#define SBS_MKR 2 /* Middle separator column */ +#define SBS_LNB 3 /* Right line number */ +#define SBS_TXTB 4 /* Right text */ + +/* + Append newlines to all columns. +*/ +static int sbsWriteNewlines(SbsLine *p){ + int i; + int rc = 0; + for( i=p->escHtml ? SBS_LNA : SBS_TXTB; !rc && i<=SBS_TXTB; i++ ){ + rc = fsl_buffer_append(p->apCols[i], "\n", 1); + } + return rc; +} + +/* + Append n spaces to the column. +*/ +static int sbsWriteSpace(SbsLine *p, int n, int col){ + return fsl_buffer_appendf(p->apCols[col], "%*s", n, ""); +} + +/* + Write the text of pLine into column iCol of p. + + If outputting HTML, write the full line. Otherwise, only write the + width characters. Translate tabs into spaces. Add newlines if col + is SBS_TXTB. Translate HTML characters if escHtml is true. Pad the + rendering to width bytes if col is SBS_TXTA and escHtml is false. + + This comment contains multibyte unicode characters (�, �, �) in order + to test the ability of the diff code to handle such characters. +*/ +static int sbsWriteText(SbsLine *p, fsl_dline *pLine, int col){ + fsl_buffer *pCol = p->apCols[col]; + int rc = 0; + int n = pLine->n; + int i; /* Number of input characters consumed */ + int k; /* Cursor position */ + int needEndSpan = 0; + const char *zIn = pLine->z; + int w = p->width; + int colorize = p->escHtml; +#if 0 + /* MISSING: re bits, but want to replace those with a predicate. */ + if( colorize && p->pRe && re_dline_match(p->pRe, pLine, 1)==0 ){ + colorize = 0; + } +#endif + for(i=k=0; !rc && (p->escHtml || kiStart ){ + rc = fsl_buffer_append(pCol, p->zStart, -1); + if(rc) break; + needEndSpan = 1; + if( p->iStart2 ){ + p->iStart = p->iStart2; + p->zStart = p->zStart2; + p->iStart2 = 0; + } + }else if( i==p->iEnd ){ + rc = fsl_buffer_append(pCol, "", 7); + if(rc) break; + needEndSpan = 0; + if( p->iEnd2 ){ + p->iEnd = p->iEnd2; + p->iEnd2 = 0; + } + } + } + if( c=='\t' && !p->escHtml ){ + rc = fsl_buffer_append(pCol, " ", 1); + while( !rc && (k&7)!=7 && (p->escHtml || kescHtml ){ + rc = fsl_buffer_append(pCol, "<", 4); + }else if( c=='&' && p->escHtml ){ + rc = fsl_buffer_append(pCol, "&", 5); + }else if( c=='>' && p->escHtml ){ + rc = fsl_buffer_append(pCol, ">", 4); + }else if( c=='"' && p->escHtml ){ + rc = fsl_buffer_append(pCol, """, 6); + }else{ + rc = fsl_buffer_append(pCol, &zIn[i], 1); + if( (c&0xc0)==0x80 ) k--; + } + } + if( !rc && needEndSpan ){ + rc = fsl_buffer_append(pCol, "", 7); + } + if(!rc){ + if( col==SBS_TXTB ){ + rc = sbsWriteNewlines(p); + }else if( !p->escHtml ){ + rc = sbsWriteSpace(p, w-k, SBS_TXTA); + } + } + return rc; +} + + +/* + Append a column to the final output blob. +*/ +static int sbsWriteColumn(DiffOutState *pOut, fsl_buffer const *pCol, int col){ + return diff_outf(pOut, + "
\n" + "
\n"
+    "%b"
+    "
\n" + "
\n", + col % 3 ? (col == SBS_MKR ? "separator" : "text") : "lineno", + pCol + ); +} + + +/* + Append a separator line to column iCol +*/ +static int sbsWriteSep(SbsLine *p, int len, int col){ + char ch = '.'; + if( len<1 ){ + len = 1; + ch = ' '; + } + return fsl_buffer_appendf(p->apCols[col], + "%.*c\n", + len, ch); +} + + +/* + Append the appropriate marker into the center column of the diff. +*/ +static int sbsWriteMarker(SbsLine *p, const char *zTxt, + const char *zHtml){ + return fsl_buffer_append(p->apCols[SBS_MKR], p->escHtml ? zHtml : zTxt, -1); +} + +/* + Append a line number to the column. +*/ +static int sbsWriteLineno(SbsLine *p, int ln, int col){ + int rc; + if( p->escHtml ){ + rc = fsl_buffer_appendf(p->apCols[col], "%d", ln+1); + }else{ + char zLn[8]; + fsl_snprintf(zLn, 8, "%5d ", ln+1); + rc = fsl_buffer_appendf(p->apCols[col], "%s ", zLn); + } + return rc; +} + + +/* + The two text segments zLeft and zRight are known to be different on + both ends, but they might have a common segment in the middle. If + they do not have a common segment, return 0. If they do have a large + common segment, return 1 and before doing so set: + + aLCS[0] = start of the common segment in zLeft + aLCS[1] = end of the common segment in zLeft + aLCS[2] = start of the common segment in zLeft + aLCS[3] = end of the common segment in zLeft + + This computation is for display purposes only and does not have to be + optimal or exact. +*/ +static int textLCS( + const char *zLeft, int nA, /* String on the left */ + const char *zRight, int nB, /* String on the right */ + int *aLCS /* Identify bounds of LCS here */ +){ + const unsigned char *zA = (const unsigned char*)zLeft; /* left string */ + const unsigned char *zB = (const unsigned char*)zRight; /* right string */ + int nt; /* Number of target points */ + int ti[3]; /* Index for start of each 4-byte target */ + unsigned int target[3]; /* 4-byte alignment targets */ + unsigned int probe; /* probe to compare against target */ + int iAS, iAE, iBS, iBE; /* Range of common segment */ + int i, j; /* Loop counters */ + int rc = 0; /* Result code. 1 for success */ + + if( nA<6 || nB<6 ) return 0; + memset(aLCS, 0, sizeof(int)*4); + ti[0] = i = nB/2-2; + target[0] = (zB[i]<<24) | (zB[i+1]<<16) | (zB[i+2]<<8) | zB[i+3]; + probe = 0; + if( nB<16 ){ + nt = 1; + }else{ + ti[1] = i = nB/4-2; + target[1] = (zB[i]<<24) | (zB[i+1]<<16) | (zB[i+2]<<8) | zB[i+3]; + ti[2] = i = (nB*3)/4-2; + target[2] = (zB[i]<<24) | (zB[i+1]<<16) | (zB[i+2]<<8) | zB[i+3]; + nt = 3; + } + probe = (zA[0]<<16) | (zA[1]<<8) | zA[2]; + for(i=3; i0 && iBS>0 && zA[iAS-1]==zB[iBS-1] ){ iAS--; iBS--; } + if( iAE-iAS > aLCS[1] - aLCS[0] ){ + aLCS[0] = iAS; + aLCS[1] = iAE; + aLCS[2] = iBS; + aLCS[3] = iBE; + rc = 1; + } + } + } + } + return rc; +} + + +/* + Try to shift iStart as far as possible to the left. +*/ +static void sbsShiftLeft(SbsLine *p, const char *z){ + int i, j; + while( (i=p->iStart)>0 && z[i-1]==z[i] ){ + for(j=i+1; jiEnd && z[j-1]==z[j]; j++){} + if( jiEnd ) break; + p->iStart--; + p->iEnd--; + } +} + +/* + Simplify iStart and iStart2: + + * If iStart is a null-change then move iStart2 into iStart + * Make sure any null-changes are in canonoical form. + * Make sure all changes are at character boundaries for + multi-byte characters. +*/ +static void sbsSimplifyLine(SbsLine *p, const char *z){ + if( p->iStart2==p->iEnd2 ){ + p->iStart2 = p->iEnd2 = 0; + }else if( p->iStart2 ){ + while( p->iStart2>0 && (z[p->iStart2]&0xc0)==0x80 ) p->iStart2--; + while( (z[p->iEnd2]&0xc0)==0x80 ) p->iEnd2++; + } + if( p->iStart==p->iEnd ){ + p->iStart = p->iStart2; + p->iEnd = p->iEnd2; + p->zStart = p->zStart2; + p->iStart2 = 0; + p->iEnd2 = 0; + } + if( p->iStart==p->iEnd ){ + p->iStart = p->iEnd = -1; + }else if( p->iStart>0 ){ + while( p->iStart>0 && (z[p->iStart]&0xc0)==0x80 ) p->iStart--; + while( (z[p->iEnd]&0xc0)==0x80 ) p->iEnd++; + } +} + + +/* + Write out lines that have been edited. Adjust the highlight to cover + only those parts of the line that actually changed. +*/ +static int sbsWriteLineChange( + SbsLine *p, /* The SBS output line */ + fsl_dline *pLeft, /* Left line of the change */ + int lnLeft, /* Line number for the left line */ + fsl_dline *pRight, /* Right line of the change */ + int lnRight /* Line number of the right line */ +){ + int rc = 0; + int nLeft; /* Length of left line in bytes */ + int nRight; /* Length of right line in bytes */ + int nShort; /* Shortest of left and right */ + int nPrefix; /* Length of common prefix */ + int nSuffix; /* Length of common suffix */ + const char *zLeft; /* Text of the left line */ + const char *zRight; /* Text of the right line */ + int nLeftDiff; /* nLeft - nPrefix - nSuffix */ + int nRightDiff; /* nRight - nPrefix - nSuffix */ + int aLCS[4] = {0,0,0,0}; /* Bounds of common middle segment */ + static const char zClassRm[] = ""; + static const char zClassAdd[] = ""; + static const char zClassChng[] = ""; + + nLeft = pLeft->n; + zLeft = pLeft->z; + nRight = pRight->n; + zRight = pRight->z; + nShort = nLeft0 && (zLeft[nPrefix]&0xc0)==0x80 ) nPrefix--; + } + nSuffix = 0; + if( nPrefix0 && (zLeft[nLeft-nSuffix]&0xc0)==0x80 ) nSuffix--; + } + if( nSuffix==nLeft || nSuffix==nRight ) nPrefix = 0; + } + if( nPrefix+nSuffix > nShort ) nPrefix = nShort - nSuffix; + + /* A single chunk of text inserted on the right */ + if( nPrefix+nSuffix==nLeft ){ + rc = sbsWriteLineno(p, lnLeft, SBS_LNA); + if(rc) return rc; + p->iStart2 = p->iEnd2 = 0; + p->iStart = p->iEnd = -1; + rc = sbsWriteText(p, pLeft, SBS_TXTA); + if( !rc && nLeft==nRight && zLeft[nLeft]==zRight[nRight] ){ + rc = sbsWriteMarker(p, " ", ""); + }else{ + rc = sbsWriteMarker(p, " | ", "|"); + } + if(!rc){ + rc = sbsWriteLineno(p, lnRight, SBS_LNB); + if(!rc){ + p->iStart = nPrefix; + p->iEnd = nRight - nSuffix; + p->zStart = zClassAdd; + rc = sbsWriteText(p, pRight, SBS_TXTB); + } + } + return rc; + } + + /* A single chunk of text deleted from the left */ + if( nPrefix+nSuffix==nRight ){ + /* Text deleted from the left */ + rc = sbsWriteLineno(p, lnLeft, SBS_LNA); + if(!rc){ + p->iStart2 = p->iEnd2 = 0; + p->iStart = nPrefix; + p->iEnd = nLeft - nSuffix; + p->zStart = zClassRm; + rc = sbsWriteText(p, pLeft, SBS_TXTA); + if(!rc){ + rc = sbsWriteMarker(p, " | ", "|"); + if(!rc){ + rc = sbsWriteLineno(p, lnRight, SBS_LNB); + if(!rc){ + p->iStart = p->iEnd = -1; + sbsWriteText(p, pRight, SBS_TXTB); + } + } + } + } + return rc; + } + + /* At this point we know that there is a chunk of text that has + changed between the left and the right. Check to see if there + is a large unchanged section in the middle of that changed block. + */ + nLeftDiff = nLeft - nSuffix - nPrefix; + nRightDiff = nRight - nSuffix - nPrefix; + if( p->escHtml + && nLeftDiff >= 6 + && nRightDiff >= 6 + && textLCS(&zLeft[nPrefix], nLeftDiff, &zRight[nPrefix], nRightDiff, aLCS) + ){ + rc = sbsWriteLineno(p, lnLeft, SBS_LNA); + if(rc) return rc; + p->iStart = nPrefix; + p->iEnd = nPrefix + aLCS[0]; + if( aLCS[2]==0 ){ + sbsShiftLeft(p, pLeft->z); + p->zStart = zClassRm; + }else{ + p->zStart = zClassChng; + } + p->iStart2 = nPrefix + aLCS[1]; + p->iEnd2 = nLeft - nSuffix; + p->zStart2 = aLCS[3]==nRightDiff ? zClassRm : zClassChng; + sbsSimplifyLine(p, zLeft+nPrefix); + rc = sbsWriteText(p, pLeft, SBS_TXTA); + if(!rc) rc = sbsWriteMarker(p, " | ", "|"); + if(!rc) rc = sbsWriteLineno(p, lnRight, SBS_LNB); + if(rc) return rc; + p->iStart = nPrefix; + p->iEnd = nPrefix + aLCS[2]; + if( aLCS[0]==0 ){ + sbsShiftLeft(p, pRight->z); + p->zStart = zClassAdd; + }else{ + p->zStart = zClassChng; + } + p->iStart2 = nPrefix + aLCS[3]; + p->iEnd2 = nRight - nSuffix; + p->zStart2 = aLCS[1]==nLeftDiff ? zClassAdd : zClassChng; + sbsSimplifyLine(p, zRight+nPrefix); + rc = sbsWriteText(p, pRight, SBS_TXTB); + return rc; + } + + /* If all else fails, show a single big change between left and right */ + rc = sbsWriteLineno(p, lnLeft, SBS_LNA); + if(!rc){ + p->iStart2 = p->iEnd2 = 0; + p->iStart = nPrefix; + p->iEnd = nLeft - nSuffix; + p->zStart = zClassChng; + rc = sbsWriteText(p, pLeft, SBS_TXTA); + if(!rc){ + rc = sbsWriteMarker(p, " | ", "|"); + if(!rc){ + rc = sbsWriteLineno(p, lnRight, SBS_LNB); + if(!rc){ + p->iEnd = nRight - nSuffix; + sbsWriteText(p, pRight, SBS_TXTB); + } + } + } + } + return rc; +} + + +/* + Return the number between 0 and 100 that is smaller the closer pA and + pB match. Return 0 for a perfect match. Return 100 if pA and pB are + completely different. + + The current algorithm is as follows: + + (1) Remove leading and trailing whitespace. + (2) Truncate both strings to at most 250 characters + (3) Find the length of the longest common subsequence + (4) Longer common subsequences yield lower scores. +*/ +static int match_dline(fsl_dline *pA, fsl_dline *pB){ + const char *zA; /* Left string */ + const char *zB; /* right string */ + int nA; /* Bytes in zA[] */ + int nB; /* Bytes in zB[] */ + int avg; /* Average length of A and B */ + int i, j, k; /* Loop counters */ + int best = 0; /* Longest match found so far */ + int score; /* Final score. 0..100 */ + unsigned char c; /* Character being examined */ + unsigned char aFirst[256]; /* aFirst[X] = index in zB[] of first char X */ + unsigned char aNext[252]; /* aNext[i] = index in zB[] of next zB[i] char */ + + zA = pA->z; + zB = pB->z; + nA = pA->n; + nB = pB->n; + while( nA>0 && fsl_isspace(zA[0]) ){ nA--; zA++; } + while( nA>0 && fsl_isspace(zA[nA-1]) ){ nA--; } + while( nB>0 && fsl_isspace(zB[0]) ){ nB--; zB++; } + while( nB>0 && fsl_isspace(zB[nB-1]) ){ nB--; } + if( nA>250 ) nA = 250; + if( nB>250 ) nB = 250; + avg = (nA+nB)/2; + if( avg==0 ) return 0; + if( nA==nB && memcmp(zA, zB, nA)==0 ) return 0; + memset(aFirst, 0, sizeof(aFirst)); + zA--; zB--; /* Make both zA[] and zB[] 1-indexed */ + for(i=nB; i>0; i--){ + c = (unsigned char)zB[i]; + aNext[i] = aFirst[c]; + aFirst[c] = i; + } + best = 0; + for(i=1; i<=nA-best; i++){ + c = (unsigned char)zA[i]; + for(j=aFirst[c]; j>0 && jbest ) best = k; + } + } + score = (best>avg) ? 0 : (avg - best)*100/avg; + +#if 0 + fprintf(stderr, "A: [%.*s]\nB: [%.*s]\nbest=%d avg=%d score=%d\n", + nA, zA+1, nB, zB+1, best, avg, score); +#endif + + /* Return the result */ + return score; +} + + +/* + There is a change block in which nLeft lines of text on the left are + converted into nRight lines of text on the right. This routine computes + how the lines on the left line up with the lines on the right. + + The return value is a buffer of unsigned characters, obtained from + fsl_malloc(). (The caller needs to free the return value using + fsl_free().) Entries in the returned array have values as follows: + + 1. Delete the next line of pLeft. + 2. Insert the next line of pRight. + 3. The next line of pLeft changes into the next line of pRight. + 4. Delete one line from pLeft and add one line to pRight. + + Values larger than three indicate better matches. + + The length of the returned array will be just large enough to cause + all elements of pLeft and pRight to be consumed. + + Algorithm: Wagner's minimum edit-distance algorithm, modified by + adding a cost to each match based on how well the two rows match + each other. Insertion and deletion costs are 50. Match costs + are between 0 and 100 where 0 is a perfect match 100 is a complete + mismatch. +*/ +static unsigned char *sbsAlignment( + fsl_dline *aLeft, int nLeft, /* Text on the left */ + fsl_dline *aRight, int nRight /* Text on the right */ +){ + int i, j, k; /* Loop counters */ + int *a; /* One row of the Wagner matrix */ + int *pToFree; /* Space that needs to be freed */ + unsigned char *aM; /* Wagner result matrix */ + int nMatch, iMatch; /* Number of matching lines and match score */ + int mnLen; /* MIN(nLeft, nRight) */ + int mxLen; /* MAX(nLeft, nRight) */ + int aBuf[100]; /* Stack space for a[] if nRight not to big */ + + aM = (unsigned char *)fsl_malloc( (nLeft+1)*(nRight+1) ); + if(!aM) return NULL; + if( nLeft==0 ){ + memset(aM, 2, nRight); + return aM; + } + if( nRight==0 ){ + memset(aM, 1, nLeft); + return aM; + } + + /* This algorithm is O(N**2). So if N is too big, bail out with a + simple (but stupid and ugly) result that doesn't take too long. */ + mnLen = nLeft100000 ){ + memset(aM, 4, mnLen); + if( nLeft>mnLen ) memset(aM+mnLen, 1, nLeft-mnLen); + if( nRight>mnLen ) memset(aM+mnLen, 2, nRight-mnLen); + return aM; + } + + if( nRight < (int)(sizeof(aBuf)/sizeof(aBuf[0]))-1 ){ + pToFree = 0; + a = aBuf; + }else{ + a = pToFree = fsl_malloc( sizeof(a[0])*(nRight+1) ); + if(!a){ + fsl_free(aM); + return NULL; + } + } + + /* Compute the best alignment */ + for(i=0; i<=nRight; i++){ + aM[i] = 2; + a[i] = i*50; + } + aM[0] = 0; + for(j=1; j<=nLeft; j++){ + int p = a[0]; + a[0] = p+50; + aM[j*(nRight+1)] = 1; + for(i=1; i<=nRight; i++){ + int m = a[i-1]+50; + int d = 2; + if( m>a[i]+50 ){ + m = a[i]+50; + d = 1; + } + if( m>p ){ + int score = match_dline(&aLeft[j-1], &aRight[i-1]); + if( (score<=63 || (ij-1)) && m>p+score ){ + m = p+score; + d = 3 | score*4; + } + } + p = a[i]; + a[i] = m; + aM[j*(nRight+1)+i] = d; + } + } + + /* Compute the lowest-cost path back through the matrix */ + i = nRight; + j = nLeft; + k = (nRight+1)*(nLeft+1)-1; + nMatch = iMatch = 0; + while( i+j>0 ){ + unsigned char c = aM[k]; + if( c>=3 ){ + assert( i>0 && j>0 ); + i--; + j--; + nMatch++; + iMatch += (c>>2); + aM[k] = 3; + }else if( c==2 ){ + assert( i>0 ); + i--; + }else{ + assert( j>0 ); + j--; + } + k--; + aM[k] = aM[j*(nRight+1)+i]; + } + k++; + i = (nRight+1)*(nLeft+1) - k; + memmove(aM, &aM[k], i); + + /* If: + (1) the alignment is more than 25% longer than the longest side, and + (2) the average match cost exceeds 15 + Then this is probably an alignment that will be difficult for humans + to read. So instead, just show all of the right side inserted followed + by all of the left side deleted. + + The coefficients for conditions (1) and (2) above are determined by + experimentation. + */ + mxLen = nLeft>nRight ? nLeft : nRight; + if( i*4>mxLen*5 && (nMatch==0 || iMatch/nMatch>15) ){ + memset(aM, 4, mnLen); + if( nLeft>mnLen ) memset(aM+mnLen, 1, nLeft-mnLen); + if( nRight>mnLen ) memset(aM+mnLen, 2, nRight-mnLen); + } + + /* Return the result */ + fsl_free(pToFree); + return aM; +} + +/* + R[] is an array of six integer, two COPY/DELETE/INSERT triples for a + pair of adjacent differences. Return true if the gap between these + two differences is so small that they should be rendered as a single + edit. +*/ +static int smallGap(int *R){ + return R[3]<=2 || R[3]<=(R[1]+R[2]+R[4]+R[5])/8; +} + +/* + Given a diff context in which the aEdit[] array has been filled + in, compute a side-by-side diff into pOut. +*/ +static int sbsDiff( + fsl_diff_cx *p, /* The computed diff */ + DiffOutState *pOut, /* Write the results here */ + ReCompiled *pRe, /* Only show changes that match this regex */ + u64 diffFlags /* Flags controlling the diff */ +){ + fsl_dline *A; /* Left side of the diff */ + fsl_dline *B; /* Right side of the diff */ + int rc = 0; + int a = 0; /* Index of next line in A[] */ + int b = 0; /* Index of next line in B[] */ + int *R; /* Array of COPY/DELETE/INSERT triples */ + int r; /* Index into R[] */ + int nr; /* Number of COPY/DELETE/INSERT triples to process */ + int mxr; /* Maximum value for r */ + int na, nb; /* Number of lines shown from A and B */ + int i, j; /* Loop counters */ + int m, ma, mb;/* Number of lines to output */ + int skip; /* Number of lines to skip */ + static int nChunk = 0; /* Number of chunks of diff output seen so far */ + SbsLine s; /* Output line buffer */ + int nContext; /* Lines of context above and below each change */ + int showDivider = 0; /* True to show the divider */ + fsl_buffer unesc = fsl_buffer_empty; + fsl_buffer aCols[5] = { /* Array of column blobs */ + fsl_buffer_empty_m, fsl_buffer_empty_m, fsl_buffer_empty_m, + fsl_buffer_empty_m, fsl_buffer_empty_m + }; + + memset(&s, 0, sizeof(s)); + s.pOut = pOut; + s.width = diff_width(diffFlags); + nContext = diff_context_lines(diffFlags); + s.escHtml = (diffFlags & DIFF_HTML)!=0; + if( s.escHtml ){ + for(i=SBS_LNA; i<=SBS_TXTB; i++){ + s.apCols[i] = &aCols[i]; + } + }else{ + for(i=SBS_LNA; i<=SBS_TXTB; i++){ + s.apCols[i] = &unesc; + } + } + s.pRe = pRe; + s.iStart = -1; + s.iStart2 = 0; + s.iEnd = -1; + A = p->aFrom; + B = p->aTo; + R = p->aEdit; + mxr = p->nEdit; + while( mxr>2 && R[mxr-1]==0 && R[mxr-2]==0 ){ mxr -= 3; } + + for(r=0; r0 && R[r+nr*3]nContext ){ + na = nb = nContext; + skip = R[r] - nContext; + }else{ + na = nb = R[r]; + skip = 0; + } + for(i=0; inContext ){ + na += nContext; + nb += nContext; + }else{ + na += R[r+nr*3]; + nb += R[r+nr*3]; + } + for(i=1; i", + nChunk); + } + + /* Show the initial common area */ + a += skip; + b += skip; + m = R[r] - skip; + for(j=0; !rc && j0; j++){ + if( alignment[j]==1 ){ + /* Delete one line from the left */ + rc = sbsWriteLineno(&s, a, SBS_LNA); + if(rc) goto end_align; + s.iStart = 0; + s.zStart = ""; + s.iEnd = LENGTH(&A[a]); + rc = sbsWriteText(&s, &A[a], SBS_TXTA); + if(!rc) rc = sbsWriteMarker(&s, " <", "<"); + if(!rc) rc = sbsWriteNewlines(&s); + if(rc) goto end_align; + assert( ma>0 ); + ma--; + a++; + }else if( alignment[j]==3 ){ + /* The left line is changed into the right line */ + rc = sbsWriteLineChange(&s, &A[a], a, &B[b], b); + if(rc) goto end_align; + assert( ma>0 && mb>0 ); + ma--; + mb--; + a++; + b++; + }else if( alignment[j]==2 ){ + /* Insert one line on the right */ + if( !s.escHtml ){ + rc = sbsWriteSpace(&s, s.width + 7, SBS_TXTA); + } + if(!rc) rc = sbsWriteMarker(&s, " > ", ">"); + if(!rc) rc = sbsWriteLineno(&s, b, SBS_LNB); + if(rc) goto end_align; + s.iStart = 0; + s.zStart = ""; + s.iEnd = LENGTH(&B[b]); + rc = sbsWriteText(&s, &B[b], SBS_TXTB); + if(rc) goto end_align; + assert( mb>0 ); + mb--; + b++; + }else{ + /* Delete from the left and insert on the right */ + rc = sbsWriteLineno(&s, a, SBS_LNA); + if(rc) goto end_align; + s.iStart = 0; + s.zStart = ""; + s.iEnd = LENGTH(&A[a]); + rc = sbsWriteText(&s, &A[a], SBS_TXTA); + if(!rc) rc = sbsWriteMarker(&s, " | ", "|"); + if(!rc) rc = sbsWriteLineno(&s, b, SBS_LNB); + if(rc) goto end_align; + s.iStart = 0; + s.zStart = ""; + s.iEnd = LENGTH(&B[b]); + rc = sbsWriteText(&s, &B[b], SBS_TXTB); + if(rc) goto end_align; + ma--; + mb--; + a++; + b++; + } + } + end_align: + fsl_free(alignment); + if(rc) goto end; + if( inContext ) m = nContext; + for(j=0; !rc && jused>0) ){ + rc = diff_out(pOut, "\n", -1); + for(i=SBS_LNA; !rc && i<=SBS_TXTB; i++){ + rc = sbsWriteColumn(pOut, s.apCols[i], i); + } + rc = diff_out(pOut, "
\n", -1); + }else if(unesc.used){ + rc = pOut->out(pOut->oState, unesc.mem, unesc.used); + } + + end: + for( i = 0; i < (int)(sizeof(aCols)/sizeof(aCols[0])); ++i ){ + fsl_buffer_clear(&aCols[i]); + } + fsl_buffer_clear(&unesc); + return rc; +} + +/** @internal + + Performs a text diff on two buffers, either streaming the + output to the 3rd argument or returning the results as an + array of copy/delete/insert triples via the final argument. + + ONE of the 3rd or final arguments must be set and the other + must be NULL + + If the 3rd argument is not NULL: + + - The 4th argument is the opaque state value passed to + the 3rd when emitting output. + + - contextLines specifies the number of lines of context for the + diff. A negative contextLines value uses a default. + + - sbsWidth, if not 0, specifies a side-by-side diff width. A + negative sbsWidth uses a default. A 0 sbsWidth indicates a + unified-style diff output. + + If the final argument is not NULL then the result array of + copy/delete/insert triples is assigned to *outRaw. Ownership is + transfered to the caller, who must eventually pass it to + fsl_free(). + + Returns 0 on success, any number of other codes on error. +*/ +static int fsl_diff_text_impl( + fsl_buffer const *pA, /* FROM file */ + fsl_buffer const *pB, /* TO file */ + fsl_output_f out, void * outState, + /* ReCompiled *pRe, */ /* Only output changes where this Regexp matches */ + short contextLines, + short sbsWidth, + int diffFlags_, /* FSL_DIFF_* flags */ + int ** outRaw +){ + int rc; + fsl_diff_cx c = fsl_diff_cx_empty; + uint64_t diffFlags = fsl_diff_flags_convert(diffFlags_) + | DIFF_CONTEXT_EX /* to shoehorn newer 0-handling semantics into + older (ported-in) code. */; + if(!pA || !pB || (out && outRaw) || (!out && !outRaw)) return FSL_RC_MISUSE; + else if(contextLines<0) contextLines = 5; + else if(contextLines & ~FSL_LINE_LENGTH_MASK){ + contextLines = (int)FSL_LINE_LENGTH_MASK; + } + diffFlags |= (FSL_LINE_LENGTH_MASK & contextLines); + /* Encode SBS width... */ + if(sbsWidth<0 + || ((DIFF_SIDEBYSIDE & diffFlags) && !sbsWidth) ) sbsWidth = 80; + if(sbsWidth) diffFlags |= DIFF_SIDEBYSIDE; + diffFlags |= ((int)(sbsWidth & 0xFF))<<16; + if( diffFlags & DIFF_INVERT ){ + fsl_buffer const *pTemp = pA; + pA = pB; + pB = pTemp; + } + + /* Prepare the input files */ + if( (diffFlags & DIFF_IGNORE_ALLWS)==DIFF_IGNORE_ALLWS ){ + c.cmpLine = fsl_dline_cmp_ignore_ws; + }else{ + c.cmpLine = fsl_dline_cmp; + } + rc = fsl_break_into_dlines(fsl_buffer_cstr(pA), (fsl_int_t)fsl_buffer_size(pA), + (uint32_t*)&c.nFrom, &c.aFrom, diffFlags); + if(rc) goto end; + rc = fsl_break_into_dlines(fsl_buffer_cstr(pB), (fsl_int_t)fsl_buffer_size(pB), + (uint32_t*)&c.nTo, &c.aTo, diffFlags); + if(rc) goto end; + + /* Compute the difference */ + rc = fsl__diff_all(&c); + //fsl__dump_triples(&c, __FILE__, __LINE__); + if(rc) goto end; + if( (diffFlags & DIFF_NOTTOOBIG)!=0 ){ + int i, m, n; + int *a = c.aEdit; + int mx = c.nEdit; + for(i=m=n=0; i10000 ){ + rc = FSL_RC_RANGE; + /* diff_errmsg(pOut, DIFF_TOO_MANY_CHANGES, diffFlags); */ + goto end; + } + } + //fsl__dump_triples(&c, __FILE__, __LINE__); + if( (diffFlags & DIFF_NOOPT)==0 ){ + fsl__diff_optimize(&c); + } + //fsl__dump_triples(&c, __FILE__, __LINE__); + + if( out ){ + /* Compute a context or side-by-side diff */ + /* MISSING: regex support */ + DiffOutState dos = DiffOutState_empty; + dos.out = out; + dos.oState = outState; + dos.ansiColor = !!(diffFlags_ & FSL_DIFF_ANSI_COLOR); + if( diffFlags & DIFF_SIDEBYSIDE ){ + rc = sbsDiff(&c, &dos, NULL/*pRe*/, diffFlags); + }else{ + rc = contextDiff(&c, &dos, NULL/*pRe*/, diffFlags); + } + }else if(outRaw){ + /* If a context diff is not requested, then return the + array of COPY/DELETE/INSERT triples. */ + *outRaw = c.aEdit; + c.aEdit = NULL; + } + end: + fsl_free(c.aFrom); + fsl_free(c.aTo); + fsl_free(c.aEdit); + return rc; +} + +int fsl_diff_text_raw(fsl_buffer const *p1, fsl_buffer const *p2, + int diffFlags, int ** outRaw){ + return fsl_diff_text_impl(p1, p2, NULL, NULL, 0, 0, diffFlags, outRaw); +} + +int fsl_diff_text(fsl_buffer const *pA, fsl_buffer const *pB, + fsl_output_f out, void * outState, + short contextLines, + short sbsWidth, + int diffFlags ){ + return fsl_diff_text_impl(pA, pB, out, outState, + contextLines, sbsWidth, diffFlags, NULL ); +} + + + +int fsl_diff_text_to_buffer(fsl_buffer const *pA, fsl_buffer const *pB, + fsl_buffer * pOut, + short contextLines, + short sbsWidth, + int diffFlags ){ + return (pA && pB && pOut) + ? fsl_diff_text_impl(pA, pB, fsl_output_f_buffer, pOut, + contextLines, sbsWidth, diffFlags, NULL ) + : FSL_RC_MISUSE; +} + +#undef MARKER +#undef LENGTH +#undef DIFF_CONTEXT_MASK +#undef DIFF_WIDTH_MASK +#undef DIFF_IGNORE_EOLWS +#undef DIFF_IGNORE_ALLWS +#undef DIFF_SIDEBYSIDE +#undef DIFF_VERBOSE +#undef DIFF_BRIEF +#undef DIFF_HTML +#undef DIFF_LINENO +#undef DIFF_NOOPT +#undef DIFF_INVERT +#undef DIFF_CONTEXT_EX +#undef DIFF_NOTTOOBIG +#undef DIFF_STRIP_EOLCR +#undef SBS_LNA +#undef SBS_TXTA +#undef SBS_MKR +#undef SBS_LNB +#undef SBS_TXTB +#undef ANN_FILE_VERS +#undef ANN_FILE_ANCEST + +#undef ANSI_COLOR_BLACK +#undef ANSI_COLOR_RED +#undef ANSI_COLOR_GREEN +#undef ANSI_COLOR_YELLOW +#undef ANSI_COLOR_BLUE +#undef ANSI_COLOR_MAGENTA +#undef ANSI_COLOR_CYAN +#undef ANSI_COLOR_WHITE +#undef ANSI_BG_BLACK +#undef ANSI_BG_RED +#undef ANSI_BG_GREEN +#undef ANSI_BG_YELLOW +#undef ANSI_BG_BLUE +#undef ANSI_BG_MAGENTA +#undef ANSI_BG_CYAN +#undef ANSI_BG_WHITE +#undef ANSI_RESET_COLOR +#undef ANSI_RESET_ALL +#undef ANSI_RESET +#undef ANSI_DIFF_ADD +#undef ANSI_DIFF_MOD +#undef ANSI_DIFF_RM +/* end of file diff.c */ +/* start of file diff2.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/** + This file houses the "2nd generation" diff-generation APIs. This + code is a straight port of those algorithms from the Fossil SCM + project, initially implemented by D. Richard Hipp, ported and + the license re-assigned to this project with this consent. +*/ +#include +#include +#include +#include /* for memmove()/strlen() */ + +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + + +const fsl_diff_opt fsl_diff_opt_empty = fsl_diff_opt_empty_m; +const fsl_diff_builder fsl_diff_builder_empty = fsl_diff_builder_empty_m; +const fsl_dline fsl_dline_empty = fsl_dline_empty_m; +const fsl_dline_change fsl_dline_change_empty = fsl_dline_change_empty_m; +const fsl_diff_cx fsl_diff_cx_empty = fsl_diff_cx_empty_m; + +void fsl__diff_cx_clean(fsl_diff_cx * const cx){ + fsl_free(cx->aFrom); + fsl_free(cx->aTo); + fsl_free(cx->aEdit); + cx->aFrom = cx->aTo = NULL; + cx->aEdit = NULL; + *cx = fsl_diff_cx_empty; +} + +/** + Counts the number of lines in the first n bytes of the given + string. If n<0 then fsl_strlen() is used to count it. + + It includes the last line in the count even if it lacks the \n + terminator. If an empty string is passed in, the number of lines + is zero. + + For the purposes of this function, a string is considered empty if + it contains no characters OR contains only NUL characters. + + If the input appears to be plain text it returns true and, if nOut + is not NULL, writes the number of lines there. If the input appears + to be binary, returns false and does not modify nOut. +*/ +FSL_EXPORT bool fsl_count_lines(const char *z, fsl_int_t n, uint32_t * nOut ); + +bool fsl_count_lines(const char *z, fsl_int_t n, uint32_t * nOut ){ + uint32_t nLine; + const char *zNL, *z2; + if(n<0) n = (fsl_int_t)fsl_strlen(z); + for(nLine=0, z2=z; (zNL = strchr(z2,'\n'))!=0; z2=zNL+1, nLine++){} + if( z2[0]!='\0' ){ + nLine++; + do{ z2++; }while( z2[0]!='\0' ); + } + if( n!=(fsl_int_t)(z2-z) ) return false; + if( nOut ) *nOut = nLine; + return true; +} + +int fsl_break_into_dlines(const char *z, fsl_int_t n, + uint32_t *pnLine, + fsl_dline **pOut, uint64_t diffFlags){ + uint32_t nLine, i, k, nn, s, x; + uint64_t h, h2; + fsl_dline *a = 0; + const char *zNL; + + if(!z || !n){ + *pnLine = 0; + *pOut = NULL; + return 0; + } + if( !fsl_count_lines(z, n, &nLine) ){ + return FSL_RC_DIFF_BINARY; + } + assert( nLine>0 || z[0]=='\0' ); + if(nLine>0){ + a = fsl_malloc( sizeof(a[0])*nLine ); + if(!a) return FSL_RC_OOM; + memset(a, 0, sizeof(a[0])*nLine); + }else{ + *pnLine = 0; + *pOut = a; + return 0; + } + assert( a ); + i = 0; + do{ + zNL = strchr(z,'\n'); + if( zNL==0 ) zNL = z+n; + nn = (uint32_t)(zNL - z); + if( nn>FSL_LINE_LENGTH_MASK ){ + fsl_free(a); + *pOut = 0; + *pnLine = 0; + return FSL_RC_DIFF_BINARY; + } + a[i].z = z; + k = nn; + if( diffFlags & FSL_DIFF2_STRIP_EOLCR ){ + if( k>0 && z[k-1]=='\r' ){ k--; } + } + a[i].n = k; + s = 0; + if( diffFlags & FSL_DIFF2_IGNORE_EOLWS ){ + while( k>0 && fsl_isspace(z[k-1]) ){ k--; } + } + if( (diffFlags & FSL_DIFF2_IGNORE_ALLWS) + ==FSL_DIFF2_IGNORE_ALLWS ){ + uint32_t numws = 0; + while( sh!=pB->h ) return 1; + return memcmp(pA->z,pB->z, pA->h&FSL_LINE_LENGTH_MASK); +} + +int fsl_dline_cmp_ignore_ws(const fsl_dline * const pA, + const fsl_dline * const pB){ + unsigned short a = pA->indent, b = pB->indent; + if( pA->h==pB->h ){ + while( an || bn ){ + if( an && bn && pA->z[a++] != pB->z[b++] ) return 1; + while( an && fsl_isspace(pA->z[a])) ++a; + while( bn && fsl_isspace(pB->z[b])) ++b; + } + return pA->n-a != pB->n-b; + } + return 1; +} + +/* +** The two text segments zLeft and zRight are known to be different on +** both ends, but they might have a common segment in the middle. If +** they do not have a common segment, return 0. If they do have a large +** common segment, return non-0 and before doing so set: +** +** aLCS[0] = start of the common segment in zLeft +** aLCS[1] = end of the common segment in zLeft +** aLCS[2] = start of the common segment in zLeft +** aLCS[3] = end of the common segment in zLeft +** +** This computation is for display purposes only and does not have to be +** optimal or exact. +*/ +static int textLCS2( + const char *zLeft, uint32_t nA, /* String on the left */ + const char *zRight, uint32_t nB, /* String on the right */ + uint32_t *aLCS /* Identify bounds of LCS here */ +){ + const unsigned char *zA = (const unsigned char*)zLeft; /* left string */ + const unsigned char *zB = (const unsigned char*)zRight; /* right string */ + uint32_t i, j, k; /* Loop counters */ + uint32_t lenBest = 0; /* Match length to beat */ + + for(i=0; ilenBest ){ + lenBest = k; + aLCS[0] = i; + aLCS[1] = i+k; + aLCS[2] = j; + aLCS[3] = j+k; + } + } + } + } + return lenBest>0; +} + +/* +** Find the smallest spans that are different between two text strings +** that are known to be different on both ends. Returns the number +** of entries in p->a which get populated. +*/ +static unsigned short textLineChanges( + const char *zLeft, uint32_t nA, /* String on the left */ + const char *zRight, uint32_t nB, /* String on the right */ + fsl_dline_change * const p /* Write results here */ +){ + p->n = 1; + p->a[0].iStart1 = 0; + p->a[0].iLen1 = nA; + p->a[0].iStart2 = 0; + p->a[0].iLen2 = nB; + p->a[0].isMin = 0; + while( p->nn; i++){ + if( p->a[i].isMin ) continue; + x = p->a[i].iLen1; + if( p->a[i].iLen2a[i].iLen2; + if( x>mxLen ){ + mxLen = x; + mxi = i; + } + } + if( mxLen<6 ) break; + x = textLCS2(zLeft + p->a[mxi].iStart1, p->a[mxi].iLen1, + zRight + p->a[mxi].iStart2, p->a[mxi].iLen2, aLCS); + if( x==0 ){ + p->a[mxi].isMin = 1; + continue; + } + a = p->a+mxi; + b = a+1; + if( mxin-1 ){ + memmove(b+1, b, sizeof(*b)*(p->n-mxi-1)); + } + p->n++; + b->iStart1 = a->iStart1 + aLCS[1]; + b->iLen1 = a->iLen1 - aLCS[1]; + a->iLen1 = aLCS[0]; + b->iStart2 = a->iStart2 + aLCS[3]; + b->iLen2 = a->iLen2 - aLCS[3]; + a->iLen2 = aLCS[2]; + b->isMin = 0; + } + return p->n; +} + +/* +** Return true if the string starts with n spaces +*/ +static int allSpaces(const char *z, int n){ + int i; + for(i=0; in<1 ) return; + + /* (1) Attempt to move indentation changes to the left margin */ + if( p->a[0].iLen1==0 + && (len = p->a[0].iLen2)>0 + && (j = p->a[0].iStart2)>0 + && zB[0]==zB[j] + && allSpaces(zB, j) + ){ + for(n=1; na[1], &p->a[0], sizeof(p->a[0])*p->n); + p->n++; + p->a[0] = p->a[1]; + p->a[1].iStart2 += n; + p->a[1].iLen2 -= n; + p->a[0].iLen2 = n; + } + p->a[0].iStart1 = 0; + p->a[0].iStart2 = 0; + }else + if( p->a[0].iLen2==0 + && (len = p->a[0].iLen1)>0 + && (j = p->a[0].iStart1)>0 + && zA[0]==zA[j] + && allSpaces(zA, j) + ){ + for(n=1; na[1], &p->a[0], sizeof(p->a[0])*p->n); + p->n++; + p->a[0] = p->a[1]; + p->a[1].iStart1 += n; + p->a[1].iLen1 -= n; + p->a[0].iLen1 = n; + } + p->a[0].iStart1 = 0; + p->a[0].iStart2 = 0; + } + + /* (2) Try to shift changes so that they begin or end with a + ** space. (TBD) */ +} + +void fsl_dline_change_spans(const fsl_dline *pLeft, const fsl_dline *pRight, + fsl_dline_change * const p){ + /* fossil(1) counterpart ==> diff.c oneLineChange() */ + int nLeft; /* Length of left line in bytes */ + int nRight; /* Length of right line in bytes */ + int nShort; /* Shortest of left and right */ + int nPrefix; /* Length of common prefix */ + int nSuffix; /* Length of common suffix */ + int nCommon; /* Total byte length of suffix and prefix */ + const char *zLeft; /* Text of the left line */ + const char *zRight; /* Text of the right line */ + int nLeftDiff; /* nLeft - nPrefix - nSuffix */ + int nRightDiff; /* nRight - nPrefix - nSuffix */ + + nLeft = pLeft->n; + zLeft = pLeft->z; + nRight = pRight->n; + zRight = pRight->z; + nShort = nLeft0 && (zLeft[nPrefix]&0xc0)==0x80 ) nPrefix--; + } + nSuffix = 0; + if( nPrefix0 && (zLeft[nLeft-nSuffix]&0xc0)==0x80 ) nSuffix--; + } + if( nSuffix==nLeft || nSuffix==nRight ) nPrefix = 0; + } + nCommon = nPrefix + nSuffix; + + /* If the prefix and suffix overlap, that means that we are dealing with + ** a pure insertion or deletion of text that can have multiple alignments. + ** Try to find an alignment to begins and ends on whitespace, or on + ** punctuation, rather than in the middle of a name or number. + */ + if( nCommon > nShort ){ + int iBest = -1; + int iBestVal = -1; + int i; + int nLong = nLeftiBestVal ){ + iBestVal = iVal; + iBest = i; + } + } + nPrefix = iBest; + nSuffix = nShort - nPrefix; + nCommon = nPrefix + nSuffix; + } + + /* A single chunk of text inserted */ + if( nCommon==nLeft ){ + p->n = 1; + p->a[0].iStart1 = nPrefix; + p->a[0].iLen1 = 0; + p->a[0].iStart2 = nPrefix; + p->a[0].iLen2 = nRight - nCommon; + improveReadability(zLeft, zRight, p); + return; + } + + /* A single chunk of text deleted */ + if( nCommon==nRight ){ + p->n = 1; + p->a[0].iStart1 = nPrefix; + p->a[0].iLen1 = nLeft - nCommon; + p->a[0].iStart2 = nPrefix; + p->a[0].iLen2 = 0; + improveReadability(zLeft, zRight, p); + return; + } + + /* At this point we know that there is a chunk of text that has + ** changed between the left and the right. Check to see if there + ** is a large unchanged section in the middle of that changed block. + */ + nLeftDiff = nLeft - nCommon; + nRightDiff = nRight - nCommon; + if( nLeftDiff >= 4 + && nRightDiff >= 4 + && textLineChanges(&zLeft[nPrefix], nLeftDiff, + &zRight[nPrefix], nRightDiff, p)>1 + ){ + int i; + for(i=0; in; i++){ + p->a[i].iStart1 += nPrefix; + p->a[i].iStart2 += nPrefix; + } + improveReadability(zLeft, zRight, p); + return; + } + + /* If all else fails, show a single big change between left and right */ + p->n = 1; + p->a[0].iStart1 = nPrefix; + p->a[0].iLen1 = nLeft - nCommon; + p->a[0].iStart2 = nPrefix; + p->a[0].iLen2 = nRight - nCommon; + improveReadability(zLeft, zRight, p); +} + + + +/* +** The threshold at which diffBlockAlignment transitions from the +** O(N*N) Wagner minimum-edit-distance algorithm to a less process +** O(NlogN) divide-and-conquer approach. +*/ +#define DIFF_ALIGN_MX 1225 + +/* +** R[] is an array of six integer, two COPY/DELETE/INSERT triples for a +** pair of adjacent differences. Return true if the gap between these +** two differences is so small that they should be rendered as a single +** edit. +*/ +static int smallGap2(const int *R, int ma, int mb){ + int m = R[3]; + ma += R[4] + m; + mb += R[5] + m; + if( ma*mb>DIFF_ALIGN_MX ) return 0; + return m<=2 || m<=(R[1]+R[2]+R[4]+R[5])/8; +} + +static unsigned short diff_opt_context_lines(fsl_diff_opt const * opt){ + const unsigned short dflt = 5; + unsigned short n = opt ? opt->contextLines : dflt; + if( !n && (opt->diffFlags & FSL_DIFF2_CONTEXT_ZERO)==0 ){ + n = dflt; + } + return n; +} + +/* +** Minimum of two values +*/ +static int diffMin(int a, int b){ return az; + zB = pB->z; + nA = pA->n; + nB = pB->n; + while( nA>0 && (unsigned char)zA[0]<=' ' ){ nA--; zA++; } + while( nA>0 && (unsigned char)zA[nA-1]<=' ' ){ nA--; } + while( nB>0 && (unsigned char)zB[0]<=' ' ){ nB--; zB++; } + while( nB>0 && (unsigned char)zB[nB-1]<=' ' ){ nB--; } + if( nA>250 ) nA = 250; + if( nB>250 ) nB = 250; + avg = (nA+nB)/2; + if( avg==0 ) return 0; + nMin = nA; + if( nB5 && nPrefix>nMin/2 ){ + best = nPrefix*3/2; + if( best>=avg - 2 ) best = avg - 2; + } + if( nA==nB && memcmp(zA, zB, nA)==0 ) return 0; + memset(aFirst, 0xff, sizeof(aFirst)); + zA--; zB--; /* Make both zA[] and zB[] 1-indexed */ + for(i=nB; i>0; i--){ + c = (unsigned char)zB[i]; + aNext[i] = aFirst[c]; + aFirst[c] = i; + } + for(i=1; i<=nA-best; i++){ + c = (unsigned char)zA[i]; + for(j=aFirst[c]; jbest ) best = k; + } + } + score = (best>=avg) ? 0 : (avg - best)*100/avg; + +#if 0 + fprintf(stderr, "A: [%.*s]\nB: [%.*s]\nbest=%d avg=%d score=%d\n", + nA, zA+1, nB, zB+1, best, avg, score); +#endif + + /* Return the result */ + return score; +} + +/* +** There is a change block in which nLeft lines of text on the left are +** converted into nRight lines of text on the right. This routine computes +** how the lines on the left line up with the lines on the right. +** +** The return value is a buffer of unsigned characters, obtained from +** fossil_malloc(). (The caller needs to free the return value using +** fossil_free().) Entries in the returned array have values as follows: +** +** 1. Delete the next line of pLeft. +** 2. Insert the next line of pRight. +** 3. The next line of pLeft changes into the next line of pRight. +** 4. Delete one line from pLeft and add one line to pRight. +** +** The length of the returned array will be at most nLeft+nRight bytes. +** If the first bytes is 4, that means we could not compute reasonable +** alignment between the two blocks. +** +** Algorithm: Wagner's minimum edit-distance algorithm, modified by +** adding a cost to each match based on how well the two rows match +** each other. Insertion and deletion costs are 50. Match costs +** are between 0 and 100 where 0 is a perfect match 100 is a complete +** mismatch. +*/ +static int diffBlockAlignment( + const fsl_dline *aLeft, int nLeft, /* Text on the left */ + const fsl_dline *aRight, int nRight, /* Text on the right */ + fsl_diff_opt *pOpt, /* Configuration options */ + unsigned char **pResult, /* Raw result */ + unsigned *pNResult /* OUTPUT: length of result */ +){ + int i, j, k; /* Loop counters */ + int *a = 0; /* One row of the Wagner matrix */ + int *pToFree = 0; /* Space that needs to be freed */ + unsigned char *aM = 0; /* Wagner result matrix */ + int nMatch, iMatch; /* Number of matching lines and match score */ + int aBuf[100]; /* Stack space for a[] if nRight not to big */ + int rc = 0; + + if( nLeft==0 ){ + aM = fsl_malloc( nRight + 2 ); + if(!aM) return FSL_RC_OOM; + memset(aM, 2, nRight); + *pNResult = nRight; + *pResult = aM; + return 0; + } + if( nRight==0 ){ + aM = fsl_malloc( nLeft + 2 ); + if(!aM) return FSL_RC_OOM; + memset(aM, 1, nLeft); + *pNResult = nLeft; + *pResult = aM; + return 0; + } + + /* For large alignments, use a divide and conquer algorithm that is + ** O(NlogN). The result is not as precise, but this whole thing is an + ** approximation anyhow, and the faster response time is an acceptable + ** trade-off for reduced precision. + */ + if( nLeft*nRight>DIFF_ALIGN_MX + && (pOpt->diffFlags & FSL_DIFF2_SLOW_SBS)==0 ){ + const fsl_dline *aSmall; /* The smaller of aLeft and aRight */ + const fsl_dline *aBig; /* The larger of aLeft and aRight */ + int nSmall, nBig; /* Size of aSmall and aBig. nSmall<=nBig */ + int iDivSmall, iDivBig; /* Divider point for aSmall and aBig */ + int iDivLeft, iDivRight; /* Divider point for aLeft and aRight */ + unsigned char *a1 = 0, *a2 = 0; /* Results of the alignments on two halves */ + unsigned int n1, n2; /* Number of entries in a1 and a2 */ + int score, bestScore; /* Score and best score seen so far */ + if( nLeft>nRight ){ + aSmall = aRight; + nSmall = nRight; + aBig = aLeft; + nBig = nLeft; + }else{ + aSmall = aLeft; + nSmall = nLeft; + aBig = aRight; + nBig = nRight; + } + iDivBig = nBig/2; + iDivSmall = nSmall/2; + bestScore = 10000; + for(i=0; ia[i]+50 ){ + m = a[i]+50; + d = 1; + } + if( m>p ){ + int const score = + match_dline2(&aLeft[j-1], &aRight[i-1]); + if( (score<=90 || (ij-1)) && m>p+score ){ + m = p+score; + d = 3 | score*4; + } + } + p = a[i]; + a[i] = m; + aM[j*(nRight+1)+i] = d; + } + } + + /* Compute the lowest-cost path back through the matrix */ + i = nRight; + j = nLeft; + k = (nRight+1)*(nLeft+1)-1; + nMatch = iMatch = 0; + while( i+j>0 ){ + unsigned char c = aM[k]; + if( c>=3 ){ + assert( i>0 && j>0 ); + i--; + j--; + nMatch++; + iMatch += (c>>2); + aM[k] = 3; + }else if( c==2 ){ + assert( i>0 ); + i--; + }else{ + assert( j>0 ); + j--; + } + k--; + aM[k] = aM[j*(nRight+1)+i]; + } + k++; + i = (nRight+1)*(nLeft+1) - k; + memmove(aM, &aM[k], i); + *pNResult = i; + *pResult = aM; + + end: + fsl_free(pToFree); + return rc; +} + + +/* +** Format a diff using a fsl_diff_builder object +*/ +static int fdb__format( + fsl_diff_cx * const cx, + fsl_diff_builder * const pBuilder +){ + const fsl_dline *A; /* Left side of the diff */ + const fsl_dline *B; /* Right side of the diff */ + fsl_diff_opt * const pOpt = pBuilder->opt; + const int *R; /* Array of COPY/DELETE/INSERT triples */ + unsigned int a; /* Index of next line in A[] */ + unsigned int b; /* Index of next line in B[] */ + unsigned int r; /* Index into R[] */ + unsigned int nr; /* Number of COPY/DELETE/INSERT triples to process */ + unsigned int mxr; /* Maximum value for r */ + unsigned int na, nb; /* Number of lines shown from A and B */ + unsigned int i, j; /* Loop counters */ + unsigned int m, ma, mb;/* Number of lines to output */ + signed int skip; /* Number of lines to skip */ + unsigned int contextLines; /* Lines of context above and below each change */ + unsigned short passNumber = 0; + int rc = 0; + +#define RC if(rc) goto end + pass_again: + contextLines = diff_opt_context_lines(pOpt); + skip = 0; + a = b = 0; + A = cx->aFrom; + B = cx->aTo; + R = cx->aEdit; + mxr = cx->nEdit; + //MARKER(("contextLines=%u, nEdit = %d, mxr=%u\n", contextLines, cx->nEdit, mxr)); + while( mxr>2 && R[mxr-1]==0 && R[mxr-2]==0 ){ mxr -= 3; } + + pBuilder->lnLHS = pBuilder->lnRHS = 0; + if(pBuilder->start){ + pBuilder->passNumber = ++passNumber; + rc = pBuilder->start(pBuilder); + RC; + } + + for(r=0; r0 && R[r+nr*3]<(int)contextLines*2; nr++){} + +#if 0 + /* MISSING: this "should" be replaced by a stateful predicate + function, probably in the fsl_diff_opt class. */ + /* If there is a regex, skip this block (generate no diff output) + ** if the regex matches or does not match both insert and delete. + ** Only display the block if one side matches but the other side does + ** not. + */ + if( pOpt->pRe ){ + int hideBlock = 1; + int xa = a, xb = b; + for(i=0; hideBlock && ipRe, &A[xa], R[r+i*3+1]); + c2 = re_dline_match(pOpt->pRe, &B[xb], R[r+i*3+2]); + hideBlock = c1==c2; + xa += R[r+i*3+1]; + xb += R[r+i*3+2]; + } + if( hideBlock ){ + a = xa; + b = xb; + continue; + } + } +#endif + + /* Figure out how many lines of A and B are to be displayed + ** for this change block. + */ + if( R[r]>(int)contextLines ){ + na = nb = contextLines; + skip = R[r] - contextLines; + }else{ + na = nb = R[r]; + skip = 0; + } + for(i=0; i(int)contextLines ){ + na += contextLines; + nb += contextLines; + }else{ + na += R[r+nr*3]; + nb += R[r+nr*3]; + } + for(i=1; ichunkHeader +#if 0 + /* The following two bits are a kludge to keep from injecting + a chunk header between chunks which are directly adjacent */ + && pBuilder->lnLHS != (uint32_t)(na ? a+skip+1 : a+skip)-1 + && pBuilder->lnRHS != (uint32_t)(nb ? b+skip+1 : b+skip)-1 +#endif + ){ + rc = pBuilder->chunkHeader(pBuilder, + (uint32_t)(na ? a+skip+1 : a+skip), + (uint32_t)na, + (uint32_t)(nb ? b+skip+1 : b+skip), + (uint32_t)nb); + RC; + } + + /* Show the initial common area */ + a += skip; + b += skip; + m = R[r] - skip; + if( r ) skip -= contextLines; + + //MARKER(("Show the initial common... a=%u, b=%u, m=%u, r=%u, skip=%d\n", a, b, m, r, skip)); + if( skip>0 ){ + if( NULL==pBuilder->chunkHeader && skip<(int)contextLines ){ + /* 2021-09-27: BUG: this is incompatible with unified diff + format. The generated header lines say we're skipping X + lines but we then end up including lines which that header + says to skip. As a workaround, we'll only run this when + pBuilder->chunkHeader is NULL, noting that fossil's + diff builder interface does not have that method. + + Without this block, our "utxt" diff builder can mimic + fossil's non-diff builder unified diff format, except that + we add Index lines (feature or bug?). With this block, + the header values output above are wrong. + */ + /* If the amount to skip is less that the context band, then + ** go ahead and show the skip band as it is not worth eliding */ + //MARKER(("skip %d < contextLines %d\n", skip, contextLines)); + /* from fossil(1) from formatDiff() */ + for(j=0; 0==rc && j<(unsigned)skip; j++){ + //MARKER(("(A) COMMON\n")); + rc = pBuilder->common(pBuilder, &A[a+j-skip]); + } + }else{ + rc = pBuilder->skip(pBuilder, skip); + } + RC; + } + for(j=0; 0==rc && jcommon(pBuilder, &A[a+j]); + } + RC; + a += m; + b += m; + //MARKER(("Show the differences... a=%d, b=%d, m=%d\n", a, b, m)); + + /* Show the differences */ + for(i=0; i0; j++){ + assert( jdeletion(pBuilder, &A[a]); + if(rc) goto bail; + ma--; + a++; + break; + } + case 2: { + /* Insert one line on the right */ + rc = pBuilder->insertion(pBuilder, &B[b]); + if(rc) goto bail; + assert( mb>0 ); + mb--; + b++; + break; + } + case 3: { + /* The left line is changed into the right line */ + if( cx->cmpLine(&A[a], &B[b])==0 ){ + rc = pBuilder->common(pBuilder, &A[a]); + }else{ + rc = pBuilder->edit(pBuilder, &A[a], &B[b]); + } + if(rc) goto bail; + assert( ma>0 && mb>0 ); + ma--; + mb--; + a++; + b++; + break; + } + case 4: { + /* Delete from left then separately insert on the right */ + rc = pBuilder->replacement(pBuilder, &A[a], &B[b]); + if(rc) goto bail; + ma--; + a++; + mb--; + b++; + break; + } + } + } + assert( nAlign==j ); + fsl_free(alignment); + if( icommon(pBuilder, &A[a+j]); + } + RC; + b += m; + a += m; + } + continue; + bail: + assert(rc); + fsl_free(alignment); + goto end; + } + + /* Show the final common area */ + assert( nr==i ); + m = R[r+nr*3]; + if( m>contextLines ) m = contextLines; + for(j=0; 0==rc && jcommon(pBuilder, &A[a+j]); + } + RC; + } + if( R[r]>(int)contextLines ){ + rc = pBuilder->skip(pBuilder, R[r] - contextLines); + } + end: +#undef RC + if(0==rc){ + if(pBuilder->finish) pBuilder->finish(pBuilder); + if(pBuilder->twoPass && 1==passNumber){ + goto pass_again; + } + } + return rc; +} + +/* MISSING(?) fossil(1) converts the diff inputs into utf8 with no + BOM. Whether we really want to do that here or rely on the caller + to is up for debate. If we do it here, we have to make the inputs + non-const, which seems "wrong" for a library API. */ +#define blob_to_utf8_no_bom(A,B) (void)0 + +/** + Performs a diff of version 1 (pA) and version 2 (pB). ONE of + pBuilder or outRaw must be non-NULL. If pBuilder is not NULL, all + output for the diff is emitted via pBuilder. If outRaw is not NULL + then on success *outRaw is set to the array of diff triples, + transfering ownership to the caller, who must eventually fsl_free() + it. On error, *outRaw is not modified but pBuilder may have emitted + partial output. That is not knowable for the general + case. Ownership of pBuilder is not changed. If pBuilder is not NULL + then pBuilder->opt must be non-NULL. +*/ +static int fsl_diff2_text_impl(fsl_buffer const *pA, + fsl_buffer const *pB, + fsl_diff_builder * const pBuilder, + fsl_diff_opt const * const opt_, + int ** outRaw){ + int rc = 0; + fsl_diff_cx c = fsl_diff_cx_empty; + bool ignoreWs = false; + int ansiOptCount = 0; + fsl_diff_opt opt = *opt_ + /*we need a copy for the sake of the FSL_DIFF2_INVERT flag*/; + if(!pA || !pB || (pBuilder && outRaw)) return FSL_RC_MISUSE; + + blob_to_utf8_no_bom(pA, 0); + blob_to_utf8_no_bom(pB, 0); + + if( opt.diffFlags & FSL_DIFF2_INVERT ){ + char const * z; + fsl_buffer const *pTemp = pA; + pA = pB; + pB = pTemp; + z = opt.hashRHS; opt.hashRHS = opt.hashLHS; opt.hashLHS = z; + z = opt.nameRHS; opt.nameRHS = opt.nameLHS; opt.nameLHS = z; + } +#define AOPT(OPT) \ + if(opt.ansiColor.OPT) ansiOptCount += (*opt.ansiColor.OPT) ? 1 : 0; \ + else opt.ansiColor.OPT = "" + AOPT(insertion); + AOPT(edit); + AOPT(deletion); +#undef AOPT + if(0==ansiOptCount){ + opt.ansiColor.reset = ""; + }else if(!opt.ansiColor.reset || !*opt.ansiColor.reset){ + opt.ansiColor.reset = "\x1b[0m"; + } + ignoreWs = (opt.diffFlags & FSL_DIFF2_IGNORE_ALLWS)!=0; + if(FSL_DIFF2_IGNORE_ALLWS==(opt.diffFlags & FSL_DIFF2_IGNORE_ALLWS)){ + c.cmpLine = fsl_dline_cmp_ignore_ws; + }else{ + c.cmpLine = fsl_dline_cmp; + } + + rc = fsl_break_into_dlines(fsl_buffer_cstr(pA), (fsl_int_t)pA->used, + (uint32_t*)&c.nFrom, &c.aFrom, opt.diffFlags); + if(rc) goto end; + rc = fsl_break_into_dlines(fsl_buffer_cstr(pB), (fsl_int_t)pB->used, + (uint32_t*)&c.nTo, &c.aTo, opt.diffFlags); + if(rc) goto end; + + /* Compute the difference */ + rc = fsl__diff_all(&c); + if(rc) goto end; + if( ignoreWs && c.nEdit==6 && c.aEdit[1]==0 && c.aEdit[2]==0 ){ + rc = FSL_RC_DIFF_WS_ONLY; + goto end; + } + if( (opt.diffFlags & FSL_DIFF2_NOTTOOBIG)!=0 ){ + int i, m, n; + int const * const a = c.aEdit; + int const mx = c.nEdit; + for(i=m=n=0; i10000 ){ + rc = FSL_RC_RANGE; + /* diff_errmsg(pOut, DIFF_TOO_MANY_CHANGES, diffFlags); */ + goto end; + } + } + //fsl__dump_triples(&c, __FILE__, __LINE__); + if( (opt.diffFlags & FSL_DIFF2_NOOPT)==0 ){ + fsl__diff_optimize(&c); + } + //fsl__dump_triples(&c, __FILE__, __LINE__); + + /** + Reference: + + https://fossil-scm.org/home/file?ci=cae7036bb7f07c1b&name=src/diff.c&ln=2749-2804 + + Noting that: + + - That function's return value is this one's *outRaw + + - DIFF_NUMSTAT flag is not implemented. For that matter, + flags which result in output going anywhere except for + pBuilder->out are not implemented here, e.g. DIFF_RAW. + + That last point makes this impl tiny compared to the original! + */ + if(pBuilder){ + fsl_diff_opt * const oldOpt = pBuilder->opt; + pBuilder->opt = &opt; + rc = fdb__format(&c, pBuilder); + pBuilder->opt = oldOpt; + } + end: + if(0==rc && outRaw){ + *outRaw = c.aEdit; + c.aEdit = 0; + } + fsl__diff_cx_clean(&c); + return rc; +} + +int fsl_diff_v2(fsl_buffer const * pv1, + fsl_buffer const * pv2, + fsl_diff_builder * const pBuilder){ + return fsl_diff2_text_impl(pv1, pv2, pBuilder, pBuilder->opt, NULL); +} + +int fsl_diff_v2_raw(fsl_buffer const * pv1, + fsl_buffer const * pv2, + fsl_diff_opt const * const opt, + int **outRaw ){ + return fsl_diff2_text_impl(pv1, pv2, NULL, opt, outRaw); +} + + +/** + Allocator for fsl_diff_builder instances. If extra is >0 then that + much extra space is allocated as part of the same block and the + pimpl member of the returned object is pointed to that space. +*/ +static fsl_diff_builder * fsl__diff_builder_alloc(fsl_size_t extra){ + fsl_diff_builder * rc = + (fsl_diff_builder*)fsl_malloc(sizeof(fsl_diff_builder) + extra); + if(rc){ + *rc = fsl_diff_builder_empty; + if(extra){ + rc->pimpl = ((unsigned char *)rc)+sizeof(fsl_diff_builder); + } + } + return rc; +} + +static int fdb__out(fsl_diff_builder *const b, + char const *z, fsl_size_t n){ + return b->opt->out(b->opt->outState, z, n); +} +static int fdb__outf(fsl_diff_builder * const b, + char const *fmt, ...){ + int rc = 0; + va_list va; + assert(b->opt->out); + va_start(va,fmt); + rc = fsl_appendfv(b->opt->out, b->opt->outState, fmt, va); + va_end(va); + return rc; +} + + +static int fdb__debug_start(fsl_diff_builder * const b){ + int rc = fdb__outf(b, "DEBUG builder starting up.\n"); + if(0==rc && b->opt->nameLHS){ + rc = fdb__outf(b,"LHS: %s\n", b->opt->nameLHS); + } + if(0==rc && b->opt->nameRHS){ + rc = fdb__outf(b,"RHS: %s\n", b->opt->nameRHS); + } + if(0==rc && b->opt->hashLHS){ + rc = fdb__outf(b,"LHS hash: %s\n", b->opt->hashLHS); + } + if(0==rc && b->opt->hashRHS){ + rc = fdb__outf(b,"RHS hash: %s\n", b->opt->hashRHS); + } + return rc; +} + + +static int fdb__debug_chunkHeader(fsl_diff_builder* const b, + uint32_t lnnoLHS, uint32_t linesLHS, + uint32_t lnnoRHS, uint32_t linesRHS ){ +#if 1 + return fdb__outf(b, "@@ -%" PRIu32 ",%" PRIu32 + " +%" PRIu32 ",%" PRIu32 " @@\n", + lnnoLHS, linesLHS, lnnoRHS, linesRHS); +#else + return 0; +#endif +} + +static int fdb__debug_skip(fsl_diff_builder * const p, uint32_t n){ + const int rc = fdb__outf(p, "SKIP %u (%u..%u left and %u..%u right)\n", + n, p->lnLHS+1, p->lnLHS+n, p->lnRHS+1, p->lnRHS+n); + p->lnLHS += n; + p->lnRHS += n; + return rc; +} +static int fdb__debug_common(fsl_diff_builder * const p, fsl_dline const * pLine){ + ++p->lnLHS; + ++p->lnRHS; + return fdb__outf(p, "COMMON %8u %8u %.*s\n", + p->lnLHS, p->lnRHS, (int)pLine->n, pLine->z); +} +static int fdb__debug_insertion(fsl_diff_builder * const p, fsl_dline const * pLine){ + p->lnRHS++; + return fdb__outf(p, "INSERT %8u %.*s\n", + p->lnRHS, (int)pLine->n, pLine->z); +} +static int fdb__debug_deletion(fsl_diff_builder * const p, fsl_dline const * pLine){ + p->lnLHS++; + return fdb__outf(p, "DELETE %8u %.*s\n", + p->lnLHS, (int)pLine->n, pLine->z); +} +static int fdb__debug_replacement(fsl_diff_builder * const p, + fsl_dline const * lineLhs, + fsl_dline const * lineRhs) { + int rc; + p->lnLHS++; + p->lnRHS++; + rc = fdb__outf(p, "REPLACE %8u %.*s\n", + p->lnLHS, (int)lineLhs->n, lineLhs->z); + if(!rc){ + rc = fdb__outf(p, " %8u %.*s\n", + p->lnRHS, (int)lineRhs->n, lineRhs->z); + } + return rc; +} + +static int fdb__debug_edit(fsl_diff_builder * const b, + fsl_dline const * pX, + fsl_dline const * pY){ + int rc = 0; + int i, j; + int x; + fsl_dline_change chng = fsl_dline_change_empty; +#define RC if(rc) goto end + b->lnLHS++; + b->lnRHS++; + rc = fdb__outf(b, "EDIT %8u %.*s\n", + b->lnLHS, (int)pX->n, pX->z); + RC; + fsl_dline_change_spans(pX, pY, &chng); + for(i=x=0; i x ){ + if( (pX->z[x]&0xc0)!=0x80 ){ + rc = fdb__out(b, " ", 1); + RC; + } + x++; + } + for(j=0; jz[x]&0xc0)!=0x80 ){ + rc = fdb__out(b, &c, 1); + RC; + } + } + } + } + if( x ){ + rc = fdb__out(b, "\n", 1); + RC; + } + rc = fdb__outf(b, " %8u %.*s\n", + b->lnRHS, (int)pY->n, pY->z); + RC; + for(i=x=0; i x ){ + if( (pY->z[x]&0xc0)!=0x80 ){ + rc = fdb__out(b, " ", 1); + RC; + } + x++; + } + for(j=0; jz[x]&0xc0)!=0x80 ){ + rc = fdb__out(b, &c, 1); + RC; + } + } + } + } + if( x ){ + rc = fdb__out(b, "\n", 1); + } + end: +#undef RC + return rc; +} + +static int fdb__debug_finish(fsl_diff_builder * const b){ + return fdb__outf(b, "END with %u lines left and %u lines right\n", + b->lnLHS, b->lnRHS); +} + +static void fdb__generic_finalize(fsl_diff_builder * const b){ + fsl_free(b); +} + +static fsl_diff_builder * fsl__diff_builder_debug(void){ + fsl_diff_builder * rc = fsl__diff_builder_alloc(0); + if(rc){ + rc->chunkHeader = fdb__debug_chunkHeader; + rc->start = fdb__debug_start; + rc->skip = fdb__debug_skip; + rc->common = fdb__debug_common; + rc->insertion = fdb__debug_insertion; + rc->deletion = fdb__debug_deletion; + rc->replacement = fdb__debug_replacement; + rc->edit = fdb__debug_edit; + rc->finish = fdb__debug_finish; + rc->finalize = fdb__generic_finalize; + assert(!rc->pimpl); + assert(0==rc->implFlags); + assert(0==rc->lnLHS); + assert(0==rc->lnRHS); + assert(NULL==rc->opt); + } + return rc; +} + +/******************** json1 diff builder ********************/ +/* Description taken verbatim from fossil(1): */ +/* +** This formatter generates a JSON array that describes the difference. +** +** The Json array consists of integer opcodes with each opcode followed +** by zero or more arguments: +** +** Syntax Mnemonic Description +** ----------- -------- -------------------------- +** 0 END This is the end of the diff +** 1 INTEGER SKIP Skip N lines from both files +** 2 STRING COMMON The line show by STRING is in both files +** 3 STRING INSERT The line STRING is in only the right file +** 4 STRING DELETE The STRING line is in only the left file +** 5 SUBARRAY EDIT One line is different on left and right. +** +** The SUBARRAY is an array of 3*N+1 strings with N>=0. The triples +** represent common-text, left-text, and right-text. The last string +** in SUBARRAY is the common-suffix. Any string can be empty if it does +** not apply. +*/ + +static int fdb__outj(fsl_diff_builder * const b, + char const *zJson, int n){ + return n<0 + ? fdb__outf(b, "%!j", zJson) + : fdb__outf(b, "%!.*j", n, zJson); +} + +static int fdb__json1_start(fsl_diff_builder * const b){ + int rc = fdb__outf(b, "{\"hashLHS\": %!j, \"hashRHS\": %!j, ", + b->opt->hashLHS, b->opt->hashRHS); + if(0==rc && b->opt->nameLHS){ + rc = fdb__outf(b, "\"nameLHS\": %!j, ", b->opt->nameLHS); + } + if(0==rc && b->opt->nameRHS){ + rc = fdb__outf(b, "\"nameRHS\": %!j, ", b->opt->nameRHS); + } + if(0==rc){ + rc = fdb__out(b, "\"diff\":[", 8); + } + return rc; +} + +static int fdb__json1_skip(fsl_diff_builder * const b, uint32_t n){ + return fdb__outf(b, "1,%" PRIu32 ",\n", n); +} +static int fdb__json1_common(fsl_diff_builder * const b, fsl_dline const * pLine){ + int rc = fdb__out(b, "2,",2); + if(!rc) { + rc = fdb__outj(b, pLine->z, (int)pLine->n); + if(!rc) rc = fdb__out(b, ",\n",2); + } + return rc; +} +static int fdb__json1_insertion(fsl_diff_builder * const b, fsl_dline const * pLine){ + int rc = fdb__out(b, "3,",2); + if(!rc){ + rc = fdb__outj(b, pLine->z, (int)pLine->n); + if(!rc) rc = fdb__out(b, ",\n",2); + } + return rc; +} +static int fdb__json1_deletion(fsl_diff_builder * const b, fsl_dline const * pLine){ + int rc = fdb__out(b, "4,",2); + if(!rc){ + rc = fdb__outj(b, pLine->z, (int)pLine->n); + if(!rc) rc = fdb__out(b, ",\n",2); + } + return rc; +} +static int fdb__json1_replacement(fsl_diff_builder * const b, + fsl_dline const * lineLhs, + fsl_dline const * lineRhs) { + int rc = fdb__out(b, "5,[\"\",",6); + if(!rc) rc = fdb__outf(b,"%!.*j", (int)lineLhs->n, lineLhs->z); + if(!rc) rc = fdb__out(b, ",", 1); + if(!rc) rc = fdb__outf(b,"%!.*j", (int)lineRhs->n, lineRhs->z); + if(!rc) rc = fdb__out(b, ",\"\"],\n",6); + return rc; +} + +static int fdb__json1_edit(fsl_diff_builder * const b, + fsl_dline const * pX, + fsl_dline const * pY){ + int rc = 0; + int i,x; + fsl_dline_change chng = fsl_dline_change_empty; + +#define RC if(rc) goto end + rc = fdb__out(b, "5,[", 3); RC; + fsl_dline_change_spans(pX, pY, &chng); + for(i=x=0; i<(int)chng.n; i++){ + rc = fdb__outj(b, pX->z + x, (int)chng.a[i].iStart1 - x); RC; + x = chng.a[i].iStart1; + rc = fdb__out(b, ",", 1); RC; + rc = fdb__outj(b, pX->z + x, (int)chng.a[i].iLen1); RC; + x += chng.a[i].iLen1; + rc = fdb__out(b, ",", 1); RC; + rc = fdb__outj(b, pY->z + chng.a[i].iStart2, + (int)chng.a[i].iLen2); RC; + } + rc = fdb__out(b, ",", 1); RC; + rc = fdb__outj(b, pX->z + x, (int)(pX->n - x)); RC; + rc = fdb__out(b, "],\n",3); RC; + end: + return rc; +#undef RC +} + +static int fdb__json1_finish(fsl_diff_builder * const b){ + return fdb__out(b, "0]}", 3); +} + +static fsl_diff_builder * fsl__diff_builder_json1(void){ + fsl_diff_builder * rc = fsl__diff_builder_alloc(0); + if(rc){ + rc->chunkHeader = NULL; + rc->start = fdb__json1_start; + rc->skip = fdb__json1_skip; + rc->common = fdb__json1_common; + rc->insertion = fdb__json1_insertion; + rc->deletion = fdb__json1_deletion; + rc->replacement = fdb__json1_replacement; + rc->edit = fdb__json1_edit; + rc->finish = fdb__json1_finish; + rc->finalize = fdb__generic_finalize; + assert(!rc->pimpl); + assert(0==rc->implFlags); + assert(0==rc->lnLHS); + assert(0==rc->lnRHS); + assert(NULL==rc->opt); + } + return rc; +} + +static int fdb__utxt_start(fsl_diff_builder * const b){ + int rc = 0; + if(0==(FSL_DIFF2_NOINDEX & b->opt->diffFlags)){ + rc = fdb__outf(b,"Index: %s\n%.66c\n", + b->opt->nameLHS/*RHS?*/, '='); + } + if(0==rc){ + rc = fdb__outf(b, "--- %s\n+++ %s\n", + b->opt->nameLHS, b->opt->nameRHS); + } + return rc; +} + +static int fdb__utxt_chunkHeader(fsl_diff_builder* const b, + uint32_t lnnoLHS, uint32_t linesLHS, + uint32_t lnnoRHS, uint32_t linesRHS ){ + if(FSL_DIFF2_LINE_NUMBERS & b->opt->diffFlags){ + return fdb__outf(b, "%.40c\n", '~'); + }else{ + return fdb__outf(b, "@@ -%" PRIu32 ",%" PRIu32 + " +%" PRIu32 ",%" PRIu32 " @@\n", + lnnoLHS, linesLHS, lnnoRHS, linesRHS); + } +} + + +static int fdb__utxt_skip(fsl_diff_builder * const b, uint32_t n){ + //MARKER(("SKIP\n")); + b->lnLHS += n; + b->lnRHS += n; + return 0; +} + +/** Outputs line numbers to b->opt->out. */ +static int fdb__utxt_lineno(fsl_diff_builder * const b, uint32_t lnL, uint32_t lnR){ + int rc = 0; + if(FSL_DIFF2_LINE_NUMBERS & b->opt->diffFlags){ + rc = lnL + ? fdb__outf(b, "%s%6" PRIu32 "%s ", + (lnR ? "" : b->opt->ansiColor.deletion), + lnL, + (lnR ? "" : b->opt->ansiColor.reset)) + : fdb__out(b, " ", 7); + if(0==rc){ + rc = lnR + ? fdb__outf(b, "%s%6" PRIu32 "%s ", + (lnL ? "" : b->opt->ansiColor.insertion), + lnR, + (lnL ? "" : b->opt->ansiColor.reset)) + : fdb__out(b, " ", 7); + } + } + return rc; +} + +static int fdb__utxt_common(fsl_diff_builder * const b, fsl_dline const * pLine){ + //MARKER(("COMMON\n")); + ++b->lnLHS; + ++b->lnRHS; + const int rc = fdb__utxt_lineno(b, b->lnLHS, b->lnRHS); + return rc ? rc : fdb__outf(b, " %.*s\n", (int)pLine->n, pLine->z); +} +static int fdb__utxt_insertion(fsl_diff_builder * const b, fsl_dline const * pLine){ + //MARKER(("INSERT\n")); + ++b->lnRHS; + const int rc = fdb__utxt_lineno(b, 0, b->lnRHS); + return rc ? rc : fdb__outf(b, "%s+%.*s%s\n", + b->opt->ansiColor.insertion, + (int)pLine->n, pLine->z, + b->opt->ansiColor.reset); +} +static int fdb__utxt_deletion(fsl_diff_builder * const b, fsl_dline const * pLine){ + //MARKER(("DELETE\n")); + ++b->lnLHS; + const int rc = fdb__utxt_lineno(b, b->lnLHS, 0); + return rc ? rc : fdb__outf(b, "%s-%.*s%s\n", + b->opt->ansiColor.deletion, + (int)pLine->n, pLine->z, + b->opt->ansiColor.reset); +} +static int fdb__utxt_replacement(fsl_diff_builder * const b, + fsl_dline const * lineLhs, + fsl_dline const * lineRhs) { + //MARKER(("REPLACE\n")); + int rc = b->deletion(b, lineLhs); + if(0==rc) rc = b->insertion(b, lineRhs); + return rc; +} +static int fdb__utxt_edit(fsl_diff_builder * const b, + fsl_dline const * lineLhs, + fsl_dline const * lineRhs){ + //MARKER(("EDIT\n")); + int rc = b->deletion(b, lineLhs); + if(0==rc) rc = b->insertion(b, lineRhs); + return rc; +} + +static void fdb__utxt_finalize(fsl_diff_builder * const b){ + fsl_free(b); +} + +static fsl_diff_builder * fsl__diff_builder_utxt(void){ + fsl_diff_builder * rc = fsl__diff_builder_alloc(0); + if(!rc) return NULL; + rc->chunkHeader = fdb__utxt_chunkHeader; + rc->start = fdb__utxt_start; + rc->skip = fdb__utxt_skip; + rc->common = fdb__utxt_common; + rc->insertion = fdb__utxt_insertion; + rc->deletion = fdb__utxt_deletion; + rc->replacement = fdb__utxt_replacement; + rc->edit = fdb__utxt_edit; + rc->finish = NULL; + rc->finalize = fdb__utxt_finalize; + return rc; +} + +struct DiBuTcl { + /** Buffer for TCL-format string conversion */ + fsl_buffer str; +}; +typedef struct DiBuTcl DiBuTcl; +static const DiBuTcl DiBuTcl_empty = {fsl_buffer_empty_m}; + +#define BR_OPEN if(FSL_DIFF2_TCL_BRACES & b->opt->diffFlags) \ + rc = fdb__out(b, "{", 1) +#define BR_CLOSE if(FSL_DIFF2_TCL_BRACES & b->opt->diffFlags) \ + rc = fdb__out(b, "}", 1) + +#define DTCL_BUFFER(B) &((DiBuTcl*)(B)->pimpl)->str +static int fdb__outtcl(fsl_diff_builder * const b, + char const *z, unsigned int n, + char chAppend ){ + int rc; + fsl_buffer * const o = DTCL_BUFFER(b); + fsl_buffer_reuse(o); + rc = fsl_buffer_append_tcl_literal(o, z, n); + if(0==rc) rc = fdb__out(b, (char const *)o->mem, o->used); + if(chAppend && 0==rc) rc = fdb__out(b, &chAppend, 1); + return rc; +} + +static int fdb__tcl_start(fsl_diff_builder * const b){ + int rc = 0; + fsl_buffer_reuse(DTCL_BUFFER(b)); + BR_OPEN; + if(0==rc) rc = fdb__out(b, "\n", 1); + if(0==rc && b->opt->nameLHS){ + char const * zRHS = + b->opt->nameRHS ? b->opt->nameRHS : b->opt->nameLHS; + BR_OPEN; + if(0==rc) rc = fdb__out(b, "FILE ", 5); + if(0==rc) rc = fdb__outtcl(b, b->opt->nameLHS, + (unsigned)fsl_strlen(b->opt->nameLHS), ' '); + if(0==rc) rc = fdb__outtcl(b, zRHS, + (unsigned)fsl_strlen(zRHS), 0); + if(0==rc) {BR_CLOSE;} + if(0==rc) rc = fdb__out(b, "\n", 1); + } + return rc; +} + +static int fdb__tcl_skip(fsl_diff_builder * const b, uint32_t n){ + int rc = 0; + BR_OPEN; + if(0==rc) rc = fdb__outf(b, "SKIP %" PRIu32, n); + if(0==rc) {BR_CLOSE;} + if(0==rc) rc = fdb__outf(b, "\n", 1); + return rc; +} + +static int fdb__tcl_common(fsl_diff_builder * const b, fsl_dline const * pLine){ + int rc = 0; + BR_OPEN; + if(0==rc) rc = fdb__out(b, "COM ", 5); + if(0==rc) rc= fdb__outtcl(b, pLine->z, pLine->n, 0); + if(0==rc) {BR_CLOSE;} + if(0==rc) rc = fdb__outf(b, "\n", 1); + return rc; +} +static int fdb__tcl_insertion(fsl_diff_builder * const b, fsl_dline const * pLine){ + int rc = 0; + BR_OPEN; + if(0==rc) rc = fdb__out(b, "INS ", 5); + if(0==rc) rc = fdb__outtcl(b, pLine->z, pLine->n, 0); + if(0==rc) {BR_CLOSE;} + if(0==rc) rc = fdb__outf(b, "\n", 1); + return rc; +} +static int fdb__tcl_deletion(fsl_diff_builder * const b, fsl_dline const * pLine){ + int rc = 0; + BR_OPEN; + if(0==rc) rc = fdb__out(b, "DEL ", 5); + if(0==rc) rc = fdb__outtcl(b, pLine->z, pLine->n, 0); + if(0==rc) {BR_CLOSE;} + if(0==rc) rc = fdb__outf(b, "\n", 1); + return rc; +} +static int fdb__tcl_replacement(fsl_diff_builder * const b, + fsl_dline const * lineLhs, + fsl_dline const * lineRhs) { + int rc = 0; + BR_OPEN; + if(0==rc) rc = fdb__out(b, "EDIT \"\" ", 8); + if(0==rc) rc = fdb__outtcl(b, lineLhs->z, lineLhs->n, ' '); + if(0==rc) rc = fdb__outtcl(b, lineRhs->z, lineRhs->n, 0); + if(0==rc) {BR_CLOSE;} + if(0==rc) rc = fdb__outf(b, "\n", 1); + return rc; +} + +static int fdb__tcl_edit(fsl_diff_builder * const b, + fsl_dline const * pX, + fsl_dline const * pY){ + int rc = 0; + int i, x; + fsl_dline_change chng = fsl_dline_change_empty; +#define RC if(rc) goto end + BR_OPEN; + rc = fdb__out(b, "EDIT", 4); RC; + fsl_dline_change_spans(pX, pY, &chng); + for(i=x=0; iz + x, chng.a[i].iStart1 - x, ' '); RC; + x = chng.a[i].iStart1; + rc = fdb__outtcl(b, pX->z + x, chng.a[i].iLen1, ' '); RC; + x += chng.a[i].iLen1; + rc = fdb__outtcl(b, pY->z + chng.a[i].iStart2, + chng.a[i].iLen2, 0); RC; + } + assert(0==rc); + if( x < pX->n ){ + rc = fdb__out(b, " ", 1); RC; + rc = fdb__outtcl(b, pX->z + x, pX->n - x, 0); RC; + } + BR_CLOSE; RC; + rc = fdb__out(b, "\n", 1); + end: +#undef RC + return rc; +} + +static int fdb__tcl_finish(fsl_diff_builder * const b){ + int rc = 0; + BR_CLOSE; + if(0==rc && FSL_DIFF2_TCL_BRACES & b->opt->diffFlags){ + rc = fdb__out(b, "\n", 1); + } + return rc; +} +#undef BR_OPEN +#undef BR_CLOSE + +static void fdb__tcl_finalize(fsl_diff_builder * const b){ + fsl_buffer_clear( &((DiBuTcl*)b->pimpl)->str ); + *b = fsl_diff_builder_empty; + fsl_free(b); +} + +static fsl_diff_builder * fsl__diff_builder_tcl(void){ + fsl_diff_builder * rc = + fsl__diff_builder_alloc((fsl_size_t)sizeof(DiBuTcl)); + if(rc){ + rc->chunkHeader = NULL; + rc->start = fdb__tcl_start; + rc->skip = fdb__tcl_skip; + rc->common = fdb__tcl_common; + rc->insertion = fdb__tcl_insertion; + rc->deletion = fdb__tcl_deletion; + rc->replacement = fdb__tcl_replacement; + rc->edit = fdb__tcl_edit; + rc->finish = fdb__tcl_finish; + rc->finalize = fdb__tcl_finalize; + assert(0!=rc->pimpl); + DiBuTcl * const dbt = (DiBuTcl*)rc->pimpl; + *dbt = DiBuTcl_empty; + if(fsl_buffer_reserve(&dbt->str, 120)){ + rc->finalize(rc); + rc = 0; + } + } + return rc; +} + +/** + Column indexes for SplitText::cols. +*/ +enum SplitTextCols { +STC_NUM1 = 0, STC_TEXT1, +STC_MOD, +STC_NUM2, STC_TEXT2, +STC_count +}; +/** + Internal state for the text-mode split diff builder. + + This builder buffers its contents in 5 buffers: 2 each for the + LHS/RHS line numbers and content and one for the "change marker" (a + center column). Each of the STC_NUM1, STC_NUM2, STC_TEXT1, and + STC_TEXT2 columns is stored as one NUL-delimited string in each of + the corresponding column buffers. STC_MOD is a simple byte array, + with each byte corresponding two one line of the diff and marking + what type of linle it is (common, insertion, deletion, or + edit/replacement). + + The STC_SKIP column is managed differently. It is zero-filled, + with a non-0 value at each line of the diff which represents a + skipped gap. Potential TODO: combine this and STC_MOD. +*/ +struct SplitTxt { + /** + Largest column width we've yet seen. These are only updated for + STC_TEXT1 and STC_TEXT2. The others currently have fixed widths. + + FIXME: these are in bytes, not text columns. The current code may + truncate multibyte characters. + */ + uint32_t maxWidths[STC_count]; +}; +typedef struct SplitTxt SplitTxt; +static const SplitTxt SplitTxt_empty = {{1,10,3,1,10}}; +#define SPLITSTATE(VNAME) SplitTxt * const VNAME = (SplitTxt *)b->pimpl + +static int maxColWidth(fsl_diff_builder const * const b, + SplitTxt const * const sst, + int mwIndex){ + static const short minColWidth = + 10/*b->opt.columnWidth values smaller than this are treated as + this value*/; + switch(mwIndex){ + case STC_NUM1: + case STC_NUM2: + case STC_MOD: return sst->maxWidths[mwIndex]; + case STC_TEXT1: case STC_TEXT2: break; + default: + assert(!"internal API misuse: invalid column index."); + break; + } + int const y = + (b->opt->columnWidth>0 + && b->opt->columnWidth<=sst->maxWidths[mwIndex]) + ? (int)b->opt->columnWidth + : (int)sst->maxWidths[mwIndex]; + return minColWidth > y ? minColWidth : y; +} + +static int fdb__splittxt_mod(fsl_diff_builder * const b, char ch){ + assert(2==b->passNumber); + return fdb__outf(b, " %c ", ch); +} + +static int fdb__splittxt_lineno(fsl_diff_builder * const b, + SplitTxt const * const sst, + bool isLeft, uint32_t n){ + assert(2==b->passNumber); + int const col = isLeft ? STC_NUM1 : STC_NUM2; + return n + ? fdb__outf(b, "%*" PRIu32 " ", sst->maxWidths[col], n) + : fdb__outf(b, "%.*c ", sst->maxWidths[col], ' '); +} + +static int fdb__splittxt_start(fsl_diff_builder * const b){ + int rc = 0; + if(1==b->passNumber){ + SPLITSTATE(sst); + *sst = SplitTxt_empty; + ++b->fileCount; + return rc; + } + if(b->fileCount>1){ + rc = fdb__out(b, "\n", 1); + } + if(0==rc){ + fsl_diff_opt const * const o = b->opt; + if(o->nameLHS || o->nameRHS + || o->hashLHS || o->hashRHS){ + rc = fdb__outf(b, "--- %s%s%s\n+++ %s%s%s\n", + o->nameLHS ? o->nameLHS : "", + (o->nameLHS && o->hashLHS) ? " " : "", + o->hashLHS ? o->hashLHS : "", + o->nameRHS ? o->nameRHS : "", + (o->nameRHS && o->hashRHS) ? " " : "", + o->hashRHS ? o->hashRHS : ""); + } + } + return rc; +} + +static int fdb__splittxt_skip(fsl_diff_builder * const b, uint32_t n){ + b->lnLHS += n; + b->lnRHS += n; + if(1==b->passNumber) return 0; + SPLITSTATE(sst); + int const maxWidth1 = maxColWidth(b, sst, STC_TEXT1); + int const maxWidth2 = maxColWidth(b, sst, STC_TEXT2); + return fdb__outf(b, "%.*c %.*c %.*c %.*c\n", + sst->maxWidths[STC_NUM1], '~', + maxWidth1, '~', + sst->maxWidths[STC_NUM2], '~', + maxWidth2, '~'); +} + +static int fdb__splittxt_color(fsl_diff_builder * const b, + int modType){ + char const *z = 0; + switch(modType){ + case (int)'i': z = b->opt->ansiColor.insertion; break; + case (int)'d': z = b->opt->ansiColor.deletion; break; + case (int)'r'/*replacement*/: + case (int)'e': z = b->opt->ansiColor.edit; break; + case 0: z = b->opt->ansiColor.reset; break; + default: + assert(!"invalid color op!"); + } + return z&&*z ? fdb__outf(b, "%s", z) : 0; +} + +static int fdb__splittxt_side(fsl_diff_builder * const b, + SplitTxt * const sst, + bool isLeft, + fsl_dline const * const pLine){ + int rc = fdb__splittxt_lineno(b, sst, isLeft, + pLine ? (isLeft ? b->lnLHS : b->lnRHS) : 0U); + if(0==rc){ + uint32_t const w = maxColWidth(b, sst, isLeft ? STC_TEXT1 : STC_TEXT2); + if(pLine){ + fsl_size_t const nU = + /* Measure column width in UTF8 characters, not bytes! */ + fsl_strlen_utf8(pLine->z, (fsl_int_t)pLine->n); + rc = fdb__outf(b, "%#.*s", (int)(w < nU ? w : nU), pLine->z); + if(0==rc && w>nU){ + rc = fdb__outf(b, "%.*c", (int)(w - nU), ' '); + } + }else{ + rc = fdb__outf(b, "%.*c", (int)w, ' '); + } + if(0==rc && !isLeft) rc = fdb__out(b, "\n", 1); + } + return rc; +} + +static void fdb__splittext_update_maxlen(SplitTxt * const sst, + int col, + char const * const z, + uint32_t n){ + if(sst->maxWidths[col]maxWidths[col] = n; +#else + n = (uint32_t)fsl_strlen_utf8(z, (fsl_int_t)n); + if(sst->maxWidths[col]maxWidths[col] = n; +#endif + } +} + +static int fdb__splittxt_common(fsl_diff_builder * const b, + fsl_dline const * const pLine){ + int rc = 0; + SPLITSTATE(sst); + ++b->lnLHS; + ++b->lnRHS; + if(1==b->passNumber){ + fdb__splittext_update_maxlen(sst, STC_TEXT1, pLine->z, pLine->n); + fdb__splittext_update_maxlen(sst, STC_TEXT2, pLine->z, pLine->n); + return 0; + } + rc = fdb__splittxt_side(b, sst, true, pLine); + if(0==rc) rc = fdb__splittxt_mod(b, ' '); + if(0==rc) rc = fdb__splittxt_side(b, sst, false, pLine); + return rc; +} + +static int fdb__splittxt_insertion(fsl_diff_builder * const b, + fsl_dline const * const pLine){ + int rc = 0; + SPLITSTATE(sst); + ++b->lnRHS; + if(1==b->passNumber){ + fdb__splittext_update_maxlen(sst, STC_TEXT1, pLine->z, pLine->n); + return rc; + } + rc = fdb__splittxt_color(b, 'i'); + if(0==rc) rc = fdb__splittxt_side(b, sst, true, NULL); + if(0==rc) rc = fdb__splittxt_mod(b, '>'); + if(0==rc) rc = fdb__splittxt_side(b, sst, false, pLine); + if(0==rc) rc = fdb__splittxt_color(b, 0); + return rc; +} + +static int fdb__splittxt_deletion(fsl_diff_builder * const b, + fsl_dline const * const pLine){ + int rc = 0; + SPLITSTATE(sst); + ++b->lnLHS; + if(1==b->passNumber){ + fdb__splittext_update_maxlen(sst, STC_TEXT2, pLine->z, pLine->n); + return rc; + } + rc = fdb__splittxt_color(b, 'd'); + if(0==rc) rc = fdb__splittxt_side(b, sst, true, pLine); + if(0==rc) rc = fdb__splittxt_mod(b, '<'); + if(0==rc) rc = fdb__splittxt_side(b, sst, false, NULL); + if(0==rc) rc = fdb__splittxt_color(b, 0); + return rc; +} + +static int fdb__splittxt_replacement(fsl_diff_builder * const b, + fsl_dline const * const lineLhs, + fsl_dline const * const lineRhs) { +#if 0 + int rc = b->deletion(b, lineLhs); + if(0==rc) rc = b->insertion(b, lineRhs); + return rc; +#else + int rc = 0; + SPLITSTATE(sst); + ++b->lnLHS; + ++b->lnRHS; + if(1==b->passNumber){ + fdb__splittext_update_maxlen(sst, STC_TEXT1, lineLhs->z, lineLhs->n); + fdb__splittext_update_maxlen(sst, STC_TEXT2, lineRhs->z, lineRhs->n); + return 0; + } + rc = fdb__splittxt_color(b, 'e'); + if(0==rc) rc = fdb__splittxt_side(b, sst, true, lineLhs); + if(0==rc) rc = fdb__splittxt_mod(b, '|'); + if(0==rc) rc = fdb__splittxt_side(b, sst, false, lineRhs); + if(0==rc) rc = fdb__splittxt_color(b, 0); + return rc; +#endif +} + +static int fdb__splittxt_finish(fsl_diff_builder * const b){ + int rc = 0; + if(1==b->passNumber){ + SPLITSTATE(sst); + uint32_t ln = b->lnLHS; + /* Calculate width of line number columns. */ + sst->maxWidths[STC_NUM1] = sst->maxWidths[STC_NUM2] = 1; + for(; ln>=10; ln/=10) ++sst->maxWidths[STC_NUM1]; + ln = b->lnRHS; + for(; ln>=10; ln/=10) ++sst->maxWidths[STC_NUM2]; + } + return rc; +} + +static void fdb__splittxt_finalize(fsl_diff_builder * const b){ + *b = fsl_diff_builder_empty; + fsl_free(b); +} + +static fsl_diff_builder * fsl__diff_builder_splittxt(void){ + fsl_diff_builder * rc = + fsl__diff_builder_alloc((fsl_size_t)sizeof(SplitTxt)); + if(rc){ + rc->twoPass = true; + rc->chunkHeader = NULL; + rc->start = fdb__splittxt_start; + rc->skip = fdb__splittxt_skip; + rc->common = fdb__splittxt_common; + rc->insertion = fdb__splittxt_insertion; + rc->deletion = fdb__splittxt_deletion; + rc->replacement = fdb__splittxt_replacement; + rc->edit = fdb__splittxt_replacement; + rc->finish = fdb__splittxt_finish; + rc->finalize = fdb__splittxt_finalize; + assert(0!=rc->pimpl); + SplitTxt * const sst = (SplitTxt*)rc->pimpl; + *sst = SplitTxt_empty; + } + return rc; +} + +int fsl_diff_builder_factory( fsl_diff_builder_e type, + fsl_diff_builder **pOut ){ + int rc = FSL_RC_TYPE; + fsl_diff_builder * (*factory)(void) = NULL; + switch(type){ + case FSL_DIFF_BUILDER_DEBUG: + factory = fsl__diff_builder_debug; + break; + case FSL_DIFF_BUILDER_JSON1: + factory = fsl__diff_builder_json1; + break; + case FSL_DIFF_BUILDER_UNIFIED_TEXT: + factory = fsl__diff_builder_utxt; + break; + case FSL_DIFF_BUILDER_TCL: + factory = fsl__diff_builder_tcl; + break; + case FSL_DIFF_BUILDER_SPLIT_TEXT: + factory = fsl__diff_builder_splittxt; + break; + } + if(NULL!=factory){ + *pOut = factory(); + rc = *pOut ? 0 : FSL_RC_OOM; + } + return rc; +} + +#undef DIFF_ALIGN_MX +#undef DIFF_CANNOT_COMPUTE_BINARY +#undef DIFF_CANNOT_COMPUTE_SYMLINK +#undef DIFF_TOO_MANY_CHANGES +#undef DIFF_WHITESPACE_ONLY +#undef fsl_dline_empty_m +#undef MARKER +#undef DTCL_BUFFER +#undef blob_to_utf8_no_bom +#undef SPLITSTATE +/* end of file diff2.c */ +/* start of file encode.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************** + This file houses some encoding/decoding API routines. +*/ +#include + +/* Only for debugging */ +#include + +/* + An array for translating single base-16 characters into a value. + Disallowed input characters have a value of 64. +*/ +static const char zDecode[] = { + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 64, 64, 64, 64, 64, 64, + 64, 10, 11, 12, 13, 14, 15, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 10, 11, 12, 13, 14, 15, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, + 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64 +}; + +int fsl_decode16(const unsigned char *zIn, unsigned char *pOut, + fsl_size_t N){ + fsl_int_t i, j; + if( (N&1)!=0 ) return FSL_RC_RANGE; + for(i=j=0; i<(fsl_int_t)N; i += 2, j++){ + fsl_int_t v1, v2, a; + a = zIn[i]; + if( (a & 0x80)!=0 || (v1 = zDecode[a])==64 ) return FSL_RC_RANGE; + a = zIn[i+1]; + if( (a & 0x80)!=0 || (v2 = zDecode[a])==64 ) return FSL_RC_RANGE; + pOut[j] = (v1<<4) + v2; + } + return 0; +} + + +bool fsl_validate16(const char *zIn, fsl_size_t nIn){ + fsl_size_t i; + for(i=0; i63 ){ + return zIn[0]==0 ? true : false; + } + } + return true; +} + +/* + The array used for encoding +*/ /* 123456789 12345 */ +static const char zEncode[] = "0123456789abcdef"; + +int fsl_encode16(const unsigned char *pIn, unsigned char *zOut, fsl_size_t N){ + fsl_size_t i; + if(!pIn || !zOut) return FSL_RC_MISUSE; + for(i=0; i>4]; + *(zOut++) = zEncode[pIn[i]&0xf]; + } + *zOut = 0; + return 0; +} + +void fsl_canonical16(char *z, fsl_size_t n){ + while( *z && n-- ){ + *z = zEncode[zDecode[(*z)&0x7f]&0x1f]; + ++z; + } +} + +void fsl_bytes_defossilize( unsigned char * z, fsl_size_t * resultLen ){ + fsl_size_t i, j, c; + for(i=0; (c=z[i])!=0 && c!='\\'; i++){} + if( c==0 ) { + if(resultLen) *resultLen = i; + return; + } + for(j=i; (c=z[i])!=0; i++){ + if( c=='\\' && z[i+1] ){ + i++; + switch( z[i] ){ + case 'n': c = '\n'; break; + case 's': c = ' '; break; + case 't': c = '\t'; break; + case 'r': c = '\r'; break; + case 'v': c = '\v'; break; + case 'f': c = '\f'; break; + case '0': c = 0; break; + case '\\': c = '\\'; break; + default: c = z[i]; break; + } + } + z[j++] = c; + } + if( z[j] ) z[j] = 0; + if(resultLen) *resultLen = j; +} + +int fsl_bytes_fossilize( unsigned char const * inp, + fsl_int_t nIn, + fsl_buffer * out ){ + fsl_size_t n, i, j, c; + unsigned char *zOut; + int rc; + fsl_size_t oldUsed; + fsl_size_t inSz; + if(!inp || !out) return FSL_RC_MISUSE; + else if( inp && (nIn<0) ) nIn = (fsl_int_t)fsl_strlen((char const *)inp); + out->used = 0; + if(!nIn) return 0; + inSz = (fsl_size_t)nIn; + /* Figure out how much space we'll need... */ + for(i=n=0; iused; + rc = fsl_buffer_reserve( out, oldUsed + (fsl_size_t)(n+1)); + if(rc) return rc; + zOut = out->mem + oldUsed; + /* Encode it... */ + for(i=j=0; i<(fsl_size_t)nIn; i++){ + unsigned char c = (unsigned char)inp[i]; + if( c==0 ){ + zOut[j++] = '\\'; + zOut[j++] = '0'; + }else if( c=='\\' ){ + zOut[j++] = '\\'; + zOut[j++] = '\\'; + }else if( fsl_isspace(c) ){ + zOut[j++] = '\\'; + switch( c ){ + case '\n': c = 'n'; break; + case ' ': c = 's'; break; + case '\t': c = 't'; break; + case '\r': c = 'r'; break; + case '\v': c = 'v'; break; + case '\f': c = 'f'; break; + } + zOut[j++] = c; + }else{ + zOut[j++] = c; + } + } + zOut[j] = 0; + out->used += j; + return 0; +} + + +fsl_size_t fsl_str_to_size(char const * str){ + fsl_size_t size, oldsize, c; + if(!str) return -1; + for(oldsize=size=0; (c = str[0])>='0' && c<='9'; str++){ + size = oldsize*10 + c - '0'; + if( size='0' && c<='9'; str++){ + size = oldsize*10 + c - '0'; + if( size': *xlate = ">"; return 4; + case '&': *xlate = "&"; return 5; + case '"': *xlate = """; return 6; + default: *xlate = NULL; return 1; + } +} + +int fsl_htmlize(fsl_output_f out, void * oState, + const char *zIn, fsl_int_t n){ + int rc = 0; + int c, i, j, len; + char const * xlate; + if(!out || !zIn) return FSL_RC_MISUSE; + else if( n<0 ) n = fsl_strlen(zIn); + for(i=j=0; !rc && (i1){ + if( jused + count + 1); + if(!rc){ + /* Now none of the fsl_buffer_append()s can fail. */ + rc = fsl_htmlize(fsl_output_f_buffer, p, zIn, n); + } + } + return rc; +} + +char *fsl_htmlize_str(const char *zIn, fsl_int_t n){ + int rc; + fsl_buffer b = fsl_buffer_empty; + rc = fsl_htmlize_to_buffer(&b, zIn, n); + if(!rc){ + return (char *)b.mem /* transfer ownership */; + }else{ + fsl_buffer_clear(&b); + return NULL; + } +} + +/* end of file encode.c */ +/* start of file event.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************ + This file implements technote (formerly known as event)-related + parts of the library. +*/ +#include + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + + +int fsl_event_ids_get( fsl_cx * f, fsl_list * tgt ){ + fsl_db * db = fsl_needs_repo(f); + if(!f || !tgt) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + else { + int rc = fsl_db_select_slist( db, tgt, + "SELECT substr(tagname,7) AS n " + "FROM tag " + "WHERE tagname GLOB 'event-*' " + "ORDER BY n"); + if(rc && db->error.code && !f->error.code){ + fsl_cx_uplift_db_error(f, db); + } + return rc; + } +} + + +#undef MARKER +/* end of file event.c */ +/* start of file fs.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +#ifdef _WIN32 +# undef __STRICT_ANSI__ /* Needed for _wfopen */ +#endif + +#include +#include /* strlen() */ +#include /* NULL on linux */ +#include +#include +#include +#ifdef _WIN32 +# define DIR _WDIR +# define dirent _wdirent +# define opendir _wopendir +# define readdir _wreaddir +# define closedir _wclosedir +# include +# include +# include +# if !defined(ELOOP) +# define ELOOP 114 /* Missing in MinGW */ +# endif +#else +# include /* access(2) */ +# include +# include +#endif +#include + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + + +FILE *fsl_fopen(const char *zName, const char *zMode){ + FILE *f; + if(zName && ('-'==*zName && !zName[1])){ + f = (strchr(zMode, 'w') || strchr(zMode,'+')) + ? stdout + : stdin + ; + }else{ +#ifdef _WIN32 + wchar_t *uMode = (wchar_t *)fsl_utf8_to_unicode(zMode); + wchar_t *uName = (wchar_t *)fsl_utf8_to_filename(zName); + f = _wfopen(uName, uMode); + fsl_filename_free(uName); + fsl_unicode_free(uMode); +#else + f = fopen(zName, zMode); +#endif + } + return f; +} + + +void fsl_fclose( FILE * f ){ + if(f && (stdin!=f) && (stdout!=f) && (stderr!=f)){ + fclose(f); + } +} + +/* + Wrapper around the access() system call. +*/ +int fsl_file_access(const char *zFilename, int flags){ + /* FIXME: port in fossil(1) win32_access() */ +#ifdef _WIN32 + wchar_t *zMbcs = (wchar_t *)fsl_utf8_to_filename(zFilename); +#define ACC _waccess +#else + char *zMbcs = (char*)fsl_utf8_to_filename(zFilename); +#define ACC access +#endif + int rc = zMbcs ? ACC(zMbcs, flags) : FSL_RC_OOM; + if(zMbcs) fsl_filename_free(zMbcs); + return rc; +#undef ACC +} + + +int fsl_getcwd(char *zBuf, fsl_size_t nBuf, fsl_size_t * outLen){ +#ifdef _WIN32 + /* FIXME: port in fossil(1) win32_getcwd() */ + char *zPwdUtf8; + fsl_size_t nPwd; + fsl_size_t i; + wchar_t zPwd[2000]; + if(!zBuf) return FSL_RC_MISUSE; + else if(!nBuf) return FSL_RC_RANGE; + /* + https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx + + It says: + + Note File I/O functions in the Windows API convert "/" to "\" as + part of converting the name to an NT-style name, except when using + the "\\?\" prefix as detailed in the following sections. + + So the path-demangling bits below might do more damage they + fix? + */ + else if( _wgetcwd(zPwd, sizeof(zPwd)/sizeof(zPwd[0])-1)==0 ){ + /* FIXME: how to determine if FSL_RC_RANGE is a better + return value? + */ + return FSL_RC_IO; + } + zPwdUtf8 = fsl_filename_to_utf8(zPwd); + if(!zPwdUtf8) return FSL_RC_OOM; + nPwd = strlen(zPwdUtf8); + if( nPwd > nBuf-1 ){ + fsl_filename_free(zPwdUtf8); + return FSL_RC_RANGE; + } + for(i=0; zPwdUtf8[i]; i++) if( zPwdUtf8[i]=='\\' ) zPwdUtf8[i] = '/'; + memcpy(zBuf, zPwdUtf8, nPwd+1); + fsl_filename_free(zPwdUtf8); + if(outLen) *outLen = nPwd; + return 0; +#else + if(!zBuf) return FSL_RC_MISUSE; + else if(!nBuf) return FSL_RC_RANGE; + else if( getcwd(zBuf, + nBuf /*-1 not necessary: getcwd() NUL-terminates*/)==0 ){ + return fsl_errno_to_rc(errno, FSL_RC_IO); + }else{ + if(outLen) *outLen = fsl_strlen(zBuf); + return 0; + } +#endif +} + +/* + The file status information from the most recent stat() call. + + Use _stati64 rather than stat on windows, in order to handle files + larger than 2GB. +*/ +#if defined(_WIN32) && (defined(__MSVCRT__) || defined(_MSC_VER)) +# undef stat +# define stat _stati64 +#endif +/* + On Windows S_ISLNK always returns FALSE. +*/ +#if !defined(S_ISLNK) +# define S_ISLNK(x) (0) +#endif + +/* Reminder: the semantics of the 3rd parameter are + reversed from v1's fossil_stat(). +*/ +int fsl_stat(const char *zFilename, fsl_fstat * fst, + bool derefSymlinks){ + /* FIXME: port in fossil(1) win32_stat() */ + if(!zFilename) return FSL_RC_MISUSE; + else if(!*zFilename) return FSL_RC_RANGE; + else{ + int rc; + struct stat buf; +#if !defined(_WIN32) + char *zMbcs = (char *)fsl_utf8_to_filename(zFilename); + if(!zMbcs) rc = FSL_RC_OOM; + else{ + if( derefSymlinks ){ + rc = stat(zMbcs, &buf); + }else{ + rc = lstat(zMbcs, &buf); + } + } +#else + wchar_t *zMbcs = (wchar_t *)fsl_utf8_to_filename(zFilename); + /*trailing pathseps are forbidden in Windows stat fxns, as per doc; sigh*/ + int nzmbcslen = wcslen ( zMbcs ); + while ( nzmbcslen > 0 && ( L'\\' == zMbcs[nzmbcslen-1] || + L'/' == zMbcs[nzmbcslen-1] ) ) { + zMbcs[nzmbcslen-1] = 0; + --nzmbcslen; + } + rc = zMbcs ? _wstati64(zMbcs, &buf) : FSL_RC_OOM; +#endif + if(zMbcs) fsl_filename_free(zMbcs); + if(fst && (0==rc)){ + *fst = fsl_fstat_empty; + fst->ctime = (fsl_time_t)buf.st_ctime; + fst->mtime = (fsl_time_t)buf.st_mtime; + fst->size = (fsl_size_t)buf.st_size; + if(S_ISDIR(buf.st_mode)) fst->type = FSL_FSTAT_TYPE_DIR; +#if !defined(_WIN32) + else if(S_ISLNK(buf.st_mode)) fst->type = FSL_FSTAT_TYPE_LINK; +#endif + else /* if(S_ISREG(buf.st_mode)) */{ + fst->type = FSL_FSTAT_TYPE_FILE; +#if defined(_WIN32) +# ifndef S_IXUSR +# define S_IXUSR _S_IEXEC +# endif + if(((S_IXUSR)&buf.st_mode)!=0){ + fst->perm |= FSL_FSTAT_PERM_EXE; + } +#else + if( ((S_IXUSR|S_IXGRP|S_IXOTH)&buf.st_mode)!=0 ){ + fst->perm |= FSL_FSTAT_PERM_EXE; + } +#if 0 + /* Porting artifact: something to consider... */ + else if( g.allowSymlinks && S_ISLNK(buf.st_mode) ) + return PERM_LNK; +#endif +#endif + } + }else if(rc){ + rc = fsl_errno_to_rc(errno, FSL_RC_IO); + } + return rc; + } +} + +fsl_int_t fsl_file_size(const char *zFilename){ + fsl_fstat fst; + return ( 0 != fsl_stat(zFilename, &fst, 1) ) + ? -1 + : (fsl_int_t)fst.size; +} + +/* + The family of 'wd' functions is historical in nature and not really + needed(???) at the library level. 'wd' == 'working directory' + (i.e. checkout). Ideally the library won't have to do any _direct_ + manipulation of directory trees, e.g. checkouts. That is essentially + app-level logic, though we'll need some level of infrastructure for + the apps to build off of. When that comes, the "wd" family of + functions (or something similar) might come back into play. +*/ + +fsl_time_t fsl_file_mtime(const char *zFilename){ + fsl_fstat fst; + return ( 0 != fsl_stat(zFilename, &fst, 1) ) + ? -1 + : (fsl_time_t)fst.mtime; +} + + +bool fsl_is_file(const char *zFilename){ + fsl_fstat fst; + return ( 0 != fsl_stat(zFilename, &fst, 1) ) + ? false + : (FSL_FSTAT_TYPE_FILE == fst.type); +} + +bool fsl_is_symlink(const char *zFilename){ + fsl_fstat fst; + return ( 0 != fsl_stat(zFilename, &fst, 0) ) + ? false + : (FSL_FSTAT_TYPE_LINK == fst.type); +} + +/* + Return true if zPath is an absolute pathname. Return false + if it is relative. +*/ +bool fsl_is_absolute_path(const char *zPath){ + if( zPath && ((zPath[0]=='/') +#if defined(_WIN32) || defined(__CYGWIN__) + || (zPath[0]=='\\') + || (fsl_isalpha(zPath[0]) && zPath[1]==':' + && (zPath[2]=='\\' || zPath[2]=='/')) +#endif + ) + ){ + return 1; + }else{ + return 0; + } +} + +bool fsl_is_simple_pathname(const char *z, bool bStrictUtf8){ + int i; + unsigned char c = (unsigned char) z[0]; + char maskNonAscii = bStrictUtf8 ? 0x80 : 0x00; + if( c=='/' || c==0 ) return 0; + if( c=='.' ){ /* Common cases: ./ and ../ */ + if( z[1]=='/' || z[1]==0 ) return 0; + if( z[1]=='.' && (z[2]=='/' || z[2]==0) ) return 0; + } + for(i=0; (c=(unsigned char)z[i])!=0; i++){ + if( c & maskNonAscii ){ + if( (z[++i]&0xc0)!=0x80 ){ + /* Invalid first continuation byte */ + return 0; + } + if( c<0xc2 ){ + /* Invalid 1-byte UTF-8 sequence, or 2-byte overlong form. */ + return 0; + }else if( (c&0xe0)==0xe0 ){ + /* 3-byte or more */ + int unicode; + if( c&0x10 ){ + /* Unicode characters > U+FFFF are not supported. + * Windows XP and earlier cannot handle them. + */ + return 0; + } + /* This is a 3-byte UTF-8 character */ + unicode = ((c&0x0f)<<12) + ((z[i]&0x3f)<<6) + (z[i+1]&0x3f); + if( unicode <= 0x07ff ){ + /* overlong form */ + return 0; + }else if( unicode>=0xe000 ){ + /* U+E000..U+FFFF */ + if( (unicode<=0xf8ff) || (unicode>=0xfffe) ){ + /* U+E000..U+F8FF are for private use. + * U+FFFE..U+FFFF are noncharacters. */ + return 0; + } else if( (unicode>=0xfdd0) && (unicode<=0xfdef) ){ + /* U+FDD0..U+FDEF are noncharacters. */ + return 0; + } + }else if( (unicode>=0xd800) && (unicode<=0xdfff) ){ + /* U+D800..U+DFFF are for surrogate pairs. */ + return 0; + } + if( (z[++i]&0xc0)!=0x80 ){ + /* Invalid second continuation byte */ + return 0; + } + } + }else if( bStrictUtf8 && (c=='\\') ){ + return 0; + } + if( c=='/' ){ + if( z[i+1]=='/' ) return 0; + if( z[i+1]=='.' ){ + if( z[i+2]=='/' || z[i+2]==0 ) return 0; + if( z[i+2]=='.' && (z[i+3]=='/' || z[i+3]==0) ) return 0; + if( z[i+3]=='.' ) return 0; + } + } + } + if( z[i-1]=='/' ) return 0; + return 1; +} + + +/* + If the last component of the pathname in z[0]..z[j-1] is something + other than ".." then back it out and return true. If the last + component is empty or if it is ".." then return false. +*/ +static bool fsl_backup_dir(const char *z, fsl_int_t *pJ){ + fsl_int_t j = *pJ; + fsl_int_t i; + if( !j ) return 0; + for(i=j-1; i>0 && z[i-1]!='/'; i--){} + if( z[i]=='.' && i==j-2 && z[i+1]=='.' ) return 0; + *pJ = i-1; + return 1; +} + + + +fsl_size_t fsl_file_simplify_name(char *z, fsl_int_t n_, bool slash){ + fsl_size_t i; + fsl_size_t n = (n_<0) ? fsl_strlen(z) : (fsl_size_t)n_; + fsl_int_t j; + /* On windows and cygwin convert all \ characters to / */ +#if defined(_WIN32) || defined(__CYGWIN__) + for(i=0; i1 && z[n-1]=='/' ){ n--; } + } + + /* Remove duplicate '/' characters. Except, two // at the beginning + of a pathname is allowed since this is important on windows. */ + for(i=j=1; i=0 ) z[j] = z[i]; + j++; + } + if( j==0 ) z[j++] = '.'; + z[j] = 0; + return (fsl_size_t)j; +} + +int fsl_file_canonical_name2(const char *zRoot, + const char *zOrigName, + fsl_buffer *pOut, bool slash){ + int rc; + if(!zOrigName || !pOut) return FSL_RC_MISUSE; + else if( fsl_is_absolute_path(zOrigName) || (zRoot && !*zRoot)){ + rc = fsl_buffer_append( pOut, zOrigName, -1 ); +#if defined(_WIN32) || defined(__CYGWIN__) + if(!rc){ + char *zOut; + /* + On Windows/cygwin, normalize the drive letter to upper case. + */ + zOut = fsl_buffer_str(pOut); + if( fsl_islower(zOut[0]) && zOut[1]==':' ){ + zOut[0] = fsl_toupper(zOut[0]); + } + } +#endif + }else if(!zRoot){ + char zPwd[2000]; + fsl_size_t nOrig = fsl_strlen(zOrigName); + assert(nOrig < sizeof(zPwd)); + rc = fsl_getcwd(zPwd, sizeof(zPwd)-nOrig, NULL); + if(!rc){ +#if defined(_WIN32) + /* + On Windows, normalize the drive letter to upper case. + */ + if( !rc && fsl_islower(zPwd[0]) && zPwd[1]==':' ){ + zPwd[0] = fsl_toupper(zPwd[0]); + } +#endif + rc = fsl_buffer_appendf(pOut, "%//%/", zPwd, zOrigName); + } + }else{ + rc = fsl_buffer_appendf(pOut, "%/%s%/", zRoot, + *zRoot ? "/" : "", + zOrigName); + } + if(!rc){ + fsl_size_t const newLen = fsl_file_simplify_name(fsl_buffer_str(pOut), + (int)pOut->used, slash); + /* Reminder to self: do NOT resize pOut to the new, + post-simplification length because pOut is almost always a + fsl_cx::scratchpad buffer and doing so forces all sorts of + downstream reallocs. */ + pOut->used = newLen; + } + return rc; +} + + +int fsl_file_canonical_name(const char *zOrigName, + fsl_buffer *pOut, bool slash){ + return fsl_file_canonical_name2(NULL, zOrigName, pOut, slash); +} + +int fsl_file_dirpart(char const * zFilename, + fsl_int_t nLen, + fsl_buffer * pOut, + bool leaveSlash){ + if(!zFilename || !*zFilename || !pOut) return FSL_RC_MISUSE; + else if(!nLen) return FSL_RC_RANGE; + else{ + fsl_size_t n = (nLen>0) ? (fsl_size_t)nLen : fsl_strlen(zFilename); + char const * z = zFilename + n; + char doBreak = 0; + if(!n) return FSL_RC_RANGE; + else while( !doBreak && (--z >= zFilename) ){ + switch(*z){ +#if defined(_WIN32) + case '\\': +#endif + case '/': + if(!leaveSlash) --z; + doBreak = 1; + break; + } + } + if(z<=zFilename){ + return (doBreak && leaveSlash) + ? fsl_buffer_append(pOut, zFilename, 1) + : fsl_buffer_append(pOut, "", 0) /* ensure a NUL terminator */; + }else{ + return fsl_buffer_append(pOut, zFilename, z-zFilename + 1); + } + } + +} + +int fsl_find_home_dir( fsl_buffer * tgt, bool requireWriteAccess ){ + char * zHome = NULL; + int rc = 0; + tgt->used = 0; +#if defined(_WIN32) || defined(__CYGWIN__) + zHome = fsl_getenv("LOCALAPPDATA"); + if( zHome==0 ){ + zHome = fsl_getenv("APPDATA"); + if( zHome==0 ){ + char *zDrive = fsl_getenv("HOMEDRIVE"); + zHome = fsl_getenv("HOMEPATH"); + if( zDrive && zHome ){ + tgt->used = 0; + rc = fsl_buffer_appendf(tgt, "%s", zDrive); + fsl_filename_free(zDrive); + if(rc){ + fsl_filename_free(zHome); + return rc; + } + } + } + } + if(NULL==zHome){ + rc = fsl_buffer_append(tgt, + "Cannot locate home directory - " + "please set the LOCALAPPDATA or " + "APPDATA or HOMEPATH " + "environment variables.", + -1); + return rc ? rc : FSL_RC_NOT_FOUND; + } + rc = fsl_buffer_appendf( tgt, "%/", zHome ); +#else + /* Unix... */ + zHome = fsl_getenv("HOME"); + if( zHome==0 ){ + rc = fsl_buffer_append(tgt, + "Cannot locate home directory - " + "please set the HOME environment " + "variable.", + -1); + return rc ? rc : FSL_RC_NOT_FOUND; + } + rc = fsl_buffer_appendf( tgt, "%s", zHome ); +#endif + + fsl_filename_free(zHome); + if(rc) return rc; + assert(0used); + zHome = fsl_buffer_str(tgt); + + if( fsl_dir_check(zHome)<1 ){ + /* assert(0==tgt->used); */ + fsl_buffer tmp = fsl_buffer_empty; + rc = fsl_buffer_appendf(&tmp, + "Invalid home directory: %s", + zHome); + fsl_buffer_swap_free(&tmp, tgt, -1); + return rc ? rc : FSL_RC_TYPE; + } + +#if !(defined(_WIN32) || defined(__CYGWIN__)) + /* Not sure why, but the is-writable check is historically only done + on Unix platforms? + + TODO: this was subsequently changed in fossil(1) to only require + that the global db dir be writable. Port the newer logic in. + */ + if( requireWriteAccess && + (0 != fsl_file_access(zHome, W_OK)) ){ + fsl_buffer tmp = fsl_buffer_empty; + rc = fsl_buffer_appendf(&tmp, + "Home directory [%s] must " + "be writeable.", + zHome); + fsl_buffer_swap_free(&tmp, tgt, -1); + return rc ? rc : FSL_RC_ACCESS; + } +#endif + + return rc; +} + +int fsl_errno_to_rc(int errNo, int dflt){ + switch(errNo){ + /* Plese expand on this as tests/use cases call for it... */ + case EINVAL: + return FSL_RC_MISUSE; + case ENOMEM: + return FSL_RC_OOM; + case EROFS: + case EACCES: + case EBUSY: + case EPERM: + case EDQUOT: + case EAGAIN: + case ETXTBSY: + return FSL_RC_ACCESS; + case EISDIR: + case ENOTDIR: + return FSL_RC_TYPE; + case ENAMETOOLONG: + case ELOOP: + case ERANGE: + return FSL_RC_RANGE; + case ENOENT: + case ESRCH: + return FSL_RC_NOT_FOUND; + case EEXIST: + case ENOTEMPTY: + return FSL_RC_ALREADY_EXISTS; + case EIO: + return FSL_RC_IO; + default: + return dflt; + } +} + +int fsl_file_unlink(const char *zFilename){ + int rc; +#ifdef _WIN32 + wchar_t *z = (wchar_t*)fsl_utf8_to_filename(zFilename); + rc = _wunlink(z) ? errno : 0; +#else + char *z = (char *)fsl_utf8_to_filename(zFilename); + rc = unlink(zFilename) ? errno : 0; +#endif + fsl_filename_free(z); + return rc ? fsl_errno_to_rc(errno, FSL_RC_IO) : 0; +} + +int fsl_mkdir(const char *zName, bool forceFlag){ + int rc = + /*file_wd_dir_check(zName)*/ + fsl_dir_check(zName) + ; + if( rc<0 ){ + if( !forceFlag ) return FSL_RC_TYPE; + rc = fsl_file_unlink(zName); + if(rc) return rc; + }else if( 0==rc ){ +#if defined(_WIN32) + typedef wchar_t char_t; +#define mkdir(F,P) _wmkdir(F) +#else + typedef char char_t; +#endif + char_t *zMbcs = (char_t*)fsl_utf8_to_filename(zName); + if(!zMbcs) return FSL_RC_OOM; + rc = mkdir(zMbcs, 0755); + fsl_filename_free(zMbcs); + return rc ? fsl_errno_to_rc(errno, FSL_RC_IO) : 0; +#if defined(_WIN32) +#undef mkdir +#endif + } + return 0; +} + +int fsl_mkdir_for_file(char const *zName, bool forceFlag){ + int rc; + fsl_buffer b = fsl_buffer_empty /* we copy zName to + simplify traversal */; + fsl_size_t n = fsl_strlen(zName); + fsl_size_t i; + char * zCan; + if(n==0) return FSL_RC_RANGE; + else if(n<2) return 0/*no dir part*/; +#if 1 + /* This variant does more work (checks dirs we know already + exist) but transforms the path into something platform-neutral. + If we use fsl_file_simplify_name() instead then we end up + having to do the trailing-slash logic here. + */ + rc = fsl_file_canonical_name(zName, &b, 1); + if(rc) goto end; +#else + rc = fsl_buffer_append(&b, zName, n); + if(rc) goto end; +#endif + zCan = fsl_buffer_str(&b); + n = b.used; + for( i = 1; i < n; ++i ){ + if( '/'==zCan[i] ){ + zCan[i] = 0; +#if defined(_WIN32) || defined(__CYGWIN__) + /* + On Windows, local path looks like: C:/develop/project/file.txt + The if stops us from trying to create a directory of a drive letter + C: in this example. + */ + if( !(i==2 && zCan[1]==':') ){ +#endif + rc = fsl_dir_check(zCan); +#if 0 + if(rc<0){ + if(forceFlag) rc = fsl_file_unlink(zCan); + else rc = FSL_RC_TYPE; + if(rc) goto end; + } +#endif + /* MARKER(("dir_check rc=%d, zCan=%s\n", rc, zCan)); */ + if(0>=rc){ + rc = fsl_mkdir(zCan, forceFlag); + /* MARKER(("mkdir(%s) rc=%s\n", zCan, fsl_rc_cstr(rc))); */ + if( 0!=rc ) goto end; + + }else{ + rc = 0; + /* Nothing to do. */ + } +#if defined(_WIN32) || defined(__CYGWIN__) + } +#endif + zCan[i] = '/'; + } + } + end: + fsl_buffer_clear(&b); + return rc; +} + +#if defined(_WIN32) +/* Taken verbatim from fossil(1), just renamed */ +/* +** Returns non-zero if the specified name represents a real directory, i.e. +** not a junction or symbolic link. This is important for some operations, +** e.g. removing directories via _wrmdir(), because its detection of empty +** directories will (apparently) not work right for junctions and symbolic +** links, etc. +*/ +static int w32_file_is_normal_dir(wchar_t *zName){ + /* + ** Mask off attributes, applicable to directories, that are harmless for + ** our purposes. This may need to be updated if other attributes should + ** be ignored by this function. + */ + DWORD dwAttributes = GetFileAttributesW(zName); + if( dwAttributes==INVALID_FILE_ATTRIBUTES ) return 0; + dwAttributes &= ~( + FILE_ATTRIBUTE_ARCHIVE | FILE_ATTRIBUTE_COMPRESSED | + FILE_ATTRIBUTE_ENCRYPTED | FILE_ATTRIBUTE_NORMAL | + FILE_ATTRIBUTE_NOT_CONTENT_INDEXED + ); + return dwAttributes==FILE_ATTRIBUTE_DIRECTORY; +} +#endif + + +int fsl_rmdir(const char *zFilename){ + int rc = fsl_dir_check(zFilename); + if(rc<1) return rc ? FSL_RC_TYPE : FSL_RC_NOT_FOUND; +#ifdef _WIN32 + wchar_t *z = (wchar_t*)fsl_utf8_to_filename(zFilename); + if(w32_file_is_normal_dir(z)){ + rc = _wunlink(z) ? errno : 0; + }else{ + rc = ENOTDIR; + } +#else + char *z = (char *)fsl_utf8_to_filename(zFilename); + rc = rmdir(zFilename) ? errno : 0; +#endif + fsl_filename_free(z); + if(rc){ + int const eno = errno; + switch(eno){ + /* ENOENT normally maps to FSL_RC_NOT_FOUND, + but in this case that's ambiguous. */ + case ENOENT: rc = FSL_RC_ACCESS; break; + default: rc = fsl_errno_to_rc(errno, FSL_RC_IO); + break; + } + } + return rc; +} + +int fsl_dir_check(const char *zFilename){ + fsl_fstat fst; + int rc; + if( zFilename ){ +#if 1 + rc = fsl_stat(zFilename, &fst, 1); +#else + char *zFN = fsl_strdup(zFilename); + if(!zFN) rc = FSL_RC_OOM; + else{ + fsl_file_simplify_name(zFN, -1, 0); + rc = fsl_stat(zFN, &fst, 1); + fsl_free(zFN); + } +#endif + }else{ + rc = -1 /*fsl_stat(zFilename, &fst, 1) historic: used static stat cache*/; + } + return rc ? 0 : ((FSL_FSTAT_TYPE_DIR == fst.type) ? 1 : -1); +} + +int fsl_chdir(const char *zChDir){ + int rc; +#ifdef _WIN32 + wchar_t *zPath = fsl_utf8_to_filename(zChDir); + errno = 0; + rc = (int)!SetCurrentDirectoryW(zPath); + fsl_filename_free(zPath); + if(rc) rc = FSL_RC_IO; +#else + char *zPath = fsl_utf8_to_filename(zChDir); + errno = 0; + rc = chdir(zPath); + fsl_filename_free(zPath); + if(rc) rc = fsl_errno_to_rc(errno, FSL_RC_IO); +#endif + return rc; +} + + +#if 0 +/* + Same as dir_check(), but takes into account symlinks. +*/ +int file_wd_dir_check(const char *zFilename){ + if(!zFilename || !*zFilename) return FSL_RC_MISUSE; + else{ + int rc; + fsl_fstat fst = fsl_fstat_empty; + char *zFN = fsl_strdup(zFilename); + if(!zFN) rc = FSL_RC_OOM; + else{ + fsl_file_simplify_name(zFN, -1, 0); + rc = fsl_stat(zFN, &fst, 0); + fsl_free(zFN); + } + return rc ? 0 : ((FSL_FSTAT_TYPE_DIR == fst.type) ? 1 : 2); + } +} +#endif + +#if 0 +/* This block requires permissions flags from v1's manifest.c. */ + +/* + Return TRUE if the named file is an executable. Return false + for directories, devices, fifos, symlinks, etc. +*/ +int fsl_wd_isexe(const char *zFilename){ + return fsl_wd_perm(zFilename)==PERM_EXE; +} + +/* + Return TRUE if the named file is a symlink and symlinks are allowed. + Return false for all other cases. + + On Windows, always return False. +*/ +int file_wd_islink(const char *zFilename){ + return file_wd_perm(zFilename)==PERM_LNK; +} +#endif + +#if 0 +/** + Same as fsl_is_file(), but takes into account symlinks. + */ +bool fsl_wd_isfile(const char *zFilename); +bool fsl_wd_isfile(const char *zFilename){ + fsl_fstat fst; + return ( 0 != fsl_stat(zFilename, &fst, 0) ) + ? 0 + : (FSL_FSTAT_TYPE_FILE == fst.type); +} +#endif +#if 0 +/** + Same as fsl_file_mtime(), but takes into account symlinks. + */ +fsl_time_t fsl_wd_mtime(const char *zFilename); +fsl_time_t fsl_wd_mtime(const char *zFilename){ + fsl_fstat fst; + return ( 0 != fsl_stat(zFilename, &fst, 0) ) + ? -1 + : (fsl_time_t)fst.mtime; +} + +bool fsl_wd_isfile_or_link(const char *zFilename){ + fsl_fstat fst; + return ( 0 != fsl_stat(zFilename, &fst, 0) ) + ? 0 + : ((FSL_FSTAT_TYPE_LINK == fst.type) + || (FSL_FSTAT_TYPE_FILE == fst.type)) + ; +} +#endif + +#if 0 +/** + Same as fsl_file_size(), but takes into account symlinks. + */ +fsl_size_t fsl_wd_size(const char *zFilename); +fsl_size_t fsl_wd_size(const char *zFilename){ + fsl_fstat fst; + return ( 0 != fsl_stat(zFilename, &fst, 0) ) + ? -1 + : fst.size; +} +#endif + +/* + Set the mtime for a file. +*/ +int fsl_file_mtime_set(const char *zFilename, fsl_time_t newMTime){ + if(!zFilename || !*zFilename) return FSL_RC_MISUSE; + else{ + int rc; + void * zMbcs; +#if !defined(_WIN32) + struct timeval tv[2]; + if(newMTime < 0) newMTime = (fsl_time_t)time(0); + zMbcs = fsl_utf8_to_filename(zFilename); + if(!zMbcs) return FSL_RC_OOM; + memset(tv, 0, sizeof(tv[0])*2); + tv[0].tv_sec = newMTime; + tv[1].tv_sec = newMTime; + rc = utimes((char const *)zMbcs, tv); +#else + struct _utimbuf tb; + if(newMTime < 0) newMTime = (fsl_time_t)time(0); + zMbcs = fsl_utf8_to_filename(zFilename); + if(!zMbcs) return FSL_RC_OOM; + tb.actime = newMTime; + tb.modtime = newMTime; + rc = _wutime((wchar_t const *)zMbcs, &tb); +#endif + fsl_filename_free(zMbcs); + return rc ? fsl_errno_to_rc(errno, FSL_RC_IO) : 0; + } +} + + + +void fsl_pathfinder_clear(fsl_pathfinder * const pf){ + if(pf){ + fsl_list_visit_free(&pf->ext, 1); + fsl_list_visit_free(&pf->dirs, 1); + fsl_buffer_clear(&pf->buf); + *pf = fsl_pathfinder_empty; + } +} + +static int fsl_pathfinder_add(fsl_list * const li, char const * str){ + char * cp = fsl_strdup(str); + int rc; + if(!cp) rc = FSL_RC_OOM; + else{ + rc = fsl_list_append(li, cp); + if(rc) fsl_free(cp); + } + return rc; +} + +int fsl_pathfinder_dir_add(fsl_pathfinder * const pf, char const * const dir){ + return (pf && dir) + ? fsl_pathfinder_add(&pf->dirs, dir) + : FSL_RC_MISUSE; +} + +int fsl_pathfinder_ext_add(fsl_pathfinder * const pf, char const * const ext){ + return (pf && ext) + ? fsl_pathfinder_add(&pf->ext, ext) + : FSL_RC_MISUSE; +} + +int fsl_pathfinder_search(fsl_pathfinder * const pf, + char const * const base, + char const ** pOut, + fsl_size_t * const outLen ){ + fsl_buffer * buf = pf ? &pf->buf : NULL; + fsl_list * ext; + fsl_list * dirs; + int rc = 0; + fsl_size_t d, x, nD, nX, resetLen = 0; + fsl_size_t baseLen; + static char const pathSep = +#if defined(_WIN32) + '\\' +#else + '/' +#endif + ; + if(!buf || !base || !*base) return FSL_RC_MISUSE; + else if(!*base) return FSL_RC_RANGE; + else if(0==fsl_file_access( base, 0 )){ + /* Special case: if base is found as-is, without a path search, + use it. This is arguable behaviour, though. + */ + if(pOut) *pOut = base; + if(outLen) *outLen = fsl_strlen(base); + return 0; + } + baseLen = fsl_strlen(base); + ext = &pf->ext; + dirs = &pf->dirs; + nD = dirs->used; + nX = ext->used; + for( d = 0; !rc && (d < nD); ++d ){ + char const * vD = (char const *)dirs->list[d]; + /* + Search breadth-first for a file/directory named by vD/base + */ + buf->used = 0; + if(vD){ + fsl_size_t const used = buf->used; + rc = fsl_buffer_append(buf, vD, -1); + if(rc) return rc; + if(used != buf->used){ + /* Only append separator if vD is non-empty. */ + rc = fsl_buffer_append(buf, &pathSep, 1); + if(rc) return rc; + } + } + rc = fsl_buffer_append(buf, base, (fsl_int_t)baseLen); + if(rc) return rc; + if(0==fsl_file_access( (char const *)buf->mem, 0 )) goto gotone; + resetLen = buf->used; + for( x = 0; !rc && (x < nX); ++x ){ + char const * vX = (char const *)ext->list[x]; + if(vX){ + buf->used = resetLen; + rc = fsl_buffer_append(buf, vX, -1); + if(rc) return rc; + } + assert(buf->used < buf->capacity); + buf->mem[buf->used] = 0; + if(0==fsl_file_access( (char const *)buf->mem, 0 )){ + goto gotone; + } + } + } + + return FSL_RC_NOT_FOUND; + + gotone: + if(outLen) *outLen = buf->used; + if(pOut) *pOut = (char const *)buf->mem; + return 0; +} + +char * fsl_file_without_drive_letter(char * zIn){ +#ifdef _WIN32 + if( zIn && fsl_isalpha(zIn[0]) && zIn[1]==':' ) zIn += 2; +#endif + return zIn; +} + +int fsl_dir_is_empty(const char *path){ + struct dirent *ent; + int retval = 0; + DIR *d = opendir(path); + if(!d){ + return -1; + } + while((ent = readdir(d))) { + const char * z = ent->d_name; + if('.'==*z && + (!z[1] || ('.'==z[1] && !z[2]))){ + // Skip "." and ".." entries + continue; + } + retval = 1; + break; + } + closedir(d); + return retval; +} + +int fsl_file_exec_set(const char *zFilename, bool isExe){ +#if FSL_PLATFORM_IS_WINDOWS + return 0; +#else + int rc = 0, err; + struct stat sb; + err = stat(zFilename, &sb); + if(0==err){ + if(!S_ISREG(sb.st_mode)) return 0; + else if(isExe){ + if( 0==(sb.st_mode & 0100) ){ + int const mode = (sb.st_mode & 0444)>>2 + /* This impl is from fossil, which is known to work, but... + what is the >>2 for?*/; + err = chmod(zFilename, (mode_t)(sb.st_mode | mode)); + } + }else if( 0!=(sb.st_mode & 0100) ){ + err = chmod(zFilename, sb.st_mode & ~0111); + } + } + if(err) rc = fsl_errno_to_rc(errno, FSL_RC_IO); + return rc; +#endif +} + +static int fsl_dircrawl_impl(fsl_buffer * dbuf, fsl_fstat * fst, + fsl_dircrawl_f cb, void * cbState, + fsl_dircrawl_state * dst, + unsigned int depth){ + int rc = 0; + DIR *dir = opendir(fsl_buffer_cstr(dbuf)); + struct dirent * dent = 0; + fsl_size_t const dPos = dbuf->used; + if(!dir){ + return fsl_errno_to_rc(errno, FSL_RC_IO); + } + if(depth>20/*arbitrary limit to try to avoid stack overflow*/){ + return FSL_RC_RANGE; + } + while(!rc && (dent = readdir(dir))){ + const char * z = dent->d_name; + if('.'==*z && + (!z[1] || ('.'==z[1] && !z[2]))){ + // Skip "." and ".." entries + continue; + } + dbuf->used = dPos; + rc = fsl_buffer_appendf(dbuf, "/%s", z); + if(rc) break; + fsl_size_t const newLen = dbuf->used; + if(fsl_stat((char const *)dbuf->mem, fst, false)){ + // Simply skip stat errors. i was once bitten by an app which did + // not do so. Scarred for life. Too soon. + rc = 0; + continue; + } + switch(fst->type){ + case FSL_FSTAT_TYPE_LINK: + case FSL_FSTAT_TYPE_DIR: + case FSL_FSTAT_TYPE_FILE: + break; + default: continue; + } + dbuf->mem[dbuf->used = dPos] = 0; + dst->absoluteDir = (char const *)dbuf->mem; + dst->entryName = z; + dst->entryType = fst->type; + dst->depth = depth; + rc = cb( dst ); + if(!rc){ + dbuf->mem[dbuf->used] = '/'; + dbuf->used = newLen; + if(FSL_FSTAT_TYPE_DIR==fst->type){ + rc = fsl_dircrawl_impl( dbuf, fst, cb, cbState, dst, depth+1 ); + } + } + } + closedir(dir); + return rc; +} + +int fsl_dircrawl(char const * dirName, fsl_dircrawl_f callback, + void * cbState){ + fsl_buffer dbuf = fsl_buffer_empty; + fsl_fstat fst = fsl_fstat_empty; + int rc = fsl_file_canonical_name(dirName, &dbuf, false); + fsl_dircrawl_state dst; + if(!rc && '/' == dbuf.mem[dbuf.used-1]){ + dbuf.mem[--dbuf.used] = 0; + } + memset(&dst, 0, sizeof(dst)); + dst.callbackState = cbState; + while(!rc){ + rc = fsl_stat((char const *)dbuf.mem, &fst, false); + if(rc) break; + else if(FSL_FSTAT_TYPE_DIR!=fst.type){ + rc = FSL_RC_TYPE; + break; + } + rc = fsl_dircrawl_impl(&dbuf, &fst, callback, cbState, &dst, 1); + if(FSL_RC_BREAK==rc) rc = 0; + break; + } + fsl_buffer_clear(&dbuf); + return rc; +} + +bool fsl_is_file_or_link(const char *zFilename){ + fsl_fstat fst = fsl_fstat_empty; + if(fsl_stat(zFilename, &fst, false)) return false; + return fst.type==FSL_FSTAT_TYPE_FILE + || fst.type==FSL_FSTAT_TYPE_LINK; +} + +fsl_size_t fsl_strip_trailing_slashes(char * name, fsl_int_t nameLen){ + fsl_size_t rc = 0; + if(nameLen < 0) nameLen = (fsl_int_t)fsl_strlen(name); + if(nameLen){ + char * z = name + nameLen - 1; + for( ; (z>=name) && ('/'==*z); --z){ + *z = 0; + ++rc; + } + } + return rc; +} + +void fsl_buffer_strip_slashes(fsl_buffer * b){ + b->used -= fsl_strip_trailing_slashes((char *)b->mem, + (fsl_int_t)b->used); +} + +int fsl_file_rename(const char *zFrom, const char *zTo){ + int rc; +#if defined(_WIN32) + /** 2021-03-24: fossil's impl of this routine has 2 additional + params (bool isFromDir, bool isToDir), which are only used on + Windows platforms and are only to allow for 12 bytes of edge + case in MAX_PATH handling. We don't need them. */ + wchar_t *zMbcsFrom = fsl_utf8_to_filename(zFrom); + wchar_t *zMbcsTo = zMbcsFrom ? fsl_utf8_to_filename(zTo) : 0; + rc = zMbcsTo + ? _wrename(zMbcsFrom, zMbcsTo) + : FSL_RC_OOM; +#else + char *zMbcsFrom = fsl_utf8_to_filename(zFrom); + char *zMbcsTo = zMbcsFrom ? fsl_utf8_to_filename(zTo) : 0; + rc = zMbcsTo + ? rename(zMbcsFrom, zMbcsTo) + : FSL_RC_OOM; +#endif + fsl_filename_free(zMbcsTo); + fsl_filename_free(zMbcsFrom); + return rc + ? (FSL_RC_OOM==rc ? rc : fsl_errno_to_rc(errno, FSL_RC_IO)) + : 0; +} + +#if 0 +int fsl_file_relative_name( char const * zRoot, char const * zPath, + fsl_buffer * pOut, char retainSlash ){ + int rc = FSL_RC_NYI; + char * zPath; + fsl_size_t rootLen; + fsl_size_t pathLen; + if(!zPath || !*zPath || !pOut) return FSL_RC_MISUSE; + + return rc; +} +#endif + + +#undef MARKER +#ifdef _WIN32 +# undef DIR +# undef dirent +# undef opendir +# undef readdir +# undef closedir +#endif +/* end of file fs.c */ +/* start of file forum.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ + +/*************************************************************************** + This file houses the code for forum-level APIS. +*/ +#include + + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +int fsl_repo_install_schema_forum(fsl_cx *f){ + int rc; + fsl_db * db = fsl_needs_repo(f); + if(!db) return FSL_RC_NOT_A_REPO; + if(fsl_db_table_exists(db, FSL_DBROLE_REPO, "forumpost")){ + return 0; + } + MARKER(("table not exists?\n")); + rc = fsl_db_exec_multi(db, "%s",fsl_schema_forum()); + if(rc){ + rc = fsl_cx_uplift_db_error(f, db); + } + return rc; +} + + + +#undef MARKER +/* end of file forum.c */ +/* start of file glob.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************* + This file contains some of the APIs dealing with globs. +*/ +#include + +bool fsl_str_glob(const char *zGlob, const char *z){ +#if 1 + return (zGlob && z) + ? (sqlite3_strglob(zGlob,z) ? 0 : 1) + : 0; +#else + int c, c2; + int invert; + int seen; + while( (c = (*(zGlob++)))!=0 ){ + if( c=='*' ){ + while( (c=(*(zGlob++))) == '*' || c=='?' ){ + if( c=='?' && (*(z++))==0 ) return 0; + } + if( c==0 ){ + return 1; + }else if( c=='[' ){ + while( *z && fsl_str_glob(zGlob-1,z)==0 ){ + z++; + } + return (*z)!=0; + } + while( (c2 = (*(z++)))!=0 ){ + while( c2!=c ){ + c2 = *(z++); + if( c2==0 ) return 0; + } + if( fsl_str_glob(zGlob,z) ) return 1; + } + return 0; + }else if( c=='?' ){ + if( (*(z++))==0 ) return 0; + }else if( c=='[' ){ + int prior_c = 0; + seen = 0; + invert = 0; + c = *(z++); + if( c==0 ) return 0; + c2 = *(zGlob++); + if( c2=='^' ){ + invert = 1; + c2 = *(zGlob++); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = *(zGlob++); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){ + c2 = *(zGlob++); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = *(zGlob++); + } + if( c2==0 || (seen ^ invert)==0 ) return 0; + }else{ + if( c!=(*(z++)) ) return 0; + } + } + return *z==0; +#endif +} + + +int fsl_glob_list_parse( fsl_list * tgt, char const * zPatternList ){ + fsl_size_t i; /* Loop counters */ + char const *z = zPatternList; + char * cp; + char delimiter; /* '\'' or '\"' or 0 */ + int rc = 0; + char const * end; + if( !tgt || !zPatternList ) return FSL_RC_MISUSE; + else if(!*zPatternList) return 0; + end = zPatternList + fsl_strlen(zPatternList); + while( (zused) return NULL; + else{ + char const * glob; + fsl_size_t i = 0; + for( ; i < globList->used; ++i){ + glob = (char const *)globList->list[i]; + if( fsl_str_glob( glob, zNeedle ) ) return glob; + } + return NULL; + } +} + +int fsl_glob_list_append( fsl_list * tgt, char const * zGlob ){ + if(!tgt || !zGlob || !*zGlob) return FSL_RC_MISUSE; + else{ + char * cp = fsl_strdup(zGlob); + int rc = cp ? 0 : FSL_RC_OOM; + if(!rc){ + rc = fsl_list_append(tgt, cp); + if(rc) fsl_free(cp); + } + return rc; + } +} + + +void fsl_glob_list_clear( fsl_list * const globList ){ + if(globList) fsl_list_visit_free(globList, 1); +} + +/* end of file glob.c */ +/* start of file io.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************* + This file implements the generic i/o-related parts of the library. +*/ +#include +#include +#include /* memcmp() */ + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +/** + fsl_appendf_f() impl which sends its output to fsl_output(). state + must be a (fsl_cx*). + */ +static int fsl_output_f_fsl_output( void * state, void const * s, + fsl_size_t n ){ + return fsl_output( (fsl_cx *)state, s, n ); +} + + +int fsl_outputfv( fsl_cx * const f, char const * fmt, va_list args ){ + if(!f || !fmt) return FSL_RC_MISUSE; + else if(!*fmt) return FSL_RC_RANGE; + return fsl_appendfv( fsl_output_f_fsl_output, f, fmt, args ); +} + +int fsl_outputf( fsl_cx * const f, char const * fmt, ... ){ + if(!f || !fmt) return FSL_RC_MISUSE; + else if(!*fmt) return FSL_RC_RANGE; + else{ + int rc; + va_list args; + va_start(args,fmt); + rc = fsl_outputfv( f, fmt, args ); + va_end(args); + return rc; + } +} + +int fsl_output( fsl_cx * const cx, void const * const src, fsl_size_t n ){ + if(!n || !cx->output.out) return 0; + else return cx->output.out( cx->output.state, src, n ); +} + +int fsl_flush( fsl_cx * const f ){ + return f->output.flush + ? f->output.flush(f->output.state) + : 0; +} + + +int fsl_flush_f_FILE(void * _FILE){ + return fflush((FILE*)_FILE) ? fsl_errno_to_rc(errno, FSL_RC_IO) : 0; +} + +int fsl_output_f_FILE( void * state, + void const * src, fsl_size_t n ){ + if(!n) return 0; + else return (1 == fwrite(src, n, 1, state ? (FILE*)state : stdout)) + ? 0 + : FSL_RC_IO; +} + +int fsl_input_f_FILE( void * state, void * dest, fsl_size_t * n ){ + if( !*n ) return FSL_RC_RANGE; + FILE * f = (FILE*) state; + *n = (fsl_size_t)fread( dest, 1, *n, f ); + return *n + ? 0 + : (feof(f) ? 0 : FSL_RC_IO); +} + +void fsl_finalizer_f_FILE( void * state, void * mem ){ + if(mem){ + fsl_fclose((FILE*)mem); + } +} + +int fsl_stream( fsl_input_f inF, void * inState, + fsl_output_f outF, void * outState ){ + if(!inF || !outF) return FSL_RC_MISUSE; + else{ + int rc = 0; + enum { BufSize = 1024 * 4 }; + unsigned char buf[BufSize]; + fsl_size_t rn = BufSize; + for( ; !rc && + (rn==BufSize) + && (0==(rc=inF(inState, buf, &rn))); + rn = BufSize){ + if(rn) rc = outF(outState, buf, rn); + else break; + } + return rc; + } +} + +int fsl_stream_compare( fsl_input_f in1, void * in1State, + fsl_input_f in2, void * in2State ){ + enum { BufSize = 1024 * 2 }; + unsigned char buf1[BufSize]; + unsigned char buf2[BufSize]; + fsl_size_t rn1 = BufSize; + fsl_size_t rn2 = BufSize; + int rc; + while(1){ + rc = in1(in1State, buf1, &rn1); + if(rc) return -1; + rc = in2(in2State, buf2, &rn2); + if(rc) return 1; + else if(rn1!=rn2){ + rc = (rn1 + + +/* Only for debugging */ +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +int fsl_repo_leaves_rebuild(fsl_cx * f){ + fsl_db * db = f ? fsl_cx_db_repo(f) : NULL; + return !db ? FSL_RC_MISUSE : fsl_db_exec_multi(db, + "DELETE FROM leaf;" + "INSERT OR IGNORE INTO leaf" + " SELECT cid FROM plink" + " EXCEPT" + " SELECT pid FROM plink" + " WHERE coalesce((SELECT value FROM tagxref" + " WHERE tagid=%d AND rid=plink.pid),'trunk')" + " == coalesce((SELECT value FROM tagxref" + " WHERE tagid=%d AND rid=plink.cid),'trunk')", + FSL_TAGID_BRANCH, FSL_TAGID_BRANCH + ); +} + +fsl_int_t fsl_count_nonbranch_children(fsl_cx * const f, fsl_id_t rid){ + int32_t rv = 0; + int rc; + fsl_db * const db = fsl_cx_db_repo(f); + if(!db || !db->dbh || (rid<=0)) return -1; + rc = fsl_db_get_int32(db, &rv, + "SELECT count(*) FROM plink " + "WHERE pid=%"FSL_ID_T_PFMT" " + "AND isprim " + "AND coalesce((SELECT value FROM tagxref " + "WHERE tagid=%d AND rid=plink.pid), 'trunk')" + "=coalesce((SELECT value FROM tagxref " + "WHERE tagid=%d AND rid=plink.cid), 'trunk')", + rid, FSL_TAGID_BRANCH, FSL_TAGID_BRANCH); + return rc ? -2 : rv; +} + +bool fsl_rid_is_leaf(fsl_cx * const f, fsl_id_t rid){ + int rv = -1; + int rc; + fsl_db * db = f ? fsl_cx_db_repo(f) : NULL; + fsl_stmt * st = NULL; + if(!db || !db->dbh || (rid<=0)) return 0; + rc = fsl_db_prepare_cached(db, &st, + "SELECT 1 FROM plink " + "WHERE pid=?1 " + "AND coalesce(" + "(SELECT value FROM tagxref " + "WHERE tagid=%d AND rid=?1), " + //"(SELECT value FROM config WHERE name='main-branch'), " + "'trunk')" + "=coalesce((SELECT value FROM tagxref " + "WHERE tagid=%d " + "AND rid=plink.cid), " + //"(SELECT value FROM config WHERE name='main-branch'), " + "'trunk')" + "/*%s()*/", + FSL_TAGID_BRANCH, FSL_TAGID_BRANCH, __func__); + if(!rc){ + rc = fsl_stmt_bind_step(st, "R", rid); + switch(rc){ + case FSL_RC_STEP_ROW: + rv = 0; + rc = 0; + break; + case 0: + rv = 1; + rc = 0; + break; + default: + break; + } + fsl_stmt_cached_yield(st); + assert(0==rv || 1==rv); + } + return rc ? 0 : (rv==1); +} + +bool fsl_rid_is_version(fsl_cx * const f, fsl_id_t rid){ + fsl_db * const db = fsl_cx_db_repo(f); + if(!db) return false; + return 1==fsl_db_g_int32(db, 0, + "SELECT 1 FROM event " + "WHERE objid=%" FSL_ID_T_PFMT + " AND type='ci'", rid); +} + +int fsl_repo_leaf_check(fsl_cx * f, fsl_id_t rid){ + fsl_db * const db = f ? fsl_cx_db_repo(f) : NULL; + if(!db || !db->dbh) return FSL_RC_MISUSE; + else if(rid<=0) return FSL_RC_RANGE; + else { + int rc = 0; + bool isLeaf; + fsl_cx_err_reset(f); + isLeaf = fsl_rid_is_leaf(f, rid); + rc = fsl_cx_err_get(f, NULL, NULL); + if(!rc){ + fsl_stmt * st = NULL; + if( isLeaf ){ + rc = fsl_db_prepare_cached(db, &st, + "INSERT OR IGNORE INTO leaf VALUES" + "(?) /*%s()*/",__func__); + }else{ + rc = fsl_db_prepare_cached(db, &st, + "DELETE FROM leaf WHERE rid=?" + "/*%s()*/",__func__); + } + if(!rc && st){ + rc = fsl_stmt_bind_step(st, "R", rid); + fsl_stmt_cached_yield(st); + if(rc) rc = fsl_cx_uplift_db_error2(f, db, rc); + } + } + return rc; + } +} + +int fsl_repo_leaf_eventually_check( fsl_cx * f, fsl_id_t rid){ + fsl_db * db = f ? fsl_cx_db_repo(f) : NULL; + if(!f) return FSL_RC_MISUSE; + else if(rid<=0) return FSL_RC_RANGE; + else if(!db) return FSL_RC_NOT_A_REPO; + else { + fsl_stmt * parentsOf = NULL; + int rc = fsl_db_prepare_cached(db, &parentsOf, + "SELECT pid FROM plink WHERE " + "cid=? AND pid>0" + "/*%s()*/",__func__); + if(rc) return rc; + rc = fsl_stmt_bind_id(parentsOf, 1, rid); + if(!rc){ + rc = fsl_id_bag_insert(&f->cache.leafCheck, rid); + while( !rc && (FSL_RC_STEP_ROW==fsl_stmt_step(parentsOf)) ){ + rc = fsl_id_bag_insert(&f->cache.leafCheck, + fsl_stmt_g_id(parentsOf, 0)); + } + } + fsl_stmt_cached_yield(parentsOf); + return rc; + } +} + + +int fsl_repo_leaf_do_pending_checks(fsl_cx *f){ + fsl_id_t rid; + int rc = 0; + for(rid=fsl_id_bag_first(&f->cache.leafCheck); + !rc && rid; rid=fsl_id_bag_next(&f->cache.leafCheck,rid)){ + rc = fsl_repo_leaf_check(f, rid); + } + fsl_id_bag_clear(&f->cache.leafCheck); + return rc; +} + +int fsl_leaves_compute(fsl_cx * f, fsl_id_t vid, + fsl_leaves_compute_e closeMode){ + fsl_db * const db = fsl_needs_repo(f); + if(!db) return FSL_RC_NOT_A_REPO; + int rc = 0; + + /* Create the LEAVES table if it does not already exist. Make sure + ** it is empty. + */ + rc = fsl_db_exec_multi(db, + "CREATE TEMP TABLE IF NOT EXISTS leaves(" + " rid INTEGER PRIMARY KEY" + ");" + "DELETE FROM leaves;" + ); + if(rc) goto dberr; + if( vid <= 0 ){ + rc = fsl_db_exec_multi(db, + "INSERT INTO leaves SELECT leaf.rid FROM leaf" + ); + if(rc) goto dberr; + } + if( vid>0 ){ + fsl_id_bag seen = fsl_id_bag_empty; /* Descendants seen */ + fsl_id_bag pending = fsl_id_bag_empty; /* Unpropagated descendants */ + fsl_stmt q1 = fsl_stmt_empty; /* Query to find children of a check-in */ + fsl_stmt isBr = fsl_stmt_empty; /* Query to check to see if a check-in starts a new branch */ + fsl_stmt ins = fsl_stmt_empty; /* INSERT statement for a new record */ + + /* Initialize the bags. */ + rc = fsl_id_bag_insert(&pending, vid); + if(rc) goto cleanup; + + /* This query returns all non-branch-merge children of check-in :rid. + ** + ** If a child is a merge of a fork within the same branch, it is + ** returned. Only merge children in different branches are excluded. + */ + rc = fsl_db_prepare(db, &q1, + "SELECT cid FROM plink" + " WHERE pid=?1" + " AND (isprim" + " OR coalesce((SELECT value FROM tagxref" + " WHERE tagid=%d AND rid=plink.pid), 'trunk')" + /* FIXME? main-branch? */ + "=coalesce((SELECT value FROM tagxref" + " WHERE tagid=%d AND rid=plink.cid), 'trunk'))" + /* FIXME? main-branch? */ + , + FSL_TAGID_BRANCH, FSL_TAGID_BRANCH + ); + if(rc) goto cleanup; + /* This query returns a single row if check-in :rid is the first + ** check-in of a new branch. + */ + rc = fsl_db_prepare(db, &isBr, + "SELECT 1 FROM tagxref" + " WHERE rid=?1 AND tagid=%d AND tagtype=2" + " AND srcid>0", + FSL_TAGID_BRANCH + ); + if(rc) goto cleanup; + + /* This statement inserts check-in :rid into the LEAVES table. + */ + rc = fsl_db_prepare(db, &ins, + "INSERT OR IGNORE INTO leaves VALUES(?1)"); + if(rc) goto cleanup; + + while( fsl_id_bag_count(&pending) ){ + fsl_id_t const rid = fsl_id_bag_first(&pending); + unsigned cnt = 0; + fsl_id_bag_remove(&pending, rid); + fsl_stmt_bind_id(&q1, 1, rid); + while( FSL_RC_STEP_ROW==(rc = fsl_stmt_step(&q1)) ){ + int const cid = fsl_stmt_g_id(&q1, 0); + rc = fsl_id_bag_insert(&seen, cid); + if(rc) break; + rc = fsl_id_bag_insert(&pending, cid); + if(rc) break; + fsl_stmt_bind_id(&isBr, 1, cid); + if( FSL_RC_STEP_DONE==fsl_stmt_step(&isBr) ){ + ++cnt; + } + fsl_stmt_reset(&isBr); + } + if(FSL_RC_STEP_DONE==rc) rc = 0; + else if(rc) break; + fsl_stmt_reset(&q1); + if( cnt==0 && !fsl_rid_is_leaf(f, rid) ){ + ++cnt; + } + if( cnt==0 ){ + fsl_stmt_bind_id(&ins, 1, rid); + rc = fsl_stmt_step(&ins); + if(FSL_RC_STEP_DONE!=rc) break; + rc = 0; + fsl_stmt_reset(&ins); + } + } + cleanup: + fsl_stmt_finalize(&ins); + fsl_stmt_finalize(&isBr); + fsl_stmt_finalize(&q1); + fsl_id_bag_clear(&pending); + fsl_id_bag_clear(&seen); + if(rc) goto dberr; + } + assert(!rc); + switch(closeMode){ + case FSL_LEAVES_COMPUTE_OPEN: + rc = + fsl_db_exec_multi(db, + "DELETE FROM leaves WHERE rid IN" + " (SELECT leaves.rid FROM leaves, tagxref" + " WHERE tagxref.rid=leaves.rid " + " AND tagxref.tagid=%d" + " AND tagxref.tagtype>0)", + FSL_TAGID_CLOSED); + if(rc) goto dberr; + break; + case FSL_LEAVES_COMPUTE_CLOSED: + rc = + fsl_db_exec_multi(db, + "DELETE FROM leaves WHERE rid NOT IN" + " (SELECT leaves.rid FROM leaves, tagxref" + " WHERE tagxref.rid=leaves.rid " + " AND tagxref.tagid=%d" + " AND tagxref.tagtype>0)", + FSL_TAGID_CLOSED); + if(rc) goto dberr; + break; + default: break; + } + + end: + return rc; + dberr: + assert(rc); + rc = fsl_cx_uplift_db_error2(f, db, rc); + goto end; +} + +bool fsl_leaves_computed_has(fsl_cx * f){ + return fsl_db_exists(fsl_cx_db_repo(f), + "SELECT 1 FROM leaves"); +} + +fsl_int_t fsl_leaves_computed_count(fsl_cx * f){ + int32_t rv = -1; + fsl_db * const db = fsl_cx_db_repo(f); + int const rc = fsl_db_get_int32(db, &rv, + "SELECT COUNT(*) FROM leaves"); + if(rc){ + fsl_cx_uplift_db_error2(f, db, rc); + assert(-1==rv); + }else{ + assert(rv>=0); + } + return rv; +} + +fsl_id_t fsl_leaves_computed_latest(fsl_cx * f){ + fsl_id_t rv = 0; + fsl_db * const db = fsl_cx_db_repo(f); + int const rc = + fsl_db_get_id(db, &rv, + "SELECT rid FROM leaves, event" + " WHERE event.objid=leaves.rid" + " ORDER BY event.mtime DESC"); + if(rc){ + fsl_cx_uplift_db_error2(f, db, rc); + assert(!rv); + }else{ + assert(rv>=0); + } + return rv; +} + +void fsl_leaves_computed_cleanup(fsl_cx * f){ + fsl_db_exec(fsl_cx_db_repo(f), "DROP TABLE IF EXISTS leaves"); +} + +#undef MARKER +/* end of file leaf.c */ +/* start of file list.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************ + This file houses the implementations for the fsl_list routines. +*/ + +#include +#include /* malloc() and friends, qsort() */ +#include /* memset() */ +int fsl_list_reserve( fsl_list * self, fsl_size_t n ) +{ + if( !self ) return FSL_RC_MISUSE; + else if(0 == n){ + if(0 == self->capacity) return 0; + fsl_free(self->list); + *self = fsl_list_empty; + return 0; + } + else if( self->capacity >= n ){ + return 0; + } + else{ + size_t const sz = sizeof(void*) * n; + void* * m = (void**)fsl_realloc( self->list, sz ); + if( !m ) return FSL_RC_OOM; + memset( m + self->capacity, 0, (sizeof(void*)*(n-self->capacity))); + self->capacity = n; + self->list = m; + return 0; + } +} + +void fsl_list_swap( fsl_list * lhs, fsl_list * rhs ){ + fsl_list tmp = *lhs; + *rhs = *lhs; + *lhs = tmp; +} + +int fsl_list_append( fsl_list * self, void* cp ){ + if( !self ) return FSL_RC_MISUSE; + assert(self->used <= self->capacity); + if(self->used == self->capacity){ + int rc; + fsl_size_t const cap = self->capacity + ? (self->capacity * 2) + : 10; + rc = fsl_list_reserve(self, cap); + if(rc) return rc; + } + self->list[self->used++] = cp; + if(self->usedcapacity) self->list[self->used]=NULL; + return 0; +} + +int fsl_list_v_fsl_free(void * obj, void * visitorState ){ + if(obj) fsl_free( obj ); + return 0; +} + +int fsl_list_clear( fsl_list * self, fsl_list_visitor_f childFinalizer, + void * finalizerState ){ + /* + TODO: manually traverse the list and set each list entry for which + the finalizer succeeds to NULL, so that we can provide + well-defined behaviour if childFinalizer() fails and we abort the + loop. + */ + int rc = fsl_list_visit(self, 0, childFinalizer, finalizerState ); + if(!rc) fsl_list_reserve(self, 0); + return rc; +} + +void fsl_list_visit_free( fsl_list * self, bool freeListMem ){ + fsl_list_visit(self, 0, fsl_list_v_fsl_free, NULL ); + if(freeListMem) fsl_list_reserve(self, 0); + else self->used = 0; +} + + +int fsl_list_visit( fsl_list const * self, int order, + fsl_list_visitor_f visitor, + void * visitorState ){ + int rc = FSL_RC_OK; + if( self && self->used && visitor ){ + fsl_int_t i = 0; + fsl_int_t pos = (order<0) ? self->used-1 : 0; + fsl_int_t step = (order<0) ? -1 : 1; + for( rc = 0; (i < (fsl_int_t)self->used) + && (0 == rc); ++i, pos+=step ){ + void* obj = self->list[pos]; + if(obj) rc = visitor( obj, visitorState ); + if( obj != self->list[pos] ){ + --i; + if(order>=0) pos -= step; + } + } + } + return rc; +} + + +int fsl_list_visit_p( fsl_list * self, int order, + bool shiftIfNulled, + fsl_list_visitor_f visitor, void * visitorState ) +{ + int rc = FSL_RC_OK; + if( self && self->used && visitor ){ + fsl_int_t i = 0; + fsl_int_t pos = (order<0) ? self->used-1 : 0; + fsl_int_t step = (order<0) ? -1 : 1; + for( rc = 0; (i < (fsl_int_t)self->used) + && (0 == rc); ++i, pos+=step ){ + void* obj = self->list[pos]; + if(obj) { + assert((order<0) && "TEST THAT THIS WORKS WITH IN-ORDER!"); + rc = visitor( &self->list[pos], visitorState ); + if( shiftIfNulled && !self->list[pos]){ + fsl_int_t x = pos; + fsl_int_t const to = self->used-pos; + + assert( to < (fsl_int_t) self->capacity ); + for( ; x < to; ++x ) self->list[x] = self->list[x+1]; + if( x < (fsl_int_t)self->capacity ) self->list[x] = 0; + --i; + --self->used; + if(order>=0) pos -= step; + } + } + } + } + return rc; +} + +void fsl_list_sort( fsl_list * li, + fsl_generic_cmp_f cmp){ + if(li && li->used>1){ + qsort( li->list, li->used, sizeof(void*), cmp ); + } +} + + + +fsl_int_t fsl_list_index_of( fsl_list const * li, + void const * key, + fsl_generic_cmp_f cmpf ){ + fsl_size_t i; + void const * p; + for(i = 0; i < li->used; ++i){ + p = li->list[i]; + if(!p){ + if(!key) return (fsl_int_t)i; + else continue; + }else if((p==key) || + (0==cmpf( key, p )) ){ + return (fsl_int_t)i; + } + } + return -1; +} + + +fsl_int_t fsl_list_index_of_cstr( fsl_list const * li, + char const * key ){ + return fsl_list_index_of(li, key, fsl_strcmp_cmp); +} +/* end of file list.c */ +/* start of file lookslike.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ + +#include /* memcmp() */ + + +/* definitions for various UTF-8 sequence lengths, encoded as start value + * and size of each valid range belonging to some lead byte*/ +#define US2A 0x80, 0x01 /* for lead byte 0xC0 */ +#define US2B 0x80, 0x40 /* for lead bytes 0xC2-0xDF */ +#define US3A 0xA0, 0x20 /* for lead byte 0xE0 */ +#define US3B 0x80, 0x40 /* for lead bytes 0xE1-0xEF */ +#define US4A 0x90, 0x30 /* for lead byte 0xF0 */ +#define US4B 0x80, 0x40 /* for lead bytes 0xF1-0xF3 */ +#define US4C 0x80, 0x10 /* for lead byte 0xF4 */ +#define US0A 0x00, 0x00 /* for any other lead byte */ + +/* a table used for quick lookup of the definition that goes with a + * particular lead byte */ +static const unsigned char lb_tab[] = { + US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, + US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, + US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, + US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, + US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, + US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, + US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, + US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A, + US2A, US0A, US2B, US2B, US2B, US2B, US2B, US2B, + US2B, US2B, US2B, US2B, US2B, US2B, US2B, US2B, + US2B, US2B, US2B, US2B, US2B, US2B, US2B, US2B, + US2B, US2B, US2B, US2B, US2B, US2B, US2B, US2B, + US3A, US3B, US3B, US3B, US3B, US3B, US3B, US3B, + US3B, US3B, US3B, US3B, US3B, US3B, US3B, US3B, + US4A, US4B, US4B, US4B, US4C, US0A, US0A, US0A, + US0A, US0A, US0A, US0A, US0A, US0A, US0A, US0A +}; + +#undef US2A +#undef US2B +#undef US3A +#undef US3B +#undef US4A +#undef US4B +#undef US4C +#undef US0A + +int fsl_looks_like_utf8(fsl_buffer const * const b, int stopFlags){ + fsl_size_t n; + const char *z = fsl_buffer_cstr2(b, &n); + int j, c, flags = FSL_LOOKSLIKE_NONE; /* Assume UTF-8 text, prove otherwise */ + + if( n==0 ) return flags; /* Empty file -> text */ + c = *z; + if( c==0 ){ + flags |= FSL_LOOKSLIKE_NUL; /* NUL character in a file -> binary */ + }else if( c=='\r' ){ + flags |= FSL_LOOKSLIKE_CR; + if( n<=1 || z[1]!='\n' ){ + flags |= FSL_LOOKSLIKE_LONE_CR; /* Not enough chars or next char not LF */ + } + } + j = (c!='\n'); + if( !j ) flags |= (FSL_LOOKSLIKE_LF | FSL_LOOKSLIKE_LONE_LF); /* Found LF as first char */ + while( !(flags&stopFlags) && --n>0 ){ + int c2 = c; + c = *++z; ++j; + if( c==0 ){ + flags |= FSL_LOOKSLIKE_NUL; /* NUL character in a file -> binary */ + }else if( c=='\n' ){ + flags |= FSL_LOOKSLIKE_LF; + if( c2=='\r' ){ + flags |= (FSL_LOOKSLIKE_CR | FSL_LOOKSLIKE_CRLF); /* Found LF preceded by CR */ + }else{ + flags |= FSL_LOOKSLIKE_LONE_LF; + } + if( j>FSL_LINE_LENGTH_MASK ){ + flags |= FSL_LOOKSLIKE_LONG; /* Very long line -> binary */ + } + j = 0; + }else if( c=='\r' ){ + flags |= FSL_LOOKSLIKE_CR; + if( n<=1 || z[1]!='\n' ){ + flags |= FSL_LOOKSLIKE_LONE_CR; /* Not enough chars or next char not LF */ + } + } + } + if( n ){ + flags |= FSL_LOOKSLIKE_SHORT; /* The whole blob was not examined */ + } + if( j>FSL_LINE_LENGTH_MASK ){ + flags |= FSL_LOOKSLIKE_LONG; /* Very long line -> binary */ + } + return flags; +} + +unsigned char const *fsl_utf8_bom(unsigned int *pnByte){ + static const unsigned char bom[] = { + 0xef, 0xbb, 0xbf, 0x00, 0x00, 0x00 + }; + if( pnByte ) *pnByte = 3; + return bom; +} + +bool fsl_starts_with_bom_utf8(fsl_buffer const * const b, + unsigned int *pBomSize){ + unsigned int bomSize; + const char * const z = fsl_buffer_cstr(b); + const unsigned char * const bom = fsl_utf8_bom(&bomSize); + if( pBomSize ) *pBomSize = bomSize; + return fsl_buffer_size(b) OK */ + c = *z; + while( --n>0 ){ + if( c>=0x80 ){ + const unsigned char *def; /* pointer to range table*/ + c <<= 1; /* multiply by 2 and get rid of highest bit */ + def = &lb_tab[c]; /* search fb's valid range in table */ + if( (unsigned int)(*++z-def[0])>=def[1] ){ + return false/*FSL_LOOKSLIKE_INVALID*/; + } + c = (c>=0xC0) ? (c|3) : ' '; /* determine next lead byte */ + } else { + c = *++z; + } + } + return c<0x80 /* Final lead byte must be ASCII. */; +} +/* end of file lookslike.c */ +/* start of file md5.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* + The code is modified for use in fossil (then libfossil). The + original header comment follows: +*/ +/* + * This code implements the MD5 message-digest algorithm. + * The algorithm is due to Ron Rivest. This code was + * written by Colin Plumb in 1993, no copyright is claimed. + * This code is in the public domain; do with it what you wish. + * + * Equivalent code is available from RSA Data Security, Inc. + * This code has been tested against that, and is equivalent, + * except that you don't need to include two pages of legalese + * with every copy. + * + * To compute the message digest of a chunk of bytes, declare an + * MD5Context structure, pass it to MD5Init, call MD5Update as + * needed on buffers full of bytes, and then call MD5Final, which + * will fill a supplied 16-byte array with the digest. + */ +#include +#include +#include + +#if defined(__i386__) || defined(__x86_64__) || defined(_WIN32) +# define byteReverse(A,B) +#else +/* + * Convert an array of integers to little-endian. + * Note: this code is a no-op on little-endian machines. + */ +static void byteReverse (unsigned char *buf, unsigned longs){ + uint32_t t; + do { + t = (uint32_t)((unsigned)buf[3]<<8 | buf[2]) << 16 | + ((unsigned)buf[1]<<8 | buf[0]); + *(uint32_t *)buf = t; + buf += 4; + } while (--longs); +} +#endif + +/* The four core functions - F1 is optimized somewhat */ + +/* #define F1(x, y, z) (x & y | ~x & z) */ +#define F1(x, y, z) (z ^ (x & (y ^ z))) +#define F2(x, y, z) F1(z, x, y) +#define F3(x, y, z) (x ^ y ^ z) +#define F4(x, y, z) (y ^ (x | ~z)) + +/* This is the central step in the MD5 algorithm. */ +#define MD5STEP(f, w, x, y, z, data, s) \ + ( w += f(x, y, z) + data, w = w<>(32-s), w += x ) + +/* + * The core of the MD5 algorithm, this alters an existing MD5 hash to + * reflect the addition of 16 longwords of new data. MD5Update blocks + * the data and converts bytes into longwords for this routine. + */ +static void MD5Transform(uint32_t buf[4], const uint32_t in[16]){ + register uint32_t a, b, c, d; + + a = buf[0]; + b = buf[1]; + c = buf[2]; + d = buf[3]; + + MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478, 7); + MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12); + MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17); + MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22); + MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf, 7); + MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12); + MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17); + MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22); + MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8, 7); + MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12); + MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17); + MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22); + MD5STEP(F1, a, b, c, d, in[12]+0x6b901122, 7); + MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12); + MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17); + MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22); + + MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562, 5); + MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340, 9); + MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14); + MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20); + MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d, 5); + MD5STEP(F2, d, a, b, c, in[10]+0x02441453, 9); + MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14); + MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20); + MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6, 5); + MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6, 9); + MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14); + MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20); + MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905, 5); + MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8, 9); + MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14); + MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20); + + MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942, 4); + MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11); + MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16); + MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23); + MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44, 4); + MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11); + MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16); + MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23); + MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6, 4); + MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11); + MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16); + MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23); + MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039, 4); + MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11); + MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16); + MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23); + + MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244, 6); + MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10); + MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15); + MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21); + MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3, 6); + MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10); + MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15); + MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21); + MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f, 6); + MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10); + MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15); + MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21); + MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82, 6); + MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10); + MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15); + MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21); + + buf[0] += a; + buf[1] += b; + buf[2] += c; + buf[3] += d; +} + +const fsl_md5_cx fsl_md5_cx_empty = fsl_md5_cx_empty_m; +/* + * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious + * initialization constants. + */ +void fsl_md5_init(fsl_md5_cx *ctx){ + *ctx = fsl_md5_cx_empty; +} + +/* + * Update context to reflect the concatenation of another buffer full + * of bytes. + */ +void fsl_md5_update(fsl_md5_cx *ctx, void const * buf_, fsl_size_t len){ + const unsigned char * buf = (const unsigned char *)buf_; + uint32_t t; + + /* Update bitcount */ + + t = ctx->bits[0]; + if ((ctx->bits[0] = t + ((uint32_t)len << 3)) < t) + ctx->bits[1]++; /* Carry from low to high */ + ctx->bits[1] += len >> 29; + + t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */ + + /* Handle any leading odd-sized chunks */ + + if ( t ) { + unsigned char *p = (unsigned char *)ctx->in + t; + + t = 64-t; + if (len < t) { + memcpy(p, buf, len); + return; + } + memcpy(p, buf, t); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32_t *)ctx->in); + buf += t; + len -= t; + } + + /* Process data in 64-byte chunks */ + + while (len >= 64) { + memcpy(ctx->in, buf, 64); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32_t *)ctx->in); + buf += 64; + len -= 64; + } + + /* Handle any remaining bytes of data. */ + + memcpy(ctx->in, buf, len); +} + +/* + * Final wrapup - pad to 64-byte boundary with the bit pattern + * 1 0* (64-bit count of bits processed, MSB-first) + */ +void fsl_md5_final(fsl_md5_cx * ctx, unsigned char * digest){ + unsigned count; + unsigned char *p; + + /* Compute number of bytes mod 64 */ + count = (ctx->bits[0] >> 3) & 0x3F; + + /* Set the first char of padding to 0x80. This is safe since there is + always at least one byte free */ + p = ctx->in + count; + *p++ = 0x80; + + /* Bytes of padding needed to make 64 bytes */ + count = 64 - 1 - count; + + /* Pad out to 56 mod 64 */ + if (count < 8) { + /* Two lots of padding: Pad the first block to 64 bytes */ + memset(p, 0, count); + byteReverse(ctx->in, 16); + MD5Transform(ctx->buf, (uint32_t *)ctx->in); + + /* Now fill the next block with 56 bytes */ + memset(ctx->in, 0, 56); + } else { + /* Pad block to 56 bytes */ + memset(p, 0, count-8); + } + byteReverse(ctx->in, 14); + + /* Append length in bits and transform */ + memcpy(&ctx->in[14*sizeof(uint32_t)], ctx->bits, 2*sizeof(uint32_t)); + + MD5Transform(ctx->buf, (uint32_t *)ctx->in); + byteReverse((unsigned char *)ctx->buf, 4); + memcpy(digest, ctx->buf, 16); + memset(ctx, 0, sizeof(*ctx)); /* In case it's sensitive */ +} + +void fsl_md5_digest_to_base16(unsigned char *digest, char *zBuf){ + static char const zEncode[] = "0123456789abcdef"; + int i, j; + + for(j=i=0; i<16; i++){ + int a = digest[i]; + zBuf[j++] = zEncode[(a>>4)&0xf]; + zBuf[j++] = zEncode[a & 0xf]; + } + zBuf[j] = 0; +} + +#if 0 +/* The symlink-hashing code here may be needed at some point... */ +int fsl_md5sum_file(const char *zFilename, fsl_buffer *pCksum){ + if(!zFilename || !pCksum) return FSL_RC_MISUSE; + else{ + /* Requires v1 code which has not yet been ported in. */ + FILE *in; + fsl_md5_cx ctx; + unsigned char zResult[20]; + char zBuf[10240]; + + if( fsl_wd_islink(zFilename) ){ + /* Instead of file content, return md5 of link destination path */ + Blob destinationPath; + int rc; + + blob_read_link(&destinationPath, zFilename); + rc = sha1sum_blob(&destinationPath, pCksum); + blob_reset(&destinationPath); + return rc; + } + + in = fossil_fopen(zFilename,"rb"); + if( in==0 ){ + return 1; + } + fsl_sha1_init(&ctx); + for(;;){ + int n; + n = fread(zBuf, 1, sizeof(zBuf), in); + if( n<=0 ) break; + fsl_sha1_update(&ctx, (unsigned char*)zBuf, (unsigned)n); + } + fsl_fclose(in); + blob_zero(pCksum); + blob_resize(pCksum, 40); + fsl_sha1_final(&ctx, zResult); + fsl_sha1_digest_to_base16(zResult, blob_buffer(pCksum)); + return 0; + } +} +#endif + +int fsl_md5sum_buffer(fsl_buffer const *pIn, fsl_buffer *pCksum){ + if(!pIn || !pCksum) return FSL_RC_MISUSE; + else{ + fsl_md5_cx ctx = fsl_md5_cx_empty; + unsigned char zResult[20]; + int rc; + fsl_md5_update(&ctx, pIn->mem, pIn->used); + fsl_buffer_reuse(pCksum); + rc = fsl_buffer_resize(pCksum, FSL_STRLEN_MD5/*resize() adds 1 for NUL*/); + if(!rc){ + fsl_md5_final(&ctx, zResult); + fsl_md5_digest_to_base16(zResult, fsl_buffer_str(pCksum)); + } + return rc; + } +} + +char *fsl_md5sum_cstr(const char *zIn, fsl_int_t len){ + if(!zIn || !len) return NULL; + else{ + fsl_md5_cx ctx; + unsigned char zResult[20]; + char * zDigest = (char *)fsl_malloc(FSL_STRLEN_MD5+1); + if(!zDigest) return NULL; + fsl_md5_init(&ctx); + fsl_md5_update(&ctx, zIn, + (len<0) ? fsl_strlen(zIn) : (fsl_size_t)len); + fsl_md5_final(&ctx, zResult); + fsl_md5_digest_to_base16(zResult, zDigest); + return zDigest; + } +} + +int fsl_md5sum_stream(fsl_input_f src, void * srcState, fsl_buffer *pCksum){ + fsl_md5_cx ctx; + int rc; + unsigned char zResult[20]; + enum { BufSize = 1024 * 4 }; + unsigned char zBuf[BufSize]; + if(!src || !pCksum) return FSL_RC_MISUSE; + fsl_md5_init(&ctx); + for(;;){ + fsl_size_t read = (fsl_size_t)BufSize; + rc = src(srcState, zBuf, &read); + if(rc) return rc; + else if(read) fsl_md5_update(&ctx, (unsigned char*)zBuf, read); + if(read < (fsl_size_t)BufSize) break; + } + fsl_buffer_reuse(pCksum); + rc = fsl_buffer_resize(pCksum, FSL_STRLEN_MD5); + if(!rc){ + fsl_md5_final(&ctx, zResult); + fsl_md5_digest_to_base16(zResult, fsl_buffer_str(pCksum)); + } + return rc; +} + + + +#if 0 +void fsl_md5_to_base16(fsl_md5_cx const * cx, char *zBuf){ + static char const zEncode[] = "0123456789abcdef"; + int i, j; + + for(j=i=0; i<16; i++){ + int a = digest[i]; + zBuf[j++] = zEncode[(a>>4)&0xf]; + zBuf[j++] = zEncode[a & 0xf]; + } + zBuf[j] = 0; +} +#endif + + +#if 0 +/* + Add the content of a blob to the incremental MD5 checksum. +*/ +void md5sum_step_blob(Blob *p){ + md5sum_step_text(blob_buffer(p), blob_size(p)); +} +#endif + +int fsl_md5sum_filename(const char *zFilename, fsl_buffer *pCksum){ + if(!zFilename || !pCksum) return FSL_RC_MISUSE; + else{ + int rc; + FILE *in = fsl_fopen(zFilename, "rb"); + if(!in) rc = FSL_RC_IO; + else{ + rc = fsl_md5sum_stream(fsl_input_f_FILE, in, pCksum); + fsl_fclose(in); + } + return rc; + } +} + +void fsl_md5_update_buffer(fsl_md5_cx *cx, fsl_buffer const * b){ + if(b->used) fsl_md5_update(cx, b->mem, b->used); +} + +void fsl_md5_update_cstr(fsl_md5_cx *cx, char const * str, fsl_int_t len){ + if(len<0) len = fsl_strlen(str); + if(len>0) fsl_md5_update(cx, str, (fsl_size_t)len); +} + +int fsl_md5_update_stream(fsl_md5_cx *ctx, + fsl_input_f src, void * srcState){ + int rc; + enum { BufSize = 1024 * 4 }; + unsigned char zBuf[BufSize]; + if(!ctx || !src) return FSL_RC_MISUSE; + for(;;){ + fsl_size_t read = (fsl_size_t)BufSize; + rc = src(srcState, zBuf, &read); + if(rc) return rc; + else if(read) fsl_md5_update(ctx, (unsigned char*)zBuf, read); + if(read < (fsl_size_t)BufSize) break; + } + return 0; +} + +int fsl_md5_update_filename(fsl_md5_cx *cx, char const * fname){ + if(!cx || !fname) return FSL_RC_MISUSE; + else{ + int rc; + FILE *in = fsl_fopen(fname, "rb"); + if(in) rc = fsl_errno_to_rc(errno,FSL_RC_IO); + else { + rc = fsl_md5_update_stream(cx, fsl_input_f_FILE, in); + fsl_fclose(in); + } + return rc; + } +} + + +#undef F1 +#undef F2 +#undef F3 +#undef F4 +#undef MD5STEP +#undef byteReverse +/* end of file md5.c */ +/* start of file merge3.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +#include +#include +#include +#include /* memmove()/strlen() */ + +#if 0 +#define FDEBUG(X) X +#define ISFDEBUG 1 +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) +#else +#define FDEBUG(X) +#define ISFDEBUG 0 +#define MARKER(pfexp) (void)0 +#endif + +// Only for testing/debugging.. + +/* The minimum of two integers */ +#define mymin(A,B) ((A)<(B)?(A):(B)) + +/** + Compare N lines of text from pV1 and pV2. If the lines are the + same, return true. Return false if one or more of the N lines are + different. + + The cursors on both pV1 and pV2 is unchanged by this comparison. +*/ +static +bool sameLines(fsl_buffer const *pV1, fsl_buffer const *pV2, + int N){ + unsigned char const *z1; + unsigned char const *z2; + fsl_size_t i; + unsigned char c; + if( !N ) return true; + z1 = pV1->mem + pV1->cursor; + z2 = pV2->mem + pV2->cursor; + for(i=0; (c=z1[i])==z2[i]; ++i){ + if( c=='\n' || !c ){ + --N; + if( !N || !c ) return true; + } + } + return false; +} + +/** + Look at the next edit triple in both aC1 and aC2. (An "edit triple" is + three integers describing the number of copies, deletes, and inserts in + moving from the original to the edited copy of the file.) If the three + integers of the edit triples describe an identical edit, then return 1. + If the edits are different, return 0. + + aC1 = the array of edit integers for pV1. + + aC2 = the array of edit integers for pV2. +*/ +static +bool sameEdit(int const *aC1, int const *aC2, + fsl_buffer const *pV1, fsl_buffer const *pV2){ +#if 0 + if( aC1[0]!=aC2[0] ) return 0; + if( aC1[1]!=aC2[1] ) return 0; + if( aC1[2]!=aC2[2] ) return 0; + if( sameLines(pV1, pV2, aC1[2]) ) return 1; + return 0; +#else + if( aC1[0]!=aC2[0] + || aC1[1]!=aC2[1] + || aC1[2]!=aC2[2] ) return false; + return sameLines(pV1, pV2, aC1[2]); +#endif +} + + +/** + The aC[] array contains triples of integers. Within each triple, + the elements are: + + (0) The number of lines to copy + (1) The number of lines to delete + (2) The number of liens to insert + + Suppose we want to advance over sz lines of the original file. + This routine returns true if that advance would land us on a copy + operation. It returns false if the advance would end on a delete. +*/ +static +bool ends_at_CPY(int *aC, int sz){ + while( sz>0 && (aC[0]>0 || aC[1]>0 || aC[2]>0) ){ + if( aC[0]>=sz ) return true; + sz -= aC[0]; + if( aC[1]>sz ) return false; + sz -= aC[1]; + aC += 3; + } + return true; +} + +/** + pSrc contains an edited file where aC[] describes the edit. Part + of pSrc has already been output. This routine outputs additional + lines of pSrc - lines that correspond to the next sz lines of the + original unedited file. + + Note that sz counts the number of lines of text in the original + file, but text is output from the edited file, so the number of + lines transfer to pOut might be different from sz. Fewer lines + appear in pOut if there are deletes. More lines appear if there + are inserts. + + The aC[] array is updated and the new index into aC[] is returned + via the final argument. + + Returns 0 on success, FSL_RC_OOM on allocation error. +*/ +static +int output_one_side(fsl_buffer *pOut, + fsl_buffer *pSrc, + int *aC, + int i, + int sz, + int *newIndex){ + int rc = 0; + while( sz>0 ){ + if( aC[i]==0 && aC[i+1]==0 && aC[i+2]==0 ) break; + if( aC[i]>=sz ){ + rc = fsl_buffer_copy_lines(pOut, pSrc, sz); + if(rc) break; + aC[i] -= sz; + break; + } + rc = fsl_buffer_copy_lines(pOut, pSrc, aC[i]); + if(!rc) rc = fsl_buffer_copy_lines(pOut, pSrc, aC[i+2]); + if(rc) break; + sz -= aC[i] + aC[i+1]; + i += 3; + } + if(!rc) *newIndex = i; + return rc; +} + +/** + Return true if the input blob contains any CR/LF pairs on the first + ten lines. This should be enough to detect files that use mainly + CR/LF line endings without causing a performance impact for LF only + files. +*/ +static +bool contains_crlf(fsl_buffer const *p){ + fsl_size_t i; + fsl_size_t j = 0; + const uint16_t maxL = 10; //Max lines to check + unsigned const char *z = p->mem; + fsl_size_t const n = p->used+1; + for(i=1; imaxL ) break; + } + return false; +} + +/** + Ensure that the text in p, if not empty, ends with a new line. If + useCrLf is true adds "\r\n" otherwise "\n". Returns 0 on success + or p is empty, and FSL_RC_OOM on OOM. +*/ +static +int ensure_line_end(fsl_buffer *p, bool useCrLf){ + int rc = 0; + if( !p->used ) return 0; + if( p->mem[p->used-1]!='\n' ){ + rc = fsl_buffer_append(p, useCrLf ? "\r\n" : "\n", useCrLf ? 2 : 1); + } + return rc; +} + +/** + Returns an array of bytes representing the byte-order-mark for + UTF-8. If pnByte is not NULL, the number of bytes in the BOM (3) + is written there. +*/ +//static +const unsigned char *get_utf8_bom(unsigned int *pnByte){ + static const unsigned char bom[] = { + 0xef, 0xbb, 0xbf, 0x00, 0x00, 0x00 + }; + if( pnByte ) *pnByte = 3; + return bom; +} + +/** + Returns true if given blob starts with a UTF-8 + byte-order-mark (BOM). +*/ +static +bool starts_with_utf8_bom(fsl_buffer const *p, unsigned int *n){ + unsigned const char *z = p->mem; + unsigned int bomSize = 0; + const unsigned char *bom = get_utf8_bom(&bomSize); + if( n ) *n = bomSize; + return (p->used>>>>>> END MERGE CONFLICT >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" +}; + +static fsl_int_t assert_mema_lengths(){ + static const fsl_int_t mmLen = 68; + static bool once = true; + if(once){ + once = false; + assert(sizeof(mergeMarker)/sizeof(mergeMarker[0]) == 4); + assert((fsl_int_t)fsl_strlen(mergeMarker[0])==mmLen); + assert((fsl_int_t)fsl_strlen(mergeMarker[1])==mmLen); + assert((fsl_int_t)fsl_strlen(mergeMarker[2])==mmLen); + assert((fsl_int_t)fsl_strlen(mergeMarker[3])==mmLen); + } + return mmLen; +} + +int fsl_buffer_merge3(fsl_buffer * const pPivot, + fsl_buffer * const pV1, + fsl_buffer * const pV2, + fsl_buffer * const pOut, + unsigned int * const conflictCount){ + int *aC1 = 0; /* Changes from pPivot to pV1 */ + int *aC2 = 0; /* Changes from pPivot to pV2 */ + int i1, i2; /* Index into aC1[] and aC2[] */ + int nCpy, nDel, nIns; /* Number of lines to copy, delete, or insert */ + int limit1, limit2; /* Sizes of aC1[] and aC2[] */ + int rc = 0; + unsigned int nConflict = 0; /* Number of merge conflicts seen so far */ + bool useCrLf = false; + const fsl_int_t mmLen = assert_mema_lengths(); + +#define RC if(rc) { \ + MARKER(("rc=%s\n", fsl_rc_cstr(rc))); goto end; } (void)0 + fsl_buffer_reuse(pOut); /* Merge results stored in pOut */ + + /* If both pV1 and pV2 start with a UTF-8 byte-order-mark (BOM), + ** keep it in the output. This should be secure enough not to cause + ** unintended changes to the merged file and consistent with what + ** users are using in their source files. + */ + if( starts_with_utf8_bom(pV1, 0) && starts_with_utf8_bom(pV2, 0) ){ + rc = fsl_buffer_append(pOut, get_utf8_bom(0), 3); + RC; + } + + /* Check once to see if both pV1 and pV2 contains CR/LF endings. + ** If true, CR/LF pair will be used later to append the + ** boundary markers for merge conflicts. + */ + if( contains_crlf(pV1) && contains_crlf(pV2) ){ + useCrLf = true; + } + + /* Compute the edits that occur from pPivot => pV1 (into aC1) + ** and pPivot => pV2 (into aC2). Each of the aC1 and aC2 arrays is + ** an array of integer triples. Within each triple, the first integer + ** is the number of lines of text to copy directly from the pivot, + ** the second integer is the number of lines of text to omit from the + ** pivot, and the third integer is the number of lines of text that are + ** inserted. The edit array ends with a triple of 0,0,0. + */ + rc = fsl_diff_text_raw(pPivot, pV1, 0, &aC1); + if(!rc) rc = fsl_diff_text_raw(pPivot, pV2, 0, &aC2); + RC; + assert(aC1 && aC2); + + /* Rewind inputs: Needed to reconstruct output */ + fsl_buffer_rewind(pV1); + fsl_buffer_rewind(pV2); + fsl_buffer_rewind(pPivot); + + /* Determine the length of the aC1[] and aC2[] change vectors */ + for(i1=0; aC1[i1] || aC1[i1+1] || aC1[i1+2]; i1+=3){} + limit1 = i1; + for(i2=0; aC2[i2] || aC2[i2+1] || aC2[i2+2]; i2+=3){} + limit2 = i2; + + FDEBUG( + for(i1=0; i10 && aC2[i2]>0 ){ + /* Output text that is unchanged in both V1 and V2 */ + nCpy = mymin(aC1[i1], aC2[i2]); + FDEBUG( printf("COPY %d\n", nCpy); ) + rc = fsl_buffer_copy_lines(pOut, pPivot, (fsl_size_t)nCpy); + if(!rc) rc = fsl_buffer_copy_lines(0, pV1, (fsl_size_t)nCpy); + if(!rc) rc = fsl_buffer_copy_lines(0, pV2, (fsl_size_t)nCpy); + RC; + aC1[i1] -= nCpy; + aC2[i2] -= nCpy; + }else + if( aC1[i1] >= aC2[i2+1] && aC1[i1]>0 && aC2[i2+1]+aC2[i2+2]>0 ){ + /* Output edits to V2 that occurs within unchanged regions of V1 */ + nDel = aC2[i2+1]; + nIns = aC2[i2+2]; + FDEBUG( printf("EDIT -%d+%d left\n", nDel, nIns); ) + rc = fsl_buffer_copy_lines(0, pPivot, (fsl_size_t)nDel); + if(!rc) rc = fsl_buffer_copy_lines(0, pV1, (fsl_size_t)nDel); + if(!rc) rc = fsl_buffer_copy_lines(pOut, pV2, (fsl_size_t)nIns); + RC; + aC1[i1] -= nDel; + i2 += 3; + }else + if( aC2[i2] >= aC1[i1+1] && aC2[i2]>0 && aC1[i1+1]+aC1[i1+2]>0 ){ + /* Output edits to V1 that occur within unchanged regions of V2 */ + nDel = aC1[i1+1]; + nIns = aC1[i1+2]; + FDEBUG( printf("EDIT -%d+%d right\n", nDel, nIns); ) + rc = fsl_buffer_copy_lines(0, pPivot, (fsl_size_t)nDel); + if(!rc) fsl_buffer_copy_lines(0, pV2, (fsl_size_t)nDel); + if(!rc) fsl_buffer_copy_lines(pOut, pV1, (fsl_size_t)nIns); + aC2[i2] -= nDel; + i1 += 3; + }else + if( sameEdit(&aC1[i1], &aC2[i2], pV1, pV2) ){ + /* Output edits that are identical in both V1 and V2. */ + assert( aC1[i1]==0 ); + nDel = aC1[i1+1]; + nIns = aC1[i1+2]; + FDEBUG( printf("EDIT -%d+%d both\n", nDel, nIns); ) + rc = fsl_buffer_copy_lines(0, pPivot, (fsl_size_t)nDel); + if(!rc) fsl_buffer_copy_lines(pOut, pV1, (fsl_size_t)nIns); + if(!rc) fsl_buffer_copy_lines(0, pV2, (fsl_size_t)nIns); + i1 += 3; + i2 += 3; + }else + { + /* We have found a region where different edits to V1 and V2 overlap. + ** This is a merge conflict. Find the size of the conflict, then + ** output both possible edits separated by distinctive marks. + */ + int sz = 1; /* Size of the conflict in lines */ + ++nConflict; + while( !ends_at_CPY(&aC1[i1], sz) || !ends_at_CPY(&aC2[i2], sz) ){ + ++sz; + } + FDEBUG( printf("CONFLICT %d\n", sz); ) + rc = ensure_line_end(pOut, useCrLf); + if(!rc) rc = fsl_buffer_append(pOut, mergeMarker[0], mmLen); + if(!rc) rc = ensure_line_end(pOut, useCrLf); + RC; + rc = output_one_side(pOut, pV1, aC1, i1, sz, &i1); + if(!rc) rc = ensure_line_end(pOut, useCrLf); + RC; + rc = fsl_buffer_append(pOut, mergeMarker[1], mmLen); + if(!rc) rc = ensure_line_end(pOut, useCrLf); + if(!rc) rc = fsl_buffer_copy_lines(pOut, pPivot, sz); + if(!rc) rc = ensure_line_end(pOut, useCrLf); + RC; + rc = fsl_buffer_append(pOut, mergeMarker[2], mmLen); + if(!rc) rc = ensure_line_end(pOut, useCrLf); + if(!rc) rc = output_one_side(pOut, pV2, aC2, i2, sz, &i2); + if(!rc) rc = ensure_line_end(pOut, useCrLf); + RC; + rc = fsl_buffer_append(pOut, mergeMarker[3], mmLen); + if(!rc) rc = ensure_line_end(pOut, useCrLf); + RC; + } + + /* If we are finished with an edit triple, advance to the next + ** triple. + */ + if( i10 ){ + FDEBUG( printf("INSERT +%d left\n", aC1[i1+2]); ) + rc = fsl_buffer_copy_lines(pOut, pV1, aC1[i1+2]); + }else if( i20 ){ + FDEBUG( printf("INSERT +%d right\n", aC2[i2+2]); ) + rc = fsl_buffer_copy_lines(pOut, pV2, aC2[i2+2]); + } + + end: + fsl_free(aC1); + fsl_free(aC2); + if(!rc && conflictCount) *conflictCount = nConflict; + return rc; +#undef RC +} + +/* +** Return true if the input string contains a merge marker on a line by +** itself. +*/ +bool fsl_buffer_contains_merge_marker(fsl_buffer const *p){ + fsl_size_t i; + fsl_size_t const len = (fsl_size_t)assert_mema_lengths(); + if(p->used <= len) return false; + fsl_size_t j; + const char * const z = (const char *)p->mem; + fsl_size_t const n = p->used - len + 1; + for(i=0; i + +#ifdef _WIN32 +#include +#include +/* + Print a fatal error and quit. +*/ +static void win32_fatal_error(const char *zMsg){ + /*fossil_fatal("%s", zMsg); TODO: what to do here? */ +} +#else +#include +#include +#include +#endif + +/* + The following macros are used to cast pointers to integers and + integers to pointers. The way you do this varies from one compiler + to the next, so we have developed the following set of #if statements + to generate appropriate macros for a wide range of compilers. + + The correct "ANSI" way to do this is to use the intptr_t type. + Unfortunately, that typedef is not available on all compilers, or + if it is available, it requires an #include of specific headers + that vary from one machine to the next. + + This code is copied out of SQLite. +*/ +#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +# define INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) +# define PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) +#elif !defined(__GNUC__) /* Works for compilers other than LLVM */ +# define INT_TO_PTR(X) ((void*)&((char*)0)[X]) +# define PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) +#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +# define INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define PTR_TO_INT(X) ((int)(intptr_t)(X)) +#else /* Generates a warning - but it always works */ +# define INT_TO_PTR(X) ((void*)(X)) +# define PTR_TO_INT(X) ((int)(X)) +#endif + + +#ifdef _WIN32 +/* + On windows, create a child process and specify the stdin, stdout, + and stderr channels for that process to use. + + Return the number of errors. +*/ +static int win32_create_child_process( + wchar_t *zCmd, /* The command that the child process will run */ + HANDLE hIn, /* Standard input */ + HANDLE hOut, /* Standard output */ + HANDLE hErr, /* Standard error */ + DWORD *pChildPid /* OUT: Child process handle */ +){ + STARTUPINFOW si; + PROCESS_INFORMATION pi; + BOOL rc; + + memset(&si, 0, sizeof(si)); + si.cb = sizeof(si); + si.dwFlags = STARTF_USESTDHANDLES; + SetHandleInformation(hIn, HANDLE_FLAG_INHERIT, TRUE); + si.hStdInput = hIn; + SetHandleInformation(hOut, HANDLE_FLAG_INHERIT, TRUE); + si.hStdOutput = hOut; + SetHandleInformation(hErr, HANDLE_FLAG_INHERIT, TRUE); + si.hStdError = hErr; + rc = CreateProcessW( + NULL, /* Application Name */ + zCmd, /* Command-line */ + NULL, /* Process attributes */ + NULL, /* Thread attributes */ + TRUE, /* Inherit Handles */ + 0, /* Create flags */ + NULL, /* Environment */ + NULL, /* Current directory */ + &si, /* Startup Info */ + &pi /* Process Info */ + ); + if( rc ){ + CloseHandle( pi.hProcess ); + CloseHandle( pi.hThread ); + *pChildPid = pi.dwProcessId; + }else{ + win32_fatal_error("cannot create child process"); + } + return rc!=0; +} +#endif + +/** + Create a child process running shell command "zCmd". *ppOut gets + assigned to a FILE that becomes the standard input of the child process. + (The caller writes to *ppOut in order to send text to the child.) + *pfdIn gets assigned to the stdout from the child process. (The caller + reads from *pfdIn in order to receive input from the child.) + Note that *pfdIn is an unbuffered file descriptor, not a FILE. + The process ID of the child is written into *pChildPid. + + On success the values returned via *pfdIn, *ppOut, and *pChildPid + must be passed to fsl_pclose2() to properly clean up. + + Return 0 on success, non-0 on error. + */ +int fsl_popen2(const char *zCmd, int *pfdIn, FILE **ppOut, int *pChildPid){ +#ifdef _WIN32 + /* FIXME: port these win32_fatal_error() bits to error codes. */ + HANDLE hStdinRd, hStdinWr, hStdoutRd, hStdoutWr, hStderr; + SECURITY_ATTRIBUTES saAttr; + DWORD childPid = 0; + int fd; + + saAttr.nLength = sizeof(saAttr); + saAttr.bInheritHandle = TRUE; + saAttr.lpSecurityDescriptor = NULL; + hStderr = GetStdHandle(STD_ERROR_HANDLE); + if( !CreatePipe(&hStdoutRd, &hStdoutWr, &saAttr, 4096) ){ + win32_fatal_error("cannot create pipe for stdout"); + } + SetHandleInformation( hStdoutRd, HANDLE_FLAG_INHERIT, FALSE); + + if( !CreatePipe(&hStdinRd, &hStdinWr, &saAttr, 4096) ){ + win32_fatal_error("cannot create pipe for stdin"); + } + SetHandleInformation( hStdinWr, HANDLE_FLAG_INHERIT, FALSE); + + win32_create_child_process(fsl_utf8_to_unicode(zCmd), + hStdinRd, hStdoutWr, hStderr,&childPid); + *pChildPid = childPid; + *pfdIn = _open_osfhandle(PTR_TO_INT(hStdoutRd), 0); + fd = _open_osfhandle(PTR_TO_INT(hStdinWr), 0); + *ppOut = _fdopen(fd, "w"); + CloseHandle(hStdinRd); + CloseHandle(hStdoutWr); + return 0; +#else + int rc; + int pin[2], pout[2]; + *pfdIn = 0; + *ppOut = 0; + *pChildPid = 0; + + if( pipe(pin)<0 ){ + return fsl_errno_to_rc(errno, FSL_RC_ERROR); + } + if( pipe(pout)<0 ){ + rc = fsl_errno_to_rc(errno, FSL_RC_ERROR); + close(pin[0]); + close(pin[1]); + return rc; + } + *pChildPid = fork(); + if( *pChildPid<0 ){ + rc = fsl_errno_to_rc(errno, FSL_RC_ERROR); + close(pin[0]); + close(pin[1]); + close(pout[0]); + close(pout[1]); + *pChildPid = 0; + return rc; + } + signal(SIGPIPE,SIG_IGN); + if( *pChildPid==0 ){ + int fd; + int nErr = 0; + /* This is the child process */ + close(0); + fd = dup(pout[0]); + if( fd!=0 ) nErr++; + close(pout[0]); + close(pout[1]); + close(1); + fd = dup(pin[1]); + if( fd!=1 ) nErr++; + close(pin[0]); + close(pin[1]); + execl("/bin/sh", "/bin/sh", "-c", zCmd, (char*)0); + return fsl_errno_to_rc(errno, FSL_RC_ERROR); + }else{ + /* This is the parent process */ + close(pin[1]); + *pfdIn = pin[0]; + close(pout[0]); + *ppOut = fdopen(pout[1], "w"); + return 0; + } +#endif +} + +/** + Close the connection to a child process previously created using + fsl_popen2(). Kill off the child process, then close the pipes. + */ +void fsl_pclose2(int fdIn, FILE *pOut, int childPid){ +#ifdef _WIN32 + /* Not implemented, yet */ + close(fdIn); + fclose(pOut); +#else + close(fdIn); + fclose(pOut); + kill(childPid, SIGINT); + while( waitpid(0, 0, WNOHANG)>0 ) {} +#endif +} + +#undef PTR_TO_INT +#undef INT_TO_PTR +/* end of file popen.c */ +/* start of file pq.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************ + This file houses the priority queue class. +*/ +#include + + +void fsl_pq_clear(fsl_pq *p){ + fsl_free(p->list); + *p = fsl_pq_empty; +} + +/* + Change the size of the queue so that it contains N slots +*/ +static int fsl_pq_resize(fsl_pq *p, fsl_size_t N){ + void * re = fsl_realloc(p->list, sizeof(fsl_pq_entry)*N); + if(!re) return FSL_RC_OOM; + else{ + p->list = (fsl_pq_entry*)re; + p->capacity = N; + return 0; + } +} + +/** + Insert element e into the queue. +*/ +int fsl_pq_insert(fsl_pq *p, fsl_id_t e, + double v, void *pData){ + fsl_size_t i, j; + if( p->used+1>p->capacity ){ + int const rc = fsl_pq_resize(p, p->used+5); + if(rc) return rc; + } + for(i=0; iused; ++i){ + if( p->list[i].priority>v ){ + for(j=p->used; j>i; --j){ + p->list[j] = p->list[j-1]; + } + break; + } + } + p->list[i].id = e; + p->list[i].data = pData; + p->list[i].priority = v; + ++p->used; + return 0; +} + +fsl_id_t fsl_pq_extract(fsl_pq *p, void **pp){ + fsl_id_t e, i; + if( p->used==0 ){ + if( pp ) *pp = 0; + return 0; + } + e = p->list[0].id; + if( pp ) *pp = p->list[0].data; + for(i=0; i<((fsl_id_t)p->used-1); ++i){ + p->list[i] = p->list[i+1]; + } + --p->used; + return e; +} +/* end of file pq.c */ +/* start of file repo.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/*********************************************************************** + This file implements most of the fsl_repo_xxx() APIs. +*/ +#include +#include /* memcpy() */ +#include /* time() */ +#include + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +/** + Calculate the youngest ancestor of the given blob.rid value that is a member of + branch zBranch. + + Returns the blob.id value of the matching record, 0 if not found, + or a negative value on error. + + Potential TODO: do we need this in the public API? +*/ +static fsl_id_t fsl_youngest_ancestor_in_branch(fsl_cx * f, fsl_id_t rid, + const char *zBranch){ + fsl_db * const db = fsl_needs_repo(f); + if(!db) return (fsl_id_t)-1; + return fsl_db_g_id(db, 0, + "WITH RECURSIVE " + " ancestor(rid, mtime) AS (" + " SELECT %"FSL_ID_T_PFMT", " + " mtime FROM event WHERE objid=%"FSL_ID_T_PFMT + " UNION " + " SELECT plink.pid, event.mtime" + " FROM ancestor, plink, event" + " WHERE plink.cid=ancestor.rid" + " AND event.objid=plink.pid" + " ORDER BY mtime DESC" + " )" + " SELECT ancestor.rid FROM ancestor" + " WHERE EXISTS(SELECT 1 FROM tagxref" + " WHERE tagid=%d AND tagxref.rid=ancestor.rid" + " AND value=%Q AND tagtype>0)" + " LIMIT 1", + rid, rid, FSL_TAGID_BRANCH, zBranch + ); +} + +/** + TODO: figure out if this needs to be in the public API and, if it does, + change its signature to: + + int fsl_branch_of_rid(fsl_cx *f, fsl_int_t rid, char **zOut ) + + So that we can distinguish "not found" from OOM errors. +*/ +static char * fsl_branch_of_rid(fsl_cx *f, fsl_int_t rid){ + char *zBr = 0; + fsl_db * const db = fsl_cx_db_repo(f); + fsl_stmt * st = 0; + int rc; + assert(db); + rc = fsl_db_prepare_cached(db, &st, + "SELECT value FROM tagxref " + "WHERE rid=? AND tagid=%d " + "AND tagtype>0 " + "/*%s()*/", FSL_TAGID_BRANCH,__func__); + if(rc) return 0; + rc = fsl_stmt_bind_id(st, 1, rid); + if(rc) goto end; + if( fsl_stmt_step(st)==FSL_RC_STEP_ROW ){ + zBr = fsl_strdup(fsl_stmt_g_text(st,0,0)); + if(!zBr) rc = FSL_RC_OOM; + } + end: + fsl_stmt_cached_yield(st); + if( !rc && zBr==0 ){ + zBr = fsl_config_get_text(f, FSL_CONFDB_REPO, "main-branch", 0); + } + return zBr; +} + +/** + morewt ==> most recent event with tag + + Comments from original fossil implementation: + + Find the RID of the most recent object with symbolic tag zTag and + having a type that matches zType. + + Return 0 if there are no matches. + + This is a tricky query to do efficiently. If the tag is very + common (ex: "trunk") then we want to use the query identified below + as Q1 - which searching the most recent EVENT table entries for the + most recent with the tag. But if the tag is relatively scarce + (anything other than "trunk", basically) then we want to do the + indexed search show below as Q2. +*/ +static fsl_id_t fsl_morewt(fsl_cx * const f, const char *zTag, fsl_satype_e type){ + char const * zType = fsl_satype_event_cstr(type); + return fsl_db_g_id(fsl_cx_db_repo(f), 0, + "SELECT objid FROM (" + /* Q1: Begin by looking for the tag in the 30 most recent events */ + "SELECT objid" + " FROM (SELECT * FROM event ORDER BY mtime DESC LIMIT 30) AS ex" + " WHERE type GLOB '%q'" + " AND EXISTS(SELECT 1 FROM tagxref, tag" + " WHERE tag.tagname='sym-%q'" + " AND tagxref.tagid=tag.tagid" + " AND tagxref.tagtype>0" + " AND tagxref.rid=ex.objid)" + " ORDER BY mtime DESC LIMIT 1" + ") UNION ALL SELECT * FROM (" + /* Q2: If the tag is not found in the 30 most recent events, then using + ** the tagxref table to index for the tag */ + "SELECT event.objid" + " FROM tag, tagxref, event" + " WHERE tag.tagname='sym-%q'" + " AND tagxref.tagid=tag.tagid" + " AND tagxref.tagtype>0" + " AND event.objid=tagxref.rid" + " AND event.type GLOB '%q'" + " ORDER BY event.mtime DESC LIMIT 1" + ") LIMIT 1;", + zType, zTag, zTag, zType + ); +} + +/** + Modes for fsl_start_of_branch(). +*/ +enum fsl_stobr_type { +/** + The check-in of the parent branch off of which + the branch containing RID originally diverged. +*/ +FSL_STOBR_ORIGIN = 0, +/** + The first check-in of the branch that contains RID. +*/ +FSL_STOBR_FIRST_CI = 1, +/** + The youngest ancestor of RID that is on the branch from which the + branch containing RID diverged. +*/ +FSL_STOBR_YOAN = 2 +}; + +/* +** Return the RID that is the "root" of the branch that contains +** check-in "rid". Details depending on eType. If not found, rid is +** returned. +*/ +static fsl_id_t fsl_start_of_branch(fsl_cx * f, fsl_id_t rid, + enum fsl_stobr_type eType){ + fsl_db * db; + fsl_stmt q = fsl_stmt_empty; + int rc; + fsl_id_t ans = rid; + char *zBr = fsl_branch_of_rid(f, rid); + if(!zBr){ + goto oom; + } + db = fsl_cx_db_repo(f); + assert(db); + rc = fsl_db_prepare(db, &q, + "SELECT pid, EXISTS(SELECT 1 FROM tagxref" + " WHERE tagid=%d AND tagtype>0" + " AND value=%Q AND rid=plink.pid)" + " FROM plink" + " WHERE cid=? AND isprim", + FSL_TAGID_BRANCH, zBr + ); + fsl_free(zBr); + zBr = 0; + if(rc){ + ans = -2; + fsl_cx_uplift_db_error(f, db); + MARKER(("Internal error: fsl_db_prepare() says: %s\n", fsl_rc_cstr(rc))); + goto end; + } + do{ + fsl_stmt_reset(&q); + fsl_stmt_bind_id(&q, 1, ans); + rc = fsl_stmt_step(&q); + if( rc!=FSL_RC_STEP_ROW ) break; + if( eType==FSL_STOBR_FIRST_CI && fsl_stmt_g_int32(&q,1)==0 ){ + break; + } + ans = fsl_stmt_g_id(&q, 0); + }while( fsl_stmt_g_int32(&q, 1)==1 && ans>0 ); + fsl_stmt_finalize(&q); + end: + if( ans>0 && eType==FSL_STOBR_YOAN ){ + zBr = fsl_branch_of_rid(f, ans); + if(zBr){ + ans = fsl_youngest_ancestor_in_branch(f, rid, zBr); + fsl_free(zBr); + }else{ + goto oom; + } + } + return ans; + oom: + fsl_cx_err_set(f, FSL_RC_OOM, NULL); + return -1; +} + +int fsl_sym_to_rid( fsl_cx * const f, char const * sym, + fsl_satype_e type, fsl_id_t * rv ){ + fsl_id_t rid = 0; + fsl_id_t vid; + fsl_size_t symLen; + /* fsl_int_t i; */ + fsl_db * dbR = fsl_cx_db_repo(f); + fsl_db * dbC = fsl_cx_db_ckout(f); + bool startOfBranch = 0; + int rc = 0; + + if(!f || !sym || !*sym || !rv) return FSL_RC_MISUSE; + else if(!dbR) return FSL_RC_NOT_A_REPO; + + if(FSL_SATYPE_BRANCH_START==type){ + /* The original implementation takes a (char const *) for the + type, and treats "b" (branch?) as a special case of + FSL_SATYPE_CHECKIN, resets the type to "ci", then sets + startOfBranch to 1. We introduced the FSL_SATYPE_BRANCH + pseudo-type for that purpose. That said: the original code + base does not, as of this writing (2021-02-15) appear to actually + use this feature anywhere. */ + type = FSL_SATYPE_CHECKIN; + startOfBranch = 1; + } + + /* special keyword: "tip" */ + if( 0==fsl_strcmp(sym,"tip") + && (FSL_SATYPE_ANY==type || FSL_SATYPE_CHECKIN==type)){ + rid = fsl_db_g_id(dbR, 0, + "SELECT objid FROM event" + " WHERE type='ci'" + " ORDER BY event.mtime DESC" + " LIMIT 1"); + if(rid>0) goto gotit; + } + /* special keywords: "prev", "previous", "current", and "next". + These require a checkout. + */ + vid = dbC ? f->ckout.rid : 0; + //MARKER(("has vid=%"FSL_ID_T_PFMT"\n", vid)); + if( vid>0){ + if( 0==fsl_strcmp(sym, "current") ){ + rid = vid; + } + else if( 0==fsl_strcmp(sym, "prev") + || 0==fsl_strcmp(sym, "previous") ){ + rid = fsl_db_g_id(dbR, 0, + "SELECT pid FROM plink WHERE " + "cid=%"FSL_ID_T_PFMT" AND isprim", + (fsl_id_t)vid); + } + else if( 0==fsl_strcmp(sym, "next") ){ + rid = fsl_db_g_id(dbR, 0, + "SELECT cid FROM plink WHERE " + "pid=%"FSL_ID_T_PFMT + " ORDER BY isprim DESC, mtime DESC", + (fsl_id_t)vid); + } + if(rid>0) goto gotit; + } + + /* Date and times */ + if( 0==memcmp(sym, "date:", 5) ){ + rid = fsl_db_g_id(dbR, 0, + "SELECT objid FROM event" + " WHERE mtime<=julianday(%Q,'utc')" + " AND type GLOB '%q'" + " ORDER BY mtime DESC LIMIT 1", + sym+5, fsl_satype_event_cstr(type)); + *rv = rid; + return 0; + } + if( fsl_str_is_date(sym) ){ + rid = fsl_db_g_id(dbR, 0, + "SELECT objid FROM event" + " WHERE mtime<=julianday(%Q,'utc')" + " AND type GLOB '%q'" + " ORDER BY mtime DESC LIMIT 1", + sym, fsl_satype_event_cstr(type)); + if(rid>0) goto gotit; + } + + /* Deprecated time formats elided: local:..., utc:... */ + + /* "tag:" + symbolic-name */ + if( memcmp(sym, "tag:", 4)==0 ){ + rid = fsl_morewt(f, sym+4, type); + if(rid>0 && startOfBranch){ + rid = fsl_start_of_branch(f, rid, FSL_STOBR_FIRST_CI); + } + goto gotit; + } + + /* root:TAG -> The origin of the branch */ + if( memcmp(sym, "root:", 5)==0 ){ + rc = fsl_sym_to_rid(f, sym+5, type, &rid); + if(!rc && rid>0){ + rid = fsl_start_of_branch(f, rid, FSL_STOBR_ORIGIN); + } + goto gotit; + } + + /* merge-in:TAG -> Most recent merge-in for the branch */ + if( memcmp(sym, "merge-in:", 9)==0 ){ + rc = fsl_sym_to_rid(f, sym+9, type, &rid); + if(!rc){ + rid = fsl_start_of_branch(f, rid, FSL_STOBR_YOAN); + } + goto gotit; + } + + symLen = fsl_strlen(sym); + /* SHA1/SHA3 hash or prefix */ + if( symLen>=4 + && symLen<=FSL_STRLEN_K256 + && fsl_validate16(sym, symLen) ){ + fsl_stmt q = fsl_stmt_empty; + char zUuid[FSL_STRLEN_K256+1]; + memcpy(zUuid, sym, symLen); + zUuid[symLen] = 0; + fsl_canonical16(zUuid, symLen); + rid = 0; + /* Reminder to self: caching these queries would be cool but it + can't work with the GLOBs. + */ + if( FSL_SATYPE_ANY==type ){ + fsl_db_prepare(dbR, &q, + "SELECT rid FROM blob WHERE uuid GLOB '%s*'", + zUuid); + }else{ + fsl_db_prepare(dbR, &q, + "SELECT blob.rid" + " FROM blob, event" + " WHERE blob.uuid GLOB '%s*'" + " AND event.objid=blob.rid" + " AND event.type GLOB '%q'", + zUuid, fsl_satype_event_cstr(type) ); + } + if( fsl_stmt_step(&q)==FSL_RC_STEP_ROW ){ + int64_t r64 = 0; + fsl_stmt_get_int64(&q, 0, &r64); + if( fsl_stmt_step(&q)==FSL_RC_STEP_ROW ) rid = -1 + /* Ambiguous results */ + ; + else rid = (fsl_id_t)r64; + } + fsl_stmt_finalize(&q); + if(rid<0){ + fsl_cx_err_set(f, FSL_RC_AMBIGUOUS, + "Symbolic name is ambiguous: %s", + sym); + } + goto gotit + /* None of the further checks against the sym can pass. */ + ; + } + + if(FSL_SATYPE_WIKI==type){ + rid = fsl_db_g_id(dbR, 0, + "SELECT event.objid, max(event.mtime)" + " FROM tag, tagxref, event" + " WHERE tag.tagname='sym-%q' " + " AND tagxref.tagid=tag.tagid AND tagxref.tagtype>0 " + " AND event.objid=tagxref.rid " + " AND event.type GLOB '%q'", + sym, fsl_satype_event_cstr(type) + ); + }else{ + rid = fsl_morewt(f, sym, type); + //MARKER(("morewt(%s,%s) == %d\n", sym, fsl_satype_cstr(type), (int)rid)); + } + + if( rid>0 ){ + if(startOfBranch) rid = fsl_start_of_branch(f, rid, + FSL_STOBR_FIRST_CI); + goto gotit; + } + + /* Undocumented: rid:### ==> rid */ + if(symLen>4 && 0==fsl_strncmp("rid:",sym,4)){ + int i; + char const * oldSym = sym; + sym += 4; + for(i=0; fsl_isdigit(sym[i]); i++){} + if( sym[i]==0 ){ + if( FSL_SATYPE_ANY==type ){ + rid = fsl_db_g_id(dbR, 0, + "SELECT rid" + " FROM blob" + " WHERE rid=%s", + sym); + }else{ + rid = fsl_db_g_id(dbR, 0, + "SELECT event.objid" + " FROM event" + " WHERE event.objid=%s" + " AND event.type GLOB '%q'", + sym, fsl_satype_event_cstr(type)); + } + if( rid>0 ) goto gotit; + } + sym = oldSym; + } + + gotit: + if(rid<=0){ + return f->error.code + ? f->error.code + : fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Could not resolve symbolic name " + "'%s' as artifact type '%s'.", + sym, fsl_satype_event_cstr(type) ); + } + assert(0==rc); + *rv = rid; + return rc; +} + +fsl_id_t fsl_uuid_to_rid2( fsl_cx * f, fsl_uuid_cstr uuid, + fsl_phantom_e mode ){ + if(!f) return -1; + else if(!fsl_is_uuid(uuid)){ + fsl_cx_err_set(f, FSL_RC_MISUSE, + "fsl_uuid_to_rid2() requires a " + "full UUID. Got: %s", uuid); + return -2; + }else{ + fsl_id_t rv; + rv = fsl_uuid_to_rid(f, uuid); + if((0==rv) && (FSL_PHANTOM_NONE!=mode) + && 0!=fsl_content_new(f, uuid, + (FSL_PHANTOM_PRIVATE==mode), + &rv)){ + assert(f->error.code); + rv = -3; + } + return rv; + } +} + +int fsl_sym_to_uuid( fsl_cx * f, char const * sym, fsl_satype_e type, + fsl_uuid_str * rv, fsl_id_t * rvId ){ + fsl_id_t rid = 0; + fsl_db * dbR = fsl_needs_repo(f); + fsl_uuid_str rvv = NULL; + int rc = dbR + ? fsl_sym_to_rid(f, sym, type, &rid) + : FSL_RC_NOT_A_REPO; + if(!rc){ + if(rvId) *rvId = rid; + rvv = fsl_rid_to_uuid(f, rid) + /* TODO: use a cached "exists" check if !rv, to avoid allocating + rvv if we don't need it. + */; + if(!rvv){ + if(!f->error.code){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Cannot find UUID for RID %"FSL_ID_T_PFMT".", + rid); + } + } + else if(rv){ + *rv = rvv; + }else{ + fsl_free( rvv ); + } + } + return rc; +} + +fsl_id_t fsl_uuid_to_rid( fsl_cx * const f, char const * uuid ){ + fsl_db * const db = fsl_needs_repo(f); + fsl_size_t const uuidLen = (uuid && db) ? fsl_strlen(uuid) : 0; + if(!uuid || !uuidLen) return -1; + else if(!db){ + /* f's error state has already been set */ + assert(FSL_RC_NOT_A_REPO == f->error.code); + return -2; + } + else if(!fsl_validate16(uuid, uuidLen)){ + fsl_cx_err_set(f, FSL_RC_RANGE, "Invalid UUID (prefix): %s", uuid); + return -3; + } + else if(uuidLen>FSL_STRLEN_K256){ + fsl_cx_err_set(f, FSL_RC_RANGE, "UUID is too long: %s", uuid); + return -4; + } + else { + fsl_id_t rid = -5; + fsl_stmt q = fsl_stmt_empty; + fsl_stmt * qS = NULL; + int rc; + rc = fsl_is_uuid_len((int)uuidLen) + /* Optimization for the common internally-used case. + + FIXME: there is an *astronomically small* chance of a prefix + collision on a v1-length uuidLen against a v2-length + blob.uuid value, leading to no match found for an existing v2 + uuid here. Like... a *REALLY* small chance. + */ + ? fsl_db_prepare_cached(db, &qS, + "SELECT rid FROM blob WHERE " + "uuid=? /*%s()*/",__func__) + : fsl_db_prepare(db, &q, + "SELECT rid FROM blob WHERE " + "uuid GLOB '%s*'", + uuid); + if(!rc){ + fsl_stmt * st = qS ? qS : &q; + if(qS){ + rc = fsl_stmt_bind_text(qS, 1, uuid, (fsl_int_t)uuidLen, 0); + } + if(!rc){ + rc = fsl_stmt_step(st); + switch(rc){ + case FSL_RC_STEP_ROW: + rc = 0; + rid = fsl_stmt_g_id(st, 0); + if(!qS){ + /* + Check for an ambiguous result. We don't need this for + the (qS==st) case because that one does an exact match + on a unique key. + */ + rc = fsl_stmt_step(st); + switch(rc){ + case FSL_RC_STEP_ROW: + rc = 0; + fsl_cx_err_set(f, FSL_RC_AMBIGUOUS, + "UUID prefix is ambiguous: %s", + uuid); + rid = -6; + break; + case FSL_RC_STEP_DONE: + /* Unambiguous UUID */ + rc = 0; + break; + default: + assert(st->db->error.code); + /* fall through and uplift the db error below... */ + } + } + break; + case FSL_RC_STEP_DONE: + /* No entry found */ + rid = 0; + rc = 0; + break; + default: + assert(st->db->error.code); + rid = -7; + break; + } + } + if(rc && db->error.code && !f->error.code){ + fsl_cx_uplift_db_error(f, db); + } + if(qS) fsl_stmt_cached_yield(qS); + else fsl_stmt_finalize(&q); + } + return rid; + } +} + +fsl_id_t fsl_repo_filename_fnid( fsl_cx * f, char const * fn ){ + fsl_id_t rv = 0; + int const rc = fsl_repo_filename_fnid2(f, fn, &rv, false); + return rv>=0 ? rv : (rc>0 ? -rc : rc); +} + +int fsl_repo_filename_fnid2( fsl_cx * f, char const * fn, fsl_id_t * rv, bool createNew ){ + fsl_db * db = fsl_cx_db_repo(f); + fsl_id_t fnid = 0; + fsl_stmt * qSel = NULL; + int rc; + assert(f); + assert(db); + assert(rv); + if(!fn || !fsl_is_simple_pathname(fn, 1)){ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "Filename is not a \"simple\" path: %s", + fn); + } + *rv = 0; + rc = fsl_db_prepare_cached(db, &qSel, + "SELECT fnid FROM filename " + "WHERE name=? " + "/*%s()*/",__func__); + if(rc){ + fsl_cx_uplift_db_error(f, db); + return rc; + } + rc = fsl_stmt_bind_text(qSel, 1, fn, -1, 0); + if(rc){ + fsl_stmt_cached_yield(qSel); + }else{ + rc = fsl_stmt_step(qSel); + if( FSL_RC_STEP_ROW == rc ){ + rc = 0; + fnid = fsl_stmt_g_id(qSel, 0); + assert(fnid>0); + }else if(FSL_RC_STEP_DONE == rc){ + rc = 0; + } + fsl_stmt_cached_yield(qSel); + if(!rc && (fnid==0) && createNew){ + fsl_stmt * qIns = NULL; + rc = fsl_db_prepare_cached(db, &qIns, + "INSERT INTO filename(name) " + "VALUES(?) /*%s()*/",__func__); + if(!rc){ + rc = fsl_stmt_bind_text(qIns, 1, fn, -1, 0); + if(!rc){ + rc = fsl_stmt_step(qIns); + if(FSL_RC_STEP_DONE==rc){ + rc = 0; + fnid = fsl_db_last_insert_id(db); + } + } + fsl_stmt_cached_yield(qIns); + } + } + } + if(!rc){ + assert(!createNew || (fnid>0)); + *rv = fnid; + }else if(db->error.code){ + fsl_cx_uplift_db_error(f, db); + } + return rc; +} + +int fsl_delta_src_id( fsl_cx * const f, fsl_id_t deltaRid, + fsl_id_t * rv ){ + fsl_db * const dbR = fsl_cx_db_repo(f); + if(!rv) return FSL_RC_MISUSE; + else if(deltaRid<=0) return FSL_RC_RANGE; + else if(!dbR) return FSL_RC_NOT_A_REPO; + else { + int rc; + fsl_stmt * q = NULL; + rc = fsl_db_prepare_cached(dbR, &q, + "SELECT srcid FROM delta " + "WHERE rid=? /*%s()*/",__func__); + if(!rc){ + rc = fsl_stmt_bind_id(q, 1, deltaRid); + if(!rc){ + if(FSL_RC_STEP_ROW==(rc=fsl_stmt_step(q))){ + rc = 0; + *rv = fsl_stmt_g_id(q, 0); + }else if(FSL_RC_STEP_DONE==rc){ + rc = 0; + *rv = 0; + } + } + fsl_stmt_cached_yield(q); + } + return rc; + } +} + + + +int fsl_repo_verify_before_commit( fsl_cx * f, fsl_id_t rid ){ + if(0){ + /* + v1 adds a commit hook here on the first entry, but it only + seems to ever use one commit hook, so the infrastructure seems + like overkill here. Thus this final verification is called from + the commit (that's where v1 calls the hook). + + If we eventually add commit hooks, this is the place to do it. + */ + } + assert( fsl_cx_db_repo(f)->beginCount > 0 ); + return rid>0 + ? fsl_id_bag_insert(&f->cache.toVerify, rid) + : FSL_RC_RANGE; +} + +void fsl_repo_verify_cancel( fsl_cx * f ){ + fsl_id_bag_clear(&f->cache.toVerify); +} + +int fsl_rid_to_uuid2(fsl_cx * const f, fsl_id_t rid, fsl_buffer *uuid){ + fsl_db * db = f ? fsl_cx_db_repo(f) : NULL; + if(!f || !db || (rid<=0)){ + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "fsl_rid_to_uuid2() requires " + "an opened repository and a " + "positive RID value. rid=%" FSL_ID_T_PFMT, + rid); + }else{ + fsl_stmt * st = NULL; + int rc; + fsl_buffer_reuse(uuid); + rc = fsl_db_prepare_cached(db, &st, + "SELECT uuid FROM blob " + "WHERE rid=? " + "/*%s()*/", __func__); + if(!rc){ + rc = fsl_stmt_bind_id(st, 1, rid); + if(!rc){ + rc = fsl_stmt_step(st); + if(FSL_RC_STEP_ROW==rc){ + fsl_size_t len = 0; + char const * x = fsl_stmt_g_text(st, 0, &len); + rc = fsl_buffer_append(uuid, x, (fsl_int_t)len); + }else if(FSL_RC_STEP_DONE){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "No blob found for rid %" FSL_ID_T_PFMT ".", + rid); + } + } + fsl_stmt_cached_yield(st); + if(rc && !f->error.code){ + if(db->error.code){ + fsl_cx_uplift_db_error(f, db); + }else{ + fsl_cx_err_set(f, rc, NULL); + } + } + } + return rc; + } +} + +fsl_uuid_str fsl_rid_to_uuid(fsl_cx * const f, fsl_id_t rid){ + fsl_buffer uuid = fsl_buffer_empty; + fsl_rid_to_uuid2(f, rid, &uuid); + return fsl_buffer_take(&uuid); +} + +fsl_uuid_str fsl_rid_to_artifact_uuid(fsl_cx * const f, fsl_id_t rid, fsl_satype_e type){ + fsl_db * db = f ? fsl_cx_db_repo(f) : NULL; + if(!f || !db || (rid<=0)) return NULL; + else{ + char * rv = NULL; + fsl_stmt * st = NULL; + int rc; + rc = fsl_db_prepare_cached(db, &st, + "SELECT uuid FROM blob " + "WHERE rid=?1 AND EXISTS " + "(SELECT 1 FROM event" + " WHERE event.objid=?1 " + " AND event.type GLOB %Q)" + "/*%s()*/", + fsl_satype_event_cstr(type), + __func__); + if(!rc){ + rc = fsl_stmt_bind_id(st, 1, rid); + if(!rc){ + rc = fsl_stmt_step(st); + if(FSL_RC_STEP_ROW==rc){ + fsl_size_t len = 0; + char const * x = fsl_stmt_g_text(st, 0, &len); + rv = x ? fsl_strndup(x, (fsl_int_t)len ) : NULL; + if(x && !rv){ + fsl_cx_err_set(f, FSL_RC_OOM, NULL); + } + }else if(FSL_RC_STEP_DONE){ + fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "No %s artifact found with rid %"FSL_ID_T_PFMT".", + fsl_satype_cstr(type), (fsl_id_t) rid); + } + } + fsl_stmt_cached_yield(st); + if(rc && !f->error.code){ + fsl_cx_uplift_db_error(f, db); + } + } + return rv; + } +} + + +/** + Load the record identified by rid. Make sure we can reproduce it + without error. + + Return non-0 and set f's error state if anything goes wrong. If + this procedure returns 0 it means that everything looks OK. + */ +static int fsl_repo_verify_rid(fsl_cx * f, fsl_id_t rid){ + fsl_uuid_str uuid = NULL; + fsl_buffer hash = fsl_buffer_empty; + fsl_buffer content = fsl_buffer_empty; + int rc; + fsl_db * db; + if( fsl_content_size(f, rid)<0 ){ + return 0 /* No way to verify phantoms */; + } + db = fsl_cx_db_repo(f); + assert(db); + uuid = fsl_rid_to_uuid(f, rid); + if(!uuid){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Could not find blob record for " + "rid #%"FSL_ID_T_PFMT".", + rid); + } + else{ + int const uuidLen = fsl_is_uuid(uuid); + if(!uuidLen){ + rc = fsl_cx_err_set(f, FSL_RC_RANGE, + "Invalid uuid for rid #%"FSL_ID_T_PFMT": %s", + (fsl_id_t)rid, uuid); + } + else if( 0==(rc=fsl_content_get(f, rid, &content)) ){ + /* This test can fail for artifacts which have an SHA1 hash in a + repo with an SHA3 policy. A test case from the main fossil + repo: c7dd1de9f9539a5a859c2b41fe4560604a774476 + + This test hashes it (in that repo) as SHA3. As a workaround, + if the hash is an SHA1 the we will temporarily force the hash + policy to SHA1, and similarly for SHA3. Lame, but nothing + better currently comes to mind. + + TODO: change the signature of fsl_cx_hash_buffer() to + optionally take a forced policy, or supply a similar function + which does what we're doing below. + */ + fsl_hashpolicy_e const oldHashP = f->cxConfig.hashPolicy; + f->cxConfig.hashPolicy = (uuidLen==FSL_STRLEN_SHA1) + ? FSL_HPOLICY_SHA1 : FSL_HPOLICY_SHA3; + rc = fsl_cx_hash_buffer(f, 0, &content, &hash); + f->cxConfig.hashPolicy = oldHashP; + if( !rc && 0!=fsl_uuidcmp(uuid, fsl_buffer_cstr(&hash)) ){ + rc = fsl_cx_err_set(f, FSL_RC_CONSISTENCY, + "Hash of rid %"FSL_ID_T_PFMT" (%b) " + "does not match its uuid (%s)", + (fsl_id_t)rid, &hash, uuid); + } + } + } + fsl_free(uuid); + fsl_buffer_clear(&hash); + fsl_buffer_clear(&content); + return rc; +} + + +int fsl_repo_verify_at_commit( fsl_cx * f ){ + fsl_id_t rid; + int rc = 0; + fsl_id_bag * bag = &f->cache.toVerify; + /* v1 does content_cache_clear() here. */ + f->cache.inFinalVerify = 1; + rid = fsl_id_bag_first(bag); + if(f->cxConfig.traceSql){ + fsl_db_exec(f->dbMain, + "SELECT 'Starting verify-at-commit.'"); + } + while( !rc && rid>0 ){ + rc = fsl_repo_verify_rid(f, rid); + if(!rc) rid = fsl_id_bag_next(bag, rid); + } + fsl_id_bag_clear(bag); + f->cache.inFinalVerify = 0; + if(rc && !f->error.code){ + fsl_cx_err_set(f, rc, + "Error #%d (%s) in fsl_repo_verify_at_commit()", + rc, fsl_rc_cstr(rc)); + } + return rc; +} + + +static int fsl_repo_create_default_users(fsl_db * db, char addOnlyUser, + char const * defaultUser ){ + int rc = fsl_db_exec(db, + "INSERT OR IGNORE INTO user(login, info) " + "VALUES(%Q,'')", defaultUser); + if(!rc){ + rc = fsl_db_exec(db, + "UPDATE user SET cap='s', pw=lower(hex(randomblob(3)))" + " WHERE login=%Q", defaultUser); + if( !rc && !addOnlyUser ){ + fsl_db_exec_multi(db, + "INSERT OR IGNORE INTO user(login,pw,cap,info)" + " VALUES('anonymous',hex(randomblob(8)),'hmncz'," + " 'Anon');" + "INSERT OR IGNORE INTO user(login,pw,cap,info)" + " VALUES('nobody','','gjor','Nobody');" + "INSERT OR IGNORE INTO user(login,pw,cap,info)" + " VALUES('developer','','dei','Dev');" + "INSERT OR IGNORE INTO user(login,pw,cap,info)" + " VALUES('reader','','kptw','Reader');" + ); + } + } + return rc; +} + +int fsl_repo_create(fsl_cx * f, fsl_repo_create_opt const * opt ){ + fsl_db * db = 0; + fsl_cx F = fsl_cx_empty /* used if !f */; + int rc = 0; + char const * userName = 0; + fsl_time_t const unixNow = (fsl_time_t)time(0); + char fileExists; + char inTrans = 0; + extern int fsl_cx_attach_role(fsl_cx * f, const char *zDbName, fsl_dbrole_e r) + /* Internal routine from fsl_cx.c */; + if(!opt || !opt->filename) return FSL_RC_MISUSE; + fileExists = 0 == fsl_file_access(opt->filename,0); + if(fileExists && !opt->allowOverwrite){ + return f + ? fsl_cx_err_set(f, FSL_RC_ALREADY_EXISTS, + "File already exists and " + "allowOverwrite is false: %s", + opt->filename) + : FSL_RC_ALREADY_EXISTS; + } + if(f){ + rc = fsl_ckout_close(f) + /* Will fail if a transaction is active! */; + switch(rc){ + case 0: + case FSL_RC_NOT_FOUND: + rc = 0; + break; + default: + return rc; + } + }else{ + f = &F; + rc = fsl_cx_init( &f, NULL ); + if(rc){ + fsl_cx_finalize(f); + return rc; + } + } + /* We probably should truncate/unlink the file here + before continuing, to ensure a clean slate. + */ + if(fileExists){ +#if 0 + FILE * file = fsl_fopen(opt->filename, "w"/*truncates it*/); + if(!file){ + rc = fsl_cx_err_set(f, fsl_errno_to_rc(errno, FSL_RC_IO), + "Cannot open '%s' for writing.", + opt->filename); + goto end2; + }else{ + fsl_fclose(file); + } +#else + rc = fsl_file_unlink(opt->filename); + if(rc){ + rc = fsl_cx_err_set(f, rc, "Cannot unlink existing repo file: %s", + opt->filename); + goto end2; + } +#endif + } + rc = fsl_cx_attach_role(f, opt->filename, FSL_DBROLE_REPO); + if(rc){ + goto end2; + } + db = fsl_cx_db(f); + if(!f->repo.user){ + f->repo.user = fsl_guess_user_name() + /* Ignore OOM error here - we'll use 'root' + by default (but if we're really OOM here then + the next op will fail). + */; + } + userName = opt->username; + + rc = fsl_db_transaction_begin(db); + if(rc) goto end1; + inTrans = 1; + /* Install the schemas... */ + rc = fsl_db_exec_multi(db, "%s; %s; %s; %s", + fsl_schema_repo1(), + fsl_schema_repo2(), + fsl_schema_ticket(), + fsl_schema_ticket_reports()); + if(rc) goto end1; + + if(1){ + /* + Set up server-code and project-code... + + in fossil this is optional, so we will presumably eventually + have to make it so here as well. Not yet sure where this routine + is used in fossil (i.e. whether the option is actually + exercised). + */ + rc = fsl_db_exec_multi(db, + "INSERT INTO repo.config (name,value,mtime) " + "VALUES ('server-code'," + "lower(hex(randomblob(20)))," + "%"PRIi64");" + "INSERT INTO repo.config (name,value,mtime) " + "VALUES ('project-code'," + "lower(hex(randomblob(20)))," + "%"PRIi64");", + (int64_t)unixNow, + (int64_t)unixNow + ); + if(rc) goto end1; + } + + + /* Set some config vars ... */ + { + fsl_stmt st = fsl_stmt_empty; + rc = fsl_db_prepare(db, &st, + "INSERT INTO repo.config (name,value,mtime) " + "VALUES (?,?,%"PRIi64")", + (int64_t)unixNow); + if(!rc){ + fsl_stmt_bind_int64(&st, 3, unixNow); +#define DBSET_STR(KEY,VAL) \ + fsl_stmt_bind_text(&st, 1, KEY, -1, 0); \ + fsl_stmt_bind_text(&st, 2, VAL, -1, 0); \ + fsl_stmt_step(&st); \ + fsl_stmt_reset(&st) + DBSET_STR("content-schema",FSL_CONTENT_SCHEMA); + DBSET_STR("aux-schema",FSL_AUX_SCHEMA); +#undef DBSET_STR + +#define DBSET_INT(KEY,VAL) \ + fsl_stmt_bind_text(&st, 1, KEY, -1, 0 ); \ + fsl_stmt_bind_int32(&st, 2, VAL); \ + fsl_stmt_step(&st); \ + fsl_stmt_reset(&st) + + DBSET_INT("autosync",1); + DBSET_INT("localauth",0); + DBSET_INT("timeline-plaintext", 1); + +#undef DBSET_INT + fsl_stmt_finalize(&st); + } + } + + rc = fsl_repo_create_default_users(db, 0, userName); + if(rc) goto end1; + + end1: + if(db->error.code && !f->error.code){ + rc = fsl_cx_uplift_db_error(f, db); + } + if(inTrans){ + if(!rc) rc = fsl_db_transaction_end(db, 0); + else fsl_db_transaction_end(db, 1); + inTrans = 0; + } + fsl_cx_close_dbs(f); + db = 0; + if(rc) goto end2; + + /** + In order for injection of the first commit to go through + cleanly (==without any ugly kludging of f->dbMain), we + need to now open the new db so that it gets connected + to f properly... + */ + rc = fsl_repo_open( f, opt->filename ); + if(rc) goto end2; + db = fsl_cx_db_repo(f); + assert(db); + assert(db == f->dbMain); + + if(!userName || !*userName){ + userName = fsl_cx_user_get(f); + if(!userName || !*userName){ + userName = "root" /* historical value */; + } + } + + /* + Copy config... + + This is done in the second phase because... + + "cannot ATTACH database within transaction" + + and installing the initial schemas outside a transaction is + horribly slow. + */ + if( opt->configRepo && *opt->configRepo ){ + bool inTrans2 = false; + char * inopConfig = fsl_config_inop_rhs(FSL_CONFIGSET_ALL); + char * inopDb = inopConfig ? fsl_db_setting_inop_rhs() : NULL; + if(!inopConfig || !inopDb){ + fsl_free(inopConfig); + rc = FSL_RC_OOM; + goto end2; + } + rc = fsl_db_attach(db, opt->configRepo, "settingSrc"); + if(rc){ + fsl_cx_uplift_db_error(f, db); + goto end2; + } + rc = fsl_db_transaction_begin(db); + if(rc){ + fsl_cx_uplift_db_error(f, db); + goto detach; + } + inTrans2 = 1; + /* + Copy all settings from the supplied template repository. + */ + rc = fsl_db_exec(db, + "INSERT OR REPLACE INTO repo.config" + " SELECT name,value,mtime FROM settingSrc.config" + " WHERE (name IN %s OR name IN %s)" + " AND name NOT GLOB 'project-*';", + inopConfig, inopDb); + if(rc) goto detach; + rc = fsl_db_exec(db, + "REPLACE INTO repo.reportfmt " + "SELECT * FROM settingSrc.reportfmt;"); + if(rc) goto detach; + + /* + Copy the user permissions, contact information, last modified + time, and photo for all the "system" users from the supplied + template repository into the one being setup. The other + columns are not copied because they contain security + information or other data specific to the other repository. + The list of columns copied by this SQL statement may need to be + revised in the future. + */ + rc = fsl_db_exec(db, "UPDATE repo.user SET" + " cap = (SELECT u2.cap FROM settingSrc.user u2" + " WHERE u2.login = user.login)," + " info = (SELECT u2.info FROM settingSrc.user u2" + " WHERE u2.login = user.login)," + " mtime = (SELECT u2.mtime FROM settingSrc.user u2" + " WHERE u2.login = user.login)," + " photo = (SELECT u2.photo FROM settingSrc.user u2" + " WHERE u2.login = user.login)" + " WHERE user.login IN ('anonymous','nobody','developer','reader');" + ); + + detach: + fsl_free(inopConfig); + fsl_free(inopDb); + if(inTrans2){ + if(!rc) rc = fsl_db_transaction_end(db,0); + else fsl_db_transaction_end(db,1); + } + fsl_db_detach(db, "settingSrc"); + if(rc) goto end2; + } + + if(opt->commitMessage && *opt->commitMessage){ + /* + Set up initial commit. Because of the historically empty P-card + on the first commit, we can't create that one using the fsl_deck + API unless we elide the P-card (not as fossil does) and insert + an empty R-card (as fossil does). We need one of P- or R-card to + unambiguously distinguish this MANIFEST from a CONTROL artifact. + + Reminder to self: fsl_deck has been adjusted to deal with the + initial-checkin(-like) case in the mean time. But this code + works, so no need to go changing it... + */ + fsl_deck d = fsl_deck_empty; + fsl_cx_err_reset(f); + fsl_deck_init(f, &d, FSL_SATYPE_CHECKIN); + rc = fsl_deck_C_set(&d, opt->commitMessage, -1); + if(!rc) rc = fsl_deck_D_set(&d, fsl_db_julian_now(db)); + if(!rc) rc = fsl_deck_R_set(&d, FSL_MD5_INITIAL_HASH); + if(!rc && opt->commitMessageMimetype && *opt->commitMessageMimetype){ + rc = fsl_deck_N_set(&d, opt->commitMessageMimetype, -1); + } + /* Reminder: setting tags in "wrong" (unsorted) order to + test/assert that the sorting gets done automatically. */ + if(!rc) rc = fsl_deck_T_add(&d, FSL_TAGTYPE_PROPAGATING, NULL, + "sym-trunk", NULL); + if(!rc) rc = fsl_deck_T_add(&d, FSL_TAGTYPE_PROPAGATING, NULL, + "branch", "trunk"); + if(!rc) rc =fsl_deck_U_set(&d, userName); + if(!rc){ + rc = fsl_deck_save(&d, 0); + } + fsl_deck_finalize(&d); + } + + end2: + if(f == &F){ + fsl_cx_finalize(f); + if(rc) fsl_file_unlink(opt->filename); + } + return rc; +} + +static int fsl_repo_dir_names_rid( fsl_cx * f, fsl_id_t rid, fsl_list * tgt, + bool addSlash){ + fsl_db * dbR = fsl_needs_repo(f); + fsl_deck D = fsl_deck_empty; + fsl_deck * d = &D; + int rc = 0; + fsl_stmt st = fsl_stmt_empty; + fsl_buffer tname = fsl_buffer_empty; + int count = 0; + fsl_card_F const * fc; + /* + This is a poor-man's impl. A more efficient one would calculate + the directory names without using the database. + */ + assert(rid>0); + assert(dbR); + rc = fsl_deck_load_rid( f, d, rid, FSL_SATYPE_CHECKIN); + if(rc){ + fsl_deck_clean(d); + return rc; + } + rc = fsl_buffer_appendf(&tname, + "tmp_filelist_for_rid_%d", + (int)rid); + if(rc) goto end; + rc = fsl_deck_F_rewind(d); + while( !rc && !(rc=fsl_deck_F_next(d, &fc)) && fc ){ + //if(!fc->name) continue; + assert(fc->name && *fc->name); + if(!st.stmt){ + rc = fsl_db_exec(dbR, "CREATE TEMP TABLE IF NOT EXISTS " + "%b(n TEXT UNIQUE ON CONFLICT IGNORE)", + &tname); + if(!rc){ + rc = fsl_db_prepare(dbR, &st, + "INSERT INTO %b(n) " + "VALUES(fsl_dirpart(?,%d))", + &tname, addSlash ? 1 : 0); + } + if(rc) goto end; + assert(st.stmt); + } + rc = fsl_stmt_bind_text(&st, 1, fc->name, -1, 0); + if(!rc){ + rc = fsl_stmt_step(&st); + if(FSL_RC_STEP_DONE==rc){ + ++count; + rc = 0; + } + } + fsl_stmt_reset(&st); + fc = 0; + } + + if(!rc && (count>0)){ + fsl_stmt_finalize(&st); + rc = fsl_db_prepare(dbR, &st, + "SELECT n FROM %b WHERE n " + "IS NOT NULL ORDER BY n %s", + &tname, + fsl_cx_filename_collation(f)); + while( !rc && (FSL_RC_STEP_ROW==(rc=fsl_stmt_step(&st))) ){ + fsl_size_t nLen = 0; + char const * name = fsl_stmt_g_text(&st, 0, &nLen); + rc = 0; + if(name){ + char * cp; + assert(nLen); + cp = fsl_strndup( name, (fsl_int_t)nLen ); + if(!cp){ + rc = FSL_RC_OOM; + break; + } + rc = fsl_list_append(tgt, cp); + if(rc){ + fsl_free(cp); + break; + } + } + } + if(FSL_RC_STEP_DONE==rc) rc = 0; + } + + end: + if(rc && !f->error.code && dbR->error.code){ + fsl_cx_uplift_db_error(f, dbR); + } + fsl_stmt_finalize(&st); + fsl_deck_clean(d); + if(tname.used){ + fsl_db_exec(dbR, "DROP TABLE IF EXISTS %b", &tname); + } + fsl_buffer_clear(&tname); + return rc; +} + +int fsl_repo_dir_names( fsl_cx * f, fsl_id_t rid, fsl_list * tgt, + bool addSlash ){ + fsl_db * db = (f && tgt) ? fsl_needs_repo(f) : NULL; + if(!f || !tgt) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + else { + int rc; + if(rid>=0){ + if(!rid){ + /* Dir list for current checkout version */ + if(f->ckout.rid>0){ + rid = f->ckout.rid; + }else{ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "The rid argument is 0 (indicating " + "the current checkout), but there is " + "no opened checkout."); + } + } + assert(rid>0); + rc = fsl_repo_dir_names_rid(f, rid, tgt, addSlash); + }else{ + /* Dir list across all versions */ + fsl_stmt s = fsl_stmt_empty; + rc = fsl_db_prepare(db, &s, + "SELECT DISTINCT(fsl_dirpart(name,%d)) dname " + "FROM filename WHERE dname IS NOT NULL " + "ORDER BY dname", addSlash ? 1 : 0); + if(rc){ + fsl_cx_uplift_db_error(f, db); + assert(!s.stmt); + return rc; + } + while( !rc && (FSL_RC_STEP_ROW==(rc=fsl_stmt_step(&s)))){ + fsl_size_t len = 0; + char const * col = fsl_stmt_g_text(&s, 0, &len); + char * cp = fsl_strndup( col, (fsl_int_t)len ); + if(!cp){ + rc = FSL_RC_OOM; + break; + } + rc = fsl_list_append(tgt, cp); + if(rc) fsl_free(cp); + } + if(FSL_RC_STEP_DONE==rc) rc = 0; + fsl_stmt_finalize(&s); + } + return rc; + } +} + +/* UNTESTED */ +char fsl_repo_is_readonly(fsl_cx const * f){ + if(!f || !f->dbMain) return 0; + else{ + int const roleId = f->ckout.db.dbh ? FSL_DBROLE_MAIN : FSL_DBROLE_REPO + /* If CKOUT is attached, it is the main DB and REPO is ATTACHed. */ + ; + char const * zRole = fsl_db_role_label(roleId); + assert(f->dbMain); + return sqlite3_db_readonly(f->dbMain->dbh, zRole) ? 1 : 0; + } +} + +int fsl_repo_record_filename(fsl_cx * f){ + fsl_buffer full = fsl_buffer_empty; + fsl_db * dbR = fsl_needs_repo(f); + fsl_db * dbC; + fsl_db * dbConf; + char const * zCDir; + char const * zName = dbR ? dbR->filename : NULL; + int rc; + if(!dbR) return FSL_RC_NOT_A_REPO; + assert(zName); + assert(f); + rc = fsl_file_canonical_name(zName, &full, 0); + if(rc){ + fsl_cx_err_set(f, rc, "Error %s canonicalizing filename: %s", zName); + goto end; + } + + /* + If global config is open, write the repo db's name to it. + */ + dbConf = fsl_cx_db_config(f); + if(dbConf){ + int const dbRole = (f->dbMain==&f->config.db) + ? FSL_DBROLE_MAIN : FSL_DBROLE_CONFIG; + rc = fsl_db_exec(dbConf, + "INSERT OR IGNORE INTO %s.global_config(name,value) " + "VALUES('repo:%q',1)", + fsl_db_role_label(dbRole), + fsl_buffer_cstr(&full)); + if(rc) goto end; + } + + dbC = fsl_cx_db_ckout(f); + if(dbC && (zCDir=f->ckout.dir)){ + /* If we have a checkout, update its repo's list of checkouts... */ + /* Assumption: if we have an opened checkout, dbR is ATTACHed with + the role REPO. */ + int ro; + assert(dbR); + ro = sqlite3_db_readonly(dbR->dbh, + fsl_db_role_label(FSL_DBROLE_REPO)); + assert(ro>=0); + if(!ro){ + fsl_buffer localRoot = fsl_buffer_empty; + rc = fsl_file_canonical_name(zCDir, &localRoot, 1); + if(0==rc){ + if(dbConf){ + /* + If global config is open, write the checkout db's name to it. + */ + int const dbRole = (f->dbMain==&f->config.db) + ? FSL_DBROLE_MAIN : FSL_DBROLE_CONFIG; + rc = fsl_db_exec(dbConf, + "REPLACE INTO INTO %s.global_config(name,value) " + "VALUES('ckout:%q',1)", + fsl_db_role_label(dbRole), + fsl_buffer_cstr(&localRoot)); + } + if(0==rc){ + /* We know that repo is ATTACHed to ckout here. */ + assert(dbR == dbC); + rc = fsl_db_exec(dbR, + "REPLACE INTO %s.config(name, value, mtime) " + "VALUES('ckout:%q', 1, now())", + fsl_db_role_label(FSL_DBROLE_REPO), + fsl_buffer_cstr(&localRoot)); + } + } + fsl_buffer_clear(&localRoot); + } + } + + end: + if(rc && !f->error.code && f->dbMain->error.code){ + fsl_cx_uplift_db_error(f, f->dbMain); + } + fsl_buffer_clear(&full); + return rc; + +} + +char fsl_rid_is_a_checkin(fsl_cx * f, fsl_id_t rid){ + fsl_db * db = f ? fsl_cx_db_repo(f) : NULL; + if(!db || (rid<0)) return 0; + else if(0==rid){ + /* Corner case: empty repo */ + return !fsl_db_exists(db, "SELECT 1 FROM blob WHERE rid>0"); + } + else{ + fsl_stmt * st = 0; + char rv = 0; + int rc = fsl_db_prepare_cached(db, &st, + "SELECT 1 FROM event WHERE " + "objid=? AND type='ci' " + "/*%s()*/",__func__); + if(!rc){ + rc = fsl_stmt_bind_id( st, 1, rid); + if(!rc){ + rc = fsl_stmt_step(st); + if(FSL_RC_STEP_ROW==rc){ + rv = 1; + } + } + fsl_stmt_cached_yield(st); + } + if(db->error.code){ + fsl_cx_uplift_db_error(f, db); + } + return rv; + } +} + +int fsl_repo_extract( fsl_cx * f, fsl_repo_extract_opt const * opt_ ){ + if(!f || !opt_->callback) return FSL_RC_MISUSE; + else if(!fsl_needs_repo(f)) return FSL_RC_NOT_A_REPO; + else if(opt_->checkinRid<=0){ + return fsl_cx_err_set(f, FSL_RC_RANGE, "RID must be positive."); + }else{ + int rc; + fsl_deck mf = fsl_deck_empty; + fsl_buffer * content = opt_->extractContent + ? &f->fileContent + : NULL; + fsl_id_t fid; + fsl_repo_extract_state xst = fsl_repo_extract_state_empty; + fsl_card_F const * fc = NULL; + fsl_repo_extract_opt const opt = *opt_ + /* Copy in case the caller modifies it via their callback. If we + find an interesting use for such modification then we can + remove this copy. */; + assert(!content || (!content->used && "Internal misuse of fsl_cx::fileContent")); + rc = fsl_deck_load_rid(f, &mf, opt.checkinRid, FSL_SATYPE_CHECKIN); + if(rc) goto end; + assert(mf.f==f); + xst.f = f; + xst.checkinRid = opt.checkinRid; + xst.callbackState = opt.callbackState; + xst.content = opt.extractContent ? content : NULL; + /* Calculate xst.count.fileCount... */ + assert(0==xst.count.fileCount); + if(mf.B.uuid){/*delta. The only way to count this reliably + is to walk though the whole card list. */ + rc = fsl_deck_F_rewind(&mf); + while( !rc && !(rc=fsl_deck_F_next(&mf, &fc)) && fc){ + ++xst.count.fileCount; + } + if(rc) goto end; + fc = NULL; + }else{ + xst.count.fileCount = mf.F.used; + } + assert(0==xst.count.fileNumber); + rc = fsl_deck_F_rewind(&mf); + while( !rc && !(rc=fsl_deck_F_next(&mf, &fc)) && fc){ + assert(fc->uuid + && "We shouldn't get F-card deletions via fsl_deck_F_next()"); + ++xst.count.fileNumber; + fid = fsl_uuid_to_rid(f, fc->uuid); + if(fid<0){ + assert(f->error.code); + rc = f->error.code; + }else if(!fid){ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "Could not resolve RID for UUID: %s", + fc->uuid); + }else if(opt.extractContent){ + fsl_buffer_reuse(content); + rc = fsl_content_get(f, fid, content); + } + if(!rc){ + /** Call the callback. */ + xst.fCard = fc; + assert(fid>0); + xst.content = content; + xst.fileRid = fid; + rc = opt.callback( &xst ); + if(FSL_RC_BREAK==rc){ + rc = 0; + break; + } + } + }/* for-each-F-card loop */ + end: + fsl_cx_content_buffer_yield(f); + fsl_deck_finalize(&mf); + return rc; + } +} + +int fsl_repo_import_blob( fsl_cx * f, fsl_input_f in, void * inState, + fsl_id_t * rid, fsl_uuid_str * uuid ){ + fsl_db * db = f ? fsl_needs_repo(f) : NULL; + if(!f || !in) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + else{ + int rc; + fsl_buffer buf = fsl_buffer_empty; + rc = fsl_buffer_fill_from(&buf, in, inState); + if(rc){ + rc = fsl_cx_err_set(f, rc, + "Error filling buffer from input source."); + }else{ + fsl_id_t theRid = 0; + rc = fsl_content_put_ex( f, &buf, NULL, 0, 0, 0, &theRid); + if(!rc){ + if(rid) *rid = theRid; + if(uuid){ + *uuid = fsl_rid_to_uuid(f, theRid); + if(!uuid) rc = FSL_RC_OOM; + } + } + } + fsl_buffer_clear(&buf); + return rc; + } +} + +int fsl_repo_import_buffer( fsl_cx * f, fsl_buffer const * in, + fsl_id_t * rid, fsl_uuid_str * uuid ){ + if(!f || !in) return FSL_RC_MISUSE; + else{ + /* Workaround: input ptr is const and input needs to modify + (only) the cursor. So we'll cheat rather than require a non-const + input... + */ + fsl_buffer cursorKludge = *in; + cursorKludge.cursor = 0; + int const rc = fsl_repo_import_blob(f, fsl_input_f_buffer, &cursorKludge, + rid, uuid ); + assert(cursorKludge.mem == in->mem); + return rc; + } +} + + +int fsl_repo_blob_lookup( fsl_cx * f, fsl_buffer const * src, fsl_id_t * ridOut, + fsl_uuid_str * hashOut ){ + int rc; + fsl_buffer hash_ = fsl_buffer_empty; + fsl_buffer * hash; + fsl_id_t rid = 0; + if(!fsl_cx_db_repo(f)) return FSL_RC_NOT_A_REPO; + hash = hashOut ? &hash_ : fsl_cx_scratchpad(f); + /* First check the auxiliary hash to see if there is already an artifact + that uses the auxiliary hash name */ + rc = fsl_cx_hash_buffer(f, true, src, hash); + if(FSL_RC_UNSUPPORTED==rc){ + // The auxiliary hash option is incompatible with our hash policy. + rc = 0; + } + else if(rc) goto end; + rid = hash->used ? fsl_uuid_to_rid(f, fsl_buffer_cstr(hash)) : 0; + if(!rid){ + /* No existing artifact with the auxiliary hash name. Therefore, use + the primary hash name. */ + fsl_buffer_reuse(hash); + rc = fsl_cx_hash_buffer(f, false, src, hash); + if(rc) goto end; + rid = fsl_uuid_to_rid(f, fsl_buffer_cstr(hash)); + if(!rid){ + rc = FSL_RC_NOT_FOUND; + } + if(rid<0){ + rc = f->error.code; + } + } + end: + if(!rc || rc==FSL_RC_NOT_FOUND){ + if(hashOut){ + assert(hash == &hash_); + *hashOut = fsl_buffer_take(hash)/*transfer*/; + } + } + if(!rc && ridOut){ + *ridOut = rid; + } + if(hash == &hash_){ + fsl_buffer_clear(hash); + }else{ + assert(!hash_.mem); + fsl_cx_scratchpad_yield(f, hash); + } + return rc; +} + +int fsl_repo_fingerprint_search( fsl_cx *f, fsl_id_t rcvid, char ** zOut ){ + int rc = 0; + fsl_db * const db = fsl_needs_repo(f); + if(!db) return FSL_RC_NOT_A_REPO; + fsl_buffer * const sql = fsl_cx_scratchpad(f); + fsl_stmt q = fsl_stmt_empty; + int version = 1 /* Fingerprint version to check: 0 or 1 */; + try_again: + /* + * We check both v1 and v0 fingerprints, in that order. From Fossil + * db.c: + * + * The original fingerprint algorithm used "quote(mtime)". But this could + * give slightly different answers depending on how the floating-point + * hardware is configured. For example, it gave different answers on + * native Linux versus running under valgrind. + */ + if(0==version){ + fsl_stmt_finalize(&q); + rc = fsl_buffer_append(sql, + "SELECT rcvid, quote(uid), quote(mtime), " + "quote(nonce), quote(ipaddr) " + "FROM rcvfrom ", -1); + }else{ + assert(1==version); + rc = fsl_buffer_append(sql, + "SELECT rcvid, quote(uid), datetime(mtime), " + "quote(nonce), quote(ipaddr) " + "FROM rcvfrom ", -1); + } + if(rc) goto end; + rc = (rcvid>0) + ? fsl_buffer_appendf(sql, "WHERE rcvid=%" FSL_ID_T_PFMT, rcvid) + : fsl_buffer_append(sql, "ORDER BY rcvid DESC LIMIT 1", -1); + if(rc) goto end; + rc = fsl_db_prepare(db, &q, "%b", sql); + if(rc) goto end; + rc = fsl_stmt_step(&q); + switch(rc){ + case FSL_RC_STEP_ROW:{ + fsl_md5_cx hash = fsl_md5_cx_empty; + fsl_size_t len = 0; + fsl_id_t const rvid = fsl_stmt_g_id(&q, 0); + unsigned char digest[16] = {0}; + char hex[FSL_STRLEN_MD5+1] = {0}; + for(int i = 1; i <= 4; ++i){ + char const * z = fsl_stmt_g_text(&q, i, &len); + fsl_md5_update(&hash, z, len); + } + fsl_md5_final(&hash, digest); + fsl_md5_digest_to_base16(digest, hex); + *zOut = fsl_mprintf("%" FSL_ID_T_PFMT "/%s", rvid, hex); + rc = *zOut ? 0 : FSL_RC_OOM; + break; + } + case FSL_RC_STEP_DONE: + if(1==version){ + version = 0; + fsl_buffer_reuse(sql); + goto try_again; + } + rc = FSL_RC_NOT_FOUND; + break; + default: + rc = fsl_cx_uplift_db_error2(f, db, rc); + break; + } + end: + fsl_cx_scratchpad_yield(f, sql); + fsl_stmt_finalize(&q); + return rc; +} + +int fsl_repo_manifest_write(fsl_cx *f, + fsl_id_t manifestRid, + fsl_buffer * const pManifest, + fsl_buffer * const pHash, + fsl_buffer * const pTags) { + fsl_db * const db = fsl_needs_repo(f); + if(!db) return FSL_RC_NOT_A_REPO; + if(manifestRid<=0){ + manifestRid = f->ckout.rid; + if(manifestRid<=0){ + return fsl_cx_err_set(f, 0==f->ckout.rid + ? FSL_RC_RANGE + : FSL_RC_NOT_A_CKOUT, + "%s(): no checkin version was specified " + "and %s.", __func__, + 0==f->ckout.rid + ? "checkout has no version" + : "no checkout is opened"); + } + } + int rc = 0; + char * str = 0; + fsl_uuid_str ridHash = 0; + fsl_buffer * bHash = 0; + assert(manifestRid>0); + + if(pManifest){ + fsl_buffer_reuse(pManifest); + rc = fsl_content_get(f, manifestRid, pManifest); + if(rc) goto end; + } + if(pHash){ + if(f->ckout.rid!=manifestRid){ + bHash = fsl_cx_scratchpad(f); + rc = fsl_rid_to_uuid2(f, manifestRid, bHash); + if(rc) goto end; + ridHash = (char *)bHash->mem; + }else{ + ridHash = f->ckout.uuid; + } + assert(ridHash); + rc = fsl_buffer_append(pHash, ridHash, -1); + if(!rc) rc = fsl_buffer_append(pHash, "\n", 1); + if(rc) goto end; + } + if(pTags){ + fsl_stmt q = fsl_stmt_empty; + fsl_db * const db = fsl_cx_db_repo(f); + assert(db && "We can't have a checkout w/o a repo."); + str = fsl_db_g_text(db, NULL, "SELECT VALUE FROM tagxref " + "WHERE rid=%" FSL_ID_T_PFMT + " AND tagid=%d /*%s()*/", + f->ckout.rid, FSL_TAGID_BRANCH, __func__); + rc = fsl_buffer_appendf(pTags, "branch %z\n", str); + str = 0; + if(rc) goto end; + rc = fsl_db_prepare(db, &q, + "SELECT substr(tagname, 5)" + " FROM tagxref, tag" + " WHERE tagxref.rid=%" FSL_ID_T_PFMT + " AND tagxref.tagtype>0" + " AND tag.tagid=tagxref.tagid" + " AND tag.tagname GLOB 'sym-*'" + " /*%s()*/", + f->ckout.rid, __func__); + if(rc) goto end; + while( FSL_RC_STEP_ROW==fsl_stmt_step(&q) ){ + const char *zName = fsl_stmt_g_text(&q, 0, NULL); + rc = fsl_buffer_appendf(pTags, "tag %s\n", zName); + if(rc) break; + } + fsl_stmt_finalize(&q); + } + end: + if(bHash){ + fsl_cx_scratchpad_yield(f, bHash); + } + return rc; +} + +/** + NOT YET IMPLEMENTED. (We have the infrastructure, just need to glue + it together.) + + Re-crosslinks all artifacts of the given type (or all artifacts if + the 2nd argument is FSL_SATYPE_ANY). This is an expensive + operation, involving dropping the contents of any corresponding + auxiliary tables, loading and parsing the appropriate artifacts, + and re-creating the auxiliary tables. + + TODO: add a way for callers to get some sort of progress feedback + and abort the process by returning non-0 from that handler. We can + possibly do that via defining an internal-use crosslink listener + which carries more state, e.g. for calculating completion progress. +*/ +//FSL_EXPORT int fsl_repo_relink_artifacts(fsl_cx *f, void * someOptionsType); + + +#undef MARKER +/* end of file repo.c */ +/* start of file schema.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 Stephan Beal (https://wanderinghorse.net). + + + + This program is free software; you can redistribute it and/or + modify it under the terms of the Simplified BSD License (also + known as the "2-Clause License" or "FreeBSD License".) + + This program is distributed in the hope that it will be useful, + but without any warranty; without even the implied warranty of + merchantability or fitness for a particular purpose. + + ***************************************************************************** + This file implements schema-related parts of the library. +*/ +#include + + +char const * fsl_schema_ckout(){ + extern char const * fsl_schema_ckout_cstr; + return fsl_schema_ckout_cstr; +} + +char const * fsl_schema_repo2(){ + extern char const * fsl_schema_repo2_cstr; + return fsl_schema_repo2_cstr; +} + +char const * fsl_schema_repo1(){ + extern char const * fsl_schema_repo1_cstr; + return fsl_schema_repo1_cstr; +} + +char const * fsl_schema_config(){ + extern char const * fsl_schema_config_cstr; + return fsl_schema_config_cstr; +} + +char const * fsl_schema_ticket_reports(){ + extern char const * fsl_schema_ticket_reports_cstr; + return fsl_schema_ticket_reports_cstr; +} + +char const * fsl_schema_ticket(){ + extern char const * fsl_schema_ticket_cstr; + return fsl_schema_ticket_cstr; +} + +char const * fsl_schema_forum(){ + extern char const * fsl_schema_forum_cstr; + return fsl_schema_forum_cstr; +} +/* end of file schema.c */ +/* start of file search.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ + +/* + This file houses some FTS-search-related functionality. + + libfossil does not aim to reproduce all search functionality + provided by fossil. Initially, at least, the only planned feature + parity is that of updating the search index as content is + added/updated. +*/ +#include +#include /* memcmp() */ + +/* Only for debugging */ +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +static bool fsl_search_ndx_exists(fsl_cx * const f){ + if(f->cache.searchIndexExists<0){ + f->cache.searchIndexExists = fsl_db_table_exists(fsl_cx_db_repo(f), + FSL_DBROLE_REPO, + "ftsdocs") + ? 1 : 0; + } + return f->cache.searchIndexExists ? true : false; +} + +static char fsl_satype_letter(fsl_satype_e t){ + switch(t){ + case FSL_SATYPE_CHECKIN: return 'c'; + case FSL_SATYPE_WIKI: return 'w'; + case FSL_SATYPE_TICKET: return 't'; + case FSL_SATYPE_FORUMPOST: return 'f'; + case FSL_SATYPE_TECHNOTE: return 'e'; + default: + assert(!"Internal misuse of fsl_satype_letter()"); + return 0; + } +} + +int fsl_search_doc_touch(fsl_cx * const f, fsl_satype_e saType, + fsl_id_t rid, const char * docName){ + if(!fsl_search_ndx_exists(f) || fsl_content_is_private(f, rid)) return 0; + char zType[2] = {0,0}; + zType[0] = fsl_satype_letter(saType); +#if 0 + /* See MARKER() call in the #else block */ + assert(*zType); + return *zType ? 0 : FSL_RC_MISUSE; +#else + /* Reminder: fossil(1) does some once-per-connection init here which + installs UDFs used by the search process. Those will be significant + for us if we add the search features to the library. */ + assert(zType[0] && "Misuse of fsl_search_doc_touch()'s 2nd parameter."); + fsl_db * const db = fsl_cx_db_repo(f); + int rc = fsl_db_exec(db, + "DELETE FROM ftsidx WHERE docid IN" + " (SELECT rowid FROM ftsdocs WHERE type=%Q AND rid=%"FSL_ID_T_PFMT" AND idxed)", + zType, rid ); + if(rc){ + // For reasons i don't understand, this query fails with "SQL logic error" + // when run from here, but succeeds fine in fossil and fossil's SQL shell. + /*MARKER(("type=%s rid=%d rc=%s\n",zType, (int)rid, fsl_rc_cstr(rc)));*/ + goto end; + } + rc = fsl_db_exec(db, + "REPLACE INTO ftsdocs(type,rid,name,idxed)" + " VALUES(%Q,%"FSL_ID_T_PFMT",%Q,0)", + zType, rid, docName ); + if(rc) goto end; + if( FSL_SATYPE_WIKI==saType || FSL_SATYPE_TECHNOTE==saType ){ + rc = fsl_db_exec(db, + "DELETE FROM ftsidx WHERE docid IN" + " (SELECT rowid FROM ftsdocs WHERE type=%Q AND name=%Q AND idxed)", + zType, docName ); + if(!rc) rc = fsl_db_exec(db, + "DELETE FROM ftsdocs WHERE type=%Q AND name=%Q AND rid!=%"FSL_ID_T_PFMT, + zType, docName, rid ); + } + /* All forum posts are always indexed */ + end: + return rc; +#endif +} + +#undef MARKER +/* end of file search.c */ +/* start of file sha1.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#include +#include /* strlen() */ +#include /* NULL on linux */ + +#include + +#if FSL_SHA1_HARDENED +/*************** File: lib/sha1.c ****************/ +/*** +* Copyright 2017 Marc Stevens , Dan Shumow (danshu@microsoft.com) +* Distributed under the MIT Software License. +* See accompanying file LICENSE.txt or copy at +* https://opensource.org/licenses/MIT +***/ +/*************** File: LICENSE.txt ***************/ +/* +** MIT License +** +** Copyright (c) 2017: +** Marc Stevens +** Cryptology Group +** Centrum Wiskunde & Informatica +** P.O. Box 94079, 1090 GB Amsterdam, Netherlands +** marc@marc-stevens.nl +** +** Dan Shumow +** Microsoft Research +** danshu@microsoft.com +** +** Permission is hereby granted, free of charge, to any person obtaining a copy +** of this software and associated documentation files (the "Software"), to deal +** in the Software without restriction, including without limitation the rights +** to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +** copies of the Software, and to permit persons to whom the Software is +** furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in all +** copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +** OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +** SOFTWARE. +*/ + +#include +#include + +#define DVMASKSIZE 1 +typedef struct { int dvType; int dvK; int dvB; int testt; int maski; int maskb; uint32_t dm[80]; } dv_info_t; +#define DOSTORESTATE58 +#define DOSTORESTATE65 +typedef void(*sha1_recompression_type)(uint32_t*, uint32_t*, const uint32_t*, const uint32_t*); +static void sha1_message_expansion(uint32_t W[80]); +static void sha1_compression_W(uint32_t ihv[5], const uint32_t W[80]); +static void sha1_compression_states(uint32_t ihv[5], const uint32_t W[80], uint32_t states[80][5]); + +/******************** File: lib/ubc_check.c **************************/ +/*** +* Copyright 2017 Marc Stevens , Dan Shumow +* Distributed under the MIT Software License. +* See accompanying file LICENSE.txt or copy at +* https://opensource.org/licenses/MIT +***/ + +/* +** this file was generated by the 'parse_bitrel' program in the tools section +** using the data files from directory 'tools/data/3565' +** +** sha1_dvs contains a list of SHA-1 Disturbance Vectors (DV) to check +** dvType, dvK and dvB define the DV: I(K,B) or II(K,B) (see the paper) +** dm[80] is the expanded message block XOR-difference defined by the DV +** testt is the step to do the recompression from for collision detection +** maski and maskb define the bit to check for each DV in the dvmask returned by ubc_check +** +** ubc_check takes as input an expanded message block and verifies the unavoidable bitconditions for all listed DVs +** it returns a dvmask where each bit belonging to a DV is set if all unavoidable bitconditions for that DV have been met +** thus one needs to do the recompression check for each DV that has its bit set +** +** ubc_check is programmatically generated and the unavoidable bitconditions have been hardcoded +** a directly verifiable version named ubc_check_verify can be found in ubc_check_verify.c +** ubc_check has been verified against ubc_check_verify using the 'ubc_check_test' program in the tools section +*/ + +static const uint32_t DV_I_43_0_bit = (uint32_t)(1) << 0; +static const uint32_t DV_I_44_0_bit = (uint32_t)(1) << 1; +static const uint32_t DV_I_45_0_bit = (uint32_t)(1) << 2; +static const uint32_t DV_I_46_0_bit = (uint32_t)(1) << 3; +static const uint32_t DV_I_46_2_bit = (uint32_t)(1) << 4; +static const uint32_t DV_I_47_0_bit = (uint32_t)(1) << 5; +static const uint32_t DV_I_47_2_bit = (uint32_t)(1) << 6; +static const uint32_t DV_I_48_0_bit = (uint32_t)(1) << 7; +static const uint32_t DV_I_48_2_bit = (uint32_t)(1) << 8; +static const uint32_t DV_I_49_0_bit = (uint32_t)(1) << 9; +static const uint32_t DV_I_49_2_bit = (uint32_t)(1) << 10; +static const uint32_t DV_I_50_0_bit = (uint32_t)(1) << 11; +static const uint32_t DV_I_50_2_bit = (uint32_t)(1) << 12; +static const uint32_t DV_I_51_0_bit = (uint32_t)(1) << 13; +static const uint32_t DV_I_51_2_bit = (uint32_t)(1) << 14; +static const uint32_t DV_I_52_0_bit = (uint32_t)(1) << 15; +static const uint32_t DV_II_45_0_bit = (uint32_t)(1) << 16; +static const uint32_t DV_II_46_0_bit = (uint32_t)(1) << 17; +static const uint32_t DV_II_46_2_bit = (uint32_t)(1) << 18; +static const uint32_t DV_II_47_0_bit = (uint32_t)(1) << 19; +static const uint32_t DV_II_48_0_bit = (uint32_t)(1) << 20; +static const uint32_t DV_II_49_0_bit = (uint32_t)(1) << 21; +static const uint32_t DV_II_49_2_bit = (uint32_t)(1) << 22; +static const uint32_t DV_II_50_0_bit = (uint32_t)(1) << 23; +static const uint32_t DV_II_50_2_bit = (uint32_t)(1) << 24; +static const uint32_t DV_II_51_0_bit = (uint32_t)(1) << 25; +static const uint32_t DV_II_51_2_bit = (uint32_t)(1) << 26; +static const uint32_t DV_II_52_0_bit = (uint32_t)(1) << 27; +static const uint32_t DV_II_53_0_bit = (uint32_t)(1) << 28; +static const uint32_t DV_II_54_0_bit = (uint32_t)(1) << 29; +static const uint32_t DV_II_55_0_bit = (uint32_t)(1) << 30; +static const uint32_t DV_II_56_0_bit = (uint32_t)(1) << 31; + +dv_info_t sha1_dvs[] = +{ + {1,43,0,58,0,0, { 0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408,0x800000e6,0x8000004c,0x00000803,0x80000161,0x80000599 } } +, {1,44,0,58,0,1, { 0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408,0x800000e6,0x8000004c,0x00000803,0x80000161 } } +, {1,45,0,58,0,2, { 0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408,0x800000e6,0x8000004c,0x00000803 } } +, {1,46,0,58,0,3, { 0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408,0x800000e6,0x8000004c } } +, {1,46,2,58,0,4, { 0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a,0x00000060,0x00000590,0x00001020,0x0000039a,0x00000132 } } +, {1,47,0,58,0,5, { 0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408,0x800000e6 } } +, {1,47,2,58,0,6, { 0x20000043,0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a,0x00000060,0x00000590,0x00001020,0x0000039a } } +, {1,48,0,58,0,7, { 0xb800000a,0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164,0x00000408 } } +, {1,48,2,58,0,8, { 0xe000002a,0x20000043,0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a,0x00000060,0x00000590,0x00001020 } } +, {1,49,0,58,0,9, { 0x18000000,0xb800000a,0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018,0x00000164 } } +, {1,49,2,58,0,10, { 0x60000000,0xe000002a,0x20000043,0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a,0x00000060,0x00000590 } } +, {1,50,0,65,0,11, { 0x0800000c,0x18000000,0xb800000a,0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202,0x00000018 } } +, {1,50,2,65,0,12, { 0x20000030,0x60000000,0xe000002a,0x20000043,0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a,0x00000060 } } +, {1,51,0,65,0,13, { 0xe8000000,0x0800000c,0x18000000,0xb800000a,0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012,0x80000202 } } +, {1,51,2,65,0,14, { 0xa0000003,0x20000030,0x60000000,0xe000002a,0x20000043,0xb0000040,0xd0000053,0xd0000022,0x20000000,0x60000032,0x60000043,0x20000040,0xe0000042,0x60000002,0x80000001,0x00000020,0x00000003,0x40000052,0x40000040,0xe0000052,0xa0000000,0x80000040,0x20000001,0x20000060,0x80000001,0x40000042,0xc0000043,0x40000022,0x00000003,0x40000042,0xc0000043,0xc0000022,0x00000001,0x40000002,0xc0000043,0x40000062,0x80000001,0x40000042,0x40000042,0x40000002,0x00000002,0x00000040,0x80000002,0x80000000,0x80000002,0x80000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000000,0x00000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000101,0x00000009,0x00000012,0x00000202,0x0000001a,0x00000124,0x0000040c,0x00000026,0x0000004a,0x0000080a } } +, {1,52,0,65,0,15, { 0x04000010,0xe8000000,0x0800000c,0x18000000,0xb800000a,0xc8000010,0x2c000010,0xf4000014,0xb4000008,0x08000000,0x9800000c,0xd8000010,0x08000010,0xb8000010,0x98000000,0x60000000,0x00000008,0xc0000000,0x90000014,0x10000010,0xb8000014,0x28000000,0x20000010,0x48000000,0x08000018,0x60000000,0x90000010,0xf0000010,0x90000008,0xc0000000,0x90000010,0xf0000010,0xb0000008,0x40000000,0x90000000,0xf0000010,0x90000018,0x60000000,0x90000010,0x90000010,0x90000000,0x80000000,0x00000010,0xa0000000,0x20000000,0xa0000000,0x20000010,0x00000000,0x20000010,0x20000000,0x00000010,0x20000000,0x00000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000040,0x40000002,0x80000004,0x80000080,0x80000006,0x00000049,0x00000103,0x80000009,0x80000012 } } +, {2,45,0,58,0,16, { 0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b,0x8000016d,0x8000041a,0x000002e4,0x80000054,0x00000967 } } +, {2,46,0,58,0,17, { 0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b,0x8000016d,0x8000041a,0x000002e4,0x80000054 } } +, {2,46,2,58,0,18, { 0x90000070,0xb0000053,0x30000008,0x00000043,0xd0000072,0xb0000010,0xf0000062,0xc0000042,0x00000030,0xe0000042,0x20000060,0xe0000041,0x20000050,0xc0000041,0xe0000072,0xa0000003,0xc0000012,0x60000041,0xc0000032,0x20000001,0xc0000002,0xe0000042,0x60000042,0x80000002,0x00000000,0x00000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000001,0x00000060,0x80000003,0x40000002,0xc0000040,0xc0000002,0x80000000,0x80000000,0x80000002,0x00000040,0x00000002,0x80000000,0x80000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000105,0x00000089,0x00000016,0x0000020b,0x0000011b,0x0000012d,0x0000041e,0x00000224,0x00000050,0x0000092e,0x0000046c,0x000005b6,0x0000106a,0x00000b90,0x00000152 } } +, {2,47,0,58,0,19, { 0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b,0x8000016d,0x8000041a,0x000002e4 } } +, {2,48,0,58,0,20, { 0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b,0x8000016d,0x8000041a } } +, {2,49,0,58,0,21, { 0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b,0x8000016d } } +, {2,49,2,58,0,22, { 0xf0000010,0xf000006a,0x80000040,0x90000070,0xb0000053,0x30000008,0x00000043,0xd0000072,0xb0000010,0xf0000062,0xc0000042,0x00000030,0xe0000042,0x20000060,0xe0000041,0x20000050,0xc0000041,0xe0000072,0xa0000003,0xc0000012,0x60000041,0xc0000032,0x20000001,0xc0000002,0xe0000042,0x60000042,0x80000002,0x00000000,0x00000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000001,0x00000060,0x80000003,0x40000002,0xc0000040,0xc0000002,0x80000000,0x80000000,0x80000002,0x00000040,0x00000002,0x80000000,0x80000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000105,0x00000089,0x00000016,0x0000020b,0x0000011b,0x0000012d,0x0000041e,0x00000224,0x00000050,0x0000092e,0x0000046c,0x000005b6 } } +, {2,50,0,65,0,23, { 0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b,0x0000011b } } +, {2,50,2,65,0,24, { 0xd0000072,0xf0000010,0xf000006a,0x80000040,0x90000070,0xb0000053,0x30000008,0x00000043,0xd0000072,0xb0000010,0xf0000062,0xc0000042,0x00000030,0xe0000042,0x20000060,0xe0000041,0x20000050,0xc0000041,0xe0000072,0xa0000003,0xc0000012,0x60000041,0xc0000032,0x20000001,0xc0000002,0xe0000042,0x60000042,0x80000002,0x00000000,0x00000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000001,0x00000060,0x80000003,0x40000002,0xc0000040,0xc0000002,0x80000000,0x80000000,0x80000002,0x00000040,0x00000002,0x80000000,0x80000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000105,0x00000089,0x00000016,0x0000020b,0x0000011b,0x0000012d,0x0000041e,0x00000224,0x00000050,0x0000092e,0x0000046c } } +, {2,51,0,65,0,25, { 0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014,0x8000024b } } +, {2,51,2,65,0,26, { 0x00000043,0xd0000072,0xf0000010,0xf000006a,0x80000040,0x90000070,0xb0000053,0x30000008,0x00000043,0xd0000072,0xb0000010,0xf0000062,0xc0000042,0x00000030,0xe0000042,0x20000060,0xe0000041,0x20000050,0xc0000041,0xe0000072,0xa0000003,0xc0000012,0x60000041,0xc0000032,0x20000001,0xc0000002,0xe0000042,0x60000042,0x80000002,0x00000000,0x00000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000000,0x00000040,0x80000001,0x00000060,0x80000003,0x40000002,0xc0000040,0xc0000002,0x80000000,0x80000000,0x80000002,0x00000040,0x00000002,0x80000000,0x80000000,0x80000000,0x00000002,0x00000040,0x00000000,0x80000040,0x80000002,0x00000000,0x80000000,0x80000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000004,0x00000080,0x00000004,0x00000009,0x00000105,0x00000089,0x00000016,0x0000020b,0x0000011b,0x0000012d,0x0000041e,0x00000224,0x00000050,0x0000092e } } +, {2,52,0,65,0,27, { 0x0c000002,0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089,0x00000014 } } +, {2,53,0,65,0,28, { 0xcc000014,0x0c000002,0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107,0x00000089 } } +, {2,54,0,65,0,29, { 0x0400001c,0xcc000014,0x0c000002,0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b,0x80000107 } } +, {2,55,0,65,0,30, { 0x00000010,0x0400001c,0xcc000014,0x0c000002,0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046,0x4000004b } } +, {2,56,0,65,0,31, { 0x2600001a,0x00000010,0x0400001c,0xcc000014,0x0c000002,0xc0000010,0xb400001c,0x3c000004,0xbc00001a,0x20000010,0x2400001c,0xec000014,0x0c000002,0xc0000010,0xb400001c,0x2c000004,0xbc000018,0xb0000010,0x0000000c,0xb8000010,0x08000018,0x78000010,0x08000014,0x70000010,0xb800001c,0xe8000000,0xb0000004,0x58000010,0xb000000c,0x48000000,0xb0000000,0xb8000010,0x98000010,0xa0000000,0x00000000,0x00000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0x20000000,0x00000010,0x60000000,0x00000018,0xe0000000,0x90000000,0x30000010,0xb0000000,0x20000000,0x20000000,0xa0000000,0x00000010,0x80000000,0x20000000,0x20000000,0x20000000,0x80000000,0x00000010,0x00000000,0x20000010,0xa0000000,0x00000000,0x20000000,0x20000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000000,0x00000001,0x00000020,0x00000001,0x40000002,0x40000041,0x40000022,0x80000005,0xc0000082,0xc0000046 } } +, {0,0,0,0,0,0, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}} +}; +void ubc_check(const uint32_t W[80], uint32_t dvmask[1]) +{ + uint32_t mask = ~((uint32_t)(0)); + mask &= (((((W[44]^W[45])>>29)&1)-1) | ~(DV_I_48_0_bit|DV_I_51_0_bit|DV_I_52_0_bit|DV_II_45_0_bit|DV_II_46_0_bit|DV_II_50_0_bit|DV_II_51_0_bit)); + mask &= (((((W[49]^W[50])>>29)&1)-1) | ~(DV_I_46_0_bit|DV_II_45_0_bit|DV_II_50_0_bit|DV_II_51_0_bit|DV_II_55_0_bit|DV_II_56_0_bit)); + mask &= (((((W[48]^W[49])>>29)&1)-1) | ~(DV_I_45_0_bit|DV_I_52_0_bit|DV_II_49_0_bit|DV_II_50_0_bit|DV_II_54_0_bit|DV_II_55_0_bit)); + mask &= ((((W[47]^(W[50]>>25))&(1<<4))-(1<<4)) | ~(DV_I_47_0_bit|DV_I_49_0_bit|DV_I_51_0_bit|DV_II_45_0_bit|DV_II_51_0_bit|DV_II_56_0_bit)); + mask &= (((((W[47]^W[48])>>29)&1)-1) | ~(DV_I_44_0_bit|DV_I_51_0_bit|DV_II_48_0_bit|DV_II_49_0_bit|DV_II_53_0_bit|DV_II_54_0_bit)); + mask &= (((((W[46]>>4)^(W[49]>>29))&1)-1) | ~(DV_I_46_0_bit|DV_I_48_0_bit|DV_I_50_0_bit|DV_I_52_0_bit|DV_II_50_0_bit|DV_II_55_0_bit)); + mask &= (((((W[46]^W[47])>>29)&1)-1) | ~(DV_I_43_0_bit|DV_I_50_0_bit|DV_II_47_0_bit|DV_II_48_0_bit|DV_II_52_0_bit|DV_II_53_0_bit)); + mask &= (((((W[45]>>4)^(W[48]>>29))&1)-1) | ~(DV_I_45_0_bit|DV_I_47_0_bit|DV_I_49_0_bit|DV_I_51_0_bit|DV_II_49_0_bit|DV_II_54_0_bit)); + mask &= (((((W[45]^W[46])>>29)&1)-1) | ~(DV_I_49_0_bit|DV_I_52_0_bit|DV_II_46_0_bit|DV_II_47_0_bit|DV_II_51_0_bit|DV_II_52_0_bit)); + mask &= (((((W[44]>>4)^(W[47]>>29))&1)-1) | ~(DV_I_44_0_bit|DV_I_46_0_bit|DV_I_48_0_bit|DV_I_50_0_bit|DV_II_48_0_bit|DV_II_53_0_bit)); + mask &= (((((W[43]>>4)^(W[46]>>29))&1)-1) | ~(DV_I_43_0_bit|DV_I_45_0_bit|DV_I_47_0_bit|DV_I_49_0_bit|DV_II_47_0_bit|DV_II_52_0_bit)); + mask &= (((((W[43]^W[44])>>29)&1)-1) | ~(DV_I_47_0_bit|DV_I_50_0_bit|DV_I_51_0_bit|DV_II_45_0_bit|DV_II_49_0_bit|DV_II_50_0_bit)); + mask &= (((((W[42]>>4)^(W[45]>>29))&1)-1) | ~(DV_I_44_0_bit|DV_I_46_0_bit|DV_I_48_0_bit|DV_I_52_0_bit|DV_II_46_0_bit|DV_II_51_0_bit)); + mask &= (((((W[41]>>4)^(W[44]>>29))&1)-1) | ~(DV_I_43_0_bit|DV_I_45_0_bit|DV_I_47_0_bit|DV_I_51_0_bit|DV_II_45_0_bit|DV_II_50_0_bit)); + mask &= (((((W[40]^W[41])>>29)&1)-1) | ~(DV_I_44_0_bit|DV_I_47_0_bit|DV_I_48_0_bit|DV_II_46_0_bit|DV_II_47_0_bit|DV_II_56_0_bit)); + mask &= (((((W[54]^W[55])>>29)&1)-1) | ~(DV_I_51_0_bit|DV_II_47_0_bit|DV_II_50_0_bit|DV_II_55_0_bit|DV_II_56_0_bit)); + mask &= (((((W[53]^W[54])>>29)&1)-1) | ~(DV_I_50_0_bit|DV_II_46_0_bit|DV_II_49_0_bit|DV_II_54_0_bit|DV_II_55_0_bit)); + mask &= (((((W[52]^W[53])>>29)&1)-1) | ~(DV_I_49_0_bit|DV_II_45_0_bit|DV_II_48_0_bit|DV_II_53_0_bit|DV_II_54_0_bit)); + mask &= ((((W[50]^(W[53]>>25))&(1<<4))-(1<<4)) | ~(DV_I_50_0_bit|DV_I_52_0_bit|DV_II_46_0_bit|DV_II_48_0_bit|DV_II_54_0_bit)); + mask &= (((((W[50]^W[51])>>29)&1)-1) | ~(DV_I_47_0_bit|DV_II_46_0_bit|DV_II_51_0_bit|DV_II_52_0_bit|DV_II_56_0_bit)); + mask &= ((((W[49]^(W[52]>>25))&(1<<4))-(1<<4)) | ~(DV_I_49_0_bit|DV_I_51_0_bit|DV_II_45_0_bit|DV_II_47_0_bit|DV_II_53_0_bit)); + mask &= ((((W[48]^(W[51]>>25))&(1<<4))-(1<<4)) | ~(DV_I_48_0_bit|DV_I_50_0_bit|DV_I_52_0_bit|DV_II_46_0_bit|DV_II_52_0_bit)); + mask &= (((((W[42]^W[43])>>29)&1)-1) | ~(DV_I_46_0_bit|DV_I_49_0_bit|DV_I_50_0_bit|DV_II_48_0_bit|DV_II_49_0_bit)); + mask &= (((((W[41]^W[42])>>29)&1)-1) | ~(DV_I_45_0_bit|DV_I_48_0_bit|DV_I_49_0_bit|DV_II_47_0_bit|DV_II_48_0_bit)); + mask &= (((((W[40]>>4)^(W[43]>>29))&1)-1) | ~(DV_I_44_0_bit|DV_I_46_0_bit|DV_I_50_0_bit|DV_II_49_0_bit|DV_II_56_0_bit)); + mask &= (((((W[39]>>4)^(W[42]>>29))&1)-1) | ~(DV_I_43_0_bit|DV_I_45_0_bit|DV_I_49_0_bit|DV_II_48_0_bit|DV_II_55_0_bit)); + if (mask & (DV_I_44_0_bit|DV_I_48_0_bit|DV_II_47_0_bit|DV_II_54_0_bit|DV_II_56_0_bit)) + mask &= (((((W[38]>>4)^(W[41]>>29))&1)-1) | ~(DV_I_44_0_bit|DV_I_48_0_bit|DV_II_47_0_bit|DV_II_54_0_bit|DV_II_56_0_bit)); + mask &= (((((W[37]>>4)^(W[40]>>29))&1)-1) | ~(DV_I_43_0_bit|DV_I_47_0_bit|DV_II_46_0_bit|DV_II_53_0_bit|DV_II_55_0_bit)); + if (mask & (DV_I_52_0_bit|DV_II_48_0_bit|DV_II_51_0_bit|DV_II_56_0_bit)) + mask &= (((((W[55]^W[56])>>29)&1)-1) | ~(DV_I_52_0_bit|DV_II_48_0_bit|DV_II_51_0_bit|DV_II_56_0_bit)); + if (mask & (DV_I_52_0_bit|DV_II_48_0_bit|DV_II_50_0_bit|DV_II_56_0_bit)) + mask &= ((((W[52]^(W[55]>>25))&(1<<4))-(1<<4)) | ~(DV_I_52_0_bit|DV_II_48_0_bit|DV_II_50_0_bit|DV_II_56_0_bit)); + if (mask & (DV_I_51_0_bit|DV_II_47_0_bit|DV_II_49_0_bit|DV_II_55_0_bit)) + mask &= ((((W[51]^(W[54]>>25))&(1<<4))-(1<<4)) | ~(DV_I_51_0_bit|DV_II_47_0_bit|DV_II_49_0_bit|DV_II_55_0_bit)); + if (mask & (DV_I_48_0_bit|DV_II_47_0_bit|DV_II_52_0_bit|DV_II_53_0_bit)) + mask &= (((((W[51]^W[52])>>29)&1)-1) | ~(DV_I_48_0_bit|DV_II_47_0_bit|DV_II_52_0_bit|DV_II_53_0_bit)); + if (mask & (DV_I_46_0_bit|DV_I_49_0_bit|DV_II_45_0_bit|DV_II_48_0_bit)) + mask &= (((((W[36]>>4)^(W[40]>>29))&1)-1) | ~(DV_I_46_0_bit|DV_I_49_0_bit|DV_II_45_0_bit|DV_II_48_0_bit)); + if (mask & (DV_I_52_0_bit|DV_II_48_0_bit|DV_II_49_0_bit)) + mask &= ((0-(((W[53]^W[56])>>29)&1)) | ~(DV_I_52_0_bit|DV_II_48_0_bit|DV_II_49_0_bit)); + if (mask & (DV_I_50_0_bit|DV_II_46_0_bit|DV_II_47_0_bit)) + mask &= ((0-(((W[51]^W[54])>>29)&1)) | ~(DV_I_50_0_bit|DV_II_46_0_bit|DV_II_47_0_bit)); + if (mask & (DV_I_49_0_bit|DV_I_51_0_bit|DV_II_45_0_bit)) + mask &= ((0-(((W[50]^W[52])>>29)&1)) | ~(DV_I_49_0_bit|DV_I_51_0_bit|DV_II_45_0_bit)); + if (mask & (DV_I_48_0_bit|DV_I_50_0_bit|DV_I_52_0_bit)) + mask &= ((0-(((W[49]^W[51])>>29)&1)) | ~(DV_I_48_0_bit|DV_I_50_0_bit|DV_I_52_0_bit)); + if (mask & (DV_I_47_0_bit|DV_I_49_0_bit|DV_I_51_0_bit)) + mask &= ((0-(((W[48]^W[50])>>29)&1)) | ~(DV_I_47_0_bit|DV_I_49_0_bit|DV_I_51_0_bit)); + if (mask & (DV_I_46_0_bit|DV_I_48_0_bit|DV_I_50_0_bit)) + mask &= ((0-(((W[47]^W[49])>>29)&1)) | ~(DV_I_46_0_bit|DV_I_48_0_bit|DV_I_50_0_bit)); + if (mask & (DV_I_45_0_bit|DV_I_47_0_bit|DV_I_49_0_bit)) + mask &= ((0-(((W[46]^W[48])>>29)&1)) | ~(DV_I_45_0_bit|DV_I_47_0_bit|DV_I_49_0_bit)); + mask &= ((((W[45]^W[47])&(1<<6))-(1<<6)) | ~(DV_I_47_2_bit|DV_I_49_2_bit|DV_I_51_2_bit)); + if (mask & (DV_I_44_0_bit|DV_I_46_0_bit|DV_I_48_0_bit)) + mask &= ((0-(((W[45]^W[47])>>29)&1)) | ~(DV_I_44_0_bit|DV_I_46_0_bit|DV_I_48_0_bit)); + mask &= (((((W[44]^W[46])>>6)&1)-1) | ~(DV_I_46_2_bit|DV_I_48_2_bit|DV_I_50_2_bit)); + if (mask & (DV_I_43_0_bit|DV_I_45_0_bit|DV_I_47_0_bit)) + mask &= ((0-(((W[44]^W[46])>>29)&1)) | ~(DV_I_43_0_bit|DV_I_45_0_bit|DV_I_47_0_bit)); + mask &= ((0-((W[41]^(W[42]>>5))&(1<<1))) | ~(DV_I_48_2_bit|DV_II_46_2_bit|DV_II_51_2_bit)); + mask &= ((0-((W[40]^(W[41]>>5))&(1<<1))) | ~(DV_I_47_2_bit|DV_I_51_2_bit|DV_II_50_2_bit)); + if (mask & (DV_I_44_0_bit|DV_I_46_0_bit|DV_II_56_0_bit)) + mask &= ((0-(((W[40]^W[42])>>4)&1)) | ~(DV_I_44_0_bit|DV_I_46_0_bit|DV_II_56_0_bit)); + mask &= ((0-((W[39]^(W[40]>>5))&(1<<1))) | ~(DV_I_46_2_bit|DV_I_50_2_bit|DV_II_49_2_bit)); + if (mask & (DV_I_43_0_bit|DV_I_45_0_bit|DV_II_55_0_bit)) + mask &= ((0-(((W[39]^W[41])>>4)&1)) | ~(DV_I_43_0_bit|DV_I_45_0_bit|DV_II_55_0_bit)); + if (mask & (DV_I_44_0_bit|DV_II_54_0_bit|DV_II_56_0_bit)) + mask &= ((0-(((W[38]^W[40])>>4)&1)) | ~(DV_I_44_0_bit|DV_II_54_0_bit|DV_II_56_0_bit)); + if (mask & (DV_I_43_0_bit|DV_II_53_0_bit|DV_II_55_0_bit)) + mask &= ((0-(((W[37]^W[39])>>4)&1)) | ~(DV_I_43_0_bit|DV_II_53_0_bit|DV_II_55_0_bit)); + mask &= ((0-((W[36]^(W[37]>>5))&(1<<1))) | ~(DV_I_47_2_bit|DV_I_50_2_bit|DV_II_46_2_bit)); + if (mask & (DV_I_45_0_bit|DV_I_48_0_bit|DV_II_47_0_bit)) + mask &= (((((W[35]>>4)^(W[39]>>29))&1)-1) | ~(DV_I_45_0_bit|DV_I_48_0_bit|DV_II_47_0_bit)); + if (mask & (DV_I_48_0_bit|DV_II_48_0_bit)) + mask &= ((0-((W[63]^(W[64]>>5))&(1<<0))) | ~(DV_I_48_0_bit|DV_II_48_0_bit)); + if (mask & (DV_I_45_0_bit|DV_II_45_0_bit)) + mask &= ((0-((W[63]^(W[64]>>5))&(1<<1))) | ~(DV_I_45_0_bit|DV_II_45_0_bit)); + if (mask & (DV_I_47_0_bit|DV_II_47_0_bit)) + mask &= ((0-((W[62]^(W[63]>>5))&(1<<0))) | ~(DV_I_47_0_bit|DV_II_47_0_bit)); + if (mask & (DV_I_46_0_bit|DV_II_46_0_bit)) + mask &= ((0-((W[61]^(W[62]>>5))&(1<<0))) | ~(DV_I_46_0_bit|DV_II_46_0_bit)); + mask &= ((0-((W[61]^(W[62]>>5))&(1<<2))) | ~(DV_I_46_2_bit|DV_II_46_2_bit)); + if (mask & (DV_I_45_0_bit|DV_II_45_0_bit)) + mask &= ((0-((W[60]^(W[61]>>5))&(1<<0))) | ~(DV_I_45_0_bit|DV_II_45_0_bit)); + if (mask & (DV_II_51_0_bit|DV_II_54_0_bit)) + mask &= (((((W[58]^W[59])>>29)&1)-1) | ~(DV_II_51_0_bit|DV_II_54_0_bit)); + if (mask & (DV_II_50_0_bit|DV_II_53_0_bit)) + mask &= (((((W[57]^W[58])>>29)&1)-1) | ~(DV_II_50_0_bit|DV_II_53_0_bit)); + if (mask & (DV_II_52_0_bit|DV_II_54_0_bit)) + mask &= ((((W[56]^(W[59]>>25))&(1<<4))-(1<<4)) | ~(DV_II_52_0_bit|DV_II_54_0_bit)); + if (mask & (DV_II_51_0_bit|DV_II_52_0_bit)) + mask &= ((0-(((W[56]^W[59])>>29)&1)) | ~(DV_II_51_0_bit|DV_II_52_0_bit)); + if (mask & (DV_II_49_0_bit|DV_II_52_0_bit)) + mask &= (((((W[56]^W[57])>>29)&1)-1) | ~(DV_II_49_0_bit|DV_II_52_0_bit)); + if (mask & (DV_II_51_0_bit|DV_II_53_0_bit)) + mask &= ((((W[55]^(W[58]>>25))&(1<<4))-(1<<4)) | ~(DV_II_51_0_bit|DV_II_53_0_bit)); + if (mask & (DV_II_50_0_bit|DV_II_52_0_bit)) + mask &= ((((W[54]^(W[57]>>25))&(1<<4))-(1<<4)) | ~(DV_II_50_0_bit|DV_II_52_0_bit)); + if (mask & (DV_II_49_0_bit|DV_II_51_0_bit)) + mask &= ((((W[53]^(W[56]>>25))&(1<<4))-(1<<4)) | ~(DV_II_49_0_bit|DV_II_51_0_bit)); + mask &= ((((W[51]^(W[50]>>5))&(1<<1))-(1<<1)) | ~(DV_I_50_2_bit|DV_II_46_2_bit)); + mask &= ((((W[48]^W[50])&(1<<6))-(1<<6)) | ~(DV_I_50_2_bit|DV_II_46_2_bit)); + if (mask & (DV_I_51_0_bit|DV_I_52_0_bit)) + mask &= ((0-(((W[48]^W[55])>>29)&1)) | ~(DV_I_51_0_bit|DV_I_52_0_bit)); + mask &= ((((W[47]^W[49])&(1<<6))-(1<<6)) | ~(DV_I_49_2_bit|DV_I_51_2_bit)); + mask &= ((((W[48]^(W[47]>>5))&(1<<1))-(1<<1)) | ~(DV_I_47_2_bit|DV_II_51_2_bit)); + mask &= ((((W[46]^W[48])&(1<<6))-(1<<6)) | ~(DV_I_48_2_bit|DV_I_50_2_bit)); + mask &= ((((W[47]^(W[46]>>5))&(1<<1))-(1<<1)) | ~(DV_I_46_2_bit|DV_II_50_2_bit)); + mask &= ((0-((W[44]^(W[45]>>5))&(1<<1))) | ~(DV_I_51_2_bit|DV_II_49_2_bit)); + mask &= ((((W[43]^W[45])&(1<<6))-(1<<6)) | ~(DV_I_47_2_bit|DV_I_49_2_bit)); + mask &= (((((W[42]^W[44])>>6)&1)-1) | ~(DV_I_46_2_bit|DV_I_48_2_bit)); + mask &= ((((W[43]^(W[42]>>5))&(1<<1))-(1<<1)) | ~(DV_II_46_2_bit|DV_II_51_2_bit)); + mask &= ((((W[42]^(W[41]>>5))&(1<<1))-(1<<1)) | ~(DV_I_51_2_bit|DV_II_50_2_bit)); + mask &= ((((W[41]^(W[40]>>5))&(1<<1))-(1<<1)) | ~(DV_I_50_2_bit|DV_II_49_2_bit)); + if (mask & (DV_I_52_0_bit|DV_II_51_0_bit)) + mask &= ((((W[39]^(W[43]>>25))&(1<<4))-(1<<4)) | ~(DV_I_52_0_bit|DV_II_51_0_bit)); + if (mask & (DV_I_51_0_bit|DV_II_50_0_bit)) + mask &= ((((W[38]^(W[42]>>25))&(1<<4))-(1<<4)) | ~(DV_I_51_0_bit|DV_II_50_0_bit)); + if (mask & (DV_I_48_2_bit|DV_I_51_2_bit)) + mask &= ((0-((W[37]^(W[38]>>5))&(1<<1))) | ~(DV_I_48_2_bit|DV_I_51_2_bit)); + if (mask & (DV_I_50_0_bit|DV_II_49_0_bit)) + mask &= ((((W[37]^(W[41]>>25))&(1<<4))-(1<<4)) | ~(DV_I_50_0_bit|DV_II_49_0_bit)); + if (mask & (DV_II_52_0_bit|DV_II_54_0_bit)) + mask &= ((0-((W[36]^W[38])&(1<<4))) | ~(DV_II_52_0_bit|DV_II_54_0_bit)); + mask &= ((0-((W[35]^(W[36]>>5))&(1<<1))) | ~(DV_I_46_2_bit|DV_I_49_2_bit)); + if (mask & (DV_I_51_0_bit|DV_II_47_0_bit)) + mask &= ((((W[35]^(W[39]>>25))&(1<<3))-(1<<3)) | ~(DV_I_51_0_bit|DV_II_47_0_bit)); +if (mask) { + + if (mask & DV_I_43_0_bit) + if ( + !((W[61]^(W[62]>>5)) & (1<<1)) + || !(!((W[59]^(W[63]>>25)) & (1<<5))) + || !((W[58]^(W[63]>>30)) & (1<<0)) + ) mask &= ~DV_I_43_0_bit; + if (mask & DV_I_44_0_bit) + if ( + !((W[62]^(W[63]>>5)) & (1<<1)) + || !(!((W[60]^(W[64]>>25)) & (1<<5))) + || !((W[59]^(W[64]>>30)) & (1<<0)) + ) mask &= ~DV_I_44_0_bit; + if (mask & DV_I_46_2_bit) + mask &= ((~((W[40]^W[42])>>2)) | ~DV_I_46_2_bit); + if (mask & DV_I_47_2_bit) + if ( + !((W[62]^(W[63]>>5)) & (1<<2)) + || !(!((W[41]^W[43]) & (1<<6))) + ) mask &= ~DV_I_47_2_bit; + if (mask & DV_I_48_2_bit) + if ( + !((W[63]^(W[64]>>5)) & (1<<2)) + || !(!((W[48]^(W[49]<<5)) & (1<<6))) + ) mask &= ~DV_I_48_2_bit; + if (mask & DV_I_49_2_bit) + if ( + !(!((W[49]^(W[50]<<5)) & (1<<6))) + || !((W[42]^W[50]) & (1<<1)) + || !(!((W[39]^(W[40]<<5)) & (1<<6))) + || !((W[38]^W[40]) & (1<<1)) + ) mask &= ~DV_I_49_2_bit; + if (mask & DV_I_50_0_bit) + mask &= ((((W[36]^W[37])<<7)) | ~DV_I_50_0_bit); + if (mask & DV_I_50_2_bit) + mask &= ((((W[43]^W[51])<<11)) | ~DV_I_50_2_bit); + if (mask & DV_I_51_0_bit) + mask &= ((((W[37]^W[38])<<9)) | ~DV_I_51_0_bit); + if (mask & DV_I_51_2_bit) + if ( + !(!((W[51]^(W[52]<<5)) & (1<<6))) + || !(!((W[49]^W[51]) & (1<<6))) + || !(!((W[37]^(W[37]>>5)) & (1<<1))) + || !(!((W[35]^(W[39]>>25)) & (1<<5))) + ) mask &= ~DV_I_51_2_bit; + if (mask & DV_I_52_0_bit) + mask &= ((((W[38]^W[39])<<11)) | ~DV_I_52_0_bit); + if (mask & DV_II_46_2_bit) + mask &= ((((W[47]^W[51])<<17)) | ~DV_II_46_2_bit); + if (mask & DV_II_48_0_bit) + if ( + !(!((W[36]^(W[40]>>25)) & (1<<3))) + || !((W[35]^(W[40]<<2)) & (1<<30)) + ) mask &= ~DV_II_48_0_bit; + if (mask & DV_II_49_0_bit) + if ( + !(!((W[37]^(W[41]>>25)) & (1<<3))) + || !((W[36]^(W[41]<<2)) & (1<<30)) + ) mask &= ~DV_II_49_0_bit; + if (mask & DV_II_49_2_bit) + if ( + !(!((W[53]^(W[54]<<5)) & (1<<6))) + || !(!((W[51]^W[53]) & (1<<6))) + || !((W[50]^W[54]) & (1<<1)) + || !(!((W[45]^(W[46]<<5)) & (1<<6))) + || !(!((W[37]^(W[41]>>25)) & (1<<5))) + || !((W[36]^(W[41]>>30)) & (1<<0)) + ) mask &= ~DV_II_49_2_bit; + if (mask & DV_II_50_0_bit) + if ( + !((W[55]^W[58]) & (1<<29)) + || !(!((W[38]^(W[42]>>25)) & (1<<3))) + || !((W[37]^(W[42]<<2)) & (1<<30)) + ) mask &= ~DV_II_50_0_bit; + if (mask & DV_II_50_2_bit) + if ( + !(!((W[54]^(W[55]<<5)) & (1<<6))) + || !(!((W[52]^W[54]) & (1<<6))) + || !((W[51]^W[55]) & (1<<1)) + || !((W[45]^W[47]) & (1<<1)) + || !(!((W[38]^(W[42]>>25)) & (1<<5))) + || !((W[37]^(W[42]>>30)) & (1<<0)) + ) mask &= ~DV_II_50_2_bit; + if (mask & DV_II_51_0_bit) + if ( + !(!((W[39]^(W[43]>>25)) & (1<<3))) + || !((W[38]^(W[43]<<2)) & (1<<30)) + ) mask &= ~DV_II_51_0_bit; + if (mask & DV_II_51_2_bit) + if ( + !(!((W[55]^(W[56]<<5)) & (1<<6))) + || !(!((W[53]^W[55]) & (1<<6))) + || !((W[52]^W[56]) & (1<<1)) + || !((W[46]^W[48]) & (1<<1)) + || !(!((W[39]^(W[43]>>25)) & (1<<5))) + || !((W[38]^(W[43]>>30)) & (1<<0)) + ) mask &= ~DV_II_51_2_bit; + if (mask & DV_II_52_0_bit) + if ( + !(!((W[59]^W[60]) & (1<<29))) + || !(!((W[40]^(W[44]>>25)) & (1<<3))) + || !(!((W[40]^(W[44]>>25)) & (1<<4))) + || !((W[39]^(W[44]<<2)) & (1<<30)) + ) mask &= ~DV_II_52_0_bit; + if (mask & DV_II_53_0_bit) + if ( + !((W[58]^W[61]) & (1<<29)) + || !(!((W[57]^(W[61]>>25)) & (1<<4))) + || !(!((W[41]^(W[45]>>25)) & (1<<3))) + || !(!((W[41]^(W[45]>>25)) & (1<<4))) + ) mask &= ~DV_II_53_0_bit; + if (mask & DV_II_54_0_bit) + if ( + !(!((W[58]^(W[62]>>25)) & (1<<4))) + || !(!((W[42]^(W[46]>>25)) & (1<<3))) + || !(!((W[42]^(W[46]>>25)) & (1<<4))) + ) mask &= ~DV_II_54_0_bit; + if (mask & DV_II_55_0_bit) + if ( + !(!((W[59]^(W[63]>>25)) & (1<<4))) + || !(!((W[57]^(W[59]>>25)) & (1<<4))) + || !(!((W[43]^(W[47]>>25)) & (1<<3))) + || !(!((W[43]^(W[47]>>25)) & (1<<4))) + ) mask &= ~DV_II_55_0_bit; + if (mask & DV_II_56_0_bit) + if ( + !(!((W[60]^(W[64]>>25)) & (1<<4))) + || !(!((W[44]^(W[48]>>25)) & (1<<3))) + || !(!((W[44]^(W[48]>>25)) & (1<<4))) + ) mask &= ~DV_II_56_0_bit; +} + + dvmask[0]=mask; +} +/******************** End Of File: lib/ubc_check.c *******************/ +/******************** Continue with: lib/sha1.c **********************/ + + +#define rotate_right(x,n) (((x)>>(n))|((x)<<(32-(n)))) +#define rotate_left(x,n) (((x)<<(n))|((x)>>(32-(n)))) + +#define sha1_f1(b,c,d) ((d)^((b)&((c)^(d)))) +#define sha1_f2(b,c,d) ((b)^(c)^(d)) +#define sha1_f3(b,c,d) (((b) & ((c)|(d))) | ((c)&(d))) +#define sha1_f4(b,c,d) ((b)^(c)^(d)) + +#define HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, m, t) \ + { e += rotate_left(a, 5) + sha1_f1(b,c,d) + 0x5A827999 + m[t]; b = rotate_left(b, 30); } +#define HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, m, t) \ + { e += rotate_left(a, 5) + sha1_f2(b,c,d) + 0x6ED9EBA1 + m[t]; b = rotate_left(b, 30); } +#define HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, m, t) \ + { e += rotate_left(a, 5) + sha1_f3(b,c,d) + 0x8F1BBCDC + m[t]; b = rotate_left(b, 30); } +#define HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, m, t) \ + { e += rotate_left(a, 5) + sha1_f4(b,c,d) + 0xCA62C1D6 + m[t]; b = rotate_left(b, 30); } + +#define HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(a, b, c, d, e, m, t) \ + { b = rotate_right(b, 30); e -= rotate_left(a, 5) + sha1_f1(b,c,d) + 0x5A827999 + m[t]; } +#define HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(a, b, c, d, e, m, t) \ + { b = rotate_right(b, 30); e -= rotate_left(a, 5) + sha1_f2(b,c,d) + 0x6ED9EBA1 + m[t]; } +#define HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(a, b, c, d, e, m, t) \ + { b = rotate_right(b, 30); e -= rotate_left(a, 5) + sha1_f3(b,c,d) + 0x8F1BBCDC + m[t]; } +#define HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(a, b, c, d, e, m, t) \ + { b = rotate_right(b, 30); e -= rotate_left(a, 5) + sha1_f4(b,c,d) + 0xCA62C1D6 + m[t]; } + +#define SHA1_STORE_STATE(i) states[i][0] = a; states[i][1] = b; states[i][2] = c; states[i][3] = d; states[i][4] = e; + +static void sha1_message_expansion(uint32_t W[80]) +{ + unsigned i; + for (i = 16; i < 80; ++i) + W[i] = rotate_left(W[i - 3] ^ W[i - 8] ^ W[i - 14] ^ W[i - 16], 1); +} + +#if 0 +static void sha1_compression(uint32_t ihv[5], const uint32_t m[16]); +static void sha1_compression(uint32_t ihv[5], const uint32_t m[16]) +{ + uint32_t W[80]; + uint32_t a,b,c,d,e; + unsigned i; + + memcpy(W, m, 16 * 4); + for (i = 16; i < 80; ++i) + W[i] = rotate_left(W[i - 3] ^ W[i - 8] ^ W[i - 14] ^ W[i - 16], 1); + + a = ihv[0]; b = ihv[1]; c = ihv[2]; d = ihv[3]; e = ihv[4]; + + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 0); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 1); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 2); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 3); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 4); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 5); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 6); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 7); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 8); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 9); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 10); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 11); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 12); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 13); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 14); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 15); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 16); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 17); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 18); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 19); + + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 20); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 21); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 22); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 23); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 24); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 25); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 26); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 27); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 28); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 29); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 30); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 31); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 32); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 33); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 34); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 35); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 36); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 37); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 38); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 39); + + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 40); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 41); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 42); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 43); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 44); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 45); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 46); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 47); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 48); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 49); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 50); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 51); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 52); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 53); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 54); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 55); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 56); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 57); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 58); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 59); + + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 60); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 61); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 62); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 63); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 64); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 65); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 66); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 67); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 68); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 69); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 70); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 71); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 72); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 73); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 74); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 75); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 76); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 77); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 78); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 79); + + ihv[0] += a; ihv[1] += b; ihv[2] += c; ihv[3] += d; ihv[4] += e; +} +#endif + + +static void sha1_compression_W(uint32_t ihv[5], const uint32_t W[80]) +{ + uint32_t a = ihv[0], b = ihv[1], c = ihv[2], d = ihv[3], e = ihv[4]; + + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 0); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 1); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 2); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 3); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 4); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 5); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 6); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 7); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 8); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 9); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 10); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 11); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 12); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 13); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 14); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 15); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 16); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 17); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 18); + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 19); + + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 20); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 21); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 22); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 23); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 24); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 25); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 26); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 27); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 28); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 29); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 30); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 31); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 32); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 33); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 34); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 35); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 36); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 37); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 38); + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 39); + + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 40); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 41); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 42); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 43); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 44); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 45); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 46); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 47); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 48); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 49); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 50); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 51); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 52); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 53); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 54); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 55); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 56); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 57); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 58); + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 59); + + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 60); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 61); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 62); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 63); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 64); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 65); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 66); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 67); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 68); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 69); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 70); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 71); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 72); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 73); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 74); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 75); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 76); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 77); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 78); + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 79); + + ihv[0] += a; ihv[1] += b; ihv[2] += c; ihv[3] += d; ihv[4] += e; +} + + + +static void sha1_compression_states(uint32_t ihv[5], const uint32_t W[80], uint32_t states[80][5]) +{ + uint32_t a = ihv[0], b = ihv[1], c = ihv[2], d = ihv[3], e = ihv[4]; + +#ifdef DOSTORESTATE00 + SHA1_STORE_STATE(0) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 0); + +#ifdef DOSTORESTATE01 + SHA1_STORE_STATE(1) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 1); + +#ifdef DOSTORESTATE02 + SHA1_STORE_STATE(2) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 2); + +#ifdef DOSTORESTATE03 + SHA1_STORE_STATE(3) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 3); + +#ifdef DOSTORESTATE04 + SHA1_STORE_STATE(4) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 4); + +#ifdef DOSTORESTATE05 + SHA1_STORE_STATE(5) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 5); + +#ifdef DOSTORESTATE06 + SHA1_STORE_STATE(6) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 6); + +#ifdef DOSTORESTATE07 + SHA1_STORE_STATE(7) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 7); + +#ifdef DOSTORESTATE08 + SHA1_STORE_STATE(8) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 8); + +#ifdef DOSTORESTATE09 + SHA1_STORE_STATE(9) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 9); + +#ifdef DOSTORESTATE10 + SHA1_STORE_STATE(10) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 10); + +#ifdef DOSTORESTATE11 + SHA1_STORE_STATE(11) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 11); + +#ifdef DOSTORESTATE12 + SHA1_STORE_STATE(12) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 12); + +#ifdef DOSTORESTATE13 + SHA1_STORE_STATE(13) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 13); + +#ifdef DOSTORESTATE14 + SHA1_STORE_STATE(14) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 14); + +#ifdef DOSTORESTATE15 + SHA1_STORE_STATE(15) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, W, 15); + +#ifdef DOSTORESTATE16 + SHA1_STORE_STATE(16) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, W, 16); + +#ifdef DOSTORESTATE17 + SHA1_STORE_STATE(17) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, W, 17); + +#ifdef DOSTORESTATE18 + SHA1_STORE_STATE(18) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, W, 18); + +#ifdef DOSTORESTATE19 + SHA1_STORE_STATE(19) +#endif + HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, W, 19); + + + +#ifdef DOSTORESTATE20 + SHA1_STORE_STATE(20) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 20); + +#ifdef DOSTORESTATE21 + SHA1_STORE_STATE(21) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 21); + +#ifdef DOSTORESTATE22 + SHA1_STORE_STATE(22) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 22); + +#ifdef DOSTORESTATE23 + SHA1_STORE_STATE(23) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 23); + +#ifdef DOSTORESTATE24 + SHA1_STORE_STATE(24) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 24); + +#ifdef DOSTORESTATE25 + SHA1_STORE_STATE(25) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 25); + +#ifdef DOSTORESTATE26 + SHA1_STORE_STATE(26) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 26); + +#ifdef DOSTORESTATE27 + SHA1_STORE_STATE(27) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 27); + +#ifdef DOSTORESTATE28 + SHA1_STORE_STATE(28) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 28); + +#ifdef DOSTORESTATE29 + SHA1_STORE_STATE(29) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 29); + +#ifdef DOSTORESTATE30 + SHA1_STORE_STATE(30) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 30); + +#ifdef DOSTORESTATE31 + SHA1_STORE_STATE(31) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 31); + +#ifdef DOSTORESTATE32 + SHA1_STORE_STATE(32) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 32); + +#ifdef DOSTORESTATE33 + SHA1_STORE_STATE(33) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 33); + +#ifdef DOSTORESTATE34 + SHA1_STORE_STATE(34) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 34); + +#ifdef DOSTORESTATE35 + SHA1_STORE_STATE(35) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, W, 35); + +#ifdef DOSTORESTATE36 + SHA1_STORE_STATE(36) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, W, 36); + +#ifdef DOSTORESTATE37 + SHA1_STORE_STATE(37) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, W, 37); + +#ifdef DOSTORESTATE38 + SHA1_STORE_STATE(38) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, W, 38); + +#ifdef DOSTORESTATE39 + SHA1_STORE_STATE(39) +#endif + HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, W, 39); + + + +#ifdef DOSTORESTATE40 + SHA1_STORE_STATE(40) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 40); + +#ifdef DOSTORESTATE41 + SHA1_STORE_STATE(41) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 41); + +#ifdef DOSTORESTATE42 + SHA1_STORE_STATE(42) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 42); + +#ifdef DOSTORESTATE43 + SHA1_STORE_STATE(43) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 43); + +#ifdef DOSTORESTATE44 + SHA1_STORE_STATE(44) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 44); + +#ifdef DOSTORESTATE45 + SHA1_STORE_STATE(45) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 45); + +#ifdef DOSTORESTATE46 + SHA1_STORE_STATE(46) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 46); + +#ifdef DOSTORESTATE47 + SHA1_STORE_STATE(47) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 47); + +#ifdef DOSTORESTATE48 + SHA1_STORE_STATE(48) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 48); + +#ifdef DOSTORESTATE49 + SHA1_STORE_STATE(49) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 49); + +#ifdef DOSTORESTATE50 + SHA1_STORE_STATE(50) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 50); + +#ifdef DOSTORESTATE51 + SHA1_STORE_STATE(51) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 51); + +#ifdef DOSTORESTATE52 + SHA1_STORE_STATE(52) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 52); + +#ifdef DOSTORESTATE53 + SHA1_STORE_STATE(53) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 53); + +#ifdef DOSTORESTATE54 + SHA1_STORE_STATE(54) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 54); + +#ifdef DOSTORESTATE55 + SHA1_STORE_STATE(55) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, W, 55); + +#ifdef DOSTORESTATE56 + SHA1_STORE_STATE(56) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, W, 56); + +#ifdef DOSTORESTATE57 + SHA1_STORE_STATE(57) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, W, 57); + +#ifdef DOSTORESTATE58 + SHA1_STORE_STATE(58) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, W, 58); + +#ifdef DOSTORESTATE59 + SHA1_STORE_STATE(59) +#endif + HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, W, 59); + + + + +#ifdef DOSTORESTATE60 + SHA1_STORE_STATE(60) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 60); + +#ifdef DOSTORESTATE61 + SHA1_STORE_STATE(61) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 61); + +#ifdef DOSTORESTATE62 + SHA1_STORE_STATE(62) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 62); + +#ifdef DOSTORESTATE63 + SHA1_STORE_STATE(63) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 63); + +#ifdef DOSTORESTATE64 + SHA1_STORE_STATE(64) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 64); + +#ifdef DOSTORESTATE65 + SHA1_STORE_STATE(65) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 65); + +#ifdef DOSTORESTATE66 + SHA1_STORE_STATE(66) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 66); + +#ifdef DOSTORESTATE67 + SHA1_STORE_STATE(67) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 67); + +#ifdef DOSTORESTATE68 + SHA1_STORE_STATE(68) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 68); + +#ifdef DOSTORESTATE69 + SHA1_STORE_STATE(69) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 69); + +#ifdef DOSTORESTATE70 + SHA1_STORE_STATE(70) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 70); + +#ifdef DOSTORESTATE71 + SHA1_STORE_STATE(71) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 71); + +#ifdef DOSTORESTATE72 + SHA1_STORE_STATE(72) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 72); + +#ifdef DOSTORESTATE73 + SHA1_STORE_STATE(73) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 73); + +#ifdef DOSTORESTATE74 + SHA1_STORE_STATE(74) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 74); + +#ifdef DOSTORESTATE75 + SHA1_STORE_STATE(75) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, W, 75); + +#ifdef DOSTORESTATE76 + SHA1_STORE_STATE(76) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, W, 76); + +#ifdef DOSTORESTATE77 + SHA1_STORE_STATE(77) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, W, 77); + +#ifdef DOSTORESTATE78 + SHA1_STORE_STATE(78) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, W, 78); + +#ifdef DOSTORESTATE79 + SHA1_STORE_STATE(79) +#endif + HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, W, 79); + + + + ihv[0] += a; ihv[1] += b; ihv[2] += c; ihv[3] += d; ihv[4] += e; +} + + + + +#define SHA1_RECOMPRESS(t) \ +void sha1recompress_fast_ ## t (uint32_t ihvin[5], uint32_t ihvout[5], const uint32_t me2[80], const uint32_t state[5]) \ +{ \ + uint32_t a = state[0], b = state[1], c = state[2], d = state[3], e = state[4]; \ + if (t > 79) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(b, c, d, e, a, me2, 79); \ + if (t > 78) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(c, d, e, a, b, me2, 78); \ + if (t > 77) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(d, e, a, b, c, me2, 77); \ + if (t > 76) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(e, a, b, c, d, me2, 76); \ + if (t > 75) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(a, b, c, d, e, me2, 75); \ + if (t > 74) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(b, c, d, e, a, me2, 74); \ + if (t > 73) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(c, d, e, a, b, me2, 73); \ + if (t > 72) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(d, e, a, b, c, me2, 72); \ + if (t > 71) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(e, a, b, c, d, me2, 71); \ + if (t > 70) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(a, b, c, d, e, me2, 70); \ + if (t > 69) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(b, c, d, e, a, me2, 69); \ + if (t > 68) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(c, d, e, a, b, me2, 68); \ + if (t > 67) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(d, e, a, b, c, me2, 67); \ + if (t > 66) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(e, a, b, c, d, me2, 66); \ + if (t > 65) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(a, b, c, d, e, me2, 65); \ + if (t > 64) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(b, c, d, e, a, me2, 64); \ + if (t > 63) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(c, d, e, a, b, me2, 63); \ + if (t > 62) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(d, e, a, b, c, me2, 62); \ + if (t > 61) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(e, a, b, c, d, me2, 61); \ + if (t > 60) HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW(a, b, c, d, e, me2, 60); \ + if (t > 59) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(b, c, d, e, a, me2, 59); \ + if (t > 58) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(c, d, e, a, b, me2, 58); \ + if (t > 57) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(d, e, a, b, c, me2, 57); \ + if (t > 56) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(e, a, b, c, d, me2, 56); \ + if (t > 55) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(a, b, c, d, e, me2, 55); \ + if (t > 54) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(b, c, d, e, a, me2, 54); \ + if (t > 53) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(c, d, e, a, b, me2, 53); \ + if (t > 52) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(d, e, a, b, c, me2, 52); \ + if (t > 51) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(e, a, b, c, d, me2, 51); \ + if (t > 50) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(a, b, c, d, e, me2, 50); \ + if (t > 49) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(b, c, d, e, a, me2, 49); \ + if (t > 48) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(c, d, e, a, b, me2, 48); \ + if (t > 47) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(d, e, a, b, c, me2, 47); \ + if (t > 46) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(e, a, b, c, d, me2, 46); \ + if (t > 45) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(a, b, c, d, e, me2, 45); \ + if (t > 44) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(b, c, d, e, a, me2, 44); \ + if (t > 43) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(c, d, e, a, b, me2, 43); \ + if (t > 42) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(d, e, a, b, c, me2, 42); \ + if (t > 41) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(e, a, b, c, d, me2, 41); \ + if (t > 40) HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW(a, b, c, d, e, me2, 40); \ + if (t > 39) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(b, c, d, e, a, me2, 39); \ + if (t > 38) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(c, d, e, a, b, me2, 38); \ + if (t > 37) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(d, e, a, b, c, me2, 37); \ + if (t > 36) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(e, a, b, c, d, me2, 36); \ + if (t > 35) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(a, b, c, d, e, me2, 35); \ + if (t > 34) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(b, c, d, e, a, me2, 34); \ + if (t > 33) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(c, d, e, a, b, me2, 33); \ + if (t > 32) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(d, e, a, b, c, me2, 32); \ + if (t > 31) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(e, a, b, c, d, me2, 31); \ + if (t > 30) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(a, b, c, d, e, me2, 30); \ + if (t > 29) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(b, c, d, e, a, me2, 29); \ + if (t > 28) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(c, d, e, a, b, me2, 28); \ + if (t > 27) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(d, e, a, b, c, me2, 27); \ + if (t > 26) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(e, a, b, c, d, me2, 26); \ + if (t > 25) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(a, b, c, d, e, me2, 25); \ + if (t > 24) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(b, c, d, e, a, me2, 24); \ + if (t > 23) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(c, d, e, a, b, me2, 23); \ + if (t > 22) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(d, e, a, b, c, me2, 22); \ + if (t > 21) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(e, a, b, c, d, me2, 21); \ + if (t > 20) HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW(a, b, c, d, e, me2, 20); \ + if (t > 19) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(b, c, d, e, a, me2, 19); \ + if (t > 18) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(c, d, e, a, b, me2, 18); \ + if (t > 17) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(d, e, a, b, c, me2, 17); \ + if (t > 16) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(e, a, b, c, d, me2, 16); \ + if (t > 15) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(a, b, c, d, e, me2, 15); \ + if (t > 14) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(b, c, d, e, a, me2, 14); \ + if (t > 13) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(c, d, e, a, b, me2, 13); \ + if (t > 12) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(d, e, a, b, c, me2, 12); \ + if (t > 11) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(e, a, b, c, d, me2, 11); \ + if (t > 10) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(a, b, c, d, e, me2, 10); \ + if (t > 9) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(b, c, d, e, a, me2, 9); \ + if (t > 8) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(c, d, e, a, b, me2, 8); \ + if (t > 7) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(d, e, a, b, c, me2, 7); \ + if (t > 6) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(e, a, b, c, d, me2, 6); \ + if (t > 5) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(a, b, c, d, e, me2, 5); \ + if (t > 4) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(b, c, d, e, a, me2, 4); \ + if (t > 3) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(c, d, e, a, b, me2, 3); \ + if (t > 2) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(d, e, a, b, c, me2, 2); \ + if (t > 1) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(e, a, b, c, d, me2, 1); \ + if (t > 0) HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW(a, b, c, d, e, me2, 0); \ + ihvin[0] = a; ihvin[1] = b; ihvin[2] = c; ihvin[3] = d; ihvin[4] = e; \ + a = state[0]; b = state[1]; c = state[2]; d = state[3]; e = state[4]; \ + if (t <= 0) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, me2, 0); \ + if (t <= 1) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, me2, 1); \ + if (t <= 2) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, me2, 2); \ + if (t <= 3) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, me2, 3); \ + if (t <= 4) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, me2, 4); \ + if (t <= 5) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, me2, 5); \ + if (t <= 6) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, me2, 6); \ + if (t <= 7) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, me2, 7); \ + if (t <= 8) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, me2, 8); \ + if (t <= 9) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, me2, 9); \ + if (t <= 10) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, me2, 10); \ + if (t <= 11) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, me2, 11); \ + if (t <= 12) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, me2, 12); \ + if (t <= 13) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, me2, 13); \ + if (t <= 14) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, me2, 14); \ + if (t <= 15) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(a, b, c, d, e, me2, 15); \ + if (t <= 16) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(e, a, b, c, d, me2, 16); \ + if (t <= 17) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(d, e, a, b, c, me2, 17); \ + if (t <= 18) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(c, d, e, a, b, me2, 18); \ + if (t <= 19) HASHCLASH_SHA1COMPRESS_ROUND1_STEP(b, c, d, e, a, me2, 19); \ + if (t <= 20) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, me2, 20); \ + if (t <= 21) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, me2, 21); \ + if (t <= 22) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, me2, 22); \ + if (t <= 23) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, me2, 23); \ + if (t <= 24) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, me2, 24); \ + if (t <= 25) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, me2, 25); \ + if (t <= 26) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, me2, 26); \ + if (t <= 27) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, me2, 27); \ + if (t <= 28) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, me2, 28); \ + if (t <= 29) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, me2, 29); \ + if (t <= 30) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, me2, 30); \ + if (t <= 31) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, me2, 31); \ + if (t <= 32) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, me2, 32); \ + if (t <= 33) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, me2, 33); \ + if (t <= 34) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, me2, 34); \ + if (t <= 35) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(a, b, c, d, e, me2, 35); \ + if (t <= 36) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(e, a, b, c, d, me2, 36); \ + if (t <= 37) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(d, e, a, b, c, me2, 37); \ + if (t <= 38) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(c, d, e, a, b, me2, 38); \ + if (t <= 39) HASHCLASH_SHA1COMPRESS_ROUND2_STEP(b, c, d, e, a, me2, 39); \ + if (t <= 40) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, me2, 40); \ + if (t <= 41) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, me2, 41); \ + if (t <= 42) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, me2, 42); \ + if (t <= 43) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, me2, 43); \ + if (t <= 44) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, me2, 44); \ + if (t <= 45) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, me2, 45); \ + if (t <= 46) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, me2, 46); \ + if (t <= 47) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, me2, 47); \ + if (t <= 48) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, me2, 48); \ + if (t <= 49) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, me2, 49); \ + if (t <= 50) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, me2, 50); \ + if (t <= 51) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, me2, 51); \ + if (t <= 52) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, me2, 52); \ + if (t <= 53) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, me2, 53); \ + if (t <= 54) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, me2, 54); \ + if (t <= 55) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(a, b, c, d, e, me2, 55); \ + if (t <= 56) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(e, a, b, c, d, me2, 56); \ + if (t <= 57) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(d, e, a, b, c, me2, 57); \ + if (t <= 58) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(c, d, e, a, b, me2, 58); \ + if (t <= 59) HASHCLASH_SHA1COMPRESS_ROUND3_STEP(b, c, d, e, a, me2, 59); \ + if (t <= 60) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, me2, 60); \ + if (t <= 61) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, me2, 61); \ + if (t <= 62) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, me2, 62); \ + if (t <= 63) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, me2, 63); \ + if (t <= 64) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, me2, 64); \ + if (t <= 65) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, me2, 65); \ + if (t <= 66) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, me2, 66); \ + if (t <= 67) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, me2, 67); \ + if (t <= 68) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, me2, 68); \ + if (t <= 69) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, me2, 69); \ + if (t <= 70) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, me2, 70); \ + if (t <= 71) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, me2, 71); \ + if (t <= 72) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, me2, 72); \ + if (t <= 73) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, me2, 73); \ + if (t <= 74) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, me2, 74); \ + if (t <= 75) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(a, b, c, d, e, me2, 75); \ + if (t <= 76) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(e, a, b, c, d, me2, 76); \ + if (t <= 77) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(d, e, a, b, c, me2, 77); \ + if (t <= 78) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(c, d, e, a, b, me2, 78); \ + if (t <= 79) HASHCLASH_SHA1COMPRESS_ROUND4_STEP(b, c, d, e, a, me2, 79); \ + ihvout[0] = ihvin[0] + a; ihvout[1] = ihvin[1] + b; ihvout[2] = ihvin[2] + c; ihvout[3] = ihvin[3] + d; ihvout[4] = ihvin[4] + e; \ +} + +SHA1_RECOMPRESS(0) +SHA1_RECOMPRESS(1) +SHA1_RECOMPRESS(2) +SHA1_RECOMPRESS(3) +SHA1_RECOMPRESS(4) +SHA1_RECOMPRESS(5) +SHA1_RECOMPRESS(6) +SHA1_RECOMPRESS(7) +SHA1_RECOMPRESS(8) +SHA1_RECOMPRESS(9) + +SHA1_RECOMPRESS(10) +SHA1_RECOMPRESS(11) +SHA1_RECOMPRESS(12) +SHA1_RECOMPRESS(13) +SHA1_RECOMPRESS(14) +SHA1_RECOMPRESS(15) +SHA1_RECOMPRESS(16) +SHA1_RECOMPRESS(17) +SHA1_RECOMPRESS(18) +SHA1_RECOMPRESS(19) + +SHA1_RECOMPRESS(20) +SHA1_RECOMPRESS(21) +SHA1_RECOMPRESS(22) +SHA1_RECOMPRESS(23) +SHA1_RECOMPRESS(24) +SHA1_RECOMPRESS(25) +SHA1_RECOMPRESS(26) +SHA1_RECOMPRESS(27) +SHA1_RECOMPRESS(28) +SHA1_RECOMPRESS(29) + +SHA1_RECOMPRESS(30) +SHA1_RECOMPRESS(31) +SHA1_RECOMPRESS(32) +SHA1_RECOMPRESS(33) +SHA1_RECOMPRESS(34) +SHA1_RECOMPRESS(35) +SHA1_RECOMPRESS(36) +SHA1_RECOMPRESS(37) +SHA1_RECOMPRESS(38) +SHA1_RECOMPRESS(39) + +SHA1_RECOMPRESS(40) +SHA1_RECOMPRESS(41) +SHA1_RECOMPRESS(42) +SHA1_RECOMPRESS(43) +SHA1_RECOMPRESS(44) +SHA1_RECOMPRESS(45) +SHA1_RECOMPRESS(46) +SHA1_RECOMPRESS(47) +SHA1_RECOMPRESS(48) +SHA1_RECOMPRESS(49) + +SHA1_RECOMPRESS(50) +SHA1_RECOMPRESS(51) +SHA1_RECOMPRESS(52) +SHA1_RECOMPRESS(53) +SHA1_RECOMPRESS(54) +SHA1_RECOMPRESS(55) +SHA1_RECOMPRESS(56) +SHA1_RECOMPRESS(57) +SHA1_RECOMPRESS(58) +SHA1_RECOMPRESS(59) + +SHA1_RECOMPRESS(60) +SHA1_RECOMPRESS(61) +SHA1_RECOMPRESS(62) +SHA1_RECOMPRESS(63) +SHA1_RECOMPRESS(64) +SHA1_RECOMPRESS(65) +SHA1_RECOMPRESS(66) +SHA1_RECOMPRESS(67) +SHA1_RECOMPRESS(68) +SHA1_RECOMPRESS(69) + +SHA1_RECOMPRESS(70) +SHA1_RECOMPRESS(71) +SHA1_RECOMPRESS(72) +SHA1_RECOMPRESS(73) +SHA1_RECOMPRESS(74) +SHA1_RECOMPRESS(75) +SHA1_RECOMPRESS(76) +SHA1_RECOMPRESS(77) +SHA1_RECOMPRESS(78) +SHA1_RECOMPRESS(79) + +static sha1_recompression_type sha1_recompression_step[80] = +{ + sha1recompress_fast_0, sha1recompress_fast_1, sha1recompress_fast_2, sha1recompress_fast_3, sha1recompress_fast_4, sha1recompress_fast_5, sha1recompress_fast_6, sha1recompress_fast_7, sha1recompress_fast_8, sha1recompress_fast_9, + sha1recompress_fast_10, sha1recompress_fast_11, sha1recompress_fast_12, sha1recompress_fast_13, sha1recompress_fast_14, sha1recompress_fast_15, sha1recompress_fast_16, sha1recompress_fast_17, sha1recompress_fast_18, sha1recompress_fast_19, + sha1recompress_fast_20, sha1recompress_fast_21, sha1recompress_fast_22, sha1recompress_fast_23, sha1recompress_fast_24, sha1recompress_fast_25, sha1recompress_fast_26, sha1recompress_fast_27, sha1recompress_fast_28, sha1recompress_fast_29, + sha1recompress_fast_30, sha1recompress_fast_31, sha1recompress_fast_32, sha1recompress_fast_33, sha1recompress_fast_34, sha1recompress_fast_35, sha1recompress_fast_36, sha1recompress_fast_37, sha1recompress_fast_38, sha1recompress_fast_39, + sha1recompress_fast_40, sha1recompress_fast_41, sha1recompress_fast_42, sha1recompress_fast_43, sha1recompress_fast_44, sha1recompress_fast_45, sha1recompress_fast_46, sha1recompress_fast_47, sha1recompress_fast_48, sha1recompress_fast_49, + sha1recompress_fast_50, sha1recompress_fast_51, sha1recompress_fast_52, sha1recompress_fast_53, sha1recompress_fast_54, sha1recompress_fast_55, sha1recompress_fast_56, sha1recompress_fast_57, sha1recompress_fast_58, sha1recompress_fast_59, + sha1recompress_fast_60, sha1recompress_fast_61, sha1recompress_fast_62, sha1recompress_fast_63, sha1recompress_fast_64, sha1recompress_fast_65, sha1recompress_fast_66, sha1recompress_fast_67, sha1recompress_fast_68, sha1recompress_fast_69, + sha1recompress_fast_70, sha1recompress_fast_71, sha1recompress_fast_72, sha1recompress_fast_73, sha1recompress_fast_74, sha1recompress_fast_75, sha1recompress_fast_76, sha1recompress_fast_77, sha1recompress_fast_78, sha1recompress_fast_79, +}; + +static void sha1_process(fsl_sha1_cx* ctx, const uint32_t block[16]) +{ + unsigned i, j; + uint32_t ubc_dv_mask[DVMASKSIZE]; + uint32_t ihvtmp[5]; + for (i=0; i < DVMASKSIZE; ++i) + ubc_dv_mask[i]=0; + ctx->ihv1[0] = ctx->ihv[0]; + ctx->ihv1[1] = ctx->ihv[1]; + ctx->ihv1[2] = ctx->ihv[2]; + ctx->ihv1[3] = ctx->ihv[3]; + ctx->ihv1[4] = ctx->ihv[4]; + memcpy(ctx->m1, block, 64); + sha1_message_expansion(ctx->m1); + if (ctx->detect_coll && ctx->ubc_check) + { + ubc_check(ctx->m1, ubc_dv_mask); + } + sha1_compression_states(ctx->ihv, ctx->m1, ctx->states); + if (ctx->detect_coll) + { + for (i = 0; sha1_dvs[i].dvType != 0; ++i) + { + if ((0 == ctx->ubc_check) || (((uint32_t)(1) << sha1_dvs[i].maskb) & ubc_dv_mask[sha1_dvs[i].maski])) + { + for (j = 0; j < 80; ++j) + ctx->m2[j] = ctx->m1[j] ^ sha1_dvs[i].dm[j]; + (sha1_recompression_step[sha1_dvs[i].testt])(ctx->ihv2, ihvtmp, ctx->m2, ctx->states[sha1_dvs[i].testt]); + /* to verify SHA-1 collision detection code with collisions for reduced-step SHA-1 */ + if ((ihvtmp[0] == ctx->ihv[0] && ihvtmp[1] == ctx->ihv[1] && ihvtmp[2] == ctx->ihv[2] && ihvtmp[3] == ctx->ihv[3] && ihvtmp[4] == ctx->ihv[4]) + || (ctx->reduced_round_coll && ctx->ihv1[0] == ctx->ihv2[0] && ctx->ihv1[1] == ctx->ihv2[1] && ctx->ihv1[2] == ctx->ihv2[2] && ctx->ihv1[3] == ctx->ihv2[3] && ctx->ihv1[4] == ctx->ihv2[4])) + { + ctx->found_collision = 1; + /* TODO: call callback */ + if (ctx->callback != NULL) + ctx->callback(ctx->total - 64, ctx->ihv1, ctx->ihv2, ctx->m1, ctx->m2); + + if (ctx->safe_hash) + { + sha1_compression_W(ctx->ihv, ctx->m1); + sha1_compression_W(ctx->ihv, ctx->m1); + } + + break; + } + } + } + } +} + +static void swap_bytes(uint32_t val[16]) +{ + unsigned i; + for (i = 0; i < 16; ++i) + { + val[i] = ((val[i] << 8) & 0xFF00FF00) | ((val[i] >> 8) & 0xFF00FF); + val[i] = (val[i] << 16) | (val[i] >> 16); + } +} + + +static const unsigned char sha1_padding[64] = +{ + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +#undef DVMASKSIZE +#undef DOSTORESTATE58 +#undef DOSTORESTATE65 +#undef rotate_right +#undef rotate_left +#undef sha1_f1 +#undef sha1_f2 +#undef sha1_f3 +#undef sha1_f4 +#undef HASHCLASH_SHA1COMPRESS_ROUND1_STEP +#undef HASHCLASH_SHA1COMPRESS_ROUND2_STEP +#undef HASHCLASH_SHA1COMPRESS_ROUND3_STEP +#undef HASHCLASH_SHA1COMPRESS_ROUND4_STEP +#undef HASHCLASH_SHA1COMPRESS_ROUND1_STEP_BW +#undef HASHCLASH_SHA1COMPRESS_ROUND2_STEP_BW +#undef HASHCLASH_SHA1COMPRESS_ROUND3_STEP_BW +#undef HASHCLASH_SHA1COMPRESS_ROUND4_STEP_BW +#undef SHA1_STORE_STATE +#undef SHA1_RECOMPRESS + +/************************************************************************/ +/* End FSL_SHA1_HARDENED */ +/************************************************************************/ +#else /* end FSL_SHA1_HARDENED */ +/************************************************************************/ +/* Standard SHA1... */ +/************************************************************************/ + +/* + The SHA1 implementation below is adapted from: + + $NetBSD: sha1.c,v 1.6 2009/11/06 20:31:18 joerg Exp $ + $OpenBSD: sha1.c,v 1.9 1997/07/23 21:12:32 kstailey Exp $ + + SHA-1 in C + By Steve Reid + 100% Public Domain +*/ + +/* + * blk0() and blk() perform the initial expand. + * I got the idea of expanding during the round function from SSLeay + * + * blk0le() for little-endian and blk0be() for big-endian. + */ +#if 0 && __GNUC__ && (defined(__i386__) || defined(__x86_64__)) +/* + * GCC by itself only generates left rotates. Use right rotates if + * possible to be kinder to dinky implementations with iterative rotate + * instructions. + */ +#define SHA_ROT(op, x, k) \ + ({ unsigned int y; asm(op " %1,%0" : "=r" (y) : "I" (k), "0" (x)); y; }) +#define rol(x,k) SHA_ROT("roll", x, k) +#define ror(x,k) SHA_ROT("rorl", x, k) + +#else +/* Generic C equivalent */ +#define SHA_ROT(x,l,r) ((x) << (l) | (x) >> (r)) +#define rol(x,k) SHA_ROT(x,k,32-(k)) +#define ror(x,k) SHA_ROT(x,32-(k),k) +#endif + + +#define blk0le(i) (block[i] = (ror(block[i],8)&0xFF00FF00) \ + |(rol(block[i],8)&0x00FF00FF)) +#define blk0be(i) block[i] +#define blk(i) (block[i&15] = rol(block[(i+13)&15]^block[(i+8)&15] \ + ^block[(i+2)&15]^block[i&15],1)) + +/* + * (R0+R1), R2, R3, R4 are the different operations (rounds) used in SHA1 + * + * Rl0() for little-endian and Rb0() for big-endian. Endianness is + * determined at run-time. + */ +#define Rl0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0le(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define Rb0(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk0be(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R1(v,w,x,y,z,i) \ + z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=ror(w,2); +#define R2(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=ror(w,2); +#define R3(v,w,x,y,z,i) \ + z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=ror(w,2); +#define R4(v,w,x,y,z,i) \ + z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=ror(w,2); + +/* + * Hash a single 512-bit block. This is the core of the algorithm. + */ +#define a qq[0] +#define b qq[1] +#define c qq[2] +#define d qq[3] +#define e qq[4] + +static void SHA1Transform(unsigned int state[5], const unsigned char buffer[64]) +{ + unsigned int qq[5]; /* a, b, c, d, e; */ + static int one = 1; + unsigned int block[16]; + memcpy(block, buffer, 64); + memcpy(qq,state,5*sizeof(unsigned int)); + + /* Copy context->state[] to working vars */ + /* + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + */ + + /* 4 rounds of 20 operations each. Loop unrolled. */ + if( 1 == *(unsigned char*)&one ){ + Rl0(a,b,c,d,e, 0); Rl0(e,a,b,c,d, 1); Rl0(d,e,a,b,c, 2); Rl0(c,d,e,a,b, 3); + Rl0(b,c,d,e,a, 4); Rl0(a,b,c,d,e, 5); Rl0(e,a,b,c,d, 6); Rl0(d,e,a,b,c, 7); + Rl0(c,d,e,a,b, 8); Rl0(b,c,d,e,a, 9); Rl0(a,b,c,d,e,10); Rl0(e,a,b,c,d,11); + Rl0(d,e,a,b,c,12); Rl0(c,d,e,a,b,13); Rl0(b,c,d,e,a,14); Rl0(a,b,c,d,e,15); + }else{ + Rb0(a,b,c,d,e, 0); Rb0(e,a,b,c,d, 1); Rb0(d,e,a,b,c, 2); Rb0(c,d,e,a,b, 3); + Rb0(b,c,d,e,a, 4); Rb0(a,b,c,d,e, 5); Rb0(e,a,b,c,d, 6); Rb0(d,e,a,b,c, 7); + Rb0(c,d,e,a,b, 8); Rb0(b,c,d,e,a, 9); Rb0(a,b,c,d,e,10); Rb0(e,a,b,c,d,11); + Rb0(d,e,a,b,c,12); Rb0(c,d,e,a,b,13); Rb0(b,c,d,e,a,14); Rb0(a,b,c,d,e,15); + } + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; +} + +/* + The state of a incremental SHA1 checksum computation. Only one + such computation can be underway at a time, of course. +*/ +/* static fsl_sha1_cx incrCtx; */ + +#if 0 +/* + Add more text to the incremental SHA1 checksum. +*/ +static void fsl_sha1sum_step_text(fsl_sha1_cx * cx, const char *zText, int nBytes){ + if( nBytes<=0 ){ + if( nBytes==0 ) return; + nBytes = fsl_strlen(zText); + } + fsl_sha1_update(cx, (unsigned char*)zText, nBytes); +} +#endif + +#if 0 +/* + Add the content of a blob to the incremental SHA1 checksum. +*/ +static void fsl_sha1sum_step_buffer(fsl_sha1_cx * cx, fsl_buffer *b){ + fsl_sha1sum_step_text(cx, fsl_buffer_cstr(b), b->used); +} +#endif + +#undef SHA_ROT +#undef rol +#undef ror +#undef blk0le +#undef blk0be +#undef blk + +#undef Rl0 +#undef Rb0 +#undef R1 +#undef R2 +#undef R3 +#undef R4 +#undef a +#undef b +#undef c +#undef d +#undef e +#endif /* Standard SHA1 */ +/************************************************************************/ +/* End main hash-type-specific code. Common code follows... */ +/************************************************************************/ + +void fsl_sha1_init(fsl_sha1_cx *ctx){ + /* SHA1 initialization constants */ +#if FSL_SHA1_HARDENED + static const union { unsigned char bytes[4]; uint32_t value; } endianness = { { 0, 1, 2, 3 } }; + static const uint32_t littleendian = 0x03020100; + memset(ctx, 0, sizeof(*ctx)); + ctx->total = 0; + ctx->ihv[0] = 0x67452301; + ctx->ihv[1] = 0xEFCDAB89; + ctx->ihv[2] = 0x98BADCFE; + ctx->ihv[3] = 0x10325476; + ctx->ihv[4] = 0xC3D2E1F0; + ctx->found_collision = 0; + ctx->safe_hash = 1; + ctx->ubc_check = 1; + ctx->detect_coll = 1; + ctx->reduced_round_coll = 0; + ctx->bigendian = (endianness.value != littleendian); + ctx->callback = NULL; +#else + *ctx = fsl_sha1_cx_empty; +#endif +} + + +/* + * Run your data through this. + */ +void fsl_sha1_update( fsl_sha1_cx *ctx, void const * data_, + fsl_size_t len ){ +#if FSL_SHA1_HARDENED + const unsigned char *buf = (const unsigned char *)data_; + unsigned left, fill; + if (len == 0) + return; + left = ctx->total & 63; + fill = 64 - left; + if (left && len >= fill) + { + ctx->total += fill; + memcpy(ctx->buffer + left, buf, fill); + if (!ctx->bigendian) + swap_bytes((uint32_t*)(ctx->buffer)); + sha1_process(ctx, (uint32_t*)(ctx->buffer)); + buf += fill; + len -= fill; + left = 0; + } + while (len >= 64) + { + ctx->total += 64; + if (!ctx->bigendian) + { + memcpy(ctx->buffer, buf, 64); + swap_bytes((uint32_t*)(ctx->buffer)); + sha1_process(ctx, (uint32_t*)(ctx->buffer)); + } + else + sha1_process(ctx, (uint32_t*)(buf)); + buf += 64; + len -= 64; + } + if (len > 0) + { + ctx->total += len; + memcpy(ctx->buffer + left, buf, len); + } +#else + const unsigned char *data = (const unsigned char *)data_; + unsigned int i, j; + j = ctx->count[0]; + if ((ctx->count[0] += len << 3) < j) + ctx->count[1] += (len>>29)+1; + j = (j >> 3) & 63; + if ((j + len) > 63) { + memcpy(&ctx->buffer[j], data, (i = 64-j)); + SHA1Transform(ctx->state, ctx->buffer); + for ( ; i + 63 < len; i += 64) + SHA1Transform(ctx->state, &data[i]); + j = 0; + } else { + i = 0; + } + (void)memcpy(&ctx->buffer[j], &data[i], len - i); +#endif +} + +/* + Convert a digest into base-16. digest should be declared as + "unsigned char digest[20]" in the calling function. The SHA1 + digest is stored in the first 20 bytes. zBuf should + be "char zBuf[41]". +*/ +void fsl_sha1_digest_to_base16(unsigned char *digest, char *zBuf){ + static char const zEncode[] = "0123456789abcdef"; + int ix; + + for(ix=0; ix>4)&0xf]; + *zBuf++ = zEncode[*digest++ & 0xf]; + } + *zBuf = '\0'; +} + + + +int fsl_sha1_final(fsl_sha1_cx *ctx, unsigned char * digest){ +#if FSL_SHA1_HARDENED + uint32_t last = ctx->total & 63; + uint32_t padn = (last < 56) ? (56 - last) : (120 - last); + uint64_t total; + fsl_sha1_update(ctx, sha1_padding, padn); + + total = ctx->total - padn; + total <<= 3; + ctx->buffer[56] = (unsigned char)(total >> 56); + ctx->buffer[57] = (unsigned char)(total >> 48); + ctx->buffer[58] = (unsigned char)(total >> 40); + ctx->buffer[59] = (unsigned char)(total >> 32); + ctx->buffer[60] = (unsigned char)(total >> 24); + ctx->buffer[61] = (unsigned char)(total >> 16); + ctx->buffer[62] = (unsigned char)(total >> 8); + ctx->buffer[63] = (unsigned char)(total); + if (!ctx->bigendian) + swap_bytes((uint32_t*)(ctx->buffer)); + sha1_process(ctx, (uint32_t*)(ctx->buffer)); + digest[0] = (unsigned char)(ctx->ihv[0] >> 24); + digest[1] = (unsigned char)(ctx->ihv[0] >> 16); + digest[2] = (unsigned char)(ctx->ihv[0] >> 8); + digest[3] = (unsigned char)(ctx->ihv[0]); + digest[4] = (unsigned char)(ctx->ihv[1] >> 24); + digest[5] = (unsigned char)(ctx->ihv[1] >> 16); + digest[6] = (unsigned char)(ctx->ihv[1] >> 8); + digest[7] = (unsigned char)(ctx->ihv[1]); + digest[8] = (unsigned char)(ctx->ihv[2] >> 24); + digest[9] = (unsigned char)(ctx->ihv[2] >> 16); + digest[10] = (unsigned char)(ctx->ihv[2] >> 8); + digest[11] = (unsigned char)(ctx->ihv[2]); + digest[12] = (unsigned char)(ctx->ihv[3] >> 24); + digest[13] = (unsigned char)(ctx->ihv[3] >> 16); + digest[14] = (unsigned char)(ctx->ihv[3] >> 8); + digest[15] = (unsigned char)(ctx->ihv[3]); + digest[16] = (unsigned char)(ctx->ihv[4] >> 24); + digest[17] = (unsigned char)(ctx->ihv[4] >> 16); + digest[18] = (unsigned char)(ctx->ihv[4] >> 8); + digest[19] = (unsigned char)(ctx->ihv[4]); + return ctx->found_collision; +#else + unsigned int i; + unsigned char finalcount[8]; + for (i = 0; i < 8; i++) { + finalcount[i] = (unsigned char)((ctx->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + fsl_sha1_update(ctx, (const unsigned char *)"\200", 1); + while ((ctx->count[0] & 504) != 448){ + fsl_sha1_update(ctx, (const unsigned char *)"\0", 1); + } + fsl_sha1_update(ctx, finalcount, 8); /* Should cause a SHA1Transform() */ + if (digest) { + for (i = 0; i < 20; i++){ + digest[i] = (unsigned char) + ((ctx->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + } + return 0; +#endif +} + +char const * fsl_sha1_final_hex(fsl_sha1_cx *context, char * zHex){ + unsigned char zResult[FSL_STRLEN_SHA1/2]; + fsl_sha1_final(context, zResult); + fsl_sha1_digest_to_base16(zResult, zHex); + return (char const *)zHex; +} + +int fsl_sha1sum_stream(fsl_input_f src, void * srcState, fsl_buffer *pCksum){ + enum { BufSize = 1024 * 4 }; + fsl_sha1_cx ctx; + int rc; + unsigned char zBuf[BufSize]; + if(!src || !pCksum) return FSL_RC_MISUSE; + fsl_sha1_init(&ctx); + for(;;){ + fsl_size_t read = (fsl_size_t)BufSize; + rc = src(srcState, zBuf, &read); + if(rc) return rc; + else if(read) fsl_sha1_update(&ctx, (unsigned char*)zBuf, read); + if(read < (fsl_size_t)BufSize) break; + } + fsl_buffer_reuse(pCksum); + rc = fsl_buffer_reserve(pCksum, FSL_STRLEN_SHA1+1/*NUL*/); + /*^^^^ DO NOT fsl_buffer_resize(), as pCksum is, more often than not, + a cached reused buffer. */ + if(!rc){ + fsl_sha1_final_hex(&ctx, fsl_buffer_str(pCksum)); + pCksum->used = (fsl_size_t)FSL_STRLEN_SHA1; + pCksum->mem[pCksum->used] = 0; + } + return rc; +} + +int fsl_sha1sum_filename(const char *zFilename, fsl_buffer *pCksum){ + if(!zFilename || !pCksum) return FSL_RC_MISUSE; + else{ +#if 1 + int rc; + FILE *in = fsl_fopen(zFilename, "rb"); + if(!in) rc = FSL_RC_IO; + else{ + rc = fsl_sha1sum_stream(fsl_input_f_FILE, in, pCksum); + fsl_fclose(in); + } + return rc; +#else + /* Requires v1 code which has not yet been ported in. */ + FILE *in; + fsl_sha1_cx ctx; + unsigned char zResult[FSL_STRLEN_SHA1/2]; + char zBuf[10240]; + + if( fsl_wd_islink(zFilename) ){ + /* Instead of file content, return sha1 of link destination path */ + Blob destinationPath; + int rc; + + blob_read_link(&destinationPath, zFilename); + rc = sha1sum_blob(&destinationPath, pCksum); + blob_reset(&destinationPath); + return rc; + } + + in = fossil_fopen(zFilename,"rb"); + if( in==0 ){ + return 1; + } + fsl_sha1_init(&ctx); + for(;;){ + int n; + n = fread(zBuf, 1, sizeof(zBuf), in); + if( n<=0 ) break; + fsl_sha1_update(&ctx, (unsigned char*)zBuf, (unsigned)n); + } + fclose_fclose(in); + blob_zero(pCksum); + blob_resize(pCksum, FSL_STRLEN_SHA1); + fsl_sha1_final(&ctx, zResult); + fsl_sha1_digest_to_base16(zResult, blob_buffer(pCksum)); + return 0; +#endif + } +} + +int fsl_sha1sum_buffer(fsl_buffer const *pIn, fsl_buffer *pCksum){ + if(!pIn || !pCksum) return FSL_RC_MISUSE; + else{ + fsl_sha1_cx ctx; + int rc; + fsl_sha1_init(&ctx); + fsl_sha1_update(&ctx, pIn->mem, pIn->used); + rc = fsl_buffer_reserve(pCksum, FSL_STRLEN_SHA1+1/*NUL*/); + if(!rc){ + fsl_buffer_reuse(pCksum); + fsl_sha1_final_hex(&ctx, fsl_buffer_str(pCksum)); + pCksum->used = (fsl_size_t)FSL_STRLEN_SHA1; + pCksum->mem[pCksum->used] = 0; + } + return rc; + } +} + +char *fsl_sha1sum_cstr(const char *zIn, fsl_int_t len){ + if(!zIn || !len) return NULL; + else{ + fsl_sha1_cx ctx; + char * zHex = (char *)fsl_malloc(FSL_STRLEN_SHA1+1); + if(!zHex) return NULL; + fsl_sha1_init(&ctx); + fsl_sha1_update(&ctx, zIn, + (len<0) ? fsl_strlen(zIn) : (fsl_size_t)len); + fsl_sha1_final_hex(&ctx, zHex); + return zHex; + } +} +/* end of file sha1.c */ +/* start of file sha3.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* +** Copyright (c) 2017 D. Richard Hipp +** +** This program is free software; you can redistribute it and/or +** modify it under the terms of the Simplified BSD License (also +** known as the "2-Clause License" or "FreeBSD License".) +** +** This program is distributed in the hope that it will be useful, +** but without any warranty; without even the implied warranty of +** merchantability or fitness for a particular purpose. +** +** Author contact information: +** drh@hwaci.com +** http://www.hwaci.com/drh/ +** +******************************************************************************* +** +** This file contains an implementation of SHA3 (Keccak) hashing. +*/ +/** + This copy was modified slightly for use with the libfossil API. +*/ +#include +#include /* strlen() */ +#include /* NULL on linux */ +#include +#include + +#if 0 +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) +#endif + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSHA3_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SHA3_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SHA3_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) +# define SHA3_BYTEORDER 4321 +# else +# define SHA3_BYTEORDER 0 +# endif +#endif + +/* +** A single step of the Keccak mixing function for a 1600-bit state +*/ +static void KeccakF1600Step(fsl_sha3_cx *p){ + int i; + uint64_t B0, B1, B2, B3, B4; + uint64_t C0, C1, C2, C3, C4; + uint64_t D0, D1, D2, D3, D4; + static const uint64_t RC[] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL + }; +# define A00 (p->u.s[0]) +# define A01 (p->u.s[1]) +# define A02 (p->u.s[2]) +# define A03 (p->u.s[3]) +# define A04 (p->u.s[4]) +# define A10 (p->u.s[5]) +# define A11 (p->u.s[6]) +# define A12 (p->u.s[7]) +# define A13 (p->u.s[8]) +# define A14 (p->u.s[9]) +# define A20 (p->u.s[10]) +# define A21 (p->u.s[11]) +# define A22 (p->u.s[12]) +# define A23 (p->u.s[13]) +# define A24 (p->u.s[14]) +# define A30 (p->u.s[15]) +# define A31 (p->u.s[16]) +# define A32 (p->u.s[17]) +# define A33 (p->u.s[18]) +# define A34 (p->u.s[19]) +# define A40 (p->u.s[20]) +# define A41 (p->u.s[21]) +# define A42 (p->u.s[22]) +# define A43 (p->u.s[23]) +# define A44 (p->u.s[24]) +# define ROL64(a,x) ((a<>(64-x))) + + for(i=0; i<24; i+=4){ + C0 = A00^A10^A20^A30^A40; + C1 = A01^A11^A21^A31^A41; + C2 = A02^A12^A22^A32^A42; + C3 = A03^A13^A23^A33^A43; + C4 = A04^A14^A24^A34^A44; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A11^D1), 44); + B2 = ROL64((A22^D2), 43); + B3 = ROL64((A33^D3), 21); + B4 = ROL64((A44^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i]; + A11 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A20^D0), 3); + B3 = ROL64((A31^D1), 45); + B4 = ROL64((A42^D2), 61); + B0 = ROL64((A03^D3), 28); + B1 = ROL64((A14^D4), 20); + A20 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A40^D0), 18); + B0 = ROL64((A01^D1), 1); + B1 = ROL64((A12^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A34^D4), 8); + A40 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A10^D0), 36); + B2 = ROL64((A21^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A43^D3), 56); + B0 = ROL64((A04^D4), 27); + A10 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A30^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A02^D2), 62); + B1 = ROL64((A13^D3), 55); + B2 = ROL64((A24^D4), 39); + A30 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + C0 = A00^A20^A40^A10^A30; + C1 = A11^A31^A01^A21^A41; + C2 = A22^A42^A12^A32^A02; + C3 = A33^A03^A23^A43^A13; + C4 = A44^A14^A34^A04^A24; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A31^D1), 44); + B2 = ROL64((A12^D2), 43); + B3 = ROL64((A43^D3), 21); + B4 = ROL64((A24^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+1]; + A31 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A40^D0), 3); + B3 = ROL64((A21^D1), 45); + B4 = ROL64((A02^D2), 61); + B0 = ROL64((A33^D3), 28); + B1 = ROL64((A14^D4), 20); + A40 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A30^D0), 18); + B0 = ROL64((A11^D1), 1); + B1 = ROL64((A42^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A04^D4), 8); + A30 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A20^D0), 36); + B2 = ROL64((A01^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A13^D3), 56); + B0 = ROL64((A44^D4), 27); + A20 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A10^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A22^D2), 62); + B1 = ROL64((A03^D3), 55); + B2 = ROL64((A34^D4), 39); + A10 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + C0 = A00^A40^A30^A20^A10; + C1 = A31^A21^A11^A01^A41; + C2 = A12^A02^A42^A32^A22; + C3 = A43^A33^A23^A13^A03; + C4 = A24^A14^A04^A44^A34; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A21^D1), 44); + B2 = ROL64((A42^D2), 43); + B3 = ROL64((A13^D3), 21); + B4 = ROL64((A34^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+2]; + A21 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A30^D0), 3); + B3 = ROL64((A01^D1), 45); + B4 = ROL64((A22^D2), 61); + B0 = ROL64((A43^D3), 28); + B1 = ROL64((A14^D4), 20); + A30 = B0 ^((~B1)& B2 ); + A01 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A10^D0), 18); + B0 = ROL64((A31^D1), 1); + B1 = ROL64((A02^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A44^D4), 8); + A10 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A40^D0), 36); + B2 = ROL64((A11^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A03^D3), 56); + B0 = ROL64((A24^D4), 27); + A40 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A20^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A12^D2), 62); + B1 = ROL64((A33^D3), 55); + B2 = ROL64((A04^D4), 39); + A20 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + C0 = A00^A30^A10^A40^A20; + C1 = A21^A01^A31^A11^A41; + C2 = A42^A22^A02^A32^A12; + C3 = A13^A43^A23^A03^A33; + C4 = A34^A14^A44^A24^A04; + D0 = C4^ROL64(C1, 1); + D1 = C0^ROL64(C2, 1); + D2 = C1^ROL64(C3, 1); + D3 = C2^ROL64(C4, 1); + D4 = C3^ROL64(C0, 1); + + B0 = (A00^D0); + B1 = ROL64((A01^D1), 44); + B2 = ROL64((A02^D2), 43); + B3 = ROL64((A03^D3), 21); + B4 = ROL64((A04^D4), 14); + A00 = B0 ^((~B1)& B2 ); + A00 ^= RC[i+3]; + A01 = B1 ^((~B2)& B3 ); + A02 = B2 ^((~B3)& B4 ); + A03 = B3 ^((~B4)& B0 ); + A04 = B4 ^((~B0)& B1 ); + + B2 = ROL64((A10^D0), 3); + B3 = ROL64((A11^D1), 45); + B4 = ROL64((A12^D2), 61); + B0 = ROL64((A13^D3), 28); + B1 = ROL64((A14^D4), 20); + A10 = B0 ^((~B1)& B2 ); + A11 = B1 ^((~B2)& B3 ); + A12 = B2 ^((~B3)& B4 ); + A13 = B3 ^((~B4)& B0 ); + A14 = B4 ^((~B0)& B1 ); + + B4 = ROL64((A20^D0), 18); + B0 = ROL64((A21^D1), 1); + B1 = ROL64((A22^D2), 6); + B2 = ROL64((A23^D3), 25); + B3 = ROL64((A24^D4), 8); + A20 = B0 ^((~B1)& B2 ); + A21 = B1 ^((~B2)& B3 ); + A22 = B2 ^((~B3)& B4 ); + A23 = B3 ^((~B4)& B0 ); + A24 = B4 ^((~B0)& B1 ); + + B1 = ROL64((A30^D0), 36); + B2 = ROL64((A31^D1), 10); + B3 = ROL64((A32^D2), 15); + B4 = ROL64((A33^D3), 56); + B0 = ROL64((A34^D4), 27); + A30 = B0 ^((~B1)& B2 ); + A31 = B1 ^((~B2)& B3 ); + A32 = B2 ^((~B3)& B4 ); + A33 = B3 ^((~B4)& B0 ); + A34 = B4 ^((~B0)& B1 ); + + B3 = ROL64((A40^D0), 41); + B4 = ROL64((A41^D1), 2); + B0 = ROL64((A42^D2), 62); + B1 = ROL64((A43^D3), 55); + B2 = ROL64((A44^D4), 39); + A40 = B0 ^((~B1)& B2 ); + A41 = B1 ^((~B2)& B3 ); + A42 = B2 ^((~B3)& B4 ); + A43 = B3 ^((~B4)& B0 ); + A44 = B4 ^((~B0)& B1 ); + } +# undef A00 +# undef A01 +# undef A02 +# undef A03 +# undef A04 +# undef A10 +# undef A11 +# undef A12 +# undef A13 +# undef A14 +# undef A20 +# undef A21 +# undef A22 +# undef A23 +# undef A24 +# undef A30 +# undef A31 +# undef A32 +# undef A33 +# undef A34 +# undef A40 +# undef A41 +# undef A42 +# undef A43 +# undef A44 +# undef ROL64 +} + +enum fsl_sha3_hash_size fsl_sha3_hash_size_for_int(int n){ + switch(n){ + case 128: return FSL_SHA3_128; case 160: return FSL_SHA3_160; + case 192: return FSL_SHA3_192; case 224: return FSL_SHA3_224; + case 256: return FSL_SHA3_256; case 288: return FSL_SHA3_288; + case 320: return FSL_SHA3_320; case 352: return FSL_SHA3_352; + case 384: return FSL_SHA3_384; case 416: return FSL_SHA3_416; + case 448: return FSL_SHA3_448; case 480: return FSL_SHA3_480; + case 512: return FSL_SHA3_512; + default: return FSL_SHA3_INVALID; + } +} + +void fsl_sha3_init(fsl_sha3_cx *cx){ + fsl_sha3_init2(cx, FSL_SHA3_DEFAULT); +} +void fsl_sha3_init2(fsl_sha3_cx *p, enum fsl_sha3_hash_size iSize){ + assert(iSize>0); + memset(p, 0, sizeof(*p)); + p->size = iSize; + if( iSize>=128 && iSize<=512 ){ + p->nRate = (1600 - ((iSize + 31)&~31)*2)/8; + }else{ + p->nRate = (1600 - 2*256)/8; + } +#if SHA3_BYTEORDER==1234 + /* Known to be little-endian at compile-time. No-op */ +#elif SHA3_BYTEORDER==4321 + p->ixMask = 7; /* Big-endian */ +#else + { + static const unsigned int one = 1; + if( 1==*(unsigned const char*)&one ){ + /* Little endian. No byte swapping. */ + p->ixMask = 0; + }else{ + /* Big endian. Byte swap. */ + p->ixMask = 7; + } + } +#endif +} + +void fsl_sha3_update( fsl_sha3_cx *p, void const *aData_, unsigned int nData ){ + unsigned char const * aData = (unsigned char const *)aData_; + unsigned int i = 0; +#if SHA3_BYTEORDER==1234 + if( (p->nLoaded % 8)==0 && ((aData - (const unsigned char*)0)&7)==0 ){ + for(; i+7u.s[p->nLoaded/8] ^= *(uint64_t*)&aData[i]; + p->nLoaded += 8; + if( p->nLoaded>=p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } + } +#endif + for(; iu.x[p->nLoaded] ^= aData[i]; +#elif SHA3_BYTEORDER==4321 + p->u.x[p->nLoaded^0x07] ^= aData[i]; +#else + p->u.x[p->nLoaded^p->ixMask] ^= aData[i]; +#endif + p->nLoaded++; + if( p->nLoaded==p->nRate ){ + KeccakF1600Step(p); + p->nLoaded = 0; + } + } +} + +/* +** Convert a digest into base-16. must be at least nBytes long +** and zBuf must be at least nBytes*2 bytes long. This routine +** writes nBytes*2 hex-encoded bytes to zBuf, but does not write +** a terminating NUL byte. +*/ +static void DigestToBase16(unsigned char *digest, unsigned char *zBuf, unsigned int nByte){ + static const unsigned char zEncode[] = "0123456789abcdef"; + unsigned int ix; + for(ix=0; ix>4)&0xf]; + *zBuf++ = zEncode[*digest++ & 0xf]; + } + /* *zBuf = '\0'; */ +} + +unsigned char const *fsl_sha3_end(fsl_sha3_cx *p){ + unsigned int i; + if( p->nLoaded==p->nRate-1 ){ + const unsigned char c1 = 0x86; + fsl_sha3_update(p, &c1, 1); + }else{ + const unsigned char c2 = 0x06; + const unsigned char c3 = 0x80; + fsl_sha3_update(p, &c2, 1); + p->nLoaded = p->nRate - 1; + fsl_sha3_update(p, &c3, 1); + } + for(i=0; inRate; i++){ + p->u.x[i+p->nRate] = p->u.x[i^p->ixMask]; + } + DigestToBase16( &p->u.x[p->nRate], p->hex, (int)p->size/8 ); + assert(0 == p->hex[(int)p->size/4+1]); + return &p->u.x[p->nRate]; +} + +void fsl_sha3_digest_to_base16(unsigned char *digest, char *zBuf){ + static char const zEncode[] = "0123456789abcdef"; + int ix; + for(ix=0; ix>4)&0xf]; + *zBuf++ = zEncode[*digest++ & 0xf]; + } + *zBuf = '\0'; +} + + +int fsl_sha3sum_stream(fsl_input_f src, void * srcState, fsl_buffer *pCksum){ + fsl_sha3_cx ctx; + int rc; + enum { BufSize = 1024 * 4 }; + unsigned char zBuf[BufSize]; + if(!src || !pCksum) return FSL_RC_MISUSE; + fsl_sha3_init(&ctx); + for(;;){ + fsl_size_t read = (fsl_size_t)BufSize; + rc = src(srcState, zBuf, &read); + if(rc) return rc; + else if(read) fsl_sha3_update(&ctx, (unsigned char*)zBuf, read); + if(read < (fsl_size_t)BufSize) break; + } + fsl_sha3_end(&ctx); + fsl_buffer_reuse(pCksum); + return fsl_buffer_append(pCksum, ctx.hex, fsl_strlen((const char *)ctx.hex)); +} + +int fsl_sha3sum_buffer(fsl_buffer const *pIn, fsl_buffer *pCksum){ + if(!pIn || !pCksum) return FSL_RC_MISUSE; + else{ + fsl_sha3_cx ctx; + int rc; + fsl_sha3_init(&ctx); + fsl_sha3_update(&ctx, pIn->mem, pIn->used); + rc = fsl_buffer_reserve(pCksum, FSL_STRLEN_K256+1/*NUL*/); + /*^^^^ DO NOT fsl_buffer_resize(), as pCksum is, more often than not, + a cached reused buffer. */ + if(!rc){ + fsl_buffer_reuse(pCksum); + fsl_sha3_end(&ctx); + assert(fsl_strlen((char const*)ctx.hex)==FSL_STRLEN_K256); + rc = fsl_buffer_append(pCksum, ctx.hex, fsl_strlen((char const*)ctx.hex)); + assert(!rc && "Cannot fail - pre-allocated"); + if(!rc){ + assert(FSL_STRLEN_K256==pCksum->used); + assert(0==pCksum->mem[FSL_STRLEN_K256]); + } + } + return rc; + } +} + +char *fsl_sha3sum_cstr(const char *zIn, fsl_int_t len){ + if(!zIn || !len) return NULL; + else{ + fsl_sha3_cx ctx; + fsl_sha3_init(&ctx); + fsl_sha3_update(&ctx, zIn, + (len<0) ? fsl_strlen(zIn) : (fsl_size_t)len); + fsl_sha3_end(&ctx); + return fsl_strdup((char const *)ctx.hex); + } +} + +int fsl_sha3sum_filename(const char *zFilename, fsl_buffer *pCksum){ + if(!zFilename || !pCksum) return FSL_RC_MISUSE; + else{ +#if 1 + int rc; + FILE *in = fsl_fopen(zFilename, "rb"); + if(!in) rc = FSL_RC_IO; + else{ + rc = fsl_sha3sum_stream(fsl_input_f_FILE, in, pCksum); + fsl_fclose(in); + } + return rc; +#else + /* Requires v1 code which has not yet been ported in. */ + FILE *in; + fsl_sha1_cx ctx; + char zBuf[10240]; + int rc; + + if( fsl_wd_islink(zFilename) ){ + /* Instead of file content, return sha3 of link destination path */ + Blob destinationPath; + + blob_read_link(&destinationPath, zFilename); + rc = fsl_sha3sum_buffer(&destinationPath, pCksum); + fsl_buffer_clear(&destinationPath); + return rc; + } + + in = fossil_fopen(zFilename,"rb"); + if( in==0 ){ + return 1; + } + fsl_sha3_init(&ctx); + for(;;){ + int n; + n = fread(zBuf, 1, sizeof(zBuf), in); + if( n<=0 ) break; + fsl_sha3_update(&ctx, (unsigned char*)zBuf, (unsigned)n); + } + fclose_fclose(in); + blob_zero(pCksum); + blob_resize(pCksum, FSL_STRLEN_SHA3); + fsl_sha3_end(&ctx); + rc = fsl_buffer_append(pCksum, ctx.hex, fsl_strlen(ctx.hex)); + return rc; +#endif + } +} + + +#undef SHA3_BYTEORDER +#undef MARKER +/* end of file sha3.c */ +/* start of file strftime.c */ +/******************************************************************************* + * The Elm Mail System - $Revision: 1.3 $ $State: Exp $ + * + * Public-domain relatively quick-and-dirty implemenation of + * ANSI library routine for System V Unix systems. + * + * Arnold Robbins + * + ***************************************************************************** + * Bug reports, patches, comments, suggestions should be sent to: + * (Note: this routine is provided as is, without support for those sites that + * do not have strftime in their library) + * + * Syd Weinstein, Elm Coordinator + * elm@DSI.COM dsinc!elm + * + ***************************************************************************** + * $Log: strftime.c,v $ + * Revision 1.3 1993/10/09 19:38:51 smace + * Update to elm 2.4 pl23 release version + * + * Revision 5.8 1993/08/23 02:46:51 syd + * Test ANSI_C, not __STDC__ (which is not set on e.g. AIX). + * From: decwrl!uunet.UU.NET!fin!chip (Chip Salzenberg) + * + * Revision 5.7 1993/08/03 19:28:39 syd + * Elm tries to replace the system toupper() and tolower() on current + * BSD systems, which is unnecessary. Even worse, the replacements + * collide during linking with routines in isctype.o. This patch adds + * a Configure test to determine whether replacements are really needed + * (BROKE_CTYPE definition). The header file is now included + * globally through hdrs/defs.h and the BROKE_CTYPE patchup is handled + * there. Inclusion of was removed from *all* the individual + * files, and the toupper() and tolower() routines in lib/opt_utils.c + * were dropped. + * From: chip@chinacat.unicom.com (Chip Rosenthal) + * + * Revision 5.6 1993/08/03 19:20:31 syd + * Implement new timezone handling. New file lib/get_tz.c with new timezone + * routines. Added new TZMINS_USE_xxxxxx and TZNAME_USE_xxxxxx configuration + * definitions. Obsoleted TZNAME, ALTCHECK, and TZ_MINUTESWEST configuration + * definitions. Updated Configure. Modified lib/getarpdate.c and + * lib/strftime.c to use new timezone routines. + * From: chip@chinacat.unicom.com (Chip Rosenthal) + * + * Revision 5.5 1993/06/10 03:17:45 syd + * Change from TZNAME_MISSING to TZNAME + * From: Syd via request from Dan Blanchard + * + * Revision 5.4 1993/05/08 19:56:45 syd + * update to newer version + * From: Syd + * + * Revision 5.3 1993/04/21 01:42:23 syd + * avoid name conflicts on min and max + * + * Revision 5.2 1993/04/16 04:29:34 syd + * attempt to bsdize a bit strftime + * From: many via syd + * + * Revision 5.1 1993/01/27 18:52:15 syd + * Initial checkin of contributed public domain routine. + * This routine is provided as is and not covered by Elm Copyright. + ****************************************************************************/ + +/* + * strftime.c + * + * Public-domain relatively quick-and-dirty implementation of + * ANSI library routine for System V Unix systems. + * + * It's written in old-style C for maximal portability. + * However, since I'm used to prototypes, I've included them too. + * + * If you want stuff in the System V ascftime routine, add the SYSV_EXT define. + * For extensions from SunOS, add SUNOS_EXT. + * For stuff needed to implement the P1003.2 date command, add POSIX2_DATE. + * For complete POSIX semantics, add POSIX_SEMANTICS. + * + * The code for %c, %x, and %X is my best guess as to what's "appropriate". + * This version ignores LOCALE information. + * It also doesn't worry about multi-byte characters. + * So there. + * + * This file is also shipped with GAWK (GNU Awk), gawk specific bits of + * code are included if GAWK is defined. + * + * Arnold Robbins + * January, February, March, 1991 + * Updated March, April 1992 + * Updated May, 1993 + * + * Fixes from ado@elsie.nci.nih.gov + * February 1991, May 1992 + * Fixes from Tor Lillqvist tor@tik.vtt.fi + * May, 1993 + * + * Fast-forward to July 2013... + * + * stephan@wanderinghorse.net copied these sources from: + * + * ftp://ftp-archive.freebsd.org/pub/FreeBSD-Archive/old-releases/i386/1.0-RELEASE/ports/elm/lib/strftime.c + * + * And made the following changes: + * + * Removed ancient non-ANSI decls. Added some headers to get it to + * compile for me. Renamed functions to fit into my project. Replaced + * hard tabs with 4 spaces. Added #undefs for all file-private #defines + * to safe-ify inclusion from/with other files. + * +*/ + +#include +/* #include */ +#include /* strchr() and friends */ +#include /* snprintf() */ +#include /* toupper(), islower() */ + + +#define HAVE_GET_TZ_NAME 0 +#if HAVE_GET_TZ_NAME +extern char *get_tz_name(); +#endif + +#if !defined(BSD) && !defined(_MSC_VER) +extern void tzset (void); +#endif +static int weeknumber (const struct tm *timeptr, int firstweekday); + +/* defaults: season to taste */ +#define SYSV_EXT 1 /* stuff in System V ascftime routine */ +#define SUNOS_EXT 1 /* stuff in SunOS strftime routine */ +#define POSIX2_DATE 1 /* stuff in Posix 1003.2 date command */ +#define VMS_EXT 1 /* include %v for VMS date format */ + +#if 0 +#define POSIX_SEMANTICS 1 /* call tzset() if TZ changes */ +#endif + +#ifdef POSIX_SEMANTICS +#include /* malloc() and friends */ +#endif + +#if defined(POSIX2_DATE) +#if ! defined(SYSV_EXT) +#define SYSV_EXT 1 +#endif +#if ! defined(SUNOS_EXT) +#define SUNOS_EXT 1 +#endif +#endif + +#if defined(POSIX2_DATE) +static int iso8601wknum(const struct tm *timeptr); +#endif + +#undef strchr /* avoid AIX weirdness */ + + +#ifdef __GNUC__ +#define inline __inline__ +#else +#define inline /**/ +#endif + +#define range(low, item, hi) maximum(low, minimum(item, hi)) + +/* minimum --- return minimum of two numbers */ + +static inline int +minimum(int a, int b) +{ + return (a < b ? a : b); +} + +/* maximum --- return maximum of two numbers */ + +static inline int +maximum(int a, int b) +{ + return (a > b ? a : b); +} + +/* strftime --- produce formatted time */ + +fsl_size_t +fsl_strftime(char *s, fsl_size_t maxsize, const char *format, const struct tm *timeptr) +{ + enum {TBufLen = 100U}; + char *endp = s + maxsize; + char *start = s; + char tbuf[TBufLen]; + int i; + static short first = 1; +#ifdef POSIX_SEMANTICS + static char *savetz = NULL; + static int savetzlen = 0; + char *tz; +#endif /* POSIX_SEMANTICS */ + + /* various tables, useful in North America */ + static char *days_a[] = { + "Sun", "Mon", "Tue", "Wed", + "Thu", "Fri", "Sat", + }; + static char *days_l[] = { + "Sunday", "Monday", "Tuesday", "Wednesday", + "Thursday", "Friday", "Saturday", + }; + static char *months_a[] = { + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", + }; + static char *months_l[] = { + "January", "February", "March", "April", + "May", "June", "July", "August", "September", + "October", "November", "December", + }; + static char *ampm[] = { "AM", "PM", }; + + if (s == NULL || format == NULL || timeptr == NULL || maxsize == 0) + return 0; + + if (strchr(format, '%') == NULL && strlen(format) + 1 >= maxsize) + return 0; + +#ifndef POSIX_SEMANTICS + if (first) { + tzset(); + first = 0; + } +#else /* POSIX_SEMANTICS */ + tz = getenv("TZ"); + if (first) { + if (tz != NULL) { + int tzlen = strlen(tz); + + savetz = (char *) malloc(tzlen + 1); + if (savetz != NULL) { + savetzlen = tzlen + 1; + strcpy(savetz, tz); + } + } + tzset(); + first = 0; + } + /* if we have a saved TZ, and it is different, recapture and reset */ + if (tz && savetz && (tz[0] != savetz[0] || strcmp(tz, savetz) != 0)) { + i = strlen(tz) + 1; + if (i > savetzlen) { + savetz = (char *) realloc(savetz, i); + if (savetz) { + savetzlen = i; + strcpy(savetz, tz); + } + } else + strcpy(savetz, tz); + tzset(); + } +#endif /* POSIX_SEMANTICS */ + + for (; *format && s < endp - 1; format++) { + tbuf[0] = '\0'; + if (*format != '%') { + *s++ = *format; + continue; + } + again: + switch (*++format) { + case '\0': + *s++ = '%'; + goto out; + + case '%': + *s++ = '%'; + continue; + + case 'a': /* abbreviated weekday name */ + if (timeptr->tm_wday < 0 || timeptr->tm_wday > 6) + strcpy(tbuf, "?"); + else + strcpy(tbuf, days_a[timeptr->tm_wday]); + break; + + case 'A': /* full weekday name */ + if (timeptr->tm_wday < 0 || timeptr->tm_wday > 6) + strcpy(tbuf, "?"); + else + strcpy(tbuf, days_l[timeptr->tm_wday]); + break; + +#ifdef SYSV_EXT + case 'h': /* abbreviated month name */ +#endif + case 'b': /* abbreviated month name */ + if (timeptr->tm_mon < 0 || timeptr->tm_mon > 11) + strcpy(tbuf, "?"); + else + strcpy(tbuf, months_a[timeptr->tm_mon]); + break; + + case 'B': /* full month name */ + if (timeptr->tm_mon < 0 || timeptr->tm_mon > 11) + strcpy(tbuf, "?"); + else + strcpy(tbuf, months_l[timeptr->tm_mon]); + break; + + case 'c': /* appropriate date and time representation */ + snprintf(tbuf, TBufLen, "%s %s %2d %02d:%02d:%02d %d", + days_a[range(0, timeptr->tm_wday, 6)], + months_a[range(0, timeptr->tm_mon, 11)], + range(1, timeptr->tm_mday, 31), + range(0, timeptr->tm_hour, 23), + range(0, timeptr->tm_min, 59), + range(0, timeptr->tm_sec, 61), + timeptr->tm_year + 1900); + break; + + case 'd': /* day of the month, 01 - 31 */ + i = range(1, timeptr->tm_mday, 31); + snprintf(tbuf, TBufLen, "%02d", i); + break; + + case 'H': /* hour, 24-hour clock, 00 - 23 */ + i = range(0, timeptr->tm_hour, 23); + snprintf(tbuf, TBufLen, "%02d", i); + break; + + case 'I': /* hour, 12-hour clock, 01 - 12 */ + i = range(0, timeptr->tm_hour, 23); + if (i == 0) + i = 12; + else if (i > 12) + i -= 12; + snprintf(tbuf, TBufLen, "%02d", i); + break; + + case 'j': /* day of the year, 001 - 366 */ + snprintf(tbuf, TBufLen, "%03d", timeptr->tm_yday + 1); + break; + + case 'm': /* month, 01 - 12 */ + i = range(0, timeptr->tm_mon, 11); + snprintf(tbuf, TBufLen, "%02d", i + 1); + break; + + case 'M': /* minute, 00 - 59 */ + i = range(0, timeptr->tm_min, 59); + snprintf(tbuf, TBufLen, "%02d", i); + break; + + case 'p': /* am or pm based on 12-hour clock */ + i = range(0, timeptr->tm_hour, 23); + if (i < 12) + strcpy(tbuf, ampm[0]); + else + strcpy(tbuf, ampm[1]); + break; + + case 'S': /* second, 00 - 61 */ + i = range(0, timeptr->tm_sec, 61); + snprintf(tbuf, TBufLen, "%02d", i); + break; + + case 'U': /* week of year, Sunday is first day of week */ + snprintf(tbuf, TBufLen, "%d", weeknumber(timeptr, 0)); + break; + + case 'w': /* weekday, Sunday == 0, 0 - 6 */ + i = range(0, timeptr->tm_wday, 6); + snprintf(tbuf, TBufLen, "%d", i); + break; + + case 'W': /* week of year, Monday is first day of week */ + snprintf(tbuf, TBufLen, "%d", weeknumber(timeptr, 1)); + break; + + case 'x': /* appropriate date representation */ + snprintf(tbuf, TBufLen, "%s %s %2d %d", + days_a[range(0, timeptr->tm_wday, 6)], + months_a[range(0, timeptr->tm_mon, 11)], + range(1, timeptr->tm_mday, 31), + timeptr->tm_year + 1900); + break; + + case 'X': /* appropriate time representation */ + snprintf(tbuf, TBufLen, "%02d:%02d:%02d", + range(0, timeptr->tm_hour, 23), + range(0, timeptr->tm_min, 59), + range(0, timeptr->tm_sec, 61)); + break; + + case 'y': /* year without a century, 00 - 99 */ + i = timeptr->tm_year % 100; + snprintf(tbuf, TBufLen, "%d", i); + break; + + case 'Y': /* year with century */ + snprintf(tbuf, TBufLen, "%d", 1900 + timeptr->tm_year); + break; + +#if HAVE_GET_TZ_NAME + case 'Z': /* time zone name or abbrevation */ + strcpy(tbuf, get_tz_name(timeptr)); + break; +#endif + +#ifdef SYSV_EXT + case 'n': /* same as \n */ + tbuf[0] = '\n'; + tbuf[1] = '\0'; + break; + + case 't': /* same as \t */ + tbuf[0] = '\t'; + tbuf[1] = '\0'; + break; + + case 'D': /* date as %m/%d/%y */ + fsl_strftime(tbuf, sizeof tbuf, "%m/%d/%y", timeptr); + break; + + case 'e': /* day of month, blank padded */ + snprintf(tbuf, TBufLen, "%2d", range(1, timeptr->tm_mday, 31)); + break; + + case 'r': /* time as %I:%M:%S %p */ + fsl_strftime(tbuf, sizeof tbuf, "%I:%M:%S %p", timeptr); + break; + + case 'R': /* time as %H:%M */ + fsl_strftime(tbuf, sizeof tbuf, "%H:%M", timeptr); + break; + + case 'T': /* time as %H:%M:%S */ + fsl_strftime(tbuf, sizeof tbuf, "%H:%M:%S", timeptr); + break; +#endif + +#ifdef SUNOS_EXT + case 'k': /* hour, 24-hour clock, blank pad */ + snprintf(tbuf, TBufLen, "%2d", range(0, timeptr->tm_hour, 23)); + break; + + case 'l': /* hour, 12-hour clock, 1 - 12, blank pad */ + i = range(0, timeptr->tm_hour, 23); + if (i == 0) + i = 12; + else if (i > 12) + i -= 12; + snprintf(tbuf, TBufLen, "%2d", i); + break; +#endif + + +#ifdef VMS_EXT + case 'v': /* date as dd-bbb-YYYY */ + snprintf(tbuf, TBufLen, "%2d-%3.3s-%4d", + range(1, timeptr->tm_mday, 31), + months_a[range(0, timeptr->tm_mon, 11)], + timeptr->tm_year + 1900); + for (i = 3; i < 6; i++) + if (islower((int)tbuf[i])) + tbuf[i] = toupper((int)tbuf[i]); + break; +#endif + + +#ifdef POSIX2_DATE + case 'C': + snprintf(tbuf, TBufLen, "%02d", (timeptr->tm_year + 1900) / 100); + break; + + + case 'E': + case 'O': + /* POSIX locale extensions, ignored for now */ + goto again; + + case 'V': /* week of year according ISO 8601 */ +#if defined(GAWK) && defined(VMS_EXT) + { + extern int do_lint; + extern void warning(); + static int warned = 0; + + if (! warned && do_lint) { + warned = 1; + warning( + "conversion %%V added in P1003.2/11.3; for VMS style date, use %%v"); + } + } +#endif + snprintf(tbuf, TBufLen, "%d", iso8601wknum(timeptr)); + break; + + case 'u': + /* ISO 8601: Weekday as a decimal number [1 (Monday) - 7] */ + snprintf(tbuf, TBufLen, "%d", timeptr->tm_wday == 0 ? 7 : + timeptr->tm_wday); + break; +#endif /* POSIX2_DATE */ + default: + tbuf[0] = '%'; + tbuf[1] = *format; + tbuf[2] = '\0'; + break; + } + i = strlen(tbuf); + if (i){ + if (s + i < endp - 1) { + strcpy(s, tbuf); + s += i; + } else return 0; + /* reminder: above IF originally had ambiguous else + placement (no braces). This placement _appears_ to be + correct.*/ + } + } +out: + if (s < endp && *format == '\0') { + *s = '\0'; + return (s - start); + } else + return 0; +} + +#ifdef POSIX2_DATE +/* iso8601wknum --- compute week number according to ISO 8601 */ + +static int +iso8601wknum(const struct tm *timeptr) +{ + /* + * From 1003.2 D11.3: + * If the week (Monday to Sunday) containing January 1 + * has four or more days in the new year, then it is week 1; + * otherwise it is week 53 of the previous year, and the + * next week is week 1. + * + * ADR: This means if Jan 1 was Monday through Thursday, + * it was week 1, otherwise week 53. + */ + + int simple_wknum, jan1day, diff, ret; + + /* get week number, Monday as first day of the week */ + simple_wknum = weeknumber(timeptr, 1) + 1; + + /* + * With thanks and tip of the hatlo to tml@tik.vtt.fi + * + * What day of the week does January 1 fall on? + * We know that + * (timeptr->tm_yday - jan1.tm_yday) MOD 7 == + * (timeptr->tm_wday - jan1.tm_wday) MOD 7 + * and that + * jan1.tm_yday == 0 + * and that + * timeptr->tm_wday MOD 7 == timeptr->tm_wday + * from which it follows that. . . + */ + jan1day = timeptr->tm_wday - (timeptr->tm_yday % 7); + if (jan1day < 0) + jan1day += 7; + + /* + * If Jan 1 was a Monday through Thursday, it was in + * week 1. Otherwise it was last year's week 53, which is + * this year's week 0. + */ + if (jan1day >= 1 && jan1day <= 4) + diff = 0; + else + diff = 1; + ret = simple_wknum - diff; + if (ret == 0) /* we're in the first week of the year */ + ret = 53; + return ret; +} +#endif + +/* weeknumber --- figure how many weeks into the year */ + +/* With thanks and tip of the hatlo to ado@elsie.nci.nih.gov */ + +static int +weeknumber(const struct tm *timeptr, int firstweekday) +{ + if (firstweekday == 0) + return (timeptr->tm_yday + 7 - timeptr->tm_wday) / 7; + else + return (timeptr->tm_yday + 7 - + (timeptr->tm_wday ? (timeptr->tm_wday - 1) : 6)) / 7; +} + +#undef SYSV_EXT +#undef SUNOS_EXT +#undef POSIX2_DATE +#undef VMS_EXT +#undef POSIX_SEMANTICS +#undef adddecl +#undef inline +#undef range +#undef maximum +#undef minimum +#undef HAVE_GET_TZ_NAME +#undef GAWK + + +/** + A convenience form of fsl_strftime() which takes its timestamp in + the form of a Unix Epoc time. +*/ +fsl_size_t fsl_strftime_unix(char * dest, fsl_size_t destLen, char const * format, + fsl_time_t epochTime, bool convertToLocal){ + time_t orig = (time_t)epochTime; + struct tm * tim = convertToLocal ? localtime(&orig) : gmtime(&orig); + return fsl_strftime( dest, destLen, format, tim ); +} +/* end of file strftime.c */ +/* start of file tag.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + + ***************************************************************************** + This file implements tag-related parts of the library. +*/ +#include + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + + +void fsl_card_T_clean(fsl_card_T *t){ + if(t){ + fsl_free(t->uuid); + t->uuid = NULL; + fsl_free(t->name); + t->name = NULL; + fsl_free(t->value); + t->value = NULL; + *t = fsl_card_T_empty; + } +} + +void fsl_card_T_free(fsl_card_T *t){ + if(t){ + fsl_card_T_clean(t); + fsl_free(t); + } +} + +fsl_card_T * fsl_card_T_malloc(fsl_tagtype_e tagType, + char const * uuid, + char const * name, + char const * value){ + fsl_card_T * t; + int const uuidLen = uuid ? fsl_is_uuid(uuid) : 0; + if(uuid && !uuidLen) return NULL; + t = (fsl_card_T *)fsl_malloc(sizeof(fsl_card_T)); + if(t){ + int rc = 0; + *t = fsl_card_T_empty; + t->type = tagType; + if(uuid && *uuid){ + t->uuid = fsl_strndup(uuid, uuidLen); + if(!t->uuid) rc = FSL_RC_OOM; + } + if(!rc && name && *name){ + t->name = fsl_strdup(name); + if(!t->name){ + rc = FSL_RC_OOM; + } + } + if(!rc && value && *value){ + t->value = fsl_strdup(value); + if(!t->value){ + rc = FSL_RC_OOM; + } + } + if(rc){ + fsl_card_T_free(t); + t = NULL; + } + } + return t; +} + +fsl_id_t fsl_tag_id( fsl_cx * f, char const * tag, bool create ){ + fsl_db * db = fsl_cx_db_repo(f); + int64_t id = 0; + int rc; + if(!db || !tag) return FSL_RC_MISUSE; + else if(!*tag) return FSL_RC_RANGE; + rc = fsl_db_get_int64( db, &id, + "SELECT tagid FROM tag WHERE tagname=%Q", + tag); + if(!rc && (0==id) && create){ + /* Not found - create one. */ + rc = fsl_db_exec(db, "INSERT INTO tag(tagname) VALUES(%Q)", + tag); + if(!rc) id = fsl_db_last_insert_id(db); + } + + if(rc){ + assert(0==id); + fsl_cx_uplift_db_error( f, db ); + id = -1; + } + return id; + +} + +int fsl_tag_propagate(fsl_cx *f, fsl_tagtype_e tagType, + fsl_id_t pid, fsl_id_t tagid, + fsl_id_t origId, const char *zValue, + double mtime){ + int rc; + fsl_pq queue = fsl_pq_empty /* Queue of artifacts to be tagged */; + fsl_stmt s = fsl_stmt_empty /* Query the children of :pid to which to propagate */; + fsl_stmt ins = fsl_stmt_empty /* INSERT INTO tagxref */; + fsl_stmt eventupdate = fsl_stmt_empty /* UPDATE event */; + fsl_db * const db = fsl_needs_repo(f); + + assert(FSL_TAGTYPE_CANCEL==tagType || FSL_TAGTYPE_PROPAGATING==tagType); + assert(f); + assert(db); + assert(pid>0); + assert(tagid>0); + + if((pid<=0 || tagid<=0) + ||(FSL_TAGTYPE_PROPAGATING!=tagType && FSL_TAGTYPE_CANCEL!=tagType) + || (FSL_TAGTYPE_PROPAGATING==tagType && origId<=0)){ + return FSL_RC_RANGE; + } + else if(!db) return FSL_RC_NOT_A_REPO; + + rc = fsl_pq_insert(&queue, pid, 0.0, NULL); + if(rc) return rc; + + rc = fsl_db_prepare(db, &s, + "SELECT cid, plink.mtime," + " coalesce(srcid=0 AND " + " tagxref.mtime<:mtime, %d) AS doit" + " FROM plink LEFT JOIN tagxref " + " ON cid=rid AND tagid=%"FSL_ID_T_PFMT + " WHERE pid=:pid AND isprim", + tagType==FSL_TAGTYPE_PROPAGATING, + (fsl_id_t)tagid); + if(rc) goto end; + rc = fsl_stmt_bind_double_name(&s, ":mtime", mtime); + + if(FSL_TAGTYPE_PROPAGATING==tagType){ + /* Set the propagated tag marker on artifact :rid */ + assert(origId>0); + rc = fsl_db_prepare(db, &ins, + "REPLACE INTO tagxref(" + " tagid, tagtype, srcid, " + " origid, value, mtime, rid" + ") VALUES(" + "%"FSL_ID_T_PFMT"," /* tagid */ + "%d," /*tagtype*/ + "0," /*srcid*/ + "%"FSL_ID_T_PFMT"," /*origId */ + "%Q," /* zValue */ + ":mtime," + ":rid" + ")", + (fsl_id_t)tagid, + (int)FSL_TAGTYPE_PROPAGATING, + (fsl_id_t)origId, zValue); + if(!rc) rc = fsl_stmt_bind_double_name(&ins, ":mtime", mtime); + }else{ + /* Remove all references to the tag from checkin :rid */ + zValue = NULL; + rc = fsl_db_prepare(db, &ins, + "DELETE FROM tagxref WHERE " + "tagid=%"FSL_ID_T_PFMT + " AND rid=:rid", (fsl_id_t)tagid); + } + if(rc) goto end; + if( tagid==FSL_TAGID_BGCOLOR ){ + rc = fsl_db_prepare(db, &eventupdate, + "UPDATE event SET bgcolor=%Q " + "WHERE objid=:rid", zValue); + if(rc) goto end; + } + + while( 0 != (pid = fsl_pq_extract(&queue,NULL))){ + fsl_stmt_bind_id_name(&s, ":pid", pid); +#if 0 + MARKER(("Walking over pid %"FSL_ID_T_PFMT + ", queue.used=%"FSL_SIZE_T_PFMT"\n", pid, queue.used)); +#endif + while( !rc && (FSL_RC_STEP_ROW == fsl_stmt_step(&s)) ){ + int32_t const doit = fsl_stmt_g_int32(&s, 2); + if(doit){ + fsl_id_t cid = fsl_stmt_g_id(&s, 0); + double mtime = fsl_stmt_g_double(&s,1); + assert(cid>0); + assert(mtime>0.0); + rc = fsl_pq_insert(&queue, cid, mtime, NULL); + if(!rc) rc = fsl_stmt_bind_id_name(&ins, ":rid", cid); + if(rc) goto end; + else { + rc = fsl_stmt_step(&ins); + if(FSL_RC_STEP_DONE != rc) goto end; + rc = 0; + } + fsl_stmt_reset(&ins); + if( FSL_TAGID_BGCOLOR == tagid ){ + rc = fsl_stmt_bind_id_name(&eventupdate, ":rid", cid); + if(!rc){ + rc = fsl_stmt_step(&eventupdate); + if(FSL_RC_STEP_DONE != rc) goto end; + rc = 0; + } + fsl_stmt_reset(&eventupdate); + }else if( FSL_TAGID_BRANCH == tagid ){ + rc = fsl_repo_leaf_eventually_check(f, cid); + } + } + } + fsl_stmt_reset(&s); + } + end: + fsl_stmt_finalize(&s); + fsl_stmt_finalize(&ins); + fsl_stmt_finalize(&eventupdate); + fsl_pq_clear(&queue); + return rc; + +} + +int fsl_tag_propagate_all(fsl_cx * f, fsl_id_t pid){ + fsl_stmt q = fsl_stmt_empty; + int rc; + fsl_db * const db = fsl_cx_db_repo(f); + if(!f) return FSL_RC_MISUSE; + else if(pid<=0) return FSL_RC_RANGE; + assert(db); + rc = fsl_db_prepare(db, &q, + "SELECT tagid, tagtype, mtime, " + "value, origid FROM tagxref" + " WHERE rid=%"FSL_ID_T_PFMT, + pid); + while( !rc && (FSL_RC_STEP_ROW == fsl_stmt_step(&q)) ){ + fsl_id_t const tagid = fsl_stmt_g_id(&q, 0); + int32_t tagtype = fsl_stmt_g_int32(&q, 1); + double const mtime = fsl_stmt_g_double(&q, 2); + const char *zValue = fsl_stmt_g_text(&q, 3, NULL); + fsl_id_t const origid = fsl_stmt_g_id(&q, 4); + if( FSL_TAGTYPE_ADD==tagtype ) tagtype = FSL_TAGTYPE_CANCEL + /* For propagating purposes */; + rc = fsl_tag_propagate(f, tagtype, pid, tagid, + origid, zValue, mtime); + } + fsl_stmt_finalize(&q); + return rc; +} + + +int fsl_tag_insert( fsl_cx * f, fsl_tagtype_e tagtype, + char const * zTag, char const * zValue, + fsl_id_t srcId, double mtime, + fsl_id_t rid, fsl_id_t *outRid ){ + fsl_db * db = f ? fsl_cx_db_repo(f) : NULL; + fsl_stmt q = fsl_stmt_empty; + fsl_id_t tagid; + int rc = 0; + char const * zCol; + if(!f || !zTag) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + tagid = fsl_tag_id(f, zTag, 1); + if(tagid<0){ + assert(f->error.code); + return f->error.code; + } + if( mtime<=0.0 ){ + mtime = fsl_db_julian_now(db); + if(mtime<0) return FSL_RC_DB; + } + + rc = fsl_db_prepare(db, &q, /* TODO: cached query */ + "SELECT 1 FROM tagxref" + " WHERE tagid=%"FSL_ID_T_PFMT + " AND rid=%"FSL_ID_T_PFMT + " AND mtime>=?", + (fsl_id_t)tagid, (fsl_id_t)rid); + if(!rc) rc = fsl_stmt_bind_double(&q, 1, mtime); + if(!rc) rc = fsl_stmt_step(&q); + if( FSL_RC_STEP_ROW == rc ){ + /* + Another entry that is more recent already exists. Do nothing. + + Reminder: the above policy is from the original implementation. + We might(?) want to return FSL_RC_ACCESS or + FSL_RC_ALREADY_EXISTS here. The current behaviour seems harmless + enough, though. + */ + if(outRid) *outRid = tagid; + fsl_stmt_finalize(&q); + return 0; + }else if(FSL_RC_STEP_DONE != rc){ + goto end; + } + fsl_stmt_finalize(&q); + rc = fsl_db_prepare(db, &q, + "REPLACE INTO tagxref" + "(tagid,tagtype,srcId,origid,value,mtime,rid) " + "VALUES(" + "%"FSL_ID_T_PFMT"," /* tagid */ + "%d," /* tagtype */ + "%"FSL_ID_T_PFMT"," /* srcid */ + "%"FSL_ID_T_PFMT"," /* rid */ + "%Q," /* zValue */ + "?," /* mtime */ + "%"FSL_ID_T_PFMT")" /* rid again */, + (fsl_id_t)tagid, (int)tagtype, + (fsl_id_t)srcId, + (fsl_id_t)rid, zValue, (fsl_id_t)rid + ); + if(!rc) fsl_stmt_bind_double(&q, 1, mtime); + if(!rc) rc = fsl_stmt_step(&q); + if(FSL_RC_STEP_DONE != rc) goto end; + rc = 0; + fsl_stmt_finalize(&q); + + if(FSL_TAGID_BRANCH == tagid ){ + rc = fsl_repo_leaf_eventually_check(f, rid); + if(rc) goto end; + } +#if 0 + /* Historical: we have valid use cases for the + value here. + */ + else if(FSL_TAGTYPE_CANCEL==tagtype){ + zValue = NULL; + } +#endif + + zCol = NULL; + switch(tagid){ + case FSL_TAGID_BGCOLOR: + zCol = "bgcolor"; + break; + case FSL_TAGID_COMMENT: + zCol = "ecomment"; + break; + case FSL_TAGID_USER: { + zCol = "euser"; + break; + } + case FSL_TAGID_PRIVATE: + rc = fsl_db_exec(db, + "INSERT OR IGNORE INTO " + "private(rid) VALUES" + "(%"FSL_ID_T_PFMT");", + (fsl_id_t)rid ); + if(rc) goto end; + else break; + } + if( zCol ){ + rc = fsl_db_exec(db, "UPDATE event SET %s=%Q " + "WHERE objid=%"FSL_ID_T_PFMT, + zCol, zValue, (fsl_id_t)rid); + if(rc) goto end; +#if 0 + /* + Legacy: i don't want this behaviour in the lib right now + (possibly never). And don't want to port it yet, either :/. + */ + if( tagid==FSL_TAGID_COMMENT ){ + char *zCopy = fsl_strdup(zValue); + wiki_extract_links(zCopy, rid, 0, mtime, 1, WIKI_INLINE); + fsl_free(zCopy); + } +#endif + } + + if( FSL_TAGID_DATE == tagid ){ + rc = fsl_db_exec(db, "UPDATE event " + "SET mtime=julianday(%Q)," + " omtime=coalesce(omtime,mtime)" + "WHERE objid=%"FSL_ID_T_PFMT, + zValue, (fsl_id_t)rid); + if(rc) goto end; + } + if( FSL_TAGTYPE_ADD == tagtype ) tagtype = FSL_TAGTYPE_CANCEL + /* For propagation purposes */; + rc = fsl_tag_propagate(f, tagtype, rid, tagid, + rid, zValue, mtime); + end: + if(rc){ + fsl_stmt_finalize(&q); + }else{ + assert(!q.stmt); + if(outRid) *outRid = tagid; + } + return rc; +} + +int fsl_tag_sym( fsl_cx * f, + fsl_tagtype_e tagType, + char const * symToTag, + char const * tagName, + char const * tagValue, + char const * userName, + double mtime, + fsl_id_t * outId ){ + if(!f || !tagName || !symToTag || !userName) return FSL_RC_MISUSE; + else if(!*tagName || !*userName || !*symToTag) return FSL_RC_RANGE; + else{ + fsl_id_t resolvedRid = 0; + int rc; + rc = fsl_sym_to_rid( f, symToTag, FSL_SATYPE_ANY, &resolvedRid ); + if(!rc){ + assert(resolvedRid>0); + rc = fsl_tag_an_rid(f, tagType, resolvedRid, + tagName, tagValue, userName, + mtime, outId); + } + return rc; + } +} + +int fsl_tag_an_rid( fsl_cx * f, + fsl_tagtype_e tagType, + fsl_id_t idToTag, + char const * tagName, + char const * tagValue, + char const * userName, + double mtime, + fsl_id_t * outId ){ + fsl_db * dbR = f ? fsl_cx_db_repo(f) : NULL; + fsl_deck c = fsl_deck_empty; + char * resolvedUuid = NULL; + fsl_buffer mfout = fsl_buffer_empty; + + int rc; + if(!f || !tagName || !userName) return FSL_RC_MISUSE; + else if(!*tagName || !*userName || (idToTag<=0)) return FSL_RC_RANGE; + else if(!dbR) return FSL_RC_NOT_A_REPO; + + if(mtime<=0) mtime = fsl_db_julian_now(dbR); + + resolvedUuid = fsl_rid_to_uuid(f, idToTag); + if(!resolvedUuid){ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "Could not resolve UUID for " + "rid %"FSL_ID_T_PFMT".", + (fsl_id_t)idToTag); + } + assert(fsl_is_uuid(resolvedUuid)); + +#if 0 + tagRid = fsl_tag_id(f, tagName, 1); + if(tagRid<=0){ + rc = f->error.rc + ? f->error.rc + : fsl_cx_err_set(f, FSL_RC_ERROR, + "Unknown error while fetching " + "ID for tag [%s].", + tagName); + goto end; + } +#endif + + fsl_deck_init(f, &c, FSL_SATYPE_CONTROL); + rc = fsl_deck_T_add( &c, tagType, resolvedUuid, + tagName, tagValue ); + if(rc) goto end; + + rc = fsl_deck_D_set( &c, mtime ); + if(rc) goto end; + + rc = fsl_deck_U_set( &c, userName ); + if(rc) goto end; + + rc = fsl_deck_save( &c, fsl_content_is_private(f, idToTag) ); + end: + fsl_free(resolvedUuid); + fsl_buffer_clear(&mfout); + if(!rc && outId){ + assert(c.rid>0); + *outId = c.rid; + } + fsl_deck_clean(&c); + return rc; +} + +int fsl_branch_create(fsl_cx * f, fsl_branch_opt const * opt, fsl_id_t * newRid ){ + int rc; + fsl_deck parent = fsl_deck_empty; + fsl_deck deck = fsl_deck_empty; + fsl_db * db = f ? fsl_needs_repo(f) : NULL; + char const * user; + char isPrivate; + if(!f || !opt) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + else if(opt->basisRid<=0 || !opt->name || !*opt->name){ + return FSL_RC_MISUSE; + } + else if(! (user = opt->user ? opt->user : f->repo.user)){ + rc = fsl_cx_err_set(f, FSL_RC_MISUSE, + "Could not determine fossil user name " + "for new branch [%s].", opt->name); + } + + + rc = fsl_deck_load_rid(f, &parent, opt->basisRid, FSL_SATYPE_CHECKIN); + if(rc) goto end; + + assert(parent.rid==opt->basisRid); + + fsl_deck_init(f, &deck, FSL_SATYPE_CHECKIN); + + if(parent.B.uuid){ + rc = fsl_deck_B_set(&deck, parent.B.uuid); + } + + rc = fsl_deck_D_set(&deck, opt->mtime>0 ? opt->mtime : fsl_db_julian_now(db)); + if(rc) goto end; + + /* + We cannot simply transfer the list of F-cards from parent to deck + because their content pointers (when the deck is parsed using fsl_deck_parse2()) + points to memory in parent.content; + */ + for( fsl_size_t i = 0; i < parent.F.used; ++i){ + fsl_card_F const * fc = &parent.F.list[i]; + rc = fsl_deck_F_add(&deck, fc->name, fc->uuid, fc->perm, fc->priorName); + if(rc) goto end; + } + + rc = fsl_deck_U_set(&deck, user); + if(rc) goto end; + + rc = fsl_deck_P_add_rid(&deck, parent.rid); + if(rc) goto end; + + if(opt->comment && *opt->comment){ + rc = fsl_deck_C_set(&deck, opt->comment, -1); + }else{ + fsl_buffer c = fsl_buffer_empty; + rc = fsl_buffer_appendf(&c, "Created branch [%s].", opt->name); + if(!rc){ + rc = fsl_deck_C_set(&deck, (char const *)c.mem, (fsl_int_t)c.used); + } + fsl_buffer_clear(&c); + } + if(rc) goto end; + + +#if 0 + /* This adds almost 18MB of allocations to my small test code! */ + if(deck.F.list.used){ + rc = fsl_deck_R_calc(&deck); + if(rc) goto end; + } +#else + rc = fsl_deck_R_set(&deck, parent.R); + if(rc) goto end; +#endif + + isPrivate = fsl_content_is_private(f, parent.rid) ? 1 : opt->isPrivate; + if(isPrivate){ + rc = fsl_deck_T_add(&deck, FSL_TAGTYPE_ADD, + NULL, "private", NULL); + if(rc) goto end; + } + + if(opt->bgColor && ('#'==*opt->bgColor)){ + rc = fsl_deck_T_add(&deck, FSL_TAGTYPE_ADD, NULL, "bgcolor", + opt->bgColor); + if(rc) goto end; + } + + rc = fsl_deck_T_add(&deck, FSL_TAGTYPE_PROPAGATING, + NULL, "branch", opt->name); + if(!rc){ + /* Add tag named sym-BRANCHNAME... */ + fsl_buffer * buf = fsl_cx_scratchpad(f); + rc = fsl_buffer_appendf(buf, "sym-%s", opt->name); + if(!rc){ + rc = fsl_deck_T_add(&deck, FSL_TAGTYPE_PROPAGATING, + NULL, fsl_buffer_cstr(buf), NULL); + } + fsl_cx_scratchpad_yield(f, buf); + } + if(rc) goto end; + +#if 1 + rc = fsl_db_transaction_begin(db); + if(rc) goto end; + else{ + /* cancel all other symbolic tags (branch tags) */ + fsl_stmt q = fsl_stmt_empty; + rc = fsl_db_prepare(db, &q, + "SELECT tagname FROM tagxref, tag" + " WHERE tagxref.rid=%"FSL_ID_T_PFMT + " AND tagxref.tagid=tag.tagid" + " AND tagtype>0 AND tagname GLOB 'sym-*'" + " ORDER BY tagname", + (fsl_id_t)parent.rid); + if(rc) goto end; + while( FSL_RC_STEP_ROW==fsl_stmt_step(&q) ){ + const char *zTag = fsl_stmt_g_text(&q, 0, NULL); + rc = fsl_deck_T_add(&deck, FSL_TAGTYPE_CANCEL, + NULL, zTag, "cancelled by branch."); + if(rc) break; + } + fsl_stmt_finalize(&q); + if(!rc){ + rc = fsl_deck_save(&deck, isPrivate); + if(!rc){ + assert(deck.rid>0); + rc = fsl_db_exec(db, "INSERT OR IGNORE INTO " + "unsent VALUES(%"FSL_ID_T_PFMT")", + (fsl_id_t)deck.rid); + if(!rc){ + /* Make the parent a delta of this one. */ + rc = fsl_content_deltify(f, parent.rid, deck.rid, 0); + } + } + } + if(!rc) rc = fsl_db_transaction_commit(db); + else fsl_db_transaction_rollback(db); + if(rc) goto end; + } +#else + MARKER(("Generating (not saving) branch artifact:\n")); + rc = fsl_deck_unshuffle(&deck, 0); + if(rc) goto end; + rc = fsl_deck_output(&deck, fsl_output_f_FILE, stdout, &f->error); + if(rc) goto end; +#endif + + end: + if(!rc && newRid) *newRid = deck.rid; + else if(rc && !f->error.code){ + if(db->error.code) fsl_cx_uplift_db_error(f,db); + } + fsl_deck_finalize(&parent); + fsl_deck_finalize(&deck); + return rc; +} + + +#undef MARKER +/* end of file tag.c */ +/* start of file ticket.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************* + This file implements ticket-related parts of the library. +*/ +#include +#include /* memcmp() */ + +int fsl_cx_ticket_create_table(fsl_cx * const f){ + fsl_db * const db = fsl_needs_repo(f); + int rc; + if(!db) return FSL_RC_NOT_A_REPO; + rc = fsl_cx_exec_multi(f, + "DROP TABLE IF EXISTS ticket;" + "DROP TABLE IF EXISTS ticketchng;" + ); + if(!rc){ + fsl_buffer * const buf = &f->fileContent; + fsl_buffer_reuse(buf); + rc = fsl_cx_schema_ticket(f, buf); + if(!rc) rc = fsl_cx_exec_multi(f, "%b", buf); + } + return rc; +} + +static int fsl_tkt_field_id(fsl_list const * jli, const char *zFieldName){ + int i; + fsl_card_J const * jc; + for(i=0; i<(int)jli->used; ++i){ + jc = (fsl_card_J const *)jli->list[i]; + if( !fsl_strcmp(zFieldName, jc->field) ) return i; + } + return -1; +} + +int fsl_cx_ticket_load_fields(fsl_cx * f, bool forceReload){ + fsl_stmt q = fsl_stmt_empty; + int i, rc = 0; + fsl_list * li = &f->ticket.customFields; + fsl_card_J * jc; + fsl_db * db; + if(li->used){ + if(!forceReload) return 0; + fsl_card_J_list_free(li, 0); + /* Fall through and reload ... */ + } + if( !(db = fsl_needs_repo(f)) ){ + return FSL_RC_NOT_A_REPO; + } + rc = fsl_db_prepare(db, &q, "PRAGMA table_info(ticket)"); + if(!rc) while( FSL_RC_STEP_ROW==fsl_stmt_step(&q) ){ + char const * zFieldName = fsl_stmt_g_text(&q, 1, NULL); + f->ticket.hasTicket = 1; + if( 0==memcmp(zFieldName,"tkt_", 4)){ + if( 0==fsl_strcmp(zFieldName,"tkt_ctime")) f->ticket.hasCTime = 1; + continue; + } + jc = fsl_card_J_malloc(0, zFieldName, NULL); + if(!jc){ + rc = FSL_RC_OOM; + break; + } + jc->flags = FSL_CARD_J_TICKET; + rc = fsl_list_append(li, jc); + if(rc){ + fsl_card_J_free(jc); + break; + } + } + fsl_stmt_finalize(&q); + if(rc) goto end; + + rc = fsl_db_prepare(db, &q, "PRAGMA table_info(ticketchng)"); + if(!rc) while( FSL_RC_STEP_ROW==fsl_stmt_step(&q) ){ + char const * zFieldName = fsl_stmt_g_text(&q, 1, NULL); + f->ticket.hasChng = 1; + if( 0==memcmp(zFieldName,"tkt_", 4)){ + if( 0==fsl_strcmp(zFieldName,"tkt_rid")) f->ticket.hasChngRid = 1; + continue; + } + if( (i=fsl_tkt_field_id(li, zFieldName)) >= 0){ + jc = (fsl_card_J*)li->list[i]; + jc->flags |= FSL_CARD_J_CHNG; + continue; + } + jc = fsl_card_J_malloc(0, zFieldName, NULL); + if(!jc){ + rc = FSL_RC_OOM; + break; + } + jc->flags = FSL_CARD_J_CHNG; + rc = fsl_list_append(li, jc); + if(rc){ + fsl_card_J_free(jc); + break; + } + } + fsl_stmt_finalize(&q); + end: + if(!rc){ + fsl_list_sort(li, fsl_qsort_cmp_J_cards); + } + return rc; +} +/* end of file ticket.c */ +/* start of file utf8.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* +** Copyright (c) 2017 D. Richard Hipp +** +** This program is free software; you can redistribute it and/or +** modify it under the terms of the Simplified BSD License (also +** known as the "2-Clause License" or "FreeBSD License".) +** +** This program is distributed in the hope that it will be useful, +** but without any warranty; without even the implied warranty of +** merchantability or fitness for a particular purpose. +** +** Author contact information: +** drh@hwaci.com +** http://www.hwaci.com/drh/ +** +******************************************************************************* +** +** This file contains an implementation of SHA3 (Keccak) hashing. +*/ +/** + This copy has been modified slightly for use with the libfossil + API. +*/ + +#include +#include /* NULL on linux */ +#include + +#ifdef _WIN32 +# include +#else +#ifdef __CYGWIN__ +# include +# define CP_UTF8 65001 + __declspec(dllimport) extern __stdcall int WideCharToMultiByte(int, int, + const char *, int, const char *, int, const char *, const char *); + __declspec(dllimport) extern __stdcall int MultiByteToWideChar(int, int, + const char *, int, wchar_t*, int); +#endif +/* /Cygwin */ +/* Assume Unix */ +#include /* getenv() */ +#endif + + +#if defined(__APPLE__) && !defined(WITHOUT_ICONV) +# include +#endif + +#ifdef _WIN32 +char *fsl_mbcs_to_utf8(const char *zMbcs){ + extern char *sqlite3_win32_mbcs_to_utf8(const char*); + return sqlite3_win32_mbcs_to_utf8(zMbcs); +} + +void fossil_mbcs_free(char *zOld){ + sqlite3_free(zOld); +} +#endif /* _WIN32 */ + +void fsl_unicode_free(void *p){ + if(p) fsl_free(p); +} + +char *fsl_unicode_to_utf8(const void *zUnicode){ +#if defined(_WIN32) || defined(__CYGWIN__) + int nByte = WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, 0, 0, 0, 0); + char *zUtf = (char *)fsl_malloc( nByte ); + if( zUtf ){ + WideCharToMultiByte(CP_UTF8, 0, zUnicode, -1, zUtf, nByte, 0, 0); + } + return zUtf; +#else + return fsl_strdup((char const *)zUnicode); /* TODO: implement for unix */ +#endif +} + +void *fsl_utf8_to_unicode(const char *zUtf8){ +#if defined(_WIN32) || defined(__CYGWIN__) + int nByte = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0); + wchar_t *zUnicode = (wchar_t *)fsl_malloc( nByte * 2 ); + if( zUnicode ){ + MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nByte); + } + return zUnicode; +#else + return fsl_strdup(zUtf8); /* TODO: implement for unix */ +#endif +} + +/* + We find that the built-in isspace() function does not work for + some international character sets. So here is a substitute. +*/ +char fsl_isspace(int c){ + return c==' ' || (c<='\r' && c>='\t'); +} + +/* + Other replacements for ctype.h functions. +*/ +char fsl_islower(int c){ return c>='a' && c<='z'; } +char fsl_isupper(int c){ return c>='A' && c<='Z'; } +char fsl_isdigit(int c){ return c>='0' && c<='9'; } +int fsl_tolower(int c){ + return fsl_isupper(c) ? c - 'A' + 'a' : c; +} +int fsl_toupper(int c){ + return fsl_islower(c) ? c - 'a' + 'A' : c; +} +char fsl_isalpha(int c){ + return (c>='a' && c<='z') || (c>='A' && c<='Z'); +} +char fsl_isalnum(int c){ + return (c>='a' && c<='z') || (c>='A' && c<='Z') || (c>='0' && c<='9'); +} + + +void fsl_filename_free(void *pOld){ +#if defined(_WIN32) + fsl_free(pOld); +#elif (defined(__APPLE__) && !defined(WITHOUT_ICONV)) || defined(__CYGWIN__) + fsl_free(pOld); +#else + /* No-op on all other unix */ +#endif +} + +char *fsl_filename_to_utf8(const void *zFilename){ +#if defined(_WIN32) + int nByte = WideCharToMultiByte(CP_UTF8, 0, zFilename, -1, 0, 0, 0, 0); + char *zUtf = fsl_malloc( nByte ); + char *pUtf, *qUtf; + if( zUtf==0 ){ + return 0; + } + WideCharToMultiByte(CP_UTF8, 0, zFilename, -1, zUtf, nByte, 0, 0); + pUtf = qUtf = zUtf; + while( *pUtf ) { + if( *pUtf == (char)0xef ){ + wchar_t c = ((pUtf[1]&0x3f)<<6)|(pUtf[2]&0x3f); + /* Only really convert it when the resulting char is in range. */ + if ( c && ((c < ' ') || wcschr(L"\"*:<>?|", c)) ){ + *qUtf++ = c; pUtf+=3; continue; + } + } + *qUtf++ = *pUtf++; + } + *qUtf = 0; + return zUtf; +#elif defined(__CYGWIN__) + char *zOut; + zOut = fsl_strdup(zFilename) + /* + Required for consistency with fsl_utf8_to_filename(), + so that fsl_filename_free() can DTRT. + */; + return zOut; +#elif defined(__APPLE__) && !defined(WITHOUT_ICONV) + char *zIn = (char*)zFilename; + char *zOut; + iconv_t cd; + size_t n, x; + for(n=0; zIn[n]>0 && zIn[n]<=0x7f; n++){} + if( zIn[n]!=0 && (cd = iconv_open("UTF-8", "UTF-8-MAC"))!=(iconv_t)-1 ){ + char *zOutx; + char *zOrig = zIn; + size_t nIn, nOutx; + nIn = n = fsl_strlen(zIn); + nOutx = nIn+100; + zOutx = zOut = (char *)fsl_malloc( nOutx+1 ); + if(!zOutx) return NULL; + x = iconv(cd, &zIn, &nIn, &zOutx, &nOutx); + if( x==(size_t)-1 ){ + fsl_free(zOut); + zOut = fsl_strdup(zOrig); + }else{ + zOut[n+100-nOutx] = 0; + } + iconv_close(cd); + }else{ + zOut = fsl_strdup(zFilename); + } + return zOut; +#else + return (char *)zFilename; /* No-op on non-mac unix */ +#endif +} + +void *fsl_utf8_to_filename(const char *zUtf8){ +#ifdef _WIN32 + /** + Maintenance note 2021-03-24: fossil's counterpart of this has + been extended since this code was ported: + + void *fossil_utf8_to_path(const char *zUtf8, int isDir) + + That isDir param is only for Windows and its only purpose is to + ensure that the translated path is not within 12 bytes of + MAX_PATH. That same effect can be had by simply always assuming + that bool is true and sacrificing those 12 bytes and that far-edge + case. + + Also, the newer code jumps through many hoops which seem + unimportant for fossil, e.g. handling UNC-style paths. + + Porting that latter bit over requires someone who can at least + test whether it compiles. + */ + int nChar = MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, 0, 0); + wchar_t *zUnicode = fsl_malloc( nChar * 2 ); + wchar_t *wUnicode = zUnicode; + if( zUnicode==0 ){ + return 0; + } + MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, nChar); + /* If path starts with ":/" or ":\", don't translate the ':' */ + if( fsl_isalpha(zUtf8[0]) && zUtf8[1]==':' + && (zUtf8[2]=='\\' || zUtf8[2]=='/')) { + zUnicode[2] = '\\'; + wUnicode += 3; + } + while( *wUnicode != '\0' ){ + if ( (*wUnicode < ' ') || wcschr(L"\"*:<>?|", *wUnicode) ){ + *wUnicode |= 0xF000; + }else if( *wUnicode == '/' ){ + *wUnicode = '\\'; + } + ++wUnicode; + } + return zUnicode; +#elif defined(__CYGWIN__) + char *zPath, *p; + if( fsl_isalpha(zUtf8[0]) && (zUtf8[1]==':') + && (zUtf8[2]=='\\' || zUtf8[2]=='/')) { + /* win32 absolute path starting with drive specifier. */ + int nByte; + wchar_t zUnicode[2000]; + wchar_t *wUnicode = zUnicode; + MultiByteToWideChar(CP_UTF8, 0, zUtf8, -1, zUnicode, + sizeof(zUnicode)/sizeof(zUnicode[0])); + while( *wUnicode != '\0' ){ + if( *wUnicode == '/' ){ + *wUnicode = '\\'; + } + ++wUnicode; + } + nByte = cygwin_conv_path(CCP_WIN_W_TO_POSIX, zUnicode, NULL, 0); + zPath = (char *)fsl_malloc(nByte); + if(!zPath) return NULL; + cygwin_conv_path(CCP_WIN_W_TO_POSIX, zUnicode, zPath, nByte); + }else{ + zPath = fsl_strdup(zUtf8); + if(!zPath) return NULL; + zUtf8 = p = zPath; + while( (*p = *zUtf8++) != 0){ + if( *p++ == '\\' ) { + p[-1] = '/'; + } + } + } + return zPath; +#elif defined(__APPLE__) && !defined(WITHOUT_ICONV) + return fsl_strdup(zUtf8) + /* Why? Why not just act like Unix? */ + ; +#else + return (void *)zUtf8; /* No-op on unix */ +#endif +} + + +char *fsl_getenv(const char *zName){ +#ifdef _WIN32 + wchar_t *uName = (wchar_t *)fsl_utf8_to_unicode(zName); + void *zValue = uName ? (void*)_wgetenv(uName) : NULL; + fsl_free(uName); +#else + char *zValue = (char *)getenv(zName); +#endif + if( zValue ) zValue = fsl_filename_to_utf8(zValue); + return zValue; +} + +fsl_size_t fsl_strlen_utf8( char const * str, fsl_int_t len ){ + if( !str || !len ) return 0; + else if(len<0){ + len = (fsl_int_t)fsl_strlen(str); + } + { + char unsigned const * x = (char unsigned const *)str; + char unsigned const * end = x + len; + fsl_size_t rc = 0; + /* Derived from: + http://www.daemonology.net/blog/2008-06-05-faster-utf8-strlen.html + */ + for( ; x < end; ++x, ++rc ){ + switch(0xF0 & *x) { + case 0xF0: /* length 4 */ + x += 3; + break; + case 0xE0: /* length 3 */ + x+= 2; + break; + case 0xC0: /* length 2 */ + x += 1; + break; + default: + break; + } + } + return rc; + } +} +/* end of file utf8.c */ +/* start of file vfile.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************ + This file contains some of the APIs dealing with the checkout state. +*/ +#include + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + +int fsl_vfile_load(fsl_cx * const f, fsl_id_t vid, + bool clearOtherVersions, + uint32_t * missingCount){ + fsl_db * dbC = f ? fsl_needs_ckout(f) : NULL; + fsl_db * dbR = dbC ? fsl_needs_repo(f) : NULL; + fsl_deck d = fsl_deck_empty; + fsl_stmt qIns = fsl_stmt_empty; + fsl_stmt qRid = fsl_stmt_empty; + int rc; + bool alreadyHad; + fsl_card_F const * fc; + assert(dbC && "Must only be called when a checkout is opened."); + assert(dbR && "Must only be called when a repo is opened."); + if(!dbC) return FSL_RC_NOT_A_CKOUT; + else if(!dbR) return FSL_RC_NOT_A_REPO; + if(vid<=0) vid = f->ckout.rid; + assert(vid>=0); + + rc = fsl_db_transaction_begin(dbC); + if(rc) return rc; + alreadyHad = fsl_db_exists(dbC, + "SELECT 1 FROM vfile " + "WHERE vid=%" FSL_ID_T_PFMT, + vid); + if(clearOtherVersions){ + /* Reminder to self: DO NOT clear vmerge here. Doing so will break + merge tracking in the checkin process. */ + rc = fsl_vfile_unload_except(f, vid); + if(rc) goto end; + } + if(alreadyHad){ + /* Already done. */ + rc = 0; + goto end; + } + assert(0==rc); + if(0==vid){ + /* This is either misuse or an empty/initial repo with no + checkins. Let's assume the latter, since that's what triggered + the addition of this check. */ + goto end; + } + + rc = fsl_deck_load_rid(f, &d, vid, FSL_SATYPE_CHECKIN); + if(rc) goto end; + assert(d.rid==vid); + rc = fsl_deck_F_rewind(&d); + if(rc) goto end; + rc = fsl_db_prepare(dbC, &qIns, + "INSERT INTO vfile" + "(vid,isexe,islink,rid,mrid,pathname,mhash) " + "VALUES(:vid,:isexe,:islink,:id,:id,:name,null)" + /* 2021-10-15: why do we not set mtime here? */); + if(rc) goto end; + rc = fsl_db_prepare(dbR, &qRid, + "SELECT rid,size FROM blob WHERE uuid=?"); + if(rc) goto end; + rc = fsl_stmt_bind_id_name(&qIns, ":vid", vid); + while( !rc && !(rc=fsl_deck_F_next(&d, &fc)) && fc){ + fsl_id_t rid; + int64_t size; + assert(fc->uuid && + "We should never see F-card deletions " + "via fsl_deck_F_next()"); + if(fsl_uuid_is_shunned(f,fc->uuid)) continue; + rc = fsl_stmt_bind_text(&qRid, 1, fc->uuid, -1, 0); + if(rc) break; + rc = fsl_stmt_step(&qRid); + if(FSL_RC_STEP_ROW==rc){ + rid = fsl_stmt_g_id(&qRid,0); + size = fsl_stmt_g_int64(&qRid,1); + }else if(FSL_RC_STEP_DONE==rc){ + rid = 0; + size = 0; + }else{ + assert(qRid.db->error.code); + rc = fsl_cx_uplift_db_error(f, qRid.db); + break; + } + fsl_stmt_reset(&qRid); + if( !rid || size<0 ){ + if(missingCount) ++*missingCount; + continue; + } + fsl_stmt_bind_int32_name(&qIns, ":isexe", + (FSL_FILE_PERM_EXE & fc->perm) ? 1 : 0); + fsl_stmt_bind_int32_name(&qIns, ":islink", + (FSL_FILE_PERM_LINK & fc->perm) ? 1 : 0); + fsl_stmt_bind_id_name(&qIns, ":id", rid); + rc = fsl_stmt_bind_text_name(&qIns, ":name", fc->name, -1, 0); + if(rc) break; + rc = fsl_stmt_step(&qIns); + if(FSL_RC_STEP_DONE!=rc) break; + else rc = 0; + fsl_stmt_reset(&qIns); + } + + end: + fsl_stmt_finalize(&qIns); + fsl_stmt_finalize(&qRid); + /* Update f->ckout state and some db bits we need + when changing the checkout. */ + if(!rc && vid>0){ + if(!alreadyHad){ + assert(d.rid>0); + } + } + fsl_deck_finalize(&d); + if(rc) fsl_db_transaction_rollback(dbC); + else rc = fsl_db_transaction_commit(dbC); + if(rc && !f->error.code){ + if(dbC->error.code) fsl_cx_uplift_db_error(f, dbC); + else if(dbR->error.code) fsl_cx_uplift_db_error(f, dbR); + } + return rc; +} + +static int fsl_vfile_unload_impl(fsl_cx * const f, fsl_id_t vid, + bool oneVersion){ + fsl_db * const db = fsl_needs_ckout(f); + if(!db) return FSL_RC_NOT_A_CKOUT; + if(vid<=0) vid = f->ckout.rid; + int const rc = fsl_db_exec(db, "DELETE FROM vfile " + "WHERE vid%s%" FSL_ID_T_PFMT + " /* %s() */", + oneVersion ? "=" : "<>", + vid, __func__); + return rc ? fsl_cx_uplift_db_error2(f, db, rc) : 0; +} +int fsl_vfile_unload(fsl_cx * const f, fsl_id_t vid){ + return fsl_vfile_unload_impl(f, vid, true); +} +int fsl_vfile_unload_except(fsl_cx * const f, fsl_id_t vid){ + return fsl_vfile_unload_impl(f, vid, false); +} + +/** + Internal code de-duplifier for places which need to re-check a + file's hash in order to be sure whether it was really + modified. hashLen must be the length of the previous (db-side) hash + of the file. This routine will hash that file using the same hash + type. The new hash is appended to pTgt. + + Returns 0 on success. +*/ +static int fsl_vfile_recheck_file_hash( fsl_cx * const f, const char * const zName, + fsl_size_t hashLen, fsl_buffer * const pTgt ){ + bool errReported = false; + int rc = 0; + if((fsl_size_t)FSL_STRLEN_SHA1==hashLen){ + rc = fsl_sha1sum_filename(zName, pTgt); + }else if((fsl_size_t)FSL_STRLEN_K256==hashLen){ + rc = fsl_sha3sum_filename(zName, pTgt); + }else{ + assert(!"This \"cannot happen\"."); + rc = fsl_cx_err_set(f, FSL_RC_CHECKSUM_MISMATCH, + "Cannot determine which hash to use for file: %s", + zName); + errReported = true; + } + if(rc && !errReported && FSL_RC_OOM != rc){ + rc = fsl_cx_err_set(f, rc, "Error %s while hashing file: %s", + fsl_rc_cstr(rc), zName); + } + return rc; +} + + +int fsl_vfile_changes_scan(fsl_cx * const f, fsl_id_t vid, unsigned cksigFlags){ + fsl_stmt * stUpdate = NULL; + fsl_stmt q = fsl_stmt_empty; + int rc = 0; + fsl_db * const db = fsl_needs_ckout(f); + fsl_fstat fst = fsl_fstat_empty; + fsl_size_t rootLen; + fsl_buffer * fileCksum = fsl_cx_scratchpad(f); + bool const useMtime = (cksigFlags & FSL_VFILE_CKSIG_HASH)==0 + && fsl_config_get_bool(f, FSL_CONFDB_REPO, true, "mtime-changes"); + if(!db) return FSL_RC_NOT_A_CKOUT; + assert(f->ckout.dir); + if(vid<=0) vid = f->ckout.rid; + assert(vid>=0); + rootLen = fsl_strlen(f->ckout.dir); + assert(rootLen); + + rc = fsl_db_transaction_begin(db); + if(rc) return rc; + if(f->ckout.rid != vid){ + rc = fsl_vfile_load(f, vid, + (FSL_VFILE_CKSIG_KEEP_OTHERS & cksigFlags) + ? false : true, NULL); + } + if(rc) goto end; + +#if 0 + MARKER(("changed/deleted vfile contents post load-from-rid:\n")); + fsl_db_each( fsl_cx_db_ckout(f), fsl_stmt_each_f_dump, NULL, + "SELECT vf.id, substr(b.uuid,0,8) hash, chnged, " + "deleted, vf.pathname " + "FROM vfile vf LEFT JOIN blob b " + "ON b.rid=vf.rid " + "WHERE vf.vid=%"FSL_ID_T_PFMT" " + "AND (chnged<>0 OR pathname<>origname OR deleted<>0)" + "ORDER BY vf.id", vid); +#endif + + rc = fsl_db_prepare(db, &q, "SELECT " + /*0*/"id," + /*1*/"%Q || pathname," + /*2*/"vfile.mrid," + /*3*/"deleted," + /*4*/"chnged," + /*5*/"uuid," + /*6*/"size," + /*7*/"mtime," + /*8*/"isexe," + /*9*/"islink, " + /*10*/"CASE WHEN isexe THEN %d " + "WHEN islink THEN %d ELSE %d END " + "FROM vfile LEFT JOIN blob ON vfile.mrid=blob.rid " + "WHERE vid=%"FSL_ID_T_PFMT, + f->ckout.dir, + FSL_FILE_PERM_EXE, FSL_FILE_PERM_LINK, + FSL_FILE_PERM_REGULAR, + (fsl_id_t)vid); + if(rc) goto end; + while( fsl_stmt_step(&q) == FSL_RC_STEP_ROW ){ + fsl_id_t id, rid; + char const * zName; +#ifndef _WIN32 + //char const * relName; +#endif + fsl_size_t nName = 0; + int isDeleted; + int64_t currentSize; + int64_t origSize; + int changed, oldChanged; + //int isExe; + fsl_time_t oldMtime, currentMtime; +#if !defined(_WIN32) + int origPerm; + int currentPerm; +#endif + id = fsl_stmt_g_id(&q, 0); + assert(id>0); + zName = fsl_stmt_g_text(&q, 1, &nName); + rid = fsl_stmt_g_id(&q, 2); + isDeleted = fsl_stmt_g_int32(&q, 3); + oldChanged = changed = fsl_stmt_g_int32(&q, 4); + origSize = fsl_stmt_g_int64(&q, 6); + oldMtime = (fsl_time_t)fsl_stmt_g_int64(&q, 7); + //isExe = fsl_stmt_g_int32(&q, 8); + rc = fsl_cx_stat( f, false, zName, &fst ); + currentSize = rc ? -1 : (int64_t)fst.size; + currentMtime = rc ? 0 : fst.mtime; + if(rc){ + fsl_cx_err_reset(f); + rc = 0; + } +#if !defined(_WIN32) + //relName = zName + rootLen; + origPerm = fsl_stmt_g_int32(&q, 10); + currentPerm = (FSL_FSTAT_PERM_EXE==fst.perm + ? FSL_FILE_PERM_EXE + : FSL_FILE_PERM_REGULAR) + /*(FSL_FSTAT_TYPE_LINK==fst.type + ? FSL_FILE_PERM_LINK + : FSL_FILE_PERM_REGULAR)*/ + /* ^^^ FIXME: this isn't right for symlinks. For those we have + to treat them as FSL_FILE_PERM_LINK when the repo has symlink + support enabled, else FSL_FILE_PERM_REGULAR. That's to fix + if/when we ever support SCM'd symlinks in the library. */ + ; +#endif + if(!changed && (isDeleted || !rid)){ + /* ADD and REMOVE operations always change the file */ + changed = FSL_VFILE_CHANGE_MOD; + } + else if( currentSize>=0 + && !(FSL_FSTAT_TYPE_FILE==fst.type + || FSL_FSTAT_TYPE_LINK==fst.type)){ + if( FSL_VFILE_CKSIG_ENOTFILE & cksigFlags ){ + rc = fsl_cx_err_set(f, FSL_RC_TYPE, + "Not an ordinary file or symlink: %s", + zName); + goto end; + } + changed = FSL_VFILE_CHANGE_MOD; + } + if(origSize!=currentSize){ + changed = FSL_VFILE_CHANGE_MOD; + /* A file size change is definitive - the file has changed. No + need to check the mtime or hash */ + }else if( changed==FSL_VFILE_CHANGE_MOD && rid!=0 && !isDeleted ){ + /* File is believed to have changed but it is the same size. + Double check that it really has changed by looking at its + content. */ + fsl_size_t nUuid = 0; + char const * uuid; + fsl_buffer_reuse(fileCksum); + assert( origSize==currentSize ); + uuid = fsl_stmt_g_text(&q, 5, &nUuid); + assert(uuid && fsl_is_uuid_len((int)nUuid)); + rc = fsl_vfile_recheck_file_hash(f, zName, (int)nUuid, fileCksum); + if(rc) goto end; + assert(fsl_is_uuid_len((int)fileCksum->used)); + if( 0 == fsl_uuidcmp(fsl_buffer_cstr(fileCksum), uuid) ){ + changed = 0; + } + }else if( (changed==FSL_VFILE_CHANGE_NONE + || changed==FSL_VFILE_CHANGE_MERGE_MOD + || changed==FSL_VFILE_CHANGE_INTEGRATE_MOD) + && (!useMtime || currentMtime!=oldMtime) ){ + /* For files that were formerly believed to be unchanged or that + were changed by merging, if their mtime changes, or + unconditionally if FSL_VFILE_CKSIG_SETMTIME is used, check to + see if they have been edited by looking at their hash sum */ + fsl_size_t nUuid = 0; + char const * uuid; + assert( origSize==currentSize ); + uuid = fsl_stmt_g_text(&q, 5, &nUuid); + assert(uuid && fsl_is_uuid_len((int)nUuid)); + fsl_buffer_reuse(fileCksum); + rc = fsl_vfile_recheck_file_hash(f, zName, nUuid, fileCksum); + if(rc) goto end; + assert(fsl_is_uuid_len((int)fileCksum->used)); + if( fsl_uuidcmp(fsl_buffer_cstr(fileCksum), uuid) ){ + changed = FSL_VFILE_CHANGE_MOD; + } + /* MARKER(("SHA compare says %d: %s\n", changed, zName)); */ + } + if( (cksigFlags & FSL_VFILE_CKSIG_SETMTIME) + && (changed==FSL_VFILE_CHANGE_NONE + || changed==FSL_VFILE_CHANGE_MERGE_MOD + || changed==FSL_VFILE_CHANGE_INTEGRATE_MOD) ){ + fsl_time_t desiredMtime = 0; + if( 0==fsl_mtime_of_manifest_file(f, vid, rid, &desiredMtime)){ + if( currentMtime != desiredMtime ){ + fsl_file_mtime_set(zName, desiredMtime); + currentMtime = fsl_file_mtime(zName); + } + } + } + /* Check for perms differences. */ +#if !defined(_WIN32) + if( origPerm!=FSL_FILE_PERM_LINK && currentPerm==FSL_FILE_PERM_LINK ){ + /* Changing to a symlink takes priority over all other change types. */ + changed = FSL_VFILE_CHANGE_BECAME_SYMLINK; + }else if( changed==0 + || changed==FSL_VFILE_CHANGE_IS_EXEC + || changed==FSL_VFILE_CHANGE_BECAME_SYMLINK + || changed==FSL_VFILE_CHANGE_NOT_EXEC + || changed==FSL_VFILE_CHANGE_NOT_SYMLINK ){ + /* Confirm metadata change types. */ + if( origPerm==currentPerm ){ + changed = 0; + }else if( currentPerm==FSL_FILE_PERM_EXE ){ + changed = FSL_VFILE_CHANGE_IS_EXEC; + }else if( origPerm==FSL_FILE_PERM_EXE ){ + changed = FSL_VFILE_CHANGE_NOT_EXEC; + }else if( origPerm==FSL_FILE_PERM_LINK ){ + changed = FSL_VFILE_CHANGE_NOT_SYMLINK; + } + } +#endif + if( currentMtime!=oldMtime || changed!=oldChanged ){ + if(!stUpdate){ + rc = fsl_db_prepare_cached(db, &stUpdate, + "UPDATE vfile SET " + "mtime=?1, chnged=?2 " + "WHERE id=?3 " + "/*%s()*/",__func__); + if(rc) goto end; + }else{ + fsl_stmt_reset(stUpdate); + } + fsl_stmt_bind_int64(stUpdate, 1, currentMtime); + fsl_stmt_bind_int32(stUpdate, 2, changed); + fsl_stmt_bind_id(stUpdate, 3, id); + rc = fsl_stmt_step(stUpdate); + if(FSL_RC_STEP_DONE!=rc) goto end; + rc = 0; + /* MARKER(("UPDATED vfile.(mtime,chnged) for: %s\n", zName)); */ + } + }/*while(step)*/ + +#if 0 + MARKER(("changed/deleted vfile contents post vfile scan:\n")); + fsl_db_each( fsl_cx_db_ckout(f), fsl_stmt_each_f_dump, NULL, + "SELECT vf.id, substr(b.uuid,0,8) hash, chnged, " + "deleted, vf.pathname " + "FROM vfile vf LEFT JOIN blob b " + "ON b.rid=vf.rid " + "WHERE vf.vid=%"FSL_ID_T_PFMT" " + "AND (chnged<>0 OR pathname<>origname OR deleted<>0)" + "ORDER BY vf.id", vid); +#endif + end: + fsl_cx_scratchpad_yield(f, fileCksum); + if(!rc && (cksigFlags & FSL_VFILE_CKSIG_WRITE_CKOUT_VERSION) + && (f->ckout.rid != vid)){ + rc = fsl_ckout_version_write(f, vid, 0); + }else if(rc){ + rc = fsl_cx_uplift_db_error2(f, db, rc); + } + if(rc) { + fsl_db_transaction_rollback(db); + }else{ + rc = fsl_db_transaction_commit(db); + if(rc){ + rc = fsl_cx_uplift_db_error2(f, db, rc); + } + } + fsl_stmt_cached_yield(stUpdate); + fsl_stmt_finalize(&q); + return rc; +} + +int fsl_vfile_to_ckout(fsl_cx * f, fsl_id_t vfileId, + int * wasWritten){ + int rc = 0; + fsl_db * const db = fsl_needs_ckout(f); + fsl_stmt q = fsl_stmt_empty; + int counter = 0; + fsl_buffer content = fsl_buffer_empty; + char const * sql; + fsl_id_t qArg; + fsl_fstat * const fst = &f->cache.fstat; + if(!db) return FSL_RC_NOT_A_CKOUT; + assert(f->ckout.rid); + if(vfileId){ + sql = "SELECT v.id, " + "%Q || v.pathname, " + "v.mrid, " + "v.isexe, v.islink, b.uuid, b.size " + "FROM vfile v, blob b" + " WHERE v.id=%" FSL_ID_T_PFMT + " AND v.mrid>0 " + " AND v.mrid=b.rid /*%s()*/"; + qArg = vfileId; + }else{ + sql = "SELECT v.id, " + "%Q || v.pathname, " + "v.mrid, " + "v.isexe, v.islink, b.uuid, b.size " + "FROM vfile v, blob b" + " WHERE v.vid=%" FSL_ID_T_PFMT + " AND v.mrid>0 " + " AND v.mrid=b.rid /*%s()*/"; + qArg = f->ckout.rid; + } +#undef VFILE_NAMEPART + assert(qArg>=0); + rc = fsl_db_prepare(db, &q, sql, f->ckout.dir, qArg, __func__); + if(rc){ + rc = fsl_cx_uplift_db_error2(f, db, rc); + goto end; + } + while(FSL_RC_STEP_ROW==(rc = fsl_stmt_step(&q))){ + //fsl_id_t const id = fsl_stmt_g_id(&q, 0); + fsl_id_t const rid = fsl_stmt_g_id(&q, 2); + int32_t const isExe = fsl_stmt_g_int32(&q, 3); + int32_t const isLink = fsl_stmt_g_int32(&q, 4); + int64_t const sz = fsl_stmt_g_int64(&q, 6); + fsl_size_t nameLen = 0; + char const * zName = fsl_stmt_g_text(&q, 1, &nameLen); + fsl_size_t hashLen = 0; + char const * zHash = fsl_stmt_g_text(&q, 5, &hashLen); + char const * zRelName = &zName[f->ckout.dirLen]; + int isMod = 0; + ++counter; + assert(nameLen > f->ckout.dirLen); + rc = fsl_ckout_safe_file_check(f, zName); + if(rc) break; + assert(fsl_is_uuid_len(hashLen)); + f->cache.fstat = fsl_fstat_empty; + rc = fsl_is_locally_modified(f, zName, sz, zHash, + (fsl_int_t)hashLen, + isExe ? FSL_FILE_PERM_EXE : + (isLink + ? FSL_FILE_PERM_LINK + : FSL_FILE_PERM_REGULAR), + &isMod) + /* that updates f->cache.fstat */; + if(rc) break; + else if(FSL_FSTAT_TYPE_DIR==fst->type){ + /* Fossil checks for this but if this happens then + we have an invalid vfile entry or someone replaced + a file with a dir. */ + rc = fsl_cx_err_set(f, FSL_RC_TYPE, + "Cannot overwrite a directory: %s", + zRelName); + break; + } + else if(!isMod) continue; + else if((rc=fsl_mkdir_for_file(zName, true))){ + rc = fsl_cx_err_set(f, rc, "mkdir() failed for file: %s", + zName); + break; + } + if(FSL_LOCALMOD_LINK & isMod){ + assert(((isLink && FSL_FILE_PERM_LINK!=fst->perm) + ||(!isLink && FSL_FILE_PERM_LINK==fst->perm)) + && "Expected fsl_is_locally_modified() to set this."); + rc = fsl_file_unlink(zName); + if(rc){ + rc = fsl_cx_err_set(f, rc, + "Error removing target to replace it: %s", + zRelName); + break; + } + } + if(isLink || (isMod & (FSL_LOCALMOD_NOTFOUND + | FSL_LOCALMOD_LINK + | FSL_LOCALMOD_CONTENT))){ + /* switched link type, content changed, or was not found in the + filesystem. */ + rc = fsl_content_get(f, rid, &content); + if(rc) break; + } + if(isLink){ + rc = fsl_ckout_symlink_create(f, zName, + fsl_buffer_cstr(&content)); + if(wasWritten && !rc) *wasWritten = 2; + }else if(isMod & (FSL_LOCALMOD_NOTFOUND | FSL_LOCALMOD_CONTENT)){ + /* Not found locally or its contents differ. */ + rc = fsl_buffer_to_filename(&content, zName); + if(rc){ + rc = fsl_cx_err_set(f, rc, "Error writing to file: %s", + zRelName); + }else if(wasWritten){ + *wasWritten = 2; + } + }else if(wasWritten && (isMod & FSL_LOCALMOD_PERM)){ + *wasWritten = 1; + } + if(rc) break; + fsl_file_exec_set(zName, !!isExe); + fsl_buffer_reuse(&content); + }/*step() loop*/ + switch(rc){ + case FSL_RC_STEP_DONE: + if(counter){ + rc = 0; + }else{ + rc = fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "No entry found: vfile.id=%" FSL_ID_T_PFMT, + vfileId); + } + break; + default: break; + } + end: + fsl_buffer_clear(&content); + fsl_stmt_finalize(&q); + return rc; +} + +#undef MARKER +/* end of file vfile.c */ +/* start of file vpath.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/********************************************************************** + This file contains routines related to working with "paths" through + Fossil SCM version history. +*/ +#include + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + + +static const fsl_vpath_node fsl_vpath_node_empty = {0,0,0,0,0,{0},0}; +const fsl_vpath fsl_vpath_empty = fsl_vpath_empty_m; + +void fsl_vpath_clear(fsl_vpath *path){ + fsl_vpath_node * p; + while(path->pAll){ + p = path->pAll; + path->pAll = p->pAll; + fsl_free(p); + } + fsl_id_bag_clear(&path->seen); + *path = fsl_vpath_empty; +} + +fsl_vpath_node * fsl_vpath_first(fsl_vpath *p){ + return p->pStart; +} + +fsl_vpath_node * fsl_vpath_last(fsl_vpath *p){ + return p->pEnd; +} + +int fsl_vpath_length(fsl_vpath const * p){ + return p->nStep; +} + +fsl_vpath_node * fsl_vpath_next(fsl_vpath_node *p){ + return p->u.pTo; +} + +fsl_vpath_node * fsl_vpath_midpoint(fsl_vpath * path){ + if( path->nStep<2 ) return 0; + else{ + fsl_vpath_node *p; + int i; + int const max = path->nStep/2; + for(p=path->pEnd, i=0; p && ipFrom, i++){} + return p; + } +} + + +void fsl_vpath_reverse(fsl_vpath * path){ + fsl_vpath_node *p; + assert( path->pEnd!=0 ); + for(p=path->pEnd; p && p->pFrom; p = p->pFrom){ + p->pFrom->u.pTo = p; + } + path->pEnd->u.pTo = 0; + assert( p==path->pStart ); +} + +/** + Adds a new node to path and returns it. Returns 0 on allocation error. + path must not be 0. rid must be greater than 0. pFrom may be 0. If + pFrom is not 0 then isParent must be true if pFrom is a parent of + rid. + + On success, sets the returned node as path->pCurrent, sets its + pFrom to the given pFrom, and sets rc->u.pPeer to the prior + path->pCurrent value. +*/ +static fsl_vpath_node * fsl_vpath_new_node(fsl_vpath * path, fsl_id_t rid, + fsl_vpath_node * pFrom, bool isParent){ + fsl_vpath_node * rc = 0; + assert(path); + assert(rid>0); + if(0 != fsl_id_bag_insert(&path->seen, rid)) return 0; + rc = (fsl_vpath_node*)fsl_malloc(sizeof(fsl_vpath_node)); + if(!rc){ + fsl_id_bag_remove(&path->seen, rid); + return 0; + } + *rc = fsl_vpath_node_empty; + rc->rid = rid; + rc->fromIsParent = pFrom ? isParent : 0; + rc->pFrom = pFrom; + rc->u.pPeer = path->pCurrent; + path->pCurrent = rc; + rc->pAll = path->pAll; + path->pAll = rc; + return rc; +} + +int fsl_vpath_shortest( fsl_cx * const f, fsl_vpath * const path, + fsl_id_t iFrom, fsl_id_t iTo, + bool directOnly, bool oneWayOnly ){ + fsl_stmt s = fsl_stmt_empty; + fsl_db * db = fsl_needs_repo(f); + int rc = 0; + fsl_vpath_node * pPrev; + fsl_vpath_node * p; + assert(db); + if(!db) return FSL_RC_NOT_A_REPO; + else if(iFrom<=0){ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "Invalid 'from' RID: %d", (int)iFrom); + }else if(iTo<=0){ + /* + Possible TODO: if iTo==0, use... what? Checkout? Tip of current + checkout branch? Trunk? The multitude of options make it impossible + to decide :/. + */ + return fsl_cx_err_set(f, FSL_RC_RANGE, + "Invalid 'to' RID: %d", (int)iTo); + } + + fsl_vpath_clear(path); + path->pStart = fsl_vpath_new_node(path, iFrom, 0, 0); + if(!path->pStart){ + return fsl_cx_err_set(f, FSL_RC_OOM, 0); + } + if( iTo == iFrom ){ + path->pEnd = path->pStart; + return 0; + } + + if( oneWayOnly ){ + if(directOnly){ + rc = fsl_db_prepare(db, &s, + "SELECT cid, 1 FROM plink WHERE pid=?1 AND isprim"); + }else{ + rc = fsl_db_prepare(db, &s, + "SELECT cid, 1 FROM plink WHERE pid=?1"); + } + }else if( directOnly ){ + rc = fsl_db_prepare(db, &s, + "SELECT cid, 1 FROM plink WHERE pid=?1 AND isprim " + "UNION ALL " + "SELECT pid, 0 FROM plink WHERE cid=?1 AND isprim"); + }else{ + rc = fsl_db_prepare(db, &s, + "SELECT cid, 1 FROM plink WHERE pid=?1 " + "UNION ALL " + "SELECT pid, 0 FROM plink WHERE cid=?1"); + } + if(rc){ + fsl_cx_uplift_db_error(f, db); + assert(f->error.code); + goto end; + } + + while(path->pCurrent){ + ++path->nStep; + pPrev = path->pCurrent; + path->pCurrent = 0; + while( pPrev ){ + rc = fsl_stmt_bind_id(&s, 1, pPrev->rid); + assert(0==rc); + while( FSL_RC_STEP_ROW == fsl_stmt_step(&s) ){ + fsl_id_t const cid = fsl_stmt_g_id(&s, 0); + int const isParent = fsl_stmt_g_int32(&s, 1); + assert((cid>0) && "fsl_id_bag_find() asserts this."); + if( fsl_id_bag_contains(&path->seen, cid) ) continue; + p = fsl_vpath_new_node(path, cid, pPrev, isParent ? 1 : 0); + if(!p){ + rc = fsl_cx_err_set(f, FSL_RC_OOM, 0); + goto end; + } + if( cid == iTo ){ + fsl_stmt_finalize(&s); + path->pEnd = p; + fsl_vpath_reverse( path ); + return 0; + } + } + fsl_stmt_reset(&s); + pPrev = pPrev->u.pPeer; + } + } + end: + fsl_stmt_finalize(&s); + fsl_vpath_clear(path); + return rc; +} + +/** + Creates, if needed, the [ancestor] table, else clears its + contents. Returns + */ +static int fsl__init_ancestor(fsl_cx * const f){ + fsl_db * const db = fsl_cx_db_repo(f); + int rc; + if(db){ + rc = fsl_db_exec_multi(db, + "CREATE TEMP TABLE IF NOT EXISTS ancestor(" + " rid INT UNIQUE," + " generation INTEGER PRIMARY KEY" + ");" + "DELETE FROM TEMP.ancestor;"); + }else{ + rc = fsl_cx_err_set(f, FSL_RC_NOT_A_REPO, + "Cannot compute ancestors without an " + "opened repository."); + } + return rc ? fsl_cx_uplift_db_error2(f, db, rc) : 0; +} + +int fsl_compute_direct_ancestors(fsl_cx * const f, fsl_id_t rid){ + int rc = fsl__init_ancestor(f); + fsl_db * const db = rc ? NULL : fsl_needs_repo(f); + if(rc) return rc; + assert(db); + return fsl_db_exec_multi(db, + "WITH RECURSIVE g(x,i) AS (" + " VALUES(%" FSL_ID_T_PFMT ",1)" + " UNION ALL" + " SELECT plink.pid, g.i+1 FROM plink, g" + " WHERE plink.cid=g.x AND plink.isprim)" + "INSERT INTO ancestor(rid,generation) SELECT x,i FROM g;", + rid + ); +} + +int fsl_vpath_shortest_store_in_ancestor(fsl_cx * const f, + fsl_id_t iFrom, + fsl_id_t iTo, + uint32_t *pSteps){ + int rc; + fsl_vpath path = fsl_vpath_empty; + fsl_stmt ins = fsl_stmt_empty; + fsl_db * const db = fsl_needs_repo(f); + fsl_vpath_node * node; + int32_t gen = 0; + if(!db) return FSL_RC_NOT_A_REPO; + rc = fsl_vpath_shortest(f, &path, iFrom, iTo, true, false); + if(rc) goto end; + rc = fsl__init_ancestor(f); + if(rc) goto end; + rc = fsl_db_prepare(db, &ins, + "INSERT INTO TEMP.ancestor(rid, generation) " + "VALUES(?,?)"); + if(rc) goto dberr; + for(node = fsl_vpath_first(&path); + node; node = fsl_vpath_next(node)){ + rc = fsl_stmt_bind_step(&ins, "Ri", node->rid, ++gen); + if(rc) goto dberr; + } + end: + if(0==rc && pSteps) *pSteps = (uint32_t)gen; + fsl_stmt_finalize(&ins); + fsl_vpath_clear(&path); + return rc; + dberr: + assert(rc!=0); + return fsl_cx_uplift_db_error2(f, db, rc); +} + +/* +** A record of a file rename operation. +*/ +typedef struct NameChange NameChange; +struct NameChange { + fsl_id_t origName; /* Original name of file */ + fsl_id_t curName; /* Current name of the file */ + fsl_id_t newName; /* Name of file in next version */ + NameChange *pNext; /* List of all name changes */ +}; +const NameChange NameChange_empty = {0,0,0,0}; + +int fsl__find_filename_changes( + fsl_cx * const f, + fsl_id_t iFrom, /* Ancestor check-in */ + fsl_id_t iTo, /* Recent check-in */ + bool revOK, /* OK to move backwards (child->parent) if true */ + uint32_t *pnChng, /* Number of name changes along the path */ + fsl_id_t **aiChng /* Name changes */ +){ + fsl_vpath_node *p = 0; /* For looping over path from iFrom to iTo */ + NameChange *pAll = 0; /* List of all name changes seen so far */ + NameChange *pChng = 0; /* For looping through the name change list */ + uint32_t nChng = 0; /* Number of files whose names have changed */ + fsl_id_t *aChng = 0; /* Two integers per name change */ + fsl_stmt q1 = fsl_stmt_empty; /* Query of name changes */ + fsl_db * const db = fsl_needs_repo(f); + fsl_vpath path = fsl_vpath_empty; + int rc = 0; + + if(!db) return FSL_RC_NOT_A_REPO; + *pnChng = 0; + *aiChng = 0; + if(iFrom<=0){ + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "Invalid 'from' RID: %" FSL_ID_T_PFMT, iFrom); + }else if(0==iTo){ + return fsl_cx_err_set(f, FSL_RC_MISUSE, + "Invalid 'to' RID: %" FSL_ID_T_PFMT, iTo); + } + if( iFrom==iTo ) return 0; + rc = fsl_vpath_shortest(f, &path, iFrom, iTo, true, !revOK); + if(rc) goto end; + else if(!path.pStart){ + goto end; + } + fsl_vpath_reverse(&path); + rc = fsl_db_prepare(db, &q1, + "SELECT pfnid, fnid FROM mlink" + " WHERE mid=?1 AND (pfnid>0 OR fid==0)" + " ORDER BY pfnid" + ); + if(rc) goto dberr; + for(p=path.pStart; p; p=p->u.pTo){ + fsl_id_t fnid = 0, pfnid = 0; + if( !p->fromIsParent && (p->u.pTo==0 || p->u.pTo->fromIsParent) ){ + /* Skip nodes where the parent is not on the path */ + continue; + } + fsl_stmt_bind_id(&q1, 1, p->rid); + while( FSL_RC_STEP_ROW==fsl_stmt_step(&q1) ){ + pfnid = fsl_stmt_g_id(&q1, 0); + fnid = fsl_stmt_g_id(&q1, 1); + if( pfnid==0 ){ + pfnid = fnid; + fnid = 0; + } + if( !p->fromIsParent ){ + fsl_id_t const t = fnid; + fnid = pfnid; + pfnid = t; + } +#if 0 + if( zDebug ){ + fossil_print("%s at %d%s %.10z: %d[%z] -> %d[%z]\n", + zDebug, p->rid, p->fromIsParent ? ">" : "<", + db_text(0, "SELECT uuid FROM blob WHERE rid=%d", p->rid), + pfnid, + db_text(0, "SELECT name FROM filename WHERE fnid=%d", pfnid), + fnid, + db_text(0, "SELECT name FROM filename WHERE fnid=%d", fnid)); + } +#endif + for(pChng=pAll; pChng; pChng=pChng->pNext){ + if( pChng->curName==pfnid ){ + pChng->newName = fnid; + break; + } + } + if( pChng==0 && fnid>0 ){ + pChng = (NameChange*)fsl_malloc( sizeof(NameChange) ); + if(!pChng){ + rc = FSL_RC_OOM; + goto end; + } + pChng->pNext = pAll; + pAll = pChng; + pChng->origName = pfnid; + pChng->curName = pfnid; + pChng->newName = fnid; + ++nChng; + } + } + for(pChng=pAll; pChng; pChng=pChng->pNext){ + pChng->curName = pChng->newName; + } + fsl_stmt_reset(&q1); + } + if( nChng ){ + /* Count effective changes. */ + uint32_t n; + for(pChng=pAll, n=0; pChng; pChng=pChng->pNext){ + if( pChng->newName==0 ) continue; + if( pChng->origName==0 ) continue; + ++n; + } + nChng = n; + } + if(nChng){ + uint32_t i; + aChng = (fsl_id_t*)fsl_malloc( nChng*2*sizeof(fsl_id_t) ); + if(!aChng){ + rc = FSL_RC_OOM; + goto end; + } + for(pChng=pAll, i=0; pChng; pChng=pChng->pNext){ + if( pChng->newName==0 ) continue; + if( pChng->origName==0 ) continue; + aChng[i] = pChng->origName; + aChng[i+1] = pChng->newName; +#if 0 + if( zDebug ){ + fossil_print("%s summary %d[%z] -> %d[%z]\n", + zDebug, + aChng[i], + db_text(0, "SELECT name FROM filename WHERE fnid=%d", aChng[i]), + aChng[i+1], + db_text(0, "SELECT name FROM filename WHERE fnid=%d", aChng[i+1])); + } +#endif + i += 2; + } + assert(nChng==i/2); + *pnChng = i/2; + *aiChng = aChng; + while( pAll ){ + pChng = pAll; + pAll = pAll->pNext; + fsl_free(pChng); + } + }else{ + *pnChng = 0; + *aiChng = 0; + } + end: + fsl_stmt_finalize(&q1); + if(rc){ + assert(!aChng); + } + fsl_vpath_clear(&path); + return rc; + dberr: + assert(rc); + rc = fsl_cx_uplift_db_error2(f, db, rc); + goto end; +} + +#undef MARKER +/* end of file vpath.c */ +/* start of file wiki.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/************************************************************************* + This file implements wiki-related parts of the library. +*/ +#include + +/* Only for debugging */ +#include +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + + +int fsl_wiki_names_get( fsl_cx * f, fsl_list * tgt ){ + fsl_db * db = fsl_needs_repo(f); + if(!f || !tgt) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + else { + int rc = fsl_db_select_slist( db, tgt, + "SELECT substr(tagname,6) AS name " + "FROM tag " + "WHERE tagname GLOB 'wiki-*' " + "ORDER BY lower(name)"); + if(rc && db->error.code && !f->error.code){ + fsl_cx_uplift_db_error(f, db); + } + return rc; + } +} + +int fsl_wiki_latest_rid( fsl_cx * f, char const * pageName, fsl_id_t * rid ){ + fsl_db * db = f ? fsl_needs_repo(f) : NULL; + if(!f || !pageName) return FSL_RC_MISUSE; + else if(!*pageName) return FSL_RC_RANGE; + else if(!db) return FSL_RC_NOT_A_REPO; + else return fsl_db_get_id(db, rid, + "SELECT x.rid FROM tag t, tagxref x " + "WHERE x.tagid=t.tagid " + "AND t.tagname='wiki-%q' " + "ORDER BY mtime DESC LIMIT 1", + pageName); +} + +bool fsl_wiki_page_exists(fsl_cx * f, char const * pageName){ + fsl_id_t rid = 0; + return (0==fsl_wiki_latest_rid(f, pageName, &rid)) + && (rid>0); +} + +int fsl_wiki_load_latest( fsl_cx * f, char const * pageName, fsl_deck * d ){ + fsl_db * db = f ? fsl_needs_repo(f) : NULL; + if(!f || !pageName || !d) return FSL_RC_MISUSE; + else if(!*pageName) return FSL_RC_RANGE; + else if(!db) return FSL_RC_NOT_A_REPO; + else{ + fsl_id_t rid = 0; + int rc = fsl_wiki_latest_rid(f, pageName, &rid); + if(rc) return rc; + else if(0==rid) return FSL_RC_NOT_FOUND; + return fsl_deck_load_rid( f, d, rid, FSL_SATYPE_WIKI); + } +} + +int fsl_wiki_foreach_page( fsl_cx * f, fsl_deck_visitor_f cb, void * state ){ + fsl_db * db = f ? fsl_needs_repo(f) : NULL; + if(!f || !cb) return FSL_RC_MISUSE; + else if(!db) return FSL_RC_NOT_A_REPO; + else{ + fsl_stmt st = fsl_stmt_empty; + fsl_stmt names = fsl_stmt_empty; + int rc; + char doBreak = 0; + rc = fsl_db_prepare(db, &names, + "SELECT substr(tagname,6) AS name " + "FROM tag " + "WHERE tagname GLOB 'wiki-*' " + "ORDER BY lower(name)"); + if(rc) return rc; + while( !doBreak && !rc + && (FSL_RC_STEP_ROW==fsl_stmt_step(&names))){ + fsl_size_t nameLen = 0; + char const * pageName = fsl_stmt_g_text(&names, 0, &nameLen); + if(!st.stmt){ + rc = fsl_db_prepare(db, &st, + "SELECT x.rid AS mrid FROM tag t, tagxref x " + "WHERE x.tagid=t.tagid " + "AND t.tagname='wiki-'||? " + "ORDER BY mtime DESC LIMIT 1"); + if(rc) goto end; + } + rc = fsl_stmt_bind_text(&st, 1, pageName, (fsl_int_t)nameLen, 0); + if(rc) break; + rc = fsl_stmt_step(&st); + assert(FSL_RC_STEP_ROW==rc); + if(FSL_RC_STEP_ROW==rc){ + fsl_deck d = fsl_deck_empty; + fsl_id_t rid = fsl_stmt_g_id(&st, 0); + rc = fsl_deck_load_rid( f, &d, rid, FSL_SATYPE_WIKI); + if(!rc){ + rc = cb(f, &d, state); + if(FSL_RC_BREAK==rc){ + rc = 0; + doBreak = 1; + } + } + fsl_deck_finalize(&d); + } + fsl_stmt_reset(&st); + } + end: + fsl_stmt_finalize(&st); + fsl_stmt_finalize(&names); + return rc; + } +} + +int fsl_wiki_save(fsl_cx * f, char const * pageName, + fsl_buffer const * b, + char const * userName, + char const * mimeType, + fsl_wiki_save_mode_t createPolicy ){ + fsl_db * db = f ? fsl_needs_repo(f) : NULL; + if(!f || !pageName || !b) return FSL_RC_MISUSE; + else if(!*pageName) return FSL_RC_RANGE; + else if(!db) return FSL_RC_NOT_A_REPO; + else{ + fsl_deck d = fsl_deck_empty; + fsl_id_t parentRid = 0; + int rc = fsl_wiki_latest_rid(f, pageName, &parentRid); + double mtime; + if(rc) return rc; + else if((FSL_WIKI_SAVE_MODE_UPDATE==createPolicy) + && !parentRid){ + return fsl_cx_err_set(f, FSL_RC_NOT_FOUND, + "No such wiki page: %s", + pageName); + } + else if((FSL_WIKI_SAVE_MODE_CREATE==createPolicy) + && (parentRid>0)){ + return fsl_cx_err_set(f, FSL_RC_ALREADY_EXISTS, + "Wiki page already exists: %s", + pageName); + } + mtime = fsl_db_julian_now(db); + fsl_deck_init(f, &d, FSL_SATYPE_WIKI); + rc = fsl_deck_D_set(&d, mtime); + assert(!rc); + rc = fsl_deck_L_set(&d, pageName, -1); + if(!rc && mimeType && *mimeType){ + rc = fsl_deck_N_set(&d, mimeType, -1); + } + if( !rc && parentRid ){ + char * zUuid = fsl_rid_to_uuid(f, parentRid); + if(!zUuid){ + rc = FSL_RC_OOM; + }else{ + rc = fsl_deck_P_add(&d,zUuid); + fsl_free(zUuid); + } + } + if(rc) goto end; + { + char * u = NULL; + if(!userName) userName = fsl_cx_user_get(f); + if(!userName){ + u = fsl_guess_user_name(); + if(!u) rc = FSL_RC_OOM; + } + if(!rc) rc = fsl_deck_U_set(&d, u ? u : userName); + if(u) fsl_free(u); + if(rc) goto end; + } + rc = fsl_deck_W_set(&d, fsl_buffer_cstr(b), (fsl_int_t)b->used); +#if 0 + fsl_deck_output(f, &d, fsl_output_f_FILE, stdout); +#endif + if(!rc) rc = fsl_deck_save(&d, 0); + end: + fsl_deck_finalize(&d); + return rc; + } +} + +#undef MARKER +/* end of file wiki.c */ +/* start of file zip.c */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* +** Copyright (c) 2017 D. Richard Hipp +** +** This program is free software; you can redistribute it and/or +** modify it under the terms of the Simplified BSD License (also +** known as the "2-Clause License" or "FreeBSD License".) +** +** This program is distributed in the hope that it will be useful, +** but without any warranty; without even the implied warranty of +** merchantability or fitness for a particular purpose. +** +** Author contact information: +** drh@hwaci.com +** http://www.hwaci.com/drh/ +** +*/ +/** + This copy has been modified slightly, and expanded, for use + with the libfossil project. +*/ +#include +#include +#include /* atoi() and friends */ +#include /* memset() */ + +#define MARKER(pfexp) \ + do{ printf("MARKER: %s:%d:%s():\t",__FILE__,__LINE__,__func__); \ + printf pfexp; \ + } while(0) + + +/* + Write a 16- or 32-bit integer as little-endian into the given buffer. +*/ +static void fzip_put16(char *z, int v){ + z[0] = v & 0xff; + z[1] = (v>>8) & 0xff; +} +static void fzip_put32(char *z, int v){ + z[0] = v & 0xff; + z[1] = (v>>8) & 0xff; + z[2] = (v>>16) & 0xff; + z[3] = (v>>24) & 0xff; +} + +/** + Set the date and time values from an ISO8601 date string. + */ +static void fzip_timestamp_from_str(fsl_zip_writer *z, const char *zDate){ + int y, m, d; + int H, M, S; + + y = atoi(zDate); + m = atoi(&zDate[5]); + d = atoi(&zDate[8]); + H = atoi(&zDate[11]); + M = atoi(&zDate[14]); + S = atoi(&zDate[17]); + z->dosTime = (H<<11) + (M<<5) + (S>>1); + z->dosDate = ((y-1980)<<9) + (m<<5) + d; +} + +fsl_buffer const * fsl_zip_body( fsl_zip_writer const * z ){ + return z ? &z->body : NULL; +} + +void fsl_zip_timestamp_set_julian(fsl_zip_writer *z, double rDate){ + char buf[20] = {0}; + fsl_julian_to_iso8601(rDate, buf, 0); + fzip_timestamp_from_str(z, buf); + z->unixTime = (fsl_time_t)((rDate - 2440587.5)*86400.0); +} + +void fsl_zip_timestamp_set_unix(fsl_zip_writer *z, fsl_time_t epochTime){ + char buf[20] = {0}; + fsl_julian_to_iso8601(fsl_unix_to_julian(epochTime), buf, 0); + fzip_timestamp_from_str(z, buf); + z->unixTime = epochTime; +} + + +/** + Adds all directories for the given file to the zip if they are not + in there already. Returns 0 on success, non-0 on error (namely + OOM). + */ +static int fzip_mkdir(fsl_zip_writer * z, char const *zName); + +/** + Adds a file entry to zw's zip output. zName is the virtual name of + the file or directory. If pSrc is NULL then it is assumed that we + are creating a directory, otherwise the zip's entry is populated + from pSrc. mPerms specify the fossil-specific permission flags + from the fsl_fileperm_e enum. If doMkDirs is true then fzip_mkdir() + is called to create the directory entries for zName, otherwise + they are not. + */ +static int fzip_file_add(fsl_zip_writer *zw, char const * zName, + fsl_buffer const * pSrc, int mPerm, + char doMkDirs){ + int rc = 0; + z_stream stream; + fsl_size_t nameLen; + int toOut = 0; + int iStart; + int iCRC = 0; + int nByte = 0; + int nByteCompr = 0; + int nBlob; /* Size of the blob */ + int iMethod; /* Compression method. */ + int iMode = 0644; /* Access permissions */ + char *z; + char zHdr[30]; + char zExTime[13]; + char zBuf[100]; + char zOutBuf[/*historical: 100000*/ 1024 * 16]; + + /* Fill in as much of the header as we know. + */ + nBlob = pSrc ? (int)pSrc->used : 0; + if( pSrc ){ /* a file entry */ + iMethod = pSrc->used ? 8 : 0 /* don't compress 0-byte files */; + switch( mPerm ){ + case FSL_FILE_PERM_LINK: iMode = 0120755; break; + case FSL_FILE_PERM_EXE: iMode = 0100755; break; + default: iMode = 0100644; break; + } + }else{ /* a directory entry */ + iMethod = 0; + iMode = 040755; + } + if(doMkDirs){ + rc = fzip_mkdir(zw, zName) + /* This causes an extraneous run of fzip_mkdir(), + but it is harmless other than the waste of search + time */; + if(rc) return rc; + } + + if(zw->rootDir){ + zw->scratch.used = 0; + rc = fsl_buffer_appendf(&zw->scratch, "%s%s", zw->rootDir, zName); + if(rc){ + assert(FSL_RC_OOM==rc); + return rc; + } + zName = fsl_buffer_cstr(&zw->scratch); + } + + nameLen = fsl_strlen(zName); + memset(zHdr, 0, sizeof(zHdr)); + fzip_put32(&zHdr[0], 0x04034b50); + fzip_put16(&zHdr[4], 0x000a); + fzip_put16(&zHdr[6], 0x0800); + fzip_put16(&zHdr[8], iMethod); + fzip_put16(&zHdr[10], zw->dosTime); + fzip_put16(&zHdr[12], zw->dosDate); + fzip_put16(&zHdr[26], nameLen); + fzip_put16(&zHdr[28], 13); + + fzip_put16(&zExTime[0], 0x5455); + fzip_put16(&zExTime[2], 9); + zExTime[4] = 3; + fzip_put32(&zExTime[5], zw->unixTime); + fzip_put32(&zExTime[9], zw->unixTime); + + + /* Write the header and filename. + */ + iStart = (int)zw->body.used; + fsl_buffer_append(&zw->body, zHdr, 30); + fsl_buffer_append(&zw->body, zName, nameLen); + fsl_buffer_append(&zw->body, zExTime, 13); + + if( nBlob>0 ){ + /* Write the compressed file. Compute the CRC as we progress. + */ + stream.zalloc = (alloc_func)0; + stream.zfree = (free_func)0; + stream.opaque = 0; + stream.avail_in = pSrc->used; + stream.next_in = /* (unsigned char*) */pSrc->mem; + stream.avail_out = sizeof(zOutBuf); + stream.next_out = (unsigned char*)zOutBuf; + deflateInit2(&stream, 9, Z_DEFLATED, -MAX_WBITS, 8, Z_DEFAULT_STRATEGY); + iCRC = crc32(0, stream.next_in, stream.avail_in); + while( stream.avail_in>0 ){ + deflate(&stream, 0); + toOut = sizeof(zOutBuf) - stream.avail_out; + fsl_buffer_append(&zw->body, zOutBuf, toOut); + stream.avail_out = sizeof(zOutBuf); + stream.next_out = (unsigned char*)zOutBuf; + } + do{ + stream.avail_out = sizeof(zOutBuf); + stream.next_out = (unsigned char*)zOutBuf; + deflate(&stream, Z_FINISH); + toOut = sizeof(zOutBuf) - stream.avail_out; + fsl_buffer_append(&zw->body, zOutBuf, toOut); + }while( stream.avail_out==0 ); + nByte = stream.total_in; + nByteCompr = stream.total_out; + deflateEnd(&stream); + + /* Go back and write the header, now that we know the compressed file size. + */ + z = (char *)zw->body.mem + iStart/* &blob_buffer(&body)[iStart] */; + fzip_put32(&z[14], iCRC); + fzip_put32(&z[18], nByteCompr); + fzip_put32(&z[22], nByte); + } + + /* Make an entry in the tables of contents + */ + memset(zBuf, 0, sizeof(zBuf)); + fzip_put32(&zBuf[0], 0x02014b50); + fzip_put16(&zBuf[4], 0x0317); + fzip_put16(&zBuf[6], 0x000a); + fzip_put16(&zBuf[8], 0x0800); + fzip_put16(&zBuf[10], iMethod); + fzip_put16(&zBuf[12], zw->dosTime); + fzip_put16(&zBuf[14], zw->dosDate); + fzip_put32(&zBuf[16], iCRC); + fzip_put32(&zBuf[20], nByteCompr); + fzip_put32(&zBuf[24], nByte); + fzip_put16(&zBuf[28], nameLen); + fzip_put16(&zBuf[30], 9); + fzip_put16(&zBuf[32], 0); + fzip_put16(&zBuf[34], 0); + fzip_put16(&zBuf[36], 0); + fzip_put32(&zBuf[38], ((unsigned)iMode)<<16); + fzip_put32(&zBuf[42], iStart); + fsl_buffer_append(&zw->toc, zBuf, 46); + fsl_buffer_append(&zw->toc, zName, nameLen); + fzip_put16(&zExTime[2], 5); + fsl_buffer_append(&zw->toc, zExTime, 9); + ++zw->entryCount; + + return rc; +} + +int fzip_mkdir(fsl_zip_writer * z, char const *zName){ + fsl_size_t i; + fsl_size_t j; + int rc = 0; + char const * dirName; + fsl_size_t nDir = z->dirs.used; + for(i=0; zName[i]; i++){ + if( zName[i]=='/' ){ + while(zName[i+1]=='/') ++i /* Skip extra slashes */; + for(j=0; jdirs.list[j]; + if( fsl_strncmp(zName, dirName, i)==0 ) break; + } + if( j>=nDir ){ + char * cp = fsl_strndup(zName, (fsl_int_t)i+1); + rc = cp ? fsl_list_append(&z->dirs, cp) : FSL_RC_OOM; + if(cp && rc){ + fsl_free(cp); + }else{ + rc = fzip_file_add(z, cp, NULL, 0, 0); + } + } + } + } + return rc; +} + +int fsl_zip_file_add(fsl_zip_writer *z, char const * zName, + fsl_buffer const * pSrc, int mPerm){ + return fzip_file_add(z, zName, pSrc, mPerm, 1); +} + +int fsl_zip_root_set(fsl_zip_writer * z, char const * zRoot ){ + if(!z) return FSL_RC_MISUSE; + else if(zRoot && *zRoot && fsl_is_absolute_path(zRoot)){ + return FSL_RC_RANGE; + }else{ + fsl_free(z->rootDir); + z->rootDir = NULL; + if(zRoot && *zRoot){ + /* + Problem: we have to mkdir zRoot before we assign z->rootDir to + avoid an interesting ROOT/ROOT dir entry on an otherwise empty + ZIP. We create the dirs here, instead of during the first file + insertion (after z->rootDir is set), to work around that. + */ + char * cp; + fsl_size_t n = fsl_strlen(zRoot); + if('/'==zRoot[n-1]){ + /* Keep the slash */ + cp = fsl_strndup(zRoot, (fsl_int_t)n); + }else{ + /* Add a slash to our copy... */ + cp = (char *)fsl_malloc(n+2); + if(cp){ + memcpy( cp, zRoot, n ); + cp[n] = '/'; + cp[n+1] = 0; + ++n; + } + } + if(!cp) return FSL_RC_OOM; + else{ + int rc; + n = fsl_file_simplify_name(cp, (fsl_int_t)n, 1); + assert(n); + assert('/'==cp[n-1]); + cp[n-1] = 0; + rc = fsl_is_simple_pathname(cp, 1); + cp[n-1] = '/'; + rc = rc + ? fzip_mkdir(z, cp) + : FSL_RC_RANGE; + z->rootDir = cp /* transfer ownership on error as well and let + normal downstream clean it up. */; + return rc; + } + } + return 0; + } +} + +static void fsl_zip_finalize_impl(fsl_zip_writer * z, char alsoBody){ + if(z){ + fsl_buffer_clear(&z->toc); + fsl_buffer_clear(&z->scratch); + fsl_list_visit_free(&z->dirs, 1); + assert(NULL==z->dirs.list); + fsl_free(z->rootDir); + if(alsoBody){ + fsl_buffer_clear(&z->body); + *z = fsl_zip_writer_empty; + }else{ + fsl_buffer cp = z->body; + *z = fsl_zip_writer_empty; + z->body = cp; + } + } +} + +void fsl_zip_finalize(fsl_zip_writer * z){ + fsl_zip_finalize_impl(z, 1); +} + + +int fsl_zip_end( fsl_zip_writer * z ){ + int rc; + fsl_int_t iTocStart; + fsl_int_t iTocEnd; + char zBuf[30]; + + iTocStart = (fsl_int_t)z->body.used; + rc = fsl_buffer_append(&z->body, z->toc.mem, z->toc.used); + if(rc) return rc; + fsl_buffer_clear(&z->toc); + iTocEnd = (fsl_int_t)z->body.used; + + memset(zBuf, 0, sizeof(zBuf)); + fzip_put32(&zBuf[0], 0x06054b50); + fzip_put16(&zBuf[4], 0); + fzip_put16(&zBuf[6], 0); + fzip_put16(&zBuf[8], (int)z->entryCount); + fzip_put16(&zBuf[10], (int)z->entryCount); + fzip_put32(&zBuf[12], iTocEnd - iTocStart); + fzip_put32(&zBuf[16], iTocStart); + fzip_put16(&zBuf[20], 0); + + rc = fsl_buffer_append(&z->body, zBuf, 22); + fsl_zip_finalize_impl(z, 0); + assert(z->body.used); + return rc; +} + +int fsl_zip_end_take( fsl_zip_writer * z, fsl_buffer * dest ){ + if(!z) return FSL_RC_MISUSE; + else{ + int rc; + if(!dest){ + rc = FSL_RC_MISUSE; + }else{ + rc = fsl_zip_end(z); + if(!rc){ + fsl_buffer_swap( &z->body, dest ); + } + } + fsl_zip_finalize( z ); + return rc; + } +} + +int fsl_zip_end_to_filename( fsl_zip_writer * z, char const * filename ){ + if(!z) return FSL_RC_MISUSE; + else{ + int rc; + if(!filename || !*filename){ + rc = FSL_RC_MISUSE; + }else{ + rc = fsl_zip_end(z); + if(!rc){ + rc = fsl_buffer_to_filename(&z->body, filename); + } + } + fsl_zip_finalize( z ); + return rc; + } +} + + + +struct ZipState{ + fsl_cx * f; + fsl_id_t vid; + fsl_card_F_visitor_f progress; + void * progressState; + fsl_zip_writer z; + fsl_buffer cbuf; +}; +typedef struct ZipState ZipState; +static const ZipState ZipState_empty = { +NULL, 0, NULL, NULL, +fsl_zip_writer_empty_m, +fsl_buffer_empty_m +}; + +static int fsl_card_F_visitor_zip(fsl_card_F const * fc, + void * state){ + ZipState * zs = (ZipState *)state; + fsl_id_t frid; + int rc = 0; + if(!fc->uuid) return 0 /* file was removed in this (delta) manifest */; + else if(zs->progress){ + rc = (*zs->progress)(fc, zs->progressState); + if(rc) return rc; + }else if(FSL_FILE_PERM_LINK == fc->perm){ + return fsl_cx_err_set(zs->f, FSL_RC_NYI, + "Symlinks are not yet supported " + "in ZIP output."); + } + frid = fsl_uuid_to_rid(zs->f, fc->uuid); + if(frid<0){ + rc = zs->f->error.code; + }else if(!frid){ + assert(zs->f->error.code); + rc = zs->f->error.code; + }else{ + fsl_time_t mTime = 0; + rc = fsl_mtime_of_manifest_file(zs->f, zs->vid, frid, &mTime); + if(!rc){ + fsl_zip_timestamp_set_unix(&zs->z, mTime); + zs->cbuf.used = 0; + rc = fsl_content_get(zs->f, frid, &zs->cbuf); + if(!rc){ + rc = fsl_zip_file_add(&zs->z, fc->name, &zs->cbuf, + FSL_FILE_PERM_REGULAR); + if(rc){ + fsl_cx_err_set(zs->f, rc, + "Error %s adding file [%s] " + "to zip.", fsl_rc_cstr(rc), + fc->name); + } + } + } + } + return rc; +} + + +int fsl_repo_zip_sym_to_filename( fsl_cx * f, char const * sym, + char const * rootDir, + char const * fileName, + fsl_card_F_visitor_f progress, + void * progressState ){ + int rc; + fsl_deck mf = fsl_deck_empty; + ZipState zs = ZipState_empty; + if(!f || !sym || !fileName || !*sym || !*fileName) return FSL_RC_MISUSE; + else if(!fsl_needs_repo(f)) return FSL_RC_NOT_A_REPO; + + rc = fsl_deck_load_sym( f, &mf, sym, FSL_SATYPE_CHECKIN ); + if(rc) goto end; + + if(rootDir && *rootDir){ + fsl_time_t rootTime = 0; + rc = fsl_mtime_of_manifest_file(f, mf.rid, 0, &rootTime); + if(rc) return rc; + fsl_zip_timestamp_set_unix(&zs.z, rootTime); + rc = fsl_zip_root_set( &zs.z, rootDir ); + if(rc) goto end; + } + + rc = fsl_deck_F_rewind(&mf); + if(rc) goto end; + + zs.f = f; + zs.vid = mf.rid; + zs.progress = progress; + zs.progressState = progressState; + rc = fsl_deck_F_foreach( &mf, fsl_card_F_visitor_zip, &zs); + if(!rc){ + if(!zs.z.entryCount){ + if(rootDir && *rootDir){ + rc = fsl_zip_file_add( &zs.z, rootDir, NULL, FSL_FILE_PERM_REGULAR ); + }else{ + rc = fsl_cx_err_set(f, FSL_RC_RANGE, "Cowardly refusing to create " + "empty ZIP file for repo version [%s].", + sym); + } + if(rc) goto end; + } + } + + /** + Always write he manifest files to the zip, regardless of + the repo-level settings. */ + if(rc) goto end; + else { + fsl_buffer * const bManifest = &f->fileContent; + fsl_buffer * const bHash = fsl_cx_scratchpad(f); + fsl_buffer * const bTags = fsl_cx_scratchpad(f); + fsl_buffer_reuse(bManifest); + rc = fsl_repo_manifest_write(f, mf.rid, bManifest, bHash, bTags); + if(rc) goto mf_end; + rc = fsl_zip_file_add(&zs.z, "manifest", bManifest, + FSL_FILE_PERM_REGULAR); + if(rc) goto mf_end; + rc = fsl_zip_file_add(&zs.z, "manifest.uuid", bHash, + FSL_FILE_PERM_REGULAR); + if(rc) goto mf_end; + rc = fsl_zip_file_add(&zs.z, "manifest.tags", bTags, + FSL_FILE_PERM_REGULAR); + mf_end: + fsl_buffer_reuse(bManifest); + fsl_cx_scratchpad_yield(f, bHash); + fsl_cx_scratchpad_yield(f, bTags); + } + if(rc) goto end; + rc = fsl_zip_end( &zs.z ); + if(!rc) rc = fsl_buffer_to_filename( fsl_zip_body(&zs.z), fileName ); + + end: + if(rc && !f->error.code){ + fsl_cx_err_set(f, rc, "Error #%d (%s) during ZIP.", + rc, fsl_rc_cstr(rc)); + } + fsl_buffer_clear(&zs.cbuf); + fsl_zip_finalize(&zs.z); + fsl_deck_clean(&mf); + return rc; +} + + + +#undef MARKER +/* end of file zip.c */ +/* start of file schema_config_cstr.c */ +/* Binary form of file ../sql/config.sql */ +/** @page page_schema_config_cstr Schema: config.sql +@code +-- This file contains the schema for the database that is kept in the +-- ~/.fossil file and that stores information about the users setup. +-- +CREATE TABLE cfg.global_config( + name TEXT PRIMARY KEY, + value TEXT +); + +-- Identifier for this file type. +-- The integer is the same as 'FSLG'. +PRAGMA cfg.application_id=252006675; + @endcode + @see schema_config() +*/ +/* auto-generated code - edit at your own risk! (Good luck with that!) */ +static char const fsl_schema_config_cstr_a[] = { +45, 45, 32, 84, 104, 105, 115, 32, 102, 105, 108, 101, 32, 99, 111, 110, 116, 97, 105, 110, +115, 32, 116, 104, 101, 32, 115, 99, 104, 101, 109, 97, 32, 102, 111, 114, 32, 116, 104, 101, +32, 100, 97, 116, 97, 98, 97, 115, 101, 32, 116, 104, 97, 116, 32, 105, 115, 32, 107, 101, +112, 116, 32, 105, 110, 32, 116, 104, 101, 10, 45, 45, 32, 126, 47, 46, 102, 111, 115, 115, +105, 108, 32, 102, 105, 108, 101, 32, 97, 110, 100, 32, 116, 104, 97, 116, 32, 115, 116, 111, +114, 101, 115, 32, 105, 110, 102, 111, 114, 109, 97, 116, 105, 111, 110, 32, 97, 98, 111, 117, +116, 32, 116, 104, 101, 32, 117, 115, 101, 114, 115, 32, 115, 101, 116, 117, 112, 46, 10, 45, +45, 10, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 99, 102, 103, 46, 103, +108, 111, 98, 97, 108, 95, 99, 111, 110, 102, 105, 103, 40, 10, 32, 32, 110, 97, 109, 101, +32, 84, 69, 88, 84, 32, 80, 82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 44, 10, 32, +32, 118, 97, 108, 117, 101, 32, 84, 69, 88, 84, 10, 41, 59, 10, 10, 45, 45, 32, 73, +100, 101, 110, 116, 105, 102, 105, 101, 114, 32, 102, 111, 114, 32, 116, 104, 105, 115, 32, 102, +105, 108, 101, 32, 116, 121, 112, 101, 46, 10, 45, 45, 32, 84, 104, 101, 32, 105, 110, 116, +101, 103, 101, 114, 32, 105, 115, 32, 116, 104, 101, 32, 115, 97, 109, 101, 32, 97, 115, 32, +39, 70, 83, 76, 71, 39, 46, 10, 80, 82, 65, 71, 77, 65, 32, 99, 102, 103, 46, 97, +112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 95, 105, 100, 61, 50, 53, 50, 48, 48, 54, +54, 55, 53, 59, 10, +0}; +char const * fsl_schema_config_cstr = fsl_schema_config_cstr_a; +/* end of ../sql/config.sql */ +/* end of file schema_config_cstr.c */ +/* start of file schema_repo1_cstr.c */ +/* Binary form of file ../sql/repo-static.sql */ +/** @page page_schema_repo1_cstr Schema: repo-static.sql +@code +-- This file contains parts of the schema that are fixed and +-- unchanging across Fossil versions. + + +-- The BLOB and DELTA tables contain all records held in the repository. +-- +-- The BLOB.CONTENT column is always compressed using zlib. This +-- column might hold the full text of the record or it might hold +-- a delta that is able to reconstruct the record from some other +-- record. If BLOB.CONTENT holds a delta, then a DELTA table entry +-- will exist for the record and that entry will point to another +-- entry that holds the source of the delta. Deltas can be chained. +-- +-- The blob and delta tables collectively hold the "global state" of +-- a Fossil repository. +-- +CREATE TABLE repo.blob( + rid INTEGER PRIMARY KEY, -- Record ID + rcvid INTEGER, -- Origin of this record + size INTEGER, -- Size of content. -1 for a phantom. + uuid TEXT UNIQUE NOT NULL, -- SHA1 hash of the content + content BLOB, -- Compressed content of this record + CHECK( length(uuid)>=40 AND rid>0 ) +); +CREATE TABLE repo.delta( + rid INTEGER PRIMARY KEY, -- Record ID + srcid INTEGER NOT NULL REFERENCES blob -- Record holding source document +); +CREATE INDEX repo.delta_i1 ON delta(srcid); + +------------------------------------------------------------------------- +-- The BLOB and DELTA tables above hold the "global state" of a Fossil +-- project; the stuff that is normally exchanged during "sync". The +-- "local state" of a repository is contained in the remaining tables of +-- the zRepositorySchema1 string. +------------------------------------------------------------------------- + +-- Whenever new blobs are received into the repository, an entry +-- in this table records the source of the blob. +-- +CREATE TABLE repo.rcvfrom( + rcvid INTEGER PRIMARY KEY, -- Received-From ID + uid INTEGER REFERENCES user, -- User login + mtime DATETIME, -- Time of receipt. Julian day. + nonce TEXT UNIQUE, -- Nonce used for login + ipaddr TEXT -- Remote IP address. NULL for direct. +); +INSERT INTO repo.rcvfrom(rcvid,uid,mtime,nonce,ipaddr) +VALUES (1, 1, julianday('now'), NULL, NULL); + +-- Information about users +-- +-- The user.pw field can be either cleartext of the password, or +-- a SHA1 hash of the password. If the user.pw field is exactly 40 +-- characters long we assume it is a SHA1 hash. Otherwise, it is +-- cleartext. The sha1_shared_secret() routine computes the password +-- hash based on the project-code, the user login, and the cleartext +-- password. +-- +CREATE TABLE repo.user( + uid INTEGER PRIMARY KEY, -- User ID + login TEXT UNIQUE, -- login name of the user + pw TEXT, -- password + cap TEXT, -- Capabilities of this user + cookie TEXT, -- WWW login cookie + ipaddr TEXT, -- IP address for which cookie is valid + cexpire DATETIME, -- Time when cookie expires + info TEXT, -- contact information + mtime DATE, -- last change. seconds since 1970 + photo BLOB -- JPEG image of this user +); + +-- The VAR table holds miscellanous information about the repository. +-- in the form of name-value pairs. +-- +CREATE TABLE repo.config( + name TEXT PRIMARY KEY NOT NULL, -- Primary name of the entry + value CLOB, -- Content of the named parameter + mtime DATE, -- last modified. seconds since 1970 + CHECK( typeof(name)='text' AND length(name)>=1 ) +); + +-- Artifacts that should not be processed are identified in the +-- "shun" table. Artifacts that are control-file forgeries or +-- spam or artifacts whose contents violate administrative policy +-- can be shunned in order to prevent them from contaminating +-- the repository. +-- +-- Shunned artifacts do not exist in the blob table. Hence they +-- have not artifact ID (rid) and we thus must store their full +-- UUID. +-- +CREATE TABLE repo.shun( + uuid UNIQUE, -- UUID of artifact to be shunned. Canonical form + mtime DATE, -- When added. seconds since 1970 + scom TEXT -- Optional text explaining why the shun occurred +); + +-- Artifacts that should not be pushed are stored in the "private" +-- table. Private artifacts are omitted from the "unclustered" and +-- "unsent" tables. +-- +CREATE TABLE repo.private(rid INTEGER PRIMARY KEY); + +-- An entry in this table describes a database query that generates a +-- table of tickets. +-- +CREATE TABLE repo.reportfmt( + rn INTEGER PRIMARY KEY, -- Report number + owner TEXT, -- Owner of this report format (not used) + title TEXT UNIQUE, -- Title of this report + mtime DATE, -- Last modified. seconds since 1970 + cols TEXT, -- A color-key specification + sqlcode TEXT -- An SQL SELECT statement for this report +); + +-- Some ticket content (such as the originators email address or contact +-- information) needs to be obscured to protect privacy. This is achieved +-- by storing an SHA1 hash of the content. For display, the hash is +-- mapped back into the original text using this table. +-- +-- This table contains sensitive information and should not be shared +-- with unauthorized users. +-- +CREATE TABLE repo.concealed( + hash TEXT PRIMARY KEY, -- The SHA1 hash of content + mtime DATE, -- Time created. Seconds since 1970 + content TEXT -- Content intended to be concealed +); + +-- The application ID helps the unix "file" command to identify the +-- database as a fossil repository. +PRAGMA repo.application_id=252006673; + @endcode + @see schema_repo1() +*/ +/* auto-generated code - edit at your own risk! (Good luck with that!) */ +static char const fsl_schema_repo1_cstr_a[] = { +45, 45, 32, 84, 104, 105, 115, 32, 102, 105, 108, 101, 32, 99, 111, 110, 116, 97, 105, 110, +115, 32, 112, 97, 114, 116, 115, 32, 111, 102, 32, 116, 104, 101, 32, 115, 99, 104, 101, 109, +97, 32, 116, 104, 97, 116, 32, 97, 114, 101, 32, 102, 105, 120, 101, 100, 32, 97, 110, 100, +10, 45, 45, 32, 117, 110, 99, 104, 97, 110, 103, 105, 110, 103, 32, 97, 99, 114, 111, 115, +115, 32, 70, 111, 115, 115, 105, 108, 32, 118, 101, 114, 115, 105, 111, 110, 115, 46, 10, 10, +10, 45, 45, 32, 84, 104, 101, 32, 66, 76, 79, 66, 32, 97, 110, 100, 32, 68, 69, 76, +84, 65, 32, 116, 97, 98, 108, 101, 115, 32, 99, 111, 110, 116, 97, 105, 110, 32, 97, 108, +108, 32, 114, 101, 99, 111, 114, 100, 115, 32, 104, 101, 108, 100, 32, 105, 110, 32, 116, 104, +101, 32, 114, 101, 112, 111, 115, 105, 116, 111, 114, 121, 46, 10, 45, 45, 10, 45, 45, 32, +84, 104, 101, 32, 66, 76, 79, 66, 46, 67, 79, 78, 84, 69, 78, 84, 32, 99, 111, 108, +117, 109, 110, 32, 105, 115, 32, 97, 108, 119, 97, 121, 115, 32, 99, 111, 109, 112, 114, 101, +115, 115, 101, 100, 32, 117, 115, 105, 110, 103, 32, 122, 108, 105, 98, 46, 32, 32, 84, 104, +105, 115, 10, 45, 45, 32, 99, 111, 108, 117, 109, 110, 32, 109, 105, 103, 104, 116, 32, 104, +111, 108, 100, 32, 116, 104, 101, 32, 102, 117, 108, 108, 32, 116, 101, 120, 116, 32, 111, 102, +32, 116, 104, 101, 32, 114, 101, 99, 111, 114, 100, 32, 111, 114, 32, 105, 116, 32, 109, 105, +103, 104, 116, 32, 104, 111, 108, 100, 10, 45, 45, 32, 97, 32, 100, 101, 108, 116, 97, 32, +116, 104, 97, 116, 32, 105, 115, 32, 97, 98, 108, 101, 32, 116, 111, 32, 114, 101, 99, 111, +110, 115, 116, 114, 117, 99, 116, 32, 116, 104, 101, 32, 114, 101, 99, 111, 114, 100, 32, 102, +114, 111, 109, 32, 115, 111, 109, 101, 32, 111, 116, 104, 101, 114, 10, 45, 45, 32, 114, 101, +99, 111, 114, 100, 46, 32, 32, 73, 102, 32, 66, 76, 79, 66, 46, 67, 79, 78, 84, 69, +78, 84, 32, 104, 111, 108, 100, 115, 32, 97, 32, 100, 101, 108, 116, 97, 44, 32, 116, 104, +101, 110, 32, 97, 32, 68, 69, 76, 84, 65, 32, 116, 97, 98, 108, 101, 32, 101, 110, 116, +114, 121, 10, 45, 45, 32, 119, 105, 108, 108, 32, 101, 120, 105, 115, 116, 32, 102, 111, 114, +32, 116, 104, 101, 32, 114, 101, 99, 111, 114, 100, 32, 97, 110, 100, 32, 116, 104, 97, 116, +32, 101, 110, 116, 114, 121, 32, 119, 105, 108, 108, 32, 112, 111, 105, 110, 116, 32, 116, 111, +32, 97, 110, 111, 116, 104, 101, 114, 10, 45, 45, 32, 101, 110, 116, 114, 121, 32, 116, 104, +97, 116, 32, 104, 111, 108, 100, 115, 32, 116, 104, 101, 32, 115, 111, 117, 114, 99, 101, 32, +111, 102, 32, 116, 104, 101, 32, 100, 101, 108, 116, 97, 46, 32, 32, 68, 101, 108, 116, 97, +115, 32, 99, 97, 110, 32, 98, 101, 32, 99, 104, 97, 105, 110, 101, 100, 46, 10, 45, 45, +10, 45, 45, 32, 84, 104, 101, 32, 98, 108, 111, 98, 32, 97, 110, 100, 32, 100, 101, 108, +116, 97, 32, 116, 97, 98, 108, 101, 115, 32, 99, 111, 108, 108, 101, 99, 116, 105, 118, 101, +108, 121, 32, 104, 111, 108, 100, 32, 116, 104, 101, 32, 34, 103, 108, 111, 98, 97, 108, 32, +115, 116, 97, 116, 101, 34, 32, 111, 102, 10, 45, 45, 32, 97, 32, 70, 111, 115, 115, 105, +108, 32, 114, 101, 112, 111, 115, 105, 116, 111, 114, 121, 46, 32, 32, 10, 45, 45, 10, 67, +82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 98, 108, 111, +98, 40, 10, 32, 32, 114, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, 82, 73, +77, 65, 82, 89, 32, 75, 69, 89, 44, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, +82, 101, 99, 111, 114, 100, 32, 73, 68, 10, 32, 32, 114, 99, 118, 105, 100, 32, 73, 78, +84, 69, 71, 69, 82, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 45, 45, 32, 79, 114, 105, 103, 105, 110, 32, 111, 102, 32, 116, 104, 105, +115, 32, 114, 101, 99, 111, 114, 100, 10, 32, 32, 115, 105, 122, 101, 32, 73, 78, 84, 69, +71, 69, 82, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 45, 45, 32, 83, 105, 122, 101, 32, 111, 102, 32, 99, 111, 110, 116, 101, 110, +116, 46, 32, 45, 49, 32, 102, 111, 114, 32, 97, 32, 112, 104, 97, 110, 116, 111, 109, 46, +10, 32, 32, 117, 117, 105, 100, 32, 84, 69, 88, 84, 32, 85, 78, 73, 81, 85, 69, 32, +78, 79, 84, 32, 78, 85, 76, 76, 44, 32, 32, 32, 32, 32, 32, 45, 45, 32, 83, 72, +65, 49, 32, 104, 97, 115, 104, 32, 111, 102, 32, 116, 104, 101, 32, 99, 111, 110, 116, 101, +110, 116, 10, 32, 32, 99, 111, 110, 116, 101, 110, 116, 32, 66, 76, 79, 66, 44, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, +67, 111, 109, 112, 114, 101, 115, 115, 101, 100, 32, 99, 111, 110, 116, 101, 110, 116, 32, 111, +102, 32, 116, 104, 105, 115, 32, 114, 101, 99, 111, 114, 100, 10, 32, 32, 67, 72, 69, 67, +75, 40, 32, 108, 101, 110, 103, 116, 104, 40, 117, 117, 105, 100, 41, 62, 61, 52, 48, 32, +65, 78, 68, 32, 114, 105, 100, 62, 48, 32, 41, 10, 41, 59, 10, 67, 82, 69, 65, 84, +69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 100, 101, 108, 116, 97, 40, 10, +32, 32, 114, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, 82, 73, 77, 65, 82, +89, 32, 75, 69, 89, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 45, 45, 32, 82, 101, 99, 111, 114, 100, 32, 73, 68, 10, 32, 32, 115, 114, +99, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 78, 79, 84, 32, 78, 85, 76, 76, +32, 82, 69, 70, 69, 82, 69, 78, 67, 69, 83, 32, 98, 108, 111, 98, 32, 32, 32, 45, +45, 32, 82, 101, 99, 111, 114, 100, 32, 104, 111, 108, 100, 105, 110, 103, 32, 115, 111, 117, +114, 99, 101, 32, 100, 111, 99, 117, 109, 101, 110, 116, 10, 41, 59, 10, 67, 82, 69, 65, +84, 69, 32, 73, 78, 68, 69, 88, 32, 114, 101, 112, 111, 46, 100, 101, 108, 116, 97, 95, +105, 49, 32, 79, 78, 32, 100, 101, 108, 116, 97, 40, 115, 114, 99, 105, 100, 41, 59, 10, +10, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, +45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, +45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, +45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 10, 45, 45, 32, 84, 104, +101, 32, 66, 76, 79, 66, 32, 97, 110, 100, 32, 68, 69, 76, 84, 65, 32, 116, 97, 98, +108, 101, 115, 32, 97, 98, 111, 118, 101, 32, 104, 111, 108, 100, 32, 116, 104, 101, 32, 34, +103, 108, 111, 98, 97, 108, 32, 115, 116, 97, 116, 101, 34, 32, 111, 102, 32, 97, 32, 70, +111, 115, 115, 105, 108, 10, 45, 45, 32, 112, 114, 111, 106, 101, 99, 116, 59, 32, 116, 104, +101, 32, 115, 116, 117, 102, 102, 32, 116, 104, 97, 116, 32, 105, 115, 32, 110, 111, 114, 109, +97, 108, 108, 121, 32, 101, 120, 99, 104, 97, 110, 103, 101, 100, 32, 100, 117, 114, 105, 110, +103, 32, 34, 115, 121, 110, 99, 34, 46, 32, 32, 84, 104, 101, 10, 45, 45, 32, 34, 108, +111, 99, 97, 108, 32, 115, 116, 97, 116, 101, 34, 32, 111, 102, 32, 97, 32, 114, 101, 112, +111, 115, 105, 116, 111, 114, 121, 32, 105, 115, 32, 99, 111, 110, 116, 97, 105, 110, 101, 100, +32, 105, 110, 32, 116, 104, 101, 32, 114, 101, 109, 97, 105, 110, 105, 110, 103, 32, 116, 97, +98, 108, 101, 115, 32, 111, 102, 10, 45, 45, 32, 116, 104, 101, 32, 122, 82, 101, 112, 111, +115, 105, 116, 111, 114, 121, 83, 99, 104, 101, 109, 97, 49, 32, 115, 116, 114, 105, 110, 103, +46, 32, 32, 10, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, +45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, +45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, +45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 10, 10, 45, +45, 32, 87, 104, 101, 110, 101, 118, 101, 114, 32, 110, 101, 119, 32, 98, 108, 111, 98, 115, +32, 97, 114, 101, 32, 114, 101, 99, 101, 105, 118, 101, 100, 32, 105, 110, 116, 111, 32, 116, +104, 101, 32, 114, 101, 112, 111, 115, 105, 116, 111, 114, 121, 44, 32, 97, 110, 32, 101, 110, +116, 114, 121, 10, 45, 45, 32, 105, 110, 32, 116, 104, 105, 115, 32, 116, 97, 98, 108, 101, +32, 114, 101, 99, 111, 114, 100, 115, 32, 116, 104, 101, 32, 115, 111, 117, 114, 99, 101, 32, +111, 102, 32, 116, 104, 101, 32, 98, 108, 111, 98, 46, 10, 45, 45, 10, 67, 82, 69, 65, +84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 114, 99, 118, 102, 114, 111, +109, 40, 10, 32, 32, 114, 99, 118, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, +82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 44, 32, 32, 32, 32, 32, 32, 45, 45, 32, +82, 101, 99, 101, 105, 118, 101, 100, 45, 70, 114, 111, 109, 32, 73, 68, 10, 32, 32, 117, +105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 82, 69, 70, 69, 82, 69, 78, 67, 69, +83, 32, 117, 115, 101, 114, 44, 32, 32, 32, 32, 45, 45, 32, 85, 115, 101, 114, 32, 108, +111, 103, 105, 110, 10, 32, 32, 109, 116, 105, 109, 101, 32, 68, 65, 84, 69, 84, 73, 77, +69, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, +45, 32, 84, 105, 109, 101, 32, 111, 102, 32, 114, 101, 99, 101, 105, 112, 116, 46, 32, 32, +74, 117, 108, 105, 97, 110, 32, 100, 97, 121, 46, 10, 32, 32, 110, 111, 110, 99, 101, 32, +84, 69, 88, 84, 32, 85, 78, 73, 81, 85, 69, 44, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 45, 45, 32, 78, 111, 110, 99, 101, 32, 117, 115, 101, 100, 32, +102, 111, 114, 32, 108, 111, 103, 105, 110, 10, 32, 32, 105, 112, 97, 100, 100, 114, 32, 84, +69, 88, 84, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 45, 45, 32, 82, 101, 109, 111, 116, 101, 32, 73, 80, 32, 97, 100, 100, +114, 101, 115, 115, 46, 32, 32, 78, 85, 76, 76, 32, 102, 111, 114, 32, 100, 105, 114, 101, +99, 116, 46, 10, 41, 59, 10, 73, 78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, 114, +101, 112, 111, 46, 114, 99, 118, 102, 114, 111, 109, 40, 114, 99, 118, 105, 100, 44, 117, 105, +100, 44, 109, 116, 105, 109, 101, 44, 110, 111, 110, 99, 101, 44, 105, 112, 97, 100, 100, 114, +41, 10, 86, 65, 76, 85, 69, 83, 32, 40, 49, 44, 32, 49, 44, 32, 106, 117, 108, 105, +97, 110, 100, 97, 121, 40, 39, 110, 111, 119, 39, 41, 44, 32, 78, 85, 76, 76, 44, 32, +78, 85, 76, 76, 41, 59, 10, 10, 45, 45, 32, 73, 110, 102, 111, 114, 109, 97, 116, 105, +111, 110, 32, 97, 98, 111, 117, 116, 32, 117, 115, 101, 114, 115, 10, 45, 45, 10, 45, 45, +32, 84, 104, 101, 32, 117, 115, 101, 114, 46, 112, 119, 32, 102, 105, 101, 108, 100, 32, 99, +97, 110, 32, 98, 101, 32, 101, 105, 116, 104, 101, 114, 32, 99, 108, 101, 97, 114, 116, 101, +120, 116, 32, 111, 102, 32, 116, 104, 101, 32, 112, 97, 115, 115, 119, 111, 114, 100, 44, 32, +111, 114, 10, 45, 45, 32, 97, 32, 83, 72, 65, 49, 32, 104, 97, 115, 104, 32, 111, 102, +32, 116, 104, 101, 32, 112, 97, 115, 115, 119, 111, 114, 100, 46, 32, 32, 73, 102, 32, 116, +104, 101, 32, 117, 115, 101, 114, 46, 112, 119, 32, 102, 105, 101, 108, 100, 32, 105, 115, 32, +101, 120, 97, 99, 116, 108, 121, 32, 52, 48, 10, 45, 45, 32, 99, 104, 97, 114, 97, 99, +116, 101, 114, 115, 32, 108, 111, 110, 103, 32, 119, 101, 32, 97, 115, 115, 117, 109, 101, 32, +105, 116, 32, 105, 115, 32, 97, 32, 83, 72, 65, 49, 32, 104, 97, 115, 104, 46, 32, 32, +79, 116, 104, 101, 114, 119, 105, 115, 101, 44, 32, 105, 116, 32, 105, 115, 10, 45, 45, 32, +99, 108, 101, 97, 114, 116, 101, 120, 116, 46, 32, 32, 84, 104, 101, 32, 115, 104, 97, 49, +95, 115, 104, 97, 114, 101, 100, 95, 115, 101, 99, 114, 101, 116, 40, 41, 32, 114, 111, 117, +116, 105, 110, 101, 32, 99, 111, 109, 112, 117, 116, 101, 115, 32, 116, 104, 101, 32, 112, 97, +115, 115, 119, 111, 114, 100, 10, 45, 45, 32, 104, 97, 115, 104, 32, 98, 97, 115, 101, 100, +32, 111, 110, 32, 116, 104, 101, 32, 112, 114, 111, 106, 101, 99, 116, 45, 99, 111, 100, 101, +44, 32, 116, 104, 101, 32, 117, 115, 101, 114, 32, 108, 111, 103, 105, 110, 44, 32, 97, 110, +100, 32, 116, 104, 101, 32, 99, 108, 101, 97, 114, 116, 101, 120, 116, 10, 45, 45, 32, 112, +97, 115, 115, 119, 111, 114, 100, 46, 10, 45, 45, 10, 67, 82, 69, 65, 84, 69, 32, 84, +65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 117, 115, 101, 114, 40, 10, 32, 32, 117, 105, +100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, 82, 73, 77, 65, 82, 89, 32, 75, 69, +89, 44, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 85, 115, 101, 114, 32, 73, 68, +10, 32, 32, 108, 111, 103, 105, 110, 32, 84, 69, 88, 84, 32, 85, 78, 73, 81, 85, 69, +44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 108, 111, +103, 105, 110, 32, 110, 97, 109, 101, 32, 111, 102, 32, 116, 104, 101, 32, 117, 115, 101, 114, +10, 32, 32, 112, 119, 32, 84, 69, 88, 84, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 112, 97, +115, 115, 119, 111, 114, 100, 10, 32, 32, 99, 97, 112, 32, 84, 69, 88, 84, 44, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 45, 45, 32, 67, 97, 112, 97, 98, 105, 108, 105, 116, 105, 101, 115, 32, 111, 102, 32, +116, 104, 105, 115, 32, 117, 115, 101, 114, 10, 32, 32, 99, 111, 111, 107, 105, 101, 32, 84, +69, 88, 84, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 45, 45, 32, 87, 87, 87, 32, 108, 111, 103, 105, 110, 32, 99, 111, 111, +107, 105, 101, 10, 32, 32, 105, 112, 97, 100, 100, 114, 32, 84, 69, 88, 84, 44, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, +32, 73, 80, 32, 97, 100, 100, 114, 101, 115, 115, 32, 102, 111, 114, 32, 119, 104, 105, 99, +104, 32, 99, 111, 111, 107, 105, 101, 32, 105, 115, 32, 118, 97, 108, 105, 100, 10, 32, 32, +99, 101, 120, 112, 105, 114, 101, 32, 68, 65, 84, 69, 84, 73, 77, 69, 44, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 84, 105, 109, 101, 32, +119, 104, 101, 110, 32, 99, 111, 111, 107, 105, 101, 32, 101, 120, 112, 105, 114, 101, 115, 10, +32, 32, 105, 110, 102, 111, 32, 84, 69, 88, 84, 44, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 99, 111, 110, +116, 97, 99, 116, 32, 105, 110, 102, 111, 114, 109, 97, 116, 105, 111, 110, 10, 32, 32, 109, +116, 105, 109, 101, 32, 68, 65, 84, 69, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 108, 97, 115, 116, 32, 99, +104, 97, 110, 103, 101, 46, 32, 32, 115, 101, 99, 111, 110, 100, 115, 32, 115, 105, 110, 99, +101, 32, 49, 57, 55, 48, 10, 32, 32, 112, 104, 111, 116, 111, 32, 66, 76, 79, 66, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 45, 45, 32, 74, 80, 69, 71, 32, 105, 109, 97, 103, 101, 32, 111, 102, 32, 116, 104, +105, 115, 32, 117, 115, 101, 114, 10, 41, 59, 10, 10, 45, 45, 32, 84, 104, 101, 32, 86, +65, 82, 32, 116, 97, 98, 108, 101, 32, 104, 111, 108, 100, 115, 32, 109, 105, 115, 99, 101, +108, 108, 97, 110, 111, 117, 115, 32, 105, 110, 102, 111, 114, 109, 97, 116, 105, 111, 110, 32, +97, 98, 111, 117, 116, 32, 116, 104, 101, 32, 114, 101, 112, 111, 115, 105, 116, 111, 114, 121, +46, 10, 45, 45, 32, 105, 110, 32, 116, 104, 101, 32, 102, 111, 114, 109, 32, 111, 102, 32, +110, 97, 109, 101, 45, 118, 97, 108, 117, 101, 32, 112, 97, 105, 114, 115, 46, 10, 45, 45, +10, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 99, +111, 110, 102, 105, 103, 40, 10, 32, 32, 110, 97, 109, 101, 32, 84, 69, 88, 84, 32, 80, +82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 32, 78, 79, 84, 32, 78, 85, 76, 76, 44, +32, 32, 45, 45, 32, 80, 114, 105, 109, 97, 114, 121, 32, 110, 97, 109, 101, 32, 111, 102, +32, 116, 104, 101, 32, 101, 110, 116, 114, 121, 10, 32, 32, 118, 97, 108, 117, 101, 32, 67, +76, 79, 66, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 45, 45, 32, 67, 111, 110, 116, 101, 110, 116, 32, 111, 102, 32, +116, 104, 101, 32, 110, 97, 109, 101, 100, 32, 112, 97, 114, 97, 109, 101, 116, 101, 114, 10, +32, 32, 109, 116, 105, 109, 101, 32, 68, 65, 84, 69, 44, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 108, 97, +115, 116, 32, 109, 111, 100, 105, 102, 105, 101, 100, 46, 32, 32, 115, 101, 99, 111, 110, 100, +115, 32, 115, 105, 110, 99, 101, 32, 49, 57, 55, 48, 10, 32, 32, 67, 72, 69, 67, 75, +40, 32, 116, 121, 112, 101, 111, 102, 40, 110, 97, 109, 101, 41, 61, 39, 116, 101, 120, 116, +39, 32, 65, 78, 68, 32, 108, 101, 110, 103, 116, 104, 40, 110, 97, 109, 101, 41, 62, 61, +49, 32, 41, 10, 41, 59, 10, 10, 45, 45, 32, 65, 114, 116, 105, 102, 97, 99, 116, 115, +32, 116, 104, 97, 116, 32, 115, 104, 111, 117, 108, 100, 32, 110, 111, 116, 32, 98, 101, 32, +112, 114, 111, 99, 101, 115, 115, 101, 100, 32, 97, 114, 101, 32, 105, 100, 101, 110, 116, 105, +102, 105, 101, 100, 32, 105, 110, 32, 116, 104, 101, 10, 45, 45, 32, 34, 115, 104, 117, 110, +34, 32, 116, 97, 98, 108, 101, 46, 32, 32, 65, 114, 116, 105, 102, 97, 99, 116, 115, 32, +116, 104, 97, 116, 32, 97, 114, 101, 32, 99, 111, 110, 116, 114, 111, 108, 45, 102, 105, 108, +101, 32, 102, 111, 114, 103, 101, 114, 105, 101, 115, 32, 111, 114, 10, 45, 45, 32, 115, 112, +97, 109, 32, 111, 114, 32, 97, 114, 116, 105, 102, 97, 99, 116, 115, 32, 119, 104, 111, 115, +101, 32, 99, 111, 110, 116, 101, 110, 116, 115, 32, 118, 105, 111, 108, 97, 116, 101, 32, 97, +100, 109, 105, 110, 105, 115, 116, 114, 97, 116, 105, 118, 101, 32, 112, 111, 108, 105, 99, 121, +10, 45, 45, 32, 99, 97, 110, 32, 98, 101, 32, 115, 104, 117, 110, 110, 101, 100, 32, 105, +110, 32, 111, 114, 100, 101, 114, 32, 116, 111, 32, 112, 114, 101, 118, 101, 110, 116, 32, 116, +104, 101, 109, 32, 102, 114, 111, 109, 32, 99, 111, 110, 116, 97, 109, 105, 110, 97, 116, 105, +110, 103, 10, 45, 45, 32, 116, 104, 101, 32, 114, 101, 112, 111, 115, 105, 116, 111, 114, 121, +46, 10, 45, 45, 10, 45, 45, 32, 83, 104, 117, 110, 110, 101, 100, 32, 97, 114, 116, 105, +102, 97, 99, 116, 115, 32, 100, 111, 32, 110, 111, 116, 32, 101, 120, 105, 115, 116, 32, 105, +110, 32, 116, 104, 101, 32, 98, 108, 111, 98, 32, 116, 97, 98, 108, 101, 46, 32, 32, 72, +101, 110, 99, 101, 32, 116, 104, 101, 121, 10, 45, 45, 32, 104, 97, 118, 101, 32, 110, 111, +116, 32, 97, 114, 116, 105, 102, 97, 99, 116, 32, 73, 68, 32, 40, 114, 105, 100, 41, 32, +97, 110, 100, 32, 119, 101, 32, 116, 104, 117, 115, 32, 109, 117, 115, 116, 32, 115, 116, 111, +114, 101, 32, 116, 104, 101, 105, 114, 32, 102, 117, 108, 108, 10, 45, 45, 32, 85, 85, 73, +68, 46, 10, 45, 45, 10, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, +101, 112, 111, 46, 115, 104, 117, 110, 40, 10, 32, 32, 117, 117, 105, 100, 32, 85, 78, 73, +81, 85, 69, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 85, 85, 73, +68, 32, 111, 102, 32, 97, 114, 116, 105, 102, 97, 99, 116, 32, 116, 111, 32, 98, 101, 32, +115, 104, 117, 110, 110, 101, 100, 46, 32, 67, 97, 110, 111, 110, 105, 99, 97, 108, 32, 102, +111, 114, 109, 10, 32, 32, 109, 116, 105, 109, 101, 32, 68, 65, 84, 69, 44, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 87, 104, 101, 110, 32, 97, 100, 100, 101, +100, 46, 32, 32, 115, 101, 99, 111, 110, 100, 115, 32, 115, 105, 110, 99, 101, 32, 49, 57, +55, 48, 10, 32, 32, 115, 99, 111, 109, 32, 84, 69, 88, 84, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 79, 112, 116, 105, 111, 110, 97, 108, 32, 116, +101, 120, 116, 32, 101, 120, 112, 108, 97, 105, 110, 105, 110, 103, 32, 119, 104, 121, 32, 116, +104, 101, 32, 115, 104, 117, 110, 32, 111, 99, 99, 117, 114, 114, 101, 100, 10, 41, 59, 10, +10, 45, 45, 32, 65, 114, 116, 105, 102, 97, 99, 116, 115, 32, 116, 104, 97, 116, 32, 115, +104, 111, 117, 108, 100, 32, 110, 111, 116, 32, 98, 101, 32, 112, 117, 115, 104, 101, 100, 32, +97, 114, 101, 32, 115, 116, 111, 114, 101, 100, 32, 105, 110, 32, 116, 104, 101, 32, 34, 112, +114, 105, 118, 97, 116, 101, 34, 10, 45, 45, 32, 116, 97, 98, 108, 101, 46, 32, 32, 80, +114, 105, 118, 97, 116, 101, 32, 97, 114, 116, 105, 102, 97, 99, 116, 115, 32, 97, 114, 101, +32, 111, 109, 105, 116, 116, 101, 100, 32, 102, 114, 111, 109, 32, 116, 104, 101, 32, 34, 117, +110, 99, 108, 117, 115, 116, 101, 114, 101, 100, 34, 32, 97, 110, 100, 10, 45, 45, 32, 34, +117, 110, 115, 101, 110, 116, 34, 32, 116, 97, 98, 108, 101, 115, 46, 10, 45, 45, 10, 67, +82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 112, 114, 105, +118, 97, 116, 101, 40, 114, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, 82, 73, +77, 65, 82, 89, 32, 75, 69, 89, 41, 59, 10, 10, 45, 45, 32, 65, 110, 32, 101, 110, +116, 114, 121, 32, 105, 110, 32, 116, 104, 105, 115, 32, 116, 97, 98, 108, 101, 32, 100, 101, +115, 99, 114, 105, 98, 101, 115, 32, 97, 32, 100, 97, 116, 97, 98, 97, 115, 101, 32, 113, +117, 101, 114, 121, 32, 116, 104, 97, 116, 32, 103, 101, 110, 101, 114, 97, 116, 101, 115, 32, +97, 10, 45, 45, 32, 116, 97, 98, 108, 101, 32, 111, 102, 32, 116, 105, 99, 107, 101, 116, +115, 46, 10, 45, 45, 10, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, +101, 112, 111, 46, 114, 101, 112, 111, 114, 116, 102, 109, 116, 40, 10, 32, 32, 32, 114, 110, +32, 73, 78, 84, 69, 71, 69, 82, 32, 80, 82, 73, 77, 65, 82, 89, 32, 75, 69, 89, +44, 32, 32, 45, 45, 32, 82, 101, 112, 111, 114, 116, 32, 110, 117, 109, 98, 101, 114, 10, +32, 32, 32, 111, 119, 110, 101, 114, 32, 84, 69, 88, 84, 44, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 79, 119, 110, 101, 114, 32, 111, 102, 32, +116, 104, 105, 115, 32, 114, 101, 112, 111, 114, 116, 32, 102, 111, 114, 109, 97, 116, 32, 40, +110, 111, 116, 32, 117, 115, 101, 100, 41, 10, 32, 32, 32, 116, 105, 116, 108, 101, 32, 84, +69, 88, 84, 32, 85, 78, 73, 81, 85, 69, 44, 32, 32, 32, 32, 32, 32, 32, 45, 45, +32, 84, 105, 116, 108, 101, 32, 111, 102, 32, 116, 104, 105, 115, 32, 114, 101, 112, 111, 114, +116, 10, 32, 32, 32, 109, 116, 105, 109, 101, 32, 68, 65, 84, 69, 44, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 76, 97, 115, 116, 32, 109, 111, +100, 105, 102, 105, 101, 100, 46, 32, 32, 115, 101, 99, 111, 110, 100, 115, 32, 115, 105, 110, +99, 101, 32, 49, 57, 55, 48, 10, 32, 32, 32, 99, 111, 108, 115, 32, 84, 69, 88, 84, +44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 65, +32, 99, 111, 108, 111, 114, 45, 107, 101, 121, 32, 115, 112, 101, 99, 105, 102, 105, 99, 97, +116, 105, 111, 110, 10, 32, 32, 32, 115, 113, 108, 99, 111, 100, 101, 32, 84, 69, 88, 84, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 65, 110, 32, 83, +81, 76, 32, 83, 69, 76, 69, 67, 84, 32, 115, 116, 97, 116, 101, 109, 101, 110, 116, 32, +102, 111, 114, 32, 116, 104, 105, 115, 32, 114, 101, 112, 111, 114, 116, 10, 41, 59, 10, 10, +45, 45, 32, 83, 111, 109, 101, 32, 116, 105, 99, 107, 101, 116, 32, 99, 111, 110, 116, 101, +110, 116, 32, 40, 115, 117, 99, 104, 32, 97, 115, 32, 116, 104, 101, 32, 111, 114, 105, 103, +105, 110, 97, 116, 111, 114, 115, 32, 101, 109, 97, 105, 108, 32, 97, 100, 100, 114, 101, 115, +115, 32, 111, 114, 32, 99, 111, 110, 116, 97, 99, 116, 10, 45, 45, 32, 105, 110, 102, 111, +114, 109, 97, 116, 105, 111, 110, 41, 32, 110, 101, 101, 100, 115, 32, 116, 111, 32, 98, 101, +32, 111, 98, 115, 99, 117, 114, 101, 100, 32, 116, 111, 32, 112, 114, 111, 116, 101, 99, 116, +32, 112, 114, 105, 118, 97, 99, 121, 46, 32, 32, 84, 104, 105, 115, 32, 105, 115, 32, 97, +99, 104, 105, 101, 118, 101, 100, 10, 45, 45, 32, 98, 121, 32, 115, 116, 111, 114, 105, 110, +103, 32, 97, 110, 32, 83, 72, 65, 49, 32, 104, 97, 115, 104, 32, 111, 102, 32, 116, 104, +101, 32, 99, 111, 110, 116, 101, 110, 116, 46, 32, 32, 70, 111, 114, 32, 100, 105, 115, 112, +108, 97, 121, 44, 32, 116, 104, 101, 32, 104, 97, 115, 104, 32, 105, 115, 10, 45, 45, 32, +109, 97, 112, 112, 101, 100, 32, 98, 97, 99, 107, 32, 105, 110, 116, 111, 32, 116, 104, 101, +32, 111, 114, 105, 103, 105, 110, 97, 108, 32, 116, 101, 120, 116, 32, 117, 115, 105, 110, 103, +32, 116, 104, 105, 115, 32, 116, 97, 98, 108, 101, 46, 32, 32, 10, 45, 45, 10, 45, 45, +32, 84, 104, 105, 115, 32, 116, 97, 98, 108, 101, 32, 99, 111, 110, 116, 97, 105, 110, 115, +32, 115, 101, 110, 115, 105, 116, 105, 118, 101, 32, 105, 110, 102, 111, 114, 109, 97, 116, 105, +111, 110, 32, 97, 110, 100, 32, 115, 104, 111, 117, 108, 100, 32, 110, 111, 116, 32, 98, 101, +32, 115, 104, 97, 114, 101, 100, 10, 45, 45, 32, 119, 105, 116, 104, 32, 117, 110, 97, 117, +116, 104, 111, 114, 105, 122, 101, 100, 32, 117, 115, 101, 114, 115, 46, 10, 45, 45, 10, 67, +82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 99, 111, 110, +99, 101, 97, 108, 101, 100, 40, 10, 32, 32, 104, 97, 115, 104, 32, 84, 69, 88, 84, 32, +80, 82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 44, 32, 32, 32, 32, 45, 45, 32, 84, +104, 101, 32, 83, 72, 65, 49, 32, 104, 97, 115, 104, 32, 111, 102, 32, 99, 111, 110, 116, +101, 110, 116, 10, 32, 32, 109, 116, 105, 109, 101, 32, 68, 65, 84, 69, 44, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 84, 105, 109, 101, 32, +99, 114, 101, 97, 116, 101, 100, 46, 32, 32, 83, 101, 99, 111, 110, 100, 115, 32, 115, 105, +110, 99, 101, 32, 49, 57, 55, 48, 10, 32, 32, 99, 111, 110, 116, 101, 110, 116, 32, 84, +69, 88, 84, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, +67, 111, 110, 116, 101, 110, 116, 32, 105, 110, 116, 101, 110, 100, 101, 100, 32, 116, 111, 32, +98, 101, 32, 99, 111, 110, 99, 101, 97, 108, 101, 100, 10, 41, 59, 10, 10, 45, 45, 32, +84, 104, 101, 32, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 32, 73, 68, 32, 104, +101, 108, 112, 115, 32, 116, 104, 101, 32, 117, 110, 105, 120, 32, 34, 102, 105, 108, 101, 34, +32, 99, 111, 109, 109, 97, 110, 100, 32, 116, 111, 32, 105, 100, 101, 110, 116, 105, 102, 121, +32, 116, 104, 101, 10, 45, 45, 32, 100, 97, 116, 97, 98, 97, 115, 101, 32, 97, 115, 32, +97, 32, 102, 111, 115, 115, 105, 108, 32, 114, 101, 112, 111, 115, 105, 116, 111, 114, 121, 46, +10, 80, 82, 65, 71, 77, 65, 32, 114, 101, 112, 111, 46, 97, 112, 112, 108, 105, 99, 97, +116, 105, 111, 110, 95, 105, 100, 61, 50, 53, 50, 48, 48, 54, 54, 55, 51, 59, 10, +0}; +char const * fsl_schema_repo1_cstr = fsl_schema_repo1_cstr_a; +/* end of ../sql/repo-static.sql */ +/* end of file schema_repo1_cstr.c */ +/* start of file schema_repo2_cstr.c */ +/* Binary form of file ../sql/repo-transient.sql */ +/** @page page_schema_repo2_cstr Schema: repo-transient.sql +@code +-- This file contains parts of the schema that can change from one +-- version to the next. The data stored in these tables is +-- reconstructed from the information in the main repo schema by the +-- "rebuild" operation. + +-- Filenames +-- +CREATE TABLE repo.filename( + fnid INTEGER PRIMARY KEY, -- Filename ID + name TEXT UNIQUE -- Name of file page +); + +-- Linkages between check-ins, files created by each check-in, and +-- the names of those files. +-- +-- Each entry represents a file that changed content from pid to fid +-- due to the check-in that goes from pmid to mid. fnid is the name +-- of the file in the mid check-in. If the file was renamed as part +-- of the mid check-in, then pfnid is the previous filename. +-- +-- There can be multiple entries for (mid,fid) if the mid check-in was +-- a merge. Entries with isaux==0 are from the primary parent. Merge +-- parents have isaux set to true. +-- +-- Field name mnemonics: +-- mid = Manifest ID. (Each check-in is stored as a "Manifest") +-- fid = File ID. +-- pmid = Parent Manifest ID. +-- pid = Parent file ID. +-- fnid = File Name ID. +-- pfnid = Parent File Name ID. +-- isaux = pmid IS AUXiliary parent, not primary parent +-- +-- pid==0 if the file is added by check-in mid. +-- pid==(-1) if the file exists in a merge parents but not in the primary +-- parent. In other words, if the file file was added by merge. +-- (TODO: confirm if/where this is used in fossil and then make sure +-- libfossil does so, too.) +-- fid==0 if the file is removed by check-in mid. +-- +CREATE TABLE repo.mlink( + mid INTEGER, -- Check-in that contains fid + fid INTEGER, -- New file content RID. 0 if deleted + pmid INTEGER, -- Check-in RID that contains pid + pid INTEGER, -- Prev file content RID. 0 if new. -1 if from a merge + fnid INTEGER REFERENCES filename, -- Name of the file + pfnid INTEGER, -- Previous name. 0 if unchanged + mperm INTEGER, -- File permissions. 1==exec + isaux BOOLEAN DEFAULT 0 -- TRUE if pmid is the primary +); +CREATE INDEX repo.mlink_i1 ON mlink(mid); +CREATE INDEX repo.mlink_i2 ON mlink(fnid); +CREATE INDEX repo.mlink_i3 ON mlink(fid); +CREATE INDEX repo.mlink_i4 ON mlink(pid); + +-- Parent/child linkages between checkins +-- +CREATE TABLE repo.plink( + pid INTEGER REFERENCES blob, -- Parent manifest + cid INTEGER REFERENCES blob, -- Child manifest + isprim BOOLEAN, -- pid is the primary parent of cid + mtime DATETIME, -- the date/time stamp on cid. Julian day. + baseid INTEGER REFERENCES blob, -- Baseline if cid is a delta manifest. + UNIQUE(pid, cid) +); +CREATE INDEX repo.plink_i2 ON plink(cid,pid); + +-- A "leaf" checkin is a checkin that has no children in the same +-- branch. The set of all leaves is easily computed with a join, +-- between the plink and tagxref tables, but it is a slower join for +-- very large repositories (repositories with 100,000 or more checkins) +-- and so it makes sense to precompute the set of leaves. There is +-- one entry in the following table for each leaf. +-- +CREATE TABLE repo.leaf(rid INTEGER PRIMARY KEY); + +-- Events used to generate a timeline +-- +CREATE TABLE repo.event( + type TEXT, -- Type of event: 'ci', 'w', 'e', 't', 'g' + mtime DATETIME, -- Time of occurrence. Julian day. + objid INTEGER PRIMARY KEY, -- Associated record ID + tagid INTEGER, -- Associated ticket or wiki name tag + uid INTEGER REFERENCES user, -- User who caused the event + bgcolor TEXT, -- Color set by 'bgcolor' property + euser TEXT, -- User set by 'user' property + user TEXT, -- Name of the user + ecomment TEXT, -- Comment set by 'comment' property + comment TEXT, -- Comment describing the event + brief TEXT, -- Short comment when tagid already seen + omtime DATETIME -- Original unchanged date+time, or NULL +); +CREATE INDEX repo.event_i1 ON event(mtime); + +-- A record of phantoms. A phantom is a record for which we know the +-- UUID but we do not (yet) know the file content. +-- +CREATE TABLE repo.phantom( + rid INTEGER PRIMARY KEY -- Record ID of the phantom +); + +-- A record of orphaned delta-manifests. An orphan is a delta-manifest +-- for which we have content, but its baseline-manifest is a phantom. +-- We have to track all orphan manifests so that when the baseline arrives, +-- we know to process the orphaned deltas. +CREATE TABLE repo.orphan( + rid INTEGER PRIMARY KEY, -- Delta manifest with a phantom baseline + baseline INTEGER -- Phantom baseline of this orphan +); +CREATE INDEX repo.orphan_baseline ON orphan(baseline); + +-- Unclustered records. An unclustered record is a record (including +-- a cluster records themselves) that is not mentioned by some other +-- cluster. +-- +-- Phantoms are usually included in the unclustered table. A new cluster +-- will never be created that contains a phantom. But another repository +-- might send us a cluster that contains entries that are phantoms to +-- us. +-- +CREATE TABLE repo.unclustered( + rid INTEGER PRIMARY KEY -- Record ID of the unclustered file +); + +-- Records which have never been pushed to another server. This is +-- used to reduce push operations to a single HTTP request in the +-- common case when one repository only talks to a single server. +-- +CREATE TABLE repo.unsent( + rid INTEGER PRIMARY KEY -- Record ID of the phantom +); + +-- Each baseline or manifest can have one or more tags. A tag +-- is defined by a row in the next table. +-- +-- Wiki pages are tagged with "wiki-NAME" where NAME is the name of +-- the wiki page. Tickets changes are tagged with "ticket-UUID" where +-- UUID is the indentifier of the ticket. Tags used to assign symbolic +-- names to baselines are branches are of the form "sym-NAME" where +-- NAME is the symbolic name. +-- +CREATE TABLE repo.tag( + tagid INTEGER PRIMARY KEY, -- Numeric tag ID + tagname TEXT UNIQUE -- Tag name. +); +INSERT INTO repo.tag VALUES(1, 'bgcolor'); -- FSL_TAGID_BGCOLOR +INSERT INTO repo.tag VALUES(2, 'comment'); -- FSL_TAGID_COMMENT +INSERT INTO repo.tag VALUES(3, 'user'); -- FSL_TAGID_USER +INSERT INTO repo.tag VALUES(4, 'date'); -- FSL_TAGID_DATE +INSERT INTO repo.tag VALUES(5, 'hidden'); -- FSL_TAGID_HIDDEN +INSERT INTO repo.tag VALUES(6, 'private'); -- FSL_TAGID_PRIVATE +INSERT INTO repo.tag VALUES(7, 'cluster'); -- FSL_TAGID_CLUSTER +INSERT INTO repo.tag VALUES(8, 'branch'); -- FSL_TAGID_BRANCH +INSERT INTO repo.tag VALUES(9, 'closed'); -- FSL_TAGID_CLOSED +INSERT INTO repo.tag VALUES(10,'parent'); -- FSL_TAGID_PARENT +INSERT INTO repo.tag VALUES(11,'note'); -- FSL_TAG_NOTE +-- arguable, to force auto-increment to start at 100: +-- INSERT INTO tag VALUES(99,'FSL_TAGID_MAX_INTERNAL'); + +-- Assignments of tags to baselines. Note that we allow tags to +-- have values assigned to them. So we are not really dealing with +-- tags here. These are really properties. But we are going to +-- keep calling them tags because in many cases the value is ignored. +-- +CREATE TABLE repo.tagxref( + tagid INTEGER REFERENCES tag, -- The tag that was added or removed + tagtype INTEGER, -- 0:-,cancel 1:+,single 2:*,propagate + srcid INTEGER REFERENCES blob, -- Artifact of tag. 0 for propagated tags + origid INTEGER REFERENCES blob, -- check-in holding propagated tag + value TEXT, -- Value of the tag. Might be NULL. + mtime TIMESTAMP, -- Time of addition or removal. Julian day + rid INTEGER REFERENCE blob, -- Artifact tag is applied to + UNIQUE(rid, tagid) +); +CREATE INDEX repo.tagxref_i1 ON tagxref(tagid, mtime); + +-- When a hyperlink occurs from one artifact to another (for example +-- when a check-in comment refers to a ticket) an entry is made in +-- the following table for that hyperlink. This table is used to +-- facilitate the display of "back links". +-- +CREATE TABLE repo.backlink( + target TEXT, -- Where the hyperlink points to + srctype INT, -- 0: check-in 1: ticket 2: wiki + srcid INT, -- rid for checkin or wiki. tkt_id for ticket. + mtime TIMESTAMP, -- time that the hyperlink was added. Julian day. + UNIQUE(target, srctype, srcid) +); +CREATE INDEX repo.backlink_src ON backlink(srcid, srctype); + +-- Each attachment is an entry in the following table. Only +-- the most recent attachment (identified by the D card) is saved. +-- +CREATE TABLE repo.attachment( + attachid INTEGER PRIMARY KEY, -- Local id for this attachment + isLatest BOOLEAN DEFAULT 0, -- True if this is the one to use + mtime TIMESTAMP, -- Last changed. Julian day. + src TEXT, -- UUID of the attachment. NULL to delete + target TEXT, -- Object attached to. Wikiname or Tkt UUID + filename TEXT, -- Filename for the attachment + comment TEXT, -- Comment associated with this attachment + user TEXT -- Name of user adding attachment +); +CREATE INDEX repo.attachment_idx1 ON attachment(target, filename, mtime); +CREATE INDEX repo.attachment_idx2 ON attachment(src); + +-- For tracking cherrypick merges +CREATE TABLE repo.cherrypick( + parentid INT, + childid INT, + isExclude BOOLEAN DEFAULT false, + PRIMARY KEY(parentid, childid) +) WITHOUT ROWID; +CREATE INDEX repo.cherrypick_cid ON cherrypick(childid); + @endcode + @see schema_repo2() +*/ +/* auto-generated code - edit at your own risk! (Good luck with that!) */ +static char const fsl_schema_repo2_cstr_a[] = { +45, 45, 32, 84, 104, 105, 115, 32, 102, 105, 108, 101, 32, 99, 111, 110, 116, 97, 105, 110, +115, 32, 112, 97, 114, 116, 115, 32, 111, 102, 32, 116, 104, 101, 32, 115, 99, 104, 101, 109, +97, 32, 116, 104, 97, 116, 32, 99, 97, 110, 32, 99, 104, 97, 110, 103, 101, 32, 102, 114, +111, 109, 32, 111, 110, 101, 10, 45, 45, 32, 118, 101, 114, 115, 105, 111, 110, 32, 116, 111, +32, 116, 104, 101, 32, 110, 101, 120, 116, 46, 32, 84, 104, 101, 32, 100, 97, 116, 97, 32, +115, 116, 111, 114, 101, 100, 32, 105, 110, 32, 116, 104, 101, 115, 101, 32, 116, 97, 98, 108, +101, 115, 32, 105, 115, 10, 45, 45, 32, 114, 101, 99, 111, 110, 115, 116, 114, 117, 99, 116, +101, 100, 32, 102, 114, 111, 109, 32, 116, 104, 101, 32, 105, 110, 102, 111, 114, 109, 97, 116, +105, 111, 110, 32, 105, 110, 32, 116, 104, 101, 32, 109, 97, 105, 110, 32, 114, 101, 112, 111, +32, 115, 99, 104, 101, 109, 97, 32, 98, 121, 32, 116, 104, 101, 10, 45, 45, 32, 34, 114, +101, 98, 117, 105, 108, 100, 34, 32, 111, 112, 101, 114, 97, 116, 105, 111, 110, 46, 10, 10, +45, 45, 32, 70, 105, 108, 101, 110, 97, 109, 101, 115, 10, 45, 45, 10, 67, 82, 69, 65, +84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 102, 105, 108, 101, 110, 97, +109, 101, 40, 10, 32, 32, 102, 110, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, +82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 44, 32, 32, 32, 32, 45, 45, 32, 70, 105, +108, 101, 110, 97, 109, 101, 32, 73, 68, 10, 32, 32, 110, 97, 109, 101, 32, 84, 69, 88, +84, 32, 85, 78, 73, 81, 85, 69, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 45, 45, 32, 78, 97, 109, 101, 32, 111, 102, 32, 102, 105, 108, 101, 32, 112, 97, 103, +101, 10, 41, 59, 10, 10, 45, 45, 32, 76, 105, 110, 107, 97, 103, 101, 115, 32, 98, 101, +116, 119, 101, 101, 110, 32, 99, 104, 101, 99, 107, 45, 105, 110, 115, 44, 32, 102, 105, 108, +101, 115, 32, 99, 114, 101, 97, 116, 101, 100, 32, 98, 121, 32, 101, 97, 99, 104, 32, 99, +104, 101, 99, 107, 45, 105, 110, 44, 32, 97, 110, 100, 10, 45, 45, 32, 116, 104, 101, 32, +110, 97, 109, 101, 115, 32, 111, 102, 32, 116, 104, 111, 115, 101, 32, 102, 105, 108, 101, 115, +46, 10, 45, 45, 10, 45, 45, 32, 69, 97, 99, 104, 32, 101, 110, 116, 114, 121, 32, 114, +101, 112, 114, 101, 115, 101, 110, 116, 115, 32, 97, 32, 102, 105, 108, 101, 32, 116, 104, 97, +116, 32, 99, 104, 97, 110, 103, 101, 100, 32, 99, 111, 110, 116, 101, 110, 116, 32, 102, 114, +111, 109, 32, 112, 105, 100, 32, 116, 111, 32, 102, 105, 100, 10, 45, 45, 32, 100, 117, 101, +32, 116, 111, 32, 116, 104, 101, 32, 99, 104, 101, 99, 107, 45, 105, 110, 32, 116, 104, 97, +116, 32, 103, 111, 101, 115, 32, 102, 114, 111, 109, 32, 112, 109, 105, 100, 32, 116, 111, 32, +109, 105, 100, 46, 32, 32, 102, 110, 105, 100, 32, 105, 115, 32, 116, 104, 101, 32, 110, 97, +109, 101, 10, 45, 45, 32, 111, 102, 32, 116, 104, 101, 32, 102, 105, 108, 101, 32, 105, 110, +32, 116, 104, 101, 32, 109, 105, 100, 32, 99, 104, 101, 99, 107, 45, 105, 110, 46, 32, 32, +73, 102, 32, 116, 104, 101, 32, 102, 105, 108, 101, 32, 119, 97, 115, 32, 114, 101, 110, 97, +109, 101, 100, 32, 97, 115, 32, 112, 97, 114, 116, 10, 45, 45, 32, 111, 102, 32, 116, 104, +101, 32, 109, 105, 100, 32, 99, 104, 101, 99, 107, 45, 105, 110, 44, 32, 116, 104, 101, 110, +32, 112, 102, 110, 105, 100, 32, 105, 115, 32, 116, 104, 101, 32, 112, 114, 101, 118, 105, 111, +117, 115, 32, 102, 105, 108, 101, 110, 97, 109, 101, 46, 10, 45, 45, 10, 45, 45, 32, 84, +104, 101, 114, 101, 32, 99, 97, 110, 32, 98, 101, 32, 109, 117, 108, 116, 105, 112, 108, 101, +32, 101, 110, 116, 114, 105, 101, 115, 32, 102, 111, 114, 32, 40, 109, 105, 100, 44, 102, 105, +100, 41, 32, 105, 102, 32, 116, 104, 101, 32, 109, 105, 100, 32, 99, 104, 101, 99, 107, 45, +105, 110, 32, 119, 97, 115, 10, 45, 45, 32, 97, 32, 109, 101, 114, 103, 101, 46, 32, 32, +69, 110, 116, 114, 105, 101, 115, 32, 119, 105, 116, 104, 32, 105, 115, 97, 117, 120, 61, 61, +48, 32, 97, 114, 101, 32, 102, 114, 111, 109, 32, 116, 104, 101, 32, 112, 114, 105, 109, 97, +114, 121, 32, 112, 97, 114, 101, 110, 116, 46, 32, 32, 77, 101, 114, 103, 101, 10, 45, 45, +32, 112, 97, 114, 101, 110, 116, 115, 32, 104, 97, 118, 101, 32, 105, 115, 97, 117, 120, 32, +115, 101, 116, 32, 116, 111, 32, 116, 114, 117, 101, 46, 10, 45, 45, 10, 45, 45, 32, 70, +105, 101, 108, 100, 32, 110, 97, 109, 101, 32, 109, 110, 101, 109, 111, 110, 105, 99, 115, 58, +10, 45, 45, 32, 32, 32, 32, 109, 105, 100, 32, 61, 32, 77, 97, 110, 105, 102, 101, 115, +116, 32, 73, 68, 46, 32, 32, 40, 69, 97, 99, 104, 32, 99, 104, 101, 99, 107, 45, 105, +110, 32, 105, 115, 32, 115, 116, 111, 114, 101, 100, 32, 97, 115, 32, 97, 32, 34, 77, 97, +110, 105, 102, 101, 115, 116, 34, 41, 10, 45, 45, 32, 32, 32, 32, 102, 105, 100, 32, 61, +32, 70, 105, 108, 101, 32, 73, 68, 46, 10, 45, 45, 32, 32, 32, 32, 112, 109, 105, 100, +32, 61, 32, 80, 97, 114, 101, 110, 116, 32, 77, 97, 110, 105, 102, 101, 115, 116, 32, 73, +68, 46, 10, 45, 45, 32, 32, 32, 32, 112, 105, 100, 32, 61, 32, 80, 97, 114, 101, 110, +116, 32, 102, 105, 108, 101, 32, 73, 68, 46, 10, 45, 45, 32, 32, 32, 32, 102, 110, 105, +100, 32, 61, 32, 70, 105, 108, 101, 32, 78, 97, 109, 101, 32, 73, 68, 46, 10, 45, 45, +32, 32, 32, 32, 112, 102, 110, 105, 100, 32, 61, 32, 80, 97, 114, 101, 110, 116, 32, 70, +105, 108, 101, 32, 78, 97, 109, 101, 32, 73, 68, 46, 10, 45, 45, 32, 32, 32, 32, 105, +115, 97, 117, 120, 32, 61, 32, 112, 109, 105, 100, 32, 73, 83, 32, 65, 85, 88, 105, 108, +105, 97, 114, 121, 32, 112, 97, 114, 101, 110, 116, 44, 32, 110, 111, 116, 32, 112, 114, 105, +109, 97, 114, 121, 32, 112, 97, 114, 101, 110, 116, 10, 45, 45, 10, 45, 45, 32, 112, 105, +100, 61, 61, 48, 32, 32, 32, 32, 105, 102, 32, 116, 104, 101, 32, 102, 105, 108, 101, 32, +105, 115, 32, 97, 100, 100, 101, 100, 32, 98, 121, 32, 99, 104, 101, 99, 107, 45, 105, 110, +32, 109, 105, 100, 46, 10, 45, 45, 32, 112, 105, 100, 61, 61, 40, 45, 49, 41, 32, 105, +102, 32, 116, 104, 101, 32, 102, 105, 108, 101, 32, 101, 120, 105, 115, 116, 115, 32, 105, 110, +32, 97, 32, 109, 101, 114, 103, 101, 32, 112, 97, 114, 101, 110, 116, 115, 32, 98, 117, 116, +32, 110, 111, 116, 32, 105, 110, 32, 116, 104, 101, 32, 112, 114, 105, 109, 97, 114, 121, 10, +45, 45, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 112, 97, 114, 101, 110, 116, 46, +32, 32, 73, 110, 32, 111, 116, 104, 101, 114, 32, 119, 111, 114, 100, 115, 44, 32, 105, 102, +32, 116, 104, 101, 32, 102, 105, 108, 101, 32, 102, 105, 108, 101, 32, 119, 97, 115, 32, 97, +100, 100, 101, 100, 32, 98, 121, 32, 109, 101, 114, 103, 101, 46, 10, 45, 45, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 40, 84, 79, 68, 79, 58, 32, 99, 111, 110, 102, 105, +114, 109, 32, 105, 102, 47, 119, 104, 101, 114, 101, 32, 116, 104, 105, 115, 32, 105, 115, 32, +117, 115, 101, 100, 32, 105, 110, 32, 102, 111, 115, 115, 105, 108, 32, 97, 110, 100, 32, 116, +104, 101, 110, 32, 109, 97, 107, 101, 32, 115, 117, 114, 101, 10, 45, 45, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 108, 105, 98, 102, 111, 115, 115, 105, 108, 32, 100, 111, 101, +115, 32, 115, 111, 44, 32, 116, 111, 111, 46, 41, 10, 45, 45, 32, 102, 105, 100, 61, 61, +48, 32, 32, 32, 32, 105, 102, 32, 116, 104, 101, 32, 102, 105, 108, 101, 32, 105, 115, 32, +114, 101, 109, 111, 118, 101, 100, 32, 98, 121, 32, 99, 104, 101, 99, 107, 45, 105, 110, 32, +109, 105, 100, 46, 10, 45, 45, 10, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, +32, 114, 101, 112, 111, 46, 109, 108, 105, 110, 107, 40, 10, 32, 32, 109, 105, 100, 32, 73, +78, 84, 69, 71, 69, 82, 44, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 67, 104, +101, 99, 107, 45, 105, 110, 32, 116, 104, 97, 116, 32, 99, 111, 110, 116, 97, 105, 110, 115, +32, 102, 105, 100, 10, 32, 32, 102, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 44, 32, +32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 78, 101, 119, 32, 102, 105, 108, 101, 32, 99, +111, 110, 116, 101, 110, 116, 32, 82, 73, 68, 46, 32, 48, 32, 105, 102, 32, 100, 101, 108, +101, 116, 101, 100, 10, 32, 32, 112, 109, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 44, +32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 67, 104, 101, 99, 107, 45, 105, 110, 32, 82, +73, 68, 32, 116, 104, 97, 116, 32, 99, 111, 110, 116, 97, 105, 110, 115, 32, 112, 105, 100, +10, 32, 32, 112, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 44, 32, 32, 32, 32, 32, +32, 32, 32, 45, 45, 32, 80, 114, 101, 118, 32, 102, 105, 108, 101, 32, 99, 111, 110, 116, +101, 110, 116, 32, 82, 73, 68, 46, 32, 48, 32, 105, 102, 32, 110, 101, 119, 46, 32, 45, +49, 32, 105, 102, 32, 102, 114, 111, 109, 32, 97, 32, 109, 101, 114, 103, 101, 10, 32, 32, +102, 110, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 82, 69, 70, 69, 82, 69, 78, +67, 69, 83, 32, 102, 105, 108, 101, 110, 97, 109, 101, 44, 32, 32, 32, 45, 45, 32, 78, +97, 109, 101, 32, 111, 102, 32, 116, 104, 101, 32, 102, 105, 108, 101, 10, 32, 32, 112, 102, +110, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 44, 32, 32, 32, 32, 32, 32, 45, 45, +32, 80, 114, 101, 118, 105, 111, 117, 115, 32, 110, 97, 109, 101, 46, 32, 48, 32, 105, 102, +32, 117, 110, 99, 104, 97, 110, 103, 101, 100, 10, 32, 32, 109, 112, 101, 114, 109, 32, 73, +78, 84, 69, 71, 69, 82, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 70, 105, 108, 101, 32, 112, 101, 114, +109, 105, 115, 115, 105, 111, 110, 115, 46, 32, 32, 49, 61, 61, 101, 120, 101, 99, 10, 32, +32, 105, 115, 97, 117, 120, 32, 66, 79, 79, 76, 69, 65, 78, 32, 68, 69, 70, 65, 85, +76, 84, 32, 48, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, +84, 82, 85, 69, 32, 105, 102, 32, 112, 109, 105, 100, 32, 105, 115, 32, 116, 104, 101, 32, +112, 114, 105, 109, 97, 114, 121, 10, 41, 59, 10, 67, 82, 69, 65, 84, 69, 32, 73, 78, +68, 69, 88, 32, 114, 101, 112, 111, 46, 109, 108, 105, 110, 107, 95, 105, 49, 32, 79, 78, +32, 109, 108, 105, 110, 107, 40, 109, 105, 100, 41, 59, 10, 67, 82, 69, 65, 84, 69, 32, +73, 78, 68, 69, 88, 32, 114, 101, 112, 111, 46, 109, 108, 105, 110, 107, 95, 105, 50, 32, +79, 78, 32, 109, 108, 105, 110, 107, 40, 102, 110, 105, 100, 41, 59, 10, 67, 82, 69, 65, +84, 69, 32, 73, 78, 68, 69, 88, 32, 114, 101, 112, 111, 46, 109, 108, 105, 110, 107, 95, +105, 51, 32, 79, 78, 32, 109, 108, 105, 110, 107, 40, 102, 105, 100, 41, 59, 10, 67, 82, +69, 65, 84, 69, 32, 73, 78, 68, 69, 88, 32, 114, 101, 112, 111, 46, 109, 108, 105, 110, +107, 95, 105, 52, 32, 79, 78, 32, 109, 108, 105, 110, 107, 40, 112, 105, 100, 41, 59, 10, +10, 45, 45, 32, 80, 97, 114, 101, 110, 116, 47, 99, 104, 105, 108, 100, 32, 108, 105, 110, +107, 97, 103, 101, 115, 32, 98, 101, 116, 119, 101, 101, 110, 32, 99, 104, 101, 99, 107, 105, +110, 115, 10, 45, 45, 10, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, +101, 112, 111, 46, 112, 108, 105, 110, 107, 40, 10, 32, 32, 112, 105, 100, 32, 73, 78, 84, +69, 71, 69, 82, 32, 82, 69, 70, 69, 82, 69, 78, 67, 69, 83, 32, 98, 108, 111, 98, +44, 32, 32, 32, 32, 45, 45, 32, 80, 97, 114, 101, 110, 116, 32, 109, 97, 110, 105, 102, +101, 115, 116, 10, 32, 32, 99, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 82, 69, +70, 69, 82, 69, 78, 67, 69, 83, 32, 98, 108, 111, 98, 44, 32, 32, 32, 32, 45, 45, +32, 67, 104, 105, 108, 100, 32, 109, 97, 110, 105, 102, 101, 115, 116, 10, 32, 32, 105, 115, +112, 114, 105, 109, 32, 66, 79, 79, 76, 69, 65, 78, 44, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 112, 105, 100, 32, 105, 115, 32, +116, 104, 101, 32, 112, 114, 105, 109, 97, 114, 121, 32, 112, 97, 114, 101, 110, 116, 32, 111, +102, 32, 99, 105, 100, 10, 32, 32, 109, 116, 105, 109, 101, 32, 68, 65, 84, 69, 84, 73, +77, 69, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +45, 45, 32, 116, 104, 101, 32, 100, 97, 116, 101, 47, 116, 105, 109, 101, 32, 115, 116, 97, +109, 112, 32, 111, 110, 32, 99, 105, 100, 46, 32, 32, 74, 117, 108, 105, 97, 110, 32, 100, +97, 121, 46, 10, 32, 32, 98, 97, 115, 101, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, +32, 82, 69, 70, 69, 82, 69, 78, 67, 69, 83, 32, 98, 108, 111, 98, 44, 32, 45, 45, +32, 66, 97, 115, 101, 108, 105, 110, 101, 32, 105, 102, 32, 99, 105, 100, 32, 105, 115, 32, +97, 32, 100, 101, 108, 116, 97, 32, 109, 97, 110, 105, 102, 101, 115, 116, 46, 10, 32, 32, +85, 78, 73, 81, 85, 69, 40, 112, 105, 100, 44, 32, 99, 105, 100, 41, 10, 41, 59, 10, +67, 82, 69, 65, 84, 69, 32, 73, 78, 68, 69, 88, 32, 114, 101, 112, 111, 46, 112, 108, +105, 110, 107, 95, 105, 50, 32, 79, 78, 32, 112, 108, 105, 110, 107, 40, 99, 105, 100, 44, +112, 105, 100, 41, 59, 10, 10, 45, 45, 32, 65, 32, 34, 108, 101, 97, 102, 34, 32, 99, +104, 101, 99, 107, 105, 110, 32, 105, 115, 32, 97, 32, 99, 104, 101, 99, 107, 105, 110, 32, +116, 104, 97, 116, 32, 104, 97, 115, 32, 110, 111, 32, 99, 104, 105, 108, 100, 114, 101, 110, +32, 105, 110, 32, 116, 104, 101, 32, 115, 97, 109, 101, 10, 45, 45, 32, 98, 114, 97, 110, +99, 104, 46, 32, 32, 84, 104, 101, 32, 115, 101, 116, 32, 111, 102, 32, 97, 108, 108, 32, +108, 101, 97, 118, 101, 115, 32, 105, 115, 32, 101, 97, 115, 105, 108, 121, 32, 99, 111, 109, +112, 117, 116, 101, 100, 32, 119, 105, 116, 104, 32, 97, 32, 106, 111, 105, 110, 44, 10, 45, +45, 32, 98, 101, 116, 119, 101, 101, 110, 32, 116, 104, 101, 32, 112, 108, 105, 110, 107, 32, +97, 110, 100, 32, 116, 97, 103, 120, 114, 101, 102, 32, 116, 97, 98, 108, 101, 115, 44, 32, +98, 117, 116, 32, 105, 116, 32, 105, 115, 32, 97, 32, 115, 108, 111, 119, 101, 114, 32, 106, +111, 105, 110, 32, 102, 111, 114, 10, 45, 45, 32, 118, 101, 114, 121, 32, 108, 97, 114, 103, +101, 32, 114, 101, 112, 111, 115, 105, 116, 111, 114, 105, 101, 115, 32, 40, 114, 101, 112, 111, +115, 105, 116, 111, 114, 105, 101, 115, 32, 119, 105, 116, 104, 32, 49, 48, 48, 44, 48, 48, +48, 32, 111, 114, 32, 109, 111, 114, 101, 32, 99, 104, 101, 99, 107, 105, 110, 115, 41, 10, +45, 45, 32, 97, 110, 100, 32, 115, 111, 32, 105, 116, 32, 109, 97, 107, 101, 115, 32, 115, +101, 110, 115, 101, 32, 116, 111, 32, 112, 114, 101, 99, 111, 109, 112, 117, 116, 101, 32, 116, +104, 101, 32, 115, 101, 116, 32, 111, 102, 32, 108, 101, 97, 118, 101, 115, 46, 32, 32, 84, +104, 101, 114, 101, 32, 105, 115, 10, 45, 45, 32, 111, 110, 101, 32, 101, 110, 116, 114, 121, +32, 105, 110, 32, 116, 104, 101, 32, 102, 111, 108, 108, 111, 119, 105, 110, 103, 32, 116, 97, +98, 108, 101, 32, 102, 111, 114, 32, 101, 97, 99, 104, 32, 108, 101, 97, 102, 46, 10, 45, +45, 10, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, +108, 101, 97, 102, 40, 114, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, 82, 73, +77, 65, 82, 89, 32, 75, 69, 89, 41, 59, 10, 10, 45, 45, 32, 69, 118, 101, 110, 116, +115, 32, 117, 115, 101, 100, 32, 116, 111, 32, 103, 101, 110, 101, 114, 97, 116, 101, 32, 97, +32, 116, 105, 109, 101, 108, 105, 110, 101, 10, 45, 45, 10, 67, 82, 69, 65, 84, 69, 32, +84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 101, 118, 101, 110, 116, 40, 10, 32, 32, +116, 121, 112, 101, 32, 84, 69, 88, 84, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 84, 121, 112, 101, 32, +111, 102, 32, 101, 118, 101, 110, 116, 58, 32, 39, 99, 105, 39, 44, 32, 39, 119, 39, 44, +32, 39, 101, 39, 44, 32, 39, 116, 39, 44, 32, 39, 103, 39, 10, 32, 32, 109, 116, 105, +109, 101, 32, 68, 65, 84, 69, 84, 73, 77, 69, 44, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 84, 105, 109, 101, 32, 111, 102, 32, +111, 99, 99, 117, 114, 114, 101, 110, 99, 101, 46, 32, 74, 117, 108, 105, 97, 110, 32, 100, +97, 121, 46, 10, 32, 32, 111, 98, 106, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, +80, 82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 44, 32, 32, 32, 32, 32, 32, 45, 45, +32, 65, 115, 115, 111, 99, 105, 97, 116, 101, 100, 32, 114, 101, 99, 111, 114, 100, 32, 73, +68, 10, 32, 32, 116, 97, 103, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 44, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 65, +115, 115, 111, 99, 105, 97, 116, 101, 100, 32, 116, 105, 99, 107, 101, 116, 32, 111, 114, 32, +119, 105, 107, 105, 32, 110, 97, 109, 101, 32, 116, 97, 103, 10, 32, 32, 117, 105, 100, 32, +73, 78, 84, 69, 71, 69, 82, 32, 82, 69, 70, 69, 82, 69, 78, 67, 69, 83, 32, 117, +115, 101, 114, 44, 32, 32, 32, 32, 45, 45, 32, 85, 115, 101, 114, 32, 119, 104, 111, 32, +99, 97, 117, 115, 101, 100, 32, 116, 104, 101, 32, 101, 118, 101, 110, 116, 10, 32, 32, 98, +103, 99, 111, 108, 111, 114, 32, 84, 69, 88, 84, 44, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 67, 111, 108, 111, 114, 32, +115, 101, 116, 32, 98, 121, 32, 39, 98, 103, 99, 111, 108, 111, 114, 39, 32, 112, 114, 111, +112, 101, 114, 116, 121, 10, 32, 32, 101, 117, 115, 101, 114, 32, 84, 69, 88, 84, 44, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +45, 45, 32, 85, 115, 101, 114, 32, 115, 101, 116, 32, 98, 121, 32, 39, 117, 115, 101, 114, +39, 32, 112, 114, 111, 112, 101, 114, 116, 121, 10, 32, 32, 117, 115, 101, 114, 32, 84, 69, +88, 84, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 45, 45, 32, 78, 97, 109, 101, 32, 111, 102, 32, 116, 104, 101, 32, +117, 115, 101, 114, 10, 32, 32, 101, 99, 111, 109, 109, 101, 110, 116, 32, 84, 69, 88, 84, +44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, +45, 32, 67, 111, 109, 109, 101, 110, 116, 32, 115, 101, 116, 32, 98, 121, 32, 39, 99, 111, +109, 109, 101, 110, 116, 39, 32, 112, 114, 111, 112, 101, 114, 116, 121, 10, 32, 32, 99, 111, +109, 109, 101, 110, 116, 32, 84, 69, 88, 84, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 67, 111, 109, 109, 101, 110, 116, +32, 100, 101, 115, 99, 114, 105, 98, 105, 110, 103, 32, 116, 104, 101, 32, 101, 118, 101, 110, +116, 10, 32, 32, 98, 114, 105, 101, 102, 32, 84, 69, 88, 84, 44, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 83, +104, 111, 114, 116, 32, 99, 111, 109, 109, 101, 110, 116, 32, 119, 104, 101, 110, 32, 116, 97, +103, 105, 100, 32, 97, 108, 114, 101, 97, 100, 121, 32, 115, 101, 101, 110, 10, 32, 32, 111, +109, 116, 105, 109, 101, 32, 68, 65, 84, 69, 84, 73, 77, 69, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 79, 114, 105, 103, 105, 110, +97, 108, 32, 117, 110, 99, 104, 97, 110, 103, 101, 100, 32, 100, 97, 116, 101, 43, 116, 105, +109, 101, 44, 32, 111, 114, 32, 78, 85, 76, 76, 10, 41, 59, 10, 67, 82, 69, 65, 84, +69, 32, 73, 78, 68, 69, 88, 32, 114, 101, 112, 111, 46, 101, 118, 101, 110, 116, 95, 105, +49, 32, 79, 78, 32, 101, 118, 101, 110, 116, 40, 109, 116, 105, 109, 101, 41, 59, 10, 10, +45, 45, 32, 65, 32, 114, 101, 99, 111, 114, 100, 32, 111, 102, 32, 112, 104, 97, 110, 116, +111, 109, 115, 46, 32, 32, 65, 32, 112, 104, 97, 110, 116, 111, 109, 32, 105, 115, 32, 97, +32, 114, 101, 99, 111, 114, 100, 32, 102, 111, 114, 32, 119, 104, 105, 99, 104, 32, 119, 101, +32, 107, 110, 111, 119, 32, 116, 104, 101, 10, 45, 45, 32, 85, 85, 73, 68, 32, 98, 117, +116, 32, 119, 101, 32, 100, 111, 32, 110, 111, 116, 32, 40, 121, 101, 116, 41, 32, 107, 110, +111, 119, 32, 116, 104, 101, 32, 102, 105, 108, 101, 32, 99, 111, 110, 116, 101, 110, 116, 46, +10, 45, 45, 10, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, +111, 46, 112, 104, 97, 110, 116, 111, 109, 40, 10, 32, 32, 114, 105, 100, 32, 73, 78, 84, +69, 71, 69, 82, 32, 80, 82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 32, 32, 32, 32, +32, 32, 32, 32, 32, 45, 45, 32, 82, 101, 99, 111, 114, 100, 32, 73, 68, 32, 111, 102, +32, 116, 104, 101, 32, 112, 104, 97, 110, 116, 111, 109, 10, 41, 59, 10, 10, 45, 45, 32, +65, 32, 114, 101, 99, 111, 114, 100, 32, 111, 102, 32, 111, 114, 112, 104, 97, 110, 101, 100, +32, 100, 101, 108, 116, 97, 45, 109, 97, 110, 105, 102, 101, 115, 116, 115, 46, 32, 32, 65, +110, 32, 111, 114, 112, 104, 97, 110, 32, 105, 115, 32, 97, 32, 100, 101, 108, 116, 97, 45, +109, 97, 110, 105, 102, 101, 115, 116, 10, 45, 45, 32, 102, 111, 114, 32, 119, 104, 105, 99, +104, 32, 119, 101, 32, 104, 97, 118, 101, 32, 99, 111, 110, 116, 101, 110, 116, 44, 32, 98, +117, 116, 32, 105, 116, 115, 32, 98, 97, 115, 101, 108, 105, 110, 101, 45, 109, 97, 110, 105, +102, 101, 115, 116, 32, 105, 115, 32, 97, 32, 112, 104, 97, 110, 116, 111, 109, 46, 10, 45, +45, 32, 87, 101, 32, 104, 97, 118, 101, 32, 116, 111, 32, 116, 114, 97, 99, 107, 32, 97, +108, 108, 32, 111, 114, 112, 104, 97, 110, 32, 109, 97, 110, 105, 102, 101, 115, 116, 115, 32, +115, 111, 32, 116, 104, 97, 116, 32, 119, 104, 101, 110, 32, 116, 104, 101, 32, 98, 97, 115, +101, 108, 105, 110, 101, 32, 97, 114, 114, 105, 118, 101, 115, 44, 10, 45, 45, 32, 119, 101, +32, 107, 110, 111, 119, 32, 116, 111, 32, 112, 114, 111, 99, 101, 115, 115, 32, 116, 104, 101, +32, 111, 114, 112, 104, 97, 110, 101, 100, 32, 100, 101, 108, 116, 97, 115, 46, 10, 67, 82, +69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 111, 114, 112, 104, +97, 110, 40, 10, 32, 32, 114, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, 82, +73, 77, 65, 82, 89, 32, 75, 69, 89, 44, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, +32, 68, 101, 108, 116, 97, 32, 109, 97, 110, 105, 102, 101, 115, 116, 32, 119, 105, 116, 104, +32, 97, 32, 112, 104, 97, 110, 116, 111, 109, 32, 98, 97, 115, 101, 108, 105, 110, 101, 10, +32, 32, 98, 97, 115, 101, 108, 105, 110, 101, 32, 73, 78, 84, 69, 71, 69, 82, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 80, 104, 97, +110, 116, 111, 109, 32, 98, 97, 115, 101, 108, 105, 110, 101, 32, 111, 102, 32, 116, 104, 105, +115, 32, 111, 114, 112, 104, 97, 110, 10, 41, 59, 10, 67, 82, 69, 65, 84, 69, 32, 73, +78, 68, 69, 88, 32, 114, 101, 112, 111, 46, 111, 114, 112, 104, 97, 110, 95, 98, 97, 115, +101, 108, 105, 110, 101, 32, 79, 78, 32, 111, 114, 112, 104, 97, 110, 40, 98, 97, 115, 101, +108, 105, 110, 101, 41, 59, 10, 10, 45, 45, 32, 85, 110, 99, 108, 117, 115, 116, 101, 114, +101, 100, 32, 114, 101, 99, 111, 114, 100, 115, 46, 32, 32, 65, 110, 32, 117, 110, 99, 108, +117, 115, 116, 101, 114, 101, 100, 32, 114, 101, 99, 111, 114, 100, 32, 105, 115, 32, 97, 32, +114, 101, 99, 111, 114, 100, 32, 40, 105, 110, 99, 108, 117, 100, 105, 110, 103, 10, 45, 45, +32, 97, 32, 99, 108, 117, 115, 116, 101, 114, 32, 114, 101, 99, 111, 114, 100, 115, 32, 116, +104, 101, 109, 115, 101, 108, 118, 101, 115, 41, 32, 116, 104, 97, 116, 32, 105, 115, 32, 110, +111, 116, 32, 109, 101, 110, 116, 105, 111, 110, 101, 100, 32, 98, 121, 32, 115, 111, 109, 101, +32, 111, 116, 104, 101, 114, 10, 45, 45, 32, 99, 108, 117, 115, 116, 101, 114, 46, 10, 45, +45, 10, 45, 45, 32, 80, 104, 97, 110, 116, 111, 109, 115, 32, 97, 114, 101, 32, 117, 115, +117, 97, 108, 108, 121, 32, 105, 110, 99, 108, 117, 100, 101, 100, 32, 105, 110, 32, 116, 104, +101, 32, 117, 110, 99, 108, 117, 115, 116, 101, 114, 101, 100, 32, 116, 97, 98, 108, 101, 46, +32, 32, 65, 32, 110, 101, 119, 32, 99, 108, 117, 115, 116, 101, 114, 10, 45, 45, 32, 119, +105, 108, 108, 32, 110, 101, 118, 101, 114, 32, 98, 101, 32, 99, 114, 101, 97, 116, 101, 100, +32, 116, 104, 97, 116, 32, 99, 111, 110, 116, 97, 105, 110, 115, 32, 97, 32, 112, 104, 97, +110, 116, 111, 109, 46, 32, 32, 66, 117, 116, 32, 97, 110, 111, 116, 104, 101, 114, 32, 114, +101, 112, 111, 115, 105, 116, 111, 114, 121, 10, 45, 45, 32, 109, 105, 103, 104, 116, 32, 115, +101, 110, 100, 32, 117, 115, 32, 97, 32, 99, 108, 117, 115, 116, 101, 114, 32, 116, 104, 97, +116, 32, 99, 111, 110, 116, 97, 105, 110, 115, 32, 101, 110, 116, 114, 105, 101, 115, 32, 116, +104, 97, 116, 32, 97, 114, 101, 32, 112, 104, 97, 110, 116, 111, 109, 115, 32, 116, 111, 10, +45, 45, 32, 117, 115, 46, 10, 45, 45, 10, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, +76, 69, 32, 114, 101, 112, 111, 46, 117, 110, 99, 108, 117, 115, 116, 101, 114, 101, 100, 40, +10, 32, 32, 114, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, 82, 73, 77, 65, +82, 89, 32, 75, 69, 89, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 82, 101, +99, 111, 114, 100, 32, 73, 68, 32, 111, 102, 32, 116, 104, 101, 32, 117, 110, 99, 108, 117, +115, 116, 101, 114, 101, 100, 32, 102, 105, 108, 101, 10, 41, 59, 10, 10, 45, 45, 32, 82, +101, 99, 111, 114, 100, 115, 32, 119, 104, 105, 99, 104, 32, 104, 97, 118, 101, 32, 110, 101, +118, 101, 114, 32, 98, 101, 101, 110, 32, 112, 117, 115, 104, 101, 100, 32, 116, 111, 32, 97, +110, 111, 116, 104, 101, 114, 32, 115, 101, 114, 118, 101, 114, 46, 32, 32, 84, 104, 105, 115, +32, 105, 115, 10, 45, 45, 32, 117, 115, 101, 100, 32, 116, 111, 32, 114, 101, 100, 117, 99, +101, 32, 112, 117, 115, 104, 32, 111, 112, 101, 114, 97, 116, 105, 111, 110, 115, 32, 116, 111, +32, 97, 32, 115, 105, 110, 103, 108, 101, 32, 72, 84, 84, 80, 32, 114, 101, 113, 117, 101, +115, 116, 32, 105, 110, 32, 116, 104, 101, 10, 45, 45, 32, 99, 111, 109, 109, 111, 110, 32, +99, 97, 115, 101, 32, 119, 104, 101, 110, 32, 111, 110, 101, 32, 114, 101, 112, 111, 115, 105, +116, 111, 114, 121, 32, 111, 110, 108, 121, 32, 116, 97, 108, 107, 115, 32, 116, 111, 32, 97, +32, 115, 105, 110, 103, 108, 101, 32, 115, 101, 114, 118, 101, 114, 46, 10, 45, 45, 10, 67, +82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 117, 110, 115, +101, 110, 116, 40, 10, 32, 32, 114, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, +82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, +45, 32, 82, 101, 99, 111, 114, 100, 32, 73, 68, 32, 111, 102, 32, 116, 104, 101, 32, 112, +104, 97, 110, 116, 111, 109, 10, 41, 59, 10, 10, 45, 45, 32, 69, 97, 99, 104, 32, 98, +97, 115, 101, 108, 105, 110, 101, 32, 111, 114, 32, 109, 97, 110, 105, 102, 101, 115, 116, 32, +99, 97, 110, 32, 104, 97, 118, 101, 32, 111, 110, 101, 32, 111, 114, 32, 109, 111, 114, 101, +32, 116, 97, 103, 115, 46, 32, 32, 65, 32, 116, 97, 103, 10, 45, 45, 32, 105, 115, 32, +100, 101, 102, 105, 110, 101, 100, 32, 98, 121, 32, 97, 32, 114, 111, 119, 32, 105, 110, 32, +116, 104, 101, 32, 110, 101, 120, 116, 32, 116, 97, 98, 108, 101, 46, 10, 45, 45, 32, 10, +45, 45, 32, 87, 105, 107, 105, 32, 112, 97, 103, 101, 115, 32, 97, 114, 101, 32, 116, 97, +103, 103, 101, 100, 32, 119, 105, 116, 104, 32, 34, 119, 105, 107, 105, 45, 78, 65, 77, 69, +34, 32, 119, 104, 101, 114, 101, 32, 78, 65, 77, 69, 32, 105, 115, 32, 116, 104, 101, 32, +110, 97, 109, 101, 32, 111, 102, 10, 45, 45, 32, 116, 104, 101, 32, 119, 105, 107, 105, 32, +112, 97, 103, 101, 46, 32, 32, 84, 105, 99, 107, 101, 116, 115, 32, 99, 104, 97, 110, 103, +101, 115, 32, 97, 114, 101, 32, 116, 97, 103, 103, 101, 100, 32, 119, 105, 116, 104, 32, 34, +116, 105, 99, 107, 101, 116, 45, 85, 85, 73, 68, 34, 32, 119, 104, 101, 114, 101, 32, 10, +45, 45, 32, 85, 85, 73, 68, 32, 105, 115, 32, 116, 104, 101, 32, 105, 110, 100, 101, 110, +116, 105, 102, 105, 101, 114, 32, 111, 102, 32, 116, 104, 101, 32, 116, 105, 99, 107, 101, 116, +46, 32, 32, 84, 97, 103, 115, 32, 117, 115, 101, 100, 32, 116, 111, 32, 97, 115, 115, 105, +103, 110, 32, 115, 121, 109, 98, 111, 108, 105, 99, 10, 45, 45, 32, 110, 97, 109, 101, 115, +32, 116, 111, 32, 98, 97, 115, 101, 108, 105, 110, 101, 115, 32, 97, 114, 101, 32, 98, 114, +97, 110, 99, 104, 101, 115, 32, 97, 114, 101, 32, 111, 102, 32, 116, 104, 101, 32, 102, 111, +114, 109, 32, 34, 115, 121, 109, 45, 78, 65, 77, 69, 34, 32, 119, 104, 101, 114, 101, 10, +45, 45, 32, 78, 65, 77, 69, 32, 105, 115, 32, 116, 104, 101, 32, 115, 121, 109, 98, 111, +108, 105, 99, 32, 110, 97, 109, 101, 46, 10, 45, 45, 10, 67, 82, 69, 65, 84, 69, 32, +84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 116, 97, 103, 40, 10, 32, 32, 116, 97, +103, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, 82, 73, 77, 65, 82, 89, 32, +75, 69, 89, 44, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 78, 117, 109, 101, 114, 105, +99, 32, 116, 97, 103, 32, 73, 68, 10, 32, 32, 116, 97, 103, 110, 97, 109, 101, 32, 84, +69, 88, 84, 32, 85, 78, 73, 81, 85, 69, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 45, 45, 32, 84, 97, 103, 32, 110, 97, 109, 101, 46, 10, 41, 59, 10, +73, 78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, 114, 101, 112, 111, 46, 116, 97, 103, +32, 86, 65, 76, 85, 69, 83, 40, 49, 44, 32, 39, 98, 103, 99, 111, 108, 111, 114, 39, +41, 59, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 70, 83, 76, 95, 84, 65, +71, 73, 68, 95, 66, 71, 67, 79, 76, 79, 82, 10, 73, 78, 83, 69, 82, 84, 32, 73, +78, 84, 79, 32, 114, 101, 112, 111, 46, 116, 97, 103, 32, 86, 65, 76, 85, 69, 83, 40, +50, 44, 32, 39, 99, 111, 109, 109, 101, 110, 116, 39, 41, 59, 32, 32, 32, 32, 32, 32, +32, 32, 32, 45, 45, 32, 70, 83, 76, 95, 84, 65, 71, 73, 68, 95, 67, 79, 77, 77, +69, 78, 84, 10, 73, 78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, 114, 101, 112, 111, +46, 116, 97, 103, 32, 86, 65, 76, 85, 69, 83, 40, 51, 44, 32, 39, 117, 115, 101, 114, +39, 41, 59, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 70, 83, +76, 95, 84, 65, 71, 73, 68, 95, 85, 83, 69, 82, 10, 73, 78, 83, 69, 82, 84, 32, +73, 78, 84, 79, 32, 114, 101, 112, 111, 46, 116, 97, 103, 32, 86, 65, 76, 85, 69, 83, +40, 52, 44, 32, 39, 100, 97, 116, 101, 39, 41, 59, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 45, 45, 32, 70, 83, 76, 95, 84, 65, 71, 73, 68, 95, 68, 65, 84, +69, 10, 73, 78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, 114, 101, 112, 111, 46, 116, +97, 103, 32, 86, 65, 76, 85, 69, 83, 40, 53, 44, 32, 39, 104, 105, 100, 100, 101, 110, +39, 41, 59, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 70, 83, 76, 95, +84, 65, 71, 73, 68, 95, 72, 73, 68, 68, 69, 78, 10, 73, 78, 83, 69, 82, 84, 32, +73, 78, 84, 79, 32, 114, 101, 112, 111, 46, 116, 97, 103, 32, 86, 65, 76, 85, 69, 83, +40, 54, 44, 32, 39, 112, 114, 105, 118, 97, 116, 101, 39, 41, 59, 32, 32, 32, 32, 32, +32, 32, 32, 32, 45, 45, 32, 70, 83, 76, 95, 84, 65, 71, 73, 68, 95, 80, 82, 73, +86, 65, 84, 69, 10, 73, 78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, 114, 101, 112, +111, 46, 116, 97, 103, 32, 86, 65, 76, 85, 69, 83, 40, 55, 44, 32, 39, 99, 108, 117, +115, 116, 101, 114, 39, 41, 59, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 70, +83, 76, 95, 84, 65, 71, 73, 68, 95, 67, 76, 85, 83, 84, 69, 82, 10, 73, 78, 83, +69, 82, 84, 32, 73, 78, 84, 79, 32, 114, 101, 112, 111, 46, 116, 97, 103, 32, 86, 65, +76, 85, 69, 83, 40, 56, 44, 32, 39, 98, 114, 97, 110, 99, 104, 39, 41, 59, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 70, 83, 76, 95, 84, 65, 71, 73, 68, +95, 66, 82, 65, 78, 67, 72, 10, 73, 78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, +114, 101, 112, 111, 46, 116, 97, 103, 32, 86, 65, 76, 85, 69, 83, 40, 57, 44, 32, 39, +99, 108, 111, 115, 101, 100, 39, 41, 59, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, +45, 32, 70, 83, 76, 95, 84, 65, 71, 73, 68, 95, 67, 76, 79, 83, 69, 68, 10, 73, +78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, 114, 101, 112, 111, 46, 116, 97, 103, 32, +86, 65, 76, 85, 69, 83, 40, 49, 48, 44, 39, 112, 97, 114, 101, 110, 116, 39, 41, 59, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 70, 83, 76, 95, 84, 65, 71, +73, 68, 95, 80, 65, 82, 69, 78, 84, 10, 73, 78, 83, 69, 82, 84, 32, 73, 78, 84, +79, 32, 114, 101, 112, 111, 46, 116, 97, 103, 32, 86, 65, 76, 85, 69, 83, 40, 49, 49, +44, 39, 110, 111, 116, 101, 39, 41, 59, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 45, 45, 32, 70, 83, 76, 95, 84, 65, 71, 95, 78, 79, 84, 69, 10, 45, 45, 32, +97, 114, 103, 117, 97, 98, 108, 101, 44, 32, 116, 111, 32, 102, 111, 114, 99, 101, 32, 97, +117, 116, 111, 45, 105, 110, 99, 114, 101, 109, 101, 110, 116, 32, 116, 111, 32, 115, 116, 97, +114, 116, 32, 97, 116, 32, 49, 48, 48, 58, 10, 45, 45, 32, 73, 78, 83, 69, 82, 84, +32, 73, 78, 84, 79, 32, 116, 97, 103, 32, 86, 65, 76, 85, 69, 83, 40, 57, 57, 44, +39, 70, 83, 76, 95, 84, 65, 71, 73, 68, 95, 77, 65, 88, 95, 73, 78, 84, 69, 82, +78, 65, 76, 39, 41, 59, 10, 10, 45, 45, 32, 65, 115, 115, 105, 103, 110, 109, 101, 110, +116, 115, 32, 111, 102, 32, 116, 97, 103, 115, 32, 116, 111, 32, 98, 97, 115, 101, 108, 105, +110, 101, 115, 46, 32, 32, 78, 111, 116, 101, 32, 116, 104, 97, 116, 32, 119, 101, 32, 97, +108, 108, 111, 119, 32, 116, 97, 103, 115, 32, 116, 111, 10, 45, 45, 32, 104, 97, 118, 101, +32, 118, 97, 108, 117, 101, 115, 32, 97, 115, 115, 105, 103, 110, 101, 100, 32, 116, 111, 32, +116, 104, 101, 109, 46, 32, 32, 83, 111, 32, 119, 101, 32, 97, 114, 101, 32, 110, 111, 116, +32, 114, 101, 97, 108, 108, 121, 32, 100, 101, 97, 108, 105, 110, 103, 32, 119, 105, 116, 104, +10, 45, 45, 32, 116, 97, 103, 115, 32, 104, 101, 114, 101, 46, 32, 32, 84, 104, 101, 115, +101, 32, 97, 114, 101, 32, 114, 101, 97, 108, 108, 121, 32, 112, 114, 111, 112, 101, 114, 116, +105, 101, 115, 46, 32, 32, 66, 117, 116, 32, 119, 101, 32, 97, 114, 101, 32, 103, 111, 105, +110, 103, 32, 116, 111, 10, 45, 45, 32, 107, 101, 101, 112, 32, 99, 97, 108, 108, 105, 110, +103, 32, 116, 104, 101, 109, 32, 116, 97, 103, 115, 32, 98, 101, 99, 97, 117, 115, 101, 32, +105, 110, 32, 109, 97, 110, 121, 32, 99, 97, 115, 101, 115, 32, 116, 104, 101, 32, 118, 97, +108, 117, 101, 32, 105, 115, 32, 105, 103, 110, 111, 114, 101, 100, 46, 10, 45, 45, 10, 67, +82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 116, 97, 103, +120, 114, 101, 102, 40, 10, 32, 32, 116, 97, 103, 105, 100, 32, 73, 78, 84, 69, 71, 69, +82, 32, 82, 69, 70, 69, 82, 69, 78, 67, 69, 83, 32, 116, 97, 103, 44, 32, 32, 32, +45, 45, 32, 84, 104, 101, 32, 116, 97, 103, 32, 116, 104, 97, 116, 32, 119, 97, 115, 32, +97, 100, 100, 101, 100, 32, 111, 114, 32, 114, 101, 109, 111, 118, 101, 100, 10, 32, 32, 116, +97, 103, 116, 121, 112, 101, 32, 73, 78, 84, 69, 71, 69, 82, 44, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 48, 58, 45, 44, 99, 97, +110, 99, 101, 108, 32, 32, 49, 58, 43, 44, 115, 105, 110, 103, 108, 101, 32, 32, 50, 58, +42, 44, 112, 114, 111, 112, 97, 103, 97, 116, 101, 10, 32, 32, 115, 114, 99, 105, 100, 32, +73, 78, 84, 69, 71, 69, 82, 32, 82, 69, 70, 69, 82, 69, 78, 67, 69, 83, 32, 98, +108, 111, 98, 44, 32, 32, 45, 45, 32, 65, 114, 116, 105, 102, 97, 99, 116, 32, 111, 102, +32, 116, 97, 103, 46, 32, 48, 32, 102, 111, 114, 32, 112, 114, 111, 112, 97, 103, 97, 116, +101, 100, 32, 116, 97, 103, 115, 10, 32, 32, 111, 114, 105, 103, 105, 100, 32, 73, 78, 84, +69, 71, 69, 82, 32, 82, 69, 70, 69, 82, 69, 78, 67, 69, 83, 32, 98, 108, 111, 98, +44, 32, 45, 45, 32, 99, 104, 101, 99, 107, 45, 105, 110, 32, 104, 111, 108, 100, 105, 110, +103, 32, 112, 114, 111, 112, 97, 103, 97, 116, 101, 100, 32, 116, 97, 103, 10, 32, 32, 118, +97, 108, 117, 101, 32, 84, 69, 88, 84, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 86, 97, 108, 117, 101, 32, +111, 102, 32, 116, 104, 101, 32, 116, 97, 103, 46, 32, 32, 77, 105, 103, 104, 116, 32, 98, +101, 32, 78, 85, 76, 76, 46, 10, 32, 32, 109, 116, 105, 109, 101, 32, 84, 73, 77, 69, +83, 84, 65, 77, 80, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 45, 45, 32, 84, 105, 109, 101, 32, 111, 102, 32, 97, 100, 100, 105, 116, 105, 111, +110, 32, 111, 114, 32, 114, 101, 109, 111, 118, 97, 108, 46, 32, 74, 117, 108, 105, 97, 110, +32, 100, 97, 121, 10, 32, 32, 114, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 82, +69, 70, 69, 82, 69, 78, 67, 69, 32, 98, 108, 111, 98, 44, 32, 32, 32, 32, 32, 45, +45, 32, 65, 114, 116, 105, 102, 97, 99, 116, 32, 116, 97, 103, 32, 105, 115, 32, 97, 112, +112, 108, 105, 101, 100, 32, 116, 111, 10, 32, 32, 85, 78, 73, 81, 85, 69, 40, 114, 105, +100, 44, 32, 116, 97, 103, 105, 100, 41, 10, 41, 59, 10, 67, 82, 69, 65, 84, 69, 32, +73, 78, 68, 69, 88, 32, 114, 101, 112, 111, 46, 116, 97, 103, 120, 114, 101, 102, 95, 105, +49, 32, 79, 78, 32, 116, 97, 103, 120, 114, 101, 102, 40, 116, 97, 103, 105, 100, 44, 32, +109, 116, 105, 109, 101, 41, 59, 10, 10, 45, 45, 32, 87, 104, 101, 110, 32, 97, 32, 104, +121, 112, 101, 114, 108, 105, 110, 107, 32, 111, 99, 99, 117, 114, 115, 32, 102, 114, 111, 109, +32, 111, 110, 101, 32, 97, 114, 116, 105, 102, 97, 99, 116, 32, 116, 111, 32, 97, 110, 111, +116, 104, 101, 114, 32, 40, 102, 111, 114, 32, 101, 120, 97, 109, 112, 108, 101, 10, 45, 45, +32, 119, 104, 101, 110, 32, 97, 32, 99, 104, 101, 99, 107, 45, 105, 110, 32, 99, 111, 109, +109, 101, 110, 116, 32, 114, 101, 102, 101, 114, 115, 32, 116, 111, 32, 97, 32, 116, 105, 99, +107, 101, 116, 41, 32, 97, 110, 32, 101, 110, 116, 114, 121, 32, 105, 115, 32, 109, 97, 100, +101, 32, 105, 110, 10, 45, 45, 32, 116, 104, 101, 32, 102, 111, 108, 108, 111, 119, 105, 110, +103, 32, 116, 97, 98, 108, 101, 32, 102, 111, 114, 32, 116, 104, 97, 116, 32, 104, 121, 112, +101, 114, 108, 105, 110, 107, 46, 32, 32, 84, 104, 105, 115, 32, 116, 97, 98, 108, 101, 32, +105, 115, 32, 117, 115, 101, 100, 32, 116, 111, 10, 45, 45, 32, 102, 97, 99, 105, 108, 105, +116, 97, 116, 101, 32, 116, 104, 101, 32, 100, 105, 115, 112, 108, 97, 121, 32, 111, 102, 32, +34, 98, 97, 99, 107, 32, 108, 105, 110, 107, 115, 34, 46, 10, 45, 45, 10, 67, 82, 69, +65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 98, 97, 99, 107, 108, +105, 110, 107, 40, 10, 32, 32, 116, 97, 114, 103, 101, 116, 32, 84, 69, 88, 84, 44, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 87, 104, 101, 114, 101, 32, 116, +104, 101, 32, 104, 121, 112, 101, 114, 108, 105, 110, 107, 32, 112, 111, 105, 110, 116, 115, 32, +116, 111, 10, 32, 32, 115, 114, 99, 116, 121, 112, 101, 32, 73, 78, 84, 44, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 48, 58, 32, 99, 104, 101, 99, 107, 45, +105, 110, 32, 32, 49, 58, 32, 116, 105, 99, 107, 101, 116, 32, 32, 50, 58, 32, 119, 105, +107, 105, 10, 32, 32, 115, 114, 99, 105, 100, 32, 73, 78, 84, 44, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 114, 105, 100, 32, 102, 111, 114, 32, 99, +104, 101, 99, 107, 105, 110, 32, 111, 114, 32, 119, 105, 107, 105, 46, 32, 32, 116, 107, 116, +95, 105, 100, 32, 102, 111, 114, 32, 116, 105, 99, 107, 101, 116, 46, 10, 32, 32, 109, 116, +105, 109, 101, 32, 84, 73, 77, 69, 83, 84, 65, 77, 80, 44, 32, 32, 32, 32, 32, 32, +32, 45, 45, 32, 116, 105, 109, 101, 32, 116, 104, 97, 116, 32, 116, 104, 101, 32, 104, 121, +112, 101, 114, 108, 105, 110, 107, 32, 119, 97, 115, 32, 97, 100, 100, 101, 100, 46, 32, 74, +117, 108, 105, 97, 110, 32, 100, 97, 121, 46, 10, 32, 32, 85, 78, 73, 81, 85, 69, 40, +116, 97, 114, 103, 101, 116, 44, 32, 115, 114, 99, 116, 121, 112, 101, 44, 32, 115, 114, 99, +105, 100, 41, 10, 41, 59, 10, 67, 82, 69, 65, 84, 69, 32, 73, 78, 68, 69, 88, 32, +114, 101, 112, 111, 46, 98, 97, 99, 107, 108, 105, 110, 107, 95, 115, 114, 99, 32, 79, 78, +32, 98, 97, 99, 107, 108, 105, 110, 107, 40, 115, 114, 99, 105, 100, 44, 32, 115, 114, 99, +116, 121, 112, 101, 41, 59, 10, 10, 45, 45, 32, 69, 97, 99, 104, 32, 97, 116, 116, 97, +99, 104, 109, 101, 110, 116, 32, 105, 115, 32, 97, 110, 32, 101, 110, 116, 114, 121, 32, 105, +110, 32, 116, 104, 101, 32, 102, 111, 108, 108, 111, 119, 105, 110, 103, 32, 116, 97, 98, 108, +101, 46, 32, 32, 79, 110, 108, 121, 10, 45, 45, 32, 116, 104, 101, 32, 109, 111, 115, 116, +32, 114, 101, 99, 101, 110, 116, 32, 97, 116, 116, 97, 99, 104, 109, 101, 110, 116, 32, 40, +105, 100, 101, 110, 116, 105, 102, 105, 101, 100, 32, 98, 121, 32, 116, 104, 101, 32, 68, 32, +99, 97, 114, 100, 41, 32, 105, 115, 32, 115, 97, 118, 101, 100, 46, 10, 45, 45, 10, 67, +82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 97, 116, 116, +97, 99, 104, 109, 101, 110, 116, 40, 10, 32, 32, 97, 116, 116, 97, 99, 104, 105, 100, 32, +73, 78, 84, 69, 71, 69, 82, 32, 80, 82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 44, +32, 32, 32, 45, 45, 32, 76, 111, 99, 97, 108, 32, 105, 100, 32, 102, 111, 114, 32, 116, +104, 105, 115, 32, 97, 116, 116, 97, 99, 104, 109, 101, 110, 116, 10, 32, 32, 105, 115, 76, +97, 116, 101, 115, 116, 32, 66, 79, 79, 76, 69, 65, 78, 32, 68, 69, 70, 65, 85, 76, +84, 32, 48, 44, 32, 32, 32, 32, 32, 45, 45, 32, 84, 114, 117, 101, 32, 105, 102, 32, +116, 104, 105, 115, 32, 105, 115, 32, 116, 104, 101, 32, 111, 110, 101, 32, 116, 111, 32, 117, +115, 101, 10, 32, 32, 109, 116, 105, 109, 101, 32, 84, 73, 77, 69, 83, 84, 65, 77, 80, +44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, +76, 97, 115, 116, 32, 99, 104, 97, 110, 103, 101, 100, 46, 32, 32, 74, 117, 108, 105, 97, +110, 32, 100, 97, 121, 46, 10, 32, 32, 115, 114, 99, 32, 84, 69, 88, 84, 44, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 45, 45, 32, 85, 85, 73, 68, 32, 111, 102, 32, 116, 104, 101, 32, 97, 116, 116, 97, +99, 104, 109, 101, 110, 116, 46, 32, 32, 78, 85, 76, 76, 32, 116, 111, 32, 100, 101, 108, +101, 116, 101, 10, 32, 32, 116, 97, 114, 103, 101, 116, 32, 84, 69, 88, 84, 44, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, +32, 79, 98, 106, 101, 99, 116, 32, 97, 116, 116, 97, 99, 104, 101, 100, 32, 116, 111, 46, +32, 87, 105, 107, 105, 110, 97, 109, 101, 32, 111, 114, 32, 84, 107, 116, 32, 85, 85, 73, +68, 10, 32, 32, 102, 105, 108, 101, 110, 97, 109, 101, 32, 84, 69, 88, 84, 44, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 70, +105, 108, 101, 110, 97, 109, 101, 32, 102, 111, 114, 32, 116, 104, 101, 32, 97, 116, 116, 97, +99, 104, 109, 101, 110, 116, 10, 32, 32, 99, 111, 109, 109, 101, 110, 116, 32, 84, 69, 88, +84, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 45, 45, 32, 67, 111, 109, 109, 101, 110, 116, 32, 97, 115, 115, 111, 99, 105, 97, 116, +101, 100, 32, 119, 105, 116, 104, 32, 116, 104, 105, 115, 32, 97, 116, 116, 97, 99, 104, 109, +101, 110, 116, 10, 32, 32, 117, 115, 101, 114, 32, 84, 69, 88, 84, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, +32, 78, 97, 109, 101, 32, 111, 102, 32, 117, 115, 101, 114, 32, 97, 100, 100, 105, 110, 103, +32, 97, 116, 116, 97, 99, 104, 109, 101, 110, 116, 10, 41, 59, 10, 67, 82, 69, 65, 84, +69, 32, 73, 78, 68, 69, 88, 32, 114, 101, 112, 111, 46, 97, 116, 116, 97, 99, 104, 109, +101, 110, 116, 95, 105, 100, 120, 49, 32, 79, 78, 32, 97, 116, 116, 97, 99, 104, 109, 101, +110, 116, 40, 116, 97, 114, 103, 101, 116, 44, 32, 102, 105, 108, 101, 110, 97, 109, 101, 44, +32, 109, 116, 105, 109, 101, 41, 59, 10, 67, 82, 69, 65, 84, 69, 32, 73, 78, 68, 69, +88, 32, 114, 101, 112, 111, 46, 97, 116, 116, 97, 99, 104, 109, 101, 110, 116, 95, 105, 100, +120, 50, 32, 79, 78, 32, 97, 116, 116, 97, 99, 104, 109, 101, 110, 116, 40, 115, 114, 99, +41, 59, 10, 10, 45, 45, 32, 70, 111, 114, 32, 116, 114, 97, 99, 107, 105, 110, 103, 32, +99, 104, 101, 114, 114, 121, 112, 105, 99, 107, 32, 109, 101, 114, 103, 101, 115, 10, 67, 82, +69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 99, 104, 101, 114, +114, 121, 112, 105, 99, 107, 40, 10, 32, 32, 112, 97, 114, 101, 110, 116, 105, 100, 32, 73, +78, 84, 44, 10, 32, 32, 99, 104, 105, 108, 100, 105, 100, 32, 73, 78, 84, 44, 10, 32, +32, 105, 115, 69, 120, 99, 108, 117, 100, 101, 32, 66, 79, 79, 76, 69, 65, 78, 32, 68, +69, 70, 65, 85, 76, 84, 32, 102, 97, 108, 115, 101, 44, 10, 32, 32, 80, 82, 73, 77, +65, 82, 89, 32, 75, 69, 89, 40, 112, 97, 114, 101, 110, 116, 105, 100, 44, 32, 99, 104, +105, 108, 100, 105, 100, 41, 10, 41, 32, 87, 73, 84, 72, 79, 85, 84, 32, 82, 79, 87, +73, 68, 59, 10, 67, 82, 69, 65, 84, 69, 32, 73, 78, 68, 69, 88, 32, 114, 101, 112, +111, 46, 99, 104, 101, 114, 114, 121, 112, 105, 99, 107, 95, 99, 105, 100, 32, 79, 78, 32, +99, 104, 101, 114, 114, 121, 112, 105, 99, 107, 40, 99, 104, 105, 108, 100, 105, 100, 41, 59, +10, +0}; +char const * fsl_schema_repo2_cstr = fsl_schema_repo2_cstr_a; +/* end of ../sql/repo-transient.sql */ +/* end of file schema_repo2_cstr.c */ +/* start of file schema_ckout_cstr.c */ +/* Binary form of file ../sql/checkout.sql */ +/** @page page_schema_ckout_cstr Schema: checkout.sql +@code +-- The VVAR table holds miscellanous information about the local database +-- in the form of name-value pairs. This is similar to the VAR table +-- table in the repository except that this table holds information that +-- is specific to the local checkout. +-- +-- Important Variables: +-- +-- repository Full pathname of the repository database +-- user-id Userid to use +-- +CREATE TABLE ckout.vvar( + name TEXT PRIMARY KEY NOT NULL, -- Primary name of the entry + value CLOB, -- Content of the named parameter + CHECK( typeof(name)='text' AND length(name)>=1 ) +); + +-- Each entry in the vfile table represents a single file in the +-- current checkout. +-- +-- The file.rid field is 0 for files or folders that have been +-- added but not yet committed. +-- +-- Vfile.chnged is 0 for unmodified files, 1 for files that have +-- been edited or which have been subjected to a 3-way merge. +-- Vfile.chnged is 2 if the file has been replaced from a different +-- version by the merge and 3 if the file has been added by a merge. +-- Vfile.chnged is 4|5 is the same as 2|3, but the operation has been +-- done by an --integrate merge. The difference between vfile.chnged==2|4 +-- and a regular add is that with vfile.chnged==2|4 we know that the +-- current version of the file is already in the repository. +-- +CREATE TABLE ckout.vfile( + id INTEGER PRIMARY KEY, -- ID of the checked out file + vid INTEGER REFERENCES blob, -- The baseline this file is part of. + chnged INT DEFAULT 0, -- 0:unchnged 1:edited 2:m-chng 3:m-add 4:i-chng 5:i-add + deleted BOOLEAN DEFAULT 0, -- True if deleted + isexe BOOLEAN, -- True if file should be executable + islink BOOLEAN, -- True if file should be symlink + rid INTEGER, -- Originally from this repository record + mrid INTEGER, -- Based on this record due to a merge + mtime INTEGER, -- Mtime of file on disk. sec since 1970 + pathname TEXT, -- Full pathname relative to root + origname TEXT, -- Original pathname. NULL if unchanged + mhash TEXT, -- Hash of mrid iff mrid!=rid. Added 2019-01-19. + UNIQUE(pathname,vid) +); + +-- This table holds a record of uncommitted merges in the local +-- file tree. If a VFILE entry with id has merged with another +-- record, there is an entry in this table with (id,merge) where +-- merge is the RECORD table entry that the file merged against. +-- An id of 0 or <-3 here means the version record itself. When +-- id==(-1) that is a cherrypick merge, id==(-2) that is a +-- backout merge and id==(-4) is a integrate merge. + +CREATE TABLE ckout.vmerge( + id INTEGER REFERENCES vfile, -- VFILE entry that has been merged + merge INTEGER, -- Merged with this record + mhash TEXT -- SHA1/SHA3 hash for merge object +); +CREATE UNIQUE INDEX ckout.vmergex1 ON vmerge(id,mhash); + +-- The following trigger will prevent older versions of Fossil that +-- do not know about the new vmerge.mhash column from updating the +-- vmerge table. This must be done with a trigger, since legacy Fossil +-- uses INSERT OR IGNORE to update vmerge, and the OR IGNORE will cause +-- a NOT NULL constraint to be silently ignored. +CREATE TRIGGER ckout.vmerge_ck1 AFTER INSERT ON vmerge +WHEN new.mhash IS NULL BEGIN + SELECT raise(FAIL, + 'trying to update a newer checkout with an older version of Fossil'); +END; + +-- Identifier for this file type. +-- The integer is the same as 'FSLC'. +PRAGMA ckout.application_id=252006674; + @endcode + @see schema_ckout() +*/ +/* auto-generated code - edit at your own risk! (Good luck with that!) */ +static char const fsl_schema_ckout_cstr_a[] = { +45, 45, 32, 84, 104, 101, 32, 86, 86, 65, 82, 32, 116, 97, 98, 108, 101, 32, 104, 111, +108, 100, 115, 32, 109, 105, 115, 99, 101, 108, 108, 97, 110, 111, 117, 115, 32, 105, 110, 102, +111, 114, 109, 97, 116, 105, 111, 110, 32, 97, 98, 111, 117, 116, 32, 116, 104, 101, 32, 108, +111, 99, 97, 108, 32, 100, 97, 116, 97, 98, 97, 115, 101, 10, 45, 45, 32, 105, 110, 32, +116, 104, 101, 32, 102, 111, 114, 109, 32, 111, 102, 32, 110, 97, 109, 101, 45, 118, 97, 108, +117, 101, 32, 112, 97, 105, 114, 115, 46, 32, 32, 84, 104, 105, 115, 32, 105, 115, 32, 115, +105, 109, 105, 108, 97, 114, 32, 116, 111, 32, 116, 104, 101, 32, 86, 65, 82, 32, 116, 97, +98, 108, 101, 10, 45, 45, 32, 116, 97, 98, 108, 101, 32, 105, 110, 32, 116, 104, 101, 32, +114, 101, 112, 111, 115, 105, 116, 111, 114, 121, 32, 101, 120, 99, 101, 112, 116, 32, 116, 104, +97, 116, 32, 116, 104, 105, 115, 32, 116, 97, 98, 108, 101, 32, 104, 111, 108, 100, 115, 32, +105, 110, 102, 111, 114, 109, 97, 116, 105, 111, 110, 32, 116, 104, 97, 116, 10, 45, 45, 32, +105, 115, 32, 115, 112, 101, 99, 105, 102, 105, 99, 32, 116, 111, 32, 116, 104, 101, 32, 108, +111, 99, 97, 108, 32, 99, 104, 101, 99, 107, 111, 117, 116, 46, 10, 45, 45, 10, 45, 45, +32, 73, 109, 112, 111, 114, 116, 97, 110, 116, 32, 86, 97, 114, 105, 97, 98, 108, 101, 115, +58, 10, 45, 45, 10, 45, 45, 32, 32, 32, 32, 32, 114, 101, 112, 111, 115, 105, 116, 111, +114, 121, 32, 32, 32, 32, 32, 32, 32, 32, 70, 117, 108, 108, 32, 112, 97, 116, 104, 110, +97, 109, 101, 32, 111, 102, 32, 116, 104, 101, 32, 114, 101, 112, 111, 115, 105, 116, 111, 114, +121, 32, 100, 97, 116, 97, 98, 97, 115, 101, 10, 45, 45, 32, 32, 32, 32, 32, 117, 115, +101, 114, 45, 105, 100, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 85, 115, 101, 114, +105, 100, 32, 116, 111, 32, 117, 115, 101, 10, 45, 45, 10, 67, 82, 69, 65, 84, 69, 32, +84, 65, 66, 76, 69, 32, 99, 107, 111, 117, 116, 46, 118, 118, 97, 114, 40, 10, 32, 32, +110, 97, 109, 101, 32, 84, 69, 88, 84, 32, 80, 82, 73, 77, 65, 82, 89, 32, 75, 69, +89, 32, 78, 79, 84, 32, 78, 85, 76, 76, 44, 32, 32, 45, 45, 32, 80, 114, 105, 109, +97, 114, 121, 32, 110, 97, 109, 101, 32, 111, 102, 32, 116, 104, 101, 32, 101, 110, 116, 114, +121, 10, 32, 32, 118, 97, 108, 117, 101, 32, 67, 76, 79, 66, 44, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, +67, 111, 110, 116, 101, 110, 116, 32, 111, 102, 32, 116, 104, 101, 32, 110, 97, 109, 101, 100, +32, 112, 97, 114, 97, 109, 101, 116, 101, 114, 10, 32, 32, 67, 72, 69, 67, 75, 40, 32, +116, 121, 112, 101, 111, 102, 40, 110, 97, 109, 101, 41, 61, 39, 116, 101, 120, 116, 39, 32, +65, 78, 68, 32, 108, 101, 110, 103, 116, 104, 40, 110, 97, 109, 101, 41, 62, 61, 49, 32, +41, 10, 41, 59, 10, 10, 45, 45, 32, 69, 97, 99, 104, 32, 101, 110, 116, 114, 121, 32, +105, 110, 32, 116, 104, 101, 32, 118, 102, 105, 108, 101, 32, 116, 97, 98, 108, 101, 32, 114, +101, 112, 114, 101, 115, 101, 110, 116, 115, 32, 97, 32, 115, 105, 110, 103, 108, 101, 32, 102, +105, 108, 101, 32, 105, 110, 32, 116, 104, 101, 10, 45, 45, 32, 99, 117, 114, 114, 101, 110, +116, 32, 99, 104, 101, 99, 107, 111, 117, 116, 46, 10, 45, 45, 10, 45, 45, 32, 84, 104, +101, 32, 102, 105, 108, 101, 46, 114, 105, 100, 32, 102, 105, 101, 108, 100, 32, 105, 115, 32, +48, 32, 102, 111, 114, 32, 102, 105, 108, 101, 115, 32, 111, 114, 32, 102, 111, 108, 100, 101, +114, 115, 32, 116, 104, 97, 116, 32, 104, 97, 118, 101, 32, 98, 101, 101, 110, 10, 45, 45, +32, 97, 100, 100, 101, 100, 32, 98, 117, 116, 32, 110, 111, 116, 32, 121, 101, 116, 32, 99, +111, 109, 109, 105, 116, 116, 101, 100, 46, 10, 45, 45, 10, 45, 45, 32, 86, 102, 105, 108, +101, 46, 99, 104, 110, 103, 101, 100, 32, 105, 115, 32, 48, 32, 102, 111, 114, 32, 117, 110, +109, 111, 100, 105, 102, 105, 101, 100, 32, 102, 105, 108, 101, 115, 44, 32, 49, 32, 102, 111, +114, 32, 102, 105, 108, 101, 115, 32, 116, 104, 97, 116, 32, 104, 97, 118, 101, 10, 45, 45, +32, 98, 101, 101, 110, 32, 101, 100, 105, 116, 101, 100, 32, 111, 114, 32, 119, 104, 105, 99, +104, 32, 104, 97, 118, 101, 32, 98, 101, 101, 110, 32, 115, 117, 98, 106, 101, 99, 116, 101, +100, 32, 116, 111, 32, 97, 32, 51, 45, 119, 97, 121, 32, 109, 101, 114, 103, 101, 46, 10, +45, 45, 32, 86, 102, 105, 108, 101, 46, 99, 104, 110, 103, 101, 100, 32, 105, 115, 32, 50, +32, 105, 102, 32, 116, 104, 101, 32, 102, 105, 108, 101, 32, 104, 97, 115, 32, 98, 101, 101, +110, 32, 114, 101, 112, 108, 97, 99, 101, 100, 32, 102, 114, 111, 109, 32, 97, 32, 100, 105, +102, 102, 101, 114, 101, 110, 116, 10, 45, 45, 32, 118, 101, 114, 115, 105, 111, 110, 32, 98, +121, 32, 116, 104, 101, 32, 109, 101, 114, 103, 101, 32, 97, 110, 100, 32, 51, 32, 105, 102, +32, 116, 104, 101, 32, 102, 105, 108, 101, 32, 104, 97, 115, 32, 98, 101, 101, 110, 32, 97, +100, 100, 101, 100, 32, 98, 121, 32, 97, 32, 109, 101, 114, 103, 101, 46, 10, 45, 45, 32, +86, 102, 105, 108, 101, 46, 99, 104, 110, 103, 101, 100, 32, 105, 115, 32, 52, 124, 53, 32, +105, 115, 32, 116, 104, 101, 32, 115, 97, 109, 101, 32, 97, 115, 32, 50, 124, 51, 44, 32, +98, 117, 116, 32, 116, 104, 101, 32, 111, 112, 101, 114, 97, 116, 105, 111, 110, 32, 104, 97, +115, 32, 98, 101, 101, 110, 10, 45, 45, 32, 100, 111, 110, 101, 32, 98, 121, 32, 97, 110, +32, 45, 45, 105, 110, 116, 101, 103, 114, 97, 116, 101, 32, 109, 101, 114, 103, 101, 46, 32, +32, 84, 104, 101, 32, 100, 105, 102, 102, 101, 114, 101, 110, 99, 101, 32, 98, 101, 116, 119, +101, 101, 110, 32, 118, 102, 105, 108, 101, 46, 99, 104, 110, 103, 101, 100, 61, 61, 50, 124, +52, 10, 45, 45, 32, 97, 110, 100, 32, 97, 32, 114, 101, 103, 117, 108, 97, 114, 32, 97, +100, 100, 32, 105, 115, 32, 116, 104, 97, 116, 32, 119, 105, 116, 104, 32, 118, 102, 105, 108, +101, 46, 99, 104, 110, 103, 101, 100, 61, 61, 50, 124, 52, 32, 119, 101, 32, 107, 110, 111, +119, 32, 116, 104, 97, 116, 32, 116, 104, 101, 10, 45, 45, 32, 99, 117, 114, 114, 101, 110, +116, 32, 118, 101, 114, 115, 105, 111, 110, 32, 111, 102, 32, 116, 104, 101, 32, 102, 105, 108, +101, 32, 105, 115, 32, 97, 108, 114, 101, 97, 100, 121, 32, 105, 110, 32, 116, 104, 101, 32, +114, 101, 112, 111, 115, 105, 116, 111, 114, 121, 46, 10, 45, 45, 10, 67, 82, 69, 65, 84, +69, 32, 84, 65, 66, 76, 69, 32, 99, 107, 111, 117, 116, 46, 118, 102, 105, 108, 101, 40, +10, 32, 32, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 80, 82, 73, 77, 65, 82, +89, 32, 75, 69, 89, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, +73, 68, 32, 111, 102, 32, 116, 104, 101, 32, 99, 104, 101, 99, 107, 101, 100, 32, 111, 117, +116, 32, 102, 105, 108, 101, 10, 32, 32, 118, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, +32, 82, 69, 70, 69, 82, 69, 78, 67, 69, 83, 32, 98, 108, 111, 98, 44, 32, 32, 32, +32, 32, 32, 45, 45, 32, 84, 104, 101, 32, 98, 97, 115, 101, 108, 105, 110, 101, 32, 116, +104, 105, 115, 32, 102, 105, 108, 101, 32, 105, 115, 32, 112, 97, 114, 116, 32, 111, 102, 46, +10, 32, 32, 99, 104, 110, 103, 101, 100, 32, 73, 78, 84, 32, 68, 69, 70, 65, 85, 76, +84, 32, 48, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, +48, 58, 117, 110, 99, 104, 110, 103, 101, 100, 32, 49, 58, 101, 100, 105, 116, 101, 100, 32, +50, 58, 109, 45, 99, 104, 110, 103, 32, 51, 58, 109, 45, 97, 100, 100, 32, 52, 58, 105, +45, 99, 104, 110, 103, 32, 53, 58, 105, 45, 97, 100, 100, 10, 32, 32, 100, 101, 108, 101, +116, 101, 100, 32, 66, 79, 79, 76, 69, 65, 78, 32, 68, 69, 70, 65, 85, 76, 84, 32, +48, 44, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 84, 114, 117, 101, 32, 105, 102, +32, 100, 101, 108, 101, 116, 101, 100, 10, 32, 32, 105, 115, 101, 120, 101, 32, 66, 79, 79, +76, 69, 65, 78, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 45, 45, 32, 84, 114, 117, 101, 32, 105, 102, 32, 102, 105, 108, 101, +32, 115, 104, 111, 117, 108, 100, 32, 98, 101, 32, 101, 120, 101, 99, 117, 116, 97, 98, 108, +101, 10, 32, 32, 105, 115, 108, 105, 110, 107, 32, 66, 79, 79, 76, 69, 65, 78, 44, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, +32, 84, 114, 117, 101, 32, 105, 102, 32, 102, 105, 108, 101, 32, 115, 104, 111, 117, 108, 100, +32, 98, 101, 32, 115, 121, 109, 108, 105, 110, 107, 10, 32, 32, 114, 105, 100, 32, 73, 78, +84, 69, 71, 69, 82, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 79, 114, 105, 103, 105, 110, 97, 108, 108, +121, 32, 102, 114, 111, 109, 32, 116, 104, 105, 115, 32, 114, 101, 112, 111, 115, 105, 116, 111, +114, 121, 32, 114, 101, 99, 111, 114, 100, 10, 32, 32, 109, 114, 105, 100, 32, 73, 78, 84, +69, 71, 69, 82, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 45, 45, 32, 66, 97, 115, 101, 100, 32, 111, 110, 32, 116, 104, +105, 115, 32, 114, 101, 99, 111, 114, 100, 32, 100, 117, 101, 32, 116, 111, 32, 97, 32, 109, +101, 114, 103, 101, 10, 32, 32, 109, 116, 105, 109, 101, 32, 73, 78, 84, 69, 71, 69, 82, +44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 45, 45, 32, 77, 116, 105, 109, 101, 32, 111, 102, 32, 102, 105, 108, 101, 32, 111, 110, +32, 100, 105, 115, 107, 46, 32, 115, 101, 99, 32, 115, 105, 110, 99, 101, 32, 49, 57, 55, +48, 10, 32, 32, 112, 97, 116, 104, 110, 97, 109, 101, 32, 84, 69, 88, 84, 44, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, +32, 70, 117, 108, 108, 32, 112, 97, 116, 104, 110, 97, 109, 101, 32, 114, 101, 108, 97, 116, +105, 118, 101, 32, 116, 111, 32, 114, 111, 111, 116, 10, 32, 32, 111, 114, 105, 103, 110, 97, +109, 101, 32, 84, 69, 88, 84, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 79, 114, 105, 103, 105, 110, 97, 108, 32, +112, 97, 116, 104, 110, 97, 109, 101, 46, 32, 78, 85, 76, 76, 32, 105, 102, 32, 117, 110, +99, 104, 97, 110, 103, 101, 100, 10, 32, 32, 109, 104, 97, 115, 104, 32, 84, 69, 88, 84, +44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 45, 45, 32, 72, 97, 115, 104, 32, 111, 102, 32, 109, 114, 105, 100, 32, +105, 102, 102, 32, 109, 114, 105, 100, 33, 61, 114, 105, 100, 46, 32, 65, 100, 100, 101, 100, +32, 50, 48, 49, 57, 45, 48, 49, 45, 49, 57, 46, 10, 32, 32, 85, 78, 73, 81, 85, +69, 40, 112, 97, 116, 104, 110, 97, 109, 101, 44, 118, 105, 100, 41, 10, 41, 59, 10, 10, +45, 45, 32, 84, 104, 105, 115, 32, 116, 97, 98, 108, 101, 32, 104, 111, 108, 100, 115, 32, +97, 32, 114, 101, 99, 111, 114, 100, 32, 111, 102, 32, 117, 110, 99, 111, 109, 109, 105, 116, +116, 101, 100, 32, 109, 101, 114, 103, 101, 115, 32, 105, 110, 32, 116, 104, 101, 32, 108, 111, +99, 97, 108, 10, 45, 45, 32, 102, 105, 108, 101, 32, 116, 114, 101, 101, 46, 32, 32, 73, +102, 32, 97, 32, 86, 70, 73, 76, 69, 32, 101, 110, 116, 114, 121, 32, 119, 105, 116, 104, +32, 105, 100, 32, 104, 97, 115, 32, 109, 101, 114, 103, 101, 100, 32, 119, 105, 116, 104, 32, +97, 110, 111, 116, 104, 101, 114, 10, 45, 45, 32, 114, 101, 99, 111, 114, 100, 44, 32, 116, +104, 101, 114, 101, 32, 105, 115, 32, 97, 110, 32, 101, 110, 116, 114, 121, 32, 105, 110, 32, +116, 104, 105, 115, 32, 116, 97, 98, 108, 101, 32, 119, 105, 116, 104, 32, 40, 105, 100, 44, +109, 101, 114, 103, 101, 41, 32, 119, 104, 101, 114, 101, 10, 45, 45, 32, 109, 101, 114, 103, +101, 32, 105, 115, 32, 116, 104, 101, 32, 82, 69, 67, 79, 82, 68, 32, 116, 97, 98, 108, +101, 32, 101, 110, 116, 114, 121, 32, 116, 104, 97, 116, 32, 116, 104, 101, 32, 102, 105, 108, +101, 32, 109, 101, 114, 103, 101, 100, 32, 97, 103, 97, 105, 110, 115, 116, 46, 10, 45, 45, +32, 65, 110, 32, 105, 100, 32, 111, 102, 32, 48, 32, 111, 114, 32, 60, 45, 51, 32, 104, +101, 114, 101, 32, 109, 101, 97, 110, 115, 32, 116, 104, 101, 32, 118, 101, 114, 115, 105, 111, +110, 32, 114, 101, 99, 111, 114, 100, 32, 105, 116, 115, 101, 108, 102, 46, 32, 32, 87, 104, +101, 110, 10, 45, 45, 32, 105, 100, 61, 61, 40, 45, 49, 41, 32, 116, 104, 97, 116, 32, +105, 115, 32, 97, 32, 99, 104, 101, 114, 114, 121, 112, 105, 99, 107, 32, 109, 101, 114, 103, +101, 44, 32, 105, 100, 61, 61, 40, 45, 50, 41, 32, 116, 104, 97, 116, 32, 105, 115, 32, +97, 10, 45, 45, 32, 98, 97, 99, 107, 111, 117, 116, 32, 109, 101, 114, 103, 101, 32, 97, +110, 100, 32, 105, 100, 61, 61, 40, 45, 52, 41, 32, 105, 115, 32, 97, 32, 105, 110, 116, +101, 103, 114, 97, 116, 101, 32, 109, 101, 114, 103, 101, 46, 10, 10, 67, 82, 69, 65, 84, +69, 32, 84, 65, 66, 76, 69, 32, 99, 107, 111, 117, 116, 46, 118, 109, 101, 114, 103, 101, +40, 10, 32, 32, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 82, 69, 70, 69, 82, +69, 78, 67, 69, 83, 32, 118, 102, 105, 108, 101, 44, 32, 32, 32, 32, 32, 32, 45, 45, +32, 86, 70, 73, 76, 69, 32, 101, 110, 116, 114, 121, 32, 116, 104, 97, 116, 32, 104, 97, +115, 32, 98, 101, 101, 110, 32, 109, 101, 114, 103, 101, 100, 10, 32, 32, 109, 101, 114, 103, +101, 32, 73, 78, 84, 69, 71, 69, 82, 44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 77, 101, 114, 103, 101, 100, 32, +119, 105, 116, 104, 32, 116, 104, 105, 115, 32, 114, 101, 99, 111, 114, 100, 10, 32, 32, 109, +104, 97, 115, 104, 32, 84, 69, 88, 84, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 83, 72, 65, 49, +47, 83, 72, 65, 51, 32, 104, 97, 115, 104, 32, 102, 111, 114, 32, 109, 101, 114, 103, 101, +32, 111, 98, 106, 101, 99, 116, 10, 41, 59, 10, 67, 82, 69, 65, 84, 69, 32, 85, 78, +73, 81, 85, 69, 32, 73, 78, 68, 69, 88, 32, 99, 107, 111, 117, 116, 46, 118, 109, 101, +114, 103, 101, 120, 49, 32, 79, 78, 32, 118, 109, 101, 114, 103, 101, 40, 105, 100, 44, 109, +104, 97, 115, 104, 41, 59, 10, 10, 45, 45, 32, 84, 104, 101, 32, 102, 111, 108, 108, 111, +119, 105, 110, 103, 32, 116, 114, 105, 103, 103, 101, 114, 32, 119, 105, 108, 108, 32, 112, 114, +101, 118, 101, 110, 116, 32, 111, 108, 100, 101, 114, 32, 118, 101, 114, 115, 105, 111, 110, 115, +32, 111, 102, 32, 70, 111, 115, 115, 105, 108, 32, 116, 104, 97, 116, 10, 45, 45, 32, 100, +111, 32, 110, 111, 116, 32, 107, 110, 111, 119, 32, 97, 98, 111, 117, 116, 32, 116, 104, 101, +32, 110, 101, 119, 32, 118, 109, 101, 114, 103, 101, 46, 109, 104, 97, 115, 104, 32, 99, 111, +108, 117, 109, 110, 32, 102, 114, 111, 109, 32, 117, 112, 100, 97, 116, 105, 110, 103, 32, 116, +104, 101, 10, 45, 45, 32, 118, 109, 101, 114, 103, 101, 32, 116, 97, 98, 108, 101, 46, 32, +32, 84, 104, 105, 115, 32, 109, 117, 115, 116, 32, 98, 101, 32, 100, 111, 110, 101, 32, 119, +105, 116, 104, 32, 97, 32, 116, 114, 105, 103, 103, 101, 114, 44, 32, 115, 105, 110, 99, 101, +32, 108, 101, 103, 97, 99, 121, 32, 70, 111, 115, 115, 105, 108, 10, 45, 45, 32, 117, 115, +101, 115, 32, 73, 78, 83, 69, 82, 84, 32, 79, 82, 32, 73, 71, 78, 79, 82, 69, 32, +116, 111, 32, 117, 112, 100, 97, 116, 101, 32, 118, 109, 101, 114, 103, 101, 44, 32, 97, 110, +100, 32, 116, 104, 101, 32, 79, 82, 32, 73, 71, 78, 79, 82, 69, 32, 119, 105, 108, 108, +32, 99, 97, 117, 115, 101, 10, 45, 45, 32, 97, 32, 78, 79, 84, 32, 78, 85, 76, 76, +32, 99, 111, 110, 115, 116, 114, 97, 105, 110, 116, 32, 116, 111, 32, 98, 101, 32, 115, 105, +108, 101, 110, 116, 108, 121, 32, 105, 103, 110, 111, 114, 101, 100, 46, 10, 67, 82, 69, 65, +84, 69, 32, 84, 82, 73, 71, 71, 69, 82, 32, 99, 107, 111, 117, 116, 46, 118, 109, 101, +114, 103, 101, 95, 99, 107, 49, 32, 65, 70, 84, 69, 82, 32, 73, 78, 83, 69, 82, 84, +32, 79, 78, 32, 118, 109, 101, 114, 103, 101, 10, 87, 72, 69, 78, 32, 110, 101, 119, 46, +109, 104, 97, 115, 104, 32, 73, 83, 32, 78, 85, 76, 76, 32, 66, 69, 71, 73, 78, 10, +32, 32, 83, 69, 76, 69, 67, 84, 32, 114, 97, 105, 115, 101, 40, 70, 65, 73, 76, 44, +10, 32, 32, 39, 116, 114, 121, 105, 110, 103, 32, 116, 111, 32, 117, 112, 100, 97, 116, 101, +32, 97, 32, 110, 101, 119, 101, 114, 32, 99, 104, 101, 99, 107, 111, 117, 116, 32, 119, 105, +116, 104, 32, 97, 110, 32, 111, 108, 100, 101, 114, 32, 118, 101, 114, 115, 105, 111, 110, 32, +111, 102, 32, 70, 111, 115, 115, 105, 108, 39, 41, 59, 10, 69, 78, 68, 59, 10, 10, 45, +45, 32, 73, 100, 101, 110, 116, 105, 102, 105, 101, 114, 32, 102, 111, 114, 32, 116, 104, 105, +115, 32, 102, 105, 108, 101, 32, 116, 121, 112, 101, 46, 10, 45, 45, 32, 84, 104, 101, 32, +105, 110, 116, 101, 103, 101, 114, 32, 105, 115, 32, 116, 104, 101, 32, 115, 97, 109, 101, 32, +97, 115, 32, 39, 70, 83, 76, 67, 39, 46, 10, 80, 82, 65, 71, 77, 65, 32, 99, 107, +111, 117, 116, 46, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 95, 105, 100, 61, 50, +53, 50, 48, 48, 54, 54, 55, 52, 59, 10, +0}; +char const * fsl_schema_ckout_cstr = fsl_schema_ckout_cstr_a; +/* end of ../sql/checkout.sql */ +/* end of file schema_ckout_cstr.c */ +/* start of file schema_ticket_cstr.c */ +/* Binary form of file ../sql/ticket.sql */ +/** @page page_schema_ticket_cstr Schema: ticket.sql +@code +-- Template for the TICKET table +CREATE TABLE repo.ticket( + -- Do not change any column that begins with tkt_ + tkt_id INTEGER PRIMARY KEY, + tkt_uuid TEXT UNIQUE, + tkt_mtime DATE, + tkt_ctime DATE, + -- Add as many field as required below this line + type TEXT, + status TEXT, + subsystem TEXT, + priority TEXT, + severity TEXT, + foundin TEXT, + private_contact TEXT, + resolution TEXT, + title TEXT, + comment TEXT +); +CREATE TABLE repo.ticketchng( + -- Do not change any column that begins with tkt_ + tkt_id INTEGER REFERENCES ticket, + tkt_rid INTEGER REFERENCES blob, + tkt_mtime DATE, + -- Add as many fields as required below this line + login TEXT, + username TEXT, + mimetype TEXT, + icomment TEXT +); +CREATE INDEX repo.ticketchng_idx1 ON ticketchng(tkt_id, tkt_mtime); + @endcode + @see schema_ticket() +*/ +/* auto-generated code - edit at your own risk! (Good luck with that!) */ +static char const fsl_schema_ticket_cstr_a[] = { +45, 45, 32, 84, 101, 109, 112, 108, 97, 116, 101, 32, 102, 111, 114, 32, 116, 104, 101, 32, +84, 73, 67, 75, 69, 84, 32, 116, 97, 98, 108, 101, 10, 67, 82, 69, 65, 84, 69, 32, +84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 116, 105, 99, 107, 101, 116, 40, 10, 32, +32, 45, 45, 32, 68, 111, 32, 110, 111, 116, 32, 99, 104, 97, 110, 103, 101, 32, 97, 110, +121, 32, 99, 111, 108, 117, 109, 110, 32, 116, 104, 97, 116, 32, 98, 101, 103, 105, 110, 115, +32, 119, 105, 116, 104, 32, 116, 107, 116, 95, 10, 32, 32, 116, 107, 116, 95, 105, 100, 32, +73, 78, 84, 69, 71, 69, 82, 32, 80, 82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 44, +10, 32, 32, 116, 107, 116, 95, 117, 117, 105, 100, 32, 84, 69, 88, 84, 32, 85, 78, 73, +81, 85, 69, 44, 10, 32, 32, 116, 107, 116, 95, 109, 116, 105, 109, 101, 32, 68, 65, 84, +69, 44, 10, 32, 32, 116, 107, 116, 95, 99, 116, 105, 109, 101, 32, 68, 65, 84, 69, 44, +10, 32, 32, 45, 45, 32, 65, 100, 100, 32, 97, 115, 32, 109, 97, 110, 121, 32, 102, 105, +101, 108, 100, 32, 97, 115, 32, 114, 101, 113, 117, 105, 114, 101, 100, 32, 98, 101, 108, 111, +119, 32, 116, 104, 105, 115, 32, 108, 105, 110, 101, 10, 32, 32, 116, 121, 112, 101, 32, 84, +69, 88, 84, 44, 10, 32, 32, 115, 116, 97, 116, 117, 115, 32, 84, 69, 88, 84, 44, 10, +32, 32, 115, 117, 98, 115, 121, 115, 116, 101, 109, 32, 84, 69, 88, 84, 44, 10, 32, 32, +112, 114, 105, 111, 114, 105, 116, 121, 32, 84, 69, 88, 84, 44, 10, 32, 32, 115, 101, 118, +101, 114, 105, 116, 121, 32, 84, 69, 88, 84, 44, 10, 32, 32, 102, 111, 117, 110, 100, 105, +110, 32, 84, 69, 88, 84, 44, 10, 32, 32, 112, 114, 105, 118, 97, 116, 101, 95, 99, 111, +110, 116, 97, 99, 116, 32, 84, 69, 88, 84, 44, 10, 32, 32, 114, 101, 115, 111, 108, 117, +116, 105, 111, 110, 32, 84, 69, 88, 84, 44, 10, 32, 32, 116, 105, 116, 108, 101, 32, 84, +69, 88, 84, 44, 10, 32, 32, 99, 111, 109, 109, 101, 110, 116, 32, 84, 69, 88, 84, 10, +41, 59, 10, 67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, +46, 116, 105, 99, 107, 101, 116, 99, 104, 110, 103, 40, 10, 32, 32, 45, 45, 32, 68, 111, +32, 110, 111, 116, 32, 99, 104, 97, 110, 103, 101, 32, 97, 110, 121, 32, 99, 111, 108, 117, +109, 110, 32, 116, 104, 97, 116, 32, 98, 101, 103, 105, 110, 115, 32, 119, 105, 116, 104, 32, +116, 107, 116, 95, 10, 32, 32, 116, 107, 116, 95, 105, 100, 32, 73, 78, 84, 69, 71, 69, +82, 32, 82, 69, 70, 69, 82, 69, 78, 67, 69, 83, 32, 116, 105, 99, 107, 101, 116, 44, +10, 32, 32, 116, 107, 116, 95, 114, 105, 100, 32, 73, 78, 84, 69, 71, 69, 82, 32, 82, +69, 70, 69, 82, 69, 78, 67, 69, 83, 32, 98, 108, 111, 98, 44, 10, 32, 32, 116, 107, +116, 95, 109, 116, 105, 109, 101, 32, 68, 65, 84, 69, 44, 10, 32, 32, 45, 45, 32, 65, +100, 100, 32, 97, 115, 32, 109, 97, 110, 121, 32, 102, 105, 101, 108, 100, 115, 32, 97, 115, +32, 114, 101, 113, 117, 105, 114, 101, 100, 32, 98, 101, 108, 111, 119, 32, 116, 104, 105, 115, +32, 108, 105, 110, 101, 10, 32, 32, 108, 111, 103, 105, 110, 32, 84, 69, 88, 84, 44, 10, +32, 32, 117, 115, 101, 114, 110, 97, 109, 101, 32, 84, 69, 88, 84, 44, 10, 32, 32, 109, +105, 109, 101, 116, 121, 112, 101, 32, 84, 69, 88, 84, 44, 10, 32, 32, 105, 99, 111, 109, +109, 101, 110, 116, 32, 84, 69, 88, 84, 10, 41, 59, 10, 67, 82, 69, 65, 84, 69, 32, +73, 78, 68, 69, 88, 32, 114, 101, 112, 111, 46, 116, 105, 99, 107, 101, 116, 99, 104, 110, +103, 95, 105, 100, 120, 49, 32, 79, 78, 32, 116, 105, 99, 107, 101, 116, 99, 104, 110, 103, +40, 116, 107, 116, 95, 105, 100, 44, 32, 116, 107, 116, 95, 109, 116, 105, 109, 101, 41, 59, +10, +0}; +char const * fsl_schema_ticket_cstr = fsl_schema_ticket_cstr_a; +/* end of ../sql/ticket.sql */ +/* end of file schema_ticket_cstr.c */ +/* start of file schema_ticket_reports_cstr.c */ +/* Binary form of file ../sql/ticket-reports.sql */ +/** @page page_schema_ticket_reports_cstr Schema: ticket-reports.sql +@code +INSERT INTO reportfmt(title,mtime,cols,sqlcode) +VALUES('All Tickets',julianday('1970-01-01'),'#ffffff Key: +#f2dcdc Active +#e8e8e8 Review +#cfe8bd Fixed +#bde5d6 Tested +#cacae5 Deferred +#c8c8c8 Closed','SELECT + CASE WHEN status IN (''Open'',''Verified'') THEN ''#f2dcdc'' + WHEN status=''Review'' THEN ''#e8e8e8'' + WHEN status=''Fixed'' THEN ''#cfe8bd'' + WHEN status=''Tested'' THEN ''#bde5d6'' + WHEN status=''Deferred'' THEN ''#cacae5'' + ELSE ''#c8c8c8'' END AS ''bgcolor'', + substr(tkt_uuid,1,10) AS ''#'', + datetime(tkt_mtime) AS ''mtime'', + type, + status, + subsystem, + title +FROM ticket'); + @endcode + @see schema_ticket_reports() +*/ +/* auto-generated code - edit at your own risk! (Good luck with that!) */ +static char const fsl_schema_ticket_reports_cstr_a[] = { +73, 78, 83, 69, 82, 84, 32, 73, 78, 84, 79, 32, 114, 101, 112, 111, 114, 116, 102, 109, +116, 40, 116, 105, 116, 108, 101, 44, 109, 116, 105, 109, 101, 44, 99, 111, 108, 115, 44, 115, +113, 108, 99, 111, 100, 101, 41, 32, 10, 86, 65, 76, 85, 69, 83, 40, 39, 65, 108, 108, +32, 84, 105, 99, 107, 101, 116, 115, 39, 44, 106, 117, 108, 105, 97, 110, 100, 97, 121, 40, +39, 49, 57, 55, 48, 45, 48, 49, 45, 48, 49, 39, 41, 44, 39, 35, 102, 102, 102, 102, +102, 102, 32, 75, 101, 121, 58, 10, 35, 102, 50, 100, 99, 100, 99, 32, 65, 99, 116, 105, +118, 101, 10, 35, 101, 56, 101, 56, 101, 56, 32, 82, 101, 118, 105, 101, 119, 10, 35, 99, +102, 101, 56, 98, 100, 32, 70, 105, 120, 101, 100, 10, 35, 98, 100, 101, 53, 100, 54, 32, +84, 101, 115, 116, 101, 100, 10, 35, 99, 97, 99, 97, 101, 53, 32, 68, 101, 102, 101, 114, +114, 101, 100, 10, 35, 99, 56, 99, 56, 99, 56, 32, 67, 108, 111, 115, 101, 100, 39, 44, +39, 83, 69, 76, 69, 67, 84, 10, 32, 32, 67, 65, 83, 69, 32, 87, 72, 69, 78, 32, +115, 116, 97, 116, 117, 115, 32, 73, 78, 32, 40, 39, 39, 79, 112, 101, 110, 39, 39, 44, +39, 39, 86, 101, 114, 105, 102, 105, 101, 100, 39, 39, 41, 32, 84, 72, 69, 78, 32, 39, +39, 35, 102, 50, 100, 99, 100, 99, 39, 39, 10, 32, 32, 32, 32, 32, 32, 32, 87, 72, +69, 78, 32, 115, 116, 97, 116, 117, 115, 61, 39, 39, 82, 101, 118, 105, 101, 119, 39, 39, +32, 84, 72, 69, 78, 32, 39, 39, 35, 101, 56, 101, 56, 101, 56, 39, 39, 10, 32, 32, +32, 32, 32, 32, 32, 87, 72, 69, 78, 32, 115, 116, 97, 116, 117, 115, 61, 39, 39, 70, +105, 120, 101, 100, 39, 39, 32, 84, 72, 69, 78, 32, 39, 39, 35, 99, 102, 101, 56, 98, +100, 39, 39, 10, 32, 32, 32, 32, 32, 32, 32, 87, 72, 69, 78, 32, 115, 116, 97, 116, +117, 115, 61, 39, 39, 84, 101, 115, 116, 101, 100, 39, 39, 32, 84, 72, 69, 78, 32, 39, +39, 35, 98, 100, 101, 53, 100, 54, 39, 39, 10, 32, 32, 32, 32, 32, 32, 32, 87, 72, +69, 78, 32, 115, 116, 97, 116, 117, 115, 61, 39, 39, 68, 101, 102, 101, 114, 114, 101, 100, +39, 39, 32, 84, 72, 69, 78, 32, 39, 39, 35, 99, 97, 99, 97, 101, 53, 39, 39, 10, +32, 32, 32, 32, 32, 32, 32, 69, 76, 83, 69, 32, 39, 39, 35, 99, 56, 99, 56, 99, +56, 39, 39, 32, 69, 78, 68, 32, 65, 83, 32, 39, 39, 98, 103, 99, 111, 108, 111, 114, +39, 39, 44, 10, 32, 32, 115, 117, 98, 115, 116, 114, 40, 116, 107, 116, 95, 117, 117, 105, +100, 44, 49, 44, 49, 48, 41, 32, 65, 83, 32, 39, 39, 35, 39, 39, 44, 10, 32, 32, +100, 97, 116, 101, 116, 105, 109, 101, 40, 116, 107, 116, 95, 109, 116, 105, 109, 101, 41, 32, +65, 83, 32, 39, 39, 109, 116, 105, 109, 101, 39, 39, 44, 10, 32, 32, 116, 121, 112, 101, +44, 10, 32, 32, 115, 116, 97, 116, 117, 115, 44, 10, 32, 32, 115, 117, 98, 115, 121, 115, +116, 101, 109, 44, 10, 32, 32, 116, 105, 116, 108, 101, 10, 70, 82, 79, 77, 32, 116, 105, +99, 107, 101, 116, 39, 41, 59, 10, +0}; +char const * fsl_schema_ticket_reports_cstr = fsl_schema_ticket_reports_cstr_a; +/* end of ../sql/ticket-reports.sql */ +/* end of file schema_ticket_reports_cstr.c */ +/* start of file schema_forum_cstr.c */ +/* Binary form of file ../sql/forum.sql */ +/** @page page_schema_forum_cstr Schema: forum.sql +@code +CREATE TABLE repo.forumpost( + fpid INTEGER PRIMARY KEY, -- BLOB.rid for the artifact + froot INT, -- fpid of the thread root + fprev INT, -- Previous version of this same post + firt INT, -- This post is in-reply-to + fmtime REAL -- When posted. Julian day +); +CREATE INDEX repo.forumthread ON forumpost(froot,fmtime); + @endcode + @see schema_forum() +*/ +/* auto-generated code - edit at your own risk! (Good luck with that!) */ +static char const fsl_schema_forum_cstr_a[] = { +67, 82, 69, 65, 84, 69, 32, 84, 65, 66, 76, 69, 32, 114, 101, 112, 111, 46, 102, 111, +114, 117, 109, 112, 111, 115, 116, 40, 10, 32, 32, 102, 112, 105, 100, 32, 73, 78, 84, 69, +71, 69, 82, 32, 80, 82, 73, 77, 65, 82, 89, 32, 75, 69, 89, 44, 32, 32, 45, 45, +32, 66, 76, 79, 66, 46, 114, 105, 100, 32, 102, 111, 114, 32, 116, 104, 101, 32, 97, 114, +116, 105, 102, 97, 99, 116, 10, 32, 32, 102, 114, 111, 111, 116, 32, 73, 78, 84, 44, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 102, +112, 105, 100, 32, 111, 102, 32, 116, 104, 101, 32, 116, 104, 114, 101, 97, 100, 32, 114, 111, +111, 116, 10, 32, 32, 102, 112, 114, 101, 118, 32, 73, 78, 84, 44, 32, 32, 32, 32, 32, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 80, 114, 101, 118, 105, +111, 117, 115, 32, 118, 101, 114, 115, 105, 111, 110, 32, 111, 102, 32, 116, 104, 105, 115, 32, +115, 97, 109, 101, 32, 112, 111, 115, 116, 10, 32, 32, 102, 105, 114, 116, 32, 73, 78, 84, +44, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, +45, 32, 84, 104, 105, 115, 32, 112, 111, 115, 116, 32, 105, 115, 32, 105, 110, 45, 114, 101, +112, 108, 121, 45, 116, 111, 10, 32, 32, 102, 109, 116, 105, 109, 101, 32, 82, 69, 65, 76, +32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, 45, 45, 32, 87, +104, 101, 110, 32, 112, 111, 115, 116, 101, 100, 46, 32, 32, 74, 117, 108, 105, 97, 110, 32, +100, 97, 121, 10, 41, 59, 10, 67, 82, 69, 65, 84, 69, 32, 73, 78, 68, 69, 88, 32, +114, 101, 112, 111, 46, 102, 111, 114, 117, 109, 116, 104, 114, 101, 97, 100, 32, 79, 78, 32, +102, 111, 114, 117, 109, 112, 111, 115, 116, 40, 102, 114, 111, 111, 116, 44, 102, 109, 116, 105, +109, 101, 41, 59, 10, +0}; +char const * fsl_schema_forum_cstr = fsl_schema_forum_cstr_a; +/* end of ../sql/forum.sql */ +/* end of file schema_forum_cstr.c */ ADDED lib/libfossil.h Index: lib/libfossil.h ================================================================== --- lib/libfossil.h +++ lib/libfossil.h @@ -0,0 +1,20975 @@ +#if !defined(FSL_AMALGAMATION_BUILD) +#define FSL_AMALGAMATION_BUILD 1 +#endif +#if defined(HAVE_CONFIG_H) +# include "config.h" +#endif +#include "libfossil-config.h" +/* start of file ../include/fossil-scm/fossil-config.h */ +#if !defined (ORG_FOSSIL_SCM_FSL_CONFIG_H_INCLUDED) +#define ORG_FOSSIL_SCM_FSL_CONFIG_H_INCLUDED +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). + +*/ +#if defined(_MSC_VER) && !defined(FSL_AMALGAMATION_BUILD) +# include "config-win32.h" /* manually generated */ +#else +#endif + +#if !defined(_ISOC99_SOURCE) +/* glibc apparently guards snprintf() on this #define, even though + snprintf() is part of C99 and we're building in C99 mode. */ +# define _ISOC99_SOURCE +#endif +#if !defined(_C99_SOURCE) +/* Some Mac enviorments guard snprintf() on this #define but do not + #define it when building in C99 mode. */ +# define _C99_SOURCE +#endif + +#ifdef _WIN32 +# if defined(BUILD_libfossil_static) || defined(FSL_AMALGAMATION_BUILD) +# define FSL_EXPORT extern +# elif defined(BUILD_libfossil) +# define FSL_EXPORT extern __declspec(dllexport) +# else +# define FSL_EXPORT extern __declspec(dllimport) +# endif +#else +# define FSL_EXPORT extern +#endif + +#if !defined(__STDC_VERSION__) || (__STDC_VERSION__ < 199901L) +# if defined(__cplusplus) && !defined(__STDC_FORMAT_MACROS) +/* inttypes.h needs this for the PRI* and SCN* macros in C++ mode. */ +# define __STDC_FORMAT_MACROS +# else +# error "This tree requires a standards-compliant C99-capable compiler." +# endif +#endif + +#include +#include + +#if !defined(FSL_AUX_SCHEMA) +#error "Expecting FSL_AUX_SCHEMA to be defined by the configuration bits." +#endif +#if !defined(FSL_LIBRARY_VERSION) +#error "Expecting FSL_LIBRARY_VERSION to be defined by the configuration bits." +#endif + + +/** @typedef some_int_type fsl_int_t + + fsl_int_t is a signed integer type used to denote "relative" + ranges and lengths, or to tell a routine that it should try to + figure out the length of some byte array itself (e.g. by using + fsl_strlen() on it). It is provided primarily for + documentation/readability purposes, to avoid confusion with the + widely varying integer semantics used by various APIs. This type + is never used as a return type for functions which use "result code + semantics." Those always use an unadorned integer type or some + API-specific enum type. + + The library typedefs this to a 64-bit type if possible, else + a 32-bit type. +*/ +typedef int64_t fsl_int_t; +/** + The unsigned counterpart of fsl_int_t. + */ +typedef uint64_t fsl_uint_t; +/** @def FSL_INT_T_PFMT + + Fossil's fsl_int_t equivalent of C99's PRIi32 and friends. + */ +#define FSL_INT_T_PFMT PRIi64 +/** @def FSL_INT_T_SFMT + + Fossil's fsl_int_t equivalent of C99's SCNi32 and friends. + */ +#define FSL_INT_T_SFMT SCNi64 +/** @def FSL_UINT_T_PFMT + + Fossil's fsl_uint_t equivalent of C99's PRIu32 and friends. + */ +#define FSL_UINT_T_PFMT PRIu64 +/** @def FSL_UINT_T_SFMT + + Fossil's fsl_uint_t equivalent of C99's SCNu32 and friends. + */ +#define FSL_UINT_T_SFMT SCNu64 + +/** @def FSL_JULIAN_T_PFMT + + An output format specifier for Julian-format doubles. + */ +#define FSL_JULIAN_T_PFMT ".17g" + +/** + fsl_size_t is an unsigned integer type used to denote absolute + ranges and lengths. It is provided primarily for + documentation/readability purposes, to avoid confusion with the + widely varying integer semantics used by various APIs. While a + 32-bit type is legal, a 64-bit type is required for "unusually + large" repos and for some metrics reporting even for mid-sized + repos. + */ +typedef uint64_t fsl_size_t; + +/** @def FSL_SIZE_T_PFMT + + Fossil's fsl_size_t equivalent of C99's PRIu32 and friends. + + ACHTUNG: when passing arguments of this type of fsl_appendf(), or + any function which uses it for formatting purposes, it is very + important if if you pass _literal integers_ OR enum values, that + they be cast to fsl_size_t, or the va_list handling might extract + the wrong number of bytes from the argument list, leading to + really weird side-effects via what is effectively memory + corruption. + + That warning applies primarily to the following typedefs and their + format specifiers: fsl_size_t, fsl_int_t, fsl_uint_t, fsl_id_t. + + The warning does not apply to strongly-typed arguments, + e.g. variables of the proper type, so long as the format specifier + string matches the argument type. + + For example: + + ``` + fsl_size_t sz = 3; + fsl_fprintf( stdout, "%"FSL_SIZE_T_PFMT" %"FSL_SIZE_T_PFMT\n", + sz, // OK! + 3 // BAD! See below... + ); + ``` + + The "fix" is to cast the literal 3 to a fsl_size_t resp. the type + appropriate for the format specifier. That ensures that there is + no (or much less ;) confusion when va_arg() extracts arguments + from the variadic array. + + Reminders to self: + + ``` + int i = 0; + f_out(("#%d: %"FSL_ID_T_PFMT" %"FSL_ID_T_PFMT" %"FSL_ID_T_PFMT"\n", + ++i, 1, 2, 3)); + f_out(("#%d: %"FSL_SIZE_T_PFMT" %"FSL_ID_T_PFMT" %"FSL_SIZE_T_PFMT"\n", + ++i, (fsl_size_t)1, (fsl_id_t)2, (fsl_size_t)3)); + // This one is the (generally) problematic case: + f_out(("#%d: %"FSL_SIZE_T_PFMT" %"FSL_ID_T_PFMT" %"FSL_SIZE_T_PFMT"\n", + ++i, 1, 2, 3)); + ``` + + The above was Tested with gcc, clang, tcc on a 32-bit linux + platform (it has not been problematic on 64-bit builds!). The + above problem was reproduced on all compiler combinations i + tried. Current code (20130824) seems to be behaving well as long + as callers always cast to help variadic arg handling DTRT. + */ +#define FSL_SIZE_T_PFMT FSL_UINT_T_PFMT + +/** @def FSL_SIZE_T_SFMT + + Fossil's fsl_int_t equivalent of C99's SCNu32 and friends. + */ +#define FSL_SIZE_T_SFMT FSL_UINT_T_SFMT + +/** + fsl_id_t is a signed integer type used to store database record + IDs. It is provided primarily for documentation/readability purposes, + to avoid confusion with the widely varying integer semantics used + by various APIs. + + This type "could" be 32-bit (instead of 64) because the + oldest/largest Fossil repo (the TCL tree, with 15 years of + history) currently (August 2013) has only 131k RIDs. HOWEVER, + changing this type can have side-effects vis-a-vis va_arg() deep + in the fsl_appendf() implementation if FSL_ID_T_PFMT is not 100% + correct for this typedef. After changing this, _make sure_ to do a + full clean rebuild and test thoroughly because changing a sizeof + can produce weird side-effects (effectively memory corruption) on + unclean rebuilds. + */ +typedef int32_t fsl_id_t; + +/** @def FSL_ID_T_PFMT + + Fossil's fsl_id_t equivalent of C99's PRIi32 and friends. + + ACHTUNG: see FSL_SIZE_T_PFMT for important details. + */ +#define FSL_ID_T_PFMT PRIi32 + +/** @def FSL_ID_T_SFMT + + Fossil's fsl_id_t equivalent of C99's SCNi32 and friends. + */ +#define FSL_ID_T_SFMT SCNi32 + +/** + The type used to represent type values. Unless noted otherwise, + the general convention is Unix Epoch. That said, Fossil internally + uses Julian Date for times, so this typedef is clearly the result + of over-specification/over-thinking the problem. THAT said, + application-level code more commonly works with Unix timestamps, + so... here it is. Over-specified, perhaps, but not 100% + unjustifiable. + */ +typedef int64_t fsl_time_t; + +/** @def FSL_TIME_T_PFMT + + Fossil's fsl_time_t equivalent of C99's PRIi32 and friends. + */ +#define FSL_TIME_T_PFMT PRIi64 + +/** @def FSL_TIME_T_SFMT + + Fossil's fsl_time_t equivalent of C99's SCNi32 and friends. + */ +#define FSL_TIME_T_SFMT SCNi64 + +/** + If true, the fsl_timer_xxx() family of functions might do something useful, + otherwise they do not. + */ +#define FSL_CONFIG_ENABLE_TIMER 1 + + +#endif +/* ORG_FOSSIL_SCM_FSL_CONFIG_H_INCLUDED */ +/* end of file ../include/fossil-scm/fossil-config.h */ +/* start of file ../include/fossil-scm/fossil.h */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#if !defined(ORG_FOSSIL_SCM_LIBFOSSIL_H_INCLUDED) +#define ORG_FOSSIL_SCM_LIBFOSSIL_H_INCLUDED +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/** @file fossil.h + + This file is the primary header for the public APIs. It includes + various other header files. They are split into multiple headers + primarily becuase my lower-end systems choke on syntax-highlighting + them and browsing their (large) Doxygen output. +*/ + +/* + fossil-config.h MUST be included first so we can set some + portability flags and config-dependent typedefs! +*/ + +#endif +/* ORG_FOSSIL_SCM_LIBFOSSIL_H_INCLUDED */ +/* end of file ../include/fossil-scm/fossil.h */ +/* start of file ../include/fossil-scm/fossil-util.h */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#if !defined(ORG_FOSSIL_SCM_FSL_UTIL_H_INCLUDED) +#define ORG_FOSSIL_SCM_FSL_UTIL_H_INCLUDED +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). + +*/ + +/** @file fossil-util.h + + This file declares a number of utility classes and routines used by + libfossil. All of them considered "public", suitable for direct use + by client code. +*/ + +#include /* FILE type */ +#include /* va_list */ +#include /* tm struct */ +#include +#if defined(__cplusplus) +extern "C" { +#endif +typedef struct fsl_allocator fsl_allocator; +typedef struct fsl_buffer fsl_buffer; +typedef struct fsl_error fsl_error; +typedef struct fsl_finalizer fsl_finalizer; +typedef struct fsl_fstat fsl_fstat; +typedef struct fsl_list fsl_list; +typedef struct fsl_outputer fsl_outputer; +typedef struct fsl_state fsl_state; +typedef struct fsl_id_bag fsl_id_bag; + +/** + fsl_uuid_str and fsl_uuid_cstr are "for documentation and + readability purposes" typedefs used to denote strings which the API + requires to be in the form of Fossil UUID strings. Such strings are + exactly FSL_STRLEN_SHA1 or FSL_STRLEN_K256 bytes long plus a + terminating NUL byte and contain only lower-case hexadecimal + bytes. Where this typedef is used, the library requires, enforces, + and/or assumes (at different times) that fsl_is_uuid() returns true + for such strings (if they are not NULL, though not all contexts + allow a NULL UUID). These typedef are _not_ used to denote + arguments which may refer to partial UUIDs or symbolic names, only + 100% bonafide Fossil UUIDs (which are different from RFC4122 + UUIDs). + + The API guarantees that this typedef will always be (char *) and + that fsl_uuid_cstr will always ben (char const *), and thus it + is safe/portable to use those type instead of these. These + typedefs serve only to improve the readability of certain APIs + by implying (through the use of this typedef) the preconditions + defined for UUID strings. + + Sidebar: fossil historically used the term UUID for blob IDs, and + still uses that term in the DB schema, but it has fallen out of + favor in documentation and discussions, with "hash" being the + preferred term. Much of the libfossil code was developed before + that happened, though, so "UUID" is still prevalent in its API and + documentation. + + @see fsl_is_uuid() + @see fsl_uuid_cstr +*/ +typedef char * fsl_uuid_str; + +/** + The const counterpart of fsl_uuid_str. + + @see fsl_is_uuid() + @see fsl_uuid_str +*/ +typedef char const * fsl_uuid_cstr; + +/** + A typedef for comparison function used by standard C + routines such as qsort(). It is provided here primarily + to simplify documentation of other APIs. Concrete + implementations must compare lhs and rhs, returning negative, + 0, or right depending on whether lhs is less than, equal to, + or greater than rhs. + + Implementations might need to be able to deal with NULL + arguments. That depends on the routine which uses the comparison + function. +*/ +typedef int (*fsl_generic_cmp_f)( void const * lhs, void const * rhs ); + +/** + If the NUL-terminated input str is exactly FSL_STRLEN_SHA1 or + FSL_STRLEN_K256 bytes long and contains only lower-case + hexadecimal characters, returns the length of the string, else + returns 0. + + Note that Fossil UUIDs are not RFC4122 UUIDs, but are SHA1 or + SHA3-256 hash strings. Don't let that disturb you. As Tim + Berners-Lee writes: + + 'The assertion that the space of URIs is a universal space + sometimes encounters opposition from those who feel there should + not be one universal space. These people need not oppose the + concept because it is not of a single universal space: Indeed, + the fact that URIs form universal space does not prevent anyone + else from forming their own universal space, which of course by + definition would be able to envelop within it as a subset the + universal URI space. Therefore the web meets the "independent + design" test, that if a similar system had been concurrently and + independently invented elsewhere, in such a way that the + arbitrary design decisions were made differently, when they met + later, the two systems could be made to interoperate.' + + Source: https://www.w3.org/DesignIssues/Axioms.html + + (Just mentally translate URI as UUID.) +*/ +FSL_EXPORT int fsl_is_uuid(char const * str); + +/** + If x is a valid fossil UUID length, it is returned, else 0 is returned. +*/ +FSL_EXPORT int fsl_is_uuid_len(int x); + +/** + Expects str to be a string containing an unsigned decimal + value. Returns its decoded value, or -1 on error. +*/ +FSL_EXPORT fsl_size_t fsl_str_to_size(char const * str); + +/** + Expects str to be a string containing a decimal value, + optionally with a leading sign. Returns its decoded value, or + dflt if !str or on error. +*/ +FSL_EXPORT fsl_int_t fsl_str_to_int(char const * str, fsl_int_t dflt); + + +/** + Generic list container type. This is used heavily by the Fossil + API for storing arrays of dynamically-allocated objects. It is + not useful as a non-pointer-array replacement. + + It is up to the APIs using this type to manage the entry count + member and use fsl_list_reserve() to manage the "capacity" + member. + + @see fsl_list_reserve() + @see fsl_list_append() + @see fsl_list_visit() +*/ +struct fsl_list { + /** + Array of entries. It contains this->capacity entries, + this->count of which are "valid" (in use). + */ + void ** list; + /** + Number of "used" entries in the list. + */ + fsl_size_t used; + /** + Number of slots allocated in this->list. Use fsl_list_reserve() + to modify this. Doing so might move the this->list pointer but + the values it points to will stay stable. + */ + fsl_size_t capacity; +}; + +/** + Empty-initialized fsl_list structure, intended for const-copy + initialization. +*/ +#define fsl_list_empty_m { NULL, 0, 0 } +/** + Empty-initialized fsl_list structure, intended for copy + initialization. +*/ +FSL_EXPORT const fsl_list fsl_list_empty; + + +/** + Generic interface for finalizing/freeing memory. Intended + primarily for use as a destructor/finalizer for high-level + structs. Implementations must semantically behave like free(mem), + regardless of whether or not they actually free the memory. At + the very least, they generally should clean up any memory owned by + mem (e.g. db resources or buffers), even if they do not free() mem. + some implementations assume that mem is stack-allocated + and they only clean up resources owned by mem. + + The state parameter is any state needed by the finalizer + (e.g. a memory allocation context) and mem is the memory which is + being finalized. + + The exact interpretaion of the state and mem are of course + implementation-specific. +*/ +typedef void (*fsl_finalizer_f)( void * state, void * mem ); + +/** + Generic interface for memory finalizers. +*/ +struct fsl_finalizer { + /** + State to be passed as the first argument to f(). + */ + void * state; + /** + Finalizer function. Should be called like this->f( this->state, ... ). + */ + fsl_finalizer_f f; +}; + +/** Empty-initialized fsl_finalizer struct. */ +#define fsl_finalizer_empty_m {NULL,NULL} + +/** + fsl_finalizer_f() impl which requires that mem be-a + (fsl_buffer*). This function frees all memory associated with + that buffer and zeroes out the structure, but does not free mem + (because it is rare that fsl_buffers are created on the + heap). The state parameter is ignored. +*/ +FSL_EXPORT int fsl_finalizer_f_buffer( void * state, void * mem ); + + +/** + Generic state-with-finalizer holder. Used for binding + client-specified state to another object, such that a + client-specified finalizer is called with the other object is + cleaned up. +*/ +struct fsl_state { + /** + Arbitrary context-dependent state. + */ + void * state; + /** + Finalizer for this->state. If used, it should be called like: + + ``` + this->finalize.f( this->finalize.state, this->state ); + ``` + + After which this->state must be treated as if it has been + free(3)'d. + */ + fsl_finalizer finalize; +}; + +/** Empty-initialized fsl_state struct. */ +#define fsl_state_empty_m {NULL,fsl_finalizer_empty_m} + +/** + Empty-initialized fsl_state struct, intended for + copy-initializing. +*/ +FSL_EXPORT const fsl_state fsl_state_empty; + + +/** + Generic interface for streaming out data. Implementations must + write n bytes from s to their destination channel and return 0 on + success, non-0 on error (assumed to be a value from the fsl_rc_e + enum). The state parameter is the implementation-specified + output channel. + + Potential TODO: change the final argument to a pointer, with + semantics similar to fsl_input_f(): at call-time n is the number + of bytes to output, and on returning n is the number of bytes + actually written. This would allow, e.g. the fsl_zip_writer APIs + to be able to stream a ZIP file (they have to know the real size + of the output, and this interface doesn't support that + operation). +*/ +typedef int (*fsl_output_f)( void * state, void const * src, fsl_size_t n ); + + +/** + Generic interface for flushing arbitrary output streams. Must + return 0 on success, non-0 on error, but the result code + "should" (to avoid downstream confusion) be one of the fsl_rc_e + values. When in doubt, return FSL_RC_IO on error. The + interpretation of the state parameter is + implementation-specific. +*/ +typedef int (*fsl_flush_f)(void * state); + +/** + Generic interface for streaming in data. Implementations must + read (at most) *n bytes from their input, copy it to dest, assign + *n to the number of bytes actually read, return 0 on success, and + return non-0 on error (assumed to be a value from the fsl_rc_e + enum). When called, *n is the max length to read. On return, *n + is the actual amount read. The state parameter is the + implementation-specified input file/buffer/whatever channel. +*/ +typedef int (*fsl_input_f)( void * state, void * dest, fsl_size_t * n ); + +/** + fsl_output_f() implementation which requires state to be a + (fsl_cx*) to which this routine simply redirects the output via + fsl_output(). Is a no-op (returning 0) if !n. Returns + FSL_RC_MISUSE if !state or !src. +*/ +FSL_EXPORT int fsl_output_f_fsl_cx(void * state, void const * src, fsl_size_t n ); + +/** + An interface which encapsulates data for managing an output + destination, primarily intended for use with fsl_output(). Why + abstract it to this level? So that we can do interesting things + like output to buffers, files, sockets, etc., using the core + output mechanism. e.g. so script bindings can send their output + to the same channel used by the library and other library + clients. +*/ +struct fsl_outputer { + /** + Output channel. + */ + fsl_output_f out; + /** + flush() implementation. This may be NULL for most uses of this + class. Cases which specifically require it must document that + requirement so. + */ + fsl_flush_f flush; + /** + State to be used when calling this->out(), namely: + this->out( this->state, ... ) and this->flush(this->state). + */ + void * state; +}; +/** Empty-initialized fsl_outputer instance. */ +#define fsl_outputer_empty_m {NULL,NULL,NULL} +/** + Empty-initialized fsl_outputer instance, intended for + copy-initializing. +*/ +FSL_EXPORT const fsl_outputer fsl_outputer_empty; + +/** + A fsl_outputer instance which is initialized to output to a + (FILE*). To use it, this value then set the copy's state + member to an opened-for-write (FILE*) handle. By default it will + use stdout. +*/ +FSL_EXPORT const fsl_outputer fsl_outputer_FILE; + +/** + fsl_outputer initializer which uses fsl_flush_f_FILE(), + fsl_output_f_FILE(), and fsl_finalizer_f_FILE(). +*/ +#define fsl_outputer_FILE_m { \ + fsl_output_f_FILE, \ + fsl_flush_f_FILE, \ + NULL \ + } +/** + Generic stateful alloc/free/realloc() interface. + + Implementations must behave as follows: + + - If 0==n then semantically behave like free(3) and return + NULL. + + - If 0!=n and !mem then semantically behave like malloc(3), returning + newly-allocated memory on success and NULL on error. + + - If 0!=n and NULL!=mem then semantically behave like + realloc(3). Note that realloc specifies: "If n was equal to 0, + either NULL or a pointer suitable to be passed to free() is + returned." Which is kind of useless, and thus implementations + MUST return NULL when n==0. +*/ +typedef void *(*fsl_realloc_f)(void * state, void * mem, fsl_size_t n); + +/** + Holds an allocator function and its related state. +*/ +struct fsl_allocator { + /** + Base allocator function. It must be passed this->state + as its first parameter. + */ + fsl_realloc_f f; + /** + State intended to be passed as the first parameter to + this->f(). + */ + void * state; +}; + +/** Empty-initialized fsl_allocator instance. */ +#define fsl_allocator_empty_m {NULL,NULL} + + +/** + A fsl_realloc_f() implementation which uses the standard + malloc()/free()/realloc(). The state parameter is ignored. +*/ +FSL_EXPORT void * fsl_realloc_f_stdalloc(void * state, void * mem, fsl_size_t n); + + +/** + Semantically behaves like malloc(3), but may introduce instrumentation, + error checking, or similar. +*/ +void * fsl_malloc( fsl_size_t n ) +#ifdef __GNUC__ + __attribute__ ((malloc)) +#endif + ; + +/** + Semantically behaves like free(3), but may introduce instrumentation, + error checking, or similar. +*/ +FSL_EXPORT void fsl_free( void * mem ); + +/** + Behaves like realloc(3). Clarifications on the behaviour (because + the standard has one case of unfortunate wording involving what + it returns when n==0): + + - If passed (NULL, n>0) then it semantically behaves like + fsl_malloc(f, n). + + - If 0==n then it semantically behaves like free(2) and returns + NULL (clarifying the aforementioned wording problem). + + - If passed (non-NULL, n) then it semantically behaves like + realloc(mem,n). + +*/ +FSL_EXPORT void * fsl_realloc( void * mem, fsl_size_t n ); + +/** + A fsl_flush_f() impl which expects _FILE to be-a (FILE*) opened + for writing, which this function passes the call on to + fflush(). If fflush() returns 0, so does this function, else it + returns non-0. +*/ +FSL_EXPORT int fsl_flush_f_FILE(void * _FILE); + +/** + A fsl_finalizer_f() impl which requires that mem be-a (FILE*). + This function passes that FILE to fsl_fclose(). The state + parameter is ignored. +*/ +FSL_EXPORT void fsl_finalizer_f_FILE( void * state, void * mem ); + +/** + A fsl_output_f() impl which requires state to be-a (FILE*), which + this function passes the call on to fwrite(). Returns 0 on + success, FSL_RC_IO on error. +*/ +FSL_EXPORT int fsl_output_f_FILE( void * state, void const * src, fsl_size_t n ); + +/** + A fsl_output_f() impl which requires state to be-a (fsl_buffer*), + which this function passes to fsl_buffer_append(). Returns 0 on + success, FSL_RC_OOM (probably) on error. +*/ +FSL_EXPORT int fsl_output_f_buffer( void * state, void const * src, fsl_size_t n ); + +/** + A fsl_input_f() implementation which requires that state be + a readable (FILE*) handle. +*/ +FSL_EXPORT int fsl_input_f_FILE( void * state, void * dest, fsl_size_t * n ); + +/** + A fsl_input_f() implementation which requires that state be a + readable (fsl_buffer*) handle. The buffer's cursor member is + updated to track input postion, but that is the only + modification made by this routine. Thus the user may need to + reset the cursor to 0 if he wishes to start consuming the buffer + at its starting point. Subsequent calls to this function will + increment the cursor by the number of bytes returned via *n. + The buffer's "used" member is used to determine the logical end + of input. + + Returns 0 on success and has no error conditions except for + invalid arguments, which result in undefined beavhiour. Results + are undefined if any argument is NULL. + + Tip (and warning): sometimes a routine might have a const buffer + handle which it would like to use in conjunction with this + routine but cannot without violating constness. Here's a crude + workaround: + + ``` + fsl_buffer kludge = *originalConstBuffer; // normally this is dangerous! + rc = some_func( fsl_input_f_buffer, &kludge, ... ); + assert(kludge.mem==originalConstBuffer->mem); // See notes below. + // DO NOT clean up the kludge buffer. Memory belongs to the original! + ``` + + That is ONLY (ONLY! ONLY!! ONLY!!!) legal because this routine + modifies only fsl_buffer::cursor. Such a workaround is STRICLY + ILLEGAL if there is ANY CHANCE WHATSOEVER that the buffer's + memory will be modified, in particular if it will be resized, + and such use will eventually leak and/or corrupt memory. +*/ +FSL_EXPORT int fsl_input_f_buffer( void * state, void * dest, fsl_size_t * n ); + + +/** + A generic streaming routine which copies data from an + fsl_input_f() to an fsl_outpuf_f(). + + Reads all data from inF() in chunks of an unspecified size and + passes them on to outF(). It reads until inF() returns fewer + bytes than requested. Returns the result of the last call to + outF() or (only if reading fails) inF(). Returns FSL_RC_MISUSE + if inF or ouF are NULL. + + Here is an example which basically does the same thing as the + cat(1) command on Unix systems: + + ``` + fsl_stream( fsl_input_f_FILE, stdin, fsl_output_f_FILE, stdout ); + ``` + + Or copy a FILE to a buffer: + + ``` + fsl_buffer myBuf = fsl_buffer_empty; + rc = fsl_stream( fsl_input_f_FILE, stdin, fsl_output_f_buffer, &myBuf ); + // Note that on error myBuf might be partially populated. + // Eventually clean up the buffer: + fsl_buffer_clear(&myBuf); + ``` +*/ +FSL_EXPORT int fsl_stream( fsl_input_f inF, void * inState, + fsl_output_f outF, void * outState ); + +/** + Consumes two input streams looking for differences. It stops + reading as soon as either or both streams run out of input or a + byte-level difference is found. It consumes input in chunks of + an unspecified size, and after this returns the input cursor of + the streams is not well-defined. i.e. the cursor probably does + not point to the exact position of the difference because this + level of abstraction does not allow that unless we read byte by + byte. + + Returns 0 if both streams emit the same amount of output and + that ouput is bitwise identical, otherwise it returns non-0. +*/ +FSL_EXPORT int fsl_stream_compare( fsl_input_f in1, void * in1State, + fsl_input_f in2, void * in2State ); + + +/** + A general-purpose buffer class, analog to Fossil's Blob + class, but it is not called fsl_blob to avoid confusion with + DB-side blobs. Buffers are used extensively in fossil to do + everything from reading files to compressing artifacts to + creating dynamically-formatted strings. Because they are such a + pervasive low-level type, and have such a simple structure, + their members (unlike most other structs in this API) may be + considered public and used directly by client code (as long as + they do not mangle their state, e.g. by setting this->capacity + smaller than this->used!). + + General conventions of this class: + + - ALWAYS initialize them by copying fsl_buffer_empty or + (depending on the context) fsl_buffer_empty_m. Failing to + initialize them properly leads to undefined behaviour. + + - ALWAYS fsl_buffer_clear() buffers when done with + them. Remember that failed routines which output to buffers + might partially populate the buffer, so be sure to clean up on + error cases. + + - The 'capacity' member specifies how much memory the buffer + current holds in its 'mem' member. + + - The 'used' member specifies how much of the memory is actually + "in use" by the client. + + - As a rule, the API tries to keep (used=20); // it may reserve more! + assert(0==b.used); + rc = fsl_buffer_append(&b, "abc", 3); + assert(0==rc); + assert(3==b.used); + assert(0==b.mem[b.used]); // API always NUL-terminates + ``` + + Potential TODO: add an allocator member which gets internally used + for allocation of the buffer member. fossil(1) uses this approach, + and swaps the allocator out as needed, to support a buffer pointing + to memory it does not own, e.g. a slice of another buffer or to + static memory, and then (re)allocate as necessary, e.g. to switch + from static memory to dynamic. That may be useful in order to + effectively port over some of the memory-intensive algos such as + merging. That would not affect [much of] the public API, just how + the buffer internally manages the memory. Certain API members would + need to specify that the memory is not owned by the blob and needs + to outlive the blob, though. We could potentially implement this + same thing without a new member by using the convention that a + `capacity` value of 0 when `mem` is!=00 means that the pointed-to + memory is owned by someone else and must be copied before + modification (effectively what fossil's approach does). + + @see fsl_buffer_reserve() + @see fsl_buffer_append() + @see fsl_buffer_appendf() + @see fsl_buffer_cstr() + @see fsl_buffer_size() + @see fsl_buffer_capacity() + @see fsl_buffer_clear() + @see fsl_buffer_reuse() +*/ +struct fsl_buffer { + /** + The raw memory owned by this buffer. It is this->capacity bytes + long, of which this->used are considered "used" by the client. + The difference beween (this->capacity - this->used) represents + space the buffer has available for use before it will require + another expansion/reallocation. + */ + unsigned char * mem; + /** + Number of bytes allocated for this buffer. + */ + fsl_size_t capacity; + /** + Number of "used" bytes in the buffer. This is generally + interpreted as the string length of this->mem, and the buffer + APIs which add data to a buffer always ensure that + this->capacity is large enough to account for a trailing NUL + byte in this->mem. + + Library routines which manipulate buffers must ensure that + (this->used<=this->capacity) is always true, expanding the + buffer if necessary. Much of the API assumes that precondition + is always met, and any violation of it opens the code to + undefined behaviour (which is okay, just don't ever break that + precondition). Most APIs ensure that (usedmem. + + TODO: factor this back out and let those cases keep their own + state. This is only used by fsl_input_f_buffer() (and that + function cannot be implemented unless we add the cursor here + or add another layer of state type specifically for it). + + TODO: No, don't do ^^^^. It turns out that the merge algo + wants this as well. + */ + fsl_size_t cursor; +}; + +/** Empty-initialized fsl_buffer instance, intended for const-copy + initialization. */ +#define fsl_buffer_empty_m {NULL,0U,0U,0U} + +/** Empty-initialized fsl_buffer instance, intended for copy + initialization. */ +FSL_EXPORT const fsl_buffer fsl_buffer_empty; + +/** + A container for storing generic error state. It is used to + propagate error state between layers of the API back to the + client. i.e. they act as basic exception containers. + + @see fsl_error_set() + @see fsl_error_get() + @see fsl_error_move() + @see fsl_error_clear() +*/ +struct fsl_error { + /** + Error message text is stored in this->msg.mem. The usable text + part is this->msg.used bytes long. + */ + fsl_buffer msg; + /** + Error code, generally assumed to be a fsl_rc_e value. The + "non-error" code is 0. + */ + int code; +}; + +/** Empty-initialized fsl_error instance, intended for const-copy + initialization. */ +#define fsl_error_empty_m {fsl_buffer_empty_m,0} + +/** Empty-initialized fsl_error instance, intended for copy + initialization. */ +FSL_EXPORT const fsl_error fsl_error_empty; + +/** + Populates err with the given code and formatted string, replacing + any existing state. If fmt==NULL then fsl_rc_cstr(rc) is used to + get the error string. + + Returns code on success, some other non-0 code on error. + + As a special case, if 0==code then fmt is ignored and the error + state is cleared. This will not free any memory held by err but + will re-set its string to start with a NUL byte, ready for + re-use later on. + + As a special case, if code==FSL_RC_OOM then fmt is ignored + to avoid a memory allocation (which would presumably fail). + + @see fsl_error_get() + @see fsl_error_clear() + @see fsl_error_move() +*/ +FSL_EXPORT int fsl_error_set( fsl_error * const err, int code, + char const * fmt, ... ); + +/** + va_list counterpart to fsl_error_set(). +*/ +FSL_EXPORT int fsl_error_setv( fsl_error * const err, int code, + char const * fmt, va_list args ); + +/** + Fetches the error state from err. If !err it returns + FSL_RC_MISUSE without side-effects, else it returns err's current + error code. + + If str is not NULL then *str will be assigned to the raw + (NUL-terminated) error string (which might be empty or even + NULL). The memory for the string is owned by err and may be + invalidated by any calls which take err as a non-const parameter + OR which might modify it indirectly through a container object, + so the client is required to copy it if it is needed for later + reference. + + If len is not NULL then *len will be assigned to the length of + the returned string (in bytes). + + @see fsl_error_set() + @see fsl_error_clear() + @see fsl_error_move() +*/ +FSL_EXPORT int fsl_error_get( fsl_error const * const err, + char const ** str, fsl_size_t * const len ); + +/** + Frees up any resources owned by err and sets its error code to 0, + but does not free err. This is harmless no-op if !err or if err + holds no dynamically allocated no memory. + + @see fsl_error_set() + @see fsl_error_get() + @see fsl_error_move() + @see fsl_error_reset() +*/ +FSL_EXPORT void fsl_error_clear( fsl_error * const err ); + +/** + Sets err->code to 0 and resets its buffer, but keeps any + err->msg memory around for later re-use. + + @see fsl_error_clear() +*/ +FSL_EXPORT void fsl_error_reset( fsl_error * const err ); + +/** + Copies the error state from src to dest. If dest contains state, it is + cleared/recycled by this operation. + + Returns 0 on success, FSL_RC_MISUSE if either argument is NULL + or if (src==dest), and FSL_RC_OOM if allocation of the message + string fails. + + As a special case, if src->code==FSL_RC_OOM, then the code is + copied but the message bytes (if any) are not (under the + assumption that we have no more memory). +*/ +FSL_EXPORT int fsl_error_copy( fsl_error const * const src, + fsl_error * const dest ); + +/** + Moves the error state from one fsl_error object to + another, intended as an allocation optimization when + propagating error state up the API. + + This "uplifts" an error from the 'from' object to the 'to' + object. After this returns 'to' will contain the prior error state + of 'from' and 'from' will contain the old error message memory of + 'to'. 'from' will be re-set to the non-error state (its buffer + memory is kept intact for later reuse, though). + + Results are undefined if either parameter is NULL or either is + not properly initialized. i.e. neither may refer to uninitialized + memory. Copying fsl_error_empty at declaration-time is a simple + way to ensure that instances are cleanly initialized. +*/ +FSL_EXPORT void fsl_error_move( fsl_error * const from, fsl_error * const to ); + +/** + Returns the given Unix Epoch timestamp value as its approximate + Julian Day value. Note that the calculation does not account for + leap seconds. +*/ +FSL_EXPORT double fsl_unix_to_julian( fsl_time_t unixEpoch ); + +/** + Returns the current Unix Epoch time converted to its approximate + Julian form. Equivalent to fsl_unix_to_julian(time(0)). See + fsl_unix_to_julian() for details. Note that the returned time + has seconds, not milliseconds, precision. +*/ +FSL_EXPORT double fsl_julian_now(); + +#if 0 +/** UNTESTED, possibly broken vis-a-vis timezone conversion. + + Returns the given Unix Epoch time value formatted as an ISO8601 + string. Returns NULL on allocation error, else a string 19 + bytes long plus a terminating NUL + (e.g. "2013-08-19T20:35:49"). The returned memory must + eventually be freed using fsl_free(). +*/ +FSL_EXPORT char * fsl_unix_to_iso8601( fsl_time_t j ); +#endif + +/** + Returns non-0 (true) if the first 10 digits of z _appear_ to + form the start of an ISO date string (YYYY-MM-DD). Whether or + not the string is really a valid date is left for downstream + code to determine. Returns 0 (false) in all other cases, + including if z is NULL. +*/ +FSL_EXPORT char fsl_str_is_date(const char *z); + + +/** + Checks if z is syntactically a time-format string in the format: + + [Y]YYYY-MM-DD + + (Yes, the year may be five-digits, left-padded with a zero for + years less than 9999.) + + Returns a positive value if the YYYYY part has five digits, a + negative value if it has four. It returns 0 (false) if z does not + match that pattern. + + If it returns a negative value, the MM part of z starts at byte offset + (z+5), and a positive value means the MM part starts at (z+6). + + z need not be NUL terminated - this function does not read past + the first invalid byte. Thus is can be used on, e.g., full + ISO8601-format strings. If z is NULL, 0 is returned. +*/ +FSL_EXPORT int fsl_str_is_date2(const char *z); + + +/** + Reserves at least n bytes of capacity in buf. Returns 0 on + success, FSL_RC_OOM if allocation fails, FSL_RC_MISUSE if !buf. + + This does not change buf->used, nor will it shrink the buffer + (reduce buf->capacity) unless n is 0, in which case it + immediately frees buf->mem and sets buf->capacity and buf->used + to 0. + + @see fsl_buffer_resize() + @see fsl_buffer_clear() +*/ +FSL_EXPORT int fsl_buffer_reserve( fsl_buffer * buf, fsl_size_t n ); + +/** + Convenience equivalent of fsl_buffer_reserve(buf,0). + This a no-op if buf==NULL. +*/ +FSL_EXPORT void fsl_buffer_clear( fsl_buffer * buf ); + +/** + Resets buf->used to 0 and sets buf->mem[0] (if buf->mem is not + NULL) to 0. Does not (de)allocate memory, only changes the + logical "used" size of the buffer. Returns its argument. + + Achtung for fossil(1) porters: this function's semantics are much + different from the fossil's blob_reset(). To get those semantics, + use fsl_buffer_reserve(buf, 0) or its convenience form + fsl_buffer_clear(). (This function _used_ to be called + fsl_buffer_reset(), but it was renamed in the hope of avoiding + related confusion.) +*/ +FSL_EXPORT fsl_buffer * fsl_buffer_reuse( fsl_buffer * buf ); + +/** + Similar to fsl_buffer_reserve() except that... + + - It does not free all memory when n==0. Instead it essentially + makes the memory a length-0, NUL-terminated string. + + - It will try to shrink (realloc) buf's memory if (ncapacity). + + - It sets buf->capacity to (n+1) and buf->used to n. This routine + allocates one extra byte to ensure that buf is always + NUL-terminated. + + - On success it always NUL-terminates the buffer at + offset buf->used. + + Returns 0 on success, FSL_RC_MISUSE if !buf, FSL_RC_OOM if + (re)allocation fails. + + @see fsl_buffer_reserve() + @see fsl_buffer_clear() +*/ +FSL_EXPORT int fsl_buffer_resize( fsl_buffer * buf, fsl_size_t n ); + +/** + Swaps the contents of the left and right arguments. Results are + undefined if either argument is NULL or points to uninitialized + memory. +*/ +FSL_EXPORT void fsl_buffer_swap( fsl_buffer * left, fsl_buffer * right ); + +/** + Similar fsl_buffer_swap() but it also optionally frees one of + the buffer's memories after swapping them. If clearWhich is + negative then the left buffer (1st arg) is cleared _after_ + swapping (i.e., the NEW left hand side gets cleared). If + clearWhich is greater than 0 then the right buffer (2nd arg) is + cleared _after_ swapping (i.e. the NEW right hand side gets + cleared). If clearWhich is 0, this function behaves identically + to fsl_buffer_swap(). + + A couple examples should clear this up: + + ``` + fsl_buffer_swap_free( &b1, &b2, -1 ); + ``` + + Swaps the contents of b1 and b2, then frees the contents + of the left-side buffer (b1). + + ``` + fsl_buffer_swap_free( &b1, &b2, 1 ); + ``` + + Swaps the contents of b1 and b2, then frees the contents + of the right-side buffer (b2). +*/ +FSL_EXPORT void fsl_buffer_swap_free( fsl_buffer * left, fsl_buffer * right, + int clearWhich ); + +/** + Appends the first n bytes of src, plus a NUL byte, to b, + expanding b as necessary and incrementing b->used by n. If n is + less than 0 then the equivalent of fsl_strlen((char const*)src) + is used to calculate the length. + + If n is 0 (or negative and !*src), this function ensures that + b->mem is not NULL and is NUL-terminated, so it may allocate + to have space for that NUL byte. + + src may only be NULL if n==0. If passed (src==NULL, n!=0) then + FSL_RC_RANGE is returned. + + Returns 0 on success, FSL_RC_MISUSE if !f, !b, or !src, + FSL_RC_OOM if allocation of memory fails. + + If this function succeeds, it guarantees that it NUL-terminates + the buffer (but that the NUL terminator is not counted in + b->used). + + @see fsl_buffer_appendf() + @see fsl_buffer_reserve() +*/ +FSL_EXPORT int fsl_buffer_append( fsl_buffer * b, + void const * src, fsl_int_t n ); + +/** + Uses fsl_appendf() to append formatted output to the given buffer. + Returns 0 on success and FSL_RC_OOM if an allocation fails while + expanding dest. Results are undefined if either of the first two + arguments are NULL. + + @see fsl_buffer_append() + @see fsl_buffer_reserve() +*/ +FSL_EXPORT int fsl_buffer_appendf( fsl_buffer * const dest, + char const * fmt, ... ); + +/** va_list counterpart to fsl_buffer_appendf(). */ +FSL_EXPORT int fsl_buffer_appendfv( fsl_buffer * const dest, + char const * fmt, va_list args ); + +/** + Compresses the first pIn->used bytes of pIn to pOut. It is ok for + pIn and pOut to be the same blob. + + pOut must either be the same as pIn or else a properly + initialized buffer. Any prior contents will be freed or their + memory reused. + + Results are undefined if any argument is NULL. + + Returns 0 on success, FSL_RC_OOM on allocation error, and FSL_RC_ERROR + if the lower-level compression routines fail. + + Use fsl_buffer_uncompress() to uncompress the data. The data is + encoded with a big-endian, unsigned 32-bit length as the first four + bytes (holding its uncomressed size), and then the data as + compressed by zlib. + + TODO: if pOut!=pIn1 then re-use pOut's memory, if it has any. + + @see fsl_buffer_compress2() + @see fsl_buffer_uncompress() + @see fsl_buffer_is_compressed() +*/ +FSL_EXPORT int fsl_buffer_compress(fsl_buffer const *pIn, fsl_buffer *pOut); + +/** + Compress the concatenation of a blobs pIn1 and pIn2 into pOut. + + pOut must be either empty (cleanly initialized or newly + recycled) or must be the same as either pIn1 or pIn2. + + Results are undefined if any argument is NULL. + + Returns 0 on success, FSL_RC_OOM on allocation error, and FSL_RC_ERROR + if the lower-level compression routines fail. + + TODO: if pOut!=(pIn1 or pIn2) then re-use its memory, if it has any. + + @see fsl_buffer_compress() + @see fsl_buffer_uncompress() + @see fsl_buffer_is_compressed() +*/ +FSL_EXPORT int fsl_buffer_compress2(fsl_buffer const *pIn1, + fsl_buffer const *pIn2, + fsl_buffer *pOut); + +/** + Uncompress buffer pIn and store the result in pOut. It is ok for + pIn and pOut to be the same buffer. Returns 0 on success. On + error pOut is not modified. + + pOut must be either cleanly initialized/empty or the same as pIn. + + Results are undefined if any argument is NULL. + + Returns 0 on success, FSL_RC_OOM on allocation error, and + FSL_RC_ERROR if the lower-level decompression routines fail. + + TODO: if pOut!=(pIn1 or pIn2) then re-use its memory, if it has any. + + @see fsl_buffer_compress() + @see fsl_buffer_compress2() + @see fsl_buffer_is_compressed() +*/ +FSL_EXPORT int fsl_buffer_uncompress(fsl_buffer const *pIn, fsl_buffer *pOut); + +/** + Returns true if this function believes that mem (which must be + at least len bytes of valid memory long) appears to have been + compressed by fsl_buffer_compress() or equivalent. This is not a + 100% reliable check - it could potentially have false positives + on certain inputs, but that is thought to be unlikely (at least + for text data). + + Returns 0 if mem is NULL. +*/ +FSL_EXPORT bool fsl_data_is_compressed(unsigned char const * mem, fsl_size_t len); + +/** + Equivalent to fsl_data_is_compressed(buf->mem, buf->used). +*/ +FSL_EXPORT bool fsl_buffer_is_compressed(fsl_buffer const * buf); + +/** + If fsl_data_is_compressed(mem,len) returns true then this function + returns the uncompressed size of the data, else it returns a negative + value. +*/ +FSL_EXPORT fsl_int_t fsl_data_uncompressed_size(unsigned char const *mem, fsl_size_t len); + +/** + The fsl_buffer counterpart of fsl_data_uncompressed_size(). +*/ +FSL_EXPORT fsl_int_t fsl_buffer_uncompressed_size(fsl_buffer const * b); + +/** + Equivalent to ((char const *)b->mem), but returns NULL if + !b. The returned string is effectively b->used bytes long unless + the user decides to apply his own conventions. Note that the buffer APIs + generally assure that buffers are NUL-terminated, meaning that strings + returned from this function can (for the vast majority of cases) + assume that the returned string is NUL-terminated (with a string length + of b->used _bytes_). + + @see fsl_buffer_str() + @see fsl_buffer_cstr2() +*/ +FSL_EXPORT char const * fsl_buffer_cstr(fsl_buffer const *b); + +/** + If buf is not NULL and has any memory allocated to it, that + memory is returned. If both b and len are not NULL then *len is + set to b->used. If b has no dynamic memory then NULL is returned + and *len (if len is not NULL) is set to 0. + + @see fsl_buffer_str() + @see fsl_buffer_cstr() +*/ +FSL_EXPORT char const * fsl_buffer_cstr2(fsl_buffer const *b, fsl_size_t * len); + +/** + Equivalent to ((char *)b->mem). The returned memory is effectively + b->used bytes long unless the user decides to apply their own + conventions. +*/ +FSL_EXPORT char * fsl_buffer_str(fsl_buffer const *b); +/** + "Takes" the memory refered to by the given buffer, transfering + ownership to the caller. After calling this, b's state will be + empty. +*/ +FSL_EXPORT char * fsl_buffer_take(fsl_buffer *b); + +/** + Returns the "used" size of b, or 0 if !b. +*/ +FSL_EXPORT fsl_size_t fsl_buffer_size(fsl_buffer const * b); + +/** + Returns the current capacity of b, or 0 if !b. +*/ +FSL_EXPORT fsl_size_t fsl_buffer_capacity(fsl_buffer const * b); + +/** + Compares the contents of buffers lhs and rhs using memcmp(3) + semantics. Return negative, zero, or positive if the first + buffer is less then, equal to, or greater than the second. + Results are undefined if either argument is NULL. + + When buffers of different length match on the first N bytes, + where N is the shorter of the two buffers' lengths, it treats the + shorter buffer as being "less than" the longer one. +*/ +FSL_EXPORT int fsl_buffer_compare(fsl_buffer const * lhs, fsl_buffer const * rhs); + +/** + Bitwise-compares the contents of b against the file named by + zFile. Returns 0 if they have the same size and contents, else + non-zero. This function has no way to report if zFile cannot be + opened, and any error results in a non-0 return value. No + interpretation/canonicalization of zFile is performed - it is + used as-is. + + This resolves symlinks and returns non-0 if zFile refers (after + symlink resolution) to a non-file. + + If zFile does not exist, is not readable, or has a different + size than b->used, non-0 is returned without opening/reading the + file contents. If a content comparison is performed, it is + streamed in chunks of an unspecified (but relatively small) + size, so it does not need to read the whole file into memory + (unless it is smaller than the chunk size). +*/ +FSL_EXPORT int fsl_buffer_compare_file( fsl_buffer const * b, char const * zFile ); + +/** + Compare two buffers in constant (a.k.a. O(1)) time and return + zero if they are equal. Constant time comparison only applies + for buffers of the same length. If lengths are different, + immediately returns 1. This operation is provided for cases + where the timing/duration of fsl_buffer_compare() (or an + equivalent memcmp()) might inadvertently leak security-relevant + information. Specifically, it address the concern that + attackers can use timing differences to check for password + misses, to narrow down an attack to passwords of a specific + length or content properties. +*/ +FSL_EXPORT int fsl_buffer_compare_O1(fsl_buffer const * lhs, fsl_buffer const * rhs); + +/** + Overwrites dest's contents with a copy of those from src + (reusing dest's memory if it has any). Results are undefined if + either pointer is NULL or invalid. Returns 0 on success, + FSL_RC_OOM on allocation error. +*/ +FSL_EXPORT int fsl_buffer_copy( fsl_buffer const * src, fsl_buffer * dest ); + + +/** + Apply the delta in pDelta to the original content pOriginal to + generate the target content pTarget. All three pointers must point + to properly initialized memory. + + If pTarget==pOriginal then this is a destructive operation, + replacing the original's content with its new form. + + Return 0 on success. + + @see fsl_buffer_delta_apply() + @see fsl_delta_apply() + @see fsl_delta_apply2() +*/ +FSL_EXPORT int fsl_buffer_delta_apply( fsl_buffer const * pOriginal, + fsl_buffer const * pDelta, + fsl_buffer * pTarget); + +/** + Identical to fsl_buffer_delta_apply() except that if delta + application fails then any error messages/codes are written to + pErr if it is not NULL. It is rare that delta application fails + (only if the inputs are invalid, e.g. do not belong together or + are corrupt), but when it does, having error information can be + useful. + + @see fsl_buffer_delta_apply() + @see fsl_delta_apply() + @see fsl_delta_apply2() +*/ +FSL_EXPORT int fsl_buffer_delta_apply2( fsl_buffer const * pOriginal, + fsl_buffer const * pDelta, + fsl_buffer * pTarget, + fsl_error * pErr); + + +/** + Uses a fsl_input_f() function to buffer input into a fsl_buffer. + + dest must be a non-NULL, initialized (though possibly empty) + fsl_buffer object. Its contents, if any, will be overwritten by + this function, and any memory it holds might be re-used. + + The src function is called, and passed the state parameter, to + fetch the input. If it returns non-0, this function returns that + error code. src() is called, possibly repeatedly, until it + reports that there is no more data. + + Whether or not this function succeeds, dest still owns any memory + pointed to by dest->mem, and the client must eventually free it + by calling fsl_buffer_reserve(dest,0). + + dest->mem might (and possibly will) be (re)allocated by this + function, so any pointers to it held from before this call might + be invalidated by this call. + + On error non-0 is returned and dest may bge partially populated. + + Errors include: + + dest or src are NULL (FSL_RC_MISUSE) + + Allocation error (FSL_RC_OOM) + + src() returns an error code + + Whether or not the state parameter may be NULL depends on the src + implementation requirements. + + On success dest will contain the contents read from the input + source. dest->used will be the length of the read-in data, and + dest->mem will point to the memory. dest->mem is automatically + NUL-terminated if this function succeeds, but dest->used does not + count that terminator. On error the state of dest->mem must be + considered incomplete, and is not guaranteed to be + NUL-terminated. + + Example usage: + + ``` + fsl_buffer buf = fsl_buffer_empty; + int rc = fsl_buffer_fill_from( &buf, + fsl_input_f_FILE, + stdin ); + if( rc ){ + fprintf(stderr,"Error %d (%s) while filling buffer.\n", + rc, fsl_rc_cstr(rc)); + fsl_buffer_reserve( &buf, 0 ); + return ...; + } + ... use the buf->mem ... + ... clean up the buffer ... + fsl_buffer_reserve( &buf, 0 ); + ``` + + To take over ownership of the buffer's memory, do: + + ``` + void * mem = buf.mem; + buf = fsl_buffer_empty; + ``` + + In which case the memory must eventually be passed to fsl_free() + to free it. +*/ +FSL_EXPORT int fsl_buffer_fill_from( fsl_buffer * const dest, + fsl_input_f src, void * const state ); + +/** + A fsl_buffer_fill_from() proxy which overwrite's dest->mem with + the contents of the given FILE handler (which must be opened for + read access). Returns 0 on success, after which dest->mem + contains dest->used bytes of content from the input source. On + error dest may be partially filled. +*/ +FSL_EXPORT int fsl_buffer_fill_from_FILE( fsl_buffer * const dest, + FILE * const src ); + +/** + A wrapper for fsl_buffer_fill_from_FILE() which gets its input + from the given file name. + + It uses fsl_fopen() to open the file, so it supports the name '-' + as an alias for stdin. +*/ +FSL_EXPORT int fsl_buffer_fill_from_filename( fsl_buffer * const dest, + char const * filename ); + +/** + Writes the given buffer to the given filename. Returns 0 on success, + FSL_RC_MISUSE if !b or !fname, FSL_RC_IO if opening or writing fails. + + Uses fsl_fopen() to open the file, so it supports the name '-' + as an alias for stdout. +*/ +FSL_EXPORT int fsl_buffer_to_filename( fsl_buffer const * b, + char const * fname ); + +/** + Copy N lines of text from pFrom into pTo. The copy begins at the + current pFrom->cursor position. pFrom->cursor is left pointing at + the first character past the last `\n` copied. (Modification of the + cursor is why pFrom is not const.) + + If pTo==NULL then this routine simply skips over N lines. + + Returns 0 if it copies lines or does nothing (because N is 0 or + pFrom's contents have been exhausted). Copying fewer lines than + requested (because of EOF) is not an error. Returns non-0 only on + allocation error. Results are undefined if pFrom is NULL or not + properly initialized. + + @see fsl_buffer_stream_lines() +*/ +FSL_EXPORT int fsl_buffer_copy_lines(fsl_buffer * const pTo, fsl_buffer * const pFrom, + fsl_size_t N); + +/** + Works identically to fsl_buffer_copy_lines() except that it sends + its output to the fTo output function. If fTo is NULL then it + simply skips over N lines. + + @see fsl_buffer_copy_lines() +*/ +FSL_EXPORT int fsl_buffer_stream_lines(fsl_output_f fTo, void * const toState, + fsl_buffer * const pFrom, + fsl_size_t N); + + +/** + Works like fsl_appendfv(), but appends all output to a + dynamically-allocated string, expanding the string as necessary + to collect all formatted data. The returned NUL-terminated string + is owned by the caller and it must be cleaned up using + fsl_free(...). If !fmt, NULL is returned. It is conceivable that + it returns NULL on a zero-length formatted string, e.g. (%.*s) + with (0,"...") as arguments, but it will only do that if the + whole format string resolves to empty. +*/ +FSL_EXPORT char * fsl_mprintf( char const * fmt, ... ); + +/** + va_list counterpart to fsl_mprintf(). +*/ +FSL_EXPORT char * fsl_mprintfv(char const * fmt, va_list vargs ); + +/** + An sprintf(3) clone which uses fsl_appendf() for the formatting. + Outputs at most n bytes to dest. Returns 0 on success, non-0 + on error. Returns 0 without side-effects if !n or !*fmt. + + If the destination buffer is long enough, this function + NUL-terminates it. +*/ +FSL_EXPORT int fsl_snprintf( char * dest, fsl_size_t n, char const * fmt, ... ); + +/** + va_list counterpart to fsl_snprintf() +*/ +FSL_EXPORT int fsl_snprintfv( char * dest, fsl_size_t n, char const * fmt, va_list args ); + +/** + Equivalent to fsl_strndup(src,-1). +*/ +FSL_EXPORT char * fsl_strdup( char const * src ); + +/** + Similar to strndup(3) but returns NULL if !src. The returned + memory must eventually be passed to fsl_free(). Returns NULL on + allocation error. If len is less than 0 and src is not NULL then + fsl_strlen() is used to calculate its length. + + If src is not NULL but len is 0 then it will return an empty + (length-0) string, as opposed to NULL. +*/ +FSL_EXPORT char * fsl_strndup( char const * src, fsl_int_t len ); + +/** + Equivalent to strlen(3) but returns 0 if src is NULL. + Note that it counts bytes, not UTF characters. +*/ +FSL_EXPORT fsl_size_t fsl_strlen( char const * src ); + +/** + Returns the number of UTF8 characters which begin within the first + n bytes of str (noting that it's possible that a multi-byte + character starts 1-3 bytes away from the end and overlaps past the + end of (str+len)). If len is negative then fsl_strlen() is used to + calculate it. + + Results are undefined if str is not UTF8 input or if str contains a + BOM marker. +*/ +FSL_EXPORT fsl_size_t fsl_strlen_utf8( char const * str, fsl_int_t n ); + +/** + Like strcmp(3) except that it accepts NULL pointers. NULL sorts + before all non-NULL string pointers. Also, this routine + performs a binary comparison that does not consider locale. +*/ +FSL_EXPORT int fsl_strcmp( char const * lhs, char const * rhs ); + +/** + Equivalent to fsl_strcmp(), but with a signature suitable + for use as a generic comparison function (e.g. for use with + qsort() and search algorithms). +*/ +FSL_EXPORT int fsl_strcmp_cmp( void const * lhs, void const * rhs ); + +/** + Case-insensitive form of fsl_strcmp(). + + @implements fsl_generic_cmp_f() +*/ +FSL_EXPORT int fsl_stricmp(const char *zA, const char *zB); + +/** + Equivalent to fsl_stricmp(), but with a signature suitable + for use as a generic comparison function (e.g. for use with + qsort() and search algorithms). + + @implements fsl_generic_cmp_f() +*/ +FSL_EXPORT int fsl_stricmp_cmp( void const * lhs, void const * rhs ); + +/** + fsl_strcmp() variant which compares at most nByte bytes of the + given strings, case-insensitively. If nByte is less than 0 then + fsl_strlen(zB) is used to obtain the length for comparision + purposes. +*/ +FSL_EXPORT int fsl_strnicmp(const char *zA, const char *zB, fsl_int_t nByte); + +/** + fsl_strcmp() variant which compares at most nByte bytes of the + given strings, case-sensitively. Returns 0 if nByte is 0. +*/ +FSL_EXPORT int fsl_strncmp(const char *zA, const char *zB, fsl_size_t nByte); + +/** + BSD strlcpy() variant which is less error prone than strncpy. Copy up to + dstsz - 1 characters from src to dst and NUL-terminate the resulting string + if dstsz is not 0. +*/ +FSL_EXPORT size_t fsl_strlcpy(char *restrict dst, const char *restrict src, size_t dstsz); + +/** + BSD strlcat() variant which is less error prone than strncat. Append src to + the end of dst. Append at most dstsz - strlen(dst - 1) characters, and + NUL-terminate unless dstsize is 0 or the passed in dst string was longer + than dstsz to begin with. +*/ +FSL_EXPORT size_t fsl_strlcat(char *restrict dst, const char *restrict src, size_t dstsz); + +/** + Equivalent to fsl_strncmp(lhs, rhs, X), where X is either + FSL_STRLEN_SHA1 or FSL_STRLEN_K256: if both lhs and rhs are + longer than FSL_STRLEN_SHA1 then they are assumed to be + FSL_STRLEN_K256 bytes long and are compared as such, else they + are assumed to be FSL_STRLEN_SHA1 bytes long and compared as + such. + + Potential FIXME/TODO: if their lengths differ, i.e. one is v1 and + one is v2, compare them up to their common length then, if they + still compare equivalent, treat the shorter one as less-than the + longer. +*/ +FSL_EXPORT int fsl_uuidcmp( fsl_uuid_cstr lhs, fsl_uuid_cstr rhs ); + +/** + Returns false if s is NULL or starts with any of (0 (NUL), '0' + (ASCII character zero), 'f', 'n', "off"), case-insensitively, + else it returns true. +*/ +FSL_EXPORT bool fsl_str_bool( char const * s ); + +/** + _Almost_ equivalent to fopen(3) but: + + - expects name to be UTF8-encoded. + + - If name=="-", it returns one of stdin or stdout, depending on + the mode string: stdout is returned if 'w' or '+' appear, + otherwise stdin. + + If it returns NULL, the global errno "should" contain a description + of the problem unless the problem was argument validation. Pass it + to fsl_errno_to_rc() to convert that into an API-conventional error + code. + + If at all possible, use fsl_close() (as opposed to fclose()) to + close these handles, as it has logic to skip closing the + standard streams. + + Potential TODOs: + + - extend mode string to support 'x', meaning "exclusive", analog + to open(2)'s O_EXCL flag. Barring race conditions, we have + enough infrastructure to implement that. (It turns out that + glibc's fopen() supports an 'x' with exactly this meaning.) + + - extend mode to support a 't', meaning "temporary". The idea + would be that we delete the file from the FS right after + opening, except that Windows can't do that. +*/ +FSL_EXPORT FILE * fsl_fopen(char const * name, char const *mode); + +/** + Passes f to fclose(3) unless f is NULL or one of the C-standard + (stdin, stdout, stderr) handles, in which cases it does nothing + at all. +*/ +FSL_EXPORT void fsl_fclose(FILE * f); + +/** + This function works similarly to classical printf + implementations, but instead of outputing somewhere specific, it + uses a callback function to push its output somewhere. This + allows it to be used for arbitrary external representations. It + can be used, for example, to output to an external string, a UI + widget, or file handle (it can also emulate printf by outputing + to stdout this way). + + INPUTS: + + pfAppend: The is a fsl_output_f function which is responsible + for accumulating the output. If pfAppend returns a negative + value then processing stops immediately. + + pfAppendArg: is ignored by this function but passed as the first + argument to pfAppend. pfAppend will presumably use it as a data + store for accumulating its string. + + fmt: This is the format string, as in the usual printf(3), except + that it supports more options (detailed below). + + ap: This is a pointer to a list of arguments. Same as in + vprintf() and friends. + + + OUTPUTS: + + ACHTUNG: this interface changed significantly in 2021-09: + + - The output function was changed from a type specific to this + interface to fsl_output_f(). + + - The return semantics where changed from printf()-like to + 0 on success, non-0 on error. + + Most printf-style specifiers work as they do in standard printf() + implementations. There might be some very minor differences, but + the more common format specifiers work as most developers expect + them to. In addition... + + Current (documented) printf extensions: + + `%s` works like conventional printf `%s` except that any precision + value can be modified via the '#' flag to counts in UTF8 characters + instead of bytes! That is, if an `%#.10s` argument has a byte + length of 20, a precision of 10, and contains only 8 UTF8, its + precision will allow it to output all 8 characters, even though + they total 20 bytes. The '#' flag works this way for both width and + precision. + + `%z` works exactly like `%s`, but takes a non-const (char *) and + deletes the string (using fsl_free()) after appending it to the + output. + + `%h` (HTML) works like `%s` but (A) does not support the '#' flag and + (B) converts certain characters (namely '<' and '&') to their HTML + escaped equivalents. + + `%t` (URL encode) works like `%h` but converts certain characters + into a representation suitable for use in an HTTP URL. (e.g. ' ' + gets converted to `%20`) + + `%T` (URL decode) does the opposite of `%t` - it decodes + URL-encoded strings and outputs their decoded form. ACHTUNG: + fossil(1) interprets this the same as `%t` except that it leaves + '/' characters unescaped (did that change at some point? This code + originally derived from that one some years ago!). It is still to + be determined whether we "really need" that behaviour (we don't + really need either one, seeing as the library is not CGI-centric + like fossil(1) is). + + `%r` requires an int and renders it in "ordinal form". That is, + the number 1 converts to "1st" and 398 converts to "398th". + + `%q` quotes a string as required for SQL. That is, '\'' + characters get doubled. It does NOT included the outer quotes + and NULL values get replaced by the string "(NULL)" (without + quotes). See `%Q`... + + `%Q` works like `%q`, but includes the outer '\'' characters and + NULL pointers get output as the string literal "NULL" (without + quotes), i.e. an SQL NULL. If modified with `%!Q` then it instead + uses double quotes, the intent being for use with identifiers. + In that form it still emits `NULL` without quotes, but it is not + intended to be used with `NULL` values. + + `%S` works like `%.16s`. It is intended for fossil hashes. The '!' + modifier removes the length limit, resulting in the whole hash + (making this formatting option equivalent to `%s`). (Sidebar: in + fossil(1) this length is runtime configurable but that requires + storing that option in global state, which is not an option for + this implementation.) + + `%/`: works mostly like `%s` but normalizes path-like strings by + replacing backslashes with the One True Slash. + + `%b`: works like `%s` but takes its input from a (fsl_buffer + const*) argument. It does not support the '#' flag. + + `%B`: works like `%Q` but takes its input from a (fsl_buffer + const*) argument. + + `%F`: works like `%s` but runs the output through + fsl_bytes_fossilize(). This requires dynamic memory allocation, so + is less efficient than re-using a client-provided buffer with + fsl_bytes_fossilize() if the client needs to fossilize more than + one element. Does not support the '#' flag. + + `%j`: works like `%s` but JSON-encodes the string. It does not + include the outer quotation marks by default, but using the '!' + flag, i.e. `%!j`, causes those to be added. The length and precision + flags are NOT supported for this format. Results are undefined if + given input which is not legal UTF8. By default non-ASCII + characters with values less than 0xffff are emitted as as literal + characters (no escaping), but the '#' modifier flag will cause it + to emit such characters in the `\u####` form. It always encodes + characters above 0xFFFF as UTF16 surrogate pairs (as JSON + requires). Invalid UTF8 characters may get converted to '?' or may + produce invalid JSON output. As a special case, if the value is NULL + pointer, it resolves to "null" without quotes (regardless of the '!' + modifier). + + Some of these extensions may be disabled by setting certain macros + when compiling appendf.c (see that file for details). + + Potential TODO: add fsl_bytes_fossilize_out() which works like + fsl_bytes_fossilize() but sends its output to an fsl_output_f(), so + that this routine doesn't need to alloc for that case. +*/ +FSL_EXPORT int fsl_appendfv(fsl_output_f pfAppend, void * pfAppendArg, + const char *fmt, va_list ap ); + +/** + Identical to fsl_appendfv() but takes an ellipses list (...) + instead of a va_list. +*/ +FSL_EXPORT int fsl_appendf(fsl_output_f pfAppend, + void * pfAppendArg, + const char *fmt, + ... ) +#if 0 +/* Would be nice, but complains about our custom format options: */ + __attribute__ ((__format__ (__printf__, 3, 4))) +#endif + ; + +/** + A fsl_output_f() impl which requires that state be an opened, + writable (FILE*) handle. +*/ +FSL_EXPORT int fsl_output_f_FILE( void * state, void const * s, + fsl_size_t n ); + + +/** + Emulates fprintf() using fsl_appendf(). Returns the result of + passing the data through fsl_appendf() to the given file handle. +*/ +FSL_EXPORT int fsl_fprintf( FILE * fp, char const * fmt, ... ); + +/** + The va_list counterpart of fsl_fprintf(). +*/ +FSL_EXPORT int fsl_fprintfv( FILE * fp, char const * fmt, va_list args ); + + +/** + Possibly reallocates self->list, changing its size. This function + ensures that self->list has at least n entries. If n is 0 then + the list is deallocated (but the self object is not), BUT THIS + DOES NOT DO ANY TYPE-SPECIFIC CLEANUP of the items. If n is less + than or equal to self->capacity then there are no side effects. If + n is greater than self->capacity, self->list is reallocated and + self->capacity is adjusted to be at least n (it might be bigger - + this function may pre-allocate a larger value). + + Passing an n of 0 when self->capacity is 0 is a no-op. + + Newly-allocated slots will be initialized with NULL pointers. + + Returns 0 on success, FSL_RC_MISUSE if !self, FSL_RC_OOM if + reservation of new elements fails. + + The return value should be used like this: + + ``` + fsl_size_t const n = number of bytes to allocate; + int const rc = fsl_list_reserve( myList, n ); + if( rc ) { ... error ... } + ``` + + @see fsl_list_clear() + @see fsl_list_visit_free() +*/ +FSL_EXPORT int fsl_list_reserve( fsl_list * self, fsl_size_t n ); + +/** + Appends a bitwise copy of cp to self->list, expanding the list as + necessary and adjusting self->used. + + Ownership of cp is unchanged by this call. cp may not be NULL. + + Returns 0 on success, FSL_RC_MISUSE if any argument is NULL, or + FSL_RC_OOM on allocation error. +*/ +FSL_EXPORT int fsl_list_append( fsl_list * self, void * cp ); + +/** + Swaps all contents of both lhs and rhs. Results are undefined if + lhs or rhs are NULL or not properly initialized (via initial copy + initialization from fsl_list_empty resp. fsl_list_empty_m). +*/ +FSL_EXPORT void fsl_list_swap( fsl_list * lhs, fsl_list * rhs ); + +/** @typedef typedef int (*fsl_list_visitor_f)(void * p, void * visitorState ) + + Generic visitor interface for fsl_list lists. Used by + fsl_list_visit(). p is the pointer held by that list entry and + visitorState is the 4th argument passed to fsl_list_visit(). + + Implementations must return 0 on success. Any other value causes + looping to stop and that value to be returned, but interpration + of the value is up to the caller (it might or might not be an + error, depending on the context). Note that client code may use + custom values, and is not strictly required to use FSL_RC_xxx + values. HOWEVER... all of the libfossil APIs which take these + as arguments may respond differently to some codes (most notable + FSL_RC_BREAK, which they tend to treat as a + stop-iteration-without-error result), so clients are strongly + encourage to return an FSL_RC_xxx value on error. +*/ +typedef int (*fsl_list_visitor_f)(void * obj, void * visitorState ); + +/** + A fsl_list_visitor_f() implementation which requires that obj be + arbitrary memory which can legally be passed to fsl_free() + (which this function does). The visitorState parameter is + ignored. +*/ +FSL_EXPORT int fsl_list_v_fsl_free(void * obj, void * visitorState ); + + +/** + For each item in self->list, visitor(item,visitorState) is + called. The item is owned by self. The visitor function MUST + NOT free the item (unless the visitor is a finalizer!), but may + manipulate its contents if application rules do not specify + otherwise. + + If order is 0 or greater then the list is traversed from start + to finish, else it is traverse from end to begin. + + Returns 0 on success, non-0 on error. + + If visitor() returns non-0 then looping stops and that code is + returned. +*/ +FSL_EXPORT int fsl_list_visit( fsl_list const * self, int order, + fsl_list_visitor_f visitor, void * visitorState ); + +/** + A list clean-up routine which takes a callback to clean up its + contents. + + Passes each element in the given list to + childFinalizer(item,finalizerState). If that returns non-0, + processing stops and that value is returned, otherwise + fsl_list_reserve(list,0) is called and 0 is returned. + + WARNING: if cleanup fails because childFinalizer() returns non-0, + the returned object is effectively left in an undefined state and + the client has no way (unless the finalizer somehow accounts for it) + to know which entries in the list were cleaned up. Thus it is highly + recommended that finalizer functions follow the conventional wisdom + of "destructors do not throw." + + @see fsl_list_visit_free() +*/ +FSL_EXPORT int fsl_list_clear( fsl_list * list, fsl_list_visitor_f childFinalizer, + void * finalizerState ); +/** + Similar to fsl_list_clear(list, fsl_list_v_fsl_free, NULL), but + only frees list->list if the second argument is true, otherwise + it sets the list's length to 0 but keep the list->list memory + intact for later use. Note that this function never frees the + list argument, only its contents. + + Be sure only to use this on lists of types for which fsl_free() + is legal. i.e. don't use it on a list of fsl_deck objects or + other types which have their own finalizers. + + Results are undefined if list is NULL. + + @see fsl_list_clear() +*/ +FSL_EXPORT void fsl_list_visit_free( fsl_list * list, bool freeListMem ); + +/** + Works similarly to the visit operation without the _p suffix + except that the pointer the visitor function gets is a (**) + pointing back to the entry within this list. That means that + callers can assign the entry in the list to another value during + the traversal process (e.g. set it to 0). If shiftIfNulled is + true then if the callback sets the list's value to 0 then it is + removed from the list and self->used is adjusted (self->capacity + is not changed). +*/ +FSL_EXPORT int fsl_list_visit_p( fsl_list * self, int order, bool shiftIfNulled, + fsl_list_visitor_f visitor, void * visitorState ); + + +/** + Sorts the given list using the given comparison function. Neither + argument may be NULL. The arugments passed to the comparison function + will be pointers to pointers to the original entries, and may (depending + on how the list is used) point to NULL. +*/ +FSL_EXPORT void fsl_list_sort( fsl_list * li, fsl_generic_cmp_f cmp); + +/** + Searches for a value in the given list, using the given + comparison function to determine equivalence. The comparison + function will never be passed a NULL value by this function - if + value is NULL then only a NULL entry will compare equal to it. + Results are undefined if li or cmpf are NULL. + + Returns the index in li of the entry, or a negative value if no + match is found. +*/ +FSL_EXPORT fsl_int_t fsl_list_index_of( fsl_list const * li, + void const * value, + fsl_generic_cmp_f cmpf); + +/** + Equivalent to fsl_list_index_of(li, key, fsl_strcmp_cmp). +*/ +FSL_EXPORT fsl_int_t fsl_list_index_of_cstr( fsl_list const * li, + char const * key ); + + +/** + Returns 0 if the given file is readable. Flags may be any values + accepted by the access(2) resp. _waccess() system calls. +*/ +FSL_EXPORT int fsl_file_access(const char *zFilename, int flags); + +/** + Computes a canonical pathname for a file or directory. Makes the + name absolute if it is relative. Removes redundant / characters. + Removes all /./ path elements. Converts /A/../ to just /. If the + slash parameter is non-zero, the trailing slash, if any, is + retained. + + If zRoot is not NULL then it is used for transforming a relative + zOrigName into an absolute path. If zRoot is NULL fsl_getcwd() + is used to determine the virtual root directory. If zRoot is + empty (starts with a NUL byte) then this function effectively + just sends zOrigName through fsl_file_simplify_name(). + + Returns 0 on success, FSL_RC_MISUSE if !zOrigName or !pOut, + FSL_RC_OOM if an allocation fails. + + pOut, if not NULL, is _appended_ to, so be sure to set pOut->used=0 + (or pass it to fsl_buffer_reuse()) before calling this to start + writing at the beginning of a re-used buffer. On error pOut might + conceivably be partially populated, but that is highly + unlikely. Nonetheless, be sure to fsl_buffer_clear() it at some + point regardless of success or failure. + + This function does no actual filesystem-level processing unless + zRoot is NULL or empty (and then only to get the current + directory). This does not confirm whether the resulting file + exists, nor that it is strictly a valid filename for the current + filesystem. It simply transforms a potentially relative path + into an absolute one. + + Example: + + ``` + int rc; + char const * zRoot = "/a/b/c"; + char const * zName = "../foo.bar"; + fsl_buffer buf = fsl_buffer_empty; + rc = fsl_file_canonical_name2(zRoot, zName, &buf, 0); + if(rc){ + fsl_buffer_clear(&buf); + return rc; + } + assert(0 == fsl_strcmp( "/a/b/foo.bar, fsl_buffer_cstr(&buf))); + fsl_buffer_clear(&buf); + ``` +*/ +FSL_EXPORT int fsl_file_canonical_name2(const char *zRoot, + const char *zOrigName, + fsl_buffer *pOut, bool slash); + +/** + Equivalent to fsl_file_canonical_name2(NULL, zOrigName, pOut, slash). + + @see fsl_file_canonical_name2() +*/ + +FSL_EXPORT int fsl_file_canonical_name(const char *zOrigName, + fsl_buffer *pOut, bool slash); + +/** + Calculates the "directory part" of zFilename and _appends_ it to + pOut. The directory part is all parts up to the final path + separator ('\\' or '/'). If leaveSlash is true (non-0) then the + separator part is appended to pOut, otherwise it is not. This + function only examines the first nLen bytes of zFilename. If + nLen is negative then fsl_strlen() is used to determine the + number of bytes to examine. + + If zFilename ends with a slash then it is considered to be its + own directory part. i.e. the dirpart of "foo/" evaluates to + "foo" (or "foo/" if leaveSlash is true), whereas the dirpart of + "foo" resolves to nothing (empty - no output except a NUL + terminator sent to pOut). + + Returns 0 on success, FSL_RC_MISUSE if !zFilename or !pOut, + FSL_RC_RANGE if 0==nLen or !*zFilename, and FSL_RC_OOM if + appending to pOut fails. If zFilename contains only a path + separator and leaveSlash is false then only a NUL terminator is + appended to pOut if it is not already NUL-terminated. + + This function does no filesystem-level validation of the the + given path - only string evaluation. +*/ +FSL_EXPORT int fsl_file_dirpart(char const * zFilename, fsl_int_t nLen, + fsl_buffer * pOut, bool leaveSlash); + + +/** + Writes the absolute path name of the current directory to zBuf, + which must be at least nBuf bytes long (nBuf includes the space + for a trailing NUL terminator). + + Returns FSL_RC_RANGE if the name would be too long for nBuf, + FSL_RC_IO if it cannot determine the current directory (e.g. a + side effect of having removed the directory at runtime or similar + things), and 0 on success. + + On success, if outLen is not NULL then the length of the string + written to zBuf is assigned to *outLen. The output string is + always NUL-terminated. + + On Windows, the name is converted from unicode to UTF8 and all '\\' + characters are converted to '/'. No conversions are needed on + Unix. +*/ +FSL_EXPORT int fsl_getcwd(char *zBuf, fsl_size_t nBuf, fsl_size_t * outLen); + + +/** + Return true if the filename given is a valid filename + for a file in a repository. Valid filenames follow all of the + following rules: + + - Does not begin with "/" + - Does not contain any path element named "." or ".." + - Does not contain "/..." (special case) + - Does not contain any of these characters in the path: "\" + - Does not end with "/". + - Does not contain two or more "/" characters in a row. + - Contains at least one character + + Invalid UTF8 characters result in a false return if bStrictUtf8 is + true. If bStrictUtf8 is false, invalid UTF8 characters are silently + ignored. See https://en.wikipedia.org/wiki/UTF-8#Invalid_byte_sequences + and https://en.wikipedia.org/wiki/Unicode (for the noncharacters). + + Fossil compatibility note: the bStrictUtf8 flag must be true + when parsing new manifests but is false when parsing legacy + manifests, for backwards compatibility. + + z must be NUL terminated. Results are undefined if !z. + + Note that periods in and of themselves are valid filename + components, with the special exceptions of "." and "..", one + implication being that "...." is, for purposes of this function, + a valid simple filename. +*/ +FSL_EXPORT bool fsl_is_simple_pathname(const char *z, bool bStrictUtf8); + +/** + Return the size of a file in bytes. Returns -1 if the file does + not exist or is not stat(2)able. +*/ +FSL_EXPORT fsl_int_t fsl_file_size(const char *zFilename); + +/** + Return the modification time for a file. Return -1 if the file + does not exist or is not stat(2)able. +*/ +FSL_EXPORT fsl_time_t fsl_file_mtime(const char *zFilename); + +#if 0 +/** + Don't use this. The wd (working directory) family of functions + might or might-not be necessary and in any case they require + a fsl_cx context argument because they require repo-specific + "allow-symlinks" setting. + + Return TRUE if the named file is an ordinary file or symlink + and symlinks are allowed. + + Return false for directories, devices, fifos, etc. +*/ +FSL_EXPORT bool fsl_wd_isfile_or_link(const char *zFilename); +#endif + +/** + Returns true if the named file is an ordinary file. Returns false + for directories, devices, fifos, symlinks, etc. The name + may be absolute or relative to the current working dir. +*/ +FSL_EXPORT bool fsl_is_file(const char *zFilename); + +/** + Returns true if the given file is a symlink, else false. The name + may be absolute or relative to the current working dir. +*/ +FSL_EXPORT bool fsl_is_symlink(const char *zFilename); + +/** + Returns true if the given filename refers to a plain file or + symlink, else returns false. The name may be absolute or relative + to the current working dir. +*/ +FSL_EXPORT bool fsl_is_file_or_link(const char *zFilename); + +/** + Returns true if the given path appears to be absolute, else + false. On Unix a path is absolute if it starts with a '/'. On + Windows a path is also absolute if it starts with a letter, a + colon, and either a backslash or forward slash. +*/ +FSL_EXPORT bool fsl_is_absolute_path(const char *zPath); + +/** + Simplify a filename by: + + * converting all \ into / on windows and cygwin + * removing any trailing and duplicate / + * removing /./ + * removing /A/../ + + Changes are made in-place. Return the new name length. If the + slash parameter is non-zero, the trailing slash, if any, is + retained. If n is <0 then fsl_strlen() is used to calculate the + length. +*/ +FSL_EXPORT fsl_size_t fsl_file_simplify_name(char *z, fsl_int_t n_, bool slash); + +/** + Return true (non-zero) if string z matches glob pattern zGlob + and zero if the pattern does not match. Always returns 0 if + either argument is NULL. Supports all globbing rules + supported by sqlite3_strglob(). +*/ +FSL_EXPORT bool fsl_str_glob(const char *zGlob, const char *z); + +/** + Parses zPatternList as a comma-and/or-fsl_isspace()-delimited + list of glob patterns (as supported by fsl_str_glob()). Each + pattern in that list is copied and appended to tgt in the form + of a new (char *) owned by that list. + + Returns 0 on success, FSL_RC_OOM if copying a pattern to tgt + fails, FSL_RC_MISUSE if !tgt or !zPatternList. An empty + zPatternList is not considered an error (to simplify usage) and + has no side-effects. On allocation error, tgt might be partially + populated. + + Elements of the glob list may be optionally enclosed in single + or double-quotes. This allows a comma to be part of a glob + pattern. + + Leading and trailing spaces on unquoted glob patterns are + ignored. + + Note that there is no separate "glob list" class. A "glob list" + is simply a fsl_list whose list entries are glob-pattern strings + owned by that list. + + Examples of a legal value for zPatternList: + + ``` + "*.c *.h, *.sh, '*.in'" + ``` + + @see fsl_glob_list_append() + @see fsl_glob_list_matches() + @see fsl_glob_list_clear() +*/ +FSL_EXPORT int fsl_glob_list_parse( fsl_list * tgt, char const * zPatternList ); + +/** + Appends a single blob pattern to tgt, in the form of a new (char *) + owned by tgt. This function copies zGlob and appends that copy + to tgt. + + Returns 0 on success, FSL_RC_MISUSE if !tgt or !zGlob or + !*zGlob, FSL_RC_OOM if appending to the list fails. + + @see fsl_glob_list_parse() + @see fsl_glob_list_matches() + @see fsl_glob_list_clear() +*/ +FSL_EXPORT int fsl_glob_list_append( fsl_list * tgt, char const * zGlob ); + +/** + Assumes globList is a list of (char [const] *) glob values and + tries to match each one against zNeedle using + fsl_str_glob(). If any glob matches, it returns a pointer to the + matched globList->list entry. If no matches are found, or if any + argument is invalid, NULL is returned. + + The returned bytes are owned by globList and may be invalidated at + its leisure. It is primarily intended to be used as a boolean, + for example: + + ``` + if( fsl_glob_list_matches(myGlobs, someFilename) ) { ... } + ``` + + @see fsl_glob_list_parse() + @see fsl_glob_list_append() + @see fsl_glob_list_clear() +*/ +FSL_EXPORT char const * fsl_glob_list_matches( fsl_list const * const globList, + char const * zNeedle ); + +/** + If globList is not NULL this is equivalent to + fsl_list_visit_free(globList, 1), otherwise it is a no-op. + + Note that this does not free the globList object itself, just + its underlying list entries and list memory. (In practice, lists + are either allocated on the stack or as part of a higher-level + structure, and not on the heap.) + + @see fsl_glob_list_parse() + @see fsl_glob_list_append() + @see fsl_glob_list_matches() +*/ +FSL_EXPORT void fsl_glob_list_clear( fsl_list * const globList ); + + +/** + Returns true if the given letter is an ASCII alphabet character. +*/ +FSL_EXPORT char fsl_isalpha(int c); + +/** + Returns true if c is a lower-case ASCII alphabet character. +*/ +FSL_EXPORT char fsl_islower(int c); + +/** + Returns true if c is an upper-case ASCII alphabet character. +*/ +FSL_EXPORT char fsl_isupper(int c); + +/** + Returns true if c is ' ', '\\r' (ASCII 13dec), or '\\t' (ASCII 9 + dec). +*/ +FSL_EXPORT char fsl_isspace(int c); + +/** + Returns true if c is an ASCII digit in the range '0' to '9'. +*/ +FSL_EXPORT char fsl_isdigit(int c); + +/** + Equivalent to fsl_isdigit(c) || fsl_isalpha(). +*/ +FSL_EXPORT char fsl_isalnum(int c); + +/** + Returns the upper-case form of c if c is an ASCII alphabet + letter, else returns c. +*/ +FSL_EXPORT int fsl_tolower(int c); + +/** + Returns the lower-case form of c if c is an ASCII alphabet + letter, else returns c. +*/ +FSL_EXPORT int fsl_toupper(int c); + +#ifdef _WIN32 +/** + Translate MBCS to UTF-8. Return a pointer to the translated + text. ACHTUNG: Call fsl_mbcs_free() (not fsl_free()) to + deallocate any memory used to store the returned pointer when + done. +*/ +FSL_EXPORT char * fsl_mbcs_to_utf8(char const * mbcs); + +/** + Frees a string allocated from fsl_mbcs_to_utf8(). Results are undefined + if mbcs was allocated using any other mechanism. +*/ +FSL_EXPORT void fsl_mbcs_free(char * mbcs); +#endif +/* _WIN32 */ + +/** + Deallocates the given memory, which must have been allocated + from fsl_unicode_to_utf8(), fsl_utf8_to_unicode(), or any + function which explicitly documents this function as being the + proper finalizer for its returned memory. +*/ +FSL_EXPORT void fsl_unicode_free(void *); + +/** + Translate UTF-8 to Unicode for use in system calls. Returns a + pointer to the translated text. The returned value must + eventually be passed to fsl_unicode_free() to deallocate any + memory used to store the returned pointer when done. + + This function exists only for Windows. On other platforms + it behaves like fsl_strdup(). + + The returned type is (wchar_t*) on Windows and (char*) + everywhere else. +*/ +FSL_EXPORT void *fsl_utf8_to_unicode(const char *zUtf8); + +/** + Translates Unicode text into UTF-8. Return a pointer to the + translated text. Call fsl_unicode_free() to deallocate any + memory used to store the returned pointer when done. + + This function exists only for Windows. On other platforms it + behaves like fsl_strdup(). +*/ +FSL_EXPORT char *fsl_unicode_to_utf8(const void *zUnicode); + +/** + Translate text from the OS's character set into UTF-8. Return a + pointer to the translated text. Call fsl_filename_free() to + deallocate any memory used to store the returned pointer when + done. + + This function must not convert '\' to '/' on Windows/Cygwin, as + it is used in places where we are not sure it's really filenames + we are handling, e.g. fsl_getenv() or handling the argv + arguments from main(). + + On Windows, translate some characters in the in the range + U+F001 - U+F07F (private use area) to ASCII. Cygwin sometimes + generates such filenames. See: + +*/ +FSL_EXPORT char *fsl_filename_to_utf8(const void *zFilename); + +/** + Translate text from UTF-8 to the OS's filename character set. + Return a pointer to the translated text. Call + fsl_filename_free() to deallocate any memory used to store the + returned pointer when done. + + On Windows, characters in the range U+0001 to U+0031 and the + characters '"', '*', ':', '<', '>', '?' and '|' are invalid in + filenames. Therefore, translate those to characters in the in the + range U+F001 - U+F07F (private use area), so those characters + never arrive in any Windows API. The filenames might look + strange in Windows explorer, but in the cygwin shell everything + looks as expected. + + See: + + The returned type is (wchar_t*) on Windows and (char*) + everywhere else. +*/ +FSL_EXPORT void *fsl_utf8_to_filename(const char *zUtf8); + + +/** + Deallocate pOld, which must have been allocated by + fsl_filename_to_utf8(), fsl_utf8_to_filename(), fsl_getenv(), or + another routine which explicitly documents this function as + being the proper finalizer for its returned memory. +*/ +FSL_EXPORT void fsl_filename_free(void *pOld); + +/** + Returns a (possible) copy of the environment variable with the + given key, or NULL if no entry is found. The returned value must + be passed to fsl_filename_free() to free it. ACHTUNG: DO NOT + MODIFY the returned value - on Unix systems it is _not_ a + copy. That interal API inconsistency "should" be resolved + (==return a copy from here, but that means doing it everywhere) + to avoid memory ownership problems later on. + + Why return a copy? Because native strings from at least one of + the more widespread OSes often have to be converted to something + portable and this requires allocation on such platforms, but + not on Unix. For API transparency, that means all platforms get + the copy(-like) behaviour. +*/ +FSL_EXPORT char *fsl_getenv(const char *zName); + +/** + Returns a positive value if zFilename is a directory, 0 if + zFilename does not exist, or a negative value if zFilename + exists but is something other than a directory. Results are + undefined if zFilename is NULL. + + This function expects zFilename to be a well-formed path - + it performs no normalization on it. +*/ +FSL_EXPORT int fsl_dir_check(const char *zFilename); + +/** + Check the given path to determine whether it is an empty directory. + Returns 0 on success (i.e., directory is empty), <0 if the provided + path is not a directory or cannot be opened, and >0 if the + directory is not empty. +*/ +FSL_EXPORT int fsl_dir_is_empty(const char *path); + +/** + Deletes the given file from the filesystem. Returns 0 on + success. If the component is a directory, this operation will + fail. If zFilename refers to a symlink, the link (not its target) + is removed. + + Results are undefined if zFilename is NULL. + + Potential TODO: if it refers to a dir, forward the call to + fsl_rmdir(). +*/ +FSL_EXPORT int fsl_file_unlink(const char *zFilename); + + +/** + Renames zFrom to zTo, as per rename(3). Returns 0 on success. On + error it returns FSL_RC_OOM on allocation error when converting the + names to platforms-specific character sets (on platforms where that + happens) or an FSL_RC_xxx value approximating an errno value, as + per fsl_errno_to_rc(). +*/ +FSL_EXPORT int fsl_file_rename(const char *zFrom, const char *zTo); + +/** + Deletes an empty directory from the filesystem. Returns 0 + on success. There are any number of reasons why deletion + of a directory can fail, some of which include: + + - It is not empty or permission denied (FSL_RC_ACCESS). + + - Not found (FSL_RC_NOT_FOUND). + + - Is not a directory or (on Windows) is a weird pseudo-dir type for + which rmdir() does not work (FSL_RC_TYPE). + + - I/O error (FSL_RC_IO). + + @see fsl_dir_is_empty() +*/ +FSL_EXPORT int fsl_rmdir(const char *zFilename); + +/** + Sets the mtime (Unix epoch) for a file. Returns 0 on success, + non-0 on error. If newMTime is less than 0 then the current + time(2) is used. This routine does not create non-existent files + (e.g. like a Unix "touch" command). +*/ +FSL_EXPORT int fsl_file_mtime_set(const char *zFilename, fsl_time_t newMTime); + +/** + On non-Windows platforms, this function sets or unsets the + executable bits on the given filename. All other permissions are + retained as-is. Returns 0 on success. On Windows this is a no-op, + returning 0. + + If the target is a directory or a symlink, this is a no-op and + returns 0. +*/ +FSL_EXPORT int fsl_file_exec_set(const char *zFilename, bool isExec); + + +/** + Create the directory with the given name if it does not already + exist. If forceFlag is true, delete any prior non-directory + object with the same name. + + Return 0 on success, non-0 on error. + + If the directory already exists then 0 is returned, not an error + (FSL_RC_ALREADY_EXISTS), because that simplifies usage. If + another filesystem entry with this name exists and forceFlag is + true then that entry is deleted before creating the directory, + and this operation fails if deletion fails. If forceFlag is + false and a non-directory entry already exists, FSL_RC_TYPE is + returned. + + For recursively creating directories, use fsl_mkdir_for_file(). + + Bug/corner case: if zFilename refers to a symlink to a + non-existent directory, this function gets slightly confused, + tries to make a dir with the symlink's name, and returns + FSL_RC_ALREADY_EXISTS. How best to resolve that is not yet + clear. The problem is that stat(2)ing the symlink says "nothing + is there" (because the link points to a non-existing thing), so + we move on to the underlying mkdir(), which then fails because + the link exists with that name. +*/ +FSL_EXPORT int fsl_mkdir(const char *zName, bool forceFlag); + +/** + A convenience form of fsl_mkdir() which can recursively create + directories. If zName has a trailing slash then the last + component is assumed to be a directory part, otherwise it is + assumed to be a file part (and no directory is created for that + part). zName may be either an absolute or relative path. + + Returns 0 on success (including if all directories already exist). + Returns FSL_RC_OOM if there is an allocation error. Returns + FSL_RC_TYPE if one of the path components already exists and is not + a directory. Returns FSL_RC_RANGE if zName is NULL or empty. If + zName is only 1 byte long, this is a no-op. + + On systems which support symlinks, a link to a directory is + considered to be a directory for purposes of this function. + + If forceFlag is true and a non-directory component is found in + the filesystem where zName calls for a directory, that component + is removed (and this function fails if removal fails). + + Examples: + + "/foo/bar" creates (if needed) /foo, but assumes "bar" is a file + component. "/foo/bar/" creates /foo/bar. However "foo" will not + create a directory - because the string has no path component, + it is assumed to be a filename. + + Both "/foo/bar/my.doc" and "/foo/bar/" result in the directories + /foo/bar. + +*/ +FSL_EXPORT int fsl_mkdir_for_file(char const *zName, bool forceFlag); + + +/** + Uses fsl_getenv() to look for the environment variables + (FOSSIL_USER, (Windows: USERNAME), (Unix: USER, LOGNAME)). If + it finds one it returns a copy of that value, which must + eventually be passed to fsl_free() to free it (NOT + fsl_filename_free(), though fsl_getenv() requires that one). If + it finds no match, or if copying the entry fails, it returns + NULL. + + @see fsl_cx_user_set() + @see fsl_cx_user_get() +*/ +FSL_EXPORT char * fsl_guess_user_name(); + +/** + Tries to find the user's home directory. If found, 0 is + returned, tgt's memory is _overwritten_ (not appended) with the + path, and tgt->used is set to the path's string length. (Design + note: the overwrite behaviour is inconsistent with most of the + API, but the implementation currently requires this.) + + If requireWriteAccess is true then the directory is checked for + write access, and FSL_RC_ACCESS is returned if that check + fails. For historical (possibly techinical?) reasons, this check + is only performed on Unix platforms. On others this argument is + ignored. When writing code on Windows, it may be necessary to + assume that write access is necessary on non-Windows platform, + and to pass 1 for the second argument even though it is ignored + on Windows. + + On error non-0 is returned and tgt is updated with an error + string OR (if the error was an allocation error while appending + to the path or allocating MBCS strings for Windows), it returns + FSL_RC_OOM and tgt "might" be updated with a partial path (up to + the allocation error), and "might" be empty (if the allocation + error happens early on). + + This routine does not canonicalize/transform the home directory + path provided by the environment, other than to convert the + string byte encoding on some platforms. i.e. if the environment + says that the home directory is "../" then this function will + return that value, possibly to the eventual disappointment of + the caller. + + Result codes include: + + - FSL_RC_OK (0) means a home directory was found and tgt is + populated with its path. + + - FSL_RC_NOT_FOUND means the home directory (platform-specific) + could not be found. + + - FSL_RC_ACCESS if the home directory is not writable and + requireWriteAccess is true. Unix platforms only - + requireWriteAccess is ignored on others. + + - FSL_RC_TYPE if the home (as determined via inspection of the + environment) is not a directory. + + - FSL_RC_OOM if a memory (re)allocation fails. +*/ +FSL_EXPORT int fsl_find_home_dir( fsl_buffer * tgt, bool requireWriteAccess ); + +/** + Values for use with the fsl_fstat::type field. +*/ +enum fsl_fstat_type_e { +/** Sentinel value for unknown/invalid filesystem entry types. */ +FSL_FSTAT_TYPE_UNKNOWN = 0, +/** Indicates a directory filesystem entry. */ +FSL_FSTAT_TYPE_DIR, +/** Indicates a non-directory, non-symlink filesystem entry. + Because fossil's scope is limited to SCM work, it assumes that + "special files" (sockets, etc.) are just files, and makes no + special effort to handle them. +*/ +FSL_FSTAT_TYPE_FILE, +/** Indicates a symlink filesystem entry. */ +FSL_FSTAT_TYPE_LINK +}; +typedef enum fsl_fstat_type_e fsl_fstat_type_e; + +/** + Bitmask values for use with the fsl_fstat::perms field. + + Only permissions which are relevant for fossil are listed here. + e.g. read-vs-write modes are irrelevant for fossil as it does not + track them. It manages only the is-executable bit. In in the + contexts of fossil manifests, it also treats "is a symlink" as a + permission flag. +*/ +enum fsl_fstat_perm_e { +/** + Sentinel value. +*/ +FSL_FSTAT_PERM_UNKNOWN = 0, +/** + The executable bit, as understood by Fossil. Fossil does not + differentiate between different +x values for user/group/other. +*/ +FSL_FSTAT_PERM_EXE = 0x01 +}; +typedef enum fsl_fstat_perm_e fsl_fstat_perm_e; + +/** + A simple wrapper around the stat(2) structure resp. _stat/_wstat + (on Windows). It exposes only the aspects of stat(2) info which + Fossil works with, and not any platform-/filesystem-specific + details except the executable bit for the permissions mode and some + handling of symlinks. +*/ +struct fsl_fstat { + /** + Indicates the type of filesystem object. + */ + fsl_fstat_type_e type; + /** + The time of the last file metadata change (owner, permissions, + etc.). The man pages (neither for Linux nor Windows) do not + specify exactly what unit this is. Let's assume seconds since the + start of the Unix Epoch. + */ + fsl_time_t ctime; + /** + Last modification time. + */ + fsl_time_t mtime; + /** + The size of the stat'd file, in bytes. + */ + fsl_size_t size; + /** + Contains the filesystem entry's permissions as a bitmask of + fsl_fstat_perm_e values. Note that only the executable bit for + _files_ (not directories) is exposed here. + */ + int perm; +}; + +/** Empty-initialized fsl_fstat structure, intended for const-copy + construction. */ +#define fsl_fstat_empty_m {FSL_FSTAT_TYPE_UNKNOWN,0,0,-1,0} + +/** Empty-initialized fsl_fstat instance, intended for non-const copy + construction. */ +FSL_EXPORT const fsl_fstat fsl_fstat_empty; + +/** + Runs the OS's stat(2) equivalent to populate fst with + information about the given file. + + Returns 0 on success, FSL_RC_MISUSE if zFilename is NULL, and + FSL_RC_RANGE if zFilename starts with a NUL byte. Returns + FSL_RC_NOT_FOUND if no filesystem entry is found for the given + name. Returns FSL_RC_IO if the underlying stat() (or equivalent) + fails for undetermined reasons inside the underlying + stat()/_wstati64() call. Note that the fst parameter may be + NULL, in which case the return value will be 0 if the name is + stat-able, but will return no other information about it. + + The derefSymlinks argument is ignored on non-Unix platforms. On + Unix platforms, if derefSymlinks is true then stat(2) is used, else + lstat(2) (if available on the platform) is used. For most cases + clients should pass true. They should only pass false if they need + to differentiate between symlinks and files. + + The fsl_fstat_type_e family of flags can be used to determine the + type of the filesystem object being stat()'d (file, directory, or + symlink). It does not apply any special logic for platform-specific + oddities other than symlinks (e.g. character devices and such). +*/ +FSL_EXPORT int fsl_stat(const char *zFilename, fsl_fstat * fst, + bool derefSymlinks); + +/** + Create a new delta between the memory zIn and zOut. + + The delta is written into a preallocated buffer, zDelta, which + must be at least 60 bytes longer than the target memory, zOut. + The delta string will be NUL-terminated, but it might also + contain embedded NUL characters if either the zSrc or zOut files + are binary. + + On success this function returns 0 and the length of the delta + string, in bytes, excluding the final NUL terminator character, + is written to *deltaSize. + + Returns FSL_RC_MISUSE if any of the pointer arguments are NULL + and FSL_RC_OOM if memory allocation fails during generation of + the delta. Returns FSL_RC_RANGE if lenSrc or lenOut are "too + big" (if they cause an overflow in the math). + + Output Format: + + The delta begins with a base64 number followed by a newline. + This number is the number of bytes in the TARGET file. Thus, + given a delta file z, a program can compute the size of the + output file simply by reading the first line and decoding the + base-64 number found there. The fsl_delta_applied_size() + routine does exactly this. + + After the initial size number, the delta consists of a series of + literal text segments and commands to copy from the SOURCE file. + A copy command looks like this: + + (Achtung: extra backslashes are for Doxygen's benefit - not + visible in the processsed docs.) + + NNN\@MMM, + + where NNN is the number of bytes to be copied and MMM is the + offset into the source file of the first byte (both base-64). + If NNN is 0 it means copy the rest of the input file. Literal + text is like this: + + NNN:TTTTT + + where NNN is the number of bytes of text (base-64) and TTTTT is + the text. + + The last term is of the form + + NNN; + + In this case, NNN is a 32-bit bigendian checksum of the output + file that can be used to verify that the delta applied + correctly. All numbers are in base-64. + + Pure text files generate a pure text delta. Binary files + generate a delta that may contain some binary data. + + Algorithm: + + The encoder first builds a hash table to help it find matching + patterns in the source file. 16-byte chunks of the source file + sampled at evenly spaced intervals are used to populate the hash + table. + + Next we begin scanning the target file using a sliding 16-byte + window. The hash of the 16-byte window in the target is used to + search for a matching section in the source file. When a match + is found, a copy command is added to the delta. An effort is + made to extend the matching section to regions that come before + and after the 16-byte hash window. A copy command is only + issued if the result would use less space that just quoting the + text literally. Literal text is added to the delta for sections + that do not match or which can not be encoded efficiently using + copy commands. + + @see fsl_delta_applied_size() + @see fsl_delta_apply() +*/ +FSL_EXPORT int fsl_delta_create( unsigned char const *zSrc, fsl_size_t lenSrc, + unsigned char const *zOut, fsl_size_t lenOut, + unsigned char *zDelta, fsl_size_t * deltaSize); + +/** + Works identically to fsl_delta_create() but sends its output to + the given output function. out(outState,...) may be called any + number of times to emit delta output. Each time it is called it + should append the new bytes to its output channel. + + The semantics of the return value and the first four arguments + are identical to fsl_delta_create(), with these ammendments + regarding the return value: + + - Returns FSL_RC_MISUSE if any of (zSrc, zOut, out) are NULL. + + - If out() returns non-0 at any time, delta generation is + aborted and that code is returned. + + Example usage: + + ``` + int rc = fsl_delta_create( v1, v1len, v2, v2len, + fsl_output_f_FILE, stdout); + ``` +*/ +FSL_EXPORT int fsl_delta_create2( unsigned char const *zSrc, fsl_size_t lenSrc, + unsigned char const *zOut, fsl_size_t lenOut, + fsl_output_f out, void * outState); + +/** + A fsl_delta_create() wrapper which uses the first two arguments + as the original and "new" content versions to delta, and outputs + the delta to the 3rd argument (overwriting any existing contents + and re-using any memory it had allocated). + + If the output buffer (delta) is the same as src or newVers, + FSL_RC_MISUSE is returned, and results are undefined if delta + indirectly refers to the same buffer as either src or newVers. + + Returns 0 on success. +*/ +FSL_EXPORT int fsl_buffer_delta_create( fsl_buffer const * src, + fsl_buffer const * newVers, + fsl_buffer * delta); + +/** + Apply a delta created using fsl_delta_create(). + + The output buffer must be big enough to hold the whole output + file and a NUL terminator at the end. The + fsl_delta_applied_size() routine can be used to determine that + size. + + zSrc represents the original sources to apply the delta to. + It must be at least lenSrc bytes of valid memory. + + zDelta holds the delta (created using fsl_delta_create()), + and it must be lenDelta bytes long. + + On success this function returns 0 and writes the applied delta + to zOut. + + Returns FSL_RC_MISUSE if any pointer argument is NULL. Returns + FSL_RC_RANGE if lenSrc or lenDelta are "too big" (if they cause + an overflow in the math). Invalid delta input can cause any of + FSL_RC_RANGE, FSL_RC_DELTA_INVALID_TERMINATOR, + FSL_RC_CHECKSUM_MISMATCH, FSL_RC_SIZE_MISMATCH, or + FSL_RC_DELTA_INVALID_OPERATOR to be returned. + + Refer to the fsl_delta_create() documentation above for a + description of the delta file format. + + @see fsl_delta_applied_size() + @see fsl_delta_create() + @see fsl_delta_apply2() +*/ +FSL_EXPORT int fsl_delta_apply( unsigned char const *zSrc, fsl_size_t lenSrc, + unsigned char const *zDelta, fsl_size_t lenDelta, + unsigned char *zOut ); + +/** + Functionally identical to fsl_delta_apply() but any errors generated + during application of the delta are described in more detail + in pErr. If pErr is NULL this behaves exactly as documented for + fsl_delta_apply(). +*/ +FSL_EXPORT int fsl_delta_apply2( unsigned char const *zSrc, + fsl_size_t lenSrc, + unsigned char const *zDelta, + fsl_size_t lenDelta, + unsigned char *zOut, + fsl_error * pErr); +/* + Calculates the size (in bytes) of the output from applying a the + given delta. On success 0 is returned and *appliedSize will be + updated with the amount of memory required for applying the + delta. zDelta must point to lenDelta bytes of memory in the + format emitted by fsl_delta_create(). It is legal for appliedSize + to point to the same memory as the 2nd argument. + + Returns FSL_RC_MISUSE if any pointer argument is NULL. Returns + FSL_RC_RANGE if lenDelta is too short to be a delta. Returns + FSL_RC_DELTA_INVALID_TERMINATOR if the delta's encoded length + is not properly terminated. + + This routine is provided so that an procedure that is able to + call fsl_delta_apply() can learn how much space is required for + the output and hence allocate nor more space that is really + needed. + + TODO?: consolidate 2nd and 3rd parameters into one i/o parameter? + + @see fsl_delta_apply() + @see fsl_delta_create() +*/ +FSL_EXPORT int fsl_delta_applied_size(unsigned char const *zDelta, + fsl_size_t lenDelta, + fsl_size_t * appliedSize); + +/** + "Fossilizes" the first len bytes of the given input string. If + (len<0) then fsl_strlen(inp) is used to calculate its length. + The output is appended to out, which is expanded as needed and + out->used is updated accordingly. Returns 0 on success, + FSL_RC_MISUSE if !inp or !out. Returns 0 without side-effects if + 0==len or (!*inp && len<0). Returns FSL_RC_OOM if reservation of + the output buffer fails (it is expanded, at most, one time by + this function). + + Fossilization replaces the following bytes/sequences with the + listed replacements: + + (Achtung: usage of doubled backslashes here it just to please + doxygen - they will show up as single slashes in the processed + output.) + + - Backslashes are doubled. + + - (\\n, \\r, \\v, \\t, \\f) are replaced with \\\\X, where X is the + conventional encoding letter for that escape sequence. + + - Spaces are replaced with \\s. + + - Embedded NULs are replaced by \\0 (numeric 0, not character + '0'). +*/ +FSL_EXPORT int fsl_bytes_fossilize( unsigned char const * inp, fsl_int_t len, + fsl_buffer * out ); +/** + "Defossilizes" bytes encoded by fsl_bytes_fossilize() in-place. + inp must be a string encoded by fsl_bytes_fossilize(), and the + decoding processes stops at the first unescaped NUL terminator. + It has no error conditions except for !inp or if inp is not + NUL-terminated, both of which invoke in undefined behaviour. + + If resultLen is not NULL then *resultLen is set to the resulting string + length. + +*/ +FSL_EXPORT void fsl_bytes_defossilize( unsigned char * inp, fsl_size_t * resultLen ); + +/** + Defossilizes the contents of b. Equivalent to: + fsl_bytes_defossilize( b->mem, &b->used ); +*/ +FSL_EXPORT void fsl_buffer_defossilize( fsl_buffer * b ); + +/** + Returns true if the input string contains only valid lower-case + base-16 digits. If any invalid characters appear in the string, + false is returned. +*/ +FSL_EXPORT bool fsl_validate16(const char *zIn, fsl_size_t nIn); + +/** + The input string is a base16 value. Convert it into its canonical + form. This means that digits are all lower case and that conversions + like "l"->"1" and "O"->"0" occur. +*/ +FSL_EXPORT void fsl_canonical16(char *z, fsl_size_t n); + +/** + Decode a N-character base-16 number into base-256. N must be a + multiple of 2. The output buffer must be at least N/2 characters + in length. Returns 0 on success. +*/ +FSL_EXPORT int fsl_decode16(const unsigned char *zIn, unsigned char *pOut, fsl_size_t N); + +/** + Encode a N-digit base-256 in base-16. N is the byte length of pIn + and zOut must be at least (N*2+1) bytes long (the extra is for a + terminating NUL). Returns zero on success, FSL_RC_MISUSE if !pIn + or !zOut. +*/ +FSL_EXPORT int fsl_encode16(const unsigned char *pIn, unsigned char *zOut, fsl_size_t N); + + +/** + Tries to convert the value of errNo, which is assumed to come + from the global errno, to a fsl_rc_e code. If it can, it returns + something approximating the errno value, else it returns dflt. + + Example usage: + + ``` + FILE * f = fsl_fopen("...", "..."); + int rc = f ? 0 : fsl_errno_to_rc(errno, FSL_RC_IO); + ... + ``` + + Why require the caller to pass in errno, instead of accessing it + directly from this function? To avoid the the off-chance that + something changes errno between the call and the conversion + (whether or not that's possible is as yet undetermined). It can + also be used by clients to map to explicit errno values to + fsl_rc_e values, e.g. fsl_errno_to_rc(EROFS,-1) returns + FSL_RC_ACCESS. + + A list of the errno-to-fossil conversions: + + - EINVAL: FSL_RC_MISUSE (could arguably be FSL_RC_RANGE, though) + + - ENOMEM: FSL_RC_OOM + + - EACCES, EBUSY, EPERM: FSL_RC_ACCESS + + - EISDIR, ENOTDIR: FSL_RC_TYPE + + - ENAMETOOLONG, ELOOP: FSL_RC_RANGE + + - ENOENT: FSL_RC_NOT_FOUND + + - EEXIST: FSL_RC_ALREADY_EXISTS + + - EIO: FSL_RC_IO + + Any other value for errNo causes dflt to be returned. + +*/ +FSL_EXPORT int fsl_errno_to_rc(int errNo, int dflt); + +/** + Make the given string safe for HTML by converting every "<" into + "<", every ">" into ">", every "&" into "&", and + encode '"' as " so that it can appear as an argument to + markup. + + The escaped output is send to out(oState,...). + + Returns 0 on success or if there is nothing to do (input has a + length of 0, in which case out() is not called). Returns + FSL_RC_MISUSE if !out or !zIn. If out() returns a non-0 code + then that value is returned to the caller. + + If n is negative, fsl_strlen() is used to calculate zIn's length. +*/ +FSL_EXPORT int fsl_htmlize(fsl_output_f out, void * oState, + const char *zIn, fsl_int_t n); + +/** + Functionally equivalent to fsl_htmlize() but optimized to perform + only a single allocation. + + Returns 0 on success or if there is nothing to do (input has a + length of 0). Returns FSL_RC_MISUSE if !p or !zIn, and + FSL_RC_OOM on allocation error. + + If n is negative, fsl_strlen() is used to calculate zIn's length. +*/ +FSL_EXPORT int fsl_htmlize_to_buffer(fsl_buffer *p, const char *zIn, fsl_int_t n); + +/** + Equivalent to fsl_htmlize_to_buffer() but returns the result as a + new string which must eventually be fsl_free()d by the caller. + + Returns NULL for invalid arguments or allocation error. +*/ +FSL_EXPORT char *fsl_htmlize_str(const char *zIn, fsl_int_t n); + +/** + If c is a character Fossil likes to HTML-escape, assigns *xlate + to its transformed form, else set it to NULL. Returns 1 for + untransformed characters and the strlen of *xlate for others. + Bytes returned via xlate are static and immutable. + + Results are undefined if xlate is NULL. +*/ +FSL_EXPORT fsl_size_t fsl_htmlize_xlate(int c, char const ** xlate); + +/** + Flags for use with text-diff generation APIs, + e.g. fsl_diff_text(). + + Maintenance reminder: these values are holy and must not be + changed without also changing the corresponding code in + diff.c. + + @deprecated in favor of fsl_diff2_flag_e and fsl_diff_v2() +*/ +enum fsl_diff_flag_e { +/** Ignore end-of-line whitespace */ +FSL_DIFF_IGNORE_EOLWS = 0x01, +/** Ignore end-of-line whitespace */ +FSL_DIFF_IGNORE_ALLWS = 0x03, +/** Generate a side-by-side diff */ +FSL_DIFF_SIDEBYSIDE = 0x04, +/** Missing shown as empty files */ +FSL_DIFF_VERBOSE = 0x08, +/** Show filenames only. Not used in this impl! */ +FSL_DIFF_BRIEF = 0x10, +/** Render HTML. */ +FSL_DIFF_HTML = 0x20, +/** Show line numbers. */ +FSL_DIFF_LINENO = 0x40, +/** Suppress optimizations (debug). */ +FSL_DIFF_NOOPT = 0x0100, +/** Invert the diff (debug). */ +FSL_DIFF_INVERT = 0x0200, +/** Only display if not "too big." */ +FSL_DIFF_NOTTOOBIG = 0x0800, +/** Strip trailing CR */ +FSL_DIFF_STRIP_EOLCR = 0x1000, +/** + This flag tells text-mode diff generation to add ANSI color + sequences to some output. The colors are currently hard-coded + and non-configurable. This has no effect for HTML output, and + that flag trumps this one. It also currently only affects + unified diffs, not side-by-side. + + Maintenance reminder: this one currently has no counterpart in + fossil(1), is not tracked in the same way, and need not map to an + internal flag value. +*/ +FSL_DIFF_ANSI_COLOR = 0x2000 +}; + +/** + Generates a textual diff from two text inputs and writes + it to the given output function. + + pA and pB are the buffers to diff. + + contextLines is the number of lines of context to output. This + parameter has a built-in limit of 2^16, and values larger than + that get truncated. A value of 0 is legal, in which case no + surrounding context is provided. A negative value translates to + some unspecified default value. + + sbsWidth specifies the width (in characters) of the side-by-side + columns. If sbsWidth is not 0 then this function behaves as if + diffFlags contains the FSL_DIFF_SIDEBYSIDE flag. If sbsWidth is + negative, OR if diffFlags explicitly contains + FSL_DIFF_SIDEBYSIDE and sbsWidth is 0, then some default width + is used. This parameter has a built-in limit of 255, and values + larger than that get truncated to 255. + + diffFlags is a mask of fsl_diff_flag_t values. Not all of the + fsl_diff_flag_t flags are yet [sup]ported. + + The output is sent to out(outState,...). If out() returns non-0 + during processing, processing stops and that result is returned + to the caller of this function. + + Returns 0 on success, FSL_RC_OOM on allocation error, + FSL_RC_MISUSE if any arguments are invalid, FSL_RC_TYPE if any + of the content appears to be binary (contains embedded NUL + bytes), FSL_RC_RANGE if some range is exceeded (e.g. the maximum + number of input lines). + + None of (pA, pB, out) may be NULL. + + TODOs: + + - Add a predicate function for outputing only matching + differences, analog to fossil(1)'s regex support (but more + flexible). + + - Expose the raw diff-generation bits via the internal API + to facilitate/enable the creation of custom diff formats. + + @see fsl_diff_v2() + @deprecated Prefer fsl_diff_v2() for new code. +*/ +FSL_EXPORT int fsl_diff_text(fsl_buffer const *pA, fsl_buffer const *pB, + fsl_output_f out, void * outState, + short contextLines, short sbsWidth, + int diffFlags ); + +/** + Functionally equivalent to: + + ``` + fsl_diff_text(pA, pB, fsl_output_f_buffer, pOut, + contextLines, sbsWidth, diffFlags); + ``` + + Except that it returns FSL_RC_MISUSE if !pOut. + + @see fsl_diff_v2() + @deprecated Prefer fsl_diff_v2() for new code. +*/ +FSL_EXPORT int fsl_diff_text_to_buffer(fsl_buffer const *pA, fsl_buffer const *pB, + fsl_buffer *pOut, short contextLines, + short sbsWidth, int diffFlags ); + +/** + Flags for use with the 2021-era text-diff generation APIs + (fsl_diff_builder and friends). This set of flags may still change + considerably. + + Maintenance reminder: some of these values are holy and must not be + changed without also changing the corresponding code in diff2.c. +*/ +enum fsl_diff2_flag_e { +/** Ignore end-of-line whitespace. Applies to + all diff builders. */ +FSL_DIFF2_IGNORE_EOLWS = 0x01, +/** Ignore end-of-line whitespace. Applies to + all diff builders. */ +FSL_DIFF2_IGNORE_ALLWS = 0x03, +/** Suppress optimizations (debug). Applies to + all diff builders. */ +FSL_DIFF2_NOOPT = 0x0100, +/** Invert the diff. Applies to all diff builders. */ +FSL_DIFF2_INVERT = 0x0200, +/** Use context line count even if it's zero. Applies to all diff + builders. Normally a value of 0 is treated as the built-in + default. */ +FSL_DIFF2_CONTEXT_ZERO = 0x0400, +/** + Only calculate diff if it's not "too big." Applies to all diff + builders and will cause the public APIs which hit this to return + FSL_RC_RANGE. +*/ +FSL_DIFF2_NOTTOOBIG = 0x0800, +/** + Strip trailing CR before diffing. Applies to all diff builders. +*/ +FSL_DIFF2_STRIP_EOLCR = 0x1000, +/** + More precise but slower side-by-side diff algorithm, for diffs + which use that. +*/ +FSL_DIFF2_SLOW_SBS = 0x2000, +/** + Tells diff builders which support it to include line numbers in + their output. +*/ +FSL_DIFF2_LINE_NUMBERS = 0x10000, +/** + Tells diff builders which optionally support an "index" line + to NOT include it in their output. +*/ +FSL_DIFF2_NOINDEX = 0x20000, + +/** + Tells the TCL diff builder that the complete output and each line + should be wrapped in {...}. +*/ +FSL_DIFF2_TCL_BRACES = 0x40000, + +/** + Reserved for client-defined diff builder use. +*/ +FSL_DIFF2_CLIENT1 = 0x01000000, +/** + Reserved for client-defined diff builder use. +*/ +FSL_DIFF2_CLIENT2 = 0x02000000, +/** + Reserved for client-defined diff builder use. +*/ +FSL_DIFF2_CLIENT3 = 0x04000000, +/** + Reserved for client-defined diff builder use. +*/ +FSL_DIFF2_CLIENT4 = 0x08000000 +}; + +/** + An instance of this class is used to convey certain state to + fsl_diff_builder objects. Some of this state is configuration + provided by the client and some is volatile, used for communicating + common state to diff builder instances during the diff rendering + process. + + Certain fsl_diff_builder implementations may require that some + ostensibly optional fields be filled out. Documenting that is TODO, + as the builders get developed. +*/ +struct fsl_diff_opt { + /** + Flags from the fsl_diff2_flag_e enum. + */ + uint32_t diffFlags; + /** + Number of lines of diff context (number of lines common to the + LHS and RHS of the diff). Library-wide default is 5. + */ + unsigned short contextLines; + /** + Maximum column width for side-by-side, a.k.a. split, diffs. + + FSL_DIFF_BUILDER_SPLIT_TEXT truncates its content columns (as + opposed to line numbers and its modification marker) to, at most, + this width. By default it uses as much width as is necessary to + render whole lines. It treats this limit as UTF8 characters, not + bytes. + + This is a hint, not a rule. A given diff builder is free to + ignore it or to ignore values which are arbitrarily deemed "too + small" or "too large." + */ + unsigned short columnWidth; + /** + The filename of the object represented by the LHS of the + diff. This is intentended for, e.g., generating header-style + output. This may be NULL. + */ + const char * nameLHS; + /** + The hash of the object represented by the LHS of the diff. This + is intentended for, e.g., generating header-style output. This + may be NULL. + */ + const char * hashLHS; + /** + The filename of the object represented by the LHS of the + diff. This is intentended for, e.g., generating header-style + output. If this is NULL but nameLHS is not then they are assumed + to have the same name. + */ + const char * nameRHS; + /** + The hash of the object represented by the RHS of the diff. This + is intentended for, e.g., generating header-style output. This + may be NULL. + */ + const char * hashRHS; + /** + Output destination. Any given builder might, depending on how it + actually constructs the diff, buffer output and delay calling + this until its finish() method is called. + */ + fsl_output_f out; + /** + State for this->out(). Ownership is unspecified by this + interface: it is for use by this->out() but what is supposed to + happen to it after this object is done with it depends on + higher-level code. + */ + void * outState; + + /** + EXPERIMENTAL AND SUBJECT TO CHANGE. + + Optional ANSI color control sequences to be injected into + text-mode diff output by diff builders which support them. All + members of this struct must either be NULL, an empty string, or a + valid ANSI escape sequence. The reset option must be the escape + sequence to reset either just the color or to reset all ANSI + control attributes, depending on how the other members are + set. If any member other than reset is set, all must be set. + + The diff driver will treat any members of this which are NULL as + empty strings to simplify diff builder color integration. The + exception is the reset member - see its docs for details. + */ + struct fsl_diff_opt_ansi { + /** + Color escape sequence for inserted lines. + */ + char const * insertion; + /** + Color escape sequence for edited or replaced lines. This option + might be ignored, depending on how the renderer works. Some + will render edits as a deletion/insertion pair. + */ + char const * edit; + /** + Color escape sequence for inserted lines. + */ + char const * deletion; + /** + Escape sequence to reset colors. If reset is empty or NULL and + any others are not then reset is automatically treated as if it + were the ANSI code to reset all color attributes. + */ + char const * reset; + } ansiColor; +}; + +/** Convenience typedef. */ +typedef struct fsl_diff_opt fsl_diff_opt; + +/** Initialized-with-defaults fsl_diff_opt structure, intended for + const-copy initialization. */ +#define fsl_diff_opt_empty_m {\ + 0/*diffFlags*/, 5/*contextLines*/, 0/*columnWidth*/,\ + NULL/*nameLHS*/,NULL/*hashLHS*/, \ + NULL/*nameRHS*/, NULL/*hashRHS*/, \ + NULL/*out*/, NULL/*outState*/, \ + {/*ansiColor*/ ""/*insertion*/,""/*edit*/,""/*deletion*/,\ + ""/*reset*/} \ +} + +/** Initialized-with-defaults fsl_diff_opt structure, intended for + non-const copy initialization. */ +extern const fsl_diff_opt fsl_diff_opt_empty; + +/** + Information about each line of a file being diffed. + + This type is only in the public API for use by the fsl_diff_builder + interface, specifically for use with fsl_dline_change_spans(). It + is not otherwise intended for public use. None of its members are + considered even remotely public except for this->z and this->n. +*/ +struct fsl_dline { + /** + The text of the line. Owned by higher-level code. Not + necessarily NUL-terminated: this->n holds its length. + */ + const char *z; + /** Number of bytes of z which belong to this line. */ + unsigned short n; + // All members after this point are strictly for internal use only. + /** Indent of the line. Only !=0 with certain options. */ + unsigned short indent; + /** Hash of the line. Lower X bits are the length. */ + uint64_t h; + /** 1+(Index of next line with same the same hash) */ + unsigned int iNext; + /** + An array of fsl_dline elements serves two purposes. The fields + above are one per line of input text. But each entry is also + a bucket in a hash table, as follows: + */ + unsigned int iHash; /* 1+(first entry in the hash chain) */ +}; + +/** + Convenience typedef. +*/ +typedef struct fsl_dline fsl_dline; +/** Initialized-with-defaults fsl_dline structure, intended for + const-copy initialization. */ +#define fsl_dline_empty_m {NULL,0U,0U,0U,0U,0U} +/** Initialized-with-defaults fsl_dline structure, intended for + non-const copy initialization. */ +extern const fsl_dline fsl_dline_empty; + +/** + Maximum number of change spans for fsl_dline_change. +*/ +#define fsl_dline_change_max_spans 8 + +/** + This "mostly-internal" type describes zero or more (up to + fsl_dline_change_max_spans) areas of difference between two lines + of text. This type is only in the public API for use with concrete + fsl_diff_builder implementations. +*/ +struct fsl_dline_change { + /** Number of change spans (number of used elements in this->a). */ + unsigned char n; + /** Array of change spans, in left-to-right order */ + struct fsl_dline_change_span { + /* Reminder: int instead of uint b/c some ported-in algos use + negatives. */ + /** Byte offset to start of a change on the left */ + int iStart1; + /** Length of the left change in bytes */ + int iLen1; + /** Byte offset to start of a change on the right */ + int iStart2; + /** Length of the change on the right in bytes */ + int iLen2; + /** True if this change is known to have no useful subdivs */ + int isMin; + } a[fsl_dline_change_max_spans]; +}; + +/** + Convenience typedef. +*/ +typedef struct fsl_dline_change fsl_dline_change; + +/** Initialized-with-defaults fsl_dline_change structure, intended for + const-copy initialization. */ +#define fsl_dline_change_empty_m { \ + 0, { \ + {0,0,0,0,0}, {0,0,0,0,0}, {0,0,0,0,0}, {0,0,0,0,0}, \ + {0,0,0,0,0}, {0,0,0,0,0}, {0,0,0,0,0}, {0,0,0,0,0} \ + } \ +} + +/** Initialized-with-defaults fsl_dline_change structure, intended for + non-const copy initialization. */ +extern const fsl_dline_change fsl_dline_change_empty; + +/** + Given two lines of a diff, this routine computes a set of changes + between those lines for display purposes and writes a description + of those changes into the 3rd argument. After returning, p->n + contains the number of elements in p->a which were populated by + this routine. + + This function is only in the public API for use with + fsl_diff_builder objects. It is not a requirement for such objects + but can be used to provide more detailed diff changes than marking + whole lines as simply changed or not. +*/ +FSL_EXPORT void fsl_dline_change_spans(const fsl_dline *pLeft, + const fsl_dline *pRight, + fsl_dline_change * const p); + +/** + Compares two fsl_dline instances using memcmp() semantics, + returning 0 if they are equivalent. + + @see fsl_dline_cmp_ignore_ws() +*/ +FSL_EXPORT int fsl_dline_cmp(const fsl_dline * const pA, + const fsl_dline * const pB); + +/** + Counterpart of fsl_dline_cmp() but ignores all whitespace + when comparing for equivalence. +*/ +FSL_EXPORT int fsl_dline_cmp_ignore_ws(const fsl_dline * const pA, + const fsl_dline * const pB); + +/** + Breaks the first n bytes of z into an array of fsl_dline records, + each of which refers back to z (so it must remain valid for their + lifetime). If n is negative, fsl_strlen() is used to calculate + z's length. + + The final argument may be any flags from the fsl_diff2_flag_e + enum, but only the following flags are honored: + + - FSL_DIFF2_STRIP_EOLCR + - FSL_DIFF2_IGNORE_EOLWS + - FSL_DIFF2_IGNORE_ALLWS + + On success, returns 0, assigns *pnLine to the number of lines, and + sets *pOut to the array of fsl_dline objects, transfering ownership + to the caller, who must eventually pass it to fsl_free() to free + it. + + If z is NULL or n is 0, the input is assumed to be empty, 0 is returned, + (*pOut) will be set to NULL, and (*pnLine) will be set to 0. + + On error, neither (*pnLine) nor (*pOut) are modified and returns one + of: + + - FSL_RC_DIFF_BINARY if the input appears to be non-text. + - FSL_RC_OOM on allocation error. +*/ +FSL_EXPORT int fsl_break_into_dlines(const char *z, fsl_int_t n, + uint32_t *pnLine, + fsl_dline **pOut, uint64_t diff2Flags); + +/** Convenience typedef. */ +typedef struct fsl_diff_builder fsl_diff_builder; + +/** + This class is the basis of libfossil's port of the diff engine + added to fossil(1) in 2021-09. + + A diff builder is an object responsible for formatting low-level + diff info another form, typically for human readability but also + for machine readability (patches). The library generates a + low-level diff then feeds that through an algorithm which + determines which methods of this class to call, delegating all + rendering of the diff to an instance of this class. + + The internal APIs which drive each instance of this class guaranty + that if any method of this class returns non-0 (an error code) then + no futher methods will be called except for finalize(). + + Potential TODO: add a two-phase option to this interface. If + builder->passes is greater than 0, builder->currentPass gets set to + the current pass number (1 or 2) then it gets passed the diff once + for each pass the builder needs, updating the currentPass flag + between runs. The first phase is simply for it to analyze what + needs to be done and the second is to do it. That would allow, + e.g. the split-mode impl to avoid buffering all of the output + because the first pass would allow it to calculate the width it + needs for outputing the text columns. It would also allow for it to + dynamically resize the line number columns more easily. +*/ +struct fsl_diff_builder { + /** + Config info, owned by higher-level routines. Every diff builder + requires one of these. Builders are prohibited from modifying + these but the diff driver will. + + Note that the diff driver may make a bitwise copy of this object + and use _that_ one for the actual diff generation. That is, + methods of this class must never assume that this member's + pointer refers to a specific object. (This leeway is necessary in + order to implement diff inversion (swapping the LHS/RHS of a + diff).) + */ + fsl_diff_opt * opt; + /** + If not NULL, this is called once per diff to give the builder a + chance to perform any bootstrapping initialization or header + output. At the point this is called, this->cfg is assumed to have + been filled out properly. Diff builder implementations which + require dynamic resource allocation may perform it here or in + their factory routine(s). + + This method should also reset any dynamic state of a builder so + that it may be reused for subsequent diffs. This enables the API + to use a single builder for a collection of logically grouped + files without having to destroy and reallocate the builder. + + Must return 0 on success, non-0 on error. If it returns non-0, + the only other method of the instance which may be legally called + is finalize(). + + The diff driver sets this->lnLHS and this->lnRHS to 0 before + calling this. + */ + int (*start)(fsl_diff_builder* const); + + /** + If this is not NULL, it is called one time at the start of each + chunk of diff for a given file and is passed the line number of + each half of the diff and the number of lines in that chunk for + that half (including insertions and deletions). This is primarily + intended for generating conventional unified diff chunk headers + in the form: + + ``` + @@ -A,B +C,D @@ + ``` + + The inclusion of this method in an object might preclude certain + other diff formatting changes which might otherwise apply. + Notably, if the span between two diff chunks is smaller than the + context lines count, the diff builder driver prefers to merge + those two chunks together. That "readability optimization" is + skipped when this method is set because this method may otherwise + report that lines are being skipped which then subsequently get + output by the driver. + + Must return 0 on success, non-0 on error. + */ + int (*chunkHeader)(fsl_diff_builder* const, + uint32_t A, uint32_t B, + uint32_t C, uint32_t D); + + /** + Tells the builder that n lines of common output are to be + skipped. How it represents this is up to the impl. Must return 0 + on success, non-0 on error. + + Typical common implementation details: + + - Increment both this->lnLHS and this->lnRHS by n. + */ + int (*skip)(fsl_diff_builder* const, uint32_t n); + /** + Tells the builder that the given line represents one line of + common output. Must return 0 on success, non-0 on error. + + Typical common implementation details: + + - Increment both this->lnLHS and this->lnRHS by 1. + */ + int (*common)(fsl_diff_builder* const, fsl_dline const * line); + /** + Tells the builder that the given line represents an "insert" into + the RHS. Must return 0 on success, non-0 on error. + + Typical common implementation details: + + - Increment this->lnRHS by 1. + */ + int (*insertion)(fsl_diff_builder* const, fsl_dline const * line); + /** + Tells the builder that the given line represents a "deletion" - a + line removed from the LHS. Must return 0 on success, non-0 on + error. + + Typical common implementation details: + + - Increment this->lnLHS by 1. + */ + int (*deletion)(fsl_diff_builder* const, fsl_dline const * line); + /** + Tells the builder that the given line represents a replacement + from the LHS to the RHS. Must return 0 on success, non-0 on + error. This differs from an "edit" in that the line being + replaced seems to have on relationship to the replacement. Even + so, builders are free to represent replacements and edits + identically, and are free to represent either or both as a pair + of deletion/insertion operations. + + Typical common implementation details: + + - Increment both this->lnLHS and this->lnRHS by 1. + */ + int (*replacement)(fsl_diff_builder* const, fsl_dline const * lineLhs, + fsl_dline const * lineRhs); + /** + Tells the builder that the given line represents an "edit" from + the LHS to the RHS. Must return 0 on success, non-0 on + error. Builders are free, syntax permitting, to usse the + fsl_dline_change_spans() API to elaborate on edits for display + purposes, to treat it identically to this->replacement(), or to + treat this as a single pair of calls to this->deletion() and + this->insertion(). In the latter case they simply need to pass + lineLhs to this->deletion() and lineRhs to this->insertion(). + + Typical common implementation details: + + - Increment both this->lnLHS and this->lnRHS by 1. + */ + int (*edit)(fsl_diff_builder* const, fsl_dline const * lineLhs, + fsl_dline const * lineRhs); + /** + Must "finish" the diff process. Depending on the diff impl, this + might flush any pending output or may be a no-op. This is only + called if the rest of the diff was generated without producing an + error result. + + This member may be NULL. + + Implementations are free to collect all of their output in an + internal representation and delay flushing it until this routine + is called. + + Must return 0 on success, non-0 on error (e.g. output flushing + fails). + */ + int (*finish)(fsl_diff_builder* const); + /** + Must free any state owned by this builder, including the builder + object. It must not generate any output. + + Must return 0 on success, non-0 on error (e.g. output flushing + fails), but must clean up its own state and the builder object in + either case. + */ + void (*finalize)(fsl_diff_builder* const); + + /** + If true, this builder gets passed through the diff generation + process twice. See this->passNumber for details. + */ + bool twoPass; + + /** + Gets set to the "pass number" immediately before this->start() is + called, starting with pass number 1. This value is only relevant + for two-pass builders, which can use this to control their mode + of operation, e.g. data collection in pass 1 and actual work in + pass 2. Note that all of the diff-building API methods are call + for both passes, including start() and finish(). Only finalize() + is not affected by this. + */ + unsigned short passNumber; + /** + Impl-specific diff-generation state. If it is owned by this + instance then this->finalize() must clean it up. + */ + void * pimpl; + /** + Impl-specific int for tracking basic output state, e.g. of + opening/closing tags. This must not be modified by clients. + */ + unsigned int implFlags; + /** + A place to store the number of files seen by this builder so far, + for builders which need to distinguish that somehow (e.g. adding + a separator before each file after the first). Implementations + which use this should increment it in their start() method. + */ + uint32_t fileCount; + /** + Number of lines seen of the LHS content. It is up to the concrete + builder impl to update this if it's needed. The core diff driver + sets this to 0 before calling this->start(). + */ + uint32_t lnLHS; + /** + Number of lines seen of the RHS content. It is up to the concrete + builder impl to update this if it's needed. The core diff driver + sets this to 0 before calling this->start(). + */ + uint32_t lnRHS; +}; + +/** Initialized-with-defaults fsl_diff_builder structure, intended for + const-copy initialization. */ +#define fsl_diff_builder_empty_m { \ + NULL/*cfg*/, \ + NULL/*start()*/,NULL/*chunkHeader()*/,NULL/*skip()*/, NULL/*common()*/, \ + NULL/*insertion()*/,NULL/*deletion()*/, NULL/*replacement()*/, \ + NULL/*edit()*/, NULL/*finish()*/, NULL/*finalize()*/, \ + false/*twoPass*/,0U/*passNumber*/, \ + NULL/*pimpl*/, 0U/*implFlags*/,0U/*fileCount*/, \ + 0/*lnLHS*/,0/*lnRHS*/ \ +} + +/** Initialized-with-defaults fsl_diff_builder structure, intended for + non-const copy initialization. */ +extern const fsl_diff_builder fsl_diff_builder_empty; + +/** + Type IDs for use with fsl_diff_builder_factory(). +*/ +enum fsl_diff_builder_e { +/** + A "dummy" diff builder intended only for testing the + fsl_diff_builder interface and related APIs. It does not produce + output which is generically useful. +*/ +FSL_DIFF_BUILDER_DEBUG = 1, +/** + Generates diffs in a compact low(ist)-level form originally + designed for use by diff renderers implemented in JavaScript. + + This diff builder outputs a JSON object with the following + properties: + + - hashLHS, hashRHS: the hashes of the LHS/RHS content. + + - nameLHS, nameRHS: the filenames of the LHS/RHS. By convention, if + the RHS is NULL but the LHS is not, both sides have the same + name. + + - diff: raw diff content, an array with the structure described + below. + + Note that it is legal for the names and hashes to be "falsy" (null, + not set, or empty strings). + + The JSON array consists of integer opcodes with each opcode + followed by zero or more arguments: + + ``` + Syntax Mnemonic Description + + ----------- -------- -------------------------- + 0 END This is the end of the diff. + 1 INTEGER SKIP Skip N lines from both files. + 2 STRING COMMON The line STRING is in both files. + 3 STRING INSERT The line STRING is in only the right file. + 4 STRING DELETE The line STRING is in only the left file. + 5 SUBARRAY EDIT One line is different on left and right. + ``` + + The SUBARRAY is an array of 3*N+1 strings with N>=0. The triples + represent common-text, left-text, and right-text. The last string + in SUBARRAY is the common-suffix. Any string can be empty if it + does not apply. +*/ +FSL_DIFF_BUILDER_JSON1, + +/** + A diff builder which produces output compatible with the patch(1) + command. Its output is functionally identical to fossil(1)'s + default diff output except that by default includes an Index line + at the top of each file (use the FSL_DIFF2_NOINDEX flag in its + fsl_diff_opt::diffFlags to disable that). + + Supported flags: + + - FSL_DIFF2_LINE_NUMBERS (makes it incompatible with patch(1)) + - FSL_DIFF2_NOINDEX +*/ +FSL_DIFF_BUILDER_UNIFIED_TEXT, + +/** + A diff builder which outputs a description of the diff in a + TCL-readable form. It requires external TCL code in order to + function. + + TODO: a flag which includes the tcl/tk script as part of the + output. We first need to compile fossil's diff.tcl into the + library. +*/ +FSL_DIFF_BUILDER_TCL, +/** + A pain-text side-by-side (a.k.a. split) diff view. This diff always + behaves as if the FSL_DIFF2_LINE_NUMBERS flag were set because its + output is fairly useless without line numbers. It optionally + supports ANSI coloring. +*/ +FSL_DIFF_BUILDER_SPLIT_TEXT +}; +typedef enum fsl_diff_builder_e fsl_diff_builder_e; + +/** + A factory for creating fsl_diff_builder instances of types which + are built in to the library. This does not preclude the creation of + client-side diff builders (e.g. ones which write to ncurses widgets + or similar special-case output). + + On success, returns 0 and assigns *pOut to a new builder instance + which must eventually be freed by calling its pOut->finalize() + method. On error, returns non-0 and *pOut is not modified. Error + codes include FSL_RC_OOM (alloc failed) and FSL_RC_TYPE (unknown + type ID), FSL_RC_TYPE (type is not (or not yet) implemented). +*/ +FSL_EXPORT int fsl_diff_builder_factory( fsl_diff_builder_e type, + fsl_diff_builder **pOut ); + +/** + This counterpart of fsl_diff_text() defines its output format in + terms of a fsl_diff_builder instance which the caller must provide. + The caller is responsible for pointing pBuilder->cfg to a + configuration object suitable for the desired diff. In particular, + pBuilder->cfg->out and (if necessary) pBuilder->cfg->outState must + be set to non-NULL values. + + This function generates a low-level diff of two versions of content, + contained in the given buffers, and passes that diff through the + given diff builder to format it. + + Returns 0 on success. On error, it is not generally knowable whether + or not any diff output was generated. + + The builder may produce any error codes it wishes, in which case + they are propagated back to the caller. Common error codes include: + + - FSL_RC_OOM if an allocation fails. + + - FSL_RC_RANGE if the diff is "too big" and + pBuilder->config->diffFlags contains the FSL_DIFF2_NOTTOOBIG flag. + + - FSL_RC_DIFF_BINARY if the to-diff content appears to be binary, + noting that "appears to be" is heuristric-driven and subject to + false positives. Specifically, files with extremely long lines will + be recognized as binary (and are, in any case, generally less than + useful for most graphical diff purposes). + + @see fsl_diff_builder_factory() + @see fsl_diff_text() + @see fsl_diff_raw_v2() +*/ +FSL_EXPORT int fsl_diff_v2(fsl_buffer const * pv1, + fsl_buffer const * pv2, + fsl_diff_builder * const pBuilder); + +/** + Performs a diff, as for fsl_diff_v2(), but returns the results in + the form of an array of COPY, DELETE, INSERT triples terminated by + 3 entries with the value 0. + + Each triple in the list specifies how many *lines* of each half of + the diff (the first 2 arguments to this function) to COPY as-is + (common code), DELETE (exists in the LHS but not in the RHS), and + INSERT (exists in the RHS but not in the LHS). By breaking the + input into lines and following these values, a line-level text-mode + diff of the two blobs can be generated. + + See fsl_diff_v2() for the details, all of which apply except for + the output: + + - cfg->out is ignored. + + - On success, *outRaw is assigned to the output array and ownership + of it is transfered to the caller, who must eventually pass it to + fsl_free() to free it. + +*/ +FSL_EXPORT int fsl_diff_v2_raw(fsl_buffer const * pv1, + fsl_buffer const * pv2, + fsl_diff_opt const * const cfg, + int **outRaw ); + + +/** + If zDate is an ISO8601-format string, optionally with a .NNN + fractional suffix, then this function returns true and sets + *pOut (if pOut is not NULL) to the corresponding Julian + value. If zDate is not an ISO8601-format string then this + returns false and pOut is not modified. + + This function does NOT confirm that zDate ends with a NUL + byte. i.e. if passed a valid date string which has trailing + bytes after it then those are simply ignored. This is so that it + can be used to read subsets of larger strings. + + Achtung: this calculation may, due to voodoo-level + floating-point behaviours, differ by a small fraction of a point + (at the millisecond level) for a given input compared to other + implementations (e.g. sqlite's strftime() _might_ differ by a + millisecond or two or _might_ not). Thus this routine should not + be used when 100% round-trip fidelity is required, but is close + enough for routines which do not require 100% millisecond-level + fidelity in time conversions. + + @see fsl_julian_to_iso8601() +*/ +FSL_EXPORT bool fsl_iso8601_to_julian( char const * zDate, double * pOut ); + +/** + Converts the Julian Day J to an ISO8601 time string. If addMs is + true then the string includes the '.NNN' fractional part, else + it will not. This function writes (on success) either 20 or 24 + bytes (including the terminating NUL byte) to pOut, depending on + the value of addMs, and it is up to the caller to ensure that + pOut is at least that long. + + Returns true (non-0) on success and the only error conditions + [it can catch] are if pOut is NULL, J is less than 0, or + evaluates to a time value which does not fit in ISO8601 + (e.g. only years 0-9999 are supported). + + @see fsl_iso8601_to_julian() +*/ +FSL_EXPORT bool fsl_julian_to_iso8601( double J, char * pOut, bool addMs ); + +/** + Returns the Julian Day time J value converted to a Unix Epoch + timestamp. It assumes 86400 seconds per day and does not account + for leap seconds, leap years, leap frogs, or any other kind of + leap, up to and including leaps of faith. +*/ +FSL_EXPORT fsl_time_t fsl_julian_to_unix( double J ); + +/** + Performs a chdir() to the directory named by zChDir. + + Returns 0 on success. On error it tries to convert the + underlying errno to one of the FSL_RC_xxx values, falling + back to FSL_RC_IO if it cannot figure out anything more + specific. +*/ +FSL_EXPORT int fsl_chdir(const char *zChDir); + +/** + A strftime() implementation. + + dest must be valid memory at least destLen bytes long. The result + will be written there. + + fmt must contain the format string. See the file fsl_strftime.c + for the complete list of format specifiers and their descriptions. + + timeptr must be the time the caller wants to format. + + Returns 0 if any arguments are NULL. + + On success it returns the number of bytes written to dest, not + counting the terminating NUL byte (which it also writes). It + returns 0 on any error, and the client may need to distinguish + between real errors and (destLen==0 or !*fmt), both of which could + also look like errors. + + TODOs: + + - Refactor this to take a callback or a fsl_buffer, so that we can + format arbitrarily long output. + + - Refactor it to return an integer error code. + + (This implementation is derived from public domain sources + dating back to the early 1990's.) +*/ +FSL_EXPORT fsl_size_t fsl_strftime(char *dest, fsl_size_t destLen, + const char *format, const struct tm *timeptr); + +/** + A convenience form of fsl_strftime() which takes its timestamp in + the form of a Unix Epoch time. See fsl_strftime() for the + semantics of the first 3 arguments and the return value. If + convertToLocal is true then epochTime gets converted to local + time (via, oddly enough, localtime(3)), otherwise gmtime(3) is + used for the conversion. + + BUG: this function uses static state and is not thread-safe. +*/ +FSL_EXPORT fsl_size_t fsl_strftime_unix(char * dest, fsl_size_t destLen, char const * format, + fsl_time_t epochTime, bool convertToLocal); + + +/** + A convenience form of fsl_strftime() which assumes that the + formatted string is of "some reasonable size" and appends its + formatted representation to b. Returns 0 on success, non-0 on + error. If any argument is NULL or !*format then FSL_RC_MISUSE is + returned. FSL_RC_RANGE is returned if the underlying call to + fsl_strftime() fails (which it will if the format string + resolves to something "unususually long"). It returns FSL_RC_OOM + if appending to b fails due to an allocation error. +*/ +FSL_EXPORT int fsl_buffer_strftime(fsl_buffer * const b, + char const * format, + const struct tm *timeptr); + +/** + "whence" values for use with fsl_buffer_seek. +*/ +enum fsl_buffer_seek_e { +FSL_BUFFER_SEEK_SET = 1, +FSL_BUFFER_SEEK_CUR = 2, +FSL_BUFFER_SEEK_END = 3 +}; +typedef enum fsl_buffer_seek_e fsl_buffer_seek_e; + +/** + "Seeks" b's internal cursor to a position specified by the given offset + from either the current cursor position (FSL_BUFFER_SEEK_CUR), the start + of the buffer (FSL_BUFFER_SEEK_SET), or the end (FSL_BUFFER_SEEK_END). + If the cursor would be placed out of bounds, it will be placed at the start + resp. end of the buffer. + + The "end" of a buffer is the value of its fsl_buffer::used member + (i.e. its one-after-the-end). + + Returns the new position. + + Note that most buffer algorithms, e.g. fsl_buffer_append(), do not + modify the cursor. Only certain special-case algorithms use it. + + @see fsl_buffer_tell() + @see fsl_buffer_rewind() +*/ +FSL_EXPORT fsl_size_t fsl_buffer_seek(fsl_buffer * const b, fsl_int_t offset, + fsl_buffer_seek_e whence); +/** + Returns the buffer's current cursor position. + + @see fsl_buffer_rewind() + @see fsl_buffer_seek() +*/ +FSL_EXPORT fsl_size_t fsl_buffer_tell(fsl_buffer const * const b); +/** + Resets b's cursor to the beginning of the buffer. + + @see fsl_buffer_tell() + @see fsl_buffer_seek() +*/ +FSL_EXPORT void fsl_buffer_rewind(fsl_buffer * const b); + +/** + The "Path Finder" class is a utility class for searching the + filesystem for files matching a set of common prefixes and/or + suffixes (i.e. directories and file extensions). + + Example usage: + + ``` + fsl_pathfinder pf = fsl_pathfinder_empty; + int rc; + char const * found = NULL; + rc = fsl_pathfinder_ext_add( &pf, ".doc" ); + if(rc) { ...error... } + // The following error checks are elided for readability: + rc = fsl_pathfinder_ext_add( &pf, ".txt" ); + rc = fsl_pathfinder_ext_add( &pf, ".wri" ); + rc = fsl_pathfinder_dir_add( &pf, "." ); + rc = fsl_pathfinder_dir_add( &pf, "/my/doc/dir" ); + rc = fsl_pathfinder_dir_add( &pf, "/other/doc/dir" ); + + rc = fsl_pathfinder_search( &pf, "MyDoc", &found, NULL); + if(0==rc){ assert(NULL!=found); } + + // Eventually clean up: + fsl_pathfinder_clear(&pf); + ``` + + @see fsl_pathfinder_dir_add() + @see fsl_pathfinder_ext_add() + @see fsl_pathfinder_clear() + @see fsl_pathfinder_search() +*/ +struct fsl_pathfinder { + /** + Holds the list of search extensions. Each entry + is a (char *) owned by this object. + */ + fsl_list ext; + /** + Holds the list of search directories. Each entry is a (char *) + owned by this object. + */ + fsl_list dirs; + /** + Used to build up a path string during fsl_pathfinder_search(), + and holds the result of a successful search. We use a buffer, + as opposed to a simple string, because (A) it simplifies the + search implementation and (B) reduces allocations (it gets + reused for each search). + */ + fsl_buffer buf; +}; + +typedef struct fsl_pathfinder fsl_pathfinder; +/** + Initialized-with-defaults fsl_pathfinder instance, intended for + const copy initialization. +*/ +#define fsl_pathfinder_empty_m {\ + fsl_list_empty_m/*ext*/,\ + fsl_list_empty_m/*dirs*/,\ + fsl_buffer_empty_m/*buf*/} + +/** + Initialized-with-defaults fsl_pathfinder instance, intended for + copy initialization. +*/ +FSL_EXPORT const fsl_pathfinder fsl_pathfinder_empty; + +/** + Frees all memory associated with pf, but does not free pf. + Is a no-op if pf is NULL. +*/ +FSL_EXPORT void fsl_pathfinder_clear(fsl_pathfinder * const pf); + +/** + Adds the given directory to pf's search path. Returns 0 on + success, FSL_RC_MISUSE if !pf or !dir (dir _may_ be an empty + string), FSL_RC_OOM if copying the string or adding it to the + list fails. + + @see fsl_pathfinder_ext_add() + @see fsl_pathfinder_search() +*/ +FSL_EXPORT int fsl_pathfinder_dir_add(fsl_pathfinder * const pf, + char const * const dir); + +/** + Adds the given directory to pf's search extensions. Returns 0 on + success, FSL_RC_MISUSE if !pf or !dir (dir _may_ be an empty + string), FSL_RC_OOM if copying the string or adding it to the + list fails. + + Note that the client is responsible for adding a "." to the + extension, if needed, as this API does not apply any special + meaning to any characters in a search extension. e.g. "-journal" + and "~" are both perfectly valid extensions for this purpose. + + @see fsl_pathfinder_dir_add() + @see fsl_pathfinder_search() +*/ +FSL_EXPORT int fsl_pathfinder_ext_add(fsl_pathfinder * const pf, + char const * const ext); + +/** + Searches for a file whose name can be constructed by some + combination of pf's directory/suffix list and the given base + name. + + It searches for files in the following manner: + + If the 2nd parameter exists as-is in the filesystem, it is + treated as a match, otherwise... Loop over all directories + in pf->dirs. Create a path with DIR/base, or just base if + the dir entry is empty (length of 0). Check for a match. + If none is found, then... Loop over each extension in + pf->ext, creating a path named DIR/baseEXT (note that it + does not add any sort of separator between the base and the + extensions, so "~" and "-foo" are legal extensions). Check + for a match. + + On success (a readable filesystem entry is found): + + - It returns 0. + + - If pOut is not NULL then *pOut is set to the path it + found. The bytes of the returned string are only valid until the + next search operation on pf, so copy them if you need them. + Note that the returned path is _not_ normalized via + fsl_file_canonical_name() or similar, and it may very well + return a relative path (if base or one of pf->dirs contains a + relative path part). + + - If outLen is not NULL, *outLen will be set to the + length of the returned string. + + On error: + + - Returns FSL_RC_MISUSE if !pf, !base, !*base. + + - Returns FSL_RC_OOM on allocation error (it uses a buffer to + hold its path combinations and return value). + + - Returns FSL_RC_NOT_FOUND if it finds no entry. + + The host platform's customary path separator is used to separate + directory/file parts ('\\' on Windows and '/' everywhere else). + + Note that it _is_ legal for pOut and outLen to both be NULL, in + which case a return of 0 signals that an entry was found, but + the client has no way of knowing what path it might be (unless, + of course, he relies on internal details of the fsl_pathfinder + API, which he most certainly should not do). + + Tip: if the client wants to be certain that this function will + not allocate memory, simply use fsl_buffer_reserve() on pf->buf + to reserve the desired amount of space in advance. As long as + the search paths never extend that length, this function will + not need to allocate. (Until/unless the following TODO is + implemented...) + + Potential TODO: use fsl_file_canonical_name() so that the search + dirs themselves do not need to be entered using + platform-specific separators. The main reason it's not done now + is that it requires another allocation. The secondary reason is + because it's sometimes useful to use relative paths in this + context (based on usage in previous trees from which this code + derives). + + @see fsl_pathfinder_dir_add() + @see fsl_pathfinder_ext_add() + @see fsl_pathfinder_clear() +*/ +FSL_EXPORT int fsl_pathfinder_search(fsl_pathfinder * const pf, + char const * const base, + char const ** pOut, + fsl_size_t * const outLen ); + + +/** + A utility class for creating ZIP-format archives. All members + are internal details and must not be mucked about with by the + client. See fsl_zip_file_add() for an example of how to use it. + + Note that it creates ZIP content in memory, as opposed to + streaming it (it is not yet certain if abstractly streaming a + ZIP is possible), so creating a ZIP file this way is exceedingly + memory-hungry. + + @see fsl_zip_file_add() + @see fsl_zip_timestamp_set_julian() + @see fsl_zip_timestamp_set_unix() + @see fsl_zip_end() + @see fsl_zip_body() + @see fsl_zip_finalize() +*/ +struct fsl_zip_writer { + /** + Number of entries (files + dirs) added to the zip file so far. + */ + fsl_size_t entryCount; + /** + Current DOS-format time of the ZIP. + */ + int32_t dosTime; + /** + Current DOS-format date of the ZIP. + */ + int32_t dosDate; + /** + Current Unix Epoch time of the ZIP. + */ + fsl_time_t unixTime; + /** + An artificial root directory which gets prefixed + to all inserted filenames. + */ + char * rootDir; + /** + The buffer for the table of contents. + */ + fsl_buffer toc; + /** + The buffer for the ZIP file body. + */ + fsl_buffer body; + /** + Internal scratchpad for ops which often allocate + small buffers. + */ + fsl_buffer scratch; + /** + The current list of directory entries (as (char *)). + */ + fsl_list dirs; +}; +typedef struct fsl_zip_writer fsl_zip_writer; + +/** + An initialized-with-defaults fsl_zip_writer instance, intended + for in-struct or const-copy initialization. +*/ +#define fsl_zip_writer_empty_m { \ + 0/*entryCount*/, \ + 0/*dosTime*/, \ + 0/*dosDate*/, \ + 0/*unixTime*/, \ + NULL/*rootDir*/, \ + fsl_buffer_empty_m/*toc*/, \ + fsl_buffer_empty_m/*body*/, \ + fsl_buffer_empty_m/*scratch*/, \ + fsl_list_empty_m/*dirs*/ \ +} + +/** + An initialized-with-defaults fsl_zip_writer instance, + intended for copy-initialization. +*/ +FSL_EXPORT const fsl_zip_writer fsl_zip_writer_empty; + +/** + Sets a virtual root directory in z, such that all files added + with fsl_zip_file_add() will get this directory prefixed to + it. + + If zRoot is NULL or empty then this clears the virtual root, + otherwise is injects any directory levels it needs to into the + being-generated ZIP. Note that zRoot may contain multiple levels + of directories, e.g. "foo/bar/baz", but it must be legal for + use in a ZIP file. + + This routine copies zRoot's bytes, so they may be transient. + + Returns 0 on success, FSL_RC_MISUSE if !z, FSL_RC_OOM on + allocation error. Returns FSL_RC_RANGE if zRoot is an absolute + path or if zRoot cannot be normalized to a "simplified name" (as + per fsl_is_simple_pathname(), with the note that this routine + will pass a copy of zRoot through fsl_file_simplify_name() + first). + + @see fsl_zip_finalize() +*/ +FSL_EXPORT int fsl_zip_root_set(fsl_zip_writer * z, char const * zRoot ); + +/** + Adds a file or directory to the ZIP writer z. zFilename is the + virtual name of the file or directory. If pContent is NULL then + it is assumed that we are creating one or more directories, + otherwise the ZIP's entry is populated from pContent. The + permsFlag argument specifies the fossil-specific permission + flags from the fsl_fileperm_e enum, but currently ignores the + permsFlag argument for directories. Not that this function + creates directory entries for any files automatically, so there + is rarely a need for client code to create them (unless they + specifically want to ZIP an empty directory entry). + + Notes of potential interest: + + - The ZIP is created in memory, and thus creating ZIPs with this + API is exceedingly memory-hungry. + + - The timestamp of any given file must be set separately from + this call using fsl_zip_timestamp_set_unix() or + fsl_zip_timestamp_set_julian(). That value is then used for + subsequent file-adds until a new time is set. + + - If a root directory has been set using fsl_zip_root_set() then + that name, plus '/' (if the root does not end with one) gets + prepended to all files added via this routine. + + An example of the ZIP-generation process: + + ``` + int rc; + fsl_zip_writer z = fsl_zip_writer_empty; + fsl_buffer buf = fsl_buffer_empty; + fsl_buffer const * zipBody; + + // ...fill the buf buffer (not shown here)... + + // Optionally set a virtual root dir for new files: + rc = fsl_zip_root_set( &z, "myRootDir" ); // trailing slash is optional + if(rc) { ... error ...; goto end; } + + // We must set a timestamp which will be used until we set another: + fsl_zip_timestamp_set_unix( &z, time(NULL) ); + + // Add a file: + rc = fsl_zip_file_add( &z, "foo/bar.txt", &buf, FSL_FILE_PERM_REGULAR ); + // Clean up our content: + fsl_buffer_reuse(&buf); // only needed if we want to re-use the buffer's memory + if(rc) goto end; + + // ... add more files the same way (not shown) ... + + // Now "seal" the ZIP file: + rc = fsl_zip_end( &z ); + if(rc) goto end; + + // Fetch the ZIP content: + zipBody = fsl_zip_body( &z ); + // zipBody now points to zipBody->used bytes of ZIP file content + // which can be sent to an arbitrary destination, e.g.: + rc = fsl_buffer_to_filename( zipBody, "my.zip" ); + + end: + fsl_buffer_clear(&buf); + // VERY important, once we're done with z: + fsl_zip_finalize( &z ); + if(rc){...we had an error...} + ``` + + @see fsl_zip_timestamp_set_julian() + @see fsl_zip_timestamp_set_unix() + @see fsl_zip_end() + @see fsl_zip_body() + @see fsl_zip_finalize() +*/ +FSL_EXPORT int fsl_zip_file_add( fsl_zip_writer * z, char const * zFilename, + fsl_buffer const * pContent, int permsFlag ); + +/** + Ends the ZIP-creation process, padding all buffers, writing all + final required values, and freeing up most of the memory owned + by z. After calling this, z->body contains the full generated + ZIP file. + + Returns 0 on success. On error z's contents may still be + partially intact (for debugging purposes) and z->body will not + hold complete/valid ZIP file contents. Results are undefined if + !z or z has not been properly initialized. + + The caller must eventually pass z to fsl_zip_finalize() to free + up any remaining resources. + + @see fsl_zip_timestamp_set_julian() + @see fsl_zip_timestamp_set_unix() + @see fsl_zip_file_add() + @see fsl_zip_body() + @see fsl_zip_finalize() + @see fsl_zip_end_take() +*/ +FSL_EXPORT int fsl_zip_end( fsl_zip_writer * z ); + +/** + This variant of fsl_zip_end() transfers the current contents + of the zip's body to dest, replacing (freeing) any contents it may + hold when this is called, then passes z to fsl_zip_finalize() + to free any other resources (which are invalidated by the removal + of the body). + + Returns 0 on success, FSL_RC_MISUSE if either pointer is NULL, + some non-0 code if the proxied fsl_zip_end() call fails. On + error, the transfer of contents to dest does NOT take place, but + z is finalized (if it is not NULL) regardless of success or + failure (even if dest is NULL). i.e. on error z is still cleaned + up. +*/ +FSL_EXPORT int fsl_zip_end_take( fsl_zip_writer * z, fsl_buffer * dest ); + +/** + This variant of fsl_zip_end_take() passes z to fsl_zip_end(), + write's the ZIP body to the given filename, passes + z to fsl_zip_finalize(), and returns the result of + either end/save combination. Saving is not attempted + if ending the ZIP fails. + + On success 0 is returned and the contents of the ZIP are in the + given file. On error z is STILL cleaned up, and the file might + have been partially populated (only on I/O error after writing + started). In either case, z is cleaned up and ready for re-use or + (in the case of a heap-allocated instance) freed. +*/ +FSL_EXPORT int fsl_zip_end_to_filename( fsl_zip_writer * z, char const * filename ); + + +/** + Returns a pointer to z's ZIP content buffer. The contents + are ONLY valid after fsl_zip_end() returns 0. + + @see fsl_zip_timestamp_set_julian() + @see fsl_zip_timestamp_set_unix() + @see fsl_zip_file_add() + @see fsl_zip_end() + @see fsl_zip_end_take() + @see fsl_zip_finalize() +*/ +FSL_EXPORT fsl_buffer const * fsl_zip_body( fsl_zip_writer const * z ); + +/** + Frees all memory owned by z and resets it to a clean state, but + does not free z. Any fsl_zip_writer instance which has been + modified via the fsl_zip_xxx() family of functions MUST + eventually be passed to this function to clean up any contents + it might have accumulated during its life. After this returns, + z is legal for re-use in creating a new ZIP archive. + + @see fsl_zip_timestamp_set_julian() + @see fsl_zip_timestamp_set_unix() + @see fsl_zip_file_add() + @see fsl_zip_end() + @see fsl_zip_body() +*/ +FSL_EXPORT void fsl_zip_finalize(fsl_zip_writer * z); + +/** + Set z's date and time from a Julian Day number. Results are + undefined if !z. Results will be invalid if rDate is negative. The + timestamp is applied to all fsl_zip_file_add() operations until it + is re-set. + + @see fsl_zip_timestamp_set_unix() + @see fsl_zip_file_add() + @see fsl_zip_end() + @see fsl_zip_body() +*/ +FSL_EXPORT void fsl_zip_timestamp_set_julian(fsl_zip_writer *z, double rDate); + +/** + Set z's date and time from a Unix Epoch time. Results are + undefined if !z. Results will be invalid if rDate is negative. The + timestamp is applied to all fsl_zip_file_add() operations until it + is re-set. +*/ +FSL_EXPORT void fsl_zip_timestamp_set_unix(fsl_zip_writer *z, fsl_time_t epochTime); + +/** + State for the fsl_timer_xxx() family of functions. + + @see fsl_timer_start() + @see fsl_timer_reset() + @see fsl_timer_stop() +*/ +struct fsl_timer_state { + /** + The amount of time (microseconds) spent in "user space." + */ + uint64_t user; + /** + The amount of time (microseconds)spent in "kernel space." + */ + uint64_t system; +}; +typedef struct fsl_timer_state fsl_timer_state; + +/** + Initialized-with-defaults fsl_timer_state_empty instance, + intended for const copy initialization. +*/ +#define fsl_timer_state_empty_m {0,0} + +/** + Initialized-with-defaults fsl_timer_state_empty instance, + intended for copy initialization. +*/ +FSL_EXPORT const fsl_timer_state fsl_timer_state_empty; + +/** + Sets t's counter state to the current CPU timer usage, as + determined by the OS. + + Achtung: timer support is only enabled if the library is built + with the FSL_CONFIG_ENABLE_TIMER macro set to a true value (it is + on by default). + + @see fsl_timer_reset() + @see fsl_timer_stop() +*/ +FSL_EXPORT void fsl_timer_start(fsl_timer_state * const t); + +/** + Returns the difference in _CPU_ times in microseconds since t was + last passed to fsl_timer_start() or fsl_timer_reset(). It might + return 0 due to system-level precision restrictions. Note that this + is not useful for measuring wall times. +*/ +FSL_EXPORT uint64_t fsl_timer_fetch(fsl_timer_state const * const t); + +/** + Resets t to the current time and returns the number of microseconds + since t was started or last reset. + + @see fsl_timer_start() + @see fsl_timer_reset() +*/ +FSL_EXPORT uint64_t fsl_timer_reset(fsl_timer_state * const t); + +/** + Clears t's state and returns the difference (in uSec) between the + last time t was started or reset, as per fsl_timer_fetch(). + + @see fsl_timer_start() + @see fsl_timer_reset() +*/ +FSL_EXPORT uint64_t fsl_timer_stop(fsl_timer_state * const t); + +/** + For the given red/green/blue values (all in the range of 0 to + 255, or truncated to be so!) this function returns the RGB + encoded in the lower 24 bits of a single number. See + fsl_gradient_color() for an explanation and example. + + For those asking themselves, "why does an SCM API have a function + for encoding RGB colors?" the answer is: fossil(1) has a long + history of using HTML color codes to set the color of branches, + and this is provided in support of such features. + + @see fsl_rgb_decode() + @see fsl_gradient_color() +*/ +FSL_EXPORT unsigned int fsl_rgb_encode( int r, int g, int b ); + +/** + Given an RGB-encoded source value, this function decodes + the lower 24 bits into r, g, and b. Any of r, g, and b may + be NULL to skip over decoding of that part. + + @see fsl_rgb_encode() + @see fsl_gradient_color() +*/ +FSL_EXPORT void fsl_rgb_decode( unsigned int src, int *r, int *g, int *b ); + +/** + For two color values encoded as RRGGBB values (see below for the + structure), this function computes a gradient somewhere between + those colors. c1 and c2 are the edges of the gradient. + numberOfSteps is the number of steps in the gradient. stepNumber + is a number less than numberOfSteps which specifies the "degree" + of the gradients. If either numberOfSteps or stepNumber is 0, c1 + is returned. stepNumber of equal to or greater than c2 returns + c2. + + The returns value is an RGB-encoded value in the lower 24 bits, + ordered in big-endian. In other words, assuming rc is the return + value: + + - red = (rc&0xFF0000)>>16 + - green = (rc&0xFF00)>>8 + - blue = (rc&0xFF) + + Or use fsl_rgb_decode() to break it into its component parts. + + It can be passed directly to a printf-like function, using the + hex-integer format specifier, e.g.: + + ``` + fsl_buffer_appendf(&myBuf, "#%06x", rc); + ``` + + Tip: for a given HTML RRGGBB value, its C representation is + identical: HTML \#F0D0A0 is 0xF0D0A0 in C. + + @see fsl_rgb_encode() + @see fsl_rgb_decode() +*/ +FSL_EXPORT unsigned int fsl_gradient_color(unsigned int c1, unsigned int c2, + unsigned int numberOfSteps, + unsigned int stepNumber); + +/** + "Simplifies" an SQL string by making the following modifications + inline: + + - Consecutive non-newline spaces outside of an SQL string are + collapsed into one space. + + - Consecutive newlines outside of an SQL string are collapsed into + one space. + + Contents of SQL strings are not transformed in any way. + + len must be the length of the sql string. If it is negative, + fsl_strlen(sql) is used to calculate the length. + + Returns the number of bytes in the modified string (its strlen) and + NUL-terminates it at the new length. Thus the input string must be + at least one byte longer than its virtual length (its NUL + terminator byte suffices, provided it is NUL-terminated, as we can + safely overwrite that byte). + + If !sql or its length resolves to 0, this function returns 0 + without side effects. +*/ +FSL_EXPORT fsl_size_t fsl_simplify_sql( char * sql, fsl_int_t len ); + +/** + Convenience form of fsl_simplify_sql() which assumes b holds an SQL + string. It gets processed by fsl_simplify_sql() and its 'used' + length potentially gets adjusted to match the adjusted SQL string. +*/ +FSL_EXPORT fsl_size_t fsl_simplify_sql_buffer( fsl_buffer * b ); + + +/** + Returns the result of calling the platform's equivalent of + isatty(fd). e.g. on Windows this is _isatty() and on Unix + isatty(). i.e. it returns a true value (non-0) if it thinks that + the given file descriptor value is attached to an interactive + terminal, else it returns false. +*/ +FSL_EXPORT char fsl_isatty(int fd); + + +/** + + A container type for lists of db record IDs. This is used in + several places as a cache for record IDs, to keep track of ones + we know about, ones we know that we don't know about, and to + avoid duplicate processing in some contexts. +*/ +struct fsl_id_bag { + /** + Number of entries of this->list which are in use (have a + positive value). They need not be contiguous! Must be <= + capacity. + */ + fsl_size_t entryCount; + /** + The number of elements allocated for this->list. + */ + fsl_size_t capacity; + /** + The number of elements in this->list which have a zero or + positive value. Must be <= capacity. + */ + fsl_size_t used; + /** + Array of IDs this->capacity elements long. "Used" elements + have a positive value. Unused ones are set to 0. + */ + fsl_id_t * list; +}; + +/** + Initialized-with-defaults fsl_id_bag structure, + intended for copy initialization. +*/ +FSL_EXPORT const fsl_id_bag fsl_id_bag_empty; + +/** + Initialized-with-defaults fsl_id_bag structure, + intended for in-struct initialization. +*/ +#define fsl_id_bag_empty_m { \ + 0/*entryCount*/, 0/*capacity*/, \ + 0/*used*/, NULL/*list*/ } + +/** + + Return the number of elements in the bag. +*/ +FSL_EXPORT fsl_size_t fsl_id_bag_count(fsl_id_bag const * const p); + +/** + + Remove element e from the bag if it exists in the bag. If e is not + in the bag, this is a no-op. Returns true if it removes an element, + else false. + + e must be positive. Results are undefined if e<=0. +*/ +FSL_EXPORT bool fsl_id_bag_remove(fsl_id_bag * const p, fsl_id_t e); + +/** + Returns true if e is in the given bag. Returns false if it is + not. It is illegal to pass an e value of 0, and that will trigger + an assertion in debug builds. In non-debug builds, behaviour if + passed 0 is undefined. +*/ +FSL_EXPORT bool fsl_id_bag_contains(fsl_id_bag const * const p, fsl_id_t e); + +/** + Insert element e into the bag if it is not there already. Returns + 0 if it actually inserts something or if it already contains such + an entry, and some other value on error (namely FSL_RC_OOM on + allocation error). + + e must be positive or an assertion is triggered in debug builds. In + non-debug builds, behaviour if passed 0 is undefined. +*/ +FSL_EXPORT int fsl_id_bag_insert(fsl_id_bag * const p, fsl_id_t e); + +/** + Returns the ID of the first element in the bag. Returns 0 if the + bag is empty. + + Example usage: + + ``` + fsl_id_t nid; + for( nid = fsl_id_bag_first(&list); + nid > 0; + nid = fsl_id_bag_next(&list, nid)){ + ...do something... + } + ``` +*/ +FSL_EXPORT fsl_id_t fsl_id_bag_first(fsl_id_bag const * const p); + +/** + Returns the next element in the bag after e. Return 0 if e is + the last element in the bag. Any insert or removal from the bag + might reorder the bag. It is illegal to pass this 0 (and will + trigger an assertion in debug builds). For the first call, pass + it the non-0 return value from fsl_id_bag_first(). For + subsequent calls, pass the previous return value from this + function. + + @see fsl_id_bag_first() +*/ +FSL_EXPORT fsl_id_t fsl_id_bag_next(fsl_id_bag const * const p, fsl_id_t e); + +/** + Swaps the contents of the given bags. +*/ +FSL_EXPORT void fsl_id_bag_swap(fsl_id_bag * const lhs, fsl_id_bag * const rhs); + +/** + Frees any memory owned by p, but does not free p. +*/ +FSL_EXPORT void fsl_id_bag_clear(fsl_id_bag * const p); + +/** + Resets p's internal list, effectively emptying it for re-use, but + does not free its memory. Immediately after calling this + fsl_id_bag_count() will return 0. +*/ +FSL_EXPORT void fsl_id_bag_reset(fsl_id_bag * const p); + + +/** + Returns true if p contains a fossil-format merge conflict marker, + else returns false. + + @see fsl_buffer_merge3() +*/ +FSL_EXPORT bool fsl_buffer_contains_merge_marker(fsl_buffer const *p); + +/** + Performs a three-way merge. + + The merge is an edit against pV2. Both pV1 and pV2 have a common + origin at pPivot. Apply the changes of pPivot ==> pV1 to pV2, + appending them to pOut. (Pedantic side-note: the input buffers are + not const because we need to manipulate their cursors, but their + buffered memory is not modified.) + + If merge conflicts are encountered, it continues as best as it can + and injects "indiscrete" markers in the output to denote the nature + of each conflict. If conflictCount is not NULL then on success the + number of merge conflicts is written to *conflictCount. + + Returns 0 on success, FSL_RC_OOM on OOM, FSL_RC_TYPE if any input + appears to be binary. + + @see fsl_buffer_contains_merge_marker() +*/ +FSL_EXPORT int fsl_buffer_merge3(fsl_buffer * const pPivot, + fsl_buffer * const pV1, + fsl_buffer * const pV2, + fsl_buffer * const pOut, + unsigned int * const conflictCount); + +/** + Appends the first n bytes of string z to buffer b in the form of + TCL-format string literal. If n<0 then fsl_strlen() is used to + determine the length. Returns 0 on success, FSL_RC_OOM on error. + */ +FSL_EXPORT int fsl_buffer_append_tcl_literal(fsl_buffer * const b, + char const * z, fsl_int_t n); + + +/** + Event IDs for use with fsl_confirm_callback_f implementations. + + The idea here is to send, via callback, events from the library to + the client when a potentially interactive response is necessary. + We define a bare minimum of information needed for the client to + prompt a user for a response. To that end, the interface passes on + 2 pieces of information to the client: the event ID and a filename. + It is up to the application to translate that ID into a + user-readable form, get a response (using a well-defined set of + response IDs), and convey that back to the + library via the callback's result pointer interface. + + This enum will be extended as the library develops new requirements + for interactive use. + + @see fsl_confirm_response_e +*/ +enum fsl_confirm_event_e { +/** + Sentinal value. +*/ +FSL_CEVENT_INVALID = 0, +/** + An operation requests permission to overwrite a locally-modified + file. e.g. when performing a checkout over a locally-edited + version. Overwrites of files which are known to be in the previous + (being-overwritten) checkout version are automatically overwritten. +*/ +FSL_CEVENT_OVERWRITE_MOD_FILE = 1, +/** + An operation requests permission to overwrite an SCM-unmanaged file + with one which is managed by SCM. This can happen, e.g., when + switching from a version which did not contain file X, but had file + X on disk, to a version which contains file X. +*/ +FSL_CEVENT_OVERWRITE_UNMGD_FILE = 2, +/** + An operation requests permission to remove a LOCALLY-MODIFIED file + which has been removed from SCM management. e.g. when performing a + checkout over a locally-edited version and an edited file was + removed from the SCM somewhere between those two versions. + UMODIFIED files which are removed from the SCM between two + checkouts are automatically removed on the grounds that it poses no + data loss risk because the other version is "somewhere" in the SCM. +*/ +FSL_CEVENT_RM_MOD_UNMGD_FILE = 3, + +/** + Indicates that the library cannot determine which of multiple + potential versions to choose from and requires the user to + select one. +*/ +FSL_CEVENT_MULTIPLE_VERSIONS = 4 + +}; +typedef enum fsl_confirm_event_e fsl_confirm_event_e; + +/** + Answers to questions posed to clients via the + fsl_confirm_callback_f() interface. + + This enum will be extended as the library develops new requirements + for interactive use. + + @see fsl_confirm_event_e +*/ +enum fsl_confirm_response_e { +/** + Sentinel/default value - not a valid answer. Guaranteed to have a + value of 0. No other entries in this enum are guaranteed to have + well-known/stable values: always use the enum symbols instead of + integer values. +*/ +FSL_CRESPONSE_INVALID = 0, +/** + Accept the current event and continue processes. +*/ +FSL_CRESPONSE_YES = 1, +/** + Reject the current event and continue processes. +*/ +FSL_CRESPONSE_NO = 2, +/** + Reject the current event and stop processesing. Cancellation is + generally considered to be a recoverable error. +*/ +FSL_CRESPONSE_CANCEL = 3, +/** + Accept the current event and all identical event types for the + current invocation of this particular SCM operation. +*/ +FSL_CRESPONSE_ALWAYS = 5, +/** + Reject the current event and all identical event types for the + current invocation of this particular SCM operation. +*/ +FSL_CRESPONSE_NEVER = 6, +/** + For events which are documented as being multiple-choice, + this answer indicates that the client has set the index of + their choice in the fsl_confirm_response::multipleChoice + field: + + - FSL_CEVENT_MULTIPLE_VERSIONS +*/ +FSL_CRESPONSE_MULTI = 7 +}; +typedef enum fsl_confirm_response_e fsl_confirm_response_e; + +/** + A response for use with the fsl_confirmer API. It is intended to + encapsulate, with a great deal of abstraction, answers to typical + questions which the library may need to interactively query a user + for. e.g. confirmation about whether to overwrite a file or which + one of 3 versions to select. + + This type will be extended as the library develops new requirements + for interactive use. +*/ +struct fsl_confirm_response { + /** + Client response to the current fsl_confirmer question. + */ + fsl_confirm_response_e response; + /** + If this->response is FSL_CRESPONSE_MULTI then this must be set to + the index of the client's multiple-choice answer. + + Events which except this in their response: + + - FSL_CEVENT_MULTIPLE_VERSIONS + */ + uint16_t multipleChoice; +}; +/** + Convenience typedef. +*/ +typedef struct fsl_confirm_response fsl_confirm_response; + +/** + Empty-initialized fsl_confirm_detail instance to be used for + const copy initialization. +*/ +#define fsl_confirm_response_empty_m {FSL_CRESPONSE_INVALID, -1} + +/** + Empty-initialized fsl_confirm_detail instance to be used for + non-const copy initialization. +*/ +FSL_EXPORT const fsl_confirm_response fsl_confirm_response_empty; + +/** + A struct for passing on interactive questions to + fsl_confirmer_callback_f implementations. +*/ +struct fsl_confirm_detail { + /** + The message ID of this confirmation request. This value + determines how the rest of this struct's values are to + be interpreted. + */ + fsl_confirm_event_e eventId; + /** + Depending on the eventId, this might be NULL or might refer to a + filename. This will be a filename for following confirmations + events: + + - FSL_CEVENT_OVERWRITE_MOD_FILE + - FSL_CEVENT_OVERWRITE_UNMGD_FILE + - FSL_CEVENT_RM_MOD_UNMGD_FILE + + For all others it will be NULL. + + Whether this name refers to an absolute or relative path is + context-dependent, and not specified by this API. In general, + relative paths should be used if/when what they are relative to + (e.g. a checkout root) is/should be clear to the user. The intent + is that applications can display that name to the user in a UI + control, so absolute paths "should" "generally" be avoided + because they can be arbitrarily long. + */ + const char * filename; + /** + Depending on the eventId, this might be NULL or might + refer to a list of details of a type specified in the + documentation for that eventId. + + Implementation of such an event is still TODO, but we have at + least one use case lined up (asking a user which of several + versions is intended when the checkout-update operation is given + an ambiguous hash prefix). + + Events for which this list will be populated: + + - FSL_CEVENT_MULTIPLE_VERSIONS: each list entry will be a (char + const*) with a version number, branch name, or similar, perhaps + with relevant metadata such as a checkin timestamp. The client is + expected to pick one answer, set its list index to the + fsl_confirm_response::multipleChoice member, and to set + fsl_confirm_response::response to FSL_CRESPONSE_MULTI. + + In all cases, a response of FSL_CRESPONSE_CANCEL will trigger a + cancellation. + + In all cases, the memory for the items in this list is owned by + (or temporarily operated on the behalf of) the routine which has + launched this query. fsl_confirm_callback_f implements must never + manipulate the list's or its content's state. + */ + const fsl_list * multi; +}; +typedef struct fsl_confirm_detail fsl_confirm_detail; + +/** + Empty-initialized fsl_confirm_detail instance to be used for + const-copy initialization. +*/ +#define fsl_confirm_detail_empty_m \ + {FSL_CEVENT_INVALID, NULL, NULL} + +/** + Empty-initialized fsl_confirm_detail instance to be used for + non-const-copy initialization. +*/ +FSL_EXPORT const fsl_confirm_detail fsl_confirm_detail_empty; + +/** + Should present the user (if appropriate) with an option of how to + handle the given event write that answer to + outAnswer->response. Return 0 on success, non-0 on error, in which + case the current operation will fail with that result code. + Answering with FSL_CRESPONSE_CANCEL is also considered failure but + recoverably so, whereas a non-cancel failure is considered + unrecoverable. +*/ +typedef int (*fsl_confirm_callback_f)(fsl_confirm_detail const * detail, + fsl_confirm_response *outAnswer, + void * confirmerState); + +/** + A fsl_confirm_callback_f and its callback state, packaged into a + neat little struct for easy copy/replace/restore of confirmers. +*/ +struct fsl_confirmer { + /** + Callback which can be used for basic interactive confirmation + purposes, within the very libfossil-centric limits of the + interface. + */ + fsl_confirm_callback_f callback; + /** + Opaque state pointer for this->callback. Its lifetime is not + managed by this object and it is assumed, if not NULL, to live at + least as long as this object. + */ + void * callbackState; +}; +typedef struct fsl_confirmer fsl_confirmer; +/** Empty-initialized fsl_confirmer instance for const-copy + initialization. */ +#define fsl_confirmer_empty_m {NULL,NULL} +/** Empty-initialized fsl_confirmer instance for non-const-copy + initialization. */ +FSL_EXPORT const fsl_confirmer fsl_confirmer_empty; + +/** + State for use with fsl_dircrawl_f() callbacks. + + @see fsl_dircrawl() +*/ +struct fsl_dircrawl_state { + /** + Absolute directory name of the being-visited directory. + */ + char const *absoluteDir; + /** + Name (no path part) of the entry being visited. + */ + char const *entryName; + /** + Filesystem entry type. + */ + fsl_fstat_type_e entryType; + /** + Opaque client-specified pointer which was passed to + fsl_dircrawl(). + */ + void * callbackState; + + /** + Directory depth of the crawl process, starting at 1 with + the directory passed to fsl_dircrawl(). + */ + unsigned int depth; +}; +typedef struct fsl_dircrawl_state fsl_dircrawl_state; + +/** + Callback type for use with fsl_dircrawl(). It gets passed the + absolute name of the target directory, the name of the directory + entry (no path part), the type of the entry, and the client state + pointer which is passed to that routine. It must return 0 on + success or another FSL_RC_xxx value on error. Returning + FSL_RC_BREAK will cause directory-crawling to stop without an + error. + + All pointers in the state argument are owned by fsl_dircrawl() and + will be invalidated as soon as the callback returns, thus they must + be copied if they are needed for later. +*/ +typedef int (*fsl_dircrawl_f)(fsl_dircrawl_state const *); + +/** + Recurses into a directory and calls a callback for each filesystem + entry. It does not change working directories, but callbacks are + free to do so as long as they restore the working directory before + returning. + + The first argument is the name of the directory to crawl. In order + to avoid any dependence on a specific working directory, if it is + not an absolute path then this function will expand it to an + absolute path before crawling begins. For each entry under the + given directory, it calls the given callback, passing it a + fsl_dircrawl_state object holding various state. All pointers in + that object, except for the callbackState pointer, are owned by + this function and may be invalidated as soon as the callback + returns. + + For each directory entry, it recurses into that directory, + depth-first _after_ passing it to the callback. + + It DOES NOT resolve/follow symlinks, instead passing them on to the + callback for processing. Note that passing a symlink to this + function will not work because this function does not resolve + symlinks. Thus it provides no way to traverse symlinks, as its + scope is only features suited for the SCM and symlinks have no + business being in an SCM. (Fossil supports symlinks, more or less, + but libfossil does not.) + + It silently skips any files for which stat() fails or is not of a + "basic" file type (e.g. character devices and such). + + Returns 0 on success, FSL_RC_TYPE if the given name is not a + directory, and FSL_RC_RANGE if it recurses "too deep," (some + "reasonable" internally hard-coded limit), in order to prevent a + stack overflow. + + If the callback returns non-0, iteration stops and returns that + result code unless the result is FSL_RC_BREAK, which stops + iteration but causes 0 to be returned from this function. +*/ +FSL_EXPORT int fsl_dircrawl(char const * dirName, fsl_dircrawl_f callback, + void * callbackState); + +/** + Strips any trailing slashes ('/') from the given string by + assigning those bytes to NUL and returns the number of slashes + NUL'd out. nameLen must be the length of the string. If nameLen is + negative, fsl_strlen() is used to calculate its length. +*/ +FSL_EXPORT fsl_size_t fsl_strip_trailing_slashes(char * name, fsl_int_t nameLen); + +/** + A convenience from of fsl_strip_trailing_slashes() which strips + trailing slashes from the given buffer and changes its b->used + value to account for any stripping. Results are undefined if b is + not properly initialized. +*/ +FSL_EXPORT void fsl_buffer_strip_slashes(fsl_buffer * b); + +/** + Appends each ID from the given bag to the given buffer using the given + separator string. Returns FSL_RC_OOM on allocation error. +*/ +FSL_EXPORT int fsl_id_bag_to_buffer(fsl_id_bag const * bag, fsl_buffer * b, + char const * separator); + +/** + Flags for use with the fsl_looks family of functions. +*/ +enum fsl_lookslike_e { +/* Nothing special was found. */ +FSL_LOOKSLIKE_NONE = 0x00000000, +/* One or more NUL chars were found. */ +FSL_LOOKSLIKE_NUL = 0x00000001, +/* One or more CR chars were found. */ +FSL_LOOKSLIKE_CR = 0x00000002, +/* An unpaired CR char was found. */ +FSL_LOOKSLIKE_LONE_CR = 0x00000004, +/* One or more LF chars were found. */ +FSL_LOOKSLIKE_LF = 0x00000008, +/* An unpaired LF char was found. */ +FSL_LOOKSLIKE_LONE_LF = 0x00000010, +/* One or more CR/LF pairs were found. */ +FSL_LOOKSLIKE_CRLF = 0x00000020, +/* An over length line was found. */ +FSL_LOOKSLIKE_LONG = 0x00000040, +/* An odd number of bytes was found. */ +FSL_LOOKSLIKE_ODD = 0x00000080, +/* Unable to perform full check. */ +FSL_LOOKSLIKE_SHORT = 0x00000100, +/* Invalid sequence was found. */ +FSL_LOOKSLIKE_INVALID = 0x00000200, +/* Might be binary. */ +FSL_LOOKSLIKE_BINARY = FSL_LOOKSLIKE_NUL | FSL_LOOKSLIKE_LONG | FSL_LOOKSLIKE_SHORT, + /* Line separators. */ +FSL_LOOKSLIKE_EOL = (FSL_LOOKSLIKE_LONE_CR | FSL_LOOKSLIKE_LONE_LF | FSL_LOOKSLIKE_CRLF) +}; + +/** + Returns true if b appears to contain "binary" (non-UTF8/16) content, + else returns true. +*/ +FSL_EXPORT bool fsl_looks_like_binary(fsl_buffer const * const b); + +/** + If b appears to contain any non-UTF8 content, returns a truthy + value: one or more values from the fsl_lookslike_e enum indicating + which sort of data was seen which triggered its conclusion: + + - FSL_LOOKSLIKE_BINARY means the content appears to be binary + because it contains embedded NUL bytes or an "extremely long" + line. This function may diagnose UTF-16 as binary. + + - !FSL_LOOKSLIKE_BINARY means the content is non-binary but may + not necessarily be valid UTF-8. + + - 0 means the contents appear to be valid UTF-8. + + It b's content is empty, returns 0. + + The 2nd argument can be a mask of any values from fsl_lookslike_e + and will cause this routine to stop inspecting the input once it + encounters any content described by those flags. + + The contents are examined until the end is reached or a condition + described by the 2nd parameter's flags is encountered. + + WARNINGS: + + - This function does not validate that the blob content is properly + formed UTF-8. It assumes that all code points are the same size. + It does not validate any code points. It makes no attempt to + detect if any [invalid] switches between UTF-8 and other + encodings occur. + + - The only code points that this function cares about are the NUL + character, carriage-return, and line-feed. +*/ +FSL_EXPORT int fsl_looks_like_utf8(fsl_buffer const * const b, int stopFlags); + +/** + Returns true if b's contents appear to contain anything other than valid + UTF8. + + It uses the approach described at: + + http://en.wikipedia.org/wiki/UTF-8#Invalid_byte_sequences + + except for the "overlong form" of \\u0000 which is not considered + invalid here: Some languages, like Java and Tcl, use it. This function + also considers valid the derivatives CESU-8 & WTF-8. +*/ +FSL_EXPORT bool fsl_invalid_utf8(fsl_buffer const * const b); + +/** + Returns a static pointer to bytes of a UTF8 BOM. If the argument is + not NULL, it is set to the strlen of those bytes (always 3). +*/ +FSL_EXPORT unsigned char const *fsl_utf8_bom(unsigned int *pnByte); + +/** + Returns true if b starts with a UTF8 BOM. If the 2nd argument is + not NULL, *pBomSize is set to the number of bytes in the BOM + (always 3), regardless of whether or not the function returns true + or false. +*/ +FSL_EXPORT bool fsl_starts_with_bom_utf8(fsl_buffer const * const b, unsigned int *pBomSize); + +#if 0 +/** + The UTF16 counterpart of fsl_looks_like_utf8(), with the addition that the + 2nd argument, if true, specifies that the 2nd argument is true then + the contents of the buffer are byte-swapped for checking purposes. + + This is not validate that the blob is valid UTF16. It assumes that all + code points are the same size and does not validate any of them, nor does + it attempt to detect (invalid) switches between big-endian and little-endian. + */ +FSL_EXPORT int fsl_looks_like_utf16(fsl_buffer const * const pContent, + bool reverse, int stopFlags); + +/** + Returns true if b'c contents begin with a UTF-16 BOM. If pBomSize is not NULL + then it is set to the byte length of the UTF-16 BOM (2 bytes). If isReversed + is not NULL, it gets assigned to true if the input appears to be in little-endian + format and false if it appears to be in big-endian format. +*/ +FSL_EXPORT bool fsl_starts_with_bom_utf16(fsl_buffer const * const b, unsigned int *pBomSize, + bool * isReversed); +/** + Returns true if b's content _might_ be valid UTF-16. If the 2nd argument is not NULL, + it gets set to true if the input appears to be little-endian and false if it appears + to be big-endian. +*/ +FSL_EXPORT bool fsl_might_be_utf16(fsl_buffer const * const b, bool * isReversed); +#endif +/* ^^^^ UTF-16 interfaces */ + +#if defined(__cplusplus) +} /*extern "C"*/ +#endif +#endif +/* ORG_FOSSIL_SCM_FSL_UTIL_H_INCLUDED */ +/* end of file ../include/fossil-scm/fossil-util.h */ +/* start of file ../include/fossil-scm/fossil-core.h */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#if !defined(ORG_FOSSIL_SCM_FSL_CORE_H_INCLUDED) +#define ORG_FOSSIL_SCM_FSL_CORE_H_INCLUDED +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ + +/** @file fossil-core.h + + This file declares the core SCM-related public APIs. +*/ + +#include /* struct tm, time_t */ +#if defined(__cplusplus) +/** + The fsl namespace is reserved for an eventual C++ wrapper for the API. +*/ +namespace fsl {} +extern "C" { +#endif + +/** + @struct fsl_cx + + The main Fossil "context" type. This is the first argument to + many Fossil library API routines, and holds all state related + to a checkout and/or repository and/or global fossil configuration + database(s). + + An instance's lifetime looks something like this: + + ``` + int rc; + fsl_cx * f = NULL; // ALWAYS initialize to NULL or results are undefined + rc = fsl_cx_init( &f, NULL ); + assert(!rc); + rc = fsl_repo_open( f, "myrepo.fsl" ); + ...use the context, and clean up when done... + fsl_cx_finalize(f); + ``` + + The contents of an fsl_cx instance are strictly private, for use + only by APIs in this library. Any client-side dependencies on + them will lead to undefined behaviour at some point. + + Design note: this type is currently opaque to client code. Having + it non-opaque also has advantages, though, and i'd generally prefer + that (to allow client-side allocation and embedding in other + structs). Binary compatibility concerns might eventually (once the + API is declared "relatively stable") force us to keep it opaque. +*/ +typedef struct fsl_cx fsl_cx; +typedef struct fsl_cx_config fsl_cx_config; +typedef struct fsl_db fsl_db; +typedef struct fsl_cx_init_opt fsl_cx_init_opt; +typedef struct fsl_stmt fsl_stmt; + +/** + This enum defines type ID tags with which the API tags fsl_db + instances so that the library can figure out which DB is + which. This is primarily important for certain queries, which + need to know whether they are accessing the repo or config db, + for example. + + As of 2021-10, the fossil context db model has long been, and is + very, very likely to remain: + + - When a context starts up, it creates a temporary db for use + as its `main` db. + + - When a repo or checkout db are opened, they are `ATTACH`ed to the + main db with the names `repo` resp. `ckout`. (Note that those + differ from the names fossil(1) uses: `repository` and + `localdb`.) + + A large amount of the library code is written _as if_ the library + (roughly) followed fossil's model of using separate sqlite3 + objects, even though it does not do so. The reason is historical: + when starting the libfossil port, it was clear that fossil's + internal connection juggling was going to be painful for the + library, so the internal API was shaped to permit two separate + approaches: + + Approach 1: the first db which gets opened (repo, checkout, or + config) becomes the `main` db and all others get attached using + their API-standard alias. That is _mostly_ what fossil does, but it + leads to fossil sometimes having to "swap" connections by closing + the db and re-opening it so that the db names are in a state which + fits the current use. That model led to the creation of this + enum. However, it also leads to unpredictable database names for + purposes of queries, as we're never sure (at the client level or + higher library-level APIs) which db is `main` and which has an + alias. + + Approach 2: open a temp (or in-memory) db and attach all of the + others to that using well-defined names. + + The latter has proven to work well enough that returning to the + first approach seems _extremely_ unlikely at this point. Thus some + newer library-level code behaves as if the latter model is in effect + (which it is). + + The long and the short of the above diversion is that APIs like + fsl_cx_db_repo() and fsl_cx_db_ckout() will always return the same + database handle, but they can tell whether a given role has been + attached or not and will fail in their documented ways when the + role's corresponding database has not yet been attached. e.g. + `fsl_cx_db_repo()` will return `NULL` if a repo database is not + attached. + + @see fsl_db_role_label() + @see fsl_cx_db_name_for_role() + @see fsl_cx_db_file_for_role() +*/ +enum fsl_dbrole_e { +/** + Sentinel "no role" value. +*/ +FSL_DBROLE_NONE = 0, +/** + The global (per user) config db. Analog to fossil's "configdb". +*/ +FSL_DBROLE_CONFIG = 0x01, +/** + The repository db. Analog to fossil's "repository". +*/ +FSL_DBROLE_REPO = 0x02, +/** + The checkout db. Analog to fossil's "localdb". +*/ +FSL_DBROLE_CKOUT = 0x04, +/** + Analog to fossil's "main", which is an alias for the first db + opened. In this API a fsl_cx instance has a temporary/transient + main db opened even if it does not have a repository, checkout, or + config db opened. +*/ +FSL_DBROLE_MAIN = 0x08, +/** + Refers to the "temp" database. This is only used by a very few APIs + and is outright invalid for most. +*/ +FSL_DBROLE_TEMP = 0x10 +}; +typedef enum fsl_dbrole_e fsl_dbrole_e; + +/** + Bitmask values specifying "configuration sets". The values in + this enum come directly from fossil(1), but they are not part of + the db structure, so may be changed over time. + + It seems very unlikely that these will ever be used at the level of + this library. They are a "porting artifact" and retained for the + time being, but will very likely be removed. +*/ +enum fsl_configset_e { +/** Sentinel value. */ +FSL_CONFSET_NONE = 0x000000, +/** Style sheet only */ +FSL_CONFIGSET_CSS = 0x000001, +/** WWW interface appearance */ +FSL_CONFIGSET_SKIN = 0x000002, +/** Ticket configuration */ +FSL_CONFIGSET_TKT = 0x000004, +/** Project name */ +FSL_CONFIGSET_PROJ = 0x000008, +/** Shun settings */ +FSL_CONFIGSET_SHUN = 0x000010, +/** The USER table */ +FSL_CONFIGSET_USER = 0x000020, +/** The CONCEALED table */ +FSL_CONFIGSET_ADDR = 0x000040, +/** Transfer configuration */ +FSL_CONFIGSET_XFER = 0x000080, +/** Everything */ +FSL_CONFIGSET_ALL = 0x0000ff, +/** Causes overwrite instead of merge */ +FSL_CONFIGSET_OVERWRITE = 0x100000, +/** Use the legacy format */ +FSL_CONFIGSET_OLDFORMAT = 0x200000 +}; +typedef enum fsl_configset_e fsl_configset_e; + +/** + Runtime-configurable flags for a fsl_cx instance. +*/ +enum fsl_cx_flags_e { +/** + The "no flags" value. Guaranteed to be 0 and this is the only entry + in this enum which is guaranteed to have a stable value. +*/ +FSL_CX_F_NONE = 0, +/** + Tells us whether or not we want to calculate R-cards by default. + Historically they were initially required but eventually made + optional due largely to their memory costs. +*/ +FSL_CX_F_CALC_R_CARD = 0x01, + +/** + When encounting artifact types in the crosslinking phase which + the library does not currently support crosslinking for, skip over + them instead of generating an error. For day-to-day use this is, + perhaps counter-intuitively, generally desirable. +*/ +FSL_CX_F_SKIP_UNKNOWN_CROSSLINKS = 0x02, + +/** + By default, fsl_reserved_fn_check() will fail if the given filename + is reserved on Windows platforms because such filenames cannot be + checked out on Windows. This flag removes that limitation. It + should only be used, if at all, for repositories which will _never_ + be used on Windows. +*/ +FSL_CX_F_ALLOW_WINDOWS_RESERVED_NAMES = 0x04, + +/** + If on (the default) then an internal cache will be used for + artifact loading to speed up operations which do lots of that. + Disabling this will save memory but will hurt performance badly for + certain operations. +*/ +FSL_CX_F_MANIFEST_CACHE = 0x08, + +/** + Internal use only to prevent duplicate initialization of some + bits. +*/ +FSL_CX_F_IS_OPENING_CKOUT = 0x100, + +/** + Default flags for all fsl_cx instances. +*/ +FSL_CX_F_DEFAULTS = FSL_CX_F_MANIFEST_CACHE + +}; +typedef enum fsl_cx_flags_e fsl_cx_flags_e; + +/** + List of hash policy values. New repositories should generally use + only SHA3 hashes, but older repos may contain SHA1 hashes (perhaps + only SHA1), so we have to support those. Repositories may contain + a mix of hash types. + + Maintenance ACHTUNG: this enum's values must align with those from + fossil(1) because their integer values are used in the + `repo.config` table. +*/ +enum fsl_hashpolicy_e { +/* Use only SHA1 hashes. */ +FSL_HPOLICY_SHA1 = 0, +/* Accept SHA1 hashes but auto-promote to SHA3. */ +FSL_HPOLICY_AUTO = 1, +/* Use SHA3 hashes. */ +FSL_HPOLICY_SHA3 = 2, +/* Use SHA3 hashes exclusively. */ +FSL_HPOLICY_SHA3_ONLY = 3, +/* With this policy, fsl_uuid_is_shunned() will always return true for + SHA1 hashes. */ +FSL_HPOLICY_SHUN_SHA1 = 4 +}; +typedef enum fsl_hashpolicy_e fsl_hashpolicy_e; + +/** + Most functions in this API which return an int type return error + codes from the fsl_rc_e enum. None of these entries are + (currently) guaranteed to have a specific value across library + versions except for FSL_RC_OK, which is guaranteed to always be + 0 (and the API guarantees that no other code shall have a value + of zero). + + The only reasons numbers are hard-coded to the values (or some of + them) is to simplify debugging during development. Clients may use + fsl_rc_cstr() to get some human-readable (or programmer-readable) + form for any given value in this enum. + + Maintenance reminder: as entries are added/changed, update + fsl_rc_cstr(). +*/ +enum fsl_rc_e { +/** + The quintessential not-an-error value. +*/ +FSL_RC_OK = 0, +/** + Generic/unknown error. +*/ +FSL_RC_ERROR = 100, +/** + A placeholder return value for "not yet implemented" functions. +*/ +FSL_RC_NYI = 101, +/** + Out of memory. Indicates that a resource allocation request + failed. +*/ +FSL_RC_OOM = 102, +/* + API misuse (invalid args) +*/ +FSL_RC_MISUSE = 103, +/** + Some range was violated (function argument, UTF character, etc.). +*/ +FSL_RC_RANGE = 104, +/** + Indicates that access to or locking of a resource was denied + by some security mechanism or other. +*/ +FSL_RC_ACCESS = 105, +/** + Indicates an I/O error. Whether it was reading or writing is + context-dependent. +*/ +FSL_RC_IO = 106, +/** + requested resource not found +*/ +FSL_RC_NOT_FOUND = 107, +/** + Indicates that a to-be-created resource already exists. +*/ +FSL_RC_ALREADY_EXISTS = 108, +/** + Data consistency problem +*/ +FSL_RC_CONSISTENCY = 109, + +/** + Indicates that the requested repo needs to be rebuilt. +*/ +FSL_RC_REPO_NEEDS_REBUILD = 110, + +/** + Indicates that the requested repo is not, in fact, a repo. Also + used by some APIs to indicate that they require a repository db + but none has been opened. +*/ +FSL_RC_NOT_A_REPO = 111, + +/** + Indicates an attempt to open a too-old or too-new repository db. +*/ +FSL_RC_REPO_VERSION = 112, + +/** + Indicates db-level error (e.g. statement prep failed). In such + cases, the error state of the related db handle (fsl_db) or + Fossilc context (fsl_cx) will be updated to contain more + information directly from the db driver. +*/ +FSL_RC_DB = 113, + +/** + Used by some iteration routines to indicate that iteration should + stop prematurely without an error. +*/ +FSL_RC_BREAK = 114, + +/** + Indicates that fsl_stmt_step() has fetched a row and the cursor + may be used to access the current row state (e.g. using + fsl_stmt_get_int32() and friends). It is strictly illegal to use + the fsl_stmt_get_xxx() APIs unless fsl_stmt_step() has returned + this code. +*/ +FSL_RC_STEP_ROW = 115, + +/** + Indicates that fsl_stmt_step() has reached the end of the result + set and that there is no row data to process. This is also the + result for non-fetching queries (INSERT and friends). It is strictly + illegal to use the fsl_stmt_get_xxx() APIs after fsl_stmt_step() has + returned this code. +*/ +FSL_RC_STEP_DONE = 116, + +/** + Indicates that a db-level error occurred during a + fsl_stmt_step() iteration. +*/ +FSL_RC_STEP_ERROR = 117, + +/** + Indicates that some data type or logical type is incorrect + (e.g. an invalid card type in conjunction with a given + fsl_deck). +*/ +FSL_RC_TYPE = 118, + +/** + Indicates that an operation which requires a checkout does not + have a checkout to work on. +*/ +FSL_RC_NOT_A_CKOUT = 119, + +/** + Indicates that a repo and checkout do not belong together. +*/ +FSL_RC_REPO_MISMATCH = 120, +/** + Indicates that a checksum comparison failed, possibly indicating + that corrupted or unexpected data was just read. +*/ +FSL_RC_CHECKSUM_MISMATCH = 121, + +/** + Indicates that a merge conflict, or some other context-dependent + type of conflict, was detected. +*/ +FSL_RC_CONFLICT, + +/** + This is a special case of FSL_RC_NOT_FOUND, intended specifically + to differentiate from "file not found in filesystem" + (FSL_RC_NOT_FOUND) and "fossil does not know about this file" in + routines for which both might be an error case. An example is a + an operation which wants to update a repo file with contents + from the filesystem - the file might not exist or it might not be + in the current repo db. + + That said, this can also be used for APIs which search for other + resources (UUIDs, tickets, etc.), but FSL_RC_NOT_FOUND is already + fairly well entrenched in those cases and is unambiguous, so this + code is only needed by APIs for which both cases described above + might happen. +*/ +FSL_RC_UNKNOWN_RESOURCE, + +/** + Indicates that a size comparison check failed. + + TODO: remove this if it is not used. +*/ +FSL_RC_SIZE_MISMATCH, + +/** + Indicates that an invalid separator was encountered while + parsing a delta. +*/ +FSL_RC_DELTA_INVALID_SEPARATOR, + +/** + Indicates that an invalid size value was encountered while + parsing a delta. +*/ +FSL_RC_DELTA_INVALID_SIZE, + +/** + Indicates that an invalid operator was encountered while parsing + a delta. +*/ +FSL_RC_DELTA_INVALID_OPERATOR, + +/** + Indicates that an invalid terminator was encountered while + parsing a delta. +*/ +FSL_RC_DELTA_INVALID_TERMINATOR, + +/** + Indicates a generic syntax error in a structural artifact. Some + types of manifest-releated errors are reported with more specific + error codes, e.g. FSL_RC_RANGE if a given card type appears too + often. +*/ +FSL_RC_SYNTAX, + +/** + Indicates that some value or expression is ambiguous. Typically + caused by trying to resolve ambiguous symbolic names or hash + prefixes to their full hashes. +*/ +FSL_RC_AMBIGUOUS, + +/** + Used by fsl_checkin_commit(), and similar operations, to indicate + that they're failing because they would be no-ops. That would + normally indicate a "non-error," but a condition the caller + certainly needs to know about. +*/ +FSL_RC_NOOP, +/** + A special case of FSL_RC_NOT_FOUND which indicates that the + requested repository blob could not be loaded because it is a + phantom. That is, the record is found but its contents are not + available. Phantoms are blobs which fossil knows should exist, + because it's seen references to their hashes, but for which it does + not yet have any content. +*/ +FSL_RC_PHANTOM, + +/** + Indicates that the requested operation is unsupported. +*/ +FSL_RC_UNSUPPORTED, + +/** + Indicates that the requested operation is missing certain required + information. +*/ +FSL_RC_MISSING_INFO, + +/** + Special case of FSL_RC_TYPE triggered in some diff APIs, indicating + that the API cannot diff what appears to be binary data. +*/ +FSL_RC_DIFF_BINARY, +/** + Triggered by some diff APIs to indicate that only whitespace + changes we found and the diff was requested to ignore whitespace. + */ +FSL_RC_DIFF_WS_ONLY, + +/** + Must be the final entry in the enum. Used for creating client-side + result codes which are guaranteed to live outside of this one's + range. +*/ +FSL_RC_end +}; +typedef enum fsl_rc_e fsl_rc_e; + +/** + File permissions flags supported by fossil manifests. Their numeric + values are a hard-coded part of the Fossil architecture and must + not be changed. Note that these refer to manifest-level permissions + and not filesystem-level permissions (though they translate to/from + filesystem-level meanings at some point). +*/ +enum fsl_fileperm_e { +/** Indicates a regular, writable file. */ +FSL_FILE_PERM_REGULAR = 0, +/** Indicates a regular file with the executable bit set. */ +FSL_FILE_PERM_EXE = 0x1, +/** + Indicates a symlink. Note that symlinks do not have the executable + bit set separately on Unix systems. Also note that libfossil does + NOT YET IMPLEMENT symlink support like fossil(1) does - it + currently treats symlinks (mostly) as Unix treats symlinks. +*/ +FSL_FILE_PERM_LINK = 0x2 +}; +typedef enum fsl_fileperm_e fsl_fileperm_e; + +/** + Returns a "standard" string form for a fsl_rc_e code. The string + is primarily intended for debugging purposes. The returned bytes + are guaranteed to be static and NUL-terminated. They are not + guaranteed to contain anything useful for any purposes other than + debugging and tracking down problems. +*/ +FSL_EXPORT char const * fsl_rc_cstr(int); + +/** + Returns the value of FSL_LIBRARY_VERSION used to compile the + library. If this value differs from the value the caller was + compiled with, Chaos might ensue. + + The API does not yet have any mechanism for determining + compatibility between repository versions and it also currently + does no explicit checking to disallow incompatible versions. +*/ +FSL_EXPORT char const * fsl_library_version(); + +/** + Returns true (non-0) if yourLibVersion compares lexically + equal to FSL_LIBRARY_VERSION, else it returns false (0). +*/ +FSL_EXPORT bool fsl_library_version_matches(char const * yourLibVersion); + +/** + This type, accessible to clients via the ::fsl_lib_configurable + global, contains configuration-related data for the library + which can be swapped out by clients. +*/ +struct fsl_lib_configurable_t { + /** + Library-wide allocator. It may be replaced by the client IFF + it is replaced before the library allocates any memory. The + default implementation uses the C-standard + de/re/allocators. Modifying this member while any memory + allocated through it is still "live" leads to undefined + results. There is an exception: a "read-only" middleman proxy + which does not change how the memory is allocated or + intepreted can safely be swapped in or out at any time + provided the underlying allocator stays the same and the + client can ensure that there are no thread-related race + conditions. e.g. it is legal to swap this out with a proxy + which logs allocation requests and then forwards the call on + to the original implementation, and it is legal to do so at + essentially any time. The important thing this that all of the + library-allocated memory goes through a single underlying + (de)allocator for the lifetime of the application. + */ + fsl_allocator allocator; +}; +typedef struct fsl_lib_configurable_t fsl_lib_configurable_t; +FSL_EXPORT fsl_lib_configurable_t fsl_lib_configurable; + +/** + A part of the configuration used by fsl_cx_init() and friends. +*/ +struct fsl_cx_config { + /** + If true, all SQL which goes through the fossil engine + will be traced to the fsl_output()-configured channel. + */ + bool traceSql; + /** + If true, the fsl_print() SQL function will output its output to the + fsl_output()-configured channel, else it is a no-op. + */ + bool sqlPrint; + + /** + Specifies the default hash policy. + */ + fsl_hashpolicy_e hashPolicy; +}; + +/** + fsl_cx_config instance initialized with defaults, intended for + in-struct initialization. +*/ +#define fsl_cx_config_empty_m { \ + 0/*traceSql*/, \ + 0/*sqlPrint*/, \ + FSL_HPOLICY_SHA3/*hashPolicy*/ \ +} + +/** + fsl_cx_config instance initialized with defaults, intended for + copy-initialization. +*/ +FSL_EXPORT const fsl_cx_config fsl_cx_config_empty; + +/** + Parameters for fsl_cx_init(). + + Reminder to self: why are fsl_cx_config and fsl_cx_init_opt + separate structs? TODO: look into consolidating them. +*/ +struct fsl_cx_init_opt { + /** + The output channel for the Fossil instance. + */ + fsl_outputer output; + /** + Basic configuration parameters. + */ + fsl_cx_config config; +}; + + +/** Empty-initialized fsl_cx_init_opt instance. */ +#define fsl_cx_init_opt_empty_m {fsl_outputer_empty_m, fsl_cx_config_empty_m} +/** + fsl_cx_init_opt instance initialized to use stdout for output and + the standard system memory allocator. +*/ +#define fsl_cx_init_opt_default_m {fsl_outputer_FILE_m, fsl_cx_config_empty_m} + +/** Empty-initialized fsl_cx_init_opt instance. */ +FSL_EXPORT const fsl_cx_init_opt fsl_cx_init_opt_empty; + +/** + fsl_cx_init_opt instance initialized to use stdout for output and + the standard system memory allocator. Used as the default when + fsl_cx_init() is passed a NULL value for this parameter. +*/ +FSL_EXPORT const fsl_cx_init_opt fsl_cx_init_opt_default; + +/** + Allocates a new fsl_cx instance, which must eventually + be passed to fsl_cx_finalize() to clean it up. + Normally clients do not need this - they can simply pass + a pointer to NULL as the first argument to fsl_cx_init() + to let it allocate an instance for them. +*/ +FSL_EXPORT fsl_cx * fsl_cx_malloc(); + +/** + Initializes a fsl_cx instance. tgt must be a pointer to NULL, + e.g.: + + ``` + fsl_cxt * f = NULL; // NULL is important - see below + int rc = fsl_cx_init( &f, NULL ); + ``` + + It is very important that f be initialized to NULL _or_ to an + instance which has been properly allocated and empty-initialized + (e.g. via fsl_cx_malloc()). If *tgt is NULL, this routine + allocates the context, else it assumes the caller did. If f + points to unitialized memory then results are undefined. + + If the second parameter is NULL then default implementations are + used for the context's output routine and other options. If it + is not NULL then param->allocator and param->output must be + initialized properly before calling this function. The contents + of param are bitwise copied by this function and ownership of + the returned value is transfered to *tgt in all cases except + one: + + If passed a pointer to a NULL context and this function cannot + allocate it, it returns FSL_RC_OOM and does not modify *tgt. In + this one case, ownership of the context is not changed (as there's + nothing to change!). On any other result (including errors), + ownership of param's contents are transfered to *tgt and the client + is responsible for passing *tgt ot fsl_cxt_finalize() when he is + done with it. Note that (like in sqlite3), *tgt may be valid memory + even if this function fails, and the caller must pass it to + fsl_cx_finalize() whether or not this function succeeds unless it + fails at the initial OOM (which the client can check by seeing if + (*tgt) is NULL, but only if he set it to NULL before calling this). + + Returns 0 on success, FSL_RC_OOM on an allocation error, + FSL_RC_MISUSE if (!tgt). If this function fails, it is illegal to + use the context object except to pass it to fsl_cx_finalize(), as + explained above. + + @see fsl_cx_finalize() + @see fsl_cx_reset() +*/ +FSL_EXPORT int fsl_cx_init( fsl_cx ** tgt, fsl_cx_init_opt const * param ); + +/** + Frees all memory associated with f, which must have been + allocated/initialized using fsl_cx_malloc(), fsl_cx_init(), or + equivalent, or created on the stack and properly initialized + (via fsl_cx_init() or copy-constructed from fsl_cx_empty). + + This function triggers any finializers set for f's client state + or output channel. + + This is a no-op if !f and is effectively a no-op if f has no + state to destruct. +*/ +FSL_EXPORT void fsl_cx_finalize( fsl_cx * f ); + + +/** + Sets or unsets one or more option flags on the given fossil + context. flags is the flag or a bitmask of flags to set (from the + fsl_cx_flags_e enum). If enable is true the flag(s) is (are) set, + else it (they) is (are) unset. Returns the _previous_ set of flags + (that is, the state they were in before this call was made). +*/ +FSL_EXPORT int fsl_cx_flag_set( fsl_cx * f, int flags, bool enable ); + +/** + Returns f's flags. +*/ +FSL_EXPORT int fsl_cx_flags_get( fsl_cx * f ); + +/** + Sets the Fossil error state to the given error code and + fsl_appendf()-style format string/arguments. On success it + returns the code parameter. It does not return 0 unless code is + 0, and if it returns a value other than code then something went + seriously wrong (e.g. allocation error: FSL_RC_OOM) or the + arguments were invalid: !f results in FSL_RC_MISUSE. + + If !fmt then fsl_rc_cstr(code) is used to create the + error string. + + As a special case, if code is FSL_RC_OOM, no error string is + allocated (because it would likely fail, assuming the OOM + is real). + + As a special case, if code is 0 (the non-error value) then fmt is + ignored and any error state is cleared. +*/ +FSL_EXPORT int fsl_cx_err_set( fsl_cx * f, int code, char const * fmt, ... ); + +/** + va_list counterpart to fsl_cx_err_set(). +*/ +FSL_EXPORT int fsl_cx_err_setv( fsl_cx * f, int code, char const * fmt, + va_list args ); + +/** + Fetches the error state from f. See fsl_error_get() for the semantics + of the parameters and return value. +*/ +FSL_EXPORT int fsl_cx_err_get( fsl_cx * f, char const ** str, fsl_size_t * len ); + +/** + Returns f's error state object. This pointer is guaranteed by the + API to be stable until f is finalized, but its contents are + modified my routines as part of the error reporting process. + + Returns NULL if !f. +*/ +FSL_EXPORT fsl_error const * fsl_cx_err_get_e(fsl_cx const * f); + +/** + Resets's f's error state, basically equivalent to + fsl_cx_err_set(f,0,NULL). Is a no-op if f is NULL. This may be + necessary for apps if they rely on looking at fsl_cx_err_get() + at the end of their app/routine, because error state survives + until it is cleared, even if the error held there was caught and + recovered. This function might keep error string memory around + for re-use later on. +*/ +FSL_EXPORT void fsl_cx_err_reset(fsl_cx * f); + +/** + Replaces f's error state with the contents of err, taking over + any memory owned by err (but not err itself). Returns the new + error state code (the value of err->code before this call) on + success. The only error case is if !f (FSL_RC_MISUSE). If err is + NULL then f's error state is cleared and 0 is returned. err's + error state is cleared by this call. +*/ +FSL_EXPORT int fsl_cx_err_set_e( fsl_cx * f, fsl_error * err ); + +/** + If f has error state then it outputs its error state to its + output channel and returns the result of fsl_output(). Returns + FSL_RC_MISUSE if !f, 0 if f has no error state our output of the + state succeeds. If addNewline is true then it adds a trailing + newline to the output, else it does not. + + This is intended for testing and debugging only, and not as an + error reporting mechanism for a full-fledged application. +*/ +FSL_EXPORT int fsl_cx_err_report( fsl_cx * const f, bool addNewline ); + +/** + Unconditionally Moves db->error's state into f. If db is NULL then + f's primary db connection is used. Returns FSL_RC_MISUSE if !f or + (!db && f-is-not-opened). On success it returns f's new error code. + + The main purpose of this function is to propagate db-level + errors up to higher-level code which deals directly with the f + object but not the underlying db(s). + + @see fsl_cx_uplift_db_error2() +*/ +FSL_EXPORT int fsl_cx_uplift_db_error( fsl_cx * const f, fsl_db * db ); + +/** + If rc is not 0 and f has no error state but db does, this calls + fsl_cx_uplift_db_error() and returns its result, else returns + rc. If db is NULL, f's main db connection is used. It is intended + to be called immediately after calling a db operation which might + have failed, and passed that operation's result. + + Results are undefined if db is NULL and f has no main db + connection. +*/ +FSL_EXPORT int fsl_cx_uplift_db_error2(fsl_cx * const f, fsl_db * db, int rc); + +/** + Outputs the first n bytes of src to f's configured output + channel. Returns 0 on success, 0 (without side effects) if !n, else + it returns the result of the underlying output call. This is a + harmless no-op if f is configured with no output channel. + + Results are undefined if f or src are NULL. + + @see fsl_outputf() + @see fsl_flush() +*/ +FSL_EXPORT int fsl_output( fsl_cx * const f, void const * const src, + fsl_size_t n ); + +/** + Flushes f's output channel. Returns 0 on success. If the flush + routine is NULL then this is a harmless no-op. Results are undefined + if f is NULL. + + @see fsl_outputf() + @see fsl_output() +*/ +FSL_EXPORT int fsl_flush( fsl_cx * const f ); + +/** + Uses fsl_appendf() to append formatted output to the channel + configured for use with fsl_output(). Returns 0 on success, + FSL_RC_MISUSE if !f or !fmt, FSL_RC_RANGE if !*fmt, and + FSL_RC_IO if the underlying fsl_appendf() operation fails. + + Note, however, that due to the printf()-style return semantics + of fsl_appendf(), it is not generically possible to distinguish + a partially-successful (i.e. failed in the middle) write from + success. e.g. if fmt contains a format specifier which performs + memory allocation and that allocation fails, it is unlikely that + this function will be able to be aware of that error. The only + way to fix that is to change the return semantics of + fsl_appendf() (and adjust any existing code which relies on + them). + + @see fsl_output() + @see fsl_flush() +*/ +FSL_EXPORT int fsl_outputf( fsl_cx * const f, char const * fmt, ... ); + +/** + va_list counterpart to fsl_outputf(). +*/ +FSL_EXPORT int fsl_outputfv( fsl_cx * const f, char const * fmt, va_list args ); + +/** + Opens the given db file name as f's repository. Returns 0 on + success. On error it sets f's error state and returns that code + unless the error was FSL_RC_MISUSE (which indicates invalid + arguments and it does not set the error state). + + Returns FSL_RC_ACCESS if f already has an opened repo db (use + fsl_repo_close() or fsl_ckout_close() to close it). + + Returns FSL_RC_NOT_FOUND if repoDbFile is not found, as this + routine cannot create a new repository db. + + Results are undefined if any argument is NULL. + + When a repository is opened, the fossil-level user name + associated with f (if any) is overwritten with the default user + from the repo's login table (the one with uid=1). Thus + fsl_cx_user_get() may return a value even if the client has not + called fsl_cx_user_set(). + + It would be nice to have a parameter specifying that the repo + should be opened read-only. That's not as straightforward as it + sounds because of how the various dbs are internally managed + (via one db handle). Until then, the permissions of the + underlying repo file will determine how it is opened. i.e. a + read-only repo will be opened read-only. + + + Potentially interesting side-effects: + + - On success this re-sets several bits of f's configuration to + match the repository-side settings. + + @see fsl_repo_create() + @see fsl_repo_close() +*/ +FSL_EXPORT int fsl_repo_open( fsl_cx * const f, char const * repoDbFile/*, char readOnlyCurrentlyIgnored*/ ); + +/** + If fsl_repo_open_xxx() has been used to open a respository db, this + call closes that db and returns 0. Returns FSL_RC_NOT_FOUND if f + has not opened a repository (that can normally be ignored but is + provided for completeness's sake). + + If a repository is opened "indirectly" via fsl_ckout_open_dir() + then attempting to close it using this function will result in + FSL_RC_MISUSE and f's error state will hold a description of the + problem (the checkout must be closed before closing its + repository). Such a repository will be closed implicitly when the + checkout db is closed. + + @see fsl_repo_open() + @see fsl_repo_create() +*/ +FSL_EXPORT int fsl_repo_close( fsl_cx * const f ); + +/** + Sets or clears (if userName is NULL or empty) the default + repository user name for operations which require one. + + Returns 0 on success, FSL_RC_MISUSE if f is NULL, + FSL_RC_OOM if copying of the userName fails. + + Example usage: + ``` + char * u = fsl_guess_user_name(); + int rc = fsl_cx_user_set(f, u); + fsl_free(u); + ``` + + (Sorry about the extra string copy there, but adding a function + which passes ownership of the name string seems like overkill.) +*/ +FSL_EXPORT int fsl_cx_user_set( fsl_cx * f, char const * userName ); + +/** + Returns the name set by fsl_cx_user_set(), or NULL if f has no + default user name set. The returned bytes are owned by f and may + be invalidated by any call to fsl_cx_user_set(). +*/ +FSL_EXPORT char const * fsl_cx_user_get( fsl_cx const * f ); + +/** + Configuration parameters for fsl_repo_create(). Always + copy-construct these from fsl_repo_create_opt_empty + resp. fsl_repo_create_opt_empty_m in order to ensure proper + behaviour vis-a-vis default values. + + TODOs: + + - Add project name/description, and possibly other + configuration bits. + + - Allow client to set password for default user (currently set + randomly, as fossil(1) does). +*/ +struct fsl_repo_create_opt { + /** + The file name for the new repository. + */ + char const * filename; + /** + Fossil user name for the admin user in the new repo. If NULL, + defaults to the Fossil context's user (see + fsl_cx_user_get()). If that is NULL, it defaults to + "root" for historical reasons. + */ + char const * username; + + /** + The comment text used for the initial commit. If NULL or empty + (starts with a NUL byte) then no initial check is + created. fossil(1) is largely untested with that scenario (but + it seems to work), so for compatibility it is not recommended + that this be set to NULL. + + The default value (when copy-initialized) is "egg". There's a + story behind the use of "egg" as the initial checkin comment, + and it all started with a typo: "initial chicken" + */ + char const * commitMessage; + + /** + Mime type for the commit message (manifest N-card). Manifests + support this but fossil(1) has never (as of 2021-02) made use of + it. It is provided for completeness but should, for + compatibility's sake, probably not be set, as the fossil UI may + not honor it. The implied default is text/x-fossil-wiki. Other + ostensibly legal values include text/plain and text/x-markdown. + This API will accept any value, but results are technically + undefined with any values other than those listed above. + */ + char const * commitMessageMimetype; + + /** + If not NULL and not empty, fsl_repo_create() will use this + repository database to copy the configuration, copying over + the following settings: + + - The reportfmt table, overwriting any existing entries. + + - The user table fields (cap, info, mtime, photo) are copied + for the "system users". The system users are: anonymous, + nobody, developer, reader. + + - The vast majority of the config table is copied, arguably + more than it should (e.g. the 'manifest' setting). + */ + char const * configRepo; + + /** + If false, fsl_repo_create() will fail if this->filename + already exists. + */ + bool allowOverwrite; + +}; +typedef struct fsl_repo_create_opt fsl_repo_create_opt; + +/** Initialized-with-defaults fsl_repo_create_opt struct, intended + for in-struct initialization. */ +#define fsl_repo_create_opt_empty_m { \ + NULL/*filename*/, \ + NULL/*username*/, \ + "egg"/*commitMessage*/, \ + NULL/*commitMessageMimetype*/, \ + NULL/*configRepo*/, \ + false/*allowOverwrite*/ \ + } + +/** Initialized-with-defaults fsl_repo_create_opt struct, intended + for copy-initialization. */ +FSL_EXPORT const fsl_repo_create_opt fsl_repo_create_opt_empty; + +/** + Creates a new repository database using the options provided in the + second argument. If f is not NULL, it must be a valid context + instance, though it need not have an opened checkout/repository. If + f has an opened repo or checkout, this routine closes them but that + closing _will fail_ if a transaction is currently active! + + If f is NULL, a temporary context is used for creating the + repository, in which case the caller will not have access to + detailed error information (only the result code) if this operation + fails. In that case, the resulting repository file will, on + success, be found at the location referred to by opt.filename. + + The opt argument may not be NULL. + + If opt->allowOverwrite is false (0) and the file exists, it fails + with FSL_RC_ALREADY_EXISTS, otherwise is creates/overwrites the + file. This is a destructive operation if opt->allowOverwrite is + true, so be careful: the existing database will be truncated and + re-created. + + This operation installs the various "static" repository schemas + into the db, sets up some default settings, and installs a + default user. + + This operation always closes any repository/checkout opened by f + because setting up the new db requires wiring it to f to set up + some of the db-side infrastructure. The one exception is if + argument validation fails, in which case f's repo/checkout-related + state are not modified. Note that closing will fail if a + transaction is currently active and that, in turn, will cause this + operation to fail. + + See the fsl_repo_create_opt docs for more details regarding the + creation options. + + On success, 0 is returned and f (if not NULL) is left with the + new repository opened and ready for use. On error, f's error + state is updated and any number of the FSL_RC_xxx codes may be + returned - there are no less than 30 different _potential_ error + conditions on the way to creating a new repository. + + If initialization of the repository fails, this routine will + attempt to remove its partially-initialize corpse from the + filesystem but will ignore any errors encountered while doing so. + + Example usage: + + ``` + fsl_repo_create_opt opt = fsl_repo_create_opt_empty; + int rc; + opt.filename = "my.fossil"; + // ... any other opt.xxx you want to set, e.g.: + // opt.user = "fred"; + // Assume fsl is a valid fsl_cx instance: + rc = fsl_repo_create(fsl, &opt); + if(rc) { ...error... } + else { + fsl_db * db = fsl_cx_db_repo(f); + assert(db); // == the new repo db + ... + } + ``` + + @see fsl_repo_open() + @see fsl_repo_close() +*/ +FSL_EXPORT int fsl_repo_create(fsl_cx * f, fsl_repo_create_opt const * opt ); + +/** + UNTESTED. + + Returns true if f has an opened repository database which is + opened in read-only mode, else returns false. +*/ +FSL_EXPORT char fsl_repo_is_readonly(fsl_cx const * f); + +/** + Tries to open a checked-out fossil repository db in the given + directory. The (dirName, checkParentDirs) parameters are passed on + as-is to fsl_ckout_db_search() to find a checkout db, so see that + routine for how it searches. + + If this routine finds/opens a checkout, it also tries to open + the repository database from which the checkout derives, and + fails if it cannot. The library never allows a checkout to be + opened without its corresponding repository partner because + a checkout has hard dependencies on the repo's state. + + Returns 0 on success. If there is an error opening or validating + the checkout or its repository db, f's error state will be + updated. Error codes/conditions include: + + - FSL_RC_MISUSE if f is NULL. + + - FSL_RC_ACCESS if f already has and opened checkout. + + - FSL_RC_OOM if an allocation fails. + + - FSL_RC_NOT_FOUND if no checkout is foud or if a checkout's + repository is not found. + + - FSL_RC_RANGE if dirname is not NULL but has a length of 0. + + - Various codes from fsl_getcwd() (if dirName is NULL). + + - Various codes if opening the associated repository DB fails. + + TODO: there's really nothing in the architecture which restricts a + checkout db to being in the same directory as the checkout, except + for some historical bits which "could" be refactored. It "might be + interesting" to eventually provide a variant which opens a checkout + db file directly. We have the infrastructure, just need some + refactoring. We would need to add the working directory path to the + checkout db's config (`vvar` table), but should otherwise require + no trickery or incompatibilities with fossil(1). +*/ +FSL_EXPORT int fsl_ckout_open_dir( fsl_cx * f, char const * dirName, + bool checkParentDirs ); + + +/** + Searches the given directory (or the current directory if dirName + is 0) for a fossil checkout database file named one of (_FOSSIL_, + .fslckout). If it finds one, it returns 0 and appends the file's + path to pOut if pOut is not 0. If neither is found AND if + checkParentDirs is true an then it moves up the path one directory + and tries again, until it hits the root of the dirPath (see below + for a note/caveat). + + If dirName is NULL then it behaves as if it had been passed the + absolute path of the current directory (as determined by + fsl_getcwd()). + + This function does no normalization of dirName. Because of that... + + Achtung: if dirName is relative, this routine might not find a + checkout where it would find one if given an absolute path (because + it traverses the path string given it instead of its canonical + form). Wether this is a bug or a feature is not yet clear. When in + doubt, use fsl_file_canonical_name() to normalize the directory + name before passing it in here. If it turns out that we always want + that behaviour, this routine will be modified to canonicalize the + name. + + This routine can return at least the following error codes: + + - FSL_RC_NOT_FOUND: either no checkout db was found or the given + directory was not found. + + - FSL_RC_RANGE if dirName is an empty string. (We could arguably + interpret this as a NULL string, i.e. the current directory.) + + - FSL_RC_OOM if allocation of a filename buffer fails. + +*/ +FSL_EXPORT int fsl_ckout_db_search( char const * dirName, + bool checkParentDirs, + fsl_buffer * const pOut ); + + +/** + If fsl_ckout_open_dir() (or similar) has been used to open a + checkout db, this call closes that db and returns 0. Returns + FSL_RC_MISUSE if f has any transactions pending, FSL_RC_NOT_FOUND + if f has not opened a checkout (which can safely be ignored and + does not update f's error state). + + This also closes the repository which was implicitly opened for the + checkout. +*/ +FSL_EXPORT int fsl_ckout_close( fsl_cx * const f ); + +/** + Attempts to close any opened databases (repo/checkout/config). This + will fail if any transactions are pending. Any databases which are + already closed are silently skipped. +*/ +FSL_EXPORT int fsl_cx_close_dbs( fsl_cx * const f ); + +/** + If f is not NULL and has a checkout db opened then this function + returns its name. The bytes are valid until that checkout db + connection is closed. If len is not NULL then *len is (on + success) assigned to the length of the returned string, in + bytes. The string is NUL-terminated, so fetching the length (by + passing a non-NULL 2nd parameter) is optional. + + Returns NULL if !f or f has no checkout opened. + + @see fsl_ckout_open_dir() + @see fsl_cx_ckout_dir_name() + @see fsl_cx_db_file_config() + @see fsl_cx_db_file_repo() +*/ +FSL_EXPORT char const * fsl_cx_db_file_ckout(fsl_cx const * f, + fsl_size_t * len); + +/** + Equivalent to fsl_ckout_db_file() except that + it applies to the name of the opened repository db, + if any. + + @see fsl_cx_db_file_ckout() + @see fsl_cx_db_file_config() +*/ +FSL_EXPORT char const * fsl_cx_db_file_repo(fsl_cx const * f, + fsl_size_t * len); + +/** + Equivalent to fsl_ckout_db_file() except that + it applies to the name of the opened config db, + if any. + + @see fsl_cx_db_file_ckout() + @see fsl_cx_db_file_repo() +*/ +FSL_EXPORT char const * fsl_cx_db_file_config(fsl_cx const * f, + fsl_size_t * len); + +/** + Similar to fsl_cx_db_file_ckout() and friends except that it + applies to db file implied by the specified role (2nd + parameter). If no such role is opened, or the role is invalid, + NULL is returned. + + Note that the role of FSL_DBROLE_TEMP is invalid here. +*/ +FSL_EXPORT char const * fsl_cx_db_file_for_role(fsl_cx const * f, + fsl_dbrole_e r, + fsl_size_t * len); + +/** + Similar to fsl_cx_db_file_ckout() and friends except that it + applies to DB name (as opposed to DB _file_ name) implied by the + specified role (2nd parameter). If no such role is opened, or the + role is invalid, NULL is returned. + + If the 3rd argument is not NULL, it is set to the length, in bytes, + of the returned string. The returned strings are static and + NUL-terminated. + + This is the "easiest" way to figure out the DB name of the given + role, independent of what order f's databases were opened + (because the first-opened DB is always called "main"). + + The Fossil-standard names of its primary databases are: "localdb" + (checkout), "repository", and "configdb" (global config DB), but + libfossil uses "ckout", "repo", and "cfg", respective. So long as + queries use table names which unambiguously refer to a given + database, the DB name is normally not needed. It is needed when + creating new non-TEMP db tables and views. By default such + tables/views would go into the "main" DB, which is actually a + transient DB in this API, so it's important to use the correct DB + name when creating such constructs. + + Note that the role of FSL_DBROLE_TEMP is invalid here. +*/ +FSL_EXPORT char const * fsl_cx_db_name_for_role(fsl_cx const * f, + fsl_dbrole_e r, + fsl_size_t * len); + +/** + If f has an opened checkout db (from fsl_ckout_open_dir()) then + this function returns the directory part of the path for the + checkout, including (for historical and internal convenience + reasons) a trailing slash. The returned bytes are valid until that + db connection is closed. If len is not NULL then *len is (on + success) assigned to the length of the returned string, in bytes. + The string is NUL-terminated, so fetching the length by passing a + non-NULL 2nd parameter is optional. + + Returns NULL if !f or f has no checkout opened. + + @see fsl_ckout_open_dir() + @see fsl_ckout_db_file() +*/ +FSL_EXPORT char const * fsl_cx_ckout_dir_name(fsl_cx const * f, + fsl_size_t * len); + +/** + Returns a handle to f's main db (which may or may not have any + relationship to the repo/checkout/config databases - that's + unspecified!), or NULL if !f. The returned object is owned by f and + the client MUST NOT do any of the following: + + - Close the db handle. + + - Use transactions without using fsl_db_transaction_begin() + and friends. + + - Fiddle with the handle's internals. Doing so might confuse its + owning context. + + Clients MAY add new user-defined functions, use the handle with + fsl_db_prepare(), and other "mundane" db-related tasks. + + Design notes: + + The current architecture uses an in-memory db as the "main" db and + attaches the repo, checkout, and config dbs using well-defined + names. Even so, it uses separate fsl_db instances to track each one + so that we "could," if needed, switch back to a multi-db-instance + approach if needed. + + @see fsl_cx_db_repo() + @see fsl_cx_db_ckout() +*/ +FSL_EXPORT fsl_db * fsl_cx_db( fsl_cx * const f ); + +/** + If f is not NULL and has had its repo opened via + fsl_repo_open(), fsl_ckout_open_dir(), or similar, this + returns a pointer to that database, else it returns NULL. + + @see fsl_cx_db() +*/ +FSL_EXPORT fsl_db * fsl_cx_db_repo( fsl_cx * const f ); + +/** + If f is not NULL and has had a checkout opened via + fsl_ckout_open_dir() or similar, this returns a pointer to that + database, else it returns NULL. + + @see fsl_cx_db() +*/ +FSL_EXPORT fsl_db * fsl_cx_db_ckout( fsl_cx * const f ); + +/** + A helper which fetches f's repository db. If f has no repo db + then it sets f's error state to FSL_RC_NOT_A_REPO with a message + describing the requirement, then returns NULL. Returns NULL if + !f. + + @see fsl_cx_db() + @see fsl_cx_db_repo() + @see fsl_needs_ckout() + @see fsl_cx_has_ckout() +*/ +FSL_EXPORT fsl_db * fsl_needs_repo(fsl_cx * const f); + +/** + The checkout-db counterpart of fsl_needs_repo(). If no checkout is + opened, f's error state is updated with a FSL_RC_NOT_A_CKOUT code + and description of the problem. + + @see fsl_cx_db() + @see fsl_needs_repo() + @see fsl_cx_db_ckout() + @see fsl_cx_has_ckout() +*/ +FSL_EXPORT fsl_db * fsl_needs_ckout(fsl_cx * const f); + +/** + Returns true if the given context has a checkout opened, else + false. + + @see fsl_needs_ckout() +*/ +FSL_EXPORT bool fsl_cx_has_ckout(fsl_cx const * const f ); + +/** + Opens the given database file as f's configuration database. If f + already has a config database opened, it is closed before opening + the new one. The database is created and populated with an + initial schema if needed. + + If dbName is NULL or empty then it uses a default db name, + "probably" under the user's home directory. To get the name of + the database after it has been opened/attached, use + fsl_cx_db_file_config(). + + TODO: strongly consider supporting non-attached + (i.e. sqlite3_open()'d) use of the config db. Comments in fossil(1) + suggest that it is possible to lock the config db for other apps + when it is attached to a long-running op by a fossil process. + + @see fsl_cx_db_config() + @see fsl_config_close() +*/ +FSL_EXPORT int fsl_config_open( fsl_cx * const f, char const * dbName ); + +/** + Closes/detaches the database connection opened by + fsl_config_open(). Returns 0 on succes, FSL_RC_MISUSE if !f, + FSL_RC_NOT_FOUND if no config db connection is opened/attached. + + @see fsl_cx_db_config() + @see fsl_config_open() +*/ +FSL_EXPORT int fsl_config_close( fsl_cx * const f ); + +/** + If f has an opened/attached configuration db then its handle is + returned, else 0 is returned. + + @see fsl_config_open() + @see fsl_config_close() +*/ +FSL_EXPORT fsl_db * fsl_cx_db_config( fsl_cx * const f ); + +/** + Convenience form of fsl_db_prepare() which uses f's main db. + Returns 0 on success. On preparation error, any db error state is + uplifted from the db object to the fsl_cx object. Returns + FSL_RC_MISUSE if f has no db opened (should never happen) or !sql, + FSL_RC_RANGE if !*sql. +*/ +FSL_EXPORT int fsl_cx_prepare( fsl_cx * const f, fsl_stmt * const tgt, + char const * sql, ... ); + +/** + va_list counterpart of fsl_cx_prepare(). +*/ +FSL_EXPORT int fsl_cx_preparev( fsl_cx * const f, fsl_stmt * const tgt, + char const * sql, va_list args ); + +/** + Convenience form of fsl_db_exec() which uses f's main db handle. + Returns 0 on success. On statement preparation or execution error, + the db's error state is uplifted into f and that result is + returned. Returns FSL_RC_MISUSE, without additional error + information if f has no database (should not be able to happen and + might assert) or sql is NULL. + */ +FSL_EXPORT int fsl_cx_exec( fsl_cx * const f, char const * sql, ... ); + +/** + va_list counterpart of fsl_cx_exec(). +*/ +FSL_EXPORT int fsl_cx_execv( fsl_cx * const f, char const * sql, + va_list args ); + +/** + The fsl_db_exec_multi() counterpart of fsl_cx_exec(). +*/ +FSL_EXPORT int fsl_cx_exec_multi( fsl_cx * const f, char const * sql, ... ); + +/** + va_list counterpart of fsl_cx_exec_multi(). +*/ +FSL_EXPORT int fsl_cx_exec_multiv( fsl_cx * const f, char const * sql, + va_list args ); + +/** + Wrapper around fsl_db_last_insert_id() which uses f's main + database. Returns -1 if !f or f has no opened db. + + @see fsl_cx_db() +*/ +FSL_EXPORT fsl_id_t fsl_cx_last_insert_id(fsl_cx * const f); + + +/** + Works similarly to fsl_stat(), except that zName must refer to a + path under f's current checkout directory. Note that this stats + local files, not repository-level content. + + If relativeToCwd is true then the filename is + resolved/canonicalized based on the current working directory (see + fsl_getcwd()), otherwise f's current checkout directory is used as + the virtual root. This makes a subtle yet important difference in + how the name is resolved. Applications taking input from users + (e.g. CLI apps) will normally want to resolve from the current + working dir (assuming the filenames were passed in from the + CLI). In a GUI environment, where the current directory is likely + not the checkout root, resolving based on the checkout root + (i.e. relativeToCwd=false) is probably saner. + + Returns 0 on success. Errors include, but are not limited to: + + - FSL_RC_MISUSE if !zName. + + - FSL_RC_NOT_A_CKOUT if f has no opened checkout. + + - If fsl_is_simple_pathname(zName) returns false then + fsl_ckout_filename_check() is used to normalize the name. If + that fails, its failure code is returned. + + - As for fsl_stat(). + + See fsl_stat() for more details regarding the tgt parameter. + + TODO: fossil-specific symlink support. Currently it does not + distinguish between symlinks and non-links. + + @see fsl_cx_stat2() +*/ +FSL_EXPORT int fsl_cx_stat( fsl_cx * const f, bool relativeToCwd, + char const * zName, fsl_fstat * const tgt ); + +/** + This works identically to fsl_cx_stat(), but provides more + information about the file being stat'd. + + If nameOut is not NULL then the resolved/normalized path to to + that file is appended to nameOut. If fullPath is true then an + absolute path is written to nameOut, otherwise a + checkout-relative path is written. + + Returns 0 on success. On stat() error, nameOut is not updated, + but after stat()'ing, allocation of memory for nameOut's buffer + may fail. + + If zName ends with a trailing slash, that slash is retained in + nameOut. + + This function DOES NOT resolve symlinks, stat()nig the link instead + of what it points to. + + @see fsl_cx_stat() +*/ +FSL_EXPORT int fsl_cx_stat2( fsl_cx * const f, bool relativeToCwd, + char const * zName, + fsl_fstat * const tgt, + fsl_buffer * const nameOut, + bool fullPath); + + +/** + Sets the case-sensitivity flag for f to the given value. This + flag alters how some filename-search/comparison operations + operate. This option is only intended to have an effect on + plaforms with case-insensitive filesystems. + + @see fsl_cx_is_case_sensitive() +*/ +FSL_EXPORT void fsl_cx_case_sensitive_set(fsl_cx * f, bool caseSensitive); + +/** + Returns true (non-0) if f is set for case-sensitive filename + handling, else 0. Returns 0 if !f. + + @see fsl_cx_case_sensitive_set() +*/ +FSL_EXPORT bool fsl_cx_is_case_sensitive(fsl_cx const * f); + +/** + If f is set to use case-sensitive filename handling, + returns a pointer to an empty string, otherwise a pointer + to the string "COLLATE nocase" is returned. + Results are undefined if f is NULL. The returned bytes + are static. + + @see fsl_cx_case_sensitive_set() + @see fsl_cx_is_case_sensitive() +*/ +FSL_EXPORT char const * fsl_cx_filename_collation(fsl_cx const * f); + +/** + An enumeration of the types of structural artifacts used by + Fossil. The numeric values of all entries before FSL_SATYPE_count, + with the exception of FSL_SATYPE_INVALID, are a hard-coded part of + the Fossil db architecture and must never be changed. Any after + FSL_SATYPE_count are libfossil extensions. +*/ +enum fsl_satype_e { +/** + Sentinel value used for some error reporting. +*/ +FSL_SATYPE_INVALID = -1, +/** + Sentinel value used to mark a deck as being "any" type. This is + a placeholder on a deck's way to completion. +*/ +FSL_SATYPE_ANY = 0, +/** + Indicates a "manifest" artifact (a checkin record). +*/ +FSL_SATYPE_CHECKIN = 1, +/** + Indicates a "cluster" artifact. These are used during synchronization. +*/ +FSL_SATYPE_CLUSTER = 2, +/** + Indicates a "control" artifact (a tag change). +*/ +FSL_SATYPE_CONTROL = 3, +/** + Indicates a "wiki" artifact. +*/ +FSL_SATYPE_WIKI = 4, +/** + Indicates a "ticket" artifact. +*/ +FSL_SATYPE_TICKET = 5, +/** + Indicates an "attachment" artifact (used in the ticketing + subsystem). +*/ +FSL_SATYPE_ATTACHMENT = 6, +/** + Indicates a technote (formerly "event") artifact (kind of like a + blog entry). +*/ +FSL_SATYPE_TECHNOTE = 7, +/** Historical (deprecated) name for FSL_SATYPE_TECHNOTE. */ +FSL_SATYPE_EVENT = 7, +/** + Indicates a forum post artifact (a close relative of wiki pages). +*/ +FSL_SATYPE_FORUMPOST = 8, +/** + The number of CATYPE entries. Must be last in the enum. Used for loop + control. +*/ +FSL_SATYPE_count, + +/** + A pseudo-type for use with fsl_sym_to_rid() which changes the + behavior of checkin lookups to return the RID of the start of the + branch rather than the tip, with the caveat that the results are + unspecified if the given symbolic name refers to multiple + branches. + + fsl_satype_event_cstr() returns the same as FSL_SATYPE_CHECKIN for + this entry. + + This entry IS NOT VALID for most APIs which require a fsl_satype_e + value. +*/ +FSL_SATYPE_BRANCH_START = 100 // MUST come after FSL_SATYPE_count +}; +typedef enum fsl_satype_e fsl_satype_e; + +/** + Returns some arbitrary but distinct string for the given + fsl_satype_e. The returned bytes are static and + NUL-terminated. Intended primarily for debugging and informative + purposes, not actual user output. +*/ +FSL_EXPORT char const * fsl_satype_cstr(fsl_satype_e t); + +/** + For a given artifact type, it returns the key string used in the + event.type db table. Returns NULL if passed an unknown value or + a type which is not used in the event table, otherwise the + returned bytes are static and NUL-terminated. + + The returned strings for a given type are as follows: + + - FSL_SATYPE_ANY returns "*" + - FSL_SATYPE_CHECKIN and FSL_SATYPE_BRANCH_START return "ci" + - FSL_SATYPE_WIKI returns "w" + - FSL_SATYPE_TAG returns "g" + - FSL_SATYPE_TICKET returns "t" + - FSL_SATYPE_EVENT returns "e" + + The other control artifact types to not have representations + in the event table, and NULL is returned for them. + + All of the returned values can be used in comparison clauses in + queries on the event table's 'type' field (but use GLOB instead + of '=' so that the "*" returned by FSL_ATYPE_ANY can match!). + For example, to get the comments from the most recent 5 commits: + + ``` + SELECT + datetime(mtime), + coalesce(ecomment,comment), + user + FROM event WHERE type='ci' + ORDER BY mtime DESC LIMIT 5; + ``` + + Where 'ci' in the SQL is the non-NULL return value from this + function. When escaping this value via fsl_buffer_appendf() (or + anything functionally similar), use the %%q/%%Q format + specifiers to escape it. +*/ +FSL_EXPORT char const * fsl_satype_event_cstr(fsl_satype_e t); +/** + A collection of bitmaskable values indicating categories + of fossil-standard glob sets. These correspond to the following + configurable settings: + + ignore-glob, crnl-glob, binary-glob +*/ +enum fsl_glob_category_e{ +/** Sentinel entry. */ +FSL_GLOBS_INVALID = 0, +/** Corresponds to the ignore-glob config setting. */ +FSL_GLOBS_IGNORE = 0x01, +/** Corresponds to the crnl-glob config setting. */ +FSL_GLOBS_CRNL = 0x02, +/** Corresponds to the binary-glob config setting. */ +FSL_GLOBS_BINARY = 0x04, +/** A superset of all config-level glob categories. */ +FSL_GLOBS_ANY = 0xFF +/* + Potential TODO: add FSL_GLOBS_CURRENT_OP for use with SQL UDFs. The + idea would be that SCM operations which could make use of + op-specific glob lists, e.g., checkin/add/merge, could set a custom + glob set as the current one and then access it via their SQL using + `fsl_glob('_', ...)` or some such. +*/ +}; +typedef enum fsl_glob_category_e fsl_glob_category_e; + +/** + Checks one or more of f's configurable glob lists to see if str + matches one of them. If it finds a match, it returns a pointer to + the matching glob (as per fsl_glob_list_matches()), the bytes + of which are owned by f and may be invalidated via modification + or reloading of the underlying glob list. In generally the return + value can be used as a boolean - clients generally do not need + to know exactly which glob matched. + + gtype specifies the glob list(s) to check in the form of a + bitmask of fsl_glob_category_e values. Note that the order of the + lists is unspecified, so if that is important for you then be + sure that gtype only specifies one glob list + (e.g. FSL_GLOBS_IGNORE) and call it again (e.g. passing + FSL_GLOBS_BINARY) if you need to distinguish between those two + cases. + + str must be a non-NULL, non-empty empty string. + + Returns NULL !str, !*str, gtype does not specify any known + glob list(s), or no glob match is found. + + Performance is, abstractly speaking, horrible, because we're + comparing arbitrarily long lists of glob patterns against an + arbitrary string. That said, it's fast enough for our purposes. +*/ +FSL_EXPORT char const * fsl_cx_glob_matches( fsl_cx * const f, int gtype, + char const * str ); + +/** + Converts a well-known fossil glob list configuration key to + a fsl_glob_category_e value: + + - "ignore-glob" = FSL_GLOBS_IGNORE + - "binary-glob" = FSL_GLOBS_BINARY + - "crnl-glob" = FSL_GLOBS_CRNL + - Anything else = FSL_GLOBS_INVALID + + To simplify this function's use via an SQL-accessible UDF, the + `*-glob` names may be passed in without their `-glob` suffix, + e.g. `"igore"` instead of `"ignore-glob"`. +*/ +FSL_EXPORT fsl_glob_category_e fsl_glob_name_to_category(char const * str); + +/** + Fetches f's glob list of the given category. If forceReload is true + then the context will check whether the list has had any content + added to its source since it was initially loaded. + + On success, returns 0 and assigns `*tgt` to the list (noting that + it may be empty). On error `*tgt` is not modified. + + Returns FSL_RC_RANGE if gtype is not one of FSL_GLOBS_IGNORE, + FSL_GLOBS_CRNL, or FSL_GLOBS_BINARY. Returns FSL_RC_OOM if there is + an allocation error during list reloading. May return lower-level + result codes from the filesystem or db layer if loading a given + list fails. +*/ +FSL_EXPORT int fsl_cx_glob_list( fsl_cx * const f, + fsl_glob_category_e gtype, + fsl_list ** tgt, + bool forceReload ); + +/** + Sets f's hash policy and returns the previous value. If f has a + repository db open then the setting is stored there and any error + in setting it is placed into f's error state but otherwise ignored + for purposes of this call. + + If p is FSL_HPOLICY_AUTO *and* the current repository contains any + SHA3-format hashes, the policy is interpreted as FSL_HPOLICY_SHA3. + + This value is a *suggestion*, and may be trumped by various + conditions, in particular in repositories containing older (SHA1) + hashes. +*/ +FSL_EXPORT fsl_hashpolicy_e fsl_cx_hash_policy_set(fsl_cx *f, fsl_hashpolicy_e p); + +/** + Returns f's current hash policy. +*/ +FSL_EXPORT fsl_hashpolicy_e fsl_cx_hash_policy_get(fsl_cx const*f); + +/** + Returns a human-friendly name for the given policy, or NULL for an + invalid policy value. The returned strings are the same ones used + by fossil's hash-policy command. +*/ +FSL_EXPORT char const * fsl_hash_policy_name(fsl_hashpolicy_e p); + +/** + Hashes all of pIn, appending the hash to pOut. Returns 0 on succes, + FSL_RC_OOM if allocation of space in pOut fails. The hash algorithm + used depends on the given fossil context's current hash policy and + the value of the 2nd argument: + + If the 2nd argument is false, the hash is performed per the first + argument's current hash policy. If the 2nd argument is true, the + hash policy is effectively inverted. e.g. if the context prefers + SHA3 hashes, the alternate form will use SHA1. + + Returns FSL_RC_UNSUPPORTED, without updating f's error state, if + the hash is not possible due to conflicting values for the policy + and its alternate. e.g. a context with policy FSL_HPOLICY_SHA3_ONLY + will refuse to apply an SHA1 hash. Whether or not this result can + be ignored is context-dependent, but it normally can be. This + result is only possible when the 2nd argument is true. + + Returns 0 on success. +*/ +FSL_EXPORT int fsl_cx_hash_buffer( const fsl_cx * f, bool useAlternate, + fsl_buffer const * pIn, + fsl_buffer * pOut); +/** + The file counterpart of fsl_cx_hash_buffer(), behaving exactly the + same except that its data source is a file and it may return + various error codes from fsl_buffer_fill_from_filename(). Note that + the contents of the file, not its name, are hashed. +*/ +FSL_EXPORT int fsl_cx_hash_filename( fsl_cx * f, bool useAlternate, + const char * zFilename, fsl_buffer * pOut); + +/** + Works like fsl_getcwd() but updates f's error state on error and + appends the current directory's name to the given buffer. Returns 0 + on success. +*/ +FSL_EXPORT int fsl_cx_getcwd(fsl_cx * f, fsl_buffer * pOut); + +/** + Returns the same as passing fsl_cx_db() to + fsl_db_transaction_level(), or 0 if f has no db opened. + + @see fsl_cx_db() +*/ +FSL_EXPORT int fsl_cx_transaction_level(fsl_cx * const f); +/** + Returns the same as passing fsl_cx_db() to + fsl_db_transaction_begin(). +*/ +FSL_EXPORT int fsl_cx_transaction_begin(fsl_cx * const f); +/** + Returns the same as passing fsl_cx_db() to + fsl_db_transaction_end(). +*/ +FSL_EXPORT int fsl_cx_transaction_end(fsl_cx * const f, bool doRollback); + +/** + Installs or (if f is NULL) uninstalls a confirmation callback for + use by operations on f which require user confirmation. The exact + implications of *not* installing a confirmer depend on the + operation in question: see fsl_cx_confirm(). + + The 2nd argument bitwise copied into f's internal confirmer + object. If the 2nd argument is NULL, f's confirmer is cleared, + which will cause fsl_cx_confirm() to use certain default responses + (see that function for details). + + If the final argument is not NULL then the previous confirmer is + bitwise copied to it. + + @see fsl_confirm_callback_f + @see fsl_cx_confirm() + @see fsl_cx_confirmer_get() +*/ +FSL_EXPORT void fsl_cx_confirmer(fsl_cx * f, + fsl_confirmer const * newConfirmer, + fsl_confirmer * prevConfirmer); +/** + Stores a bitwise copy of f's current confirmer object into *dest. Can + be used to save the confirmer before temporarily swapping it out. + + @see fsl_cx_confirmer() +*/ +FSL_EXPORT void fsl_cx_confirmer_get(fsl_cx const * f, fsl_confirmer * dest); + +/** + If fsl_cx_confirmer() was used to install a confirmer callback in f + then this routine calls that confirmer and returns its result code + and its answer via *outAnswer. If no confirmer is currently + installed, it responds with default answers, depending on the + eventId: + + - FSL_CEVENT_OVERWRITE_MOD_FILE: FSL_CRESPONSE_NEVER + + - FSL_CEVENT_OVERWRITE_UNMGD_FILE: FSL_CRESPONSE_NEVER + + - FSL_CEVENT_RM_MOD_UNMGD_FILE: FSL_CRESPONSE_NEVER + + - FSL_CEVENT_MULTIPLE_VERSIONS: FSL_CRESPONSE_CANCEL + + Those are not 100% set in stone and are up for reconsideration. + + If a confirmer has been installed, this function does not modify + outAnswer->response if the installed confirmer does not. Thus + routines should set it to some acceptable default/sentinel value + before calling this, to account for callbacks which ignore the + given detail->eventId. + + If a confirmer callback responds with FSL_CRESPONSE_ALWAYS or + FSL_CRESPONSE_NEVER, the code which is requesting confirmation must + honor that by *NOT* calling the callback again for the current + processing step of that eventId. e.g. if a loop asks for + confirmation of FSL_CEVENT_RM_MOD_FILE and any response is one of + the above, that one loop must not ask for confirmation again, and + must instead accept that response for future queries within the + same logical library operation (e.g. one checkout-update + cycle). This is particularly important for applications which + interactively present the question to the user for confirmation so + that users have a way to *not* get spammed with a confirmation + message showing up for each and every one of an arbitrary number of + confirmations. + + @see fsl_confirm_callback_f + @see fsl_cx_confirmer() +*/ +FSL_EXPORT int fsl_cx_confirm(fsl_cx *f, fsl_confirm_detail const * detail, + fsl_confirm_response *outAnswer); + +#if 0 +/** + DO NOT USE - not yet tested and ready. + + Returns the result of either localtime(clock) or gmtime(clock), + depending on f: + + - If f is NULL, returns localtime(clock). + + - If f has had its FSL_CX_F_LOCALTIME_GMT flag set (see + fsl_cx_flag_set()) then returns gmtime(clock), else + localtime(clock). + + If clock is NULL, NULL is returned. + + Note that fsl_cx instances default to using UTC for everything, + which is the opposite of fossil(1). +*/ +FSL_EXPORT struct tm * fsl_cx_localtime( fsl_cx const * f, const time_t * clock ); + +/** + Equivalent to fsl_cx_localtime(NULL, clock). +*/ +FSL_EXPORT struct tm * fsl_localtime( const time_t * clock ); + +/** + DO NOT USE - not yet tested and ready. + + This function passes (f, clock) to fsl_cx_localtime(), + then returns the result of mktime(3) on it. So... + it adjusts a UTC Unix timestamp to either the same UTC + local timestamp or to the local time. +*/ +FSL_EXPORT time_t fsl_cx_time_adj(fsl_cx const * f, time_t clock); +#endif + +#if defined(__cplusplus) +} /*extern "C"*/ +#endif +#endif +/* ORG_FOSSIL_SCM_FSL_CORE_H_INCLUDED */ +/* end of file ../include/fossil-scm/fossil-core.h */ +/* start of file ../include/fossil-scm/fossil-db.h */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#if !defined(ORG_FOSSIL_SCM_FSL_DB_H_INCLUDED) +#define ORG_FOSSIL_SCM_FSL_DB_H_INCLUDED +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). + + + ****************************************************************************** + This file declares public APIs for working with fossil's database + abstraction layer. +*/ + +/* + We don't _really_ want to include sqlite3.h at this point, but if we + do not then we have to typedef the sqlite3 struct here and that + breaks when client code includes both this file and sqlite3.h. +*/ +#include "sqlite3.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +#if 0 +/** + Potential TODO. Maybe not needed - v1 uses only(?) 1 hook and we + can do that w/o hooks. +*/ +typedef int (*fsl_commit_hook_f)( void * state ); +/** Potential TODO */ +struct fsl_commit_hook { + fsl_commit_hook_f hook; + int sequence; + void * state; +}; +#define fsl_commit_hook_empty_m {NULL,0,NULL} +typedef struct fsl_commit_hook fsl_commit_hook; +/* extern const fsl_commit_hook fsl_commit_hook_empty; */ + +/** + Potential TODO. +*/ +FSL_EXPORT int fsl_db_before_commit_hook( fsl_db * db, fsl_commit_hook_f f, + int sequence, void * state ); +#endif + +#if 0 +/* We can't do this because it breaks when clients include both + this header and sqlite3.h. Is there a solution which lets us + _not_ include sqlite3.h from this file and also compiles when + clients include both? +*/ +#if !defined(SQLITE_OK) +/** + Placeholder for sqlite3/4 type. We currently use v3 but will + almost certainly switch to v4 at some point. Before we can do + that we need an upgrade/migration path. +*/ +typedef struct sqlite3 sqlite3; +#endif +#endif + + +/** + Flags for use with fsl_db_open() and friends. +*/ +enum fsl_open_flags_e { +/** + The "no flags" value. +*/ +FSL_OPEN_F_NONE = 0, +/** + Flag for fsl_db_open() specifying that the db should be opened + in read-only mode. +*/ +FSL_OPEN_F_RO = 0x01, +/** + Flag for fsl_db_open() specifying that the db should be opened + in read-write mode, but should not create the db if it does + not already exist. +*/ +FSL_OPEN_F_RW = 0x02, +/** + Flag for fsl_db_open() specifying that the db should be opened in + read-write mode, creating the db if it does not already exist. +*/ +FSL_OPEN_F_CREATE = 0x04, +/** + Shorthand for RW+CREATE flags. +*/ +FSL_OPEN_F_RWC = FSL_OPEN_F_RW | FSL_OPEN_F_CREATE, +/** + Tells fsl_repo_open_xxx() to confirm that the db + is a repository. +*/ +FSL_OPEN_F_SCHEMA_VALIDATE = 0x20, + +/** + Used by fsl_db_open() to to tell 1the underlying db connection to + trace all SQL to stdout. This is often useful for testing, + debugging, and learning about what's going on behind the scenes. +*/ +FSL_OPEN_F_TRACE_SQL = 0x40 +}; + +/** + A level of indirection to "hide" the actual db driver + implementation from the public API. Whether or not the API + uses/will use sqlite3 or 4 is "officially unspecified." We + currently use 3 because (A) it bootstraps development and + testing by letting us use existing fossil repos for and (B) it + reduces the number of potential problems when porting SQL-heavy + code from the fossil(1) tree. Clients should try not to rely on the + underlying db driver API, but may need it for some uses + (e.g. binding custom SQL functions). + + Sidebar: at the time this port was initiated, sqlite4 was in an + experimental stage to explore a new storage engine which, it was + hoped, would speed up sqlite considerably. It ended up never + leaving that stage because the performance gains of the storage + engine did not justify such a significant upheaval. Even so, + sqlite4 "might" come around sometime within the lifetime of this + project, so it behooves us to abstract away this type from the + public API. +*/ +typedef sqlite3 fsl_dbh_t; + +/** + Db handle wrapper class. Each instance wraps a single sqlite + database handle. + + Fossil is built upon sqlite3, but this abstraction is intended to + hide that, insofar as possible, from clients so as to simplify an + eventual port from v3 to v4. Clients should avoid relying on the + underlying db being sqlite (or at least not rely on a specific + version), but may want to register custom functions with the driver + (or perform similar low-level operations) and the option is left + open for them to access that handle via the fsl_db::dbh member. In + practice, library clients do not ever need to use the underlying + sqlite API. + + @see fsl_db_open(); + @see fsl_db_close(); + @see fsl_stmt +*/ +struct fsl_db { + /** + Fossil Context on whose behalf this instance is operating, if + any. Certain db operations behave differently depending + on whether or not this is NULL. + */ + fsl_cx * f; + + /** + Describes what role(s) this db connection plays in fossil (if + any). This is a bitmask of fsl_dbrole_e values, and a db + connection may have multiple roles. This is only used by the + fsl_cx-internal API. + */ + int role; + + /** + Underlying db driver handle. + */ + fsl_dbh_t * dbh; + + /** + Holds error state from the underlying driver. fsl_db and + fsl_stmt operations which fail at the driver level "should" + update this state to include error info from the driver. + fsl_cx APIs which fail at the DB level uplift this (using + fsl_error_move()) so that they can pass it on up the call chain. + */ + fsl_error error; + + /** + Holds the file name used when opening this db. Might not refer to + a real file (e.g. might be ":memory:" or "" (similar to + ":memory:" but may swap to temp storage). + + Design note: we currently hold the name as it is passed to the + db-open routine, without canonicalizing it. That is very possibly + a mistake, as it makes it impossible to properly compare the name + to another arbitrary checkout-relative name for purposes of + fsl_reserved_fn_check(). For purposes of fsl_cx we currently + (2021-03-12) canonicalize db's which we fsl_db_open(), but not + those which we ATTACH (which includes the repo and checkout + dbs). We cannot reasonably canonicalize the repo db filename + because it gets written into the checkout db so that the checkout + knows where to find the repository. History has shown that that + path needs to be stored exactly as a user entered it, which is + often relative. + + Memory is owned by this object. + */ + char * filename; + + /** + Holds the database name for use in creating queries. + Might or might not be set/needed, depending on + the context. + + Memory is owned by this object. + */ + char * name; + + /** + Debugging/test counter. Closing a db with opened statements + might assert() or trigger debug output when the db is closed. + */ + int openStatementCount; + + /** + Counter for fsl_db_transaction_begin/end(). + */ + int beginCount; + + /** + Internal flag for communicating rollback state through the + call stack. If this is set to a true value, + fsl_db_transaction_end() calls will behave like a rollback + regardless of the value of the 2nd argument passed to that + function. i.e. it propagates a rollback through nested + transactions. + + Potential TODO: instead of treating this like a boolean, store + the error number which caused the rollback here. We'd have to + go fix a lot of code for that, though :/. + */ + int doRollback; + + /** + Internal change counter. Set when a transaction is + started/committed. + + Maintenance note: it's an int because that's what + sqlite3_total_changes() returns. + */ + int priorChanges; + + /** + List of SQL commands (char *) which should be executed prior + to a commit. This list is cleared when the transaction counter + drops to zero as the result of fsl_db_transaction_end() + or fsl_db_rollback_force(). + + TODO? Use (fsl_stmt*) objects instead of strings? Depends on + how much data we need to bind here (want to avoid an extra + copy if we need to bind big stuff). That was implemented in + [9d9375ac2d], but that approach prohibits multi-statement + pre-commit triggers, so it was not trunked. It's still unknown + whether we need multi-statement SQL in this context + (==fossil's infrastructure). + + @see fsl_db_before_commit() + */ + fsl_list beforeCommit; + + /** + Internal buffer to reduce, potentially drasticlaly, reallocations + caused via fsl_db_prepare_cached(). + */ + fsl_buffer cachePrepBuf; + + /** + An internal cache of "static" queries - those which do not rely + on call-time state unless that state can be bind()ed. Holds a + linked list of (fsl_stmt*) instances, managed by the + fsl_db_prepare_cached() and fsl_stmt_cached_yield() APIs. + + @see fsl_db_prepare_cached() + */ + fsl_stmt * cacheHead; + + /** + A marker which tells fsl_db_close() whether or not + fsl_db_malloc() allocated this instance (in which case + fsl_db_close() will fsl_free() it) or not (in which case it + does not fsl_free() it). + */ + void const * allocStamp; +}; +/** + Empty-initialized fsl_db structure, intended for const-copy + initialization. +*/ +#define fsl_db_empty_m { \ + NULL/*f*/, \ + FSL_DBROLE_NONE, \ + NULL/*dbh*/, \ + fsl_error_empty_m /*error*/, \ + NULL/*filename*/, \ + NULL/*name*/, \ + 0/*openStatementCount*/, \ + 0/*beginCount*/, \ + 0/*doRollback*/, \ + 0/*priorChanges*/, \ + fsl_list_empty_m/*beforeCommit*/, \ + fsl_buffer_empty_m/*cachePrepBuf*/, \ + NULL/*cacheHead*/, \ + NULL/*allocStamp*/ \ + } + +/** + Empty-initialized fsl_db structure, intended for copy + initialization. +*/ +FSL_EXPORT const fsl_db fsl_db_empty; + +/** + If db is not NULL then this function returns its name (the one used + to open it). The bytes are valid until the db connection is closed + or until someone mucks with db->filename. If len is not NULL then + *len is (on success) assigned to the length of the returned string, + in bytes. The string is NUL-terminated, so fetching the length (by + passing a non-NULL 2nd parameter) is optional but sometimes useful + to eliminate a downstream call to fsl_strlen(). + + Results are undefined if db is NULL or was improperly initialized. + Will return NULL if db was properly initialized (via copying + fsl_db_empty) but has not yet been opened. +*/ +FSL_EXPORT char const * fsl_db_filename(fsl_db const * db, fsl_size_t * len); + +typedef sqlite3_stmt fsl_stmt_t; +/** + Represents a prepared statement handle. + Intended usage: + + ``` + fsl_stmt st = fsl_stmt_empty; + int rc = fsl_db_prepare( db, &st, "..." ); + if(rc){ // Error! + assert(!st.stmt); + // db->error might hold driver-level error details. + }else{ + // use st and eventually finalize it: + fsl_stmt_finalize( &st ); + } + ``` + + + Script binding implementations can largely avoid exposing the + statement handle (and its related cleanup ordering requirements) + to script code. They need to have some mechanism for binding + values to SQL (or implement all the escaping themselves), but + that can be done without exposing all of the statement class if + desired. For example, here's some hypothetical script code: + + ``` + var st = db.prepare(".... where i=:i and x=:x"); + // st is-a Statement, but we need not add script bindings for + // the whole Statement.bind() API. We can instead simplify that + // to something like: + try { + st.exec( {i: 42, x: 3} ) + // or, for a SELECT query: + st.each({ + bind{i:42, x:3}, + rowType: 'array', // or 'object' + callback: function(row,state,colNames){ print(row.join('\t')); }, + state: {...callback function state...} + }); + } finally { + st.finalize(); + // It is critical that st gets finalized before its DB, and + // that'shard to guaranty if we leave st to the garbage collector! + } + // see below for another (less messy) alternative + ``` + + Ideally, script code should not have direct access to the + Statement because managing lifetimes can be difficult in the + face of flow-control changes caused by exceptions (as the above + example demonstrates). Statements can be completely hidden from + clients if the DB wrapper is written to support it. For example, + in pseudo-JavaScript that might look like: + + ``` + db.exec("...where i=? AND x=?", 42, 3); + db.each({sql:"select ... where id0) or not + (fsl_stmt_col_count()==0) without having to know the contents + of the query. + + - fsl_db_prepare_cached() can be used to cache often-used or + expensive-to-prepare queries within the context of their parent + db handle. +*/ +FSL_EXPORT int fsl_db_prepare( fsl_db * const db, fsl_stmt * const tgt, + char const * sql, ... ); + +/** + va_list counterpart of fsl_db_prepare(). +*/ +FSL_EXPORT int fsl_db_preparev( fsl_db * const db, fsl_stmt * const tgt, + char const * sql, va_list args ); + +/** + A special-purpose variant of fsl_db_prepare() which caches + statements based on their SQL code. This works very much like + fsl_db_prepare() and friends except that it can return the same + statement (via *st) multiple times (statements with identical + SQL are considered equivalent for caching purposes). Clients + need not explicitly pass the returned statement to + fsl_stmt_finalize() - the db holds these statements and will + finalize them when it is closed. It is legal to pass them to + finalize, in which case they will be cleaned up immediately but + that also invalidates _all_ pointers to the shared instances. + + If client code does not call fsl_stmt_finalize(), it MUST pass + the statement pointer to fsl_stmt_cached_yield(st) after is done + with it. That makes the query available for use again with this + routine. If a cached query is not yielded via + fsl_stmt_cached_yield() then this routine will return + FSL_RC_ACCESS on subsequent requests for that SQL to prevent + that recursive (mis)use of the statement causes problems. + + This routine is intended to be used in oft-called routines + where the cost of re-creating statements on each execution could + be prohibitive (or at least a bummer). + + Returns 0 on success, FSL_RC_MISUSE if any arguments are + invalid. On error, *st is not written to. On other error's + db->error might be updated with more useful information. See the + Caveats section below for more details. + + Its intended usage looks like: + + ``` + fsl_stmt * st = NULL; + int rc = fsl_db_prepare_cached(myDb, &st, "SELECT ..."); + if(rc) { assert(!st); ...error... } + else { + ...use it, and _be sure_ to yield it when done:... + fsl_stmt_cached_yield(st); + } + ``` + + Though this function allows a formatted SQL string, caching is + generally only useful with statements which have "static" SQL, + i.e. no call-dependent values embedded within the SQL. It _can_, + however, contain bind() placeholders which get reset for each + use. Note that fsl_stmt_cached_yield() resets the statement, so + most uses of cached statements do not require that the client + explicitly reset cached statements (doing so is harmless, + however). + + Caveats: + + Cached queries must not be used in contexts where recursion + might cause the same query to be returned from this function + while it is being processed at another level in the execution + stack. Results would be undefined. Caching is primarily intended + for often-used routines which bind and fetch simple values, and + not for queries which bind large inlined values or might invoke + recursion. Because of the potential for recursive breakage, this + function flags queries it doles out and requires that clients + call fsl_stmt_cached_yield() to un-flag them for re-use. It will + return FSL_RC_ACCESS if an attempt is made to (re)prepare a + statement for which a fsl_stmt_cached_yield() is pending, and + db->error will be populated with a (long) error string + descripting the problem and listing the SQL which caused the + collision/misuse. + + + Design note: for the recursion/parallel use case we "could" + reimplement this to dole out a new statement (e.g. by appending + " -- a_number" to the SQL to bypass the collision) and free it in + fsl_stmt_cached_yield(), but that (A) gets uglier than it needs + to be and (B) is not needed unless/until we really need cached + queries in spots which would normally break them. The whole + recursion problem is still theoretical at this point but could + easily affect small, often-used queries without recursion. + + @see fsl_db_stmt_cache_clear() + @see fsl_stmt_cached_yield() +*/ +FSL_EXPORT int fsl_db_prepare_cached( fsl_db * const db, fsl_stmt ** st, + char const * sql, ... ); + +/** + The va_list counterpart of fsl_db_prepare_cached(). +*/ +FSL_EXPORT int fsl_db_preparev_cached( fsl_db * const db, fsl_stmt ** st, + char const * sql, va_list args ); + +/** + "Yields" a statement which was prepared with + fsl_db_prepare_cached(), such that that routine can once again + use/re-issue that statement. Statements prepared this way must + be yielded in order to prevent that recursion causes + difficult-to-track errors when a given cached statement is used + concurrently in different code contexts. + + If st is not NULL then this also calls fsl_stmt_reset() on the + statement (because that simplifies usage of cached statements). + + Returns 0 on success, FSL_RC_MISUSE if !st or if st does not + appear to have been doled out from fsl_db_prepare_cached(). + + @see fsl_db_prepare_cached() + @see fsl_db_stmt_cache_clear() +*/ +FSL_EXPORT int fsl_stmt_cached_yield( fsl_stmt * const st ); + +/** + A special-purposes utility which schedules SQL to be executed + the next time fsl_db_transaction_end() commits a transaction for + the given db. A commit or rollback will clear all before-commit + SQL whether it executes them or not. This should not be used as + a general-purpose trick, and is intended only for use in very + limited parts of the Fossil infrastructure. + + Before-commit code is only executed if the db has made changes + since the transaction began. If no changes are recorded + then before-commit triggers are _not_ run. This is a historical + behaviour which is up for debate. + + This function does not prepare the SQL, so it does not catch + errors which happen at prepare-time. Preparation is done (if + ever) just before the next transaction is committed. + + Returns 0 on success, non-0 on error. + + Potential TODO: instead of storing the raw SQL, prepare the + statements here and store the statement handles. The main + benefit would be that this routine could report preport + preparation errors (which otherwise cause the the commit to + fail). The down-side is that it prohibits the use of + multi-statement pre-commit code. We have an implementation of + this somewhere early on in the libfossil tree, but it was not + integrated because of the inability to use multi-statement SQL + with it. +*/ +FSL_EXPORT int fsl_db_before_commit( fsl_db * const db, char const * const sql, ... ); + +/** + va_list counterpart to fsl_db_before_commit(). +*/ +FSL_EXPORT int fsl_db_before_commitv( fsl_db * const db, char const * const sql, va_list args ); + + +/** + Frees memory associated with stmt but does not free stmt unless + it was allocated by fsl_stmt_malloc() (these objects are + normally stack-allocated, and such object must be initialized by + copying fsl_stmt_empty so that this function knows whether or + not to fsl_free() them). Returns FSL_RC_MISUSE if !stmt or it + has already been finalized (but was not freed). +*/ +FSL_EXPORT int fsl_stmt_finalize( fsl_stmt * const stmt ); + +/** + "Steps" the given SQL cursor one time and returns one of the + following: FSL_RC_STEP_ROW, FSL_RC_STEP_DONE, FSL_RC_STEP_ERROR. + On a db error this will update the underlying db's error state. + This function increments stmt->rowCount by 1 if it returns + FSL_RC_STEP_ROW. + + Returns FSL_RC_MISUSE if !stmt or stmt has not been prepared. + + It is only legal to call the fsl_stmt_g_xxx() and + fsl_stmt_get_xxx() functions if this functon returns + FSL_RC_STEP_ROW. FSL_RC_STEP_DONE is returned upon successfully + ending iteration or if there is no iteration to perform (e.g. a + UPDATE or INSERT). + + + @see fsl_stmt_reset() + @see fsl_stmt_reset2() + @see fsl_stmt_each() +*/ +FSL_EXPORT int fsl_stmt_step( fsl_stmt * const stmt ); + +/** + A callback interface for use with fsl_stmt_each() and + fsl_db_each(). It will be called one time for each row fetched, + passed the statement object and the state parameter passed to + fsl_stmt_each() resp. fsl_db_each(). If it returns non-0 then + iteration stops and that code is returned UNLESS it returns + FSL_RC_BREAK, in which case fsl_stmt_each() stops iteration and + returns 0. i.e. implementations may return FSL_RC_BREAK to + prematurly end iteration without causing an error. + + This callback is not called for non-fetching queries or queries + which return no results, though it might (or might not) be + interesting for it to do so, passing a NULL stmt for that case. + + stmt->rowCount can be used to determine how many times the + statement has called this function. Its counting starts at 1. + + It is strictly illegal for a callback to pass stmt to + fsl_stmt_step(), fsl_stmt_reset(), fsl_stmt_finalize(), or any + similar routine which modifies its state. It must only read the + current column data (or similar metatdata, e.g. column names) + from the statement, e.g. using fsl_stmt_g_int32(), + fsl_stmt_get_text(), or similar. +*/ +typedef int (*fsl_stmt_each_f)( fsl_stmt * stmt, void * state ); + +/** + Calls the given callback one time for each result row in the + given statement, iterating over stmt using fsl_stmt_step(). It + applies no meaning to the callbackState parameter, which gets + passed as-is to the callback. See fsl_stmt_each_f() for the + semantics of the callback. + + Returns 0 on success. Returns FSL_RC_MISUSE if !stmt or + !callback. +*/ +FSL_EXPORT int fsl_stmt_each( fsl_stmt * const stmt, fsl_stmt_each_f callback, + void * callbackState ); + +/** + Resets the given statement, analog to sqlite3_reset(). Should be + called one time between fsl_stmt_step() iterations when running + multiple INSERTS, UPDATES, etc. via the same statement. If + resetRowCounter is true then the statement's row counter + (st->rowCount) is also reset to 0, else it is left + unmodified. (Most use cases don't use the row counter.) + + Returns 0 on success, FSL_RC_MISUSE if stmt has not been prepared + or has not been cleanly initialized via copying from fsl_stmt_empty + or fsl_stmt_empty_m, FSL_RC_DB if the underlying reset fails (in + which case the error state of the stmt->db handle is updated to + contain the error information). + + @see fsl_stmt_db() + @see fsl_stmt_reset() +*/ +FSL_EXPORT int fsl_stmt_reset2( fsl_stmt * const stmt, bool resetRowCounter ); + +/** + Equivalent to fsl_stmt_reset2(stmt, 0). +*/ +FSL_EXPORT int fsl_stmt_reset( fsl_stmt * const stmt ); + +/** + Returns the db handle which prepared the given statement, or + NULL if !stmt or stmt has not been prepared. +*/ +FSL_EXPORT fsl_db * fsl_stmt_db( fsl_stmt * const stmt ); + +/** + Returns the SQL string used to prepare the given statement, or + NULL if !stmt or stmt has not been prepared. If len is not NULL + then *len is set to the length of the returned string (which is + NUL-terminated). The returned bytes are owned by stmt and are + invalidated when it is finalized. +*/ +FSL_EXPORT char const * fsl_stmt_sql( fsl_stmt * const stmt, + fsl_size_t * const len ); + +/** + Returns the name of the given 0-based result column index, or + NULL if !stmt, stmt is not prepared, or index is out out of + range. The returned bytes are owned by the statement object and + may be invalidated shortly after this is called, so the caller + must copy the returned value if it needs to have any useful + lifetime guarantees. It's a bit more complicated than this, but + assume that any API calls involving the statement handle might + invalidate the column name bytes. + + The API guarantees that the returned value is either NULL or + NUL-terminated. + + @see fsl_stmt_param_count() + @see fsl_stmt_col_count() +*/ +FSL_EXPORT char const * fsl_stmt_col_name(fsl_stmt * const stmt, int index); + +/** + Returns the result column count for the given statement, or -1 if + !stmt or it has not been prepared. Note that this value is cached + when the statement is created. Note that non-fetching queries + (e.g. INSERT and UPDATE) have a column count of 0. Some non-SELECT + constructs, e.g. PRAGMA table_info(tname), behave like SELECT + and have a positive column count. + + @see fsl_stmt_param_count() + @see fsl_stmt_col_name() +*/ +FSL_EXPORT int fsl_stmt_col_count( fsl_stmt const * const stmt ); + +/** + Returns the bound parameter count for the given statement, or -1 + if !stmt or it has not been prepared. Note that this value is + cached when the statement is created. + + @see fsl_stmt_col_count() + @see fsl_stmt_col_name() +*/ +FSL_EXPORT int fsl_stmt_param_count( fsl_stmt const * const stmt ); + +/** + Returns the index of the given named parameter for the given + statement, or -1 if !stmt or stmt is not prepared. +*/ +FSL_EXPORT int fsl_stmt_param_index( fsl_stmt * const stmt, char const * const param); + +/** + Binds NULL to the given 1-based parameter index. Returns 0 on + succcess. Sets the DB's error state on error. +*/ +FSL_EXPORT int fsl_stmt_bind_null( fsl_stmt * const stmt, int index ); + +/** + Equivalent to fsl_stmt_bind_null_name() but binds to + a named parameter. +*/ +FSL_EXPORT int fsl_stmt_bind_null_name( fsl_stmt * const stmt, char const * param ); + +/** + Binds v to the given 1-based parameter index. Returns 0 on + succcess. Sets the DB's error state on error. +*/ +FSL_EXPORT int fsl_stmt_bind_int32( fsl_stmt * const stmt, int index, int32_t v ); + +/** + Equivalent to fsl_stmt_bind_int32() but binds to a named + parameter. +*/ +FSL_EXPORT int fsl_stmt_bind_int32_name( fsl_stmt * const stmt, char const * param, int32_t v ); + +/** + Binds v to the given 1-based parameter index. Returns 0 on + succcess. Sets the DB's error state on error. +*/ +FSL_EXPORT int fsl_stmt_bind_int64( fsl_stmt * const stmt, int index, int64_t v ); + +/** + Equivalent to fsl_stmt_bind_int64() but binds to a named + parameter. +*/ +FSL_EXPORT int fsl_stmt_bind_int64_name( fsl_stmt * const stmt, char const * param, int64_t v ); + +/** + Binds v to the given 1-based parameter index. Returns 0 on + succcess. Sets the Fossil context's error state on error. +*/ +FSL_EXPORT int fsl_stmt_bind_double( fsl_stmt * const stmt, int index, double v ); + +/** + Equivalent to fsl_stmt_bind_double() but binds to a named + parameter. +*/ +FSL_EXPORT int fsl_stmt_bind_double_name( fsl_stmt * const stmt, char const * param, double v ); + +/** + Binds v to the given 1-based parameter index. Returns 0 on + succcess. Sets the DB's error state on error. +*/ +FSL_EXPORT int fsl_stmt_bind_id( fsl_stmt * const stmt, int index, fsl_id_t v ); + +/** + Equivalent to fsl_stmt_bind_id() but binds to a named + parameter. +*/ +FSL_EXPORT int fsl_stmt_bind_id_name( fsl_stmt * const stmt, char const * param, fsl_id_t v ); + +/** + Binds the first n bytes of v as text to the given 1-based bound + parameter column in the given statement. If makeCopy is true then + the binding makes an copy of the data. Set makeCopy to false ONLY + if you KNOW that the bytes will outlive the binding. + + Returns 0 on success. On error stmt's underlying db's error state + is updated, hopefully with a useful error message. +*/ +FSL_EXPORT int fsl_stmt_bind_text( fsl_stmt * const stmt, int index, + char const * v, fsl_int_t n, + bool makeCopy ); + +/** + Equivalent to fsl_stmt_bind_text() but binds to a named + parameter. +*/ +FSL_EXPORT int fsl_stmt_bind_text_name( fsl_stmt * const stmt, char const * param, + char const * v, fsl_int_t n, + bool makeCopy ); +/** + Binds the first n bytes of v as a blob to the given 1-based bound + parameter column in the given statement. See fsl_stmt_bind_text() + for the semantics of the makeCopy parameter and return value. +*/ +FSL_EXPORT int fsl_stmt_bind_blob( fsl_stmt * const stmt, int index, + void const * v, fsl_size_t len, + bool makeCopy ); + +/** + Equivalent to fsl_stmt_bind_blob() but binds to a named + parameter. +*/ +FSL_EXPORT int fsl_stmt_bind_blob_name( fsl_stmt * const stmt, char const * param, + void const * v, fsl_int_t len, + bool makeCopy ); + +/** + Gets an integer value from the given 0-based result set column, + assigns *v to that value, and returns 0 on success. + + Returns FSL_RC_RANGE if index is out of range for stmt, FSL_RC_MISUSE + if stmt has no result columns. +*/ +FSL_EXPORT int fsl_stmt_get_int32( fsl_stmt * const stmt, int index, int32_t * v ); + +/** + Gets an integer value from the given 0-based result set column, + assigns *v to that value, and returns 0 on success. + + Returns FSL_RC_RANGE if index is out of range for stmt, FSL_RC_MISUSE + if stmt has no result columns. +*/ +FSL_EXPORT int fsl_stmt_get_int64( fsl_stmt * const stmt, int index, int64_t * v ); + +/** + The fsl_id_t counterpart of fsl_stmt_get_int32(). Depending on + the sizeof(fsl_id_t), it behaves as one of fsl_stmt_get_int32() + or fsl_stmt_get_int64(). +*/ +FSL_EXPORT int fsl_stmt_get_id( fsl_stmt * const stmt, int index, fsl_id_t * v ); + +/** + Convenience form of fsl_stmt_get_id() which returns the value + directly but cannot report errors. It returns -1 on error, but + that is not unambiguously an error value. +*/ +FSL_EXPORT fsl_id_t fsl_stmt_g_id( fsl_stmt * const stmt, int index ); + +/** + Convenience form of fsl_stmt_get_int32() which returns the value + directly but cannot report errors. It returns 0 on error, but + that is not unambiguously an error. +*/ +FSL_EXPORT int32_t fsl_stmt_g_int32( fsl_stmt * const stmt, int index ); + +/** + Convenience form of fsl_stmt_get_int64() which returns the value + directly but cannot report errors. It returns 0 on error, but + that is not unambiguously an error. +*/ +FSL_EXPORT int64_t fsl_stmt_g_int64( fsl_stmt * const stmt, int index ); + +/** + Convenience form of fsl_stmt_get_double() which returns the value + directly but cannot report errors. It returns 0 on error, but + that is not unambiguously an error. +*/ +FSL_EXPORT double fsl_stmt_g_double( fsl_stmt * const stmt, int index ); + +/** + Convenience form of fsl_stmt_get_text() which returns the value + directly but cannot report errors. It returns NULL on error, but + that is not unambiguously an error because it also returns NULL + if the column contains an SQL NULL value. If outLen is not NULL + then it is set to the byte length of the returned string. +*/ +FSL_EXPORT char const * fsl_stmt_g_text( fsl_stmt * const stmt, int index, fsl_size_t * outLen ); + +/** + Gets double value from the given 0-based result set column, + assigns *v to that value, and returns 0 on success. + + Returns FSL_RC_RANGE if index is out of range for stmt, FSL_RC_MISUSE + if stmt has no result columns. +*/ +FSL_EXPORT int fsl_stmt_get_double( fsl_stmt * const stmt, int index, double * v ); + +/** + Gets a string value from the given 0-based result set column, + assigns *out (if out is not NULL) to that value, assigns *outLen + (if outLen is not NULL) to *out's length in bytes, and returns 0 + on success. Ownership of the string memory is unchanged - it is owned + by the statement and the caller should immediately copy it if + it will be needed for much longer. + + Returns FSL_RC_RANGE if index is out of range for stmt, FSL_RC_MISUSE + if stmt has no result columns. +*/ +FSL_EXPORT int fsl_stmt_get_text( fsl_stmt * const stmt, int index, char const **out, + fsl_size_t * outLen ); + +/** + The Blob counterpart of fsl_stmt_get_text(). Identical to that + function except that its output result (3rd paramter) type + differs, and it fetches the data as a raw blob, without any sort + of string interpretation. + + Returns FSL_RC_RANGE if index is out of range for stmt, FSL_RC_MISUSE + if stmt has no result columns. +*/ +FSL_EXPORT int fsl_stmt_get_blob( fsl_stmt * const stmt, int index, void const **out, fsl_size_t * outLen ); + +/** + Executes multiple SQL statements, ignoring any results they might + collect. Returns 0 on success, non-0 on error. On error + db->error might be updated to report the problem. +*/ +FSL_EXPORT int fsl_db_exec_multi( fsl_db * const db, const char * sql, ...); + +/** + va_list counterpart of db_exec_multi(). +*/ +FSL_EXPORT int fsl_db_exec_multiv( fsl_db * const db, const char * sql, va_list args); + +/** + Executes a single SQL statement, skipping over any results + it may have. Returns 0 on success. On error db's error state + may be updated. +*/ +FSL_EXPORT int fsl_db_exec( fsl_db * const db, char const * sql, ... ); + +/** + va_list counterpart of fs_db_exec(). +*/ +FSL_EXPORT int fsl_db_execv( fsl_db * const db, char const * sql, va_list args ); + +/** + Begins a transaction on the given db. Nested transactions are + not directly supported but the db handle keeps track of + open/close counts, such that fsl_db_transaction_end() will not + actually do anything until the transaction begin/end counter + goes to 0. Returns FSL_RC_MISUSE if !db or the db is not + connected, else the result of the underlying db call(s). + + Transactions are an easy way to implement "dry-run" mode for + some types of applications. For example: + + ``` + char dryRunMode = ...; + fsl_db_transaction_begin(db); + ...do your stuff... + fsl_db_transaction_end(db, dryRunMode ? 1 : 0); + ``` + + Here's a tip for propagating error codes when using + transactions: + + ``` + ... + if(rc) fsl_db_transaction_end(db, 1); + else rc = fsl_db_transaction_end(db, 0); + ``` + + That ensures that we propagate rc in the face of a rollback but + we also capture the rc for a commit (which might yet fail). Note + that a rollback in and of itself is not an error (though it also + might fail, that would be "highly unusual" and indicative of + other problems), and we certainly don't want to overwrite that + precious non-0 rc with a successful return result from a + rollback (which would, in effect, hide the error from the + client). +*/ +FSL_EXPORT int fsl_db_transaction_begin(fsl_db * const db); + +/** + Equivalent to fsl_db_transaction_end(db, 0). +*/ +FSL_EXPORT int fsl_db_transaction_commit(fsl_db * const db); + +/** + Equivalent to fsl_db_transaction_end(db, 1). +*/ +FSL_EXPORT int fsl_db_transaction_rollback(fsl_db * const db); + +/** + Forces a rollback of any pending transaction in db, regardless + of the internal transaction begin/end counter. Returns + FSL_RC_MISUSE if !db or db is not opened, else returns the value + of the underlying ROLLBACK call. This also re-sets/frees any + transaction-related state held by db (e.g. db->beforeCommit). + Use with care, as this mucks about with db state in a way which + is not all that pretty and it may confuse downstream code. + + Returns 0 on success. +*/ +FSL_EXPORT int fsl_db_rollback_force(fsl_db * const db); + +/** + Decrements the transaction counter incremented by + fsl_db_transaction_begin() and commits or rolls back the + transaction if the counter goes to 0. + + If doRollback is true then this rolls back (or schedules a + rollback of) a transaction started by + fsl_db_transaction_begin(). If doRollback is false is commits + (or schedules a commit). + + If db fsl_db_transaction_begin() is used in a nested manner and + doRollback is true for any one of the nested calls, then that + value will be remembered, such that the downstream calls to this + function within the same transaction will behave like a rollback + even if they pass 0 for the second argument. + + Returns FSL_RC_MISUSE if !db or the db is not opened, 0 if + the transaction counter is above 0, else the result of the + (potentially many) underlying database operations. + + Unfortunate low-level co-dependency: if db->f is not NULL and + (db->role & FSL_DBROLE_REPO) then this function may perform + extra repository-related post-processing on any commit, and + checking the result code is particularly important for those + cases. +*/ +FSL_EXPORT int fsl_db_transaction_end(fsl_db * const db, bool doRollback); + +/** + Returns the given db's current transaction depth. If the value is + negative, its absolute value represents the depth but indicates + that a rollback is pending. If it is positive, the transaction is + still in a "good" state. If it is 0, no transaction is active. +*/ +FSL_EXPORT int fsl_db_transaction_level(fsl_db * const db); + +/** + Runs the given SQL query on the given db and returns true if the + query returns any rows, else false. Returns 0 for any error as + well. +*/ +FSL_EXPORT bool fsl_db_exists(fsl_db * const db, char const * sql, ... ); + +/** + va_list counterpart of fsl_db_exists(). +*/ +FSL_EXPORT bool fsl_db_existsv(fsl_db * const db, char const * sql, va_list args ); + +/** + Runs a fetch-style SQL query against DB and returns the first + column of the first result row via *rv. If the query returns no + rows, *rv is not modified. The intention is that the caller sets + *rv to his preferred default (or sentinel) value before calling + this. + + The format string (the sql parameter) accepts all formatting + options supported by fsl_appendf(). + + Returns 0 on success. On error db's error state is updated and + *rv is not modified. + + Returns FSL_RC_MISUSE without side effects if !db, !rv, !sql, + or !*sql. +*/ +FSL_EXPORT int fsl_db_get_int32( fsl_db * const db, int32_t * rv, + char const * sql, ... ); + +/** + va_list counterpart of fsl_db_get_int32(). +*/ +FSL_EXPORT int fsl_db_get_int32v( fsl_db * const db, int32_t * rv, + char const * sql, va_list args); + +/** + Convenience form of fsl_db_get_int32() which returns the value + directly but provides no way of checking for errors. On error, + or if no result is found, defaultValue is returned. +*/ +FSL_EXPORT int32_t fsl_db_g_int32( fsl_db * const db, + int32_t defaultValue, + char const * sql, ... ); + +/** + The int64 counterpart of fsl_db_get_int32(). See that function + for the semantics. +*/ +FSL_EXPORT int fsl_db_get_int64( fsl_db * db, int64_t * rv, + char const * sql, ... ); + +/** + va_list counterpart of fsl_db_get_int64(). +*/ +FSL_EXPORT int fsl_db_get_int64v( fsl_db * db, int64_t * rv, + char const * sql, va_list args); + +/** + Convenience form of fsl_db_get_int64() which returns the value + directly but provides no way of checking for errors. On error, + or if no result is found, defaultValue is returned. +*/ +FSL_EXPORT int64_t fsl_db_g_int64( fsl_db * db, int64_t defaultValue, + char const * sql, ... ); + + +/** + The fsl_id_t counterpart of fsl_db_get_int32(). See that function + for the semantics. +*/ +FSL_EXPORT int fsl_db_get_id( fsl_db * db, fsl_id_t * rv, + char const * sql, ... ); + +/** + va_list counterpart of fsl_db_get_id(). +*/ +FSL_EXPORT int fsl_db_get_idv( fsl_db * db, fsl_id_t * rv, + char const * sql, va_list args); + +/** + Convenience form of fsl_db_get_id() which returns the value + directly but provides no way of checking for errors. On error, + or if no result is found, defaultValue is returned. +*/ +FSL_EXPORT fsl_id_t fsl_db_g_id( fsl_db * db, fsl_id_t defaultValue, + char const * sql, ... ); + + +/** + The fsl_size_t counterpart of fsl_db_get_int32(). See that + function for the semantics. If this function would fetch a + negative value, it returns FSL_RC_RANGE and *rv is not modified. +*/ +FSL_EXPORT int fsl_db_get_size( fsl_db * db, fsl_size_t * rv, + char const * sql, ... ); + +/** + va_list counterpart of fsl_db_get_size(). +*/ +FSL_EXPORT int fsl_db_get_sizev( fsl_db * db, fsl_size_t * rv, + char const * sql, va_list args); + +/** + Convenience form of fsl_db_get_size() which returns the value + directly but provides no way of checking for errors. On error, + or if no result is found, defaultValue is returned. +*/ +FSL_EXPORT fsl_size_t fsl_db_g_size( fsl_db * db, fsl_size_t defaultValue, + char const * sql, ... ); + + +/** + The double counterpart of fsl_db_get_int32(). See that function + for the semantics. +*/ +FSL_EXPORT int fsl_db_get_double( fsl_db * db, double * rv, + char const * sql, ... ); + +/** + va_list counterpart of fsl_db_get_double(). +*/ +FSL_EXPORT int fsl_db_get_doublev( fsl_db * db, double * rv, + char const * sql, va_list args); + +/** + Convenience form of fsl_db_get_double() which returns the value + directly but provides no way of checking for errors. On error, + or if no result is found, defaultValue is returned. +*/ +FSL_EXPORT double fsl_db_g_double( fsl_db * db, double defaultValue, + char const * sql, ... ); + +/** + The C-string counterpart of fsl_db_get_int32(). On success *rv + will be set to a dynamically allocated string copied from the + first column of the first result row. If rvLen is not NULL then + *rvLen will be assigned the byte-length of that string. If no + row is found, *rv is set to NULL and *rvLen (if not NULL) is set + to 0, and 0 is returned. Note that NULL is also a legal result + (an SQL NULL translates as a NULL string), The caller must + eventually free the returned string value using fsl_free(). +*/ +FSL_EXPORT int fsl_db_get_text( fsl_db * db, char ** rv, fsl_size_t * rvLen, + char const * sql, ... ); + +/** + va_list counterpart of fsl_db_get_text(). +*/ +FSL_EXPORT int fsl_db_get_textv( fsl_db * db, char ** rv, fsl_size_t * rvLen, + char const * sql, va_list args ); + +/** + Convenience form of fsl_db_get_text() which returns the value + directly but provides no way of checking for errors. On error, + or if no result is found, NULL is returned. The returned string + must eventually be passed to fsl_free() to free it. If len is + not NULL then if non-NULL is returned, *len will be assigned the + byte-length of the returned string. +*/ +FSL_EXPORT char * fsl_db_g_text( fsl_db * db, fsl_size_t * len, + char const * sql, + ... ); + +/** + The Blob counterpart of fsl_db_get_text(). Identical to that + function except that its output result (2nd paramter) type + differs, and it fetches the data as a raw blob, without any sort + of string interpretation. The returned *rv memory must + eventually be passed to fsl_free() to free it. If len is not + NULL then on success *len will be set to the byte length of the + returned blob. If no row is found, *rv is set to NULL and *rvLen + (if not NULL) is set to 0, and 0 is returned. Note that NULL is + also a legal result (an SQL NULL translates as a NULL string), +*/ +FSL_EXPORT int fsl_db_get_blob( fsl_db * db, void ** rv, fsl_size_t * len, + char const * sql, ... ); + + +/** + va_list counterpart of fsl_db_get_blob(). +*/ +FSL_EXPORT int fsl_db_get_blobv( fsl_db * db, void ** rv, fsl_size_t * stmtLen, + char const * sql, va_list args ); + +/** + Convenience form of fsl_db_get_blob() which returns the value + directly but provides no way of checking for errors. On error, + or if no result is found, NULL is returned. +*/ +FSL_EXPORT void * fsl_db_g_blob( fsl_db * db, fsl_size_t * len, + char const * sql, + ... ); +/** + Similar to fsl_db_get_text() and fsl_db_get_blob(), but writes + its result to tgt, overwriting (not appennding to) any existing + memory it might hold. + + If asBlob is true then the underlying BLOB API is used to + populate the buffer, else the underlying STRING/TEXT API is + used. For many purposes there will be no difference, but if you + know you might have binary data, be sure to pass a true value + for asBlob to avoid any potential encoding-related problems. +*/ +FSL_EXPORT int fsl_db_get_buffer( fsl_db * db, fsl_buffer * tgt, + char asBlob, + char const * sql, ... ); + +/** + va_list counterpart of fsl_db_get_buffer(). +*/ +FSL_EXPORT int fsl_db_get_bufferv( fsl_db * db, fsl_buffer * tgt, + char asBlob, + char const * sql, va_list args ); + + +/** + Expects sql to be a SELECT-style query which (potentially) + returns a result set. For each row in the set callback() is + called, as described for fsl_stmt_each(). Returns 0 on success. + The callback is _not_ called for queries which return no + rows. If clients need to know if rows were returned, they can + add a counter to their callbackState and increment it from the + callback. + + Returns FSL_RC_MISUSE if !db, db is not opened, !callback, + !sql. Returns FSL_RC_RANGE if !*sql. +*/ +FSL_EXPORT int fsl_db_each( fsl_db * db, fsl_stmt_each_f callback, + void * callbackState, char const * sql, ... ); + +/** + va_list counterpart to fsl_db_each(). +*/ +FSL_EXPORT int fsl_db_eachv( fsl_db * db, fsl_stmt_each_f callback, + void * callbackState, char const * sql, va_list args ); + + +/** + Returns the given Julian date value formatted as an ISO8601 + string (with a fractional seconds part if msPrecision is true, + else without it). Returns NULL if !db, db is not connected, j + is less than 0, or on allocation error. The returned memory must + eventually be freed using fsl_free(). + + If localTime is true then the value is converted to the local time, + otherwise it is not. + + @see fsl_db_unix_to_iso8601() + @see fsl_julian_to_iso8601() + @see fsl_iso8601_to_julian() +*/ +FSL_EXPORT char * fsl_db_julian_to_iso8601( fsl_db * const db, double j, + bool msPrecision, bool localTime ); + +/** + Returns the given Julian date value formatted as an ISO8601 + string (with a fractional seconds part if msPrecision is true, + else without it). Returns NULL if !db, db is not connected, j + is less than 0, or on allocation error. The returned memory must + eventually be freed using fsl_free(). + + If localTime is true then the value is converted to the local time, + otherwise it is not. + + @see fsl_db_julian_to_iso8601() + @see fsl_julian_to_iso8601() + @see fsl_iso8601_to_julian() +*/ +FSL_EXPORT char * fsl_db_unix_to_iso8601( fsl_db * const db, fsl_time_t j, + bool localTime ); + + +/** + Returns the current time in Julian Date format. Returns a negative + value if !db or db is not opened. +*/ +FSL_EXPORT double fsl_db_julian_now(fsl_db * db); + +/** + Uses the given db to convert the given time string to Julian Day + format. If it cannot be converted, a negative value is returned. + The str parameter can be anything suitable for passing to sqlite's: + + SELECT julianday(str) + + Note that this routine will escape str for use with SQL - the + caller must not do so. + + @see fsl_julian_to_iso8601() + @see fsl_iso8601_to_julian() +*/ +FSL_EXPORT double fsl_db_string_to_julian(fsl_db * db, char const * str); + +/** + Opens the given db file and populates db with its handle. db + must have been cleanly initialized by copy-initializing it from + fsl_db_empty (or fsl_db_empty_m) or by allocating it using + fsl_db_malloc(). Failure to do so will lead to undefined + behaviour. + + openFlags may be a mask of FSL_OPEN_F_xxx values, but not all + are used/supported here. If FSL_OPEN_F_CREATE is _not_ set in + openFlags and dbFile does not exist, it will return + FSL_RC_NOT_FOUND. The existence of FSL_OPEN_F_CREATE in the + flags will cause this routine to try to create the file if + needed. If conflicting flags are specified (e.g. FSL_OPEN_F_RO + and FSL_OPEN_F_RWC) then which one takes precedence is + unspecified and possibly unpredictable. + + As a special case, if dbFile is ":memory:" (for an in-memory + database) or "" (empty string, for a "temporary" database) then it + is is passed through without any filesystem-related checks and the + openFlags are ignored. + + See this page for the differences between ":memory:" and "": + + https://www.sqlite.org/inmemorydb.html + + Returns FSL_RC_MISUSE if !db, !dbFile, !*dbFile, or if db->dbh + is not NULL (i.e. if it is already opened or its memory was + default-initialized (use fsl_db_empty to cleanly copy-initialize + new stack-allocated instances). + + On error db->dbh will be NULL, but db->error might contain error + details. + + Regardless of success or failure, db should be passed to + fsl_db_close() to free up all memory associated with it. It is + not closed automatically by this function because doing so cleans + up the error state, which the caller will presumably want to + have. + + If db->f is not NULL when this is called then it is assumed that + db should be plugged in to the Fossil repository system, and the + following additional things happen: + + - A number of SQL functions are registered with the db. Details + are below. + + - If FSL_OPEN_F_SCHEMA_VALIDATE is set in openFlags then the + db is validated to see if it has a fossil schema. If that + validation fails, FSL_RC_REPO_NEEDS_REBUILD or FSL_RC_NOT_A_REPO + will be returned and db's error state will be updated. db->f + does not need to be set for that check to work. + + + If db->f is not NULL when this function is called then a number of + fossil-specific SQL-accessible functions are installed. See the + file doc/db-udf.md in the libfossil source tree for complete + docs. Note that functions in those docs described as "triggering a + db error" will propagate that error, such that fsl_db_err_get() can + report it to the client. + + @see fsl_db_close() + @see fsl_db_prepare() + @see fsl_db_malloc() +*/ +FSL_EXPORT int fsl_db_open( fsl_db * const db, char const * dbFile, + int openFlags ); + +/** + Closes the given db handle and frees any resources owned by + db. This is a no-op if db is NULL. + + If db was allocated using fsl_db_malloc() (as determined by + examining db->allocStamp) then this routine also fsl_free()s it, + otherwise it is assumed to either be on the stack or part of a + larger struct and is not freed. + + If db has any pending transactions, they are rolled + back by this function. +*/ +FSL_EXPORT void fsl_db_close( fsl_db * const db ); + +/** + If db is an opened db handle, this registers a debugging + function with the db which traces all SQL to the given FILE + handle (defaults to stdout if outStream is NULL). + + This mechanism is only intended for debugging and exploration of + how Fossil works. Tracing is often as easy way to ensure that a + given code block is getting run. + + As a special case, if db->f is not NULL _before_ it is is + fsl_db_open()ed, then this function automatically gets installed + if the SQL tracing option is enabled for that fsl_cx instance + before the db is opened. + + This is a no-op if !db or db is not opened. +*/ +FSL_EXPORT void fsl_db_sqltrace_enable( fsl_db * const db, FILE * outStream ); + +/** + Returns the row ID of the most recent insertion, + or -1 if !db, db is not connected, or 0 if no inserts + have been performed. +*/ +FSL_EXPORT fsl_id_t fsl_db_last_insert_id(fsl_db * const db); + +/** + Returns non-0 (true) if the database (which must be open) table + identified by zTableName has a column named zColName + (case-sensitive), else returns 0. +*/ +FSL_EXPORT bool fsl_db_table_has_column( fsl_db * const db, + char const *zTableName, + char const *zColName ); + +/** + If a db name has been associated with db then it is returned, + otherwise NULL is returned. A db has no name by default, but + fsl_cx-used ones get their database name assigned to them + (e.g. "main" for the main db). +*/ +FSL_EXPORT char const * fsl_db_name(fsl_db const * const db); + + +/** + Returns a db name string for the given fsl_db_role value. The + string is static, guaranteed to live as long as the app. It + returns NULL (or asserts in debug builds) if passed + FSL_DBROLE_NONE or some value out of range for the enum. +*/ +FSL_EXPORT const char * fsl_db_role_label(enum fsl_dbrole_e r); + + +/** + Allocates a new fsl_db instance(). Returns NULL on allocation + error. Note that fsl_db instances can often be used from the + stack - allocating them dynamically is an uncommon case necessary + for script bindings. + + Achtung: the returned value's allocStamp member is used for + determining if fsl_db_close() should free the value or not. Thus + if clients copy over this value without adjusting allocStamp back + to its original value, the library will likely leak the instance. + Been there, done that. +*/ +FSL_EXPORT fsl_db * fsl_db_malloc(); + +/** + The fsl_stmt counterpart of fsl_db_malloc(). See that function + for when you might want to use this and a caveat involving the + allocStamp member of the returned value. fsl_stmt_finalize() will + free statements created with this function. +*/ +FSL_EXPORT fsl_stmt * fsl_stmt_malloc(); + +/** + ATTACHes the file zDbName to db using the databbase name + zLabel. Returns 0 on success. Returns FSL_RC_MISUSE if any + argument is NULL or any string argument starts with a NUL byte, + else it returns the result of fsl_db_exec() which attaches the + db. On db-level errors db's error state will be updated. +*/ +FSL_EXPORT int fsl_db_attach(fsl_db * const db, const char *zDbName, + const char *zLabel); + +/** + The converse of fsl_db_detach(). Must be passed the same arguments + which were passed as the 1st and 3rd arguments to fsl_db_attach(). + Returns 0 on success, FSL_RC_MISUSE if !db, !zLabel, or !*zLabel, + else it returns the result of the underlying fsl_db_exec() + call. +*/ +FSL_EXPORT int fsl_db_detach(fsl_db * const db, const char *zLabel); + +/** + Expects fmt to be a SELECT-style query. For each row in the + query, the first column is fetched as a string and appended to + the tgt list. + + Returns 0 on success, FSL_RC_MISUSE if !db, !tgt, or !fmt, any + number of potential FSL_RC_OOM or db-related errors. + + Results rows with a NULL value (resulting from an SQL NULL) are + added to the list as NULL entries. + + Each entry appended to the list is a (char *) which must + be freed using fsl_free(). To easiest way to clean up + the list and its contents is: + + ``` + fsl_list_visit_free(tgt,...); + ``` + + On error the list may be partially populated. + + Complete example: + + ``` + fsl_list li = fsl_list_empty; + int rc = fsl_db_select_slist(db, &li, + "SELECT uuid FROM blob WHERE rid<20"); + if(!rc){ + fsl_size_t i; + for(i = 0;i < li.used; ++i){ + char const * uuid = (char const *)li.list[i]; + fsl_fprintf(stdout, "UUID: %s\n", uuid); + } + } + fsl_list_visit_free(&li, 1); + ``` + + Of course fsl_list_visit() may be used to traverse the list as + well, as long as the visitor expects (char [const]*) list + elements. +*/ +FSL_EXPORT int fsl_db_select_slist( fsl_db * const db, fsl_list * tgt, + char const * fmt, ... ); + +/** + The va_list counterpart of fsl_db_select_slist(). +*/ +FSL_EXPORT int fsl_db_select_slistv( fsl_db * const db, fsl_list * tgt, + char const * fmt, va_list args ); + +/** + Returns n bytes of random lower-case hexidecimal characters + using the given db as its data source, plus a terminating NUL + byte. The returned memory must eventually be freed using + fsl_free(). Returns NULL if !db, !n, or on a db-level error. +*/ +FSL_EXPORT char * fsl_db_random_hex(fsl_db * db, fsl_size_t n); + +/** + Returns the "number of database rows that were changed or + inserted or deleted by the most recently completed SQL statement" + (to quote the underlying APIs). Returns 0 if !db or if db is not + opened. + + + See: https://sqlite.org/c3ref/changes.html +*/ +FSL_EXPORT int fsl_db_changes_recent(fsl_db * const db); + +/** + Returns "the number of row changes caused by INSERT, UPDATE or + DELETE statements since the database connection was opened" (to + quote the underlying APIs). Returns 0 if !db or if db is not + opened. + + See; https://sqlite.org/c3ref/total_changes.html +*/ +FSL_EXPORT int fsl_db_changes_total(fsl_db * const db); + +/** + Initializes the given database file. zFilename is the name of + the db file. It is created if needed, but any directory + components are not created. zSchema is the base schema to + install. The following arguments may be (char const *) SQL + code, each of which gets run against the db after the main + schema is called. The variadic argument list MUST end with NULL + (0), even if there are no non-NULL entries. + + Returns 0 on success. + + On error, if err is not NULL then it is populated with any error + state from the underlying (temporary) db handle. +*/ +FSL_EXPORT int fsl_db_init( fsl_error * err, char const * zFilename, + char const * zSchema, ... ); + +/** + A fsl_stmt_each_f() impl, intended primarily for debugging, which + simply outputs row data in tabular form via fsl_output(). The + state argument is ignored. This only works if stmt was prepared + by a fsl_db instance which has an associated fsl_cx instance. On + the first row, the column names are output. +*/ +FSL_EXPORT int fsl_stmt_each_f_dump( fsl_stmt * const stmt, void * state ); + +/** + Returns true if the table name specified by the final argument + exists in the fossil database specified by the 2nd argument on the + db connection specified by the first argument, else returns false. + + Trivia: this is one of the few libfossil APIs which makes use of + FSL_DBROLE_TEMP. + + Potential TODO: this is a bit of a wonky interface. Consider + changing it to eliminate the role argument, which is only really + needed if we have duplicate table names across attached dbs or if + we internally mess up and write a table to the wrong db. +*/ +FSL_EXPORT bool fsl_db_table_exists(fsl_db * const db, fsl_dbrole_e whichDb, + const char *zTable); + +/** + The elipsis counterpart of fsl_stmt_bind_fmtv(). +*/ +FSL_EXPORT int fsl_stmt_bind_fmt( fsl_stmt * const st, char const * fmt, ... ); + +/** + Binds a series of values using a formatting string. + + The string may contain the following characters, each of which + refers to the next argument in the args list: + + '-': binds a NULL and expects a NULL placeholder + in the argument list (for consistency's sake). + + 'i': binds an int32 + + 'I': binds an int64 + + 'R': binds a fsl_id_t ('R' as in 'RID') + + 'f': binds a double + + 's': binds a (char const *) as text or NULL. + + 'S': binds a (char const *) as a blob or NULL. + + 'b': binds a (fsl_buffer const *) as text or NULL. + + 'B': binds a (fsl_buffer const *) as a blob or NULL. + + ' ': spaces are allowed for readability and are ignored. + + Returns 0 on success, any number of other FSL_RC_xxx codes on + error. + + ACHTUNG: the "sSbB" bindings assume, because of how this API is + normally used, that the memory pointed to by the given argument + will outlive the pending step of the given statement, so that + memory is NOT copied by the binding. Thus results are undefined if + such an argument's memory is invalidated before the statement is + done with it. +*/ +FSL_EXPORT int fsl_stmt_bind_fmtv( fsl_stmt * const st, char const * fmt, + va_list args ); + +/** + Works like fsl_stmt_bind_fmt() but: + + 1) It calls fsl_stmt_reset() before binding the arguments. + + 2) If binding succeeds then it steps the given statement a single + time. + + 3) If the result is NOT FSL_RC_STEP_ROW then it also resets the + statement before returning. It does not do so for FSL_RC_STEP_ROW + because doing so would remove the fetched columns (and this is why + it resets in step (1)). + + Returns 0 if stepping results in FSL_RC_STEP_DONE, FSL_RC_STEP_ROW + if it produces a result row, or any number of other potential non-0 + codes on error. On error, the error state of st->db is updated. + + Design note: the return value for FSL_RC_STEP_ROW, as opposed to + returning 0, is necessary for proper statement use if the client + wants to fetch any result data from the statement afterwards (which + is illegal if FSL_RC_STEP_ROW was not the result). This is also why + it cannot reset the statement if that result is returned. +*/ +FSL_EXPORT int fsl_stmt_bind_stepv( fsl_stmt * const st, char const * fmt, + va_list args ); + +/** + The elipsis counterpart of fsl_stmt_bind_stepv(). +*/ +FSL_EXPORT int fsl_stmt_bind_step( fsl_stmt * st, char const * fmt, ... ); + +#if defined(__cplusplus) +} /*extern "C"*/ +#endif +#endif +/* ORG_FOSSIL_SCM_FSL_DB_H_INCLUDED */ +/* end of file ../include/fossil-scm/fossil-db.h */ +/* start of file ../include/fossil-scm/fossil-hash.h */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#if !defined(ORG_FOSSIL_SCM_FSL_HASH_H_INCLUDED) +#define ORG_FOSSIL_SCM_FSL_HASH_H_INCLUDED +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). + + ***************************************************************************** + This file declares public APIs relating to hashing. +*/ + +#if !defined(FSL_SHA1_HARDENED) +# define FSL_SHA1_HARDENED 1 +#endif +#if defined(__cplusplus) +extern "C" { +#endif + +/** + Various set-in-stone constants used by the API. +*/ +enum fsl_hash_constants { +/** + The length, in bytes, of fossil's hex-form SHA1 UUID strings. +*/ +FSL_STRLEN_SHA1 = 40, +/** + The length, in bytes, of fossil's hex-form SHA3-256 UUID strings. +*/ +FSL_STRLEN_K256 = 64, +/** + The length, in bytes, of a hex-form MD5 hash. +*/ +FSL_STRLEN_MD5 = 32, + +/** Minimum length of a full UUID. */ +FSL_UUID_STRLEN_MIN = FSL_STRLEN_SHA1, +/** Maximum length of a full UUID. */ +FSL_UUID_STRLEN_MAX = FSL_STRLEN_K256 +}; + +/** + Unique IDs for artifact hash types supported by fossil. +*/ +enum fsl_hash_types_e { +/** Invalid hash type. */ +FSL_HTYPE_ERROR = 0, +/** SHA1. */ +FSL_HTYPE_SHA1 = 1, +/** SHA3-256. */ +FSL_HTYPE_K256 = 2 +}; +typedef enum fsl_hash_types_e fsl_hash_types_e; + +typedef struct fsl_md5_cx fsl_md5_cx; +typedef struct fsl_sha1_cx fsl_sha1_cx; +typedef struct fsl_sha3_cx fsl_sha3_cx; + +/** + The hash string of the initial MD5 state. Used as an + optimization for some places where we need an MD5 but know it + will not hash any data. + + Equivalent to what the md5sum command outputs for empty input: + + ``` + # md5sum < /dev/null + d41d8cd98f00b204e9800998ecf8427e - + ``` +*/ +#define FSL_MD5_INITIAL_HASH "d41d8cd98f00b204e9800998ecf8427e" + +/** + Holds state for MD5 calculations. It is intended to be used like + this: + + ``` + unsigned char digest[16]; + char hex[FSL_STRLEN_MD5+1]; + fsl_md5_cx cx = fsl_md5_cx_empty; + // alternately: fsl_md5_init(&cx); + ...call fsl_md5_update(&cx,...) any number of times to + ...incrementally calculate the hash. + fsl_md5_final(&cx, digest); // ends the calculation + fsl_md5_digest_to_base16(digest, hex); + // digest now contains the raw 16-byte MD5 digest. + // hex now contains the 32-byte MD5 + a trailing NUL + ``` +*/ +struct fsl_md5_cx { + int isInit; + uint32_t buf[4]; + uint32_t bits[2]; + unsigned char in[64]; +}; +#define fsl_md5_cx_empty_m { \ + 1/*isInit*/, \ + {/*buf*/0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476 }, \ + {/*bits*/0,0}, \ + {0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, \ + 0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, \ + 0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0, \ + 0,0,0,0}} + +/** + A fsl_md5_cx instance which holds the initial state + used for md5 calculations. Instances must either be + copy-initialized from this instance or they must be + passed to fsl_md5_init() before they are used. +*/ +FSL_EXPORT const fsl_md5_cx fsl_md5_cx_empty; + +/** + Initializes the given context pointer. It must not be NULL. This + must be the first routine called on any fsl_md5_cx instances. + Alternately, copy-constructing fsl_md5_cx_empty has the same effect. + + @see fsl_md5_update() + @see fsl_md5_final() +*/ +FSL_EXPORT void fsl_md5_init(fsl_md5_cx *cx); + +/** + Updates cx's state to reflect the addition of the data + specified by the range (buf, buf+len]. Neither cx nor buf may + be NULL. This may be called an arbitrary number of times between + fsl_md5_init() and fsl_md5_final(). + + @see fsl_md5_init() + @see fsl_md5_final() +*/ +FSL_EXPORT void fsl_md5_update(fsl_md5_cx *cx, void const * buf, fsl_size_t len); + +/** + Finishes up the calculation of the md5 for the given context and + writes a 16-byte digest value to the 2nd parameter. Use + fsl_md5_digest_to_base16() to convert the digest output value to + hexadecimal form. + + @see fsl_md5_init() + @see fsl_md5_update() + @see fsl_md5_digest_to_base16() +*/ +FSL_EXPORT void fsl_md5_final(fsl_md5_cx * cx, unsigned char * digest); + +/** + Converts an md5 digest value (from fsl_md5_final()'s 2nd + parameter) to a 32-byte (FSL_STRLEN_MD5) CRC string plus a + terminating NUL byte. i.e. zBuf must be at least + (FSL_STRLEN_MD5+1) bytes long. + + @see fsl_md5_final() +*/ +FSL_EXPORT void fsl_md5_digest_to_base16(unsigned char *digest, char *zBuf); + +/** + The md5 counterpart of fsl_sha1sum_buffer(), identical in + semantics except that its result is an MD5 hash instead of an + SHA1 hash and the resulting hex string is FSL_STRLEN_MD5 bytes + long plus a terminating NUL. +*/ +FSL_EXPORT int fsl_md5sum_buffer(fsl_buffer const *pIn, fsl_buffer *pCksum); + +/** + The md5 counterpart of fsl_sha1sum_cstr(), identical in + semantics except that its result is an MD5 hash instead of an + SHA1 hash and the resulting string is FSL_STRLEN_MD5 bytes long + plus a terminating NUL. +*/ +FSL_EXPORT char *fsl_md5sum_cstr(const char *zIn, fsl_int_t len); + +/** + The MD5 counter part to fsl_sha1sum_stream(), with identical + semantics except that the generated hash is an MD5 string + instead of SHA1. +*/ +FSL_EXPORT int fsl_md5sum_stream(fsl_input_f src, void * srcState, fsl_buffer *pCksum); + +/** + Reads all input from src() and passes it through fsl_md5_update(cx,...). + Returns 0 on success, FSL_RC_MISUSE if !cx or !src. If src returns + a non-0 code, that code is returned from here. +*/ +FSL_EXPORT int fsl_md5_update_stream(fsl_md5_cx *cx, fsl_input_f src, void * srcState); + +/** + Equivalent to fsl_md5_update(cx, b->mem, b->used). Results are undefined + if either pointer is invalid or NULL. +*/ +FSL_EXPORT void fsl_md5_update_buffer(fsl_md5_cx *cx, fsl_buffer const * b); + +/** + Passes the first len bytes of str to fsl_md5_update(cx). If len + is less than 0 then fsl_strlen() is used to calculate the + length. Results are undefined if either pointer is invalid or + NULL. This is a no-op if !len or (len<0 && !*str). +*/ +FSL_EXPORT void fsl_md5_update_cstr(fsl_md5_cx *cx, char const * str, fsl_int_t len); + +/** + A fsl_md5_update_stream() proxy which updates cx to include the + contents of the given file. +*/ +FSL_EXPORT int fsl_md5_update_filename(fsl_md5_cx *cx, char const * fname); + +/** + The MD5 counter part to fsl_sha1sum_filename(), with identical + semantics except that the generated hash is an MD5 string + instead of SHA1. +*/ +FSL_EXPORT int fsl_md5sum_filename(const char *zFilename, fsl_buffer *pCksum); + + +#if FSL_SHA1_HARDENED +typedef void(*fsl_sha1h_collision_callback)(uint64_t, const uint32_t*, const uint32_t*, const uint32_t*, const uint32_t*); +#endif +/** + Holds state for SHA1 calculations. It is intended to be used + like this: + + ``` + unsigned char digest[20] + char hex[FSL_STRLEN_SHA1+1]; + fsl_sha1_cx cx = fsl_sha1_cx_empty; + // alternately: fsl_sha1_init(&cx) + ...call fsl_sha1_update(&cx,...) any number of times to + ...incrementally calculate the hash. + fsl_sha1_final(&cx, digest); // ends the calculation + fsl_sha1_digest_to_base16(digest, hex); + // digest now contains the raw 20-byte SHA1 digest. + // hex now contains the 40-byte SHA1 + a trailing NUL + ``` +*/ +struct fsl_sha1_cx { +#if FSL_SHA1_HARDENED + uint64_t total; + uint32_t ihv[5]; + unsigned char buffer[64]; + int bigendian; + int found_collision; + int safe_hash; + int detect_coll; + int ubc_check; + int reduced_round_coll; + fsl_sha1h_collision_callback callback; + uint32_t ihv1[5]; + uint32_t ihv2[5]; + uint32_t m1[80]; + uint32_t m2[80]; + uint32_t states[80][5]; +#else + unsigned int state[5]; + unsigned int count[2]; + unsigned char buffer[64]; +#endif +}; +/** + fsl_sha1_cx instance intended for in-struct copy initialization. +*/ +#if FSL_SHA1_HARDENED +#define fsl_sha1_cx_empty_m {0} +#else +#define fsl_sha1_cx_empty_m { \ + {0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0 }, \ + {0,0}, \ + {0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, \ + 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0 \ + } \ + } +#endif +/** + fsl_sha1_cx instance intended for copy initialization. For build + config portability, the copied-to object must still be passed to + fsl_sha1_init() to initialize it. +*/ +FSL_EXPORT const fsl_sha1_cx fsl_sha1_cx_empty; + +/** + Initializes the given context with the initial SHA1 state. This + must be the first routine called on an SHA1 context, and passing + this context to other SHA1 routines without first having passed + it to this will lead to undefined results. + + @see fsl_sha1_update() + @see fsl_sha1_final() +*/ +FSL_EXPORT void fsl_sha1_init(fsl_sha1_cx *context); + +/** + Updates the given context to include the hash of the first len + bytes of the given data. + + @see fsl_sha1_init() + @see fsl_sha1_final() +*/ +FSL_EXPORT void fsl_sha1_update( fsl_sha1_cx *context, void const *data, fsl_size_t len); + +/** + Add padding and finalizes the message digest. If digest is not NULL + then it writes 20 bytes of digest to the 2nd parameter. If this + library is configured with hardened SHA1 hashes, this function + returns non-0 if a collision was detected while hashing. If it is + not configured for hardened SHA1, or no collision was detected, it + returns 0. + + @see fsl_sha1_update() + @see fsl_sha1_digest_to_base16() +*/ +FSL_EXPORT int fsl_sha1_final(fsl_sha1_cx *context, unsigned char * digest); + +/** + A convenience form of fsl_sha1_final() which writes + FSL_STRLEN_SHA1+1 bytes (hash plus terminating NUL byte) to the + 2nd argument and returns a (const char *)-type cast of the 2nd + argument. +*/ +FSL_EXPORT const char * fsl_sha1_final_hex(fsl_sha1_cx *context, char * zHex); + +/** + Convert a digest into base-16. digest must be at least 20 bytes + long and hold an SHA1 digest. zBuf must be at least (FSL_STRLEN_SHA1 + + 1) bytes long, to which FSL_STRLEN_SHA1 characters of + hexidecimal-form SHA1 hash and 1 NUL byte will be written. + + @see fsl_sha1_final() +*/ +FSL_EXPORT void fsl_sha1_digest_to_base16(unsigned char *digest, char *zBuf); + +/** + Computes the SHA1 checksum of pIn and stores the resulting + checksum in the buffer pCksum. pCksum's memory is re-used if is + has any allocated to it. pCksum may == pIn, in which case this + is a destructive operation (replacing the hashed data with its + hash code). + + Return 0 on success, FSL_RC_OOM if (re)allocating pCksum fails. +*/ +FSL_EXPORT int fsl_sha1sum_buffer(fsl_buffer const *pIn, fsl_buffer *pCksum); + +/** + Computes the SHA1 checksum of the first len bytes of the given + string. If len is negative then zIn must be NUL-terminated and + fsl_strlen() is used to find its length. The result is a + FSL_UUID_STRLEN-byte string (+NUL byte) returned in memory + obtained from fsl_malloc(), so it must be passed to fsl_free() + to free it. If NULL==zIn or !len then NULL is returned. +*/ +FSL_EXPORT char *fsl_sha1sum_cstr(const char *zIn, fsl_int_t len); + +/** + Consumes all input from src and calculates its SHA1 hash. The + result is set in pCksum (its contents, if any, are overwritten, + not appended to). Returns 0 on success. Returns FSL_RC_MISUSE if + !src or !pCksum. It keeps consuming input from src() until that + function reads fewer bytes than requested, at which point EOF is + assumed. If src() returns non-0, that code is returned from this + function. +*/ +FSL_EXPORT int fsl_sha1sum_stream(fsl_input_f src, void * srcState, fsl_buffer *pCksum); + + +/** + A fsl_sha1sum_stream() wrapper which calculates the SHA1 of + given file. + + Returns FSL_RC_IO if the file cannot be opened, FSL_RC_MISUSE if + !zFilename or !pCksum, else as per fsl_sha1sum_stream(). + + TODO: the v1 impl has special behaviour for symlinks which this + function lacks. For that support we need a variant of this + function which takes a fsl_cx parameter (for the allow-symlinks + setting). +*/ +FSL_EXPORT int fsl_sha1sum_filename(const char *zFilename, fsl_buffer *pCksum); + +/** + Legal values for SHA3 hash sizes, in bits: an increment of 32 bits + in the inclusive range (128..512). + + The hexidecimal-code size, in bytes, of any given bit size in this + enum is the bit size/4. +*/ +enum fsl_sha3_hash_size { +/** Sentinel value. Must be 0. */ +FSL_SHA3_INVALID = 0, +FSL_SHA3_128 = 128, FSL_SHA3_160 = 160, FSL_SHA3_192 = 192, +FSL_SHA3_224 = 224, FSL_SHA3_256 = 256, FSL_SHA3_288 = 288, +FSL_SHA3_320 = 320, FSL_SHA3_352 = 352, FSL_SHA3_384 = 384, +FSL_SHA3_416 = 416, FSL_SHA3_448 = 448, FSL_SHA3_480 = 480, +FSL_SHA3_512 = 512, +/* Default SHA3 flavor */ +FSL_SHA3_DEFAULT = 256 +}; + +/** + Type for holding SHA3 processing state. Each instance must be + initialized with fsl_sha3_init(), populated with fsl_sha3_update(), + and "sealed" with fsl_sha3_end(). + + Sample usage: + + ``` + fsl_sha3_cx cx; + fsl_sha3_init(&cx, FSL_SHA3_DEFAULT); + fsl_sha3_update(&cx, memory, lengthOfMemory); + fsl_sha3_end(&cx); + printf("Hash = %s\n", (char const *)cx.hex); + ``` + + After fsl_sha3_end() is called cx.hex contains the hex-string forms + of the digest. Note that fsl_sha3_update() may be called an arbitrary + number of times to feed in chunks of memory (e.g. to stream in + arbitrarily large data). +*/ +struct fsl_sha3_cx { + union { + uint64_t s[25]; /* Keccak state. 5x5 lines of 64 bits each */ + unsigned char x[1600]; /* ... or 1600 bytes */ + } u; + unsigned nRate; /* Bytes of input accepted per Keccak iteration */ + unsigned nLoaded; /* Input bytes loaded into u.x[] so far this cycle */ + unsigned ixMask; /* Insert next input into u.x[nLoaded^ixMask]. */ + enum fsl_sha3_hash_size size; /* Size of the hash, in bits. */ + unsigned char hex[132]; /* Hex form of final digest: 56-128 bytes + plus terminating NUL. */ +}; + +/** + If the given number is a valid fsl_sha3_hash_size value, its enum + entry is returned, else FSL_SHA3_INVALID is returned. + + @see fsl_sha3_init() +*/ +FSL_EXPORT enum fsl_sha3_hash_size fsl_sha3_hash_size_for_int(int); + +/** + Initialize a new hash. The second argument specifies the size of + the hash in bits. Results are undefined if cx is NULL or sz is not + a valid positive value. + + After calling this, use fsl_sha3_update() to hash data and + fsl_sha3_end() to finalize the hashing process and generate a digest. +*/ +FSL_EXPORT void fsl_sha3_init2(fsl_sha3_cx *cx, enum fsl_sha3_hash_size sz); + +/** + Equivalent to fsl_sha3_init2(cx, FSL_SHA3_DEFAULT). +*/ +FSL_EXPORT void fsl_sha3_init(fsl_sha3_cx *cx); + +/** + Updates cx's state to include the first len bytes of data. + + If cx is NULL results are undefined (segfault!). If mem is not + NULL then it must be at least n bytes long. If n is 0 then this + function has no side-effects. + + @see fsl_sha3_init() + @see fsl_sha3_end() +*/ +FSL_EXPORT void fsl_sha3_update( fsl_sha3_cx *cx, void const *data, unsigned int len); + +/** + To be called when SHA3 hashing is complete: finishes the hash + calculation and populates cx->hex with the final hash code in + hexidecimal-string form. Returns the binary-form digest value, + which refers to cx->size/8 bytes of memory which lives in the cx + object. After this call cx->hex will be populated with cx->size/4 + bytes of lower-case ASCII hex codes plus a terminating NUL byte. + + Potential TODO: change fsl_sha1_final() and fsl_md5_final() to use + these same return semantics. + + @see fsl_sha3_init() + @see fsl_sha3_update() +*/ +FSL_EXPORT unsigned char const * fsl_sha3_end(fsl_sha3_cx *cx); + +/** + SHA3-256 counterpart of fsl_sha1_digest_to_base16(). digest must be at least + 32 bytes long and hold an SHA3 digest. zBuf must be at least (FSL_STRLEN_K256+1) + bytes long, to which FSL_STRLEN_K256 characters of + hexidecimal-form SHA3 hash and 1 NUL byte will be written + + @see fsl_sha3_end(). +*/ +FSL_EXPORT void fsl_sha3_digest_to_base16(unsigned char *digest, char *zBuf); +/** + SHA3 counter part of fsl_sha1sum_buffer(). +*/ +FSL_EXPORT int fsl_sha3sum_buffer(fsl_buffer const *pIn, fsl_buffer *pCksum); +/** + SHA3 counter part of fsl_sha1sum_cstr(). +*/ +FSL_EXPORT char *fsl_sha3sum_cstr(const char *zIn, fsl_int_t len); +/** + SHA3 counterpart of fsl_sha1sum_stream(). + */ +FSL_EXPORT int fsl_sha3sum_stream(fsl_input_f src, void * srcState, fsl_buffer *pCksum); +/** + SHA3 counterpart of fsl_sha1sum_filename(). + */ +FSL_EXPORT int fsl_sha3sum_filename(const char *zFilename, fsl_buffer *pCksum); + +/** + Expects zHash to be a full-length hash value of one of the + fsl_hash_types_t-specified types, and nHash to be the length, in + bytes, of zHash's contents (which must be the full hash length, not + a prefix). If zHash can be validated as a hash, its corresponding + hash type is returned, else FSL_HTYPE_ERROR is returned. +*/ +FSL_EXPORT fsl_hash_types_e fsl_validate_hash(const char *zHash, int nHash); + +/** + Expects (zHash, nHash) to refer to a full hash (of a supported + content hash type) of pIn's contents. This routine hashes pIn's + contents and, if it compares equivalent to zHash then the ID of the + hash type is returned. On a mismatch, FSL_HTYPE_ERROR is returned. +*/ +FSL_EXPORT fsl_hash_types_e fsl_verify_blob_hash(fsl_buffer const * pIn, + const char *zHash, int nHash); + +/** + Returns a human-readable name for the given hash type, or its + second argument h is not a supported hash type. + */ +FSL_EXPORT const char * fsl_hash_type_name(fsl_hash_types_e h, const char *zUnknown); + +#if defined(__cplusplus) +} /*extern "C"*/ +#endif +#endif +/* ORG_FOSSIL_SCM_FSL_HASH_H_INCLUDED */ +/* end of file ../include/fossil-scm/fossil-hash.h */ +/* start of file ../include/fossil-scm/fossil-repo.h */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#if !defined(ORG_FOSSIL_SCM_FSL_REPO_H_INCLUDED) +#define ORG_FOSSIL_SCM_FSL_REPO_H_INCLUDED +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ + +/** @file fossil-repo.h + + fossil-repo.h declares APIs specifically dealing with + repository-db-side state, as opposed to specifically checkout-side + state or non-content-related APIs. +*/ + + +#if defined(__cplusplus) +extern "C" { +#endif + +typedef struct fsl_card_F fsl_card_F; +typedef struct fsl_card_J fsl_card_J; +typedef struct fsl_card_Q fsl_card_Q; +typedef struct fsl_card_T fsl_card_T; +typedef struct fsl_checkin_opt fsl_checkin_opt; +typedef struct fsl_deck fsl_deck; + +/** + This function is a programmatic interpretation of + this table: + + https://fossil-scm.org/index.html/doc/trunk/www/fileformat.wiki#summary + + For a given control artifact type and a card name in the form of + the card name's letter (e.g. 'A', 'W', ...), this function + returns 0 (false) if that card type is not permitted in this + control artifact type, a negative value if the card is optional + for this artifact type, and a positive value if the card type is + required for this artifact type. + + As a special case, if t==FSL_SATYPE_ANY then this function + always returns a negative value as long as card is a valid card + letter. + + Another special case: when t==FSL_SATYPE_CHECKIN and card=='F', + this returns a negative value because the table linked to above + says F-cards are optional. In practice we have yet to find a use + for checkins with no F-cards, so this library currently requires + F-cards at checkin-time even though this function reports that + they are optional. +*/ +FSL_EXPORT int fsl_card_is_legal( fsl_satype_e t, char card ); + +/** + Artifact tag types used by the Fossil framework. Their values + are a hard-coded part of the Fossil format, and not subject to + change (only extension, possibly). +*/ +enum fsl_tagtype_e { +/** + Sentinel value for use with constructors/initializers. +*/ +FSL_TAGTYPE_INVALID = -1, +/** + The "cancel tag" indicator, a.k.a. an anti-tag. +*/ +FSL_TAGTYPE_CANCEL = 0, +/** + The "add tag" indicator, a.k.a. a singleton tag. +*/ +FSL_TAGTYPE_ADD = 1, +/** + The "propagating tag" indicator. +*/ +FSL_TAGTYPE_PROPAGATING = 2 +}; +typedef enum fsl_tagtype_e fsl_tagtype_e; + +/** + Hard-coded IDs used by the 'tag' table of repository DBs. These + values get installed as part of the base Fossil db schema in new + repos, and they must never change. +*/ +enum fsl_tagid_e { +/** + DB string tagname='bgcolor'. +*/ +FSL_TAGID_BGCOLOR = 1, +/** + DB: tag.tagname='comment'. +*/ +FSL_TAGID_COMMENT = 2, +/** + DB: tag.tagname='user'. +*/ +FSL_TAGID_USER = 3, +/** + DB: tag.tagname='date'. +*/ +FSL_TAGID_DATE = 4, +/** + DB: tag.tagname='hidden'. +*/ +FSL_TAGID_HIDDEN = 5, +/** + DB: tag.tagname='private'. +*/ +FSL_TAGID_PRIVATE = 6, +/** + DB: tag.tagname='cluster'. +*/ +FSL_TAGID_CLUSTER = 7, +/** + DB: tag.tagname='branch'. +*/ +FSL_TAGID_BRANCH = 8, +/** + DB: tag.tagname='closed'. +*/ +FSL_TAGID_CLOSED = 9, +/** + DB: tag.tagname='parent'. +*/ +FSL_TAGID_PARENT = 10, +/** + DB: tag.tagname='note' + + Extra text appended to a check-in comment. +*/ +FSL_TAGID_NOTE = 11, + +/** + Largest tag ID reserved for internal use. +*/ +FSL_TAGID_MAX_INTERNAL = 99 +}; + + +/** + Returns one of '-', '+', or '*' for a valid input parameter, 0 + for any other value. +*/ +FSL_EXPORT char fsl_tag_prefix_char( fsl_tagtype_e t ); + + +/** + A list of fsl_card_F objects. F-cards used a custom list type, + instead of the framework's generic fsl_list, because experience has + shown that the number of (de)allocations we need for F-card lists + has a large negative impact when parsing and creating artifacts en + masse. This list type, unlike fsl_list, uses a conventional object + array approach to storage, as opposed to an array of pointers (each + entry of which has to be separately allocated). + + These lists, and F-cards in generally, are typically maintained + internally in the library. There's probably "no good reason" for + clients to manipulate them. +*/ +struct fsl_card_F_list { + /** + The list of F-cards. The first this->used elements are in-use. + This pointer may change any time the list is reallocated.a + */ + fsl_card_F * list; + /** + The number of entries in this->list which are in use. + */ + uint32_t used; + /** + The number of entries currently allocated in this->list. + */ + uint32_t capacity; + /** + An internal cursor into this->list, used primarily for + properly traversing the file list in delta manifests. + + Maintenance notes: internal updates to this member are the only + reason some of the deck APIs require a non-const deck. This type + needs to be signed for compatibility with some of the older + algos, e.g. fsl_deck_F_seek_base(). + */ + int32_t cursor; + /** + Internal flags. Never, ever modify these from client code. + */ + uint32_t flags; +}; +typedef struct fsl_card_F_list fsl_card_F_list; +/** Empty-initialized fsl_card_F instance for const copy + initialization */ +#define fsl_card_F_list_empty_m {NULL, 0, 0, 0, 0} +/** Empty-initialized fsl_card_F instance for non-const copy + initialization */ +FSL_EXPORT const fsl_card_F_list fsl_card_F_list_empty; + +/** + A "deck" stores (predictably enough) a collection of "cards." + Cards are constructs embedded within Fossil's Structural Artifacts + to denote various sorts of changes in a Fossil repository, and a + Deck encapsulates the cards for a single Structural Artifact of an + arbitrary type, e.g. Manifest (a.k.a. "checkin") or Cluster. A card + is basically a command with a single-letter name and a well-defined + signature for its arguments. Each card is represented by a member + of this struct whose name is the same as the card type + (e.g. fsl_card::C holds a C-card and fsl_card::F holds a list of + F-card). Each type of artifact only allows certain types of + card. The complete list of valid card/construct combinations can be + found here: + + https://fossil-scm.org/home/doc/trunk/www/fileformat.wiki#summary + + fsl_card_is_legal() can be used determine if a given card type + is legal (per the above chart) with a given Control Artifiact + type (as stored in the fsl_deck::type member). + + The type member is used by some algorithms to determine which + operations are legal on a given artifact type, so that they can + fail long before the user gets a chance to add a malformed artifact + to the database. Clients who bypass the fsl_deck APIs and + manipulate the deck's members "by hand" (so to say) effectively + invoke undefined behaviour. + + The various routines to add/set cards in the deck are named + fsl_deck_CARDNAME_add() resp. fsl_deck_CARDNAME_set(). The "add" + functions represent cards which may appear multiple times + (e.g. the 'F' card) or have multiple values (the 'P' card), and + those named "set" represent unique or optional cards. The R-card + is the outlier, with fsl_deck_R_calc(). NEVER EVER EVER directly + modify a member of this struct - always use the APIs. The + library performs some optimizations which can lead to corrupt + memory and invalid free()s if certain members' values are + directly replaced by the client (as opposed to via the APIs). + + Note that the 'Z' card is not in this structure because it is a + hash of the other inputs and is calculated incrementally and + appended automatically by fsl_deck_output(). + + All non-const pointer members of this structure are owned by the + structure instance unless noted otherwise (the fsl_deck::f member + being the notable exception). + + Maintenance reminder: please keep the card members alpha sorted to + simplify eyeball-searching through their docs. + + @see fsl_deck_malloc() + @see fsl_deck_init() + @see fsl_deck_parse() + @see fsl_deck_load_rid() + @see fsl_deck_finalize() + @see fsl_deck_clean() + @see fsl_deck_save() + @see fsl_deck_A_set() + @see fsl_deck_B_set() + @see fsl_deck_D_set() + @see fsl_deck_E_set() + @see fsl_deck_F_add() + @see fsl_deck_J_add() + @see fsl_deck_K_set() + @see fsl_deck_L_set() + @see fsl_deck_M_add() + @see fsl_deck_N_set() + @see fsl_deck_P_add() + @see fsl_deck_Q_add() + @see fsl_deck_R_set() + @see fsl_deck_T_add() + @see fsl_deck_branch_set() + @see fsl_deck_U_set() + @see fsl_deck_W_set() +*/ +struct fsl_deck { + /** + Specifies the the type (or eventual type) of this + artifact. The function fsl_card_is_legal() can be used to + determined if a given card type is legal for a given value of + this member. APIs which add/set cards use that to determine if + the operation requested by the client is semantically legal. + */ + fsl_satype_e type; + + /** + DB repo.blob.rid value. Normally set by fsl_deck_load_rid(). + */ + fsl_id_t rid; + + /** + The Fossil context responsible for this deck. Though this data + type is normally, at least conceptually, free of any given fossil + context, many related algorithms need a context in order to + perform db- or caching-related work, as well as to simplify error + message propagation. We store this as a struct member to keep all + such algorithms from redundantly requiring both pieces of + information as arguments. + + This object does not own the context and the context object must + outlive this deck instance. + */ + fsl_cx * f; + + /** + The 'A' (attachment) card. Only used by FSL_SATYPE_ATTACHMENT + decks. The spec currently specifies only 1 A-card per + manifest, but conceptually this could/should be a list. + */ + struct { + /** + Filename of the A-card. + */ + char * name; + + /** + Name of wiki page, or UUID of ticket or event (technote), to + which the attachment applies. + */ + char * tgt; + + /** + UUID of the file being attached via the A-card. + */ + fsl_uuid_str src; + } A; + + struct { + /** + The 'B' (baseline) card holds the UUID of baseline manifest. + This is empty for baseline manifests and holds the UUID of + the parent for delta manifests. + */ + fsl_uuid_str uuid; + + /** + Baseline manifest corresponding to this->B. It is loaded on + demand by routines which need it, typically by calling + fsl_deck_F_rewind() (unintuitively enough!). The + parent/child relationship in Fossil is the reverse of + conventional - children own their parents, not the other way + around. i.e. this->baseline will get cleaned up + (recursively) when this instance is cleaned up (when the + containing deck is cleaned up). + */ + fsl_deck * baseline; + } B; + /** + The 'C' (comment) card. + */ + char * C; + + /** + The 'D' (date) card, in Julian format. + */ + double D; + + /** + The 'E' (event) card. + */ + struct { + /** + The 'E' card's date in Julian Day format. + */ + double julian; + + /** + The 'E' card's UUID. + */ + fsl_uuid_str uuid; + } E; + + /** + The 'F' (file) card container. + */ + fsl_card_F_list F; + + /** + UUID for the 'G' (forum thread-root) card. + */ + fsl_uuid_str G; + + /** + The H (forum title) card. + */ + char * H; + + /** + UUID for the 'I' (forum in-response-to) card. + */ + fsl_uuid_str I; + + /** + The 'J' card specifies changes to "value" of "fields" in + tickets (FSL_SATYPE_TICKET). + + Holds (fsl_card_J*) entries. + */ + fsl_list J; + + /** + UUID for the 'K' (ticket) card. + */ + fsl_uuid_str K; + + /** + The 'L' (wiki name/title) card. + */ + char * L; + + /** + List of UUIDs (fsl_uuid_str) in a cluster ('M' cards). + */ + fsl_list M; + + /** + The 'N' (comment mime type) card. Note that this is only + ostensibly supported by fossil, but fossil does not (as of + 2021-04-13) honor this value and always assumes that its value is + "text/x-fossil-wiki". + */ + char * N; + + /** + List of UUIDs of parents ('P' cards). Entries are of type + (fsl_uuid_str). + */ + fsl_list P; + + /** + 'Q' (cherry pick) cards. Holds (fsl_card_Q*) entries. + */ + fsl_list Q; + + /** + The R-card holds an MD5 hash which is calculated based on the + names, sizes, and contents of the files included in a + manifest. See the class-level docs for a link to a page which + describes how this is calculated. + */ + char * R; + + /** + List of 'T' (tag) cards. Holds (fsl_card_T*) instances. + */ + fsl_list T; + + /** + The U (user) card. + */ + char * U; + + /** + The W (wiki content) card. + */ + fsl_buffer W; + + /** + This is part of an optimization used when parsing fsl_deck + instances from source text. For most types of card we re-use + string values in the raw source text rather than duplicate them, + and that requires storing the original text (as passed to + fsl_deck_parse()). This requires that clients never tinker + directly with values in a fsl_deck, in particular never assign + over them or assume they know who allocated the memory for that + bit. + */ + fsl_buffer content; + + /** + To potentially be used for a manifest cache. + */ + fsl_deck * next; + + /** + A marker which tells fsl_deck_finalize() whether or not + fsl_deck_malloc() allocated this instance (in which case + fsl_deck_finalize() will fsl_free() it) or not (in which case + it does not fsl_free() it). + */ + void const * allocStamp; +}; + +/** + Initialized-with-defaults fsl_deck structure, intended for copy + initialization. +*/ +FSL_EXPORT const fsl_deck fsl_deck_empty; + +/** + Initialized-with-defaults fsl_deck structure, intended for + in-struct and const copy initialization. +*/ +#define fsl_deck_empty_m { \ + FSL_SATYPE_ANY /*type*/, \ + 0/*rid*/, \ + NULL/*f*/, \ + {/*A*/ NULL /* name */, \ + NULL /* tgt */, \ + NULL /* src */}, \ + {/*B*/ NULL /*uuid*/, \ + NULL /*baseline*/}, \ + NULL /* C */, \ + 0.0 /*D*/, \ + {/*E*/ 0.0 /* julian */, \ + NULL /* uuid */}, \ + /*F*/ fsl_card_F_list_empty_m, \ + 0/*G*/,0/*H*/,0/*I*/, \ + fsl_list_empty_m /* J */, \ + NULL /* K */, \ + NULL /* L */, \ + fsl_list_empty_m /* M */, \ + NULL /* N */, \ + fsl_list_empty_m /* P */, \ + fsl_list_empty_m /* Q */, \ + NULL /* R */, \ + fsl_list_empty_m /* T */, \ + NULL /* U */, \ + fsl_buffer_empty_m /* W */, \ + fsl_buffer_empty_m/*content*/, \ + NULL/*next*/, \ + NULL/*allocStamp*/ \ + } + + +/** + Allocates a new fsl_deck instance. Returns NULL on allocation + error. The returned value must eventually be passed to + fsl_deck_finalize() to free its resources. + + @see fsl_deck_finalize() + @see fsl_deck_clean() +*/ +FSL_EXPORT fsl_deck * fsl_deck_malloc(); + +/** + Frees all resources belonging to the given deck's members + (including its parents, recursively), and wipes deck clean of most + state, but does not free() deck. Is a no-op if deck is NULL. As a + special case, the (allocStamp, f) members of deck are kept intact. + + @see fsl_deck_finalize() + @see fsl_deck_malloc() + @see fsl_deck_clean2() +*/ +FSL_EXPORT void fsl_deck_clean(fsl_deck * const deck); + +/** + A variant of fsl_deck_clean() which "returns" its content buffer + for re-use by transferring, after ensuring proper cleanup of its + internals, its own content buffer's bytes into the given target + buffer. Note that decks created "manually" do not have any content + buffer contents, but those loaded via fsl_deck_load_rid() do. This + function will fsl_buffer_swap() the contents of the given buffer + (if any) with its own buffer, clean up its newly-acquired memory + (tgt's previous contents, if any), and fsl_buffer_reuse() the + output buffer. + + If tgt is NULL, this behaves exactly like fsl_deck_clean(). +*/ +FSL_EXPORT void fsl_deck_clean2(fsl_deck * const deck, fsl_buffer * const tgt); + +/** + Frees all memory owned by deck (see fsl_deck_clean()). If deck + was allocated using fsl_deck_malloc() then this function + fsl_free()'s it, otherwise it does not free it. + + @see fsl_deck_malloc() + @see fsl_deck_clean() +*/ +FSL_EXPORT void fsl_deck_finalize(fsl_deck *deck); + +/** + Sets the A-card for an Attachment (FSL_SATYPE_ATTACHMENT) + deck. Returns 0 on success. + + Returns FSL_RC_MISUSE if any of (mf, filename, target) are NULL, + FSL_RC_RANGE if !*filename or if uuidSrc is not NULL and + fsl_is_uuid(uuidSrc) returns false. + + Returns FSL_RC_TYPE if mf is not (as determined by its mf->type + member) of a deck type capable of holding 'A' cards. (Only decks + of type FSL_SATYPE_ATTACHMENT may hold an 'A' card.) If uuidSrc + is NULL or starts with a NUL byte then it is ignored, otherwise + the same restrictions apply to it as to target. + + The target parameter represents the "name" of the + wiki/ticket/event record to which the attachment applies. For + wiki pages this is their normal name (e.g. "MyWikiPage"). For + events and tickets it is their full 40-byte UUID. + + uuidSrc is the UUID of the attachment blob itself. If it is NULL + or empty then this card indicates that the attachment will be + "deleted" (insofar as anything is ever deleted in Fossil). +*/ +FSL_EXPORT int fsl_deck_A_set( fsl_deck * const mf, char const * filename, + char const * target, + fsl_uuid_cstr uuidSrc); + +/** + Sets or unsets (if uuidBaseline is NULL or empty) the B-card for + the given manifest to a copy of the given UUID. Returns 0 on + success, FSL_RC_MISUSE if !mf, FSL_RC_OOM on allocation + error. Setting this will free any prior values in mf->B, including + a previously loaded mf->B.baseline. + + If uuidBaseline is not NULL and fsl_is_uuid() returns false, + FSL_RC_SYNTAX is returned. If it is NULL the current value is + freed (semantically, though the deck may still own the memory), the + B card is effectively removed, and 0 is returned. + + Returns FSL_RC_TYPE if mf is not syntactically allowed to have + this card card (as determined by + fsl_card_is_legal(mf->type,...)). + + Sidebar: the ability to unset this card is unusual within this API, + and is a requirement the library-internal delta manifest creation + process. Most of the card-setting APIs, even when they are + described as working like this one, do not accept NULL hash values. +*/ +FSL_EXPORT int fsl_deck_B_set( fsl_deck * const mf, fsl_uuid_cstr uuidBaseline); + +/** + Semantically identical to fsl_deck_B_set() but sets the C-card and + does not place a practical limit on the comment's length. comment + must be the comment text for the change being applied. If the + given length is negative, fsl_strlen() is used to determine its + length. +*/ +FSL_EXPORT int fsl_deck_C_set( fsl_deck * const mf, char const * comment, fsl_int_t cardLen); + +/** + Sets mf's D-card as a Julian Date value. Returns FSL_RC_MISUSE if + !mf, FSL_RC_RANGE if date is negative, FSL_RC_TYPE if a D-card is + not valid for the given deck, else 0. Passing a value of 0 + effectively unsets the card. +*/ +FSL_EXPORT int fsl_deck_D_set( fsl_deck * const mf, double date); + +/** + Sets the E-card in the given deck. date may not be negative - + use fsl_db_julian_now() or fsl_julian_now() to get a default + time if needed. Retursn FSL_RC_MISUSE if !mf or !uuid, + FSL_RC_RANGE if date is not positive, FSL_RC_RANGE if uuid is + not a valid UUID string. + + Note that the UUID for an event, unlike most other UUIDs, need + not be calculated - it may be a random hex string, but it must + pass the fsl_is_uuid() test. Use fsl_db_random_hex() to generate + random UUIDs. When editing events, e.g. using the HTML UI, only + the most recent event with the same UUID is shown. So when + updating events, be sure to apply the same UUID to the edited + copies before saving them. +*/ +FSL_EXPORT int fsl_deck_E_set( fsl_deck * const mf, double date, fsl_uuid_cstr uuid); + +/** + Adds a new F-card to the given deck. The uuid argument is required + to be NULL or pass the fsl_is_uuid() test. The name must be a + "simplified path name" (as determined by fsl_is_simple_pathname()), + or FSL_RC_RANGE is returned. Note that a NULL uuid is only valid + when constructing a delta manifest, and this routine will return + FSL_RC_MISUSE and update d->f's error state if uuid is NULL and + d->B.uuid is also NULL. + + perms should be one of the fsl_fileperm_e values (0 is the usual + case). + + priorName must only be non-NULL when renaming a file, and it must + follow the same naming rules as the name parameter. + + Returns 0 on success. + + @see fsl_deck_F_set() +*/ +FSL_EXPORT int fsl_deck_F_add( fsl_deck * d, char const * name, + fsl_uuid_cstr uuid, + fsl_fileperm_e perm, + char const * priorName); + +/** + Works mostly like fsl_deck_F_add() except that: + + 1) It enables replacing an existing F-card with a new one matching + the same name. + + 2) It enables removing an F-card by passing a NULL uuid. + + 3) It refuses to work on a deck for which d->uuid is not NULL or + d->rid!=0, returning FSL_RC_MISUSE if either of those apply. + + If d contains no F-card matching the given name (case-sensitivity + depends on d->f's fsl_cx_is_case_sensitive() value) then: + + - If the 3rd argument is NULL, it returns FSL_RC_NOT_FOUND with + (effectively) no side effects (aside, perhaps, from sorting d->F + if needed to perform the search). + + - If the 3rd argument is not NULL then it behaves identically to + fsl_deck_F_add(). + + If a match is found, then: + + - If the 3rd argument is NULL, it removes that entry from the + F-card list and returns 0. + + - If the 3rd argument is not NULL, the fields of the resulting + F-card are modified to match the arguments passed to this + function, copying the values of all C-string arguments. (Sidebar: + we may need to copy the name, despite already knowing it, because + of how fsl_deck instances manage F-card memory.) + + In all cases, if the 3rd argument is NULL then the 4th and 5th + arguments are ignored. + + Returns 0 on success, FSL_RC_OOM if an allocation fails. See + fsl_deck_F_add() for other failure modes. On error, d's F-card list + may be left in an inconsistent state and it must not be used + further. + + @see fsl_deck_F_add() + @see fsl_deck_F_set_content() +*/ +FSL_EXPORT int fsl_deck_F_set( fsl_deck * d, char const * name, + fsl_uuid_cstr uuid, + fsl_fileperm_e perm, + char const * priorName); +/** + UNDER CONSTRUCTION! EXPERIMENTAL! + + This variant of fsl_deck_F_set() accepts a buffer of content to + store as the file's contents. Its hash is derived from that + content, using fsl_repo_blob_lookup() to hash the given content. + Thus this routine can replace existing F-cards and save their + content at the same time. When doing so, it will try to make the + parent version (if this is a replacement F-card) a delta of the new + content version (it may refuse to do so for various resources, but + any heuristics which forbid that will not trigger an error). + + The intended use of this routine is for adding or replacing content + in a deck which has been prepared using fsl_deck_derive(). + + Returns 0 on success, else an error code propagated by + fsl_deck_F_set(), fsl_repo_blob_lookup(), or some other lower-level + routine. This routine requires that a transaction is active and + returns FSL_RC_MISUSE if none is active. For any non-trivial + error's, d->f's error state will be updated with a description of + the problem. + + TODO: add a fsl_cx-level or fsl_deck-level API for marking content + saved this way as private. This type of content is intended for use + cases which do not have a checkout, and thus cannot be processed + with fsl_checkin_commit() (which includes a flag to mark its + content as private). + + @see fsl_deck_F_set() + @see fsl_deck_F_add() + @see fsl_deck_derive() +*/ +FSL_EXPORT int fsl_deck_F_set_content( fsl_deck * d, char const * name, + fsl_buffer const * src, + fsl_fileperm_e perm, + char const * priorName); + +/** + UNDER CONSTRUCTION! EXPERIMENTAL! + + This routine rewires d such that it becomes the basis for a derived + version of itself. Requires that d be a loaded + from a repository, complete with a UUID and an RID, else + FSL_RC_MISUSE is returned. + + In short, this function peforms the following: + + - Clears d->P + - Moves d->uuid into d->P + - Clears d->rid + - Clears any other members which need to be (re)set by the new + child/derived version. + - It specifically keeps d->F intact OR creates a new one (see below). + + Returns 0 on success, FSL_RC_OOM on an allocation error, + FSL_RC_MISUSE if d->rid<=0 (i.e. the deck has never been saved or + was not loaded from the db. If d->type is not FSL_SATYPE_CHECKIN, + FSL_RC_TYPE is returned. On error, d may be left in an inconsistent + state and must not be used further except to pass it to + fsl_deck_finalize(). + + The intention of this function is to simplify creation of decks + which are to be used for creating checkins without requiring a + checkin. + + To avoid certain corner cases, this function does not allow + creation of delta manifests. If d has a B-card then it is a delta. + This function clears its B-card and recreates the F-card list using + the B-card's F-card list and any F-cards from the current delta. In + other words, it creates a new baseline manifest. + + TODO: extend this to support other inheritable deck types, e.g. + wiki, forum posts, and technotes. + + @see fsl_deck_F_set_content() +*/ +FSL_EXPORT int fsl_deck_derive(fsl_deck * const d); + +/** + Callback type for use with fsl_deck_F_foreach() and + friends. Implementations must return 0 on success, FSL_RC_BREAK + to abort looping without an error, and any other value on error. +*/ +typedef int (*fsl_card_F_visitor_f)(fsl_card_F const * fc, + void * state); + +/** + For each F-card in d, cb(card,visitorState) is called. Returns + the result of that loop. If cb returns FSL_RC_BREAK, the + visitation loop stops immediately and this function returns + 0. If cb returns any other non-0 code, looping stops and that + code is returned immediately. + + This routine calls fsl_deck_F_rewind() to reset the F-card cursor + and/or load d's baseline manifest (if any). If loading the baseline + fails, an error code from fsl_deck_baseline_fetch() is returned. + + The F-cards will be visited in the order they are declared in + d. For loaded-from-a-repo manifests this is always lexical order + (for delta manifests, consistent across the delta and + baseline). For hand-created decks which have not yet been + fsl_deck_unshuffle()'d, the order is unspecified. +*/ +FSL_EXPORT int fsl_deck_F_foreach( fsl_deck * d, fsl_card_F_visitor_f cb, + void * visitorState ); + +/** + Fetches the next F-card entry from d. fsl_deck_F_rewind() must + have be successfully executed one time before calling this, as + that routine ensures that the baseline is loaded (if needed), + which is needed for proper iteration over delta manifests. + + This routine always assigns *f to NULL before starting its work, so + the client can be assured that it will never contain the same value + as before calling this (unless that value was NULL). + + On success 0 is returned and *f is assigned to the next F-card. + If *f is NULL when returning 0 then the end of the list has been + reached (fsl_deck_F_rewind() can be used to re-set it). + + Example usage: + + ``` + int rc; + fsl_card_F const * fc = NULL; + rc = fsl_deck_F_rewind(d); + if(!rc) while( !(rc=fsl_deck_F_next(d, &fc)) && fc) {...} + ``` + + Note that files which were deleted in a given version are not + recorded in baseline manifests but are in deltas. To avoid + inconsistencies, this routine does NOT include deleted files in its + results, regardless of whether d is a baseline or delta. (It used + to, but that turned out to be a design flaw.) + + Implementation notes: for baseline manifests this is a very + fast and simple operation. For delta manifests it gets + rather complicated. +*/ +FSL_EXPORT int fsl_deck_F_next( fsl_deck * d, fsl_card_F const **f ); + +/** + Rewinds d's F-card traversal iterator and loads d's baseline + manifest, if it has one (i.e. if d->B.uuid is not NULL) and it is + not loaded already (i.e. if d->B.baseline is NULL). Returns 0 on + success. The only error condition is if loading of the a baseline + manifest fails, noting that only delta manifests have baselines. + + Results are undefined if d->f is NULL, and that may trigger an + assert() in debug builds. +*/ +FSL_EXPORT int fsl_deck_F_rewind( fsl_deck * d ); + +/** + Looks for a file in a manifest or (for a delta manifest) its + baseline. No normalization of the given filename is performed - + it is assumed to be relative to the root of the checkout. + + It requires that d->type be FSL_SATYPE_CHECKIN and that d be + loaded from a stored manifest or have been fsl_deck_unshuffle()'d + (if called on an under-construction deck). Specifically, this + routine requires that d->F be sorted properly or results are + undefined. + + d->f is assumed to be the fsl_cx instance which deck was loaded + from, which impacts the search process as follows: + + - The search take's d->f's underlying case-insensitive option into + account. i.e. if case-insensitivy is on then files in any case + will match. + + - If no match is found in d and is a delta manifest (d->B.uuid + is set) then d's baseline is lazily loaded (if needed) and + the search continues there. (Delta manifests are only one level + deep, so this is not recursive.) + + Returns NULL if !d, !d->f, or d->type!=FSL_SATYPE_CHECKIN, if no + entry is found, or if delayed loading of the parent manifest (if + needed) of a delta manifest fails (in which case d->f's error + state should hold more information about the problem). + + In debug builds this function asserts that d is not NULL. + + Design note: d "should" be const, but search optimizations for + the typical use case require potentially lazy-loading + d->B.baseline and updating d->F. +*/ +FSL_EXPORT fsl_card_F const * fsl_deck_F_search(fsl_deck *d, const char *zName); + +/** + Given two F-card instances, this function compares their names + (case-sensitively). Returns a negative value if lhs is + lexically less than rhs, a positive value if lhs is lexically + greater than rhs, and 0 if they are lexically equivalent (or are + the same pointer). + + + Though fossil repositories may be case-insensitive, the F-cards use + a stable casing unless a file is removed and re-added with a + different case, so this comparison is case-sensitive.. + + Results are undefined if either argument is NULL. +*/ +FSL_EXPORT int fsl_card_F_compare_name( fsl_card_F const * const lhs, + fsl_card_F const * const rhs); + +/** + If fc->uuid refers to a blob in f's repository database then that + content is placed into dest (as per fsl_content_get()) and 0 is + returned. Returns FSL_RC_NOT_FOUND if fc->uuid is not + found. Returns FSL_RC_MISUSE if any argument is NULL. If + fc->uuid is NULL (indicating that it refers to a file deleted in + a delta manifest) then FSL_RC_RANGE is returned. Returns + FSL_RC_NOT_A_REPO if f has no repository opened. + + On any error but FSL_RC_MISUSE (basic argument validation) f's + error state is updated to describe the error. + + @see fsl_content_get() +*/ +FSL_EXPORT int fsl_card_F_content( fsl_cx * f, fsl_card_F const * fc, + fsl_buffer * dest ); + +/** + Sets the 'G' card on a forum-post deck to a copy of the given + UUID. +*/ +FSL_EXPORT int fsl_deck_G_set( fsl_deck * const mf, fsl_uuid_cstr uuid); +/** + Sets the 'H' card on a forum-post deck to a copy of the given + comment. If cardLen is negative then fsl_strlen() is used to + calculate its length. + */ +FSL_EXPORT int fsl_deck_H_set( fsl_deck * const mf, char const * comment, fsl_int_t cardLen); +/** + Sets the 'I' card on a forum-post deck to a copy of the given + UUID. +*/ +FSL_EXPORT int fsl_deck_I_set( fsl_deck * const mf, fsl_uuid_cstr uuid); + +/** + Adds a J-card to the given deck, setting/updating the given ticket + property key to the given value. The key is required but the value + is optional (may be NULL). If isAppend then the value is appended + to any existing value, otherwise it replaces any existing value. + + It is currently unclear whether it is legal to include multiple + J cards for the same key in the same control artifact, in + particular if their isAppend values differ. + + Returns 0 on success, FSL_RC_MISUSE if !mf or !key, FSL_RC_RANGE + if !*field, FSL_RC_TYPE if mf is of a type for which J cards are + not legal (see fsl_card_is_legal()), FSL_RC_OOM on allocation + error. +*/ +FSL_EXPORT int fsl_deck_J_add( fsl_deck * const mf, char isAppend, + char const * key, char const * value ); + +/** + Semantically identical fsl_deck_B_set() but sets the K-card and + does not accept a NULL value. uuid must be the UUID of the ticket + this change is being applied to. +*/ +FSL_EXPORT int fsl_deck_K_set( fsl_deck * const mf, fsl_uuid_cstr uuid); + +/** + Semantically identical fsl_deck_B_set() but sets the L-card. + title must be the wiki page title text of the wiki page this + change is being applied to. +*/ +FSL_EXPORT int fsl_deck_L_set( fsl_deck * const mf, char const *title, fsl_int_t len); + +/** + Adds the given UUID as an M-card entry. Returns 0 on success, or: + + FSL_RC_MISUSE if !mf or !uuid + + FSL_RC_TYPE if fsl_deck_check_type(mf,'M') returns false. + + FSL_RC_RANGE if !fsl_is_uuid(uuid). + + FSL_RC_OOM if memory allocation fails while adding the entry. +*/ +FSL_EXPORT int fsl_deck_M_add( fsl_deck * const mf, fsl_uuid_cstr uuid ); + +/** + Semantically identical to fsl_deck_B_set() but sets the N card. + mimeType must be the content mime type for comment text of the + change being applied. +*/ +FSL_EXPORT int fsl_deck_N_set( fsl_deck * const mf, char const *mimeType, fsl_int_t len); + +/** + Adds the given UUID as a parent of the given change record. If len + is less than 0 then fsl_strlen(parentUuid) is used to determine its + length. Returns FSL_RC_MISUE if !*parentUuid. Returns FSL_RC_RANGE + if parentUuid does not pass fsl_is_uuid(). + + Results are undefined if parentUuid is NULL. + + The first P-card added to a deck MUST be the UUID of its primary + parent (one which was not involved in a merge operation). All + others (from merges) are considered "non-primary." + +*/ +FSL_EXPORT int fsl_deck_P_add( fsl_deck * const mf, fsl_uuid_cstr parentUuid); +/** + A convenience wrapper around fsl_deck_P_add() which resolves the given + RID to its UUID and passes it on to fsl_deck_P_add(). Returns non-0 + on error. +*/ +FSL_EXPORT int fsl_deck_P_add_rid( fsl_deck * const mf, fsl_id_t parentRid ); + +/** + If d contains a P card with the given index, this returns the RID + corresponding to the UUID at that index. Returns a negative value + on error, 0 if there is no entry for that index (the index is out + of bounds). +*/ +FSL_EXPORT fsl_id_t fsl_deck_P_get_id(fsl_deck * const d, int index); + +/** + Adds a Q-card record to the given deck. The type argument must + be negative for a backed-out change, positive for a cherrypicked + change. target must be a valid UUID string. If baseline is not + NULL then it also must be a valid UUID. + + Returns 0 on success, non-0 on error. FSL_RC_MISUSE if !mf + or !target, FSL_RC_RANGE if target/baseline are not valid + UUID strings (baseline may be NULL). +*/ +FSL_EXPORT int fsl_deck_Q_add( fsl_deck * const mf, int type, + fsl_uuid_cstr target, + fsl_uuid_cstr baseline ); + +/** + Functionally identical to fsl_deck_B_set() except that it sets + the R-card. Returns 0 on succes, FSL_RC_RANGE if md5 is not NULL + or exactly FSL_STRLEN_MD5 bytes long (not including trailing + NUL). If md5==NULL the current R value is cleared. + + It would be highly unusual to have to set the R-card manually, + as its calculation is quite intricate/intensive. See + fsl_deck_R_calc() and fsl_deck_unshuffle() for details +*/ +FSL_EXPORT int fsl_deck_R_set( fsl_deck * const mf, char const *md5); + +/** + Adds a new T-card (tag) entry to the given deck. + + If uuid is not NULL and fsl_is_uuid(uuid) returns false then + this function returns FSL_RC_RANGE. If uuid is NULL then it is + assumed to be the UUID of the currently-being-constructed + artifact in which the tag is contained (which appears as the '*' + character in generated artifacts). + + Returns 0 on success. Returns FSL_RC_MISUE if !mf or + !name. Returns FSL_RC_TYPE (and update's mf's error state with a + message) if the T card is not legal for mf (see + fsl_card_is_legal()). Returns FSL_RC_RANGE if !*name, tagType + is invalid, or if uuid is not NULL and fsl_is_uuid(uuid) + return false. Returns FSL_RC_OOM if an allocation fails. +*/ +FSL_EXPORT int fsl_deck_T_add( fsl_deck * const mf, fsl_tagtype_e tagType, + fsl_uuid_cstr uuid, char const * name, + char const * value); + +/** + Adds the given tag instance to the given manifest. + Returns 0 on success, FSL_RC_MISUSE if either argument + is NULL, FSL_RC_OOM if appending the tag to the list + fails. + + On success ownership of t is passed to mf. On error ownership is + not modified. +*/ +FSL_EXPORT int fsl_deck_T_add2( fsl_deck * const mf, fsl_card_T * t); + +/** + A convenience form of fsl_deck_T_add() which adds two propagating + tags to the given deck: "branch" with a value of branchName and + "sym-branchName" with no value. + + Returns 0 on success. Returns FSL_RC_OOM on allocation error and + FSL_RC_RANGE if branchName is empty or contains any characters with + ASCII values <=32d. It natively assumes that any characters >=128 + are part of multibyte UTF8 characters. +*/ +FSL_EXPORT int fsl_deck_branch_set( fsl_deck * d, char const * branchName ); + +/** + Calculates the value of d's R-card based on its F-cards and updates + d->R. It may also, as a side-effect, sort d->F.list lexically (a + requirement of a R-card calculation). + + Returns 0 on success. Requires that d->f have an opened + repository db, else FSL_RC_NOT_A_REPO is returned. If d's type is + not legal for an R-card then FSL_RC_TYPE is returned and d->f's + error state is updated with a description of the error. If d is of + type FSL_SATYPE_CHECKIN and has no F-cards then the R-card's value + is that of the initial MD5 hash state. Various other codes can be + returned if fetching file content from the db fails. + + Note that this calculation is exceedingly memory-hungry. While + Fossil originally required R-cards, the cost of calculation + eventually caused the R-card to be made optional. This API + allows the client to decide on whether to use them (for more + (admittedly redundant!) integrity checking) or not (much faster + but "not strictly historically correct"), but defaults to having + them enabled for symmetry with fossil(1). + + @see fsl_deck_R_calc2() +*/ +FSL_EXPORT int fsl_deck_R_calc(fsl_deck * d); + +/** + A variant of fsl_deck_R_calc() which calculates the given deck's + R-card but does not assign it to the deck, instead returning it + via the 2nd argument: + + If *tgt is not NULL when this function is called, it is required to + point to at least FSL_STRLEN_MD5+1 bytes of memory to which the + NUL-terminated R-card hash will be written. If *tgt is NULL then + this function assigns (on success) *tgt to a dynamically-allocated + R-card hash and transfers ownership of it to the caller (who must + eventually fsl_free() it). On error, *tgt is not modified. + + Results are undefined if either argument is NULL. + + Returns 0 on success. See fsl_deck_R_calc() for information about + possible errors, with the addition that FSL_RC_OOM is returned + if *tgt is NULL and allocating a new *tgt value fails. + + Calculating the R-card necessarily requires that d's F-card list be + sorted, which this routine does if it seems necessary. The + calculation also necessarily mutates the deck's F-card-traversal + cursor, which requires loading the deck's B-card, if it has + one. Aside from the F-card sorting, and potentially B-card, and the + cursor resets, this routine does not modify the deck. On success, + the deck's F-card iteration cursor (and that of d->B, if it's + loaded) is rewound. +*/ +FSL_EXPORT int fsl_deck_R_calc2(fsl_deck *d, char ** tgt); + +/** + Semantically identical fsl_deck_B_set() but sets the U-card. + userName must be the user who's name should be recorded for + this change. +*/ +FSL_EXPORT int fsl_deck_U_set( fsl_deck * const mf, char const *userName); + +/** + Semantically identical fsl_deck_B_set() but sets the W-card. + content must be the page content of the Wiki page or Event this + change is being applied to. +*/ +FSL_EXPORT int fsl_deck_W_set( fsl_deck * const mf, char const *content, fsl_int_t len); + +/** + Must be called to initialize a newly-created/allocated deck + instance. This function clears out all contents of the d + parameter except for its (f, type, allocStamp) members, sets its + (f, type) members, and leaves d->allocStamp intact. +*/ +FSL_EXPORT void fsl_deck_init( fsl_cx * cx, fsl_deck * d, fsl_satype_e type ); + +/** + Returns true if d contains data for all _required_ cards, as + determined by the value of d->type, else returns false. It returns + false if d->type==FSL_SATYPE_ANY, as that is a placeholder value + intended to be re-set by the deck's user. + + If it returns false, d->f's error state will help a description of + the problem. + + The library calls this as needed, but clients may, if they want + to. Note, however, that for FSL_SATYPE_CHECKIN decks it may fail + if the deck has not been fsl_deck_unshuffle()d yet because the + R-card gets calculated there (if needed). + + As a special case, d->f is configured to calculate R-cards, + d->type==FSL_SATYPE_CHECKIN, AND d->R is not set, this will fail + (with a descriptive error message). + + Another special case: for FSL_SATYPE_CHECKIN decks, if no + F-cards are in th deck then an R-card is required to avoid a + potental (admittedly harmless) syntactic ambiguity with + FSL_SATYPE_CONTROL artifacts. The only legal R-card for a + checkin with no F-cards has the initial MD5 hash state value + (defined in the constant FSL_MD5_INITIAL_HASH), and that + precondition is checked in this routine. fsl_deck_unshuffle() + recognizes this case and adds the initial-state R-card, so + clients normally need not concern themselves with this. If d has + F-cards, whether or not an R-card is required depends on + whether d->f is configured to require them or not. + + Enough about the R-card. In all other cases not described above, + R-cards are not required (and they are only ever required on + FSL_SATYPE_CHECKIN manifests). + + Though fossil(1) does not technically require F-cards in + FSL_SATYPE_CHECKIN decks, so far none of the Fossil developers + have found a use for a checkin without F-cards except the + initial empty checkin. Additionally, a checkin without F-cards + is potentially syntactically ambiguous (it could be an EVENT or + ATTACHMENT artifact if it has no F- or R-card). So... this + library _normally_ requires that CHECKIN decks have at least one + F-card. This function, however, does not consider F-cards to be + strictly required. +*/ +FSL_EXPORT bool fsl_deck_has_required_cards( fsl_deck const * d ); + +/** + Prepares the given deck for output by ensuring that cards + which need to be sorted are sorted, and it may run some + last-minute validation checks. + + The cards which get sorted are: F, J, M, Q, T. The P-card list is + _not_ sorted - the client is responsible for ensuring that the + primary parent is added to that list first, and after that the + ordering is largely irrelevant. It is not possible for the library + to determine a proper order for P-cards, nor to validate that order + at input-time. + + If calculateRCard is true and fsl_card_is_legal(d,'R') then this + function calculates the R-card for the deck. The R-card + calculation is _extremely_ memory-hungry but adds another level + of integrity checking to Fossil. If d->type is not + FSL_SATYPE_MANIFEST then calculateRCard is ignored. + + If calculateRCard is true but no F-cards are present AND d->type is + FSL_SATYPE_CHECKIN then the R-card is set to the initial MD5 hash + state (the only legal R-card value for an empty F-card list). (This + is necessary in order to prevent a deck-type ambiguity in one + corner case.) + + The R-card, if used, must be calculated before + fsl_deck_output()ing a deck containing F-cards. Clients may + alternately call fsl_deck_R_calc() to calculate the R card + separately, but there is little reason to do so. There are rare + cases where the client can call fsl_deck_R_set() + legally. Historically speaking the R-card was required when + F-cards were used, but it was eventually made optional because + (A) the memory cost and (B) it's part of a 3rd or 4th level of + integrity-related checks, and is somewhat superfluous. + + @see fsl_deck_output() + @see fsl_deck_save() +*/ +FSL_EXPORT int fsl_deck_unshuffle( fsl_deck * d, bool calculateRCard ); + +/** + Renders the given control artifact's contents to the given output + function and calculates any cards which cannot be calculated until + the contents are complete (namely the R-card and Z-card). + + The given deck is "logically const" but traversal over F-cards and + baselines requires non-const operations. To keep this routine from + requiring an undue amount of pre-call effort on the client's part, + it also takes care of calling fsl_deck_unshuffle() to ensure that + all of the deck's cards are in order. (If the deck has no R card, + but has F-cards, and d->f is configured to generate R-cards, then + unshuffling will also calculate the R-card.) + + Returns 0 on success, FSL_RC_MISUSE if !d or !d->f or !out. If + out() returns non-0, output stops and that code is + returned. outputState is passed as the first argument to out() and + out() may be called an arbitrary number of times by this routine. + + Returns FSL_RC_SYNTAX if fsl_deck_has_required_cards() + returns false. + + On errors more serious than argument validation, the deck's + context's (d->f) error state is updated. + + The exact structure of the ouput depends on the value of + mf->type, and FSL_RC_TYPE is returned if this function cannot + figure out what to do with the given deck's type. + + @see fsl_deck_unshuffle() + @see fsl_deck_save() +*/ +FSL_EXPORT int fsl_deck_output( fsl_deck * d, fsl_output_f out, + void * outputState ); + + +/** + Saves the given deck into f's repository database as new control + artifact content. If isPrivate is true then the content is + marked as private, otherwise it is not. Note that isPrivate is a + suggestion and might be trumped by existing state within f or + its repository, and such a trumping is not treated as an + error. e.g. tags are automatically private when they tag private + content. + + Before saving, the deck is passed through fsl_deck_unshuffle() + and fsl_deck_output(), which will fail for a variety of + easy-to-make errors such as the deck missing required cards. + For unshuffle purposes, the R-card gets calculated if the deck + has any F-cards AND if the caller has not already set/calculated + it AND if f's FSL_CX_F_CALC_R_CARD flag is set (it is on by + default for historical reasons, but this may change at some + point). + + Returns 0 on success, the usual non-0 suspects on error. + + - If d->rid is positive, it is assumed to refer to a valid + `blob.rid` and this function returns 0 without side-effects. It + does _not_ attempt to re-save such a record, as that could very + possible result in a new hash (and would definitely do so if the + deck's state had been modified in the slightest bit (literally + "bit")). It should arguably return FSL_RC_ALREADY_EXISTS for this + case, but does not do so for historical reasons. + + - PREVIOUSLY (prior to 2021-10-20), we assumed such a deck referred + to a phantom which we would then save, but that behaviour seems to + be misguided, as we only ever want to save newly-created decks, not + decks created from existing content. + + On success, d->rid be set to the new record's RID. It will only be + set on success because it would otherwise refer to a db record + which get destroyed when the transaction rolls back. + + After saving, the deck will be cross-linked to update any + relationships described by the deck. + + The save operation happens within a transaction, of course, and + on any sort of error, db-side changes are rolled back. Note that + it _is_ legal to start the transaction before calling this, + which effectively makes this operation part of that transaction. + + This function will fail with FSL_RC_ACCESS if d is a delta manifest + (has a B-card) d->f's forbid-delta-manifests configuration option + is set to a truthy value. See fsl_repo_forbids_delta_manifests(). + + Maintenance reminder: this function also does a very small bit of + artifact-type-specific processing. + + @see fsl_deck_output() + @see fsl_content_put_ex() +*/ +FSL_EXPORT int fsl_deck_save( fsl_deck * const d, bool isPrivate ); + +/** + This starts a transaction (possibly nested) on the repository db + and initializes some temporary db state needed for the + crosslinking certain artifact types. It "should" (see below) be + called at the start of the crosslinking process. Crosslinking + *can* work without this but certain steps for certain (subject to + change) artifact types will be skipped, possibly leading to + unexpected timeline data or similar funkiness. No permanent + SCM-relevant state will be missing, but the timeline might not be + updated and tickets might not be fully processed. This should be + used before crosslinking any artifact types, but will only have + significant side effects for certain (subject to change) types. + + Returns 0 on success. + + If this function succeeds, the caller is OBLIGATED to either call + fsl_crosslink_end() or fsl_db_transaction_rollback(), depending + on whether the work done after this call succeeds + resp. fails. This process may install temporary tables and/or + triggers, so failing to call one or the other of those will result + in misbehavior. + + @see fsl_deck_crosslink() +*/ +int fsl_crosslink_begin(fsl_cx * f); + +/** + Must not be called unless fsl_crosslink_begin() has + succeeded. This performs crosslink post-processing on certain + artifact types and cleans up any temporary db state initialized by + fsl_crosslink_begin(). + + Returns 0 on success. On error it initiates (or propagates) a + rollback for the current transaction. +*/ +int fsl_crosslink_end(fsl_cx * f); + +/** + Parses src as Control Artifact content and populates d with it. + + d will be cleaned up before parsing if it has any contents, + retaining its d->f member (which must be non-NULL for + error-reporting purposes). + + This function _might_ take over the contents of the source + buffer on success or it _might_ leave it for the caller to clean + up or re-use, as he sees fit. If the caller does not intend to + re-use the buffer, he should simply pass it to + fsl_buffer_clear() after calling this (no need to check if it + has contents or not first). + + When taking over the contents then on success, after returning + src->mem will be NULL, and all other members will be reset to + their default state. This function only takes over the contents + if it decides to implement certain memory optimizations. + + Ownership of src itself is never changed by this function, only + (possibly!) the ownership of its contents. + + In any case, the content of the source buffer is modified by + this function because (A) that simplifies tokenization greatly, + (B) saves us having to make another copy to work on, (C) the + original implementation did it that way, (D) because in + historical use the source is normally thrown away after parsing, + anyway, and (E) in combination with taking ownership of src's + contents it allows us to optimize away some memory allocations + by re-using the internal memory of the buffer. This function + never changes src's size, but it mutilates its contents + (injecting NUL bytes as token delimiters). + + If d->type is _not_ FSL_SATYPE_ANY when this is called, then + this function requires that the input to be of that type. We can + fail relatively quickly in that case, and this can be used to + save some downstream code some work. Note that the initial type + for decks created using fsl_deck_malloc() or copy-initialized + from ::fsl_deck_empty is FSL_SATYPE_ANY, so normally clients do + not need to set this (unless they want to, as a small + optimization). + + On success it returns 0 and d will be updated with the state + from the input artifact. (Ideally, outputing d via + fsl_deck_output() will produce a lossless copy of the original.) + + On error, if there is error information to propagate beyond the + result code then it is stored in d->f (if that is not NULL), + else in d->error. Whether or not such error info is propagated + depends on the type of error, but anything more trivial than + invalid arguments will be noted there. + + d might be partially populated on error, so regardless of success + or failure, the client must eventually pass d to + fsl_deck_finalize() to free its memory. + + Error result codes include: + + - FSL_RC_MISUSE if any pointer argument is NULL. + + - FSL_RC_SYNTAX on syntax errors. + + - FSL_RC_CONSISTENCY if validation of a Z-card fails. + + - Any number of errors coming from the allocator, database, or + fsl_deck APIs used here. + + ACHTUNG API CHANGE: prior to 2021-10-20, this routine set d->rid + (and the now-removed d->uuid) based on the hash of the input buffer + if a matching record could be found in the db. That proved to be + a huge performance hit and was removed. + + @see fsl_deck_parse2() +*/ +FSL_EXPORT int fsl_deck_parse(fsl_deck * const d, fsl_buffer * const src); + +/** + This variant of fsl_deck_parse() works identically to that + function except for the 3rd argument. + + If you happen to know the _correct_ RID for the deck being parsed, + pass it as the rid argument, else pass 0. A negative value will + result in a FSL_RC_RANGE error. This value is (or will be) only + used as an optimization in other places. Passing a positive value + has no effect on how the content is parsed or on the result - it + only affects internal details/optimizations. +*/ +FSL_EXPORT int fsl_deck_parse2(fsl_deck * const d, fsl_buffer * const src, fsl_id_t rid); + +/** + Quickly determines whether the content held by the given buffer + "might" be a structural artifact. It performs a fast sanity check + for prominent features which can be checked either in O(1) or very + short O(N) time (with a fixed N). If it returns false then the + given buffer's contents are, with 100% certainty, *not* a + structural artifact. If it returns true then they *might* be, but + being 100% certain requires passing the contents to + fsl_deck_parse() to fully parse them. +*/ +FSL_EXPORT bool fsl_might_be_artifact(fsl_buffer const * src); + +/** + Loads the content from given rid and tries to parse it as a + Fossil artifact. If rid==0 the current checkout (if opened) is + used. (Trivia: there can never be a checkout with rid==0 but + rid==0 is sometimes valid for an new/empty repo devoid of + commits). If type==FSL_SATYPE_ANY then it will allow any type of + control artifact, else it returns FSL_RC_TYPE if the loaded + artifact is of the wrong type. + + Returns 0 on success. Results are undefined if f or d are NULL. The + potential error result codes include, but are not limited to: + + - FSL_RC_OOM + + - FSL_RC_RANGE if rid is negative or is 0 and no checkout is + opened. + + - FSL_RC_TYPE if `type` is not `FSL_SATYPE_ANY` and the loaded + result is of any artifact type other than `type`. + + d may be partially populated on error, and the caller must + eventually pass it to fsl_deck_finalize() resp. fsl_deck_clean() + regardless of success or error. This function "could" clean it + up on error, but leaving it partially populated makes debugging + easier. If the error was an artifact type mismatch then d will + "probably" be properly populated but will not hold the type of + artifact requested. It "should" otherwise be well-formed because + parsing errors occur before the type check can happen, but + parsing of invalid manifests might also trigger a FSL_RC_TYPE + error of a different nature. The morale of the storage is: if + this function returns non-0, assume d is useless and needs to be + cleaned up. + + f's error state may be updated on error (for anything more + serious than basic argument validation errors). + + On success this function sets d->rid to rid. + + @see fsl_deck_load_sym() +*/ +FSL_EXPORT int fsl_deck_load_rid( fsl_cx * const f, fsl_deck * const d, + fsl_id_t rid, fsl_satype_e type ); + +/** + A convenience form of fsl_deck_load_rid() which uses + fsl_sym_to_rid() to convert symbolicName into an artifact RID. See + fsl_deck_load_rid() for the symantics of the first, second, and + fourth arguments, as well as the return value. See fsl_sym_to_rid() + for the allowable values of symbolicName. + + @see fsl_deck_load_rid() +*/ +FSL_EXPORT int fsl_deck_load_sym( fsl_cx * const f, fsl_deck * const d, + char const * symbolicName, + fsl_satype_e type ); + +/** + Loads the baseline manifest specified in d->B.uuid, if any and if + necessary. Returns 0 on success. If d->B.baseline is already loaded + or d->B.uuid is NULL (in which case there is no baseline), it + returns 0 and has no side effects. + + Neither argument may be NULL and d must be a fully-populated + object, complete with a proper d->rid, before calling this. + + On success 0 is returned. If d->B.baseline is NULL then + it means that d has no baseline manifest (and d->B.uuid will be NULL + in that case). If d->B.baseline is not NULL then it is owned by + d and will be cleaned up when d is cleaned/finalized. + + Error codes include, but are not limited to: + + - FSL_RC_MISUSE if !d->f. + + - FSL_RC_NOT_A_REPO if d->f has no opened repo db. + + - FSL_RC_RANGE if d->rid<=0, but that code might propagate up from + a lower-level call as well. + + On non-trivial errors d->f's error state will be updated to hold + a description of the problem. + + Some misuses trigger assertions in debug builds. +*/ +FSL_EXPORT int fsl_deck_baseline_fetch( fsl_deck * d ); + +/** + A callback interface for manifest crosslinking, so that we can farm + out the updating of the event table. Each callback registered via + fsl_xlink_listener() will be called at the end of the so-called + crosslinking process, which is run every time a control artifact is + processed for d->f's repository database, passed the deck being + crosslinked and the client-provided state which was registered with + fsl_xlink_listener(). Note that the deck object itself holds other + state useful for crosslinking, like the blob.rid value of the deck + and its fsl_cx instance. + + If an implementation is only interested in a specific type of + artifact, it must check d->type and return 0 if it's an + "uninteresting" type. + + Implementations must return 0 on success or some other fsl_rc_e + value on error. Returning non-0 causes the database transaction + for the crosslinking operation to roll back, effectively + cancelling whatever pending operation triggered the + crosslink. If any callback fails, processing stops immediately - + no other callbacks are executed. + + Implementations which want to report more info than an integer + should call fsl_cx_err_set() to set d->f's error state, as that + will be propagated up to the code which initiated the failed + crosslink. + + ACHTUNG and WARNING: the fsl_deck parameter "really should" be + const, but certain operations on a deck are necessarily non-const + operations. That includes, but may not be limited to: + + - Iterating over F-cards, which requires calling + fsl_deck_F_rewind() before doing so. + + - Loading a checkin's baseline (required for F-card iteration and + performed automatically by fsl_deck_F_rewind()). + + Aside from such iteration-related mutable state, it is STRICTLY + ILLEGAL to modify a deck's artifact-related state while it is + undergoing crosslinking. It is legal to modify its error state. + + + Potential TODO: add some client-opaque state to decks so that they + can be flagged as "being crosslinked" and fail mutation operations + such as card adders/setters. + + @see fsl_xlink_listener() +*/ +typedef int (*fsl_deck_xlink_f)(fsl_deck * d, void * state); + +/** + A type for holding a callback/state pair for manifest + crosslinking callbacks. +*/ +struct fsl_xlinker { + char const * name; + /** Callback function. */ + fsl_deck_xlink_f f; + /** State for this->f's last argument. */ + void * state; +}; +typedef struct fsl_xlinker fsl_xlinker; + +/** Empty-initialized fsl_xlinker struct, intended for const-copy + intialization. */ +#define fsl_xlinker_empty_m {NULL,NULL,NULL} + +/** Empty-initialized fsl_xlinker struct, intended for copy intialization. */ +extern const fsl_xlinker fsl_xlinker_empty; + +/** + A list of fsl_xlinker instances. +*/ +struct fsl_xlinker_list { + /** Number of used items in this->list. */ + fsl_size_t used; + /** Number of slots allocated in this->list. */ + fsl_size_t capacity; + /** Array of this->used elements. */ + fsl_xlinker * list; +}; +typedef struct fsl_xlinker_list fsl_xlinker_list; + +/** Empty-initializes fsl_xlinker_list struct, intended for + const-copy intialization. */ +#define fsl_xlinker_list_empty_m {0,0,NULL} + +/** Empty-initializes fsl_xlinker_list struct, intended for copy intialization. */ +extern const fsl_xlinker_list fsl_xlinker_list_empty; + +/** + Searches f's crosslink callbacks for an entry with the given + name and returns that entry, or NULL if no match is found. The + returned object is owned by f. +*/ +fsl_xlinker * fsl_xlinker_by_name( fsl_cx * f, char const * name ); + +/** + Adds the given function as a "crosslink callback" for the given + Fossil context. The callback is called at the end of a + successfull fsl_deck_crosslink() operation and provides a way + for the client to perform their own work based on the app having + crosslinked an artifact. Crosslinking happens when artifacts are + saved or upon a rebuild operation. + + This function returns 0 on success, non-0 on error. Behaviour is + undefined if any of the first 3 arguments are NULL. + + Crosslink callbacks are called at the end of the core crosslink + steps, in the order they are registered, with the caveat that if a + listener is overwritten by another with the same name, the new + entry retains the older one's position in the list. The library may + register its own before the client gets a chance to. + + If _any_ crosslinking callback fails (returns non-0) then the + _whole_ crosslinking fails and is rolled back (which may very + well include pending tags/commits/whatever getting rolled back). + + The state parameter has no meaning for this function, but is + passed on as the final argument to cb(). If not NULL, cbState + "may" be required to outlive f, depending on cbState's exact + client-side internal semantics/use, as there is currently no API + to remove registered crosslink listeners. + + The name must be non-NULL/empty. If a listener is registered with a + duplicate name then the first one is replaced. This function does + not copy the name bytes - they are assumed to be static or + otherwise to live at least as long as f. The name may be + arbitrarily long, but must have a terminating NUL byte. It is + recommended that clients choose a namespace/prefix to apply to the + names they register. The library reserves the prefix "fsl/" for + its own use, and will happily overwrite client-registered entries + with the same names. The name string need not be stable across + application sessions and maybe be a randomly-generated string. + + Caveat: some obscure artifact crosslinking steps do not happen + unless crosslinking takes place in the context of a + fsl_crosslink_begin() and fsl_crosslink_end() + session. Thus, at the time client-side crosslinker callbacks are + called, certain crosslinking state in the database may still be + pending. It is as yet unclear how best to resolve that minor + discrepancy, or whether it even needs resolving. + + + Default (overrideable) crosslink handlers: + + The library internally splits crosslinking of artifacts into two + parts: the main one (which clients cannot modify) handles the + database-level linking of relational state implied by a given + artifact. The secondary one adds an entry to the "event" table, + which is where Fossil's timeline lives. The crosslinkers for the + timeline updates may be overridden by clients by registering + a crosslink listener with the following names: + + - Attachment artifacts: "fsl/attachment/timeline" + + - Checkin artifacts: "fsl/checkin/timeline" + + - Control artifacts: "fsl/control/timeline" + + - Forum post artifacts: "fsl/forumpost/timeline" + + - Technote artifacts: "fsl/technote/timeline" + + - Wiki artifacts: "fsl/wiki/timeline" + + A context registers listeners under those names when it + initializes, and clients may override them at any point after that. + + Caveat: updating the timeline requires a bit of knowledge about the + Fossil DB schema and/or conventions. Updates for certain types, + e.g. attachment/control/forum post, is somewhat more involved and + updating the timeline for wiki comments requires observing a "quirk + of conventions" for labeling such comments, such that they will + appear properly when the main fossil app renders them. That said, + the only tricky parts of those updates involve generating the + "correct" comment text. So long as the non-comment parts are + updated properly (that part is easy to do), fossil can function + with it. The timeline comment text/links are soley for human + consumption. Fossil makes much use of the "event" table internally, + however, so the rest of that table must be properly populated. + + Because of that caveat, clients may, rather than overriding the + defaults, install their own crosslink listners which ammend the + state applied by the default ones. e.g. add a listener which + watches for checkin updates and replace the default-installed + comment with one suitable for your application, leaving the rest of + the db state in place. At its simplest, that looks more or less + like the following code (inside a fsl_deck_xlink_f() callback): + + ``` + int rc = fsl_db_exec(fsl_cx_db_repo(deck->f), + "UPDATE event SET comment=%Q " + "WHERE objid=%"FSL_ID_T_PFMT, + "the new comment.", deck->rid); + ``` +*/ +FSL_EXPORT int fsl_xlink_listener( fsl_cx * const f, char const * name, + fsl_deck_xlink_f cb, void * cbState ); + + +/** + For the given blob.rid value, returns the blob.size value of + that record via *rv. Returns 0 or higher on success, -1 if a + phantom record is found, -2 if no entry is found, or a smaller + negative value on error (dig around the sources to decode them - + this is not expected to fail unless the system is undergoing a + catastrophe). + + @see fsl_content_blob() + @see fsl_content_get() +*/ +FSL_EXPORT fsl_int_t fsl_content_size( fsl_cx * f, fsl_id_t blobRid ); + +/** + For the given blob.rid value, fetches the content field of that + record and overwrites tgt's contents with it (reusing tgt's + memory if it has any and if it can). The blob's contents are + uncompressed if they were stored in compressed form. This + extracts a raw blob and does not apply any deltas - use + fsl_content_get() to fully expand a delta-stored blob. + + Returns 0 on success. On error tgt might be partially updated, + e.g. it might be populated with compressed data instead of + uncompressed. On error tgt's contents should be recycled + (e.g. fsl_buffer_reuse()) or discarded (e.g. fsl_buffer_clear()) by + the client. Returns FSL_RC_RANGE if blobRid<=0, FSL_RC_NOT_A_REPO + if f has no repo opened, FSL_RC_OOM on allocation error, or + potentially any number of other codes via the db layer. + + Results are undefined if any pointer argument is NULL. + + @see fsl_content_get() + @see fsl_content_size() +*/ +FSL_EXPORT int fsl_content_blob( fsl_cx * const f, fsl_id_t blobRid, + fsl_buffer * const tgt ); + +/** + Functionally similar to fsl_content_blob() but does a lot of + work to ensure that the returned blob is expanded from its + deltas, if any. The tgt buffer's memory, if any, will be + replaced/reused if it has any. + + Returns 0 on success. There are no less than 50 potental + different errors, so we won't bother to list them all. On error + tgt might be partially populated. The basic error cases are: + + - FSL_RC_MISUSE if !tgt or !f. + + - FSL_RC_RANGE if rid<=0 or if an infinite loop is discovered in + the repo delta table links (that is a consistency check to avoid + an infinite loop - that condition "cannot happen" because the + verify-before-commit logic catches that error case). + + - FSL_RC_NOT_A_REPO if f has no repo db opened. + + - FSL_RC_NOT_FOUND if the given rid is not in the repo db. + + - FSL_RC_OOM if an allocation fails. + + + @see fsl_content_blob() + @see fsl_content_size() +*/ +FSL_EXPORT int fsl_content_get( fsl_cx * const f, fsl_id_t blobRid, + fsl_buffer * const tgt ); + +/** + Uses fsl_sym_to_rid() to convert sym to a record ID, then + passes that to fsl_content_get(). Returns 0 on success. +*/ +FSL_EXPORT int fsl_content_get_sym( fsl_cx * const f, char const * sym, + fsl_buffer * const tgt ); + +/** + Returns true if the given rid is marked as PRIVATE in f's current + repository. Returns false (0) on error or if the content is not + marked as private. +*/ +FSL_EXPORT bool fsl_content_is_private(fsl_cx * const f, fsl_id_t rid); + +/** + Marks the given rid public, if it was previously marked as + private. Returns 0 on success, non-0 on error. + + Note that it is not possible to make public content private. +*/ +FSL_EXPORT int fsl_content_make_public(fsl_cx * const f, fsl_id_t rid); + +/** + Generic callback interface for visiting decks. The interface + does not generically require that d survive after this call + returns. + + Implementations must return 0 on success, non-0 on error. Some + APIs using this interface may specify that FSL_RC_BREAK can be + used to stop iteration over a loop without signaling an error. + In such cases the APIs will translate FSL_RC_BREAK to 0 for + result purposes, but will stop looping over whatever it is they + are looping over. +*/ +typedef int (*fsl_deck_visitor_f)( fsl_cx * f, fsl_deck const * d, + void * state ); + +/** + For each unique wiki page name in f's repostory, this calls + cb(), passing it the manifest of the most recent version of that + page. The callback should return 0 on success, FSL_RC_BREAK to + stop looping without an error, or any other non-0 code + (preferably a value from fsl_rc_e) on error. + + The 3rd parameter has no meaning for this function but it is + passed on as-is to the callback. + + ACHTUNG: the deck passed to the callback is transient and will + be cleaned up after the callback has returned, so the callback + must not hold a pointer to it or its contents. + + @see fsl_wiki_load_latest() + @see fsl_wiki_latest_rid() + @see fsl_wiki_names_get() + @see fsl_wiki_page_exists() +*/ +FSL_EXPORT int fsl_wiki_foreach_page( fsl_cx * f, fsl_deck_visitor_f cb, void * state ); + +/** + Fetches the most recent RID for the given wiki page name and + assigns *newId (if it is not NULL) to that value. Returns 0 on + success, FSL_RC_MISUSE if !f or !pageName, FSL_RC_RANGE if + !*pageName, and a host of other potential db-side errors + indicating more serious problems. If no such page is found, + newRid is not modified and this function returns 0 (as opposed + to FSL_RC_NOT_FOUND) because that simplifies usage (so far). + + On error *newRid is not modified. + + @see fsl_wiki_load_latest() + @see fsl_wiki_foreach_page() + @see fsl_wiki_names_get() + @see fsl_wiki_page_exists() +*/ +FSL_EXPORT int fsl_wiki_latest_rid( fsl_cx * f, char const * pageName, fsl_id_t * newRid ); + +/** + Loads the artifact for the most recent version of the given wiki page, + populating d with its contents. + + Returns 0 on success. On error d might be partially populated, + so it needs to be passed to fsl_deck_finalize() regardless of + whether this function succeeds or fails. + + Returns FSL_RC_NOT_FOUND if no page with that name is found. + + @see fsl_wiki_latest_rid() + @see fsl_wiki_names_get() + @see fsl_wiki_page_exists() +*/ +FSL_EXPORT int fsl_wiki_load_latest( fsl_cx * f, char const * pageName, fsl_deck * d ); + +/** + Returns true (non-0) if f's repo database contains a page with the + given name, else false. + + @see fsl_wiki_load_latest() + @see fsl_wiki_latest_rid() + @see fsl_wiki_names_get() + @see fsl_wiki_names_get() +*/ +FSL_EXPORT bool fsl_wiki_page_exists(fsl_cx * f, char const * pageName); + +/** + A helper type for use with fsl_wiki_save(), intended primarily + to help client-side code readability somewhat. +*/ +enum fsl_wiki_save_mode_t { +/** + Indicates that fsl_wiki_save() must only allow the creation of + a new page, and must fail if such an entry already exists. +*/ +FSL_WIKI_SAVE_MODE_CREATE = -1, +/** + Indicates that fsl_wiki_save() must only allow the update of an + existing page, and will not create a branch new page. +*/ +FSL_WIKI_SAVE_MODE_UPDATE = 0, +/** + Indicates that fsl_wiki_save() must allow both the update and + creation of pages. Trivia: "upsert" is a common SQL slang + abbreviation for "update or insert." +*/ +FSL_WIKI_SAVE_MODE_UPSERT = 1 +}; + +typedef enum fsl_wiki_save_mode_t fsl_wiki_save_mode_t; + +/** + Saves wiki content to f's repository db. + + pageName is the name of the page to update or create. + + b contains the content for the page. + + userName specifies the user name to apply to the change. If NULL + or empty then fsl_cx_user_get() or fsl_guess_user_name() are + used (in that order) to determine the name. + + mimeType specifies the mime type for the content (may be NULL). + Mime type names supported directly by fossil(1) include (as of + this writing): text/x-fossil-wiki, text/x-markdown, + text/plain + + Whether or not this function is allowed to create a new page is + determined by creationPolicy. If it is + FSL_WIKI_SAVE_MODE_UPDATE, this function will fail with + FSL_RC_NOT_FOUND if no page with the given name already exists. + If it is FSL_WIKI_SAVE_MODE_CREATE and a previous version _does_ + exist, it fails with FSL_RC_ALREADY_EXISTS. If it is + FSL_WIKI_SAVE_MODE_UPSERT then both the save-exiting and + create-new cases are allowed. In summary: + + - use FSL_WIKI_SAVE_MODE_UPDATE to allow updates to existing pages + but disallow creation of new pages, + + - use FSL_WIKI_SAVE_MODE_CREATE to allow creating of new pages + but not of updating an existing page. + + - FSL_WIKI_SAVE_MODE_UPSERT allows both updating and creating + a new page on demand. + + Returns 0 on success, or any number fsl_rc_e codes on error. On + error no content changes are saved, and any transaction is + rolled back or a rollback is scheduled if this function is + called while a transaction is active. + + + Potential TODO: add an optional (fsl_id_t*) output parameter + which gets set to the new record's RID. + + @see fsl_wiki_page_exists() + @see fsl_wiki_names_get() +*/ +FSL_EXPORT int fsl_wiki_save(fsl_cx * f, char const * pageName, + fsl_buffer const * b, char const * userName, + char const * mimeType, fsl_wiki_save_mode_t creationPolicy ); + +/** + Fetches the list of all wiki page names in f's current repo db + and appends them as new (char *) strings to tgt. On error tgt + might be partially populated (but this will only happen on an + OOM or serious system-level error). + + It is up to the caller free the entries added to the list. Some + of the possibilities include: + + ``` + fsl_list_visit( list, 0, fsl_list_v_fsl_free, NULL ); + fsl_list_reserve(list,0); + // Or: + fsl_list_clear(list, fsl_list_v_fsl_free, NULL); + // Or simply: + fsl_list_visit_free( list, 1 ); + ``` + +*/ +FSL_EXPORT int fsl_wiki_names_get( fsl_cx * f, fsl_list * tgt ); + +/** + F-cards each represent one file entry in a Manifest Artifact (i.e., + a checkin version). + + All of the non-const pointers in this class are owned by the + respective instance of the class OR by the fsl_deck which created + it, and must neither be modified nor freed except via the + appropriate APIs. +*/ +struct fsl_card_F { + /** + UUID of the underlying blob record for the file. NULL for + removed entries. + */ + fsl_uuid_str uuid; + /** + Name of the file. + */ + char * name; + /** + Previous name if the file was renamed, else NULL. + */ + char * priorName; + /** + File permissions. Fossil only supports one "permission" per + file, and it does not necessarily map to a real + filesystem-level permission. + + @see fsl_fileperm_e + */ + fsl_fileperm_e perm; + + /** + An internal optimization. Do not mess with this. When this is + true, the various string members of this struct are not owned + by this struct, but by the deck which created this struct. This + is used when loading decks from storage - the strings are + pointed to the original content data, rather than strdup()'d + copies of it. fsl_card_F_clean() will DTRT and delete the + strings (or not). + */ + bool deckOwnsStrings; +}; +/** + Empty-initialized fsl_card_F structure, intended for use in + initialization when embedding fsl_card_F in another struct or + copy-initializing a const struct. +*/ +#define fsl_card_F_empty_m { \ + NULL/*uuid*/, \ + NULL/*name*/, \ + NULL/*priorName*/, \ + 0/*perm*/, \ + false/*deckOwnsStrings*/ \ +} +FSL_EXPORT const fsl_card_F fsl_card_F_empty; + +/** + Represents a J card in a Ticket Control Artifact. +*/ +struct fsl_card_J { + /** + If true, the new value should be appended to any existing one + with the same key, else it will replace any old one. + */ + char append; + /** + For internal use only. + */ + unsigned char flags; + /** + The ticket field to update. The bytes are owned by this object. + */ + char * field; + /** + The value for the field. The bytes are owned by this object. + */ + char * value; +}; +/** Empty-initialized fsl_card_J struct. */ +#define fsl_card_J_empty_m {0,0,NULL, NULL} +/** Empty-initialized fsl_card_J struct. */ +FSL_EXPORT const fsl_card_J fsl_card_J_empty; + +/** + Represents a tag in a Manifest or Control Artifact. +*/ +struct fsl_card_T { + /** + The type of tag. + */ + fsl_tagtype_e type; + /** + UUID of the artifact this tag is tagging. When applying a tag to + a new checkin, this value is left empty (=NULL) and gets replaced + by a '*' in the resulting control artifact. + */ + fsl_uuid_str uuid; + /** + The tag's name. The bytes are owned by this object. + */ + char * name; + /** + The tag's value. May be NULL/empty. The bytes are owned by + this object. + */ + char * value; +}; +/** Defaults-initialized fsl_card_T instance. */ +#define fsl_card_T_empty_m {FSL_TAGTYPE_INVALID, NULL, NULL,NULL} +/** Defaults-initialized fsl_card_T instance. */ +FSL_EXPORT const fsl_card_T fsl_card_T_empty; + +/** + Types of cherrypick merges. +*/ +enum fsl_cherrypick_type_e { +/** Sentinel value. */ +FSL_CHERRYPICK_INVALID = 0, +/** Indicates a cherrypick merge. */ +FSL_CHERRYPICK_ADD = 1, +/** Indicates a cherrypick backout. */ +FSL_CHERRYPICK_BACKOUT = -1 +}; +typedef enum fsl_cherrypick_type_e fsl_cherrypick_type_e; + +/** + Represents a Q card in a Manifest or Control Artifact. +*/ +struct fsl_card_Q { + /** 0==invalid, negative==backed out, positive=cherrypicked. */ + fsl_cherrypick_type_e type; + /** + UUID of the target of the cherrypick. The bytes are owned by + this object. + */ + fsl_uuid_str target; + /** + UUID of the baseline for the cherrypick. The bytes are owned by + this object. + */ + fsl_uuid_str baseline; +}; +/** Empty-initialized fsl_card_Q struct. */ +#define fsl_card_Q_empty_m {FSL_CHERRYPICK_INVALID, NULL, NULL} +/** Empty-initialized fsl_card_Q struct. */ +FSL_EXPORT const fsl_card_Q fsl_card_Q_empty; + +/** + Allocates a new J-card record instance + + On success it returns a new record which must eventually be + passed to fsl_card_J_free() to free its resources. On + error (invalid arguments or allocation error) it returns NULL. + field may not be NULL or empty but value may be either. + + These records are immutable - the API provides no way to change + them once they are instantiated. +*/ +FSL_EXPORT fsl_card_J * fsl_card_J_malloc(bool isAppend, + char const * field, + char const * value); +/** + Frees a J-card record created by fsl_card_J_malloc(). + Is a no-op if cp is NULL. +*/ +FSL_EXPORT void fsl_card_J_free( fsl_card_J * cp ); + +/** + Allocates a new fsl_card_T instance. If any of the pointer + parameters are non-NULL, their values are assumed to be + NUL-terminated strings, which this function copies. Returns NULL + on allocation error. The returned value must eventually be passed + to fsl_card_T_clean() or fsl_card_T_free() to free its resources. + + If uuid is not NULL and fsl_is_uuid(uuid) returns false then + this function returns NULL. If it is NULL and gets assigned + later, it must conform to fsl_is_uuid()'s rules or downstream + results are undefined. + + @see fsl_card_T_free() + @see fsl_card_T_clean() + @see fsl_deck_T_add() +*/ +FSL_EXPORT fsl_card_T * fsl_card_T_malloc(fsl_tagtype_e tagType, + fsl_uuid_cstr uuid, + char const * name, + char const * value); +/** + If t is not NULL, calls fsl_card_T_clean(t) and then passes t to + fsl_free(). + + @see fsl_card_T_clean() +*/ +FSL_EXPORT void fsl_card_T_free(fsl_card_T *t); + +/** + Frees up any memory owned by t and clears out t's state, + but does not free t. + + @see fsl_card_T_free() +*/ +FSL_EXPORT void fsl_card_T_clean(fsl_card_T *t); + +/** + Allocates a new cherrypick record instance. The type argument must + be one of FSL_CHERRYPICK_ADD or FSL_CHERRYPICK_BACKOUT. target + must be a valid UUID string. If baseline is not NULL then it also + must be a valid UUID. + + On success it returns a new record which must eventually be + passed to fsl_card_Q_free() to free its resources. On + error (invalid arguments or allocation error) it returns NULL. + + These records are immutable - the API provides no way to change + them once they are instantiated. +*/ +FSL_EXPORT fsl_card_Q * fsl_card_Q_malloc(fsl_cherrypick_type_e type, + fsl_uuid_cstr target, + fsl_uuid_cstr baseline); +/** + Frees a cherrypick record created by fsl_card_Q_malloc(). + Is a no-op if cp is NULL. +*/ +FSL_EXPORT void fsl_card_Q_free( fsl_card_Q * cp ); + +/** + Returns true (non-0) if f is not NULL and f has an opened repo + which contains a checkin with the given rid, else it returns + false. + + As a special case, if rid==0 then this only returns true + if the repository currently has no content in the blob + table. +*/ +FSL_EXPORT char fsl_rid_is_a_checkin(fsl_cx * f, fsl_id_t rid); + +/** + Fetches the list of all directory names for a given checkin record + id or (if rid is negative) the whole repo over all of its combined + history. Each name entry in the list is appended to tgt. The + results are reduced to unique names only and are sorted + lexically. If addSlash is true then each entry will include a + trailing slash character, else it will not. The list does not + include an entry for the top-most directory. + + If rid is less than 0 then the directory list across _all_ + versions is returned. If it is 0 then the current checkout's RID + is used (if a checkout is opened, otherwise a usage error is + triggered). If it is positive then only directories for the + given checkin RID are returned. If rid is specified, it is + assumed to be the record ID of a commit (manifest) record, and + it is impossible to distinguish between the results "invalid + rid" and "empty directory list" (which is a legal result). + + On success it returns 0 and tgt will have a number of (char *) + entries appended to it equal to the number of subdirectories in + the repo (possibly 0). + + Returns non-0 on error, FSL_RC_MISUSE if !f, !tgt. On other + errors error tgt might have been partially populated and the + list contents should not be considered valid/complete. + + Ownership of the returned strings is transfered to the caller, + who must eventually free each one using + fsl_free(). fsl_list_visit_free() is the simplest way to free + them all at once. +*/ +FSL_EXPORT int fsl_repo_dir_names( fsl_cx * f, fsl_id_t rid, + fsl_list * tgt, bool addSlash ); + + +/** + ZIPs up a copy of the contents of a specific version from f's + opened repository db. sym is the symbolic name for the checkin + to ZIP. filename is the name of the ZIP file to output the + result to. See fsl_zip_writer for details and caveats of this + library's ZIP creation. If vRootDir is not NULL and not empty + then each file injected into the ZIP gets that directory + prepended to its name. + + If progressVisitor is not NULL then it is called once just + before each file is processed, passed the F-card for the file + about to be zipped and the progressState parameter. If it + returns non-0, ZIPping is cancelled and that error code is + returned. This is intended primarily for providing feedback on + the update process, but could also be used to cancel the + operation between files. + + As of 2021-09-05 this routine automatically adds the files + (manifest, manifest.uuid, manifest.tags) to the zip file, + regardless of repository-level settings regarding those + pseudo-files (see fsl_ckout_manifest_write()). As there are no + F-cards associated with those non-files, the progressVisitor is not + called for those. + + BUG: this function does not honor symlink content in a + fossil-compatible fashion. If it encounters a symlink entry + during ZIP generation, it will fail and f's error state will be + updated with an explanation of this shortcoming. + + @see fsl_zip_writer + @see fsl_card_F_visitor_f() +*/ +FSL_EXPORT int fsl_repo_zip_sym_to_filename( fsl_cx * f, char const * sym, + char const * vRootDir, + char const * fileName, + fsl_card_F_visitor_f progressVisitor, + void * progressState); + + +/** + Callback state for use with fsl_repo_extract_f() implementations + to stream a given version of a repository's file's, one file at a + time, to a client. Instances are never created by client code, + only by fsl_repo_extract() and its delegates, which pass them to + client-provided fsl_repo_extract_f() functions. +*/ +struct fsl_repo_extract_state { + /** + The associated Fossil context. + */ + fsl_cx * f; + /** + RID of the checkin version for this file. For a given call to + fsl_repo_extract(), this number will be the same across all + calls to the callback function. + */ + fsl_id_t checkinRid; + /** + File-level blob.rid for fc. Can be used with, e.g., + fsl_mtime_of_manifest_file(). + */ + fsl_id_t fileRid; + /** + Client state passed to fsl_repo_extract(). Its interpretation + is callback-implementation-dependent. + */ + void * callbackState; + /** + The F-card being iterated over. This holds the repo-level + metadata associated with the file, other than its RID, which is + available via this->fileRid. + + Deleted files are NOT reported via the extraction process + because reporting them accurately is trickier and more + expensive than it could be. Thus this member's uuid field + will always be non-NULL. + + Certain operations which use this class, e.g. fsl_repo_ckout() + and fsl_ckout_update(), will temporarily synthesize an F-card to + represent the state of a file update, in which case this object's + contents might not 100% reflect any given db-side state. e.g. + fsl_ckout_update() synthesizes an F-card which reflects the + current state of a file after applying an update operation to it. + In such cases, the fCard->uuid may refer to a repository-side + file even though the hash of the on-disk file contents may differ + because of, e.g., a merge. + */ + fsl_card_F const * fCard; + + /** + If the fsl_repo_extract_opt object which was used to initiate the + current extraction has the extractContent member set to false, + this will be a NULL pointer. If it's true, this member points to + a transient buffer which holds the full, undelta'd/uncompressed + content of fc's file record. The content bytes are owned by + fsl_repo_extract() and are invalidated as soon as this callback + returns, so the callback must copy/consume them immediately if + needed. + */ + fsl_buffer const * content; + + /** + These counters can be used by an extraction callback to calculate + a progress percentage. + */ + struct { + /** The current file number, starting at 1. */ + uint32_t fileNumber; + /** Total number of files to extract. */ + uint32_t fileCount; + } count; +}; +typedef struct fsl_repo_extract_state fsl_repo_extract_state; + +/** + Initialized-with-defaults fsl_repo_extract_state instance, intended + for const-copy initialization. +*/ +#define fsl_repo_extract_state_empty_m {\ + NULL/*f*/, 0/*checkinRid*/, 0/*fileRid*/, \ + NULL/*state*/, NULL/*fCard*/, NULL/*content*/, \ + {/*count*/0,0} \ +} +/** + Initialized-with-defaults fsl_repo_extract_state instance, + intended for non-const copy initialization. +*/ +FSL_EXPORT const fsl_repo_extract_state fsl_repo_extract_state_empty; + +/** + A callback type for use with fsl_repo_extract(). See + fsl_repo_extract_state for the meanings of xstate's various + members. The xstate memory must be considered invalidated + immediately after this function returns, thus implementations + must copy or consume anything they need from xstate before + returning. + + Implementations must return 0 on success. As a special case, if + FSL_RC_BREAK is returned then fsl_repo_extract() will stop + looping over files but will report it as success (by returning + 0). Any other code causes extraction looping to stop and is + returned as-is to the caller of fsl_repo_extract(). + + When returning an error, the client may use fsl_cx_err_set() to + populate state->f with a useful error message which will + propagate back up through the call stack. + + @see fsl_repo_extract() +*/ +typedef int (*fsl_repo_extract_f)( fsl_repo_extract_state const * xstate ); + +/** + Options for use with fsl_repo_extract(). +*/ +struct fsl_repo_extract_opt { + /** + The version of the repostitory to check out. This must be + the blob.rid of a checkin artifact. + */ + fsl_id_t checkinRid; + /** + The callback to call for each extracted file in the checkin. + May not be NULL. + */ + fsl_repo_extract_f callback; + /** + Optional state pointer to pass to the callback when extracting. + Its interpretation is client-dependent. + */ + void * callbackState; + /** + If true, the fsl_repo_extract_state::content pointer passed to + the callback will be non-NULL and will contain the content of the + file. If false, that pointer will be NULL. Such extraction is a + relatively costly operation, so should only be enabled when + necessary. Some uses cases can delay this decision until the + callback and only fetch the content for cases which need it. + */ + bool extractContent; +}; + +typedef struct fsl_repo_extract_opt fsl_repo_extract_opt; +/** + Initialized-with-defaults fsl_repo_extract_opt instance, intended + for intializing via const-copy initialization. +*/ +#define fsl_repo_extract_opt_empty_m \ + {0/*checkinRid*/,NULL/*callback*/, \ + NULL/*callbackState*/,false/*extractContent*/} +/** + Initialized-with-defaults fsl_repo_extract_opt instance, + intended for intializing new non-const instances. +*/ +FSL_EXPORT const fsl_repo_extract_opt fsl_repo_extract_opt_empty; + +/** + Extracts the contents of a single checkin from a repository, + sending the appropriate version of each file's contents to a + client-specified callback. + + For each file in the given checkin, opt->callback() is passed a + fsl_repo_extract_state instance containing enough information to, + e.g., unpack the contents to a working directory, add it to a + compressed archive, or send it to some other destination. + + Returns 0 on success, non-0 on error. It will fail if f has no + opened repository db. + + If the callback returns any code other than 0 or FSL_RC_BREAK, + looping over the list of files ends and this function returns + that value. FSL_RC_BREAK causes looping to stop but 0 is + returned. + + Files deleted by the given version are NOT reported to the callback + (because getting sane semantics has proven to be tricker and more + costly than it's worth). + + See fsl_repo_extract_f() for more details about the semantics of + the callback. See fsl_repo_extract_opt for the documentation of the + various options. + + Fossil's internal metadata format guarantees that files will passed + be passed to the callback in "lexical order" (as defined by + fossil's manifest format definition). i.e. the files will be passed + in case-sensitive, alphabetical order. Note that upper-case letters + sort before lower-case ones. + + Sidebar: this function makes a bitwise copy of the 2nd argument + before starting its work, just in case the caller gets the crazy + idea to modify it from the extraction callback. Whether or not + there are valid/interesting uses for such modification remains to + be seen. If any are found, this copy behavior may change. +*/ +FSL_EXPORT int fsl_repo_extract( fsl_cx * f, + fsl_repo_extract_opt const * opt ); + +/** + Equivalent to fsl_tag_an_rid() except that it takes a symbolic + artifact name in place of an artifact ID as the third + argumemnt. + + This function passes symToTag to fsl_sym_to_rid(), and on + success passes the rest of the parameters as-is to + fsl_tag_an_rid(). See that function the semantics of the other + arguments and the return value, as well as a description of the + side effects. +*/ +FSL_EXPORT int fsl_tag_sym( fsl_cx * f, fsl_tagtype_e tagType, + char const * symToTag, char const * tagName, + char const * tagValue, char const * userName, + double mtime, fsl_id_t * newId ); + +/** + Adds a control record to f's repositoriy that either creates or + cancels a tag. + + artifactRidToTag is the RID of the record to be tagged. + + tagType is the type (add, cancel, or propagate) of tag. + + tagName is the name of the tag. Must not be NULL/empty. + + tagValue is the optional value for the tag. May be NULL. + + userName is the user's name to apply to the artifact. May not be + empty/NULL. Use fsl_guess_user_name() to try to figure out a + proper user name based on the environment. See also: + fsl_cx_user_get(), but note that the application must first + use fsl_cx_user_set() to set a context's user name. + + mtime is the Julian Day timestamp for the new artifact. Pass a + value <=0 to use the current time. + + If newId is not NULL then on success the rid of the new tag control + artifact is assigned to *newId. + + Returns 0 on success and has about a million and thirteen + possible error conditions. On success a new artifact record is + written to the db, its RID being written into newId as described + above. + + If the artifact being tagged is private, the new tag is also + marked as private. + +*/ +FSL_EXPORT int fsl_tag_an_rid( fsl_cx * f, fsl_tagtype_e tagType, + fsl_id_t artifactRidToTag, char const * tagName, + char const * tagValue, char const * userName, + double mtime, fsl_id_t * newId ); + +/** + Searches for a repo.tag entry given name in the given context's + repository db. If found, it returns the record's id. If no + record is found and create is true (non-0) then a tag is created + and its entry id is returned. Returns 0 if it finds no entry, a + negative value on error. On db-level error, f's error state is + updated. +*/ +FSL_EXPORT fsl_id_t fsl_tag_id( fsl_cx * f, char const * tag, bool create ); + + +/** + Returns true if the checkin with the given rid is a leaf, false if + not. Returns false if f has no repo db opened, the query fails + (likely indicating that it is not a repository db), or just about + any other conceivable non-success case. + + A leaf, by the way, is a commit which has no children in the same + branch. + + Sidebar: this function calculates whether the RID is a leaf, as + opposed to checking the "static" (pre-calculated) list of leaves in + the [leaf] table. +*/ +FSL_EXPORT bool fsl_rid_is_leaf(fsl_cx * const f, fsl_id_t rid); +/** + Returns true if, according to f's current repo's [event] table, + rid refers to a checkin, else false. +*/ +FSL_EXPORT bool fsl_rid_is_version(fsl_cx * const f, fsl_id_t rid); + +/** + Counts the number of primary non-branch children for the given + check-in. + + A primary child is one where the parent is the primary parent, not + a merge parent. A "leaf" is a node that has zero children of any + kind. This routine counts only primary children. + + A non-branch child is one which is on the same branch as the parent. + + Returns a negative value on error. +*/ +FSL_EXPORT fsl_int_t fsl_count_nonbranch_children(fsl_cx * const f, + fsl_id_t rid); + +/** + Looks for the delta table record where rid==deltaRid, and + returns that record's srcid via *rv. Returns 0 on success, non-0 + on error. If no record is found, *rv is set to 0 and 0 is + returned (as opposed to FSL_RC_NOT_FOUND) because that generally + simplifies the error checking. +*/ +FSL_EXPORT int fsl_delta_src_id( fsl_cx * const f, fsl_id_t deltaRid, + fsl_id_t * rv ); + + +/** + Return true if the given artifact ID should is listed in f's + shun table, else false. +*/ +FSL_EXPORT int fsl_uuid_is_shunned(fsl_cx * const f, fsl_uuid_cstr zUuid); + + +/** + Compute the "mtime" of the file given whose blob.rid is "fid" + that is part of check-in "vid". The mtime will be the mtime on + vid or some ancestor of vid where fid first appears. Note that + fossil does not track the "real" mtimes of files, it only + computes reasonable estimates for those files based on the + timestamps of their most recent checkin in the ancestry of vid. + + On success, if pMTime is not null then the result is written to + *pMTime. + + If fid is 0 or less then the checkin time of vid is written to + pMTime (this is a much less expensive operation, by the way). + In this particular case, FSL_RC_NOT_FOUND is returned if vid is + not a valid checkin version. + + Returns 0 on success, non-0 on error. Returns FSL_RC_NOT_FOUND + if fid is not found in vid. + + This routine is much more efficient if used to answer several + queries in a row for the same manifest (the vid parameter). It + is least efficient when it is passed intermixed manifest IDs, + e.g. (1, 3, 1, 4, 1,...). This is a side-effect of the caching + used in the computation of ancestors for a given vid. +*/ +FSL_EXPORT int fsl_mtime_of_manifest_file(fsl_cx * f, fsl_id_t vid, fsl_id_t fid, fsl_time_t *pMTime); + +/** + A convenience form of fsl_mtime_of_manifest_file() which looks up + fc's RID based on its UUID. vid must be the RID of the checkin + version fc originates from. See fsl_mtime_of_manifest_file() for + full details - this function simply calculates the 3rd argument + for that one. +*/ +FSL_EXPORT int fsl_mtime_of_F_card(fsl_cx * f, fsl_id_t vid, fsl_card_F const * fc, fsl_time_t *pMTime); + +/** + Ensures that the given list has capacity for at least n entries. If + the capacity is currently equal to or less than n, this is a no-op + unless n is 0, in which case li->list is freed and the list is + zeroed out. Else li->list is expanded to hold at least n + elements. Returns 0 on success, FSL_RC_OOM on allocation error. + */ +FSL_EXPORT int fsl_card_F_list_reserve( fsl_card_F_list * li, uint32_t n ); + +/** + Frees all memory owned by li and the F-cards it contains. Does not + free the li pointer. +*/ +FSL_EXPORT void fsl_card_F_list_finalize( fsl_card_F_list * li ); + +/** + Holds options for use with fsl_branch_create(). +*/ +struct fsl_branch_opt { + /** + The checkin RID from which the branch should originate. + */ + fsl_id_t basisRid; + /** + The name of the branch. May not be NULL or empty. + */ + char const * name; + /** + User name for the branch. If NULL, fsl_cx_user_get() will + be used. + */ + char const * user; + /** + Optional comment (may be NULL). If NULL or empty, a default + comment is generated (because fossil requires a non-empty + comment string). + */ + char const * comment; + /** + Optional background color for the fossil(1) HTML timeline + view. Must be in \#RRGGBB format, but this API does not + validate it as such. + */ + char const * bgColor; + /** + The julian time of the branch. If 0 or less, default is the + current time. + */ + double mtime; + /** + If true, the branch will be marked as private. + */ + char isPrivate; +}; +typedef struct fsl_branch_opt fsl_branch_opt; +#define fsl_branch_opt_empty_m { \ + 0/*basisRid*/, NULL/*name*/, \ + NULL/*user*/, NULL/*comment*/, \ + NULL/*bgColor*/, \ + 0.0/*mtime*/, 0/*isPrivate*/ \ + } +FSL_EXPORT const fsl_branch_opt fsl_branch_opt_empty; + +/** + Creates a new branch in f's repository. The 2nd paramter holds + the options describing the branch. The 3rd parameter may be + NULL, but if it is not then on success the RID of the new + manifest is assigned to *newRid. + + In Fossil branches are implemented as tags. The branch name + provided by the client will cause the creation of a tag with + name name plus a "sym-" prefix to be created (if needed). + "sym-" denotes that it is a "symbolic tag" (fossil's term for + "symbolic name applying to one or more checkins," + i.e. branches). + + Creating a branch cancels all other branch tags which the new + branch would normally inherit. + + Returns 0 on success, non-0 on error. +*/ +FSL_EXPORT int fsl_branch_create(fsl_cx * f, fsl_branch_opt const * opt, fsl_id_t * newRid ); + + +/** + Tries to determine the [filename.fnid] value for the given + filename. Returns a positive value if it finds one, 0 if it + finds none, and some unspecified negative value(s) for any sort + of error. filename must be a normalized, relative filename (as it + is recorded by a repo). +*/ +FSL_EXPORT fsl_id_t fsl_repo_filename_fnid( fsl_cx * f, char const * filename ); + + +/** + Imports content to f's opened repository's BLOB table using a + client-provided input source. f must have an opened repository + db. inFunc is the source of the data and inState is the first + argument passed to inFunc(). If inFunc() succeeds in fetching all + data (i.e. if it always returns 0 when called by this function) + then that data is inserted into the blob table _if_ no existing + record with the same hash is already in the table. If such a record + exists, it is assumed that the content is identical and this + function has no side-effects vis-a-vis the db in that case. + + If rid is not NULL then the BLOB.RID record value (possibly of an + older record!) is stored in *rid. If uuid is not NULL then the + BLOB.UUID record value is stored in *uuid and the caller takes + ownership of those bytes, which must eventually be passed to + fsl_free() to release them. + + rid and uuid are only modified on success and only if they are + not NULL. + + Returns 0 on success, non-0 on error. For errors other than basic + argument validation and OOM conditions, f's error state is + updated with a description of the problem. Returns FSL_RC_MISUSE + if either f or inFunc are NULL. Whether or not inState may be + NULL depends on inFunc's concrete implementation. + + Be aware that BLOB.RID values can (but do not necessarily) change + in the life of a repod db (via a reconstruct, a full re-clone, or + similar, or simply when referring to different clones of the same + repo). Thus clients should always store the full UUID, as opposed + to the RID, for later reference. RIDs should, in general, be + treated as session-transient values. That said, for purposes of + linking tables in the db, the RID is used exclusively (clients are + free to link their own extension tables using UUIDs, but doing so + has a performance penalty comared to RIDs). For long-term storage + of external links, and to guaranty that the data be usable with + other copies of the same repo, the UUID is required. + + Note that Fossil may deltify, compress, or otherwise modify + content on its way into the blob table, and it may even modify + content long after its insertion (e.g. to make it a delta against + a newer version). Thus clients should normally never try + to read back the blob directly from the database, but should + instead read it using fsl_content_get(). + + That said: this routine has no way of associating and older version + (if any) of the same content with this newly-imported version, and + therefore cannot delta-compress the older version. + + Maintenance reminder: this is basically just a glorified form of + the internal fsl_content_put(). Interestingly, fsl_content_put() + always sets content to public (by default - the f object may + override that later). It is not yet clear whether this routine + needs to have a flag to set the blob private or not. Generally + speaking, privacy is applied to fossil artifacts, as opposed to + content blobs. + + @see fsl_repo_import_buffer() +*/ +FSL_EXPORT int fsl_repo_import_blob( fsl_cx * f, fsl_input_f inFunc, + void * inState, fsl_id_t * rid, + fsl_uuid_str * uuid ); + +/** + A convenience form of fsl_repo_import_blob(), equivalent to: + + ``` + fsl_repo_import_blob(f, fsl_input_f_buffer, bIn, rid, uuid ) + ``` + + except that (A) bIn is const in this call and non-const in the + other form (due to cursor traversal requirements) and (B) it + returns FSL_RC_MISUSE if bIn is NULL. +*/ +FSL_EXPORT int fsl_repo_import_buffer( fsl_cx * f, fsl_buffer const * bIn, + fsl_id_t * rid, fsl_uuid_str * uuid ); + +/** + Resolves client-provided symbol as an artifact's db record ID. + f must have an opened repository db, and some symbols can only + be looked up if it has an opened checkout (see the list below). + + Returns 0 and sets *rv to the id if it finds an unambiguous + match. + + Returns FSL_RC_MISUSE if !f, !sym, !*sym, or !rv. + + Returns FSL_RC_NOT_A_REPO if f has no opened repository. + + Returns FSL_RC_AMBIGUOUS if sym is a partial UUID which matches + multiple full UUIDs. + + Returns FSL_RC_NOT_FOUND if it cannot find anything. + + Symbols supported by this function: + + - SHA1/3 hash + - SHA1/3 hash prefix of at least 4 characters + - Symbolic Name + - "tag:" + symbolic name + - Date or date-time + - "date:" + Date or date-time + - symbolic-name ":" date-time + - "tip" + + - "rid:###" resolves to the hash of blob.rid ### if that RID is in + the database + + The following additional forms are available in local checkouts: + + - "current" + - "prev" or "previous" + - "next" + + The following prefix may be applied to the above to modify how + they are resolved: + + - "root:" prefix resolves to the checkin of the parent branch from + which the record's branch divered. i.e. the version from which it + was branched. In the trunk this will always resolve to the first + checkin. + + - "merge-in:" TODO - document this once its implications are + understood. + + If type is not FSL_SATYPE_ANY then it will only match artifacts + of the specified type. In order to resolve arbitrary UUIDs, e.g. + those of arbitrary blob content, type needs to be + FSL_SATYPE_ANY. + +*/ +FSL_EXPORT int fsl_sym_to_rid( fsl_cx * const f, char const * sym, + fsl_satype_e type, fsl_id_t * rv ); + +/** + Similar to fsl_sym_to_rid() but on success it returns a UUID string + by assigning it to *rv (if rv is not NULL). If rid is not NULL then + on success the db record ID corresponding to the returned UUID is + assigned to *rid. The caller must eventually free the returned + string memory by passing it to fsl_free(). Returns 0 if it finds a + match and any number of result codes on error. +*/ +FSL_EXPORT int fsl_sym_to_uuid( fsl_cx * f, char const * sym, + fsl_satype_e type, fsl_uuid_str * rv, + fsl_id_t * rid ); + + +/** + Searches f's repo database for the a blob with the given uuid + (any unique UUID prefix). On success a positive record ID is + returned. On error one of several unspecified negative values is + returned. If no uuid match is found 0 is returned. + + Error cases include: either argument is NULL, uuid does not + appear to be a full or partial UUID (or is too long), + uuid is ambiguous (try providing a longer one) + + This implementation is more efficient when given a full, + valid UUID (one for which fsl_is_uuid() returns true). +*/ +FSL_EXPORT fsl_id_t fsl_uuid_to_rid( fsl_cx * const f, char const * uuid ); + +/** + The opposite of fsl_uuid_to_rid(), this returns the UUID string + of the given blob record ID. Ownership of the string is passed + to the caller and it must eventually be freed using + fsl_free(). Returns NULL on error (invalid arguments or f has no + repo opened) or if no blob record is found. If no record is + found, f's error state is updated with an explanation of the + problem. +*/ +FSL_EXPORT fsl_uuid_str fsl_rid_to_uuid(fsl_cx * const f, fsl_id_t rid); + +/** + Works like fsl_rid_to_uuid() but assigns the UUID to the given + buffer, re-using its memory, if any. Returns 0 on success, + FSL_RC_MISUSE if rid is not positive, FSL_RC_OOM on allocation + error, and FSL_RC_NOT_FOUND if no blob entry matching the given rid + is found. +*/ +FSL_EXPORT int fsl_rid_to_uuid2(fsl_cx * const f, fsl_id_t rid, fsl_buffer *uuid); + +/** + This works identically to fsl_rid_to_uuid() except that it will + only resolve to a UUID if an artifact matching the given type has + that UUID. If no entry is found, f's error state gets updated + with a description of the problem. + + This can be used to distinguish artifact UUIDs from file blob + content UUIDs by passing the type FSL_SATYPE_ANY. A non-artifact + blob will return NULL in that case, but any artifact type will + match (assuming rid is valid). +*/ +FSL_EXPORT fsl_uuid_str fsl_rid_to_artifact_uuid(fsl_cx * f, fsl_id_t rid, + fsl_satype_e type); +/** + Returns the raw SQL code for a Fossil global config database. + + TODO: add optional (fsl_size_t*) to return the length. +*/ +FSL_EXPORT char const * fsl_schema_config(); + +/** + Returns the raw SQL code for the "static" parts of a Fossil + repository database. These are the parts which are immutable + (for the most part) between Fossil versions. They change _very_ + rarely. + + TODO: add optional (fsl_size_t*) to return the length. +*/ +FSL_EXPORT char const * fsl_schema_repo1(); + +/** + Returns the raw SQL code for the "transient" parts of a Fossil + repository database - any parts which can be calculated via data + held in the primary "static" schemas. These parts are + occassionally recreated, e.g. via a 'rebuild' of a repository. + + TODO: add optional (fsl_size_t*) to return the length. +*/ +FSL_EXPORT char const * fsl_schema_repo2(); + +/** + Returns the raw SQL code for a Fossil checkout database. + + TODO: add optional (fsl_size_t*) to return the length. +*/ +FSL_EXPORT char const * fsl_schema_ckout(); + +/** + Returns the raw SQL code for a Fossil checkout db's + _default_ core ticket-related tables. + + TODO: add optional (fsl_size_t*) to return the length. + + @see fsl_cx_schema_ticket() +*/ +FSL_EXPORT char const * fsl_schema_ticket(); + +/** + Returns the raw SQL code for the "forum" parts of a Fossil + repository database. + + TODO: add optional (fsl_size_t*) to return the length. +*/ +FSL_EXPORT char const * fsl_schema_forum(); + +/** + If f's opened repository has a non-empty config entry named + 'ticket-table', this returns its text via appending it to + pOut. If no entry is found, fsl_schema_ticket() is appended to + pOut. + + Returns 0 on success. On error the contents of pOut must not be + considered valid but pOut might be partially populated. +*/ +FSL_EXPORT int fsl_cx_schema_ticket(fsl_cx * f, fsl_buffer * pOut); + +/** + Returns the raw SQL code for Fossil ticket reports schemas. + This gets installed as needed into repository databases. + + TODO: add optional (fsl_size_t*) to return the length. +*/ +FSL_EXPORT char const * fsl_schema_ticket_reports(); + +/** + This is a wrapper around fsl_cx_hash_buffer() which looks for a + matching artifact for the given input blob. It first hashes src + using f's "alternate" hash and then, if no match is found, tries + again with f's preferred hash. + + On success (a match is found): + + - Returns 0. + + - If ridOut is not NULL, *ridOut is set to the RID of the matching blob. + + - If hashOut is not NULL, *hashOut is set to the hash of the + blob. Its ownership is transferred to the caller, who must + eventually pass it to fsl_free(). + + If no matching blob is found in the repository, FSL_RC_NOT_FOUND is + returned (but f's error state is not annotated with more + information). Returns FSL_RC_NOT_A_REPO if f has no repository + opened. For more serious errors, e.g. allocation error or db + problems, another (more serious) result code is returned, + e.g. FSL_RC_OOM or FSL_RC_DB. + + If FSL_RC_NOT_FOUND is returned and hashOut is not NULL, *hashOut + is set to the value of f's preferred hash. *ridOut is only modified + if 0 is returned, in which case *ridOut will have a positive value. +*/ +FSL_EXPORT int fsl_repo_blob_lookup( fsl_cx * f, fsl_buffer const * src, fsl_id_t * ridOut, + fsl_uuid_str * hashOut ); + +/** + Returns true if the specified file name ends with any reserved + name, e.g.: _FOSSIL_ or .fslckout. + + For the sake of efficiency, zFilename must be a canonical name, + e.g. an absolute or checkout-relative path using only forward slash + ('/') as a directory separator. + + On Windows builds, this also checks for reserved Windows filenames, + e.g. "CON" and "PRN". + + nameLen must be the length of zFilename. If it is negative, + fsl_strlen() is used to calculate it. +*/ +FSL_EXPORT bool fsl_is_reserved_fn(const char *zFilename, + fsl_int_t nameLen ); + +/** + Uses fsl_is_reserved_fn() to determine whether the filename part of + zPath is legal for use as an in-repository filename. If it is, 0 is + returned, else FSL_RC_RANGE (or FSL_RC_OOM) is returned and f's + error state is updated to indicate the nature of the problem. nFile + is the length of zPath. If negative, fsl_strlen() is used to + determine its length. + + If relativeToCwd is true then zPath, if not absolute, is + canonicalized as if were relative to the current working directory + (see fsl_getcwd()), else it is assumed to be relative to the + current checkout (if any - falling back to the current working + directory). This flag is only relevant if zPath is not absolute and + if f has a checkout opened. An absolute zPath is used as-is and if + no checkout is opened then relativeToCwd is always treated as if it + were true. + + This routine does not validate that zPath lives inside a checkout + nor that the file actually exists. It does only name comparison and + only uses the filesystem for purposes of canonicalizing (if needed) + zPath. + + This routine does not require that f have an opened repo, but if it + does then this routine compares the canonicalized forms of both the + repository db and the given path and fails if zPath refers to the + repository db. Be aware that the relativeToCwd flag may influence + that test. + + TODO/FIXME: if f's 'manifest' config setting is set to true AND + zPath refers to the top of the checkout root, treat the files + (manifest, manifest.uuid, manifest.tags) as reserved. If it is a + string with any of the letters "r", "u", or "t", check only the + file(s) which those letters represent (see + add.c:fossil_reserved_name() in fossil). Apply these only at the top + of the tree - allow them in subdirectories. +*/ +FSL_EXPORT int fsl_reserved_fn_check(fsl_cx *f, const char *zPath, + fsl_int_t nFile, bool relativeToCwd); + +/** + Recompute/rebuild the entire repo.leaf table. This is not normally + needed, as leaf tracking is part of the crosslinking process, but + "just in case," here it is. + + This can supposedly be expensive (in time) for a really large + repository. Testing implies otherwise. + + Returns 0 on success. Error may indicate that f has no repo db + opened. On error f's error state may be updated. +*/ +FSL_EXPORT int fsl_repo_leaves_rebuild(fsl_cx * f); + +/** + Flags for use with fsl_leaves_compute(). +*/ +enum fsl_leaves_compute_e { +/** + Compute all leaves regardless of the "closed" tag. +*/ +FSL_LEAVES_COMPUTE_ALL = 0, +/** + Compute only leaves without the "closed" tag. +*/ +FSL_LEAVES_COMPUTE_OPEN = 1, +/** + Compute only leaves with the "closed" tag. +*/ +FSL_LEAVES_COMPUTE_CLOSED = 2 +}; +typedef enum fsl_leaves_compute_e fsl_leaves_compute_e; + +/** + Creates a temporary table named "leaves" if it does not already + exist, else empties it. Populates that table with the RID of all + check-ins that are leaves which are descended from the checkin + referred to by vid. + + A "leaf" is a check-in that has no children in the same branch. + There is a separate permanent table named [leaf] that contains all + leaves in the tree. This routine is used to compute a subset of + that table consisting of leaves that are descended from a single + check-in. + + The leafMode flag determines behavior associated with the "closed" + tag, as documented for the fsl_leaves_compute_e enum. + + If vid is <=0 then this function, after setting up or cleaning out + the [leaves] table, simply copies the list of leaves from the + repository's pre-computed [leaf] table (see + fsl_repo_leaves_rebuild()). + + @see fsl_leaves_computed_has() + @see fsl_leaves_computed_count() + @see fsl_leaves_computed_latest() + @see fsl_leaves_computed_cleanup() +*/ +FSL_EXPORT int fsl_leaves_compute(fsl_cx * f, fsl_id_t vid, + fsl_leaves_compute_e leafMode); + +/** + Requires that a prior call to fsl_leaves_compute() has succeeded, + else results are undefined. + + Returns true if the leaves list computed by fsl_leaves_compute() is + not empty, else false. This is more efficient than checking + against fsl_leaves_computed_count()>0. +*/ +FSL_EXPORT bool fsl_leaves_computed_has(fsl_cx * f); + +/** + Requires that a prior call to fsl_leaves_compute() has succeeded, + else results are undefined. + + Returns a count of the leaves list computed by + fsl_leaves_compute(), or a negative value if a db-level error is + encountered. On errors other than FSL_RC_OOM, f's error state will + be updated with information about the error. +*/ +FSL_EXPORT fsl_int_t fsl_leaves_computed_count(fsl_cx * f); + +/** + Requires that a prior call to fsl_leaves_compute() has succeeded, + else results are undefined. + + Returns the RID of the most recent checkin from those computed by + fsl_leaves_compute(), 0 if no entries are found, or a negative + value if a db-level error is encountered. On errors other than + FSL_RC_OOM, f's error state will be updated with information about + the error. +*/ +FSL_EXPORT fsl_id_t fsl_leaves_computed_latest(fsl_cx * f); + +/** + Cleans up any db-side resources created by fsl_leaves_compute(). + e.g. drops the temporary table created by that routine. Any errors + are silenty ignored. +*/ +FSL_EXPORT void fsl_leaves_computed_cleanup(fsl_cx * f); + +/** + Returns true if f's current repository has the + forbid-delta-manifests setting set to a truthy value. Results are + undefined if f has no opened repository. Some routines behave + differently if this setting is enabled. e.g. fsl_checkin_commit() + will never generate a delta manifest and fsl_deck_save() will + refuse to save a delta. This does not affect parsing or deltas or + those which are injected into the db via lower-level means (e.g. a + direct blob import or from a remote sync). + + Results are undefined if f has no opened repository. +*/ +FSL_EXPORT bool fsl_repo_forbids_delta_manifests(fsl_cx * f); + +/** + This is a variant of fsl_ckout_manifest_write() which writes data + regarding the given manifest RID to the given blobs. If manifestRid + is 0 or less then the current checkout is assumed and + FSL_RC_NOT_A_CKOUT is returned if no checkout is opened (or + FSL_RC_RANGE if an empty checkout is opened - a freshly-created + repository with no checkins). + + For each buffer argument which is not NULL, the corresponding + checkin-related data are appended to it. All such blobs will end + in a terminating newline character. + + Returns 0 on success, any of numerious non-0 fsl_rc_e codes on + error. +*/ +FSL_EXPORT int fsl_repo_manifest_write(fsl_cx *f, + fsl_id_t manifestRid, + fsl_buffer * const manifest, + fsl_buffer * const manifestUuid, + fsl_buffer * const manifestTags ); + + +/** + "Stage" flag for use with fsl_annotate_step and + fsl_annotate_step_f(). +*/ +enum fsl_annotate_step_e { +/** + Indicates that the current fsl_annotate_step_f() call is + part of the "version dump" stage of the annotation. +*/ +FSL_ANNOTATE_STEP_VERSION, +/** + Indicates that the current fsl_annotate_step_f() call has + complete version information for the line it is reporting + about. +*/ +FSL_ANNOTATE_STEP_FULL, +/** + Indicates that the current fsl_annotate_step_f() call has + only partial version information for the line it is reporting + about, as a result of a limited-run annotation. +*/ +FSL_ANNOTATE_STEP_LIMITED +}; +typedef enum fsl_annotate_step_e fsl_annotate_step_e; + +/** + Callback state for use by fsl_annotate_step_f(). + + ACHTUNG: this state gets repopulated for each call to the + fsl_annote_step_f() callback and any pointers it holds must be + treated as if they are invalidated as soon as the callback returns + (whether or not that is the case is undefined, though). + + Not all state is set on each call involving this object. See the + stepType member for details. +*/ +struct fsl_annotate_step { + /** + Tells the caller the "type" of annotation step this call + represents. The type of step will determine which fields + of this object are populated when it is passed to a + fsl_annotate_step_f() callback: + + Always set: + + stepType, stepNumber (but interpretation varies - see that + member's docs for details). + + FSL_ANNOTATE_STEP_VERSION: fileHash, versionHash, mtime + + FSL_ANNOTATE_STEP_LIMITED: stepNumber (always 0 for the limited + case), lineNumber, line, lineNength. + + FSL_ANNOTATE_STEP_FULL: as for FSL_ANNOTATE_STEP_LIMITED plus: + fileHash, versionHash, mtime, username. + */ + fsl_annotate_step_e stepType; + /** + Step number in this annotation run. When this->stepType is + FSL_ANNOTATE_STEP_VERSION, this value is the relative number of + the version, starting at 0 and incremented by 1 on each call. In + the other modes it is 0-based relative version from which the + current line is from, or negative if that information is + incomplete due to a limited annotation run. e.g. a value of 3 + indicates that this line is from 3 version away from the starting + version. + */ + int stepNumber; + /** + Line number for the current file. + */ + uint32_t lineNumber; + /** + NUL-terminated current line of the input file, minus any + newline and/or carriage return. + */ + char const * line; + /** + The number of bytes in this->line. + */ + uint32_t lineLength; + /** + The hash of the file version from which this->line was + pulled. + */ + fsl_uuid_cstr fileHash; + /** + The hash of the checkin version from which this->line was + pulled. + */ + fsl_uuid_cstr versionHash; + + /** + The mtime field from the [event] table (timeline) entry + associated with this version. This is a Julian Date and, because + it is updated by annotations which modify timestamps in the + [event] table, it reflects any "edited" time (if any). + + @see fsl_julian_to_iso8601() + */ + double mtime; + /** + The user name this change was attributed to, noting that merges + are attributed to the one who did the merge. + */ + char const * username; +}; + +/** Convenience typedef. */ +typedef struct fsl_annotate_step fsl_annotate_step; +/** Forward decl and convenience typedef. */ +typedef struct fsl_annotate_opt fsl_annotate_opt; + +/** + Callback for use with fsl_annotate(). Implementations receive + state about each step of an annotation process. They must return 0 + on success. On error, their non-0 result code is propagated back + to the fsl_annotate() caller. +*/ +typedef int (*fsl_annotate_step_f)(void * state, + fsl_annotate_opt const * const opt, + fsl_annotate_step const * const step); + +/** + A fsl_annotate_step_f() impl. which requires that its first argument + be a fsl_outputer. It formats each step of the annotation + in a manner similar to fossil(1) and forwards the result to + state->out(state->state, ...), returning that function's result code. +*/ +int fsl_annotate_step_f_fossilesque(void * state, + fsl_annotate_opt const * const opt, + fsl_annotate_step const * const step); + +/** + Configuration for use with fsl_annotate(). + + This structure holds options for the "annotate" operation and its + close cousin, "blame" a.k.a. "praise." Annotation takes a given + file version and builds a line-by-line history, showing when each + line was last modified. The "blame" a.k.a. "praise" option includes + *who* modified that line. +*/ +struct fsl_annotate_opt { + /** + The repository-root-relative NUL-terminated filename to annotate. + */ + char const * filename; + /** + The checkin from which the file's version should be selected. A + value of 0 or less means the current checkout, if in a checkout, + and is otherwise an error. + */ + fsl_id_t versionRid; + /** + The origin checkin version. A value of 0 or less means the "root of the + tree." + + TODO: figure out and explain the difference between versionRid + and originRid. + */ + fsl_id_t originRid; + /** + The maximum number of versions to search through. + + Note that fossil(1) offers the ability to limit the calculation + based on processing time, e.g. to 1500ms. We may or may not add + that in this library. + */ + uint32_t limitVersions; + + /** + An approximate number of milliseconds of processing time to limit + the annotation to. Note that this is measured in CPU time, not + "wall clock" time. This value is rough minimum approximation, + and the annotation will stop at the first processing step after which + this limit has been hit or surpassed. + + Even with this limit in place, the annotation engine may impose a + minimum number of versions to step through before it enforces + this limit. + + If both this and limitVersions are set to positive values, the + first limit which is exceeded is applied. + */ + uint32_t limitMs; + /** + - 0 = do not ignore any spaces. + - <0 = ignore trailing end-of-line spaces. + - >1 = ignore all spaces + */ + int16_t spacePolicy; + /** + If true, include the name of the user for which each change is + attributed (noting that merges show whoever merged the change, + which may differ from the original committer, and amended user + names will be used over those in the initial commit). If false, + show only version information. + + This option is alternately known as "blame". + + For reasons lost to history, blame/praise mode does not include + line numbers. That may change in the future. + */ + bool praise; + /** + Output file blob versions, instead of checkin versions. + */ + bool fileVersions; + + /** + If true, annotation output will start with a list of all + versions analyzed by the annotation process. + */ + bool dumpVersions; + /** + The output channel for the resulting annotation. + */ + fsl_annotate_step_f out; + /** + State for passing as the first argument to this->out(). + */ + void * outState; +}; + +/** Initialized-with-defaults fsl_annotate_opt structure, intended for + const-copy initialization. */ +#define fsl_annotate_opt_empty_m {\ + NULL/*filename*/, \ + 0/*versionRid*/,0/*originRid*/, \ + 0U/*limitVersions*/, 0U/*limitMs*/,\ + 0/*spacePolicy*/, \ + false/*praise*/, false/*fileVersions*/, \ + false/*dumpVersions*/, \ + NULL/*out*/, NULL/*outState*/ \ +} + +/** Initialized-with-defaults fsl_annotate_opt structure, intended for + non-const copy initialization. */ +extern const fsl_annotate_opt fsl_annotate_opt_empty; + +/** + UNDER CONSTRUCTION. Not yet known to be fully functional or + bug-free. + + Runs an "annotation" of an SCM-controled file and sends the results + to opt->out(). + + Returns 0 on success. On error, returns one of: + + - FSL_RC_OOM on OOM + + - FSL_RC_NOT_A_CKOUT if opt->versionRid<=0 and f has no opened checkout. + + - FSL_RC_NOT_FOUND if the given filename cannot be found in the + repository OR a given version ID does not resolve to a blob. (Sorry + about this ambiguity!) + + - FSL_RC_PHANTOM if a phantom blob is encountered while trying to + annotate. + + opt->out() may return arbitrary non-0 result codes, in which case + the returned code is propagated to the caller of this function. + + Results are undefined if either argument is invalid or opt->out is + NULL. +*/ +FSL_EXPORT int fsl_annotate( fsl_cx * const f, + fsl_annotate_opt const * const opt ); + +#if defined(__cplusplus) +} /*extern "C"*/ +#endif +#endif +/* ORG_FOSSIL_SCM_FSL_REPO_H_INCLUDED */ +/* end of file ../include/fossil-scm/fossil-repo.h */ +/* start of file ../include/fossil-scm/fossil-checkout.h */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#if !defined(ORG_FOSSIL_SCM_FSL_CHECKOUT_H_INCLUDED) +#define ORG_FOSSIL_SCM_FSL_CHECKOUT_H_INCLUDED +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ + +/** @file fossil-checkout.h + + fossil-checkout.h declares APIs specifically dealing with + checkout-side state, as opposed to purely repository-db-side state + or non-content-related APIs. +*/ + + +#if defined(__cplusplus) +extern "C" { +#endif + + +/** + Returns version information for the current checkout. + + If f has an opened checkout then... + + If uuid is not NULL then *uuid is set to the UUID of the opened + checkout, or NULL if there is no checkout. If rid is not NULL, *rid + is set to the record ID of that checkout, or 0 if there is no + checkout (or the current checkout is from an empty repository). The + returned uuid bytes and rid are owned by f and valid until the + library updates its checkout state to a newer checkout version + (essentially unpredictably). When in doubt about lifetime issues, + copy the UUID immediately after calling this if they will be needed + later. + + Corner case: a new repo with no checkins has an RID of 0 and a UUID + of NULL. That does not happen with fossil-generated repositories, + as those always "seed" the database with an initial commit artifact + containing no files. +*/ +FSL_EXPORT void fsl_ckout_version_info(fsl_cx * const f, fsl_id_t * const rid, + fsl_uuid_cstr * const uuid ); + +/** + Given a fsl_cx with an opened checkout, and a filename, this + function canonicalizes zOrigName to a form suitable for use as + an in-repo filename, _appending_ the results to pOut. If pOut is + NULL, it performs its normal checking but does not write a + result, other than to return 0 for success. + + As a special case, if zOrigName refers to the top-level checkout + directory, it resolves to either "." or "./", depending on whether + zOrigName contains a trailing slash. + + If relativeToCwd is true then the filename is canonicalized + based on the current working directory (see fsl_getcwd()), + otherwise f's current checkout directory is used as the virtual + root. + + If the input name contains a trailing slash, it is retained in + the output sent to pOut except in the top-dir case mentioned + above. + + Returns 0 on success, meaning that the value appended to pOut + (if not NULL) is a syntactically valid checkout-relative path. + + Returns FSL_RC_RANGE if zOrigName points to a path outside + of f's current checkout root. + + Returns FSL_RC_NOT_A_CKOUT if f has no checkout opened. + + Returns FSL_RC_MISUSE if !zOrigName, FSL_RC_OOM on an allocation + error. + + This function does not validate whether or not the file actually + exists, only that its name is potentially valid as a filename + for use in a checkout (though other, downstream rules might prohibit that, e.g. + the filename "..../...." is not valid but is not seen as invalid by + this function). (Reminder to self: we could run the end result through + fsl_is_simple_pathname() to catch that?) +*/ +FSL_EXPORT int fsl_ckout_filename_check( fsl_cx * f, bool relativeToCwd, + char const * zOrigName, fsl_buffer * pOut ); + +/** + Callback type for use with fsl_ckout_manage_opt(). It should + inspect the given filename using whatever criteria it likes, set + *include to true or false to indicate whether the filename is okay + to include the current add-file-to-repo operation, and return 0. + + If it returns non-0 the add-file-to-repo process will end and that + error code will be reported to its caller. Such result codes must + come from the FSL_RC_xxx family. + + It will be passed a name which is relative to the top-most checkout + directory. + + The final argument is not used by the library, but is passed on + as-is from the fsl_ckout_manage_opt::callbackState pointer which + is passed to fsl_ckout_manage(). +*/ +typedef int (*fsl_ckout_manage_f)(const char *zFilename, + bool *include, + void *state); +/** + Options for use with fsl_ckout_manage(). +*/ +struct fsl_ckout_manage_opt { + /** + The file or directory name to add. If it is a directory, the add + process will recurse into it. + */ + char const * filename; + /** + Whether to evaluate the given name as relative to the current working + directory or to the current checkout root. + + This makes a subtle yet important difference in how the name is + resolved. CLI apps which take file names from the user from + within a checkout directory will generally want to set + relativeToCwd to true. GUI apps, OTOH, will possibly need it to + be false, depending on how they resolve and pass on the + filenames. + */ + bool relativeToCwd; + /** + Whether or not to check the name(s) against the 'ignore-globs' + config setting (if set). + */ + bool checkIgnoreGlobs; + /** + Optional predicate function which may be called for each + to-be-added filename. It is only called if: + + - It is not NULL (obviously) and... + + - The file is not already in the checkout database and... + + - The is-internal-name check passes (see + fsl_reserved_fn_check()) and... + + - If checkIgnoreGlobs is false or the name does not match one of + the ignore-globs values. + + The name it is passed is relative to the checkout root. + + Because the callback is called only if other options have not + already excluded the file, the client may use the callback to + report to the user (or otherwise record) exactly which files + get added. + */ + fsl_ckout_manage_f callback; + + /** + State to be passed to this->callback. + */ + void * callbackState; + + /** + These counts are updated by fsl_ckout_manage() to report + what it did. + */ + struct { + /** + Number of files actually added by fsl_ckout_manage(). + */ + uint32_t added; + /** + Number of files which were requested to be added but were only + updated because they had previously been added. Updates set the + vfile table entry's current mtime, executable-bit state, and + is-it-a-symlink state. (That said, this code currently ignores + symlinks altogether!) + */ + uint32_t updated; + /** + The number of files skipped over for addition. This includes + files which meet any of these criteria: + + - fsl_reserved_fn_check() fails. + + - If the checkIgnoreGlobs option is true and a filename matches + any of those globs. + + - The client-provided callback says not to include the file. + */ + uint32_t skipped; + } counts; +}; +typedef struct fsl_ckout_manage_opt fsl_ckout_manage_opt; +/** + Initialized-with-defaults fsl_ckout_manage_opt instance, + intended for use in const-copy initialization. +*/ +#define fsl_ckout_manage_opt_empty_m {\ + NULL/*filename*/, true/*relativeToCwd*/, true/*checkIgnoreGlobs*/, \ + NULL/*callback*/, NULL/*callbackState*/, \ + {/*counts*/ 0/*added*/, 0/*updated*/, 0/*skipped*/} \ + } +/** + Initialized-with-defaults fsl_ckout_manage_opt instance, + intended for use in non-const copy initialization. +*/ +FSL_EXPORT const fsl_ckout_manage_opt fsl_ckout_manage_opt_empty; + +/** + Adds the given filename or directory (recursively) to the current + checkout vfile list of files as a to-be-added file, or updates an + existing record if one exists. + + This function ensures that opt->filename gets canonicalized and can + be found under the checkout directory, and fails if no such file + exists (checking against the canonicalized name). Filenames are all + filtered through fsl_reserved_fn_check() and may have other filters + applied to them, as determined by the options object. + + Each filename which passes through the filters is passed to + the opt->callback (if not NULL), which may perform a final + filtering check and/or alert the client about the file being + queued. + + The options object is non-const because this routine updates + opt->counts when it adds, updates, or skips a file. On each call, + it updated opt->counts without resetting it (as this function is + typically called in a loop). This function does not modify any + other entries of that object and it requires that the object not be + modified (e.g. via opt->callback()) while it is recursively + processing. To reset the counts between calls, if needed: + + ``` + opt->counts = fsl_ckout_manage_opt_empty.counts; + ``` + + Returns 0 on success, non-0 on error. + + Files queued for addition this way can be unqueued before they are + committed using fsl_ckout_unmanage(). + + @see fsl_ckout_unmanage() + @see fsl_reserved_fn_check() +*/ +FSL_EXPORT int fsl_ckout_manage( fsl_cx * const f, + fsl_ckout_manage_opt * const opt ); + + +/** + Callback type for use with fsl_ckout_unmanage(). It is called + by the removal process, immediately after a file is "removed" + from SCM management (a.k.a. when the file becomes "unmanaged"). + + If it returns non-0 the unmanage process will end and that + error code will be reported to its caller. Such result codes must + come from the FSL_RC_xxx family. + + It will be passed a name which is relative to the top-most checkout + directory. The client is free to unlink the file from the filesystem + if they like - the library does not do so automatically + + The final argument is not used by the library, but is passed on + as-is from the callbackState pointer which + is passed to fsl_ckout_unmanage(). +*/ +typedef int (*fsl_ckout_unmanage_f)(const char *zFilename, void *state); + +/** + Options for use with fsl_ckout_unmanage(). +*/ +struct fsl_ckout_unmanage_opt { + /** + The file or directory name to add. If it is a directory, the add + process will recurse into it. See also this->vfileIds. + */ + char const * filename; + /** + An alternative to assigning this->filename is to point + this->vfileIds to a bag of vfile.id values. If this member is not + NULL, fsl_ckout_revert() will ignore this->filename. + + @see fsl_filename_to_vfile_ids() + */ + fsl_id_bag const * vfileIds; + /** + Whether to evaluate this->filename as relative to the current + working directory (true) or to the current checkout root + (false). This is ignored when this->vfileIds is not NULL. + + This makes a subtle yet important difference in how the name is + resolved. CLI apps which take file names from the user from + within a checkout directory will generally want to set + relativeToCwd to true. GUI apps, OTOH, will possibly need it to + be false, depending on how they resolve and pass on the + filenames. + */ + bool relativeToCwd; + /** + If true, fsl_vfile_changes_scan() is called to ensure that + the filesystem and vfile tables agree. If the client code has + called that function, or its equivalent, since any changes were + made to the checkout then this may be set to false to speed up + the rm process. + */ + bool scanForChanges; + /** + Optional predicate function which will be called after each + file is made unmanaged. + + The name it is passed is relative to the checkout root. + */ + fsl_ckout_unmanage_f callback; + /** + State to be passed to this->callback. + */ + void * callbackState; + +}; +typedef struct fsl_ckout_unmanage_opt fsl_ckout_unmanage_opt; +/** + Initialized-with-defaults fsl_ckout_unmanage_opt instance, + intended for use in const-copy initialization. +*/ +#define fsl_ckout_unmanage_opt_empty_m {\ + NULL/*filename*/, NULL/*vfileIds*/,\ + true/*relativeToCwd*/,true/*scanForChanges*/, \ + NULL/*callback*/, NULL/*callbackState*/ \ +} +/** + Initialized-with-defaults fsl_ckout_unmanage_opt instance, + intended for use in non-const copy initialization. +*/ +FSL_EXPORT const fsl_ckout_unmanage_opt fsl_ckout_unmanage_opt_empty; + +/** + The converse of fsl_ckout_manage(), this queues a file for removal + from the current checkout. Unlike fsl_ckout_manage(), this routine + does not ensure that opt->filename actually exists - it only + normalizes zFilename into its repository-friendly form and passes + it through the vfile table. + + If opt->filename refers to a directory then this operation queues + all files under that directory (recursively) for removal. In this + case, it is irrelevant whether or not opt->filename ends in a + trailing slash. + + Returns 0 on success, any of a number of non-0 codes on error. + Returns FSL_RC_MISUSE if !opt->filename or !*opt->filename. + Returns FSL_RC_NOT_A_CKOUT if f has no opened checkout. + + If opt->callback is not NULL, it is called for each + newly-unamanaged entry. The intention is to provide it the + opportunity to notify the user, record the filename for later use, + remove the file from the filesystem, etc. If it returns non-0, the + unmanaging process will fail with that code and any pending + transaction will be placed into a rollback state. + + This routine does not actually remove any files from the + filesystem, it only modifies the vfile table entry so that the + file(s) will be removed from the SCM by the commit process. If + opt->filename is an entry which was previously + fsl_ckout_manage()'d, but not yet committed, or any such entries + are found under directory opt->filename, they are removed from the + vfile table. i.e. this effective undoes the add operation. + + @see fsl_ckout_manage() +*/ +FSL_EXPORT int fsl_ckout_unmanage( fsl_cx * f, + fsl_ckout_unmanage_opt const * opt ); + +/** + Hard-coded range of values of the vfile.chnged db field. + These values are part of the fossil schema and must not + be modified. +*/ +enum fsl_vfile_change_e { + /** File is unchanged. */ + FSL_VFILE_CHANGE_NONE = 0, + /** File edit. */ + FSL_VFILE_CHANGE_MOD = 1, + /** File changed due to a merge. */ + FSL_VFILE_CHANGE_MERGE_MOD = 2, + /** File added by a merge. */ + FSL_VFILE_CHANGE_MERGE_ADD = 3, + /** File changed due to an integrate merge. */ + FSL_VFILE_CHANGE_INTEGRATE_MOD = 4, + /** File added by an integrate merge. */ + FSL_VFILE_CHANGE_INTEGRATE_ADD = 5, + /** File became executable but has unmodified contents. */ + FSL_VFILE_CHANGE_IS_EXEC = 6, + /** File became a symlink whose target equals its old contents. */ + FSL_VFILE_CHANGE_BECAME_SYMLINK = 7, + /** File lost executable status but has unmodified contents. */ + FSL_VFILE_CHANGE_NOT_EXEC = 8, + /** File lost symlink status and has contents equal to its old target. */ + FSL_VFILE_CHANGE_NOT_SYMLINK = 9 +}; +typedef enum fsl_vfile_change_e fsl_vfile_change_e; + +/** + Change-type flags for use with fsl_ckout_changes_visit() and + friends. + + TODO: consolidate this with fsl_vfile_change_e insofar as possible. + There are a few checkout change statuses not reflected in + fsl_vfile_change_e. +*/ +enum fsl_ckout_change_e { +/** + Sentinel placeholder value. +*/ +FSL_CKOUT_CHANGE_NONE = 0, +/** + Indicates that a file was modified in some unspecified way. +*/ +FSL_CKOUT_CHANGE_MOD = FSL_VFILE_CHANGE_MOD, +/** + Indicates that a file was modified as the result of a merge. +*/ +FSL_CKOUT_CHANGE_MERGE_MOD = FSL_VFILE_CHANGE_MERGE_MOD, +/** + Indicates that a file was added as the result of a merge. +*/ +FSL_CKOUT_CHANGE_MERGE_ADD = FSL_VFILE_CHANGE_MERGE_ADD, +/** + Indicates that a file was modified as the result of an + integrate-merge. +*/ +FSL_CKOUT_CHANGE_INTEGRATE_MOD = FSL_VFILE_CHANGE_INTEGRATE_MOD, +/** + Indicates that a file was added as the result of an + integrate-merge. +*/ +FSL_CKOUT_CHANGE_INTEGRATE_ADD = FSL_VFILE_CHANGE_INTEGRATE_ADD, +/** + Indicates that the file gained the is-executable trait + but is otherwise unmodified. +*/ +FSL_CKOUT_CHANGE_IS_EXEC = FSL_VFILE_CHANGE_IS_EXEC, +/** + Indicates that the file has changed to a symlink. +*/ +FSL_CKOUT_CHANGE_BECAME_SYMLINK = FSL_VFILE_CHANGE_BECAME_SYMLINK, +/** + Indicates that the file lost the is-executable trait + but is otherwise unmodified. +*/ +FSL_CKOUT_CHANGE_NOT_EXEC = FSL_VFILE_CHANGE_NOT_EXEC, +/** + Indicates that the file was previously a symlink but is + now a plain file. +*/ +FSL_CKOUT_CHANGE_NOT_SYMLINK = FSL_VFILE_CHANGE_NOT_SYMLINK, +/** + Indicates that a file was added. +*/ +FSL_CKOUT_CHANGE_ADDED = FSL_CKOUT_CHANGE_NOT_SYMLINK + 1000, +/** + Indicates that a file was removed from SCM management. +*/ +FSL_CKOUT_CHANGE_REMOVED, +/** + Indicates that a file is missing from the local checkout. +*/ +FSL_CKOUT_CHANGE_MISSING, +/** + Indicates that a file was renamed. +*/ +FSL_CKOUT_CHANGE_RENAMED +}; + +typedef enum fsl_ckout_change_e fsl_ckout_change_e; + +/** + This is equivalent to calling fsl_vfile_changes_scan() with the + arguments (f, -1, 0). + + @see fsl_ckout_changes_visit() + @see fsl_vfile_changes_scan() +*/ +FSL_EXPORT int fsl_ckout_changes_scan(fsl_cx * f); + +/** + A typedef for visitors of checkout status information via + fsl_ckout_changes_visit(). Implementions will receive the + last argument passed to fsl_ckout_changes_visit() as their + first argument. The second argument indicates the type of change + and the third holds the repository-relative name of the file. + + If changes is FSL_CKOUT_CHANGE_RENAMED then origName will hold + the original name, else it will be NULL. + + Implementations must return 0 on success, non-zero on error. On + error any looping performed by fsl_ckout_changes_visit() will + stop and this function's result code will be returned. + + @see fsl_ckout_changes_visit() +*/ +typedef int (*fsl_ckout_changes_f)(void * state, fsl_ckout_change_e change, + char const * filename, + char const * origName); + +/** + Compares the changes of f's local checkout against repository + version vid (checkout version if vid is negative). For each + change detected it calls visitor(state,...) to report the + change. If visitor() returns non-0, that code is returned from + this function. If doChangeScan is true then + fsl_ckout_changes_scan() is called by this function before + iterating, otherwise it is assumed that the caller has called + that or has otherwise ensured that the checkout db's vfile table + has been populated. + + If the callback returns FSL_RC_BREAK, this function stops iteration + and returns 0. + + Returns 0 on success. + + @see fsl_ckout_changes_scan() +*/ +FSL_EXPORT int fsl_ckout_changes_visit( fsl_cx * f, fsl_id_t vid, + bool doChangeScan, + fsl_ckout_changes_f visitor, + void * state ); +/** + A bitmask of flags for fsl_vfile_changes_scan(). +*/ +enum fsl_ckout_sig_e { +/** + The empty flags set. +*/ +FSL_VFILE_CKSIG_NONE = 0, + +/** + Non-file/non-link FS objects trigger an error. +*/ +FSL_VFILE_CKSIG_ENOTFILE = 0x001, +/** + Verify file content using hashing, regardless of whether or not + file timestamps differ. +*/ +FSL_VFILE_CKSIG_HASH = 0x002, +/** + For unchanged or changed-by-merge files, set the mtime to last + check-out time, as determined by fsl_mtime_of_manifest_file(). +*/ +FSL_VFILE_CKSIG_SETMTIME = 0x004, +/** + Indicates that when populating the vfile table, it should be not be + cleared of entries for other checkins. Normally we want to clear + all versions except for the one we're working with, but at least + a couple of use cases call for having multiple versions in vfile at + once. Many algorithms generally assume only a single checkin's + worth of state is in vfile and can get confused if that is not the + case. +*/ +FSL_VFILE_CKSIG_KEEP_OTHERS = 0x008, +/** + If set and fsl_vfile_changes_scan() is passed a version other than + the pre-call checkout version, it will, when finished, write the + given version in the "checkout" setting of the ckout.vvar table, + effectively switching the checkout to that version. It does not do + this by default because it is sometimes necessary to have two + versions in the vfile table at once and the operation doing so + needs to control which version number is the current checkout. +*/ +FSL_VFILE_CKSIG_WRITE_CKOUT_VERSION = 0x010 +}; + +/** + This function populates (if needed) the vfile table of f's + checkout db for the given checkin version ID then compares files + listed in it against files in the checkout directory, updating + vfile's status for the current checkout version id as its goes. If + vid is<=0 then the current checkout's RID is used in its place + (note that 0 is the RID of an initial empty repository!). + + cksigFlags must be 0 or a bitmask of fsl_ckout_sig_e values. + + This is a relatively memory- and filesystem-intensive operation, + and should not be performed more often than necessary. Many SCM + algorithms rely on its state being correct, however, so it's + generally better to err on the side of running it once too often + rather than once too few times. + + Returns 0 on success, non-0 on error. + + BUG: this does not properly catch one particular corner-case + change, where a file has been replaced by a same-named non-file + (symlink or directory). +*/ +FSL_EXPORT int fsl_vfile_changes_scan(fsl_cx * const f, fsl_id_t vid, + unsigned cksigFlags); + +/** + If f has an opened checkout which has local changes noted in its + checkout db state (the vfile table), returns true, else returns + false. Note that this function does not do the filesystem scan to + check for changes, but checks only the db state. Use + fsl_vfile_changes_scan() to perform the actual scan (noting that + library-side APIs which update that state may also record + individual changes or automatically run a scan). +*/ +FSL_EXPORT bool fsl_ckout_has_changes(fsl_cx * const f); + +/** + Callback type for use with fsl_checkin_queue_opt for alerting a + client about exactly which files get enqueued/dequeued via + fsl_checkin_enqueue() and fsl_checkin_dequeue(). + + This function gets passed the checkout-relative name of the file + being enqueued/dequeued and the client-provided state pointer which + was passed to the relevant API. It must return 0 on success. If it + returns non-0, the API on whose behalf this callback is invoked + will propagate that error code back to the caller. + + The intent of this callback is simply to report changes to the + client, not to perform validation. Thus such callbacks "really + should not fail" unless, e.g., they encounter an OOM condition or + some such. Any validation required by the client should be + performed before calling fsl_checkin_enqueue() + resp. fsl_checkin_dequeue(). +*/ +typedef int (*fsl_checkin_queue_f)(const char * filename, void * state); + +/** + Options object type used by fsl_checkin_enqueue() and + fsl_checkin_dequeue(). +*/ +struct fsl_checkin_queue_opt { + /** + File or directory name to enqueue/dequeue to/from a pending + checkin. + */ + char const * filename; + /** + If true, filename (if not absolute) is interpreted as relative to + the current working directory, else it is assumed to be relative + to the top of the current checkout directory. + */ + bool relativeToCwd; + + /** + If not NULL then this->filename and this->relativeToCwd are + IGNORED and any to-queue filename(s) is/are added from this + container. It is an error (FSL_RC_MISUSE) to pass an empty bag. + (Should that be FSL_RC_RANGE instead?) + + The bag is assumed to contain values from the vfile.id checkout + db field, refering to one or more files which should be queued + for the pending checkin. It is okay to pass IDs for unmodified + files or to queue the same files multiple times. Unmodified files + may be enqueued but will be ignored by the checkin process if, at + the time the checkin is processed, they are still unmodified. + Duplicated entries are simply ignored for the 2nd and subsequent + inclusion. + + @see fsl_ckout_vfile_ids() + */ + fsl_id_bag const * vfileIds; + + /** + If true, fsl_vfile_changes_scan() is called to ensure that the + filesystem and vfile tables agree. If the client code has called + that function, or its equivalent, since any changes were made to + the checkout then this may be set to false to speed up the + enqueue process. This is only used by fsl_checkin_enqueue(), not + fsl_checkin_dequeue(). + */ + bool scanForChanges; + + /** + If true, only flagged-as-modified files will be enqueued by + fsl_checkin_enqueue(). By and large, this should be set to + true. Setting this to false is generally only intended/useful for + testing. + */ + bool onlyModifiedFiles; + + /** + It not NULL, is pass passed the checkout-relative filename of + each enqueued/dequeued file and this->callbackState. See the + callback type's docs for more details. + */ + fsl_checkin_queue_f callback; + + /** + Opaque client-side state for use as the 2nd argument to + this->callback. + */ + void * callbackState; +}; + +/** Convenience typedef. */ +typedef struct fsl_checkin_queue_opt fsl_checkin_queue_opt; + +/** Initialized-with-defaults fsl_checkin_queue_opt structure, intended for + const-copy initialization. */ +#define fsl_checkin_queue_opt_empty_m { \ + NULL/*filename*/,true/*relativeToCwd*/, \ + NULL/*vfileIds*/, \ + true/*scanForChanges*/,true/*onlyModifiedFiles*/, \ + NULL/*callback*/,NULL/*callbackState*/ \ +} + +/** Initialized-with-defaults fsl_checkin_queue_opt structure, intended for + non-const copy initialization. */ +extern const fsl_checkin_queue_opt fsl_checkin_queue_opt_empty; + +/** + Adds one or more files to f's list of "selected" files - those + which should be included in the next commit (see + fsl_checkin_commit()). + + Warning: if this function is not called before + fsl_checkin_commit(), then fsl_checkin_commit() will select all + modified, fsl_ckout_manage()'d, fsl_ckout_unmanage()'d, or renamed + files by default. + + opt->filename must be a non-empty NUL-terminated string. The + filename is canonicalized via fsl_ckout_filename_check() - see that + function for the meaning of the opt->relativeToCwd parameter. To + queue all modified files in a checkout, set opt->filename to ".", + opt->relativeToCwd to false, and opt->onlyModifiedFiles to true. + "Modified" includes any which are pending deletion, are + newly-added, or for which a rename is pending. + + The resolved name must refer to either a single vfile.pathname + value in the current vfile table or to a checkout-root-relative + directory. All matching filenames which refer to modified files (as + recorded in the vfile table) are queued up for the next commit. + If opt->filename is NULL, empty, or ("." and opt->relativeToCwd is false) + then all files in the vfile table are checked for changes. + + If opt->scanForChanges is true then fsl_vfile_changes_scan() is + called before starting to ensure that the vfile entries are up to + date. If the client app has "recently" run that (or its + equivalent), that (slow) step can be skipped by setting + opt->scanForChanges to false before calling this + + Note that after this returns, any given file may still be modified + by the client before the commit takes place, and the changes on + disk at the point of the fsl_checkin_commit() are the ones which + get saved (or not). + + For each resolved entry which actually gets enqueued (i.e. was not + already enqueued and which is marked as modified), opt->callback + (if it is not NULL) is passed the checkout-relative file name and + the opt->callbackState pointer. + + Returns 0 on success, FSL_RC_MISUSE if either pointer is NULL, or + *zName is NUL. Returns FSL_RC_OOM on allocation error. It is not + inherently an error for opt->filename to resolve to no queue-able + entries. A client can check for that case, if needed, by assigning + opt->callback and incrementing a counter in that callback. If the + callback is never called, no queue-able entries were found. + + On error f's error state might (depending on the nature of the + problem) contain more details. + + @see fsl_checkin_is_enqueued() + @see fsl_checkin_dequeue() + @see fsl_checkin_discard() + @see fsl_checkin_commit() +*/ +FSL_EXPORT int fsl_checkin_enqueue(fsl_cx * f, + fsl_checkin_queue_opt const * opt); + +/** + The opposite of fsl_checkin_enqueue(), this opt->filename, + which may resolve to a single name or a directory, from the checkin + queue. Returns 0 on succes. This function does no validation on + whether a given file(s) actually exist(s), it simply asks the + internals to clean up matching strings from the checkout's vfile + table. Specifically, it does not return an error if this operation + finds no entries to dequeue. + + If opt->filename is empty or NULL then ALL files are unqueued from + the pending checkin. + + If opt->relativeToCwd is true (non-0) then opt->filename is + resolved based on the current directory, otherwise it is resolved + based on the checkout's root directory. + + If opt->filename is not NULL or empty, this functions runs the + given path through fsl_ckout_filename_check() and will fail if that + function fails, propagating any error from that function. Ergo, + opt->filename must refer to a path within the current checkout. + + @see fsl_checkin_enqueue() + @see fsl_checkin_is_enqueued() + @see fsl_checkin_discard() + @see fsl_checkin_commit() +*/ +FSL_EXPORT int fsl_checkin_dequeue(fsl_cx * f, + fsl_checkin_queue_opt const * opt); + +/** + Returns true (non-0) if the file named by zName is in f's current + file checkin queue. If NO files are in the current selection + queue then this routine assumes that ALL files are implicitely + selected. As long as at least one file is enqueued (via + fsl_checkin_enqueue()) then this function only returns true + for files which have been explicitly enqueued. + + If relativeToCwd then zName is resolved based on the current + directory, otherwise it assumed to be related to the checkout's + root directory. + + This function returning true does not necessarily indicate that + the file _will_ be checked in at the next commit. If the file has + not been modified at commit-time then it will not be part of the + commit. + + This function honors the fsl_cx_is_case_sensitive() setting + when comparing names. + + Achtung: this does not resolve directory names like + fsl_checkin_enqueue() and fsl_checkin_dequeue() do. It + only works with file names. + + @see fsl_checkin_enqueue() + @see fsl_checkin_dequeue() + @see fsl_checkin_discard() + @see fsl_checkin_commit() +*/ +FSL_EXPORT bool fsl_checkin_is_enqueued(fsl_cx * f, char const * zName, + bool relativeToCwd); + +/** + Discards any state accumulated for a pending checking, + including any files queued via fsl_checkin_enqueue() + and tags added via fsl_checkin_T_add(). + + @see fsl_checkin_enqueue() + @see fsl_checkin_dequeue() + @see fsl_checkin_is_enqueued() + @see fsl_checkin_commit() + @see fsl_checkin_T_add() +*/ +FSL_EXPORT void fsl_checkin_discard(fsl_cx * f); + +/** + Parameters for fsl_checkin_commit(). + + Checkins are created in a multi-step process: + + - fsl_checkin_enqueue() queues up a file or directory for + commit at the next commit. + + - fsl_checkin_dequeue() removes an entry, allowing + UIs to toggle files in and out of a checkin before + committing it. + + - fsl_checkin_is_enqueued() can be used to determine whether + a given name is already enqueued or not. + + - fsl_checkin_T_add() can be used to T-cards (tags) to a + deck. Branch tags are intended to be applied via the + fsl_checkin_opt::branch member. + + - fsl_checkin_discard() can be used to cancel any pending file + enqueuings, effectively cancelling a commit (which can be + re-started by enqueuing another file). + + - fsl_checkin_commit() creates a checkin for the list of enqueued + files (defaulting to all modified files in the checkout!). It + takes an object of this type to specify a variety of parameters + for the check. + + Note that this API uses the terms "enqueue" and "unqueue" rather + than "add" and "remove" because those both have very specific + (and much different) meanings in the overall SCM scheme. +*/ +struct fsl_checkin_opt { + /** + The commit message. May not be empty - the library + forbids empty checkin messages. + */ + char const * message; + + /** + The optional mime type for the message. Only set + this if you know what you're doing. + */ + char const * messageMimeType; + + /** + The user name for the checkin. If NULL or empty, it defaults to + fsl_cx_user_get(). If that is NULL, a FSL_RC_RANGE error is + triggered. + */ + char const * user; + + /** + If not NULL, makes the checkin the start of a new branch with + this name. + */ + char const * branch; + + /** + If this->branch is not NULL, this is applied as its "bgcolor" + propagating property. If this->branch is NULL then this is + applied as a one-time color tag to the checkin. + + It must be NULL, empty, or in a form usable by HTML/CSS, + preferably \#RRGGBB form. Length-0 values are ignored (as if + they were NULL). + */ + char const * bgColor; + + /** + If true, the checkin will be marked as private, otherwise it + will be marked as private or public, depending on whether or + not it inherits private content. + */ + bool isPrivate; + + /** + Whether or not to calculate an R-card. Doing so is very + expensive (memory and I/O) but it adds another layer of + consistency checking to manifest files. In practice, the R-card + is somewhat superfluous and the cost of calculating it has + proven painful on very large repositories. fossil(1) creates an + R-card for all checkins but does not require that one be set + when it reads a manifest. + */ + bool calcRCard; + + /** + Tells the checkin to close merged-in branches (merge type of + 0). INTEGRATE merges (type=-4) are always closed by a + checkin. This does not apply to CHERRYPICK (type=-1) and + BACKOUT (type=-2) merges. + */ + bool integrate; + + /** + If true, allow a file to be checked in if it contains + fossil-style merge conflict markers, else fail if an attempt is + made to commit any files with such markers. + */ + bool allowMergeConflict; + + /** + A hint to fsl_checkin_commit() about whether it needs to scan the + checkout for changes. Set this to false ONLY if the calling code + calls fsl_ckout_changes_scan() (or equivalent, + e.g. fsl_vfile_changes_scan()) immediately before calling + fsl_checkin_commit(). fsl_checkin_commit() requires a non-stale + changes scan in order to function properly, but it's a + computationally slow operation so the checkin process does not + want to duplicate it if the application has recently done so. + */ + bool scanForChanges; + + /** + NOT YET IMPLEMENTED! TODO! + + If true, files which are removed from the SCM by this checkin + should be removed from the filesystem. + + Reminder to self: when we do this, incorporate + fsl_rm_empty_dirs(). + */ + bool rmRemovedFiles; + + /** + Whether to allow (or try to force) a delta manifest or not. 0 + means no deltas allowed - it will generate a baseline + manifest. Greater than 0 forces generation of a delta if + possible (if one can be readily found) even if doing so would not + save a notable amount of space. Less than 0 means to + decide via some heuristics. + + A "readily available" baseline means either the current checkout + is a baseline or has a baseline. In either case, we can use that + as a baseline for a delta. i.e. a baseline "should" generally be + available except on the initial checkin, which has neither a + parent checkin nor a baseline. + + The current behaviour for "auto-detect" mode is: it will generate + a delta if a baseline is "readily available" _and_ the repository + has at least one delta already. Once it calculates a delta form, + it calculates whether that form saves any appreciable + space/overhead compared to whether a baseline manifest was + generated. If so, it discards the delta and re-generates the + manifest as a baseline. The "force" behaviour (deltaPolicy>0) + bypasses the "is it too big?" test, and is only intended for + testing, not real-life use. + + Caveat: if the repository has the "forbid-delta-manifests" set to + a true value, this option is ignored: that setting takes + priority. Similarly, it will not create a delta in a repository + unless a delta has been "seen" in that repository before or this + policy is set to >0. When a checkin is created with a delta + manifest, that fact gets recorded in the repository's config + table. + + Note that delta manifests have some advantages and may not + actually save much (if any) repository space because the + lower-level delta framework already compresses parent versions of + artifacts tightly. For more information see: + + https://fossil-scm.org/home/doc/tip/www/delta-manifests.md + */ + int deltaPolicy; + + /** + Time of the checkin. If 0 or less, the time of the + fsl_checkin_commit() call is used. + */ + double julianTime; + + /** + If this is not NULL then the committed manifest will include a + tag which closes the branch. The value of this string will be + the value of the "closed" tag, and the value may be an empty + string. The intention is that this gets set to a comment about + why the branch is closed, but it is in no way mandatory. + */ + char const * closeBranch; + + /** + Tells fsl_checkin_commit() to dump the generated manifest to + this file. Intended only for debugging and testing. Checking in + will fail if this file cannot be opened for writing. + */ + char const * dumpManifestFile; + /* + fossil(1) has many more options. We might want to wrap some of + it up in the "incremental" state (f->ckin.mf). + + TODOs: + + A callback mechanism which supports the user cancelling + the checkin. It is (potentially) needed for ops like + confirming the commit of CRNL-only changes. + + 2021-03-09: we now have fsl_confirmer for this but currently no + part of the checkin code needs a prompt. + */ +}; + +/** + Empty-initialized fsl_checkin_opt instance, intended for use in + const-copy constructing. +*/ +#define fsl_checkin_opt_empty_m { \ + NULL/*message*/, \ + NULL/*messageMimeType*/, \ + NULL/*user*/, \ + NULL/*branch*/, \ + NULL/*bgColor*/, \ + false/*isPrivate*/, \ + true/*calcRCard*/, \ + false/*integrate*/, \ + false/*allowMergeConflict*/,\ + true/*scanForChanges*/,\ + false/*rmRemovedFiles*/,\ + 0/*deltaPolicy*/, \ + 0.0/*julianTime*/, \ + NULL/*closeBranch*/, \ + NULL/*dumpManifestFile*/ \ +} + +/** + Empty-initialized fsl_checkin_opt instance, intended for use in + copy-constructing. It is important that clients copy this value + (or fsl_checkin_opt_empty_m) to cleanly initialize their + fsl_checkin_opt instances, as this may set default values which + (e.g.) a memset() would not. +*/ +FSL_EXPORT const fsl_checkin_opt fsl_checkin_opt_empty; + +/** + This creates and saves a "checkin manifest" for the current + checkout. + + Its primary inputs is a list of files to commit. This list is + provided by the client by calling fsl_checkin_enqueue() one or + more times. If no files are explicitely selected (enqueued) then + it calculates which local files have changed vs the current + checkout and selects all of those. + + Non-file inputs are provided via the opt parameter. + + On success, it returns 0 and... + + - If newRid is not NULL, it is assigned the new checkin's RID + value. + + - If newUuid is not NULL, it is assigned the new checkin's UUID + value. Ownership of the bytes is passed to the caller, who must + eventually pass them to fsl_free() to free them. + + Note that the new RID and UUID can also be fetched afterwards by + calling fsl_ckout_version_info(). + + On error non-0 is returned and f's error state may (depending on + the nature of the problem) contain details about the problem. + Note, however, that any error codes returned here may have arrived + from several layers down in the internals, and may not have a + single specific interpretation here. When possible/practical, f's + error state gets updated with a human-readable description of the + problem. + + ACHTUNG: all pending checking state is cleaned if this function + fails for any reason other than basic argument validation. This + means any queued files or tags need to be re-applied if the client + wants to try again. That is somewhat of a bummer, but this + behaviour is the only way we can ensure that then the pending + checkin state does not get garbled on a second use. When in doubt + about the state, the client should call fsl_checkin_discard() to + clear it before try to re-commit. (Potential TODO: add a + success/fail state flag to the checkin state and only clean up on + success? OTOH, since something in the state likely caused the + problem, we might not want to do that.) + + This operation does all of its db-related work in a transaction, so + it rolls back any db changes if it fails. To implement a "dry-run" + mode, simply wrap this call in a transaction started on the + fsl_cx_db_ckout() db handle (passing it to + fsl_db_transaction_begin()), then, after this call, either cal; + fsl_db_transaction_rollback() (to implement dry-run mode) or + fsl_db_transaction_commit() (for "wet-run" mode). If this function + returns non-0 due to anything more serious than basic argument + validation, such a transaction will be in a roll-back state. + + Some of the more notable, potentially not obvious, error + conditions: + + - Trying to commit against a closed leaf: FSL_RC_ACCESS. Doing so + is not permitted by fossil(1), so we disallow it here. + + - An empty/NULL user name or commit message, or no files were + selected which actually changed: FSL_RC_MISSING_INFO. In these + cases f's error state describes the problem. + + - Some resource is not found (e.g. an expected RID/UUID could not + be resolved): FSL_RC_NOT_FOUND. This would generally indicate + some sort of data consistency problem. i.e. it's quite possibly + very bad if this is returned. + + - If the checkin would result in no file-level changes vis-a-vis + the current checkout, FSL_RC_NOOP is returned. + + BUGS: + + - It cannot currently properly distinguish a "no-op" commit, one in + which no files were modified or only their permissions were + modifed. + + @see fsl_checkin_enqueue() + @see fsl_checkin_dequeue() + @see fsl_checkin_discard() + @see fsl_checkin_T_add() +*/ +FSL_EXPORT int fsl_checkin_commit(fsl_cx * f, fsl_checkin_opt const * opt, + fsl_id_t * newRid, fsl_uuid_str * newUuid); + +/** + Works like fsl_deck_T_add(), adding the given tag information to + the pending checkin state. Returns 0 on success, non-0 on error. A + checkin may, in principal, have any number of tags, and this may be + called any number of times to add new tags to the pending + commit. This list of tags gets cleared by a successful + fsl_checkin_commit() or by fsl_checkin_discard(). Decks require + that each tag be distinct from each other (none may compare + equivalent), but that check is delayed until the deck is output + into its final artifact form. + + @see fsl_checkin_enqueue() + @see fsl_checkin_dequeue() + @see fsl_checkin_commit() + @see fsl_checkin_discard() + @see fsl_checkin_T_add2() +*/ +FSL_EXPORT int fsl_checkin_T_add( fsl_cx * f, fsl_tagtype_e tagType, + fsl_uuid_cstr uuid, char const * name, + char const * value); + +/** + Works identically to fsl_checkin_T_add() except that it takes its + argument in the form of a T-card object. + + On success ownership of t is passed to mf. On error (see + fsl_deck_T_add()) ownership is not modified. + + Results are undefined if either argument is NULL or improperly + initialized. +*/ +FSL_EXPORT int fsl_checkin_T_add2( fsl_cx * f, fsl_card_T * t ); + +/** + Clears all contents from f's checkout database, including the vfile + table, vmerge table, and some of the vvar table. The tables are + left intact. Returns 0 on success, non-0 if f has no checkout or for + a database error. + */ +FSL_EXPORT int fsl_ckout_clear_db(fsl_cx *f); + +/** + Returns the base name of the current platform's checkout database + file. That is "_FOSSIL_" on Windows and ".fslckout" everywhere + else. The returned bytes are static. + + TODO: an API which takes a dir name and looks for either name +*/ +FSL_EXPORT char const *fsl_preferred_ckout_db_name(); + +/** + File-overwrite policy values for use with fsl_ckup_opt and friends. +*/ +enum fsl_file_overwrite_policy_e { +/** Indicates that an error should be triggered if a file would be + overwritten. */ +FSL_OVERWRITE_ERROR = 0, +/** + Indicates that files should always be overwritten by +*/ +FSL_OVERWRITE_ALWAYS, +/** + Indicates that files should never be overwritten, and silently + skipped over. This is almost never what one wants to do. +*/ +FSL_OVERWRITE_NEVER +}; +typedef enum fsl_file_overwrite_policy_e fsl_file_overwrite_policy_e; + +/** + State values for use with fsl_ckup_state::fileRmInfo. +*/ +enum fsl_ckup_rm_state_e { +/** + Indicates that the file was not removed in a given checkout. + Guaranteed to have the value 0 so that it is treated as boolean + false. No other entries in this enum have well-defined values. +*/ +FSL_CKUP_RM_NOT = 0, +/** + Indicates that a file was removed from a checkout but kept + in the filesystem because it was locally modified. +*/ +FSL_CKUP_RM_KEPT, +/** + Indicates that a file was removed from a checkout and the + filesystem, with the caveat that failed attempts to remove from the + filesystem are ignored for Reasons but will be reported as if the + unlink worked. +*/ +FSL_CKUP_RM +}; +typedef enum fsl_ckup_rm_state_e fsl_ckup_rm_state_e; + +/** + Under construction. Work in progress... + + Options for "opening" a fossil repository database. That is, + creating a new fossil checkout database and populating its schema, + _without_ checking out any files. (That latter part is up for + reconsideration and this API might change in the future to check + out files after creating/opening the db.) +*/ +struct fsl_repo_open_ckout_opt { + /** + Name of the target directory, which must already exist. May be + relative, e.g. ".". The repo-open operation will chdir to this + directory for the duration of the operation. May be NULL, in + which case the current directory is assumed and no chdir is + performed. + */ + char const * targetDir; + + /** + The filename, with no directory components, of the desired + checkout db name. For the time being, always leave this NULL and + let the library decide. It "might" (but probably won't) be + interesting at some point to allow the client to specify a + different name (noting that that would be directly incompatible + with fossil(1)). + */ + char const * ckoutDbFile; + + /** + Policy for how to handle overwrites of files extracted from a + newly-opened checkout. + + Potential TODO: replace this with a fsl_confirmer, though that + currently seems like overkill for this particular case. + */ + fsl_file_overwrite_policy_e fileOverwritePolicy; + + /** + fsl_repo_open_ckout() installs the fossil checkout schema. If + this is true it will forcibly replace any existing relevant + schema components in the checkout db, otherwise it will fail when + it tries to overwrite an existing schema and cannot. + */ + bool dbOverwritePolicy; + + /** + Of true, the checkout-open process will look for an opened + checkout in the target directory and its parents (recursively) + and fail with FSL_RC_ALREADY_EXISTS if one is found. + */ + bool checkForOpenedCkout; +}; +typedef struct fsl_repo_open_ckout_opt fsl_repo_open_ckout_opt; + +/** + Empty-initialized fsl_repo_open_ckout_opt const-copy constructer. +*/ +#define fsl_repo_open_ckout_opt_m { \ + NULL/*targetDir*/, NULL/*ckoutDbFile*/, \ + FSL_OVERWRITE_ERROR/*fileOverwritePolicy*/, \ + false/*dbOverwritePolicy*/, \ + -1/*checkForOpenedCkout*/ \ +} + +/** + Empty-initialised fsl_repo_open_ckout_opt instance. Clients should copy + this value (or fsl_repo_open_ckout_opt_empty_m) to initialise + fsl_repo_open_ckout_opt instances for sane default values. +*/ +FSL_EXPORT const fsl_repo_open_ckout_opt fsl_repo_open_ckout_opt_empty; + +/** + Work in progress... + + Opens a checkout db for use with the currently-connected repository + or creates a new one. If opening an existing one, it gets "stolen" + from any repository it might have been previously mapped to. + + - Requires that f have an opened repository db and no opened + checkout. Returns FSL_RC_NOT_A_REPO if no repo is opened and + FSL_RC_MISUSE if a checkout *is* opened. + + - Creates/re-uses a .fslckout DB in the dir opt->targetDir. The + directory must be NULL or already exist, else FSL_RC_NOT_FOUND is + returned. If opt->dbOverwritePolicy is false then it fails with + FSL_RC_ALREADY_EXISTS if that directory already contains a + checkout db. + + Note that this does not extract any SCM'd files from the + repository, it only opens (and possibly creates) the checkout + database. + + Pending: + + - If opening an existing checkout db for a different repo then + delete the STASH and UNDO entries, as they're not valid for a + different repo. +*/ +FSL_EXPORT int fsl_repo_open_ckout( fsl_cx * f, fsl_repo_open_ckout_opt const * opt ); + +typedef struct fsl_ckup_state fsl_ckup_state; +/** + A callback type for use with fsl_ckup_state. It gets called via + fsl_repo_ckout() and fsl_ckout_update() to report progress of the + extraction process. It gets called after one of those functions has + successfully extracted a file or skipped over it because the file + existed and the checkout options specified to leave existing files + in place. It must return 0 on success, and non-0 will end the + extraction process, propagating that result code back to the + caller. If this callback fails, the checkout's contents may be left + in an undefined state, with some files updated and others not. All + database-side data will be consistent (the transaction is rolled + back) but filesystem-side changes may not be. +*/ +typedef int (*fsl_ckup_f)( fsl_ckup_state const * cState ); + +/** + This enum lists the various types of individual file change states + which can happen during a checkout, update, or merge. +*/ +enum fsl_ckup_fchange_e { +/** Sentinel value. */ +FSL_CKUP_FCHANGE_INVALID = -1, +/** Was unchanged between the previous and updated-to version, + so no change was made to the on-disk file. This is the + only entry in the enum which is guaranteed to have a specific + value: 0, so that it can be used as a boolean false. */ +FSL_CKUP_FCHANGE_NONE = 0, +/** + Added to SCM in the updated-to version. +*/ +FSL_CKUP_FCHANGE_ADDED, +/** + Added to SCM in the current checkout version and carried over into + the updated-to version. +*/ +FSL_CKUP_FCHANGE_ADD_PROPAGATED, +/** + Removed from SCM in the updated-to to version OR in the checked-out + version but not yet committed. a.k.a. it became "unmanaged." + + Do we need to differentiate between those cases? +*/ +FSL_CKUP_FCHANGE_RM, +/** + Removed from the checked-out version but not yet commited, + so was carried over to the updated-to version. +*/ +FSL_CKUP_FCHANGE_RM_PROPAGATED, +/** Updated or replaced without a merge by the checkout/update + process. */ +FSL_CKUP_FCHANGE_UPDATED, +/** Merge was not performed because at least one of the inputs appears + to be binary. The updated-to version overwrites the previous + version in this case. +*/ +FSL_CKUP_FCHANGE_UPDATED_BINARY, +/** Updated with a merge by the update process. */ +FSL_CKUP_FCHANGE_MERGED, +/** Special case of FSL_CKUP_FCHANGE_UPDATED. Merge was performed + and conflicts were detected. The newly-updated file will contain + conflict markers. + + @see fsl_buffer_contains_merge_marker() +*/ +FSL_CKUP_FCHANGE_CONFLICT_MERGED, +/** Added in the current checkout but also contained in the + updated-to version. The local copy takes precedence. +*/ +FSL_CKUP_FCHANGE_CONFLICT_ADDED, +/** + Added by the updated-to version but a local unmanaged copy exists. + The local copy is overwritten, per historical fossil(1) convention + (noting that fossil has undo support to allow one to avoid loss of + such a file's contents). + + TODO: use confirmer here to ask user whether to overwrite. +*/ +FSL_CKUP_FCHANGE_CONFLICT_ADDED_UNMANAGED, +/** Edited locally but removed from updated-to version. Local + edits will be left in the checkout tree. */ +FSL_CKUP_FCHANGE_CONFLICT_RM, +/** Cannot merge if one or both of the update/updating verions of a + file is a symlink The updated-to version overwrites the previous + version in this case. + + We probably need a better name for this. +*/ +FSL_CKUP_FCHANGE_CONFLICT_SYMLINK, +/** File was renamed in the updated-to version. If a file is both + modified and renamed, it is flagged as renamed instead + of modified. */ +FSL_CKUP_FCHANGE_RENAMED, +/** Locally modified. This state appears only when + "updating" a checkout to the same version. */ +FSL_CKUP_FCHANGE_EDITED +}; +typedef enum fsl_ckup_fchange_e fsl_ckup_fchange_e; + +/** + State to be passed to fsl_ckup_f() implementations via + calls to fsl_repo_ckout() and fsl_ckout_update(). +*/ +struct fsl_ckup_state { + /** + The core SCM state for the just-extracted file. Note that its + content member will be NULL: the content is not passed on via + this interface because it is only loaded for files which require + overwriting. + + An update process may synthesize content for extractState->fCard + which do not 100% reflect the file on disk. Of primary note here: + + 1) fCard->uuid will refer to the hash of the updated-to + version, as opposed to the hash of the on-disk file (which may + differ due to having local edits merged in). + + 2) For the update process, fCard->priorName will be NULL unless + the file was renamed between the original and updated-to + versions, in which case priorName will refer to the original + version's name. + */ + fsl_repo_extract_state const * extractState; + /** + Optional client-dependent state for use in the fsl_ckup_f() + callback. This is copies from the corresponding + fsl_ckup_opt::callbackState member. + */ + void * callbackState; + + /** + Vaguely describes the type of change the current call into + the fsl_ckup_f() represents. The full range of values is + not valid for all operations. Specifically: + + Checkout only uses: + + FSL_CKUP_FCHANGE_NONE + FSL_CKUP_FCHANGE_UPDATED + FSL_CKUP_FCHANGE_RM + + For update operations all (or most) values are potentially + possible. + + If this has a value of FSL_CKUP_FCHANGE_RM, + this->fileRmInfo will provide a bit more detail. + */ + fsl_ckup_fchange_e fileChangeType; + + /** + Indicates whether the file was removed by the process: + + - FSL_CKUP_RM_NOT = Was not removed. + + - FSL_CKUP_RM_KEPT = Was removed from the checked-out version but + left in the filesystem because the confirmer said to. + + - FSL_CKUP_RM = Was removed from the checkout and the filesystem. + + When this->dryRun is true, this specifies whether the file would + have been removed. + */ + fsl_ckup_rm_state_e fileRmInfo; + + /** + If fsl_repo_ckout()'s or fsl_ckout_update()'s options specified + that the mtime should be set on each updated file, this holds + that time. If the file existed and was not overwritten, it is set + to that file's time. Else it is set to the current time (which + may differ by a small fraction of a second from the file-write + time because we avoid stat()'ing it again after writing). If + this->fileRmInfo indicates that a file was removed, this might + (depending on availability of the file in the filesystem at the + time) be set to 0. + + When running in dry-run mode, this value may be 0, as we may not + have a file in place which we can stat() to get it, nor a db + entry from which to fetch it. + + This option is ignored for merge operations. + */ + fsl_time_t mtime; + + /** + The size of the extracted file, in bytes. If the file was removed + from the filesystem (or removal was at least attempted) then this + is set to -1. + */ + fsl_int_t size; + + /** + True if the current checkout/update is running in dry-run mode, + else false. See fsl_ckup_opt::dryRun for details. + */ + bool dryRun; +}; + +/** + Options for use with fsl_repo_ckout() and + fsl_ckout_update(). i.e. for checkout and update. +*/ +struct fsl_ckup_opt { + /** + The version of the repostitory to check out or update. This must + be the blob.rid of a checkin artifact. + */ + fsl_id_t checkinRid; + + /** + Gets called once per checked-out or updated file, passed a + fsl_ckup_state instance with information about the + checked-out file and related metadata. May be NULL. + */ + fsl_ckup_f callback; + + /** + State to be passed to this->callback via the + fsl_ckup_state::callbackState member. + */ + void * callbackState; + + /** + An optional "confirmer" for answering questions about file + overwrites and deletions posed by the checkout process. + By default this confirmer of the associated fsl_cx instance + is used. + + Caveats: + + - This is not currently used by the update process, only + checkout. + + - If this->setMTime is true, the mtime is NOT set for any files + which already exist and are skipped due to the confirmer saying + to leave them in place. + + - Similarly, if the confirmer says to never overwrite files, + permissions on existing files are not modified. fsl_repo_ckout() + does not (re)write unmodified files, and thus may leave such + files with different permissions. That's on the to-fix list. + */ + fsl_confirmer confirmer; + + /** + If true, the checkout/update processes will calculate the + (synthetic) mtime of each extracted file and set its mtime. This + is a relatively expensive operation which calculates the + "effective mtime" of each file by calculating it: Fossil does not + record file timestamps, instead treating files as if they had the + timestamp of the most recent checkin in which they were added or + modified. + + It's generally a good idea to let the update process stamp the + _current_ time on modified files, in order to avoid any hiccups + with build processes which rely on accurate times + (e.g. Makefiles). When doing a clean checkout, it's often + interesting to see the "original" times, though. + */ + bool setMtime; + + /** + A hint to fsl_repo_ckout() and fsl_ckout_update() about whether + it needs to scan the checkout for changes. Set this to false ONLY + if the calling code calls fsl_ckout_changes_scan() (or + equivalent, e.g. fsl_vfile_changes_scan()) immediately before + calling fsl_repo_ckout() or fsl_ckout_update(), as those require + a non-stale changes scan in order to function properly. + */ + bool scanForChanges; + + /** + If true, the extraction process will "go through the motions" but + will not write any files to disk. It will perform I/O such as + stat()'ing to see, e.g., if it would have needed to overwrite a + file. + */ + bool dryRun; +}; +typedef struct fsl_ckup_opt fsl_ckup_opt; + +/** + Empty-initialized fsl_ckup_opt const-copy constructor. +*/ +#define fsl_ckup_opt_m {\ + -1/*checkinRid*/, NULL/*callback*/, NULL/*callbackState*/, \ + fsl_confirmer_empty_m/*confirmer*/,\ + false/*setMtime*/, true/*scanForChanges*/,false/*dryRun*/ \ +} + +/** + Empty-initialised fsl_ckup_opt instance. Clients should copy + this value (or fsl_ckup_opt_empty_m) to initialise + fsl_ckup_opt instances for sane default values. +*/ +FSL_EXPORT const fsl_ckup_opt fsl_ckup_opt_empty; + +/** + A fsl_repo_extract() proxy which extracts the contents of the + repository version specified by opt->checkinRid to the root + directory of f's currently-opened checkout. i.e. it performs a + "checkout" operation. + + For each extracted entry, cOpt->callback (if not NULL) will be + passed a (fsl_ckup_state const*) which contains a pointer + to the fsl_repo_extract_state and some additional metadata + regarding the extraction. The value of cOpt->callbackState will be + set as the callbackState member of that fsl_ckup_state + struct, so that the client has a way of passing around app-specific + state to that callback. + + After successful completion, the process will report (see below) + any files which were part of the previous checkout version but are + not part of the current version, optionally removing them from the + filesystem (depending on the value of opt->rmMissingPolicy). It + will IGNORE ANY DELETION FAILURE of files it attempts to + delete. The reason it does not fail on removal error is because + doing so would require rolling back the transaction, effectively + undoing the checkout, but it cannot roll back any prior deletions + which succeeded. Similarly, after all file removal is complete, it + attempts to remove any now-empty directories left over by that + process, also silently ignoring any errors. If the cOpt->dryRun + option is specified, it will "go through the motions" of removing + files but will not actually attempt filesystem removal. For + purposes of the callback, however, it will report deletions as + having happened (but will also set the dryRun flag on the object + passed to the callback). + + After unpacking the SCM-side files, it may write out one or more + manifest files, as described for fsl_ckout_manifest_write(), if the + 'manifest' config setting says to do so. + + As part of the file-removal process, AFTER all "existing" files are + processed, it calls cOpt->callback() (if not NULL) for each removed + file, noting the following peculiarities in the + fsl_ckup_state object which is passed to it for those + calls: + + - It is called after the processing of "existing" files. Thus the + file names passed during this step may appear "out of order" with + regards to the others (which are guaranteed to be passed in lexical + order, though whether it is case-sensitive or not depends on the + repository's case-sensitivity setting). + + - fileRmInfo will indicate that the file was removed from the + checkout, and whether it was actually removed or retained in the + filesystem. This will indicate filesystem-level removal even when + in dry-run mode, though in that case no filesystem-level removal is + actually attempted. + + - extractState->fileRid will refer to the file's blob's RID for the + previous checkout version. + + - extractState->content will be NULL. + + - extractState->callbackState will be NULL. + + - extractState->fCard will refer to the pre-removal state of the + file. i.e. the state as it was in the checkout prior to this + function being called. + + Returns 0 on success. Returns FSL_RC_NOT_A_REPO if f has no opened + repo, FSL_RC_NOT_A_CKOUT if no checkout is opened. If + cOpt->callback is not NULL and returns a non-0 result code, + extraction ends and that result is returned. If it returns non-0 at + any point after basic argument validation, it rolls back all + changes or sets the current transaction stack into a rollback + state. + + @see fsl_repo_ckout_open() +*/ +FSL_EXPORT int fsl_repo_ckout(fsl_cx * f, fsl_ckup_opt const * cOpt); + + +/** + UNDER CONSTRUCTION. + + Performs an "update" operation on f's currenly-opened + checkout. Performing an update is similar to performing a checkout, + the primary difference being that an update will merge local file + modifications into any newly-updated files, whereas a checkout will + overwrite them. + + TODO?: fossil(1)'s update permits a list of files, in which case it + behaves differently: it updates the given files to the version + requested but leaves the checkout at its current version. To be + able to implement that we either need clients to call this in a + loop, changing opt->filename on each call (like how we do + fsl_ckout_manage()) or we need a way for them to pass on the list + of files/dir in the opt object. + + @see fsl_repo_ckout(). +*/ +FSL_EXPORT int fsl_ckout_update(fsl_cx * f, fsl_ckup_opt const *opt); + + +/** + Tries to calculate a version to update the current checkout version + to, preferring the tip of the current checkout's branch. + + On success, 0 is returned and *outRid is set to the calculated RID, + which may be 0, indicating that no errors were encountered but no + version could be calculated. + + On error, non-0 is returned, outRid is not modified, and f's error + state is updated. + + Returns FSL_RC_NOT_A_CKOUT if f has no checkout opened and + FSL_RC_NOT_A_REPO if no repo is opened. + + If it calculates that there are multiple viable descendants it + returns FSL_RC_AMBIGUOUS and f's error state will contain a list of + the UUIDs (or UUID prefixes) of those descendants. + + Sidebar: to get the absolute latest version, irrespective of the + branch, use fsl_sym_to_rid() to resolve the symbolic name "tip". +*/ +FSL_EXPORT int fsl_ckout_calc_update_version(fsl_cx * f, fsl_id_t * outRid); + +/** + Bitmask used by fsl_ckout_manifest_setting() and + fsl_ckout_manifest_write(). +*/ +enum fsl_cx_manifest_mask_e { +FSL_MANIFEST_MAIN = 0x001, +FSL_MANIFEST_UUID = 0x010, +FSL_MANIFEST_TAGS = 0x100 +}; +typedef enum fsl_cx_manifest_mask_e fsl_cx_manifest_mask_e; + +/** + Returns a bitmask representing which manifest files, if any, will + be written when opening or updating a checkout directory, as + specified by the repository's 'manifest' configuration setting, and + sets *m to a bitmask indicating which of those are enabled. It + first checks for a versioned setting then, if no versioned setting + is found, a repository-level setting. + + A truthy setting value (1, "on", "true") means to write the + manifest and manifest.uuid files. A string with any of the letters + 'r', 'u', or 't' means to write the [r]aw, [u]uid, and/or [t]ags + file(s), respectively. + + If the manifest setting is falsy or not set, *m is set to 0, else + *m is set to a bitmask representing which file(s) are considered to + be auto-generated for this repository: + + - FSL_MANIFEST_MAIN = manifest + - FSL_MANIFEST_UUID = manifest.uuid + - FSL_MANIFEST_TAGS = manifest.tags + + Any db-related or allocation errors while trying to fetch the + setting are silently ignored. + + For performance's sake, since this is potentially called often from + fsl_reserved_fn_check(), this setting is currently cached by this + routine (in the fsl_cx object), but that ignores the fact that the + manifest setting can be modified at any time, either in a versioned + setting file or the repository db, and may be modified from outside + the library. There's a tiny back-door for working around that: if m + is NULL, the cache will be flushed and no other work will be + performed. Thus the following approach can be used to force a fresh + check for that setting: + + ``` + fsl_ckout_manifest_setting(f, NULL); // clears caches, does nothing else + fsl_ckout_manifest_setting(f, &myInt); // loads/caches the setting + ``` +*/ +FSL_EXPORT void fsl_ckout_manifest_setting(fsl_cx *f, int *m); + +/** + Might write out the files manifest, manifest.uuid, and/or + manifest.tags for the current checkout to the the checkout's root + directory. The 2nd-4th arguments are interpreted as follows: + + 0: Do not write that file. + + >0: Always write that file. + + <0: Use the value of the "manifest" config setting (see + fsl_ckout_manifest_setting()) to determine whether or not to write + that file. + + As each file is written, its mtime is set to that of the checkout + version. (Forewarning: that behaviour may change if it proves to be + problematic vis a vis build processes.) + + Returns 0 on success, non-0 on error: + + - FSL_RC_NOT_A_CKOUT if no checkout is opened. + + - FSL_RC_RANGE if the current checkout RID is 0 (indicating a fresh, + empty repository). + + - Various potential DB/IO-related error codes. + + If the final argument is not NULL then it will be updated to + contain a bitmask representing which files, if any, were written: + see fsl_ckout_manifest_setting() for the values. It is updated + regardless of success or failure and will indicate which file(s) + was/were written before the error was triggered. + + Each file implied by the various manifest settings which is NOT + written by this routine and is also not part of the current + checkout (i.e. not listed in the vfile table) will be removed from + disk, but a failure while attempting to do so will be silently + ignored. + + @see fsl_repo_manifest_write() +*/ +FSL_EXPORT int fsl_ckout_manifest_write(fsl_cx *f, + int manifest, + int manifestUuid, + int manifestTags, + int *wroteWhat ); + +/** + Returns true if f has an opened checkout and the given absolute + path is rooted in that checkout, else false. As a special case, it + returns false if the path _is_ the checkout root unless zAbsPath + has a trailing slash. (The checkout root is always stored with a + trailing slash because that simplifies its internal usage.) + + Note that this is strictly a string comparison, not a + filesystem-level operation. +*/ +FSL_EXPORT bool fsl_is_rooted_in_ckout(fsl_cx *f, char const *zAbsPath); + +/** + Works like fsl_is_rooted_in_ckout() except that it returns 0 on + success, and on error updates f with a description of the problem + and returns non-0: FSL_RC_RANGE or (if updating the error state + fails) FSL_RC_OOM. + */ +FSL_EXPORT int fsl_is_rooted_in_ckout2(fsl_cx *f, char const *zAbsPath); + +/** + Change-type values for use with fsl_ckout_revert_f() callbacks. +*/ +enum fsl_ckout_revert_e { +/** Sentinel value. */ +FSL_REVERT_NONE = 0, +/** + File was previously queued for addition but unqueued + by the revert process. +*/ +FSL_REVERT_UNMANAGE, +/** + File was previously queued for removal but unqueued by the revert + process. If the file's contents or permissions were also reverted + then the file is reported as FSL_REVERT_PERMISSIONS or + FSL_REVERT_CONTENTS instead. +*/ +FSL_REVERT_REMOVE, +/** + File was previously scheduled to be renamed, but the rename was + reverted. The name reported to the callback is the original one. + If a file was both modified and renamed, it will be flagged as + renamed instead of modified, for consistency with the usage of + fsl_ckup_fchange_e's FSL_CKUP_FCHANGE_RENAMED. + + FIXME: this does not mean that the file on disk was actually + renamed (if needed). That is TODO, pending addition of code to + perform renames. +*/ +FSL_REVERT_RENAME, +/** File's permissions (only) were reverted. */ +FSL_REVERT_PERMISSIONS, +/** + File's contents reverted. This value trumps any others in this + enum. Thus if a file's permissions and contents were reverted, + or it was un-renamed and its contents reverted, it will be + reported using this enum entry. +*/ +FSL_REVERT_CONTENTS +}; +typedef enum fsl_ckout_revert_e fsl_ckout_revert_e; +/** + Callback type for use with fsl_ckout_revert(). For each reverted + file it gets passed the checkout-relative filename, type of change, + and the callback state pointer which was passed to + fsl_ckout_revert(). If it returns non-0, the revert process will + end in an error and that code will be propagated back to the + caller. In such cases, any files reverted up until that point will + still be reverted on disk but the reversion in the database will be + rolled back. A change scan (e.g. fsl_ckout_changes_scan()) will + restore balance to that equation, but these callbacks should only + return non-0 in for catastrophic failure. +*/ +typedef int (*fsl_ckout_revert_f)( char const *zFilename, + fsl_ckout_revert_e changeType, + void * callbackState ); + +/** + Options for passing to fsl_ckout_revert(). +*/ +struct fsl_ckout_revert_opt { + /** + File or directory name to revert. See also this->vfileIds. + */ + char const * filename; + /** + An alternative to assigning this->filename is to point + this->vfileIds to a bag of vfile.id values. If this member is not + NULL, fsl_ckout_revert() will ignore this->filename. + + @see fsl_filename_to_vfile_ids() + */ + fsl_id_bag const * vfileIds; + /** + Interpret filename as relative to cwd if true, else relative to + the current checkout root. This is ignored when this->vfileIds is + not NULL. + */ + bool relativeToCwd; + /** + If true, fsl_vfile_changes_scan() is called to ensure that + the filesystem and vfile tables agree. If the client code has + called that function, or its equivalent, since any changes were + made to the checkout then this may be set to false to speed up + the revert process. + */ + bool scanForChanges; + /** + Optional callback to notify the client of what gets reverted. + */ + fsl_ckout_revert_f callback; + /** + State for this->callback. + */ + void * callbackState; +}; +typedef struct fsl_ckout_revert_opt fsl_ckout_revert_opt; +/** + Initialized-with-defaults fsl_ckout_revert_opt instance, + intended for use in const-copy initialization. +*/ +#define fsl_ckout_revert_opt_empty_m { \ + NULL/*filename*/,NULL/*vfileIds*/,true/*relativeToCwd*/,true/*scanForChanges*/, \ + NULL/*callback*/,NULL/*callbackState*/ \ +} +/** + Initialized-with-defaults fsl_ckout_revert_opt instance, + intended for use in non-const copy initialization. +*/ +FSL_EXPORT const fsl_ckout_revert_opt fsl_ckout_revert_opt_empty; + +/** + Reverts changes to checked-out files, replacing their + on-disk versions with the current checkout's version. + + If zFilename refers to a directory, all managed files under that + directory are reverted (if modified). If zFilename is NULL or + empty, all modifications in the current checkout are reverted. + + If a file has been added but not yet committed, the add + is un-queued but the file is otherwise untouched. If the + file has been queued for removal, this removes it from + that queue as well as restores its contents. + + If a rename is pending for the given filename, the name may match + either its original name or new name. Whether or not that will + actually work when file A is renamed to B and file C is renamed to + A is anyone's guess. (Noting that (fossil mv) won't allow that + situation to exist in the vfile table for a single checkout + version, so it seems safe enough.) + + If the 4th argument is not NULL then it is called for each revert + (_after_ the revert happens) to report what was done with that + file. It gets passed the checkout-relative name of each reverted + file. The 5th argument is not interpreted by this function but is + passed on as-is as the final argument to the callback. If the + callback returns non-0, the revert process is cancelled, any + pending transaction is set to roll back, and that error code is + returned. Note that cancelling a revert mid-process will leave file + changes made by the revert so far in place, and thus the checkout + db and filesystem will be in an inconsistent state until + fsl_vfile_changes_scan() (or equivalent) is called to restore + balance to the world. + + Files which are not actually reverted because their contents or + permissions were not modified on disk are not reported to the + callback unless the reversion was the un-queuing of an ADD or + REMOVE operation. + + Returns 0 on success, any number of non-0 results on error. +*/ +FSL_EXPORT int fsl_ckout_revert( fsl_cx * f, + fsl_ckout_revert_opt const * opt ); + +/** + Expects f to have an opened checkout and zName to be the name of + an entry in the vfile table where vfile.vid == vid. If vid<=0 then + the current checkout RID is used. This function does not do any + path resolution or normalization on zName and checks only for an + exact match (honoring f's case-sensitivity setting - see + fsl_cx_case_sensitive_set()). + + On success it returns 0 and assigns *vfid to the vfile.id value of + the matching file. If no match is found, 0 is returned and *vfile + is set to 0. Returns FSL_RC_NOT_A_CKOUT if no checkout is opened, + FSL_RC_RANGE if zName is not a simple path (see + fsl_is_simple_pathname()), and any number of codes for db-related + errors. + + This function matches only vfile.pathname, not vfile.origname, + because it is possible for a given name to be in both fields (in + different records) at the same time. +*/ +FSL_EXPORT int fsl_filename_to_vfile_id( fsl_cx * f, fsl_id_t vid, + char const * zName, + fsl_id_t * vfid ); + +/** + Searches the `vfile` table where `vfile.vid=vid` for a name which + matches `zName` or all `vfile` entries found under a subdirectory + named `zName` (with no trailing slash). `zName` must be relative to + the checkout root. As a special case, if `zName` is `NULL`, empty, + or `"."` then all files in `vfile` with the given `vid` are + selected. For each entry it finds, it adds the `vfile.id` to + `dest`. If `vid<=0` then the current checkout RID is used. + + If `changedOnly` is `true` then only entries which have been marked + in the `vfile` table as having some sort of change are included, so + if `true` then fsl_ckout_changes_scan() (or equivalent) must have + been "recently" called to ensure that state is up to do. This routine + only checks the `vfile` table for "is changed" state, it does not + do filesystem-level checks on the files. + + This search honors the context-level case-sensitivity setting (see + fsl_cx_case_sensitive_set()). + + Returns 0 on success. Not finding anything is not treated as an + error, though we could arguably return `FSL_RC_NOT_FOUND` for the + cases which use this function. In order to determine whether or not + any results were actually found, compare `dest->entryCount` before + and after calling this. + + This function matches only `vfile.pathname`, not `vfile.origname`, + because it is possible for a given name to be in both fields (in + different records) at the same time. + + @see fsl_ckout_vfile_ids() +*/ +FSL_EXPORT int fsl_filename_to_vfile_ids( fsl_cx * f, fsl_id_t vid, + fsl_id_bag * dest, + char const * zName, + bool changedOnly); + +/** + This is a variant of fsl_filename_to_vfile_ids() which accepts + filenames in a more flexible form than that routine. This routine + works exactly like that one except for the following differences: + + 1) The given filename and the relativeToCwd arguments are passed to + by fsl_ckout_filename_check() to canonicalize the name and ensure + that it points to someplace within f's current checkout. + + 2) Because of (1), zName may not be NULL or empty. To fetch all of + the vfile IDs for the current checkout, pass a zName of "." and + relativeToCwd=false. + + Returns 0 on success, FSL_RC_MISUSE if zName is NULL or empty, + FSL_RC_OOM on allocation error, FSL_RC_NOT_A_CKOUT if f has no + opened checkout. +*/ +FSL_EXPORT int fsl_ckout_vfile_ids( fsl_cx * f, fsl_id_t vid, + fsl_id_bag * dest, char const * zName, + bool relativeToCwd, bool changedOnly ); + + +/** + This "mostly internal" routine (re)populates f's checkout vfile + table with all files from the given checkin manifest. If + manifestRid is 0 or less then the current checkout's RID is + used. If vfile already contains any content for the given checkin, + it is left intact (and several processes rely on that behavior to + keep it from nuking, e.g., as-yet-uncommitted queued add/rm + entries). + + Returns 0 on success, any number of codes on any of many potential + errors. + + f must not be NULL and must have opened checkout and repository + databases. In debug builds it will assert that that is so. + + If the 3rd argument is true, any entries in vfile for checkin + versions other than the one specified in the 2nd argument are + cleared from the vfile table. That is _almost_ always the desired + behavior, but there are rare cases where vfile needs to temporarily + (for the duration of a single transaction) hold state for multiple + versions. + + If the 4th argument is not NULL, it gets assigned the number of + blobs from the given version which are currently missing from the + repository due to being phantoms (as opposed to being shunned). + + Returns 0 on success, FSL_RC_NOT_A_CKOUT if no checkout is opened, + FSL_RC_OOM on allocation error, FSL_RC_DB for db-related problems, + et.al. + + Misc. notes: + + - This does NOT update the "checkout" vvar table entry because this + routine is sometimes used in contexts where we need to briefly + maintain two vfile versions and keep the previous checkout version. + + - Apps must take care to not leave more than one version in the + vfile table for longer than absolutely needed. They "really should" + use fsl_vfile_unload() to clear out any version they load with this + routine. + + @see fsl_vfile_unload() + @see fsl_vfile_unload_except() +*/ +FSL_EXPORT int fsl_vfile_load(fsl_cx * const f, fsl_id_t manifestRid, + bool clearOtherVersions, + uint32_t * missingCount); + +/** + Clears out all entries in the current checkout's vfile table with + the given vfile.vid value. If vid<=0 then the current checkout RID + is used (which is never a good idea from client-side code!). + + ACHTUNG: never do this without understanding the consequences. It + can ruin the current checkout state. + + Returns 0 on success, FSL_RC_NOT_A_CKOUT if f has no checkout + opened, FSL_RC_DB on any sort of db-related error (in which case + f's error state is updated with a description of the problem). + + @see fsl_vfile_load() + @see fsl_vfile_unload_except() +*/ +FSL_EXPORT int fsl_vfile_unload(fsl_cx * const f, fsl_id_t vid); + +/** + A counterpart of fsl_vfile_unload() which removes all vfile + entries where vfile.vid is not the given vid. If vid is <=0 then + the current checkout RID is used. + + Returns 0 on success, FSL_RC_NOT_A_CKOUT if f has no checkout + opened, FSL_RC_DB on any sort of db-related error (in which case + f's error state is updated with a description of the problem). + + @see fsl_vfile_load() + @see fsl_vfile_unload() +*/ +FSL_EXPORT int fsl_vfile_unload_except(fsl_cx * const f, fsl_id_t vid); + + +/** + Performs a "fingerprint check" between f's current checkout and + repository databases. Returns 0 if either there is no checkout, no + mismatch, or it is impossible to determine because the checkout is + missing a fingerprint (which is legal for "older" checkout + databases). + + If a mismatch is found, FSL_RC_REPO_MISMATCH is returned. Returns + some other non-0 code on a lower-level error (db access, OOM, + etc.). + + A mismatch can happen when the repository to which a checkout + belongs is replaced, either with a completely different repository + or a copy/clone of that same repository. Each repository copy may + have differing blob.rid values, and those are what the checkout + database uses to refer to repository-side data. If those RIDs + change, then the checkout is left pointing to data other than what + it should be. + + TODO: currently the library offers no automated recovery mechanism + from a mismatch, the only remedy being to close the checkout + database, destroy it, and re-create it. fossil(1) is able, in some cases, + to automatically recover from this situation. +*/ +FSL_EXPORT int fsl_ckout_fingerprint_check(fsl_cx * f); + +/** + Looks for the given file in f's current checkout. If relativeToCwd + then the name is resolved from the current directory, otherwise it is + assumed to be relative to the checkout root or an absolute path + with the checkout dir as a prefix of that path. + + On success, 0 is returned and dest's gets populated with the + content of the file. + + On error, non-0 is returned and, depending on the error type, dest + might be partially populated. f's error state will be updated to + describe the error. + + Results are undefined if any pointer argument is NULL. + + This function currently resolves symlinks on its way to the + content, but that behaviour may change in the future to reflect f's + symlink preferences. +*/ +FSL_EXPORT int fsl_ckout_file_content(fsl_cx * const f, bool relativeToCwd, + char const * zName, + fsl_buffer * const dest); + +/** + Fetches the timestamp of the given F-card's name against the + filesystem and/or the most recent checkin in which it was modified + (as reported by fsl_mtime_of_manifest()). vid is the checkin + version to look at. If it's 0, the current checkout will be used. + + On success, returns 0 and: + + - If repoMtime is not NULL then (*repoMtime) is assigned to the + result of fsl_mtime_of_manifest_file() for the given file. + + - If localMtime is not NULL then (*localMtime) is assigned to + the checkout-local timestamp of the file. + + Returns non-0 on error. Some of the potential results include: + + - FSL_RC_NOT_A_CKOUT. + + - FSL_RC_NOT_FOUND if the filename cannot be resolved in the + requested version or cannot be stat()'d. + + - FSL_RC_OOM. +*/ +FSL_EXPORT int fsl_card_F_ckout_mtime(fsl_cx * const f, fsl_id_t vid, + fsl_card_F const * const fc, + fsl_time_t * repoMtime, + fsl_time_t * localMtime); + +/** + UNDER CONSTRUCTION! INCOMPLETE! + */ +struct fsl_merge_state { + /** + State to be passed to this->callback via the + fsl_merge_state::callbackState member. + */ + void * callbackState; + + fsl_ckup_fchange_e fileChangeType; + + fsl_ckup_rm_state_e fileRmInfo; +}; +typedef struct fsl_merge_state fsl_merge_state; +/** + Callback type for use with fsl_merge_opt and fsl_merge(). + + TODO: figure out whether we can use fsl_ckout_state for this or + whether we need a new type. Merge and update are quite similar, so + the current thinking is that we can recycling this. +*/ +typedef int (*fsl_merge_f)(fsl_merge_state const * const); + +/** + Merge type enum for use with fsl_merge_opt::mergeType. +*/ +enum fsl_merge_type_e { +/** + Indicates a normal merge. +*/ +FSL_MERGE_TYPE_NORMAL, +/** + Indicates an "integrate" merge, which tells the next checkin + operation to apply a "closed" tag to the checkin from which this + merge is performed (effectively closing its branch). + + Certain merge-time state will force this merge type to behave like + FSL_MERGE_TYPE_NORMAL: + + - If the being-merged-in content is marked as private. + - If the being-merged-in content is not a leaf. +*/ +FSL_MERGE_TYPE_INTEGRATE, +/** + Indicates a cherrypick merge, pulling in only the changes made to a + specific checkin without otherwise inheriting its lineage. +*/ +FSL_MERGE_TYPE_CHERRYPICK, +/** + Indicates a backout merge, a reverse cherrypick, backing out any + changes which were added by this->checkinRid. +*/ +FSL_MERGE_TYPE_BACKOUT +}; +typedef enum fsl_merge_type_e fsl_merge_type_e; + +/** + UNDER CONSTRUCTION. + + Options for use with fsl_ckout_merge(). +*/ +struct fsl_merge_opt { + /** + The version of the repostitory to merge into the current + checkout. This must be the blob.rid of a checkin artifact. + */ + fsl_id_t mergeRid; + /** + The version of the most recent common ancestor. Must normally be + 0. The default is calculated automatically based on + this->mergeRid and this->mergeType. + + This corresponds to fossil(1)'s --baseline merge flag. + */ + fsl_id_t baselineRid; + /** + Specifies the merge type to perform. + */ + fsl_merge_type_e mergeType; + /** + Gets called once per merge-updated file, passed a fsl_ckup_state + instance with information about the merged file and related + metadata. May be NULL. + */ + fsl_merge_f callback; + /** + State to be passed to this->callback via the + fsl_merge_state::callbackState member. + */ + void * callbackState; + /** + A hint to fsl_merge() about whether it needs to scan the checkout + for changes. Set this to false ONLY if the calling code calls + fsl_ckout_changes_scan() (or equivalent, + e.g. fsl_vfile_changes_scan()) immediately before calling + fsl_merge(), as that function require a non-stale changes scan in + order to function properly. + */ + bool scanForChanges; + /** + If true, on merge conflict retain the temporary files used for + mergin: `*-baseline`, `*-original`, and `*-merge`. By default + these are removed because they're very rarely useful. + */ + bool keepMergeFiles; + /** + If true, the extraction process will "go through the motions" but + will not write any files to disk. It will perform I/O such as + stat()'ing to see, e.g., if it would have needed to overwrite a + file. + */ + bool dryRun; + + /** + This flag is not part of the public API and will be removed + once the merge operation's development has settled down. + */ + unsigned short debug; + /** + TODO: + + - How to handle fossil's --binary GLOBPATTERN flag. Plain string + or a glob list object or a stateful predicate function or... ? + */ +}; +/** Convenience typedef. */ +typedef struct fsl_merge_opt fsl_merge_opt; +/** Initialized-with-defaults fsl_merge_opt structure, intended for + const-copy initialization. */ +#define fsl_merge_opt_empty_m \ + {-1/*mergeRid*/,0/*baselineRid*/, \ + FSL_MERGE_TYPE_NORMAL/*mergeType*/, \ + NULL/*callback*/, NULL/*callbackState*/, \ + true/*scanForChanges*/, false/*keepMergeFiles*/, \ + false/*dryRun*/,0/*debug*/} +/** Initialized-with-defaults fsl_merge_opt structure, intended for + non-const copy initialization. */ +extern const fsl_merge_opt fsl_merge_opt_empty; + +/** + UNDER CONSTRUCTION and NOT YET IMPLEMENTED for the foreseeable + future. + + Performs a "merge" operation on the current checkout, merging in + version opt->mergeRid. If that version has already been merged, + this call has no SCM-related side effects. + + Returns 0 on success, any number of non-0 codes on error, + including, but not limited to: + + - FSL_RC_NOT_A_CKOUT if f has no opened checkout. + + - FSL_RC_OOM on allocation error. + + - FSL_RC_TYPE if opt->mergeRid does to refer to a checkin. + + - FSL_RC_PHANTOM if a file participating in the merge is + a phantom. + + - FSL_RC_RANGE if the to-be-merged-in RID is the same as the + current checkout RID or the same as the pivot/baseline of + the merge. + + - FSL_RC_NOT_FOUND if no common ancestor can be found for use as a + basis for the merge. + + - FSL_RC_MISUSE if the current checkout is empty (has an RID of 0). + + - Any number of DB- or I/O-related codes, as well as codes from + underlying APIs such as fsl_vfile_changes_scan(). + + For most errors, f's error state will be updated with a description + of the problem. + + Reminder: there are certain illegal combinations of merge state + which may require adding new result codes for. e.g. a no-op merge. +*/ +FSL_EXPORT int fsl_ckout_merge(fsl_cx * const f, fsl_merge_opt const * const opt); + +#if defined(__cplusplus) +} /*extern "C"*/ +#endif +#endif +/* ORG_FOSSIL_SCM_FSL_CHECKOUT_H_INCLUDED */ +/* end of file ../include/fossil-scm/fossil-checkout.h */ +/* start of file ../include/fossil-scm/fossil-confdb.h */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#if !defined(ORG_FOSSIL_SCM_FSL_CONFDB_H_INCLUDED) +#define ORG_FOSSIL_SCM_FSL_CONFDB_H_INCLUDED +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). +*/ +/** @file fossil-confdb.h + + fossil-confdb.h declares APIs dealing with fossil's various + configuration option storage backends. +*/ + + +#if defined(__cplusplus) +extern "C" { +#endif + +/** + A flag type for specifying which configuration backend a given API + should be applied to. Used by most of the fsl_config_XXX() APIS, + e.g. fsl_config_get_int32() and friends. + + This type is named fsl_confdb_e (not the "db" part) because 3 of + the 4 fossil-supported config storage backends are databases. The + "versioned setting" backend, which deals with non-db local files, + was encapsulated into this API long after the db-related options + were +*/ +enum fsl_confdb_e { +/** + Signfies the global-level (per system user) configuration area. +*/ +FSL_CONFDB_GLOBAL = 1, +/** + Signfies the repository-level configuration area. +*/ +FSL_CONFDB_REPO = 2, +/** + Signfies the checkout-level (a.k.a. "local") configuration area. +*/ +FSL_CONFDB_CKOUT = 3, + +/** + The obligatory special case... + + Versionable settings are stored directly in SCM-controlled files, + each of which has the same name as the setting and lives in the + .fossil-settings directory of a checkout. Though versionable + settings _can_ be read from a non-checked-out repository, doing so + requires knowning which version to fetch and is horribly + inefficient, so there are currently no APIs for doing so. + + Note that the APIs which read and write versioned settings do not + care whether those settings are valid for fossil(1). + + Reminder to self: SQL to find the checkins in which a versioned + settings file was added or modified (ignoring renames and branches + and whatnot). + + ``` +select strftime('%Y-%m-%d %H:%M:%S',e.mtime) mtime, +b.rid, b.uuid from mlink m, filename f, blob b, event e +where f.fnid=m.fnid +and m.mid=b.rid +and b.rid=e.objid +and f.name='.fossil-settings/ignore-glob' +order by e.mtime desc +; + ``` + +*/ +FSL_CONFDB_VERSIONABLE = 4 +}; +typedef enum fsl_confdb_e fsl_confdb_e; + +/** + Returns the name of the db table associated with the given + mode. Results are undefined if mode is an invalid value. The + returned bytes are static and constant. + + Returns NULL for the role FSL_CONFDB_VERSIONABLE. +*/ +FSL_EXPORT char const * fsl_config_table_for_role(fsl_confdb_e mode); + +/** + Returns a handle to the db associates with the given fsl_confdb_e + value. Returns NULL if !f or if f has no db opened for that + configuration role. Results are undefined if mode is an invalid + value. + + For FSL_CONFDB_VERSIONABLE it returns the results of fsl_cx_db(), + even though there is no database-side support for versionable files + (which live in files in a checkout). +*/ +FSL_EXPORT fsl_db * fsl_config_for_role(fsl_cx * f, fsl_confdb_e mode); + +/** + Returns the int32 value of a property from one of f's config + dbs, as specified by the mode parameter. Returns dflt if !f, f + does not have the requested config db opened, no entry is found, + or on db-level errors. +*/ +FSL_EXPORT int32_t fsl_config_get_int32( fsl_cx * f, fsl_confdb_e mode, + int32_t dflt, char const * key ); +/** + int64_t counterpart of fsl_config_get_int32(). +*/ +FSL_EXPORT int64_t fsl_config_get_int64( fsl_cx * f, fsl_confdb_e mode, + int64_t dflt, char const * key ); + +/** + fsl_id_t counterpart of fsl_config_get_int32(). +*/ +FSL_EXPORT fsl_id_t fsl_config_get_id( fsl_cx * f, fsl_confdb_e mode, + fsl_id_t dflt, char const * key ); +/** + double counterpart of fsl_config_get_int32(). +*/ +FSL_EXPORT double fsl_config_get_double( fsl_cx * f, fsl_confdb_e mode, + double dflt, char const * key ); + + +/** + Boolean countertpart of fsl_config_get_int32(). + + fsl_str_bool() is used to determine the booleanness (booleanity?) + of a given config option. +*/ +FSL_EXPORT bool fsl_config_get_bool( fsl_cx * f, fsl_confdb_e mode, + bool dflt, char const * key ); + +/** + A convenience form of fsl_config_get_buffer(). If it finds a + config entry it returns its value. If *len is not NULL then *len is + assigned the length of the returned string, in bytes (and is set to + 0 if NULL is returned). Any errors encounters while looking for + the entry are suppressed and NULL is returned. + + The returned memory must eventually be freed using fsl_free(). If + len is not NULL then it is set to the length of the returned + string. Returns NULL for any sort of error or for a NULL db value. + + If capturing error state is important for a given use case, use + fsl_config_get_buffer() instead, which provides the same features + as this one but propagates any error state. +*/ +FSL_EXPORT char * fsl_config_get_text( fsl_cx * f, fsl_confdb_e mode, + char const * key, + fsl_size_t * len ); + +/** + The fsl_buffer-type counterpart of fsl_config_get_int32(). + + Replaces the contents of the given buffer (re-using any memory it + might have) with a value from a config region. Returns 0 on + success, FSL_RC_NOT_FOUND if no entry was found or the requested db + is not opened, FSL_RC_OOM on allocation errror. + + If mode is FSL_CONFDB_VERSIONABLE, this operation requires + a checkout and returns (if possible) the contents of the file + named {CHECKOUT_ROOT}/.fossil-settings/{key}. + + On any sort of error, including the inability to find or open + a versionable-settings file, non-0 is returned: + + - FSL_RC_OOM on allocation error + + - FSL_RC_NOT_FOUND if either the database referred to by mode is + not opened or a matching setting cannot be found. + + - FSL_RC_ACCESS is a versionable setting file is found but cannot + be opened for reading. + + - FSL_RC_NOT_A_CKOUT if mode is FSL_CONFDB_VERSIONABLE and no + checkout is opened. + + - Potentially one of several db-related codes if reading a + non-versioned setting fails. + + In the grand scheme of things, the inability to load a setting is + not generally an error, so clients are not expected to treat it as + fatal unless perhaps it returns FSL_RC_OOM, in which case it likely + is. + + @see fsl_config_has_versionable() +*/ +FSL_EXPORT int fsl_config_get_buffer( fsl_cx * f, fsl_confdb_e mode, + char const * key, fsl_buffer * b ); + + +/** + If f has an opened checkout, this replaces b's contents (re-using + any existing memory) with an absolute path to the filename for that + setting: {CHECKOUT_ROOT}/.fossil-settings/{key} + + This routine neither verifies that the given key is a valid + versionable setting name nor that the file exists: it's purely a + string operation. + + Returns 0 on success, FSL_RC_NOT_A_CKOUT if f has no checkout + opened, FSL_RC_MISUSE if key is NULL, empty, or if + fsl_is_simple_pathname() returns false for the key, and FSL_RC_OOM + if appending to the buffer fails. + */ +FSL_EXPORT int fsl_config_versionable_filename(fsl_cx *f, char const * key, fsl_buffer *b); + +/** + Sets a configuration variable in one of f's config databases, as + specified by the mode parameter. Returns 0 on success. val may + be NULL. Returns FSL_RC_MISUSE if !f, f does not have that + database opened, or !key, FSL_RC_RANGE if !key. + + If mode is FSL_CONFDB_VERSIONABLE, it attempts to write/overwrite + the corresponding file in the current checkout. + + If mem is NULL and mode is not FSL_CONFDB_VERSIONABLE then an SQL + NULL is bound instead of an empty blob. For FSL_CONFDB_VERSIONABLE + an empty file will be written for that case. + + If mode is FSL_CONFDB_VERSIONABLE, this does NOT queue any + newly-created versionable setting file for inclusion into the + SCM. That is up to the caller. See + fsl_config_versionable_filename() for info about an additional + potential usage error case with FSL_CONFDB_VERSIONABLE. + + Pedantic side-note: the input text is saved as-is. No trailing + newline is added when saving to FSL_CONFDB_VERSIONABLE because + doing so would require making a copy of the input bytes just to add + a newline to it. The non-text fsl_config_set_XXX() APIs add a + newline when writing to their values out to a versionable config + file because it costs them nothing to do so and text files "should" + have a trailing newline. + + Potential TODO: if mode is FSL_CONFDB_VERSIONABLE and the key + contains directory components, e,g, "app/x", we should arguably use + fsl_mkdir_for_file() to create those components. As of this writing + (2021-03-14), no such config keys have ever been used in fossil. + + @see fsl_config_versionable_filename() +*/ +FSL_EXPORT int fsl_config_set_text( fsl_cx * f, fsl_confdb_e mode, char const * key, char const * val ); + +/** + The blob counterpart of fsl_config_set_text(). If len is + negative then fsl_strlen(mem) is used to determine the length of + the memory. + + If mem is NULL and mode is not FSL_CONFDB_VERSIONABLE then an SQL + NULL is bound instead of an empty blob. For FSL_CONFDB_VERSIONABLE + an empty file will be written for that case. +*/ +FSL_EXPORT int fsl_config_set_blob( fsl_cx * f, fsl_confdb_e mode, char const * key, + void const * mem, fsl_int_t len ); +/** + int32 counterpart of fsl_config_set_text(). +*/ +FSL_EXPORT int fsl_config_set_int32( fsl_cx * f, fsl_confdb_e mode, + char const * key, int32_t val ); +/** + int64 counterpart of fsl_config_set_text(). +*/ +FSL_EXPORT int fsl_config_set_int64( fsl_cx * f, fsl_confdb_e mode, + char const * key, int64_t val ); +/** + fsl_id_t counterpart of fsl_config_set_text(). +*/ +FSL_EXPORT int fsl_config_set_id( fsl_cx * f, fsl_confdb_e mode, + char const * key, fsl_id_t val ); +/** + fsl_double counterpart of fsl_config_set_text(). +*/ +FSL_EXPORT int fsl_config_set_double( fsl_cx * f, fsl_confdb_e mode, + char const * key, double val ); +/** + Boolean counterpart of fsl_config_set_text(). + + For compatibility with fossil conventions, the value will be saved + in the string form "on" or "off". When mode is + FSL_CONFDB_VERSIONABLE, that value will include a trailing newline. +*/ +FSL_EXPORT int fsl_config_set_bool( fsl_cx * f, fsl_confdb_e mode, + char const * key, bool val ); + +/** + "Unsets" (removes) the given key from the given configuration database. + It is not considered to be an error if the config table does not + contain that key. + + Returns FSL_RC_UNSUPPORTED, without side effects, if mode is + FSL_CONFDB_VERSIONABLE. It "could" hypothetically remove a + checked-out copy of a versioned setting, then queue the file for + removal in the next checkin, but it does not do so. It might, in + the future, be changed to do so, or at least to remove the local + settings file. +*/ +FSL_EXPORT int fsl_config_unset( fsl_cx * f, fsl_confdb_e mode, + char const * key ); + +/** + Begins (or recurses) a transaction on the given configuration + database. Returns 0 on success, non-0 on error. On success, + fsl_config_transaction_end() must eventually be called with the + same parameters to pop the transaction stack. Returns + FSL_RC_MISUSE if no db handle is opened for the given + configuration mode. Assuming all arguments are valid, this + returns the result of fsl_db_transaction_end() and propagates + any db-side error into the f object's error state. + + This is primarily intended as an optimization when an app is + making many changes to a config database. It is not needed when + the app is only making one or two changes. + + @see fsl_config_transaction_end() + @see fsl_db_transaction_begin() +*/ +FSL_EXPORT int fsl_config_transaction_begin(fsl_cx * f, fsl_confdb_e mode); + +/** + Pops the transaction stack pushed by + fsl_config_transaction_begin(). If rollback is true then the + transaction is set roll back, otherwise it is allowed to + continue (if recursive) or committed immediately (if not + recursive). Returns 0 on success, non-0 on error. Returns + FSL_RC_MISUSE if no db handle is opened for the given + configuration mode. Assuming all arguments are valid, this + returns the result of fsl_db_transaction_end() and propagates + any db-side error into the f object's error state. + + @see fsl_config_transaction_begin() + @see fsl_db_transaction_end() +*/ +FSL_EXPORT int fsl_config_transaction_end(fsl_cx * f, fsl_confdb_e mode, char rollback); + +/** + Populates li as a glob list from the given configuration key. + Uses (versionable/repo/global) config settings, in that order. + It is not an error if one or more of those sources is missing - + they are simply skipped. + + Note that gets any new globs appended to it, as per + fsl_glob_list_append(), as opposed to replacing any existing + contents. + + Returns 0 on success, but that only means that there were no + errors, not that any entries were necessarily added to li. + + Arguably a bug: this function does not open the global config if + it was not already opened, but will use it if it is opened. This + function should arbuably open and close it in that case. +*/ +FSL_EXPORT int fsl_config_globs_load(fsl_cx * f, fsl_list * li, char const * key); + +/** + Fetches the preferred name of the "global" db file for the current + user by assigning it to *zOut. Returns 0 on success, in which case + *zOut is updated and non-0 on error, in which case *zOut is not + modified. On success, ownership of *zOut is transferred to the + caller, who must eventually free it using fsl_free(). + + The locations searched for the database file are + platform-dependent... + + Unix-like systems are searched in the following order: + + 1) If the FOSSIL_HOME environment var is set, use + $FOSSIL_HOME/.fossil. + + 2) If $HOME/.fossil already exists, use that. + + 3) If XDG_CONFIG_HOME environment var is set, use + $XDG_CONFIG_HOME/fossil.db. + + 4) If $HOME/.config is a directory, use $HOME/.config/fossil.db + + 5) Fall back to $HOME/.fossil (historical name). + + Except where listed above, this function does not check whether the + file already exists or is a database. + + Windows: + + - We need a Windows port of this routine. Currently it simply uses + the Windows home directory + "/_fossil" or "/.fossil", depending on + the build-time environment. +*/ +FSL_EXPORT int fsl_config_global_preferred_name(char ** zOut); + +#if defined(__cplusplus) +} /*extern "C"*/ +#endif +#endif +/* ORG_FOSSIL_SCM_FSL_CONFDB_H_INCLUDED */ +/* end of file ../include/fossil-scm/fossil-confdb.h */ +/* start of file ../include/fossil-scm/fossil-vpath.h */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#if !defined(ORG_FOSSIL_SCM_FSL_VPATH_H_INCLUDED) +#define ORG_FOSSIL_SCM_FSL_VPATH_H_INCLUDED +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). + + ***************************************************************************** + This file declares public APIs relating to calculating paths via + Fossil SCM version history. +*/ + + +#if defined(__cplusplus) +extern "C" { +#endif + +typedef struct fsl_vpath_node fsl_vpath_node; +typedef struct fsl_vpath fsl_vpath; + +/** + Holds information for a single node in a path of checkin versions. + + @see fsl_vpath +*/ +struct fsl_vpath_node { + /** ID for this node */ + fsl_id_t rid; + /** True if pFrom is the parent of rid */ + bool fromIsParent; + /** True if primary side of common ancestor */ + bool isPrimary; + /* HISTORICAL: Abbreviate output in "fossil bisect ls" */ + bool isHidden; + /** Node this one came from. */ + fsl_vpath_node *pFrom; + union { + /** List of nodes of the same generation */ + fsl_vpath_node *pPeer; + /** Next on path from beginning to end */ + fsl_vpath_node *pTo; + } u; + /** List of all nodes */ + fsl_vpath_node *pAll; +}; + +/** + A utility type for collecting "paths" between two checkin versions. +*/ +struct fsl_vpath{ + /** Current generation of nodes */ + fsl_vpath_node *pCurrent; + /** All nodes */ + fsl_vpath_node *pAll; + /** Nodes seen before */ + fsl_id_bag seen; + /** Number of steps from first to last. */ + int nStep; + /** Earliest node in the path. */ + fsl_vpath_node *pStart; + /** Common ancestor of pStart and pEnd */ + fsl_vpath_node *pPivot; + /** Most recent node in the path. */ + fsl_vpath_node *pEnd; +}; + +/** + An empty-initialize fsl_vpath object, intended for const-copy + initialization. +*/ +#define fsl_vpath_empty_m {0,0,fsl_id_bag_empty_m,0,0,0,0} + +/** + An empty-initialize fsl_vpath object, intended for copy + initialization. +*/ +FSL_EXPORT const fsl_vpath fsl_vpath_empty; + + +/** + Returns the first node in p's path. + + The returned node is owned by path may be invalidated by any APIs + which manipulate path. +*/ +FSL_EXPORT fsl_vpath_node * fsl_vpath_first(fsl_vpath *p); + +/** + Returns the last node in p's path. + + The returned node is owned by path may be invalidated by any APIs + which manipulate path. +*/ +FSL_EXPORT fsl_vpath_node * fsl_vpath_last(fsl_vpath *p); + +/** + Returns the next node p's path. + + The returned node is owned by path may be invalidated by any APIs + which manipulate path. + + Intended to be used like this: + + ``` + for( p = fsl_vpath_first(path) ; + p ; + p = fsl_vpath_next(p)){ + ... + } + ``` +*/ +FSL_EXPORT fsl_vpath_node * fsl_vpath_next(fsl_vpath_node *p); + +/** + Returns p's path length. +*/ +FSL_EXPORT int fsl_vpath_length(fsl_vpath const * p); + +/** + Frees all nodes in path (which must not be NULL) and resets all + state in path. Does not free path. +*/ +FSL_EXPORT void fsl_vpath_clear(fsl_vpath *path); + +/** + Find the mid-point of the path. If the path contains fewer than 2 + steps, returns 0. The returned node is owned by path and may be + invalidated by any APIs which manipulate path. +*/ +FSL_EXPORT fsl_vpath_node * fsl_vpath_midpoint(fsl_vpath * path); + + +/** + Computes the shortest path from checkin versions iFrom to iTo + (inclusive), storing the result state in path. If path has + state before this is called, it is cleared by this call. + + iFrom and iTo must both be valid checkin version RIDs. + + If directOnly is true, then use only the "primary" links from + parent to child. In other words, ignore merges. + + On success, returns 0 and path->pStart will point to the + beginning of the path (the iFrom node). If pStart is 0 then no + path could be found but 0 is still returned. + + Elements of the path can be traversed like so: + + ``` + fsl_vpath path = fsl_vpath_empty; + fsl_vpath_node * n = 0; + int rc = fsl_vpath_shortest(f, &path, versionFrom, versionTo, 1, 0); + if(rc) { ... error ... } + for( n = fsl_vpath_first(&path); n; n = fsl_vpath_next(n) ){ + ... + } + fsl_vpath_clear(&path); + ``` + + On error, f's error state may be updated with a description of the + problem. +*/ +FSL_EXPORT int fsl_vpath_shortest( fsl_cx * const f, fsl_vpath * const path, + fsl_id_t iFrom, fsl_id_t iTo, + bool directOnly, bool oneWayOnly ); + +/** + This variant of fsl_vpath_shortest() stores the shortest direct + path from version iFrom to version iTo in the ANCESTOR temporary + table using f's current repo db handle. That table gets created, if + needed, else cleared by this call. + + The ANCESTOR temp table has the following interface: + + ``` + rid INT UNIQUE + generation INTEGER PRIMARY KEY + ``` + + Where [rid] is a checkin version RID and [generation] is the + 1-based number of steps from iFrom, including iFrom (so iFrom's own + generation is 1). + + On error returns 0 and, if pSteps is not NULL, assigns *pSteps to + the number of entries added to the ancestor table. On error, pSteps + is never modifed, any number of various non-0 codes may be + returned, and f's error state will, if possible (not an OOM), be + updated to describe the problem. + + This function's name is far too long and descriptive. We might want + to consider something shorter. + + Maintenance reminder: this impl swaps the 2nd and 3rd parameters + compared to fossil(1)'s version! +*/ +FSL_EXPORT int fsl_vpath_shortest_store_in_ancestor(fsl_cx * const f, + fsl_id_t iFrom, + fsl_id_t iTo, + uint32_t *pSteps); + +/** + Computes a list of direct (non-merge) ancestors of the given + checkin RID and stores it in the TEMP table [ancestor], which it + creates if needed or clears if it currently exists. + + The [ancestor] schema is described in + fsl_vpath_shortest_store_in_ancestor(). The [generation] value of + the record corresponding to rid is 1, increasing by 1 for each + generation back in the history. + + Returns 0 on success, FSL_RC_NOT_A_REPO if f has no repo db opened, + and any number of lower-level result codes if something goes wrong. +*/ +FSL_EXPORT int fsl_compute_direct_ancestors(fsl_cx * const f, fsl_id_t rid); + +/** + Reconstructs path from path->pStart to path->pEnd, reversing its + order by fiddling with the u->pTo fields. + + Unfortunately does not reverse after the initial creation/reversal + :/. +*/ +FSL_EXPORT void fsl_vpath_reverse(fsl_vpath * path); + + +#if defined(__cplusplus) +} /*extern "C"*/ +#endif +#endif +/* ORG_FOSSIL_SCM_FSL_VPATH_H_INCLUDED */ +/* end of file ../include/fossil-scm/fossil-vpath.h */ +/* start of file ../include/fossil-scm/fossil-internal.h */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#if !defined(ORG_FOSSIL_SCM_FSL_INTERNAL_H_INCLUDED) +#define ORG_FOSSIL_SCM_FSL_INTERNAL_H_INCLUDED +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). + + ***************************************************************************** + This file declares library-level internal APIs which are shared + across the library. +*/ + + +#if defined(__cplusplus) +extern "C" { +#endif + +typedef struct fsl_acache fsl_acache; +typedef struct fsl_acache_line fsl_acache_line; +typedef struct fsl_pq fsl_pq; +typedef struct fsl_pq_entry fsl_pq_entry; + +/** @internal + + Queue entry type for the fsl_pq class. + + Potential TODO: we don't currently use the (data) member. We can + probably remove it. +*/ +struct fsl_pq_entry { + /** RID of the entry. */ + fsl_id_t id; + /** Raw data associated with this entry. */ + void * data; + /** Priority of this element. */ + double priority; +}; +/** @internal + Empty-initialized fsl_pq_entry structure. +*/ +#define fsl_pq_entry_empty_m {0,NULL,0.0} + +/** @internal + + A simple priority queue class. Instances _must_ be initialized + by copying fsl_pq_empty or fsl_pq_empty_m (depending on where + the instance lives). +*/ +struct fsl_pq { + /** Number of items allocated in this->list. */ + uint16_t capacity; + /** Number of items used in this->list. */ + uint16_t used; + /** The queue. It is kept sorted by entry->priority. */ + fsl_pq_entry * list; +}; + +/** @internal + Empty-initialized fsl_pq struct, intended for const-copy initialization. +*/ +#define fsl_pq_empty_m {0,0,NULL} + +/** @internal + Empty-initialized fsl_pq struct, intended for copy initialization. +*/ +extern const fsl_pq fsl_pq_empty; + +/** @internal + + Clears the contents of p, freeing any memory it owns, but not + freeing p. Results are undefined if !p. +*/ +void fsl_pq_clear(fsl_pq * p); + +/** @internal + + Insert element e into the queue. Returns 0 on success, FSL_RC_OOM + on error. Results are undefined if !p. pData may be NULL. +*/ +int fsl_pq_insert(fsl_pq *p, fsl_id_t e, + double v, void *pData); + +/** @internal + + Extracts (removes) the first element from the queue (the element + with the smallest value) and return its ID. Return 0 if the queue + is empty. If pp is not NULL then *pp is (on success) assigned to + opaquedata pointer mapped to the entry. +*/ +fsl_id_t fsl_pq_extract(fsl_pq *p, void **pp); + +/** @internal + + Holds one "line" of a fsl_acache cache. +*/ +struct fsl_acache_line { + /** + RID of the cached record. + */ + fsl_id_t rid; + /** + Age. Newer is larger. + */ + fsl_int_t age; + /** + Content of the artifact. + */ + fsl_buffer content; +}; +/** @internal + + Empty-initialized fsl_acache_line structure. +*/ +#define fsl_acache_line_empty_m { 0,0,fsl_buffer_empty_m } + + +/** @internal + + A cache for tracking the existence of artifacts while the + internal goings-on of control artifacts are going on. + + Currently the artifact cache is unused because it costs much + more than it gives us. Once the library supports certain + operations (like rebuild and sync) caching will become more + useful. + + Historically fossil caches artifacts as their blob content, but + libfossil will likely (at some point) to instead cache fsl_deck + instances, which contain all of the same data in pre-parsed form. + It cost more memory, though. That approach also precludes caching + non-structural artifacts (i.e. opaque client blobs). + + Potential TODO: the limits of the cache size are hard-coded in + fsl_acache_insert. Those really should be part of this struct. +*/ +struct fsl_acache { + /** + Total amount of buffer memory (in bytes) used by cached content. + This does not account for memory held by this->list. + */ + fsl_size_t szTotal; + /** + Limit on the (approx.) amount of memory (in bytes) which can be + taken up by the cached buffers at one time. Fossil's historical + value is 50M. + */ + fsl_size_t szLimit; + /** + Number of entries "used" in this->list. + */ + uint16_t used; + /** + Approximate upper limit on the number of entries in this->list. + This limit may be violated slightly. + + This number should ideally be relatively small: 3 digits or less. + Fossil's historical value is 500. + */ + uint16_t usedLimit; + /** + Number of allocated slots in this->list. + */ + uint16_t capacity; + /** + Next cache counter age. Higher is newer. + */ + fsl_int_t nextAge; + /** + List of cached content, ordered by age. + */ + fsl_acache_line * list; + /** + All artifacts currently in the cache. + */ + fsl_id_bag inCache; + /** + Cache of known-missing content. + */ + fsl_id_bag missing; + /** + Cache of of known-existing content. + */ + fsl_id_bag available; +}; +/** @internal + + Empty-initialized fsl_acache structure, intended + for const-copy initialization. +*/ +#define fsl_acache_empty_m { \ + 0/*szTotal*/, \ + 20000000/*szLimit. Historical fossil value=50M*/, \ + 0/*used*/,300U/*usedLimit. Historical fossil value=500*/,\ + 0/*capacity*/, \ + 0/*nextAge*/,NULL/*list*/, \ + fsl_id_bag_empty_m/*inCache*/, \ + fsl_id_bag_empty_m/*missing*/, \ + fsl_id_bag_empty_m/*available*/ \ +} +/** @internal + + Empty-initialized fsl_acache structure, intended + for copy initialization. +*/ +extern const fsl_acache fsl_acache_empty; + +/** @internal + + Very internal. + + "Manifest cache" for fsl_deck entries encountered during + crosslinking. This type is intended only to be embedded in fsl_cx. + + The routines for managing this cache are static in deck.c: + fsl_cx_mcache_insert() and fsl_cx_mcache_search(). + + The array members in this struct MUST have the same length + or results are undefined. +*/ +struct fsl_mcache { + /** Next age value. No clue how the cache will react once this + overflows. */ + unsigned nextAge; + /** The virtual age of each deck in the cache. They get evicted + oldest first. */ + unsigned aAge[4]; + /** Counts the number of cache hits. */ + unsigned hits; + /** Counts the number of cache misses. */ + unsigned misses; + /** + Stores bitwise copies of decks. Storing a fsl_deck_malloc() deck + into the cache requires bitwise-copying is contents, wiping out + its contents via assignment from fsl_deck_empty, then + fsl_free()'ing it (as opposed to fsl_deck_finalize(), which would + attempt to clean up memory which now belongs to the cache's + copy). + + Array sizes of 6 and 10 do not appreciably change the hit rate + compared to 4, at least not for current (2021-03-26) uses. + */ + fsl_deck decks[4]; +}; + +/** Convenience typedef. */ +typedef struct fsl_mcache fsl_mcache; + +/** Initialized-with-defaults fsl_mcache structure, intended for + const-copy initialization. */ +#define fsl_mcache_empty_m {\ + 0, \ + {0,0,0,0},\ + 0,0, \ + {fsl_deck_empty_m,fsl_deck_empty_m,fsl_deck_empty_m, \ + fsl_deck_empty_m} \ +} + +/** Initialized-with-defaults fsl_mcache structure, intended for + non-const copy initialization. */ +extern const fsl_mcache fsl_mcache_empty; + + +/* The fsl_cx class is documented in main public header. */ +struct fsl_cx { + /** + A pointer to the "main" db handle. Exactly which db IS the + main db is, because we have three DBs, not generally knowble. + + As of this writing (20141027, updated 20211018) the following + applies: + + dbMain always points to &this->dbMem (a temp or ":memory:" + (unspecified!) db opened by fsl_cx_init()), and the + repo/ckout/config DBs get ATTACHed to that one. Their separate + handles (this->{repo,ckout,config}.db) are used to store the name + and file path to each one (even though they have no real db + handle associated with them). + + Internal code should rely as little as possible on the actual + arrangement of internal DB handles, and should use + fsl_cx_db_repo(), fsl_cx_db_ckout(), and fsl_cx_db_config() to + get a handle to the specific db they want. Currently they will + always return NULL or the same handle, but that design decision + might change at some point, so the public API treats them as + separate entities. _That said_: at this point (2021-10-18), + treating them as separate handles often proves to be annoying in + their usage, and newer code will sometimes use (e.g.) + fsl_cx_prepare() in lieu of explicitely using fsl_db_prepare() + with the ostensibly db-specific handle when it knows that the + required db is indeed attached. In other words: the internals, in + some places, are starting to rely on this long-established + convention of having a single sqlite3 object and multiple + attached databases, _and that's okay_. + */ + fsl_db * dbMain; + + /** + Marker which tells us whether fsl_cx_finalize() needs + to fsl_free() this instance or not. + */ + void const * allocStamp; + + /** + A ":memory:" (or "") db to work around + open-vs-attach-vs-main-vs-real-name problems wrt to the + repo/ckout/config dbs. This db handle gets opened automatically + at startup and all others which a fsl_cx manages get ATTACHed to + it. Thus the other internal fsl_db objects, e.g. this->repo.db, + may hold state, such as the path to the current repo db, but they + do NOT hold an sqlite3 db handle. Assigning them the handle of + this->dbMain would indeed simplify certain work but it would + require special care to ensure that we never sqlite3_close() + those out from under this->dbMain. + */ + fsl_db dbMem; + + /** + Holds info directly related to a checkout database. + */ + struct { + /** + Holds the filename of the current checkout db and possibly + related state. Historically (very historically) it could also + be the `main` db, but that is no longer the case. + */ + fsl_db db; + /** + The directory part of an opened checkout db. This is currently + only set by fsl_ckout_open_dir(). It contains a trailing slash, + largely because that simplifies porting fossil(1) code and + appending filenames to this directory name to create absolute + paths (a frequently-needed option). + + Useful for doing absolute-to-relative path conversions for + checking file lists. + */ + char * dir; + /** + Optimization: fsl_strlen() of dir. Guaranteed to be set to + dir's length if dir is not NULL. + */ + fsl_size_t dirLen; + /** + The rid of the current checkout. May be 0 for an empty + repo/checkout. Must be negative if not yet known. + */ + fsl_id_t rid; + /** + The UUID of the current checkout. Only set if this->rid is + positive. Owned by the containing fsl_cx object. + */ + fsl_uuid_str uuid; + /** + Julian mtime of the checkout version, as reported by the + [event] table. + */ + double mtime; + } ckout; + + /** + Holds info directly related to a repo database. + */ + struct { + /** + Holds the filename of the current repo db and possibly related + state. Historically (very historically) it could also be the + `main` db, but that is no longer the case. + */ + fsl_db db; + /** + The default user name, for operations which need one. + See fsl_cx_user_set(). + */ + char * user; + } repo; + + /** + Holds info directly related to a global config database. + */ + struct { + /** + Holds the filename of the current global config db and possibly + related state. Historically (very historically) it could also + be the `main` db, but that is no longer the case. + */ + fsl_db db; + } config; + + /** + State for incrementally proparing a checkin operation. + */ + struct { + /** + Holds a list of "selected files" in the form + of vfile.id values. + */ + fsl_id_bag selectedIds; + + /** + The deck used for incrementally building certain parts of a + checkin. + */ + fsl_deck mf; + } ckin; + + /** + Confirmation callback. Used by routines which may have to + interactively ask a user to confirm things. + */ + fsl_confirmer confirmer; + + /** + Output channel used by fsl_output() and friends. + + This was added primarily so that fossil client apps can share a + single output channel which the user can swap out, e.g. to direct + all output to a UI widget or a file. + */ + fsl_outputer output; + + /** + Can be used to tie client-specific data to the context. Its + finalizer is called when fsl_cx_finalize() cleans up. The library + does not use this state. It is intended primarily for tying, + e.g., scripting-engine information to the context, e.g. mapping a + scripting engine context to this one for later use in fossil-side + callbacks. + */ + fsl_state clientState; + + /** + Holds error state. As a general rule, this information is updated + only by routines which need to return more info than a simple + integer error code. e.g. this is often used to hold + db-driver-provided error state. It is not used by "simple" + routines for which an integer code always suffices. APIs which + set this should denote it with a comment like "updates the + context's error state on error." + */ + fsl_error error; + + /** + A place for temporarily holding file content. We use this in + places where we have to loop over files and read their entire + contents, so that we can reuse this buffer's memory if possible. + The loop and the reading might be happening in different + functions, though, and some care must be taken to avoid use in + two functions concurrently. + */ + fsl_buffer fileContent; + + /** + Reuseable scratchpads for low-level file canonicalization + buffering and whatnot. Not intended for huge content: use + this->fileContent for that. This list should stay relatively + short. + + @see fsl_cx_scratchpad() + @see fsl_cx_scratchpad_yield() + */ + struct { + /** + Strictly-internal temporary buffers we intend to reuse many + times, mostly for filename canonicalization, holding hash + values, and small encoding/decoding tasks. These must never be + used for values which will be long-lived, nor are they intended + to be used for large content, e.g. reading files, with the + possible exception of holding versioned config settings, as + those are typically rather small. + + If needed, the lengths of this->buf[] and this->used[] may be + extended, but anything beyond 8, maybe 10, seems a bit extreme. + They should only be increased if we find code paths which + require it. As of this writing (2021-03-17), the peak + concurrently used was 5. In any case fsl_cx_scratchpad() fails + fatally if it needs more than it has, so we won't fail to + overlook such a case. + */ + fsl_buffer buf[8]; + /** + Flags telling us which of this->buf is currenly in use. + */ + bool used[8]; + /** + A cursor _hint_ to try to speed up fsl_cx_scratchpad() by about + half a nanosecond, making it O(1) instead of O(small N) for the + common case. + */ + short next; + } scratchpads; + + /** + A bitwise copy of the config object passed to fsl_cx_init() (or + some default). + */ + fsl_cx_config cxConfig; + + /** + Flags, some (or one) of which is runtime-configurable by the + client (see fsl_cx_flags_e). We can get rid of this and add the + flags to the cache member along with the rest of them. + */ + int flags; + + /** + List of callbacks for deck crosslinking purposes. + */ + fsl_xlinker_list xlinkers; + + /** + A place for caching generic things. + */ + struct { + /** + If true, SOME repository-level file-name comparisons/searches + will work case-insensitively. + */ + bool caseInsensitive; + + /** + If true, skip "dephantomization" of phantom blobs. This is a + detail from fossil(1) with as-yet-undetermined utility. It's + apparently only used during the remote-sync process, which this + API does not (as of 2021-10) yet have. + */ + bool ignoreDephantomizations; + + /** + Whether or not a running commit process should be marked as + private. This flag is used for communicating this flag through + multiple levels of API. + */ + bool markPrivate; + + /** + True if fsl_crosslink_begin() has been called but + fsl_crosslink_end() is still pending. + */ + bool isCrosslinking; + + /** + Flag indicating that only cluster control artifacts should be + processed by manifest crosslinking. This will only be relevant + if/when the sync protocol is implemented. + */ + bool xlinkClustersOnly; + + /** + Is used to tell the content-save internals that a "final + verification" (a.k.a. verify-before-commit) is underway. + */ + bool inFinalVerify; + + /** + Cached copy of the allow-symlinks config option, because it is + (hypothetically) needed on many stat() call. Negative + value=="not yet determined", 0==no, positive==yes. The negative + value means we need to check the repo config resp. the global + config to see if this is on. + + As of late 2020, fossil(1) is much more restrictive with + symlinks due to vulnerabilities which were discovered by a + security researcher, and we definitely must not default any + symlink-related features to enabled/on. As of Feb. 2021, my + personal preference, and very likely plan of attack, is to only + treat SCM'd symlinks as if symlinks support is disabled. It's + very unlikely that i will implement "real" symlink support but + would, *solely* for compatibility with fossil(1), be convinced + to permit such changes if someone else wants to implement them. + Patches are joyfully considered! + */ + short allowSymlinks; + + /** + Indicates whether or not this repo has ever seen a delta + manifest. If none has ever been seen then the repository will + prefer to use baseline (non-delta) manifests. Once a delta is + seen in the repository, the checkin algorithm is free to choose + deltas later on unless its otherwise prohibited, e.g. by the + forbid-delta-manifests config db setting. + + This article provides an overview to the topic delta manifests + and essentially debunks their ostensible benefits: + + https://fossil-scm.org/home/doc/tip/www/delta-manifests.md + + Values: negative==undetermined, 0==no, positive==yes. This is + updated when a repository is first opened and when new content + is written to it. + */ + short seenDeltaManifest; + + /** + Records whether this repository has an FTS search + index. <0=undetermined, 0=no, >0=yes. + */ + short searchIndexExists; + + /** + Cache for the "manifest" config setting, as used by + fsl_ckout_manifest_setting(), with the caveat that + if the setting changes after it is cached, we won't necessarily + see that here! + */ + short manifestSetting; + + /** + Record ID of rcvfrom entry during commits. This is likely to + remain unused in libf until/unless the sync protocol is + implemented. + */ + fsl_id_t rcvId; + + /** + Artifact cache used during processing of manifests. + */ + fsl_acache arty; + /** + Used during manifest parsing to keep track of artifacts we have + seen. Whether that's really necessary or is now an unnecessary + porting artifact (haha) is unclear. + */ + fsl_id_bag mfSeen; + /** + Used during the processing of manifests to keep track of + "leaf checks" which need to be done downstream. + */ + fsl_id_bag leafCheck; + /** + Holds the RID of every record awaiting verification + during the verify-at-commit checks. + */ + fsl_id_bag toVerify; + /** + Infrastructure for fsl_mtime_of_manifest_file(). It + remembers the previous RID so that it knows when it has to + invalidate/rebuild its ancestry cache. + */ + fsl_id_t mtimeManifest; + /** + The "project-code" config option. We do not currently (2021-03) + use this but it will be important if/when the sync protocol is + implemented or we want to create hashes, e.g. for user + passwords, which depend in part on the project code. + */ + char * projectCode; + + /** + Internal optimization to avoid duplicate stat() calls across + two functions in some cases. + */ + fsl_fstat fstat; + + /** + Parsed-deck cache. + */ + fsl_mcache mcache; + + /** + Holds various glob lists. That said... these features are + actually app-level stuff which the library itself does not + resp. should not enforce. We can keep track of these for users + but the library internals _generally_ have no business using + them. + + _THAT_ said... these have enough generic utility that we can + justify storing them and _optionally_ applying them. See + fsl_checkin_opt for an example of where we do this. + */ + struct { + /** + Holds the "ignore-glob" globs. + */ + fsl_list ignore; + /** + Holds the "binary-glob" globs. + */ + fsl_list binary; + /** + Holds the "crnl-glob" globs. + */ + fsl_list crnl; + } globs; + } cache; + + /** + Ticket-related information. + */ + struct { + /** + Holds a list of (fsl_card_J*) records representing custom + ticket table fields available in the db. + + Each entry's flags member denote (using fsl_card_J_flags) + whether that field is used by the ticket or ticketchng + tables. + + TODO, eventually: add a separate type for these entries. We + use fsl_card_J because the infrastructure is there and they + provide what we need, but fsl_card_J::flags only exists for + this list. A custom type would be smaller than fsl_card_J + (only two members) but adding it requires adding some + infrastructure which isn't worth the effort at the moment. + */ + fsl_list customFields; + + /** + Gets set to true (at some point) if the client has the + ticket db table. + */ + bool hasTicket; + /** + Gets set to true (at some point) if the client has the + ticket.tkt_ctime db field. + */ + bool hasCTime; + + /** + Gets set to true (at some point) if the client has the + ticketchng db table. + */ + bool hasChng; + /** + Gets set to true (at some point) if the client has the + ticketchng.rid db field. + */ + bool hasChngRid; + } ticket; + + /* + Note: no state related to server/user/etc. That is higher-level + stuff. We might need to allow the user to set a default user + name to avoid that he has to explicitly set it on all of the + various Control Artifact-generation bits which need it. + */ +}; + +/** @internal + + Initialized-with-defaults fsl_cx struct. +*/ +#define fsl_cx_empty_m { \ + NULL /*dbMain*/, \ + NULL/*allocStamp*/, \ + fsl_db_empty_m /* dbMem */, \ + {/*ckout*/ \ + fsl_db_empty_m /*db*/, \ + NULL /*dir*/, 0/*dirLen*/, \ + -1/*rid*/, NULL/*uuid*/, 0/*mtime*/ \ + }, \ + {/*repo*/ fsl_db_empty_m /*db*/, \ + 0/*user*/ \ + }, \ + {/*config*/ fsl_db_empty_m /*db*/ }, \ + {/*ckin*/ \ + fsl_id_bag_empty_m/*selectedIds*/, \ + fsl_deck_empty_m/*mf*/ \ + }, \ + fsl_confirmer_empty_m/*confirmer*/, \ + fsl_outputer_FILE_m /*output*/, \ + fsl_state_empty_m /*clientState*/, \ + fsl_error_empty_m /*error*/, \ + fsl_buffer_empty_m /*fileContent*/, \ + {/*scratchpads*/ \ + {fsl_buffer_empty_m,fsl_buffer_empty_m, \ + fsl_buffer_empty_m,fsl_buffer_empty_m, \ + fsl_buffer_empty_m,fsl_buffer_empty_m}, \ + {false,false,false,false,false,false}, \ + 0/*next*/ \ + }, \ + fsl_cx_config_empty_m /*cxConfig*/, \ + FSL_CX_F_DEFAULTS/*flags*/, \ + fsl_xlinker_list_empty_m/*xlinkers*/, \ + {/*cache*/ \ + false/*caseInsensitive*/, \ + false/*ignoreDephantomizations*/, \ + false/*markPrivate*/, \ + false/*isCrosslinking*/, \ + false/*xlinkClustersOnly*/, \ + false/*inFinalVerify*/, \ + -1/*allowSymlinks*/, \ + -1/*seenDeltaManifest*/, \ + -1/*searchIndexExists*/, \ + -1/*manifestSetting*/,\ + 0/*rcvId*/, \ + fsl_acache_empty_m/*arty*/, \ + fsl_id_bag_empty_m/*mfSeen*/, \ + fsl_id_bag_empty_m/*leafCheck*/, \ + fsl_id_bag_empty_m/*toVerify*/, \ + 0/*mtimeManifest*/, \ + NULL/*projectCode*/, \ + fsl_fstat_empty_m/*fstat*/, \ + fsl_mcache_empty_m/*mcache*/, \ + {/*globs*/ \ + fsl_list_empty_m/*ignore*/, \ + fsl_list_empty_m/*binary*/, \ + fsl_list_empty_m/*crnl*/ \ + } \ + }/*cache*/, \ + {/*ticket*/ \ + fsl_list_empty_m/*customFields*/, \ + 0/*hasTicket*/, \ + 0/*hasCTime*/, \ + 0/*hasChng*/, \ + 0/*hasCngRid*/ \ + } \ + } + +/** @internal + Initialized-with-defaults fsl_cx instance. +*/ +FSL_EXPORT const fsl_cx fsl_cx_empty; + +/* + TODO: + + int fsl_buffer_append_getenv( fsl_buffer * b, char const * env ) + + Fetches the given env var and appends it to b. Returns FSL_RC_NOT_FOUND + if the env var is not set. The primary use for this would be to simplify + the Windows implementation of fsl_find_home_dir(). +*/ + + +/** @internal + + Expires the single oldest entry in c. Returns true if it removes + an item, else false. +*/ +FSL_EXPORT bool fsl_acache_expire_oldest(fsl_acache * c); + +/** @internal + + Add an entry to the content cache. + + This routines transfers the contents of pBlob over to c, + regardless of success or failure. The cache will deallocate the + memory when it has finished with it. + + If the cache cannot add the entry due to cache-internal + constraints, as opposed to allocation errors, it clears the buffer + (for consistency's sake) and returned 0. + + Returns 0 on success, FSL_RC_OOM on allocation error. Has undefined + behaviour if !c, rid is not semantically valid, !pBlob. An empty + blob is normally semantically illegal but is not strictly illegal + for this cache's purposes. +*/ +FSL_EXPORT int fsl_acache_insert(fsl_acache * c, fsl_id_t rid, fsl_buffer *pBlob); + +/** @internal + + Frees all memory held by c, and clears out c's state, but does + not free c. Results are undefined if !c. +*/ +FSL_EXPORT void fsl_acache_clear(fsl_acache * c); + +/** @internal + + Checks f->cache.arty to see if rid is available in the + repository opened by f. + + Returns 0 if the content for the given rid is available in the + repo or the cache. Returns FSL_RC_NOT_FOUND if it is not in the + repo nor the cache. Returns some other non-0 code for "real + errors," e.g. FSL_RC_OOM if a cache allocation fails. This + operation may update the cache's contents. + + If this function detects a loop in artifact lineage, it fails an + assert() in debug builds and returns FSL_RC_CONSISTENCY in + non-debug builds. That doesn't happen in real life, though. +*/ +FSL_EXPORT int fsl_acache_check_available(fsl_cx * f, fsl_id_t rid); + +/** @internal + + This is THE ONLY routine which adds content to the blob table. + + This writes the given buffer content into the repository + database's blob tabe. It Returns the record ID via outRid (if it + is not NULL). If the content is already in the database (as + determined by a lookup of its hash against blob.uuid), this + routine fetches the RID (via *outRid) but has no side effects in + the repo. + + If srcId is >0 then pBlob must contain delta content from + the srcId record. srcId might be a phantom. + + pBlob is normally uncompressed text, but if uncompSize>0 then + the pBlob value is assumed to be compressed (via fsl_buffer_compress() + or equivalent) and uncompSize is + its uncompressed size. If uncompSize>0 then zUuid must be valid + and refer to the hash of the _uncompressed_ data (which is why + this routine does not calculate it for the client). + + Sidebar: we "could" use fsl_buffer_is_compressed() and friends + to determine if pBlob is compressed and get its decompressed + size, then remove the uncompSize parameter, but that would + require that this function decompress the content to calculate + the hash. Since the caller likely just compressed it, that seems + like a huge waste. + + zUuid is the UUID of the artifact, if it is not NULL. When + srcId is specified then zUuid must always be specified. If + srcId is zero, and zUuid is zero then the correct zUuid is + computed from pBlob. If zUuid is not NULL then this function + asserts (in debug builds) that fsl_is_uuid() returns true for + zUuid. + + If isPrivate is true, the blob is created as a private record. + + If the record already exists but is a phantom, the pBlob content + is inserted and the phatom becomes a real record. + + The original content of pBlob is not disturbed. The caller continues + to be responsible for pBlob. This routine does *not* take over + responsibility for freeing pBlob. + + If outRid is not NULL then on success *outRid is assigned to the + RID of the underlying blob record. + + Returns 0 on success and there are too many potential error cases + to name - this function is a massive beast. + + Potential TODO: we don't really need the uncompSize param - we + can deduce it, if needed, based on pBlob's content. We cannot, + however, know the UUID of the decompressed content unless the + client passes it in to us. + + @see fsl_content_put() +*/ +FSL_EXPORT int fsl_content_put_ex( fsl_cx * const f, + fsl_buffer const * pBlob, + fsl_uuid_cstr zUuid, fsl_id_t srcId, + fsl_size_t uncompSize, bool isPrivate, + fsl_id_t * outRid); +/** @internal + + Equivalent to fsl_content_put_ex(f,pBlob,NULL,0,0,0,newRid). + + This must only be used for saving raw (non-delta) content. + + @see fsl_content_put_ex() +*/ +FSL_EXPORT int fsl_content_put( fsl_cx * const f, + fsl_buffer const * pBlob, + fsl_id_t * newRid); + + +/** @internal + + If the given blob ID refers to deltified repo content, this routine + undeltifies it and replaces its content with its expanded + form. + + Returns 0 on success, FSL_RC_MISUSE if !f, FSL_RC_NOT_A_REPO if + f has no opened repository, FSL_RC_RANGE if rid is not positive, + and any number of other potential errors during the db and + content operations. This function treats already unexpanded + content as success. + + @see fsl_content_deltify() +*/ +FSL_EXPORT int fsl_content_undeltify(fsl_cx * const f, fsl_id_t rid); + + +/** @internal + + The converse of fsl_content_undeltify(), this replaces the storage + of the given blob record so that it is a delta of srcid. + + If rid is already a delta from some other place then no + conversion occurs and this is a no-op unless force is true. + + If rid's contents are not available because the the rid is a + phantom or depends to one, no delta is generated and 0 is + returned. + + It never generates a delta that carries a private artifact into + a public artifact. Otherwise, when we go to send the public + artifact on a sync operation, the other end of the sync will + never be able to receive the source of the delta. It is OK to + delta private->private, public->private, and public->public. + Just no private->public delta. For such cases this function + returns 0, as opposed to FSL_RC_ACCESS or some similar code, and + leaves the content untouched. + + If srcid is a delta that depends on rid, then srcid is + converted to undelta'd text. + + If either rid or srcid contain less than some "small, + unspecified number" of bytes (currently 50), or if the resulting + delta does not achieve a compression of at least 25%, the rid is + left untouched. + + Returns 0 if a delta is successfully made or none needs to be + made, non-0 on error. + + @see fsl_content_undeltify() +*/ +FSL_EXPORT int fsl_content_deltify(fsl_cx * f, fsl_id_t rid, + fsl_id_t srcid, bool force); + + +/** @internal + + Creates a new phantom blob with the given UUID and return its + artifact ID via *newId. Returns 0 on success, FSL_RC_MISUSE if + !f or !uuid, FSL_RC_RANGE if fsl_is_uuid(uuid) returns false, + FSL_RC_NOT_A_REPO if f has no repository opened, FSL_RC_ACCESS + if the given uuid has been shunned, and about 20 other potential + error codes from the underlying db calls. If isPrivate is true + _or_ f has been flagged as being in "private mode" then the new + content is flagged as private. newId may be NULL, but if it is + then the caller will have to find the record id himself by using + the UUID (see fsl_uuid_to_rid()). +*/ +FSL_EXPORT int fsl_content_new( fsl_cx * f, fsl_uuid_cstr uuid, bool isPrivate, + fsl_id_t * newId ); + +/** @internal + + Check to see if checkin "rid" is a leaf and either add it to the LEAF + table if it is, or remove it if it is not. + + Returns 0 on success, FSL_RC_MISUSE if !f or f has no repo db + opened, FSL_RC_RANGE if pid is <=0. Other errors + (e.g. FSL_RC_DB) may indicate that db is not a repo. On error + db's error state may be updated. +*/ +FSL_EXPORT int fsl_repo_leaf_check(fsl_cx * f, fsl_id_t pid); + +/** @internal + + Schedules a leaf check for "rid" and its parents. Returns 0 on + success. +*/ +FSL_EXPORT int fsl_repo_leaf_eventually_check( fsl_cx * f, fsl_id_t rid); + +/** @internal + + Perform all pending leaf checks. Returns 0 on success or if it + has nothing to do. +*/ +FSL_EXPORT int fsl_repo_leaf_do_pending_checks(fsl_cx *f); + +/** @internal + + Inserts a tag into f's repo db. It does not create the related + control artifact - use fsl_tag_add_artifact() for that. + + rid is the artifact to which the tag is being applied. + + srcId is the artifact that contains the tag. It is often, but + not always, the same as rid. This is often the RID of the + manifest containing tags added as part of the commit, in which + case rid==srcId. A Control Artifact which tags a different + artifact will have rid!=srcId. + + mtime is the Julian timestamp for the tag. Defaults to the + current time if mtime <= 0.0. + + If outRid is not NULL then on success *outRid is assigned the + record ID of the generated tag (the tag.tagid db field). + + If a more recent (compared to mtime) entry already exists for + this tag/rid combination then its tag.tagid is returned via + *outRid (if outRid is not NULL) and no new entry is created. + + Returns 0 on success, and has a huge number of potential error + codes. +*/ +FSL_EXPORT int fsl_tag_insert( fsl_cx * f, + fsl_tagtype_e tagtype, + char const * zTag, + char const * zValue, + fsl_id_t srcId, + double mtime, + fsl_id_t rid, + fsl_id_t *outRid ); +/** @internal + + Propagate all propagatable tags in artifact pid to the children of + pid. Returns 0 on... non-error. Returns FSL_RC_RANGE if pid<=0. +*/ +FSL_EXPORT int fsl_tag_propagate_all(fsl_cx * f, fsl_id_t pid); + +/** @internal + + Propagates a tag through the various internal pipelines. + + pid is the artifact id to whose children the tag should be + propagated. + + tagid is the id of the tag to propagate (the tag.tagid db value). + + tagType is the type of tag to propagate. Must be either + FSL_TAGTYPE_CANCEL or FSL_TAGTYPE_PROPAGATING. Note that + FSL_TAGTYPE_ADD is not permitted. The tag-handling internals + (other than this function) translate ADD to CANCEL for propagation + purposes. A CANCEL tag is used to stop propagation. (That's a + historical behaviour inherited from fossil(1).) A potential TODO + is for this function to simply treat ADD as CANCEL, without + requiring that the caller be sure to never pass an ADD tag. + + origId is the artifact id of the origin tag if tagType == + FSL_TAGTYPE_PROPAGATING, otherwise it is ignored. + + zValue is the optional value for the tag. May be NULL. + + mtime is the Julian timestamp for the tag. Must be a valid time + (no defaults here). + + This function is unforgiving of invalid values/ranges, and may assert + in debug mode if passed invalid ids (values<=0), a NULL f, or if f has + no opened repo. +*/ +FSL_EXPORT int fsl_tag_propagate(fsl_cx *f, + fsl_tagtype_e tagType, + fsl_id_t pid, + fsl_id_t tagid, + fsl_id_t origId, + const char *zValue, + double mtime ); + +/** @internal + + Remove the PGP signature from a raw artifact, if there is one. + + Expects *pz to point to *pn bytes of string memory which might + or might not be prefixed by a PGP signature. If the string is + enveloped in a signature, then upon returning *pz will point to + the first byte after the end of the PGP header and *pn will + contain the length of the content up to, but not including, the + PGP footer. + + If *pz does not look like a PGP header then this is a no-op. + + Neither pointer may be NULL and *pz must point to *pn bytes of + valid memory. If *pn is initially less than 59, this is a no-op. +*/ +FSL_EXPORT void fsl_remove_pgp_signature(unsigned char const **pz, fsl_size_t *pn); + +/** @internal + + Clears the "seen" cache used by manifest parsing. Should be + called by routines which initialize parsing, but not until their + work has finished all parsing (so that recursive parsing can + use it). +*/ +FSL_EXPORT void fsl_cx_clear_mf_seen(fsl_cx * f); + +/** @internal + + Generates an fsl_appendf()-formatted message to stderr and + fatally aborts the application by calling exit(). This is only + (ONLY!) intended for us as a placeholder for certain test cases + and is neither thread-safe nor reantrant. + + fmt may be empty or NULL, in which case only the code and its + fsl_rc_cstr() representation are output. + + This function does not return. +*/ +FSL_EXPORT void fsl_fatal( int code, char const * fmt, ... ) +#ifdef __GNUC__ + __attribute__ ((noreturn)) +#endif + ; + +/** @internal + + Translate a normalized, repo-relative filename into a + filename-id (fnid). Create a new fnid if none previously exists + and createNew is true. On success returns 0 and sets *rv to the + filename.fnid record value. If createNew is false and no match + is found, 0 is returned but *rv will be set to 0. Returns non-0 + on error. Results are undefined if any parameter is NULL. + + + In debug builds, this function asserts that no pointer arguments + are NULL and that f has an opened repository. +*/ +FSL_EXPORT int fsl_repo_filename_fnid2( fsl_cx * f, char const * filename, + fsl_id_t * rv, bool createNew ); + + +/** @internal + + Clears and frees all (char*) members of db but leaves the rest + intact. If alsoErrorState is true then the error state is also + freed, else it is kept as well. +*/ +FSL_EXPORT void fsl_db_clear_strings(fsl_db * const db, bool alsoErrorState ); + +/** @internal + + Returns 0 if db appears to have a current repository schema, 1 + if it appears to have an out of date schema, and -1 if it + appears to not be a repository. Results are undefined if db is + NULL or not opened. +*/ +FSL_EXPORT int fsl_db_repo_verify_schema(fsl_db * db); + + +/** @internal + + Flags for APIs which add phantom blobs to the repository. The + values in this enum derive from fossil(1) code and should not be + changed without careful forethought and (afterwards) testing. A + phantom blob is a blob about whose existence we know but for which + we have no content. This normally happens during sync or rebuild + operations, but can also happen when artifacts are stored directly + as files in a repo (like this project's repository does, storing + artifacts from *other* projects for testing purposes). +*/ +enum fsl_phantom_e { +/** + Indicates to fsl_uuid_to_rid2() that no phantom artifact + should be created. +*/ +FSL_PHANTOM_NONE = 0, +/** + Indicates to fsl_uuid_to_rid2() that a public phantom + artifact should be created if no artifact is found. +*/ +FSL_PHANTOM_PUBLIC = 1, +/** + Indicates to fsl_uuid_to_rid2() that a private phantom + artifact should be created if no artifact is found. +*/ +FSL_PHANTOM_PRIVATE = 2 +}; +typedef enum fsl_phantom_e fsl_phantom_e; + +/** @internal + + Works like fsl_uuid_to_rid(), with these differences: + + - uuid is required to be a complete UUID, not a prefix. + + - If it finds no entry and the mode argument specifies so then + it will add either a public or private phantom entry and return + its new rid. If mode is FSL_PHANTOM_NONE then this this behaves + just like fsl_uuid_to_rid(). + + Returns a positive value on success, 0 if it finds no entry and + mode==FSL_PHANTOM_NONE, and a negative value on error (e.g. if + fsl_is_uuid(uuid) returns false). Errors which happen after + argument validation will "most likely" update f's error state + with details. +*/ +FSL_EXPORT fsl_id_t fsl_uuid_to_rid2( fsl_cx * f, fsl_uuid_cstr uuid, + fsl_phantom_e mode ); + +/** @internal + + Schedules the given rid to be verified at the next commit. This + is used by routines which add artifact records to the blob + table. + + The only error case, assuming the arguments are valid, is an + allocation error while appending rid to the internal to-verify + queue. + + @see fsl_repo_verify_at_commit() + @see fsl_repo_verify_cancel() +*/ +FSL_EXPORT int fsl_repo_verify_before_commit( fsl_cx * f, fsl_id_t rid ); + +/** @internal + + Clears f's verify-at-commit list of RIDs. + + @see fsl_repo_verify_at_commit() + @see fsl_repo_verify_before_commit() +*/ +FSL_EXPORT void fsl_repo_verify_cancel( fsl_cx * f ); + +/** @internal + + Processes all pending verify-at-commit entries and clears the + to-verify list. Returns 0 on success. On error f's error state + will likely be updated. + + ONLY call this from fsl_db_transaction_end() or its delegate (if + refactored). + + Verification calls fsl_content_get() to "unpack" content added in + the current transaction. If fetching the content (which applies + any deltas it may need to) fails or a checksum does not match then + this routine fails and returns non-0. On error f's error state + will be updated. + + @see fsl_repo_verify_cancel() + @see fsl_repo_verify_before_commit() +*/ +FSL_EXPORT int fsl_repo_verify_at_commit( fsl_cx * f ); + +/** @internal + + Removes all entries from the repo's blob table which are listed + in the shun table. Returns 0 on success. This operation is + wrapped in a transaction. Delta contant which depend on + to-be-shunned content are replaced with their undeltad forms. + + Returns 0 on success. +*/ +FSL_EXPORT int fsl_repo_shun_artifacts(fsl_cx * f); + +/** @internal. + + Return a pointer to a string that contains the RHS of an SQL IN + operator which will select config.name values that are part of + the configuration that matches iMatch (a bitmask of + fsl_configset_e values). Ownership of the returned string is + passed to the caller, who must eventually pass it to + fsl_free(). Returns NULL on allocation error. + + Reminder to self: this is part of the infrastructure for copying + config state from an existing repo when creating new repo. +*/ +FSL_EXPORT char *fsl_config_inop_rhs(int iMask); + +/** @internal + + Return a pointer to a string that contains the RHS of an IN + operator that will select config.name values that are in the + list of control settings. Ownership of the returned string is + passed to the caller, who must eventually pass it to + fsl_free(). Returns NULL on allocation error. + + Reminder to self: this is part of the infrastructure for copying + config state from an existing repo when creating new repo. +*/ +FSL_EXPORT char *fsl_db_setting_inop_rhs(); + +/** @internal + + Creates the ticket and ticketchng tables in f's repository db, + DROPPING them if they already exist. The schema comes from + fsl_schema_ticket(). + + TODO? Add a flag specifying whether to drop or keep existing + copies. + + Returns 0 on success. +*/ +FSL_EXPORT int fsl_cx_ticket_create_table(fsl_cx * const f); + +/** @internal + + Frees all J-card entries in the given list. + + li is assumed to be empty or contain (fsl_card_J*) + instances. If alsoListMem is true then any memory owned + by li is also freed. li itself is not freed. + + Results are undefined if li is NULL. +*/ +FSL_EXPORT void fsl_card_J_list_free( fsl_list * li, bool alsoListMem ); + +/** @internal + + Values for fsl_card_J::flags. +*/ +enum fsl_card_J_flags { +/** + Sentinel value. +*/ +FSL_CARD_J_INVALID = 0, +/** + Indicates that the field is used by the ticket table. +*/ +FSL_CARD_J_TICKET = 0x01, +/** + Indicates that the field is used by the ticketchng table. +*/ +FSL_CARD_J_CHNG = 0x02, +/** + Indicates that the field is used by both the ticket and + ticketchng tables. +*/ +FSL_CARD_J_BOTH = FSL_CARD_J_TICKET | FSL_CARD_J_CHNG +}; + +/** @internal + + Loads all custom/customizable ticket fields from f's repo's + ticket table info f. If f has already loaded the list and + forceReload is false, this is a no-op. + + Returns 0 on success. + + @see fsl_cx::ticket::customFields +*/ +FSL_EXPORT int fsl_cx_ticket_load_fields(fsl_cx * f, bool forceReload); + +/** @internal + + A comparison routine for qsort(3) which compares fsl_card_J + instances in a lexical manner based on their names. The order is + important for card ordering in generated manifests. + + This routine expects to get passed (fsl_card_J**) (namely from + fsl_list entries), and will not work on an array of J-cards. +*/ +FSL_EXPORT int fsl_qsort_cmp_J_cards( void const * lhs, void const * rhs ); + +/** @internal + + This function updates the repo and/or global config databases + with links between the dbs intended for various fossil-level + bookkeeping and housecleaning. These links are not essential to + fossil's functionality but assist in certain "global" + operations. + + If no checkout is opened but a repo is, the global config (if + opened) is updated to know about the opened repo db. + + If a checkout is opened, global config (if opened) and the + repo are updated to point to the checked-out db. +*/ +FSL_EXPORT int fsl_repo_record_filename(fsl_cx * f); + +/** @internal + + Updates f->ckout.uuid and f->ckout.rid to reflect the current + checkout state. If no checkout is opened, the uuid is freed/NULLed + and the rid is set to 0. Returns 0 on success. If it returns an + error (OOM or db-related), the f->ckout state is left in a + potentially inconsistent state, and it should not be relied upon + until/unless the error is resolved. + + This is done when a checkout db is opened, when performing a + checkin, and otherwise as needed, and so calling it from other + code is normally not necessary. + + @see fsl_ckout_version_write() +*/ +FSL_EXPORT int fsl_ckout_version_fetch( fsl_cx *f ); + +/** @internal + + Updates f->ckout's state to reflect the given version info and + writes the 'checkout' and 'checkout-hash' properties to the + currently-opened checkout db. Returns 0 on success, + FSL_RC_NOT_A_CKOUT if no checkout is opened (may assert() in that + case!), or some other code if writing to the db fails. + + If vid is 0 then the version info is null'd out. Else if uuid is + NULL then fsl_rid_to_uuid() is used to fetch the UUID for vid. + + If the RID differs from f->ckout.rid then f->ckout's version state + is updated to the new values. + + This routine also updates or removes the checkout's manifest + files, as per fsl_ckout_manifest_write(). If vid is 0 then it + removes any such files which themselves are not part of the + current checkout. + + @see fsl_ckout_version_fetch() + @see fsl_cx_ckout_version_set() +*/ +FSL_EXPORT int fsl_ckout_version_write( fsl_cx *f, fsl_id_t vid, + fsl_uuid_cstr uuid ); + +/** + @internal + + Exports the file with the given [vfile].[id] to the checkout, + overwriting (if possible) anything which gets in its way. If + the file is determined to have not been modified, it is + unchanged. + + If the final argument is not NULL then it is set to 0 if the file + was not modified, 1 if only its permissions were modified, and 2 if + its contents were updated (which also requires resetting its + permissions to match their repo-side state). + + Returns 0 on success, any number of potential non-0 codes on + error, including, but not limited to: + + - FSL_RC_NOT_A_CKOUT - no opened checkout. + - FSL_RC_NOT_FOUND - no matching vfile entry. + - FSL_RC_OOM - we cannot escape this eventuality. + + Trivia: + + - fossil(1)'s vfile_to_disk() is how it exports a whole vfile, or a + single vfile entry, to disk. e.g. it performs a checkout that way, + whereas we currently perform a checkout using the "repo extraction" + API. The checkout mechanism was probably the first major core + fossil feature which was structured radically differently in + libfossil, compared to the feature's fossil counterpart, when it + was ported over. + + - This routine always writes to the vfile.pathname entry, as + opposed to vfile.origname. + + Maintenance reminders: internally this code supports handling + multiple files at once, but (A) that's not part of the interface + and may change and (B) the 3rd parameter makes little sense in that + case unless maybe we change it to a callback, which seems like + overkill for our use cases. +*/ +FSL_EXPORT int fsl_vfile_to_ckout(fsl_cx * f, fsl_id_t vfileId, + int * wasWritten); + +/** @internal + + On Windows platforms (only), if fsl_isalpha(*zFile) + and ':' == zFile[1] then this returns zFile+2, + otherwise it returns zFile. +*/ +FSL_EXPORT char * fsl_file_without_drive_letter(char * zFile); + +/** @internal + + This is identical to the public-API member fsl_deck_F_search(), + except that it returns a non-const F-card. + + Locate a file named zName in d->F.list. Return a pointer to the + appropriate fsl_card_F object. Return NULL if not found. + + If d->f is set (as it is when loading decks via + fsl_deck_load_rid() and friends), this routine works even if p is + a delta-manifest. The pointer returned might be to the baseline + and d->B.baseline is loaded on demand if needed. + + If the returned card's uuid member is NULL, it means that the file + was removed in the checkin represented by d. + + If !d, zName is NULL or empty, or FSL_SATYPE_CHECKIN!=d->type, it + asserts in debug builds and returns NULL in non-debug builds. + + We assume that filenames are in sorted order and use a binary + search. As an optimization, to support the most common use case, + searches through a deck update d->F.cursor to the last position a + search was found. Because searches are normally done in lexical + order (because of architectural reasons), this is normally an O(1) + operation. It degrades to O(N) if out-of-lexical-order searches + are performed. +*/ +FSL_EXPORT fsl_card_F * fsl_deck_F_seek(fsl_deck * const d, const char *zName); + +/** @internal + + Part of the fsl_cx::fileContent optimization. This sets + f->fileContent.used to 0 and if its capacity is over a certain + (unspecified, unconfigurable) size then it is trimmed to that + size. +*/ +FSL_EXPORT void fsl_cx_content_buffer_yield(fsl_cx * const f); + +/** @internal + + Currently disabled (always returns 0) pending resolution of a + "wha???" result from one of the underlying queries. + + Queues up the given artifact for a search index update. This is + only intended to be called from crosslinking steps and similar + content updates. Returns 0 on success. + + The final argument is intended only for wiki titles (the L-card of + a wiki post). + + If the repository database has no search index or the given content + is marked as private, this function returns 0 and makes no changes + to the db. +*/ +FSL_EXPORT int fsl_search_doc_touch(fsl_cx * const f, fsl_satype_e saType, + fsl_id_t rid, const char * docName); + +/** @internal + + Performs the same job as fsl_diff_text() but produces the results + in the low-level form of an array of "copy/delete/insert triples." + This is primarily intended for internal use in other + library-internal algorithms, not for client code. Note all + FSL_DIFF_xxx flags apply to this form. + + Returns 0 on success, any number of non-0 codes on error. On + success *outRaw will contain the resulting array, which must + eventually be fsl_free()'d by the caller. On error *outRaw is not + modified. + + @deprecated Use fsl_diff_v2_raw() instead. +*/ +FSL_EXPORT int fsl_diff_text_raw(fsl_buffer const *p1, fsl_buffer const *p2, + int diffFlags, int ** outRaw); + +/** @internal + + If the given file name is a reserved filename (case-insensitive) on + Windows platforms, a pointer to the reserved part of the name, else + NULL is returned. + + zPath must be a canonical path with forward-slash directory + separators. nameLen is the length of zPath. If negative, fsl_strlen() + is used to determine its length. +*/ +FSL_EXPORT bool fsl_is_reserved_fn_windows(const char *zPath, fsl_int_t nameLen); + +/** @internal + + Clears any pending merge state from the checkout db's vmerge table. + Returns 0 on success. +*/ +FSL_EXPORT int fsl_ckout_clear_merge_state( fsl_cx *f ); + + +/** @internal + + Installs or reinstalls the checkout database schema into f's open + checkout db. Returns 0 on success, FSL_RC_NOT_A_CKOUT if f has + no opened checkout, or an code if a lower-level operation fails. + + If dropIfExists is true then all affected tables are dropped + beforehand if they exist. "It's the only way to be sure." + + If dropIfExists is false and the schema appears to already exists + (without actually validating its validity), 0 is returned. +*/ +FSL_EXPORT int fsl_ckout_install_schema(fsl_cx *f, bool dropIfExists); + +/** @internal + + Attempts to remove empty directories from under a checkout, + starting with tgtDir and working upwards until it either cannot + remove one or it reaches the top of the checkout dir. + + The second argument must be the canonicalized absolute path to some + directory under the checkout root. The contents of the buffer may, + for efficiency's sake, be modified by this routine as it traverses + the directory tree. It will never grow the buffer but may mutate + its memory's contents. + + Returns the number of directories it is able to remove. + + Results are undefined if tgtDir is not an absolute path rooted in + f's current checkout. + + There are any number of valid reasons removal of a directory might + fail, and this routine stops at the first one which does. +*/ +FSL_EXPORT unsigned int fsl_ckout_rm_empty_dirs(fsl_cx * f, fsl_buffer * tgtDir); + +/** @internal + + This is intended to be passed the name of a file which was just + deleted and "might" have left behind an empty directory. The name + _must_ an absolute path based in f's current checkout. This routine + uses fsl_file_dirpart() to strip path components from the string + and remove directories until either removing one fails or the top + of the checkout is reach. Since removal of a directory can fail for + any given reason, this routine ignores such errors. It returns 0 on + success, FSL_RC_OOM if allocation of the working buffer for the + filename hackery fails, and FSL_RC_MISUSE if zFilename is not + rooted in the checkout (in which case it may assert(), so don't do + that). + + @see fsl_is_rooted_in_ckout() + @see fsl_rm_empty_dirs() +*/ +FSL_EXPORT int fsl_ckout_rm_empty_dirs_for_file(fsl_cx * f, char const *zAbsPath); + +/** @internal + + If f->cache.seenDeltaManifest<=0 then this routine sets it to 1 + and sets the 'seen-delta-manifest' repository config setting to 1, + else this has no side effects. Returns 0 on success, non-0 if + there is an error while writing to the repository config. +*/ +FSL_EXPORT int fsl_cx_update_seen_delta_mf(fsl_cx *f); + +/** @internal + + Very, VERY internal. + + Returns the next available buffer from f->scratchpads. Fatally + aborts if there are no free buffers because "that should not + happen." Calling this obligates the caller to eventually pass + its result to fsl_cx_scratchpad_yield(). + + This function guarantees the returned buffer's 'used' member will be + set to 0. + + Maintenance note: the number of buffers is hard-coded in the + fsl_cx::scratchpads anonymous struct. +*/ +FSL_EXPORT fsl_buffer * fsl_cx_scratchpad(fsl_cx *f); + +/** @internal + + Very, VERY internal. + + "Yields" a buffer which was returned from fsl_cx_scratchpad(), + making it available for re-use. The caller must treat the buffer as + if this routine frees it: using the buffer after having passed it + to this function will internally be flagged as explicit misuse and + will lead to a fatal crash the next time that buffer is fetched via + fsl_cx_scratchpad(). So don't do that. +*/ +FSL_EXPORT void fsl_cx_scratchpad_yield(fsl_cx *f, fsl_buffer * b); + + +/** @internal + + Run automatically by fsl_deck_save(), so it needn't normally be run + aside from that, at least not from average client code. + + Runs postprocessing on the Structural Artifact represented by + d. d->f must be set, d->rid must be set and valid and d's contents + must accurately represent the stored manifest for the given + rid. This is normally run just after the insertion of a new + manifest, but is sometimes also run after reading a deck from the + database (in order to rebuild all db relations and add/update the + timeline entry). + + Returns 0 on succes, FSL_RC_MISUSE !d->f, FSL_RC_RANGE if + d->rid<=0, FSL_RC_MISUSE (with more error info in f) if d does not + contain all required cards for its d->type value. It may return + various other codes from the many routines it delegates work to. + + Crosslinking of ticket artifacts is currently (2021-03) missing. + + Design note: d "really should" be const here but some internals + (d->F.cursor and delayed baseline loading) prohibit it. + + @see fsl_deck_crosslink_one() +*/ +FSL_EXPORT int fsl_deck_crosslink( fsl_deck /* const */ * const d ); + +/** @internal + + Run automatically by fsl_deck_save(), so it needn't normally be run + aside from that, at least not from average client code. + + This is a convience form of crosslinking which must only be used + when a single deck (and only a single deck) is to be crosslinked. + This function wraps the crosslinking in fsl_crosslink_begin() + and fsl_crosslink_end(), but otherwise behaves the same as + fsl_deck_crosslink(). If crosslinking fails, any in-progress + transaction will be flagged as failed. + + Returns 0 on success. +*/ +FSL_EXPORT int fsl_deck_crosslink_one( fsl_deck * const d ); + +/** @internal + + Checks whether the given filename is "safe" for writing to within + f's current checkout. + + zFilename must be in canonical form: only '/' directory separators. + If zFilename is not absolute, it is assumed to be relative to the top + of the current checkout, else it must point to a file under the current + checkout. + + Checks made on the filename include: + + - It must refer to a file under the current checkout. + + - Ensure that each directory listed in the file's path is actually + a directory, and fail if any part other than the final one is a + non-directory. + + If the name refers to something not (yet) in the filesystem, that + is not considered an error. + + Returns 0 on success. On error f's error state is updated with + information about the problem. +*/ +FSL_EXPORT int fsl_ckout_safe_file_check(fsl_cx *f, char const * zFilename); + +/** @internal + UNTESTED! + + Creates a file named zLinkFile and populates its contents with a + single line: zTgtFile. This behaviour corresponds to how fossil + manages SCM'd symlink entries on Windows and on other platforms + when the 'allow-symlinks' repo-level config setting is disabled. + (In late 2020 fossil defaulted that setting to disabled and made it + non-versionable.) + + zLinkFile may be an absolute path rooted at f's current checkout or + may be a checkout-relative path. + + Returns 0 on success, non-0 on error: + + - FSL_RC_NOT_A_CKOUT if f has no opened checkout. + + - FSL_RC_MISUSE if zLinkFile refers to a path outside of the + current checkout. + + Potential TODO (maybe!): handle symlinks as described above or + "properly" on systems which natively support them iff f's + 'allow-symlinks' repo-level config setting is true. That said: the + addition of symlinks support into fossil was, IMHO, a poor decision + for $REASONS. That might (might) be reflected long-term in this API + by only supporting them in the way fossil does for platforms which + do not support symlinks. +*/ +FSL_EXPORT int fsl_ckout_symlink_create(fsl_cx * f, char const *zTgtFile, + char const * zLinkFile); + + +/** + Compute all file name changes that occur going from check-in iFrom + to check-in iTo. Requires an opened repository. + + If revOK is true, the algorithm is free to move backwards in the + chain. This is the opposite of the oneWayOnly parameter for + fsl_vpath_shortest(). + + On success, the number of name changes is written into *pnChng. + For each name change, two integers are allocated for *piChng. The + first is the filename.fnid for the original name as seen in + check-in iFrom and the second is for new name as it is used in + check-in iTo. If *pnChng is 0 then *aiChng will be NULL. + + On error returns non-0, pnChng and aiChng are not modified, and + f's error state might (depending on the error) contain a description + of the problem. + + Space to hold *aiChng is obtained from fsl_malloc() and must + be released by the caller. +*/ +FSL_EXPORT int fsl__find_filename_changes(fsl_cx * const f, + fsl_id_t iFrom, + fsl_id_t iTo, + bool revOK, + uint32_t *pnChng, + fsl_id_t **aiChng); + +/** + Bitmask of file change types for use with + fsl_is_locally_modified(). + */ +enum fsl_localmod_e { +/** Sentinel value. */ +FSL_LOCALMOD_NONE = 0, +/** + Permissions changed. +*/ +FSL_LOCALMOD_PERM = 0x01, +/** + File size or hash (i.e. content) differ. +*/ +FSL_LOCALMOD_CONTENT = 0x02, +/** + The file type was switched between symlink and normal file. In + this case, no check for content change, beyond the file size + change, is performed. +*/ +FSL_LOCALMOD_LINK = 0x04, +/** + File was not found in the local checkout. + */ +FSL_LOCALMOD_NOTFOUND = 0x10 +}; +typedef enum fsl_localmod_e fsl_localmod_e; +/** @internal + + Checks whether the given file has been locally modified compared to + a known size, hash value, and permissions. Requires that f has an + opened checkout. + + If zFilename is not an absolute path, it is assumed to be relative + to the checkout root (as opposed to the current directory) and is + canonicalized into an absolute path for purposes of this function. + + fileSize is the "original" version's file size. zOrigHash is the + initial hash of the file to use as a basis for comparison. + zOrigHashLen is the length of zOrigHash, or a negative value if + this function should use fsl_is_uuid() to determine the length. If + the hash length is not that of one of the supported hash types, + FSL_RC_RANGE is returned and f's error state is updated. This + length is used to determine which hash to use for the comparison. + + If the file's current size differs from the given size, it is + quickly considered modified, otherwise the file's contents get + hashed and compared to zOrigHash. + + Because this is used for comparing local files to their state from + the fossil database, where files have no timestamps, the local + file's timestamp is never considered for purposes of modification + checking. + + If isModified is not NULL then on success it is set to a bitmask of + values from the fsl_localmod_e enum specifying the type(s) of + change(s) detected: + + - FSL_LOCALMOD_PERM = permissions changed. + + - FSL_LOCALMOD_CONTENT = file size or hash (i.e. content) differ. + + - FSL_LOCALMOD_LINK = the file type was switched between symlink + and normal file. In this case, no check for content change, + beyond the file size change, is performed. + + - FSL_LOCALMOD_NOFOUND = file was not found in the local checkout. + + Noting that: + + - Combined values of (FSL_LOCALMOD_PERM | FSL_LOCALMOD_CONTENT) are + possible, but FSL_LOCALMOD_NOFOUND will never be combined with one + of the other values. + + If stat() fails for any reason other than file-not-found + (e.g. permissions), an error is triggered. + + Returns 0 on success. On error, returns non-0 and f's error state + will be updated and isModified... isNotModified. Errors include, + but are not limited to: + + - Invalid hash length: FSL_RC_RANGE + - f has no opened checkout: FSL_RC_NOT_A_CKOUT + - Cannot find the file: FSL_RC_NOT_FOUND + - Error accessing the file: FSL_RC_ACCESS + - Allocation error: FSL_RC_OOM + - I/O error during hashing: FSL_RC_IO + + And potentially other errors, roughly translated from errno values, + for corner cases such as passing a directory name instead of a + file. + + Results are undefined if any pointer argument is NULL or invalid. + + This function currently does NOT follow symlinks for purposes of + resolving zFilename, but that behavior may change in the future or + may become dependent on the repository's 'allow-symlinks' setting. + + Internal detail, not relevant for clients: this updates f's + cache stat entry. +*/ +FSL_EXPORT int fsl_is_locally_modified(fsl_cx * f, + const char * zFilename, + fsl_size_t fileSize, + const char * zOrigHash, + fsl_int_t zOrigHashLen, + fsl_fileperm_e origPerm, + int * isModified); + +/** @internal + + This routine cleans up the state of selected cards in the given + deck. The 2nd argument is an list of upper-case letters + representing the cards which should be cleaned up, e.g. "ADG". If + it is NULL, all cards are cleaned up but d has non-card state + which is not cleaned up by this routine. Unknown letters are simply + ignored. +*/ +FSL_EXPORT void fsl_deck_clean_cards(fsl_deck * d, char const * letters); + +/** @internal + + Searches the current repository database for a fingerprint and + returns it as a string in *zOut. + + If rcvid<=0 then the fingerprint matches the last entry in the + [rcvfrom] table, where "last" means highest-numbered rcvid (as + opposed to most recent mtime, for whatever reason). If rcvid>0 then + it searches for an exact match. + + Returns 0 on non-error, where finding no matching rcvid causes + FSL_RC_NOT_FOUND to be returned. If 0 is returned then *zOut will + be non-NULL and ownership of that value is transferred to the + caller, who must eventually pass it to fsl_free(). On error, *zOut + is not modified. + + Returns FSL_RC_NOT_A_REPO if f has no opened repository, FSL_RC_OOM + on allocation error, or any number of potential db-related codes if + something goes wrong at the db level. + + This API internally first checks for "version 1" fossil + fingerprints and falls back to "version 0" fingerprint if a v1 + fingerprint is not found. Version 0 was very short-lived and is not + expected to be in many repositories which are accessed via this + library. Practice has, however, revealed some. + + @see fsl_ckout_fingerprint_check() +*/ +FSL_EXPORT int fsl_repo_fingerprint_search(fsl_cx *f, fsl_id_t rcvid, char ** zOut); + +/** + A context for running a raw diff. + + The aEdit[] array describes the raw diff. Each triple of integers in + aEdit[] means: + + (1) COPY: Number of lines aFrom and aTo have in common + (2) DELETE: Number of lines found only in aFrom + (3) INSERT: Number of lines found only in aTo + + The triples repeat until all lines of both aFrom and aTo are accounted + for. +*/ +struct fsl_diff_cx { + /*TODO unsigned*/ int *aEdit; /* Array of copy/delete/insert triples */ + /*TODO unsigned*/ int nEdit; /* Number of integers (3x num of triples) in aEdit[] */ + /*TODO unsigned*/ int nEditAlloc; /* Space allocated for aEdit[] */ + fsl_dline *aFrom; /* File on left side of the diff */ + /*TODO unsigned*/ int nFrom; /* Number of lines in aFrom[] */ + fsl_dline *aTo; /* File on right side of the diff */ + /*TODO unsigned*/ int nTo; /* Number of lines in aTo[] */ + int (*cmpLine)(const fsl_dline * const, const fsl_dline *const); /* Function to be used for comparing */ +}; +/** + Convenience typeef. +*/ +typedef struct fsl_diff_cx fsl_diff_cx; +/** Initialized-with-defaults fsl_diff_cx structure, intended for + const-copy initialization. */ +#define fsl_diff_cx_empty_m {\ + NULL,0,0,NULL,0,NULL,0,fsl_dline_cmp \ +} +/** Initialized-with-defaults fsl_diff_cx structure, intended for + non-const copy initialization. */ +extern const fsl_diff_cx fsl_diff_cx_empty; + + + +/** @internal + + Compute the differences between two files already loaded into + the fsl_diff_cx structure. + + A divide and conquer technique is used. We look for a large + block of common text that is in the middle of both files. Then + compute the difference on those parts of the file before and + after the common block. This technique is fast, but it does + not necessarily generate the minimum difference set. On the + other hand, we do not need a minimum difference set, only one + that makes sense to human readers, which this algorithm does. + + Any common text at the beginning and end of the two files is + removed before starting the divide-and-conquer algorithm. + + Returns 0 on succes, FSL_RC_OOM on an allocation error. +*/ +int fsl__diff_all(fsl_diff_cx * const p); + +/** @internal + */ +void fsl__diff_optimize(fsl_diff_cx * const p); + +/** @internal + */ +void fsl__diff_cx_clean(fsl_diff_cx * const cx); + +/** @internal + + Undocumented. For internal debugging only. + */ +void fsl__dump_triples(fsl_diff_cx const * const p, + char const * zFile, int ln ); + +/** @internal + + Removes from the BLOB table all artifacts that are in the SHUN + table. Returns 0 on success. Requires (asserts) that a repo is + opened. +*/ +int fsl__shunned_remove(fsl_cx * const f); + + +/** @internal + + Maximum length of a line in a text file, in bytes. (2**15 = 32k) +*/ +#define FSL_LINE_LENGTH_MASK_SZ 15 +/** @internal + + */ +#define FSL_LINE_LENGTH_MASK ((1<0). fsl_int_t also guarantees that + it will be 64-bit if available, so can be used for places where + large values are needed but a negative value is legal (or handy), + e.g. fsl_strndup()'s second argument. The use of the fsl_xxx_t + typedefs, rather than (unsigned) int, is primarily for + readability/documentation, e.g. so that readers can know + immediately that the function uses a given argument or return + value following certain API-wide semantics. It also allows us to + better define platform-portable printf/scanf-style format + modifiers for them (analog to C99's PRIi32 and friends), which + often come in handy. + + - Signed vs. unsigned types for size/length arguments: use the + fsl_int_t (signed) argument type when the client may legally pass + in a negative value as a hint that the API should use fsl_strlen() + (or similar) to determine a byte array's length. Use fsl_size_t + when no automatic length determination is possible (or desired), + to "force" the client to pass the proper length. Internally + fsl_int_t is used in some places where fsl_size_t "should" be used + because some ported-in logic relies on loop control vars being + able to go negative. Additionally, fossil internally uses negative + blob lengths to mark phantom blobs, and care must be taken when + using fsl_size_t with those. + + - Functions taking elipses (...) are accompanied by a va_list + counterpart named the same as the (...) form plus a trailing + 'v'. e.g. fsl_appendf() and fsl_appendfv(). We do not use the + printf()/vprintf() convention because that hoses sorting of the + functions in generated/filtered API documentation. + + - Error handling/reporting: please keep in mind that the core code + is a library, not an application. The main implication is that + all lib-level code needs to check for errors whereever they can + happen (e.g. on every single memory allocation, of which there are + many) and propagate errors to the caller, to be handled at his + discretion. The app-level code (::fcli) is not particularly strict + in this regard, and installs its own allocator which abort()s on + allocation error, which simplifies app-side code somewhat + vis-a-vis lib-level code. When reporting an error can be improved + by the inclusion of an error string, functions like + fsl_cx_err_set() can be used to report the error. Several of the + high-level types in the API have fsl_error object member which + contains such error state. The APIs which use that state take care + to use-use the error string memory whenever possible, so setting + an error string is often a non-allocating operation. +*/ + + +/** @page page_fossil_arch Fossil Architecture Overview + + An introduction to the Fossil architecture. These docs + are basically just a reformulation of other, more detailed, + docs which can be found via the main Fossil site, e.g.: + + - https://fossil-scm.org/home/doc/trunk/www/concepts.wiki + + - https://fossil-scm.org/home/doc/trunk/www/fileformat.wiki + + Fossil's internals are fundamentally broken down into two basic + parts. The first is a "collection of blobs." The simplest way to + think of this (and it's not far from the full truth) is a + directory containing lots of files, each one named after a hash of + its contents. This pool contains ALL content required for a + repository - all other data can be generated from data contained + here. Included in the blob pool are so-called Artifacts. Artifacts + are simple text files with a very strict format, which hold + information regarding the idententies of, relationships involving, + and other metadata for each type of blob in the pool. The most + fundamental Artifact type is called a Manifest, and a Manifest + tells us, amongst other things, which of the hash-based file names + has which "real" file name, which version the parent (or parents!) + is (or are), and other data required for a "commit" operation. + + The blob pool and the Manifests are all a Fossil repository really + needs in order to function. On top of that basis, other forms of + Artifacts provide features such as tagging (which is the basis of + branching and merging), wiki pages, and tickets. From those + Artifacts, Fossil can create/calculate all sorts of + information. For example, as new Artifacts are inserted it + transforms the Artifact's metadata into a relational model which + sqlite can work with. That leads us to what is conceptually the + next-higher-up level, but is in practice a core-most component... + + Storage. Fossil's core model is agnostic about how its blobs are + stored, but libfossil and fossil(1) both make heavy use of sqlite + to implement many of their features. These include: + + - Transaction-capable storage. It's almost impossible to corrupt a + Fossil db in normal use. sqlite3 offers literally the most robust + general-purpose file format on the planet. + + - The storage of the raw blobs. + + - Artifact metadata is transformed into various DB structures + which allow libfossil to traverse historical data much more + efficiently than would be possible without a db-like + infrastructure (and everything that implies). These structures are + kept up to date as new Artifacts are stored in a repository, + either via local edits or synching in remote content. These data + are incrementally updated as changes are made to a repo. + + - A tremendous amount of the "leg-work" in processing the + repository state is handled by SQL queries, without which the + library would easily require 5-10x more code in the form of + equivalent hard-coded data structures and corresponding + functionality. The db approach allows us to ad-hoc structures as + we need them, providing us a great deal of flexibility. + + All content in a Fossil repository is in fact stored in a single + database file. Fossil additionally uses another database (a + "checkout" db) to keep track of local changes, but the repo + contains all "fossilized" content. Each copy of a repo is a + full-fledged repo, each capable of acting as a central copy for + any number of clones or checkouts. + + That's really all there is to understand about Fossil. How it does + its magic, keeping everything aligned properly, merging in + content, how it stores content, etc., is all internal details + which most clients will not need to know anything about in order + to make use of fossil(1). Using libfossil effectively, though, + does require learning _some_ amount of how Fossil works. That will + require taking some time with _other_ docs, however: see the + links at the top of this section for some starting points. + + + Sidebar: + + - The only file-level permission Fossil tracks is the "executable" + (a.k.a. "+x") bit. It internally marks symlinks as a permission + attribute, but that is applied much differently than the + executable bit and only does anything useful on platforms which + support symlinks. + +*/ + +#endif +/* ORG_FOSSIL_SCM_PAGES_H_INCLUDED */ +/* end of file ../include/fossil-scm/fossil-pages.h */ +/* start of file ../include/fossil-scm/fossil-cli.h */ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +#if !defined(_ORG_FOSSIL_SCM_FCLI_H_INCLUDED_) +#define _ORG_FOSSIL_SCM_FCLI_H_INCLUDED_ +/* + Copyright 2013-2021 The Libfossil Authors, see LICENSES/BSD-2-Clause.txt + + SPDX-License-Identifier: BSD-2-Clause-FreeBSD + SPDX-FileCopyrightText: 2021 The Libfossil Authors + SPDX-ArtifactOfProjectName: Libfossil + SPDX-FileType: Code + + Heavily indebted to the Fossil SCM project (https://fossil-scm.org). + + ***************************************************************************** + This file provides a basis for basic libfossil-using apps. It + attempts to provide basic services required by a wide variety of + fossil-using apps, with the intent of simplifying their creation. +*/ + +/* Force assert() to always work... */ +#if defined(NDEBUG) +#undef NDEBUG +#define DEBUG 1 +#endif +#include /* for the benefit of test apps */ +#include /* EXIT_SUCCESS and friends */ + +/** @page page_fcli fcli (formerly FossilApp) + + ::fcli (formerly FossilApp) provides a small framework for + bootstrapping simple libfossil applications which only need a + single fsl_cx instance managing a single checkout and/or + repository. It is primarily intended for use with CLI apps + implementing features similar to those in fossil(1), but can also + be used with GUI apps. It provides the following basic services to + applications: + + - The global ::fcli struct holds global state. + + - fcli_setup() bootstraps the environment. This must be the + first call made into the API, as this replaces the libfossil + memory allocator with a fail-fast variant to simplify app-level + code a bit (by removing the need to check for OOM errors). This + also registers an atexit(3) handler to clean up fcli-owned + resources at app shutdown. + + - Automatically tries to open a checkout (and its associated + repository) under the current directory, but not finding one + is not an error (the app needs to check for this if it should + be an error: use fsl_cx_db_repo() and fsl_cx_db_ckout()). + + - fcli_flag(), fcli_next_arg(), and friends provide uniform access + to CLI arguments. + + - A (very) basic help subsystem, triggered by the --help or -? CLI + flags, or if the first non-flag argument is "help". Applications + may optionally assign fcli.appHelp to a function which outputs + app-specific help. + + - Basic error reporting mechanism. See fcli_err_report(). + + The source tree contains several examples of using fcli + in the files named f-*.c. +*/ + +/** + Ouputs the given printf-type expression (2nd arg) to + fcli_printf() if fcli.verbose>=N, else it does nothing. + + Reminder: this uses a while(0) loop so that the macro can end + with a semicolon without triggering a warning. +*/ +#define FCLI_VN(N,pfexp) \ + if(fcli.clientFlags.verbose>=(N)) do { \ + fcli_printf("VERBOSE %d: ", (int)(N)); \ + fcli_printf pfexp; \ + } while(0) + +/** + Convenience form of FCLI_VN for level-2 verbosity. +*/ +#define FCLI_V2(pfexp) FCLI_VN(2,pfexp) + +/** + Convenience form of FCLI_VN for level-1 verbosity. Levels 1 and 2 + are intended for application-level use. Levels 3+ are intended for + fcli use. +*/ +#define FCLI_V(pfexp) FCLI_VN(1,pfexp) + +#if defined(__cplusplus) +extern "C" { +#endif + + +/** + Result codes specific to the fcli API. +*/ +enum fcli_rc_e { +/** + For use with fcli_flag_callback_f() implementations to indicate + that the flag processor should check for that flag again. +*/ +FCLI_RC_FLAG_AGAIN = FSL_RC_end + 1, +/** + Returned from fcli_setup() if flag processing invokes the help + system. This is an indication that the app should exit immediately + with a 0 result code. +*/ +FCLI_RC_HELP +}; + +/** + Types for use with the fcli_cliflag::flagType field. The value of + that field determines how the CLI flag handling interprets the + fcli_cliflag::flagValue void pointer. +*/ +enum fcli_cliflag_type_e { +/** Sentinel placeholder. */ +FCLI_FLAG_TYPE_INVALID = 0, +/** Represents bool. */ +FCLI_FLAG_TYPE_BOOL, +/** Represents bool, but gets set to false if the flag is set. */ +FCLI_FLAG_TYPE_BOOL_INVERT, +/** Represents int32_t. */ +FCLI_FLAG_TYPE_INT32, +/** Represents int64_t. */ +FCLI_FLAG_TYPE_INT64, +/** Represents fsl_id_t, which might be either int32_t or int64_t. */ +FCLI_FLAG_TYPE_ID, +/** Represents double. */ +FCLI_FLAG_TYPE_DOUBLE, +/** Represents (char const *). */ +FCLI_FLAG_TYPE_CSTR +}; + +typedef struct fcli_cliflag fcli_cliflag; +/** + Callback handler for CLI flag handling. It is passed the flag in + question, containing its state after fcli has processed its + flagValue part. + + If the callback returns FCLI_RC_FLAG_AGAIN, the flag handler will + check for that flag again (for as long as the handler keeps + returning that value and as long as the flag repeats in the + argument list). This can be used to implement repeatable flags. + + If it returns any other non-0 value, fcli_setup() + resp. fcli_process_flags() will fail with that result code. + + Achtung: during fcli_setup(), at the time these are processed, fcli + will not yet have opened up a repository or checkout, so the + callback will not have access to things like symbolic-name-to-UUID + conversion at this level. Apps which need to do such work on their + arguments must first queue up the input and process it after + fcli_setup() returns. Apps which process the flags using + fcli_process_flags() after setup is complete will have access to + such features. +*/ +typedef int (*fcli_flag_callback_f)(fcli_cliflag const *); + +/** + Under construction. A reworking of fcli's CLI flag handling, + primarily so that we can unify the generation of app help text and + make it consistent across apps. + + @see fcli_process_flags() +*/ +struct fcli_cliflag { + /** + "Short-form" for this flag, noting that there's no restriction on + its length. Either of flagShort and flagLong, but not both, may be + NULL. + */ + const char * flagShort; + /** + "Long-form" for this flag, noting that there's no restriction on + its length. + */ + const char * flagLong; + + /** + Specifies how to interpret the data pointed to by + this->flagValue. See that member for details. + */ + enum fcli_cliflag_type_e flagType; + + /** + If not NULL, the member is the target of the flag's value. + Its exact interpretation depends on the value of this->flagType: + + - FCLI_FLAG_TYPE_BOOL: a pointer to a bool. Existence + of the flag causes it to be set to true. + + - FCLI_FLAG_TYPE_BOOL_INVERT: a pointer to a bool, but + existence of the flag causes it to be set to false. + + - FCLI_FLAG_TYPE_INT32: a pointer to an int32_t. The flag + will be converted from string to int using atoi(). + + - FCLI_FLAG_TYPE_INT64: a pointer to an int64_t. The flag + will be converted from string to int using atoll(). + + - FCLI_FLAG_TYPE_ID: a pointer to a fsl_id_t, which + can be either int32_t or int64_t. + + - FCLI_FLAG_TYPE_DOUBLE: a pointer to a double. The flag + will be converted from string to int using strtod(). + + - FCLI_FLAG_TYPE_CSTR: a pointer to a (const char *). The flag's + value will be assigned on without further interpretation. + + If this member is not NULL: If the flag is set, this value will + be assigned to. If the flag is not set in the CLI args, this + value is not written to. + + If this member is NULL then fcli_process_flags() will simply skip + over it, but the help-text generator will process it. This can be + used to set up flags which will appear in the --help text but + which are processed separately (or outright ignored) by the app. + + The underlying flag value's string memory is owned by fcli and is + valid until the app exits. + */ + void * flagValue; + + /** + Optional descriptive label to be used when rendering help text: + + -x|--x-flag=ABCD + + If it is NULL, the --help text generator will choose a value of + the ABCD part which depends on this->flagType. + + This is only used if this->flagType is not one of + (FCLI_FLAG_TYPE_BOOL, FCLI_FLAG_TYPE_BOOL_INVERT). For those + types (which have no client-given values), this member is + ignored when generating help text. + */ + const char * flagValueLabel; + + /** + Optional callback which gets passed the flag, if it is set, after + fcli has assigned the flagValue entry (if it is not NULL). + See this data type's docs for more details. + */ + fcli_flag_callback_f callback; + + /** + Help text for the flag. Intended to be displayed in the context + of a --help listing of flags. + */ + const char * helpText; +}; +#define fcli_cliflag_empty_m {\ + 0/*flagShort*/,0/*flagLong*/,\ + FCLI_FLAG_TYPE_INVALID/*flagType*/,\ + NULL/*flagValue*/,\ + NULL/*flagValueLabel*/,NULL/*callback*/,NULL/*helpText*/} +/** Non-const-copyable counterpart of fcli_cliflag_empty_m. */ +FSL_EXPORT const fcli_cliflag fcli_cliflag_empty; + +/** @def FCLI_FLAG_xxx + + The various FCLI_FLAG_xxx macros are convenience-form initializers + for fcli_cliflag instances for use in initializing a fcli_cliflag + array. Example usage: + + ``` + bool flag1 = true, flag2 = false; + int32_t flag3 = 0; + const char * flag4 = 0; + const fcli_cliflag cliFlags[] = { + FCLI_FLAG_BOOL("x","xyz",&flag1,"Flag 1."), + FCLI_FLAG_BOOL_INVERT(NULL,"yzx",&flag2,"Flag 2."), + FCLI_FLAG_INT32("z",NULL,"value",&flag3,"Flag 3"), + FCLI_FLAG("f","file","filename",&flag4, + "Input file. May optionally be passed as the first " + "non-flag argument."), + fcli_cliflag_empty_m // list MUST end with this (or equivalent) + }; + fcli.cliFlags = cliFlags; + ``` + + BE CAREFUL with the data types, as we're using (void*) to access + data of an arbitrary type, the type being defined by a separate + field in the fcli_cliflag object. +*/ +#define FCLI_FLAG_xxx /* for doc purposes only */ +//Members: +//{short, long, type, tgt, valueDescr., callback, help} +/** Bool-type flag. */ +#define FCLI_FLAG_BOOL(S,L,TGT,HELP) \ + {S,L,FCLI_FLAG_TYPE_BOOL,TGT,NULL,NULL,HELP} +/** Bool-type flag, but set to false if flag is set. */ +#define FCLI_FLAG_BOOL_INVERT(S,L,TGT,HELP) \ + {S,L,FCLI_FLAG_TYPE_BOOL_INVERT,TGT,NULL,NULL,HELP} +/** Bool-type flag with a callback. */ +#define FCLI_FLAG_BOOL_X(S,L,TGT,CALLBACK,HELP) \ + {S,L,FCLI_FLAG_TYPE_BOOL,TGT,NULL,CALLBACK,HELP} +/** Bool-type flag with a callback, but set to false if the flag is set. */ +#define FCLI_FLAG_BOOL_INVERT_X(S,L,TGT,CALLBACK,HELP) \ + {S,L,FCLI_FLAG_TYPE_BOOL_INVERT,TGT,NULL,CALLBACK,HELP} +/** String-type flag. */ +#define FCLI_FLAG_CSTR(S,L,LBL,TGT,HELP) \ + {S,L,FCLI_FLAG_TYPE_CSTR,TGT,LBL,NULL,HELP} +#define FCLI_FLAG FCLI_FLAG_CSTR +/** String-type flag with a callback. */ +#define FCLI_FLAG_CSTR_X(S,L,LBL,TGT,CALLBACK,HELP) \ + {S,L,FCLI_FLAG_TYPE_CSTR,TGT,LBL,CALLBACK,HELP} +#define FCLI_FLAG_X FCLI_FLAG_CSTR_X +/** int32-type flag. */ +#define FCLI_FLAG_I32(S,L,LBL,TGT,HELP) \ + {S,L,FCLI_FLAG_TYPE_INT32,TGT,LBL,NULL,HELP} +/** int32-type flag with a callback. */ +#define FCLI_FLAG_I32_X(S,L,LBL,TGT,CALLBACK,HELP) \ + {S,L,FCLI_FLAG_TYPE_INT32,TGT,LBL,CALLBACK,HELP} +/** int32-type flag. */ +#define FCLI_FLAG_I64(S,L,LBL,TGT,HELP) \ + {S,L,FCLI_FLAG_TYPE_INT64,TGT,LBL,NULL,HELP} +/** int32-type flag with a callback. */ +#define FCLI_FLAG_I64_X(S,L,LBL,TGT,CALLBACK,HELP) \ + {S,L,FCLI_FLAG_TYPE_INT64,TGT,LBL,CALLBACK,HELP} +/** fsl_id_t-type flag. */ +#define FCLI_FLAG_ID(S,L,LBL,TGT,HELP) \ + {S,L,FCLI_FLAG_TYPE_ID,TGT,LBL,NULL,HELP} +/** fsl_id_t-type flag with a callback. */ +#define FCLI_FLAG_ID_X(S,L,LBL,TGT,CALLBACK,HELP) \ + {S,L,FCLI_FLAG_TYPE_ID,TGT,LBL,CALLBACK,HELP} +/** double-type flag. */ +#define FCLI_FLAG_DBL(S,L,LBL,TGT,HELP) \ + {S,L,FCLI_FLAG_TYPE_DOUBLE,TGT,LBL,NULL,HELP} +/** double-type flag with a callback. */ +#define FCLI_FLAG_DBL_X(S,L,LBL,TGT,CALLBACK,HELP) \ + {S,L,FCLI_FLAG_TYPE_DOUBLE,TGT,LBL,CALLBACK,HELP} + +/** + Structure for holding app-level --help info. +*/ +struct fcli_help_info { + /** Brief description of what the app does. */ + char const * briefDescription; + /** Brief usage text. It will be prefixed by the app's name and + "[options]". e.g. "file1 ... fileN". + */ + char const * briefUsage; + /** + If not 0 then this is called after outputing any flags' help. + It should output any additional help text using f_out(). + */ + void (*callback)(void); +}; +typedef struct fcli_help_info fcli_help_info; + +/** + The fcli_t type, accessed by clients via the ::fcli + global instance, contains various data for managing a basic + Fossil SCM application build using libfossil. + + Usage notes: + + - All non-const pointer members are owned and managed by the + fcli API. Clients may reference them but must be aware of + lifetimes if they plan to hold the reference for long. + +*/ +struct fcli_t { + /** + If not NULL, it must be a pointer to fcli_help_info holding help + info for the app. It will be formated and output when --help + is triggered. + + Should be set by client applications BEFORE calling fcli_setup() + so that the ::fcli help subsystem can integrate the + app. fcli.appName will be set by the time this is used. + */ + fcli_help_info const * appHelp; + + /** + May be set to an array of CLI flag objects, which fcli_setup() + will use for parsing the CLI flags. The array MUST end with an + entry which has NULL values for its (flagShort, flagLong) + members. When creating the array it is simplest to use + fcli_cliflag_empty_m as the initializer for that sentinel entry. + + The elements in this array are traversed in the order they are + provided, and any which have a callback which returns + FCLI_RC_FLAG_AGAIN will be traversed repeatedly before moving on + to the next flag. This ordering may have side-effects on how the + app sets up its flag handling. In particular, this arrangement + makes the common cases easy to implement but makes certain more + complicated situations effectively impossible to implement. For + example: + + ``` + -x=1 -y=2 -x=3 -y=4 + ``` + + Assuming the definition of the -y flag is first in this array and + has a callback which returns FCLI_RC_FLAG_AGAIN, the two -x flags + will be processed before either of the -y flags. Thus it is + impossible (using this method of traversal) to change the + behavior of the -y flags based on the left-closest -x value. + + We "could," but probably never will, instead walk the input argv + looking for things which look like flags, then looking back into + this list for a match, rather than the other way around. That + would require a new args processing implementation, though, for + relatively little benefit. + */ + const fcli_cliflag * cliFlags; + + /** + The shared fsl_cx instance. It gets initialized by + fcli_setup() and cleaned up post-main(). + + this->f is owned by this object and will be cleaned up at app + shutdown (post-main). + */ + fsl_cx * f; + /** + The current list of CLI arguments. This list gets modified by + fcli_flag() and friends. Its memory is owned by fcli. + */ + char ** argv; + /** + Current number of items in this->argv. + */ + int argc; + /** + Application's name. Currently argv[0] but needs to be + adjusted for Windows platforms. + */ + char const * appName; + /** + The flags in this struct are "public," meaning that client + applications may query and safely manipulate them directly if + they know what they're doing. + */ + struct { + /** + If not NULL then fcli_setup() will attempt to open the + checkout for the given dir, including its associated repo + db. By default this is "." (the current directory). + + Applications can set this to NULL _before_ calling + fcli_setup() in order to disable the automatic attemp to + open a checkout under the current directory. Doing so is + equivalent to using the --no-checkout|-C flags. The global + --checkout-dir flag will trump that setting, though. + */ + char const * checkoutDir; + + /** + A verbosity level counter. Starts at 0 (no verbosity) and goes + up for higher verbosity levels. Currently levels 1 and 2 are + intended for app-level use and level 3 for library-level use. + */ + unsigned short verbose; + } clientFlags; + /** + Transient settings and flags. These are bits which are used + during (or very shortly after) fcli_setup() but have no effect + if modified after that. + */ + struct { + /** + repo db name string from -R/--repo CLI flag. + */ + const char * repoDbArg; + /** + User name from the -U/--user CLI flag. + */ + const char * userArg; + /** + Incremented if fcli_setup() detects -? or --help in the + argument list, or if the first non-flag argument is "help". + */ + short helpRequested; + /** + True if the --lib-version CLI flag was used. This causes + fcli setup to exit as if --help had been invoked. + */ + bool versionRequested; + } transient; + /** + Holds bits which can/should be configured by clients BEFORE + calling fcli_setup(). + */ + struct { + /** + Whether or not to enable fossil's SQL tracing. This should + start with a negative value, which helps fcli_setup() process + it. Setting this after initialization has no effect. + */ + int traceSql; + /** + This output channel is used when initializing this->f. The + default implementation uses fsl_outputer_FILE to output to + stdout. + */ + fsl_outputer outputer; + + } config; + /** + For holding pre-this->f-init error state. Once this->f is + initialized, all errors reported via fcli_err_set() are stored in + that object's error state. + */ + fsl_error err; +}; +typedef struct fcli_t fcli_t; + +/** @var fcli + + This fcli_t instance is intended to act as a singleton. It holds + all fcli-related global state. It gets initialized with + default/empty state at app startup and gets fully initialized via + fcli_setup(). See that routine for an example of how it is + typically initialized. + + fcli_cx() returns the API's fsl_cx instance. It will be non-NULL + (but might not have an opened checkout/repository) if fsl_setup() + succeeds. +*/ +FSL_EXPORT fcli_t fcli; + +/** + Equivalent to `fcli_setup_v2(argc,argv,NULL,NULL)`. + + @see fcli_pre_setup() + @see fcli_setup_v2() + @see fcli_end_of_main() + @deprecated Its signature will change to fcli_setup_v2()'s at some point. +*/ +FSL_EXPORT int fcli_setup(int argc, char const * const * argv ); + +/** + Initializes fcli's state and CLI flags processing. + + MUST BE CALLED BEFORE fsl_malloc() and friends are used, as this + swaps out the allocator with one which aborts on OOM. (But see + fcli_pre_setup() for a workaround for that.) + + Should be called early on in main(), passed the arguments passed to + main(). Returns 0 on success. This sets up the ::fcli instance and + opens any checkout found in/above the current dir by default. To + disable the automatic search for a checkout, assign + `fcli.clientFlags.checkoutDir` to `NULL` before calling this. + + If the 3rd argument is not NULL, this sets `fcli.cliFlags` to its + value. If the 4th argument is not NULL, this sets `fcli.appHelp` to + that value. + + If argument processing finds either of the (--help, -?) flags, + or the first non-flag argument is "help", it sets + fcli.transient.helpRequested to a true value, calls fcli_help(), + and returns FCLI_RC_HELP, in which case the application should + exit/return from main with code 0 immediately. + + This function behaves significantly differently if fcli.cliFlags + has been set before it is called. In that case, it parses the CLI + flags using that type's rules and sets up fcli_help() to use those + flags for generating the help. It parses the global flags first, + then the app-specific flags. + + Returns 0 on success. Results other than FCLI_RC_HELP should be + treated as fatal to the app, and fcli.f's error state _might_ + contain info about the error. If this function returns non-0, the + convention is that the app immediately returns the result of + fcli_end_of_main(THE_RESULT_CODE) from main(). That function will + treat FCLI_RC_HELP as a non-error and will report any error state + pending in the fcli_cx() object. + + Example of intended basic usage: + + ``` + int main(int argc, char const * const * argv){ + ... + // Optional fcli_cliflag setup: + fcli_cliflag const cliFlags[] = { + ..., + fcli_cliflag_empty_m + }; + // Optional fcli_help_info setup: + fcli_help_info const help = { + "Fnoobs the borts and rustles the feathers.", + "file1 [... fileN]", + NULL // optional callback to display extra help + }; + // Initialize... + int rc = fcli_setup_v2(argc, argv, cliFlags, &help); + if(rc) goto end; + + ... app logic ... + + end: + return fcli_end_of_main(rc); + } + ``` + + @see fcli_pre_setup() + @see fcli_setup() + @see fcli_end_of_main() +*/ +FSL_EXPORT int fcli_setup_v2(int argc, char const * const * argv, + fcli_cliflag const * const cliFlags, + fcli_help_info const * const helpInfo ); + +/** + The first time this is called, it swaps out libfossil's default + allocator with a fail-fast one which abort()s on allocation + error. This is normally called by fcli_setup(), but that also means + that it's illegal to use fsl_malloc() and friends before calling + that routine. If an application really needs to use fsl_malloc() + before calling fcli_setup(), it must call this first in order to + get the allocator initialization out of the way. + + Calls after the first are no-ops, but the check for that is not + thread-safe. Neither is fcli, though, so that's okay. +*/ +FSL_EXPORT void fcli_pre_setup(void); + +/** + Returns the libfossil context associated with the fcli API. + This will be NULL until fcli_setup() is called. +*/ +FSL_EXPORT fsl_cx * fcli_cx(void); + +/** + Works like printf() but sends its output to fsl_outputf() using + the fcli.f fossil conext (if set) or fsl_fprintf() (to + stdout). +*/ +FSL_EXPORT void fcli_printf(char const * fmt, ...) +#if 0 +/* Would be nice, but complains about our custom format options: */ + __attribute__ ((__format__ (__printf__, 1, 2))) +#endif + ; + +/** + f_out() is a shorthand for fcli_printf(). +*/ +#define f_out fcli_printf + +/** + Returns the verbosity level set via CLI args. 0 is no verbosity, + and one level is added each time the --verbose/-V CLI flag is + encountered by fcli_setup(). +*/ +FSL_EXPORT unsigned short fcli_is_verbose(void); + +/** + Searches fcli.argv for the given flag (pass it without leading + dashes). If found, this function returns true, else it returns + false. If value is not NULL then the flag, if found, is assumed to + have a value, otherwise the flag is assumed to be a boolean. A flag + with a value may take either one of these forms: + + -flag=value + -flag value + + *value gets assigned to a COPY OF the value part of the first form + or a COPY OF the subsequent argument for the second form (copies + are required in order to avoid trickier memory management + here). That copy is owned by fcli and will be cleaned up at app + exit. On success it removes the flag (and its value, if any) from + fcli.argv. Thus by removing all flags early on, the CLI arguments + are left only with non-flag arguments to sift through. + + Flags may start with either one or two dashes - they are + equivalent. + + This function may update the fcli error state if. Specifically, if + passed a non-NULL 2nd argument and the flag is found at the end of + the argument list or the value immediately after the flag starts + with '-' then the error state will be updated and false will be + returned. Apps don't normally need to be quite that picky with + their error checking after looking for a flag, but the state is + there if needed. It may get reset by any future calls into the API, + though. fcli_process_flags() does check this state and will fail if + such an error is triggered. +*/ +FSL_EXPORT bool fcli_flag(char const * opt, const char ** value); + +/** + Works like fcli_flag() but tries two argument forms, in order. It + is intended to be passed short and long forms, but can be passed + two aliases or similar. It accepts NULL for either form. + + ``` + const char * v = NULL; + fcli_flag2("n", "limit", &v); + if(v) { ... } + ``` +*/ +FSL_EXPORT bool fcli_flag2(char const * opt1, char const * opt2, + const char ** value); + +/** + Works similarly to fcli_flag2(), but if no flag is found and + value is not NULL then *value is assigned to the return value of + fcli_next_arg(true). In that case: + + - The return value will specify whether or not fcli_next_arg() + returned a value or not. + + - If it returns true then *value is owned by fcli and will be + cleaned up at app exit. + + - If it returns false, *value is not modified. + + The opt2 parameter may be NULL, but op1 may not. +*/ +FSL_EXPORT bool fcli_flag_or_arg(char const * opt1, char const * opt2, + const char ** value); + + +/** + Clears any error state in fcli.f. +*/ +FSL_EXPORT void fcli_err_reset(void); + +/** + Sets fcli.f's error state, analog to fsl_cx_err_set(). + Returns the code argument on success, some other non-0 value on + a more serious error (e.g. FSL_RC_OOM when formatting the + string). +*/ +FSL_EXPORT int fcli_err_set(int code, char const * fmt, ...); + +/** + Returns the internally-used fsl_error instance which is used for + propagating errors. The object is owned by ::fcli and MUST NOT + be freed or otherwise abused by clients. It may, however, be + passed to routines which take a fsl_error parameter to report + errors (e.g. fsl_deck_output(). + + Returns NULL if fcli_setup() has not yet been called or after + fcli has been cleaned up (post-main()). + +*/ +FSL_EXPORT fsl_error * fcli_error(void); + +/** + If ::fcli has any error state, this outputs it and returns the + error code, else returns 0. If clear is true the error state is + cleared/reset, otherwise it is left in place. Returns 0 if + ::fcli has not been initialized. The 2nd and 3rd arguments are + assumed to be the __FILE__ and __LINE__ macro values of the call + point. See fcli_err_report() for a convenience form of this + function. + + The format of the output depends partially on fcli_is_verbose(). In + verbose mode, the file/line info is included, otherwise it is + elided. + + @see fcli_err_report() +*/ +FSL_EXPORT int fcli_err_report2(bool clear, char const * file, int line); + +/** + Convenience macro for using fcli_err_report2(). +*/ +#define fcli_err_report(CLEAR) fcli_err_report2((CLEAR), __FILE__, __LINE__) + +/** + Peeks at or takes the next argument from the CLI args. If the + argument is true, it is removed from the args list. It is owned by + fcli and will be freed when the app exits. +*/ +FSL_EXPORT const char * fcli_next_arg(bool remove); + +/** + If fcli.argv contains what looks like any flag arguments, this + updates the fossil error state and returns FSL_RC_MISUSE, else + returns 0. If outputError is true and an unused flag is found + then the error state is immediately output (but not cleared). +*/ +FSL_EXPORT int fcli_has_unused_flags(bool outputError); +/** + If fcli.argv contains any entries, returns FSL_RC_MISUSE and + updates the error state with a message about unusued extra + arguments, else returns 0. If outputError is true and an unconsumed + argument is found then the error state is immediately output (but + not cleared). +*/ +FSL_EXPORT int fcli_has_unused_args(bool outputError); + +typedef struct fcli_command fcli_command; +/** + Typedef for general-purpose fcli call-by-name commands. + + It gets passed its own command definition, primarily for each of + access to the flags member for CLI flags processing. + + @see fcli_dispatch_commands() +*/ +typedef int (*fcli_command_f)(fcli_command const *); + +/** + Describes a named callback command. + + @see fcli_dispatch_commands() +*/ +struct fcli_command { + /** The name of the command. */ + char const * name; + /** + NUL-delimited string containing optional aliases of the above command + name. This string _must_ be double-NUL terminated (e.g., "alias\0\al\0"). + */ + char const * aliases; + /** Brief description, for use in generating help text. */ + char const * briefDescription; + /** The callback for this command. */ + fcli_command_f f; + /** Optional usage callback for this command. */ + void (*usage)(void); + /** + Must be NULL or an array compatible with fcli_process_flags(). + Can be used from within this->f for command-specific flag + dispatching, as well as help text generation. + */ + fcli_cliflag const * flags; +}; + +/** + Expects an array of fcli_commands which contain a trailing + sentry entry with a NULL name and callback. It searches the list + for a command matching fcli_next_arg(). If found, it + removes that argument from the list, calls the callback, and + returns its result. If no command is found FSL_RC_NOT_FOUND is + returned, the argument list is not modified, and the error state + is updated with a description of the problem and a list of all + command names in cmdList. + + If reportErrors is true then on error this function outputs + the error result but it keeps the error state in place + for the downstream use. + + As a special case: when a command matches the first argument and + that object has a non-NULL flags member, this function checks the + _next_ argument, and if it is "help" then this function passes + that flags member to fcli_command_help() to output help, then + returns 0. +*/ +FSL_EXPORT int fcli_dispatch_commands( fcli_command const * cmdList, + bool reportErrors); + +/** + Parse cmd->aliases for an alias (i.e., NUL-terminated string semantically + equivalent to cmd->name) matching arg. Return true if found. Note that + cmd->aliases _must_ be double-NUL terminated. +*/ +FSL_EXPORT bool fcli_cmd_aliascmp( fcli_command const * cmd, char const * arg ); + +/** + A minor helper function intended to be passed the pending result + code of the main() routine. This function outputs any pending + error state in fcli. Returns one of EXIT_SUCCESS if mainRc is 0 + and fcli had no pending error report, otherwise it returns + EXIT_FAILURE. This function does not clean up fcli - that is + handled via an atexit() handler. + + It is intended to be called once at the very end of main: + + ``` + int main(){ + int rc; + + ...set up fcli...assign rc... + + return fcli_end_of_main(rc); + } + ``` + + As a special case, if mainRc is FCLI_RC_HELP, it is assumed to + be the result of the fcli --help flag handling, and is treated as + if it were 0. + + @see fcli_error() + @see fcli_err_set() + @see fcli_err_report() + @see fcli_err_reset() +*/ +FSL_EXPORT int fcli_end_of_main(int mainRc); + +/** + If mem is not NULL, this routine appends mem to a list of pointers + which will be passed to fsl_free() during the atexit() shutdown + phase of the app. Because fcli uses a fail-fast allocator, failure + to append the entry will itself cause a crash. This is only useful + for values for which fsl_free() suffices to clean them up, not + complex values like multi-dimensional arrays. + + "fax" is short for "free at exit." + + Results are undefined if the same address or overlapping addresses + are queued more than once. Once an entry is in this queue, there is + no way to remove it. +*/ +FSL_EXPORT void fcli_fax(void * mem); + +/** + Requires an array of fcli_cliflag objects terminated with an + instance with NULL values for the (flagShort, flagLong) members + (fcli_cliflag_empty_m is an easy way to get that). + + If fcli.cliFlags is set before fcli_setup() is called, this routine + is called and passed those flags. Thus most apps can simply assign + their flags there and let setup() do the work. Apps which have + multiple dispatch paths, each with differing flags, may find it + easier to use this. + + As a special case, if a given entry has NULL values for both of its + (flagValue, callback) members, it is assumed to exist purely for + use with the help-generating mechanisms and the flag is NOT + processed or consumed by fcli_process_flags(). That can be used + when the client needs to process the flag in ways beyond what is + capable via this routine. In such cases, add an appropriate entry + to the fcli_cliflag array for --help purposes and then process the + flag using fcli_flag() (or similar) either before or after calling + this. + + Returns 0 on success. Returns non-0 only if a callback() member of + one of the entries returns a value other than FCLI_RC_FLAG_AGAIN. +*/ +FSL_EXPORT int fcli_process_flags( fcli_cliflag const * defs ); + +/** + Requires an array of fcli_cliflag objects as described for + fcli_process_flags(). This routine outputs their flags + and help text in a framework-conventional manner. + + If fcli.cliFlags is set before fcli_setup() is called, this routine + is called and passed those flags. Thus most apps can simply assign + their flags there and let fcli_setup() do the work. Apps which have + multiple dispatch paths, each with differing flags, may find it + easier to use this. + + Such apps, in order to get the full help listing for all dispatch + paths, may need to assign fcli.appHelp to a helper which + dispatches to an app-local help routine which, in turn, passes each + of their separate fcli_cliflag lists to this routine. +*/ +FSL_EXPORT void fcli_cliflag_help(fcli_cliflag const *defs); + +/** + Requires that cmd be an array of fcli_command objects with a + trailing entry which has a NULL name. This function iterates over + them and outputs help text based on each one's (name, + briefDescription, flags) members. + + If the 2nd argument is true AND the fcli_command->usage() callback + has been set, it will be called for each object in the cmd array + after outputting the fcli_command->flags help text. + + If the 3rd argument is true, only the help for the single given + object is output, not any adjacent array members (if any). +*/ +FSL_EXPORT void fcli_command_help(fcli_command const * cmd, bool showUsage, bool onlyOne); + +/** + Pretty print fcli_command->aliases like: " (aliases: cmd, com)". +*/ +FSL_EXPORT void fcli_help_show_aliases(char const * aliases); + +/** + If fcli has a checkout opened, this dumps various info about it + to its output channel. Returns 0 on success, FSL_RC_NOT_A_CKOUT + if no checkout is opened, or some other non-0 code on error. + + If the useUtc argument is true, it uses UTC timestamps, else + localtime. (Potential TODO: add an fcli-level CLI flag for that + instead?) +*/ +FSL_EXPORT int fcli_ckout_show_info(bool useUtc); + +/** + Given a hash prefix, lists (via f_out()) all blob table entries + which have this prefix. This is intended to be used by apps which + accept a from a user and fsl_sym_to_rid() returns FSL_RC_AMBIGUOUS. + + The first argument is an optional output label/header to output. + If it is NULL, a default is used. If it is "" then no header is + output. +*/ +FSL_EXPORT void fcli_list_ambiguous_artifacts(char const * label, char const *prefix); + +/** + If fcli has an opened checkout, that db handle is returned, else NULL + is returned. +*/ +FSL_EXPORT fsl_db * fcli_db_ckout(void); + +/** + If fcli has an opened repository, that db handle is returned, else + NULL is returned. +*/ +FSL_EXPORT fsl_db * fcli_db_repo(void); + +/** + If fcli has an opened checkout, that db handle is returned, else NULL + is returned and fcli's error state is updated with a description + of the problem. +*/ +FSL_EXPORT fsl_db * fcli_needs_ckout(void); + +/** + If fcli has an opened repository, that db handle is returned, else NULL + is returned and fcli's error state is updated with a description + of the problem. +*/ +FSL_EXPORT fsl_db * fcli_needs_repo(void); + +/** + Processes all remaining CLI arguments as potential file or directory + names, collects their vfile.id values, and stores them in the given + target bag. It requires an opened checkout. + + vid is the vfile.vid value to filter on. If vid<=0 then the current + checkout version is used. (Unless the app has explicitly loaded + another version, that will be the only option available.) + + If relativeToCwd is true then each argument is resolved as if + referenced from the current working directory, else each is assumed + to be relative to the top of the checkout directory. (For CLI apps, + a value of true is almost always the right choice.) + + If changedFilesOnly is true then only files which are "changed", + according to the vfile table (as opposed to a filesystem check) are + considered for addition. For that to work, vfile must be up to + date, so fsl_vfile_changes_scan() must have been recently called to + update that state. This function does not call it automatically + because it's relatively slow and many apps already have to call it + on their own. + + This function matches only vfile.pathname, not vfile.origname, + because it is possible for a given name to be in both fields (in + different records) at the same time. + + Returns 0 on success. If there are no more CLI arguments when it is + called then it returns FSL_RC_MISUSE and updates the fcli error + state with a description of the problem. It may return any number + of non-0 codes from the underlying operations. + + Sidebar: fsl_filename_to_vfile_ids() requires that directory names + passed to it have no trailing slashes, and routine strips trailing + slashes from its arguments before passing them on to that routine, + so they may be entered with slashes without ill effect. + + @see fsl_filename_to_vfile_ids() +*/ +FSL_EXPORT int fcli_args_to_vfile_ids(fsl_id_bag *tgt, fsl_id_t vid, + bool relativeToCwd, + bool changedFilesOnly); + +/** + Performs a "fingerprint check" on the current checkout/repo + combination, as per fsl_ckout_fingerprint_check(). If the check + fails and reportImmediately is true then an error report is + immediately output. Returns 0 if the fingerprint check is okay, + else a non-0 value as per fsl_ckout_fingerprint_check(). + + Passing true here may output more information than the underlying + fsl_cx-level error state provides. e.g. it may provide a hint about + how to recover. +*/ +FSL_EXPORT int fcli_fingerprint_check(bool reportImmediately); + +/** + Returns the "tail" part of the argv[0] string which was passed to + fcli_setup(), or NULL if neither of those have yet been called. The + "tail" part is the part immediately after the final '/' or '\\' + character. +*/ +FSL_EXPORT char const * fcli_progname(); + +/** + Color theme IDs for use with fcli_diff_colors. + */ +enum fcli_diff_colors_e{ +/** + Tells fcli_diff_colors() to NULL out the ANSI color state of its + target object. +*/ +FCLI_DIFF_COLORS_NONE = 0, +/** + Basic red (deletion) and green (insertion) diff colors. +*/ +FCLI_DIFF_COLORS_RG, +FCLI_DIFF_COLORS_DEFAULT = FCLI_DIFF_COLORS_RG +}; +typedef enum fcli_diff_colors_e fcli_diff_colors_e; + +/** + Populates the given fsl_diff_opt::ansiColors state with values + dependend on the second argument. +*/ +FSL_EXPORT void fcli_diff_colors(fsl_diff_opt * const tgt, + fcli_diff_colors_e theme); + +/** @internal + + This function is intented for use in development of libfossil. It + dumps the current state of cached SQL statements to fcli_printf(). + + Normally its output detail level is determined by + fcli_is_verbose(), but if forceVerbose is true then it cranks up + the detail all the way. + */ +FSL_EXPORT void fcli_dump_stmt_cache(bool forceVerbose); + +#if defined(__cplusplus) +} /*extern "C"*/ +#endif + + +#endif +/* _ORG_FOSSIL_SCM_FCLI_H_INCLUDED_ */ +/* end of file ../include/fossil-scm/fossil-cli.h */ ADDED lib/sqlite3.c Index: lib/sqlite3.c ================================================================== --- lib/sqlite3.c +++ lib/sqlite3.c @@ -0,0 +1,236391 @@ +/****************************************************************************** +** This file is an amalgamation of many separate C source files from SQLite +** version 3.37.0. By combining all the individual C code files into this +** single large file, the entire code can be compiled as a single translation +** unit. This allows many compilers to do optimizations that would not be +** possible if the files were compiled separately. Performance improvements +** of 5% or more are commonly seen when SQLite is compiled as a single +** translation unit. +** +** This file is all you need to compile SQLite. To use SQLite in other +** programs, you need this file and the "sqlite3.h" header file that defines +** the programming interface to the SQLite library. (If you do not have +** the "sqlite3.h" header file at hand, you will find a copy embedded within +** the text of this file. Search for "Begin file sqlite3.h" to find the start +** of the embedded sqlite3.h header file.) Additional code files may be needed +** if you want a wrapper to interface SQLite with your choice of programming +** language. The code for the "sqlite3" command-line shell is also in a +** separate file. This file contains only code for the core SQLite library. +*/ +#define SQLITE_CORE 1 +#define SQLITE_AMALGAMATION 1 +#ifndef SQLITE_PRIVATE +# define SQLITE_PRIVATE static +#endif +/************** Begin file sqliteInt.h ***************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Internal interface definitions for SQLite. +** +*/ +#ifndef SQLITEINT_H +#define SQLITEINT_H + +/* Special Comments: +** +** Some comments have special meaning to the tools that measure test +** coverage: +** +** NO_TEST - The branches on this line are not +** measured by branch coverage. This is +** used on lines of code that actually +** implement parts of coverage testing. +** +** OPTIMIZATION-IF-TRUE - This branch is allowed to alway be false +** and the correct answer is still obtained, +** though perhaps more slowly. +** +** OPTIMIZATION-IF-FALSE - This branch is allowed to alway be true +** and the correct answer is still obtained, +** though perhaps more slowly. +** +** PREVENTS-HARMLESS-OVERREAD - This branch prevents a buffer overread +** that would be harmless and undetectable +** if it did occur. +** +** In all cases, the special comment must be enclosed in the usual +** slash-asterisk...asterisk-slash comment marks, with no spaces between the +** asterisks and the comment text. +*/ + +/* +** Make sure the Tcl calling convention macro is defined. This macro is +** only used by test code and Tcl integration code. +*/ +#ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI +#endif + +/* +** Include the header file used to customize the compiler options for MSVC. +** This should be done first so that it can successfully prevent spurious +** compiler warnings due to subsequent content in this file and other files +** that are included by this file. +*/ +/************** Include msvc.h in the middle of sqliteInt.h ******************/ +/************** Begin file msvc.h ********************************************/ +/* +** 2015 January 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to MSVC. +*/ +#ifndef SQLITE_MSVC_H +#define SQLITE_MSVC_H + +#if defined(_MSC_VER) +#pragma warning(disable : 4054) +#pragma warning(disable : 4055) +#pragma warning(disable : 4100) +#pragma warning(disable : 4127) +#pragma warning(disable : 4130) +#pragma warning(disable : 4152) +#pragma warning(disable : 4189) +#pragma warning(disable : 4206) +#pragma warning(disable : 4210) +#pragma warning(disable : 4232) +#pragma warning(disable : 4244) +#pragma warning(disable : 4305) +#pragma warning(disable : 4306) +#pragma warning(disable : 4702) +#pragma warning(disable : 4706) +#endif /* defined(_MSC_VER) */ + +#if defined(_MSC_VER) && !defined(_WIN64) +#undef SQLITE_4_BYTE_ALIGNED_MALLOC +#define SQLITE_4_BYTE_ALIGNED_MALLOC +#endif /* defined(_MSC_VER) && !defined(_WIN64) */ + +#endif /* SQLITE_MSVC_H */ + +/************** End of msvc.h ************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + +/* +** Special setup for VxWorks +*/ +/************** Include vxworks.h in the middle of sqliteInt.h ***************/ +/************** Begin file vxworks.h *****************************************/ +/* +** 2015-03-02 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Wind River's VxWorks +*/ +#if defined(__RTP__) || defined(_WRS_KERNEL) +/* This is VxWorks. Set up things specially for that OS +*/ +#include +#include /* amalgamator: dontcache */ +#define OS_VXWORKS 1 +#define SQLITE_OS_OTHER 0 +#define SQLITE_HOMEGROWN_RECURSIVE_MUTEX 1 +#define SQLITE_OMIT_LOAD_EXTENSION 1 +#define SQLITE_ENABLE_LOCKING_STYLE 0 +#define HAVE_UTIME 1 +#else +/* This is not VxWorks. */ +#define OS_VXWORKS 0 +#define HAVE_FCHOWN 1 +#define HAVE_READLINK 1 +#define HAVE_LSTAT 1 +#endif /* defined(_WRS_KERNEL) */ + +/************** End of vxworks.h *********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + +/* +** These #defines should enable >2GB file support on POSIX if the +** underlying operating system supports it. If the OS lacks +** large file support, or if the OS is windows, these should be no-ops. +** +** Ticket #2739: The _LARGEFILE_SOURCE macro must appear before any +** system #includes. Hence, this block of code must be the very first +** code in all source files. +** +** Large file support can be disabled using the -DSQLITE_DISABLE_LFS switch +** on the compiler command line. This is necessary if you are compiling +** on a recent machine (ex: Red Hat 7.2) but you want your code to work +** on an older machine (ex: Red Hat 6.0). If you compile on Red Hat 7.2 +** without this option, LFS is enable. But LFS does not exist in the kernel +** in Red Hat 6.0, so the code won't work. Hence, for maximum binary +** portability you should omit LFS. +** +** The previous paragraph was written in 2005. (This paragraph is written +** on 2008-11-28.) These days, all Linux kernels support large files, so +** you should probably leave LFS enabled. But some embedded platforms might +** lack LFS in which case the SQLITE_DISABLE_LFS macro might still be useful. +** +** Similar is true for Mac OS X. LFS is only supported on Mac OS X 9 and later. +*/ +#ifndef SQLITE_DISABLE_LFS +# define _LARGE_FILE 1 +# ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +# endif +# define _LARGEFILE_SOURCE 1 +#endif + +/* The GCC_VERSION and MSVC_VERSION macros are used to +** conditionally include optimizations for each of these compilers. A +** value of 0 means that compiler is not being used. The +** SQLITE_DISABLE_INTRINSIC macro means do not use any compiler-specific +** optimizations, and hence set all compiler macros to 0 +** +** There was once also a CLANG_VERSION macro. However, we learn that the +** version numbers in clang are for "marketing" only and are inconsistent +** and unreliable. Fortunately, all versions of clang also recognize the +** gcc version numbers and have reasonable settings for gcc version numbers, +** so the GCC_VERSION macro will be set to a correct non-zero value even +** when compiling with clang. +*/ +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif + +/* +** Some C99 functions in "math.h" are only present for MSVC when its version +** is associated with Visual Studio 2013 or higher. +*/ +#ifndef SQLITE_HAVE_C99_MATH_FUNCS +# if MSVC_VERSION==0 || MSVC_VERSION>=1800 +# define SQLITE_HAVE_C99_MATH_FUNCS (1) +# else +# define SQLITE_HAVE_C99_MATH_FUNCS (0) +# endif +#endif + +/* Needed for various definitions... */ +#if defined(__GNUC__) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE +#endif + +#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) +# define _BSD_SOURCE +#endif + +/* +** Macro to disable warnings about missing "break" at the end of a "case". +*/ +#if GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +#else +# define deliberate_fall_through +#endif + +/* +** For MinGW, check to see if we can include the header file containing its +** version information, among other things. Normally, this internal MinGW +** header file would [only] be included automatically by other MinGW header +** files; however, the contained version information is now required by this +** header file to work around binary compatibility issues (see below) and +** this is the only known way to reliably obtain it. This entire #if block +** would be completely unnecessary if there was any other way of detecting +** MinGW via their preprocessor (e.g. if they customized their GCC to define +** some MinGW-specific macros). When compiling for MinGW, either the +** _HAVE_MINGW_H or _HAVE__MINGW_H (note the extra underscore) macro must be +** defined; otherwise, detection of conditions specific to MinGW will be +** disabled. +*/ +#if defined(_HAVE_MINGW_H) +# include "mingw.h" +#elif defined(_HAVE__MINGW_H) +# include "_mingw.h" +#endif + +/* +** For MinGW version 4.x (and higher), check to see if the _USE_32BIT_TIME_T +** define is required to maintain binary compatibility with the MSVC runtime +** library in use (e.g. for Windows XP). +*/ +#if !defined(_USE_32BIT_TIME_T) && !defined(_USE_64BIT_TIME_T) && \ + defined(_WIN32) && !defined(_WIN64) && \ + defined(__MINGW_MAJOR_VERSION) && __MINGW_MAJOR_VERSION >= 4 && \ + defined(__MSVCRT__) +# define _USE_32BIT_TIME_T +#endif + +/* Optionally #include a user-defined header, whereby compilation options +** may be set prior to where they take effect, but after platform setup. +** If SQLITE_CUSTOM_INCLUDE=? is defined, its value names the #include +** file. +*/ +#ifdef SQLITE_CUSTOM_INCLUDE +# define INC_STRINGIFY_(f) #f +# define INC_STRINGIFY(f) INC_STRINGIFY_(f) +# include INC_STRINGIFY(SQLITE_CUSTOM_INCLUDE) +#endif + +/* The public SQLite interface. The _FILE_OFFSET_BITS macro must appear +** first in QNX. Also, the _USE_32BIT_TIME_T macro must appear first for +** MinGW. +*/ +/************** Include sqlite3.h in the middle of sqliteInt.h ***************/ +/************** Begin file sqlite3.h *****************************************/ +/* +** 2001-09-15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the SQLite library +** presents to client programs. If a C-function, structure, datatype, +** or constant definition does not appear in this file, then it is +** not a published API of SQLite, is subject to change without +** notice, and should not be referenced by programs that use SQLite. +** +** Some of the definitions that are in this file are marked as +** "experimental". Experimental interfaces are normally new +** features recently added to SQLite. We do not anticipate changes +** to experimental interfaces but reserve the right to make minor changes +** if experience from use "in the wild" suggest such changes are prudent. +** +** The official C-language API documentation for SQLite is derived +** from comments in this file. This file is the authoritative source +** on how SQLite interfaces are supposed to operate. +** +** The name of this file under configuration management is "sqlite.h.in". +** The makefile makes some minor changes to this file (such as inserting +** the version number) and changes its name to "sqlite3.h" as +** part of the build process. +*/ +#ifndef SQLITE3_H +#define SQLITE3_H +#include /* Needed for the definition of va_list */ + +/* +** Make sure we can call this stuff from C++. +*/ +#if 0 +extern "C" { +#endif + + +/* +** Facilitate override of interface linkage and calling conventions. +** Be aware that these macros may not be used within this particular +** translation of the amalgamation and its associated header file. +** +** The SQLITE_EXTERN and SQLITE_API macros are used to instruct the +** compiler that the target identifier should have external linkage. +** +** The SQLITE_CDECL macro is used to set the calling convention for +** public functions that accept a variable number of arguments. +** +** The SQLITE_APICALL macro is used to set the calling convention for +** public functions that accept a fixed number of arguments. +** +** The SQLITE_STDCALL macro is no longer used and is now deprecated. +** +** The SQLITE_CALLBACK macro is used to set the calling convention for +** function pointers. +** +** The SQLITE_SYSAPI macro is used to set the calling convention for +** functions provided by the operating system. +** +** Currently, the SQLITE_CDECL, SQLITE_APICALL, SQLITE_CALLBACK, and +** SQLITE_SYSAPI macros are used only when building for environments +** that require non-default calling conventions. +*/ +#ifndef SQLITE_EXTERN +# define SQLITE_EXTERN extern +#endif +#ifndef SQLITE_API +# define SQLITE_API +#endif +#ifndef SQLITE_CDECL +# define SQLITE_CDECL +#endif +#ifndef SQLITE_APICALL +# define SQLITE_APICALL +#endif +#ifndef SQLITE_STDCALL +# define SQLITE_STDCALL SQLITE_APICALL +#endif +#ifndef SQLITE_CALLBACK +# define SQLITE_CALLBACK +#endif +#ifndef SQLITE_SYSAPI +# define SQLITE_SYSAPI +#endif + +/* +** These no-op macros are used in front of interfaces to mark those +** interfaces as either deprecated or experimental. New applications +** should not use deprecated interfaces - they are supported for backwards +** compatibility only. Application writers should be aware that +** experimental interfaces are subject to change in point releases. +** +** These macros used to resolve to various kinds of compiler magic that +** would generate warning messages when they were used. But that +** compiler magic ended up generating such a flurry of bug reports +** that we have taken it all out and gone back to using simple +** noop macros. +*/ +#define SQLITE_DEPRECATED +#define SQLITE_EXPERIMENTAL + +/* +** Ensure these symbols were not defined by some previous header file. +*/ +#ifdef SQLITE_VERSION +# undef SQLITE_VERSION +#endif +#ifdef SQLITE_VERSION_NUMBER +# undef SQLITE_VERSION_NUMBER +#endif + +/* +** CAPI3REF: Compile-Time Library Version Numbers +** +** ^(The [SQLITE_VERSION] C preprocessor macro in the sqlite3.h header +** evaluates to a string literal that is the SQLite version in the +** format "X.Y.Z" where X is the major version number (always 3 for +** SQLite3) and Y is the minor version number and Z is the release number.)^ +** ^(The [SQLITE_VERSION_NUMBER] C preprocessor macro resolves to an integer +** with the value (X*1000000 + Y*1000 + Z) where X, Y, and Z are the same +** numbers used in [SQLITE_VERSION].)^ +** The SQLITE_VERSION_NUMBER for any given release of SQLite will also +** be larger than the release from which it is derived. Either Y will +** be held constant and Z will be incremented or else Y will be incremented +** and Z will be reset to zero. +** +** Since [version 3.6.18] ([dateof:3.6.18]), +** SQLite source code has been stored in the +** Fossil configuration management +** system. ^The SQLITE_SOURCE_ID macro evaluates to +** a string which identifies a particular check-in of SQLite +** within its configuration management system. ^The SQLITE_SOURCE_ID +** string contains the date and time of the check-in (UTC) and a SHA1 +** or SHA3-256 hash of the entire source tree. If the source code has +** been edited in any way since it was last checked in, then the last +** four hexadecimal digits of the hash may be modified. +** +** See also: [sqlite3_libversion()], +** [sqlite3_libversion_number()], [sqlite3_sourceid()], +** [sqlite_version()] and [sqlite_source_id()]. +*/ +#define SQLITE_VERSION "3.37.0" +#define SQLITE_VERSION_NUMBER 3037000 +#define SQLITE_SOURCE_ID "2021-10-04 11:10:15 8b24c177061c38361588f419eda9b7943b72a0c6b2855b6f39272451b8a1b813" + +/* +** CAPI3REF: Run-Time Library Version Numbers +** KEYWORDS: sqlite3_version sqlite3_sourceid +** +** These interfaces provide the same information as the [SQLITE_VERSION], +** [SQLITE_VERSION_NUMBER], and [SQLITE_SOURCE_ID] C preprocessor macros +** but are associated with the library instead of the header file. ^(Cautious +** programmers might include assert() statements in their application to +** verify that values returned by these interfaces match the macros in +** the header, and thus ensure that the application is +** compiled with matching library and header files. +** +** 
+** assert( sqlite3_libversion_number()==SQLITE_VERSION_NUMBER );
+** assert( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,80)==0 );
+** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 );
+**
)^ +** +** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] +** macro. ^The sqlite3_libversion() function returns a pointer to the +** to the sqlite3_version[] string constant. The sqlite3_libversion() +** function is provided for use in DLLs since DLL users usually do not have +** direct access to string constants within the DLL. ^The +** sqlite3_libversion_number() function returns an integer equal to +** [SQLITE_VERSION_NUMBER]. ^(The sqlite3_sourceid() function returns +** a pointer to a string constant whose value is the same as the +** [SQLITE_SOURCE_ID] C preprocessor macro. Except if SQLite is built +** using an edited copy of [the amalgamation], then the last four characters +** of the hash might be different from [SQLITE_SOURCE_ID].)^ +** +** See also: [sqlite_version()] and [sqlite_source_id()]. +*/ +SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; +SQLITE_API const char *sqlite3_libversion(void); +SQLITE_API const char *sqlite3_sourceid(void); +SQLITE_API int sqlite3_libversion_number(void); + +/* +** CAPI3REF: Run-Time Library Compilation Options Diagnostics +** +** ^The sqlite3_compileoption_used() function returns 0 or 1 +** indicating whether the specified option was defined at +** compile time. ^The SQLITE_ prefix may be omitted from the +** option name passed to sqlite3_compileoption_used(). +** +** ^The sqlite3_compileoption_get() function allows iterating +** over the list of options that were defined at compile time by +** returning the N-th compile time option string. ^If N is out of range, +** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ +** prefix is omitted from any strings returned by +** sqlite3_compileoption_get(). +** +** ^Support for the diagnostic functions sqlite3_compileoption_used() +** and sqlite3_compileoption_get() may be omitted by specifying the +** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. +** +** See also: SQL functions [sqlite_compileoption_used()] and +** [sqlite_compileoption_get()] and the [compile_options pragma]. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +SQLITE_API int sqlite3_compileoption_used(const char *zOptName); +SQLITE_API const char *sqlite3_compileoption_get(int N); +#else +# define sqlite3_compileoption_used(X) 0 +# define sqlite3_compileoption_get(X) ((void*)0) +#endif + +/* +** CAPI3REF: Test To See If The Library Is Threadsafe +** +** ^The sqlite3_threadsafe() function returns zero if and only if +** SQLite was compiled with mutexing code omitted due to the +** [SQLITE_THREADSAFE] compile-time option being set to 0. +** +** SQLite can be compiled with or without mutexes. When +** the [SQLITE_THREADSAFE] C preprocessor macro is 1 or 2, mutexes +** are enabled and SQLite is threadsafe. When the +** [SQLITE_THREADSAFE] macro is 0, +** the mutexes are omitted. Without the mutexes, it is not safe +** to use SQLite concurrently from more than one thread. +** +** Enabling mutexes incurs a measurable performance penalty. +** So if speed is of utmost importance, it makes sense to disable +** the mutexes. But for maximum safety, mutexes should be enabled. +** ^The default behavior is for mutexes to be enabled. +** +** This interface can be used by an application to make sure that the +** version of SQLite that it is linking against was compiled with +** the desired setting of the [SQLITE_THREADSAFE] macro. +** +** This interface only reports on the compile-time mutex setting +** of the [SQLITE_THREADSAFE] flag. If SQLite is compiled with +** SQLITE_THREADSAFE=1 or =2 then mutexes are enabled by default but +** can be fully or partially disabled using a call to [sqlite3_config()] +** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD], +** or [SQLITE_CONFIG_SERIALIZED]. ^(The return value of the +** sqlite3_threadsafe() function shows only the compile-time setting of +** thread safety, not any run-time changes to that setting made by +** sqlite3_config(). In other words, the return value from sqlite3_threadsafe() +** is unchanged by calls to sqlite3_config().)^ +** +** See the [threading mode] documentation for additional information. +*/ +SQLITE_API int sqlite3_threadsafe(void); + +/* +** CAPI3REF: Database Connection Handle +** KEYWORDS: {database connection} {database connections} +** +** Each open SQLite database is represented by a pointer to an instance of +** the opaque structure named "sqlite3". It is useful to think of an sqlite3 +** pointer as an object. The [sqlite3_open()], [sqlite3_open16()], and +** [sqlite3_open_v2()] interfaces are its constructors, and [sqlite3_close()] +** and [sqlite3_close_v2()] are its destructors. There are many other +** interfaces (such as +** [sqlite3_prepare_v2()], [sqlite3_create_function()], and +** [sqlite3_busy_timeout()] to name but three) that are methods on an +** sqlite3 object. +*/ +typedef struct sqlite3 sqlite3; + +/* +** CAPI3REF: 64-Bit Integer Types +** KEYWORDS: sqlite_int64 sqlite_uint64 +** +** Because there is no cross-platform way to specify 64-bit integer types +** SQLite includes typedefs for 64-bit signed and unsigned integers. +** +** The sqlite3_int64 and sqlite3_uint64 are the preferred type definitions. +** The sqlite_int64 and sqlite_uint64 types are supported for backwards +** compatibility only. +** +** ^The sqlite3_int64 and sqlite_int64 types can store integer values +** between -9223372036854775808 and +9223372036854775807 inclusive. ^The +** sqlite3_uint64 and sqlite_uint64 types can store integer values +** between 0 and +18446744073709551615 inclusive. +*/ +#ifdef SQLITE_INT64_TYPE + typedef SQLITE_INT64_TYPE sqlite_int64; +# ifdef SQLITE_UINT64_TYPE + typedef SQLITE_UINT64_TYPE sqlite_uint64; +# else + typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +# endif +#elif defined(_MSC_VER) || defined(__BORLANDC__) + typedef __int64 sqlite_int64; + typedef unsigned __int64 sqlite_uint64; +#else + typedef long long int sqlite_int64; + typedef unsigned long long int sqlite_uint64; +#endif +typedef sqlite_int64 sqlite3_int64; +typedef sqlite_uint64 sqlite3_uint64; + +/* +** If compiling for a processor that lacks floating point support, +** substitute integer for floating-point. +*/ +#ifdef SQLITE_OMIT_FLOATING_POINT +# define double sqlite3_int64 +#endif + +/* +** CAPI3REF: Closing A Database Connection +** DESTRUCTOR: sqlite3 +** +** ^The sqlite3_close() and sqlite3_close_v2() routines are destructors +** for the [sqlite3] object. +** ^Calls to sqlite3_close() and sqlite3_close_v2() return [SQLITE_OK] if +** the [sqlite3] object is successfully destroyed and all associated +** resources are deallocated. +** +** Ideally, applications should [sqlite3_finalize | finalize] all +** [prepared statements], [sqlite3_blob_close | close] all [BLOB handles], and +** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated +** with the [sqlite3] object prior to attempting to close the object. +** ^If the database connection is associated with unfinalized prepared +** statements, BLOB handlers, and/or unfinished sqlite3_backup objects then +** sqlite3_close() will leave the database connection open and return +** [SQLITE_BUSY]. ^If sqlite3_close_v2() is called with unfinalized prepared +** statements, unclosed BLOB handlers, and/or unfinished sqlite3_backups, +** it returns [SQLITE_OK] regardless, but instead of deallocating the database +** connection immediately, it marks the database connection as an unusable +** "zombie" and makes arrangements to automatically deallocate the database +** connection after all prepared statements are finalized, all BLOB handles +** are closed, and all backups have finished. The sqlite3_close_v2() interface +** is intended for use with host languages that are garbage collected, and +** where the order in which destructors are called is arbitrary. +** +** ^If an [sqlite3] object is destroyed while a transaction is open, +** the transaction is automatically rolled back. +** +** The C parameter to [sqlite3_close(C)] and [sqlite3_close_v2(C)] +** must be either a NULL +** pointer or an [sqlite3] object pointer obtained +** from [sqlite3_open()], [sqlite3_open16()], or +** [sqlite3_open_v2()], and not previously closed. +** ^Calling sqlite3_close() or sqlite3_close_v2() with a NULL pointer +** argument is a harmless no-op. +*/ +SQLITE_API int sqlite3_close(sqlite3*); +SQLITE_API int sqlite3_close_v2(sqlite3*); + +/* +** The type for a callback function. +** This is legacy and deprecated. It is included for historical +** compatibility and is not documented. +*/ +typedef int (*sqlite3_callback)(void*,int,char**, char**); + +/* +** CAPI3REF: One-Step Query Execution Interface +** METHOD: sqlite3 +** +** The sqlite3_exec() interface is a convenience wrapper around +** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()], +** that allows an application to run multiple statements of SQL +** without having to use a lot of C code. +** +** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, +** semicolon-separate SQL statements passed into its 2nd argument, +** in the context of the [database connection] passed in as its 1st +** argument. ^If the callback function of the 3rd argument to +** sqlite3_exec() is not NULL, then it is invoked for each result row +** coming out of the evaluated SQL statements. ^The 4th argument to +** sqlite3_exec() is relayed through to the 1st argument of each +** callback invocation. ^If the callback pointer to sqlite3_exec() +** is NULL, then no callback is ever invoked and result rows are +** ignored. +** +** ^If an error occurs while evaluating the SQL statements passed into +** sqlite3_exec(), then execution of the current statement stops and +** subsequent statements are skipped. ^If the 5th parameter to sqlite3_exec() +** is not NULL then any error message is written into memory obtained +** from [sqlite3_malloc()] and passed back through the 5th parameter. +** To avoid memory leaks, the application should invoke [sqlite3_free()] +** on error message strings returned through the 5th parameter of +** sqlite3_exec() after the error message string is no longer needed. +** ^If the 5th parameter to sqlite3_exec() is not NULL and no errors +** occur, then sqlite3_exec() sets the pointer in its 5th parameter to +** NULL before returning. +** +** ^If an sqlite3_exec() callback returns non-zero, the sqlite3_exec() +** routine returns SQLITE_ABORT without invoking the callback again and +** without running any subsequent SQL statements. +** +** ^The 2nd argument to the sqlite3_exec() callback function is the +** number of columns in the result. ^The 3rd argument to the sqlite3_exec() +** callback is an array of pointers to strings obtained as if from +** [sqlite3_column_text()], one for each column. ^If an element of a +** result row is NULL then the corresponding string pointer for the +** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the +** sqlite3_exec() callback is an array of pointers to strings where each +** entry represents the name of corresponding result column as obtained +** from [sqlite3_column_name()]. +** +** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer +** to an empty string, or a pointer that contains only whitespace and/or +** SQL comments, then no SQL statements are evaluated and the database +** is not changed. +** +** Restrictions: +** +**
    +**
  • The application must ensure that the 1st parameter to sqlite3_exec() +** is a valid and open [database connection]. +**
  • The application must not close the [database connection] specified by +** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not modify the SQL statement text passed into +** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
+*/ +SQLITE_API int sqlite3_exec( + sqlite3*, /* An open database */ + const char *sql, /* SQL to be evaluated */ + int (*callback)(void*,int,char**,char**), /* Callback function */ + void *, /* 1st argument to callback */ + char **errmsg /* Error msg written here */ +); + +/* +** CAPI3REF: Result Codes +** KEYWORDS: {result code definitions} +** +** Many SQLite functions return an integer result code from the set shown +** here in order to indicate success or failure. +** +** New error codes may be added in future versions of SQLite. +** +** See also: [extended result code definitions] +*/ +#define SQLITE_OK 0 /* Successful result */ +/* beginning-of-error-codes */ +#define SQLITE_ERROR 1 /* Generic error */ +#define SQLITE_INTERNAL 2 /* Internal logic error in SQLite */ +#define SQLITE_PERM 3 /* Access permission denied */ +#define SQLITE_ABORT 4 /* Callback routine requested an abort */ +#define SQLITE_BUSY 5 /* The database file is locked */ +#define SQLITE_LOCKED 6 /* A table in the database is locked */ +#define SQLITE_NOMEM 7 /* A malloc() failed */ +#define SQLITE_READONLY 8 /* Attempt to write a readonly database */ +#define SQLITE_INTERRUPT 9 /* Operation terminated by sqlite3_interrupt()*/ +#define SQLITE_IOERR 10 /* Some kind of disk I/O error occurred */ +#define SQLITE_CORRUPT 11 /* The database disk image is malformed */ +#define SQLITE_NOTFOUND 12 /* Unknown opcode in sqlite3_file_control() */ +#define SQLITE_FULL 13 /* Insertion failed because database is full */ +#define SQLITE_CANTOPEN 14 /* Unable to open the database file */ +#define SQLITE_PROTOCOL 15 /* Database lock protocol error */ +#define SQLITE_EMPTY 16 /* Internal use only */ +#define SQLITE_SCHEMA 17 /* The database schema changed */ +#define SQLITE_TOOBIG 18 /* String or BLOB exceeds size limit */ +#define SQLITE_CONSTRAINT 19 /* Abort due to constraint violation */ +#define SQLITE_MISMATCH 20 /* Data type mismatch */ +#define SQLITE_MISUSE 21 /* Library used incorrectly */ +#define SQLITE_NOLFS 22 /* Uses OS features not supported on host */ +#define SQLITE_AUTH 23 /* Authorization denied */ +#define SQLITE_FORMAT 24 /* Not used */ +#define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ +#define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ +#define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ +#define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ +/* end-of-error-codes */ + +/* +** CAPI3REF: Extended Result Codes +** KEYWORDS: {extended result code definitions} +** +** In its default configuration, SQLite API routines return one of 30 integer +** [result codes]. However, experience has shown that many of +** these result codes are too coarse-grained. They do not provide as +** much information about problems as programmers might like. In an effort to +** address this, newer versions of SQLite (version 3.3.8 [dateof:3.3.8] +** and later) include +** support for additional result codes that provide more detailed information +** about errors. These [extended result codes] are enabled or disabled +** on a per database connection basis using the +** [sqlite3_extended_result_codes()] API. Or, the extended code for +** the most recent error can be obtained using +** [sqlite3_extended_errcode()]. +*/ +#define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) +#define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) +#define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) +#define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) +#define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) +#define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) +#define SQLITE_IOERR_FSYNC (SQLITE_IOERR | (4<<8)) +#define SQLITE_IOERR_DIR_FSYNC (SQLITE_IOERR | (5<<8)) +#define SQLITE_IOERR_TRUNCATE (SQLITE_IOERR | (6<<8)) +#define SQLITE_IOERR_FSTAT (SQLITE_IOERR | (7<<8)) +#define SQLITE_IOERR_UNLOCK (SQLITE_IOERR | (8<<8)) +#define SQLITE_IOERR_RDLOCK (SQLITE_IOERR | (9<<8)) +#define SQLITE_IOERR_DELETE (SQLITE_IOERR | (10<<8)) +#define SQLITE_IOERR_BLOCKED (SQLITE_IOERR | (11<<8)) +#define SQLITE_IOERR_NOMEM (SQLITE_IOERR | (12<<8)) +#define SQLITE_IOERR_ACCESS (SQLITE_IOERR | (13<<8)) +#define SQLITE_IOERR_CHECKRESERVEDLOCK (SQLITE_IOERR | (14<<8)) +#define SQLITE_IOERR_LOCK (SQLITE_IOERR | (15<<8)) +#define SQLITE_IOERR_CLOSE (SQLITE_IOERR | (16<<8)) +#define SQLITE_IOERR_DIR_CLOSE (SQLITE_IOERR | (17<<8)) +#define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) +#define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) +#define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) +#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) +#define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) +#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) +#define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) +#define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) +#define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8)) +#define SQLITE_IOERR_AUTH (SQLITE_IOERR | (28<<8)) +#define SQLITE_IOERR_BEGIN_ATOMIC (SQLITE_IOERR | (29<<8)) +#define SQLITE_IOERR_COMMIT_ATOMIC (SQLITE_IOERR | (30<<8)) +#define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) +#define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) +#define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) +#define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) +#define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) +#define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) +#define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) +#define SQLITE_BUSY_TIMEOUT (SQLITE_BUSY | (3<<8)) +#define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) +#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) +#define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) /* Not Used */ +#define SQLITE_CANTOPEN_SYMLINK (SQLITE_CANTOPEN | (6<<8)) +#define SQLITE_CANTOPEN_EXISTS (SQLITE_CANTOPEN | (7<<8)) +#define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) +#define SQLITE_CORRUPT_SEQUENCE (SQLITE_CORRUPT | (2<<8)) +#define SQLITE_CORRUPT_INDEX (SQLITE_CORRUPT | (3<<8)) +#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) +#define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) +#define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (5<<8)) +#define SQLITE_READONLY_DIRECTORY (SQLITE_READONLY | (6<<8)) +#define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_CONSTRAINT_ROWID (SQLITE_CONSTRAINT |(10<<8)) +#define SQLITE_CONSTRAINT_PINNED (SQLITE_CONSTRAINT |(11<<8)) +#define SQLITE_CONSTRAINT_DATATYPE (SQLITE_CONSTRAINT |(12<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) +#define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) +#define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) + +/* +** CAPI3REF: Flags For File Open Operations +** +** These bit values are intended for use in the +** 3rd parameter to the [sqlite3_open_v2()] interface and +** in the 4th parameter to the [sqlite3_vfs.xOpen] method. +*/ +#define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_READWRITE 0x00000002 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_CREATE 0x00000004 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_DELETEONCLOSE 0x00000008 /* VFS only */ +#define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ +#define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ +#define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MEMORY 0x00000080 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ +#define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ +#define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ +#define SQLITE_OPEN_MAIN_JOURNAL 0x00000800 /* VFS only */ +#define SQLITE_OPEN_TEMP_JOURNAL 0x00001000 /* VFS only */ +#define SQLITE_OPEN_SUBJOURNAL 0x00002000 /* VFS only */ +#define SQLITE_OPEN_SUPER_JOURNAL 0x00004000 /* VFS only */ +#define SQLITE_OPEN_NOMUTEX 0x00008000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_FULLMUTEX 0x00010000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ +#define SQLITE_OPEN_NOFOLLOW 0x01000000 /* Ok for sqlite3_open_v2() */ + +/* Reserved: 0x00F00000 */ +/* Legacy compatibility: */ +#define SQLITE_OPEN_MASTER_JOURNAL 0x00004000 /* VFS only */ + + +/* +** CAPI3REF: Device Characteristics +** +** The xDeviceCharacteristics method of the [sqlite3_io_methods] +** object returns an integer which is a vector of these +** bit values expressing I/O characteristics of the mass storage +** device that holds the file that the [sqlite3_io_methods] +** refers to. +** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). The SQLITE_IOCAP_POWERSAFE_OVERWRITE property means that +** after reboot following a crash or power loss, the only bytes in a +** file that were written at the application level might have changed +** and that adjacent bytes, even bytes within the same sector are +** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN +** flag indicates that a file cannot be deleted when open. The +** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on +** read-only media and cannot be changed even by processes with +** elevated privileges. +** +** The SQLITE_IOCAP_BATCH_ATOMIC property means that the underlying +** filesystem supports doing multiple write operations atomically when those +** write operations are bracketed by [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] and +** [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. +*/ +#define SQLITE_IOCAP_ATOMIC 0x00000001 +#define SQLITE_IOCAP_ATOMIC512 0x00000002 +#define SQLITE_IOCAP_ATOMIC1K 0x00000004 +#define SQLITE_IOCAP_ATOMIC2K 0x00000008 +#define SQLITE_IOCAP_ATOMIC4K 0x00000010 +#define SQLITE_IOCAP_ATOMIC8K 0x00000020 +#define SQLITE_IOCAP_ATOMIC16K 0x00000040 +#define SQLITE_IOCAP_ATOMIC32K 0x00000080 +#define SQLITE_IOCAP_ATOMIC64K 0x00000100 +#define SQLITE_IOCAP_SAFE_APPEND 0x00000200 +#define SQLITE_IOCAP_SEQUENTIAL 0x00000400 +#define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 +#define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 +#define SQLITE_IOCAP_IMMUTABLE 0x00002000 +#define SQLITE_IOCAP_BATCH_ATOMIC 0x00004000 + +/* +** CAPI3REF: File Locking Levels +** +** SQLite uses one of these integer values as the second +** argument to calls it makes to the xLock() and xUnlock() methods +** of an [sqlite3_io_methods] object. +*/ +#define SQLITE_LOCK_NONE 0 +#define SQLITE_LOCK_SHARED 1 +#define SQLITE_LOCK_RESERVED 2 +#define SQLITE_LOCK_PENDING 3 +#define SQLITE_LOCK_EXCLUSIVE 4 + +/* +** CAPI3REF: Synchronization Type Flags +** +** When SQLite invokes the xSync() method of an +** [sqlite3_io_methods] object it uses a combination of +** these integer values as the second argument. +** +** When the SQLITE_SYNC_DATAONLY flag is used, it means that the +** sync operation only needs to flush data to mass storage. Inode +** information need not be flushed. If the lower four bits of the flag +** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics. +** If the lower four bits equal SQLITE_SYNC_FULL, that means +** to use Mac OS X style fullsync instead of fsync(). +** +** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags +** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL +** settings. The [synchronous pragma] determines when calls to the +** xSync VFS method occur and applies uniformly across all platforms. +** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how +** energetic or rigorous or forceful the sync operations are and +** only make a difference on Mac OSX for the default SQLite code. +** (Third-party VFS implementations might also make the distinction +** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the +** operating systems natively supported by SQLite, only Mac OSX +** cares about the difference.) +*/ +#define SQLITE_SYNC_NORMAL 0x00002 +#define SQLITE_SYNC_FULL 0x00003 +#define SQLITE_SYNC_DATAONLY 0x00010 + +/* +** CAPI3REF: OS Interface Open File Handle +** +** An [sqlite3_file] object represents an open file in the +** [sqlite3_vfs | OS interface layer]. Individual OS interface +** implementations will +** want to subclass this object by appending additional fields +** for their own use. The pMethods entry is a pointer to an +** [sqlite3_io_methods] object that defines methods for performing +** I/O operations on the open file. +*/ +typedef struct sqlite3_file sqlite3_file; +struct sqlite3_file { + const struct sqlite3_io_methods *pMethods; /* Methods for an open file */ +}; + +/* +** CAPI3REF: OS Interface File Virtual Methods Object +** +** Every file opened by the [sqlite3_vfs.xOpen] method populates an +** [sqlite3_file] object (or, more commonly, a subclass of the +** [sqlite3_file] object) with a pointer to an instance of this object. +** This object defines the methods used to perform various operations +** against the open file represented by the [sqlite3_file] object. +** +** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element +** to a non-NULL pointer, then the sqlite3_io_methods.xClose method +** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed. The +** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen] +** is for the [sqlite3_vfs.xOpen] to set the sqlite3_file.pMethods element +** to NULL. +** +** The flags argument to xSync may be one of [SQLITE_SYNC_NORMAL] or +** [SQLITE_SYNC_FULL]. The first choice is the normal fsync(). +** The second choice is a Mac OS X style fullsync. The [SQLITE_SYNC_DATAONLY] +** flag may be ORed in to indicate that only the data of the file +** and not its inode needs to be synced. +** +** The integer values to xLock() and xUnlock() are one of +**
    +**
  • [SQLITE_LOCK_NONE], +**
  • [SQLITE_LOCK_SHARED], +**
  • [SQLITE_LOCK_RESERVED], +**
  • [SQLITE_LOCK_PENDING], or +**
  • [SQLITE_LOCK_EXCLUSIVE]. +**
+** xLock() increases the lock. xUnlock() decreases the lock. +** The xCheckReservedLock() method checks whether any database connection, +** either in this process or in some other process, is holding a RESERVED, +** PENDING, or EXCLUSIVE lock on the file. It returns true +** if such a lock exists and false otherwise. +** +** The xFileControl() method is a generic interface that allows custom +** VFS implementations to directly control an open file using the +** [sqlite3_file_control()] interface. The second "op" argument is an +** integer opcode. The third argument is a generic pointer intended to +** point to a structure that may contain arguments or space in which to +** write return values. Potential uses for xFileControl() might be +** functions to enable blocking locks with timeouts, to change the +** locking strategy (for example to use dot-file locks), to inquire +** about the status of a lock, or to break stale locks. The SQLite +** core reserves all opcodes less than 100 for its own use. +** A [file control opcodes | list of opcodes] less than 100 is available. +** Applications that define a custom xFileControl method should use opcodes +** greater than 100 to avoid conflicts. VFS implementations should +** return [SQLITE_NOTFOUND] for file control opcodes that they do not +** recognize. +** +** The xSectorSize() method returns the sector size of the +** device that underlies the file. The sector size is the +** minimum write that can be performed without disturbing +** other bytes in the file. The xDeviceCharacteristics() +** method returns a bit vector describing behaviors of the +** underlying device: +** +**
    +**
  • [SQLITE_IOCAP_ATOMIC] +**
  • [SQLITE_IOCAP_ATOMIC512] +**
  • [SQLITE_IOCAP_ATOMIC1K] +**
  • [SQLITE_IOCAP_ATOMIC2K] +**
  • [SQLITE_IOCAP_ATOMIC4K] +**
  • [SQLITE_IOCAP_ATOMIC8K] +**
  • [SQLITE_IOCAP_ATOMIC16K] +**
  • [SQLITE_IOCAP_ATOMIC32K] +**
  • [SQLITE_IOCAP_ATOMIC64K] +**
  • [SQLITE_IOCAP_SAFE_APPEND] +**
  • [SQLITE_IOCAP_SEQUENTIAL] +**
  • [SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN] +**
  • [SQLITE_IOCAP_POWERSAFE_OVERWRITE] +**
  • [SQLITE_IOCAP_IMMUTABLE] +**
  • [SQLITE_IOCAP_BATCH_ATOMIC] +**
+** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). +** +** If xRead() returns SQLITE_IOERR_SHORT_READ it must also fill +** in the unread portions of the buffer with zeros. A VFS that +** fails to zero-fill short reads might seem to work. However, +** failure to zero-fill short reads will eventually lead to +** database corruption. +*/ +typedef struct sqlite3_io_methods sqlite3_io_methods; +struct sqlite3_io_methods { + int iVersion; + int (*xClose)(sqlite3_file*); + int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); + int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst); + int (*xTruncate)(sqlite3_file*, sqlite3_int64 size); + int (*xSync)(sqlite3_file*, int flags); + int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize); + int (*xLock)(sqlite3_file*, int); + int (*xUnlock)(sqlite3_file*, int); + int (*xCheckReservedLock)(sqlite3_file*, int *pResOut); + int (*xFileControl)(sqlite3_file*, int op, void *pArg); + int (*xSectorSize)(sqlite3_file*); + int (*xDeviceCharacteristics)(sqlite3_file*); + /* Methods above are valid for version 1 */ + int (*xShmMap)(sqlite3_file*, int iPg, int pgsz, int, void volatile**); + int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); + void (*xShmBarrier)(sqlite3_file*); + int (*xShmUnmap)(sqlite3_file*, int deleteFlag); + /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ + /* Additional methods may be added in future releases */ +}; + +/* +** CAPI3REF: Standard File Control Opcodes +** KEYWORDS: {file control opcodes} {file control opcode} +** +** These integer constants are opcodes for the xFileControl method +** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] +** interface. +** +**
    +**
  • [[SQLITE_FCNTL_LOCKSTATE]] +** The [SQLITE_FCNTL_LOCKSTATE] opcode is used for debugging. This +** opcode causes the xFileControl method to write the current state of +** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], +** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) +** into an integer that the pArg argument points to. This capability +** is used during testing and is only available when the SQLITE_TEST +** compile-time option is used. +** +**
  • [[SQLITE_FCNTL_SIZE_HINT]] +** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS +** layer a hint of how large the database file will grow to be during the +** current transaction. This hint is not guaranteed to be accurate but it +** is often close. The underlying VFS might choose to preallocate database +** file space based on this hint in order to help writes to the database +** file run faster. +** +**
  • [[SQLITE_FCNTL_SIZE_LIMIT]] +** The [SQLITE_FCNTL_SIZE_LIMIT] opcode is used by in-memory VFS that +** implements [sqlite3_deserialize()] to set an upper bound on the size +** of the in-memory database. The argument is a pointer to a [sqlite3_int64]. +** If the integer pointed to is negative, then it is filled in with the +** current limit. Otherwise the limit is set to the larger of the value +** of the integer pointed to and the current database size. The integer +** pointed to is set to the new limit. +** +**
  • [[SQLITE_FCNTL_CHUNK_SIZE]] +** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS +** extends and truncates the database file in chunks of a size specified +** by the user. The fourth argument to [sqlite3_file_control()] should +** point to an integer (type int) containing the new chunk-size to use +** for the nominated database. Allocating database file space in large +** chunks (say 1MB at a time), may reduce file-system fragmentation and +** improve performance on some systems. +** +**
  • [[SQLITE_FCNTL_FILE_POINTER]] +** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with a particular database +** connection. See also [SQLITE_FCNTL_JOURNAL_POINTER]. +** +**
  • [[SQLITE_FCNTL_JOURNAL_POINTER]] +** The [SQLITE_FCNTL_JOURNAL_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with the journal file (either +** the [rollback journal] or the [write-ahead log]) for a particular database +** connection. See also [SQLITE_FCNTL_FILE_POINTER]. +** +**
  • [[SQLITE_FCNTL_SYNC_OMITTED]] +** No longer in use. +** +**
  • [[SQLITE_FCNTL_SYNC]] +** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and +** sent to the VFS immediately before the xSync method is invoked on a +** database file descriptor. Or, if the xSync method is not invoked +** because the user has configured SQLite with +** [PRAGMA synchronous | PRAGMA synchronous=OFF] it is invoked in place +** of the xSync method. In most cases, the pointer argument passed with +** this file-control is NULL. However, if the database file is being synced +** as part of a multi-database commit, the argument points to a nul-terminated +** string containing the transactions super-journal file name. VFSes that +** do not need this signal should silently ignore this opcode. Applications +** should not call [sqlite3_file_control()] with this opcode as doing so may +** disrupt the operation of the specialized VFSes that do require it. +** +**
  • [[SQLITE_FCNTL_COMMIT_PHASETWO]] +** The [SQLITE_FCNTL_COMMIT_PHASETWO] opcode is generated internally by SQLite +** and sent to the VFS after a transaction has been committed immediately +** but before the database is unlocked. VFSes that do not need this signal +** should silently ignore this opcode. Applications should not call +** [sqlite3_file_control()] with this opcode as doing so may disrupt the +** operation of the specialized VFSes that do require it. +** +**
  • [[SQLITE_FCNTL_WIN32_AV_RETRY]] +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic +** retry counts and intervals for certain disk I/O operations for the +** windows [VFS] in order to provide robustness in the presence of +** anti-virus programs. By default, the windows VFS will retry file read, +** file write, and file delete operations up to 10 times, with a delay +** of 25 milliseconds before the first retry and with the delay increasing +** by an additional 25 milliseconds with each subsequent retry. This +** opcode allows these two values (10 retries and 25 milliseconds of delay) +** to be adjusted. The values are changed for all database connections +** within the same process. The argument is a pointer to an array of two +** integers where the first integer is the new retry count and the second +** integer is the delay. If either integer is negative, then the setting +** is not changed but instead the prior value of that setting is written +** into the array entry, allowing the current retry settings to be +** interrogated. The zDbName parameter is ignored. +** +**
  • [[SQLITE_FCNTL_PERSIST_WAL]] +** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the +** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary +** write ahead log ([WAL file]) and shared memory +** files used for transaction control +** are automatically deleted when the latest connection to the database +** closes. Setting persistent WAL mode causes those files to persist after +** close. Persisting the files is useful when other processes that do not +** have write permission on the directory containing the database file want +** to read the database file, as the WAL and shared memory files must exist +** in order for the database to be readable. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable persistent WAL mode or 1 to enable persistent +** WAL mode. If the integer is -1, then it is overwritten with the current +** WAL persistence setting. +** +**
  • [[SQLITE_FCNTL_POWERSAFE_OVERWRITE]] +** ^The [SQLITE_FCNTL_POWERSAFE_OVERWRITE] opcode is used to set or query the +** persistent "powersafe-overwrite" or "PSOW" setting. The PSOW setting +** determines the [SQLITE_IOCAP_POWERSAFE_OVERWRITE] bit of the +** xDeviceCharacteristics methods. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable zero-damage mode or 1 to enable zero-damage +** mode. If the integer is -1, then it is overwritten with the current +** zero-damage mode setting. +** +**
  • [[SQLITE_FCNTL_OVERWRITE]] +** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening +** a write transaction to indicate that, unless it is rolled back for some +** reason, the entire database file will be overwritten by the current +** transaction. This is used by VACUUM operations. +** +**
  • [[SQLITE_FCNTL_VFSNAME]] +** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of +** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** final bottom-level VFS are written into memory obtained from +** [sqlite3_malloc()] and the result is stored in the char* variable +** that the fourth parameter of [sqlite3_file_control()] points to. +** The caller is responsible for freeing the memory when done. As with +** all file-control actions, there is no guarantee that this will actually +** do anything. Callers should initialize the char* variable to a NULL +** pointer in case this file-control is not implemented. This file-control +** is intended for diagnostic use only. +** +**
  • [[SQLITE_FCNTL_VFS_POINTER]] +** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level +** [VFSes] currently in use. ^(The argument X in +** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be +** of type "[sqlite3_vfs] **". This opcodes will set *X +** to a pointer to the top-level VFS.)^ +** ^When there are multiple VFS shims in the stack, this opcode finds the +** upper-most shim only. +** +**
  • [[SQLITE_FCNTL_PRAGMA]] +** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] +** file control is sent to the open [sqlite3_file] object corresponding +** to the database file to which the pragma statement refers. ^The argument +** to the [SQLITE_FCNTL_PRAGMA] file control is an array of +** pointers to strings (char**) in which the second element of the array +** is the name of the pragma and the third element is the argument to the +** pragma or NULL if the pragma has no argument. ^The handler for an +** [SQLITE_FCNTL_PRAGMA] file control can optionally make the first element +** of the char** argument point to a string obtained from [sqlite3_mprintf()] +** or the equivalent and that string will become the result of the pragma or +** the error message if the pragma fails. ^If the +** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal +** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] +** file control returns [SQLITE_OK], then the parser assumes that the +** VFS has handled the PRAGMA itself and the parser generates a no-op +** prepared statement if result string is NULL, or that returns a copy +** of the result string if the string is non-NULL. +** ^If the [SQLITE_FCNTL_PRAGMA] file control returns +** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means +** that the VFS encountered an error while handling the [PRAGMA] and the +** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] +** file control occurs at the beginning of pragma statement analysis and so +** it is able to override built-in [PRAGMA] statements. +** +**
  • [[SQLITE_FCNTL_BUSYHANDLER]] +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connection's busy-handler callback. The argument is of type (void**) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connection's +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. +** +**
  • [[SQLITE_FCNTL_TEMPFILENAME]] +** ^Applications can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a +** temporary filename using the same algorithm that is followed to generate +** temporary filenames for TEMP tables and other internal uses. The +** argument should be a char** which will be filled with the filename +** written into memory obtained from [sqlite3_malloc()]. The caller should +** invoke [sqlite3_free()] on the result to avoid a memory leak. +** +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** +**
  • [[SQLITE_FCNTL_TRACE]] +** The [SQLITE_FCNTL_TRACE] file control provides advisory information +** to the VFS about what the higher layers of the SQLite stack are doing. +** This file control is used by some VFS activity tracing [shims]. +** The argument is a zero-terminated string. Higher layers in the +** SQLite stack may generate instances of this file control if +** the [SQLITE_USE_FCNTL_TRACE] compile-time option is enabled. +** +**
  • [[SQLITE_FCNTL_HAS_MOVED]] +** The [SQLITE_FCNTL_HAS_MOVED] file control interprets its argument as a +** pointer to an integer and it writes a boolean into that integer depending +** on whether or not the file has been renamed, moved, or deleted since it +** was first opened. +** +**
  • [[SQLITE_FCNTL_WIN32_GET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_GET_HANDLE] opcode can be used to obtain the +** underlying native file handle associated with a file handle. This file +** control interprets its argument as a pointer to a native file handle and +** writes the resulting value there. +** +**
  • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This +** opcode causes the xFileControl method to swap the file handle with the one +** pointed to by the pArg argument. This capability is used during testing +** and only needs to be supported when SQLITE_TEST is defined. +** +**
  • [[SQLITE_FCNTL_WAL_BLOCK]] +** The [SQLITE_FCNTL_WAL_BLOCK] is a signal to the VFS layer that it might +** be advantageous to block on the next WAL lock if the lock is not immediately +** available. The WAL subsystem issues this signal during rare +** circumstances in order to fix a problem with priority inversion. +** Applications should not use this file-control. +** +**
  • [[SQLITE_FCNTL_ZIPVFS]] +** The [SQLITE_FCNTL_ZIPVFS] opcode is implemented by zipvfs only. All other +** VFS should return SQLITE_NOTFOUND for this opcode. +** +**
  • [[SQLITE_FCNTL_RBU]] +** The [SQLITE_FCNTL_RBU] opcode is implemented by the special VFS used by +** the RBU extension only. All other VFS should return SQLITE_NOTFOUND for +** this opcode. +** +**
  • [[SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]] +** If the [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] opcode returns SQLITE_OK, then +** the file descriptor is placed in "batch write mode", which +** means all subsequent write operations will be deferred and done +** atomically at the next [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. Systems +** that do not support batch atomic writes will return SQLITE_NOTFOUND. +** ^Following a successful SQLITE_FCNTL_BEGIN_ATOMIC_WRITE and prior to +** the closing [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE] or +** [SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE], SQLite will make +** no VFS interface calls on the same [sqlite3_file] file descriptor +** except for calls to the xWrite method and the xFileControl method +** with [SQLITE_FCNTL_SIZE_HINT]. +** +**
  • [[SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]] +** The [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE] opcode causes all write +** operations since the previous successful call to +** [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] to be performed atomically. +** This file control returns [SQLITE_OK] if and only if the writes were +** all performed successfully and have been committed to persistent storage. +** ^Regardless of whether or not it is successful, this file control takes +** the file descriptor out of batch write mode so that all subsequent +** write operations are independent. +** ^SQLite will never invoke SQLITE_FCNTL_COMMIT_ATOMIC_WRITE without +** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. +** +**
  • [[SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE]] +** The [SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE] opcode causes all write +** operations since the previous successful call to +** [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] to be rolled back. +** ^This file control takes the file descriptor out of batch write mode +** so that all subsequent write operations are independent. +** ^SQLite will never invoke SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE without +** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. +** +**
  • [[SQLITE_FCNTL_LOCK_TIMEOUT]] +** The [SQLITE_FCNTL_LOCK_TIMEOUT] opcode is used to configure a VFS +** to block for up to M milliseconds before failing when attempting to +** obtain a file lock using the xLock or xShmLock methods of the VFS. +** The parameter is a pointer to a 32-bit signed integer that contains +** the value that M is to be set to. Before returning, the 32-bit signed +** integer is overwritten with the previous value of M. +** +**
  • [[SQLITE_FCNTL_DATA_VERSION]] +** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to +** a database file. The argument is a pointer to a 32-bit unsigned integer. +** The "data version" for the pager is written into the pointer. The +** "data version" changes whenever any change occurs to the corresponding +** database file, either through SQL statements on the same database +** connection or through transactions committed by separate database +** connections possibly in other processes. The [sqlite3_total_changes()] +** interface can be used to find if any database on the connection has changed, +** but that interface responds to changes on TEMP as well as MAIN and does +** not provide a mechanism to detect changes to MAIN only. Also, the +** [sqlite3_total_changes()] interface responds to internal changes only and +** omits changes made by other database connections. The +** [PRAGMA data_version] command provides a mechanism to detect changes to +** a single attached database that occur due to other database connections, +** but omits changes implemented by the database connection on which it is +** called. This file control is the only mechanism to detect changes that +** happen either internally or externally and that are associated with +** a particular attached database. +** +**
  • [[SQLITE_FCNTL_CKPT_START]] +** The [SQLITE_FCNTL_CKPT_START] opcode is invoked from within a checkpoint +** in wal mode before the client starts to copy pages from the wal +** file to the database file. +** +**
  • [[SQLITE_FCNTL_CKPT_DONE]] +** The [SQLITE_FCNTL_CKPT_DONE] opcode is invoked from within a checkpoint +** in wal mode after the client has finished copying pages from the wal +** file to the database file, but before the *-shm file is updated to +** record the fact that the pages have been checkpointed. +**
+** +**
  • [[SQLITE_FCNTL_EXTERNAL_READER]] +** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect +** whether or not there is a database client in another process with a wal-mode +** transaction open on the database or not. It is only available on unix.The +** (void*) argument passed with this file-control should be a pointer to a +** value of type (int). The integer value is set to 1 if the database is a wal +** mode database and there exists at least one client in another process that +** currently has an SQL transaction open on the database. It is set to 0 if +** the database is not a wal-mode db, or if there is no such connection in any +** other process. This opcode cannot be used to detect transactions opened +** by clients within the current process, only within other processes. +** +** +**
  • [[SQLITE_FCNTL_CKSM_FILE]] +** Used by the cksmvfs VFS module only. +** +*/ +#define SQLITE_FCNTL_LOCKSTATE 1 +#define SQLITE_FCNTL_GET_LOCKPROXYFILE 2 +#define SQLITE_FCNTL_SET_LOCKPROXYFILE 3 +#define SQLITE_FCNTL_LAST_ERRNO 4 +#define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 +#define SQLITE_FCNTL_SYNC_OMITTED 8 +#define SQLITE_FCNTL_WIN32_AV_RETRY 9 +#define SQLITE_FCNTL_PERSIST_WAL 10 +#define SQLITE_FCNTL_OVERWRITE 11 +#define SQLITE_FCNTL_VFSNAME 12 +#define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 +#define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 +#define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 +#define SQLITE_FCNTL_TRACE 19 +#define SQLITE_FCNTL_HAS_MOVED 20 +#define SQLITE_FCNTL_SYNC 21 +#define SQLITE_FCNTL_COMMIT_PHASETWO 22 +#define SQLITE_FCNTL_WIN32_SET_HANDLE 23 +#define SQLITE_FCNTL_WAL_BLOCK 24 +#define SQLITE_FCNTL_ZIPVFS 25 +#define SQLITE_FCNTL_RBU 26 +#define SQLITE_FCNTL_VFS_POINTER 27 +#define SQLITE_FCNTL_JOURNAL_POINTER 28 +#define SQLITE_FCNTL_WIN32_GET_HANDLE 29 +#define SQLITE_FCNTL_PDB 30 +#define SQLITE_FCNTL_BEGIN_ATOMIC_WRITE 31 +#define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32 +#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33 +#define SQLITE_FCNTL_LOCK_TIMEOUT 34 +#define SQLITE_FCNTL_DATA_VERSION 35 +#define SQLITE_FCNTL_SIZE_LIMIT 36 +#define SQLITE_FCNTL_CKPT_DONE 37 +#define SQLITE_FCNTL_RESERVE_BYTES 38 +#define SQLITE_FCNTL_CKPT_START 39 +#define SQLITE_FCNTL_EXTERNAL_READER 40 +#define SQLITE_FCNTL_CKSM_FILE 41 + +/* deprecated names */ +#define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE +#define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE +#define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO + + +/* +** CAPI3REF: Mutex Handle +** +** The mutex module within SQLite defines [sqlite3_mutex] to be an +** abstract type for a mutex object. The SQLite core never looks +** at the internal representation of an [sqlite3_mutex]. It only +** deals with pointers to the [sqlite3_mutex] object. +** +** Mutexes are created using [sqlite3_mutex_alloc()]. +*/ +typedef struct sqlite3_mutex sqlite3_mutex; + +/* +** CAPI3REF: Loadable Extension Thunk +** +** A pointer to the opaque sqlite3_api_routines structure is passed as +** the third parameter to entry points of [loadable extensions]. This +** structure must be typedefed in order to work around compiler warnings +** on some platforms. +*/ +typedef struct sqlite3_api_routines sqlite3_api_routines; + +/* +** CAPI3REF: OS Interface Object +** +** An instance of the sqlite3_vfs object defines the interface between +** the SQLite core and the underlying operating system. The "vfs" +** in the name of the object stands for "virtual file system". See +** the [VFS | VFS documentation] for further information. +** +** The VFS interface is sometimes extended by adding new methods onto +** the end. Each time such an extension occurs, the iVersion field +** is incremented. The iVersion value started out as 1 in +** SQLite [version 3.5.0] on [dateof:3.5.0], then increased to 2 +** with SQLite [version 3.7.0] on [dateof:3.7.0], and then increased +** to 3 with SQLite [version 3.7.6] on [dateof:3.7.6]. Additional fields +** may be appended to the sqlite3_vfs object and the iVersion value +** may increase again in future versions of SQLite. +** Note that due to an oversight, the structure +** of the sqlite3_vfs object changed in the transition from +** SQLite [version 3.5.9] to [version 3.6.0] on [dateof:3.6.0] +** and yet the iVersion field was not increased. +** +** The szOsFile field is the size of the subclassed [sqlite3_file] +** structure used by this VFS. mxPathname is the maximum length of +** a pathname in this VFS. +** +** Registered sqlite3_vfs objects are kept on a linked list formed by +** the pNext pointer. The [sqlite3_vfs_register()] +** and [sqlite3_vfs_unregister()] interfaces manage this list +** in a thread-safe way. The [sqlite3_vfs_find()] interface +** searches the list. Neither the application code nor the VFS +** implementation should use the pNext pointer. +** +** The pNext field is the only field in the sqlite3_vfs +** structure that SQLite will ever modify. SQLite will only access +** or modify this field while holding a particular static mutex. +** The application should never modify anything within the sqlite3_vfs +** object once the object has been registered. +** +** The zName field holds the name of the VFS module. The name must +** be unique across all VFS modules. +** +** [[sqlite3_vfs.xOpen]] +** ^SQLite guarantees that the zFilename parameter to xOpen +** is either a NULL pointer or string obtained +** from xFullPathname() with an optional suffix added. +** ^If a suffix is added to the zFilename parameter, it will +** consist of a single "-" character followed by no more than +** 11 alphanumeric and/or "-" characters. +** ^SQLite further guarantees that +** the string will be valid and unchanged until xClose() is +** called. Because of the previous sentence, +** the [sqlite3_file] can safely store a pointer to the +** filename if it needs to remember the filename for some reason. +** If the zFilename parameter to xOpen is a NULL pointer then xOpen +** must invent its own temporary name for the file. ^Whenever the +** xFilename parameter is NULL it will also be the case that the +** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. +** +** The flags argument to xOpen() includes all bits set in +** the flags argument to [sqlite3_open_v2()]. Or if [sqlite3_open()] +** or [sqlite3_open16()] is used, then flags includes at least +** [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]. +** If xOpen() opens a file read-only then it sets *pOutFlags to +** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. +** +** ^(SQLite will also add one of the following flags to the xOpen() +** call, depending on the object being opened: +** +**
      +**
    • [SQLITE_OPEN_MAIN_DB] +**
    • [SQLITE_OPEN_MAIN_JOURNAL] +**
    • [SQLITE_OPEN_TEMP_DB] +**
    • [SQLITE_OPEN_TEMP_JOURNAL] +**
    • [SQLITE_OPEN_TRANSIENT_DB] +**
    • [SQLITE_OPEN_SUBJOURNAL] +**
    • [SQLITE_OPEN_SUPER_JOURNAL] +**
    • [SQLITE_OPEN_WAL] +**
    )^ +** +** The file I/O implementation can use the object type flags to +** change the way it deals with files. For example, an application +** that does not care about crash recovery or rollback might make +** the open of a journal file a no-op. Writes to this journal would +** also be no-ops, and any attempt to read the journal would return +** SQLITE_IOERR. Or the implementation might recognize that a database +** file will be doing page-aligned sector reads and writes in a random +** order and set up its I/O subsystem accordingly. +** +** SQLite might also add one of the following flags to the xOpen method: +** +**
      +**
    • [SQLITE_OPEN_DELETEONCLOSE] +**
    • [SQLITE_OPEN_EXCLUSIVE] +**
    +** +** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be +** deleted when it is closed. ^The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases and their journals, transient +** databases, and subjournals. +** +** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction +** with the [SQLITE_OPEN_CREATE] flag, which are both directly +** analogous to the O_EXCL and O_CREAT flags of the POSIX open() +** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the +** SQLITE_OPEN_CREATE, is used to indicate that file should always +** be created, and that it is an error if it already exists. +** It is not used to indicate the file should be opened +** for exclusive access. +** +** ^At least szOsFile bytes of memory are allocated by SQLite +** to hold the [sqlite3_file] structure passed as the third +** argument to xOpen. The xOpen method does not have to +** allocate the structure; it should just fill it in. Note that +** the xOpen method must set the sqlite3_file.pMethods to either +** a valid [sqlite3_io_methods] object or to NULL. xOpen must do +** this even if the open fails. SQLite expects that the sqlite3_file.pMethods +** element will be valid after xOpen returns regardless of the success +** or failure of the xOpen call. +** +** [[sqlite3_vfs.xAccess]] +** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to +** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] +** to test whether a file is at least readable. The SQLITE_ACCESS_READ +** flag is never actually used and is not implemented in the built-in +** VFSes of SQLite. The file is named by the second argument and can be a +** directory. The xAccess method returns [SQLITE_OK] on success or some +** non-zero error code if there is an I/O error or if the name of +** the file given in the second argument is illegal. If SQLITE_OK +** is returned, then non-zero or zero is written into *pResOut to indicate +** whether or not the file is accessible. +** +** ^SQLite will always allocate at least mxPathname+1 bytes for the +** output buffer xFullPathname. The exact size of the output buffer +** is also passed as a parameter to both methods. If the output buffer +** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is +** handled as a fatal error by SQLite, vfs implementations should endeavor +** to prevent this by setting mxPathname to a sufficiently large value. +** +** The xRandomness(), xSleep(), xCurrentTime(), and xCurrentTimeInt64() +** interfaces are not strictly a part of the filesystem, but they are +** included in the VFS structure for completeness. +** The xRandomness() function attempts to return nBytes bytes +** of good-quality randomness into zOut. The return value is +** the actual number of bytes of randomness obtained. +** The xSleep() method causes the calling thread to sleep for at +** least the number of microseconds given. ^The xCurrentTime() +** method returns a Julian Day Number for the current date and time as +** a floating point value. +** ^The xCurrentTimeInt64() method returns, as an integer, the Julian +** Day Number multiplied by 86400000 (the number of milliseconds in +** a 24-hour day). +** ^SQLite will use the xCurrentTimeInt64() method to get the current +** date and time if that method is available (if iVersion is 2 or +** greater and the function pointer is not NULL) and will fall back +** to xCurrentTime() if xCurrentTimeInt64() is unavailable. +** +** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces +** are not used by the SQLite core. These optional interfaces are provided +** by some VFSes to facilitate testing of the VFS code. By overriding +** system calls with functions under its control, a test program can +** simulate faults and error conditions that would otherwise be difficult +** or impossible to induce. The set of system calls that can be overridden +** varies from one VFS to another, and from one version of the same VFS to the +** next. Applications that use these interfaces must be prepared for any +** or all of these interfaces to be NULL or for their behavior to change +** from one release to the next. Applications must not attempt to access +** any of these methods if the iVersion of the VFS is less than 3. +*/ +typedef struct sqlite3_vfs sqlite3_vfs; +typedef void (*sqlite3_syscall_ptr)(void); +struct sqlite3_vfs { + int iVersion; /* Structure version number (currently 3) */ + int szOsFile; /* Size of subclassed sqlite3_file */ + int mxPathname; /* Maximum file pathname length */ + sqlite3_vfs *pNext; /* Next registered VFS */ + const char *zName; /* Name of this virtual file system */ + void *pAppData; /* Pointer to application-specific data */ + int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int flags, int *pOutFlags); + int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); + int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); + int (*xFullPathname)(sqlite3_vfs*, const char *zName, int nOut, char *zOut); + void *(*xDlOpen)(sqlite3_vfs*, const char *zFilename); + void (*xDlError)(sqlite3_vfs*, int nByte, char *zErrMsg); + void (*(*xDlSym)(sqlite3_vfs*,void*, const char *zSymbol))(void); + void (*xDlClose)(sqlite3_vfs*, void*); + int (*xRandomness)(sqlite3_vfs*, int nByte, char *zOut); + int (*xSleep)(sqlite3_vfs*, int microseconds); + int (*xCurrentTime)(sqlite3_vfs*, double*); + int (*xGetLastError)(sqlite3_vfs*, int, char *); + /* + ** The methods above are in version 1 of the sqlite_vfs object + ** definition. Those that follow are added in version 2 or later + */ + int (*xCurrentTimeInt64)(sqlite3_vfs*, sqlite3_int64*); + /* + ** The methods above are in versions 1 and 2 of the sqlite_vfs object. + ** Those below are for version 3 and greater. + */ + int (*xSetSystemCall)(sqlite3_vfs*, const char *zName, sqlite3_syscall_ptr); + sqlite3_syscall_ptr (*xGetSystemCall)(sqlite3_vfs*, const char *zName); + const char *(*xNextSystemCall)(sqlite3_vfs*, const char *zName); + /* + ** The methods above are in versions 1 through 3 of the sqlite_vfs object. + ** New fields may be appended in future versions. The iVersion + ** value will increment whenever this happens. + */ +}; + +/* +** CAPI3REF: Flags for the xAccess VFS method +** +** These integer constants can be used as the third parameter to +** the xAccess method of an [sqlite3_vfs] object. They determine +** what kind of permissions the xAccess method is looking for. +** With SQLITE_ACCESS_EXISTS, the xAccess method +** simply checks whether the file exists. +** With SQLITE_ACCESS_READWRITE, the xAccess method +** checks whether the named directory is both readable and writable +** (in other words, if files can be added, removed, and renamed within +** the directory). +** The SQLITE_ACCESS_READWRITE constant is currently used only by the +** [temp_store_directory pragma], though this could change in a future +** release of SQLite. +** With SQLITE_ACCESS_READ, the xAccess method +** checks whether the file is readable. The SQLITE_ACCESS_READ constant is +** currently unused, though it might be used in a future release of +** SQLite. +*/ +#define SQLITE_ACCESS_EXISTS 0 +#define SQLITE_ACCESS_READWRITE 1 /* Used by PRAGMA temp_store_directory */ +#define SQLITE_ACCESS_READ 2 /* Unused */ + +/* +** CAPI3REF: Flags for the xShmLock VFS method +** +** These integer constants define the various locking operations +** allowed by the xShmLock method of [sqlite3_io_methods]. The +** following are the only legal combinations of flags to the +** xShmLock method: +** +**
      +**
    • SQLITE_SHM_LOCK | SQLITE_SHM_SHARED +**
    • SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE +**
    • SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED +**
    • SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE +**
    +** +** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as +** was given on the corresponding lock. +** +** The xShmLock method can transition between unlocked and SHARED or +** between unlocked and EXCLUSIVE. It cannot transition between SHARED +** and EXCLUSIVE. +*/ +#define SQLITE_SHM_UNLOCK 1 +#define SQLITE_SHM_LOCK 2 +#define SQLITE_SHM_SHARED 4 +#define SQLITE_SHM_EXCLUSIVE 8 + +/* +** CAPI3REF: Maximum xShmLock index +** +** The xShmLock method on [sqlite3_io_methods] may use values +** between 0 and this upper bound as its "offset" argument. +** The SQLite core will never attempt to acquire or release a +** lock outside of this range +*/ +#define SQLITE_SHM_NLOCK 8 + + +/* +** CAPI3REF: Initialize The SQLite Library +** +** ^The sqlite3_initialize() routine initializes the +** SQLite library. ^The sqlite3_shutdown() routine +** deallocates any resources that were allocated by sqlite3_initialize(). +** These routines are designed to aid in process initialization and +** shutdown on embedded systems. Workstation applications using +** SQLite normally do not need to invoke either of these routines. +** +** A call to sqlite3_initialize() is an "effective" call if it is +** the first time sqlite3_initialize() is invoked during the lifetime of +** the process, or if it is the first time sqlite3_initialize() is invoked +** following a call to sqlite3_shutdown(). ^(Only an effective call +** of sqlite3_initialize() does any initialization. All other calls +** are harmless no-ops.)^ +** +** A call to sqlite3_shutdown() is an "effective" call if it is the first +** call to sqlite3_shutdown() since the last sqlite3_initialize(). ^(Only +** an effective call to sqlite3_shutdown() does any deinitialization. +** All other valid calls to sqlite3_shutdown() are harmless no-ops.)^ +** +** The sqlite3_initialize() interface is threadsafe, but sqlite3_shutdown() +** is not. The sqlite3_shutdown() interface must only be called from a +** single thread. All open [database connections] must be closed and all +** other SQLite resources must be deallocated prior to invoking +** sqlite3_shutdown(). +** +** Among other things, ^sqlite3_initialize() will invoke +** sqlite3_os_init(). Similarly, ^sqlite3_shutdown() +** will invoke sqlite3_os_end(). +** +** ^The sqlite3_initialize() routine returns [SQLITE_OK] on success. +** ^If for some reason, sqlite3_initialize() is unable to initialize +** the library (perhaps it is unable to allocate a needed resource such +** as a mutex) it returns an [error code] other than [SQLITE_OK]. +** +** ^The sqlite3_initialize() routine is called internally by many other +** SQLite interfaces so that an application usually does not need to +** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] +** calls sqlite3_initialize() so the SQLite library will be automatically +** initialized when [sqlite3_open()] is called if it has not be initialized +** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] +** compile-time option, then the automatic calls to sqlite3_initialize() +** are omitted and the application must call sqlite3_initialize() directly +** prior to using any other SQLite interface. For maximum portability, +** it is recommended that applications always invoke sqlite3_initialize() +** directly prior to using any other SQLite interface. Future releases +** of SQLite may require this. In other words, the behavior exhibited +** when SQLite is compiled with [SQLITE_OMIT_AUTOINIT] might become the +** default behavior in some future release of SQLite. +** +** The sqlite3_os_init() routine does operating-system specific +** initialization of the SQLite library. The sqlite3_os_end() +** routine undoes the effect of sqlite3_os_init(). Typical tasks +** performed by these routines include allocation or deallocation +** of static resources, initialization of global variables, +** setting up a default [sqlite3_vfs] module, or setting up +** a default configuration using [sqlite3_config()]. +** +** The application should never invoke either sqlite3_os_init() +** or sqlite3_os_end() directly. The application should only invoke +** sqlite3_initialize() and sqlite3_shutdown(). The sqlite3_os_init() +** interface is called automatically by sqlite3_initialize() and +** sqlite3_os_end() is called by sqlite3_shutdown(). Appropriate +** implementations for sqlite3_os_init() and sqlite3_os_end() +** are built into SQLite when it is compiled for Unix, Windows, or OS/2. +** When [custom builds | built for other platforms] +** (using the [SQLITE_OS_OTHER=1] compile-time +** option) the application must supply a suitable implementation for +** sqlite3_os_init() and sqlite3_os_end(). An application-supplied +** implementation of sqlite3_os_init() or sqlite3_os_end() +** must return [SQLITE_OK] on success and some other [error code] upon +** failure. +*/ +SQLITE_API int sqlite3_initialize(void); +SQLITE_API int sqlite3_shutdown(void); +SQLITE_API int sqlite3_os_init(void); +SQLITE_API int sqlite3_os_end(void); + +/* +** CAPI3REF: Configuring The SQLite Library +** +** The sqlite3_config() interface is used to make global configuration +** changes to SQLite in order to tune SQLite to the specific needs of +** the application. The default configuration is recommended for most +** applications and so this routine is usually not necessary. It is +** provided to support rare applications with unusual needs. +** +** The sqlite3_config() interface is not threadsafe. The application +** must ensure that no other SQLite interfaces are invoked by other +** threads while sqlite3_config() is running. +** +** The sqlite3_config() interface +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** +** The first argument to sqlite3_config() is an integer +** [configuration option] that determines +** what property of SQLite is to be configured. Subsequent arguments +** vary depending on the [configuration option] +** in the first argument. +** +** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. +** ^If the option is unknown or SQLite is unable to set the option +** then this routine returns a non-zero [error code]. +*/ +SQLITE_API int sqlite3_config(int, ...); + +/* +** CAPI3REF: Configure database connections +** METHOD: sqlite3 +** +** The sqlite3_db_config() interface is used to make configuration +** changes to a [database connection]. The interface is similar to +** [sqlite3_config()] except that the changes apply to a single +** [database connection] (specified in the first argument). +** +** The second argument to sqlite3_db_config(D,V,...) is the +** [SQLITE_DBCONFIG_LOOKASIDE | configuration verb] - an integer code +** that indicates what aspect of the [database connection] is being configured. +** Subsequent arguments vary depending on the configuration verb. +** +** ^Calls to sqlite3_db_config() return SQLITE_OK if and only if +** the call is considered successful. +*/ +SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Memory Allocation Routines +** +** An instance of this object defines the interface between SQLite +** and low-level memory allocation routines. +** +** This object is used in only one place in the SQLite interface. +** A pointer to an instance of this object is the argument to +** [sqlite3_config()] when the configuration option is +** [SQLITE_CONFIG_MALLOC] or [SQLITE_CONFIG_GETMALLOC]. +** By creating an instance of this object +** and passing it to [sqlite3_config]([SQLITE_CONFIG_MALLOC]) +** during configuration, an application can specify an alternative +** memory allocation subsystem for SQLite to use for all of its +** dynamic memory needs. +** +** Note that SQLite comes with several [built-in memory allocators] +** that are perfectly adequate for the overwhelming majority of applications +** and that this object is only useful to a tiny minority of applications +** with specialized memory allocation requirements. This object is +** also used during testing of SQLite in order to specify an alternative +** memory allocator that simulates memory out-of-memory conditions in +** order to verify that SQLite recovers gracefully from such +** conditions. +** +** The xMalloc, xRealloc, and xFree methods must work like the +** malloc(), realloc() and free() functions from the standard C library. +** ^SQLite guarantees that the second argument to +** xRealloc is always a value returned by a prior call to xRoundup. +** +** xSize should return the allocated size of a memory allocation +** previously obtained from xMalloc or xRealloc. The allocated size +** is always at least as big as the requested size but may be larger. +** +** The xRoundup method returns what would be the allocated size of +** a memory allocation given a particular requested size. Most memory +** allocators round up memory allocations at least to the next multiple +** of 8. Some allocators round up to a larger multiple or to a power of 2. +** Every memory allocation request coming in through [sqlite3_malloc()] +** or [sqlite3_realloc()] first calls xRoundup. If xRoundup returns 0, +** that causes the corresponding memory allocation to fail. +** +** The xInit method initializes the memory allocator. For example, +** it might allocate any required mutexes or initialize internal data +** structures. The xShutdown method is invoked (indirectly) by +** [sqlite3_shutdown()] and should deallocate any resources acquired +** by xInit. The pAppData pointer is used as the only parameter to +** xInit and xShutdown. +** +** SQLite holds the [SQLITE_MUTEX_STATIC_MAIN] mutex when it invokes +** the xInit method, so the xInit method need not be threadsafe. The +** xShutdown method is only called from [sqlite3_shutdown()] so it does +** not need to be threadsafe either. For all other methods, SQLite +** holds the [SQLITE_MUTEX_STATIC_MEM] mutex as long as the +** [SQLITE_CONFIG_MEMSTATUS] configuration option is turned on (which +** it is by default) and so the methods are automatically serialized. +** However, if [SQLITE_CONFIG_MEMSTATUS] is disabled, then the other +** methods must be threadsafe or else make their own arrangements for +** serialization. +** +** SQLite will never invoke xInit() more than once without an intervening +** call to xShutdown(). +*/ +typedef struct sqlite3_mem_methods sqlite3_mem_methods; +struct sqlite3_mem_methods { + void *(*xMalloc)(int); /* Memory allocation function */ + void (*xFree)(void*); /* Free a prior allocation */ + void *(*xRealloc)(void*,int); /* Resize an allocation */ + int (*xSize)(void*); /* Return the size of an allocation */ + int (*xRoundup)(int); /* Round up request size to allocation size */ + int (*xInit)(void*); /* Initialize the memory allocator */ + void (*xShutdown)(void*); /* Deinitialize the memory allocator */ + void *pAppData; /* Argument to xInit() and xShutdown() */ +}; + +/* +** CAPI3REF: Configuration Options +** KEYWORDS: {configuration option} +** +** These constants are the available integer configuration options that +** can be passed as the first argument to the [sqlite3_config()] interface. +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_config()] to make sure that +** the call worked. The [sqlite3_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
    +** [[SQLITE_CONFIG_SINGLETHREAD]]
    SQLITE_CONFIG_SINGLETHREAD
    +**
    There are no arguments to this option. ^This option sets the +** [threading mode] to Single-thread. In other words, it disables +** all mutexing and puts SQLite into a mode where it can only be used +** by a single thread. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to change the [threading mode] from its default +** value of Single-thread and so [sqlite3_config()] will return +** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD +** configuration option.
    +** +** [[SQLITE_CONFIG_MULTITHREAD]]
    SQLITE_CONFIG_MULTITHREAD
    +**
    There are no arguments to this option. ^This option sets the +** [threading mode] to Multi-thread. In other words, it disables +** mutexing on [database connection] and [prepared statement] objects. +** The application is responsible for serializing access to +** [database connections] and [prepared statements]. But other mutexes +** are enabled so that SQLite will be safe to use in a multi-threaded +** environment as long as no two threads attempt to use the same +** [database connection] at the same time. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Multi-thread [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_MULTITHREAD configuration option.
    +** +** [[SQLITE_CONFIG_SERIALIZED]]
    SQLITE_CONFIG_SERIALIZED
    +**
    There are no arguments to this option. ^This option sets the +** [threading mode] to Serialized. In other words, this option enables +** all mutexes including the recursive +** mutexes on [database connection] and [prepared statement] objects. +** In this mode (which is the default when SQLite is compiled with +** [SQLITE_THREADSAFE=1]) the SQLite library will itself serialize access +** to [database connections] and [prepared statements] so that the +** application is free to use the same [database connection] or the +** same [prepared statement] in different threads at the same time. +** ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Serialized [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_SERIALIZED configuration option.
    +** +** [[SQLITE_CONFIG_MALLOC]]
    SQLITE_CONFIG_MALLOC
    +**
    ^(The SQLITE_CONFIG_MALLOC option takes a single argument which is +** a pointer to an instance of the [sqlite3_mem_methods] structure. +** The argument specifies +** alternative low-level memory allocation routines to be used in place of +** the memory allocation routines built into SQLite.)^ ^SQLite makes +** its own private copy of the content of the [sqlite3_mem_methods] structure +** before the [sqlite3_config()] call returns.
    +** +** [[SQLITE_CONFIG_GETMALLOC]]
    SQLITE_CONFIG_GETMALLOC
    +**
    ^(The SQLITE_CONFIG_GETMALLOC option takes a single argument which +** is a pointer to an instance of the [sqlite3_mem_methods] structure. +** The [sqlite3_mem_methods] +** structure is filled with the currently defined memory allocation routines.)^ +** This option can be used to overload the default memory allocation +** routines with a wrapper that simulations memory allocation failure or +** tracks memory usage, for example.
    +** +** [[SQLITE_CONFIG_SMALL_MALLOC]]
    SQLITE_CONFIG_SMALL_MALLOC
    +**
    ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of +** type int, interpreted as a boolean, which if true provides a hint to +** SQLite that it should avoid large memory allocations if possible. +** SQLite will run faster if it is free to make large memory allocations, +** but some application might prefer to run slower in exchange for +** guarantees about memory fragmentation that are possible if large +** allocations are avoided. This hint is normally off. +**
    +** +** [[SQLITE_CONFIG_MEMSTATUS]]
    SQLITE_CONFIG_MEMSTATUS
    +**
    ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +** interpreted as a boolean, which enables or disables the collection of +** memory allocation statistics. ^(When memory allocation statistics are +** disabled, the following SQLite interfaces become non-operational: +**
      +**
    • [sqlite3_hard_heap_limit64()] +**
    • [sqlite3_memory_used()] +**
    • [sqlite3_memory_highwater()] +**
    • [sqlite3_soft_heap_limit64()] +**
    • [sqlite3_status64()] +**
    )^ +** ^Memory allocation statistics are enabled by default unless SQLite is +** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory +** allocation statistics are disabled by default. +**
    +** +** [[SQLITE_CONFIG_SCRATCH]]
    SQLITE_CONFIG_SCRATCH
    +**
    The SQLITE_CONFIG_SCRATCH option is no longer used. +**
    +** +** [[SQLITE_CONFIG_PAGECACHE]]
    SQLITE_CONFIG_PAGECACHE
    +**
    ^The SQLITE_CONFIG_PAGECACHE option specifies a memory pool +** that SQLite can use for the database page cache with the default page +** cache implementation. +** This configuration option is a no-op if an application-defined page +** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2]. +** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to +** 8-byte aligned memory (pMem), the size of each page cache line (sz), +** and the number of cache lines (N). +** The sz argument should be the size of the largest database page +** (a power of two between 512 and 65536) plus some extra bytes for each +** page header. ^The number of extra bytes needed by the page header +** can be determined using [SQLITE_CONFIG_PCACHE_HDRSZ]. +** ^It is harmless, apart from the wasted memory, +** for the sz parameter to be larger than necessary. The pMem +** argument must be either a NULL pointer or a pointer to an 8-byte +** aligned block of memory of at least sz*N bytes, otherwise +** subsequent behavior is undefined. +** ^When pMem is not NULL, SQLite will strive to use the memory provided +** to satisfy page cache needs, falling back to [sqlite3_malloc()] if +** a page cache line is larger than sz bytes or if all of the pMem buffer +** is exhausted. +** ^If pMem is NULL and N is non-zero, then each database connection +** does an initial bulk allocation for page cache memory +** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or +** of -1024*N bytes if N is negative, . ^If additional +** page cache memory is needed beyond what is provided by the initial +** allocation, then SQLite goes to [sqlite3_malloc()] separately for each +** additional cache line.
    +** +** [[SQLITE_CONFIG_HEAP]]
    SQLITE_CONFIG_HEAP
    +**
    ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer +** that SQLite will use for all of its dynamic memory allocation needs +** beyond those provided for by [SQLITE_CONFIG_PAGECACHE]. +** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled +** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns +** [SQLITE_ERROR] if invoked otherwise. +** ^There are three arguments to SQLITE_CONFIG_HEAP: +** An 8-byte aligned pointer to the memory, +** the number of bytes in the memory buffer, and the minimum allocation size. +** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts +** to using its default memory allocator (the system malloc() implementation), +** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. ^If the +** memory pointer is not NULL then the alternative memory +** allocator is engaged to handle all of SQLites memory allocation needs. +** The first pointer (the memory pointer) must be aligned to an 8-byte +** boundary or subsequent behavior of SQLite will be undefined. +** The minimum allocation size is capped at 2**12. Reasonable values +** for the minimum allocation size are 2**5 through 2**8.
    +** +** [[SQLITE_CONFIG_MUTEX]]
    SQLITE_CONFIG_MUTEX
    +**
    ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a +** pointer to an instance of the [sqlite3_mutex_methods] structure. +** The argument specifies alternative low-level mutex routines to be used +** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** the content of the [sqlite3_mutex_methods] structure before the call to +** [sqlite3_config()] returns. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will +** return [SQLITE_ERROR].
    +** +** [[SQLITE_CONFIG_GETMUTEX]]
    SQLITE_CONFIG_GETMUTEX
    +**
    ^(The SQLITE_CONFIG_GETMUTEX option takes a single argument which +** is a pointer to an instance of the [sqlite3_mutex_methods] structure. The +** [sqlite3_mutex_methods] +** structure is filled with the currently defined mutex routines.)^ +** This option can be used to overload the default mutex allocation +** routines with a wrapper used to track mutex usage for performance +** profiling or testing, for example. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_GETMUTEX configuration option will +** return [SQLITE_ERROR].
    +** +** [[SQLITE_CONFIG_LOOKASIDE]]
    SQLITE_CONFIG_LOOKASIDE
    +**
    ^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine +** the default size of lookaside memory on each [database connection]. +** The first argument is the +** size of each lookaside buffer slot and the second is the number of +** slots allocated to each database connection.)^ ^(SQLITE_CONFIG_LOOKASIDE +** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] +** option to [sqlite3_db_config()] can be used to change the lookaside +** configuration on individual connections.)^
    +** +** [[SQLITE_CONFIG_PCACHE2]]
    SQLITE_CONFIG_PCACHE2
    +**
    ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is +** a pointer to an [sqlite3_pcache_methods2] object. This object specifies +** the interface to a custom page cache implementation.)^ +** ^SQLite makes a copy of the [sqlite3_pcache_methods2] object.
    +** +** [[SQLITE_CONFIG_GETPCACHE2]]
    SQLITE_CONFIG_GETPCACHE2
    +**
    ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** the current page cache implementation into that object.)^
    +** +** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    +**
    The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +** function with a call signature of void(*)(void*,int,const char*), +** and a pointer to void. ^If the function pointer is not NULL, it is +** invoked by [sqlite3_log()] to process each logging event. ^If the +** function pointer is NULL, the [sqlite3_log()] interface becomes a no-op. +** ^The void pointer that is the second argument to SQLITE_CONFIG_LOG is +** passed through as the first parameter to the application-defined logger +** function whenever that function is invoked. ^The second parameter to +** the logger function is a copy of the first parameter to the corresponding +** [sqlite3_log()] call and is intended to be a [result code] or an +** [extended result code]. ^The third parameter passed to the logger is +** log message after formatting via [sqlite3_snprintf()]. +** The SQLite logging interface is not reentrant; the logger function +** supplied by the application must not invoke any SQLite interface. +** In a multi-threaded application, the application-defined logger +** function must be threadsafe.
    +** +** [[SQLITE_CONFIG_URI]]
    SQLITE_CONFIG_URI +**
    ^(The SQLITE_CONFIG_URI option takes a single argument of type int. +** If non-zero, then URI handling is globally enabled. If the parameter is zero, +** then URI handling is globally disabled.)^ ^If URI handling is globally +** enabled, all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], +** [sqlite3_open16()] or +** specified as part of [ATTACH] commands are interpreted as URIs, regardless +** of whether or not the [SQLITE_OPEN_URI] flag is set when the database +** connection is opened. ^If it is globally disabled, filenames are +** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the +** database connection is opened. ^(By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** [SQLITE_USE_URI] symbol defined.)^ +** +** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
    SQLITE_CONFIG_COVERING_INDEX_SCAN +**
    ^The SQLITE_CONFIG_COVERING_INDEX_SCAN option takes a single integer +** argument which is interpreted as a boolean in order to enable or disable +** the use of covering indices for full table scans in the query optimizer. +** ^The default setting is determined +** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" +** if that compile-time option is omitted. +** The ability to disable the use of covering indices for full table scans +** is because some incorrectly coded legacy applications might malfunction +** when the optimization is enabled. Providing the ability to +** disable the optimization allows the older, buggy application code to work +** without change even with newer versions of SQLite. +** +** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] +**
    SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE +**
    These options are obsolete and should not be used by new code. +** They are retained for backwards compatibility but are now no-ops. +**
    +** +** [[SQLITE_CONFIG_SQLLOG]] +**
    SQLITE_CONFIG_SQLLOG +**
    This option is only available if sqlite is compiled with the +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
    +** +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
    SQLITE_CONFIG_MMAP_SIZE +**
    ^SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** ^The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size +** will be silently truncated if necessary so that it does not exceed the +** compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ +** ^If either argument to this option is negative, then that argument is +** changed to its compile-time default. +** +** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] +**
    SQLITE_CONFIG_WIN32_HEAPSIZE +**
    ^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is +** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro +** defined. ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +** that specifies the maximum size of the created heap. +** +** [[SQLITE_CONFIG_PCACHE_HDRSZ]] +**
    SQLITE_CONFIG_PCACHE_HDRSZ +**
    ^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which +** is a pointer to an integer and writes into that integer the number of extra +** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. +** The amount of extra space required can change depending on the compiler, +** target platform, and SQLite version. +** +** [[SQLITE_CONFIG_PMASZ]] +**
    SQLITE_CONFIG_PMASZ +**
    ^The SQLITE_CONFIG_PMASZ option takes a single parameter which +** is an unsigned integer and sets the "Minimum PMA Size" for the multithreaded +** sorter to that integer. The default minimum PMA Size is set by the +** [SQLITE_SORTER_PMASZ] compile-time option. New threads are launched +** to help with sort operations when multithreaded sorting +** is enabled (using the [PRAGMA threads] command) and the amount of content +** to be sorted exceeds the page size times the minimum of the +** [PRAGMA cache_size] setting and this value. +** +** [[SQLITE_CONFIG_STMTJRNL_SPILL]] +**
    SQLITE_CONFIG_STMTJRNL_SPILL +**
    ^The SQLITE_CONFIG_STMTJRNL_SPILL option takes a single parameter which +** becomes the [statement journal] spill-to-disk threshold. +** [Statement journals] are held in memory until their size (in bytes) +** exceeds this threshold, at which point they are written to disk. +** Or if the threshold is -1, statement journals are always held +** exclusively in memory. +** Since many statement journals never become large, setting the spill +** threshold to a value such as 64KiB can greatly reduce the amount of +** I/O required to support statement rollback. +** The default value for this setting is controlled by the +** [SQLITE_STMTJRNL_SPILL] compile-time option. +** +** [[SQLITE_CONFIG_SORTERREF_SIZE]] +**
    SQLITE_CONFIG_SORTERREF_SIZE +**
    The SQLITE_CONFIG_SORTERREF_SIZE option accepts a single parameter +** of type (int) - the new value of the sorter-reference size threshold. +** Usually, when SQLite uses an external sort to order records according +** to an ORDER BY clause, all fields required by the caller are present in the +** sorted records. However, if SQLite determines based on the declared type +** of a table column that its values are likely to be very large - larger +** than the configured sorter-reference size threshold - then a reference +** is stored in each sorted record and the required column values loaded +** from the database as records are returned in sorted order. The default +** value for this option is to never use this optimization. Specifying a +** negative value for this option restores the default behaviour. +** This option is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. +** +** [[SQLITE_CONFIG_MEMDB_MAXSIZE]] +**
    SQLITE_CONFIG_MEMDB_MAXSIZE +**
    The SQLITE_CONFIG_MEMDB_MAXSIZE option accepts a single parameter +** [sqlite3_int64] parameter which is the default maximum size for an in-memory +** database created using [sqlite3_deserialize()]. This default maximum +** size can be adjusted up or down for individual databases using the +** [SQLITE_FCNTL_SIZE_LIMIT] [sqlite3_file_control|file-control]. If this +** configuration setting is never used, then the default maximum is determined +** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that +** compile-time option is not set, then the default maximum is 1073741824. +**
    +*/ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ +#define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ +#define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ +#define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ +#define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ +#define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ +#define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ + +/* +** CAPI3REF: Database Connection Configuration Options +** +** These constants are the available integer configuration options that +** can be passed as the second argument to the [sqlite3_db_config()] interface. +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_db_config()] to make sure that +** the call worked. ^The [sqlite3_db_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
    +** [[SQLITE_DBCONFIG_LOOKASIDE]] +**
    SQLITE_DBCONFIG_LOOKASIDE
    +**
    ^This option takes three additional arguments that determine the +** [lookaside memory allocator] configuration for the [database connection]. +** ^The first argument (the third parameter to [sqlite3_db_config()] is a +** pointer to a memory buffer to use for lookaside memory. +** ^The first argument after the SQLITE_DBCONFIG_LOOKASIDE verb +** may be NULL in which case SQLite will allocate the +** lookaside buffer itself using [sqlite3_malloc()]. ^The second argument is the +** size of each lookaside buffer slot. ^The third argument is the number of +** slots. The size of the buffer in the first argument must be greater than +** or equal to the product of the second and third arguments. The buffer +** must be aligned to an 8-byte boundary. ^If the second argument to +** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally +** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** configuration for a database connection can only be changed when that +** connection is not currently using lookaside memory, or in other words +** when the "current value" returned by +** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** Any attempt to change the lookaside memory configuration when lookaside +** memory is in use leaves the configuration unchanged and returns +** [SQLITE_BUSY].)^
    +** +** [[SQLITE_DBCONFIG_ENABLE_FKEY]] +**
    SQLITE_DBCONFIG_ENABLE_FKEY
    +**
    ^This option is used to enable or disable the enforcement of +** [foreign key constraints]. There should be two additional arguments. +** The first argument is an integer which is 0 to disable FK enforcement, +** positive to enable FK enforcement or negative to leave FK enforcement +** unchanged. The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether FK enforcement is off or on +** following this call. The second parameter may be a NULL pointer, in +** which case the FK enforcement setting is not reported back.
    +** +** [[SQLITE_DBCONFIG_ENABLE_TRIGGER]] +**
    SQLITE_DBCONFIG_ENABLE_TRIGGER
    +**
    ^This option is used to enable or disable [CREATE TRIGGER | triggers]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable triggers, +** positive to enable triggers or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether triggers are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the trigger setting is not reported back. +** +**

    Originally this option disabled all triggers. ^(However, since +** SQLite version 3.35.0, TEMP triggers are still allowed even if +** this option is off. So, in other words, this option now only disables +** triggers in the main database schema or in the schemas of ATTACH-ed +** databases.)^

    +** +** [[SQLITE_DBCONFIG_ENABLE_VIEW]] +**
    SQLITE_DBCONFIG_ENABLE_VIEW
    +**
    ^This option is used to enable or disable [CREATE VIEW | views]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable views, +** positive to enable views or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether views are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the view setting is not reported back. +** +**

    Originally this option disabled all views. ^(However, since +** SQLite version 3.35.0, TEMP views are still allowed even if +** this option is off. So, in other words, this option now only disables +** views in the main database schema or in the schemas of ATTACH-ed +** databases.)^

    +** +** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] +**
    SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
    +**
    ^This option is used to enable or disable the +** [fts3_tokenizer()] function which is part of the +** [FTS3] full-text search engine extension. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable fts3_tokenizer() or +** positive to enable fts3_tokenizer() or negative to leave the setting +** unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the new setting is not reported back.
    +** +** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]] +**
    SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
    +**
    ^This option is used to enable or disable the [sqlite3_load_extension()] +** interface independently of the [load_extension()] SQL function. +** The [sqlite3_enable_load_extension()] API enables or disables both the +** C-API [sqlite3_load_extension()] and the SQL function [load_extension()]. +** There should be two additional arguments. +** When the first argument to this interface is 1, then only the C-API is +** enabled and the SQL function remains disabled. If the first argument to +** this interface is 0, then both the C-API and the SQL function are disabled. +** If the first argument is -1, then no changes are made to state of either the +** C-API or the SQL function. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface +** is disabled or enabled following this call. The second parameter may +** be a NULL pointer, in which case the new setting is not reported back. +**
    +** +** [[SQLITE_DBCONFIG_MAINDBNAME]]
    SQLITE_DBCONFIG_MAINDBNAME
    +**
    ^This option is used to change the name of the "main" database +** schema. ^The sole argument is a pointer to a constant UTF8 string +** which will become the new schema name in place of "main". ^SQLite +** does not make a copy of the new main schema name string, so the application +** must ensure that the argument passed into this DBCONFIG option is unchanged +** until after the database connection closes. +**
    +** +** [[SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE]] +**
    SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE
    +**
    Usually, when a database in wal mode is closed or detached from a +** database handle, SQLite checks if this will mean that there are now no +** connections at all to the database. If so, it performs a checkpoint +** operation before closing the connection. This option may be used to +** override this behaviour. The first parameter passed to this operation +** is an integer - positive to disable checkpoints-on-close, or zero (the +** default) to enable them, and negative to leave the setting unchanged. +** The second parameter is a pointer to an integer +** into which is written 0 or 1 to indicate whether checkpoints-on-close +** have been disabled - 0 if they are not disabled, 1 if they are. +**
    +** +** [[SQLITE_DBCONFIG_ENABLE_QPSG]]
    SQLITE_DBCONFIG_ENABLE_QPSG
    +**
    ^(The SQLITE_DBCONFIG_ENABLE_QPSG option activates or deactivates +** the [query planner stability guarantee] (QPSG). When the QPSG is active, +** a single SQL query statement will always use the same algorithm regardless +** of values of [bound parameters].)^ The QPSG disables some query optimizations +** that look at the values of bound parameters, which can make some queries +** slower. But the QPSG has the advantage of more predictable behavior. With +** the QPSG active, SQLite will always use the same query plan in the field as +** was used during testing in the lab. +** The first argument to this setting is an integer which is 0 to disable +** the QPSG, positive to enable QPSG, or negative to leave the setting +** unchanged. The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether the QPSG is disabled or enabled +** following this call. +**
    +** +** [[SQLITE_DBCONFIG_TRIGGER_EQP]]
    SQLITE_DBCONFIG_TRIGGER_EQP
    +**
    By default, the output of EXPLAIN QUERY PLAN commands does not +** include output for any operations performed by trigger programs. This +** option is used to set or clear (the default) a flag that governs this +** behavior. The first parameter passed to this operation is an integer - +** positive to enable output for trigger programs, or zero to disable it, +** or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which is written +** 0 or 1 to indicate whether output-for-triggers has been disabled - 0 if +** it is not disabled, 1 if it is. +**
    +** +** [[SQLITE_DBCONFIG_RESET_DATABASE]]
    SQLITE_DBCONFIG_RESET_DATABASE
    +**
    Set the SQLITE_DBCONFIG_RESET_DATABASE flag and then run +** [VACUUM] in order to reset a database back to an empty database +** with no schema and no content. The following process works even for +** a badly corrupted database file: +**
      +**
    1. If the database connection is newly opened, make sure it has read the +** database schema by preparing then discarding some query against the +** database, or calling sqlite3_table_column_metadata(), ignoring any +** errors. This step is only necessary if the application desires to keep +** the database in WAL mode after the reset if it was in WAL mode before +** the reset. +**
    2. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); +**
    3. [sqlite3_exec](db, "[VACUUM]", 0, 0, 0); +**
    4. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); +**
    +** Because resetting a database is destructive and irreversible, the +** process requires the use of this obscure API and multiple steps to help +** ensure that it does not happen by accident. +** +** [[SQLITE_DBCONFIG_DEFENSIVE]]
    SQLITE_DBCONFIG_DEFENSIVE
    +**
    The SQLITE_DBCONFIG_DEFENSIVE option activates or deactivates the +** "defensive" flag for a database connection. When the defensive +** flag is enabled, language features that allow ordinary SQL to +** deliberately corrupt the database file are disabled. The disabled +** features include but are not limited to the following: +**
      +**
    • The [PRAGMA writable_schema=ON] statement. +**
    • The [PRAGMA journal_mode=OFF] statement. +**
    • Writes to the [sqlite_dbpage] virtual table. +**
    • Direct writes to [shadow tables]. +**
    +**
    +** +** [[SQLITE_DBCONFIG_WRITABLE_SCHEMA]]
    SQLITE_DBCONFIG_WRITABLE_SCHEMA
    +**
    The SQLITE_DBCONFIG_WRITABLE_SCHEMA option activates or deactivates the +** "writable_schema" flag. This has the same effect and is logically equivalent +** to setting [PRAGMA writable_schema=ON] or [PRAGMA writable_schema=OFF]. +** The first argument to this setting is an integer which is 0 to disable +** the writable_schema, positive to enable writable_schema, or negative to +** leave the setting unchanged. The second parameter is a pointer to an +** integer into which is written 0 or 1 to indicate whether the writable_schema +** is enabled or disabled following this call. +**
    +** +** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] +**
    SQLITE_DBCONFIG_LEGACY_ALTER_TABLE
    +**
    The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates +** the legacy behavior of the [ALTER TABLE RENAME] command such it +** behaves as it did prior to [version 3.24.0] (2018-06-04). See the +** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for +** additional information. This feature can also be turned on and off +** using the [PRAGMA legacy_alter_table] statement. +**
    +** +** [[SQLITE_DBCONFIG_DQS_DML]] +**
    SQLITE_DBCONFIG_DQS_DML +**
    The SQLITE_DBCONFIG_DQS_DML option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DML statements +** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
    +** +** [[SQLITE_DBCONFIG_DQS_DDL]] +**
    SQLITE_DBCONFIG_DQS_DDL +**
    The SQLITE_DBCONFIG_DQS option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DDL statements, +** such as CREATE TABLE and CREATE INDEX. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
    +** +** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] +**
    SQLITE_DBCONFIG_TRUSTED_SCHEMA +**
    The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to +** assume that database schemas are untainted by malicious content. +** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite +** takes additional defensive steps to protect the application from harm +** including: +**
      +**
    • Prohibit the use of SQL functions inside triggers, views, +** CHECK constraints, DEFAULT clauses, expression indexes, +** partial indexes, or generated columns +** unless those functions are tagged with [SQLITE_INNOCUOUS]. +**
    • Prohibit the use of virtual tables inside of triggers or views +** unless those virtual tables are tagged with [SQLITE_VTAB_INNOCUOUS]. +**
    +** This setting defaults to "on" for legacy compatibility, however +** all applications are advised to turn it off if possible. This setting +** can also be controlled using the [PRAGMA trusted_schema] statement. +**
    +** +** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] +**
    SQLITE_DBCONFIG_LEGACY_FILE_FORMAT +**
    The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates +** the legacy file format flag. When activated, this flag causes all newly +** created database file to have a schema format version number (the 4-byte +** integer found at offset 44 into the database header) of 1. This in turn +** means that the resulting database file will be readable and writable by +** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, +** newly created databases are generally not understandable by SQLite versions +** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there +** is now scarcely any need to generated database files that are compatible +** all the way back to version 3.0.0, and so this setting is of little +** practical use, but is provided so that SQLite can continue to claim the +** ability to generate new database files that are compatible with version +** 3.0.0. +**

    Note that when the SQLITE_DBCONFIG_LEGACY_FILE_FORMAT setting is on, +** the [VACUUM] command will fail with an obscure error when attempting to +** process a table with generated columns and a descending index. This is +** not considered a bug since SQLite versions 3.3.0 and earlier do not support +** either generated columns or decending indexes. +**

    +**
    +*/ +#define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ +#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ +#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */ +#define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE 1006 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_QPSG 1007 /* int int* */ +#define SQLITE_DBCONFIG_TRIGGER_EQP 1008 /* int int* */ +#define SQLITE_DBCONFIG_RESET_DATABASE 1009 /* int int* */ +#define SQLITE_DBCONFIG_DEFENSIVE 1010 /* int int* */ +#define SQLITE_DBCONFIG_WRITABLE_SCHEMA 1011 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_ALTER_TABLE 1012 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DML 1013 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DDL 1014 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ +#define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ + +/* +** CAPI3REF: Enable Or Disable Extended Result Codes +** METHOD: sqlite3 +** +** ^The sqlite3_extended_result_codes() routine enables or disables the +** [extended result codes] feature of SQLite. ^The extended result +** codes are disabled by default for historical compatibility. +*/ +SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); + +/* +** CAPI3REF: Last Insert Rowid +** METHOD: sqlite3 +** +** ^Each entry in most SQLite tables (except for [WITHOUT ROWID] tables) +** has a unique 64-bit signed +** integer key called the [ROWID | "rowid"]. ^The rowid is always available +** as an undeclared column named ROWID, OID, or _ROWID_ as long as those +** names are not also used by explicitly declared columns. ^If +** the table has a column of type [INTEGER PRIMARY KEY] then that column +** is another alias for the rowid. +** +** ^The sqlite3_last_insert_rowid(D) interface usually returns the [rowid] of +** the most recent successful [INSERT] into a rowid table or [virtual table] +** on database connection D. ^Inserts into [WITHOUT ROWID] tables are not +** recorded. ^If no successful [INSERT]s into rowid tables have ever occurred +** on the database connection D, then sqlite3_last_insert_rowid(D) returns +** zero. +** +** As well as being set automatically as rows are inserted into database +** tables, the value returned by this function may be set explicitly by +** [sqlite3_set_last_insert_rowid()] +** +** Some virtual table implementations may INSERT rows into rowid tables as +** part of committing a transaction (e.g. to flush data accumulated in memory +** to disk). In this case subsequent calls to this function return the rowid +** associated with these internal INSERT operations, which leads to +** unintuitive results. Virtual table implementations that do write to rowid +** tables in this way can avoid this problem by restoring the original +** rowid value using [sqlite3_set_last_insert_rowid()] before returning +** control to the user. +** +** ^(If an [INSERT] occurs within a trigger then this routine will +** return the [rowid] of the inserted row as long as the trigger is +** running. Once the trigger program ends, the value returned +** by this routine reverts to what it was before the trigger was fired.)^ +** +** ^An [INSERT] that fails due to a constraint violation is not a +** successful [INSERT] and does not change the value returned by this +** routine. ^Thus INSERT OR FAIL, INSERT OR IGNORE, INSERT OR ROLLBACK, +** and INSERT OR ABORT make no changes to the return value of this +** routine when their insertion fails. ^(When INSERT OR REPLACE +** encounters a constraint violation, it does not fail. The +** INSERT continues to completion after deleting rows that caused +** the constraint problem so INSERT OR REPLACE will always change +** the return value of this interface.)^ +** +** ^For the purposes of this routine, an [INSERT] is considered to +** be successful even if it is subsequently rolled back. +** +** This function is accessible to SQL statements via the +** [last_insert_rowid() SQL function]. +** +** If a separate thread performs a new [INSERT] on the same +** database connection while the [sqlite3_last_insert_rowid()] +** function is running and thus changes the last insert [rowid], +** then the value returned by [sqlite3_last_insert_rowid()] is +** unpredictable and might not equal either the old or the new +** last insert [rowid]. +*/ +SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); + +/* +** CAPI3REF: Set the Last Insert Rowid value. +** METHOD: sqlite3 +** +** The sqlite3_set_last_insert_rowid(D, R) method allows the application to +** set the value returned by calling sqlite3_last_insert_rowid(D) to R +** without inserting a row into the database. +*/ +SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); + +/* +** CAPI3REF: Count The Number Of Rows Modified +** METHOD: sqlite3 +** +** ^These functions return the number of rows modified, inserted or +** deleted by the most recently completed INSERT, UPDATE or DELETE +** statement on the database connection specified by the only parameter. +** The two functions are identical except for the type of the return value +** and that if the number of rows modified by the most recent INSERT, UPDATE +** or DELETE is greater than the maximum value supported by type "int", then +** the return value of sqlite3_changes() is undefined. ^Executing any other +** type of SQL statement does not modify the value returned by these functions. +** +** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are +** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], +** [foreign key actions] or [REPLACE] constraint resolution are not counted. +** +** Changes to a view that are intercepted by +** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value +** returned by sqlite3_changes() immediately after an INSERT, UPDATE or +** DELETE statement run on a view is always zero. Only changes made to real +** tables are counted. +** +** Things are more complicated if the sqlite3_changes() function is +** executed while a trigger program is running. This may happen if the +** program uses the [changes() SQL function], or if some other callback +** function invokes sqlite3_changes() directly. Essentially: +** +**
      +**
    • ^(Before entering a trigger program the value returned by +** sqlite3_changes() function is saved. After the trigger program +** has finished, the original value is restored.)^ +** +**
    • ^(Within a trigger program each INSERT, UPDATE and DELETE +** statement sets the value returned by sqlite3_changes() +** upon completion as normal. Of course, this value will not include +** any changes performed by sub-triggers, as the sqlite3_changes() +** value will be saved and restored after each sub-trigger has run.)^ +**
    +** +** ^This means that if the changes() SQL function (or similar) is used +** by the first INSERT, UPDATE or DELETE statement within a trigger, it +** returns the value as set when the calling statement began executing. +** ^If it is used by the second or subsequent such statement within a trigger +** program, the value returned reflects the number of rows modified by the +** previous INSERT, UPDATE or DELETE statement within the same trigger. +** +** If a separate thread makes changes on the same database connection +** while [sqlite3_changes()] is running then the value returned +** is unpredictable and not meaningful. +** +** See also: +**
      +**
    • the [sqlite3_total_changes()] interface +**
    • the [count_changes pragma] +**
    • the [changes() SQL function] +**
    • the [data_version pragma] +**
    +*/ +SQLITE_API int sqlite3_changes(sqlite3*); +SQLITE_API sqlite3_int64 sqlite3_changes64(sqlite3*); + +/* +** CAPI3REF: Total Number Of Rows Modified +** METHOD: sqlite3 +** +** ^These functions return the total number of rows inserted, modified or +** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed +** since the database connection was opened, including those executed as +** part of trigger programs. The two functions are identical except for the +** type of the return value and that if the number of rows modified by the +** connection exceeds the maximum value supported by type "int", then +** the return value of sqlite3_total_changes() is undefined. ^Executing +** any other type of SQL statement does not affect the value returned by +** sqlite3_total_changes(). +** +** ^Changes made as part of [foreign key actions] are included in the +** count, but those made as part of REPLACE constraint resolution are +** not. ^Changes to a view that are intercepted by INSTEAD OF triggers +** are not counted. +** +** The [sqlite3_total_changes(D)] interface only reports the number +** of rows that changed due to SQL statement run against database +** connection D. Any changes by other database connections are ignored. +** To detect changes against a database file from other database +** connections use the [PRAGMA data_version] command or the +** [SQLITE_FCNTL_DATA_VERSION] [file control]. +** +** If a separate thread makes changes on the same database connection +** while [sqlite3_total_changes()] is running then the value +** returned is unpredictable and not meaningful. +** +** See also: +**
      +**
    • the [sqlite3_changes()] interface +**
    • the [count_changes pragma] +**
    • the [changes() SQL function] +**
    • the [data_version pragma] +**
    • the [SQLITE_FCNTL_DATA_VERSION] [file control] +**
    +*/ +SQLITE_API int sqlite3_total_changes(sqlite3*); +SQLITE_API sqlite3_int64 sqlite3_total_changes64(sqlite3*); + +/* +** CAPI3REF: Interrupt A Long-Running Query +** METHOD: sqlite3 +** +** ^This function causes any pending database operation to abort and +** return at its earliest opportunity. This routine is typically +** called in response to a user action such as pressing "Cancel" +** or Ctrl-C where the user wants a long query operation to halt +** immediately. +** +** ^It is safe to call this routine from a thread different from the +** thread that is currently running the database operation. But it +** is not safe to call this routine with a [database connection] that +** is closed or might close before sqlite3_interrupt() returns. +** +** ^If an SQL operation is very nearly finished at the time when +** sqlite3_interrupt() is called, then it might not have an opportunity +** to be interrupted and might continue to completion. +** +** ^An SQL operation that is interrupted will return [SQLITE_INTERRUPT]. +** ^If the interrupted SQL operation is an INSERT, UPDATE, or DELETE +** that is inside an explicit transaction, then the entire transaction +** will be rolled back automatically. +** +** ^The sqlite3_interrupt(D) call is in effect until all currently running +** SQL statements on [database connection] D complete. ^Any new SQL statements +** that are started after the sqlite3_interrupt() call and before the +** running statement count reaches zero are interrupted as if they had been +** running prior to the sqlite3_interrupt() call. ^New SQL statements +** that are started after the running statement count reaches zero are +** not effected by the sqlite3_interrupt(). +** ^A call to sqlite3_interrupt(D) that occurs when there are no running +** SQL statements is a no-op and has no effect on SQL statements +** that are started after the sqlite3_interrupt() call returns. +*/ +SQLITE_API void sqlite3_interrupt(sqlite3*); + +/* +** CAPI3REF: Determine If An SQL Statement Is Complete +** +** These routines are useful during command-line input to determine if the +** currently entered text seems to form a complete SQL statement or +** if additional input is needed before sending the text into +** SQLite for parsing. ^These routines return 1 if the input string +** appears to be a complete SQL statement. ^A statement is judged to be +** complete if it ends with a semicolon token and is not a prefix of a +** well-formed CREATE TRIGGER statement. ^Semicolons that are embedded within +** string literals or quoted identifier names or comments are not +** independent tokens (they are part of the token in which they are +** embedded) and thus do not count as a statement terminator. ^Whitespace +** and comments that follow the final semicolon are ignored. +** +** ^These routines return 0 if the statement is incomplete. ^If a +** memory allocation fails, then SQLITE_NOMEM is returned. +** +** ^These routines do not parse the SQL statements thus +** will not detect syntactically incorrect SQL. +** +** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior +** to invoking sqlite3_complete16() then sqlite3_initialize() is invoked +** automatically by sqlite3_complete16(). If that initialization fails, +** then the return value from sqlite3_complete16() will be non-zero +** regardless of whether or not the input SQL is complete.)^ +** +** The input to [sqlite3_complete()] must be a zero-terminated +** UTF-8 string. +** +** The input to [sqlite3_complete16()] must be a zero-terminated +** UTF-16 string in native byte order. +*/ +SQLITE_API int sqlite3_complete(const char *sql); +SQLITE_API int sqlite3_complete16(const void *sql); + +/* +** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** KEYWORDS: {busy-handler callback} {busy handler} +** METHOD: sqlite3 +** +** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X +** that might be invoked with argument P whenever +** an attempt is made to access a database table associated with +** [database connection] D when another thread +** or process has the table locked. +** The sqlite3_busy_handler() interface is used to implement +** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. +** +** ^If the busy callback is NULL, then [SQLITE_BUSY] +** is returned immediately upon encountering the lock. ^If the busy callback +** is not NULL, then the callback might be invoked with two arguments. +** +** ^The first argument to the busy handler is a copy of the void* pointer which +** is the third argument to sqlite3_busy_handler(). ^The second argument to +** the busy handler callback is the number of times that the busy handler has +** been invoked previously for the same locking event. ^If the +** busy callback returns 0, then no additional attempts are made to +** access the database and [SQLITE_BUSY] is returned +** to the application. +** ^If the callback returns non-zero, then another attempt +** is made to access the database and the cycle repeats. +** +** The presence of a busy handler does not guarantee that it will be invoked +** when there is lock contention. ^If SQLite determines that invoking the busy +** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] +** to the application instead of invoking the +** busy handler. +** Consider a scenario where one process is holding a read lock that +** it is trying to promote to a reserved lock and +** a second process is holding a reserved lock that it is trying +** to promote to an exclusive lock. The first process cannot proceed +** because it is blocked by the second and the second process cannot +** proceed because it is blocked by the first. If both processes +** invoke the busy handlers, neither will make any progress. Therefore, +** SQLite returns [SQLITE_BUSY] for the first process, hoping that this +** will induce the first process to release its read lock and allow +** the second process to proceed. +** +** ^The default busy callback is NULL. +** +** ^(There can only be a single busy handler defined for each +** [database connection]. Setting a new busy handler clears any +** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] +** or evaluating [PRAGMA busy_timeout=N] will change the +** busy handler and thus clear any previously set busy handler. +** +** The busy callback should not take any actions which modify the +** database connection that invoked the busy handler. In other words, +** the busy handler is not reentrant. Any such actions +** result in undefined behavior. +** +** A busy handler must not close the database connection +** or [prepared statement] that invoked the busy handler. +*/ +SQLITE_API int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*); + +/* +** CAPI3REF: Set A Busy Timeout +** METHOD: sqlite3 +** +** ^This routine sets a [sqlite3_busy_handler | busy handler] that sleeps +** for a specified amount of time when a table is locked. ^The handler +** will sleep multiple times until at least "ms" milliseconds of sleeping +** have accumulated. ^After at least "ms" milliseconds of sleeping, +** the handler returns 0 which causes [sqlite3_step()] to return +** [SQLITE_BUSY]. +** +** ^Calling this routine with an argument less than or equal to zero +** turns off all busy handlers. +** +** ^(There can only be a single busy handler for a particular +** [database connection] at any given moment. If another busy handler +** was defined (using [sqlite3_busy_handler()]) prior to calling +** this routine, that other busy handler is cleared.)^ +** +** See also: [PRAGMA busy_timeout] +*/ +SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); + +/* +** CAPI3REF: Convenience Routines For Running Queries +** METHOD: sqlite3 +** +** This is a legacy interface that is preserved for backwards compatibility. +** Use of this interface is not recommended. +** +** Definition: A result table is memory data structure created by the +** [sqlite3_get_table()] interface. A result table records the +** complete query results from one or more queries. +** +** The table conceptually has a number of rows and columns. But +** these numbers are not part of the result table itself. These +** numbers are obtained separately. Let N be the number of rows +** and M be the number of columns. +** +** A result table is an array of pointers to zero-terminated UTF-8 strings. +** There are (N+1)*M elements in the array. The first M pointers point +** to zero-terminated strings that contain the names of the columns. +** The remaining entries all point to query results. NULL values result +** in NULL pointers. All other values are in their UTF-8 zero-terminated +** string representation as returned by [sqlite3_column_text()]. +** +** A result table might consist of one or more memory allocations. +** It is not safe to pass a result table directly to [sqlite3_free()]. +** A result table should be deallocated using [sqlite3_free_table()]. +** +** ^(As an example of the result table format, suppose a query result +** is as follows: +** +** 
    +**        Name        | Age
+**        -----------------------
+**        Alice       | 43
+**        Bob         | 28
+**        Cindy       | 21
+**
    +** +** There are two columns (M==2) and three rows (N==3). Thus the +** result table has 8 entries. Suppose the result table is stored +** in an array named azResult. Then azResult holds this content: +** +** 
    +**        azResult[0] = "Name";
+**        azResult[1] = "Age";
+**        azResult[2] = "Alice";
+**        azResult[3] = "43";
+**        azResult[4] = "Bob";
+**        azResult[5] = "28";
+**        azResult[6] = "Cindy";
+**        azResult[7] = "21";
+**
    )^ +** +** ^The sqlite3_get_table() function evaluates one or more +** semicolon-separated SQL statements in the zero-terminated UTF-8 +** string of its 2nd parameter and returns a result table to the +** pointer given in its 3rd parameter. +** +** After the application has finished with the result from sqlite3_get_table(), +** it must pass the result table pointer to sqlite3_free_table() in order to +** release the memory that was malloced. Because of the way the +** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling +** function must not try to call [sqlite3_free()] directly. Only +** [sqlite3_free_table()] is able to release the memory properly and safely. +** +** The sqlite3_get_table() interface is implemented as a wrapper around +** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access +** to any internal data structures of SQLite. It uses only the public +** interface defined here. As a consequence, errors that occur in the +** wrapper layer outside of the internal [sqlite3_exec()] call are not +** reflected in subsequent calls to [sqlite3_errcode()] or +** [sqlite3_errmsg()]. +*/ +SQLITE_API int sqlite3_get_table( + sqlite3 *db, /* An open database */ + const char *zSql, /* SQL to be evaluated */ + char ***pazResult, /* Results of the query */ + int *pnRow, /* Number of result rows written here */ + int *pnColumn, /* Number of result columns written here */ + char **pzErrmsg /* Error msg written here */ +); +SQLITE_API void sqlite3_free_table(char **result); + +/* +** CAPI3REF: Formatted String Printing Functions +** +** These routines are work-alikes of the "printf()" family of functions +** from the standard C library. +** These routines understand most of the common formatting options from +** the standard library printf() +** plus some additional non-standard formats ([%q], [%Q], [%w], and [%z]). +** See the [built-in printf()] documentation for details. +** +** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their +** results into memory obtained from [sqlite3_malloc64()]. +** The strings returned by these two routines should be +** released by [sqlite3_free()]. ^Both routines return a +** NULL pointer if [sqlite3_malloc64()] is unable to allocate enough +** memory to hold the resulting string. +** +** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from +** the standard C library. The result is written into the +** buffer supplied as the second parameter whose size is given by +** the first parameter. Note that the order of the +** first two parameters is reversed from snprintf().)^ This is an +** historical accident that cannot be fixed without breaking +** backwards compatibility. ^(Note also that sqlite3_snprintf() +** returns a pointer to its buffer instead of the number of +** characters actually written into the buffer.)^ We admit that +** the number of characters written would be a more useful return +** value but we cannot change the implementation of sqlite3_snprintf() +** now without breaking compatibility. +** +** ^As long as the buffer size is greater than zero, sqlite3_snprintf() +** guarantees that the buffer is always zero-terminated. ^The first +** parameter "n" is the total size of the buffer, including space for +** the zero terminator. So the longest string that can be completely +** written will be n-1 characters. +** +** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf(). +** +** See also: [built-in printf()], [printf() SQL function] +*/ +SQLITE_API char *sqlite3_mprintf(const char*,...); +SQLITE_API char *sqlite3_vmprintf(const char*, va_list); +SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); +SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); + +/* +** CAPI3REF: Memory Allocation Subsystem +** +** The SQLite core uses these three routines for all of its own +** internal memory allocation needs. "Core" in the previous sentence +** does not include operating-system specific [VFS] implementation. The +** Windows VFS uses native malloc() and free() for some operations. +** +** ^The sqlite3_malloc() routine returns a pointer to a block +** of memory at least N bytes in length, where N is the parameter. +** ^If sqlite3_malloc() is unable to obtain sufficient free +** memory, it returns a NULL pointer. ^If the parameter N to +** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns +** a NULL pointer. +** +** ^The sqlite3_malloc64(N) routine works just like +** sqlite3_malloc(N) except that N is an unsigned 64-bit integer instead +** of a signed 32-bit integer. +** +** ^Calling sqlite3_free() with a pointer previously returned +** by sqlite3_malloc() or sqlite3_realloc() releases that memory so +** that it might be reused. ^The sqlite3_free() routine is +** a no-op if is called with a NULL pointer. Passing a NULL pointer +** to sqlite3_free() is harmless. After being freed, memory +** should neither be read nor written. Even reading previously freed +** memory might result in a segmentation fault or other severe error. +** Memory corruption, a segmentation fault, or other severe error +** might result if sqlite3_free() is called with a non-NULL pointer that +** was not obtained from sqlite3_malloc() or sqlite3_realloc(). +** +** ^The sqlite3_realloc(X,N) interface attempts to resize a +** prior memory allocation X to be at least N bytes. +** ^If the X parameter to sqlite3_realloc(X,N) +** is a NULL pointer then its behavior is identical to calling +** sqlite3_malloc(N). +** ^If the N parameter to sqlite3_realloc(X,N) is zero or +** negative then the behavior is exactly the same as calling +** sqlite3_free(X). +** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation +** of at least N bytes in size or NULL if insufficient memory is available. +** ^If M is the size of the prior allocation, then min(N,M) bytes +** of the prior allocation are copied into the beginning of buffer returned +** by sqlite3_realloc(X,N) and the prior allocation is freed. +** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the +** prior allocation is not freed. +** +** ^The sqlite3_realloc64(X,N) interfaces works the same as +** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead +** of a 32-bit signed integer. +** +** ^If X is a memory allocation previously obtained from sqlite3_malloc(), +** sqlite3_malloc64(), sqlite3_realloc(), or sqlite3_realloc64(), then +** sqlite3_msize(X) returns the size of that memory allocation in bytes. +** ^The value returned by sqlite3_msize(X) might be larger than the number +** of bytes requested when X was allocated. ^If X is a NULL pointer then +** sqlite3_msize(X) returns zero. If X points to something that is not +** the beginning of memory allocation, or if it points to a formerly +** valid memory allocation that has now been freed, then the behavior +** of sqlite3_msize(X) is undefined and possibly harmful. +** +** ^The memory returned by sqlite3_malloc(), sqlite3_realloc(), +** sqlite3_malloc64(), and sqlite3_realloc64() +** is always aligned to at least an 8 byte boundary, or to a +** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time +** option is used. +** +** The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()] +** must be either NULL or else pointers obtained from a prior +** invocation of [sqlite3_malloc()] or [sqlite3_realloc()] that have +** not yet been released. +** +** The application must not read or write any part of +** a block of memory after it has been released using +** [sqlite3_free()] or [sqlite3_realloc()]. +*/ +SQLITE_API void *sqlite3_malloc(int); +SQLITE_API void *sqlite3_malloc64(sqlite3_uint64); +SQLITE_API void *sqlite3_realloc(void*, int); +SQLITE_API void *sqlite3_realloc64(void*, sqlite3_uint64); +SQLITE_API void sqlite3_free(void*); +SQLITE_API sqlite3_uint64 sqlite3_msize(void*); + +/* +** CAPI3REF: Memory Allocator Statistics +** +** SQLite provides these two interfaces for reporting on the status +** of the [sqlite3_malloc()], [sqlite3_free()], and [sqlite3_realloc()] +** routines, which form the built-in memory allocation subsystem. +** +** ^The [sqlite3_memory_used()] routine returns the number of bytes +** of memory currently outstanding (malloced but not freed). +** ^The [sqlite3_memory_highwater()] routine returns the maximum +** value of [sqlite3_memory_used()] since the high-water mark +** was last reset. ^The values returned by [sqlite3_memory_used()] and +** [sqlite3_memory_highwater()] include any overhead +** added by SQLite in its implementation of [sqlite3_malloc()], +** but not overhead added by the any underlying system library +** routines that [sqlite3_malloc()] may call. +** +** ^The memory high-water mark is reset to the current value of +** [sqlite3_memory_used()] if and only if the parameter to +** [sqlite3_memory_highwater()] is true. ^The value returned +** by [sqlite3_memory_highwater(1)] is the high-water mark +** prior to the reset. +*/ +SQLITE_API sqlite3_int64 sqlite3_memory_used(void); +SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); + +/* +** CAPI3REF: Pseudo-Random Number Generator +** +** SQLite contains a high-quality pseudo-random number generator (PRNG) used to +** select random [ROWID | ROWIDs] when inserting new records into a table that +** already uses the largest possible [ROWID]. The PRNG is also used for +** the built-in random() and randomblob() SQL functions. This interface allows +** applications to access the same PRNG for other purposes. +** +** ^A call to this routine stores N bytes of randomness into buffer P. +** ^The P parameter can be a NULL pointer. +** +** ^If this routine has not been previously called or if the previous +** call had N less than one or a NULL pointer for P, then the PRNG is +** seeded using randomness obtained from the xRandomness method of +** the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more and a +** non-NULL P then the pseudo-randomness is generated +** internally and without recourse to the [sqlite3_vfs] xRandomness +** method. +*/ +SQLITE_API void sqlite3_randomness(int N, void *P); + +/* +** CAPI3REF: Compile-Time Authorization Callbacks +** METHOD: sqlite3 +** KEYWORDS: {authorizer callback} +** +** ^This routine registers an authorizer callback with a particular +** [database connection], supplied in the first argument. +** ^The authorizer callback is invoked as SQL statements are being compiled +** by [sqlite3_prepare()] or its variants [sqlite3_prepare_v2()], +** [sqlite3_prepare_v3()], [sqlite3_prepare16()], [sqlite3_prepare16_v2()], +** and [sqlite3_prepare16_v3()]. ^At various +** points during the compilation process, as logic is being created +** to perform various actions, the authorizer callback is invoked to +** see if those actions are allowed. ^The authorizer callback should +** return [SQLITE_OK] to allow the action, [SQLITE_IGNORE] to disallow the +** specific action but allow the SQL statement to continue to be +** compiled, or [SQLITE_DENY] to cause the entire SQL statement to be +** rejected with an error. ^If the authorizer callback returns +** any value other than [SQLITE_IGNORE], [SQLITE_OK], or [SQLITE_DENY] +** then the [sqlite3_prepare_v2()] or equivalent call that triggered +** the authorizer will fail with an error message. +** +** When the callback returns [SQLITE_OK], that means the operation +** requested is ok. ^When the callback returns [SQLITE_DENY], the +** [sqlite3_prepare_v2()] or equivalent call that triggered the +** authorizer will fail with an error message explaining that +** access is denied. +** +** ^The first parameter to the authorizer callback is a copy of the third +** parameter to the sqlite3_set_authorizer() interface. ^The second parameter +** to the callback is an integer [SQLITE_COPY | action code] that specifies +** the particular action to be authorized. ^The third through sixth parameters +** to the callback are either NULL pointers or zero-terminated strings +** that contain additional details about the action to be authorized. +** Applications must always be prepared to encounter a NULL pointer in any +** of the third through the sixth parameters of the authorization callback. +** +** ^If the action code is [SQLITE_READ] +** and the callback returns [SQLITE_IGNORE] then the +** [prepared statement] statement is constructed to substitute +** a NULL value in place of the table column that would have +** been read if [SQLITE_OK] had been returned. The [SQLITE_IGNORE] +** return can be used to deny an untrusted user access to individual +** columns of a table. +** ^When a table is referenced by a [SELECT] but no column values are +** extracted from that table (for example in a query like +** "SELECT count(*) FROM tab") then the [SQLITE_READ] authorizer callback +** is invoked once for that table with a column name that is an empty string. +** ^If the action code is [SQLITE_DELETE] and the callback returns +** [SQLITE_IGNORE] then the [DELETE] operation proceeds but the +** [truncate optimization] is disabled and all rows are deleted individually. +** +** An authorizer is used when [sqlite3_prepare | preparing] +** SQL statements from an untrusted source, to ensure that the SQL statements +** do not try to access data they are not allowed to see, or that they do not +** try to execute malicious statements that damage the database. For +** example, an application may allow a user to enter arbitrary +** SQL queries for evaluation by a database. But the application does +** not want the user to be able to make arbitrary changes to the +** database. An authorizer could then be put in place while the +** user-entered SQL is being [sqlite3_prepare | prepared] that +** disallows everything except [SELECT] statements. +** +** Applications that need to process SQL from untrusted sources +** might also consider lowering resource limits using [sqlite3_limit()] +** and limiting database size using the [max_page_count] [PRAGMA] +** in addition to using an authorizer. +** +** ^(Only a single authorizer can be in place on a database connection +** at a time. Each call to sqlite3_set_authorizer overrides the +** previous call.)^ ^Disable the authorizer by installing a NULL callback. +** The authorizer is disabled by default. +** +** The authorizer callback must not do anything that will modify +** the database connection that invoked the authorizer callback. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** ^When [sqlite3_prepare_v2()] is used to prepare a statement, the +** statement might be re-prepared during [sqlite3_step()] due to a +** schema change. Hence, the application should ensure that the +** correct authorizer callback remains in place during the [sqlite3_step()]. +** +** ^Note that the authorizer callback is invoked only during +** [sqlite3_prepare()] or its variants. Authorization is not +** performed during statement evaluation in [sqlite3_step()], unless +** as stated in the previous paragraph, sqlite3_step() invokes +** sqlite3_prepare_v2() to reprepare a statement after a schema change. +*/ +SQLITE_API int sqlite3_set_authorizer( + sqlite3*, + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), + void *pUserData +); + +/* +** CAPI3REF: Authorizer Return Codes +** +** The [sqlite3_set_authorizer | authorizer callback function] must +** return either [SQLITE_OK] or one of these two constants in order +** to signal SQLite whether or not the action is permitted. See the +** [sqlite3_set_authorizer | authorizer documentation] for additional +** information. +** +** Note that SQLITE_IGNORE is also used as a [conflict resolution mode] +** returned from the [sqlite3_vtab_on_conflict()] interface. +*/ +#define SQLITE_DENY 1 /* Abort the SQL statement with an error */ +#define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ + +/* +** CAPI3REF: Authorizer Action Codes +** +** The [sqlite3_set_authorizer()] interface registers a callback function +** that is invoked to authorize certain SQL statement actions. The +** second parameter to the callback is an integer code that specifies +** what action is being authorized. These are the integer action codes that +** the authorizer callback may be passed. +** +** These action code values signify what kind of operation is to be +** authorized. The 3rd and 4th parameters to the authorization +** callback function will be parameters or NULL depending on which of these +** codes is used as the second parameter. ^(The 5th parameter to the +** authorizer callback is the name of the database ("main", "temp", +** etc.) if applicable.)^ ^The 6th parameter to the authorizer callback +** is the name of the inner-most trigger or view that is responsible for +** the access attempt or NULL if this access attempt is directly from +** top-level SQL code. +*/ +/******************************************* 3rd ************ 4th ***********/ +#define SQLITE_CREATE_INDEX 1 /* Index Name Table Name */ +#define SQLITE_CREATE_TABLE 2 /* Table Name NULL */ +#define SQLITE_CREATE_TEMP_INDEX 3 /* Index Name Table Name */ +#define SQLITE_CREATE_TEMP_TABLE 4 /* Table Name NULL */ +#define SQLITE_CREATE_TEMP_TRIGGER 5 /* Trigger Name Table Name */ +#define SQLITE_CREATE_TEMP_VIEW 6 /* View Name NULL */ +#define SQLITE_CREATE_TRIGGER 7 /* Trigger Name Table Name */ +#define SQLITE_CREATE_VIEW 8 /* View Name NULL */ +#define SQLITE_DELETE 9 /* Table Name NULL */ +#define SQLITE_DROP_INDEX 10 /* Index Name Table Name */ +#define SQLITE_DROP_TABLE 11 /* Table Name NULL */ +#define SQLITE_DROP_TEMP_INDEX 12 /* Index Name Table Name */ +#define SQLITE_DROP_TEMP_TABLE 13 /* Table Name NULL */ +#define SQLITE_DROP_TEMP_TRIGGER 14 /* Trigger Name Table Name */ +#define SQLITE_DROP_TEMP_VIEW 15 /* View Name NULL */ +#define SQLITE_DROP_TRIGGER 16 /* Trigger Name Table Name */ +#define SQLITE_DROP_VIEW 17 /* View Name NULL */ +#define SQLITE_INSERT 18 /* Table Name NULL */ +#define SQLITE_PRAGMA 19 /* Pragma Name 1st arg or NULL */ +#define SQLITE_READ 20 /* Table Name Column Name */ +#define SQLITE_SELECT 21 /* NULL NULL */ +#define SQLITE_TRANSACTION 22 /* Operation NULL */ +#define SQLITE_UPDATE 23 /* Table Name Column Name */ +#define SQLITE_ATTACH 24 /* Filename NULL */ +#define SQLITE_DETACH 25 /* Database Name NULL */ +#define SQLITE_ALTER_TABLE 26 /* Database Name Table Name */ +#define SQLITE_REINDEX 27 /* Index Name NULL */ +#define SQLITE_ANALYZE 28 /* Table Name NULL */ +#define SQLITE_CREATE_VTABLE 29 /* Table Name Module Name */ +#define SQLITE_DROP_VTABLE 30 /* Table Name Module Name */ +#define SQLITE_FUNCTION 31 /* NULL Function Name */ +#define SQLITE_SAVEPOINT 32 /* Operation Savepoint Name */ +#define SQLITE_COPY 0 /* No longer used */ +#define SQLITE_RECURSIVE 33 /* NULL NULL */ + +/* +** CAPI3REF: Tracing And Profiling Functions +** METHOD: sqlite3 +** +** These routines are deprecated. Use the [sqlite3_trace_v2()] interface +** instead of the routines described here. +** +** These routines register callback functions that can be used for +** tracing and profiling the execution of SQL statements. +** +** ^The callback function registered by sqlite3_trace() is invoked at +** various times when an SQL statement is being run by [sqlite3_step()]. +** ^The sqlite3_trace() callback is invoked with a UTF-8 rendering of the +** SQL statement text as the statement first begins executing. +** ^(Additional sqlite3_trace() callbacks might occur +** as each triggered subprogram is entered. The callbacks for triggers +** contain a UTF-8 SQL comment that identifies the trigger.)^ +** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** +** ^The callback function registered by sqlite3_profile() is invoked +** as each SQL statement finishes. ^The profile callback contains +** the original statement text and an estimate of wall-clock time +** of how long that statement took to run. ^The profile callback +** time is in units of nanoseconds, however the current implementation +** is only capable of millisecond resolution so the six least significant +** digits in the time are meaningless. Future versions of SQLite +** might provide greater resolution on the profiler callback. Invoking +** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the +** profile callback. +*/ +SQLITE_API SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*, + void(*xTrace)(void*,const char*), void*); +SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, + void(*xProfile)(void*,const char*,sqlite3_uint64), void*); + +/* +** CAPI3REF: SQL Trace Event Codes +** KEYWORDS: SQLITE_TRACE +** +** These constants identify classes of events that can be monitored +** using the [sqlite3_trace_v2()] tracing logic. The M argument +** to [sqlite3_trace_v2(D,M,X,P)] is an OR-ed combination of one or more of +** the following constants. ^The first argument to the trace callback +** is one of the following constants. +** +** New tracing constants may be added in future releases. +** +** ^A trace callback has four arguments: xCallback(T,C,P,X). +** ^The T argument is one of the integer type codes above. +** ^The C argument is a copy of the context pointer passed in as the +** fourth argument to [sqlite3_trace_v2()]. +** The P and X arguments are pointers whose meanings depend on T. +** +**
    +** [[SQLITE_TRACE_STMT]]
    SQLITE_TRACE_STMT
    +**
    ^An SQLITE_TRACE_STMT callback is invoked when a prepared statement +** first begins running and possibly at other times during the +** execution of the prepared statement, such as at the start of each +** trigger subprogram. ^The P argument is a pointer to the +** [prepared statement]. ^The X argument is a pointer to a string which +** is the unexpanded SQL text of the prepared statement or an SQL comment +** that indicates the invocation of a trigger. ^The callback can compute +** the same text that would have been returned by the legacy [sqlite3_trace()] +** interface by using the X argument when X begins with "--" and invoking +** [sqlite3_expanded_sql(P)] otherwise. +** +** [[SQLITE_TRACE_PROFILE]]
    SQLITE_TRACE_PROFILE
    +**
    ^An SQLITE_TRACE_PROFILE callback provides approximately the same +** information as is provided by the [sqlite3_profile()] callback. +** ^The P argument is a pointer to the [prepared statement] and the +** X argument points to a 64-bit integer which is the estimated of +** the number of nanosecond that the prepared statement took to run. +** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes. +** +** [[SQLITE_TRACE_ROW]]
    SQLITE_TRACE_ROW
    +**
    ^An SQLITE_TRACE_ROW callback is invoked whenever a prepared +** statement generates a single row of result. +** ^The P argument is a pointer to the [prepared statement] and the +** X argument is unused. +** +** [[SQLITE_TRACE_CLOSE]]
    SQLITE_TRACE_CLOSE
    +**
    ^An SQLITE_TRACE_CLOSE callback is invoked when a database +** connection closes. +** ^The P argument is a pointer to the [database connection] object +** and the X argument is unused. +**
    +*/ +#define SQLITE_TRACE_STMT 0x01 +#define SQLITE_TRACE_PROFILE 0x02 +#define SQLITE_TRACE_ROW 0x04 +#define SQLITE_TRACE_CLOSE 0x08 + +/* +** CAPI3REF: SQL Trace Hook +** METHOD: sqlite3 +** +** ^The sqlite3_trace_v2(D,M,X,P) interface registers a trace callback +** function X against [database connection] D, using property mask M +** and context pointer P. ^If the X callback is +** NULL or if the M mask is zero, then tracing is disabled. The +** M argument should be the bitwise OR-ed combination of +** zero or more [SQLITE_TRACE] constants. +** +** ^Each call to either sqlite3_trace() or sqlite3_trace_v2() overrides +** (cancels) any prior calls to sqlite3_trace() or sqlite3_trace_v2(). +** +** ^The X callback is invoked whenever any of the events identified by +** mask M occur. ^The integer return value from the callback is currently +** ignored, though this may change in future releases. Callback +** implementations should return zero to ensure future compatibility. +** +** ^A trace callback is invoked with four arguments: callback(T,C,P,X). +** ^The T argument is one of the [SQLITE_TRACE] +** constants to indicate why the callback was invoked. +** ^The C argument is a copy of the context pointer. +** The P and X arguments are pointers whose meanings depend on T. +** +** The sqlite3_trace_v2() interface is intended to replace the legacy +** interfaces [sqlite3_trace()] and [sqlite3_profile()], both of which +** are deprecated. +*/ +SQLITE_API int sqlite3_trace_v2( + sqlite3*, + unsigned uMask, + int(*xCallback)(unsigned,void*,void*,void*), + void *pCtx +); + +/* +** CAPI3REF: Query Progress Callbacks +** METHOD: sqlite3 +** +** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback +** function X to be invoked periodically during long running calls to +** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for +** database connection D. An example use for this +** interface is to keep a GUI updated during a large query. +** +** ^The parameter P is passed through as the only parameter to the +** callback function X. ^The parameter N is the approximate number of +** [virtual machine instructions] that are evaluated between successive +** invocations of the callback X. ^If N is less than one then the progress +** handler is disabled. +** +** ^Only a single progress handler may be defined at one time per +** [database connection]; setting a new progress handler cancels the +** old one. ^Setting parameter X to NULL disables the progress handler. +** ^The progress handler is also disabled by setting N to a value less +** than 1. +** +** ^If the progress callback returns non-zero, the operation is +** interrupted. This feature can be used to implement a +** "Cancel" button on a GUI progress dialog box. +** +** The progress handler callback must not do anything that will modify +** the database connection that invoked the progress handler. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +*/ +SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); + +/* +** CAPI3REF: Opening A New Database Connection +** CONSTRUCTOR: sqlite3 +** +** ^These routines open an SQLite database file as specified by the +** filename argument. ^The filename argument is interpreted as UTF-8 for +** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte +** order for sqlite3_open16(). ^(A [database connection] handle is usually +** returned in *ppDb, even if an error occurs. The only exception is that +** if SQLite is unable to allocate memory to hold the [sqlite3] object, +** a NULL will be written into *ppDb instead of a pointer to the [sqlite3] +** object.)^ ^(If the database is opened (and/or created) successfully, then +** [SQLITE_OK] is returned. Otherwise an [error code] is returned.)^ ^The +** [sqlite3_errmsg()] or [sqlite3_errmsg16()] routines can be used to obtain +** an English language description of the error following a failure of any +** of the sqlite3_open() routines. +** +** ^The default encoding will be UTF-8 for databases created using +** sqlite3_open() or sqlite3_open_v2(). ^The default encoding for databases +** created using sqlite3_open16() will be UTF-16 in the native byte order. +** +** Whether or not an error occurs when it is opened, resources +** associated with the [database connection] handle should be released by +** passing it to [sqlite3_close()] when it is no longer required. +** +** The sqlite3_open_v2() interface works like sqlite3_open() +** except that it accepts two additional parameters for additional control +** over the new database connection. ^(The flags parameter to +** sqlite3_open_v2() must include, at a minimum, one of the following +** three flag combinations:)^ +** +**
    +** ^(
    [SQLITE_OPEN_READONLY]
    +**
    The database is opened in read-only mode. If the database does not +** already exist, an error is returned.
    )^ +** +** ^(
    [SQLITE_OPEN_READWRITE]
    +**
    The database is opened for reading and writing if possible, or reading +** only if the file is write protected by the operating system. In either +** case the database must already exist, otherwise an error is returned.
    )^ +** +** ^(
    [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]
    +**
    The database is opened for reading and writing, and is created if +** it does not already exist. This is the behavior that is always used for +** sqlite3_open() and sqlite3_open16().
    )^ +**
    +** +** In addition to the required flags, the following optional flags are +** also supported: +** +**
    +** ^(
    [SQLITE_OPEN_URI]
    +**
    The filename can be interpreted as a URI if this flag is set.
    )^ +** +** ^(
    [SQLITE_OPEN_MEMORY]
    +**
    The database will be opened as an in-memory database. The database +** is named by the "filename" argument for the purposes of cache-sharing, +** if shared cache mode is enabled, but the "filename" is otherwise ignored. +**
    )^ +** +** ^(
    [SQLITE_OPEN_NOMUTEX]
    +**
    The new database connection will use the "multi-thread" +** [threading mode].)^ This means that separate threads are allowed +** to use SQLite at the same time, as long as each thread is using +** a different [database connection]. +** +** ^(
    [SQLITE_OPEN_FULLMUTEX]
    +**
    The new database connection will use the "serialized" +** [threading mode].)^ This means the multiple threads can safely +** attempt to use the same database connection at the same time. +** (Mutexes will block any actual concurrency, but in this mode +** there is no harm in trying.) +** +** ^(
    [SQLITE_OPEN_SHAREDCACHE]
    +**
    The database is opened [shared cache] enabled, overriding +** the default shared cache setting provided by +** [sqlite3_enable_shared_cache()].)^ +** +** ^(
    [SQLITE_OPEN_PRIVATECACHE]
    +**
    The database is opened [shared cache] disabled, overriding +** the default shared cache setting provided by +** [sqlite3_enable_shared_cache()].)^ +** +** [[OPEN_NOFOLLOW]] ^(
    [SQLITE_OPEN_NOFOLLOW]
    +**
    The database filename is not allowed to be a symbolic link
    +** +** [[OPEN_EXCLUSIVE]] ^(
    [SQLITE_OPEN_EXCLUSIVE]
    +**
    This flag causes the open to fail if the database file already +** exists. The open will only be success if this flag is used in combination +** with the SQLITE_OPEN_CREATE and SQLITE_OPEN_READWRITE flags and if +** the file does not previously exist.
    +**
    )^ +** +** If the 3rd parameter to sqlite3_open_v2() is not one of the +** required combinations shown above optionally combined with other +** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits] +** then the behavior is undefined. +** +** ^The fourth parameter to sqlite3_open_v2() is the name of the +** [sqlite3_vfs] object that defines the operating system interface that +** the new database connection should use. ^If the fourth parameter is +** a NULL pointer then the default [sqlite3_vfs] object is used. +** +** ^If the filename is ":memory:", then a private, temporary in-memory database +** is created for the connection. ^This in-memory database will vanish when +** the database connection is closed. Future versions of SQLite might +** make use of additional special filenames that begin with the ":" character. +** It is recommended that when a database filename actually does begin with +** a ":" character you should prefix the filename with a pathname such as +** "./" to avoid ambiguity. +** +** ^If the filename is an empty string, then a private, temporary +** on-disk database will be created. ^This private database will be +** automatically deleted as soon as the database connection is closed. +** +** [[URI filenames in sqlite3_open()]]

    URI Filenames

    +** +** ^If [URI filename] interpretation is enabled, and the filename argument +** begins with "file:", then the filename is interpreted as a URI. ^URI +** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is +** set in the third argument to sqlite3_open_v2(), or if it has +** been enabled globally using the [SQLITE_CONFIG_URI] option with the +** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option. +** URI filename interpretation is turned off +** by default, but future releases of SQLite might enable URI filename +** interpretation by default. See "[URI filenames]" for additional +** information. +** +** URI filenames are parsed according to RFC 3986. ^If the URI contains an +** authority, then it must be either an empty string or the string +** "localhost". ^If the authority is not an empty string or "localhost", an +** error is returned to the caller. ^The fragment component of a URI, if +** present, is ignored. +** +** ^SQLite uses the path component of the URI as the name of the disk file +** which contains the database. ^If the path begins with a '/' character, +** then it is interpreted as an absolute path. ^If the path does not begin +** with a '/' (meaning that the authority section is omitted from the URI) +** then the path is interpreted as a relative path. +** ^(On windows, the first component of an absolute path +** is a drive specification (e.g. "C:").)^ +** +** [[core URI query parameters]] +** The query component of a URI may contain parameters that are interpreted +** either by SQLite itself, or by a [VFS | custom VFS implementation]. +** SQLite and its built-in [VFSes] interpret the +** following query parameters: +** +**
      +**
    • vfs: ^The "vfs" parameter may be used to specify the name of +** a VFS object that provides the operating system interface that should +** be used to access the database file on disk. ^If this option is set to +** an empty string the default VFS object is used. ^Specifying an unknown +** VFS is an error. ^If sqlite3_open_v2() is used and the vfs option is +** present, then the VFS specified by the option takes precedence over +** the value passed as the fourth parameter to sqlite3_open_v2(). +** +**
    • mode: ^(The mode parameter may be set to either "ro", "rw", +** "rwc", or "memory". Attempting to set it to any other value is +** an error)^. +** ^If "ro" is specified, then the database is opened for read-only +** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the +** third argument to sqlite3_open_v2(). ^If the mode option is set to +** "rw", then the database is opened for read-write (but not create) +** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had +** been set. ^Value "rwc" is equivalent to setting both +** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If the mode option is +** set to "memory" then a pure [in-memory database] that never reads +** or writes from disk is used. ^It is an error to specify a value for +** the mode parameter that is less restrictive than that specified by +** the flags passed in the third parameter to sqlite3_open_v2(). +** +**
    • cache: ^The cache parameter may be set to either "shared" or +** "private". ^Setting it to "shared" is equivalent to setting the +** SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed to +** sqlite3_open_v2(). ^Setting the cache parameter to "private" is +** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. +** ^If sqlite3_open_v2() is used and the "cache" parameter is present in +** a URI filename, its value overrides any behavior requested by setting +** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
    • psow: ^The psow parameter indicates whether or not the +** [powersafe overwrite] property does or does not apply to the +** storage media on which the database file resides. +** +**
    • nolock: ^The nolock parameter is a boolean query parameter +** which if set disables file locking in rollback journal modes. This +** is useful for accessing a database on a filesystem that does not +** support locking. Caution: Database corruption might result if two +** or more processes write to the same database and any one of those +** processes uses nolock=1. +** +**
    • immutable: ^The immutable parameter is a boolean query +** parameter that indicates that the database file is stored on +** read-only media. ^When immutable is set, SQLite assumes that the +** database file cannot be changed, even by a process with higher +** privilege, and so the database is opened read-only and all locking +** and change detection is disabled. Caution: Setting the immutable +** property on a database file that does in fact change can result +** in incorrect query results and/or [SQLITE_CORRUPT] errors. +** See also: [SQLITE_IOCAP_IMMUTABLE]. +** +**
    +** +** ^Specifying an unknown parameter in the query component of a URI is not an +** error. Future versions of SQLite might understand additional query +** parameters. See "[query parameters with special meaning to SQLite]" for +** additional information. +** +** [[URI filename examples]]

    URI filename examples

    +** +** +**
    URI filenames Results +**
    file:data.db +** Open the file "data.db" in the current directory. +**
    file:/home/fred/data.db
    +** file:///home/fred/data.db
    +** file://localhost/home/fred/data.db
    		+** Open the database file "/home/fred/data.db". +**
    file://darkstar/home/fred/data.db +** An error. "darkstar" is not a recognized authority. +**
    +** file:///C:/Documents%20and%20Settings/fred/Desktop/data.db +** Windows only: Open the file "data.db" on fred's desktop on drive +** C:. Note that the %20 escaping in this example is not strictly +** necessary - space characters can be used literally +** in URI filenames. +**
    file:data.db?mode=ro&cache=private +** Open file "data.db" in the current directory for read-only access. +** Regardless of whether or not shared-cache mode is enabled by +** default, use a private cache. +**
    file:/home/fred/data.db?vfs=unix-dotfile +** Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" +** that uses dot-files in place of posix advisory locking. +**
    file:data.db?mode=readonly +** An error. "readonly" is not a valid option for the "mode" parameter. +** Use "ro" instead: "file:data.db?mode=ro". +**
    +** +** ^URI hexadecimal escape sequences (%HH) are supported within the path and +** query components of a URI. A hexadecimal escape sequence consists of a +** percent sign - "%" - followed by exactly two hexadecimal digits +** specifying an octet value. ^Before the path or query components of a +** URI filename are interpreted, they are encoded using UTF-8 and all +** hexadecimal escape sequences replaced by a single byte containing the +** corresponding octet. If this process generates an invalid UTF-8 encoding, +** the results are undefined. +** +** Note to Windows users: The encoding used for the filename argument +** of sqlite3_open() and sqlite3_open_v2() must be UTF-8, not whatever +** codepage is currently defined. Filenames containing international +** characters must be converted to UTF-8 prior to passing them into +** sqlite3_open() or sqlite3_open_v2(). +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling sqlite3_open() or sqlite3_open_v2(). Otherwise, various +** features that require the use of temporary files may fail. +** +** See also: [sqlite3_temp_directory] +*/ +SQLITE_API int sqlite3_open( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb /* OUT: SQLite db handle */ +); +SQLITE_API int sqlite3_open16( + const void *filename, /* Database filename (UTF-16) */ + sqlite3 **ppDb /* OUT: SQLite db handle */ +); +SQLITE_API int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +); + +/* +** CAPI3REF: Obtain Values For URI Parameters +** +** These are utility routines, useful to [VFS|custom VFS implementations], +** that check if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of that query parameter. +** +** The first parameter to these interfaces (hereafter referred to +** as F) must be one of: +**
      +**
    • A database filename pointer created by the SQLite core and +** passed into the xOpen() method of a VFS implemention, or +**
    • A filename obtained from [sqlite3_db_filename()], or +**
    • A new filename constructed using [sqlite3_create_filename()]. +**
    +** If the F parameter is not one of the above, then the behavior is +** undefined and probably undesirable. Older versions of SQLite were +** more tolerant of invalid F parameters than newer versions. +** +** If F is a suitable filename (as described in the previous paragraph) +** and if P is the name of the query parameter, then +** sqlite3_uri_parameter(F,P) returns the value of the P +** parameter if it exists or a NULL pointer if P does not appear as a +** query parameter on F. If P is a query parameter of F and it +** has no explicit value, then sqlite3_uri_parameter(F,P) returns +** a pointer to an empty string. +** +** The sqlite3_uri_boolean(F,P,B) routine assumes that P is a boolean +** parameter and returns true (1) or false (0) according to the value +** of P. The sqlite3_uri_boolean(F,P,B) routine returns true (1) if the +** value of query parameter P is one of "yes", "true", or "on" in any +** case or if the value begins with a non-zero number. The +** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of +** query parameter P is one of "no", "false", or "off" in any case or +** if the value begins with a numeric zero. If P is not a query +** parameter on F or if the value of P does not match any of the +** above, then sqlite3_uri_boolean(F,P,B) returns (B!=0). +** +** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a +** 64-bit signed integer and returns that integer, or D if P does not +** exist. If the value of P is something other than an integer, then +** zero is returned. +** +** The sqlite3_uri_key(F,N) returns a pointer to the name (not +** the value) of the N-th query parameter for filename F, or a NULL +** pointer if N is less than zero or greater than the number of query +** parameters minus 1. The N value is zero-based so N should be 0 to obtain +** the name of the first query parameter, 1 for the second parameter, and +** so forth. +** +** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and +** sqlite3_uri_boolean(F,P,B) returns B. If F is not a NULL pointer and +** is not a database file pathname pointer that the SQLite core passed +** into the xOpen VFS method, then the behavior of this routine is undefined +** and probably undesirable. +** +** Beginning with SQLite [version 3.31.0] ([dateof:3.31.0]) the input F +** parameter can also be the name of a rollback journal file or WAL file +** in addition to the main database file. Prior to version 3.31.0, these +** routines would only work if F was the name of the main database file. +** When the F parameter is the name of the rollback journal or WAL file, +** it has access to all the same query parameters as were found on the +** main database file. +** +** See the [URI filename] documentation for additional information. +*/ +SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); +SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); + +/* +** CAPI3REF: Translate filenames +** +** These routines are available to [VFS|custom VFS implementations] for +** translating filenames between the main database file, the journal file, +** and the WAL file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** passed by the SQLite core into the VFS, then sqlite3_filename_database(F) +** returns the name of the corresponding database file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** passed by the SQLite core into the VFS, or if F is a database filename +** obtained from [sqlite3_db_filename()], then sqlite3_filename_journal(F) +** returns the name of the corresponding rollback journal file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** that was passed by the SQLite core into the VFS, or if F is a database +** filename obtained from [sqlite3_db_filename()], then +** sqlite3_filename_wal(F) returns the name of the corresponding +** WAL file. +** +** In all of the above, if F is not the name of a database, journal or WAL +** filename passed into the VFS from the SQLite core and F is not the +** return value from [sqlite3_db_filename()], then the result is +** undefined and is likely a memory access violation. +*/ +SQLITE_API const char *sqlite3_filename_database(const char*); +SQLITE_API const char *sqlite3_filename_journal(const char*); +SQLITE_API const char *sqlite3_filename_wal(const char*); + +/* +** CAPI3REF: Database File Corresponding To A Journal +** +** ^If X is the name of a rollback or WAL-mode journal file that is +** passed into the xOpen method of [sqlite3_vfs], then +** sqlite3_database_file_object(X) returns a pointer to the [sqlite3_file] +** object that represents the main database file. +** +** This routine is intended for use in custom [VFS] implementations +** only. It is not a general-purpose interface. +** The argument sqlite3_file_object(X) must be a filename pointer that +** has been passed into [sqlite3_vfs].xOpen method where the +** flags parameter to xOpen contains one of the bits +** [SQLITE_OPEN_MAIN_JOURNAL] or [SQLITE_OPEN_WAL]. Any other use +** of this routine results in undefined and probably undesirable +** behavior. +*/ +SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); + +/* +** CAPI3REF: Create and Destroy VFS Filenames +** +** These interfces are provided for use by [VFS shim] implementations and +** are not useful outside of that context. +** +** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of +** database filename D with corresponding journal file J and WAL file W and +** with N URI parameters key/values pairs in the array P. The result from +** sqlite3_create_filename(D,J,W,N,P) is a pointer to a database filename that +** is safe to pass to routines like: +**
      +**
    • [sqlite3_uri_parameter()], +**
    • [sqlite3_uri_boolean()], +**
    • [sqlite3_uri_int64()], +**
    • [sqlite3_uri_key()], +**
    • [sqlite3_filename_database()], +**
    • [sqlite3_filename_journal()], or +**
    • [sqlite3_filename_wal()]. +**
    +** If a memory allocation error occurs, sqlite3_create_filename() might +** return a NULL pointer. The memory obtained from sqlite3_create_filename(X) +** must be released by a corresponding call to sqlite3_free_filename(Y). +** +** The P parameter in sqlite3_create_filename(D,J,W,N,P) should be an array +** of 2*N pointers to strings. Each pair of pointers in this array corresponds +** to a key and value for a query parameter. The P parameter may be a NULL +** pointer if N is zero. None of the 2*N pointers in the P array may be +** NULL pointers and key pointers should not be empty strings. +** None of the D, J, or W parameters to sqlite3_create_filename(D,J,W,N,P) may +** be NULL pointers, though they can be empty strings. +** +** The sqlite3_free_filename(Y) routine releases a memory allocation +** previously obtained from sqlite3_create_filename(). Invoking +** sqlite3_free_filename(Y) where Y is a NULL pointer is a harmless no-op. +** +** If the Y parameter to sqlite3_free_filename(Y) is anything other +** than a NULL pointer or a pointer previously acquired from +** sqlite3_create_filename(), then bad things such as heap +** corruption or segfaults may occur. The value Y should not be +** used again after sqlite3_free_filename(Y) has been called. This means +** that if the [sqlite3_vfs.xOpen()] method of a VFS has been called using Y, +** then the corresponding [sqlite3_module.xClose() method should also be +** invoked prior to calling sqlite3_free_filename(Y). +*/ +SQLITE_API char *sqlite3_create_filename( + const char *zDatabase, + const char *zJournal, + const char *zWal, + int nParam, + const char **azParam +); +SQLITE_API void sqlite3_free_filename(char*); + +/* +** CAPI3REF: Error Codes And Messages +** METHOD: sqlite3 +** +** ^If the most recent sqlite3_* API call associated with +** [database connection] D failed, then the sqlite3_errcode(D) interface +** returns the numeric [result code] or [extended result code] for that +** API call. +** ^The sqlite3_extended_errcode() +** interface is the same except that it always returns the +** [extended result code] even when extended result codes are +** disabled. +** +** The values returned by sqlite3_errcode() and/or +** sqlite3_extended_errcode() might change with each API call. +** Except, there are some interfaces that are guaranteed to never +** change the value of the error code. The error-code preserving +** interfaces are: +** +**
      +**
    • sqlite3_errcode() +**
    • sqlite3_extended_errcode() +**
    • sqlite3_errmsg() +**
    • sqlite3_errmsg16() +**
    +** +** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language +** text that describes the error, as either UTF-8 or UTF-16 respectively. +** ^(Memory to hold the error message string is managed internally. +** The application does not need to worry about freeing the result. +** However, the error string might be overwritten or deallocated by +** subsequent calls to other SQLite interface functions.)^ +** +** ^The sqlite3_errstr() interface returns the English-language text +** that describes the [result code], as UTF-8. +** ^(Memory to hold the error message string is managed internally +** and must not be freed by the application)^. +** +** When the serialized [threading mode] is in use, it might be the +** case that a second error occurs on a separate thread in between +** the time of the first error and the call to these interfaces. +** When that happens, the second error will be reported since these +** interfaces always report the most recent result. To avoid +** this, each thread can obtain exclusive use of the [database connection] D +** by invoking [sqlite3_mutex_enter]([sqlite3_db_mutex](D)) before beginning +** to use D and invoking [sqlite3_mutex_leave]([sqlite3_db_mutex](D)) after +** all calls to the interfaces listed here are completed. +** +** If an interface fails with SQLITE_MISUSE, that means the interface +** was invoked incorrectly by the application. In that case, the +** error code and message may or may not be set. +*/ +SQLITE_API int sqlite3_errcode(sqlite3 *db); +SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); +SQLITE_API const char *sqlite3_errmsg(sqlite3*); +SQLITE_API const void *sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *sqlite3_errstr(int); + +/* +** CAPI3REF: Prepared Statement Object +** KEYWORDS: {prepared statement} {prepared statements} +** +** An instance of this object represents a single SQL statement that +** has been compiled into binary form and is ready to be evaluated. +** +** Think of each SQL statement as a separate computer program. The +** original SQL text is source code. A prepared statement object +** is the compiled object code. All SQL must be converted into a +** prepared statement before it can be run. +** +** The life-cycle of a prepared statement object usually goes like this: +** +**
      +**
    1. Create the prepared statement object using [sqlite3_prepare_v2()]. +**
    2. Bind values to [parameters] using the sqlite3_bind_*() +** interfaces. +**
    3. Run the SQL by calling [sqlite3_step()] one or more times. +**
    4. Reset the prepared statement using [sqlite3_reset()] then go back +** to step 2. Do this zero or more times. +**
    5. Destroy the object using [sqlite3_finalize()]. +**
    +*/ +typedef struct sqlite3_stmt sqlite3_stmt; + +/* +** CAPI3REF: Run-time Limits +** METHOD: sqlite3 +** +** ^(This interface allows the size of various constructs to be limited +** on a connection by connection basis. The first parameter is the +** [database connection] whose limit is to be set or queried. The +** second parameter is one of the [limit categories] that define a +** class of constructs to be size limited. The third parameter is the +** new limit for that construct.)^ +** +** ^If the new limit is a negative number, the limit is unchanged. +** ^(For each limit category SQLITE_LIMIT_NAME there is a +** [limits | hard upper bound] +** set at compile-time by a C preprocessor macro called +** [limits | SQLITE_MAX_NAME]. +** (The "_LIMIT_" in the name is changed to "_MAX_".))^ +** ^Attempts to increase a limit above its hard upper bound are +** silently truncated to the hard upper bound. +** +** ^Regardless of whether or not the limit was changed, the +** [sqlite3_limit()] interface returns the prior value of the limit. +** ^Hence, to find the current value of a limit without changing it, +** simply invoke this interface with the third parameter set to -1. +** +** Run-time limits are intended for use in applications that manage +** both their own internal database and also databases that are controlled +** by untrusted external sources. An example application might be a +** web browser that has its own databases for storing history and +** separate databases controlled by JavaScript applications downloaded +** off the Internet. The internal databases can be given the +** large, default limits. Databases managed by external sources can +** be given much smaller limits designed to prevent a denial of service +** attack. Developers might also want to use the [sqlite3_set_authorizer()] +** interface to further control untrusted SQL. The size of the database +** created by an untrusted script can be contained using the +** [max_page_count] [PRAGMA]. +** +** New run-time limit categories may be added in future releases. +*/ +SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); + +/* +** CAPI3REF: Run-Time Limit Categories +** KEYWORDS: {limit category} {*limit categories} +** +** These constants define various performance limits +** that can be lowered at run-time using [sqlite3_limit()]. +** The synopsis of the meanings of the various limits is shown below. +** Additional information is available at [limits | Limits in SQLite]. +** +**
    +** [[SQLITE_LIMIT_LENGTH]] ^(
    SQLITE_LIMIT_LENGTH
    +**
    The maximum size of any string or BLOB or table row, in bytes.
    )^ +** +** [[SQLITE_LIMIT_SQL_LENGTH]] ^(
    SQLITE_LIMIT_SQL_LENGTH
    +**
    The maximum length of an SQL statement, in bytes.
    )^ +** +** [[SQLITE_LIMIT_COLUMN]] ^(
    SQLITE_LIMIT_COLUMN
    +**
    The maximum number of columns in a table definition or in the +** result set of a [SELECT] or the maximum number of columns in an index +** or in an ORDER BY or GROUP BY clause.
    )^ +** +** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(
    SQLITE_LIMIT_EXPR_DEPTH
    +**
    The maximum depth of the parse tree on any expression.
    )^ +** +** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(
    SQLITE_LIMIT_COMPOUND_SELECT
    +**
    The maximum number of terms in a compound SELECT statement.
    )^ +** +** [[SQLITE_LIMIT_VDBE_OP]] ^(
    SQLITE_LIMIT_VDBE_OP
    +**
    The maximum number of instructions in a virtual machine program +** used to implement an SQL statement. If [sqlite3_prepare_v2()] or +** the equivalent tries to allocate space for more than this many opcodes +** in a single prepared statement, an SQLITE_NOMEM error is returned.
    )^ +** +** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
    SQLITE_LIMIT_FUNCTION_ARG
    +**
    The maximum number of arguments on a function.
    )^ +** +** [[SQLITE_LIMIT_ATTACHED]] ^(
    SQLITE_LIMIT_ATTACHED
    +**
    The maximum number of [ATTACH | attached databases].)^
    +** +** [[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]] +** ^(
    SQLITE_LIMIT_LIKE_PATTERN_LENGTH
    +**
    The maximum length of the pattern argument to the [LIKE] or +** [GLOB] operators.
    )^ +** +** [[SQLITE_LIMIT_VARIABLE_NUMBER]] +** ^(
    SQLITE_LIMIT_VARIABLE_NUMBER
    +**
    The maximum index number of any [parameter] in an SQL statement.)^ +** +** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
    SQLITE_LIMIT_TRIGGER_DEPTH
    +**
    The maximum depth of recursion for triggers.
    )^ +** +** [[SQLITE_LIMIT_WORKER_THREADS]] ^(
    SQLITE_LIMIT_WORKER_THREADS
    +**
    The maximum number of auxiliary worker threads that a single +** [prepared statement] may start.
    )^ +**
    +*/ +#define SQLITE_LIMIT_LENGTH 0 +#define SQLITE_LIMIT_SQL_LENGTH 1 +#define SQLITE_LIMIT_COLUMN 2 +#define SQLITE_LIMIT_EXPR_DEPTH 3 +#define SQLITE_LIMIT_COMPOUND_SELECT 4 +#define SQLITE_LIMIT_VDBE_OP 5 +#define SQLITE_LIMIT_FUNCTION_ARG 6 +#define SQLITE_LIMIT_ATTACHED 7 +#define SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8 +#define SQLITE_LIMIT_VARIABLE_NUMBER 9 +#define SQLITE_LIMIT_TRIGGER_DEPTH 10 +#define SQLITE_LIMIT_WORKER_THREADS 11 + +/* +** CAPI3REF: Prepare Flags +** +** These constants define various flags that can be passed into +** "prepFlags" parameter of the [sqlite3_prepare_v3()] and +** [sqlite3_prepare16_v3()] interfaces. +** +** New flags may be added in future releases of SQLite. +** +**
    +** [[SQLITE_PREPARE_PERSISTENT]] ^(
    SQLITE_PREPARE_PERSISTENT
    +**
    The SQLITE_PREPARE_PERSISTENT flag is a hint to the query planner +** that the prepared statement will be retained for a long time and +** probably reused many times.)^ ^Without this flag, [sqlite3_prepare_v3()] +** and [sqlite3_prepare16_v3()] assume that the prepared statement will +** be used just once or at most a few times and then destroyed using +** [sqlite3_finalize()] relatively soon. The current implementation acts +** on this hint by avoiding the use of [lookaside memory] so as not to +** deplete the limited store of lookaside memory. Future versions of +** SQLite may act on this hint differently. +** +** [[SQLITE_PREPARE_NORMALIZE]]
    SQLITE_PREPARE_NORMALIZE
    +**
    The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used +** to be required for any prepared statement that wanted to use the +** [sqlite3_normalized_sql()] interface. However, the +** [sqlite3_normalized_sql()] interface is now available to all +** prepared statements, regardless of whether or not they use this +** flag. +** +** [[SQLITE_PREPARE_NO_VTAB]]
    SQLITE_PREPARE_NO_VTAB
    +**
    The SQLITE_PREPARE_NO_VTAB flag causes the SQL compiler +** to return an error (error code SQLITE_ERROR) if the statement uses +** any virtual tables. +**
    +*/ +#define SQLITE_PREPARE_PERSISTENT 0x01 +#define SQLITE_PREPARE_NORMALIZE 0x02 +#define SQLITE_PREPARE_NO_VTAB 0x04 + +/* +** CAPI3REF: Compiling An SQL Statement +** KEYWORDS: {SQL statement compiler} +** METHOD: sqlite3 +** CONSTRUCTOR: sqlite3_stmt +** +** To execute an SQL statement, it must first be compiled into a byte-code +** program using one of these routines. Or, in other words, these routines +** are constructors for the [prepared statement] object. +** +** The preferred routine to use is [sqlite3_prepare_v2()]. The +** [sqlite3_prepare()] interface is legacy and should be avoided. +** [sqlite3_prepare_v3()] has an extra "prepFlags" option that is used +** for special purposes. +** +** The use of the UTF-8 interfaces is preferred, as SQLite currently +** does all parsing using UTF-8. The UTF-16 interfaces are provided +** as a convenience. The UTF-16 interfaces work by converting the +** input text into UTF-8, then invoking the corresponding UTF-8 interface. +** +** The first argument, "db", is a [database connection] obtained from a +** prior successful call to [sqlite3_open()], [sqlite3_open_v2()] or +** [sqlite3_open16()]. The database connection must not have been closed. +** +** The second argument, "zSql", is the statement to be compiled, encoded +** as either UTF-8 or UTF-16. The sqlite3_prepare(), sqlite3_prepare_v2(), +** and sqlite3_prepare_v3() +** interfaces use UTF-8, and sqlite3_prepare16(), sqlite3_prepare16_v2(), +** and sqlite3_prepare16_v3() use UTF-16. +** +** ^If the nByte argument is negative, then zSql is read up to the +** first zero terminator. ^If nByte is positive, then it is the +** number of bytes read from zSql. ^If nByte is zero, then no prepared +** statement is generated. +** If the caller knows that the supplied string is nul-terminated, then +** there is a small performance advantage to passing an nByte parameter that +** is the number of bytes in the input string including +** the nul-terminator. +** +** ^If pzTail is not NULL then *pzTail is made to point to the first byte +** past the end of the first SQL statement in zSql. These routines only +** compile the first statement in zSql, so *pzTail is left pointing to +** what remains uncompiled. +** +** ^*ppStmt is left pointing to a compiled [prepared statement] that can be +** executed using [sqlite3_step()]. ^If there is an error, *ppStmt is set +** to NULL. ^If the input text contains no SQL (if the input is an empty +** string or a comment) then *ppStmt is set to NULL. +** The calling procedure is responsible for deleting the compiled +** SQL statement using [sqlite3_finalize()] after it has finished with it. +** ppStmt may not be NULL. +** +** ^On success, the sqlite3_prepare() family of routines return [SQLITE_OK]; +** otherwise an [error code] is returned. +** +** The sqlite3_prepare_v2(), sqlite3_prepare_v3(), sqlite3_prepare16_v2(), +** and sqlite3_prepare16_v3() interfaces are recommended for all new programs. +** The older interfaces (sqlite3_prepare() and sqlite3_prepare16()) +** are retained for backwards compatibility, but their use is discouraged. +** ^In the "vX" interfaces, the prepared statement +** that is returned (the [sqlite3_stmt] object) contains a copy of the +** original SQL text. This causes the [sqlite3_step()] interface to +** behave differently in three ways: +** +**
      +**
    1. +** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it +** always used to do, [sqlite3_step()] will automatically recompile the SQL +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. +**
      2. +** +**
    2. +** ^When an error occurs, [sqlite3_step()] will return one of the detailed +** [error codes] or [extended error codes]. ^The legacy behavior was that +** [sqlite3_step()] would only return a generic [SQLITE_ERROR] result code +** and the application would have to make a second call to [sqlite3_reset()] +** in order to find the underlying cause of the problem. With the "v2" prepare +** interfaces, the underlying reason for the error is returned immediately. +**
      3. +** +**
    3. +** ^If the specific value bound to a [parameter | host parameter] in the +** WHERE clause might influence the choice of query plan for a statement, +** then the statement will be automatically recompiled, as if there had been +** a schema change, on the first [sqlite3_step()] call following any change +** to the [sqlite3_bind_text | bindings] of that [parameter]. +** ^The specific value of a WHERE-clause [parameter] might influence the +** choice of query plan if the parameter is the left-hand side of a [LIKE] +** or [GLOB] operator or if the parameter is compared to an indexed column +** and the [SQLITE_ENABLE_STAT4] compile-time option is enabled. +**
      4. +**
    +** +**

    ^sqlite3_prepare_v3() differs from sqlite3_prepare_v2() only in having +** the extra prepFlags parameter, which is a bit array consisting of zero or +** more of the [SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_*] flags. ^The +** sqlite3_prepare_v2() interface works exactly the same as +** sqlite3_prepare_v3() with a zero prepFlags parameter. +*/ +SQLITE_API int sqlite3_prepare( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare_v3( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_ flags */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16_v2( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16_v3( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_ flags */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); + +/* +** CAPI3REF: Retrieving Statement SQL +** METHOD: sqlite3_stmt +** +** ^The sqlite3_sql(P) interface returns a pointer to a copy of the UTF-8 +** SQL text used to create [prepared statement] P if P was +** created by [sqlite3_prepare_v2()], [sqlite3_prepare_v3()], +** [sqlite3_prepare16_v2()], or [sqlite3_prepare16_v3()]. +** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8 +** string containing the SQL text of prepared statement P with +** [bound parameters] expanded. +** ^The sqlite3_normalized_sql(P) interface returns a pointer to a UTF-8 +** string containing the normalized SQL text of prepared statement P. The +** semantics used to normalize a SQL statement are unspecified and subject +** to change. At a minimum, literal values will be replaced with suitable +** placeholders. +** +** ^(For example, if a prepared statement is created using the SQL +** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345 +** and parameter :xyz is unbound, then sqlite3_sql() will return +** the original string, "SELECT $abc,:xyz" but sqlite3_expanded_sql() +** will return "SELECT 2345,NULL".)^ +** +** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory +** is available to hold the result, or if the result would exceed the +** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. +** +** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of +** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time +** option causes sqlite3_expanded_sql() to always return NULL. +** +** ^The strings returned by sqlite3_sql(P) and sqlite3_normalized_sql(P) +** are managed by SQLite and are automatically freed when the prepared +** statement is finalized. +** ^The string returned by sqlite3_expanded_sql(P), on the other hand, +** is obtained from [sqlite3_malloc()] and must be freed by the application +** by passing it to [sqlite3_free()]. +** +** ^The sqlite3_normalized_sql() interface is only available if +** the [SQLITE_ENABLE_NORMALIZE] compile-time option is defined. +*/ +SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); +SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt); +#ifdef SQLITE_ENABLE_NORMALIZE +SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt); +#endif + +/* +** CAPI3REF: Determine If An SQL Statement Writes The Database +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if +** and only if the [prepared statement] X makes no direct changes to +** the content of the database file. +** +** Note that [application-defined SQL functions] or +** [virtual tables] might change the database indirectly as a side effect. +** ^(For example, if an application defines a function "eval()" that +** calls [sqlite3_exec()], then the following SQL statement would +** change the database file through side-effects: +** +** 

    +**    SELECT eval('DELETE FROM t1') FROM t2;
+**
    +** +** But because the [SELECT] statement does not change the database file +** directly, sqlite3_stmt_readonly() would still return true.)^ +** +** ^Transaction control statements such as [BEGIN], [COMMIT], [ROLLBACK], +** [SAVEPOINT], and [RELEASE] cause sqlite3_stmt_readonly() to return true, +** since the statements themselves do not actually modify the database but +** rather they control the timing of when other statements modify the +** database. ^The [ATTACH] and [DETACH] statements also cause +** sqlite3_stmt_readonly() to return true since, while those statements +** change the configuration of a database connection, they do not make +** changes to the content of the database files on disk. +** ^The sqlite3_stmt_readonly() interface returns true for [BEGIN] since +** [BEGIN] merely sets internal flags, but the [BEGIN|BEGIN IMMEDIATE] and +** [BEGIN|BEGIN EXCLUSIVE] commands do touch the database and so +** sqlite3_stmt_readonly() returns false for those commands. +** +** ^This routine returns false if there is any possibility that the +** statement might change the database file. ^A false return does +** not guarantee that the statement will change the database file. +** ^For example, an UPDATE statement might have a WHERE clause that +** makes it a no-op, but the sqlite3_stmt_readonly() result would still +** be false. ^Similarly, a CREATE TABLE IF NOT EXISTS statement is a +** read-only no-op if the table already exists, but +** sqlite3_stmt_readonly() still returns false for such a statement. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Query The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_isexplain(S) interface returns 1 if the +** prepared statement S is an EXPLAIN statement, or 2 if the +** statement S is an EXPLAIN QUERY PLAN. +** ^The sqlite3_stmt_isexplain(S) interface returns 0 if S is +** an ordinary statement or a NULL pointer. +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Determine If A Prepared Statement Has Been Reset +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the +** [prepared statement] S has been stepped at least once using +** [sqlite3_step(S)] but has neither run to completion (returned +** [SQLITE_DONE] from [sqlite3_step(S)]) nor +** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) +** interface returns false if S is a NULL pointer. If S is not a +** NULL pointer and is not a pointer to a valid [prepared statement] +** object, then the behavior is undefined and probably undesirable. +** +** This interface can be used in combination [sqlite3_next_stmt()] +** to locate all prepared statements associated with a database +** connection that are in need of being reset. This can be used, +** for example, in diagnostic routines to search for prepared +** statements that are holding a transaction open. +*/ +SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); + +/* +** CAPI3REF: Dynamically Typed Value Object +** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} +** +** SQLite uses the sqlite3_value object to represent all values +** that can be stored in a database table. SQLite uses dynamic typing +** for the values it stores. ^Values stored in sqlite3_value objects +** can be integers, floating point values, strings, BLOBs, or NULL. +** +** An sqlite3_value object may be either "protected" or "unprotected". +** Some interfaces require a protected sqlite3_value. Other interfaces +** will accept either a protected or an unprotected sqlite3_value. +** Every interface that accepts sqlite3_value arguments specifies +** whether or not it requires a protected sqlite3_value. The +** [sqlite3_value_dup()] interface can be used to construct a new +** protected sqlite3_value from an unprotected sqlite3_value. +** +** The terms "protected" and "unprotected" refer to whether or not +** a mutex is held. An internal mutex is held for a protected +** sqlite3_value object but no mutex is held for an unprotected +** sqlite3_value object. If SQLite is compiled to be single-threaded +** (with [SQLITE_THREADSAFE=0] and with [sqlite3_threadsafe()] returning 0) +** or if SQLite is run in one of reduced mutex modes +** [SQLITE_CONFIG_SINGLETHREAD] or [SQLITE_CONFIG_MULTITHREAD] +** then there is no distinction between protected and unprotected +** sqlite3_value objects and they can be used interchangeably. However, +** for maximum code portability it is recommended that applications +** still make the distinction between protected and unprotected +** sqlite3_value objects even when not strictly required. +** +** ^The sqlite3_value objects that are passed as parameters into the +** implementation of [application-defined SQL functions] are protected. +** ^The sqlite3_value object returned by +** [sqlite3_column_value()] is unprotected. +** Unprotected sqlite3_value objects may only be used as arguments +** to [sqlite3_result_value()], [sqlite3_bind_value()], and +** [sqlite3_value_dup()]. +** The [sqlite3_value_blob | sqlite3_value_type()] family of +** interfaces require protected sqlite3_value objects. +*/ +typedef struct sqlite3_value sqlite3_value; + +/* +** CAPI3REF: SQL Function Context Object +** +** The context in which an SQL function executes is stored in an +** sqlite3_context object. ^A pointer to an sqlite3_context object +** is always first parameter to [application-defined SQL functions]. +** The application-defined SQL function implementation will pass this +** pointer through into calls to [sqlite3_result_int | sqlite3_result()], +** [sqlite3_aggregate_context()], [sqlite3_user_data()], +** [sqlite3_context_db_handle()], [sqlite3_get_auxdata()], +** and/or [sqlite3_set_auxdata()]. +*/ +typedef struct sqlite3_context sqlite3_context; + +/* +** CAPI3REF: Binding Values To Prepared Statements +** KEYWORDS: {host parameter} {host parameters} {host parameter name} +** KEYWORDS: {SQL parameter} {SQL parameters} {parameter binding} +** METHOD: sqlite3_stmt +** +** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants, +** literals may be replaced by a [parameter] that matches one of following +** templates: +** +**
      +**
    • ? +**
    • ?NNN +**
    • :VVV +**
    • @VVV +**
    • $VVV +**
    +** +** In the templates above, NNN represents an integer literal, +** and VVV represents an alphanumeric identifier.)^ ^The values of these +** parameters (also called "host parameter names" or "SQL parameters") +** can be set using the sqlite3_bind_*() routines defined here. +** +** ^The first argument to the sqlite3_bind_*() routines is always +** a pointer to the [sqlite3_stmt] object returned from +** [sqlite3_prepare_v2()] or its variants. +** +** ^The second argument is the index of the SQL parameter to be set. +** ^The leftmost SQL parameter has an index of 1. ^When the same named +** SQL parameter is used more than once, second and subsequent +** occurrences have the same index as the first occurrence. +** ^The index for named parameters can be looked up using the +** [sqlite3_bind_parameter_index()] API if desired. ^The index +** for "?NNN" parameters is the value of NNN. +** ^The NNN value must be between 1 and the [sqlite3_limit()] +** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 32766). +** +** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). +** ^If the third parameter to sqlite3_bind_text() is not NULL, then +** it should be a pointer to well-formed UTF8 text. +** ^If the third parameter to sqlite3_bind_text16() is not NULL, then +** it should be a pointer to well-formed UTF16 text. +** ^If the third parameter to sqlite3_bind_text64() is not NULL, then +** it should be a pointer to a well-formed unicode string that is +** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 +** otherwise. +** +** [[byte-order determination rules]] ^The byte-order of +** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) +** found in first character, which is removed, or in the absence of a BOM +** the byte order is the native byte order of the host +** machine for sqlite3_bind_text16() or the byte order specified in +** the 6th parameter for sqlite3_bind_text64().)^ +** ^If UTF16 input text contains invalid unicode +** characters, then SQLite might change those invalid characters +** into the unicode replacement character: U+FFFD. +** +** ^(In those routines that have a fourth argument, its value is the +** number of bytes in the parameter. To be clear: the value is the +** number of bytes in the value, not the number of characters.)^ +** ^If the fourth parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** is negative, then the length of the string is +** the number of bytes up to the first zero terminator. +** If the fourth parameter to sqlite3_bind_blob() is negative, then +** the behavior is undefined. +** If a non-negative fourth parameter is provided to sqlite3_bind_text() +** or sqlite3_bind_text16() or sqlite3_bind_text64() then +** that parameter must be the byte offset +** where the NUL terminator would occur assuming the string were NUL +** terminated. If any NUL characters occurs at byte offsets less than +** the value of the fourth parameter then the resulting string value will +** contain embedded NULs. The result of expressions involving strings +** with embedded NULs is undefined. +** +** ^The fifth argument to the BLOB and string binding interfaces controls +** or indicates the lifetime of the object referenced by the third parameter. +** These three options exist: +** ^ (1) A destructor to dispose of the BLOB or string after SQLite has finished +** with it may be passed. ^It is called to dispose of the BLOB or string even +** if the call to the bind API fails, except the destructor is not called if +** the third parameter is a NULL pointer or the fourth parameter is negative. +** ^ (2) The special constant, [SQLITE_STATIC], may be passsed to indicate that +** the application remains responsible for disposing of the object. ^In this +** case, the object and the provided pointer to it must remain valid until +** either the prepared statement is finalized or the same SQL parameter is +** bound to something else, whichever occurs sooner. +** ^ (3) The constant, [SQLITE_TRANSIENT], may be passed to indicate that the +** object is to be copied prior to the return from sqlite3_bind_*(). ^The +** object and pointer to it must remain valid until then. ^SQLite will then +** manage the lifetime of its private copy. +** +** ^The sixth argument to sqlite3_bind_text64() must be one of +** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] +** to specify the encoding of the text in the third parameter. If +** the sixth argument to sqlite3_bind_text64() is not one of the +** allowed values shown above, or if the text encoding is different +** from the encoding specified by the sixth parameter, then the behavior +** is undefined. +** +** ^The sqlite3_bind_zeroblob() routine binds a BLOB of length N that +** is filled with zeroes. ^A zeroblob uses a fixed amount of memory +** (just an integer to hold its size) while it is being processed. +** Zeroblobs are intended to serve as placeholders for BLOBs whose +** content is later written using +** [sqlite3_blob_open | incremental BLOB I/O] routines. +** ^A negative value for the zeroblob results in a zero-length BLOB. +** +** ^The sqlite3_bind_pointer(S,I,P,T,D) routine causes the I-th parameter in +** [prepared statement] S to have an SQL value of NULL, but to also be +** associated with the pointer P of type T. ^D is either a NULL pointer or +** a pointer to a destructor function for P. ^SQLite will invoke the +** destructor D with a single argument of P when it is finished using +** P. The T parameter should be a static string, preferably a string +** literal. The sqlite3_bind_pointer() routine is part of the +** [pointer passing interface] added for SQLite 3.20.0. +** +** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer +** for the [prepared statement] or with a prepared statement for which +** [sqlite3_step()] has been called more recently than [sqlite3_reset()], +** then the call will return [SQLITE_MISUSE]. If any sqlite3_bind_() +** routine is passed a [prepared statement] that has been finalized, the +** result is undefined and probably harmful. +** +** ^Bindings are not cleared by the [sqlite3_reset()] routine. +** ^Unbound parameters are interpreted as NULL. +** +** ^The sqlite3_bind_* routines return [SQLITE_OK] on success or an +** [error code] if anything goes wrong. +** ^[SQLITE_TOOBIG] might be returned if the size of a string or BLOB +** exceeds limits imposed by [sqlite3_limit]([SQLITE_LIMIT_LENGTH]) or +** [SQLITE_MAX_LENGTH]. +** ^[SQLITE_RANGE] is returned if the parameter +** index is out of range. ^[SQLITE_NOMEM] is returned if malloc() fails. +** +** See also: [sqlite3_bind_parameter_count()], +** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +SQLITE_API int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, + void(*)(void*)); +SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double); +SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int); +SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); +SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int); +SQLITE_API int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); +SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +SQLITE_API int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, + void(*)(void*), unsigned char encoding); +SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); +SQLITE_API int sqlite3_bind_pointer(sqlite3_stmt*, int, void*, const char*,void(*)(void*)); +SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); +SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64); + +/* +** CAPI3REF: Number Of SQL Parameters +** METHOD: sqlite3_stmt +** +** ^This routine can be used to find the number of [SQL parameters] +** in a [prepared statement]. SQL parameters are tokens of the +** form "?", "?NNN", ":AAA", "$AAA", or "@AAA" that serve as +** placeholders for values that are [sqlite3_bind_blob | bound] +** to the parameters at a later time. +** +** ^(This routine actually returns the index of the largest (rightmost) +** parameter. For all forms except ?NNN, this will correspond to the +** number of unique parameters. If parameters of the ?NNN form are used, +** there may be gaps in the list.)^ +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_name()], and +** [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*); + +/* +** CAPI3REF: Name Of A Host Parameter +** METHOD: sqlite3_stmt +** +** ^The sqlite3_bind_parameter_name(P,N) interface returns +** the name of the N-th [SQL parameter] in the [prepared statement] P. +** ^(SQL parameters of the form "?NNN" or ":AAA" or "@AAA" or "$AAA" +** have a name which is the string "?NNN" or ":AAA" or "@AAA" or "$AAA" +** respectively. +** In other words, the initial ":" or "$" or "@" or "?" +** is included as part of the name.)^ +** ^Parameters of the form "?" without a following integer have no name +** and are referred to as "nameless" or "anonymous parameters". +** +** ^The first host parameter has an index of 1, not 0. +** +** ^If the value N is out of range or if the N-th parameter is +** nameless, then NULL is returned. ^The returned string is +** always in UTF-8 encoding even if the named parameter was +** originally specified as UTF-16 in [sqlite3_prepare16()], +** [sqlite3_prepare16_v2()], or [sqlite3_prepare16_v3()]. +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_count()], and +** [sqlite3_bind_parameter_index()]. +*/ +SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); + +/* +** CAPI3REF: Index Of A Parameter With A Given Name +** METHOD: sqlite3_stmt +** +** ^Return the index of an SQL parameter given its name. ^The +** index value returned is suitable for use as the second +** parameter to [sqlite3_bind_blob|sqlite3_bind()]. ^A zero +** is returned if no matching parameter is found. ^The parameter +** name must be given in UTF-8 even if the original statement +** was prepared from UTF-16 text using [sqlite3_prepare16_v2()] or +** [sqlite3_prepare16_v3()]. +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_count()], and +** [sqlite3_bind_parameter_name()]. +*/ +SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); + +/* +** CAPI3REF: Reset All Bindings On A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^Contrary to the intuition of many, [sqlite3_reset()] does not reset +** the [sqlite3_bind_blob | bindings] on a [prepared statement]. +** ^Use this routine to reset all host parameters to NULL. +*/ +SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); + +/* +** CAPI3REF: Number Of Columns In A Result Set +** METHOD: sqlite3_stmt +** +** ^Return the number of columns in the result set returned by the +** [prepared statement]. ^If this routine returns 0, that means the +** [prepared statement] returns no data (for example an [UPDATE]). +** ^However, just because this routine returns a positive number does not +** mean that one or more rows of data will be returned. ^A SELECT statement +** will always have a positive sqlite3_column_count() but depending on the +** WHERE clause constraints and the table content, it might return no rows. +** +** See also: [sqlite3_data_count()] +*/ +SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Column Names In A Result Set +** METHOD: sqlite3_stmt +** +** ^These routines return the name assigned to a particular column +** in the result set of a [SELECT] statement. ^The sqlite3_column_name() +** interface returns a pointer to a zero-terminated UTF-8 string +** and sqlite3_column_name16() returns a pointer to a zero-terminated +** UTF-16 string. ^The first parameter is the [prepared statement] +** that implements the [SELECT] statement. ^The second parameter is the +** column number. ^The leftmost column is number 0. +** +** ^The returned string pointer is valid until either the [prepared statement] +** is destroyed by [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the next call to +** sqlite3_column_name() or sqlite3_column_name16() on the same column. +** +** ^If sqlite3_malloc() fails during the processing of either routine +** (for example during a conversion from UTF-8 to UTF-16) then a +** NULL pointer is returned. +** +** ^The name of a result column is the value of the "AS" clause for +** that column, if there is an AS clause. If there is no AS clause +** then the name of the column is unspecified and may change from +** one release of SQLite to the next. +*/ +SQLITE_API const char *sqlite3_column_name(sqlite3_stmt*, int N); +SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); + +/* +** CAPI3REF: Source Of Data In A Query Result +** METHOD: sqlite3_stmt +** +** ^These routines provide a means to determine the database, table, and +** table column that is the origin of a particular result column in +** [SELECT] statement. +** ^The name of the database or table or column can be returned as +** either a UTF-8 or UTF-16 string. ^The _database_ routines return +** the database name, the _table_ routines return the table name, and +** the origin_ routines return the column name. +** ^The returned string is valid until the [prepared statement] is destroyed +** using [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the same information is requested +** again in a different encoding. +** +** ^The names returned are the original un-aliased names of the +** database, table, and column. +** +** ^The first argument to these interfaces is a [prepared statement]. +** ^These functions return information about the Nth result column returned by +** the statement, where N is the second function argument. +** ^The left-most column is column 0 for these routines. +** +** ^If the Nth column returned by the statement is an expression or +** subquery and is not a column value, then all of these functions return +** NULL. ^These routines might also return NULL if a memory allocation error +** occurs. ^Otherwise, they return the name of the attached database, table, +** or column that query result column was extracted from. +** +** ^As with all other SQLite APIs, those whose names end with "16" return +** UTF-16 encoded strings and the other functions return UTF-8. +** +** ^These APIs are only available if the library was compiled with the +** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol. +** +** If two or more threads call one or more +** [sqlite3_column_database_name | column metadata interfaces] +** for the same [prepared statement] and result column +** at the same time then the results are undefined. +*/ +SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); + +/* +** CAPI3REF: Declared Datatype Of A Query Result +** METHOD: sqlite3_stmt +** +** ^(The first parameter is a [prepared statement]. +** If this statement is a [SELECT] statement and the Nth column of the +** returned result set of that [SELECT] is a table column (not an +** expression or subquery) then the declared type of the table +** column is returned.)^ ^If the Nth column of the result set is an +** expression or subquery, then a NULL pointer is returned. +** ^The returned string is always UTF-8 encoded. +** +** ^(For example, given the database schema: +** +** CREATE TABLE t1(c1 VARIANT); +** +** and the following statement to be compiled: +** +** SELECT c1 + 1, c1 FROM t1; +** +** this routine would return the string "VARIANT" for the second result +** column (i==1), and a NULL pointer for the first result column (i==0).)^ +** +** ^SQLite uses dynamic run-time typing. ^So just because a column +** is declared to contain a particular type does not mean that the +** data stored in that column is of the declared type. SQLite is +** strongly typed, but the typing is dynamic not static. ^Type +** is associated with individual values, not with the containers +** used to hold those values. +*/ +SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); + +/* +** CAPI3REF: Evaluate An SQL Statement +** METHOD: sqlite3_stmt +** +** After a [prepared statement] has been prepared using any of +** [sqlite3_prepare_v2()], [sqlite3_prepare_v3()], [sqlite3_prepare16_v2()], +** or [sqlite3_prepare16_v3()] or one of the legacy +** interfaces [sqlite3_prepare()] or [sqlite3_prepare16()], this function +** must be called one or more times to evaluate the statement. +** +** The details of the behavior of the sqlite3_step() interface depend +** on whether the statement was prepared using the newer "vX" interfaces +** [sqlite3_prepare_v3()], [sqlite3_prepare_v2()], [sqlite3_prepare16_v3()], +** [sqlite3_prepare16_v2()] or the older legacy +** interfaces [sqlite3_prepare()] and [sqlite3_prepare16()]. The use of the +** new "vX" interface is recommended for new applications but the legacy +** interface will continue to be supported. +** +** ^In the legacy interface, the return value will be either [SQLITE_BUSY], +** [SQLITE_DONE], [SQLITE_ROW], [SQLITE_ERROR], or [SQLITE_MISUSE]. +** ^With the "v2" interface, any of the other [result codes] or +** [extended result codes] might be returned as well. +** +** ^[SQLITE_BUSY] means that the database engine was unable to acquire the +** database locks it needs to do its job. ^If the statement is a [COMMIT] +** or occurs outside of an explicit transaction, then you can retry the +** statement. If the statement is not a [COMMIT] and occurs within an +** explicit transaction then you should rollback the transaction before +** continuing. +** +** ^[SQLITE_DONE] means that the statement has finished executing +** successfully. sqlite3_step() should not be called again on this virtual +** machine without first calling [sqlite3_reset()] to reset the virtual +** machine back to its initial state. +** +** ^If the SQL statement being executed returns any data, then [SQLITE_ROW] +** is returned each time a new row of data is ready for processing by the +** caller. The values may be accessed using the [column access functions]. +** sqlite3_step() is called again to retrieve the next row of data. +** +** ^[SQLITE_ERROR] means that a run-time error (such as a constraint +** violation) has occurred. sqlite3_step() should not be called again on +** the VM. More information may be found by calling [sqlite3_errmsg()]. +** ^With the legacy interface, a more specific error code (for example, +** [SQLITE_INTERRUPT], [SQLITE_SCHEMA], [SQLITE_CORRUPT], and so forth) +** can be obtained by calling [sqlite3_reset()] on the +** [prepared statement]. ^In the "v2" interface, +** the more specific error code is returned directly by sqlite3_step(). +** +** [SQLITE_MISUSE] means that the this routine was called inappropriately. +** Perhaps it was called on a [prepared statement] that has +** already been [sqlite3_finalize | finalized] or on one that had +** previously returned [SQLITE_ERROR] or [SQLITE_DONE]. Or it could +** be the case that the same database connection is being used by two or +** more threads at the same moment in time. +** +** For all versions of SQLite up to and including 3.6.23.1, a call to +** [sqlite3_reset()] was required after sqlite3_step() returned anything +** other than [SQLITE_ROW] before any subsequent invocation of +** sqlite3_step(). Failure to reset the prepared statement using +** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from +** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1], +** sqlite3_step() began +** calling [sqlite3_reset()] automatically in this circumstance rather +** than returning [SQLITE_MISUSE]. This is not considered a compatibility +** break because any application that ever receives an SQLITE_MISUSE error +** is broken by definition. The [SQLITE_OMIT_AUTORESET] compile-time option +** can be used to restore the legacy behavior. +** +** Goofy Interface Alert: In the legacy interface, the sqlite3_step() +** API always returns a generic error code, [SQLITE_ERROR], following any +** error other than [SQLITE_BUSY] and [SQLITE_MISUSE]. You must call +** [sqlite3_reset()] or [sqlite3_finalize()] in order to find one of the +** specific [error codes] that better describes the error. +** We admit that this is a goofy design. The problem has been fixed +** with the "v2" interface. If you prepare all of your SQL statements +** using [sqlite3_prepare_v3()] or [sqlite3_prepare_v2()] +** or [sqlite3_prepare16_v2()] or [sqlite3_prepare16_v3()] instead +** of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()] interfaces, +** then the more specific [error codes] are returned directly +** by sqlite3_step(). The use of the "vX" interfaces is recommended. +*/ +SQLITE_API int sqlite3_step(sqlite3_stmt*); + +/* +** CAPI3REF: Number of columns in a result set +** METHOD: sqlite3_stmt +** +** ^The sqlite3_data_count(P) interface returns the number of columns in the +** current row of the result set of [prepared statement] P. +** ^If prepared statement P does not have results ready to return +** (via calls to the [sqlite3_column_int | sqlite3_column()] family of +** interfaces) then sqlite3_data_count(P) returns 0. +** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** ^The sqlite3_data_count(P) routine returns 0 if the previous call to +** [sqlite3_step](P) returned [SQLITE_DONE]. ^The sqlite3_data_count(P) +** will return non-zero if previous call to [sqlite3_step](P) returned +** [SQLITE_ROW], except in the case of the [PRAGMA incremental_vacuum] +** where it always returns zero since each step of that multi-step +** pragma returns 0 columns of data. +** +** See also: [sqlite3_column_count()] +*/ +SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Fundamental Datatypes +** KEYWORDS: SQLITE_TEXT +** +** ^(Every value in SQLite has one of five fundamental datatypes: +** +**
      +**
    • 64-bit signed integer +**
    • 64-bit IEEE floating point number +**
    • string +**
    • BLOB +**
    • NULL +**
    )^ +** +** These constants are codes for each of those types. +** +** Note that the SQLITE_TEXT constant was also used in SQLite version 2 +** for a completely different meaning. Software that links against both +** SQLite version 2 and SQLite version 3 should use SQLITE3_TEXT, not +** SQLITE_TEXT. +*/ +#define SQLITE_INTEGER 1 +#define SQLITE_FLOAT 2 +#define SQLITE_BLOB 4 +#define SQLITE_NULL 5 +#ifdef SQLITE_TEXT +# undef SQLITE_TEXT +#else +# define SQLITE_TEXT 3 +#endif +#define SQLITE3_TEXT 3 + +/* +** CAPI3REF: Result Values From A Query +** KEYWORDS: {column access functions} +** METHOD: sqlite3_stmt +** +** Summary: +**
    +**
    sqlite3_column_blob→BLOB result +**
    sqlite3_column_double→REAL result +**
    sqlite3_column_int→32-bit INTEGER result +**
    sqlite3_column_int64→64-bit INTEGER result +**
    sqlite3_column_text→UTF-8 TEXT result +**
    sqlite3_column_text16→UTF-16 TEXT result +**
    sqlite3_column_value→The result as an +** [sqlite3_value|unprotected sqlite3_value] object. +**
        +**
    sqlite3_column_bytes→Size of a BLOB +** or a UTF-8 TEXT result in bytes +**
    sqlite3_column_bytes16   +** →  Size of UTF-16 +** TEXT in bytes +**
    sqlite3_column_type→Default +** datatype of the result +**
    +** +** Details: +** +** ^These routines return information about a single column of the current +** result row of a query. ^In every case the first argument is a pointer +** to the [prepared statement] that is being evaluated (the [sqlite3_stmt*] +** that was returned from [sqlite3_prepare_v2()] or one of its variants) +** and the second argument is the index of the column for which information +** should be returned. ^The leftmost column of the result set has the index 0. +** ^The number of columns in the result can be determined using +** [sqlite3_column_count()]. +** +** If the SQL statement does not currently point to a valid row, or if the +** column index is out of range, the result is undefined. +** These routines may only be called when the most recent call to +** [sqlite3_step()] has returned [SQLITE_ROW] and neither +** [sqlite3_reset()] nor [sqlite3_finalize()] have been called subsequently. +** If any of these routines are called after [sqlite3_reset()] or +** [sqlite3_finalize()] or after [sqlite3_step()] has returned +** something other than [SQLITE_ROW], the results are undefined. +** If [sqlite3_step()] or [sqlite3_reset()] or [sqlite3_finalize()] +** are called from a different thread while any of these routines +** are pending, then the results are undefined. +** +** The first six interfaces (_blob, _double, _int, _int64, _text, and _text16) +** each return the value of a result column in a specific data format. If +** the result column is not initially in the requested format (for example, +** if the query returns an integer but the sqlite3_column_text() interface +** is used to extract the value) then an automatic type conversion is performed. +** +** ^The sqlite3_column_type() routine returns the +** [SQLITE_INTEGER | datatype code] for the initial data type +** of the result column. ^The returned value is one of [SQLITE_INTEGER], +** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL]. +** The return value of sqlite3_column_type() can be used to decide which +** of the first six interface should be used to extract the column value. +** The value returned by sqlite3_column_type() is only meaningful if no +** automatic type conversions have occurred for the value in question. +** After a type conversion, the result of calling sqlite3_column_type() +** is undefined, though harmless. Future +** versions of SQLite may change the behavior of sqlite3_column_type() +** following a type conversion. +** +** If the result is a BLOB or a TEXT string, then the sqlite3_column_bytes() +** or sqlite3_column_bytes16() interfaces can be used to determine the size +** of that BLOB or string. +** +** ^If the result is a BLOB or UTF-8 string then the sqlite3_column_bytes() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-16 string, then sqlite3_column_bytes() converts +** the string to UTF-8 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes() uses +** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes() returns zero. +** +** ^If the result is a BLOB or UTF-16 string then the sqlite3_column_bytes16() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-8 string, then sqlite3_column_bytes16() converts +** the string to UTF-16 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes16() uses +** [sqlite3_snprintf()] to convert that value to a UTF-16 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes16() returns zero. +** +** ^The values returned by [sqlite3_column_bytes()] and +** [sqlite3_column_bytes16()] do not include the zero terminators at the end +** of the string. ^For clarity: the values returned by +** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of +** bytes in the string, not the number of characters. +** +** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), +** even empty strings, are always zero-terminated. ^The return +** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. +** +** Warning: ^The object returned by [sqlite3_column_value()] is an +** [unprotected sqlite3_value] object. In a multithreaded environment, +** an unprotected sqlite3_value object may only be used safely with +** [sqlite3_bind_value()] and [sqlite3_result_value()]. +** If the [unprotected sqlite3_value] object returned by +** [sqlite3_column_value()] is used in any other way, including calls +** to routines like [sqlite3_value_int()], [sqlite3_value_text()], +** or [sqlite3_value_bytes()], the behavior is not threadsafe. +** Hence, the sqlite3_column_value() interface +** is normally only useful within the implementation of +** [application-defined SQL functions] or [virtual tables], not within +** top-level application code. +** +** The these routines may attempt to convert the datatype of the result. +** ^For example, if the internal representation is FLOAT and a text result +** is requested, [sqlite3_snprintf()] is used internally to perform the +** conversion automatically. ^(The following table details the conversions +** that are applied: +** +**
    +** +**
    Internal
    Type     		Requested
    Type     		Conversion +** +**
    NULL INTEGER Result is 0 +**
    NULL FLOAT Result is 0.0 +**
    NULL TEXT Result is a NULL pointer +**
    NULL BLOB Result is a NULL pointer +**
    INTEGER FLOAT Convert from integer to float +**
    INTEGER TEXT ASCII rendering of the integer +**
    INTEGER BLOB Same as INTEGER->TEXT +**
    FLOAT INTEGER [CAST] to INTEGER +**
    FLOAT TEXT ASCII rendering of the float +**
    FLOAT BLOB [CAST] to BLOB +**
    TEXT INTEGER [CAST] to INTEGER +**
    TEXT FLOAT [CAST] to REAL +**
    TEXT BLOB No change +**
    BLOB INTEGER [CAST] to INTEGER +**
    BLOB FLOAT [CAST] to REAL +**
    BLOB TEXT Add a zero terminator if needed +**
    +**
    )^ +** +** Note that when type conversions occur, pointers returned by prior +** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or +** sqlite3_column_text16() may be invalidated. +** Type conversions and pointer invalidations might occur +** in the following cases: +** +**
      +**
    • The initial content is a BLOB and sqlite3_column_text() or +** sqlite3_column_text16() is called. A zero-terminator might +** need to be added to the string.
      • +**
    • The initial content is UTF-8 text and sqlite3_column_bytes16() or +** sqlite3_column_text16() is called. The content must be converted +** to UTF-16.
      • +**
    • The initial content is UTF-16 text and sqlite3_column_bytes() or +** sqlite3_column_text() is called. The content must be converted +** to UTF-8.
      • +**
    +** +** ^Conversions between UTF-16be and UTF-16le are always done in place and do +** not invalidate a prior pointer, though of course the content of the buffer +** that the prior pointer references will have been modified. Other kinds +** of conversion are done in place when it is possible, but sometimes they +** are not possible and in those cases prior pointers are invalidated. +** +** The safest policy is to invoke these routines +** in one of the following ways: +** +**
      +**
    • sqlite3_column_text() followed by sqlite3_column_bytes()
      • +**
    • sqlite3_column_blob() followed by sqlite3_column_bytes()
      • +**
    • sqlite3_column_text16() followed by sqlite3_column_bytes16()
      • +**
    +** +** In other words, you should call sqlite3_column_text(), +** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result +** into the desired format, then invoke sqlite3_column_bytes() or +** sqlite3_column_bytes16() to find the size of the result. Do not mix calls +** to sqlite3_column_text() or sqlite3_column_blob() with calls to +** sqlite3_column_bytes16(), and do not mix calls to sqlite3_column_text16() +** with calls to sqlite3_column_bytes(). +** +** ^The pointers returned are valid until a type conversion occurs as +** described above, or until [sqlite3_step()] or [sqlite3_reset()] or +** [sqlite3_finalize()] is called. ^The memory space used to hold strings +** and BLOBs is freed automatically. Do not pass the pointers returned +** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into +** [sqlite3_free()]. +** +** As long as the input parameters are correct, these routines will only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
      +**
    • sqlite3_column_blob() +**
    • sqlite3_column_text() +**
    • sqlite3_column_text16() +**
    • sqlite3_column_bytes() +**
    • sqlite3_column_bytes16() +**
    +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. +*/ +SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); +SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_int(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); +SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); +SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); + +/* +** CAPI3REF: Destroy A Prepared Statement Object +** DESTRUCTOR: sqlite3_stmt +** +** ^The sqlite3_finalize() function is called to delete a [prepared statement]. +** ^If the most recent evaluation of the statement encountered no errors +** or if the statement is never been evaluated, then sqlite3_finalize() returns +** SQLITE_OK. ^If the most recent evaluation of statement S failed, then +** sqlite3_finalize(S) returns the appropriate [error code] or +** [extended error code]. +** +** ^The sqlite3_finalize(S) routine can be called at any point during +** the life cycle of [prepared statement] S: +** before statement S is ever evaluated, after +** one or more calls to [sqlite3_reset()], or after any call +** to [sqlite3_step()] regardless of whether or not the statement has +** completed execution. +** +** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. +** +** The application must finalize every [prepared statement] in order to avoid +** resource leaks. It is a grievous error for the application to try to use +** a prepared statement after it has been finalized. Any use of a prepared +** statement after it has been finalized can result in undefined and +** undesirable behavior such as segfaults and heap corruption. +*/ +SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Reset A Prepared Statement Object +** METHOD: sqlite3_stmt +** +** The sqlite3_reset() function is called to reset a [prepared statement] +** object back to its initial state, ready to be re-executed. +** ^Any SQL statement variables that had values bound to them using +** the [sqlite3_bind_blob | sqlite3_bind_*() API] retain their values. +** Use [sqlite3_clear_bindings()] to reset the bindings. +** +** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S +** back to the beginning of its program. +** +** ^If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], +** or if [sqlite3_step(S)] has never before been called on S, +** then [sqlite3_reset(S)] returns [SQLITE_OK]. +** +** ^If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S indicated an error, then +** [sqlite3_reset(S)] returns an appropriate [error code]. +** +** ^The [sqlite3_reset(S)] interface does not change the values +** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. +*/ +SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Create Or Redefine SQL Functions +** KEYWORDS: {function creation routines} +** METHOD: sqlite3 +** +** ^These functions (collectively known as "function creation routines") +** are used to add SQL functions or aggregates or to redefine the behavior +** of existing SQL functions or aggregates. The only differences between +** the three "sqlite3_create_function*" routines are the text encoding +** expected for the second parameter (the name of the function being +** created) and the presence or absence of a destructor callback for +** the application data pointer. Function sqlite3_create_window_function() +** is similar, but allows the user to supply the extra callback functions +** needed by [aggregate window functions]. +** +** ^The first parameter is the [database connection] to which the SQL +** function is to be added. ^If an application uses more than one database +** connection then application-defined SQL functions must be added +** to each database connection separately. +** +** ^The second parameter is the name of the SQL function to be created or +** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 +** representation, exclusive of the zero-terminator. ^Note that the name +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** ^Any attempt to create a function with a longer name +** will result in [SQLITE_MISUSE] being returned. +** +** ^The third parameter (nArg) +** is the number of arguments that the SQL function or +** aggregate takes. ^If this parameter is -1, then the SQL function or +** aggregate may take any number of arguments between 0 and the limit +** set by [sqlite3_limit]([SQLITE_LIMIT_FUNCTION_ARG]). If the third +** parameter is less than -1 or greater than 127 then the behavior is +** undefined. +** +** ^The fourth parameter, eTextRep, specifies what +** [SQLITE_UTF8 | text encoding] this SQL function prefers for +** its parameters. The application should set this parameter to +** [SQLITE_UTF16LE] if the function implementation invokes +** [sqlite3_value_text16le()] on an input, or [SQLITE_UTF16BE] if the +** implementation invokes [sqlite3_value_text16be()] on an input, or +** [SQLITE_UTF16] if [sqlite3_value_text16()] is used, or [SQLITE_UTF8] +** otherwise. ^The same SQL function may be registered multiple times using +** different preferred text encodings, with different implementations for +** each encoding. +** ^When multiple implementations of the same function are available, SQLite +** will pick the one that involves the least amount of data conversion. +** +** ^The fourth parameter may optionally be ORed with [SQLITE_DETERMINISTIC] +** to signal that the function will always return the same result given +** the same inputs within a single SQL statement. Most SQL functions are +** deterministic. The built-in [random()] SQL function is an example of a +** function that is not deterministic. The SQLite query planner is able to +** perform additional optimizations on deterministic functions, so use +** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. +** +** ^The fourth parameter may also optionally include the [SQLITE_DIRECTONLY] +** flag, which if present prevents the function from being invoked from +** within VIEWs, TRIGGERs, CHECK constraints, generated column expressions, +** index expressions, or the WHERE clause of partial indexes. +** +** For best security, the [SQLITE_DIRECTONLY] flag is recommended for +** all application-defined SQL functions that do not need to be +** used inside of triggers, view, CHECK constraints, or other elements of +** the database schema. This flags is especially recommended for SQL +** functions that have side effects or reveal internal application state. +** Without this flag, an attacker might be able to modify the schema of +** a database file to include invocations of the function with parameters +** chosen by the attacker, which the application will then execute when +** the database file is opened and read. +** +** ^(The fifth parameter is an arbitrary pointer. The implementation of the +** function can gain access to this pointer using [sqlite3_user_data()].)^ +** +** ^The sixth, seventh and eighth parameters passed to the three +** "sqlite3_create_function*" functions, xFunc, xStep and xFinal, are +** pointers to C-language functions that implement the SQL function or +** aggregate. ^A scalar SQL function requires an implementation of the xFunc +** callback only; NULL pointers must be passed as the xStep and xFinal +** parameters. ^An aggregate SQL function requires an implementation of xStep +** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing +** SQL function or aggregate, pass NULL pointers for all three function +** callbacks. +** +** ^The sixth, seventh, eighth and ninth parameters (xStep, xFinal, xValue +** and xInverse) passed to sqlite3_create_window_function are pointers to +** C-language callbacks that implement the new function. xStep and xFinal +** must both be non-NULL. xValue and xInverse may either both be NULL, in +** which case a regular aggregate function is created, or must both be +** non-NULL, in which case the new function may be used as either an aggregate +** or aggregate window function. More details regarding the implementation +** of aggregate window functions are +** [user-defined window functions|available here]. +** +** ^(If the final parameter to sqlite3_create_function_v2() or +** sqlite3_create_window_function() is not NULL, then it is destructor for +** the application data pointer. The destructor is invoked when the function +** is deleted, either by being overloaded or when the database connection +** closes.)^ ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. ^When the destructor callback is +** invoked, it is passed a single argument which is a copy of the application +** data pointer which was the fifth parameter to sqlite3_create_function_v2(). +** +** ^It is permitted to register multiple implementations of the same +** functions with the same name but with either differing numbers of +** arguments or differing preferred text encodings. ^SQLite will use +** the implementation that most closely matches the way in which the +** SQL function is used. ^A function implementation with a non-negative +** nArg parameter is a better match than a function implementation with +** a negative nArg. ^A function where the preferred text encoding +** matches the database encoding is a better +** match than a function where the encoding is different. +** ^A function where the encoding difference is between UTF16le and UTF16be +** is a closer match than a function where the encoding difference is +** between UTF8 and UTF16. +** +** ^Built-in functions may be overloaded by new application-defined functions. +** +** ^An application-defined function is permitted to call other +** SQLite interfaces. However, such calls must not +** close the database connection nor finalize or reset the prepared +** statement in which the function is running. +*/ +SQLITE_API int sqlite3_create_function( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +); +SQLITE_API int sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +); +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*) +); +SQLITE_API int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) +); + +/* +** CAPI3REF: Text Encodings +** +** These constant define integer codes that represent the various +** text encodings supported by SQLite. +*/ +#define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ +#define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ +#define SQLITE_UTF16BE 3 /* IMP: R-51971-34154 */ +#define SQLITE_UTF16 4 /* Use native byte order */ +#define SQLITE_ANY 5 /* Deprecated */ +#define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ + +/* +** CAPI3REF: Function Flags +** +** These constants may be ORed together with the +** [SQLITE_UTF8 | preferred text encoding] as the fourth argument +** to [sqlite3_create_function()], [sqlite3_create_function16()], or +** [sqlite3_create_function_v2()]. +** +**
    +** [[SQLITE_DETERMINISTIC]]
    SQLITE_DETERMINISTIC
    +** The SQLITE_DETERMINISTIC flag means that the new function always gives +** the same output when the input parameters are the same. +** The [abs|abs() function] is deterministic, for example, but +** [randomblob|randomblob()] is not. Functions must +** be deterministic in order to be used in certain contexts such as +** with the WHERE clause of [partial indexes] or in [generated columns]. +** SQLite might also optimize deterministic functions by factoring them +** out of inner loops. +**
    +** +** [[SQLITE_DIRECTONLY]]
    SQLITE_DIRECTONLY
    +** The SQLITE_DIRECTONLY flag means that the function may only be invoked +** from top-level SQL, and cannot be used in VIEWs or TRIGGERs nor in +** schema structures such as [CHECK constraints], [DEFAULT clauses], +** [expression indexes], [partial indexes], or [generated columns]. +** The SQLITE_DIRECTONLY flags is a security feature which is recommended +** for all [application-defined SQL functions], and especially for functions +** that have side-effects or that could potentially leak sensitive +** information. +**
    +** +** [[SQLITE_INNOCUOUS]]
    SQLITE_INNOCUOUS
    +** The SQLITE_INNOCUOUS flag means that the function is unlikely +** to cause problems even if misused. An innocuous function should have +** no side effects and should not depend on any values other than its +** input parameters. The [abs|abs() function] is an example of an +** innocuous function. +** The [load_extension() SQL function] is not innocuous because of its +** side effects. +**

    SQLITE_INNOCUOUS is similar to SQLITE_DETERMINISTIC, but is not +** exactly the same. The [random|random() function] is an example of a +** function that is innocuous but not deterministic. +**

    Some heightened security settings +** ([SQLITE_DBCONFIG_TRUSTED_SCHEMA] and [PRAGMA trusted_schema=OFF]) +** disable the use of SQL functions inside views and triggers and in +** schema structures such as [CHECK constraints], [DEFAULT clauses], +** [expression indexes], [partial indexes], and [generated columns] unless +** the function is tagged with SQLITE_INNOCUOUS. Most built-in functions +** are innocuous. Developers are advised to avoid using the +** SQLITE_INNOCUOUS flag for application-defined functions unless the +** function has been carefully audited and found to be free of potentially +** security-adverse side-effects and information-leaks. +**

    +** +** [[SQLITE_SUBTYPE]]
    SQLITE_SUBTYPE
    +** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call +** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. +** Specifying this flag makes no difference for scalar or aggregate user +** functions. However, if it is not specified for a user-defined window +** function, then any sub-types belonging to arguments passed to the window +** function may be discarded before the window function is called (i.e. +** sqlite3_value_subtype() will always return 0). +**
    +**
    +*/ +#define SQLITE_DETERMINISTIC 0x000000800 +#define SQLITE_DIRECTONLY 0x000080000 +#define SQLITE_SUBTYPE 0x000100000 +#define SQLITE_INNOCUOUS 0x000200000 + +/* +** CAPI3REF: Deprecated Functions +** DEPRECATED +** +** These functions are [deprecated]. In order to maintain +** backwards compatibility with older code, these functions continue +** to be supported. However, new applications should avoid +** the use of these functions. To encourage programmers to avoid +** these functions, we will not explain what they do. +*/ +#ifndef SQLITE_OMIT_DEPRECATED +SQLITE_API SQLITE_DEPRECATED int sqlite3_aggregate_count(sqlite3_context*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); +SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); +SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); +#endif + +/* +** CAPI3REF: Obtaining SQL Values +** METHOD: sqlite3_value +** +** Summary: +**
    +**
    sqlite3_value_blob→BLOB value +**
    sqlite3_value_double→REAL value +**
    sqlite3_value_int→32-bit INTEGER value +**
    sqlite3_value_int64→64-bit INTEGER value +**
    sqlite3_value_pointer→Pointer value +**
    sqlite3_value_text→UTF-8 TEXT value +**
    sqlite3_value_text16→UTF-16 TEXT value in +** the native byteorder +**
    sqlite3_value_text16be→UTF-16be TEXT value +**
    sqlite3_value_text16le→UTF-16le TEXT value +**
        +**
    sqlite3_value_bytes→Size of a BLOB +** or a UTF-8 TEXT in bytes +**
    sqlite3_value_bytes16   +** →  Size of UTF-16 +** TEXT in bytes +**
    sqlite3_value_type→Default +** datatype of the value +**
    sqlite3_value_numeric_type   +** →  Best numeric datatype of the value +**
    sqlite3_value_nochange   +** →  True if the column is unchanged in an UPDATE +** against a virtual table. +**
    sqlite3_value_frombind   +** →  True if value originated from a [bound parameter] +**
    +** +** Details: +** +** These routines extract type, size, and content information from +** [protected sqlite3_value] objects. Protected sqlite3_value objects +** are used to pass parameter information into the functions that +** implement [application-defined SQL functions] and [virtual tables]. +** +** These routines work only with [protected sqlite3_value] objects. +** Any attempt to use these routines on an [unprotected sqlite3_value] +** is not threadsafe. +** +** ^These routines work just like the corresponding [column access functions] +** except that these routines take a single [protected sqlite3_value] object +** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. +** +** ^The sqlite3_value_text16() interface extracts a UTF-16 string +** in the native byte-order of the host machine. ^The +** sqlite3_value_text16be() and sqlite3_value_text16le() interfaces +** extract UTF-16 strings as big-endian and little-endian respectively. +** +** ^If [sqlite3_value] object V was initialized +** using [sqlite3_bind_pointer(S,I,P,X,D)] or [sqlite3_result_pointer(C,P,X,D)] +** and if X and Y are strings that compare equal according to strcmp(X,Y), +** then sqlite3_value_pointer(V,Y) will return the pointer P. ^Otherwise, +** sqlite3_value_pointer(V,Y) returns a NULL. The sqlite3_bind_pointer() +** routine is part of the [pointer passing interface] added for SQLite 3.20.0. +** +** ^(The sqlite3_value_type(V) interface returns the +** [SQLITE_INTEGER | datatype code] for the initial datatype of the +** [sqlite3_value] object V. The returned value is one of [SQLITE_INTEGER], +** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL].)^ +** Other interfaces might change the datatype for an sqlite3_value object. +** For example, if the datatype is initially SQLITE_INTEGER and +** sqlite3_value_text(V) is called to extract a text value for that +** integer, then subsequent calls to sqlite3_value_type(V) might return +** SQLITE_TEXT. Whether or not a persistent internal datatype conversion +** occurs is undefined and may change from one release of SQLite to the next. +** +** ^(The sqlite3_value_numeric_type() interface attempts to apply +** numeric affinity to the value. This means that an attempt is +** made to convert the value to an integer or floating point. If +** such a conversion is possible without loss of information (in other +** words, if the value is a string that looks like a number) +** then the conversion is performed. Otherwise no conversion occurs. +** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ +** +** ^Within the [xUpdate] method of a [virtual table], the +** sqlite3_value_nochange(X) interface returns true if and only if +** the column corresponding to X is unchanged by the UPDATE operation +** that the xUpdate method call was invoked to implement and if +** and the prior [xColumn] method call that was invoked to extracted +** the value for that column returned without setting a result (probably +** because it queried [sqlite3_vtab_nochange()] and found that the column +** was unchanging). ^Within an [xUpdate] method, any value for which +** sqlite3_value_nochange(X) is true will in all other respects appear +** to be a NULL value. If sqlite3_value_nochange(X) is invoked anywhere other +** than within an [xUpdate] method call for an UPDATE statement, then +** the return value is arbitrary and meaningless. +** +** ^The sqlite3_value_frombind(X) interface returns non-zero if the +** value X originated from one of the [sqlite3_bind_int|sqlite3_bind()] +** interfaces. ^If X comes from an SQL literal value, or a table column, +** or an expression, then sqlite3_value_frombind(X) returns zero. +** +** Please pay particular attention to the fact that the pointer returned +** from [sqlite3_value_blob()], [sqlite3_value_text()], or +** [sqlite3_value_text16()] can be invalidated by a subsequent call to +** [sqlite3_value_bytes()], [sqlite3_value_bytes16()], [sqlite3_value_text()], +** or [sqlite3_value_text16()]. +** +** These routines must be called from the same thread as +** the SQL function that supplied the [sqlite3_value*] parameters. +** +** As long as the input parameter is correct, these routines can only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
      +**
    • sqlite3_value_blob() +**
    • sqlite3_value_text() +**
    • sqlite3_value_text16() +**
    • sqlite3_value_text16le() +**
    • sqlite3_value_text16be() +**
    • sqlite3_value_bytes() +**
    • sqlite3_value_bytes16() +**
    +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. +*/ +SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); +SQLITE_API double sqlite3_value_double(sqlite3_value*); +SQLITE_API int sqlite3_value_int(sqlite3_value*); +SQLITE_API sqlite3_int64 sqlite3_value_int64(sqlite3_value*); +SQLITE_API void *sqlite3_value_pointer(sqlite3_value*, const char*); +SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16le(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16be(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); +SQLITE_API int sqlite3_value_type(sqlite3_value*); +SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); +SQLITE_API int sqlite3_value_nochange(sqlite3_value*); +SQLITE_API int sqlite3_value_frombind(sqlite3_value*); + +/* +** CAPI3REF: Finding The Subtype Of SQL Values +** METHOD: sqlite3_value +** +** The sqlite3_value_subtype(V) function returns the subtype for +** an [application-defined SQL function] argument V. The subtype +** information can be used to pass a limited amount of context from +** one SQL function to another. Use the [sqlite3_result_subtype()] +** routine to set the subtype for the return value of an SQL function. +*/ +SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); + +/* +** CAPI3REF: Copy And Free SQL Values +** METHOD: sqlite3_value +** +** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value] +** object D and returns a pointer to that copy. ^The [sqlite3_value] returned +** is a [protected sqlite3_value] object even if the input is not. +** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a +** memory allocation fails. +** +** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object +** previously obtained from [sqlite3_value_dup()]. ^If V is a NULL pointer +** then sqlite3_value_free(V) is a harmless no-op. +*/ +SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value*); +SQLITE_API void sqlite3_value_free(sqlite3_value*); + +/* +** CAPI3REF: Obtain Aggregate Function Context +** METHOD: sqlite3_context +** +** Implementations of aggregate SQL functions use this +** routine to allocate memory for storing their state. +** +** ^The first time the sqlite3_aggregate_context(C,N) routine is called +** for a particular aggregate function, SQLite allocates +** N bytes of memory, zeroes out that memory, and returns a pointer +** to the new memory. ^On second and subsequent calls to +** sqlite3_aggregate_context() for the same aggregate function instance, +** the same buffer is returned. Sqlite3_aggregate_context() is normally +** called once for each invocation of the xStep callback and then one +** last time when the xFinal callback is invoked. ^(When no rows match +** an aggregate query, the xStep() callback of the aggregate function +** implementation is never called and xFinal() is called exactly once. +** In those cases, sqlite3_aggregate_context() might be called for the +** first time from within xFinal().)^ +** +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. +** +** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is +** determined by the N parameter on first successful call. Changing the +** value of N in any subsequent call to sqlite3_aggregate_context() within +** the same aggregate function instance will not resize the memory +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. +** +** ^SQLite automatically frees the memory allocated by +** sqlite3_aggregate_context() when the aggregate query concludes. +** +** The first parameter must be a copy of the +** [sqlite3_context | SQL function context] that is the first parameter +** to the xStep or xFinal callback routine that implements the aggregate +** function. +** +** This routine must be called from the same thread in which +** the aggregate SQL function is running. +*/ +SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); + +/* +** CAPI3REF: User Data For Functions +** METHOD: sqlite3_context +** +** ^The sqlite3_user_data() interface returns a copy of +** the pointer that was the pUserData parameter (the 5th parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. +** +** This routine must be called from the same thread in which +** the application-defined function is running. +*/ +SQLITE_API void *sqlite3_user_data(sqlite3_context*); + +/* +** CAPI3REF: Database Connection For Functions +** METHOD: sqlite3_context +** +** ^The sqlite3_context_db_handle() interface returns a copy of +** the pointer to the [database connection] (the 1st parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. +*/ +SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); + +/* +** CAPI3REF: Function Auxiliary Data +** METHOD: sqlite3_context +** +** These functions may be used by (non-aggregate) SQL functions to +** associate metadata with argument values. If the same value is passed to +** multiple invocations of the same SQL function during query execution, under +** some circumstances the associated metadata may be preserved. An example +** of where this might be useful is in a regular-expression matching +** function. The compiled version of the regular expression can be stored as +** metadata associated with the pattern string. +** Then as long as the pattern string remains the same, +** the compiled regular expression can be reused on multiple +** invocations of the same function. +** +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the metadata +** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument +** value to the application-defined function. ^N is zero for the left-most +** function argument. ^If there is no metadata +** associated with the function argument, the sqlite3_get_auxdata(C,N) interface +** returns a NULL pointer. +** +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th +** argument of the application-defined function. ^Subsequent +** calls to sqlite3_get_auxdata(C,N) return P from the most recent +** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or +** NULL if the metadata has been discarded. +** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, +** SQLite will invoke the destructor function X with parameter P exactly +** once, when the metadata is discarded. +** SQLite is free to discard the metadata at any time, including:
      +**
    • ^(when the corresponding function parameter changes)^, or +**
    • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement)^, or +**
    • ^(when sqlite3_set_auxdata() is invoked again on the same +** parameter)^, or +**
    • ^(during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.)^
    +** +** Note the last bullet in particular. The destructor X in +** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the +** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() +** should be called near the end of the function implementation and the +** function implementation should not make any use of P after +** sqlite3_set_auxdata() has been called. +** +** ^(In practice, metadata is preserved between function calls for +** function parameters that are compile-time constants, including literal +** values and [parameters] and expressions composed from the same.)^ +** +** The value of the N parameter to these interfaces should be non-negative. +** Future enhancements may make use of negative N values to define new +** kinds of function caching behavior. +** +** These routines must be called from the same thread in which +** the SQL function is running. +*/ +SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); +SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); + + +/* +** CAPI3REF: Constants Defining Special Destructor Behavior +** +** These are special values for the destructor that is passed in as the +** final argument to routines like [sqlite3_result_blob()]. ^If the destructor +** argument is SQLITE_STATIC, it means that the content pointer is constant +** and will never change. It does not need to be destroyed. ^The +** SQLITE_TRANSIENT value means that the content will likely change in +** the near future and that SQLite should make its own private copy of +** the content before returning. +** +** The typedef is necessary to work around problems in certain +** C++ compilers. +*/ +typedef void (*sqlite3_destructor_type)(void*); +#define SQLITE_STATIC ((sqlite3_destructor_type)0) +#define SQLITE_TRANSIENT ((sqlite3_destructor_type)-1) + +/* +** CAPI3REF: Setting The Result Of An SQL Function +** METHOD: sqlite3_context +** +** These routines are used by the xFunc or xFinal callbacks that +** implement SQL functions and aggregates. See +** [sqlite3_create_function()] and [sqlite3_create_function16()] +** for additional information. +** +** These functions work very much like the [parameter binding] family of +** functions used to bind values to host parameters in prepared statements. +** Refer to the [SQL parameter] documentation for additional information. +** +** ^The sqlite3_result_blob() interface sets the result from +** an application-defined function to be the BLOB whose content is pointed +** to by the second parameter and which is N bytes long where N is the +** third parameter. +** +** ^The sqlite3_result_zeroblob(C,N) and sqlite3_result_zeroblob64(C,N) +** interfaces set the result of the application-defined function to be +** a BLOB containing all zero bytes and N bytes in size. +** +** ^The sqlite3_result_double() interface sets the result from +** an application-defined function to be a floating point value specified +** by its 2nd argument. +** +** ^The sqlite3_result_error() and sqlite3_result_error16() functions +** cause the implemented SQL function to throw an exception. +** ^SQLite uses the string pointed to by the +** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() +** as the text of an error message. ^SQLite interprets the error +** message string from sqlite3_result_error() as UTF-8. ^SQLite +** interprets the string from sqlite3_result_error16() as UTF-16 using +** the same [byte-order determination rules] as [sqlite3_bind_text16()]. +** ^If the third parameter to sqlite3_result_error() +** or sqlite3_result_error16() is negative then SQLite takes as the error +** message all text up through the first zero character. +** ^If the third parameter to sqlite3_result_error() or +** sqlite3_result_error16() is non-negative then SQLite takes that many +** bytes (not characters) from the 2nd parameter as the error message. +** ^The sqlite3_result_error() and sqlite3_result_error16() +** routines make a private copy of the error message text before +** they return. Hence, the calling function can deallocate or +** modify the text after they return without harm. +** ^The sqlite3_result_error_code() function changes the error code +** returned by SQLite as a result of an error in a function. ^By default, +** the error code is SQLITE_ERROR. ^A subsequent call to sqlite3_result_error() +** or sqlite3_result_error16() resets the error code to SQLITE_ERROR. +** +** ^The sqlite3_result_error_toobig() interface causes SQLite to throw an +** error indicating that a string or BLOB is too long to represent. +** +** ^The sqlite3_result_error_nomem() interface causes SQLite to throw an +** error indicating that a memory allocation failed. +** +** ^The sqlite3_result_int() interface sets the return value +** of the application-defined function to be the 32-bit signed integer +** value given in the 2nd argument. +** ^The sqlite3_result_int64() interface sets the return value +** of the application-defined function to be the 64-bit signed integer +** value given in the 2nd argument. +** +** ^The sqlite3_result_null() interface sets the return value +** of the application-defined function to be NULL. +** +** ^The sqlite3_result_text(), sqlite3_result_text16(), +** sqlite3_result_text16le(), and sqlite3_result_text16be() interfaces +** set the return value of the application-defined function to be +** a text string which is represented as UTF-8, UTF-16 native byte order, +** UTF-16 little endian, or UTF-16 big endian, respectively. +** ^The sqlite3_result_text64() interface sets the return value of an +** application-defined function to be a text string in an encoding +** specified by the fifth (and last) parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. +** ^SQLite takes the text result from the application from +** the 2nd parameter of the sqlite3_result_text* interfaces. +** ^If the 3rd parameter to the sqlite3_result_text* interfaces +** is negative, then SQLite takes result text from the 2nd parameter +** through the first zero character. +** ^If the 3rd parameter to the sqlite3_result_text* interfaces +** is non-negative, then as many bytes (not characters) of the text +** pointed to by the 2nd parameter are taken as the application-defined +** function result. If the 3rd parameter is non-negative, then it +** must be the byte offset into the string where the NUL terminator would +** appear if the string where NUL terminated. If any NUL characters occur +** in the string at a byte offset that is less than the value of the 3rd +** parameter, then the resulting string will contain embedded NULs and the +** result of expressions operating on strings with embedded NULs is undefined. +** ^If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is a non-NULL pointer, then SQLite calls that +** function as the destructor on the text or BLOB result when it has +** finished using that result. +** ^If the 4th parameter to the sqlite3_result_text* interfaces or to +** sqlite3_result_blob is the special constant SQLITE_STATIC, then SQLite +** assumes that the text or BLOB result is in constant space and does not +** copy the content of the parameter nor call a destructor on the content +** when it has finished using that result. +** ^If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is the special constant SQLITE_TRANSIENT +** then SQLite makes a copy of the result into space obtained +** from [sqlite3_malloc()] before it returns. +** +** ^For the sqlite3_result_text16(), sqlite3_result_text16le(), and +** sqlite3_result_text16be() routines, and for sqlite3_result_text64() +** when the encoding is not UTF8, if the input UTF16 begins with a +** byte-order mark (BOM, U+FEFF) then the BOM is removed from the +** string and the rest of the string is interpreted according to the +** byte-order specified by the BOM. ^The byte-order specified by +** the BOM at the beginning of the text overrides the byte-order +** specified by the interface procedure. ^So, for example, if +** sqlite3_result_text16le() is invoked with text that begins +** with bytes 0xfe, 0xff (a big-endian byte-order mark) then the +** first two bytes of input are skipped and the remaining input +** is interpreted as UTF16BE text. +** +** ^For UTF16 input text to the sqlite3_result_text16(), +** sqlite3_result_text16be(), sqlite3_result_text16le(), and +** sqlite3_result_text64() routines, if the text contains invalid +** UTF16 characters, the invalid characters might be converted +** into the unicode replacement character, U+FFFD. +** +** ^The sqlite3_result_value() interface sets the result of +** the application-defined function to be a copy of the +** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The +** sqlite3_result_value() interface makes a copy of the [sqlite3_value] +** so that the [sqlite3_value] specified in the parameter may change or +** be deallocated after sqlite3_result_value() returns without harm. +** ^A [protected sqlite3_value] object may always be used where an +** [unprotected sqlite3_value] object is required, so either +** kind of [sqlite3_value] object can be used with this interface. +** +** ^The sqlite3_result_pointer(C,P,T,D) interface sets the result to an +** SQL NULL value, just like [sqlite3_result_null(C)], except that it +** also associates the host-language pointer P or type T with that +** NULL value such that the pointer can be retrieved within an +** [application-defined SQL function] using [sqlite3_value_pointer()]. +** ^If the D parameter is not NULL, then it is a pointer to a destructor +** for the P parameter. ^SQLite invokes D with P as its only argument +** when SQLite is finished with P. The T parameter should be a static +** string and preferably a string literal. The sqlite3_result_pointer() +** routine is part of the [pointer passing interface] added for SQLite 3.20.0. +** +** If these routines are called from within the different thread +** than the one containing the application-defined function that received +** the [sqlite3_context] pointer, the results are undefined. +*/ +SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_blob64(sqlite3_context*,const void*, + sqlite3_uint64,void(*)(void*)); +SQLITE_API void sqlite3_result_double(sqlite3_context*, double); +SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); +SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); +SQLITE_API void sqlite3_result_error_toobig(sqlite3_context*); +SQLITE_API void sqlite3_result_error_nomem(sqlite3_context*); +SQLITE_API void sqlite3_result_error_code(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); +SQLITE_API void sqlite3_result_null(sqlite3_context*); +SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, + void(*)(void*), unsigned char encoding); +SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_value(sqlite3_context*, sqlite3_value*); +SQLITE_API void sqlite3_result_pointer(sqlite3_context*, void*,const char*,void(*)(void*)); +SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); +SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); + + +/* +** CAPI3REF: Setting The Subtype Of An SQL Function +** METHOD: sqlite3_context +** +** The sqlite3_result_subtype(C,T) function causes the subtype of +** the result from the [application-defined SQL function] with +** [sqlite3_context] C to be the value T. Only the lower 8 bits +** of the subtype T are preserved in current versions of SQLite; +** higher order bits are discarded. +** The number of subtype bytes preserved by SQLite might increase +** in future releases of SQLite. +*/ +SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); + +/* +** CAPI3REF: Define New Collating Sequences +** METHOD: sqlite3 +** +** ^These functions add, remove, or modify a [collation] associated +** with the [database connection] specified as the first argument. +** +** ^The name of the collation is a UTF-8 string +** for sqlite3_create_collation() and sqlite3_create_collation_v2() +** and a UTF-16 string in native byte order for sqlite3_create_collation16(). +** ^Collation names that compare equal according to [sqlite3_strnicmp()] are +** considered to be the same name. +** +** ^(The third argument (eTextRep) must be one of the constants: +**
      +**
    • [SQLITE_UTF8], +**
    • [SQLITE_UTF16LE], +**
    • [SQLITE_UTF16BE], +**
    • [SQLITE_UTF16], or +**
    • [SQLITE_UTF16_ALIGNED]. +**
    )^ +** ^The eTextRep argument determines the encoding of strings passed +** to the collating function callback, xCompare. +** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep +** force strings to be UTF16 with native byte order. +** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +** on an even byte address. +** +** ^The fourth argument, pArg, is an application data pointer that is passed +** through as the first argument to the collating function callback. +** +** ^The fifth argument, xCompare, is a pointer to the collating function. +** ^Multiple collating functions can be registered using the same name but +** with different eTextRep parameters and SQLite will use whichever +** function requires the least amount of data transformation. +** ^If the xCompare argument is NULL then the collating function is +** deleted. ^When all collating functions having the same name are deleted, +** that collation is no longer usable. +** +** ^The collating function callback is invoked with a copy of the pArg +** application data pointer and with two strings in the encoding specified +** by the eTextRep argument. The two integer parameters to the collating +** function callback are the length of the two strings, in bytes. The collating +** function must return an integer that is negative, zero, or positive +** if the first string is less than, equal to, or greater than the second, +** respectively. A collating function must always return the same answer +** given the same inputs. If two or more collating functions are registered +** to the same collation name (using different eTextRep values) then all +** must give an equivalent answer when invoked with equivalent strings. +** The collating function must obey the following properties for all +** strings A, B, and C: +** +**
      +**
    1. If A==B then B==A. +**
    2. If A==B and B==C then A==C. +**
    3. If A<B THEN B>A. +**
    4. If A<B and B<C then A<C. +**
    +** +** If a collating function fails any of the above constraints and that +** collating function is registered and used, then the behavior of SQLite +** is undefined. +** +** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() +** with the addition that the xDestroy callback is invoked on pArg when +** the collating function is deleted. +** ^Collating functions are deleted when they are overridden by later +** calls to the collation creation functions or when the +** [database connection] is closed using [sqlite3_close()]. +** +** ^The xDestroy callback is not called if the +** sqlite3_create_collation_v2() function fails. Applications that invoke +** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should +** check the return code and dispose of the application data pointer +** themselves rather than expecting SQLite to deal with it for them. +** This is different from every other SQLite interface. The inconsistency +** is unfortunate but cannot be changed without breaking backwards +** compatibility. +** +** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. +*/ +SQLITE_API int sqlite3_create_collation( + sqlite3*, + const char *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*) +); +SQLITE_API int sqlite3_create_collation_v2( + sqlite3*, + const char *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDestroy)(void*) +); +SQLITE_API int sqlite3_create_collation16( + sqlite3*, + const void *zName, + int eTextRep, + void *pArg, + int(*xCompare)(void*,int,const void*,int,const void*) +); + +/* +** CAPI3REF: Collation Needed Callbacks +** METHOD: sqlite3 +** +** ^To avoid having to register all collation sequences before a database +** can be used, a single callback function may be registered with the +** [database connection] to be invoked whenever an undefined collation +** sequence is required. +** +** ^If the function is registered using the sqlite3_collation_needed() API, +** then it is passed the names of undefined collation sequences as strings +** encoded in UTF-8. ^If sqlite3_collation_needed16() is used, +** the names are passed as UTF-16 in machine native byte order. +** ^A call to either function replaces the existing collation-needed callback. +** +** ^(When the callback is invoked, the first argument passed is a copy +** of the second argument to sqlite3_collation_needed() or +** sqlite3_collation_needed16(). The second argument is the database +** connection. The third argument is one of [SQLITE_UTF8], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE], indicating the most desirable form of the collation +** sequence function required. The fourth parameter is the name of the +** required collation sequence.)^ +** +** The callback function should register the desired collation using +** [sqlite3_create_collation()], [sqlite3_create_collation16()], or +** [sqlite3_create_collation_v2()]. +*/ +SQLITE_API int sqlite3_collation_needed( + sqlite3*, + void*, + void(*)(void*,sqlite3*,int eTextRep,const char*) +); +SQLITE_API int sqlite3_collation_needed16( + sqlite3*, + void*, + void(*)(void*,sqlite3*,int eTextRep,const void*) +); + +#ifdef SQLITE_ENABLE_CEROD +/* +** Specify the activation key for a CEROD database. Unless +** activated, none of the CEROD routines will work. +*/ +SQLITE_API void sqlite3_activate_cerod( + const char *zPassPhrase /* Activation phrase */ +); +#endif + +/* +** CAPI3REF: Suspend Execution For A Short Time +** +** The sqlite3_sleep() function causes the current thread to suspend execution +** for at least a number of milliseconds specified in its parameter. +** +** If the operating system does not support sleep requests with +** millisecond time resolution, then the time will be rounded up to +** the nearest second. The number of milliseconds of sleep actually +** requested from the operating system is returned. +** +** ^SQLite implements this interface by calling the xSleep() +** method of the default [sqlite3_vfs] object. If the xSleep() method +** of the default VFS is not implemented correctly, or not implemented at +** all, then the behavior of sqlite3_sleep() may deviate from the description +** in the previous paragraphs. +*/ +SQLITE_API int sqlite3_sleep(int); + +/* +** CAPI3REF: Name Of The Folder Holding Temporary Files +** +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all temporary files +** created by SQLite when using a built-in [sqlite3_vfs | VFS] +** will be placed in that directory.)^ ^If this variable +** is a NULL pointer, then SQLite performs a search for an appropriate +** temporary file directory. +** +** Applications are strongly discouraged from using this global variable. +** It is required to set a temporary folder on Windows Runtime (WinRT). +** But for all other platforms, it is highly recommended that applications +** neither read nor write this variable. This global variable is a relic +** that exists for backwards compatibility of legacy applications and should +** be avoided in new projects. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [temp_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [temp_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [temp_store_directory pragma] should be avoided. +** Except when requested by the [temp_store_directory pragma], SQLite +** does not free the memory that sqlite3_temp_directory points to. If +** the application wants that memory to be freed, it must do +** so itself, taking care to only do so after all [database connection] +** objects have been destroyed. +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling [sqlite3_open] or [sqlite3_open_v2]. Otherwise, various +** features that require the use of temporary files may fail. Here is an +** example of how to do this using C++ with the Windows Runtime: +** +** 
    +** LPCWSTR zPath = Windows::Storage::ApplicationData::Current->
+**       TemporaryFolder->Path->Data();
+** char zPathBuf[MAX_PATH + 1];
+** memset(zPathBuf, 0, sizeof(zPathBuf));
+** WideCharToMultiByte(CP_UTF8, 0, zPath, -1, zPathBuf, sizeof(zPathBuf),
+**       NULL, NULL);
+** sqlite3_temp_directory = sqlite3_mprintf("%s", zPathBuf);
+**
    +*/ +SQLITE_API char *sqlite3_temp_directory; + +/* +** CAPI3REF: Name Of The Folder Holding Database Files +** +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all database files +** specified with a relative pathname and created or accessed by +** SQLite when using a built-in windows [sqlite3_vfs | VFS] will be assumed +** to be relative to that directory.)^ ^If this variable is a NULL +** pointer, then SQLite assumes that all database files specified +** with a relative pathname are relative to the current directory +** for the process. Only the windows VFS makes use of this global +** variable; it is ignored by the unix VFS. +** +** Changing the value of this variable while a database connection is +** open can result in a corrupt database. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [data_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [data_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [data_store_directory pragma] should be avoided. +*/ +SQLITE_API char *sqlite3_data_directory; + +/* +** CAPI3REF: Win32 Specific Interface +** +** These interfaces are available only on Windows. The +** [sqlite3_win32_set_directory] interface is used to set the value associated +** with the [sqlite3_temp_directory] or [sqlite3_data_directory] variable, to +** zValue, depending on the value of the type parameter. The zValue parameter +** should be NULL to cause the previous value to be freed via [sqlite3_free]; +** a non-NULL value will be copied into memory obtained from [sqlite3_malloc] +** prior to being used. The [sqlite3_win32_set_directory] interface returns +** [SQLITE_OK] to indicate success, [SQLITE_ERROR] if the type is unsupported, +** or [SQLITE_NOMEM] if memory could not be allocated. The value of the +** [sqlite3_data_directory] variable is intended to act as a replacement for +** the current directory on the sub-platforms of Win32 where that concept is +** not present, e.g. WinRT and UWP. The [sqlite3_win32_set_directory8] and +** [sqlite3_win32_set_directory16] interfaces behave exactly the same as the +** sqlite3_win32_set_directory interface except the string parameter must be +** UTF-8 or UTF-16, respectively. +*/ +SQLITE_API int sqlite3_win32_set_directory( + unsigned long type, /* Identifier for directory being set or reset */ + void *zValue /* New value for directory being set or reset */ +); +SQLITE_API int sqlite3_win32_set_directory8(unsigned long type, const char *zValue); +SQLITE_API int sqlite3_win32_set_directory16(unsigned long type, const void *zValue); + +/* +** CAPI3REF: Win32 Directory Types +** +** These macros are only available on Windows. They define the allowed values +** for the type argument to the [sqlite3_win32_set_directory] interface. +*/ +#define SQLITE_WIN32_DATA_DIRECTORY_TYPE 1 +#define SQLITE_WIN32_TEMP_DIRECTORY_TYPE 2 + +/* +** CAPI3REF: Test For Auto-Commit Mode +** KEYWORDS: {autocommit mode} +** METHOD: sqlite3 +** +** ^The sqlite3_get_autocommit() interface returns non-zero or +** zero if the given database connection is or is not in autocommit mode, +** respectively. ^Autocommit mode is on by default. +** ^Autocommit mode is disabled by a [BEGIN] statement. +** ^Autocommit mode is re-enabled by a [COMMIT] or [ROLLBACK]. +** +** If certain kinds of errors occur on a statement within a multi-statement +** transaction (errors including [SQLITE_FULL], [SQLITE_IOERR], +** [SQLITE_NOMEM], [SQLITE_BUSY], and [SQLITE_INTERRUPT]) then the +** transaction might be rolled back automatically. The only way to +** find out whether SQLite automatically rolled back the transaction after +** an error is to use this function. +** +** If another thread changes the autocommit status of the database +** connection while this routine is running, then the return value +** is undefined. +*/ +SQLITE_API int sqlite3_get_autocommit(sqlite3*); + +/* +** CAPI3REF: Find The Database Handle Of A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^The sqlite3_db_handle interface returns the [database connection] handle +** to which a [prepared statement] belongs. ^The [database connection] +** returned by sqlite3_db_handle is the same [database connection] +** that was the first argument +** to the [sqlite3_prepare_v2()] call (or its variants) that was used to +** create the statement in the first place. +*/ +SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); + +/* +** CAPI3REF: Return The Filename For A Database Connection +** METHOD: sqlite3 +** +** ^The sqlite3_db_filename(D,N) interface returns a pointer to the filename +** associated with database N of connection D. +** ^If there is no attached database N on the database +** connection D, or if database N is a temporary or in-memory database, then +** this function will return either a NULL pointer or an empty string. +** +** ^The string value returned by this routine is owned and managed by +** the database connection. ^The value will be valid until the database N +** is [DETACH]-ed or until the database connection closes. +** +** ^The filename returned by this function is the output of the +** xFullPathname method of the [VFS]. ^In other words, the filename +** will be an absolute pathname, even if the filename used +** to open the database originally was a URI or relative pathname. +** +** If the filename pointer returned by this routine is not NULL, then it +** can be used as the filename input parameter to these routines: +**
      +**
    • [sqlite3_uri_parameter()] +**
    • [sqlite3_uri_boolean()] +**
    • [sqlite3_uri_int64()] +**
    • [sqlite3_filename_database()] +**
    • [sqlite3_filename_journal()] +**
    • [sqlite3_filename_wal()] +**
    +*/ +SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine if a database is read-only +** METHOD: sqlite3 +** +** ^The sqlite3_db_readonly(D,N) interface returns 1 if the database N +** of connection D is read-only, 0 if it is read/write, or -1 if N is not +** the name of a database on connection D. +*/ +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine the transaction state of a database +** METHOD: sqlite3 +** +** ^The sqlite3_txn_state(D,S) interface returns the current +** [transaction state] of schema S in database connection D. ^If S is NULL, +** then the highest transaction state of any schema on database connection D +** is returned. Transaction states are (in order of lowest to highest): +**
      +**
    1. SQLITE_TXN_NONE +**
    2. SQLITE_TXN_READ +**
    3. SQLITE_TXN_WRITE +**
    +** ^If the S argument to sqlite3_txn_state(D,S) is not the name of +** a valid schema, then -1 is returned. +*/ +SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); + +/* +** CAPI3REF: Allowed return values from [sqlite3_txn_state()] +** KEYWORDS: {transaction state} +** +** These constants define the current transaction state of a database file. +** ^The [sqlite3_txn_state(D,S)] interface returns one of these +** constants in order to describe the transaction state of schema S +** in [database connection] D. +** +**
    +** [[SQLITE_TXN_NONE]]
    SQLITE_TXN_NONE
    +**
    The SQLITE_TXN_NONE state means that no transaction is currently +** pending.
    +** +** [[SQLITE_TXN_READ]]
    SQLITE_TXN_READ
    +**
    The SQLITE_TXN_READ state means that the database is currently +** in a read transaction. Content has been read from the database file +** but nothing in the database file has changed. The transaction state +** will advanced to SQLITE_TXN_WRITE if any changes occur and there are +** no other conflicting concurrent write transactions. The transaction +** state will revert to SQLITE_TXN_NONE following a [ROLLBACK] or +** [COMMIT].
    +** +** [[SQLITE_TXN_WRITE]]
    SQLITE_TXN_WRITE
    +**
    The SQLITE_TXN_WRITE state means that the database is currently +** in a write transaction. Content has been written to the database file +** but has not yet committed. The transaction state will change to +** to SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].
    +*/ +#define SQLITE_TXN_NONE 0 +#define SQLITE_TXN_READ 1 +#define SQLITE_TXN_WRITE 2 + +/* +** CAPI3REF: Find the next prepared statement +** METHOD: sqlite3 +** +** ^This interface returns a pointer to the next [prepared statement] after +** pStmt associated with the [database connection] pDb. ^If pStmt is NULL +** then this interface returns a pointer to the first prepared statement +** associated with the database connection pDb. ^If no prepared statement +** satisfies the conditions of this routine, it returns NULL. +** +** The [database connection] pointer D in a call to +** [sqlite3_next_stmt(D,S)] must refer to an open database +** connection and in particular must not be a NULL pointer. +*/ +SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Commit And Rollback Notification Callbacks +** METHOD: sqlite3 +** +** ^The sqlite3_commit_hook() interface registers a callback +** function to be invoked whenever a transaction is [COMMIT | committed]. +** ^Any callback set by a previous call to sqlite3_commit_hook() +** for the same database connection is overridden. +** ^The sqlite3_rollback_hook() interface registers a callback +** function to be invoked whenever a transaction is [ROLLBACK | rolled back]. +** ^Any callback set by a previous call to sqlite3_rollback_hook() +** for the same database connection is overridden. +** ^The pArg argument is passed through to the callback. +** ^If the callback on a commit hook function returns non-zero, +** then the commit is converted into a rollback. +** +** ^The sqlite3_commit_hook(D,C,P) and sqlite3_rollback_hook(D,C,P) functions +** return the P argument from the previous call of the same function +** on the same [database connection] D, or NULL for +** the first call for each function on D. +** +** The commit and rollback hook callbacks are not reentrant. +** The callback implementation must not do anything that will modify +** the database connection that invoked the callback. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the commit +** or rollback hook in the first place. +** Note that running any other SQL statements, including SELECT statements, +** or merely calling [sqlite3_prepare_v2()] and [sqlite3_step()] will modify +** the database connections for the meaning of "modify" in this paragraph. +** +** ^Registering a NULL function disables the callback. +** +** ^When the commit hook callback routine returns zero, the [COMMIT] +** operation is allowed to continue normally. ^If the commit hook +** returns non-zero, then the [COMMIT] is converted into a [ROLLBACK]. +** ^The rollback hook is invoked on a rollback that results from a commit +** hook returning non-zero, just as it would be with any other rollback. +** +** ^For the purposes of this API, a transaction is said to have been +** rolled back if an explicit "ROLLBACK" statement is executed, or +** an error or constraint causes an implicit rollback to occur. +** ^The rollback callback is not invoked if a transaction is +** automatically rolled back because the database connection is closed. +** +** See also the [sqlite3_update_hook()] interface. +*/ +SQLITE_API void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); +SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); + +/* +** CAPI3REF: Data Change Notification Callbacks +** METHOD: sqlite3 +** +** ^The sqlite3_update_hook() interface registers a callback function +** with the [database connection] identified by the first argument +** to be invoked whenever a row is updated, inserted or deleted in +** a [rowid table]. +** ^Any callback set by a previous call to this function +** for the same database connection is overridden. +** +** ^The second argument is a pointer to the function to invoke when a +** row is updated, inserted or deleted in a rowid table. +** ^The first argument to the callback is a copy of the third argument +** to sqlite3_update_hook(). +** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], +** or [SQLITE_UPDATE], depending on the operation that caused the callback +** to be invoked. +** ^The third and fourth arguments to the callback contain pointers to the +** database and table name containing the affected row. +** ^The final callback parameter is the [rowid] of the row. +** ^In the case of an update, this is the [rowid] after the update takes place. +** +** ^(The update hook is not invoked when internal system tables are +** modified (i.e. sqlite_sequence).)^ +** ^The update hook is not invoked when [WITHOUT ROWID] tables are modified. +** +** ^In the current implementation, the update hook +** is not invoked when conflicting rows are deleted because of an +** [ON CONFLICT | ON CONFLICT REPLACE] clause. ^Nor is the update hook +** invoked when rows are deleted using the [truncate optimization]. +** The exceptions defined in this paragraph might change in a future +** release of SQLite. +** +** The update hook implementation must not do anything that will modify +** the database connection that invoked the update hook. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the update hook. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** ^The sqlite3_update_hook(D,C,P) function +** returns the P argument from the previous call +** on the same [database connection] D, or NULL for +** the first call on D. +** +** See also the [sqlite3_commit_hook()], [sqlite3_rollback_hook()], +** and [sqlite3_preupdate_hook()] interfaces. +*/ +SQLITE_API void *sqlite3_update_hook( + sqlite3*, + void(*)(void *,int ,char const *,char const *,sqlite3_int64), + void* +); + +/* +** CAPI3REF: Enable Or Disable Shared Pager Cache +** +** ^(This routine enables or disables the sharing of the database cache +** and schema data structures between [database connection | connections] +** to the same database. Sharing is enabled if the argument is true +** and disabled if the argument is false.)^ +** +** ^Cache sharing is enabled and disabled for an entire process. +** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). +** In prior versions of SQLite, +** sharing was enabled or disabled for each thread separately. +** +** ^(The cache sharing mode set by this interface effects all subsequent +** calls to [sqlite3_open()], [sqlite3_open_v2()], and [sqlite3_open16()]. +** Existing database connections continue to use the sharing mode +** that was in effect at the time they were opened.)^ +** +** ^(This routine returns [SQLITE_OK] if shared cache was enabled or disabled +** successfully. An [error code] is returned otherwise.)^ +** +** ^Shared cache is disabled by default. It is recommended that it stay +** that way. In other words, do not use this routine. This interface +** continues to be provided for historical compatibility, but its use is +** discouraged. Any use of shared cache is discouraged. If shared cache +** must be used, it is recommended that shared cache only be enabled for +** individual database connections using the [sqlite3_open_v2()] interface +** with the [SQLITE_OPEN_SHAREDCACHE] flag. +** +** Note: This method is disabled on MacOS X 10.7 and iOS version 5.0 +** and will always return SQLITE_MISUSE. On those systems, +** shared cache mode should be enabled per-database connection via +** [sqlite3_open_v2()] with [SQLITE_OPEN_SHAREDCACHE]. +** +** This interface is threadsafe on processors where writing a +** 32-bit integer is atomic. +** +** See Also: [SQLite Shared-Cache Mode] +*/ +SQLITE_API int sqlite3_enable_shared_cache(int); + +/* +** CAPI3REF: Attempt To Free Heap Memory +** +** ^The sqlite3_release_memory() interface attempts to free N bytes +** of heap memory by deallocating non-essential memory allocations +** held by the database library. Memory used to cache database +** pages to improve performance is an example of non-essential memory. +** ^sqlite3_release_memory() returns the number of bytes actually freed, +** which might be more or less than the amount requested. +** ^The sqlite3_release_memory() routine is a no-op returning zero +** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. +** +** See also: [sqlite3_db_release_memory()] +*/ +SQLITE_API int sqlite3_release_memory(int); + +/* +** CAPI3REF: Free Memory Used By A Database Connection +** METHOD: sqlite3 +** +** ^The sqlite3_db_release_memory(D) interface attempts to free as much heap +** memory as possible from database connection D. Unlike the +** [sqlite3_release_memory()] interface, this interface is in effect even +** when the [SQLITE_ENABLE_MEMORY_MANAGEMENT] compile-time option is +** omitted. +** +** See also: [sqlite3_release_memory()] +*/ +SQLITE_API int sqlite3_db_release_memory(sqlite3*); + +/* +** CAPI3REF: Impose A Limit On Heap Size +** +** These interfaces impose limits on the amount of heap memory that will be +** by all database connections within a single process. +** +** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the +** soft limit on the amount of heap memory that may be allocated by SQLite. +** ^SQLite strives to keep heap memory utilization below the soft heap +** limit by reducing the number of pages held in the page cache +** as heap memory usages approaches the limit. +** ^The soft heap limit is "soft" because even though SQLite strives to stay +** below the limit, it will exceed the limit rather than generate +** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** is advisory only. +** +** ^The sqlite3_hard_heap_limit64(N) interface sets a hard upper bound of +** N bytes on the amount of memory that will be allocated. ^The +** sqlite3_hard_heap_limit64(N) interface is similar to +** sqlite3_soft_heap_limit64(N) except that memory allocations will fail +** when the hard heap limit is reached. +** +** ^The return value from both sqlite3_soft_heap_limit64() and +** sqlite3_hard_heap_limit64() is the size of +** the heap limit prior to the call, or negative in the case of an +** error. ^If the argument N is negative +** then no change is made to the heap limit. Hence, the current +** size of heap limits can be determined by invoking +** sqlite3_soft_heap_limit64(-1) or sqlite3_hard_heap_limit(-1). +** +** ^Setting the heap limits to zero disables the heap limiter mechanism. +** +** ^The soft heap limit may not be greater than the hard heap limit. +** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) +** is invoked with a value of N that is greater than the hard heap limit, +** the the soft heap limit is set to the value of the hard heap limit. +** ^The soft heap limit is automatically enabled whenever the hard heap +** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and +** the soft heap limit is outside the range of 1..N, then the soft heap +** limit is set to N. ^Invoking sqlite3_soft_heap_limit64(0) when the +** hard heap limit is enabled makes the soft heap limit equal to the +** hard heap limit. +** +** The memory allocation limits can also be adjusted using +** [PRAGMA soft_heap_limit] and [PRAGMA hard_heap_limit]. +** +** ^(The heap limits are not enforced in the current implementation +** if one or more of following conditions are true: +** +**
      +**
    • The limit value is set to zero. +**
    • Memory accounting is disabled using a combination of the +** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and +** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. +**
    • An alternative page cache implementation is specified using +** [sqlite3_config]([SQLITE_CONFIG_PCACHE2],...). +**
    • The page cache allocates from its own memory pool supplied +** by [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],...) rather than +** from the heap. +**
    )^ +** +** The circumstances under which SQLite will enforce the heap limits may +** changes in future releases of SQLite. +*/ +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); +SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N); + +/* +** CAPI3REF: Deprecated Soft Heap Limit Interface +** DEPRECATED +** +** This is a deprecated version of the [sqlite3_soft_heap_limit64()] +** interface. This routine is provided for historical compatibility +** only. All new applications should use the +** [sqlite3_soft_heap_limit64()] interface rather than this one. +*/ +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); + + +/* +** CAPI3REF: Extract Metadata About A Column Of A Table +** METHOD: sqlite3 +** +** ^(The sqlite3_table_column_metadata(X,D,T,C,....) routine returns +** information about column C of table T in database D +** on [database connection] X.)^ ^The sqlite3_table_column_metadata() +** interface returns SQLITE_OK and fills in the non-NULL pointers in +** the final five arguments with appropriate values if the specified +** column exists. ^The sqlite3_table_column_metadata() interface returns +** SQLITE_ERROR if the specified column does not exist. +** ^If the column-name parameter to sqlite3_table_column_metadata() is a +** NULL pointer, then this routine simply checks for the existence of the +** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it +** does not. If the table name parameter T in a call to +** sqlite3_table_column_metadata(X,D,T,C,...) is NULL then the result is +** undefined behavior. +** +** ^The column is identified by the second, third and fourth parameters to +** this function. ^(The second parameter is either the name of the database +** (i.e. "main", "temp", or an attached database) containing the specified +** table or NULL.)^ ^If it is NULL, then all attached databases are searched +** for the table using the same algorithm used by the database engine to +** resolve unqualified table references. +** +** ^The third and fourth parameters to this function are the table and column +** name of the desired column, respectively. +** +** ^Metadata is returned by writing to the memory locations passed as the 5th +** and subsequent parameters to this function. ^Any of these arguments may be +** NULL, in which case the corresponding element of metadata is omitted. +** +** ^(
    +** +**
    Parameter Output
    Type     		Description +** +**
    5th const char* Data type +**
    6th const char* Name of default collation sequence +**
    7th int True if column has a NOT NULL constraint +**
    8th int True if column is part of the PRIMARY KEY +**
    9th int True if column is [AUTOINCREMENT] +**
    +**
    )^ +** +** ^The memory pointed to by the character pointers returned for the +** declaration type and collation sequence is valid until the next +** call to any SQLite API function. +** +** ^If the specified table is actually a view, an [error code] is returned. +** +** ^If the specified column is "rowid", "oid" or "_rowid_" and the table +** is not a [WITHOUT ROWID] table and an +** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output +** parameters are set for the explicitly declared column. ^(If there is no +** [INTEGER PRIMARY KEY] column, then the outputs +** for the [rowid] are set as follows: +** +** 
    +**     data type: "INTEGER"
+**     collation sequence: "BINARY"
+**     not null: 0
+**     primary key: 1
+**     auto increment: 0
+**
    )^ +** +** ^This function causes all database schemas to be read from disk and +** parsed, if that has not already been done, and returns an error if +** any errors are encountered while loading the schema. +*/ +SQLITE_API int sqlite3_table_column_metadata( + sqlite3 *db, /* Connection handle */ + const char *zDbName, /* Database name or NULL */ + const char *zTableName, /* Table name */ + const char *zColumnName, /* Column name */ + char const **pzDataType, /* OUTPUT: Declared data type */ + char const **pzCollSeq, /* OUTPUT: Collation sequence name */ + int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ + int *pPrimaryKey, /* OUTPUT: True if column part of PK */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ +); + +/* +** CAPI3REF: Load An Extension +** METHOD: sqlite3 +** +** ^This interface loads an SQLite extension library from the named file. +** +** ^The sqlite3_load_extension() interface attempts to load an +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. +** +** ^The entry point is zProc. +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ +** ^The sqlite3_load_extension() interface returns +** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. +** ^If an error occurs and pzErrMsg is not 0, then the +** [sqlite3_load_extension()] interface shall attempt to +** fill *pzErrMsg with error message text stored in memory +** obtained from [sqlite3_malloc()]. The calling function +** should free this memory by calling [sqlite3_free()]. +** +** ^Extension loading must be enabled using +** [sqlite3_enable_load_extension()] or +** [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],1,NULL) +** prior to calling this API, +** otherwise an error will be returned. +** +** Security warning: It is recommended that the +** [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method be used to enable only this +** interface. The use of the [sqlite3_enable_load_extension()] interface +** should be avoided. This will keep the SQL function [load_extension()] +** disabled and prevent SQL injections from giving attackers +** access to extension loading capabilities. +** +** See also the [load_extension() SQL function]. +*/ +SQLITE_API int sqlite3_load_extension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Derived from zFile if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ +); + +/* +** CAPI3REF: Enable Or Disable Extension Loading +** METHOD: sqlite3 +** +** ^So as not to open security holes in older applications that are +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API +** is provided to turn the [sqlite3_load_extension()] mechanism on and off. +** +** ^Extension loading is off by default. +** ^Call the sqlite3_enable_load_extension() routine with onoff==1 +** to turn extension loading on and call it with onoff==0 to turn +** it back off again. +** +** ^This interface enables or disables both the C-API +** [sqlite3_load_extension()] and the SQL function [load_extension()]. +** ^(Use [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],..) +** to enable or disable only the C-API.)^ +** +** Security warning: It is recommended that extension loading +** be enabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method +** rather than this interface, so the [load_extension()] SQL function +** remains disabled. This will prevent SQL injections from giving attackers +** access to extension loading capabilities. +*/ +SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); + +/* +** CAPI3REF: Automatically Load Statically Linked Extensions +** +** ^This interface causes the xEntryPoint() function to be invoked for +** each new [database connection] that is created. The idea here is that +** xEntryPoint() is the entry point for a statically linked [SQLite extension] +** that is to be automatically loaded into all new database connections. +** +** ^(Even though the function prototype shows that xEntryPoint() takes +** no arguments and returns void, SQLite invokes xEntryPoint() with three +** arguments and expects an integer result as if the signature of the +** entry point where as follows: +** +** 
    +**    int xEntryPoint(
+**      sqlite3 *db,
+**      const char **pzErrMsg,
+**      const struct sqlite3_api_routines *pThunk
+**    );
+**
    )^ +** +** If the xEntryPoint routine encounters an error, it should make *pzErrMsg +** point to an appropriate error message (obtained from [sqlite3_mprintf()]) +** and return an appropriate [error code]. ^SQLite ensures that *pzErrMsg +** is NULL before calling the xEntryPoint(). ^SQLite will invoke +** [sqlite3_free()] on *pzErrMsg after xEntryPoint() returns. ^If any +** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail. +** +** ^Calling sqlite3_auto_extension(X) with an entry point X that is already +** on the list of automatic extensions is a harmless no-op. ^No entry point +** will be called more than once for each database connection that is opened. +** +** See also: [sqlite3_reset_auto_extension()] +** and [sqlite3_cancel_auto_extension()] +*/ +SQLITE_API int sqlite3_auto_extension(void(*xEntryPoint)(void)); + +/* +** CAPI3REF: Cancel Automatic Extension Loading +** +** ^The [sqlite3_cancel_auto_extension(X)] interface unregisters the +** initialization routine X that was registered using a prior call to +** [sqlite3_auto_extension(X)]. ^The [sqlite3_cancel_auto_extension(X)] +** routine returns 1 if initialization routine X was successfully +** unregistered and it returns 0 if X was not on the list of initialization +** routines. +*/ +SQLITE_API int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void)); + +/* +** CAPI3REF: Reset Automatic Extension Loading +** +** ^This interface disables all automatic extensions previously +** registered using [sqlite3_auto_extension()]. +*/ +SQLITE_API void sqlite3_reset_auto_extension(void); + +/* +** The interface to the virtual-table mechanism is currently considered +** to be experimental. The interface might change in incompatible ways. +** If this is a problem for you, do not use the interface at this time. +** +** When the virtual-table mechanism stabilizes, we will declare the +** interface fixed, support it indefinitely, and remove this comment. +*/ + +/* +** Structures used by the virtual table interface +*/ +typedef struct sqlite3_vtab sqlite3_vtab; +typedef struct sqlite3_index_info sqlite3_index_info; +typedef struct sqlite3_vtab_cursor sqlite3_vtab_cursor; +typedef struct sqlite3_module sqlite3_module; + +/* +** CAPI3REF: Virtual Table Object +** KEYWORDS: sqlite3_module {virtual table module} +** +** This structure, sometimes called a "virtual table module", +** defines the implementation of a [virtual table]. +** This structure consists mostly of methods for the module. +** +** ^A virtual table module is created by filling in a persistent +** instance of this structure and passing a pointer to that instance +** to [sqlite3_create_module()] or [sqlite3_create_module_v2()]. +** ^The registration remains valid until it is replaced by a different +** module or until the [database connection] closes. The content +** of this structure must not change while it is registered with +** any database connection. +*/ +struct sqlite3_module { + int iVersion; + int (*xCreate)(sqlite3*, void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, char**); + int (*xConnect)(sqlite3*, void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, char**); + int (*xBestIndex)(sqlite3_vtab *pVTab, sqlite3_index_info*); + int (*xDisconnect)(sqlite3_vtab *pVTab); + int (*xDestroy)(sqlite3_vtab *pVTab); + int (*xOpen)(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor); + int (*xClose)(sqlite3_vtab_cursor*); + int (*xFilter)(sqlite3_vtab_cursor*, int idxNum, const char *idxStr, + int argc, sqlite3_value **argv); + int (*xNext)(sqlite3_vtab_cursor*); + int (*xEof)(sqlite3_vtab_cursor*); + int (*xColumn)(sqlite3_vtab_cursor*, sqlite3_context*, int); + int (*xRowid)(sqlite3_vtab_cursor*, sqlite3_int64 *pRowid); + int (*xUpdate)(sqlite3_vtab *, int, sqlite3_value **, sqlite3_int64 *); + int (*xBegin)(sqlite3_vtab *pVTab); + int (*xSync)(sqlite3_vtab *pVTab); + int (*xCommit)(sqlite3_vtab *pVTab); + int (*xRollback)(sqlite3_vtab *pVTab); + int (*xFindFunction)(sqlite3_vtab *pVtab, int nArg, const char *zName, + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), + void **ppArg); + int (*xRename)(sqlite3_vtab *pVtab, const char *zNew); + /* The methods above are in version 1 of the sqlite_module object. Those + ** below are for version 2 and greater. */ + int (*xSavepoint)(sqlite3_vtab *pVTab, int); + int (*xRelease)(sqlite3_vtab *pVTab, int); + int (*xRollbackTo)(sqlite3_vtab *pVTab, int); + /* The methods above are in versions 1 and 2 of the sqlite_module object. + ** Those below are for version 3 and greater. */ + int (*xShadowName)(const char*); +}; + +/* +** CAPI3REF: Virtual Table Indexing Information +** KEYWORDS: sqlite3_index_info +** +** The sqlite3_index_info structure and its substructures is used as part +** of the [virtual table] interface to +** pass information into and receive the reply from the [xBestIndex] +** method of a [virtual table module]. The fields under **Inputs** are the +** inputs to xBestIndex and are read-only. xBestIndex inserts its +** results into the **Outputs** fields. +** +** ^(The aConstraint[] array records WHERE clause constraints of the form: +** +**
    column OP expr
    +** +** where OP is =, <, <=, >, or >=.)^ ^(The particular operator is +** stored in aConstraint[].op using one of the +** [SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_ values].)^ +** ^(The index of the column is stored in +** aConstraint[].iColumn.)^ ^(aConstraint[].usable is TRUE if the +** expr on the right-hand side can be evaluated (and thus the constraint +** is usable) and false if it cannot.)^ +** +** ^The optimizer automatically inverts terms of the form "expr OP column" +** and makes other simplifications to the WHERE clause in an attempt to +** get as many WHERE clause terms into the form shown above as possible. +** ^The aConstraint[] array only reports WHERE clause terms that are +** relevant to the particular virtual table being queried. +** +** ^Information about the ORDER BY clause is stored in aOrderBy[]. +** ^Each term of aOrderBy records a column of the ORDER BY clause. +** +** The colUsed field indicates which columns of the virtual table may be +** required by the current scan. Virtual table columns are numbered from +** zero in the order in which they appear within the CREATE TABLE statement +** passed to sqlite3_declare_vtab(). For the first 63 columns (columns 0-62), +** the corresponding bit is set within the colUsed mask if the column may be +** required by SQLite. If the table has at least 64 columns and any column +** to the right of the first 63 is required, then bit 63 of colUsed is also +** set. In other words, column iCol may be required if the expression +** (colUsed & ((sqlite3_uint64)1 << (iCol>=63 ? 63 : iCol))) evaluates to +** non-zero. +** +** The [xBestIndex] method must fill aConstraintUsage[] with information +** about what parameters to pass to xFilter. ^If argvIndex>0 then +** the right-hand side of the corresponding aConstraint[] is evaluated +** and becomes the argvIndex-th entry in argv. ^(If aConstraintUsage[].omit +** is true, then the constraint is assumed to be fully handled by the +** virtual table and might not be checked again by the byte code.)^ ^(The +** aConstraintUsage[].omit flag is an optimization hint. When the omit flag +** is left in its default setting of false, the constraint will always be +** checked separately in byte code. If the omit flag is change to true, then +** the constraint may or may not be checked in byte code. In other words, +** when the omit flag is true there is no guarantee that the constraint will +** not be checked again using byte code.)^ +** +** ^The idxNum and idxPtr values are recorded and passed into the +** [xFilter] method. +** ^[sqlite3_free()] is used to free idxPtr if and only if +** needToFreeIdxPtr is true. +** +** ^The orderByConsumed means that output from [xFilter]/[xNext] will occur in +** the correct order to satisfy the ORDER BY clause so that no separate +** sorting step is required. +** +** ^The estimatedCost value is an estimate of the cost of a particular +** strategy. A cost of N indicates that the cost of the strategy is similar +** to a linear scan of an SQLite table with N rows. A cost of log(N) +** indicates that the expense of the operation is similar to that of a +** binary search on a unique indexed field of an SQLite table with N rows. +** +** ^The estimatedRows value is an estimate of the number of rows that +** will be returned by the strategy. +** +** The xBestIndex method may optionally populate the idxFlags field with a +** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag - +** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite +** assumes that the strategy may visit at most one row. +** +** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then +** SQLite also assumes that if a call to the xUpdate() method is made as +** part of the same statement to delete or update a virtual table row and the +** implementation returns SQLITE_CONSTRAINT, then there is no need to rollback +** any database changes. In other words, if the xUpdate() returns +** SQLITE_CONSTRAINT, the database contents must be exactly as they were +** before xUpdate was called. By contrast, if SQLITE_INDEX_SCAN_UNIQUE is not +** set and xUpdate returns SQLITE_CONSTRAINT, any database changes made by +** the xUpdate method are automatically rolled back by SQLite. +** +** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info +** structure for SQLite [version 3.8.2] ([dateof:3.8.2]). +** If a virtual table extension is +** used with an SQLite version earlier than 3.8.2, the results of attempting +** to read or write the estimatedRows field are undefined (but are likely +** to include crashing the application). The estimatedRows field should +** therefore only be used if [sqlite3_libversion_number()] returns a +** value greater than or equal to 3008002. Similarly, the idxFlags field +** was added for [version 3.9.0] ([dateof:3.9.0]). +** It may therefore only be used if +** sqlite3_libversion_number() returns a value greater than or equal to +** 3009000. +*/ +struct sqlite3_index_info { + /* Inputs */ + int nConstraint; /* Number of entries in aConstraint */ + struct sqlite3_index_constraint { + int iColumn; /* Column constrained. -1 for ROWID */ + unsigned char op; /* Constraint operator */ + unsigned char usable; /* True if this constraint is usable */ + int iTermOffset; /* Used internally - xBestIndex should ignore */ + } *aConstraint; /* Table of WHERE clause constraints */ + int nOrderBy; /* Number of terms in the ORDER BY clause */ + struct sqlite3_index_orderby { + int iColumn; /* Column number */ + unsigned char desc; /* True for DESC. False for ASC. */ + } *aOrderBy; /* The ORDER BY clause */ + /* Outputs */ + struct sqlite3_index_constraint_usage { + int argvIndex; /* if >0, constraint is part of argv to xFilter */ + unsigned char omit; /* Do not code a test for this constraint */ + } *aConstraintUsage; + int idxNum; /* Number used to identify the index */ + char *idxStr; /* String, possibly obtained from sqlite3_malloc */ + int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */ + int orderByConsumed; /* True if output is already ordered */ + double estimatedCost; /* Estimated cost of using this index */ + /* Fields below are only available in SQLite 3.8.2 and later */ + sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ + /* Fields below are only available in SQLite 3.9.0 and later */ + int idxFlags; /* Mask of SQLITE_INDEX_SCAN_* flags */ + /* Fields below are only available in SQLite 3.10.0 and later */ + sqlite3_uint64 colUsed; /* Input: Mask of columns used by statement */ +}; + +/* +** CAPI3REF: Virtual Table Scan Flags +** +** Virtual table implementations are allowed to set the +** [sqlite3_index_info].idxFlags field to some combination of +** these bits. +*/ +#define SQLITE_INDEX_SCAN_UNIQUE 1 /* Scan visits at most 1 row */ + +/* +** CAPI3REF: Virtual Table Constraint Operator Codes +** +** These macros define the allowed values for the +** [sqlite3_index_info].aConstraint[].op field. Each value represents +** an operator that is part of a constraint term in the wHERE clause of +** a query that uses a [virtual table]. +*/ +#define SQLITE_INDEX_CONSTRAINT_EQ 2 +#define SQLITE_INDEX_CONSTRAINT_GT 4 +#define SQLITE_INDEX_CONSTRAINT_LE 8 +#define SQLITE_INDEX_CONSTRAINT_LT 16 +#define SQLITE_INDEX_CONSTRAINT_GE 32 +#define SQLITE_INDEX_CONSTRAINT_MATCH 64 +#define SQLITE_INDEX_CONSTRAINT_LIKE 65 +#define SQLITE_INDEX_CONSTRAINT_GLOB 66 +#define SQLITE_INDEX_CONSTRAINT_REGEXP 67 +#define SQLITE_INDEX_CONSTRAINT_NE 68 +#define SQLITE_INDEX_CONSTRAINT_ISNOT 69 +#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70 +#define SQLITE_INDEX_CONSTRAINT_ISNULL 71 +#define SQLITE_INDEX_CONSTRAINT_IS 72 +#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150 + +/* +** CAPI3REF: Register A Virtual Table Implementation +** METHOD: sqlite3 +** +** ^These routines are used to register a new [virtual table module] name. +** ^Module names must be registered before +** creating a new [virtual table] using the module and before using a +** preexisting [virtual table] for the module. +** +** ^The module name is registered on the [database connection] specified +** by the first parameter. ^The name of the module is given by the +** second parameter. ^The third parameter is a pointer to +** the implementation of the [virtual table module]. ^The fourth +** parameter is an arbitrary client data pointer that is passed through +** into the [xCreate] and [xConnect] methods of the virtual table module +** when a new virtual table is be being created or reinitialized. +** +** ^The sqlite3_create_module_v2() interface has a fifth parameter which +** is a pointer to a destructor for the pClientData. ^SQLite will +** invoke the destructor function (if it is not NULL) when SQLite +** no longer needs the pClientData pointer. ^The destructor will also +** be invoked if the call to sqlite3_create_module_v2() fails. +** ^The sqlite3_create_module() +** interface is equivalent to sqlite3_create_module_v2() with a NULL +** destructor. +** +** ^If the third parameter (the pointer to the sqlite3_module object) is +** NULL then no new module is create and any existing modules with the +** same name are dropped. +** +** See also: [sqlite3_drop_modules()] +*/ +SQLITE_API int sqlite3_create_module( + sqlite3 *db, /* SQLite connection to register module with */ + const char *zName, /* Name of the module */ + const sqlite3_module *p, /* Methods for the module */ + void *pClientData /* Client data for xCreate/xConnect */ +); +SQLITE_API int sqlite3_create_module_v2( + sqlite3 *db, /* SQLite connection to register module with */ + const char *zName, /* Name of the module */ + const sqlite3_module *p, /* Methods for the module */ + void *pClientData, /* Client data for xCreate/xConnect */ + void(*xDestroy)(void*) /* Module destructor function */ +); + +/* +** CAPI3REF: Remove Unnecessary Virtual Table Implementations +** METHOD: sqlite3 +** +** ^The sqlite3_drop_modules(D,L) interface removes all virtual +** table modules from database connection D except those named on list L. +** The L parameter must be either NULL or a pointer to an array of pointers +** to strings where the array is terminated by a single NULL pointer. +** ^If the L parameter is NULL, then all virtual table modules are removed. +** +** See also: [sqlite3_create_module()] +*/ +SQLITE_API int sqlite3_drop_modules( + sqlite3 *db, /* Remove modules from this connection */ + const char **azKeep /* Except, do not remove the ones named here */ +); + +/* +** CAPI3REF: Virtual Table Instance Object +** KEYWORDS: sqlite3_vtab +** +** Every [virtual table module] implementation uses a subclass +** of this object to describe a particular instance +** of the [virtual table]. Each subclass will +** be tailored to the specific needs of the module implementation. +** The purpose of this superclass is to define certain fields that are +** common to all module implementations. +** +** ^Virtual tables methods can set an error message by assigning a +** string obtained from [sqlite3_mprintf()] to zErrMsg. The method should +** take care that any prior string is freed by a call to [sqlite3_free()] +** prior to assigning a new string to zErrMsg. ^After the error message +** is delivered up to the client application, the string will be automatically +** freed by sqlite3_free() and the zErrMsg field will be zeroed. +*/ +struct sqlite3_vtab { + const sqlite3_module *pModule; /* The module for this virtual table */ + int nRef; /* Number of open cursors */ + char *zErrMsg; /* Error message from sqlite3_mprintf() */ + /* Virtual table implementations will typically add additional fields */ +}; + +/* +** CAPI3REF: Virtual Table Cursor Object +** KEYWORDS: sqlite3_vtab_cursor {virtual table cursor} +** +** Every [virtual table module] implementation uses a subclass of the +** following structure to describe cursors that point into the +** [virtual table] and are used +** to loop through the virtual table. Cursors are created using the +** [sqlite3_module.xOpen | xOpen] method of the module and are destroyed +** by the [sqlite3_module.xClose | xClose] method. Cursors are used +** by the [xFilter], [xNext], [xEof], [xColumn], and [xRowid] methods +** of the module. Each module implementation will define +** the content of a cursor structure to suit its own needs. +** +** This superclass exists in order to define fields of the cursor that +** are common to all implementations. +*/ +struct sqlite3_vtab_cursor { + sqlite3_vtab *pVtab; /* Virtual table of this cursor */ + /* Virtual table implementations will typically add additional fields */ +}; + +/* +** CAPI3REF: Declare The Schema Of A Virtual Table +** +** ^The [xCreate] and [xConnect] methods of a +** [virtual table module] call this interface +** to declare the format (the names and datatypes of the columns) of +** the virtual tables they implement. +*/ +SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); + +/* +** CAPI3REF: Overload A Function For A Virtual Table +** METHOD: sqlite3 +** +** ^(Virtual tables can provide alternative implementations of functions +** using the [xFindFunction] method of the [virtual table module]. +** But global versions of those functions +** must exist in order to be overloaded.)^ +** +** ^(This API makes sure a global version of a function with a particular +** name and number of parameters exists. If no such function exists +** before this API is called, a new function is created.)^ ^The implementation +** of the new function always causes an exception to be thrown. So +** the new function is not good for anything by itself. Its only +** purpose is to be a placeholder function that can be overloaded +** by a [virtual table]. +*/ +SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); + +/* +** The interface to the virtual-table mechanism defined above (back up +** to a comment remarkably similar to this one) is currently considered +** to be experimental. The interface might change in incompatible ways. +** If this is a problem for you, do not use the interface at this time. +** +** When the virtual-table mechanism stabilizes, we will declare the +** interface fixed, support it indefinitely, and remove this comment. +*/ + +/* +** CAPI3REF: A Handle To An Open BLOB +** KEYWORDS: {BLOB handle} {BLOB handles} +** +** An instance of this object represents an open BLOB on which +** [sqlite3_blob_open | incremental BLOB I/O] can be performed. +** ^Objects of this type are created by [sqlite3_blob_open()] +** and destroyed by [sqlite3_blob_close()]. +** ^The [sqlite3_blob_read()] and [sqlite3_blob_write()] interfaces +** can be used to read or write small subsections of the BLOB. +** ^The [sqlite3_blob_bytes()] interface returns the size of the BLOB in bytes. +*/ +typedef struct sqlite3_blob sqlite3_blob; + +/* +** CAPI3REF: Open A BLOB For Incremental I/O +** METHOD: sqlite3 +** CONSTRUCTOR: sqlite3_blob +** +** ^(This interfaces opens a [BLOB handle | handle] to the BLOB located +** in row iRow, column zColumn, table zTable in database zDb; +** in other words, the same BLOB that would be selected by: +** +** 
    +**     SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow;
+**
    )^ +** +** ^(Parameter zDb is not the filename that contains the database, but +** rather the symbolic name of the database. For attached databases, this is +** the name that appears after the AS keyword in the [ATTACH] statement. +** For the main database file, the database name is "main". For TEMP +** tables, the database name is "temp".)^ +** +** ^If the flags parameter is non-zero, then the BLOB is opened for read +** and write access. ^If the flags parameter is zero, the BLOB is opened for +** read-only access. +** +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is stored +** in *ppBlob. Otherwise an [error code] is returned and, unless the error +** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided +** the API is not misused, it is always safe to call [sqlite3_blob_close()] +** on *ppBlob after this function it returns. +** +** This function fails with SQLITE_ERROR if any of the following are true: +**
      +**
    • ^(Database zDb does not exist)^, +**
    • ^(Table zTable does not exist within database zDb)^, +**
    • ^(Table zTable is a WITHOUT ROWID table)^, +**
    • ^(Column zColumn does not exist)^, +**
    • ^(Row iRow is not present in the table)^, +**
    • ^(The specified column of row iRow contains a value that is not +** a TEXT or BLOB value)^, +**
    • ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE +** constraint and the blob is being opened for read/write access)^, +**
    • ^([foreign key constraints | Foreign key constraints] are enabled, +** column zColumn is part of a [child key] definition and the blob is +** being opened for read/write access)^. +**
    +** +** ^Unless it returns SQLITE_MISUSE, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. +** +** A BLOB referenced by sqlite3_blob_open() may be read using the +** [sqlite3_blob_read()] interface and modified by using +** [sqlite3_blob_write()]. The [BLOB handle] can be moved to a +** different row of the same table using the [sqlite3_blob_reopen()] +** interface. However, the column, table, or database of a [BLOB handle] +** cannot be changed after the [BLOB handle] is opened. +** +** ^(If the row that a BLOB handle points to is modified by an +** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects +** then the BLOB handle is marked as "expired". +** This is true if any column of the row is changed, even a column +** other than the one the BLOB handle is open on.)^ +** ^Calls to [sqlite3_blob_read()] and [sqlite3_blob_write()] for +** an expired BLOB handle fail with a return code of [SQLITE_ABORT]. +** ^(Changes written into a BLOB prior to the BLOB expiring are not +** rolled back by the expiration of the BLOB. Such changes will eventually +** commit if the transaction continues to completion.)^ +** +** ^Use the [sqlite3_blob_bytes()] interface to determine the size of +** the opened blob. ^The size of a blob may not be changed by this +** interface. Use the [UPDATE] SQL command to change the size of a +** blob. +** +** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces +** and the built-in [zeroblob] SQL function may be used to create a +** zero-filled blob to read or write using the incremental-blob interface. +** +** To avoid a resource leak, every open [BLOB handle] should eventually +** be released by a call to [sqlite3_blob_close()]. +** +** See also: [sqlite3_blob_close()], +** [sqlite3_blob_reopen()], [sqlite3_blob_read()], +** [sqlite3_blob_bytes()], [sqlite3_blob_write()]. +*/ +SQLITE_API int sqlite3_blob_open( + sqlite3*, + const char *zDb, + const char *zTable, + const char *zColumn, + sqlite3_int64 iRow, + int flags, + sqlite3_blob **ppBlob +); + +/* +** CAPI3REF: Move a BLOB Handle to a New Row +** METHOD: sqlite3_blob +** +** ^This function is used to move an existing [BLOB handle] so that it points +** to a different row of the same database table. ^The new row is identified +** by the rowid value passed as the second argument. Only the row can be +** changed. ^The database, table and column on which the blob handle is open +** remain the same. Moving an existing [BLOB handle] to a new row is +** faster than closing the existing handle and opening a new one. +** +** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - +** it must exist and there must be either a blob or text value stored in +** the nominated column.)^ ^If the new row is not present in the table, or if +** it does not contain a blob or text value, or if another error occurs, an +** SQLite error code is returned and the blob handle is considered aborted. +** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or +** [sqlite3_blob_reopen()] on an aborted blob handle immediately return +** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle +** always returns zero. +** +** ^This function sets the database handle error code and message. +*/ +SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); + +/* +** CAPI3REF: Close A BLOB Handle +** DESTRUCTOR: sqlite3_blob +** +** ^This function closes an open [BLOB handle]. ^(The BLOB handle is closed +** unconditionally. Even if this routine returns an error code, the +** handle is still closed.)^ +** +** ^If the blob handle being closed was opened for read-write access, and if +** the database is in auto-commit mode and there are no other open read-write +** blob handles or active write statements, the current transaction is +** committed. ^If an error occurs while committing the transaction, an error +** code is returned and the transaction rolled back. +** +** Calling this function with an argument that is not a NULL pointer or an +** open blob handle results in undefined behaviour. ^Calling this routine +** with a null pointer (such as would be returned by a failed call to +** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function +** is passed a valid open blob handle, the values returned by the +** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning. +*/ +SQLITE_API int sqlite3_blob_close(sqlite3_blob *); + +/* +** CAPI3REF: Return The Size Of An Open BLOB +** METHOD: sqlite3_blob +** +** ^Returns the size in bytes of the BLOB accessible via the +** successfully opened [BLOB handle] in its only argument. ^The +** incremental blob I/O routines can only read or overwriting existing +** blob content; they cannot change the size of a blob. +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +*/ +SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *); + +/* +** CAPI3REF: Read Data From A BLOB Incrementally +** METHOD: sqlite3_blob +** +** ^(This function is used to read data from an open [BLOB handle] into a +** caller-supplied buffer. N bytes of data are copied into buffer Z +** from the open BLOB, starting at offset iOffset.)^ +** +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is read. ^If N or iOffset is +** less than zero, [SQLITE_ERROR] is returned and no data is read. +** ^The size of the blob (and hence the maximum value of N+iOffset) +** can be determined using the [sqlite3_blob_bytes()] interface. +** +** ^An attempt to read from an expired [BLOB handle] fails with an +** error code of [SQLITE_ABORT]. +** +** ^(On success, sqlite3_blob_read() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +** +** See also: [sqlite3_blob_write()]. +*/ +SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); + +/* +** CAPI3REF: Write Data Into A BLOB Incrementally +** METHOD: sqlite3_blob +** +** ^(This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. N bytes of data are copied from the buffer Z +** into the open BLOB, starting at offset iOffset.)^ +** +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** ^Unless SQLITE_MISUSE is returned, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. +** +** ^If the [BLOB handle] passed as the first argument was not opened for +** writing (the flags parameter to [sqlite3_blob_open()] was zero), +** this function returns [SQLITE_READONLY]. +** +** This function may only modify the contents of the BLOB; it is +** not possible to increase the size of a BLOB using this API. +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is written. The size of the +** BLOB (and hence the maximum value of N+iOffset) can be determined +** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less +** than zero [SQLITE_ERROR] is returned and no data is written. +** +** ^An attempt to write to an expired [BLOB handle] fails with an +** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred +** before the [BLOB handle] expired are not rolled back by the +** expiration of the handle, though of course those changes might +** have been overwritten by the statement that expired the BLOB handle +** or by other independent statements. +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +** +** See also: [sqlite3_blob_read()]. +*/ +SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); + +/* +** CAPI3REF: Virtual File System Objects +** +** A virtual filesystem (VFS) is an [sqlite3_vfs] object +** that SQLite uses to interact +** with the underlying operating system. Most SQLite builds come with a +** single default VFS that is appropriate for the host computer. +** New VFSes can be registered and existing VFSes can be unregistered. +** The following interfaces are provided. +** +** ^The sqlite3_vfs_find() interface returns a pointer to a VFS given its name. +** ^Names are case sensitive. +** ^Names are zero-terminated UTF-8 strings. +** ^If there is no match, a NULL pointer is returned. +** ^If zVfsName is NULL then the default VFS is returned. +** +** ^New VFSes are registered with sqlite3_vfs_register(). +** ^Each new VFS becomes the default VFS if the makeDflt flag is set. +** ^The same VFS can be registered multiple times without injury. +** ^To make an existing VFS into the default VFS, register it again +** with the makeDflt flag set. If two different VFSes with the +** same name are registered, the behavior is undefined. If a +** VFS is registered with a name that is NULL or an empty string, +** then the behavior is undefined. +** +** ^Unregister a VFS with the sqlite3_vfs_unregister() interface. +** ^(If the default VFS is unregistered, another VFS is chosen as +** the default. The choice for the new VFS is arbitrary.)^ +*/ +SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName); +SQLITE_API int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); +SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); + +/* +** CAPI3REF: Mutexes +** +** The SQLite core uses these routines for thread +** synchronization. Though they are intended for internal +** use by SQLite, code that links against SQLite is +** permitted to use any of these routines. +** +** The SQLite source code contains multiple implementations +** of these mutex routines. An appropriate implementation +** is selected automatically at compile-time. The following +** implementations are available in the SQLite core: +** +**
      +**
    • SQLITE_MUTEX_PTHREADS +**
    • SQLITE_MUTEX_W32 +**
    • SQLITE_MUTEX_NOOP +**
    +** +** The SQLITE_MUTEX_NOOP implementation is a set of routines +** that does no real locking and is appropriate for use in +** a single-threaded application. The SQLITE_MUTEX_PTHREADS and +** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix +** and Windows. +** +** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex +** implementation is included with the library. In this case the +** application must supply a custom mutex implementation using the +** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function +** before calling sqlite3_initialize() or any other public sqlite3_ +** function that calls sqlite3_initialize(). +** +** ^The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() +** routine returns NULL if it is unable to allocate the requested +** mutex. The argument to sqlite3_mutex_alloc() must one of these +** integer constants: +** +**
      +**
    • SQLITE_MUTEX_FAST +**
    • SQLITE_MUTEX_RECURSIVE +**
    • SQLITE_MUTEX_STATIC_MAIN +**
    • SQLITE_MUTEX_STATIC_MEM +**
    • SQLITE_MUTEX_STATIC_OPEN +**
    • SQLITE_MUTEX_STATIC_PRNG +**
    • SQLITE_MUTEX_STATIC_LRU +**
    • SQLITE_MUTEX_STATIC_PMEM +**
    • SQLITE_MUTEX_STATIC_APP1 +**
    • SQLITE_MUTEX_STATIC_APP2 +**
    • SQLITE_MUTEX_STATIC_APP3 +**
    • SQLITE_MUTEX_STATIC_VFS1 +**
    • SQLITE_MUTEX_STATIC_VFS2 +**
    • SQLITE_MUTEX_STATIC_VFS3 +**
    +** +** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) +** cause sqlite3_mutex_alloc() to create +** a new mutex. ^The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other +** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return +** a pointer to a static preexisting mutex. ^Nine static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. ^For the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +** +** ^The sqlite3_mutex_free() routine deallocates a previously +** allocated dynamic mutex. Attempting to deallocate a static +** mutex results in undefined behavior. +** +** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. ^If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] +** upon successful entry. ^(Mutexes created using +** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. +** In such cases, the +** mutex must be exited an equal number of times before another thread +** can enter.)^ If the same thread tries to enter any mutex other +** than an SQLITE_MUTEX_RECURSIVE more than once, the behavior is undefined. +** +** ^(Some systems (for example, Windows 95) do not support the operation +** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() +** will always return SQLITE_BUSY. The SQLite core only ever uses +** sqlite3_mutex_try() as an optimization so this is acceptable +** behavior.)^ +** +** ^The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered by the +** calling thread or is not currently allocated. +** +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or +** sqlite3_mutex_leave() is a NULL pointer, then all three routines +** behave as no-ops. +** +** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. +*/ +SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int); +SQLITE_API void sqlite3_mutex_free(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_try(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); + +/* +** CAPI3REF: Mutex Methods Object +** +** An instance of this structure defines the low-level routines +** used to allocate and use mutexes. +** +** Usually, the default mutex implementations provided by SQLite are +** sufficient, however the application has the option of substituting a custom +** implementation for specialized deployments or systems for which SQLite +** does not provide a suitable implementation. In this case, the application +** creates and populates an instance of this structure to pass +** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option. +** Additionally, an instance of this structure can be used as an +** output variable when querying the system for the current mutex +** implementation, using the [SQLITE_CONFIG_GETMUTEX] option. +** +** ^The xMutexInit method defined by this structure is invoked as +** part of system initialization by the sqlite3_initialize() function. +** ^The xMutexInit routine is called by SQLite exactly once for each +** effective call to [sqlite3_initialize()]. +** +** ^The xMutexEnd method defined by this structure is invoked as +** part of system shutdown by the sqlite3_shutdown() function. The +** implementation of this method is expected to release all outstanding +** resources obtained by the mutex methods implementation, especially +** those obtained by the xMutexInit method. ^The xMutexEnd() +** interface is invoked exactly once for each call to [sqlite3_shutdown()]. +** +** ^(The remaining seven methods defined by this structure (xMutexAlloc, +** xMutexFree, xMutexEnter, xMutexTry, xMutexLeave, xMutexHeld and +** xMutexNotheld) implement the following interfaces (respectively): +** +**
      +**
    • [sqlite3_mutex_alloc()]
      • +**
    • [sqlite3_mutex_free()]
      • +**
    • [sqlite3_mutex_enter()]
      • +**
    • [sqlite3_mutex_try()]
      • +**
    • [sqlite3_mutex_leave()]
      • +**
    • [sqlite3_mutex_held()]
      • +**
    • [sqlite3_mutex_notheld()]
      • +**
    )^ +** +** The only difference is that the public sqlite3_XXX functions enumerated +** above silently ignore any invocations that pass a NULL pointer instead +** of a valid mutex handle. The implementations of the methods defined +** by this structure are not required to handle this case. The results +** of passing a NULL pointer instead of a valid mutex handle are undefined +** (i.e. it is acceptable to provide an implementation that segfaults if +** it is passed a NULL pointer). +** +** The xMutexInit() method must be threadsafe. It must be harmless to +** invoke xMutexInit() multiple times within the same process and without +** intervening calls to xMutexEnd(). Second and subsequent calls to +** xMutexInit() must be no-ops. +** +** xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). Similarly, xMutexAlloc() must not use SQLite memory +** allocation for a static mutex. ^However xMutexAlloc() may use SQLite +** memory allocation for a fast or recursive mutex. +** +** ^SQLite will invoke the xMutexEnd() method when [sqlite3_shutdown()] is +** called, but only if the prior call to xMutexInit returned SQLITE_OK. +** If xMutexInit fails in any way, it is expected to clean up after itself +** prior to returning. +*/ +typedef struct sqlite3_mutex_methods sqlite3_mutex_methods; +struct sqlite3_mutex_methods { + int (*xMutexInit)(void); + int (*xMutexEnd)(void); + sqlite3_mutex *(*xMutexAlloc)(int); + void (*xMutexFree)(sqlite3_mutex *); + void (*xMutexEnter)(sqlite3_mutex *); + int (*xMutexTry)(sqlite3_mutex *); + void (*xMutexLeave)(sqlite3_mutex *); + int (*xMutexHeld)(sqlite3_mutex *); + int (*xMutexNotheld)(sqlite3_mutex *); +}; + +/* +** CAPI3REF: Mutex Verification Routines +** +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines +** are intended for use inside assert() statements. The SQLite core +** never uses these routines except inside an assert() and applications +** are advised to follow the lead of the core. The SQLite core only +** provides implementations for these routines when it is compiled +** with the SQLITE_DEBUG flag. External mutex implementations +** are only required to provide these routines if SQLITE_DEBUG is +** defined and if NDEBUG is not defined. +** +** These routines should return true if the mutex in their argument +** is held or not held, respectively, by the calling thread. +** +** The implementation is not required to provide versions of these +** routines that actually work. If the implementation does not provide working +** versions of these routines, it should at least provide stubs that always +** return true so that one does not get spurious assertion failures. +** +** If the argument to sqlite3_mutex_held() is a NULL pointer then +** the routine should return 1. This seems counter-intuitive since +** clearly the mutex cannot be held if it does not exist. But +** the reason the mutex does not exist is because the build is not +** using mutexes. And we do not want the assert() containing the +** call to sqlite3_mutex_held() to fail, so a non-zero return is +** the appropriate thing to do. The sqlite3_mutex_notheld() +** interface should also return 1 when given a NULL pointer. +*/ +#ifndef NDEBUG +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); +#endif + +/* +** CAPI3REF: Mutex Types +** +** The [sqlite3_mutex_alloc()] interface takes a single argument +** which is one of these integer constants. +** +** The set of static mutexes may change from one SQLite release to the +** next. Applications that override the built-in mutex logic must be +** prepared to accommodate additional static mutexes. +*/ +#define SQLITE_MUTEX_FAST 0 +#define SQLITE_MUTEX_RECURSIVE 1 +#define SQLITE_MUTEX_STATIC_MAIN 2 +#define SQLITE_MUTEX_STATIC_MEM 3 /* sqlite3_malloc() */ +#define SQLITE_MUTEX_STATIC_MEM2 4 /* NOT USED */ +#define SQLITE_MUTEX_STATIC_OPEN 4 /* sqlite3BtreeOpen() */ +#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_randomness() */ +#define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ +#define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ +#define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ +#define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ +#define SQLITE_MUTEX_STATIC_VFS1 11 /* For use by built-in VFS */ +#define SQLITE_MUTEX_STATIC_VFS2 12 /* For use by extension VFS */ +#define SQLITE_MUTEX_STATIC_VFS3 13 /* For use by application VFS */ + +/* Legacy compatibility: */ +#define SQLITE_MUTEX_STATIC_MASTER 2 + + +/* +** CAPI3REF: Retrieve the mutex for a database connection +** METHOD: sqlite3 +** +** ^This interface returns a pointer the [sqlite3_mutex] object that +** serializes access to the [database connection] given in the argument +** when the [threading mode] is Serialized. +** ^If the [threading mode] is Single-thread or Multi-thread then this +** routine returns a NULL pointer. +*/ +SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); + +/* +** CAPI3REF: Low-Level Control Of Database Files +** METHOD: sqlite3 +** KEYWORDS: {file control} +** +** ^The [sqlite3_file_control()] interface makes a direct call to the +** xFileControl method for the [sqlite3_io_methods] object associated +** with a particular database identified by the second argument. ^The +** name of the database is "main" for the main database or "temp" for the +** TEMP database, or the name that appears after the AS keyword for +** databases that are added using the [ATTACH] SQL command. +** ^A NULL pointer can be used in place of "main" to refer to the +** main database file. +** ^The third and fourth parameters to this routine +** are passed directly through to the second and third parameters of +** the xFileControl method. ^The return value of the xFileControl +** method becomes the return value of this routine. +** +** A few opcodes for [sqlite3_file_control()] are handled directly +** by the SQLite core and never invoke the +** sqlite3_io_methods.xFileControl method. +** ^The [SQLITE_FCNTL_FILE_POINTER] value for the op parameter causes +** a pointer to the underlying [sqlite3_file] object to be written into +** the space pointed to by the 4th parameter. The +** [SQLITE_FCNTL_JOURNAL_POINTER] works similarly except that it returns +** the [sqlite3_file] object associated with the journal file instead of +** the main database. The [SQLITE_FCNTL_VFS_POINTER] opcode returns +** a pointer to the underlying [sqlite3_vfs] object for the file. +** The [SQLITE_FCNTL_DATA_VERSION] returns the data version counter +** from the pager. +** +** ^If the second parameter (zDbName) does not match the name of any +** open database file, then SQLITE_ERROR is returned. ^This error +** code is not remembered and will not be recalled by [sqlite3_errcode()] +** or [sqlite3_errmsg()]. The underlying xFileControl method might +** also return SQLITE_ERROR. There is no way to distinguish between +** an incorrect zDbName and an SQLITE_ERROR return from the underlying +** xFileControl method. +** +** See also: [file control opcodes] +*/ +SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); + +/* +** CAPI3REF: Testing Interface +** +** ^The sqlite3_test_control() interface is used to read out internal +** state of SQLite and to inject faults into SQLite for testing +** purposes. ^The first parameter is an operation code that determines +** the number, meaning, and operation of all subsequent parameters. +** +** This interface is not for use by applications. It exists solely +** for verifying the correct operation of the SQLite library. Depending +** on how the SQLite library is compiled, this interface might not exist. +** +** The details of the operation codes, their meanings, the parameters +** they take, and what they do are all subject to change without notice. +** Unlike most of the SQLite API, this function is not guaranteed to +** operate consistently from one release to the next. +*/ +SQLITE_API int sqlite3_test_control(int op, ...); + +/* +** CAPI3REF: Testing Interface Operation Codes +** +** These constants are the valid operation code parameters used +** as the first argument to [sqlite3_test_control()]. +** +** These parameters and their meanings are subject to change +** without notice. These values are for testing purposes only. +** Applications should not use any of these parameters or the +** [sqlite3_test_control()] interface. +*/ +#define SQLITE_TESTCTRL_FIRST 5 +#define SQLITE_TESTCTRL_PRNG_SAVE 5 +#define SQLITE_TESTCTRL_PRNG_RESTORE 6 +#define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ +#define SQLITE_TESTCTRL_BITVEC_TEST 8 +#define SQLITE_TESTCTRL_FAULT_INSTALL 9 +#define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 +#define SQLITE_TESTCTRL_PENDING_BYTE 11 +#define SQLITE_TESTCTRL_ASSERT 12 +#define SQLITE_TESTCTRL_ALWAYS 13 +#define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ +#define SQLITE_TESTCTRL_OPTIMIZATIONS 15 +#define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ +#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ +#define SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 17 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 +#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ +#define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD 19 +#define SQLITE_TESTCTRL_NEVER_CORRUPT 20 +#define SQLITE_TESTCTRL_VDBE_COVERAGE 21 +#define SQLITE_TESTCTRL_BYTEORDER 22 +#define SQLITE_TESTCTRL_ISINIT 23 +#define SQLITE_TESTCTRL_SORTER_MMAP 24 +#define SQLITE_TESTCTRL_IMPOSTER 25 +#define SQLITE_TESTCTRL_PARSER_COVERAGE 26 +#define SQLITE_TESTCTRL_RESULT_INTREAL 27 +#define SQLITE_TESTCTRL_PRNG_SEED 28 +#define SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS 29 +#define SQLITE_TESTCTRL_SEEK_COUNT 30 +#define SQLITE_TESTCTRL_TRACEFLAGS 31 +#define SQLITE_TESTCTRL_TUNE 32 +#define SQLITE_TESTCTRL_LAST 32 /* Largest TESTCTRL */ + +/* +** CAPI3REF: SQL Keyword Checking +** +** These routines provide access to the set of SQL language keywords +** recognized by SQLite. Applications can uses these routines to determine +** whether or not a specific identifier needs to be escaped (for example, +** by enclosing in double-quotes) so as not to confuse the parser. +** +** The sqlite3_keyword_count() interface returns the number of distinct +** keywords understood by SQLite. +** +** The sqlite3_keyword_name(N,Z,L) interface finds the N-th keyword and +** makes *Z point to that keyword expressed as UTF8 and writes the number +** of bytes in the keyword into *L. The string that *Z points to is not +** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns +** SQLITE_OK if N is within bounds and SQLITE_ERROR if not. If either Z +** or L are NULL or invalid pointers then calls to +** sqlite3_keyword_name(N,Z,L) result in undefined behavior. +** +** The sqlite3_keyword_check(Z,L) interface checks to see whether or not +** the L-byte UTF8 identifier that Z points to is a keyword, returning non-zero +** if it is and zero if not. +** +** The parser used by SQLite is forgiving. It is often possible to use +** a keyword as an identifier as long as such use does not result in a +** parsing ambiguity. For example, the statement +** "CREATE TABLE BEGIN(REPLACE,PRAGMA,END);" is accepted by SQLite, and +** creates a new table named "BEGIN" with three columns named +** "REPLACE", "PRAGMA", and "END". Nevertheless, best practice is to avoid +** using keywords as identifiers. Common techniques used to avoid keyword +** name collisions include: +**
      +**
    • Put all identifier names inside double-quotes. This is the official +** SQL way to escape identifier names. +**
    • Put identifier names inside [...]. This is not standard SQL, +** but it is what SQL Server does and so lots of programmers use this +** technique. +**
    • Begin every identifier with the letter "Z" as no SQL keywords start +** with "Z". +**
    • Include a digit somewhere in every identifier name. +**
    +** +** Note that the number of keywords understood by SQLite can depend on +** compile-time options. For example, "VACUUM" is not a keyword if +** SQLite is compiled with the [-DSQLITE_OMIT_VACUUM] option. Also, +** new keywords may be added to future releases of SQLite. +*/ +SQLITE_API int sqlite3_keyword_count(void); +SQLITE_API int sqlite3_keyword_name(int,const char**,int*); +SQLITE_API int sqlite3_keyword_check(const char*,int); + +/* +** CAPI3REF: Dynamic String Object +** KEYWORDS: {dynamic string} +** +** An instance of the sqlite3_str object contains a dynamically-sized +** string under construction. +** +** The lifecycle of an sqlite3_str object is as follows: +**
      +**
    1. ^The sqlite3_str object is created using [sqlite3_str_new()]. +**
    2. ^Text is appended to the sqlite3_str object using various +** methods, such as [sqlite3_str_appendf()]. +**
    3. ^The sqlite3_str object is destroyed and the string it created +** is returned using the [sqlite3_str_finish()] interface. +**
    +*/ +typedef struct sqlite3_str sqlite3_str; + +/* +** CAPI3REF: Create A New Dynamic String Object +** CONSTRUCTOR: sqlite3_str +** +** ^The [sqlite3_str_new(D)] interface allocates and initializes +** a new [sqlite3_str] object. To avoid memory leaks, the object returned by +** [sqlite3_str_new()] must be freed by a subsequent call to +** [sqlite3_str_finish(X)]. +** +** ^The [sqlite3_str_new(D)] interface always returns a pointer to a +** valid [sqlite3_str] object, though in the event of an out-of-memory +** error the returned object might be a special singleton that will +** silently reject new text, always return SQLITE_NOMEM from +** [sqlite3_str_errcode()], always return 0 for +** [sqlite3_str_length()], and always return NULL from +** [sqlite3_str_finish(X)]. It is always safe to use the value +** returned by [sqlite3_str_new(D)] as the sqlite3_str parameter +** to any of the other [sqlite3_str] methods. +** +** The D parameter to [sqlite3_str_new(D)] may be NULL. If the +** D parameter in [sqlite3_str_new(D)] is not NULL, then the maximum +** length of the string contained in the [sqlite3_str] object will be +** the value set for [sqlite3_limit](D,[SQLITE_LIMIT_LENGTH]) instead +** of [SQLITE_MAX_LENGTH]. +*/ +SQLITE_API sqlite3_str *sqlite3_str_new(sqlite3*); + +/* +** CAPI3REF: Finalize A Dynamic String +** DESTRUCTOR: sqlite3_str +** +** ^The [sqlite3_str_finish(X)] interface destroys the sqlite3_str object X +** and returns a pointer to a memory buffer obtained from [sqlite3_malloc64()] +** that contains the constructed string. The calling application should +** pass the returned value to [sqlite3_free()] to avoid a memory leak. +** ^The [sqlite3_str_finish(X)] interface may return a NULL pointer if any +** errors were encountered during construction of the string. ^The +** [sqlite3_str_finish(X)] interface will also return a NULL pointer if the +** string in [sqlite3_str] object X is zero bytes long. +*/ +SQLITE_API char *sqlite3_str_finish(sqlite3_str*); + +/* +** CAPI3REF: Add Content To A Dynamic String +** METHOD: sqlite3_str +** +** These interfaces add content to an sqlite3_str object previously obtained +** from [sqlite3_str_new()]. +** +** ^The [sqlite3_str_appendf(X,F,...)] and +** [sqlite3_str_vappendf(X,F,V)] interfaces uses the [built-in printf] +** functionality of SQLite to append formatted text onto the end of +** [sqlite3_str] object X. +** +** ^The [sqlite3_str_append(X,S,N)] method appends exactly N bytes from string S +** onto the end of the [sqlite3_str] object X. N must be non-negative. +** S must contain at least N non-zero bytes of content. To append a +** zero-terminated string in its entirety, use the [sqlite3_str_appendall()] +** method instead. +** +** ^The [sqlite3_str_appendall(X,S)] method appends the complete content of +** zero-terminated string S onto the end of [sqlite3_str] object X. +** +** ^The [sqlite3_str_appendchar(X,N,C)] method appends N copies of the +** single-byte character C onto the end of [sqlite3_str] object X. +** ^This method can be used, for example, to add whitespace indentation. +** +** ^The [sqlite3_str_reset(X)] method resets the string under construction +** inside [sqlite3_str] object X back to zero bytes in length. +** +** These methods do not return a result code. ^If an error occurs, that fact +** is recorded in the [sqlite3_str] object and can be recovered by a +** subsequent call to [sqlite3_str_errcode(X)]. +*/ +SQLITE_API void sqlite3_str_appendf(sqlite3_str*, const char *zFormat, ...); +SQLITE_API void sqlite3_str_vappendf(sqlite3_str*, const char *zFormat, va_list); +SQLITE_API void sqlite3_str_append(sqlite3_str*, const char *zIn, int N); +SQLITE_API void sqlite3_str_appendall(sqlite3_str*, const char *zIn); +SQLITE_API void sqlite3_str_appendchar(sqlite3_str*, int N, char C); +SQLITE_API void sqlite3_str_reset(sqlite3_str*); + +/* +** CAPI3REF: Status Of A Dynamic String +** METHOD: sqlite3_str +** +** These interfaces return the current status of an [sqlite3_str] object. +** +** ^If any prior errors have occurred while constructing the dynamic string +** in sqlite3_str X, then the [sqlite3_str_errcode(X)] method will return +** an appropriate error code. ^The [sqlite3_str_errcode(X)] method returns +** [SQLITE_NOMEM] following any out-of-memory error, or +** [SQLITE_TOOBIG] if the size of the dynamic string exceeds +** [SQLITE_MAX_LENGTH], or [SQLITE_OK] if there have been no errors. +** +** ^The [sqlite3_str_length(X)] method returns the current length, in bytes, +** of the dynamic string under construction in [sqlite3_str] object X. +** ^The length returned by [sqlite3_str_length(X)] does not include the +** zero-termination byte. +** +** ^The [sqlite3_str_value(X)] method returns a pointer to the current +** content of the dynamic string under construction in X. The value +** returned by [sqlite3_str_value(X)] is managed by the sqlite3_str object X +** and might be freed or altered by any subsequent method on the same +** [sqlite3_str] object. Applications must not used the pointer returned +** [sqlite3_str_value(X)] after any subsequent method call on the same +** object. ^Applications may change the content of the string returned +** by [sqlite3_str_value(X)] as long as they do not write into any bytes +** outside the range of 0 to [sqlite3_str_length(X)] and do not read or +** write any byte after any subsequent sqlite3_str method call. +*/ +SQLITE_API int sqlite3_str_errcode(sqlite3_str*); +SQLITE_API int sqlite3_str_length(sqlite3_str*); +SQLITE_API char *sqlite3_str_value(sqlite3_str*); + +/* +** CAPI3REF: SQLite Runtime Status +** +** ^These interfaces are used to retrieve runtime status information +** about the performance of SQLite, and optionally to reset various +** highwater marks. ^The first argument is an integer code for +** the specific parameter to measure. ^(Recognized integer codes +** are of the form [status parameters | SQLITE_STATUS_...].)^ +** ^The current value of the parameter is returned into *pCurrent. +** ^The highest recorded value is returned in *pHighwater. ^If the +** resetFlag is true, then the highest record value is reset after +** *pHighwater is written. ^(Some parameters do not record the highest +** value. For those parameters +** nothing is written into *pHighwater and the resetFlag is ignored.)^ +** ^(Other parameters record only the highwater mark and not the current +** value. For these latter parameters nothing is written into *pCurrent.)^ +** +** ^The sqlite3_status() and sqlite3_status64() routines return +** SQLITE_OK on success and a non-zero [error code] on failure. +** +** If either the current value or the highwater mark is too large to +** be represented by a 32-bit integer, then the values returned by +** sqlite3_status() are undefined. +** +** See also: [sqlite3_db_status()] +*/ +SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); +SQLITE_API int sqlite3_status64( + int op, + sqlite3_int64 *pCurrent, + sqlite3_int64 *pHighwater, + int resetFlag +); + + +/* +** CAPI3REF: Status Parameters +** KEYWORDS: {status parameters} +** +** These integer constants designate various run-time status parameters +** that can be returned by [sqlite3_status()]. +** +**
    +** [[SQLITE_STATUS_MEMORY_USED]] ^(
    SQLITE_STATUS_MEMORY_USED
    +**
    This parameter is the current amount of memory checked out +** using [sqlite3_malloc()], either directly or indirectly. The +** figure includes calls made to [sqlite3_malloc()] by the application +** and internal memory usage by the SQLite library. Auxiliary page-cache +** memory controlled by [SQLITE_CONFIG_PAGECACHE] is not included in +** this parameter. The amount returned is the sum of the allocation +** sizes as reported by the xSize method in [sqlite3_mem_methods].
    )^ +** +** [[SQLITE_STATUS_MALLOC_SIZE]] ^(
    SQLITE_STATUS_MALLOC_SIZE
    +**
    This parameter records the largest memory allocation request +** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their +** internal equivalents). Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
    )^ +** +** [[SQLITE_STATUS_MALLOC_COUNT]] ^(
    SQLITE_STATUS_MALLOC_COUNT
    +**
    This parameter records the number of separate memory allocations +** currently checked out.
    )^ +** +** [[SQLITE_STATUS_PAGECACHE_USED]] ^(
    SQLITE_STATUS_PAGECACHE_USED
    +**
    This parameter returns the number of pages used out of the +** [pagecache memory allocator] that was configured using +** [SQLITE_CONFIG_PAGECACHE]. The +** value returned is in pages, not in bytes.
    )^ +** +** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] +** ^(
    SQLITE_STATUS_PAGECACHE_OVERFLOW
    +**
    This parameter returns the number of bytes of page cache +** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] +** buffer and where forced to overflow to [sqlite3_malloc()]. The +** returned value includes allocations that overflowed because they +** where too large (they were larger than the "sz" parameter to +** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because +** no space was left in the page cache.
    )^ +** +** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(
    SQLITE_STATUS_PAGECACHE_SIZE
    +**
    This parameter records the largest memory allocation request +** handed to the [pagecache memory allocator]. Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
    )^ +** +** [[SQLITE_STATUS_SCRATCH_USED]]
    SQLITE_STATUS_SCRATCH_USED
    +**
    No longer used.
    +** +** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(
    SQLITE_STATUS_SCRATCH_OVERFLOW
    +**
    No longer used.
    +** +** [[SQLITE_STATUS_SCRATCH_SIZE]]
    SQLITE_STATUS_SCRATCH_SIZE
    +**
    No longer used.
    +** +** [[SQLITE_STATUS_PARSER_STACK]] ^(
    SQLITE_STATUS_PARSER_STACK
    +**
    The *pHighwater parameter records the deepest parser stack. +** The *pCurrent value is undefined. The *pHighwater value is only +** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
    )^ +**
    +** +** New status parameters may be added from time to time. +*/ +#define SQLITE_STATUS_MEMORY_USED 0 +#define SQLITE_STATUS_PAGECACHE_USED 1 +#define SQLITE_STATUS_PAGECACHE_OVERFLOW 2 +#define SQLITE_STATUS_SCRATCH_USED 3 /* NOT USED */ +#define SQLITE_STATUS_SCRATCH_OVERFLOW 4 /* NOT USED */ +#define SQLITE_STATUS_MALLOC_SIZE 5 +#define SQLITE_STATUS_PARSER_STACK 6 +#define SQLITE_STATUS_PAGECACHE_SIZE 7 +#define SQLITE_STATUS_SCRATCH_SIZE 8 /* NOT USED */ +#define SQLITE_STATUS_MALLOC_COUNT 9 + +/* +** CAPI3REF: Database Connection Status +** METHOD: sqlite3 +** +** ^This interface is used to retrieve runtime status information +** about a single [database connection]. ^The first argument is the +** database connection object to be interrogated. ^The second argument +** is an integer constant, taken from the set of +** [SQLITE_DBSTATUS options], that +** determines the parameter to interrogate. The set of +** [SQLITE_DBSTATUS options] is likely +** to grow in future releases of SQLite. +** +** ^The current value of the requested parameter is written into *pCur +** and the highest instantaneous value is written into *pHiwtr. ^If +** the resetFlg is true, then the highest instantaneous value is +** reset back down to the current value. +** +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. +** +** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. +*/ +SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); + +/* +** CAPI3REF: Status Parameters for database connections +** KEYWORDS: {SQLITE_DBSTATUS options} +** +** These constants are the available integer "verbs" that can be passed as +** the second argument to the [sqlite3_db_status()] interface. +** +** New verbs may be added in future releases of SQLite. Existing verbs +** might be discontinued. Applications should check the return code from +** [sqlite3_db_status()] to make sure that the call worked. +** The [sqlite3_db_status()] interface will return a non-zero error code +** if a discontinued or unsupported verb is invoked. +** +**
    +** [[SQLITE_DBSTATUS_LOOKASIDE_USED]] ^(
    SQLITE_DBSTATUS_LOOKASIDE_USED
    +**
    This parameter returns the number of lookaside memory slots currently +** checked out.
    )^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
    SQLITE_DBSTATUS_LOOKASIDE_HIT
    +**
    This parameter returns the number of malloc attempts that were +** satisfied using lookaside memory. Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] +** ^(
    SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
    +**
    This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to the amount of +** memory requested being larger than the lookaside slot size. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] +** ^(
    SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
    +**
    This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to all lookaside +** memory already being in use. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
    SQLITE_DBSTATUS_CACHE_USED
    +**
    This parameter returns the approximate number of bytes of heap +** memory used by all pager caches associated with the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +** +** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] +** ^(
    SQLITE_DBSTATUS_CACHE_USED_SHARED
    +**
    This parameter is similar to DBSTATUS_CACHE_USED, except that if a +** pager cache is shared between two or more connections the bytes of heap +** memory used by that pager cache is divided evenly between the attached +** connections.)^ In other words, if none of the pager caches associated +** with the database connection are shared, this request returns the same +** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are +** shared, the value returned by this call will be smaller than that returned +** by DBSTATUS_CACHE_USED. ^The highwater mark associated with +** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. +** +** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    +**
    This parameter returns the approximate number of bytes of heap +** memory used to store the schema for all databases associated +** with the connection - main, temp, and any [ATTACH]-ed databases.)^ +** ^The full amount of memory used by the schemas is reported, even if the +** schema memory is shared with other database connections due to +** [shared cache mode] being enabled. +** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +** +** [[SQLITE_DBSTATUS_STMT_USED]] ^(
    SQLITE_DBSTATUS_STMT_USED
    +**
    This parameter returns the approximate number of bytes of heap +** and lookaside memory used by all prepared statements associated with +** the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. +**
    +** +** [[SQLITE_DBSTATUS_CACHE_HIT]] ^(
    SQLITE_DBSTATUS_CACHE_HIT
    +**
    This parameter returns the number of pager cache hits that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT +** is always 0. +**
    +** +** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(
    SQLITE_DBSTATUS_CACHE_MISS
    +**
    This parameter returns the number of pager cache misses that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS +** is always 0. +**
    +** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(
    SQLITE_DBSTATUS_CACHE_WRITE
    +**
    This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**
    +** +** [[SQLITE_DBSTATUS_CACHE_SPILL]] ^(
    SQLITE_DBSTATUS_CACHE_SPILL
    +**
    This parameter returns the number of dirty cache entries that have +** been written to disk in the middle of a transaction due to the page +** cache overflowing. Transactions are more efficient if they are written +** to disk all at once. When pages spill mid-transaction, that introduces +** additional overhead. This parameter can be used help identify +** inefficiencies that can be resolved by increasing the cache size. +**
    +** +** [[SQLITE_DBSTATUS_DEFERRED_FKS]] ^(
    SQLITE_DBSTATUS_DEFERRED_FKS
    +**
    This parameter returns zero for the current value if and only if +** all foreign key constraints (deferred or immediate) have been +** resolved.)^ ^The highwater mark is always 0. +**
    +**
    +*/ +#define SQLITE_DBSTATUS_LOOKASIDE_USED 0 +#define SQLITE_DBSTATUS_CACHE_USED 1 +#define SQLITE_DBSTATUS_SCHEMA_USED 2 +#define SQLITE_DBSTATUS_STMT_USED 3 +#define SQLITE_DBSTATUS_LOOKASIDE_HIT 4 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 +#define SQLITE_DBSTATUS_CACHE_HIT 7 +#define SQLITE_DBSTATUS_CACHE_MISS 8 +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_DEFERRED_FKS 10 +#define SQLITE_DBSTATUS_CACHE_USED_SHARED 11 +#define SQLITE_DBSTATUS_CACHE_SPILL 12 +#define SQLITE_DBSTATUS_MAX 12 /* Largest defined DBSTATUS */ + + +/* +** CAPI3REF: Prepared Statement Status +** METHOD: sqlite3_stmt +** +** ^(Each prepared statement maintains various +** [SQLITE_STMTSTATUS counters] that measure the number +** of times it has performed specific operations.)^ These counters can +** be used to monitor the performance characteristics of the prepared +** statements. For example, if the number of table steps greatly exceeds +** the number of table searches or result rows, that would tend to indicate +** that the prepared statement is using a full table scan rather than +** an index. +** +** ^(This interface is used to retrieve and reset counter values from +** a [prepared statement]. The first argument is the prepared statement +** object to be interrogated. The second argument +** is an integer code for a specific [SQLITE_STMTSTATUS counter] +** to be interrogated.)^ +** ^The current value of the requested counter is returned. +** ^If the resetFlg is true, then the counter is reset to zero after this +** interface call returns. +** +** See also: [sqlite3_status()] and [sqlite3_db_status()]. +*/ +SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); + +/* +** CAPI3REF: Status Parameters for prepared statements +** KEYWORDS: {SQLITE_STMTSTATUS counter} {SQLITE_STMTSTATUS counters} +** +** These preprocessor macros define integer codes that name counter +** values associated with the [sqlite3_stmt_status()] interface. +** The meanings of the various counters are as follows: +** +**
    +** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]]
    SQLITE_STMTSTATUS_FULLSCAN_STEP
    +**
    ^This is the number of times that SQLite has stepped forward in +** a table as part of a full table scan. Large numbers for this counter +** may indicate opportunities for performance improvement through +** careful use of indices.
    +** +** [[SQLITE_STMTSTATUS_SORT]]
    SQLITE_STMTSTATUS_SORT
    +**
    ^This is the number of sort operations that have occurred. +** A non-zero value in this counter may indicate an opportunity to +** improvement performance through careful use of indices.
    +** +** [[SQLITE_STMTSTATUS_AUTOINDEX]]
    SQLITE_STMTSTATUS_AUTOINDEX
    +**
    ^This is the number of rows inserted into transient indices that +** were created automatically in order to help joins run faster. +** A non-zero value in this counter may indicate an opportunity to +** improvement performance by adding permanent indices that do not +** need to be reinitialized each time the statement is run.
    +** +** [[SQLITE_STMTSTATUS_VM_STEP]]
    SQLITE_STMTSTATUS_VM_STEP
    +**
    ^This is the number of virtual machine operations executed +** by the prepared statement if that number is less than or equal +** to 2147483647. The number of virtual machine operations can be +** used as a proxy for the total work done by the prepared statement. +** If the number of virtual machine operations exceeds 2147483647 +** then the value returned by this statement status code is undefined. +** +** [[SQLITE_STMTSTATUS_REPREPARE]]
    SQLITE_STMTSTATUS_REPREPARE
    +**
    ^This is the number of times that the prepare statement has been +** automatically regenerated due to schema changes or changes to +** [bound parameters] that might affect the query plan. +** +** [[SQLITE_STMTSTATUS_RUN]]
    SQLITE_STMTSTATUS_RUN
    +**
    ^This is the number of times that the prepared statement has +** been run. A single "run" for the purposes of this counter is one +** or more calls to [sqlite3_step()] followed by a call to [sqlite3_reset()]. +** The counter is incremented on the first [sqlite3_step()] call of each +** cycle. +** +** [[SQLITE_STMTSTATUS_MEMUSED]]
    SQLITE_STMTSTATUS_MEMUSED
    +**
    ^This is the approximate number of bytes of heap memory +** used to store the prepared statement. ^This value is not actually +** a counter, and so the resetFlg parameter to sqlite3_stmt_status() +** is ignored when the opcode is SQLITE_STMTSTATUS_MEMUSED. +**
    +**
    +*/ +#define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 +#define SQLITE_STMTSTATUS_SORT 2 +#define SQLITE_STMTSTATUS_AUTOINDEX 3 +#define SQLITE_STMTSTATUS_VM_STEP 4 +#define SQLITE_STMTSTATUS_REPREPARE 5 +#define SQLITE_STMTSTATUS_RUN 6 +#define SQLITE_STMTSTATUS_MEMUSED 99 + +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache type is opaque. It is implemented by +** the pluggable module. The SQLite core has no knowledge of +** its size or internal structure and never deals with the +** sqlite3_pcache object except by holding and passing pointers +** to the object. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache sqlite3_pcache; + +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache_page object represents a single page in the +** page cache. The page cache will allocate instances of this +** object. Various methods of the page cache use pointers to instances +** of this object as parameters or as their return value. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache_page sqlite3_pcache_page; +struct sqlite3_pcache_page { + void *pBuf; /* The content of the page */ + void *pExtra; /* Extra information associated with the page */ +}; + +/* +** CAPI3REF: Application Defined Page Cache. +** KEYWORDS: {page cache} +** +** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE2], ...) interface can +** register an alternative page cache implementation by passing in an +** instance of the sqlite3_pcache_methods2 structure.)^ +** In many applications, most of the heap memory allocated by +** SQLite is used for the page cache. +** By implementing a +** custom page cache using this API, an application can better control +** the amount of memory consumed by SQLite, the way in which +** that memory is allocated and released, and the policies used to +** determine exactly which parts of a database file are cached and for +** how long. +** +** The alternative page cache mechanism is an +** extreme measure that is only needed by the most demanding applications. +** The built-in page cache is recommended for most uses. +** +** ^(The contents of the sqlite3_pcache_methods2 structure are copied to an +** internal buffer by SQLite within the call to [sqlite3_config]. Hence +** the application may discard the parameter after the call to +** [sqlite3_config()] returns.)^ +** +** [[the xInit() page cache method]] +** ^(The xInit() method is called once for each effective +** call to [sqlite3_initialize()])^ +** (usually only once during the lifetime of the process). ^(The xInit() +** method is passed a copy of the sqlite3_pcache_methods2.pArg value.)^ +** The intent of the xInit() method is to set up global data structures +** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the +** built-in default page cache is used instead of the application defined +** page cache.)^ +** +** [[the xShutdown() page cache method]] +** ^The xShutdown() method is called by [sqlite3_shutdown()]. +** It can be used to clean up +** any outstanding resources before process shutdown, if required. +** ^The xShutdown() method may be NULL. +** +** ^SQLite automatically serializes calls to the xInit method, +** so the xInit method need not be threadsafe. ^The +** xShutdown method is only called from [sqlite3_shutdown()] so it does +** not need to be threadsafe either. All other methods must be threadsafe +** in multithreaded applications. +** +** ^SQLite will never invoke xInit() more than once without an intervening +** call to xShutdown(). +** +** [[the xCreate() page cache methods]] +** ^SQLite invokes the xCreate() method to construct a new cache instance. +** SQLite will typically create one cache instance for each open database file, +** though this is not guaranteed. ^The +** first parameter, szPage, is the size in bytes of the pages that must +** be allocated by the cache. ^szPage will always a power of two. ^The +** second parameter szExtra is a number of bytes of extra storage +** associated with each page cache entry. ^The szExtra parameter will +** a number less than 250. SQLite will use the +** extra szExtra bytes on each page to store metadata about the underlying +** database page on disk. The value passed into szExtra depends +** on the SQLite version, the target platform, and how SQLite was compiled. +** ^The third argument to xCreate(), bPurgeable, is true if the cache being +** created will be used to cache database pages of a file stored on disk, or +** false if it is used for an in-memory database. The cache implementation +** does not have to do anything special based with the value of bPurgeable; +** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will +** never invoke xUnpin() except to deliberately delete a page. +** ^In other words, calls to xUnpin() on a cache with bPurgeable set to +** false will always have the "discard" flag set to true. +** ^Hence, a cache created with bPurgeable false will +** never contain any unpinned pages. +** +** [[the xCachesize() page cache method]] +** ^(The xCachesize() method may be called at any time by SQLite to set the +** suggested maximum cache-size (number of pages stored by) the cache +** instance passed as the first argument. This is the value configured using +** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable +** parameter, the implementation is not required to do anything with this +** value; it is advisory only. +** +** [[the xPagecount() page cache methods]] +** The xPagecount() method must return the number of pages currently +** stored in the cache, both pinned and unpinned. +** +** [[the xFetch() page cache methods]] +** The xFetch() method locates a page in the cache and returns a pointer to +** an sqlite3_pcache_page object associated with that page, or a NULL pointer. +** The pBuf element of the returned sqlite3_pcache_page object will be a +** pointer to a buffer of szPage bytes used to store the content of a +** single database page. The pExtra element of sqlite3_pcache_page will be +** a pointer to the szExtra bytes of extra storage that SQLite has requested +** for each entry in the page cache. +** +** The page to be fetched is determined by the key. ^The minimum key value +** is 1. After it has been retrieved using xFetch, the page is considered +** to be "pinned". +** +** If the requested page is already in the page cache, then the page cache +** implementation must return a pointer to the page buffer with its content +** intact. If the requested page is not already in the cache, then the +** cache implementation should use the value of the createFlag +** parameter to help it determined what action to take: +** +** +**
    createFlag Behavior when page is not already in cache +**
    0 Do not allocate a new page. Return NULL. +**
    1 Allocate a new page if it easy and convenient to do so. +** Otherwise return NULL. +**
    2 Make every effort to allocate a new page. Only return +** NULL if allocating a new page is effectively impossible. +**
    +** +** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite +** will only use a createFlag of 2 after a prior call with a createFlag of 1 +** failed.)^ In between the xFetch() calls, SQLite may +** attempt to unpin one or more cache pages by spilling the content of +** pinned pages to disk and synching the operating system disk cache. +** +** [[the xUnpin() page cache method]] +** ^xUnpin() is called by SQLite with a pointer to a currently pinned page +** as its second argument. If the third parameter, discard, is non-zero, +** then the page must be evicted from the cache. +** ^If the discard parameter is +** zero, then the page may be discarded or retained at the discretion of +** page cache implementation. ^The page cache implementation +** may choose to evict unpinned pages at any time. +** +** The cache must not perform any reference counting. A single +** call to xUnpin() unpins the page regardless of the number of prior calls +** to xFetch(). +** +** [[the xRekey() page cache methods]] +** The xRekey() method is used to change the key value associated with the +** page passed as the second argument. If the cache +** previously contains an entry associated with newKey, it must be +** discarded. ^Any prior cache entry associated with newKey is guaranteed not +** to be pinned. +** +** When SQLite calls the xTruncate() method, the cache must discard all +** existing cache entries with page numbers (keys) greater than or equal +** to the value of the iLimit parameter passed to xTruncate(). If any +** of these pages are pinned, they are implicitly unpinned, meaning that +** they can be safely discarded. +** +** [[the xDestroy() page cache method]] +** ^The xDestroy() method is used to delete a cache allocated by xCreate(). +** All resources associated with the specified cache should be freed. ^After +** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] +** handle invalid, and will not use it with any other sqlite3_pcache_methods2 +** functions. +** +** [[the xShrink() page cache method]] +** ^SQLite invokes the xShrink() method when it wants the page cache to +** free up as much of heap memory as possible. The page cache implementation +** is not obligated to free any memory, but well-behaved implementations should +** do their best. +*/ +typedef struct sqlite3_pcache_methods2 sqlite3_pcache_methods2; +struct sqlite3_pcache_methods2 { + int iVersion; + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int szExtra, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + sqlite3_pcache_page *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, sqlite3_pcache_page*, int discard); + void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*, + unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); + void (*xShrink)(sqlite3_pcache*); +}; + +/* +** This is the obsolete pcache_methods object that has now been replaced +** by sqlite3_pcache_methods2. This object is not used by SQLite. It is +** retained in the header file for backwards compatibility only. +*/ +typedef struct sqlite3_pcache_methods sqlite3_pcache_methods; +struct sqlite3_pcache_methods { + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + void *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, void*, int discard); + void (*xRekey)(sqlite3_pcache*, void*, unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); +}; + + +/* +** CAPI3REF: Online Backup Object +** +** The sqlite3_backup object records state information about an ongoing +** online backup operation. ^The sqlite3_backup object is created by +** a call to [sqlite3_backup_init()] and is destroyed by a call to +** [sqlite3_backup_finish()]. +** +** See Also: [Using the SQLite Online Backup API] +*/ +typedef struct sqlite3_backup sqlite3_backup; + +/* +** CAPI3REF: Online Backup API. +** +** The backup API copies the content of one database into another. +** It is useful either for creating backups of databases or +** for copying in-memory databases to or from persistent files. +** +** See Also: [Using the SQLite Online Backup API] +** +** ^SQLite holds a write transaction open on the destination database file +** for the duration of the backup operation. +** ^The source database is read-locked only while it is being read; +** it is not locked continuously for the entire backup operation. +** ^Thus, the backup may be performed on a live source database without +** preventing other database connections from +** reading or writing to the source database while the backup is underway. +** +** ^(To perform a backup operation: +**
      +**
    1. sqlite3_backup_init() is called once to initialize the +** backup, +**
    2. sqlite3_backup_step() is called one or more times to transfer +** the data between the two databases, and finally +**
    3. sqlite3_backup_finish() is called to release all resources +** associated with the backup operation. +**
    )^ +** There should be exactly one call to sqlite3_backup_finish() for each +** successful call to sqlite3_backup_init(). +** +** [[sqlite3_backup_init()]] sqlite3_backup_init() +** +** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the +** [database connection] associated with the destination database +** and the database name, respectively. +** ^The database name is "main" for the main database, "temp" for the +** temporary database, or the name specified after the AS keyword in +** an [ATTACH] statement for an attached database. +** ^The S and M arguments passed to +** sqlite3_backup_init(D,N,S,M) identify the [database connection] +** and database name of the source database, respectively. +** ^The source and destination [database connections] (parameters S and D) +** must be different or else sqlite3_backup_init(D,N,S,M) will fail with +** an error. +** +** ^A call to sqlite3_backup_init() will fail, returning NULL, if +** there is already a read or read-write transaction open on the +** destination database. +** +** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is +** returned and an error code and error message are stored in the +** destination [database connection] D. +** ^The error code and message for the failed call to sqlite3_backup_init() +** can be retrieved using the [sqlite3_errcode()], [sqlite3_errmsg()], and/or +** [sqlite3_errmsg16()] functions. +** ^A successful call to sqlite3_backup_init() returns a pointer to an +** [sqlite3_backup] object. +** ^The [sqlite3_backup] object may be used with the sqlite3_backup_step() and +** sqlite3_backup_finish() functions to perform the specified backup +** operation. +** +** [[sqlite3_backup_step()]] sqlite3_backup_step() +** +** ^Function sqlite3_backup_step(B,N) will copy up to N pages between +** the source and destination databases specified by [sqlite3_backup] object B. +** ^If N is negative, all remaining source pages are copied. +** ^If sqlite3_backup_step(B,N) successfully copies N pages and there +** are still more pages to be copied, then the function returns [SQLITE_OK]. +** ^If sqlite3_backup_step(B,N) successfully finishes copying all pages +** from source to destination, then it returns [SQLITE_DONE]. +** ^If an error occurs while running sqlite3_backup_step(B,N), +** then an [error code] is returned. ^As well as [SQLITE_OK] and +** [SQLITE_DONE], a call to sqlite3_backup_step() may return [SQLITE_READONLY], +** [SQLITE_NOMEM], [SQLITE_BUSY], [SQLITE_LOCKED], or an +** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX] extended error code. +** +** ^(The sqlite3_backup_step() might return [SQLITE_READONLY] if +**
      +**
    1. the destination database was opened read-only, or +**
    2. the destination database is using write-ahead-log journaling +** and the destination and source page sizes differ, or +**
    3. the destination database is an in-memory database and the +** destination and source page sizes differ. +**
    )^ +** +** ^If sqlite3_backup_step() cannot obtain a required file-system lock, then +** the [sqlite3_busy_handler | busy-handler function] +** is invoked (if one is specified). ^If the +** busy-handler returns non-zero before the lock is available, then +** [SQLITE_BUSY] is returned to the caller. ^In this case the call to +** sqlite3_backup_step() can be retried later. ^If the source +** [database connection] +** is being used to write to the source database when sqlite3_backup_step() +** is called, then [SQLITE_LOCKED] is returned immediately. ^Again, in this +** case the call to sqlite3_backup_step() can be retried later on. ^(If +** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX], [SQLITE_NOMEM], or +** [SQLITE_READONLY] is returned, then +** there is no point in retrying the call to sqlite3_backup_step(). These +** errors are considered fatal.)^ The application must accept +** that the backup operation has failed and pass the backup operation handle +** to the sqlite3_backup_finish() to release associated resources. +** +** ^The first call to sqlite3_backup_step() obtains an exclusive lock +** on the destination file. ^The exclusive lock is not released until either +** sqlite3_backup_finish() is called or the backup operation is complete +** and sqlite3_backup_step() returns [SQLITE_DONE]. ^Every call to +** sqlite3_backup_step() obtains a [shared lock] on the source database that +** lasts for the duration of the sqlite3_backup_step() call. +** ^Because the source database is not locked between calls to +** sqlite3_backup_step(), the source database may be modified mid-way +** through the backup process. ^If the source database is modified by an +** external process or via a database connection other than the one being +** used by the backup operation, then the backup will be automatically +** restarted by the next call to sqlite3_backup_step(). ^If the source +** database is modified by the using the same database connection as is used +** by the backup operation, then the backup database is automatically +** updated at the same time. +** +** [[sqlite3_backup_finish()]] sqlite3_backup_finish() +** +** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the +** application wishes to abandon the backup operation, the application +** should destroy the [sqlite3_backup] by passing it to sqlite3_backup_finish(). +** ^The sqlite3_backup_finish() interfaces releases all +** resources associated with the [sqlite3_backup] object. +** ^If sqlite3_backup_step() has not yet returned [SQLITE_DONE], then any +** active write-transaction on the destination database is rolled back. +** The [sqlite3_backup] object is invalid +** and may not be used following a call to sqlite3_backup_finish(). +** +** ^The value returned by sqlite3_backup_finish is [SQLITE_OK] if no +** sqlite3_backup_step() errors occurred, regardless or whether or not +** sqlite3_backup_step() completed. +** ^If an out-of-memory condition or IO error occurred during any prior +** sqlite3_backup_step() call on the same [sqlite3_backup] object, then +** sqlite3_backup_finish() returns the corresponding [error code]. +** +** ^A return of [SQLITE_BUSY] or [SQLITE_LOCKED] from sqlite3_backup_step() +** is not a permanent error and does not affect the return value of +** sqlite3_backup_finish(). +** +** [[sqlite3_backup_remaining()]] [[sqlite3_backup_pagecount()]] +** sqlite3_backup_remaining() and sqlite3_backup_pagecount() +** +** ^The sqlite3_backup_remaining() routine returns the number of pages still +** to be backed up at the conclusion of the most recent sqlite3_backup_step(). +** ^The sqlite3_backup_pagecount() routine returns the total number of pages +** in the source database at the conclusion of the most recent +** sqlite3_backup_step(). +** ^(The values returned by these functions are only updated by +** sqlite3_backup_step(). If the source database is modified in a way that +** changes the size of the source database or the number of pages remaining, +** those changes are not reflected in the output of sqlite3_backup_pagecount() +** and sqlite3_backup_remaining() until after the next +** sqlite3_backup_step().)^ +** +** Concurrent Usage of Database Handles +** +** ^The source [database connection] may be used by the application for other +** purposes while a backup operation is underway or being initialized. +** ^If SQLite is compiled and configured to support threadsafe database +** connections, then the source database connection may be used concurrently +** from within other threads. +** +** However, the application must guarantee that the destination +** [database connection] is not passed to any other API (by any thread) after +** sqlite3_backup_init() is called and before the corresponding call to +** sqlite3_backup_finish(). SQLite does not currently check to see +** if the application incorrectly accesses the destination [database connection] +** and so no error code is reported, but the operations may malfunction +** nevertheless. Use of the destination database connection while a +** backup is in progress might also also cause a mutex deadlock. +** +** If running in [shared cache mode], the application must +** guarantee that the shared cache used by the destination database +** is not accessed while the backup is running. In practice this means +** that the application must guarantee that the disk file being +** backed up to is not accessed by any connection within the process, +** not just the specific connection that was passed to sqlite3_backup_init(). +** +** The [sqlite3_backup] object itself is partially threadsafe. Multiple +** threads may safely make multiple concurrent calls to sqlite3_backup_step(). +** However, the sqlite3_backup_remaining() and sqlite3_backup_pagecount() +** APIs are not strictly speaking threadsafe. If they are invoked at the +** same time as another thread is invoking sqlite3_backup_step() it is +** possible that they return invalid values. +*/ +SQLITE_API sqlite3_backup *sqlite3_backup_init( + sqlite3 *pDest, /* Destination database handle */ + const char *zDestName, /* Destination database name */ + sqlite3 *pSource, /* Source database handle */ + const char *zSourceName /* Source database name */ +); +SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage); +SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); + +/* +** CAPI3REF: Unlock Notification +** METHOD: sqlite3 +** +** ^When running in shared-cache mode, a database operation may fail with +** an [SQLITE_LOCKED] error if the required locks on the shared-cache or +** individual tables within the shared-cache cannot be obtained. See +** [SQLite Shared-Cache Mode] for a description of shared-cache locking. +** ^This API may be used to register a callback that SQLite will invoke +** when the connection currently holding the required lock relinquishes it. +** ^This API is only available if the library was compiled with the +** [SQLITE_ENABLE_UNLOCK_NOTIFY] C-preprocessor symbol defined. +** +** See Also: [Using the SQLite Unlock Notification Feature]. +** +** ^Shared-cache locks are released when a database connection concludes +** its current transaction, either by committing it or rolling it back. +** +** ^When a connection (known as the blocked connection) fails to obtain a +** shared-cache lock and SQLITE_LOCKED is returned to the caller, the +** identity of the database connection (the blocking connection) that +** has locked the required resource is stored internally. ^After an +** application receives an SQLITE_LOCKED error, it may call the +** sqlite3_unlock_notify() method with the blocked connection handle as +** the first argument to register for a callback that will be invoked +** when the blocking connections current transaction is concluded. ^The +** callback is invoked from within the [sqlite3_step] or [sqlite3_close] +** call that concludes the blocking connection's transaction. +** +** ^(If sqlite3_unlock_notify() is called in a multi-threaded application, +** there is a chance that the blocking connection will have already +** concluded its transaction by the time sqlite3_unlock_notify() is invoked. +** If this happens, then the specified callback is invoked immediately, +** from within the call to sqlite3_unlock_notify().)^ +** +** ^If the blocked connection is attempting to obtain a write-lock on a +** shared-cache table, and more than one other connection currently holds +** a read-lock on the same table, then SQLite arbitrarily selects one of +** the other connections to use as the blocking connection. +** +** ^(There may be at most one unlock-notify callback registered by a +** blocked connection. If sqlite3_unlock_notify() is called when the +** blocked connection already has a registered unlock-notify callback, +** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is +** called with a NULL pointer as its second argument, then any existing +** unlock-notify callback is canceled. ^The blocked connections +** unlock-notify callback may also be canceled by closing the blocked +** connection using [sqlite3_close()]. +** +** The unlock-notify callback is not reentrant. If an application invokes +** any sqlite3_xxx API functions from within an unlock-notify callback, a +** crash or deadlock may be the result. +** +** ^Unless deadlock is detected (see below), sqlite3_unlock_notify() always +** returns SQLITE_OK. +** +** Callback Invocation Details +** +** When an unlock-notify callback is registered, the application provides a +** single void* pointer that is passed to the callback when it is invoked. +** However, the signature of the callback function allows SQLite to pass +** it an array of void* context pointers. The first argument passed to +** an unlock-notify callback is a pointer to an array of void* pointers, +** and the second is the number of entries in the array. +** +** When a blocking connection's transaction is concluded, there may be +** more than one blocked connection that has registered for an unlock-notify +** callback. ^If two or more such blocked connections have specified the +** same callback function, then instead of invoking the callback function +** multiple times, it is invoked once with the set of void* context pointers +** specified by the blocked connections bundled together into an array. +** This gives the application an opportunity to prioritize any actions +** related to the set of unblocked database connections. +** +** Deadlock Detection +** +** Assuming that after registering for an unlock-notify callback a +** database waits for the callback to be issued before taking any further +** action (a reasonable assumption), then using this API may cause the +** application to deadlock. For example, if connection X is waiting for +** connection Y's transaction to be concluded, and similarly connection +** Y is waiting on connection X's transaction, then neither connection +** will proceed and the system may remain deadlocked indefinitely. +** +** To avoid this scenario, the sqlite3_unlock_notify() performs deadlock +** detection. ^If a given call to sqlite3_unlock_notify() would put the +** system in a deadlocked state, then SQLITE_LOCKED is returned and no +** unlock-notify callback is registered. The system is said to be in +** a deadlocked state if connection A has registered for an unlock-notify +** callback on the conclusion of connection B's transaction, and connection +** B has itself registered for an unlock-notify callback when connection +** A's transaction is concluded. ^Indirect deadlock is also detected, so +** the system is also considered to be deadlocked if connection B has +** registered for an unlock-notify callback on the conclusion of connection +** C's transaction, where connection C is waiting on connection A. ^Any +** number of levels of indirection are allowed. +** +** The "DROP TABLE" Exception +** +** When a call to [sqlite3_step()] returns SQLITE_LOCKED, it is almost +** always appropriate to call sqlite3_unlock_notify(). There is however, +** one exception. When executing a "DROP TABLE" or "DROP INDEX" statement, +** SQLite checks if there are any currently executing SELECT statements +** that belong to the same connection. If there are, SQLITE_LOCKED is +** returned. In this case there is no "blocking connection", so invoking +** sqlite3_unlock_notify() results in the unlock-notify callback being +** invoked immediately. If the application then re-attempts the "DROP TABLE" +** or "DROP INDEX" query, an infinite loop might be the result. +** +** One way around this problem is to check the extended error code returned +** by an sqlite3_step() call. ^(If there is a blocking connection, then the +** extended error code is set to SQLITE_LOCKED_SHAREDCACHE. Otherwise, in +** the special "DROP TABLE/INDEX" case, the extended error code is just +** SQLITE_LOCKED.)^ +*/ +SQLITE_API int sqlite3_unlock_notify( + sqlite3 *pBlocked, /* Waiting connection */ + void (*xNotify)(void **apArg, int nArg), /* Callback function to invoke */ + void *pNotifyArg /* Argument to pass to xNotify */ +); + + +/* +** CAPI3REF: String Comparison +** +** ^The [sqlite3_stricmp()] and [sqlite3_strnicmp()] APIs allow applications +** and extensions to compare the contents of two buffers containing UTF-8 +** strings in a case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. +*/ +SQLITE_API int sqlite3_stricmp(const char *, const char *); +SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); + +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if and only if +** string X matches the [GLOB] pattern P. +** ^The definition of [GLOB] pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect understood by SQLite. ^The [sqlite3_strglob(P,X)] function +** is case sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +** +** See also: [sqlite3_strlike()]. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); + +/* +** CAPI3REF: String LIKE Matching +* +** ^The [sqlite3_strlike(P,X,E)] interface returns zero if and only if +** string X matches the [LIKE] pattern P with escape character E. +** ^The definition of [LIKE] pattern matching used in +** [sqlite3_strlike(P,X,E)] is the same as for the "X LIKE P ESCAPE E" +** operator in the SQL dialect understood by SQLite. ^For "X LIKE P" without +** the ESCAPE clause, set the E parameter of [sqlite3_strlike(P,X,E)] to 0. +** ^As with the LIKE operator, the [sqlite3_strlike(P,X,E)] function is case +** insensitive - equivalent upper and lower case ASCII characters match +** one another. +** +** ^The [sqlite3_strlike(P,X,E)] function matches Unicode characters, though +** only ASCII characters are case folded. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +** +** See also: [sqlite3_strglob()]. +*/ +SQLITE_API int sqlite3_strlike(const char *zGlob, const char *zStr, unsigned int cEsc); + +/* +** CAPI3REF: Error Logging Interface +** +** ^The [sqlite3_log()] interface writes a message into the [error log] +** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. +** ^If logging is enabled, the zFormat string and subsequent arguments are +** used with [sqlite3_snprintf()] to generate the final output string. +** +** The sqlite3_log() interface is intended for use by extensions such as +** virtual tables, collating functions, and SQL functions. While there is +** nothing to prevent an application from calling sqlite3_log(), doing so +** is considered bad form. +** +** The zFormat string must not be NULL. +** +** To avoid deadlocks and other threading problems, the sqlite3_log() routine +** will not use dynamically allocated memory. The log message is stored in +** a fixed-length buffer on the stack. If the log message is longer than +** a few hundred characters, it will be truncated to the length of the +** buffer. +*/ +SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); + +/* +** CAPI3REF: Write-Ahead Log Commit Hook +** METHOD: sqlite3 +** +** ^The [sqlite3_wal_hook()] function is used to register a callback that +** is invoked each time data is committed to a database in wal mode. +** +** ^(The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released)^, so the implementation +** may read, write or [checkpoint] the database as required. +** +** ^The first parameter passed to the callback function when it is invoked +** is a copy of the third parameter passed to sqlite3_wal_hook() when +** registering the callback. ^The second is a copy of the database handle. +** ^The third parameter is the name of the database that was written to - +** either "main" or the name of an [ATTACH]-ed database. ^The fourth parameter +** is the number of pages currently in the write-ahead log file, +** including those that were just committed. +** +** The callback function should normally return [SQLITE_OK]. ^If an error +** code is returned, that error will propagate back up through the +** SQLite code base to cause the statement that provoked the callback +** to report an error, though the commit will have still occurred. If the +** callback returns [SQLITE_ROW] or [SQLITE_DONE], or if it returns a value +** that does not correspond to any valid SQLite error code, the results +** are undefined. +** +** A single database handle may have at most a single write-ahead log callback +** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any +** previously registered write-ahead log callback. ^The return value is +** a copy of the third parameter from the previous call, if any, or 0. +** ^Note that the [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will +** overwrite any prior [sqlite3_wal_hook()] settings. +*/ +SQLITE_API void *sqlite3_wal_hook( + sqlite3*, + int(*)(void *,sqlite3*,const char*,int), + void* +); + +/* +** CAPI3REF: Configure an auto-checkpoint +** METHOD: sqlite3 +** +** ^The [sqlite3_wal_autocheckpoint(D,N)] is a wrapper around +** [sqlite3_wal_hook()] that causes any database on [database connection] D +** to automatically [checkpoint] +** after committing a transaction if there are N or +** more frames in the [write-ahead log] file. ^Passing zero or +** a negative value as the nFrame parameter disables automatic +** checkpoints entirely. +** +** ^The callback registered by this function replaces any existing callback +** registered using [sqlite3_wal_hook()]. ^Likewise, registering a callback +** using [sqlite3_wal_hook()] disables the automatic checkpoint mechanism +** configured by this function. +** +** ^The [wal_autocheckpoint pragma] can be used to invoke this interface +** from SQL. +** +** ^Checkpoints initiated by this mechanism are +** [sqlite3_wal_checkpoint_v2|PASSIVE]. +** +** ^Every new [database connection] defaults to having the auto-checkpoint +** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] +** pages. The use of this interface +** is only necessary if the default setting is found to be suboptimal +** for a particular application. +*/ +SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); + +/* +** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 +** +** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to +** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^ +** +** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the +** [write-ahead log] for database X on [database connection] D to be +** transferred into the database file and for the write-ahead log to +** be reset. See the [checkpointing] documentation for addition +** information. +** +** This interface used to be the only way to cause a checkpoint to +** occur. But then the newer and more powerful [sqlite3_wal_checkpoint_v2()] +** interface was added. This interface is retained for backwards +** compatibility and as a convenience for applications that need to manually +** start a callback but which do not need the full power (and corresponding +** complication) of [sqlite3_wal_checkpoint_v2()]. +*/ +SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); + +/* +** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 +** +** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint +** operation on database X of [database connection] D in mode M. Status +** information is written back into integers pointed to by L and C.)^ +** ^(The M parameter must be a valid [checkpoint mode]:)^ +** +**
    +**
    SQLITE_CHECKPOINT_PASSIVE
    +** ^Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish, then sync the database file if all frames +** in the log were checkpointed. ^The [busy-handler callback] +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. +** ^On the other hand, passive mode might leave the checkpoint unfinished +** if there are concurrent readers or writers. +** +**
    SQLITE_CHECKPOINT_FULL
    +** ^This mode blocks (it invokes the +** [sqlite3_busy_handler|busy-handler callback]) until there is no +** database writer and all readers are reading from the most recent database +** snapshot. ^It then checkpoints all frames in the log file and syncs the +** database file. ^This mode blocks new database writers while it is pending, +** but new database readers are allowed to continue unimpeded. +** +**
    SQLITE_CHECKPOINT_RESTART
    +** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition +** that after checkpointing the log file it blocks (calls the +** [busy-handler callback]) +** until all readers are reading from the database file only. ^This ensures +** that the next writer will restart the log file from the beginning. +** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new +** database writer attempts while it is pending, but does not impede readers. +** +**
    SQLITE_CHECKPOINT_TRUNCATE
    +** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the +** addition that it also truncates the log file to zero bytes just prior +** to a successful return. +**
    +** +** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file or to -1 if the checkpoint could not run because +** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not +** NULL,then *pnCkpt is set to the total number of checkpointed frames in the +** log file (including any that were already checkpointed before the function +** was called) or to -1 if the checkpoint could not run due to an error or +** because the database is not in WAL mode. ^Note that upon successful +** completion of an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been +** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero. +** +** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If +** any other process is running a checkpoint operation at the same time, the +** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a +** busy-handler configured, it will not be invoked in this case. +** +** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. ^If the writer lock cannot be +** obtained immediately, and a busy-handler is configured, it is invoked and +** the writer lock retried until either the busy-handler returns 0 or the lock +** is successfully obtained. ^The busy-handler is also invoked while waiting for +** database readers as described above. ^If the busy-handler returns 0 before +** the writer lock is obtained or while waiting for database readers, the +** checkpoint operation proceeds from that point in the same way as +** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible +** without blocking any further. ^SQLITE_BUSY is returned in this case. +** +** ^If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases [attached] to +** [database connection] db. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. ^If +** an SQLITE_BUSY error is encountered when processing one or more of the +** attached WAL databases, the operation is still attempted on any remaining +** attached databases and SQLITE_BUSY is returned at the end. ^If any other +** error occurs while processing an attached database, processing is abandoned +** and the error code is returned to the caller immediately. ^If no error +** (SQLITE_BUSY or otherwise) is encountered while processing the attached +** databases, SQLITE_OK is returned. +** +** ^If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. ^If +** zDb is not NULL (or a zero length string) and is not the name of any +** attached database, SQLITE_ERROR is returned to the caller. +** +** ^Unless it returns SQLITE_MISUSE, +** the sqlite3_wal_checkpoint_v2() interface +** sets the error information that is queried by +** [sqlite3_errcode()] and [sqlite3_errmsg()]. +** +** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface +** from SQL. +*/ +SQLITE_API int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +); + +/* +** CAPI3REF: Checkpoint Mode Values +** KEYWORDS: {checkpoint mode} +** +** These constants define all valid values for the "checkpoint mode" passed +** as the third parameter to the [sqlite3_wal_checkpoint_v2()] interface. +** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the +** meaning of each of these checkpoint modes. +*/ +#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ +#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ + +/* +** CAPI3REF: Virtual Table Interface Configuration +** +** This function may be called by either the [xConnect] or [xCreate] method +** of a [virtual table] implementation to configure +** various facets of the virtual table interface. +** +** If this interface is invoked outside the context of an xConnect or +** xCreate virtual table method then the behavior is undefined. +** +** In the call sqlite3_vtab_config(D,C,...) the D parameter is the +** [database connection] in which the virtual table is being created and +** which is passed in as the first argument to the [xConnect] or [xCreate] +** method that is invoking sqlite3_vtab_config(). The C parameter is one +** of the [virtual table configuration options]. The presence and meaning +** of parameters after C depend on which [virtual table configuration option] +** is used. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Virtual Table Configuration Options +** KEYWORDS: {virtual table configuration options} +** KEYWORDS: {virtual table configuration option} +** +** These macros define the various options to the +** [sqlite3_vtab_config()] interface that [virtual table] implementations +** can use to customize and optimize their behavior. +** +**
    +** [[SQLITE_VTAB_CONSTRAINT_SUPPORT]] +**
    SQLITE_VTAB_CONSTRAINT_SUPPORT
    +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, +** where X is an integer. If X is zero, then the [virtual table] whose +** [xCreate] or [xConnect] method invoked [sqlite3_vtab_config()] does not +** support constraints. In this configuration (which is the default) if +** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire +** statement is rolled back as if [ON CONFLICT | OR ABORT] had been +** specified as part of the users SQL statement, regardless of the actual +** ON CONFLICT mode specified. +** +** If X is non-zero, then the virtual table implementation guarantees +** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before +** any modifications to internal or persistent data structures have been made. +** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite +** is able to roll back a statement or database transaction, and abandon +** or continue processing the current SQL statement as appropriate. +** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns +** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode +** had been ABORT. +** +** Virtual table implementations that are required to handle OR REPLACE +** must do so within the [xUpdate] method. If a call to the +** [sqlite3_vtab_on_conflict()] function indicates that the current ON +** CONFLICT policy is REPLACE, the virtual table implementation should +** silently replace the appropriate rows within the xUpdate callback and +** return SQLITE_OK. Or, if this is not possible, it may return +** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT +** constraint handling. +**
    +** +** [[SQLITE_VTAB_DIRECTONLY]]
    SQLITE_VTAB_DIRECTONLY
    +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** prohibits that virtual table from being used from within triggers and +** views. +**
    +** +** [[SQLITE_VTAB_INNOCUOUS]]
    SQLITE_VTAB_INNOCUOUS
    +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** identify that virtual table as being safe to use from within triggers +** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the +** virtual table can do no serious harm even if it is controlled by a +** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS +** flag unless absolutely necessary. +**
    +**
    +*/ +#define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 +#define SQLITE_VTAB_INNOCUOUS 2 +#define SQLITE_VTAB_DIRECTONLY 3 + +/* +** CAPI3REF: Determine The Virtual Table Conflict Policy +** +** This function may only be called from within a call to the [xUpdate] method +** of a [virtual table] implementation for an INSERT or UPDATE operation. ^The +** value returned is one of [SQLITE_ROLLBACK], [SQLITE_IGNORE], [SQLITE_FAIL], +** [SQLITE_ABORT], or [SQLITE_REPLACE], according to the [ON CONFLICT] mode +** of the SQL statement that triggered the call to the [xUpdate] method of the +** [virtual table]. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); + +/* +** CAPI3REF: Determine If Virtual Table Column Access Is For UPDATE +** +** If the sqlite3_vtab_nochange(X) routine is called within the [xColumn] +** method of a [virtual table], then it might return true if the +** column is being fetched as part of an UPDATE operation during which the +** column value will not change. The virtual table implementation can use +** this hint as permission to substitute a return value that is less +** expensive to compute and that the corresponding +** [xUpdate] method understands as a "no-change" value. +** +** If the [xColumn] method calls sqlite3_vtab_nochange() and finds that +** the column is not changed by the UPDATE statement, then the xColumn +** method can optionally return without setting a result, without calling +** any of the [sqlite3_result_int|sqlite3_result_xxxxx() interfaces]. +** In that case, [sqlite3_value_nochange(X)] will return true for the +** same column in the [xUpdate] method. +** +** The sqlite3_vtab_nochange() routine is an optimization. Virtual table +** implementations should continue to give a correct answer even if the +** sqlite3_vtab_nochange() interface were to always return false. In the +** current implementation, the sqlite3_vtab_nochange() interface does always +** returns false for the enhanced [UPDATE FROM] statement. +*/ +SQLITE_API int sqlite3_vtab_nochange(sqlite3_context*); + +/* +** CAPI3REF: Determine The Collation For a Virtual Table Constraint +** +** This function may only be called from within a call to the [xBestIndex] +** method of a [virtual table]. +** +** The first argument must be the sqlite3_index_info object that is the +** first parameter to the xBestIndex() method. The second argument must be +** an index into the aConstraint[] array belonging to the sqlite3_index_info +** structure passed to xBestIndex. This function returns a pointer to a buffer +** containing the name of the collation sequence for the corresponding +** constraint. +*/ +SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_info*,int); + +/* +** CAPI3REF: Conflict resolution modes +** KEYWORDS: {conflict resolution mode} +** +** These constants are returned by [sqlite3_vtab_on_conflict()] to +** inform a [virtual table] implementation what the [ON CONFLICT] mode +** is for the SQL statement being evaluated. +** +** Note that the [SQLITE_IGNORE] constant is also used as a potential +** return value from the [sqlite3_set_authorizer()] callback and that +** [SQLITE_ABORT] is also a [result code]. +*/ +#define SQLITE_ROLLBACK 1 +/* #define SQLITE_IGNORE 2 // Also used by sqlite3_authorizer() callback */ +#define SQLITE_FAIL 3 +/* #define SQLITE_ABORT 4 // Also an error code */ +#define SQLITE_REPLACE 5 + +/* +** CAPI3REF: Prepared Statement Scan Status Opcodes +** KEYWORDS: {scanstatus options} +** +** The following constants can be used for the T parameter to the +** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a +** different metric for sqlite3_stmt_scanstatus() to return. +** +** When the value returned to V is a string, space to hold that string is +** managed by the prepared statement S and will be automatically freed when +** S is finalized. +** +**
    +** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    +**
    ^The [sqlite3_int64] variable pointed to by the V parameter will be +** set to the total number of times that the X-th loop has run.
    +** +** [[SQLITE_SCANSTAT_NVISIT]]
    SQLITE_SCANSTAT_NVISIT
    +**
    ^The [sqlite3_int64] variable pointed to by the V parameter will be set +** to the total number of rows examined by all iterations of the X-th loop.
    +** +** [[SQLITE_SCANSTAT_EST]]
    SQLITE_SCANSTAT_EST
    +**
    ^The "double" variable pointed to by the V parameter will be set to the +** query planner's estimate for the average number of rows output from each +** iteration of the X-th loop. If the query planner's estimates was accurate, +** then this value will approximate the quotient NVISIT/NLOOP and the +** product of this value for all prior loops with the same SELECTID will +** be the NLOOP value for the current loop. +** +** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    +**
    ^The "const char *" variable pointed to by the V parameter will be set +** to a zero-terminated UTF-8 string containing the name of the index or table +** used for the X-th loop. +** +** [[SQLITE_SCANSTAT_EXPLAIN]]
    SQLITE_SCANSTAT_EXPLAIN
    +**
    ^The "const char *" variable pointed to by the V parameter will be set +** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] +** description for the X-th loop. +** +** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    +**
    ^The "int" variable pointed to by the V parameter will be set to the +** "select-id" for the X-th loop. The select-id identifies which query or +** subquery the loop is part of. The main query has a select-id of zero. +** The select-id is the same value as is output in the first column +** of an [EXPLAIN QUERY PLAN] query. +**
    +*/ +#define SQLITE_SCANSTAT_NLOOP 0 +#define SQLITE_SCANSTAT_NVISIT 1 +#define SQLITE_SCANSTAT_EST 2 +#define SQLITE_SCANSTAT_NAME 3 +#define SQLITE_SCANSTAT_EXPLAIN 4 +#define SQLITE_SCANSTAT_SELECTID 5 + +/* +** CAPI3REF: Prepared Statement Scan Status +** METHOD: sqlite3_stmt +** +** This interface returns information about the predicted and measured +** performance for pStmt. Advanced applications can use this +** interface to compare the predicted and the measured performance and +** issue warnings and/or rerun [ANALYZE] if discrepancies are found. +** +** Since this interface is expected to be rarely used, it is only +** available if SQLite is compiled using the [SQLITE_ENABLE_STMT_SCANSTATUS] +** compile-time option. +** +** The "iScanStatusOp" parameter determines which status information to return. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior +** of this interface is undefined. +** ^The requested measurement is written into a variable pointed to by +** the "pOut" parameter. +** Parameter "idx" identifies the specific loop to retrieve statistics for. +** Loops are numbered starting from zero. ^If idx is out of range - less than +** zero or greater than or equal to the total number of loops used to implement +** the statement - a non-zero value is returned and the variable that pOut +** points to is unchanged. +** +** ^Statistics might not be available for all loops in all statements. ^In cases +** where there exist loops with no available statistics, this function behaves +** as if the loop did not exist - it returns non-zero and leave the variable +** that pOut points to unchanged. +** +** See also: [sqlite3_stmt_scanstatus_reset()] +*/ +SQLITE_API int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Zero Scan-Status Counters +** METHOD: sqlite3_stmt +** +** ^Zero all [sqlite3_stmt_scanstatus()] related event counters. +** +** This API is only available if the library is built with pre-processor +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. +*/ +SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); + +/* +** CAPI3REF: Flush caches to disk mid-transaction +** METHOD: sqlite3 +** +** ^If a write-transaction is open on [database connection] D when the +** [sqlite3_db_cacheflush(D)] interface invoked, any dirty +** pages in the pager-cache that are not currently in use are written out +** to disk. A dirty page may be in use if a database cursor created by an +** active SQL statement is reading from it, or if it is page 1 of a database +** file (page 1 is always "in use"). ^The [sqlite3_db_cacheflush(D)] +** interface flushes caches for all schemas - "main", "temp", and +** any [attached] databases. +** +** ^If this function needs to obtain extra database locks before dirty pages +** can be flushed to disk, it does so. ^If those locks cannot be obtained +** immediately and there is a busy-handler callback configured, it is invoked +** in the usual manner. ^If the required lock still cannot be obtained, then +** the database is skipped and an attempt made to flush any dirty pages +** belonging to the next (if any) database. ^If any databases are skipped +** because locks cannot be obtained, but no other error occurs, this +** function returns SQLITE_BUSY. +** +** ^If any other error occurs while flushing dirty pages to disk (for +** example an IO error or out-of-memory condition), then processing is +** abandoned and an SQLite [error code] is returned to the caller immediately. +** +** ^Otherwise, if no error occurs, [sqlite3_db_cacheflush()] returns SQLITE_OK. +** +** ^This function does not set the database handle error code or message +** returned by the [sqlite3_errcode()] and [sqlite3_errmsg()] functions. +*/ +SQLITE_API int sqlite3_db_cacheflush(sqlite3*); + +/* +** CAPI3REF: The pre-update hook. +** METHOD: sqlite3 +** +** ^These interfaces are only available if SQLite is compiled using the +** [SQLITE_ENABLE_PREUPDATE_HOOK] compile-time option. +** +** ^The [sqlite3_preupdate_hook()] interface registers a callback function +** that is invoked prior to each [INSERT], [UPDATE], and [DELETE] operation +** on a database table. +** ^At most one preupdate hook may be registered at a time on a single +** [database connection]; each call to [sqlite3_preupdate_hook()] overrides +** the previous setting. +** ^The preupdate hook is disabled by invoking [sqlite3_preupdate_hook()] +** with a NULL pointer as the second parameter. +** ^The third parameter to [sqlite3_preupdate_hook()] is passed through as +** the first parameter to callbacks. +** +** ^The preupdate hook only fires for changes to real database tables; the +** preupdate hook is not invoked for changes to [virtual tables] or to +** system tables like sqlite_sequence or sqlite_stat1. +** +** ^The second parameter to the preupdate callback is a pointer to +** the [database connection] that registered the preupdate hook. +** ^The third parameter to the preupdate callback is one of the constants +** [SQLITE_INSERT], [SQLITE_DELETE], or [SQLITE_UPDATE] to identify the +** kind of update operation that is about to occur. +** ^(The fourth parameter to the preupdate callback is the name of the +** database within the database connection that is being modified. This +** will be "main" for the main database or "temp" for TEMP tables or +** the name given after the AS keyword in the [ATTACH] statement for attached +** databases.)^ +** ^The fifth parameter to the preupdate callback is the name of the +** table that is being modified. +** +** For an UPDATE or DELETE operation on a [rowid table], the sixth +** parameter passed to the preupdate callback is the initial [rowid] of the +** row being modified or deleted. For an INSERT operation on a rowid table, +** or any operation on a WITHOUT ROWID table, the value of the sixth +** parameter is undefined. For an INSERT or UPDATE on a rowid table the +** seventh parameter is the final rowid value of the row being inserted +** or updated. The value of the seventh parameter passed to the callback +** function is not defined for operations on WITHOUT ROWID tables, or for +** DELETE operations on rowid tables. +** +** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()], +** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces +** provide additional information about a preupdate event. These routines +** may only be called from within a preupdate callback. Invoking any of +** these routines from outside of a preupdate callback or with a +** [database connection] pointer that is different from the one supplied +** to the preupdate callback results in undefined and probably undesirable +** behavior. +** +** ^The [sqlite3_preupdate_count(D)] interface returns the number of columns +** in the row that is being inserted, updated, or deleted. +** +** ^The [sqlite3_preupdate_old(D,N,P)] interface writes into P a pointer to +** a [protected sqlite3_value] that contains the value of the Nth column of +** the table row before it is updated. The N parameter must be between 0 +** and one less than the number of columns or the behavior will be +** undefined. This must only be used within SQLITE_UPDATE and SQLITE_DELETE +** preupdate callbacks; if it is used by an SQLITE_INSERT callback then the +** behavior is undefined. The [sqlite3_value] that P points to +** will be destroyed when the preupdate callback returns. +** +** ^The [sqlite3_preupdate_new(D,N,P)] interface writes into P a pointer to +** a [protected sqlite3_value] that contains the value of the Nth column of +** the table row after it is updated. The N parameter must be between 0 +** and one less than the number of columns or the behavior will be +** undefined. This must only be used within SQLITE_INSERT and SQLITE_UPDATE +** preupdate callbacks; if it is used by an SQLITE_DELETE callback then the +** behavior is undefined. The [sqlite3_value] that P points to +** will be destroyed when the preupdate callback returns. +** +** ^The [sqlite3_preupdate_depth(D)] interface returns 0 if the preupdate +** callback was invoked as a result of a direct insert, update, or delete +** operation; or 1 for inserts, updates, or deletes invoked by top-level +** triggers; or 2 for changes resulting from triggers called by top-level +** triggers; and so forth. +** +** When the [sqlite3_blob_write()] API is used to update a blob column, +** the pre-update hook is invoked with SQLITE_DELETE. This is because the +** in this case the new values are not available. In this case, when a +** callback made with op==SQLITE_DELETE is actuall a write using the +** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns +** the index of the column being written. In other cases, where the +** pre-update hook is being invoked for some other reason, including a +** regular DELETE, sqlite3_preupdate_blobwrite() returns -1. +** +** See also: [sqlite3_update_hook()] +*/ +#if defined(SQLITE_ENABLE_PREUPDATE_HOOK) +SQLITE_API void *sqlite3_preupdate_hook( + sqlite3 *db, + void(*xPreUpdate)( + void *pCtx, /* Copy of third arg to preupdate_hook() */ + sqlite3 *db, /* Database handle */ + int op, /* SQLITE_UPDATE, DELETE or INSERT */ + char const *zDb, /* Database name */ + char const *zName, /* Table name */ + sqlite3_int64 iKey1, /* Rowid of row about to be deleted/updated */ + sqlite3_int64 iKey2 /* New rowid value (for a rowid UPDATE) */ + ), + void* +); +SQLITE_API int sqlite3_preupdate_old(sqlite3 *, int, sqlite3_value **); +SQLITE_API int sqlite3_preupdate_count(sqlite3 *); +SQLITE_API int sqlite3_preupdate_depth(sqlite3 *); +SQLITE_API int sqlite3_preupdate_new(sqlite3 *, int, sqlite3_value **); +SQLITE_API int sqlite3_preupdate_blobwrite(sqlite3 *); +#endif + +/* +** CAPI3REF: Low-level system error code +** METHOD: sqlite3 +** +** ^Attempt to return the underlying operating system error code or error +** number that caused the most recent I/O error or failure to open a file. +** The return value is OS-dependent. For example, on unix systems, after +** [sqlite3_open_v2()] returns [SQLITE_CANTOPEN], this interface could be +** called to get back the underlying "errno" that caused the problem, such +** as ENOSPC, EAUTH, EISDIR, and so forth. +*/ +SQLITE_API int sqlite3_system_errno(sqlite3*); + +/* +** CAPI3REF: Database Snapshot +** KEYWORDS: {snapshot} {sqlite3_snapshot} +** +** An instance of the snapshot object records the state of a [WAL mode] +** database for some specific point in history. +** +** In [WAL mode], multiple [database connections] that are open on the +** same database file can each be reading a different historical version +** of the database file. When a [database connection] begins a read +** transaction, that connection sees an unchanging copy of the database +** as it existed for the point in time when the transaction first started. +** Subsequent changes to the database from other connections are not seen +** by the reader until a new read transaction is started. +** +** The sqlite3_snapshot object records state information about an historical +** version of the database file so that it is possible to later open a new read +** transaction that sees that historical version of the database rather than +** the most recent version. +*/ +typedef struct sqlite3_snapshot { + unsigned char hidden[48]; +} sqlite3_snapshot; + +/* +** CAPI3REF: Record A Database Snapshot +** CONSTRUCTOR: sqlite3_snapshot +** +** ^The [sqlite3_snapshot_get(D,S,P)] interface attempts to make a +** new [sqlite3_snapshot] object that records the current state of +** schema S in database connection D. ^On success, the +** [sqlite3_snapshot_get(D,S,P)] interface writes a pointer to the newly +** created [sqlite3_snapshot] object into *P and returns SQLITE_OK. +** If there is not already a read-transaction open on schema S when +** this function is called, one is opened automatically. +** +** The following must be true for this function to succeed. If any of +** the following statements are false when sqlite3_snapshot_get() is +** called, SQLITE_ERROR is returned. The final value of *P is undefined +** in this case. +** +**
      +**
    • The database handle must not be in [autocommit mode]. +** +**
    • Schema S of [database connection] D must be a [WAL mode] database. +** +**
    • There must not be a write transaction open on schema S of database +** connection D. +** +**
    • One or more transactions must have been written to the current wal +** file since it was created on disk (by any connection). This means +** that a snapshot cannot be taken on a wal mode database with no wal +** file immediately after it is first opened. At least one transaction +** must be written to it first. +**
    +** +** This function may also return SQLITE_NOMEM. If it is called with the +** database handle in autocommit mode but fails for some other reason, +** whether or not a read transaction is opened on schema S is undefined. +** +** The [sqlite3_snapshot] object returned from a successful call to +** [sqlite3_snapshot_get()] must be freed using [sqlite3_snapshot_free()] +** to avoid a memory leak. +** +** The [sqlite3_snapshot_get()] interface is only available when the +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( + sqlite3 *db, + const char *zSchema, + sqlite3_snapshot **ppSnapshot +); + +/* +** CAPI3REF: Start a read transaction on an historical snapshot +** METHOD: sqlite3_snapshot +** +** ^The [sqlite3_snapshot_open(D,S,P)] interface either starts a new read +** transaction or upgrades an existing one for schema S of +** [database connection] D such that the read transaction refers to +** historical [snapshot] P, rather than the most recent change to the +** database. ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK +** on success or an appropriate [error code] if it fails. +** +** ^In order to succeed, the database connection must not be in +** [autocommit mode] when [sqlite3_snapshot_open(D,S,P)] is called. If there +** is already a read transaction open on schema S, then the database handle +** must have no active statements (SELECT statements that have been passed +** to sqlite3_step() but not sqlite3_reset() or sqlite3_finalize()). +** SQLITE_ERROR is returned if either of these conditions is violated, or +** if schema S does not exist, or if the snapshot object is invalid. +** +** ^A call to sqlite3_snapshot_open() will fail to open if the specified +** snapshot has been overwritten by a [checkpoint]. In this case +** SQLITE_ERROR_SNAPSHOT is returned. +** +** If there is already a read transaction open when this function is +** invoked, then the same read transaction remains open (on the same +** database snapshot) if SQLITE_ERROR, SQLITE_BUSY or SQLITE_ERROR_SNAPSHOT +** is returned. If another error code - for example SQLITE_PROTOCOL or an +** SQLITE_IOERR error code - is returned, then the final state of the +** read transaction is undefined. If SQLITE_OK is returned, then the +** read transaction is now open on database snapshot P. +** +** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the +** database connection D does not know that the database file for +** schema S is in [WAL mode]. A database connection might not know +** that the database file is in [WAL mode] if there has been no prior +** I/O on that database connection, or if the database entered [WAL mode] +** after the most recent I/O on the database connection.)^ +** (Hint: Run "[PRAGMA application_id]" against a newly opened +** database connection in order to make it ready to use snapshots.) +** +** The [sqlite3_snapshot_open()] interface is only available when the +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( + sqlite3 *db, + const char *zSchema, + sqlite3_snapshot *pSnapshot +); + +/* +** CAPI3REF: Destroy a snapshot +** DESTRUCTOR: sqlite3_snapshot +** +** ^The [sqlite3_snapshot_free(P)] interface destroys [sqlite3_snapshot] P. +** The application must eventually free every [sqlite3_snapshot] object +** using this routine to avoid a memory leak. +** +** The [sqlite3_snapshot_free()] interface is only available when the +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); + +/* +** CAPI3REF: Compare the ages of two snapshot handles. +** METHOD: sqlite3_snapshot +** +** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages +** of two valid snapshot handles. +** +** If the two snapshot handles are not associated with the same database +** file, the result of the comparison is undefined. +** +** Additionally, the result of the comparison is only valid if both of the +** snapshot handles were obtained by calling sqlite3_snapshot_get() since the +** last time the wal file was deleted. The wal file is deleted when the +** database is changed back to rollback mode or when the number of database +** clients drops to zero. If either snapshot handle was obtained before the +** wal file was last deleted, the value returned by this function +** is undefined. +** +** Otherwise, this API returns a negative value if P1 refers to an older +** snapshot than P2, zero if the two handles refer to the same database +** snapshot, and a positive value if P1 is a newer snapshot than P2. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( + sqlite3_snapshot *p1, + sqlite3_snapshot *p2 +); + +/* +** CAPI3REF: Recover snapshots from a wal file +** METHOD: sqlite3_snapshot +** +** If a [WAL file] remains on disk after all database connections close +** (either through the use of the [SQLITE_FCNTL_PERSIST_WAL] [file control] +** or because the last process to have the database opened exited without +** calling [sqlite3_close()]) and a new connection is subsequently opened +** on that database and [WAL file], the [sqlite3_snapshot_open()] interface +** will only be able to open the last transaction added to the WAL file +** even though the WAL file contains other valid transactions. +** +** This function attempts to scan the WAL file associated with database zDb +** of database handle db and make all valid snapshots available to +** sqlite3_snapshot_open(). It is an error if there is already a read +** transaction open on the database, or if the database is not a WAL mode +** database. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); + +/* +** CAPI3REF: Serialize a database +** +** The sqlite3_serialize(D,S,P,F) interface returns a pointer to memory +** that is a serialization of the S database on [database connection] D. +** If P is not a NULL pointer, then the size of the database in bytes +** is written into *P. +** +** For an ordinary on-disk database file, the serialization is just a +** copy of the disk file. For an in-memory database or a "TEMP" database, +** the serialization is the same sequence of bytes which would be written +** to disk if that database where backed up to disk. +** +** The usual case is that sqlite3_serialize() copies the serialization of +** the database into memory obtained from [sqlite3_malloc64()] and returns +** a pointer to that memory. The caller is responsible for freeing the +** returned value to avoid a memory leak. However, if the F argument +** contains the SQLITE_SERIALIZE_NOCOPY bit, then no memory allocations +** are made, and the sqlite3_serialize() function will return a pointer +** to the contiguous memory representation of the database that SQLite +** is currently using for that database, or NULL if the no such contiguous +** memory representation of the database exists. A contiguous memory +** representation of the database will usually only exist if there has +** been a prior call to [sqlite3_deserialize(D,S,...)] with the same +** values of D and S. +** The size of the database is written into *P even if the +** SQLITE_SERIALIZE_NOCOPY bit is set but no contiguous copy +** of the database exists. +** +** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the +** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory +** allocation error occurs. +** +** This interface is omitted if SQLite is compiled with the +** [SQLITE_OMIT_DESERIALIZE] option. +*/ +SQLITE_API unsigned char *sqlite3_serialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which DB to serialize. ex: "main", "temp", ... */ + sqlite3_int64 *piSize, /* Write size of the DB here, if not NULL */ + unsigned int mFlags /* Zero or more SQLITE_SERIALIZE_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3_serialize +** +** Zero or more of the following constants can be OR-ed together for +** the F argument to [sqlite3_serialize(D,S,P,F)]. +** +** SQLITE_SERIALIZE_NOCOPY means that [sqlite3_serialize()] will return +** a pointer to contiguous in-memory database that it is currently using, +** without making a copy of the database. If SQLite is not currently using +** a contiguous in-memory database, then this option causes +** [sqlite3_serialize()] to return a NULL pointer. SQLite will only be +** using a contiguous in-memory database if it has been initialized by a +** prior call to [sqlite3_deserialize()]. +*/ +#define SQLITE_SERIALIZE_NOCOPY 0x001 /* Do no memory allocations */ + +/* +** CAPI3REF: Deserialize a database +** +** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the +** [database connection] D to disconnect from database S and then +** reopen S as an in-memory database based on the serialization contained +** in P. The serialized database P is N bytes in size. M is the size of +** the buffer P, which might be larger than N. If M is larger than N, and +** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is +** permitted to add content to the in-memory database as long as the total +** size does not exceed M bytes. +** +** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will +** invoke sqlite3_free() on the serialization buffer when the database +** connection closes. If the SQLITE_DESERIALIZE_RESIZEABLE bit is set, then +** SQLite will try to increase the buffer size using sqlite3_realloc64() +** if writes on the database cause it to grow larger than M bytes. +** +** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the +** database is currently in a read transaction or is involved in a backup +** operation. +** +** It is not possible to deserialized into the TEMP database. If the +** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the +** function returns SQLITE_ERROR. +** +** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the +** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then +** [sqlite3_free()] is invoked on argument P prior to returning. +** +** This interface is omitted if SQLite is compiled with the +** [SQLITE_OMIT_DESERIALIZE] option. +*/ +SQLITE_API int sqlite3_deserialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which DB to reopen with the deserialization */ + unsigned char *pData, /* The serialized database content */ + sqlite3_int64 szDb, /* Number bytes in the deserialization */ + sqlite3_int64 szBuf, /* Total size of buffer pData[] */ + unsigned mFlags /* Zero or more SQLITE_DESERIALIZE_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3_deserialize() +** +** The following are allowed values for 6th argument (the F argument) to +** the [sqlite3_deserialize(D,S,P,N,M,F)] interface. +** +** The SQLITE_DESERIALIZE_FREEONCLOSE means that the database serialization +** in the P argument is held in memory obtained from [sqlite3_malloc64()] +** and that SQLite should take ownership of this memory and automatically +** free it when it has finished using it. Without this flag, the caller +** is responsible for freeing any dynamically allocated memory. +** +** The SQLITE_DESERIALIZE_RESIZEABLE flag means that SQLite is allowed to +** grow the size of the database using calls to [sqlite3_realloc64()]. This +** flag should only be used if SQLITE_DESERIALIZE_FREEONCLOSE is also used. +** Without this flag, the deserialized database cannot increase in size beyond +** the number of bytes specified by the M parameter. +** +** The SQLITE_DESERIALIZE_READONLY flag means that the deserialized database +** should be treated as read-only. +*/ +#define SQLITE_DESERIALIZE_FREEONCLOSE 1 /* Call sqlite3_free() on close */ +#define SQLITE_DESERIALIZE_RESIZEABLE 2 /* Resize using sqlite3_realloc64() */ +#define SQLITE_DESERIALIZE_READONLY 4 /* Database is read-only */ + +/* +** Undo the hack that converts floating point types to integer for +** builds on processors without floating point support. +*/ +#ifdef SQLITE_OMIT_FLOATING_POINT +# undef double +#endif + +#if 0 +} /* End of the 'extern "C"' block */ +#endif +#endif /* SQLITE3_H */ + +/******** Begin file sqlite3rtree.h *********/ +/* +** 2010 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE3RTREE_H_ +#define _SQLITE3RTREE_H_ + + +#if 0 +extern "C" { +#endif + +typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif + +/* +** Register a geometry callback named zGeom that can be used as part of an +** R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zGeom(... params ...) +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), + void *pContext +); + + +/* +** A pointer to a structure of the following type is passed as the first +** argument to callbacks registered using rtree_geometry_callback(). +*/ +struct sqlite3_rtree_geometry { + void *pContext; /* Copy of pContext passed to s_r_g_c() */ + int nParam; /* Size of array aParam[] */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ + void *pUser; /* Callback implementation user data */ + void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ +}; + +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visibility */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ + /* The following fields are only available in 3.8.11 and later */ + sqlite3_value **apSqlParam; /* Original SQL values of parameters */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE3RTREE_H_ */ + +/******** End of sqlite3rtree.h *********/ +/******** Begin file sqlite3session.h *********/ + +#if !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) +#define __SQLITESESSION_H_ 1 + +/* +** Make sure we can call this stuff from C++. +*/ +#if 0 +extern "C" { +#endif + + +/* +** CAPI3REF: Session Object Handle +** +** An instance of this object is a [session] that can be used to +** record changes to a database. +*/ +typedef struct sqlite3_session sqlite3_session; + +/* +** CAPI3REF: Changeset Iterator Handle +** +** An instance of this object acts as a cursor for iterating +** over the elements of a [changeset] or [patchset]. +*/ +typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; + +/* +** CAPI3REF: Create A New Session Object +** CONSTRUCTOR: sqlite3_session +** +** Create a new session object attached to database handle db. If successful, +** a pointer to the new object is written to *ppSession and SQLITE_OK is +** returned. If an error occurs, *ppSession is set to NULL and an SQLite +** error code (e.g. SQLITE_NOMEM) is returned. +** +** It is possible to create multiple session objects attached to a single +** database handle. +** +** Session objects created using this function should be deleted using the +** [sqlite3session_delete()] function before the database handle that they +** are attached to is itself closed. If the database handle is closed before +** the session object is deleted, then the results of calling any session +** module function, including [sqlite3session_delete()] on the session object +** are undefined. +** +** Because the session module uses the [sqlite3_preupdate_hook()] API, it +** is not possible for an application to register a pre-update hook on a +** database handle that has one or more session objects attached. Nor is +** it possible to create a session object attached to a database handle for +** which a pre-update hook is already defined. The results of attempting +** either of these things are undefined. +** +** The session object will be used to create changesets for tables in +** database zDb, where zDb is either "main", or "temp", or the name of an +** attached database. It is not an error if database zDb is not attached +** to the database when the session object is created. +*/ +SQLITE_API int sqlite3session_create( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (e.g. "main") */ + sqlite3_session **ppSession /* OUT: New session object */ +); + +/* +** CAPI3REF: Delete A Session Object +** DESTRUCTOR: sqlite3_session +** +** Delete a session object previously allocated using +** [sqlite3session_create()]. Once a session object has been deleted, the +** results of attempting to use pSession with any other session module +** function are undefined. +** +** Session objects must be deleted before the database handle to which they +** are attached is closed. Refer to the documentation for +** [sqlite3session_create()] for details. +*/ +SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); + +/* +** CAPIREF: Conigure a Session Object +** METHOD: sqlite3_session +** +** This method is used to configure a session object after it has been +** created. At present the only valid value for the second parameter is +** [SQLITE_SESSION_OBJCONFIG_SIZE]. +** +** Arguments for sqlite3session_object_config() +** +** The following values may passed as the the 4th parameter to +** sqlite3session_object_config(). +** +**
    SQLITE_SESSION_OBJCONFIG_SIZE
    +** This option is used to set, clear or query the flag that enables +** the [sqlite3session_changeset_size()] API. Because it imposes some +** computational overhead, this API is disabled by default. Argument +** pArg must point to a value of type (int). If the value is initially +** 0, then the sqlite3session_changeset_size() API is disabled. If it +** is greater than 0, then the same API is enabled. Or, if the initial +** value is less than zero, no change is made. In all cases the (int) +** variable is set to 1 if the sqlite3session_changeset_size() API is +** enabled following the current call, or 0 otherwise. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. +*/ +SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); + +/* +*/ +#define SQLITE_SESSION_OBJCONFIG_SIZE 1 + +/* +** CAPI3REF: Enable Or Disable A Session Object +** METHOD: sqlite3_session +** +** Enable or disable the recording of changes by a session object. When +** enabled, a session object records changes made to the database. When +** disabled - it does not. A newly created session object is enabled. +** Refer to the documentation for [sqlite3session_changeset()] for further +** details regarding how enabling and disabling a session object affects +** the eventual changesets. +** +** Passing zero to this function disables the session. Passing a value +** greater than zero enables it. Passing a value less than zero is a +** no-op, and may be used to query the current state of the session. +** +** The return value indicates the final state of the session object: 0 if +** the session is disabled, or 1 if it is enabled. +*/ +SQLITE_API int sqlite3session_enable(sqlite3_session *pSession, int bEnable); + +/* +** CAPI3REF: Set Or Clear the Indirect Change Flag +** METHOD: sqlite3_session +** +** Each change recorded by a session object is marked as either direct or +** indirect. A change is marked as indirect if either: +** +**
      +**
    • The session object "indirect" flag is set when the change is +** made, or +**
    • The change is made by an SQL trigger or foreign key action +** instead of directly as a result of a users SQL statement. +**
    +** +** If a single row is affected by more than one operation within a session, +** then the change is considered indirect if all operations meet the criteria +** for an indirect change above, or direct otherwise. +** +** This function is used to set, clear or query the session object indirect +** flag. If the second argument passed to this function is zero, then the +** indirect flag is cleared. If it is greater than zero, the indirect flag +** is set. Passing a value less than zero does not modify the current value +** of the indirect flag, and may be used to query the current state of the +** indirect flag for the specified session object. +** +** The return value indicates the final state of the indirect flag: 0 if +** it is clear, or 1 if it is set. +*/ +SQLITE_API int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); + +/* +** CAPI3REF: Attach A Table To A Session Object +** METHOD: sqlite3_session +** +** If argument zTab is not NULL, then it is the name of a table to attach +** to the session object passed as the first argument. All subsequent changes +** made to the table while the session object is enabled will be recorded. See +** documentation for [sqlite3session_changeset()] for further details. +** +** Or, if argument zTab is NULL, then changes are recorded for all tables +** in the database. If additional tables are added to the database (by +** executing "CREATE TABLE" statements) after this call is made, changes for +** the new tables are also recorded. +** +** Changes can only be recorded for tables that have a PRIMARY KEY explicitly +** defined as part of their CREATE TABLE statement. It does not matter if the +** PRIMARY KEY is an "INTEGER PRIMARY KEY" (rowid alias) or not. The PRIMARY +** KEY may consist of a single column, or may be a composite key. +** +** It is not an error if the named table does not exist in the database. Nor +** is it an error if the named table does not have a PRIMARY KEY. However, +** no changes will be recorded in either of these scenarios. +** +** Changes are not recorded for individual rows that have NULL values stored +** in one or more of their PRIMARY KEY columns. +** +** SQLITE_OK is returned if the call completes without error. Or, if an error +** occurs, an SQLite error code (e.g. SQLITE_NOMEM) is returned. +** +**

    Special sqlite_stat1 Handling

    +** +** As of SQLite version 3.22.0, the "sqlite_stat1" table is an exception to +** some of the rules above. In SQLite, the schema of sqlite_stat1 is: +** 
    +**        CREATE TABLE sqlite_stat1(tbl,idx,stat)
+**
    +** +** Even though sqlite_stat1 does not have a PRIMARY KEY, changes are +** recorded for it as if the PRIMARY KEY is (tbl,idx). Additionally, changes +** are recorded for rows for which (idx IS NULL) is true. However, for such +** rows a zero-length blob (SQL value X'') is stored in the changeset or +** patchset instead of a NULL value. This allows such changesets to be +** manipulated by legacy implementations of sqlite3changeset_invert(), +** concat() and similar. +** +** The sqlite3changeset_apply() function automatically converts the +** zero-length blob back to a NULL value when updating the sqlite_stat1 +** table. However, if the application calls sqlite3changeset_new(), +** sqlite3changeset_old() or sqlite3changeset_conflict on a changeset +** iterator directly (including on a changeset iterator passed to a +** conflict-handler callback) then the X'' value is returned. The application +** must translate X'' to NULL itself if required. +** +** Legacy (older than 3.22.0) versions of the sessions module cannot capture +** changes made to the sqlite_stat1 table. Legacy versions of the +** sqlite3changeset_apply() function silently ignore any modifications to the +** sqlite_stat1 table that are part of a changeset or patchset. +*/ +SQLITE_API int sqlite3session_attach( + sqlite3_session *pSession, /* Session object */ + const char *zTab /* Table name */ +); + +/* +** CAPI3REF: Set a table filter on a Session Object. +** METHOD: sqlite3_session +** +** The second argument (xFilter) is the "filter callback". For changes to rows +** in tables that are not attached to the Session object, the filter is called +** to determine whether changes to the table's rows should be tracked or not. +** If xFilter returns 0, changes are not tracked. Note that once a table is +** attached, xFilter will not be called again. +*/ +SQLITE_API void sqlite3session_table_filter( + sqlite3_session *pSession, /* Session object */ + int(*xFilter)( + void *pCtx, /* Copy of third arg to _filter_table() */ + const char *zTab /* Table name */ + ), + void *pCtx /* First argument passed to xFilter */ +); + +/* +** CAPI3REF: Generate A Changeset From A Session Object +** METHOD: sqlite3_session +** +** Obtain a changeset containing changes to the tables attached to the +** session object passed as the first argument. If successful, +** set *ppChangeset to point to a buffer containing the changeset +** and *pnChangeset to the size of the changeset in bytes before returning +** SQLITE_OK. If an error occurs, set both *ppChangeset and *pnChangeset to +** zero and return an SQLite error code. +** +** A changeset consists of zero or more INSERT, UPDATE and/or DELETE changes, +** each representing a change to a single row of an attached table. An INSERT +** change contains the values of each field of a new database row. A DELETE +** contains the original values of each field of a deleted database row. An +** UPDATE change contains the original values of each field of an updated +** database row along with the updated values for each updated non-primary-key +** column. It is not possible for an UPDATE change to represent a change that +** modifies the values of primary key columns. If such a change is made, it +** is represented in a changeset as a DELETE followed by an INSERT. +** +** Changes are not recorded for rows that have NULL values stored in one or +** more of their PRIMARY KEY columns. If such a row is inserted or deleted, +** no corresponding change is present in the changesets returned by this +** function. If an existing row with one or more NULL values stored in +** PRIMARY KEY columns is updated so that all PRIMARY KEY columns are non-NULL, +** only an INSERT is appears in the changeset. Similarly, if an existing row +** with non-NULL PRIMARY KEY values is updated so that one or more of its +** PRIMARY KEY columns are set to NULL, the resulting changeset contains a +** DELETE change only. +** +** The contents of a changeset may be traversed using an iterator created +** using the [sqlite3changeset_start()] API. A changeset may be applied to +** a database with a compatible schema using the [sqlite3changeset_apply()] +** API. +** +** Within a changeset generated by this function, all changes related to a +** single table are grouped together. In other words, when iterating through +** a changeset or when applying a changeset to a database, all changes related +** to a single table are processed before moving on to the next table. Tables +** are sorted in the same order in which they were attached (or auto-attached) +** to the sqlite3_session object. The order in which the changes related to +** a single table are stored is undefined. +** +** Following a successful call to this function, it is the responsibility of +** the caller to eventually free the buffer that *ppChangeset points to using +** [sqlite3_free()]. +** +**

    Changeset Generation

    +** +** Once a table has been attached to a session object, the session object +** records the primary key values of all new rows inserted into the table. +** It also records the original primary key and other column values of any +** deleted or updated rows. For each unique primary key value, data is only +** recorded once - the first time a row with said primary key is inserted, +** updated or deleted in the lifetime of the session. +** +** There is one exception to the previous paragraph: when a row is inserted, +** updated or deleted, if one or more of its primary key columns contain a +** NULL value, no record of the change is made. +** +** The session object therefore accumulates two types of records - those +** that consist of primary key values only (created when the user inserts +** a new record) and those that consist of the primary key values and the +** original values of other table columns (created when the users deletes +** or updates a record). +** +** When this function is called, the requested changeset is created using +** both the accumulated records and the current contents of the database +** file. Specifically: +** +**
      +**
    • For each record generated by an insert, the database is queried +** for a row with a matching primary key. If one is found, an INSERT +** change is added to the changeset. If no such row is found, no change +** is added to the changeset. +** +**
    • For each record generated by an update or delete, the database is +** queried for a row with a matching primary key. If such a row is +** found and one or more of the non-primary key fields have been +** modified from their original values, an UPDATE change is added to +** the changeset. Or, if no such row is found in the table, a DELETE +** change is added to the changeset. If there is a row with a matching +** primary key in the database, but all fields contain their original +** values, no change is added to the changeset. +**
    +** +** This means, amongst other things, that if a row is inserted and then later +** deleted while a session object is active, neither the insert nor the delete +** will be present in the changeset. Or if a row is deleted and then later a +** row with the same primary key values inserted while a session object is +** active, the resulting changeset will contain an UPDATE change instead of +** a DELETE and an INSERT. +** +** When a session object is disabled (see the [sqlite3session_enable()] API), +** it does not accumulate records when rows are inserted, updated or deleted. +** This may appear to have some counter-intuitive effects if a single row +** is written to more than once during a session. For example, if a row +** is inserted while a session object is enabled, then later deleted while +** the same session object is disabled, no INSERT record will appear in the +** changeset, even though the delete took place while the session was disabled. +** Or, if one field of a row is updated while a session is disabled, and +** another field of the same row is updated while the session is enabled, the +** resulting changeset will contain an UPDATE change that updates both fields. +*/ +SQLITE_API int sqlite3session_changeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +); + +/* +** CAPI3REF: Return An Upper-limit For The Size Of The Changeset +** METHOD: sqlite3_session +** +** By default, this function always returns 0. For it to return +** a useful result, the sqlite3_session object must have been configured +** to enable this API using sqlite3session_object_config() with the +** SQLITE_SESSION_OBJCONFIG_SIZE verb. +** +** When enabled, this function returns an upper limit, in bytes, for the size +** of the changeset that might be produced if sqlite3session_changeset() were +** called. The final changeset size might be equal to or smaller than the +** size in bytes returned by this function. +*/ +SQLITE_API sqlite3_int64 sqlite3session_changeset_size(sqlite3_session *pSession); + +/* +** CAPI3REF: Load The Difference Between Tables Into A Session +** METHOD: sqlite3_session +** +** If it is not already attached to the session object passed as the first +** argument, this function attaches table zTbl in the same manner as the +** [sqlite3session_attach()] function. If zTbl does not exist, or if it +** does not have a primary key, this function is a no-op (but does not return +** an error). +** +** Argument zFromDb must be the name of a database ("main", "temp" etc.) +** attached to the same database handle as the session object that contains +** a table compatible with the table attached to the session by this function. +** A table is considered compatible if it: +** +**
      +**
    • Has the same name, +**
    • Has the same set of columns declared in the same order, and +**
    • Has the same PRIMARY KEY definition. +**
    +** +** If the tables are not compatible, SQLITE_SCHEMA is returned. If the tables +** are compatible but do not have any PRIMARY KEY columns, it is not an error +** but no changes are added to the session object. As with other session +** APIs, tables without PRIMARY KEYs are simply ignored. +** +** This function adds a set of changes to the session object that could be +** used to update the table in database zFrom (call this the "from-table") +** so that its content is the same as the table attached to the session +** object (call this the "to-table"). Specifically: +** +**
      +**
    • For each row (primary key) that exists in the to-table but not in +** the from-table, an INSERT record is added to the session object. +** +**
    • For each row (primary key) that exists in the to-table but not in +** the from-table, a DELETE record is added to the session object. +** +**
    • For each row (primary key) that exists in both tables, but features +** different non-PK values in each, an UPDATE record is added to the +** session. +**
    +** +** To clarify, if this function is called and then a changeset constructed +** using [sqlite3session_changeset()], then after applying that changeset to +** database zFrom the contents of the two compatible tables would be +** identical. +** +** It an error if database zFrom does not exist or does not contain the +** required compatible table. +** +** If the operation is successful, SQLITE_OK is returned. Otherwise, an SQLite +** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to free this buffer using +** sqlite3_free(). +*/ +SQLITE_API int sqlite3session_diff( + sqlite3_session *pSession, + const char *zFromDb, + const char *zTbl, + char **pzErrMsg +); + + +/* +** CAPI3REF: Generate A Patchset From A Session Object +** METHOD: sqlite3_session +** +** The differences between a patchset and a changeset are that: +** +**
      +**
    • DELETE records consist of the primary key fields only. The +** original values of other fields are omitted. +**
    • The original values of any modified fields are omitted from +** UPDATE records. +**
    +** +** A patchset blob may be used with up to date versions of all +** sqlite3changeset_xxx API functions except for sqlite3changeset_invert(), +** which returns SQLITE_CORRUPT if it is passed a patchset. Similarly, +** attempting to use a patchset blob with old versions of the +** sqlite3changeset_xxx APIs also provokes an SQLITE_CORRUPT error. +** +** Because the non-primary key "old.*" fields are omitted, no +** SQLITE_CHANGESET_DATA conflicts can be detected or reported if a patchset +** is passed to the sqlite3changeset_apply() API. Other conflict types work +** in the same way as for changesets. +** +** Changes within a patchset are ordered in the same way as for changesets +** generated by the sqlite3session_changeset() function (i.e. all changes for +** a single table are grouped together, tables appear in the order in which +** they were attached to the session object). +*/ +SQLITE_API int sqlite3session_patchset( + sqlite3_session *pSession, /* Session object */ + int *pnPatchset, /* OUT: Size of buffer at *ppPatchset */ + void **ppPatchset /* OUT: Buffer containing patchset */ +); + +/* +** CAPI3REF: Test if a changeset has recorded any changes. +** +** Return non-zero if no changes to attached tables have been recorded by +** the session object passed as the first argument. Otherwise, if one or +** more changes have been recorded, return zero. +** +** Even if this function returns zero, it is possible that calling +** [sqlite3session_changeset()] on the session handle may still return a +** changeset that contains no changes. This can happen when a row in +** an attached table is modified and then later on the original values +** are restored. However, if this function returns non-zero, then it is +** guaranteed that a call to sqlite3session_changeset() will return a +** changeset containing zero changes. +*/ +SQLITE_API int sqlite3session_isempty(sqlite3_session *pSession); + +/* +** CAPI3REF: Query for the amount of heap memory used by a session object. +** +** This API returns the total amount of heap memory in bytes currently +** used by the session object passed as the only argument. +*/ +SQLITE_API sqlite3_int64 sqlite3session_memory_used(sqlite3_session *pSession); + +/* +** CAPI3REF: Create An Iterator To Traverse A Changeset +** CONSTRUCTOR: sqlite3_changeset_iter +** +** Create an iterator used to iterate through the contents of a changeset. +** If successful, *pp is set to point to the iterator handle and SQLITE_OK +** is returned. Otherwise, if an error occurs, *pp is set to zero and an +** SQLite error code is returned. +** +** The following functions can be used to advance and query a changeset +** iterator created by this function: +** +**
      +**
    • [sqlite3changeset_next()] +**
    • [sqlite3changeset_op()] +**
    • [sqlite3changeset_new()] +**
    • [sqlite3changeset_old()] +**
    +** +** It is the responsibility of the caller to eventually destroy the iterator +** by passing it to [sqlite3changeset_finalize()]. The buffer containing the +** changeset (pChangeset) must remain valid until after the iterator is +** destroyed. +** +** Assuming the changeset blob was created by one of the +** [sqlite3session_changeset()], [sqlite3changeset_concat()] or +** [sqlite3changeset_invert()] functions, all changes within the changeset +** that apply to a single table are grouped together. This means that when +** an application iterates through a changeset using an iterator created by +** this function, all changes that relate to a single table are visited +** consecutively. There is no chance that the iterator will visit a change +** the applies to table X, then one for table Y, and then later on visit +** another change for table X. +** +** The behavior of sqlite3changeset_start_v2() and its streaming equivalent +** may be modified by passing a combination of +** [SQLITE_CHANGESETSTART_INVERT | supported flags] as the 4th parameter. +** +** Note that the sqlite3changeset_start_v2() API is still experimental +** and therefore subject to change. +*/ +SQLITE_API int sqlite3changeset_start( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset /* Pointer to blob containing changeset */ +); +SQLITE_API int sqlite3changeset_start_v2( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset, /* Pointer to blob containing changeset */ + int flags /* SESSION_CHANGESETSTART_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3changeset_start_v2 +** +** The following flags may passed via the 4th parameter to +** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]: +** +**
    SQLITE_CHANGESETAPPLY_INVERT
    +** Invert the changeset while iterating through it. This is equivalent to +** inverting a changeset using sqlite3changeset_invert() before applying it. +** It is an error to specify this flag with a patchset. +*/ +#define SQLITE_CHANGESETSTART_INVERT 0x0002 + + +/* +** CAPI3REF: Advance A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** This function may only be used with iterators created by the function +** [sqlite3changeset_start()]. If it is called on an iterator passed to +** a conflict-handler callback by [sqlite3changeset_apply()], SQLITE_MISUSE +** is returned and the call has no effect. +** +** Immediately after an iterator is created by sqlite3changeset_start(), it +** does not point to any change in the changeset. Assuming the changeset +** is not empty, the first call to this function advances the iterator to +** point to the first change in the changeset. Each subsequent call advances +** the iterator to point to the next change in the changeset (if any). If +** no error occurs and the iterator points to a valid change after a call +** to sqlite3changeset_next() has advanced it, SQLITE_ROW is returned. +** Otherwise, if all changes in the changeset have already been visited, +** SQLITE_DONE is returned. +** +** If an error occurs, an SQLite error code is returned. Possible error +** codes include SQLITE_CORRUPT (if the changeset buffer is corrupt) or +** SQLITE_NOMEM. +*/ +SQLITE_API int sqlite3changeset_next(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Obtain The Current Operation From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned [SQLITE_ROW]. If this +** is not the case, this function returns [SQLITE_MISUSE]. +** +** Arguments pOp, pnCol and pzTab may not be NULL. Upon return, three +** outputs are set through these pointers: +** +** *pOp is set to one of [SQLITE_INSERT], [SQLITE_DELETE] or [SQLITE_UPDATE], +** depending on the type of change that the iterator currently points to; +** +** *pnCol is set to the number of columns in the table affected by the change; and +** +** *pzTab is set to point to a nul-terminated utf-8 encoded string containing +** the name of the table affected by the current change. The buffer remains +** valid until either sqlite3changeset_next() is called on the iterator +** or until the conflict-handler function returns. +** +** If pbIndirect is not NULL, then *pbIndirect is set to true (1) if the change +** is an indirect change, or false (0) otherwise. See the documentation for +** [sqlite3session_indirect()] for a description of direct and indirect +** changes. +** +** If no error occurs, SQLITE_OK is returned. If an error does occur, an +** SQLite error code is returned. The values of the output variables may not +** be trusted in this case. +*/ +SQLITE_API int sqlite3changeset_op( + sqlite3_changeset_iter *pIter, /* Iterator object */ + const char **pzTab, /* OUT: Pointer to table name */ + int *pnCol, /* OUT: Number of columns in table */ + int *pOp, /* OUT: SQLITE_INSERT, DELETE or UPDATE */ + int *pbIndirect /* OUT: True for an 'indirect' change */ +); + +/* +** CAPI3REF: Obtain The Primary Key Definition Of A Table +** METHOD: sqlite3_changeset_iter +** +** For each modified table, a changeset includes the following: +** +**
      +**
    • The number of columns in the table, and +**
    • Which of those columns make up the tables PRIMARY KEY. +**
    +** +** This function is used to find which columns comprise the PRIMARY KEY of +** the table modified by the change that iterator pIter currently points to. +** If successful, *pabPK is set to point to an array of nCol entries, where +** nCol is the number of columns in the table. Elements of *pabPK are set to +** 0x01 if the corresponding column is part of the tables primary key, or +** 0x00 if it is not. +** +** If argument pnCol is not NULL, then *pnCol is set to the number of columns +** in the table. +** +** If this function is called when the iterator does not point to a valid +** entry, SQLITE_MISUSE is returned and the output variables zeroed. Otherwise, +** SQLITE_OK is returned and the output variables populated as described +** above. +*/ +SQLITE_API int sqlite3changeset_pk( + sqlite3_changeset_iter *pIter, /* Iterator object */ + unsigned char **pabPK, /* OUT: Array of boolean - true for PK cols */ + int *pnCol /* OUT: Number of entries in output array */ +); + +/* +** CAPI3REF: Obtain old.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_DELETE] or [SQLITE_UPDATE]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** original row values stored as part of the UPDATE or DELETE change and +** returns SQLITE_OK. The name of the function comes from the fact that this +** is similar to the "old.*" columns available to update or delete triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +SQLITE_API int sqlite3changeset_old( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Old value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain new.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_UPDATE] or [SQLITE_INSERT]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** new row values stored as part of the UPDATE or INSERT change and +** returns SQLITE_OK. If the change is an UPDATE and does not include +** a new value for the requested column, *ppValue is set to NULL and +** SQLITE_OK returned. The name of the function comes from the fact that +** this is similar to the "new.*" columns available to update or delete +** triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +SQLITE_API int sqlite3changeset_new( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: New value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** This function should only be used with iterator objects passed to a +** conflict-handler callback by [sqlite3changeset_apply()] with either +** [SQLITE_CHANGESET_DATA] or [SQLITE_CHANGESET_CONFLICT]. If this function +** is called on any other iterator, [SQLITE_MISUSE] is returned and *ppValue +** is set to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the +** "conflicting row" associated with the current conflict-handler callback +** and returns SQLITE_OK. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +SQLITE_API int sqlite3changeset_conflict( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Value from conflicting row */ +); + +/* +** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations +** METHOD: sqlite3_changeset_iter +** +** This function may only be called with an iterator passed to an +** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case +** it sets the output variable to the total number of known foreign key +** violations in the destination database and returns SQLITE_OK. +** +** In all other cases this function returns SQLITE_MISUSE. +*/ +SQLITE_API int sqlite3changeset_fk_conflicts( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int *pnOut /* OUT: Number of FK violations */ +); + + +/* +** CAPI3REF: Finalize A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** This function is used to finalize an iterator allocated with +** [sqlite3changeset_start()]. +** +** This function should only be called on iterators created using the +** [sqlite3changeset_start()] function. If an application calls this +** function with an iterator passed to a conflict-handler by +** [sqlite3changeset_apply()], [SQLITE_MISUSE] is immediately returned and the +** call has no effect. +** +** If an error was encountered within a call to an sqlite3changeset_xxx() +** function (for example an [SQLITE_CORRUPT] in [sqlite3changeset_next()] or an +** [SQLITE_NOMEM] in [sqlite3changeset_new()]) then an error code corresponding +** to that error is returned by this function. Otherwise, SQLITE_OK is +** returned. This is to allow the following pattern (pseudo-code): +** +** 
    +**   sqlite3changeset_start();
+**   while( SQLITE_ROW==sqlite3changeset_next() ){
+**     // Do something with change.
+**   }
+**   rc = sqlite3changeset_finalize();
+**   if( rc!=SQLITE_OK ){
+**     // An error has occurred
+**   }
+**
    +*/ +SQLITE_API int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Invert A Changeset +** +** This function is used to "invert" a changeset object. Applying an inverted +** changeset to a database reverses the effects of applying the uninverted +** changeset. Specifically: +** +**
      +**
    • Each DELETE change is changed to an INSERT, and +**
    • Each INSERT change is changed to a DELETE, and +**
    • For each UPDATE change, the old.* and new.* values are exchanged. +**
    +** +** This function does not change the order in which changes appear within +** the changeset. It merely reverses the sense of each individual change. +** +** If successful, a pointer to a buffer containing the inverted changeset +** is stored in *ppOut, the size of the same buffer is stored in *pnOut, and +** SQLITE_OK is returned. If an error occurs, both *pnOut and *ppOut are +** zeroed and an SQLite error code returned. +** +** It is the responsibility of the caller to eventually call sqlite3_free() +** on the *ppOut pointer to free the buffer allocation following a successful +** call to this function. +** +** WARNING/TODO: This function currently assumes that the input is a valid +** changeset. If it is not, the results are undefined. +*/ +SQLITE_API int sqlite3changeset_invert( + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + +/* +** CAPI3REF: Concatenate Two Changeset Objects +** +** This function is used to concatenate two changesets, A and B, into a +** single changeset. The result is a changeset equivalent to applying +** changeset A followed by changeset B. +** +** This function combines the two input changesets using an +** sqlite3_changegroup object. Calling it produces similar results as the +** following code fragment: +** +** 
    +**   sqlite3_changegroup *pGrp;
+**   rc = sqlite3_changegroup_new(&pGrp);
+**   if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA);
+**   if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nB, pB);
+**   if( rc==SQLITE_OK ){
+**     rc = sqlite3changegroup_output(pGrp, pnOut, ppOut);
+**   }else{
+**     *ppOut = 0;
+**     *pnOut = 0;
+**   }
+**
    +** +** Refer to the sqlite3_changegroup documentation below for details. +*/ +SQLITE_API int sqlite3changeset_concat( + int nA, /* Number of bytes in buffer pA */ + void *pA, /* Pointer to buffer containing changeset A */ + int nB, /* Number of bytes in buffer pB */ + void *pB, /* Pointer to buffer containing changeset B */ + int *pnOut, /* OUT: Number of bytes in output changeset */ + void **ppOut /* OUT: Buffer containing output changeset */ +); + + +/* +** CAPI3REF: Changegroup Handle +** +** A changegroup is an object used to combine two or more +** [changesets] or [patchsets] +*/ +typedef struct sqlite3_changegroup sqlite3_changegroup; + +/* +** CAPI3REF: Create A New Changegroup Object +** CONSTRUCTOR: sqlite3_changegroup +** +** An sqlite3_changegroup object is used to combine two or more changesets +** (or patchsets) into a single changeset (or patchset). A single changegroup +** object may combine changesets or patchsets, but not both. The output is +** always in the same format as the input. +** +** If successful, this function returns SQLITE_OK and populates (*pp) with +** a pointer to a new sqlite3_changegroup object before returning. The caller +** should eventually free the returned object using a call to +** sqlite3changegroup_delete(). If an error occurs, an SQLite error code +** (i.e. SQLITE_NOMEM) is returned and *pp is set to NULL. +** +** The usual usage pattern for an sqlite3_changegroup object is as follows: +** +**
      +**
    • It is created using a call to sqlite3changegroup_new(). +** +**
    • Zero or more changesets (or patchsets) are added to the object +** by calling sqlite3changegroup_add(). +** +**
    • The result of combining all input changesets together is obtained +** by the application via a call to sqlite3changegroup_output(). +** +**
    • The object is deleted using a call to sqlite3changegroup_delete(). +**
    +** +** Any number of calls to add() and output() may be made between the calls to +** new() and delete(), and in any order. +** +** As well as the regular sqlite3changegroup_add() and +** sqlite3changegroup_output() functions, also available are the streaming +** versions sqlite3changegroup_add_strm() and sqlite3changegroup_output_strm(). +*/ +SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); + +/* +** CAPI3REF: Add A Changeset To A Changegroup +** METHOD: sqlite3_changegroup +** +** Add all changes within the changeset (or patchset) in buffer pData (size +** nData bytes) to the changegroup. +** +** If the buffer contains a patchset, then all prior calls to this function +** on the same changegroup object must also have specified patchsets. Or, if +** the buffer contains a changeset, so must have the earlier calls to this +** function. Otherwise, SQLITE_ERROR is returned and no changes are added +** to the changegroup. +** +** Rows within the changeset and changegroup are identified by the values in +** their PRIMARY KEY columns. A change in the changeset is considered to +** apply to the same row as a change already present in the changegroup if +** the two rows have the same primary key. +** +** Changes to rows that do not already appear in the changegroup are +** simply copied into it. Or, if both the new changeset and the changegroup +** contain changes that apply to a single row, the final contents of the +** changegroup depends on the type of each change, as follows: +** +** +** +** +**
    Existing Change New Change Output Change +**
    INSERT INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    INSERT UPDATE +** The INSERT change remains in the changegroup. The values in the +** INSERT change are modified as if the row was inserted by the +** existing change and then updated according to the new change. +**
    INSERT DELETE +** The existing INSERT is removed from the changegroup. The DELETE is +** not added. +**
    UPDATE INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    UPDATE UPDATE +** The existing UPDATE remains within the changegroup. It is amended +** so that the accompanying values are as if the row was updated once +** by the existing change and then again by the new change. +**
    UPDATE DELETE +** The existing UPDATE is replaced by the new DELETE within the +** changegroup. +**
    DELETE INSERT +** If one or more of the column values in the row inserted by the +** new change differ from those in the row deleted by the existing +** change, the existing DELETE is replaced by an UPDATE within the +** changegroup. Otherwise, if the inserted row is exactly the same +** as the deleted row, the existing DELETE is simply discarded. +**
    DELETE UPDATE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    DELETE DELETE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
    +** +** If the new changeset contains changes to a table that is already present +** in the changegroup, then the number of columns and the position of the +** primary key columns for the table must be consistent. If this is not the +** case, this function fails with SQLITE_SCHEMA. If the input changeset +** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is +** returned. Or, if an out-of-memory condition occurs during processing, this +** function returns SQLITE_NOMEM. In all cases, if an error occurs the state +** of the final contents of the changegroup is undefined. +** +** If no error occurs, SQLITE_OK is returned. +*/ +SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); + +/* +** CAPI3REF: Obtain A Composite Changeset From A Changegroup +** METHOD: sqlite3_changegroup +** +** Obtain a buffer containing a changeset (or patchset) representing the +** current contents of the changegroup. If the inputs to the changegroup +** were themselves changesets, the output is a changeset. Or, if the +** inputs were patchsets, the output is also a patchset. +** +** As with the output of the sqlite3session_changeset() and +** sqlite3session_patchset() functions, all changes related to a single +** table are grouped together in the output of this function. Tables appear +** in the same order as for the very first changeset added to the changegroup. +** If the second or subsequent changesets added to the changegroup contain +** changes for tables that do not appear in the first changeset, they are +** appended onto the end of the output changeset, again in the order in +** which they are first encountered. +** +** If an error occurs, an SQLite error code is returned and the output +** variables (*pnData) and (*ppData) are set to 0. Otherwise, SQLITE_OK +** is returned and the output variables are set to the size of and a +** pointer to the output buffer, respectively. In this case it is the +** responsibility of the caller to eventually free the buffer using a +** call to sqlite3_free(). +*/ +SQLITE_API int sqlite3changegroup_output( + sqlite3_changegroup*, + int *pnData, /* OUT: Size of output buffer in bytes */ + void **ppData /* OUT: Pointer to output buffer */ +); + +/* +** CAPI3REF: Delete A Changegroup Object +** DESTRUCTOR: sqlite3_changegroup +*/ +SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); + +/* +** CAPI3REF: Apply A Changeset To A Database +** +** Apply a changeset or patchset to a database. These functions attempt to +** update the "main" database attached to handle db with the changes found in +** the changeset passed via the second and third arguments. +** +** The fourth argument (xFilter) passed to these functions is the "filter +** callback". If it is not NULL, then for each table affected by at least one +** change in the changeset, the filter callback is invoked with +** the table name as the second argument, and a copy of the context pointer +** passed as the sixth argument as the first. If the "filter callback" +** returns zero, then no attempt is made to apply any changes to the table. +** Otherwise, if the return value is non-zero or the xFilter argument to +** is NULL, all changes related to the table are attempted. +** +** For each table that is not excluded by the filter callback, this function +** tests that the target database contains a compatible table. A table is +** considered compatible if all of the following are true: +** +**
      +**
    • The table has the same name as the name recorded in the +** changeset, and +**
    • The table has at least as many columns as recorded in the +** changeset, and +**
    • The table has primary key columns in the same position as +** recorded in the changeset. +**
    +** +** If there is no compatible table, it is not an error, but none of the +** changes associated with the table are applied. A warning message is issued +** via the sqlite3_log() mechanism with the error code SQLITE_SCHEMA. At most +** one such warning is issued for each table in the changeset. +** +** For each change for which there is a compatible table, an attempt is made +** to modify the table contents according to the UPDATE, INSERT or DELETE +** change. If a change cannot be applied cleanly, the conflict handler +** function passed as the fifth argument to sqlite3changeset_apply() may be +** invoked. A description of exactly when the conflict handler is invoked for +** each type of change is below. +** +** Unlike the xFilter argument, xConflict may not be passed NULL. The results +** of passing anything other than a valid function pointer as the xConflict +** argument are undefined. +** +** Each time the conflict handler function is invoked, it must return one +** of [SQLITE_CHANGESET_OMIT], [SQLITE_CHANGESET_ABORT] or +** [SQLITE_CHANGESET_REPLACE]. SQLITE_CHANGESET_REPLACE may only be returned +** if the second argument passed to the conflict handler is either +** SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If the conflict-handler +** returns an illegal value, any changes already made are rolled back and +** the call to sqlite3changeset_apply() returns SQLITE_MISUSE. Different +** actions are taken by sqlite3changeset_apply() depending on the value +** returned by each invocation of the conflict-handler function. Refer to +** the documentation for the three +** [SQLITE_CHANGESET_OMIT|available return values] for details. +** +**
    +**
    DELETE Changes
    +** For each DELETE change, the function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all non-primary key columns also match the values stored in +** the changeset the row is deleted from the target database. +** +** If a row with matching primary key values is found, but one or more of +** the non-primary key fields contains a value different from the original +** row value stored in the changeset, the conflict-handler function is +** invoked with [SQLITE_CHANGESET_DATA] as the second argument. If the +** database table has more columns than are recorded in the changeset, +** only the values of those non-primary key fields are compared against +** the current database contents - any trailing database table columns +** are ignored. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the DELETE operation is attempted, but SQLite returns SQLITE_CONSTRAINT +** (which can only happen if a foreign key constraint is violated), the +** conflict-handler function is invoked with [SQLITE_CHANGESET_CONSTRAINT] +** passed as the second argument. This includes the case where the DELETE +** operation is attempted because an earlier call to the conflict handler +** function returned [SQLITE_CHANGESET_REPLACE]. +** +**
    INSERT Changes
    +** For each INSERT change, an attempt is made to insert the new row into +** the database. If the changeset row contains fewer fields than the +** database table, the trailing fields are populated with their default +** values. +** +** If the attempt to insert the row fails because the database already +** contains a row with the same primary key values, the conflict handler +** function is invoked with the second argument set to +** [SQLITE_CHANGESET_CONFLICT]. +** +** If the attempt to insert the row fails because of some other constraint +** violation (e.g. NOT NULL or UNIQUE), the conflict handler function is +** invoked with the second argument set to [SQLITE_CHANGESET_CONSTRAINT]. +** This includes the case where the INSERT operation is re-attempted because +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +** +**
    UPDATE Changes
    +** For each UPDATE change, the function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all modified non-primary key columns also match the values +** stored in the changeset the row is updated within the target database. +** +** If a row with matching primary key values is found, but one or more of +** the modified non-primary key fields contains a value different from an +** original row value stored in the changeset, the conflict-handler function +** is invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since +** UPDATE changes only contain values for non-primary key fields that are +** to be modified, only those fields need to match the original values to +** avoid the SQLITE_CHANGESET_DATA conflict-handler callback. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the UPDATE operation is attempted, but SQLite returns +** SQLITE_CONSTRAINT, the conflict-handler function is invoked with +** [SQLITE_CHANGESET_CONSTRAINT] passed as the second argument. +** This includes the case where the UPDATE operation is attempted after +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +**
    +** +** It is safe to execute SQL statements, including those that write to the +** table that the callback related to, from within the xConflict callback. +** This can be used to further customize the application's conflict +** resolution strategy. +** +** All changes made by these functions are enclosed in a savepoint transaction. +** If any other error (aside from a constraint failure when attempting to +** write to the target database) occurs, then the savepoint transaction is +** rolled back, restoring the target database to its original state, and an +** SQLite error code returned. +** +** If the output parameters (ppRebase) and (pnRebase) are non-NULL and +** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2() +** may set (*ppRebase) to point to a "rebase" that may be used with the +** sqlite3_rebaser APIs buffer before returning. In this case (*pnRebase) +** is set to the size of the buffer in bytes. It is the responsibility of the +** caller to eventually free any such buffer using sqlite3_free(). The buffer +** is only allocated and populated if one or more conflicts were encountered +** while applying the patchset. See comments surrounding the sqlite3_rebaser +** APIs for further details. +** +** The behavior of sqlite3changeset_apply_v2() and its streaming equivalent +** may be modified by passing a combination of +** [SQLITE_CHANGESETAPPLY_NOSAVEPOINT | supported flags] as the 9th parameter. +** +** Note that the sqlite3changeset_apply_v2() API is still experimental +** and therefore subject to change. +*/ +SQLITE_API int sqlite3changeset_apply( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); +SQLITE_API int sqlite3changeset_apply_v2( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, /* OUT: Rebase data */ + int flags /* SESSION_CHANGESETAPPLY_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3changeset_apply_v2 +** +** The following flags may passed via the 9th parameter to +** [sqlite3changeset_apply_v2] and [sqlite3changeset_apply_v2_strm]: +** +**
    +**
    SQLITE_CHANGESETAPPLY_NOSAVEPOINT
    +** Usually, the sessions module encloses all operations performed by +** a single call to apply_v2() or apply_v2_strm() in a [SAVEPOINT]. The +** SAVEPOINT is committed if the changeset or patchset is successfully +** applied, or rolled back if an error occurs. Specifying this flag +** causes the sessions module to omit this savepoint. In this case, if the +** caller has an open transaction or savepoint when apply_v2() is called, +** it may revert the partially applied changeset by rolling it back. +** +**
    SQLITE_CHANGESETAPPLY_INVERT
    +** Invert the changeset before applying it. This is equivalent to inverting +** a changeset using sqlite3changeset_invert() before applying it. It is +** an error to specify this flag with a patchset. +*/ +#define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 +#define SQLITE_CHANGESETAPPLY_INVERT 0x0002 + +/* +** CAPI3REF: Constants Passed To The Conflict Handler +** +** Values that may be passed as the second argument to a conflict-handler. +** +**
    +**
    SQLITE_CHANGESET_DATA
    +** The conflict handler is invoked with CHANGESET_DATA as the second argument +** when processing a DELETE or UPDATE change if a row with the required +** PRIMARY KEY fields is present in the database, but one or more other +** (non primary-key) fields modified by the update do not contain the +** expected "before" values. +** +** The conflicting row, in this case, is the database row with the matching +** primary key. +** +**
    SQLITE_CHANGESET_NOTFOUND
    +** The conflict handler is invoked with CHANGESET_NOTFOUND as the second +** argument when processing a DELETE or UPDATE change if a row with the +** required PRIMARY KEY fields is not present in the database. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
    SQLITE_CHANGESET_CONFLICT
    +** CHANGESET_CONFLICT is passed as the second argument to the conflict +** handler while processing an INSERT change if the operation would result +** in duplicate primary key values. +** +** The conflicting row in this case is the database row with the matching +** primary key. +** +**
    SQLITE_CHANGESET_FOREIGN_KEY
    +** If foreign key handling is enabled, and applying a changeset leaves the +** database in a state containing foreign key violations, the conflict +** handler is invoked with CHANGESET_FOREIGN_KEY as the second argument +** exactly once before the changeset is committed. If the conflict handler +** returns CHANGESET_OMIT, the changes, including those that caused the +** foreign key constraint violation, are committed. Or, if it returns +** CHANGESET_ABORT, the changeset is rolled back. +** +** No current or conflicting row information is provided. The only function +** it is possible to call on the supplied sqlite3_changeset_iter handle +** is sqlite3changeset_fk_conflicts(). +** +**
    SQLITE_CHANGESET_CONSTRAINT
    +** If any other constraint violation occurs while applying a change (i.e. +** a UNIQUE, CHECK or NOT NULL constraint), the conflict handler is +** invoked with CHANGESET_CONSTRAINT as the second argument. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
    +*/ +#define SQLITE_CHANGESET_DATA 1 +#define SQLITE_CHANGESET_NOTFOUND 2 +#define SQLITE_CHANGESET_CONFLICT 3 +#define SQLITE_CHANGESET_CONSTRAINT 4 +#define SQLITE_CHANGESET_FOREIGN_KEY 5 + +/* +** CAPI3REF: Constants Returned By The Conflict Handler +** +** A conflict handler callback must return one of the following three values. +** +**
    +**
    SQLITE_CHANGESET_OMIT
    +** If a conflict handler returns this value no special action is taken. The +** change that caused the conflict is not applied. The session module +** continues to the next change in the changeset. +** +**
    SQLITE_CHANGESET_REPLACE
    +** This value may only be returned if the second argument to the conflict +** handler was SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If this +** is not the case, any changes applied so far are rolled back and the +** call to sqlite3changeset_apply() returns SQLITE_MISUSE. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_DATA conflict +** handler, then the conflicting row is either updated or deleted, depending +** on the type of change. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_CONFLICT conflict +** handler, then the conflicting row is removed from the database and a +** second attempt to apply the change is made. If this second attempt fails, +** the original row is restored to the database before continuing. +** +**
    SQLITE_CHANGESET_ABORT
    +** If this value is returned, any changes applied so far are rolled back +** and the call to sqlite3changeset_apply() returns SQLITE_ABORT. +**
    +*/ +#define SQLITE_CHANGESET_OMIT 0 +#define SQLITE_CHANGESET_REPLACE 1 +#define SQLITE_CHANGESET_ABORT 2 + +/* +** CAPI3REF: Rebasing changesets +** EXPERIMENTAL +** +** Suppose there is a site hosting a database in state S0. And that +** modifications are made that move that database to state S1 and a +** changeset recorded (the "local" changeset). Then, a changeset based +** on S0 is received from another site (the "remote" changeset) and +** applied to the database. The database is then in state +** (S1+"remote"), where the exact state depends on any conflict +** resolution decisions (OMIT or REPLACE) made while applying "remote". +** Rebasing a changeset is to update it to take those conflict +** resolution decisions into account, so that the same conflicts +** do not have to be resolved elsewhere in the network. +** +** For example, if both the local and remote changesets contain an +** INSERT of the same key on "CREATE TABLE t1(a PRIMARY KEY, b)": +** +** local: INSERT INTO t1 VALUES(1, 'v1'); +** remote: INSERT INTO t1 VALUES(1, 'v2'); +** +** and the conflict resolution is REPLACE, then the INSERT change is +** removed from the local changeset (it was overridden). Or, if the +** conflict resolution was "OMIT", then the local changeset is modified +** to instead contain: +** +** UPDATE t1 SET b = 'v2' WHERE a=1; +** +** Changes within the local changeset are rebased as follows: +** +**
    +**
    Local INSERT
    +** This may only conflict with a remote INSERT. If the conflict +** resolution was OMIT, then add an UPDATE change to the rebased +** changeset. Or, if the conflict resolution was REPLACE, add +** nothing to the rebased changeset. +** +**
    Local DELETE
    +** This may conflict with a remote UPDATE or DELETE. In both cases the +** only possible resolution is OMIT. If the remote operation was a +** DELETE, then add no change to the rebased changeset. If the remote +** operation was an UPDATE, then the old.* fields of change are updated +** to reflect the new.* values in the UPDATE. +** +**
    Local UPDATE
    +** This may conflict with a remote UPDATE or DELETE. If it conflicts +** with a DELETE, and the conflict resolution was OMIT, then the update +** is changed into an INSERT. Any undefined values in the new.* record +** from the update change are filled in using the old.* values from +** the conflicting DELETE. Or, if the conflict resolution was REPLACE, +** the UPDATE change is simply omitted from the rebased changeset. +** +** If conflict is with a remote UPDATE and the resolution is OMIT, then +** the old.* values are rebased using the new.* values in the remote +** change. Or, if the resolution is REPLACE, then the change is copied +** into the rebased changeset with updates to columns also updated by +** the conflicting remote UPDATE removed. If this means no columns would +** be updated, the change is omitted. +**
    +** +** A local change may be rebased against multiple remote changes +** simultaneously. If a single key is modified by multiple remote +** changesets, they are combined as follows before the local changeset +** is rebased: +** +**
      +**
    • If there has been one or more REPLACE resolutions on a +** key, it is rebased according to a REPLACE. +** +**
    • If there have been no REPLACE resolutions on a key, then +** the local changeset is rebased according to the most recent +** of the OMIT resolutions. +**
    +** +** Note that conflict resolutions from multiple remote changesets are +** combined on a per-field basis, not per-row. This means that in the +** case of multiple remote UPDATE operations, some fields of a single +** local change may be rebased for REPLACE while others are rebased for +** OMIT. +** +** In order to rebase a local changeset, the remote changeset must first +** be applied to the local database using sqlite3changeset_apply_v2() and +** the buffer of rebase information captured. Then: +** +**
      +**
    1. An sqlite3_rebaser object is created by calling +** sqlite3rebaser_create(). +**
    2. The new object is configured with the rebase buffer obtained from +** sqlite3changeset_apply_v2() by calling sqlite3rebaser_configure(). +** If the local changeset is to be rebased against multiple remote +** changesets, then sqlite3rebaser_configure() should be called +** multiple times, in the same order that the multiple +** sqlite3changeset_apply_v2() calls were made. +**
    3. Each local changeset is rebased by calling sqlite3rebaser_rebase(). +**
    4. The sqlite3_rebaser object is deleted by calling +** sqlite3rebaser_delete(). +**
    +*/ +typedef struct sqlite3_rebaser sqlite3_rebaser; + +/* +** CAPI3REF: Create a changeset rebaser object. +** EXPERIMENTAL +** +** Allocate a new changeset rebaser object. If successful, set (*ppNew) to +** point to the new object and return SQLITE_OK. Otherwise, if an error +** occurs, return an SQLite error code (e.g. SQLITE_NOMEM) and set (*ppNew) +** to NULL. +*/ +SQLITE_API int sqlite3rebaser_create(sqlite3_rebaser **ppNew); + +/* +** CAPI3REF: Configure a changeset rebaser object. +** EXPERIMENTAL +** +** Configure the changeset rebaser object to rebase changesets according +** to the conflict resolutions described by buffer pRebase (size nRebase +** bytes), which must have been obtained from a previous call to +** sqlite3changeset_apply_v2(). +*/ +SQLITE_API int sqlite3rebaser_configure( + sqlite3_rebaser*, + int nRebase, const void *pRebase +); + +/* +** CAPI3REF: Rebase a changeset +** EXPERIMENTAL +** +** Argument pIn must point to a buffer containing a changeset nIn bytes +** in size. This function allocates and populates a buffer with a copy +** of the changeset rebased according to the configuration of the +** rebaser object passed as the first argument. If successful, (*ppOut) +** is set to point to the new buffer containing the rebased changeset and +** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the +** responsibility of the caller to eventually free the new buffer using +** sqlite3_free(). Otherwise, if an error occurs, (*ppOut) and (*pnOut) +** are set to zero and an SQLite error code returned. +*/ +SQLITE_API int sqlite3rebaser_rebase( + sqlite3_rebaser*, + int nIn, const void *pIn, + int *pnOut, void **ppOut +); + +/* +** CAPI3REF: Delete a changeset rebaser object. +** EXPERIMENTAL +** +** Delete the changeset rebaser object and all associated resources. There +** should be one call to this function for each successful invocation +** of sqlite3rebaser_create(). +*/ +SQLITE_API void sqlite3rebaser_delete(sqlite3_rebaser *p); + +/* +** CAPI3REF: Streaming Versions of API functions. +** +** The six streaming API xxx_strm() functions serve similar purposes to the +** corresponding non-streaming API functions: +** +** +** +**
    Streaming functionNon-streaming equivalent
    sqlite3changeset_apply_strm[sqlite3changeset_apply] +**
    sqlite3changeset_apply_strm_v2[sqlite3changeset_apply_v2] +**
    sqlite3changeset_concat_strm[sqlite3changeset_concat] +**
    sqlite3changeset_invert_strm[sqlite3changeset_invert] +**
    sqlite3changeset_start_strm[sqlite3changeset_start] +**
    sqlite3session_changeset_strm[sqlite3session_changeset] +**
    sqlite3session_patchset_strm[sqlite3session_patchset] +**
    +** +** Non-streaming functions that accept changesets (or patchsets) as input +** require that the entire changeset be stored in a single buffer in memory. +** Similarly, those that return a changeset or patchset do so by returning +** a pointer to a single large buffer allocated using sqlite3_malloc(). +** Normally this is convenient. However, if an application running in a +** low-memory environment is required to handle very large changesets, the +** large contiguous memory allocations required can become onerous. +** +** In order to avoid this problem, instead of a single large buffer, input +** is passed to a streaming API functions by way of a callback function that +** the sessions module invokes to incrementally request input data as it is +** required. In all cases, a pair of API function parameters such as +** +** 
    +**        int nChangeset,
+**        void *pChangeset,
+**
    +** +** Is replaced by: +** +** 
    +**        int (*xInput)(void *pIn, void *pData, int *pnData),
+**        void *pIn,
+**
    +** +** Each time the xInput callback is invoked by the sessions module, the first +** argument passed is a copy of the supplied pIn context pointer. The second +** argument, pData, points to a buffer (*pnData) bytes in size. Assuming no +** error occurs the xInput method should copy up to (*pnData) bytes of data +** into the buffer and set (*pnData) to the actual number of bytes copied +** before returning SQLITE_OK. If the input is completely exhausted, (*pnData) +** should be set to zero to indicate this. Or, if an error occurs, an SQLite +** error code should be returned. In all cases, if an xInput callback returns +** an error, all processing is abandoned and the streaming API function +** returns a copy of the error code to the caller. +** +** In the case of sqlite3changeset_start_strm(), the xInput callback may be +** invoked by the sessions module at any point during the lifetime of the +** iterator. If such an xInput callback returns an error, the iterator enters +** an error state, whereby all subsequent calls to iterator functions +** immediately fail with the same error code as returned by xInput. +** +** Similarly, streaming API functions that return changesets (or patchsets) +** return them in chunks by way of a callback function instead of via a +** pointer to a single large buffer. In this case, a pair of parameters such +** as: +** +** 
    +**        int *pnChangeset,
+**        void **ppChangeset,
+**
    +** +** Is replaced by: +** +** 
    +**        int (*xOutput)(void *pOut, const void *pData, int nData),
+**        void *pOut
+**
    +** +** The xOutput callback is invoked zero or more times to return data to +** the application. The first parameter passed to each call is a copy of the +** pOut pointer supplied by the application. The second parameter, pData, +** points to a buffer nData bytes in size containing the chunk of output +** data being returned. If the xOutput callback successfully processes the +** supplied data, it should return SQLITE_OK to indicate success. Otherwise, +** it should return some other SQLite error code. In this case processing +** is immediately abandoned and the streaming API function returns a copy +** of the xOutput error code to the application. +** +** The sessions module never invokes an xOutput callback with the third +** parameter set to a value less than or equal to zero. Other than this, +** no guarantees are made as to the size of the chunks of data returned. +*/ +SQLITE_API int sqlite3changeset_apply_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); +SQLITE_API int sqlite3changeset_apply_v2_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +); +SQLITE_API int sqlite3changeset_concat_strm( + int (*xInputA)(void *pIn, void *pData, int *pnData), + void *pInA, + int (*xInputB)(void *pIn, void *pData, int *pnData), + void *pInB, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3changeset_invert_strm( + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3changeset_start_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +SQLITE_API int sqlite3changeset_start_v2_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int flags +); +SQLITE_API int sqlite3session_changeset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3session_patchset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3changegroup_add_strm(sqlite3_changegroup*, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +SQLITE_API int sqlite3changegroup_output_strm(sqlite3_changegroup*, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3rebaser_rebase_strm( + sqlite3_rebaser *pRebaser, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); + +/* +** CAPI3REF: Configure global parameters +** +** The sqlite3session_config() interface is used to make global configuration +** changes to the sessions module in order to tune it to the specific needs +** of the application. +** +** The sqlite3session_config() interface is not threadsafe. If it is invoked +** while any other thread is inside any other sessions method then the +** results are undefined. Furthermore, if it is invoked after any sessions +** related objects have been created, the results are also undefined. +** +** The first argument to the sqlite3session_config() function must be one +** of the SQLITE_SESSION_CONFIG_XXX constants defined below. The +** interpretation of the (void*) value passed as the second parameter and +** the effect of calling this function depends on the value of the first +** parameter. +** +**
    +**
    SQLITE_SESSION_CONFIG_STRMSIZE
    +** By default, the sessions module streaming interfaces attempt to input +** and output data in approximately 1 KiB chunks. This operand may be used +** to set and query the value of this configuration setting. The pointer +** passed as the second argument must point to a value of type (int). +** If this value is greater than 0, it is used as the new streaming data +** chunk size for both input and output. Before returning, the (int) value +** pointed to by pArg is set to the final value of the streaming interface +** chunk size. +**
    +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +SQLITE_API int sqlite3session_config(int op, void *pArg); + +/* +** CAPI3REF: Values for sqlite3session_config(). +*/ +#define SQLITE_SESSION_CONFIG_STRMSIZE 1 + +/* +** Make sure we can call this stuff from C++. +*/ +#if 0 +} +#endif + +#endif /* !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) */ + +/******** End of sqlite3session.h *********/ +/******** Begin file fts5.h *********/ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ + + +#ifndef _FTS5_H +#define _FTS5_H + + +#if 0 +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the context pointer the extension function was +** registered with. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** This function may be quite inefficient if used with an FTS5 table +** created with the "columnsize=0" option. +** +** xColumnText: +** This function attempts to retrieve the text of column iCol of the +** current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** Returns the number of tokens in phrase iPhrase of the query. Phrases +** are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always returns 0. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). +** +** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** to the column in which it occurs and *piOff the token offset of the +** first token of the phrase. Returns SQLITE_OK if successful, or an error +** code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. Any column filter that applies to +** phrase iPhrase of the current query is included in $p. For each +** row visited, the callback function passed as the fourth argument +** is invoked. The context and API objects passed to the callback +** function may be used to access the properties of each matched row. +** Invoking Api.xUserData() returns a copy of the pointer passed as +** the third argument to pUserData. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension function's +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iCol>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods (and by +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always iterates +** through an empty set (all calls to xPhraseFirst() set iCol to -1). +** +** xPhraseNext() +** See xPhraseFirst above. +** +** xPhraseFirstColumn() +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() +** and xPhraseNext() APIs described above. The difference is that instead +** of iterating through all instances of a phrase in the current row, these +** APIs are used to iterate through the set of columns in the current row +** that contain one or more instances of a specified phrase. For example: +** +** Fts5PhraseIter iter; +** int iCol; +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); +** iCol>=0; +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) +** ){ +** // Column iCol contains at least one instance of phrase iPhrase +** } +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to +** xPhraseFirstColumn() set iCol to -1). +** +** The information accessed using this API and its companion +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext +** (or xInst/xInstCount). The chief advantage of this API is that it is +** significantly more efficient than those alternatives when used with +** "detail=column" tables. +** +** xPhraseNextColumn() +** See xPhraseFirstColumn above. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 3 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and initialize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The second argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**
    • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +**
    • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +**
    • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +**
    • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +**
    +** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
    1. By mapping all synonyms to a single token. In this case, using +** the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
    2. By querying the index for all synonyms of each query term +** separately. In this case, when tokenizing query text, the +** tokenizer may provide multiple synonyms for a single term +** within the document. FTS5 then queries the index for each +** synonym individually. For example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
    3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do so would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entries in the +** FTS index corresponding to both forms of the first token. +**
    +** +** Whether it is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is substituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (2)) or query +** text (method (3)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 2 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppContext, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pContext, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ + +/******** End of fts5.h *********/ + +/************** End of sqlite3.h *********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + +/* +** Include the configuration header output by 'configure' if we're using the +** autoconf-based build +*/ +#if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) +#include "config.h" +#define SQLITECONFIG_H 1 +#endif + +/************** Include sqliteLimit.h in the middle of sqliteInt.h ***********/ +/************** Begin file sqliteLimit.h *************************************/ +/* +** 2007 May 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file defines various limits of what SQLite can process. +*/ + +/* +** The maximum length of a TEXT or BLOB in bytes. This also +** limits the size of a row in a table or index. +** +** The hard limit is the ability of a 32-bit signed integer +** to count the size: 2^31-1 or 2147483647. +*/ +#ifndef SQLITE_MAX_LENGTH +# define SQLITE_MAX_LENGTH 1000000000 +#endif + +/* +** This is the maximum number of +** +** * Columns in a table +** * Columns in an index +** * Columns in a view +** * Terms in the SET clause of an UPDATE statement +** * Terms in the result set of a SELECT statement +** * Terms in the GROUP BY or ORDER BY clauses of a SELECT statement. +** * Terms in the VALUES clause of an INSERT statement +** +** The hard upper limit here is 32676. Most database people will +** tell you that in a well-normalized database, you usually should +** not have more than a dozen or so columns in any table. And if +** that is the case, there is no point in having more than a few +** dozen values in any of the other situations described above. +*/ +#ifndef SQLITE_MAX_COLUMN +# define SQLITE_MAX_COLUMN 2000 +#endif + +/* +** The maximum length of a single SQL statement in bytes. +** +** It used to be the case that setting this value to zero would +** turn the limit off. That is no longer true. It is not possible +** to turn this limit off. +*/ +#ifndef SQLITE_MAX_SQL_LENGTH +# define SQLITE_MAX_SQL_LENGTH 1000000000 +#endif + +/* +** The maximum depth of an expression tree. This is limited to +** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might +** want to place more severe limits on the complexity of an +** expression. A value of 0 means that there is no limit. +*/ +#ifndef SQLITE_MAX_EXPR_DEPTH +# define SQLITE_MAX_EXPR_DEPTH 1000 +#endif + +/* +** The maximum number of terms in a compound SELECT statement. +** The code generator for compound SELECT statements does one +** level of recursion for each term. A stack overflow can result +** if the number of terms is too large. In practice, most SQL +** never has more than 3 or 4 terms. Use a value of 0 to disable +** any limit on the number of terms in a compount SELECT. +*/ +#ifndef SQLITE_MAX_COMPOUND_SELECT +# define SQLITE_MAX_COMPOUND_SELECT 500 +#endif + +/* +** The maximum number of opcodes in a VDBE program. +** Not currently enforced. +*/ +#ifndef SQLITE_MAX_VDBE_OP +# define SQLITE_MAX_VDBE_OP 250000000 +#endif + +/* +** The maximum number of arguments to an SQL function. +*/ +#ifndef SQLITE_MAX_FUNCTION_ARG +# define SQLITE_MAX_FUNCTION_ARG 127 +#endif + +/* +** The suggested maximum number of in-memory pages to use for +** the main database table and for temporary tables. +** +** IMPLEMENTATION-OF: R-30185-15359 The default suggested cache size is -2000, +** which means the cache size is limited to 2048000 bytes of memory. +** IMPLEMENTATION-OF: R-48205-43578 The default suggested cache size can be +** altered using the SQLITE_DEFAULT_CACHE_SIZE compile-time options. +*/ +#ifndef SQLITE_DEFAULT_CACHE_SIZE +# define SQLITE_DEFAULT_CACHE_SIZE -2000 +#endif + +/* +** The default number of frames to accumulate in the log file before +** checkpointing the database in WAL mode. +*/ +#ifndef SQLITE_DEFAULT_WAL_AUTOCHECKPOINT +# define SQLITE_DEFAULT_WAL_AUTOCHECKPOINT 1000 +#endif + +/* +** The maximum number of attached databases. This must be between 0 +** and 125. The upper bound of 125 is because the attached databases are +** counted using a signed 8-bit integer which has a maximum value of 127 +** and we have to allow 2 extra counts for the "main" and "temp" databases. +*/ +#ifndef SQLITE_MAX_ATTACHED +# define SQLITE_MAX_ATTACHED 10 +#endif + + +/* +** The maximum value of a ?nnn wildcard that the parser will accept. +** If the value exceeds 32767 then extra space is required for the Expr +** structure. But otherwise, we believe that the number can be as large +** as a signed 32-bit integer can hold. +*/ +#ifndef SQLITE_MAX_VARIABLE_NUMBER +# define SQLITE_MAX_VARIABLE_NUMBER 32766 +#endif + +/* Maximum page size. The upper bound on this value is 65536. This a limit +** imposed by the use of 16-bit offsets within each page. +** +** Earlier versions of SQLite allowed the user to change this value at +** compile time. This is no longer permitted, on the grounds that it creates +** a library that is technically incompatible with an SQLite library +** compiled with a different limit. If a process operating on a database +** with a page-size of 65536 bytes crashes, then an instance of SQLite +** compiled with the default page-size limit will not be able to rollback +** the aborted transaction. This could lead to database corruption. +*/ +#ifdef SQLITE_MAX_PAGE_SIZE +# undef SQLITE_MAX_PAGE_SIZE +#endif +#define SQLITE_MAX_PAGE_SIZE 65536 + + +/* +** The default size of a database page. +*/ +#ifndef SQLITE_DEFAULT_PAGE_SIZE +# define SQLITE_DEFAULT_PAGE_SIZE 4096 +#endif +#if SQLITE_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE +# undef SQLITE_DEFAULT_PAGE_SIZE +# define SQLITE_DEFAULT_PAGE_SIZE SQLITE_MAX_PAGE_SIZE +#endif + +/* +** Ordinarily, if no value is explicitly provided, SQLite creates databases +** with page size SQLITE_DEFAULT_PAGE_SIZE. However, based on certain +** device characteristics (sector-size and atomic write() support), +** SQLite may choose a larger value. This constant is the maximum value +** SQLite will choose on its own. +*/ +#ifndef SQLITE_MAX_DEFAULT_PAGE_SIZE +# define SQLITE_MAX_DEFAULT_PAGE_SIZE 8192 +#endif +#if SQLITE_MAX_DEFAULT_PAGE_SIZE>SQLITE_MAX_PAGE_SIZE +# undef SQLITE_MAX_DEFAULT_PAGE_SIZE +# define SQLITE_MAX_DEFAULT_PAGE_SIZE SQLITE_MAX_PAGE_SIZE +#endif + + +/* +** Maximum number of pages in one database file. +** +** This is really just the default value for the max_page_count pragma. +** This value can be lowered (or raised) at run-time using that the +** max_page_count macro. +*/ +#ifndef SQLITE_MAX_PAGE_COUNT +# define SQLITE_MAX_PAGE_COUNT 1073741823 +#endif + +/* +** Maximum length (in bytes) of the pattern in a LIKE or GLOB +** operator. +*/ +#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +#endif + +/* +** Maximum depth of recursion for triggers. +** +** A value of 1 means that a trigger program will not be able to itself +** fire any triggers. A value of 0 means that no trigger programs at all +** may be executed. +*/ +#ifndef SQLITE_MAX_TRIGGER_DEPTH +# define SQLITE_MAX_TRIGGER_DEPTH 1000 +#endif + +/************** End of sqliteLimit.h *****************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + +/* Disable nuisance warnings on Borland compilers */ +#if defined(__BORLANDC__) +#pragma warn -rch /* unreachable code */ +#pragma warn -ccc /* Condition is always true or false */ +#pragma warn -aus /* Assigned value is never used */ +#pragma warn -csu /* Comparing signed and unsigned */ +#pragma warn -spa /* Suspicious pointer arithmetic */ +#endif + +/* +** WAL mode depends on atomic aligned 32-bit loads and stores in a few +** places. The following macros try to make this explicit. +*/ +#ifndef __has_extension +# define __has_extension(x) 0 /* compatibility with non-clang compilers */ +#endif +#if GCC_VERSION>=4007000 || __has_extension(c_atomic) +# define SQLITE_ATOMIC_INTRINSICS 1 +# define AtomicLoad(PTR) __atomic_load_n((PTR),__ATOMIC_RELAXED) +# define AtomicStore(PTR,VAL) __atomic_store_n((PTR),(VAL),__ATOMIC_RELAXED) +#else +# define SQLITE_ATOMIC_INTRINSICS 0 +# define AtomicLoad(PTR) (*(PTR)) +# define AtomicStore(PTR,VAL) (*(PTR) = (VAL)) +#endif + +/* +** Include standard header files as necessary +*/ +#ifdef HAVE_STDINT_H +#include +#endif +#ifdef HAVE_INTTYPES_H +#include +#endif + +/* +** The following macros are used to cast pointers to integers and +** integers to pointers. The way you do this varies from one compiler +** to the next, so we have developed the following set of #if statements +** to generate appropriate macros for a wide range of compilers. +** +** The correct "ANSI" way to do this is to use the intptr_t type. +** Unfortunately, that typedef is not available on all compilers, or +** if it is available, it requires an #include of specific headers +** that vary from one machine to the next. +** +** Ticket #3860: The llvm-gcc-4.2 compiler from Apple chokes on +** the ((void*)&((char*)0)[X]) construct. But MSVC chokes on ((void*)(X)). +** So we have to define the macros in different ways depending on the +** compiler. +*/ +#if defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) +#elif defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +# define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) +#elif !defined(__GNUC__) /* Works for compilers other than LLVM */ +# define SQLITE_INT_TO_PTR(X) ((void*)&((char*)0)[X]) +# define SQLITE_PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) +#else /* Generates a warning - but it always works */ +# define SQLITE_INT_TO_PTR(X) ((void*)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(X)) +#endif + +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define SQLITE_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define SQLITE_NOINLINE __declspec(noinline) +#else +# define SQLITE_NOINLINE +#endif + +/* +** Make sure that the compiler intrinsics we desire are enabled when +** compiling with an appropriate version of MSVC unless prevented by +** the SQLITE_DISABLE_INTRINSIC define. +*/ +#if !defined(SQLITE_DISABLE_INTRINSIC) +# if defined(_MSC_VER) && _MSC_VER>=1400 +# if !defined(_WIN32_WCE) +# include +# pragma intrinsic(_byteswap_ushort) +# pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_byteswap_uint64) +# pragma intrinsic(_ReadWriteBarrier) +# else +# include +# endif +# endif +#endif + +/* +** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. +** 0 means mutexes are permanently disable and the library is never +** threadsafe. 1 means the library is serialized which is the highest +** level of threadsafety. 2 means the library is multithreaded - multiple +** threads can use SQLite as long as no two threads try to use the same +** database connection at the same time. +** +** Older versions of SQLite used an optional THREADSAFE macro. +** We support that for legacy. +** +** To ensure that the correct value of "THREADSAFE" is reported when querying +** for compile-time options at runtime (e.g. "PRAGMA compile_options"), this +** logic is partially replicated in ctime.c. If it is updated here, it should +** also be updated there. +*/ +#if !defined(SQLITE_THREADSAFE) +# if defined(THREADSAFE) +# define SQLITE_THREADSAFE THREADSAFE +# else +# define SQLITE_THREADSAFE 1 /* IMP: R-07272-22309 */ +# endif +#endif + +/* +** Powersafe overwrite is on by default. But can be turned off using +** the -DSQLITE_POWERSAFE_OVERWRITE=0 command-line option. +*/ +#ifndef SQLITE_POWERSAFE_OVERWRITE +# define SQLITE_POWERSAFE_OVERWRITE 1 +#endif + +/* +** EVIDENCE-OF: R-25715-37072 Memory allocation statistics are enabled by +** default unless SQLite is compiled with SQLITE_DEFAULT_MEMSTATUS=0 in +** which case memory allocation statistics are disabled by default. +*/ +#if !defined(SQLITE_DEFAULT_MEMSTATUS) +# define SQLITE_DEFAULT_MEMSTATUS 1 +#endif + +/* +** Exactly one of the following macros must be defined in order to +** specify which memory allocation subsystem to use. +** +** SQLITE_SYSTEM_MALLOC // Use normal system malloc() +** SQLITE_WIN32_MALLOC // Use Win32 native heap API +** SQLITE_ZERO_MALLOC // Use a stub allocator that always fails +** SQLITE_MEMDEBUG // Debugging version of system malloc() +** +** On Windows, if the SQLITE_WIN32_MALLOC_VALIDATE macro is defined and the +** assert() macro is enabled, each call into the Win32 native heap subsystem +** will cause HeapValidate to be called. If heap validation should fail, an +** assertion will be triggered. +** +** If none of the above are defined, then set SQLITE_SYSTEM_MALLOC as +** the default. +*/ +#if defined(SQLITE_SYSTEM_MALLOC) \ + + defined(SQLITE_WIN32_MALLOC) \ + + defined(SQLITE_ZERO_MALLOC) \ + + defined(SQLITE_MEMDEBUG)>1 +# error "Two or more of the following compile-time configuration options\ + are defined but at most one is allowed:\ + SQLITE_SYSTEM_MALLOC, SQLITE_WIN32_MALLOC, SQLITE_MEMDEBUG,\ + SQLITE_ZERO_MALLOC" +#endif +#if defined(SQLITE_SYSTEM_MALLOC) \ + + defined(SQLITE_WIN32_MALLOC) \ + + defined(SQLITE_ZERO_MALLOC) \ + + defined(SQLITE_MEMDEBUG)==0 +# define SQLITE_SYSTEM_MALLOC 1 +#endif + +/* +** If SQLITE_MALLOC_SOFT_LIMIT is not zero, then try to keep the +** sizes of memory allocations below this value where possible. +*/ +#if !defined(SQLITE_MALLOC_SOFT_LIMIT) +# define SQLITE_MALLOC_SOFT_LIMIT 1024 +#endif + +/* +** We need to define _XOPEN_SOURCE as follows in order to enable +** recursive mutexes on most Unix systems and fchmod() on OpenBSD. +** But _XOPEN_SOURCE define causes problems for Mac OS X, so omit +** it. +*/ +#if !defined(_XOPEN_SOURCE) && !defined(__DARWIN__) && !defined(__APPLE__) +# define _XOPEN_SOURCE 600 +#endif + +/* +** NDEBUG and SQLITE_DEBUG are opposites. It should always be true that +** defined(NDEBUG)==!defined(SQLITE_DEBUG). If this is not currently true, +** make it true by defining or undefining NDEBUG. +** +** Setting NDEBUG makes the code smaller and faster by disabling the +** assert() statements in the code. So we want the default action +** to be for NDEBUG to be set and NDEBUG to be undefined only if SQLITE_DEBUG +** is set. Thus NDEBUG becomes an opt-in rather than an opt-out +** feature. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif + +/* +** Enable SQLITE_ENABLE_EXPLAIN_COMMENTS if SQLITE_DEBUG is turned on. +*/ +#if !defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) && defined(SQLITE_DEBUG) +# define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 +#endif + +/* +** The testcase() macro is used to aid in coverage testing. When +** doing coverage testing, the condition inside the argument to +** testcase() must be evaluated both true and false in order to +** get full branch coverage. The testcase() macro is inserted +** to help ensure adequate test coverage in places where simple +** condition/decision coverage is inadequate. For example, testcase() +** can be used to make sure boundary values are tested. For +** bitmask tests, testcase() can be used to make sure each bit +** is significant and used at least once. On switch statements +** where multiple cases go to the same block of code, testcase() +** can insure that all cases are evaluated. +** +*/ +#ifdef SQLITE_COVERAGE_TEST +SQLITE_PRIVATE void sqlite3Coverage(int); +# define testcase(X) if( X ){ sqlite3Coverage(__LINE__); } +#else +# define testcase(X) +#endif + +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if !defined(NDEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +/* +** Sometimes we need a small amount of code such as a variable initialization +** to setup for a later assert() statement. We do not want this code to +** appear when assert() is disabled. The following macro is therefore +** used to contain that setup code. The "VVA" acronym stands for +** "Verification, Validation, and Accreditation". In other words, the +** code within VVA_ONLY() will only run during verification processes. +*/ +#ifndef NDEBUG +# define VVA_ONLY(X) X +#else +# define VVA_ONLY(X) +#endif + +/* +** The ALWAYS and NEVER macros surround boolean expressions which +** are intended to always be true or false, respectively. Such +** expressions could be omitted from the code completely. But they +** are included in a few cases in order to enhance the resilience +** of SQLite to unexpected behavior - to make the code "self-healing" +** or "ductile" rather than being "brittle" and crashing at the first +** hint of unplanned behavior. +** +** In other words, ALWAYS and NEVER are added for defensive code. +** +** When doing coverage testing ALWAYS and NEVER are hard-coded to +** be true and false so that the unreachable code they specify will +** not be counted as untested code. +*/ +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif + +/* +** The harmless(X) macro indicates that expression X is usually false +** but can be true without causing any problems, but we don't know of +** any way to cause X to be true. +** +** In debugging and testing builds, this macro will abort if X is ever +** true. In this way, developers are alerted to a possible test case +** that causes X to be true. If a harmless macro ever fails, that is +** an opportunity to change the macro into a testcase() and add a new +** test case to the test suite. +** +** For normal production builds, harmless(X) is a no-op, since it does +** not matter whether expression X is true or false. +*/ +#ifdef SQLITE_DEBUG +# define harmless(X) assert(!(X)); +#else +# define harmless(X) +#endif + +/* +** Some conditionals are optimizations only. In other words, if the +** conditionals are replaced with a constant 1 (true) or 0 (false) then +** the correct answer is still obtained, though perhaps not as quickly. +** +** The following macros mark these optimizations conditionals. +*/ +#if defined(SQLITE_MUTATION_TEST) +# define OK_IF_ALWAYS_TRUE(X) (1) +# define OK_IF_ALWAYS_FALSE(X) (0) +#else +# define OK_IF_ALWAYS_TRUE(X) (X) +# define OK_IF_ALWAYS_FALSE(X) (X) +#endif + +/* +** Some malloc failures are only possible if SQLITE_TEST_REALLOC_STRESS is +** defined. We need to defend against those failures when testing with +** SQLITE_TEST_REALLOC_STRESS, but we don't want the unreachable branches +** during a normal build. The following macro can be used to disable tests +** that are always false except when SQLITE_TEST_REALLOC_STRESS is set. +*/ +#if defined(SQLITE_TEST_REALLOC_STRESS) +# define ONLY_IF_REALLOC_STRESS(X) (X) +#elif !defined(NDEBUG) +# define ONLY_IF_REALLOC_STRESS(X) ((X)?(assert(0),1):0) +#else +# define ONLY_IF_REALLOC_STRESS(X) (0) +#endif + +/* +** Declarations used for tracing the operating system interfaces. +*/ +#if defined(SQLITE_FORCE_OS_TRACE) || defined(SQLITE_TEST) || \ + (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) + extern int sqlite3OSTrace; +# define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X +# define SQLITE_HAVE_OS_TRACE +#else +# define OSTRACE(X) +# undef SQLITE_HAVE_OS_TRACE +#endif + +/* +** Is the sqlite3ErrName() function needed in the build? Currently, +** it is needed by "mutex_w32.c" (when debugging), "os_win.c" (when +** OSTRACE is enabled), and by several "test*.c" files (which are +** compiled using SQLITE_TEST). +*/ +#if defined(SQLITE_HAVE_OS_TRACE) || defined(SQLITE_TEST) || \ + (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) +# define SQLITE_NEED_ERR_NAME +#else +# undef SQLITE_NEED_ERR_NAME +#endif + +/* +** SQLITE_ENABLE_EXPLAIN_COMMENTS is incompatible with SQLITE_OMIT_EXPLAIN +*/ +#ifdef SQLITE_OMIT_EXPLAIN +# undef SQLITE_ENABLE_EXPLAIN_COMMENTS +#endif + +/* +** SQLITE_OMIT_VIRTUALTABLE implies SQLITE_OMIT_ALTERTABLE +*/ +#if defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_ALTERTABLE) +# define SQLITE_OMIT_ALTERTABLE +#endif + +/* +** Return true (non-zero) if the input is an integer that is too large +** to fit in 32-bits. This macro is used inside of various testcase() +** macros to verify that we have tested SQLite for large-file support. +*/ +#define IS_BIG_INT(X) (((X)&~(i64)0xffffffff)!=0) + +/* +** The macro unlikely() is a hint that surrounds a boolean +** expression that is usually false. Macro likely() surrounds +** a boolean expression that is usually true. These hints could, +** in theory, be used by the compiler to generate better code, but +** currently they are just comments for human readers. +*/ +#define likely(X) (X) +#define unlikely(X) (X) + +/************** Include hash.h in the middle of sqliteInt.h ******************/ +/************** Begin file hash.h ********************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implementation +** used in SQLite. +*/ +#ifndef SQLITE_HASH_H +#define SQLITE_HASH_H + +/* Forward declarations of structures. */ +typedef struct Hash Hash; +typedef struct HashElem HashElem; + +/* A complete hash table is an instance of the following structure. +** The internals of this structure are intended to be opaque -- client +** code should not attempt to access or modify the fields of this structure +** directly. Change this structure only by using the routines below. +** However, some of the "procedures" and "functions" for modifying and +** accessing this structure are really macros, so we can't really make +** this structure opaque. +** +** All elements of the hash table are on a single doubly-linked list. +** Hash.first points to the head of this list. +** +** There are Hash.htsize buckets. Each bucket points to a spot in +** the global doubly-linked list. The contents of the bucket are the +** element pointed to plus the next _ht.count-1 elements in the list. +** +** Hash.htsize and Hash.ht may be zero. In that case lookup is done +** by a linear search of the global list. For small tables, the +** Hash.ht table is never allocated because if there are few elements +** in the table, it is faster to do a linear search than to manage +** the hash table. +*/ +struct Hash { + unsigned int htsize; /* Number of buckets in the hash table */ + unsigned int count; /* Number of entries in this table */ + HashElem *first; /* The first element of the array */ + struct _ht { /* the hash table */ + unsigned int count; /* Number of entries with this hash */ + HashElem *chain; /* Pointer to first entry with this hash */ + } *ht; +}; + +/* Each element in the hash table is an instance of the following +** structure. All elements are stored on a single doubly-linked list. +** +** Again, this structure is intended to be opaque, but it can't really +** be opaque because it is used by macros. +*/ +struct HashElem { + HashElem *next, *prev; /* Next and previous elements in the table */ + void *data; /* Data associated with this element */ + const char *pKey; /* Key associated with this element */ +}; + +/* +** Access routines. To delete, insert a NULL pointer. +*/ +SQLITE_PRIVATE void sqlite3HashInit(Hash*); +SQLITE_PRIVATE void *sqlite3HashInsert(Hash*, const char *pKey, void *pData); +SQLITE_PRIVATE void *sqlite3HashFind(const Hash*, const char *pKey); +SQLITE_PRIVATE void sqlite3HashClear(Hash*); + +/* +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Hash h; +** HashElem *p; +** ... +** for(p=sqliteHashFirst(&h); p; p=sqliteHashNext(p)){ +** SomeStructure *pData = sqliteHashData(p); +** // do something with pData +** } +*/ +#define sqliteHashFirst(H) ((H)->first) +#define sqliteHashNext(E) ((E)->next) +#define sqliteHashData(E) ((E)->data) +/* #define sqliteHashKey(E) ((E)->pKey) // NOT USED */ +/* #define sqliteHashKeysize(E) ((E)->nKey) // NOT USED */ + +/* +** Number of entries in a hash table +*/ +/* #define sqliteHashCount(H) ((H)->count) // NOT USED */ + +#endif /* SQLITE_HASH_H */ + +/************** End of hash.h ************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include parse.h in the middle of sqliteInt.h *****************/ +/************** Begin file parse.h *******************************************/ +#define TK_SEMI 1 +#define TK_EXPLAIN 2 +#define TK_QUERY 3 +#define TK_PLAN 4 +#define TK_BEGIN 5 +#define TK_TRANSACTION 6 +#define TK_DEFERRED 7 +#define TK_IMMEDIATE 8 +#define TK_EXCLUSIVE 9 +#define TK_COMMIT 10 +#define TK_END 11 +#define TK_ROLLBACK 12 +#define TK_SAVEPOINT 13 +#define TK_RELEASE 14 +#define TK_TO 15 +#define TK_TABLE 16 +#define TK_CREATE 17 +#define TK_IF 18 +#define TK_NOT 19 +#define TK_EXISTS 20 +#define TK_TEMP 21 +#define TK_LP 22 +#define TK_RP 23 +#define TK_AS 24 +#define TK_COMMA 25 +#define TK_WITHOUT 26 +#define TK_ABORT 27 +#define TK_ACTION 28 +#define TK_AFTER 29 +#define TK_ANALYZE 30 +#define TK_ASC 31 +#define TK_ATTACH 32 +#define TK_BEFORE 33 +#define TK_BY 34 +#define TK_CASCADE 35 +#define TK_CAST 36 +#define TK_CONFLICT 37 +#define TK_DATABASE 38 +#define TK_DESC 39 +#define TK_DETACH 40 +#define TK_EACH 41 +#define TK_FAIL 42 +#define TK_OR 43 +#define TK_AND 44 +#define TK_IS 45 +#define TK_MATCH 46 +#define TK_LIKE_KW 47 +#define TK_BETWEEN 48 +#define TK_IN 49 +#define TK_ISNULL 50 +#define TK_NOTNULL 51 +#define TK_NE 52 +#define TK_EQ 53 +#define TK_GT 54 +#define TK_LE 55 +#define TK_LT 56 +#define TK_GE 57 +#define TK_ESCAPE 58 +#define TK_ID 59 +#define TK_COLUMNKW 60 +#define TK_DO 61 +#define TK_FOR 62 +#define TK_IGNORE 63 +#define TK_INITIALLY 64 +#define TK_INSTEAD 65 +#define TK_NO 66 +#define TK_KEY 67 +#define TK_OF 68 +#define TK_OFFSET 69 +#define TK_PRAGMA 70 +#define TK_RAISE 71 +#define TK_RECURSIVE 72 +#define TK_REPLACE 73 +#define TK_RESTRICT 74 +#define TK_ROW 75 +#define TK_ROWS 76 +#define TK_TRIGGER 77 +#define TK_VACUUM 78 +#define TK_VIEW 79 +#define TK_VIRTUAL 80 +#define TK_WITH 81 +#define TK_NULLS 82 +#define TK_FIRST 83 +#define TK_LAST 84 +#define TK_CURRENT 85 +#define TK_FOLLOWING 86 +#define TK_PARTITION 87 +#define TK_PRECEDING 88 +#define TK_RANGE 89 +#define TK_UNBOUNDED 90 +#define TK_EXCLUDE 91 +#define TK_GROUPS 92 +#define TK_OTHERS 93 +#define TK_TIES 94 +#define TK_GENERATED 95 +#define TK_ALWAYS 96 +#define TK_MATERIALIZED 97 +#define TK_REINDEX 98 +#define TK_RENAME 99 +#define TK_CTIME_KW 100 +#define TK_ANY 101 +#define TK_BITAND 102 +#define TK_BITOR 103 +#define TK_LSHIFT 104 +#define TK_RSHIFT 105 +#define TK_PLUS 106 +#define TK_MINUS 107 +#define TK_STAR 108 +#define TK_SLASH 109 +#define TK_REM 110 +#define TK_CONCAT 111 +#define TK_COLLATE 112 +#define TK_BITNOT 113 +#define TK_ON 114 +#define TK_INDEXED 115 +#define TK_STRING 116 +#define TK_JOIN_KW 117 +#define TK_CONSTRAINT 118 +#define TK_DEFAULT 119 +#define TK_NULL 120 +#define TK_PRIMARY 121 +#define TK_UNIQUE 122 +#define TK_CHECK 123 +#define TK_REFERENCES 124 +#define TK_AUTOINCR 125 +#define TK_INSERT 126 +#define TK_DELETE 127 +#define TK_UPDATE 128 +#define TK_SET 129 +#define TK_DEFERRABLE 130 +#define TK_FOREIGN 131 +#define TK_DROP 132 +#define TK_UNION 133 +#define TK_ALL 134 +#define TK_EXCEPT 135 +#define TK_INTERSECT 136 +#define TK_SELECT 137 +#define TK_VALUES 138 +#define TK_DISTINCT 139 +#define TK_DOT 140 +#define TK_FROM 141 +#define TK_JOIN 142 +#define TK_USING 143 +#define TK_ORDER 144 +#define TK_GROUP 145 +#define TK_HAVING 146 +#define TK_LIMIT 147 +#define TK_WHERE 148 +#define TK_RETURNING 149 +#define TK_INTO 150 +#define TK_NOTHING 151 +#define TK_FLOAT 152 +#define TK_BLOB 153 +#define TK_INTEGER 154 +#define TK_VARIABLE 155 +#define TK_CASE 156 +#define TK_WHEN 157 +#define TK_THEN 158 +#define TK_ELSE 159 +#define TK_INDEX 160 +#define TK_ALTER 161 +#define TK_ADD 162 +#define TK_WINDOW 163 +#define TK_OVER 164 +#define TK_FILTER 165 +#define TK_COLUMN 166 +#define TK_AGG_FUNCTION 167 +#define TK_AGG_COLUMN 168 +#define TK_TRUEFALSE 169 +#define TK_ISNOT 170 +#define TK_FUNCTION 171 +#define TK_UMINUS 172 +#define TK_UPLUS 173 +#define TK_TRUTH 174 +#define TK_REGISTER 175 +#define TK_VECTOR 176 +#define TK_SELECT_COLUMN 177 +#define TK_IF_NULL_ROW 178 +#define TK_ASTERISK 179 +#define TK_SPAN 180 +#define TK_ERROR 181 +#define TK_SPACE 182 +#define TK_ILLEGAL 183 + +/************** End of parse.h ***********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +#include +#include +#include +#include +#include + +/* +** Use a macro to replace memcpy() if compiled with SQLITE_INLINE_MEMCPY. +** This allows better measurements of where memcpy() is used when running +** cachegrind. But this macro version of memcpy() is very slow so it +** should not be used in production. This is a performance measurement +** hack only. +*/ +#ifdef SQLITE_INLINE_MEMCPY +# define memcpy(D,S,N) {char*xxd=(char*)(D);const char*xxs=(const char*)(S);\ + int xxn=(N);while(xxn-->0)*(xxd++)=*(xxs++);} +#endif + +/* +** If compiling for a processor that lacks floating point support, +** substitute integer for floating-point +*/ +#ifdef SQLITE_OMIT_FLOATING_POINT +# define double sqlite_int64 +# define float sqlite_int64 +# define LONGDOUBLE_TYPE sqlite_int64 +# ifndef SQLITE_BIG_DBL +# define SQLITE_BIG_DBL (((sqlite3_int64)1)<<50) +# endif +# define SQLITE_OMIT_DATETIME_FUNCS 1 +# define SQLITE_OMIT_TRACE 1 +# undef SQLITE_MIXED_ENDIAN_64BIT_FLOAT +# undef SQLITE_HAVE_ISNAN +#endif +#ifndef SQLITE_BIG_DBL +# define SQLITE_BIG_DBL (1e99) +#endif + +/* +** OMIT_TEMPDB is set to 1 if SQLITE_OMIT_TEMPDB is defined, or 0 +** afterward. Having this macro allows us to cause the C compiler +** to omit code used by TEMP tables without messy #ifndef statements. +*/ +#ifdef SQLITE_OMIT_TEMPDB +#define OMIT_TEMPDB 1 +#else +#define OMIT_TEMPDB 0 +#endif + +/* +** The "file format" number is an integer that is incremented whenever +** the VDBE-level file format changes. The following macros define the +** the default file format for new databases and the maximum file format +** that the library can read. +*/ +#define SQLITE_MAX_FILE_FORMAT 4 +#ifndef SQLITE_DEFAULT_FILE_FORMAT +# define SQLITE_DEFAULT_FILE_FORMAT 4 +#endif + +/* +** Determine whether triggers are recursive by default. This can be +** changed at run-time using a pragma. +*/ +#ifndef SQLITE_DEFAULT_RECURSIVE_TRIGGERS +# define SQLITE_DEFAULT_RECURSIVE_TRIGGERS 0 +#endif + +/* +** Provide a default value for SQLITE_TEMP_STORE in case it is not specified +** on the command-line +*/ +#ifndef SQLITE_TEMP_STORE +# define SQLITE_TEMP_STORE 1 +#endif + +/* +** If no value has been provided for SQLITE_MAX_WORKER_THREADS, or if +** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it +** to zero. +*/ +#if SQLITE_TEMP_STORE==3 || SQLITE_THREADSAFE==0 +# undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 0 +#endif +#ifndef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 8 +#endif +#ifndef SQLITE_DEFAULT_WORKER_THREADS +# define SQLITE_DEFAULT_WORKER_THREADS 0 +#endif +#if SQLITE_DEFAULT_WORKER_THREADS>SQLITE_MAX_WORKER_THREADS +# undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS +#endif + +/* +** The default initial allocation for the pagecache when using separate +** pagecaches for each database connection. A positive number is the +** number of pages. A negative number N translations means that a buffer +** of -1024*N bytes is allocated and used for as many pages as it will hold. +** +** The default value of "20" was choosen to minimize the run-time of the +** speedtest1 test program with options: --shrink-memory --reprepare +*/ +#ifndef SQLITE_DEFAULT_PCACHE_INITSZ +# define SQLITE_DEFAULT_PCACHE_INITSZ 20 +#endif + +/* +** Default value for the SQLITE_CONFIG_SORTERREF_SIZE option. +*/ +#ifndef SQLITE_DEFAULT_SORTERREF_SIZE +# define SQLITE_DEFAULT_SORTERREF_SIZE 0x7fffffff +#endif + +/* +** The compile-time options SQLITE_MMAP_READWRITE and +** SQLITE_ENABLE_BATCH_ATOMIC_WRITE are not compatible with one another. +** You must choose one or the other (or neither) but not both. +*/ +#if defined(SQLITE_MMAP_READWRITE) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +#error Cannot use both SQLITE_MMAP_READWRITE and SQLITE_ENABLE_BATCH_ATOMIC_WRITE +#endif + +/* +** GCC does not define the offsetof() macro so we'll have to do it +** ourselves. +*/ +#ifndef offsetof +#define offsetof(STRUCTURE,FIELD) ((int)((char*)&((STRUCTURE*)0)->FIELD)) +#endif + +/* +** Macros to compute minimum and maximum of two numbers. +*/ +#ifndef MIN +# define MIN(A,B) ((A)<(B)?(A):(B)) +#endif +#ifndef MAX +# define MAX(A,B) ((A)>(B)?(A):(B)) +#endif + +/* +** Swap two objects of type TYPE. +*/ +#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} + +/* +** Check to see if this machine uses EBCDIC. (Yes, believe it or +** not, there are still machines out there that use EBCDIC.) +*/ +#if 'A' == '\301' +# define SQLITE_EBCDIC 1 +#else +# define SQLITE_ASCII 1 +#endif + +/* +** Integers of known sizes. These typedefs might change for architectures +** where the sizes very. Preprocessor macros are available so that the +** types can be conveniently redefined at compile-type. Like this: +** +** cc '-DUINTPTR_TYPE=long long int' ... +*/ +#ifndef UINT32_TYPE +# ifdef HAVE_UINT32_T +# define UINT32_TYPE uint32_t +# else +# define UINT32_TYPE unsigned int +# endif +#endif +#ifndef UINT16_TYPE +# ifdef HAVE_UINT16_T +# define UINT16_TYPE uint16_t +# else +# define UINT16_TYPE unsigned short int +# endif +#endif +#ifndef INT16_TYPE +# ifdef HAVE_INT16_T +# define INT16_TYPE int16_t +# else +# define INT16_TYPE short int +# endif +#endif +#ifndef UINT8_TYPE +# ifdef HAVE_UINT8_T +# define UINT8_TYPE uint8_t +# else +# define UINT8_TYPE unsigned char +# endif +#endif +#ifndef INT8_TYPE +# ifdef HAVE_INT8_T +# define INT8_TYPE int8_t +# else +# define INT8_TYPE signed char +# endif +#endif +#ifndef LONGDOUBLE_TYPE +# define LONGDOUBLE_TYPE long double +#endif +typedef sqlite_int64 i64; /* 8-byte signed integer */ +typedef sqlite_uint64 u64; /* 8-byte unsigned integer */ +typedef UINT32_TYPE u32; /* 4-byte unsigned integer */ +typedef UINT16_TYPE u16; /* 2-byte unsigned integer */ +typedef INT16_TYPE i16; /* 2-byte signed integer */ +typedef UINT8_TYPE u8; /* 1-byte unsigned integer */ +typedef INT8_TYPE i8; /* 1-byte signed integer */ + +/* +** SQLITE_MAX_U32 is a u64 constant that is the maximum u64 value +** that can be stored in a u32 without loss of data. The value +** is 0x00000000ffffffff. But because of quirks of some compilers, we +** have to specify the value in the less intuitive manner shown: +*/ +#define SQLITE_MAX_U32 ((((u64)1)<<32)-1) + +/* +** The datatype used to store estimates of the number of rows in a +** table or index. This is an unsigned integer type. For 99.9% of +** the world, a 32-bit integer is sufficient. But a 64-bit integer +** can be used at compile-time if desired. +*/ +#ifdef SQLITE_64BIT_STATS + typedef u64 tRowcnt; /* 64-bit only if requested at compile-time */ +#else + typedef u32 tRowcnt; /* 32-bit is the default */ +#endif + +/* +** Estimated quantities used for query planning are stored as 16-bit +** logarithms. For quantity X, the value stored is 10*log2(X). This +** gives a possible range of values of approximately 1.0e986 to 1e-986. +** But the allowed values are "grainy". Not every value is representable. +** For example, quantities 16 and 17 are both represented by a LogEst +** of 40. However, since LogEst quantities are suppose to be estimates, +** not exact values, this imprecision is not a problem. +** +** "LogEst" is short for "Logarithmic Estimate". +** +** Examples: +** 1 -> 0 20 -> 43 10000 -> 132 +** 2 -> 10 25 -> 46 25000 -> 146 +** 3 -> 16 100 -> 66 1000000 -> 199 +** 4 -> 20 1000 -> 99 1048576 -> 200 +** 10 -> 33 1024 -> 100 4294967296 -> 320 +** +** The LogEst can be negative to indicate fractional values. +** Examples: +** +** 0.5 -> -10 0.1 -> -33 0.0625 -> -40 +*/ +typedef INT16_TYPE LogEst; + +/* +** Set the SQLITE_PTRSIZE macro to the number of bytes in a pointer +*/ +#ifndef SQLITE_PTRSIZE +# if defined(__SIZEOF_POINTER__) +# define SQLITE_PTRSIZE __SIZEOF_POINTER__ +# elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(_M_ARM) || defined(__arm__) || defined(__x86) || \ + (defined(__APPLE__) && defined(__POWERPC__)) || \ + (defined(__TOS_AIX__) && !defined(__64BIT__)) +# define SQLITE_PTRSIZE 4 +# else +# define SQLITE_PTRSIZE 8 +# endif +#endif + +/* The uptr type is an unsigned integer large enough to hold a pointer +*/ +#if defined(HAVE_STDINT_H) + typedef uintptr_t uptr; +#elif SQLITE_PTRSIZE==4 + typedef u32 uptr; +#else + typedef u64 uptr; +#endif + +/* +** The SQLITE_WITHIN(P,S,E) macro checks to see if pointer P points to +** something between S (inclusive) and E (exclusive). +** +** In other words, S is a buffer and E is a pointer to the first byte after +** the end of buffer S. This macro returns true if P points to something +** contained within the buffer S. +*/ +#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) + + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSQLITE_BYTEORDER=0 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SQLITE_BYTEORDER +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ppc__) || \ + defined(__ARMEB__) || defined(__AARCH64EB__) +# define SQLITE_BYTEORDER 4321 +# else +# define SQLITE_BYTEORDER 0 +# endif +#endif +#if SQLITE_BYTEORDER==4321 +# define SQLITE_BIGENDIAN 1 +# define SQLITE_LITTLEENDIAN 0 +# define SQLITE_UTF16NATIVE SQLITE_UTF16BE +#elif SQLITE_BYTEORDER==1234 +# define SQLITE_BIGENDIAN 0 +# define SQLITE_LITTLEENDIAN 1 +# define SQLITE_UTF16NATIVE SQLITE_UTF16LE +#else +# ifdef SQLITE_AMALGAMATION + const int sqlite3one = 1; +# else + extern const int sqlite3one; +# endif +# define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) +# define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) +# define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE) +#endif + +/* +** Constants for the largest and smallest possible 64-bit signed integers. +** These macros are designed to work correctly on both 32-bit and 64-bit +** compilers. +*/ +#define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +#define LARGEST_UINT64 (0xffffffff|(((u64)0xffffffff)<<32)) +#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) + +/* +** Round up a number to the next larger multiple of 8. This is used +** to force 8-byte alignment on 64-bit architectures. +*/ +#define ROUND8(x) (((x)+7)&~7) + +/* +** Round down to the nearest multiple of 8 +*/ +#define ROUNDDOWN8(x) ((x)&~7) + +/* +** Assert that the pointer X is aligned to an 8-byte boundary. This +** macro is used only within assert() to verify that the code gets +** all alignment restrictions correct. +** +** Except, if SQLITE_4_BYTE_ALIGNED_MALLOC is defined, then the +** underlying malloc() implementation might return us 4-byte aligned +** pointers. In that case, only verify 4-byte alignment. +*/ +#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC +# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&3)==0) +#else +# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) +#endif + +/* +** Disable MMAP on platforms where it is known to not work +*/ +#if defined(__OpenBSD__) || defined(__QNXNTO__) +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +#endif + +/* +** Default maximum size of memory used by memory-mapped I/O in the VFS +*/ +#ifdef __APPLE__ +# include +#endif +#ifndef SQLITE_MAX_MMAP_SIZE +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) \ + || defined(__FreeBSD__) \ + || defined(__DragonFly__) +# define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ +# else +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +#endif + +/* +** The default MMAP_SIZE is zero on all platforms. Or, even if a larger +** default MMAP_SIZE is specified at compile-time, make sure that it does +** not exceed the maximum mmap size. +*/ +#ifndef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE 0 +#endif +#if SQLITE_DEFAULT_MMAP_SIZE>SQLITE_MAX_MMAP_SIZE +# undef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE +#endif + +/* +** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not +** the Select query generator tracing logic is turned on. +*/ +#if !defined(SQLITE_AMALGAMATION) +SQLITE_PRIVATE u32 sqlite3SelectTrace; +#endif +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_SELECTTRACE)) +# define SELECTTRACE_ENABLED 1 +# define SELECTTRACE(K,P,S,X) \ + if(sqlite3SelectTrace&(K)) \ + sqlite3DebugPrintf("%u/%d/%p: ",(S)->selId,(P)->addrExplain,(S)),\ + sqlite3DebugPrintf X +#else +# define SELECTTRACE(K,P,S,X) +# define SELECTTRACE_ENABLED 0 +#endif + +/* +** Macros for "wheretrace" +*/ +SQLITE_PRIVATE u32 sqlite3WhereTrace; +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) +# define WHERETRACE(K,X) if(sqlite3WhereTrace&(K)) sqlite3DebugPrintf X +# define WHERETRACE_ENABLED 1 +#else +# define WHERETRACE(K,X) +#endif + + +/* +** An instance of the following structure is used to store the busy-handler +** callback for a given sqlite handle. +** +** The sqlite.busyHandler member of the sqlite struct contains the busy +** callback for the database handle. Each pager opened via the sqlite +** handle is passed a pointer to sqlite.busyHandler. The busy-handler +** callback is currently invoked only from within pager.c. +*/ +typedef struct BusyHandler BusyHandler; +struct BusyHandler { + int (*xBusyHandler)(void *,int); /* The busy callback */ + void *pBusyArg; /* First arg to busy callback */ + int nBusy; /* Incremented with each busy call */ +}; + +/* +** Name of table that holds the database schema. +*/ +#define DFLT_SCHEMA_TABLE "sqlite_master" +#define DFLT_TEMP_SCHEMA_TABLE "sqlite_temp_master" +#define ALT_SCHEMA_TABLE "sqlite_schema" +#define ALT_TEMP_SCHEMA_TABLE "sqlite_temp_schema" + + +/* +** The root-page of the schema table. +*/ +#define SCHEMA_ROOT 1 + +/* +** The name of the schema table. The name is different for TEMP. +*/ +#define SCHEMA_TABLE(x) \ + ((!OMIT_TEMPDB)&&(x==1)?DFLT_TEMP_SCHEMA_TABLE:DFLT_SCHEMA_TABLE) + +/* +** A convenience macro that returns the number of elements in +** an array. +*/ +#define ArraySize(X) ((int)(sizeof(X)/sizeof(X[0]))) + +/* +** Determine if the argument is a power of two +*/ +#define IsPowerOfTwo(X) (((X)&((X)-1))==0) + +/* +** The following value as a destructor means to use sqlite3DbFree(). +** The sqlite3DbFree() routine requires two parameters instead of the +** one parameter that destructors normally want. So we have to introduce +** this magic value that the code knows to handle differently. Any +** pointer will work here as long as it is distinct from SQLITE_STATIC +** and SQLITE_TRANSIENT. +*/ +#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3OomFault) + +/* +** When SQLITE_OMIT_WSD is defined, it means that the target platform does +** not support Writable Static Data (WSD) such as global and static variables. +** All variables must either be on the stack or dynamically allocated from +** the heap. When WSD is unsupported, the variable declarations scattered +** throughout the SQLite code must become constants instead. The SQLITE_WSD +** macro is used for this purpose. And instead of referencing the variable +** directly, we use its constant as a key to lookup the run-time allocated +** buffer that holds real variable. The constant is also the initializer +** for the run-time allocated buffer. +** +** In the usual case where WSD is supported, the SQLITE_WSD and GLOBAL +** macros become no-ops and have zero performance impact. +*/ +#ifdef SQLITE_OMIT_WSD + #define SQLITE_WSD const + #define GLOBAL(t,v) (*(t*)sqlite3_wsd_find((void*)&(v), sizeof(v))) + #define sqlite3GlobalConfig GLOBAL(struct Sqlite3Config, sqlite3Config) +SQLITE_API int sqlite3_wsd_init(int N, int J); +SQLITE_API void *sqlite3_wsd_find(void *K, int L); +#else + #define SQLITE_WSD + #define GLOBAL(t,v) v + #define sqlite3GlobalConfig sqlite3Config +#endif + +/* +** The following macros are used to suppress compiler warnings and to +** make it clear to human readers when a function parameter is deliberately +** left unused within the body of a function. This usually happens when +** a function is called via a function pointer. For example the +** implementation of an SQL aggregate step callback may not use the +** parameter indicating the number of arguments passed to the aggregate, +** if it knows that this is enforced elsewhere. +** +** When a function parameter is not used at all within the body of a function, +** it is generally named "NotUsed" or "NotUsed2" to make things even clearer. +** However, these macros may also be used to suppress warnings related to +** parameters that may or may not be used depending on compilation options. +** For example those parameters only used in assert() statements. In these +** cases the parameters are named as per the usual conventions. +*/ +#define UNUSED_PARAMETER(x) (void)(x) +#define UNUSED_PARAMETER2(x,y) UNUSED_PARAMETER(x),UNUSED_PARAMETER(y) + +/* +** Forward references to structures +*/ +typedef struct AggInfo AggInfo; +typedef struct AuthContext AuthContext; +typedef struct AutoincInfo AutoincInfo; +typedef struct Bitvec Bitvec; +typedef struct CollSeq CollSeq; +typedef struct Column Column; +typedef struct Cte Cte; +typedef struct CteUse CteUse; +typedef struct Db Db; +typedef struct DbFixer DbFixer; +typedef struct Schema Schema; +typedef struct Expr Expr; +typedef struct ExprList ExprList; +typedef struct FKey FKey; +typedef struct FuncDestructor FuncDestructor; +typedef struct FuncDef FuncDef; +typedef struct FuncDefHash FuncDefHash; +typedef struct IdList IdList; +typedef struct Index Index; +typedef struct IndexSample IndexSample; +typedef struct KeyClass KeyClass; +typedef struct KeyInfo KeyInfo; +typedef struct Lookaside Lookaside; +typedef struct LookasideSlot LookasideSlot; +typedef struct Module Module; +typedef struct NameContext NameContext; +typedef struct Parse Parse; +typedef struct ParseCleanup ParseCleanup; +typedef struct PreUpdate PreUpdate; +typedef struct PrintfArguments PrintfArguments; +typedef struct RenameToken RenameToken; +typedef struct Returning Returning; +typedef struct RowSet RowSet; +typedef struct Savepoint Savepoint; +typedef struct Select Select; +typedef struct SQLiteThread SQLiteThread; +typedef struct SelectDest SelectDest; +typedef struct SrcItem SrcItem; +typedef struct SrcList SrcList; +typedef struct sqlite3_str StrAccum; /* Internal alias for sqlite3_str */ +typedef struct Table Table; +typedef struct TableLock TableLock; +typedef struct Token Token; +typedef struct TreeView TreeView; +typedef struct Trigger Trigger; +typedef struct TriggerPrg TriggerPrg; +typedef struct TriggerStep TriggerStep; +typedef struct UnpackedRecord UnpackedRecord; +typedef struct Upsert Upsert; +typedef struct VTable VTable; +typedef struct VtabCtx VtabCtx; +typedef struct Walker Walker; +typedef struct WhereInfo WhereInfo; +typedef struct Window Window; +typedef struct With With; + + +/* +** The bitmask datatype defined below is used for various optimizations. +** +** Changing this from a 64-bit to a 32-bit type limits the number of +** tables in a join to 32 instead of 64. But it also reduces the size +** of the library by 738 bytes on ix86. +*/ +#ifdef SQLITE_BITMASK_TYPE + typedef SQLITE_BITMASK_TYPE Bitmask; +#else + typedef u64 Bitmask; +#endif + +/* +** The number of bits in a Bitmask. "BMS" means "BitMask Size". +*/ +#define BMS ((int)(sizeof(Bitmask)*8)) + +/* +** A bit in a Bitmask +*/ +#define MASKBIT(n) (((Bitmask)1)<<(n)) +#define MASKBIT64(n) (((u64)1)<<(n)) +#define MASKBIT32(n) (((unsigned int)1)<<(n)) +#define ALLBITS ((Bitmask)-1) + +/* A VList object records a mapping between parameters/variables/wildcards +** in the SQL statement (such as $abc, @pqr, or :xyz) and the integer +** variable number associated with that parameter. See the format description +** on the sqlite3VListAdd() routine for more information. A VList is really +** just an array of integers. +*/ +typedef int VList; + +/* +** Defer sourcing vdbe.h and btree.h until after the "u8" and +** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque +** pointer types (i.e. FuncDef) defined above. +*/ +/************** Include pager.h in the middle of sqliteInt.h *****************/ +/************** Begin file pager.h *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the sqlite page cache +** subsystem. The page cache subsystem reads and writes a file a page +** at a time and provides a journal for rollback. +*/ + +#ifndef SQLITE_PAGER_H +#define SQLITE_PAGER_H + +/* +** Default maximum size for persistent journal files. A negative +** value means no limit. This value may be overridden using the +** sqlite3PagerJournalSizeLimit() API. See also "PRAGMA journal_size_limit". +*/ +#ifndef SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT + #define SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT -1 +#endif + +/* +** The type used to represent a page number. The first page in a file +** is called page 1. 0 is used to represent "not a page". +*/ +typedef u32 Pgno; + +/* +** Each open file is managed by a separate instance of the "Pager" structure. +*/ +typedef struct Pager Pager; + +/* +** Handle type for pages. +*/ +typedef struct PgHdr DbPage; + +/* +** Page number PAGER_MJ_PGNO is never used in an SQLite database (it is +** reserved for working around a windows/posix incompatibility). It is +** used in the journal to signify that the remainder of the journal file +** is devoted to storing a super-journal name - there are no more pages to +** roll back. See comments for function writeSuperJournal() in pager.c +** for details. +*/ +#define PAGER_MJ_PGNO(x) ((Pgno)((PENDING_BYTE/((x)->pageSize))+1)) + +/* +** Allowed values for the flags parameter to sqlite3PagerOpen(). +** +** NOTE: These values must match the corresponding BTREE_ values in btree.h. +*/ +#define PAGER_OMIT_JOURNAL 0x0001 /* Do not use a rollback journal */ +#define PAGER_MEMORY 0x0002 /* In-memory database */ + +/* +** Valid values for the second argument to sqlite3PagerLockingMode(). +*/ +#define PAGER_LOCKINGMODE_QUERY -1 +#define PAGER_LOCKINGMODE_NORMAL 0 +#define PAGER_LOCKINGMODE_EXCLUSIVE 1 + +/* +** Numeric constants that encode the journalmode. +** +** The numeric values encoded here (other than PAGER_JOURNALMODE_QUERY) +** are exposed in the API via the "PRAGMA journal_mode" command and +** therefore cannot be changed without a compatibility break. +*/ +#define PAGER_JOURNALMODE_QUERY (-1) /* Query the value of journalmode */ +#define PAGER_JOURNALMODE_DELETE 0 /* Commit by deleting journal file */ +#define PAGER_JOURNALMODE_PERSIST 1 /* Commit by zeroing journal header */ +#define PAGER_JOURNALMODE_OFF 2 /* Journal omitted. */ +#define PAGER_JOURNALMODE_TRUNCATE 3 /* Commit by truncating journal */ +#define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ +#define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ + +/* +** Flags that make up the mask passed to sqlite3PagerGet(). +*/ +#define PAGER_GET_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_GET_READONLY 0x02 /* Read-only page is acceptable */ + +/* +** Flags for sqlite3PagerSetFlags() +** +** Value constraints (enforced via assert()): +** PAGER_FULLFSYNC == SQLITE_FullFSync +** PAGER_CKPT_FULLFSYNC == SQLITE_CkptFullFSync +** PAGER_CACHE_SPILL == SQLITE_CacheSpill +*/ +#define PAGER_SYNCHRONOUS_OFF 0x01 /* PRAGMA synchronous=OFF */ +#define PAGER_SYNCHRONOUS_NORMAL 0x02 /* PRAGMA synchronous=NORMAL */ +#define PAGER_SYNCHRONOUS_FULL 0x03 /* PRAGMA synchronous=FULL */ +#define PAGER_SYNCHRONOUS_EXTRA 0x04 /* PRAGMA synchronous=EXTRA */ +#define PAGER_SYNCHRONOUS_MASK 0x07 /* Mask for four values above */ +#define PAGER_FULLFSYNC 0x08 /* PRAGMA fullfsync=ON */ +#define PAGER_CKPT_FULLFSYNC 0x10 /* PRAGMA checkpoint_fullfsync=ON */ +#define PAGER_CACHESPILL 0x20 /* PRAGMA cache_spill=ON */ +#define PAGER_FLAGS_MASK 0x38 /* All above except SYNCHRONOUS */ + +/* +** The remainder of this file contains the declarations of the functions +** that make up the Pager sub-system API. See source code comments for +** a detailed description of each routine. +*/ + +/* Open and close a Pager connection. */ +SQLITE_PRIVATE int sqlite3PagerOpen( + sqlite3_vfs*, + Pager **ppPager, + const char*, + int, + int, + int, + void(*)(DbPage*) +); +SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3*); +SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); + +/* Functions used to configure a Pager object. */ +SQLITE_PRIVATE void sqlite3PagerSetBusyHandler(Pager*, int(*)(void *), void *); +SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); +SQLITE_PRIVATE Pgno sqlite3PagerMaxPageCount(Pager*, Pgno); +SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); +SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager*, int); +SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); +SQLITE_PRIVATE void sqlite3PagerShrink(Pager*); +SQLITE_PRIVATE void sqlite3PagerSetFlags(Pager*,unsigned); +SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *, int); +SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *, int); +SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager*); +SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager*); +SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *, i64); +SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager*); +SQLITE_PRIVATE int sqlite3PagerFlush(Pager*); + +/* Functions used to obtain and release page references. */ +SQLITE_PRIVATE int sqlite3PagerGet(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); +SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); +SQLITE_PRIVATE void sqlite3PagerRef(DbPage*); +SQLITE_PRIVATE void sqlite3PagerUnref(DbPage*); +SQLITE_PRIVATE void sqlite3PagerUnrefNotNull(DbPage*); +SQLITE_PRIVATE void sqlite3PagerUnrefPageOne(DbPage*); + +/* Operations on page references. */ +SQLITE_PRIVATE int sqlite3PagerWrite(DbPage*); +SQLITE_PRIVATE void sqlite3PagerDontWrite(DbPage*); +SQLITE_PRIVATE int sqlite3PagerMovepage(Pager*,DbPage*,Pgno,int); +SQLITE_PRIVATE int sqlite3PagerPageRefcount(DbPage*); +SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *); +SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *); + +/* Functions used to manage pager transactions and savepoints. */ +SQLITE_PRIVATE void sqlite3PagerPagecount(Pager*, int*); +SQLITE_PRIVATE int sqlite3PagerBegin(Pager*, int exFlag, int); +SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(Pager*,const char *zSuper, int); +SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager*); +SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zSuper); +SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager*); +SQLITE_PRIVATE int sqlite3PagerRollback(Pager*); +SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int n); +SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); +SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager); + +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, sqlite3*, int, int*, int*); +SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); +SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3*); +# ifdef SQLITE_ENABLE_SNAPSHOT +SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager*, sqlite3_snapshot **ppSnapshot); +SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager*, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE int sqlite3PagerSnapshotRecover(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager); +# endif +#endif + +#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_ENABLE_SETLK_TIMEOUT) +SQLITE_PRIVATE int sqlite3PagerWalWriteLock(Pager*, int); +SQLITE_PRIVATE void sqlite3PagerWalDb(Pager*, sqlite3*); +#else +# define sqlite3PagerWalWriteLock(y,z) SQLITE_OK +# define sqlite3PagerWalDb(x,y) +#endif + +#ifdef SQLITE_DIRECT_OVERFLOW_READ +SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno); +#endif + +#ifdef SQLITE_ENABLE_ZIPVFS +SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager); +#endif + +/* Functions used to query pager state and configuration. */ +SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager*); +SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); +#endif +SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager*); +SQLITE_PRIVATE const char *sqlite3PagerFilename(const Pager*, int); +SQLITE_PRIVATE sqlite3_vfs *sqlite3PagerVfs(Pager*); +SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager*); +SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager*); +SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager*); +SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*); +SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); +SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); +SQLITE_PRIVATE void sqlite3PagerClearCache(Pager*); +SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *); + +/* Functions used to truncate the database file. */ +SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno); + +SQLITE_PRIVATE void sqlite3PagerRekey(DbPage*, Pgno, u16); + +/* Functions to support testing and debugging. */ +#if !defined(NDEBUG) || defined(SQLITE_TEST) +SQLITE_PRIVATE Pgno sqlite3PagerPagenumber(DbPage*); +SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage*); +#endif +#ifdef SQLITE_TEST +SQLITE_PRIVATE int *sqlite3PagerStats(Pager*); +SQLITE_PRIVATE void sqlite3PagerRefdump(Pager*); + void disable_simulated_io_errors(void); + void enable_simulated_io_errors(void); +#else +# define disable_simulated_io_errors() +# define enable_simulated_io_errors() +#endif + +#endif /* SQLITE_PAGER_H */ + +/************** End of pager.h ***********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include btree.h in the middle of sqliteInt.h *****************/ +/************** Begin file btree.h *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the sqlite B-Tree file +** subsystem. See comments in the source code for a detailed description +** of what each interface routine does. +*/ +#ifndef SQLITE_BTREE_H +#define SQLITE_BTREE_H + +/* TODO: This definition is just included so other modules compile. It +** needs to be revisited. +*/ +#define SQLITE_N_BTREE_META 16 + +/* +** If defined as non-zero, auto-vacuum is enabled by default. Otherwise +** it must be turned on for each database using "PRAGMA auto_vacuum = 1". +*/ +#ifndef SQLITE_DEFAULT_AUTOVACUUM + #define SQLITE_DEFAULT_AUTOVACUUM 0 +#endif + +#define BTREE_AUTOVACUUM_NONE 0 /* Do not do auto-vacuum */ +#define BTREE_AUTOVACUUM_FULL 1 /* Do full auto-vacuum */ +#define BTREE_AUTOVACUUM_INCR 2 /* Incremental vacuum */ + +/* +** Forward declarations of structure +*/ +typedef struct Btree Btree; +typedef struct BtCursor BtCursor; +typedef struct BtShared BtShared; +typedef struct BtreePayload BtreePayload; + + +SQLITE_PRIVATE int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use with this b-tree */ + const char *zFilename, /* Name of database file to open */ + sqlite3 *db, /* Associated database connection */ + Btree **ppBtree, /* Return open Btree* here */ + int flags, /* Flags */ + int vfsFlags /* Flags passed through to VFS open */ +); + +/* The flags parameter to sqlite3BtreeOpen can be the bitwise or of the +** following values. +** +** NOTE: These values must match the corresponding PAGER_ values in +** pager.h. +*/ +#define BTREE_OMIT_JOURNAL 1 /* Do not create or use a rollback journal */ +#define BTREE_MEMORY 2 /* This is an in-memory DB */ +#define BTREE_SINGLE 4 /* The file contains at most 1 b-tree */ +#define BTREE_UNORDERED 8 /* Use of a hash implementation is OK */ + +SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); +SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeSetSpillSize(Btree*,int); +#if SQLITE_MAX_MMAP_SIZE>0 +SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#endif +SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags(Btree*,unsigned); +SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); +SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); +SQLITE_PRIVATE Pgno sqlite3BtreeMaxPageCount(Btree*,Pgno); +SQLITE_PRIVATE Pgno sqlite3BtreeLastPage(Btree*); +SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeGetRequestedReserve(Btree*); +SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p); +SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); +SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int,int*); +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree*, const char*); +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree*, int); +SQLITE_PRIVATE int sqlite3BtreeCommit(Btree*); +SQLITE_PRIVATE int sqlite3BtreeRollback(Btree*,int,int); +SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree*, Pgno*, int flags); +SQLITE_PRIVATE int sqlite3BtreeTxnState(Btree*); +SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree*); + +SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *, int, void(*)(void *)); +SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *pBtree); +#ifndef SQLITE_OMIT_SHARED_CACHE +SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *pBtree, int iTab, u8 isWriteLock); +#endif + +/* Savepoints are named, nestable SQL transactions mostly implemented */ +/* in vdbe.c and pager.c See https://sqlite.org/lang_savepoint.html */ +SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *, int, int); + +/* "Checkpoint" only refers to WAL. See https://sqlite.org/wal.html#ckpt */ +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); +#endif + +SQLITE_PRIVATE const char *sqlite3BtreeGetFilename(Btree *); +SQLITE_PRIVATE const char *sqlite3BtreeGetJournalname(Btree *); +SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *, Btree *); + +SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *); + +/* The flags parameter to sqlite3BtreeCreateTable can be the bitwise OR +** of the flags shown below. +** +** Every SQLite table must have either BTREE_INTKEY or BTREE_BLOBKEY set. +** With BTREE_INTKEY, the table key is a 64-bit integer and arbitrary data +** is stored in the leaves. (BTREE_INTKEY is used for SQL tables.) With +** BTREE_BLOBKEY, the key is an arbitrary BLOB and no content is stored +** anywhere - the key is the content. (BTREE_BLOBKEY is used for SQL +** indices.) +*/ +#define BTREE_INTKEY 1 /* Table has only 64-bit signed integer keys */ +#define BTREE_BLOBKEY 2 /* Table has keys only - no data */ + +SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree*, int, int*); +SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree*, int, i64*); +SQLITE_PRIVATE int sqlite3BtreeClearTableOfCursor(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree*, int, int); + +SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); +SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); + +SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); + +/* +** The second parameter to sqlite3BtreeGetMeta or sqlite3BtreeUpdateMeta +** should be one of the following values. The integer values are assigned +** to constants so that the offset of the corresponding field in an +** SQLite database header may be found using the following formula: +** +** offset = 36 + (idx * 4) +** +** For example, the free-page-count field is located at byte offset 36 of +** the database file header. The incr-vacuum-flag field is located at +** byte offset 64 (== 36+4*7). +** +** The BTREE_DATA_VERSION value is not really a value stored in the header. +** It is a read-only number computed by the pager. But we merge it with +** the header value access routines since its access pattern is the same. +** Call it a "virtual meta value". +*/ +#define BTREE_FREE_PAGE_COUNT 0 +#define BTREE_SCHEMA_VERSION 1 +#define BTREE_FILE_FORMAT 2 +#define BTREE_DEFAULT_CACHE_SIZE 3 +#define BTREE_LARGEST_ROOT_PAGE 4 +#define BTREE_TEXT_ENCODING 5 +#define BTREE_USER_VERSION 6 +#define BTREE_INCR_VACUUM 7 +#define BTREE_APPLICATION_ID 8 +#define BTREE_DATA_VERSION 15 /* A virtual meta-value */ + +/* +** Kinds of hints that can be passed into the sqlite3BtreeCursorHint() +** interface. +** +** BTREE_HINT_RANGE (arguments: Expr*, Mem*) +** +** The first argument is an Expr* (which is guaranteed to be constant for +** the lifetime of the cursor) that defines constraints on which rows +** might be fetched with this cursor. The Expr* tree may contain +** TK_REGISTER nodes that refer to values stored in the array of registers +** passed as the second parameter. In other words, if Expr.op==TK_REGISTER +** then the value of the node is the value in Mem[pExpr.iTable]. Any +** TK_COLUMN node in the expression tree refers to the Expr.iColumn-th +** column of the b-tree of the cursor. The Expr tree will not contain +** any function calls nor subqueries nor references to b-trees other than +** the cursor being hinted. +** +** The design of the _RANGE hint is aid b-tree implementations that try +** to prefetch content from remote machines - to provide those +** implementations with limits on what needs to be prefetched and thereby +** reduce network bandwidth. +** +** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by +** standard SQLite. The other hints are provided for extentions that use +** the SQLite parser and code generator but substitute their own storage +** engine. +*/ +#define BTREE_HINT_RANGE 0 /* Range constraints on queries */ + +/* +** Values that may be OR'd together to form the argument to the +** BTREE_HINT_FLAGS hint for sqlite3BtreeCursorHint(): +** +** The BTREE_BULKLOAD flag is set on index cursors when the index is going +** to be filled with content that is already in sorted order. +** +** The BTREE_SEEK_EQ flag is set on cursors that will get OP_SeekGE or +** OP_SeekLE opcodes for a range search, but where the range of entries +** selected will all have the same key. In other words, the cursor will +** be used only for equality key searches. +** +*/ +#define BTREE_BULKLOAD 0x00000001 /* Used to full index in sorted order */ +#define BTREE_SEEK_EQ 0x00000002 /* EQ seeks only - no range seeks */ + +/* +** Flags passed as the third argument to sqlite3BtreeCursor(). +** +** For read-only cursors the wrFlag argument is always zero. For read-write +** cursors it may be set to either (BTREE_WRCSR|BTREE_FORDELETE) or just +** (BTREE_WRCSR). If the BTREE_FORDELETE bit is set, then the cursor will +** only be used by SQLite for the following: +** +** * to seek to and then delete specific entries, and/or +** +** * to read values that will be used to create keys that other +** BTREE_FORDELETE cursors will seek to and delete. +** +** The BTREE_FORDELETE flag is an optimization hint. It is not used by +** by this, the native b-tree engine of SQLite, but it is available to +** alternative storage engines that might be substituted in place of this +** b-tree system. For alternative storage engines in which a delete of +** the main table row automatically deletes corresponding index rows, +** the FORDELETE flag hint allows those alternative storage engines to +** skip a lot of work. Namely: FORDELETE cursors may treat all SEEK +** and DELETE operations as no-ops, and any READ operation against a +** FORDELETE cursor may return a null row: 0x01 0x00. +*/ +#define BTREE_WRCSR 0x00000004 /* read-write cursor */ +#define BTREE_FORDELETE 0x00000008 /* Cursor is for seek/delete only */ + +SQLITE_PRIVATE int sqlite3BtreeCursor( + Btree*, /* BTree containing table to open */ + Pgno iTable, /* Index of root page */ + int wrFlag, /* 1 for writing. 0 for read-only */ + struct KeyInfo*, /* First argument to compare function */ + BtCursor *pCursor /* Space to write cursor structure */ +); +SQLITE_PRIVATE BtCursor *sqlite3BtreeFakeValidCursor(void); +SQLITE_PRIVATE int sqlite3BtreeCursorSize(void); +SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned); +#ifdef SQLITE_ENABLE_CURSOR_HINTS +SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor*, int, ...); +#endif + +SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeTableMoveto( + BtCursor*, + i64 intKey, + int bias, + int *pRes +); +SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( + BtCursor*, + UnpackedRecord *pUnKey, + int *pRes +); +SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor*, int*); +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*, u8 flags); + +/* Allowed flags for sqlite3BtreeDelete() and sqlite3BtreeInsert() */ +#define BTREE_SAVEPOSITION 0x02 /* Leave cursor pointing at NEXT or PREV */ +#define BTREE_AUXDELETE 0x04 /* not the primary delete operation */ +#define BTREE_APPEND 0x08 /* Insert is likely an append */ +#define BTREE_PREFORMAT 0x80 /* Inserted data is a preformated cell */ + +/* An instance of the BtreePayload object describes the content of a single +** entry in either an index or table btree. +** +** Index btrees (used for indexes and also WITHOUT ROWID tables) contain +** an arbitrary key and no data. These btrees have pKey,nKey set to the +** key and the pData,nData,nZero fields are uninitialized. The aMem,nMem +** fields give an array of Mem objects that are a decomposition of the key. +** The nMem field might be zero, indicating that no decomposition is available. +** +** Table btrees (used for rowid tables) contain an integer rowid used as +** the key and passed in the nKey field. The pKey field is zero. +** pData,nData hold the content of the new entry. nZero extra zero bytes +** are appended to the end of the content when constructing the entry. +** The aMem,nMem fields are uninitialized for table btrees. +** +** Field usage summary: +** +** Table BTrees Index Btrees +** +** pKey always NULL encoded key +** nKey the ROWID length of pKey +** pData data not used +** aMem not used decomposed key value +** nMem not used entries in aMem +** nData length of pData not used +** nZero extra zeros after pData not used +** +** This object is used to pass information into sqlite3BtreeInsert(). The +** same information used to be passed as five separate parameters. But placing +** the information into this object helps to keep the interface more +** organized and understandable, and it also helps the resulting code to +** run a little faster by using fewer registers for parameter passing. +*/ +struct BtreePayload { + const void *pKey; /* Key content for indexes. NULL for tables */ + sqlite3_int64 nKey; /* Size of pKey for indexes. PRIMARY KEY for tabs */ + const void *pData; /* Data for tables. */ + sqlite3_value *aMem; /* First of nMem value in the unpacked pKey */ + u16 nMem; /* Number of aMem[] value. Might be zero */ + int nData; /* Size of pData. 0 if none. */ + int nZero; /* Extra zero data appended after pData,nData */ +}; + +SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, + int flags, int seekResult); +SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor*, int *pRes); +SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor*, int *pRes); +SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int flags); +SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int flags); +SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeCursorPin(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor*); +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC +SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor*); +#endif +SQLITE_PRIVATE int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); +SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); +SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor*); +SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor*); + +SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(sqlite3*,Btree*,Pgno*aRoot,int nRoot,int,int*); +SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); +SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor*); + +#ifndef SQLITE_OMIT_INCRBLOB +SQLITE_PRIVATE int sqlite3BtreePayloadChecked(BtCursor*, u32 offset, u32 amt, void*); +SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); +SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *); +#endif +SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *); +SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); +SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); +SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *pBt); +SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void); + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE sqlite3_uint64 sqlite3BtreeSeekCount(Btree*); +#else +# define sqlite3BtreeSeekCount(X) 0 +#endif + +#ifndef NDEBUG +SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor*); +#endif +SQLITE_PRIVATE int sqlite3BtreeCursorIsValidNN(BtCursor*); + +SQLITE_PRIVATE int sqlite3BtreeCount(sqlite3*, BtCursor*, i64*); + +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3BtreeCursorInfo(BtCursor*, int*, int); +SQLITE_PRIVATE void sqlite3BtreeCursorList(Btree*); +#endif + +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); +#endif + +SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor*, BtCursor*, i64); + +/* +** If we are not using shared cache, then there is no need to +** use mutexes to access the BtShared structures. So make the +** Enter and Leave procedures no-ops. +*/ +#ifndef SQLITE_OMIT_SHARED_CACHE +SQLITE_PRIVATE void sqlite3BtreeEnter(Btree*); +SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3*); +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree*); +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeConnectionCount(Btree*); +#else +# define sqlite3BtreeEnter(X) +# define sqlite3BtreeEnterAll(X) +# define sqlite3BtreeSharable(X) 0 +# define sqlite3BtreeEnterCursor(X) +# define sqlite3BtreeConnectionCount(X) 1 +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE +SQLITE_PRIVATE void sqlite3BtreeLeave(Btree*); +SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3*); +#ifndef NDEBUG + /* These routines are used inside assert() statements only. */ +SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree*); +SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3*); +SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); +#endif +#else + +# define sqlite3BtreeLeave(X) +# define sqlite3BtreeLeaveCursor(X) +# define sqlite3BtreeLeaveAll(X) + +# define sqlite3BtreeHoldsMutex(X) 1 +# define sqlite3BtreeHoldsAllMutexes(X) 1 +# define sqlite3SchemaMutexHeld(X,Y,Z) 1 +#endif + + +#endif /* SQLITE_BTREE_H */ + +/************** End of btree.h ***********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include vdbe.h in the middle of sqliteInt.h ******************/ +/************** Begin file vdbe.h ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Header file for the Virtual DataBase Engine (VDBE) +** +** This header defines the interface to the virtual database engine +** or VDBE. The VDBE implements an abstract machine that runs a +** simple program to access and modify the underlying database. +*/ +#ifndef SQLITE_VDBE_H +#define SQLITE_VDBE_H +/* #include */ + +/* +** A single VDBE is an opaque structure named "Vdbe". Only routines +** in the source file sqliteVdbe.c are allowed to see the insides +** of this structure. +*/ +typedef struct Vdbe Vdbe; + +/* +** The names of the following types declared in vdbeInt.h are required +** for the VdbeOp definition. +*/ +typedef struct sqlite3_value Mem; +typedef struct SubProgram SubProgram; + +/* +** A single instruction of the virtual machine has an opcode +** and as many as three operands. The instruction is recorded +** as an instance of the following structure: +*/ +struct VdbeOp { + u8 opcode; /* What operation to perform */ + signed char p4type; /* One of the P4_xxx constants for p4 */ + u16 p5; /* Fifth parameter is an unsigned 16-bit integer */ + int p1; /* First operand */ + int p2; /* Second parameter (often the jump destination) */ + int p3; /* The third parameter */ + union p4union { /* fourth parameter */ + int i; /* Integer value if p4type==P4_INT32 */ + void *p; /* Generic pointer */ + char *z; /* Pointer to data for string (char array) types */ + i64 *pI64; /* Used when p4type is P4_INT64 */ + double *pReal; /* Used when p4type is P4_REAL */ + FuncDef *pFunc; /* Used when p4type is P4_FUNCDEF */ + sqlite3_context *pCtx; /* Used when p4type is P4_FUNCCTX */ + CollSeq *pColl; /* Used when p4type is P4_COLLSEQ */ + Mem *pMem; /* Used when p4type is P4_MEM */ + VTable *pVtab; /* Used when p4type is P4_VTAB */ + KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ + u32 *ai; /* Used when p4type is P4_INTARRAY */ + SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ + Table *pTab; /* Used when p4type is P4_TABLE */ +#ifdef SQLITE_ENABLE_CURSOR_HINTS + Expr *pExpr; /* Used when p4type is P4_EXPR */ +#endif + int (*xAdvance)(BtCursor *, int); + } p4; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + char *zComment; /* Comment to improve readability */ +#endif +#ifdef VDBE_PROFILE + u32 cnt; /* Number of times this instruction was executed */ + u64 cycles; /* Total time spent executing this instruction */ +#endif +#ifdef SQLITE_VDBE_COVERAGE + u32 iSrcLine; /* Source-code line that generated this opcode + ** with flags in the upper 8 bits */ +#endif +}; +typedef struct VdbeOp VdbeOp; + + +/* +** A sub-routine used to implement a trigger program. +*/ +struct SubProgram { + VdbeOp *aOp; /* Array of opcodes for sub-program */ + int nOp; /* Elements in aOp[] */ + int nMem; /* Number of memory cells required */ + int nCsr; /* Number of cursors required */ + u8 *aOnce; /* Array of OP_Once flags */ + void *token; /* id that may be used to recursive triggers */ + SubProgram *pNext; /* Next sub-program already visited */ +}; + +/* +** A smaller version of VdbeOp used for the VdbeAddOpList() function because +** it takes up less space. +*/ +struct VdbeOpList { + u8 opcode; /* What operation to perform */ + signed char p1; /* First operand */ + signed char p2; /* Second parameter (often the jump destination) */ + signed char p3; /* Third parameter */ +}; +typedef struct VdbeOpList VdbeOpList; + +/* +** Allowed values of VdbeOp.p4type +*/ +#define P4_NOTUSED 0 /* The P4 parameter is not used */ +#define P4_TRANSIENT 0 /* P4 is a pointer to a transient string */ +#define P4_STATIC (-1) /* Pointer to a static string */ +#define P4_COLLSEQ (-2) /* P4 is a pointer to a CollSeq structure */ +#define P4_INT32 (-3) /* P4 is a 32-bit signed integer */ +#define P4_SUBPROGRAM (-4) /* P4 is a pointer to a SubProgram structure */ +#define P4_ADVANCE (-5) /* P4 is a pointer to BtreeNext() or BtreePrev() */ +#define P4_TABLE (-6) /* P4 is a pointer to a Table structure */ +/* Above do not own any resources. Must free those below */ +#define P4_FREE_IF_LE (-7) +#define P4_DYNAMIC (-7) /* Pointer to memory from sqliteMalloc() */ +#define P4_FUNCDEF (-8) /* P4 is a pointer to a FuncDef structure */ +#define P4_KEYINFO (-9) /* P4 is a pointer to a KeyInfo structure */ +#define P4_EXPR (-10) /* P4 is a pointer to an Expr tree */ +#define P4_MEM (-11) /* P4 is a pointer to a Mem* structure */ +#define P4_VTAB (-12) /* P4 is a pointer to an sqlite3_vtab structure */ +#define P4_REAL (-13) /* P4 is a 64-bit floating point value */ +#define P4_INT64 (-14) /* P4 is a 64-bit signed integer */ +#define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ +#define P4_FUNCCTX (-16) /* P4 is a pointer to an sqlite3_context object */ +#define P4_DYNBLOB (-17) /* Pointer to memory from sqliteMalloc() */ + +/* Error message codes for OP_Halt */ +#define P5_ConstraintNotNull 1 +#define P5_ConstraintUnique 2 +#define P5_ConstraintCheck 3 +#define P5_ConstraintFK 4 + +/* +** The Vdbe.aColName array contains 5n Mem structures, where n is the +** number of columns of data returned by the statement. +*/ +#define COLNAME_NAME 0 +#define COLNAME_DECLTYPE 1 +#define COLNAME_DATABASE 2 +#define COLNAME_TABLE 3 +#define COLNAME_COLUMN 4 +#ifdef SQLITE_ENABLE_COLUMN_METADATA +# define COLNAME_N 5 /* Number of COLNAME_xxx symbols */ +#else +# ifdef SQLITE_OMIT_DECLTYPE +# define COLNAME_N 1 /* Store only the name */ +# else +# define COLNAME_N 2 /* Store the name and decltype */ +# endif +#endif + +/* +** The following macro converts a label returned by sqlite3VdbeMakeLabel() +** into an index into the Parse.aLabel[] array that contains the resolved +** address of that label. +*/ +#define ADDR(X) (~(X)) + +/* +** The makefile scans the vdbe.c source file and creates the "opcodes.h" +** header file that defines a number for each opcode used by the VDBE. +*/ +/************** Include opcodes.h in the middle of vdbe.h ********************/ +/************** Begin file opcodes.h *****************************************/ +/* Automatically generated. Do not edit */ +/* See the tool/mkopcodeh.tcl script for details */ +#define OP_Savepoint 0 +#define OP_AutoCommit 1 +#define OP_Transaction 2 +#define OP_SorterNext 3 /* jump */ +#define OP_Prev 4 /* jump */ +#define OP_Next 5 /* jump */ +#define OP_Checkpoint 6 +#define OP_JournalMode 7 +#define OP_Vacuum 8 +#define OP_VFilter 9 /* jump, synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 10 /* synopsis: data=r[P3@P2] */ +#define OP_Goto 11 /* jump */ +#define OP_Gosub 12 /* jump */ +#define OP_InitCoroutine 13 /* jump */ +#define OP_Yield 14 /* jump */ +#define OP_MustBeInt 15 /* jump */ +#define OP_Jump 16 /* jump */ +#define OP_Once 17 /* jump */ +#define OP_If 18 /* jump */ +#define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ +#define OP_IfNot 20 /* jump */ +#define OP_IsNullOrType 21 /* jump, synopsis: if typeof(r[P1]) IN (P3,5) goto P2 */ +#define OP_IfNullRow 22 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ +#define OP_SeekLT 23 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekLE 24 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGE 25 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGT 26 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfNotOpen 27 /* jump, synopsis: if( !csr[P1] ) goto P2 */ +#define OP_IfNoHope 28 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NoConflict 29 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NotFound 30 /* jump, synopsis: key=r[P3@P4] */ +#define OP_Found 31 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekRowid 32 /* jump, synopsis: intkey=r[P3] */ +#define OP_NotExists 33 /* jump, synopsis: intkey=r[P3] */ +#define OP_Last 34 /* jump */ +#define OP_IfSmaller 35 /* jump */ +#define OP_SorterSort 36 /* jump */ +#define OP_Sort 37 /* jump */ +#define OP_Rewind 38 /* jump */ +#define OP_IdxLE 39 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGT 40 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxLT 41 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGE 42 /* jump, synopsis: key=r[P3@P4] */ +#define OP_Or 43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ +#define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ +#define OP_RowSetRead 45 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 46 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 47 /* jump */ +#define OP_FkIfZero 48 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_IfPos 49 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IsNull 50 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ +#define OP_NotNull 51 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ +#define OP_Ne 52 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ +#define OP_Eq 53 /* jump, same as TK_EQ, synopsis: IF r[P3]==r[P1] */ +#define OP_Gt 54 /* jump, same as TK_GT, synopsis: IF r[P3]>r[P1] */ +#define OP_Le 55 /* jump, same as TK_LE, synopsis: IF r[P3]<=r[P1] */ +#define OP_Lt 56 /* jump, same as TK_LT, synopsis: IF r[P3]=r[P1] */ +#define OP_ElseEq 58 /* jump, same as TK_ESCAPE */ +#define OP_IfNotZero 59 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +#define OP_DecrJumpZero 60 /* jump, synopsis: if (--r[P1])==0 goto P2 */ +#define OP_IncrVacuum 61 /* jump */ +#define OP_VNext 62 /* jump */ +#define OP_Init 63 /* jump, synopsis: Start at P2 */ +#define OP_PureFunc 64 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Function 65 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Return 66 +#define OP_EndCoroutine 67 +#define OP_HaltIfNull 68 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 69 +#define OP_Integer 70 /* synopsis: r[P2]=P1 */ +#define OP_Int64 71 /* synopsis: r[P2]=P4 */ +#define OP_String 72 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 73 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 74 /* synopsis: r[P1]=NULL */ +#define OP_Blob 75 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 76 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 77 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 78 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 79 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 80 /* synopsis: r[P2]=r[P1] */ +#define OP_ChngCntRow 81 /* synopsis: output=r[P1] */ +#define OP_ResultRow 82 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 83 +#define OP_AddImm 84 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_RealAffinity 85 +#define OP_Cast 86 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 87 +#define OP_Compare 88 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_IsTrue 89 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ +#define OP_ZeroOrNull 90 /* synopsis: r[P2] = 0 OR NULL */ +#define OP_Offset 91 /* synopsis: r[P3] = sqlite_offset(P1) */ +#define OP_Column 92 /* synopsis: r[P3]=PX */ +#define OP_TypeCheck 93 /* synopsis: typecheck(r[P1@P2]) */ +#define OP_Affinity 94 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 95 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 96 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 97 +#define OP_SetCookie 98 +#define OP_ReopenIdx 99 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 100 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 101 /* synopsis: root=P2 iDb=P3 */ +#define OP_BitAnd 102 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 103 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 104 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ +#define OP_Add 106 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 107 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 108 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 109 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 110 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 111 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +#define OP_OpenDup 112 +#define OP_BitNot 113 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ +#define OP_OpenAutoindex 114 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 115 /* synopsis: nColumn=P2 */ +#define OP_String8 116 /* same as TK_STRING, synopsis: r[P2]='P4' */ +#define OP_SorterOpen 117 +#define OP_SequenceTest 118 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 119 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 120 +#define OP_ColumnsUsed 121 +#define OP_SeekScan 122 /* synopsis: Scan-ahead up to P1 rows */ +#define OP_SeekHit 123 /* synopsis: set P2<=seekHit<=P3 */ +#define OP_Sequence 124 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 125 /* synopsis: r[P2]=rowid */ +#define OP_Insert 126 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_RowCell 127 +#define OP_Delete 128 +#define OP_ResetCount 129 +#define OP_SorterCompare 130 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 131 /* synopsis: r[P2]=data */ +#define OP_RowData 132 /* synopsis: r[P2]=data */ +#define OP_Rowid 133 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 134 +#define OP_SeekEnd 135 +#define OP_IdxInsert 136 /* synopsis: key=r[P2] */ +#define OP_SorterInsert 137 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 138 /* synopsis: key=r[P2@P3] */ +#define OP_DeferredSeek 139 /* synopsis: Move P3 to P1.rowid if needed */ +#define OP_IdxRowid 140 /* synopsis: r[P2]=rowid */ +#define OP_FinishSeek 141 +#define OP_Destroy 142 +#define OP_Clear 143 +#define OP_ResetSorter 144 +#define OP_CreateBtree 145 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ +#define OP_SqlExec 146 +#define OP_ParseSchema 147 +#define OP_LoadAnalysis 148 +#define OP_DropTable 149 +#define OP_DropIndex 150 +#define OP_DropTrigger 151 +#define OP_Real 152 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_IntegrityCk 153 +#define OP_RowSetAdd 154 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 155 +#define OP_FkCounter 156 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 157 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 158 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 159 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 160 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 161 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 162 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 163 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 164 +#define OP_CursorLock 165 +#define OP_CursorUnlock 166 +#define OP_TableLock 167 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 168 +#define OP_VCreate 169 +#define OP_VDestroy 170 +#define OP_VOpen 171 +#define OP_VColumn 172 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 173 +#define OP_Pagecount 174 +#define OP_MaxPgcnt 175 +#define OP_Trace 176 +#define OP_CursorHint 177 +#define OP_ReleaseReg 178 /* synopsis: release r[P1@P2] mask P3 */ +#define OP_Noop 179 +#define OP_Explain 180 +#define OP_Abortable 181 + +/* Properties such as "out2" or "jump" that are specified in +** comments following the "case" for each opcode in the vdbe.c +** are encoded into bitvectors as follows: +*/ +#define OPFLG_JUMP 0x01 /* jump: P2 holds jmp target */ +#define OPFLG_IN1 0x02 /* in1: P1 is an input */ +#define OPFLG_IN2 0x04 /* in2: P2 is an input */ +#define OPFLG_IN3 0x08 /* in3: P3 is an input */ +#define OPFLG_OUT2 0x10 /* out2: P2 is an output */ +#define OPFLG_OUT3 0x20 /* out3: P3 is an output */ +#define OPFLG_INITIALIZER {\ +/* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x00, 0x10,\ +/* 8 */ 0x00, 0x01, 0x00, 0x01, 0x01, 0x01, 0x03, 0x03,\ +/* 16 */ 0x01, 0x01, 0x03, 0x12, 0x03, 0x03, 0x01, 0x09,\ +/* 24 */ 0x09, 0x09, 0x09, 0x01, 0x09, 0x09, 0x09, 0x09,\ +/* 32 */ 0x09, 0x09, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\ +/* 40 */ 0x01, 0x01, 0x01, 0x26, 0x26, 0x23, 0x0b, 0x01,\ +/* 48 */ 0x01, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ +/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x01, 0x01, 0x01,\ +/* 64 */ 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10, 0x10,\ +/* 72 */ 0x10, 0x10, 0x00, 0x10, 0x10, 0x00, 0x00, 0x10,\ +/* 80 */ 0x10, 0x00, 0x00, 0x00, 0x02, 0x02, 0x02, 0x00,\ +/* 88 */ 0x00, 0x12, 0x1e, 0x20, 0x00, 0x00, 0x00, 0x00,\ +/* 96 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x26, 0x26,\ +/* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ +/* 112 */ 0x00, 0x12, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00,\ +/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ +/* 128 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00,\ +/* 136 */ 0x04, 0x04, 0x00, 0x00, 0x10, 0x00, 0x10, 0x00,\ +/* 144 */ 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 152 */ 0x10, 0x00, 0x06, 0x10, 0x00, 0x04, 0x1a, 0x00,\ +/* 160 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x10,\ +/* 176 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,} + +/* The resolve3P2Values() routine is able to run faster if it knows +** the value of the largest JUMP opcode. The smaller the maximum +** JUMP opcode the better, so the mkopcodeh.tcl script that +** generated this include file strives to group all JUMP opcodes +** together near the beginning of the list. +*/ +#define SQLITE_MX_JUMP_OPCODE 63 /* Maximum JUMP opcode */ + +/************** End of opcodes.h *********************************************/ +/************** Continuing where we left off in vdbe.h ***********************/ + +/* +** Additional non-public SQLITE_PREPARE_* flags +*/ +#define SQLITE_PREPARE_SAVESQL 0x80 /* Preserve SQL text */ +#define SQLITE_PREPARE_MASK 0x0f /* Mask of public flags */ + +/* +** Prototypes for the VDBE interface. See comments on the implementation +** for a description of what each of these routines does. +*/ +SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); +SQLITE_PRIVATE Parse *sqlite3VdbeParser(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe*,int,const char*); +SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...); +SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall(Parse*,int,int,int,int,const FuncDef*,int); +SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe*,int); +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) +SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N); +SQLITE_PRIVATE void sqlite3VdbeVerifyNoResultRow(Vdbe *p); +#else +# define sqlite3VdbeVerifyNoMallocRequired(A,B) +# define sqlite3VdbeVerifyNoResultRow(A) +#endif +#if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3VdbeVerifyAbortable(Vdbe *p, int); +#else +# define sqlite3VdbeVerifyAbortable(A,B) +#endif +SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp,int iLineno); +#ifndef SQLITE_OMIT_EXPLAIN +SQLITE_PRIVATE void sqlite3VdbeExplain(Parse*,u8,const char*,...); +SQLITE_PRIVATE void sqlite3VdbeExplainPop(Parse*); +SQLITE_PRIVATE int sqlite3VdbeExplainParent(Parse*); +# define ExplainQueryPlan(P) sqlite3VdbeExplain P +# define ExplainQueryPlanPop(P) sqlite3VdbeExplainPop(P) +# define ExplainQueryPlanParent(P) sqlite3VdbeExplainParent(P) +#else +# define ExplainQueryPlan(P) +# define ExplainQueryPlanPop(P) +# define ExplainQueryPlanParent(P) 0 +# define sqlite3ExplainBreakpoint(A,B) /*no-op*/ +#endif +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN) +SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char*,const char*); +#else +# define sqlite3ExplainBreakpoint(A,B) /*no-op*/ +#endif +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*, int, char*, u16); +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, int addr, u8); +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u16 P5); +SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); +SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe*, int addr); +SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); +SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbeReleaseRegisters(Parse*,int addr, int n, u32 mask, int); +#else +# define sqlite3VdbeReleaseRegisters(P,A,N,M,F) +#endif +SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); +SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type); +SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); +SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); +SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse*); +SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3*,Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); +SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); +SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *, int); +#endif +SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); +SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe*); +SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe*); +SQLITE_PRIVATE u8 sqlite3VdbePrepareFlags(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8); +#ifdef SQLITE_ENABLE_NORMALIZE +SQLITE_PRIVATE void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*); +SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*); +#endif +SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe*,Vdbe*); +SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); +SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8); +SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int); +#ifndef SQLITE_OMIT_TRACE +SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); +#endif +SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); +SQLITE_PRIVATE int sqlite3BlobCompare(const Mem*, const Mem*); + +SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); +SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int); +SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo*); + +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); +SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); + +SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); +SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe*); + +SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context*); +#ifdef SQLITE_ENABLE_BYTECODE_VTAB +SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3*); +#endif + +/* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on +** each VDBE opcode. +** +** Use the SQLITE_ENABLE_MODULE_COMMENTS macro to see some extra no-op +** comments in VDBE programs that show key decision points in the code +** generator. +*/ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe*, const char*, ...); +# define VdbeComment(X) sqlite3VdbeComment X +SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); +# define VdbeNoopComment(X) sqlite3VdbeNoopComment X +# ifdef SQLITE_ENABLE_MODULE_COMMENTS +# define VdbeModuleComment(X) sqlite3VdbeNoopComment X +# else +# define VdbeModuleComment(X) +# endif +#else +# define VdbeComment(X) +# define VdbeNoopComment(X) +# define VdbeModuleComment(X) +#endif + +/* +** The VdbeCoverage macros are used to set a coverage testing point +** for VDBE branch instructions. The coverage testing points are line +** numbers in the sqlite3.c source file. VDBE branch coverage testing +** only works with an amalagmation build. That's ok since a VDBE branch +** coverage build designed for testing the test suite only. No application +** should ever ship with VDBE branch coverage measuring turned on. +** +** VdbeCoverage(v) // Mark the previously coded instruction +** // as a branch +** +** VdbeCoverageIf(v, conditional) // Mark previous if conditional true +** +** VdbeCoverageAlwaysTaken(v) // Previous branch is always taken +** +** VdbeCoverageNeverTaken(v) // Previous branch is never taken +** +** VdbeCoverageNeverNull(v) // Previous three-way branch is only +** // taken on the first two ways. The +** // NULL option is not possible +** +** VdbeCoverageEqNe(v) // Previous OP_Jump is only interested +** // in distingishing equal and not-equal. +** +** Every VDBE branch operation must be tagged with one of the macros above. +** If not, then when "make test" is run with -DSQLITE_VDBE_COVERAGE and +** -DSQLITE_DEBUG then an ALWAYS() will fail in the vdbeTakeBranch() +** routine in vdbe.c, alerting the developer to the missed tag. +** +** During testing, the test application will invoke +** sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE,...) to set a callback +** routine that is invoked as each bytecode branch is taken. The callback +** contains the sqlite3.c source line number ov the VdbeCoverage macro and +** flags to indicate whether or not the branch was taken. The test application +** is responsible for keeping track of this and reporting byte-code branches +** that are never taken. +** +** See the VdbeBranchTaken() macro and vdbeTakeBranch() function in the +** vdbe.c source file for additional information. +*/ +#ifdef SQLITE_VDBE_COVERAGE +SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); +# define VdbeCoverage(v) sqlite3VdbeSetLineNumber(v,__LINE__) +# define VdbeCoverageIf(v,x) if(x)sqlite3VdbeSetLineNumber(v,__LINE__) +# define VdbeCoverageAlwaysTaken(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x5000000); +# define VdbeCoverageNeverTaken(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x6000000); +# define VdbeCoverageNeverNull(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x4000000); +# define VdbeCoverageNeverNullIf(v,x) \ + if(x)sqlite3VdbeSetLineNumber(v,__LINE__|0x4000000); +# define VdbeCoverageEqNe(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x8000000); +# define VDBE_OFFSET_LINENO(x) (__LINE__+x) +#else +# define VdbeCoverage(v) +# define VdbeCoverageIf(v,x) +# define VdbeCoverageAlwaysTaken(v) +# define VdbeCoverageNeverTaken(v) +# define VdbeCoverageNeverNull(v) +# define VdbeCoverageNeverNullIf(v,x) +# define VdbeCoverageEqNe(v) +# define VDBE_OFFSET_LINENO(x) 0 +#endif + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); +#else +# define sqlite3VdbeScanStatus(a,b,c,d,e) +#endif + +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, VdbeOp*); +#endif + +#endif /* SQLITE_VDBE_H */ + +/************** End of vdbe.h ************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include pcache.h in the middle of sqliteInt.h ****************/ +/************** Begin file pcache.h ******************************************/ +/* +** 2008 August 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the sqlite page cache +** subsystem. +*/ + +#ifndef _PCACHE_H_ + +typedef struct PgHdr PgHdr; +typedef struct PCache PCache; + +/* +** Every page in the cache is controlled by an instance of the following +** structure. +*/ +struct PgHdr { + sqlite3_pcache_page *pPage; /* Pcache object page handle */ + void *pData; /* Page data */ + void *pExtra; /* Extra content */ + PCache *pCache; /* PRIVATE: Cache that owns this page */ + PgHdr *pDirty; /* Transient list of dirty sorted by pgno */ + Pager *pPager; /* The pager this page is part of */ + Pgno pgno; /* Page number for this page */ +#ifdef SQLITE_CHECK_PAGES + u32 pageHash; /* Hash of page content */ +#endif + u16 flags; /* PGHDR flags defined below */ + + /********************************************************************** + ** Elements above, except pCache, are public. All that follow are + ** private to pcache.c and should not be accessed by other modules. + ** pCache is grouped with the public elements for efficiency. + */ + i16 nRef; /* Number of users of this page */ + PgHdr *pDirtyNext; /* Next element in list of dirty pages */ + PgHdr *pDirtyPrev; /* Previous element in list of dirty pages */ + /* NB: pDirtyNext and pDirtyPrev are undefined if the + ** PgHdr object is not dirty */ +}; + +/* Bit values for PgHdr.flags */ +#define PGHDR_CLEAN 0x001 /* Page not on the PCache.pDirty list */ +#define PGHDR_DIRTY 0x002 /* Page is on the PCache.pDirty list */ +#define PGHDR_WRITEABLE 0x004 /* Journaled and ready to modify */ +#define PGHDR_NEED_SYNC 0x008 /* Fsync the rollback journal before + ** writing this page to the database */ +#define PGHDR_DONT_WRITE 0x010 /* Do not write content to disk */ +#define PGHDR_MMAP 0x020 /* This is an mmap page object */ + +#define PGHDR_WAL_APPEND 0x040 /* Appended to wal file */ + +/* Initialize and shutdown the page cache subsystem */ +SQLITE_PRIVATE int sqlite3PcacheInitialize(void); +SQLITE_PRIVATE void sqlite3PcacheShutdown(void); + +/* Page cache buffer management: +** These routines implement SQLITE_CONFIG_PAGECACHE. +*/ +SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *, int sz, int n); + +/* Create a new pager cache. +** Under memory stress, invoke xStress to try to make pages clean. +** Only clean and unpinned pages can be reclaimed. +*/ +SQLITE_PRIVATE int sqlite3PcacheOpen( + int szPage, /* Size of every page */ + int szExtra, /* Extra space associated with each page */ + int bPurgeable, /* True if pages are on backing store */ + int (*xStress)(void*, PgHdr*), /* Call to try to make pages clean */ + void *pStress, /* Argument to xStress */ + PCache *pToInit /* Preallocated space for the PCache */ +); + +/* Modify the page-size after the cache has been created. */ +SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *, int); + +/* Return the size in bytes of a PCache object. Used to preallocate +** storage space. +*/ +SQLITE_PRIVATE int sqlite3PcacheSize(void); + +/* One release per successful fetch. Page is pinned until released. +** Reference counted. +*/ +SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch(PCache*, Pgno, int createFlag); +SQLITE_PRIVATE int sqlite3PcacheFetchStress(PCache*, Pgno, sqlite3_pcache_page**); +SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish(PCache*, Pgno, sqlite3_pcache_page *pPage); +SQLITE_PRIVATE void sqlite3PcacheRelease(PgHdr*); + +SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ +SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr*); /* Make sure page is marked dirty */ +SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr*); /* Mark a single page as clean */ +SQLITE_PRIVATE void sqlite3PcacheCleanAll(PCache*); /* Mark all dirty list pages as clean */ +SQLITE_PRIVATE void sqlite3PcacheClearWritable(PCache*); + +/* Change a page number. Used by incr-vacuum. */ +SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr*, Pgno); + +/* Remove all pages with pgno>x. Reset the cache if x==0 */ +SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache*, Pgno x); + +/* Get a list of all dirty pages in the cache, sorted by page number */ +SQLITE_PRIVATE PgHdr *sqlite3PcacheDirtyList(PCache*); + +/* Reset and close the cache object */ +SQLITE_PRIVATE void sqlite3PcacheClose(PCache*); + +/* Clear flags from pages of the page cache */ +SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *); + +/* Discard the contents of the cache */ +SQLITE_PRIVATE void sqlite3PcacheClear(PCache*); + +/* Return the total number of outstanding page references */ +SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache*); + +/* Increment the reference count of an existing page */ +SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr*); + +SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr*); + +/* Return the total number of pages stored in the cache */ +SQLITE_PRIVATE int sqlite3PcachePagecount(PCache*); + +#if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) +/* Iterate through all dirty pages currently stored in the cache. This +** interface is only available if SQLITE_CHECK_PAGES is defined when the +** library is built. +*/ +SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)); +#endif + +#if defined(SQLITE_DEBUG) +/* Check invariants on a PgHdr object */ +SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr*); +#endif + +/* Set and get the suggested cache-size for the specified pager-cache. +** +** If no global maximum is configured, then the system attempts to limit +** the total number of pages cached by purgeable pager-caches to the sum +** of the suggested cache-sizes. +*/ +SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *, int); +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *); +#endif + +/* Set or get the suggested spill-size for the specified pager-cache. +** +** The spill-size is the minimum number of pages in cache before the cache +** will attempt to spill dirty pages by calling xStress. +*/ +SQLITE_PRIVATE int sqlite3PcacheSetSpillsize(PCache *, int); + +/* Free up as much memory as possible from the page cache */ +SQLITE_PRIVATE void sqlite3PcacheShrink(PCache*); + +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* Try to return memory used by the pcache module to the main memory heap */ +SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int); +#endif + +#ifdef SQLITE_TEST +SQLITE_PRIVATE void sqlite3PcacheStats(int*,int*,int*,int*); +#endif + +SQLITE_PRIVATE void sqlite3PCacheSetDefault(void); + +/* Return the header size */ +SQLITE_PRIVATE int sqlite3HeaderSizePcache(void); +SQLITE_PRIVATE int sqlite3HeaderSizePcache1(void); + +/* Number of dirty pages as a percentage of the configured cache size */ +SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache*); + +#ifdef SQLITE_DIRECT_OVERFLOW_READ +SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache); +#endif + +#endif /* _PCACHE_H_ */ + +/************** End of pcache.h **********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include os.h in the middle of sqliteInt.h ********************/ +/************** Begin file os.h **********************************************/ +/* +** 2001 September 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file (together with is companion C source-code file +** "os.c") attempt to abstract the underlying operating system so that +** the SQLite library will work on both POSIX and windows systems. +** +** This header file is #include-ed by sqliteInt.h and thus ends up +** being included by every source file. +*/ +#ifndef _SQLITE_OS_H_ +#define _SQLITE_OS_H_ + +/* +** Attempt to automatically detect the operating system and setup the +** necessary pre-processor macros for it. +*/ +/************** Include os_setup.h in the middle of os.h *********************/ +/************** Begin file os_setup.h ****************************************/ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains pre-processor directives related to operating system +** detection and/or setup. +*/ +#ifndef SQLITE_OS_SETUP_H +#define SQLITE_OS_SETUP_H + +/* +** Figure out if we are dealing with Unix, Windows, or some other operating +** system. +** +** After the following block of preprocess macros, all of SQLITE_OS_UNIX, +** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of +** the three will be 1. The other two will be 0. +*/ +#if defined(SQLITE_OS_OTHER) +# if SQLITE_OS_OTHER==1 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# else +# undef SQLITE_OS_OTHER +# endif +#endif +#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) +# define SQLITE_OS_OTHER 0 +# ifndef SQLITE_OS_WIN +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# endif +# else +# define SQLITE_OS_UNIX 0 +# endif +#else +# ifndef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# endif +#endif + +#endif /* SQLITE_OS_SETUP_H */ + +/************** End of os_setup.h ********************************************/ +/************** Continuing where we left off in os.h *************************/ + +/* If the SET_FULLSYNC macro is not defined above, then make it +** a no-op +*/ +#ifndef SET_FULLSYNC +# define SET_FULLSYNC(x,y) +#endif + +/* Maximum pathname length. Note: FILENAME_MAX defined by stdio.h +*/ +#ifndef SQLITE_MAX_PATHLEN +# define SQLITE_MAX_PATHLEN FILENAME_MAX +#endif + +/* +** The default size of a disk sector +*/ +#ifndef SQLITE_DEFAULT_SECTOR_SIZE +# define SQLITE_DEFAULT_SECTOR_SIZE 4096 +#endif + +/* +** Temporary files are named starting with this prefix followed by 16 random +** alphanumeric characters, and no file extension. They are stored in the +** OS's standard temporary file directory, and are deleted prior to exit. +** If sqlite is being embedded in another program, you may wish to change the +** prefix to reflect your program's name, so that if your program exits +** prematurely, old temporary files can be easily identified. This can be done +** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. +** +** 2006-10-31: The default prefix used to be "sqlite_". But then +** Mcafee started using SQLite in their anti-virus product and it +** started putting files with the "sqlite" name in the c:/temp folder. +** This annoyed many windows users. Those users would then do a +** Google search for "sqlite", find the telephone numbers of the +** developers and call to wake them up at night and complain. +** For this reason, the default name prefix is changed to be "sqlite" +** spelled backwards. So the temp files are still identified, but +** anybody smart enough to figure out the code is also likely smart +** enough to know that calling the developer will not help get rid +** of the file. +*/ +#ifndef SQLITE_TEMP_FILE_PREFIX +# define SQLITE_TEMP_FILE_PREFIX "etilqs_" +#endif + +/* +** The following values may be passed as the second argument to +** sqlite3OsLock(). The various locks exhibit the following semantics: +** +** SHARED: Any number of processes may hold a SHARED lock simultaneously. +** RESERVED: A single process may hold a RESERVED lock on a file at +** any time. Other processes may hold and obtain new SHARED locks. +** PENDING: A single process may hold a PENDING lock on a file at +** any one time. Existing SHARED locks may persist, but no new +** SHARED locks may be obtained by other processes. +** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks. +** +** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a +** process that requests an EXCLUSIVE lock may actually obtain a PENDING +** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to +** sqlite3OsLock(). +*/ +#define NO_LOCK 0 +#define SHARED_LOCK 1 +#define RESERVED_LOCK 2 +#define PENDING_LOCK 3 +#define EXCLUSIVE_LOCK 4 + +/* +** File Locking Notes: (Mostly about windows but also some info for Unix) +** +** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because +** those functions are not available. So we use only LockFile() and +** UnlockFile(). +** +** LockFile() prevents not just writing but also reading by other processes. +** A SHARED_LOCK is obtained by locking a single randomly-chosen +** byte out of a specific range of bytes. The lock byte is obtained at +** random so two separate readers can probably access the file at the +** same time, unless they are unlucky and choose the same lock byte. +** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range. +** There can only be one writer. A RESERVED_LOCK is obtained by locking +** a single byte of the file that is designated as the reserved lock byte. +** A PENDING_LOCK is obtained by locking a designated byte different from +** the RESERVED_LOCK byte. +** +** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available, +** which means we can use reader/writer locks. When reader/writer locks +** are used, the lock is placed on the same range of bytes that is used +** for probabilistic locking in Win95/98/ME. Hence, the locking scheme +** will support two or more Win95 readers or two or more WinNT readers. +** But a single Win95 reader will lock out all WinNT readers and a single +** WinNT reader will lock out all other Win95 readers. +** +** The following #defines specify the range of bytes used for locking. +** SHARED_SIZE is the number of bytes available in the pool from which +** a random byte is selected for a shared lock. The pool of bytes for +** shared locks begins at SHARED_FIRST. +** +** The same locking strategy and +** byte ranges are used for Unix. This leaves open the possibility of having +** clients on win95, winNT, and unix all talking to the same shared file +** and all locking correctly. To do so would require that samba (or whatever +** tool is being used for file sharing) implements locks correctly between +** windows and unix. I'm guessing that isn't likely to happen, but by +** using the same locking range we are at least open to the possibility. +** +** Locking in windows is manditory. For this reason, we cannot store +** actual data in the bytes used for locking. The pager never allocates +** the pages involved in locking therefore. SHARED_SIZE is selected so +** that all locks will fit on a single page even at the minimum page size. +** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE +** is set high so that we don't have to allocate an unused page except +** for very large databases. But one should test the page skipping logic +** by setting PENDING_BYTE low and running the entire regression suite. +** +** Changing the value of PENDING_BYTE results in a subtly incompatible +** file format. Depending on how it is changed, you might not notice +** the incompatibility right away, even running a full regression test. +** The default location of PENDING_BYTE is the first byte past the +** 1GB boundary. +** +*/ +#ifdef SQLITE_OMIT_WSD +# define PENDING_BYTE (0x40000000) +#else +# define PENDING_BYTE sqlite3PendingByte +#endif +#define RESERVED_BYTE (PENDING_BYTE+1) +#define SHARED_FIRST (PENDING_BYTE+2) +#define SHARED_SIZE 510 + +/* +** Wrapper around OS specific sqlite3_os_init() function. +*/ +SQLITE_PRIVATE int sqlite3OsInit(void); + +/* +** Functions for accessing sqlite3_file methods +*/ +SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*); +SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); +SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); +SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); +SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*); +#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 +SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); +SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); +SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); +#endif /* SQLITE_OMIT_WAL */ +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); + + +/* +** Functions for accessing sqlite3_vfs methods +*/ +SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); +SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int); +SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); +#ifndef SQLITE_OMIT_LOAD_EXTENSION +SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); +SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); +SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); +SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); +SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); + +/* +** Convenience functions for opening and closing files using +** sqlite3_malloc() to obtain space for the file-handle structure. +*/ +SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); +SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); + +#endif /* _SQLITE_OS_H_ */ + +/************** End of os.h **************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include mutex.h in the middle of sqliteInt.h *****************/ +/************** Begin file mutex.h *******************************************/ +/* +** 2007 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the common header for all mutex implementations. +** The sqliteInt.h header #includes this file so that it is available +** to all source files. We break it out in an effort to keep the code +** better organized. +** +** NOTE: source files should *not* #include this header file directly. +** Source files should #include the sqliteInt.h file and let that file +** include this one indirectly. +*/ + + +/* +** Figure out what version of the code to use. The choices are +** +** SQLITE_MUTEX_OMIT No mutex logic. Not even stubs. The +** mutexes implementation cannot be overridden +** at start-time. +** +** SQLITE_MUTEX_NOOP For single-threaded applications. No +** mutual exclusion is provided. But this +** implementation can be overridden at +** start-time. +** +** SQLITE_MUTEX_PTHREADS For multi-threaded applications on Unix. +** +** SQLITE_MUTEX_W32 For multi-threaded applications on Win32. +*/ +#if !SQLITE_THREADSAFE +# define SQLITE_MUTEX_OMIT +#endif +#if SQLITE_THREADSAFE && !defined(SQLITE_MUTEX_NOOP) +# if SQLITE_OS_UNIX +# define SQLITE_MUTEX_PTHREADS +# elif SQLITE_OS_WIN +# define SQLITE_MUTEX_W32 +# else +# define SQLITE_MUTEX_NOOP +# endif +#endif + +#ifdef SQLITE_MUTEX_OMIT +/* +** If this is a no-op implementation, implement everything as macros. +*/ +#define sqlite3_mutex_alloc(X) ((sqlite3_mutex*)8) +#define sqlite3_mutex_free(X) +#define sqlite3_mutex_enter(X) +#define sqlite3_mutex_try(X) SQLITE_OK +#define sqlite3_mutex_leave(X) +#define sqlite3_mutex_held(X) ((void)(X),1) +#define sqlite3_mutex_notheld(X) ((void)(X),1) +#define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) +#define sqlite3MutexInit() SQLITE_OK +#define sqlite3MutexEnd() +#define MUTEX_LOGIC(X) +#else +#define MUTEX_LOGIC(X) X +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); +#endif /* defined(SQLITE_MUTEX_OMIT) */ + +/************** End of mutex.h ***********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ + +/* The SQLITE_EXTRA_DURABLE compile-time option used to set the default +** synchronous setting to EXTRA. It is no longer supported. +*/ +#ifdef SQLITE_EXTRA_DURABLE +# warning Use SQLITE_DEFAULT_SYNCHRONOUS=3 instead of SQLITE_EXTRA_DURABLE +# define SQLITE_DEFAULT_SYNCHRONOUS 3 +#endif + +/* +** Default synchronous levels. +** +** Note that (for historcal reasons) the PAGER_SYNCHRONOUS_* macros differ +** from the SQLITE_DEFAULT_SYNCHRONOUS value by 1. +** +** PAGER_SYNCHRONOUS DEFAULT_SYNCHRONOUS +** OFF 1 0 +** NORMAL 2 1 +** FULL 3 2 +** EXTRA 4 3 +** +** The "PRAGMA synchronous" statement also uses the zero-based numbers. +** In other words, the zero-based numbers are used for all external interfaces +** and the one-based values are used internally. +*/ +#ifndef SQLITE_DEFAULT_SYNCHRONOUS +# define SQLITE_DEFAULT_SYNCHRONOUS 2 +#endif +#ifndef SQLITE_DEFAULT_WAL_SYNCHRONOUS +# define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS +#endif + +/* +** Each database file to be accessed by the system is an instance +** of the following structure. There are normally two of these structures +** in the sqlite.aDb[] array. aDb[0] is the main database file and +** aDb[1] is the database file used to hold temporary tables. Additional +** databases may be attached. +*/ +struct Db { + char *zDbSName; /* Name of this database. (schema name, not filename) */ + Btree *pBt; /* The B*Tree structure for this database file */ + u8 safety_level; /* How aggressive at syncing data to disk */ + u8 bSyncSet; /* True if "PRAGMA synchronous=N" has been run */ + Schema *pSchema; /* Pointer to database schema (possibly shared) */ +}; + +/* +** An instance of the following structure stores a database schema. +** +** Most Schema objects are associated with a Btree. The exception is +** the Schema for the TEMP databaes (sqlite3.aDb[1]) which is free-standing. +** In shared cache mode, a single Schema object can be shared by multiple +** Btrees that refer to the same underlying BtShared object. +** +** Schema objects are automatically deallocated when the last Btree that +** references them is destroyed. The TEMP Schema is manually freed by +** sqlite3_close(). +* +** A thread must be holding a mutex on the corresponding Btree in order +** to access Schema content. This implies that the thread must also be +** holding a mutex on the sqlite3 connection pointer that owns the Btree. +** For a TEMP Schema, only the connection mutex is required. +*/ +struct Schema { + int schema_cookie; /* Database schema version number for this file */ + int iGeneration; /* Generation counter. Incremented with each change */ + Hash tblHash; /* All tables indexed by name */ + Hash idxHash; /* All (named) indices indexed by name */ + Hash trigHash; /* All triggers indexed by name */ + Hash fkeyHash; /* All foreign keys by referenced table name */ + Table *pSeqTab; /* The sqlite_sequence table used by AUTOINCREMENT */ + u8 file_format; /* Schema format version for this file */ + u8 enc; /* Text encoding used by this database */ + u16 schemaFlags; /* Flags associated with this schema */ + int cache_size; /* Number of pages to use in the cache */ +}; + +/* +** These macros can be used to test, set, or clear bits in the +** Db.pSchema->flags field. +*/ +#define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))==(P)) +#define DbHasAnyProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))!=0) +#define DbSetProperty(D,I,P) (D)->aDb[I].pSchema->schemaFlags|=(P) +#define DbClearProperty(D,I,P) (D)->aDb[I].pSchema->schemaFlags&=~(P) + +/* +** Allowed values for the DB.pSchema->flags field. +** +** The DB_SchemaLoaded flag is set after the database schema has been +** read into internal hash tables. +** +** DB_UnresetViews means that one or more views have column names that +** have been filled out. If the schema changes, these column names might +** changes and so the view will need to be reset. +*/ +#define DB_SchemaLoaded 0x0001 /* The schema has been loaded */ +#define DB_UnresetViews 0x0002 /* Some views have defined column names */ +#define DB_ResetWanted 0x0008 /* Reset the schema when nSchemaLock==0 */ + +/* +** The number of different kinds of things that can be limited +** using the sqlite3_limit() interface. +*/ +#define SQLITE_N_LIMIT (SQLITE_LIMIT_WORKER_THREADS+1) + +/* +** Lookaside malloc is a set of fixed-size buffers that can be used +** to satisfy small transient memory allocation requests for objects +** associated with a particular database connection. The use of +** lookaside malloc provides a significant performance enhancement +** (approx 10%) by avoiding numerous malloc/free requests while parsing +** SQL statements. +** +** The Lookaside structure holds configuration information about the +** lookaside malloc subsystem. Each available memory allocation in +** the lookaside subsystem is stored on a linked list of LookasideSlot +** objects. +** +** Lookaside allocations are only allowed for objects that are associated +** with a particular database connection. Hence, schema information cannot +** be stored in lookaside because in shared cache mode the schema information +** is shared by multiple database connections. Therefore, while parsing +** schema information, the Lookaside.bEnabled flag is cleared so that +** lookaside allocations are not used to construct the schema objects. +** +** New lookaside allocations are only allowed if bDisable==0. When +** bDisable is greater than zero, sz is set to zero which effectively +** disables lookaside without adding a new test for the bDisable flag +** in a performance-critical path. sz should be set by to szTrue whenever +** bDisable changes back to zero. +** +** Lookaside buffers are initially held on the pInit list. As they are +** used and freed, they are added back to the pFree list. New allocations +** come off of pFree first, then pInit as a fallback. This dual-list +** allows use to compute a high-water mark - the maximum number of allocations +** outstanding at any point in the past - by subtracting the number of +** allocations on the pInit list from the total number of allocations. +** +** Enhancement on 2019-12-12: Two-size-lookaside +** The default lookaside configuration is 100 slots of 1200 bytes each. +** The larger slot sizes are important for performance, but they waste +** a lot of space, as most lookaside allocations are less than 128 bytes. +** The two-size-lookaside enhancement breaks up the lookaside allocation +** into two pools: One of 128-byte slots and the other of the default size +** (1200-byte) slots. Allocations are filled from the small-pool first, +** failing over to the full-size pool if that does not work. Thus more +** lookaside slots are available while also using less memory. +** This enhancement can be omitted by compiling with +** SQLITE_OMIT_TWOSIZE_LOOKASIDE. +*/ +struct Lookaside { + u32 bDisable; /* Only operate the lookaside when zero */ + u16 sz; /* Size of each buffer in bytes */ + u16 szTrue; /* True value of sz, even if disabled */ + u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */ + u32 nSlot; /* Number of lookaside slots allocated */ + u32 anStat[3]; /* 0: hits. 1: size misses. 2: full misses */ + LookasideSlot *pInit; /* List of buffers not previously used */ + LookasideSlot *pFree; /* List of available buffers */ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + LookasideSlot *pSmallInit; /* List of small buffers not prediously used */ + LookasideSlot *pSmallFree; /* List of available small buffers */ + void *pMiddle; /* First byte past end of full-size buffers and + ** the first byte of LOOKASIDE_SMALL buffers */ +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + void *pStart; /* First byte of available memory space */ + void *pEnd; /* First byte past end of available space */ +}; +struct LookasideSlot { + LookasideSlot *pNext; /* Next buffer in the list of free buffers */ +}; + +#define DisableLookaside db->lookaside.bDisable++;db->lookaside.sz=0 +#define EnableLookaside db->lookaside.bDisable--;\ + db->lookaside.sz=db->lookaside.bDisable?0:db->lookaside.szTrue + +/* Size of the smaller allocations in two-size lookside */ +#ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE +# define LOOKASIDE_SMALL 0 +#else +# define LOOKASIDE_SMALL 128 +#endif + +/* +** A hash table for built-in function definitions. (Application-defined +** functions use a regular table table from hash.h.) +** +** Hash each FuncDef structure into one of the FuncDefHash.a[] slots. +** Collisions are on the FuncDef.u.pHash chain. Use the SQLITE_FUNC_HASH() +** macro to compute a hash on the function name. +*/ +#define SQLITE_FUNC_HASH_SZ 23 +struct FuncDefHash { + FuncDef *a[SQLITE_FUNC_HASH_SZ]; /* Hash table for functions */ +}; +#define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ) + +#ifdef SQLITE_USER_AUTHENTICATION +/* +** Information held in the "sqlite3" database connection object and used +** to manage user authentication. +*/ +typedef struct sqlite3_userauth sqlite3_userauth; +struct sqlite3_userauth { + u8 authLevel; /* Current authentication level */ + int nAuthPW; /* Size of the zAuthPW in bytes */ + char *zAuthPW; /* Password used to authenticate */ + char *zAuthUser; /* User name used to authenticate */ +}; + +/* Allowed values for sqlite3_userauth.authLevel */ +#define UAUTH_Unknown 0 /* Authentication not yet checked */ +#define UAUTH_Fail 1 /* User authentication failed */ +#define UAUTH_User 2 /* Authenticated as a normal user */ +#define UAUTH_Admin 3 /* Authenticated as an administrator */ + +/* Functions used only by user authorization logic */ +SQLITE_PRIVATE int sqlite3UserAuthTable(const char*); +SQLITE_PRIVATE int sqlite3UserAuthCheckLogin(sqlite3*,const char*,u8*); +SQLITE_PRIVATE void sqlite3UserAuthInit(sqlite3*); +SQLITE_PRIVATE void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); + +#endif /* SQLITE_USER_AUTHENTICATION */ + +/* +** typedef for the authorization callback function. +*/ +#ifdef SQLITE_USER_AUTHENTICATION + typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, + const char*, const char*); +#else + typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, + const char*); +#endif + +#ifndef SQLITE_OMIT_DEPRECATED +/* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing +** in the style of sqlite3_trace() +*/ +#define SQLITE_TRACE_LEGACY 0x40 /* Use the legacy xTrace */ +#define SQLITE_TRACE_XPROFILE 0x80 /* Use the legacy xProfile */ +#else +#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_XPROFILE 0 +#endif /* SQLITE_OMIT_DEPRECATED */ +#define SQLITE_TRACE_NONLEGACY_MASK 0x0f /* Normal flags */ + +/* +** Maximum number of sqlite3.aDb[] entries. This is the number of attached +** databases plus 2 for "main" and "temp". +*/ +#define SQLITE_MAX_DB (SQLITE_MAX_ATTACHED+2) + +/* +** Each database connection is an instance of the following structure. +*/ +struct sqlite3 { + sqlite3_vfs *pVfs; /* OS Interface */ + struct Vdbe *pVdbe; /* List of active virtual machines */ + CollSeq *pDfltColl; /* BINARY collseq for the database encoding */ + sqlite3_mutex *mutex; /* Connection mutex */ + Db *aDb; /* All backends */ + int nDb; /* Number of backends currently in use */ + u32 mDbFlags; /* flags recording internal state */ + u64 flags; /* flags settable by pragmas. See below */ + i64 lastRowid; /* ROWID of most recent insert (see above) */ + i64 szMmap; /* Default mmap_size setting */ + u32 nSchemaLock; /* Do not reset the schema when non-zero */ + unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ + int errCode; /* Most recent error code (SQLITE_*) */ + int errMask; /* & result codes with this before returning */ + int iSysErrno; /* Errno value from last system error */ + u32 dbOptFlags; /* Flags to enable/disable optimizations */ + u8 enc; /* Text encoding */ + u8 autoCommit; /* The auto-commit flag. */ + u8 temp_store; /* 1: file 2: memory 0: default */ + u8 mallocFailed; /* True if we have seen a malloc failure */ + u8 bBenignMalloc; /* Do not require OOMs if true */ + u8 dfltLockMode; /* Default locking-mode for attached dbs */ + signed char nextAutovac; /* Autovac setting after VACUUM if >=0 */ + u8 suppressErr; /* Do not issue error messages if true */ + u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ + u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ + u8 mTrace; /* zero or more SQLITE_TRACE flags */ + u8 noSharedCache; /* True if no shared-cache backends */ + u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ + u8 eOpenState; /* Current condition of the connection */ + int nextPagesize; /* Pagesize after VACUUM if >0 */ + i64 nChange; /* Value returned by sqlite3_changes() */ + i64 nTotalChange; /* Value returned by sqlite3_total_changes() */ + int aLimit[SQLITE_N_LIMIT]; /* Limits */ + int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ + struct sqlite3InitInfo { /* Information used during initialization */ + Pgno newTnum; /* Rootpage of table being initialized */ + u8 iDb; /* Which db file is being initialized */ + u8 busy; /* TRUE if currently initializing */ + unsigned orphanTrigger : 1; /* Last statement is orphaned TEMP trigger */ + unsigned imposterTable : 1; /* Building an imposter table */ + unsigned reopenMemdb : 1; /* ATTACH is really a reopen using MemDB */ + const char **azInit; /* "type", "name", and "tbl_name" columns */ + } init; + int nVdbeActive; /* Number of VDBEs currently running */ + int nVdbeRead; /* Number of active VDBEs that read or write */ + int nVdbeWrite; /* Number of active VDBEs that read and write */ + int nVdbeExec; /* Number of nested calls to VdbeExec() */ + int nVDestroy; /* Number of active OP_VDestroy operations */ + int nExtension; /* Number of loaded extensions */ + void **aExtension; /* Array of shared library handles */ + union { + void (*xLegacy)(void*,const char*); /* Legacy trace function */ + int (*xV2)(u32,void*,void*,void*); /* V2 Trace function */ + } trace; + void *pTraceArg; /* Argument to the trace function */ +#ifndef SQLITE_OMIT_DEPRECATED + void (*xProfile)(void*,const char*,u64); /* Profiling function */ + void *pProfileArg; /* Argument to profile function */ +#endif + void *pCommitArg; /* Argument to xCommitCallback() */ + int (*xCommitCallback)(void*); /* Invoked at every commit. */ + void *pRollbackArg; /* Argument to xRollbackCallback() */ + void (*xRollbackCallback)(void*); /* Invoked at every commit. */ + void *pUpdateArg; + void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); + Parse *pParse; /* Current parse */ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + void *pPreUpdateArg; /* First argument to xPreUpdateCallback */ + void (*xPreUpdateCallback)( /* Registered using sqlite3_preupdate_hook() */ + void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64 + ); + PreUpdate *pPreUpdate; /* Context for active pre-update callback */ +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ +#ifndef SQLITE_OMIT_WAL + int (*xWalCallback)(void *, sqlite3 *, const char *, int); + void *pWalArg; +#endif + void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*); + void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*); + void *pCollNeededArg; + sqlite3_value *pErr; /* Most recent error message */ + union { + volatile int isInterrupted; /* True if sqlite3_interrupt has been called */ + double notUsed1; /* Spacer */ + } u1; + Lookaside lookaside; /* Lookaside malloc configuration */ +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth; /* Access authorization function */ + void *pAuthArg; /* 1st argument to the access auth function */ +#endif +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + int (*xProgress)(void *); /* The progress callback */ + void *pProgressArg; /* Argument to the progress callback */ + unsigned nProgressOps; /* Number of opcodes for progress callback */ +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + int nVTrans; /* Allocated size of aVTrans */ + Hash aModule; /* populated by sqlite3_create_module() */ + VtabCtx *pVtabCtx; /* Context for active vtab connect/create */ + VTable **aVTrans; /* Virtual tables with open transactions */ + VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ +#endif + Hash aFunc; /* Hash table of connection functions */ + Hash aCollSeq; /* All collating sequences */ + BusyHandler busyHandler; /* Busy callback */ + Db aDbStatic[2]; /* Static space for the 2 default backends */ + Savepoint *pSavepoint; /* List of active savepoints */ + int nAnalysisLimit; /* Number of index rows to ANALYZE */ + int busyTimeout; /* Busy handler timeout, in msec */ + int nSavepoint; /* Number of non-transaction savepoints */ + int nStatement; /* Number of nested statement-transactions */ + i64 nDeferredCons; /* Net deferred constraints this transaction. */ + i64 nDeferredImmCons; /* Net deferred immediate constraints */ + int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + /* The following variables are all protected by the STATIC_MAIN + ** mutex, not by sqlite3.mutex. They are used by code in notify.c. + ** + ** When X.pUnlockConnection==Y, that means that X is waiting for Y to + ** unlock so that it can proceed. + ** + ** When X.pBlockingConnection==Y, that means that something that X tried + ** tried to do recently failed with an SQLITE_LOCKED error due to locks + ** held by Y. + */ + sqlite3 *pBlockingConnection; /* Connection that caused SQLITE_LOCKED */ + sqlite3 *pUnlockConnection; /* Connection to watch for unlock */ + void *pUnlockArg; /* Argument to xUnlockNotify */ + void (*xUnlockNotify)(void **, int); /* Unlock notify callback */ + sqlite3 *pNextBlocked; /* Next in list of all blocked connections */ +#endif +#ifdef SQLITE_USER_AUTHENTICATION + sqlite3_userauth auth; /* User authentication information */ +#endif +}; + +/* +** A macro to discover the encoding of a database. +*/ +#define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc) +#define ENC(db) ((db)->enc) + +/* +** A u64 constant where the lower 32 bits are all zeros. Only the +** upper 32 bits are included in the argument. Necessary because some +** C-compilers still do not accept LL integer literals. +*/ +#define HI(X) ((u64)(X)<<32) + +/* +** Possible values for the sqlite3.flags. +** +** Value constraints (enforced via assert()): +** SQLITE_FullFSync == PAGER_FULLFSYNC +** SQLITE_CkptFullFSync == PAGER_CKPT_FULLFSYNC +** SQLITE_CacheSpill == PAGER_CACHE_SPILL +*/ +#define SQLITE_WriteSchema 0x00000001 /* OK to update SQLITE_SCHEMA */ +#define SQLITE_LegacyFileFmt 0x00000002 /* Create new databases in format 1 */ +#define SQLITE_FullColNames 0x00000004 /* Show full column names on SELECT */ +#define SQLITE_FullFSync 0x00000008 /* Use full fsync on the backend */ +#define SQLITE_CkptFullFSync 0x00000010 /* Use full fsync for checkpoint */ +#define SQLITE_CacheSpill 0x00000020 /* OK to spill pager cache */ +#define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ +#define SQLITE_TrustedSchema 0x00000080 /* Allow unsafe functions and + ** vtabs in the schema definition */ +#define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ + /* result set is empty */ +#define SQLITE_IgnoreChecks 0x00000200 /* Do not enforce check constraints */ +#define SQLITE_ReadUncommit 0x00000400 /* READ UNCOMMITTED in shared-cache */ +#define SQLITE_NoCkptOnClose 0x00000800 /* No checkpoint on close()/DETACH */ +#define SQLITE_ReverseOrder 0x00001000 /* Reverse unordered SELECTs */ +#define SQLITE_RecTriggers 0x00002000 /* Enable recursive triggers */ +#define SQLITE_ForeignKeys 0x00004000 /* Enforce foreign key constraints */ +#define SQLITE_AutoIndex 0x00008000 /* Enable automatic indexes */ +#define SQLITE_LoadExtension 0x00010000 /* Enable load_extension */ +#define SQLITE_LoadExtFunc 0x00020000 /* Enable load_extension() SQL func */ +#define SQLITE_EnableTrigger 0x00040000 /* True to enable triggers */ +#define SQLITE_DeferFKs 0x00080000 /* Defer all FK constraints */ +#define SQLITE_QueryOnly 0x00100000 /* Disable database changes */ +#define SQLITE_CellSizeCk 0x00200000 /* Check btree cell sizes on load */ +#define SQLITE_Fts3Tokenizer 0x00400000 /* Enable fts3_tokenizer(2) */ +#define SQLITE_EnableQPSG 0x00800000 /* Query Planner Stability Guarantee*/ +#define SQLITE_TriggerEQP 0x01000000 /* Show trigger EXPLAIN QUERY PLAN */ +#define SQLITE_ResetDatabase 0x02000000 /* Reset the database */ +#define SQLITE_LegacyAlter 0x04000000 /* Legacy ALTER TABLE behaviour */ +#define SQLITE_NoSchemaError 0x08000000 /* Do not report schema parse errors*/ +#define SQLITE_Defensive 0x10000000 /* Input SQL is likely hostile */ +#define SQLITE_DqsDDL 0x20000000 /* dbl-quoted strings allowed in DDL*/ +#define SQLITE_DqsDML 0x40000000 /* dbl-quoted strings allowed in DML*/ +#define SQLITE_EnableView 0x80000000 /* Enable the use of views */ +#define SQLITE_CountRows HI(0x00001) /* Count rows changed by INSERT, */ + /* DELETE, or UPDATE and return */ + /* the count using a callback. */ + +/* Flags used only if debugging */ +#ifdef SQLITE_DEBUG +#define SQLITE_SqlTrace HI(0x0100000) /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing HI(0x0200000) /* Debug listings of VDBE progs */ +#define SQLITE_VdbeTrace HI(0x0400000) /* True to trace VDBE execution */ +#define SQLITE_VdbeAddopTrace HI(0x0800000) /* Trace sqlite3VdbeAddOp() calls */ +#define SQLITE_VdbeEQP HI(0x1000000) /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_ParserTrace HI(0x2000000) /* PRAGMA parser_trace=ON */ +#endif + +/* +** Allowed values for sqlite3.mDbFlags +*/ +#define DBFLAG_SchemaChange 0x0001 /* Uncommitted Hash table changes */ +#define DBFLAG_PreferBuiltin 0x0002 /* Preference to built-in funcs */ +#define DBFLAG_Vacuum 0x0004 /* Currently in a VACUUM */ +#define DBFLAG_VacuumInto 0x0008 /* Currently running VACUUM INTO */ +#define DBFLAG_SchemaKnownOk 0x0010 /* Schema is known to be valid */ +#define DBFLAG_InternalFunc 0x0020 /* Allow use of internal functions */ +#define DBFLAG_EncodingFixed 0x0040 /* No longer possible to change enc. */ + +/* +** Bits of the sqlite3.dbOptFlags field that are used by the +** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to +** selectively disable various optimizations. +*/ +#define SQLITE_QueryFlattener 0x00000001 /* Query flattening */ +#define SQLITE_WindowFunc 0x00000002 /* Use xInverse for window functions */ +#define SQLITE_GroupByOrder 0x00000004 /* GROUPBY cover of ORDERBY */ +#define SQLITE_FactorOutConst 0x00000008 /* Constant factoring */ +#define SQLITE_DistinctOpt 0x00000010 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 0x00000020 /* Covering index scans */ +#define SQLITE_OrderByIdxJoin 0x00000040 /* ORDER BY of joins via index */ +#define SQLITE_Transitive 0x00000080 /* Transitive constraints */ +#define SQLITE_OmitNoopJoin 0x00000100 /* Omit unused tables in joins */ +#define SQLITE_CountOfView 0x00000200 /* The count-of-view optimization */ +#define SQLITE_CursorHints 0x00000400 /* Add OP_CursorHint opcodes */ +#define SQLITE_Stat4 0x00000800 /* Use STAT4 data */ + /* TH3 expects this value ^^^^^^^^^^ to be 0x0000800. Don't change it */ +#define SQLITE_PushDown 0x00001000 /* The push-down optimization */ +#define SQLITE_SimplifyJoin 0x00002000 /* Convert LEFT JOIN to JOIN */ +#define SQLITE_SkipScan 0x00004000 /* Skip-scans */ +#define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */ +#define SQLITE_MinMaxOpt 0x00010000 /* The min/max optimization */ +#define SQLITE_SeekScan 0x00020000 /* The OP_SeekScan optimization */ +#define SQLITE_OmitOrderBy 0x00040000 /* Omit pointless ORDER BY */ + /* TH3 expects this value ^^^^^^^^^^ to be 0x40000. Coordinate any change */ +#define SQLITE_AllOpts 0xffffffff /* All optimizations */ + +/* +** Macros for testing whether or not optimizations are enabled or disabled. +*/ +#define OptimizationDisabled(db, mask) (((db)->dbOptFlags&(mask))!=0) +#define OptimizationEnabled(db, mask) (((db)->dbOptFlags&(mask))==0) + +/* +** Return true if it OK to factor constant expressions into the initialization +** code. The argument is a Parse object for the code generator. +*/ +#define ConstFactorOk(P) ((P)->okConstFactor) + +/* Possible values for the sqlite3.eOpenState field. +** The numbers are randomly selected such that a minimum of three bits must +** change to convert any number to another or to zero +*/ +#define SQLITE_STATE_OPEN 0x76 /* Database is open */ +#define SQLITE_STATE_CLOSED 0xce /* Database is closed */ +#define SQLITE_STATE_SICK 0xba /* Error and awaiting close */ +#define SQLITE_STATE_BUSY 0x6d /* Database currently in use */ +#define SQLITE_STATE_ERROR 0xd5 /* An SQLITE_MISUSE error occurred */ +#define SQLITE_STATE_ZOMBIE 0xa7 /* Close with last statement close */ + +/* +** Each SQL function is defined by an instance of the following +** structure. For global built-in functions (ex: substr(), max(), count()) +** a pointer to this structure is held in the sqlite3BuiltinFunctions object. +** For per-connection application-defined functions, a pointer to this +** structure is held in the db->aHash hash table. +** +** The u.pHash field is used by the global built-ins. The u.pDestructor +** field is used by per-connection app-def functions. +*/ +struct FuncDef { + i8 nArg; /* Number of arguments. -1 means unlimited */ + u32 funcFlags; /* Some combination of SQLITE_FUNC_* */ + void *pUserData; /* User data parameter */ + FuncDef *pNext; /* Next function with same name */ + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**); /* func or agg-step */ + void (*xFinalize)(sqlite3_context*); /* Agg finalizer */ + void (*xValue)(sqlite3_context*); /* Current agg value */ + void (*xInverse)(sqlite3_context*,int,sqlite3_value**); /* inverse agg-step */ + const char *zName; /* SQL name of the function. */ + union { + FuncDef *pHash; /* Next with a different name but the same hash */ + FuncDestructor *pDestructor; /* Reference counted destructor function */ + } u; +}; + +/* +** This structure encapsulates a user-function destructor callback (as +** configured using create_function_v2()) and a reference counter. When +** create_function_v2() is called to create a function with a destructor, +** a single object of this type is allocated. FuncDestructor.nRef is set to +** the number of FuncDef objects created (either 1 or 3, depending on whether +** or not the specified encoding is SQLITE_ANY). The FuncDef.pDestructor +** member of each of the new FuncDef objects is set to point to the allocated +** FuncDestructor. +** +** Thereafter, when one of the FuncDef objects is deleted, the reference +** count on this object is decremented. When it reaches 0, the destructor +** is invoked and the FuncDestructor structure freed. +*/ +struct FuncDestructor { + int nRef; + void (*xDestroy)(void *); + void *pUserData; +}; + +/* +** Possible values for FuncDef.flags. Note that the _LENGTH and _TYPEOF +** values must correspond to OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG. And +** SQLITE_FUNC_CONSTANT must be the same as SQLITE_DETERMINISTIC. There +** are assert() statements in the code to verify this. +** +** Value constraints (enforced via assert()): +** SQLITE_FUNC_MINMAX == NC_MinMaxAgg == SF_MinMaxAgg +** SQLITE_FUNC_ANYORDER == NC_OrderAgg == SF_OrderByReqd +** SQLITE_FUNC_LENGTH == OPFLAG_LENGTHARG +** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG +** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API +** SQLITE_FUNC_DIRECT == SQLITE_DIRECTONLY from the API +** SQLITE_FUNC_UNSAFE == SQLITE_INNOCUOUS +** SQLITE_FUNC_ENCMASK depends on SQLITE_UTF* macros in the API +*/ +#define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ +#define SQLITE_FUNC_LIKE 0x0004 /* Candidate for the LIKE optimization */ +#define SQLITE_FUNC_CASE 0x0008 /* Case-sensitive LIKE-type function */ +#define SQLITE_FUNC_EPHEM 0x0010 /* Ephemeral. Delete with VDBE */ +#define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/ +#define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ +#define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ +#define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ +/* 0x0200 -- available for reuse */ +#define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ +#define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ +#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ +#define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a + ** single query - might change over time */ +#define SQLITE_FUNC_TEST 0x4000 /* Built-in testing functions */ +#define SQLITE_FUNC_OFFSET 0x8000 /* Built-in sqlite_offset() function */ +#define SQLITE_FUNC_WINDOW 0x00010000 /* Built-in window-only function */ +#define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */ +#define SQLITE_FUNC_DIRECT 0x00080000 /* Not for use in TRIGGERs or VIEWs */ +#define SQLITE_FUNC_SUBTYPE 0x00100000 /* Result likely to have sub-type */ +#define SQLITE_FUNC_UNSAFE 0x00200000 /* Function has side effects */ +#define SQLITE_FUNC_INLINE 0x00400000 /* Functions implemented in-line */ +#define SQLITE_FUNC_ANYORDER 0x08000000 /* count/min/max aggregate */ + +/* Identifier numbers for each in-line function */ +#define INLINEFUNC_coalesce 0 +#define INLINEFUNC_implies_nonnull_row 1 +#define INLINEFUNC_expr_implies_expr 2 +#define INLINEFUNC_expr_compare 3 +#define INLINEFUNC_affinity 4 +#define INLINEFUNC_iif 5 +#define INLINEFUNC_unlikely 99 /* Default case */ + +/* +** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are +** used to create the initializers for the FuncDef structures. +** +** FUNCTION(zName, nArg, iArg, bNC, xFunc) +** Used to create a scalar function definition of a function zName +** implemented by C function xFunc that accepts nArg arguments. The +** value passed as iArg is cast to a (void*) and made available +** as the user-data (sqlite3_user_data()) for the function. If +** argument bNC is true, then the SQLITE_FUNC_NEEDCOLL flag is set. +** +** VFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. +** +** SFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and +** adds the SQLITE_DIRECTONLY flag. +** +** INLINE_FUNC(zName, nArg, iFuncId, mFlags) +** zName is the name of a function that is implemented by in-line +** byte code rather than by the usual callbacks. The iFuncId +** parameter determines the function id. The mFlags parameter is +** optional SQLITE_FUNC_ flags for this function. +** +** TEST_FUNC(zName, nArg, iFuncId, mFlags) +** zName is the name of a test-only function implemented by in-line +** byte code rather than by the usual callbacks. The iFuncId +** parameter determines the function id. The mFlags parameter is +** optional SQLITE_FUNC_ flags for this function. +** +** DFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and +** adds the SQLITE_FUNC_SLOCHNG flag. Used for date & time functions +** and functions like sqlite_version() that can change, but not during +** a single query. The iArg is ignored. The user-data is always set +** to a NULL pointer. The bNC parameter is not used. +** +** MFUNCTION(zName, nArg, xPtr, xFunc) +** For math-library functions. xPtr is an arbitrary pointer. +** +** PURE_DATE(zName, nArg, iArg, bNC, xFunc) +** Used for "pure" date/time functions, this macro is like DFUNCTION +** except that it does set the SQLITE_FUNC_CONSTANT flags. iArg is +** ignored and the user-data for these functions is set to an +** arbitrary non-NULL pointer. The bNC parameter is not used. +** +** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) +** Used to create an aggregate function definition implemented by +** the C functions xStep and xFinal. The first four parameters +** are interpreted in the same way as the first 4 parameters to +** FUNCTION(). +** +** WFUNCTION(zName, nArg, iArg, xStep, xFinal, xValue, xInverse) +** Used to create an aggregate function definition implemented by +** the C functions xStep and xFinal. The first four parameters +** are interpreted in the same way as the first 4 parameters to +** FUNCTION(). +** +** LIKEFUNC(zName, nArg, pArg, flags) +** Used to create a scalar function definition of a function zName +** that accepts nArg arguments and is implemented by a call to C +** function likeFunc. Argument pArg is cast to a (void *) and made +** available as the function user-data (sqlite3_user_data()). The +** FuncDef.flags variable is set to the value passed as the flags +** parameter. +*/ +#define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } +#define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } +#define SFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_UTF8|SQLITE_DIRECTONLY|SQLITE_FUNC_UNSAFE, \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } +#define MFUNCTION(zName, nArg, xPtr, xFunc) \ + {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \ + xPtr, 0, xFunc, 0, 0, 0, #zName, {0} } +#define INLINE_FUNC(zName, nArg, iArg, mFlags) \ + {nArg, SQLITE_UTF8|SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \ + SQLITE_INT_TO_PTR(iArg), 0, noopFunc, 0, 0, 0, #zName, {0} } +#define TEST_FUNC(zName, nArg, iArg, mFlags) \ + {nArg, SQLITE_UTF8|SQLITE_FUNC_INTERNAL|SQLITE_FUNC_TEST| \ + SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \ + SQLITE_INT_TO_PTR(iArg), 0, noopFunc, 0, 0, 0, #zName, {0} } +#define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8, \ + 0, 0, xFunc, 0, 0, 0, #zName, {0} } +#define PURE_DATE(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|SQLITE_FUNC_CONSTANT, \ + (void*)&sqlite3Config, 0, xFunc, 0, 0, 0, #zName, {0} } +#define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ + {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } +#define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + pArg, 0, xFunc, 0, 0, 0, #zName, } +#define LIKEFUNC(zName, nArg, arg, flags) \ + {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ + (void *)arg, 0, likeFunc, 0, 0, 0, #zName, {0} } +#define WAGGREGATE(zName, nArg, arg, nc, xStep, xFinal, xValue, xInverse, f) \ + {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|f, \ + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xValue,xInverse,#zName, {0}} +#define INTERNAL_FUNCTION(zName, nArg, xFunc) \ + {nArg, SQLITE_FUNC_INTERNAL|SQLITE_UTF8|SQLITE_FUNC_CONSTANT, \ + 0, 0, xFunc, 0, 0, 0, #zName, {0} } + + +/* +** All current savepoints are stored in a linked list starting at +** sqlite3.pSavepoint. The first element in the list is the most recently +** opened savepoint. Savepoints are added to the list by the vdbe +** OP_Savepoint instruction. +*/ +struct Savepoint { + char *zName; /* Savepoint name (nul-terminated) */ + i64 nDeferredCons; /* Number of deferred fk violations */ + i64 nDeferredImmCons; /* Number of deferred imm fk. */ + Savepoint *pNext; /* Parent savepoint (if any) */ +}; + +/* +** The following are used as the second parameter to sqlite3Savepoint(), +** and as the P1 argument to the OP_Savepoint instruction. +*/ +#define SAVEPOINT_BEGIN 0 +#define SAVEPOINT_RELEASE 1 +#define SAVEPOINT_ROLLBACK 2 + + +/* +** Each SQLite module (virtual table definition) is defined by an +** instance of the following structure, stored in the sqlite3.aModule +** hash table. +*/ +struct Module { + const sqlite3_module *pModule; /* Callback pointers */ + const char *zName; /* Name passed to create_module() */ + int nRefModule; /* Number of pointers to this object */ + void *pAux; /* pAux passed to create_module() */ + void (*xDestroy)(void *); /* Module destructor function */ + Table *pEpoTab; /* Eponymous table for this module */ +}; + +/* +** Information about each column of an SQL table is held in an instance +** of the Column structure, in the Table.aCol[] array. +** +** Definitions: +** +** "table column index" This is the index of the column in the +** Table.aCol[] array, and also the index of +** the column in the original CREATE TABLE stmt. +** +** "storage column index" This is the index of the column in the +** record BLOB generated by the OP_MakeRecord +** opcode. The storage column index is less than +** or equal to the table column index. It is +** equal if and only if there are no VIRTUAL +** columns to the left. +** +** Notes on zCnName: +** The zCnName field stores the name of the column, the datatype of the +** column, and the collating sequence for the column, in that order, all in +** a single allocation. Each string is 0x00 terminated. The datatype +** is only included if the COLFLAG_HASTYPE bit of colFlags is set and the +** collating sequence name is only included if the COLFLAG_HASCOLL bit is +** set. +*/ +struct Column { + char *zCnName; /* Name of this column */ + unsigned notNull :4; /* An OE_ code for handling a NOT NULL constraint */ + unsigned eCType :4; /* One of the standard types */ + char affinity; /* One of the SQLITE_AFF_... values */ + u8 szEst; /* Est size of value in this column. sizeof(INT)==1 */ + u8 hName; /* Column name hash for faster lookup */ + u16 iDflt; /* 1-based index of DEFAULT. 0 means "none" */ + u16 colFlags; /* Boolean properties. See COLFLAG_ defines below */ +}; + +/* Allowed values for Column.eCType. +** +** Values must match entries in the global constant arrays +** sqlite3StdTypeLen[] and sqlite3StdType[]. Each value is one more +** than the offset into these arrays for the corresponding name. +** Adjust the SQLITE_N_STDTYPE value if adding or removing entries. +*/ +#define COLTYPE_CUSTOM 0 /* Type appended to zName */ +#define COLTYPE_ANY 1 +#define COLTYPE_BLOB 2 +#define COLTYPE_INT 3 +#define COLTYPE_INTEGER 4 +#define COLTYPE_REAL 5 +#define COLTYPE_TEXT 6 +#define SQLITE_N_STDTYPE 6 /* Number of standard types */ + +/* Allowed values for Column.colFlags. +** +** Constraints: +** TF_HasVirtual == COLFLAG_VIRTUAL +** TF_HasStored == COLFLAG_STORED +** TF_HasHidden == COLFLAG_HIDDEN +*/ +#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ +#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ +#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ +#define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ +#define COLFLAG_SORTERREF 0x0010 /* Use sorter-refs with this column */ +#define COLFLAG_VIRTUAL 0x0020 /* GENERATED ALWAYS AS ... VIRTUAL */ +#define COLFLAG_STORED 0x0040 /* GENERATED ALWAYS AS ... STORED */ +#define COLFLAG_NOTAVAIL 0x0080 /* STORED column not yet calculated */ +#define COLFLAG_BUSY 0x0100 /* Blocks recursion on GENERATED columns */ +#define COLFLAG_HASCOLL 0x0200 /* Has collating sequence name in zCnName */ +#define COLFLAG_GENERATED 0x0060 /* Combo: _STORED, _VIRTUAL */ +#define COLFLAG_NOINSERT 0x0062 /* Combo: _HIDDEN, _STORED, _VIRTUAL */ + +/* +** A "Collating Sequence" is defined by an instance of the following +** structure. Conceptually, a collating sequence consists of a name and +** a comparison routine that defines the order of that sequence. +** +** If CollSeq.xCmp is NULL, it means that the +** collating sequence is undefined. Indices built on an undefined +** collating sequence may not be read or written. +*/ +struct CollSeq { + char *zName; /* Name of the collating sequence, UTF-8 encoded */ + u8 enc; /* Text encoding handled by xCmp() */ + void *pUser; /* First argument to xCmp() */ + int (*xCmp)(void*,int, const void*, int, const void*); + void (*xDel)(void*); /* Destructor for pUser */ +}; + +/* +** A sort order can be either ASC or DESC. +*/ +#define SQLITE_SO_ASC 0 /* Sort in ascending order */ +#define SQLITE_SO_DESC 1 /* Sort in ascending order */ +#define SQLITE_SO_UNDEFINED -1 /* No sort order specified */ + +/* +** Column affinity types. +** +** These used to have mnemonic name like 'i' for SQLITE_AFF_INTEGER and +** 't' for SQLITE_AFF_TEXT. But we can save a little space and improve +** the speed a little by numbering the values consecutively. +** +** But rather than start with 0 or 1, we begin with 'A'. That way, +** when multiple affinity types are concatenated into a string and +** used as the P4 operand, they will be more readable. +** +** Note also that the numeric types are grouped together so that testing +** for a numeric type is a single comparison. And the BLOB type is first. +*/ +#define SQLITE_AFF_NONE 0x40 /* '@' */ +#define SQLITE_AFF_BLOB 0x41 /* 'A' */ +#define SQLITE_AFF_TEXT 0x42 /* 'B' */ +#define SQLITE_AFF_NUMERIC 0x43 /* 'C' */ +#define SQLITE_AFF_INTEGER 0x44 /* 'D' */ +#define SQLITE_AFF_REAL 0x45 /* 'E' */ + +#define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) + +/* +** The SQLITE_AFF_MASK values masks off the significant bits of an +** affinity value. +*/ +#define SQLITE_AFF_MASK 0x47 + +/* +** Additional bit values that can be ORed with an affinity without +** changing the affinity. +** +** The SQLITE_NOTNULL flag is a combination of NULLEQ and JUMPIFNULL. +** It causes an assert() to fire if either operand to a comparison +** operator is NULL. It is added to certain comparison operators to +** prove that the operands are always NOT NULL. +*/ +#define SQLITE_JUMPIFNULL 0x10 /* jumps if either operand is NULL */ +#define SQLITE_NULLEQ 0x80 /* NULL=NULL */ +#define SQLITE_NOTNULL 0x90 /* Assert that operands are never NULL */ + +/* +** An object of this type is created for each virtual table present in +** the database schema. +** +** If the database schema is shared, then there is one instance of this +** structure for each database connection (sqlite3*) that uses the shared +** schema. This is because each database connection requires its own unique +** instance of the sqlite3_vtab* handle used to access the virtual table +** implementation. sqlite3_vtab* handles can not be shared between +** database connections, even when the rest of the in-memory database +** schema is shared, as the implementation often stores the database +** connection handle passed to it via the xConnect() or xCreate() method +** during initialization internally. This database connection handle may +** then be used by the virtual table implementation to access real tables +** within the database. So that they appear as part of the callers +** transaction, these accesses need to be made via the same database +** connection as that used to execute SQL operations on the virtual table. +** +** All VTable objects that correspond to a single table in a shared +** database schema are initially stored in a linked-list pointed to by +** the Table.pVTable member variable of the corresponding Table object. +** When an sqlite3_prepare() operation is required to access the virtual +** table, it searches the list for the VTable that corresponds to the +** database connection doing the preparing so as to use the correct +** sqlite3_vtab* handle in the compiled query. +** +** When an in-memory Table object is deleted (for example when the +** schema is being reloaded for some reason), the VTable objects are not +** deleted and the sqlite3_vtab* handles are not xDisconnect()ed +** immediately. Instead, they are moved from the Table.pVTable list to +** another linked list headed by the sqlite3.pDisconnect member of the +** corresponding sqlite3 structure. They are then deleted/xDisconnected +** next time a statement is prepared using said sqlite3*. This is done +** to avoid deadlock issues involving multiple sqlite3.mutex mutexes. +** Refer to comments above function sqlite3VtabUnlockList() for an +** explanation as to why it is safe to add an entry to an sqlite3.pDisconnect +** list without holding the corresponding sqlite3.mutex mutex. +** +** The memory for objects of this type is always allocated by +** sqlite3DbMalloc(), using the connection handle stored in VTable.db as +** the first argument. +*/ +struct VTable { + sqlite3 *db; /* Database connection associated with this table */ + Module *pMod; /* Pointer to module implementation */ + sqlite3_vtab *pVtab; /* Pointer to vtab instance */ + int nRef; /* Number of pointers to this structure */ + u8 bConstraint; /* True if constraints are supported */ + u8 eVtabRisk; /* Riskiness of allowing hacker access */ + int iSavepoint; /* Depth of the SAVEPOINT stack */ + VTable *pNext; /* Next in linked list (see above) */ +}; + +/* Allowed values for VTable.eVtabRisk +*/ +#define SQLITE_VTABRISK_Low 0 +#define SQLITE_VTABRISK_Normal 1 +#define SQLITE_VTABRISK_High 2 + +/* +** The schema for each SQL table, virtual table, and view is represented +** in memory by an instance of the following structure. +*/ +struct Table { + char *zName; /* Name of the table or view */ + Column *aCol; /* Information about each column */ + Index *pIndex; /* List of SQL indexes on this table. */ + char *zColAff; /* String defining the affinity of each column */ + ExprList *pCheck; /* All CHECK constraints */ + /* ... also used as column name list in a VIEW */ + Pgno tnum; /* Root BTree page for this table */ + u32 nTabRef; /* Number of pointers to this Table */ + u32 tabFlags; /* Mask of TF_* values */ + i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ + i16 nCol; /* Number of columns in this table */ + i16 nNVCol; /* Number of columns that are not VIRTUAL */ + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ + LogEst szTabRow; /* Estimated size of each table row in bytes */ +#ifdef SQLITE_ENABLE_COSTMULT + LogEst costMult; /* Cost multiplier for using this table */ +#endif + u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ + u8 eTabType; /* 0: normal, 1: virtual, 2: view */ + union { + struct { /* Used by ordinary tables: */ + int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ + FKey *pFKey; /* Linked list of all foreign keys in this table */ + ExprList *pDfltList; /* DEFAULT clauses on various columns. + ** Or the AS clause for generated columns. */ + } tab; + struct { /* Used by views: */ + Select *pSelect; /* View definition */ + } view; + struct { /* Used by virtual tables only: */ + int nArg; /* Number of arguments to the module */ + char **azArg; /* 0: module 1: schema 2: vtab name 3...: args */ + VTable *p; /* List of VTable objects. */ + } vtab; + } u; + Trigger *pTrigger; /* List of triggers on this object */ + Schema *pSchema; /* Schema that contains this table */ +}; + +/* +** Allowed values for Table.tabFlags. +** +** TF_OOOHidden applies to tables or view that have hidden columns that are +** followed by non-hidden columns. Example: "CREATE VIRTUAL TABLE x USING +** vtab1(a HIDDEN, b);". Since "b" is a non-hidden column but "a" is hidden, +** the TF_OOOHidden attribute would apply in this case. Such tables require +** special handling during INSERT processing. The "OOO" means "Out Of Order". +** +** Constraints: +** +** TF_HasVirtual == COLFLAG_VIRTUAL +** TF_HasStored == COLFLAG_STORED +** TF_HasHidden == COLFLAG_HIDDEN +*/ +#define TF_Readonly 0x00000001 /* Read-only system table */ +#define TF_HasHidden 0x00000002 /* Has one or more hidden columns */ +#define TF_HasPrimaryKey 0x00000004 /* Table has a primary key */ +#define TF_Autoincrement 0x00000008 /* Integer primary key is autoincrement */ +#define TF_HasStat1 0x00000010 /* nRowLogEst set from sqlite_stat1 */ +#define TF_HasVirtual 0x00000020 /* Has one or more VIRTUAL columns */ +#define TF_HasStored 0x00000040 /* Has one or more STORED columns */ +#define TF_HasGenerated 0x00000060 /* Combo: HasVirtual + HasStored */ +#define TF_WithoutRowid 0x00000080 /* No rowid. PRIMARY KEY is the key */ +#define TF_StatsUsed 0x00000100 /* Query planner decisions affected by + ** Index.aiRowLogEst[] values */ +#define TF_NoVisibleRowid 0x00000200 /* No user-visible "rowid" column */ +#define TF_OOOHidden 0x00000400 /* Out-of-Order hidden columns */ +#define TF_HasNotNull 0x00000800 /* Contains NOT NULL constraints */ +#define TF_Shadow 0x00001000 /* True for a shadow table */ +#define TF_HasStat4 0x00002000 /* STAT4 info available for this table */ +#define TF_Ephemeral 0x00004000 /* An ephemeral table */ +#define TF_Eponymous 0x00008000 /* An eponymous virtual table */ +#define TF_Strict 0x00010000 /* STRICT mode */ + +/* +** Allowed values for Table.eTabType +*/ +#define TABTYP_NORM 0 /* Ordinary table */ +#define TABTYP_VTAB 1 /* Virtual table */ +#define TABTYP_VIEW 2 /* A view */ + +#define IsView(X) ((X)->eTabType==TABTYP_VIEW) +#define IsOrdinaryTable(X) ((X)->eTabType==TABTYP_NORM) + +/* +** Test to see whether or not a table is a virtual table. This is +** done as a macro so that it will be optimized out when virtual +** table support is omitted from the build. +*/ +#ifndef SQLITE_OMIT_VIRTUALTABLE +# define IsVirtual(X) ((X)->eTabType==TABTYP_VTAB) +# define ExprIsVtab(X) \ + ((X)->op==TK_COLUMN && (X)->y.pTab!=0 && (X)->y.pTab->eTabType==TABTYP_VTAB) +#else +# define IsVirtual(X) 0 +# define ExprIsVtab(X) 0 +#endif + +/* +** Macros to determine if a column is hidden. IsOrdinaryHiddenColumn() +** only works for non-virtual tables (ordinary tables and views) and is +** always false unless SQLITE_ENABLE_HIDDEN_COLUMNS is defined. The +** IsHiddenColumn() macro is general purpose. +*/ +#if defined(SQLITE_ENABLE_HIDDEN_COLUMNS) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) +# define IsOrdinaryHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) +#elif !defined(SQLITE_OMIT_VIRTUALTABLE) +# define IsHiddenColumn(X) (((X)->colFlags & COLFLAG_HIDDEN)!=0) +# define IsOrdinaryHiddenColumn(X) 0 +#else +# define IsHiddenColumn(X) 0 +# define IsOrdinaryHiddenColumn(X) 0 +#endif + + +/* Does the table have a rowid */ +#define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) +#define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) + +/* +** Each foreign key constraint is an instance of the following structure. +** +** A foreign key is associated with two tables. The "from" table is +** the table that contains the REFERENCES clause that creates the foreign +** key. The "to" table is the table that is named in the REFERENCES clause. +** Consider this example: +** +** CREATE TABLE ex1( +** a INTEGER PRIMARY KEY, +** b INTEGER CONSTRAINT fk1 REFERENCES ex2(x) +** ); +** +** For foreign key "fk1", the from-table is "ex1" and the to-table is "ex2". +** Equivalent names: +** +** from-table == child-table +** to-table == parent-table +** +** Each REFERENCES clause generates an instance of the following structure +** which is attached to the from-table. The to-table need not exist when +** the from-table is created. The existence of the to-table is not checked. +** +** The list of all parents for child Table X is held at X.pFKey. +** +** A list of all children for a table named Z (which might not even exist) +** is held in Schema.fkeyHash with a hash key of Z. +*/ +struct FKey { + Table *pFrom; /* Table containing the REFERENCES clause (aka: Child) */ + FKey *pNextFrom; /* Next FKey with the same in pFrom. Next parent of pFrom */ + char *zTo; /* Name of table that the key points to (aka: Parent) */ + FKey *pNextTo; /* Next with the same zTo. Next child of zTo. */ + FKey *pPrevTo; /* Previous with the same zTo */ + int nCol; /* Number of columns in this key */ + /* EV: R-30323-21917 */ + u8 isDeferred; /* True if constraint checking is deferred till COMMIT */ + u8 aAction[2]; /* ON DELETE and ON UPDATE actions, respectively */ + Trigger *apTrigger[2];/* Triggers for aAction[] actions */ + struct sColMap { /* Mapping of columns in pFrom to columns in zTo */ + int iFrom; /* Index of column in pFrom */ + char *zCol; /* Name of column in zTo. If NULL use PRIMARY KEY */ + } aCol[1]; /* One entry for each of nCol columns */ +}; + +/* +** SQLite supports many different ways to resolve a constraint +** error. ROLLBACK processing means that a constraint violation +** causes the operation in process to fail and for the current transaction +** to be rolled back. ABORT processing means the operation in process +** fails and any prior changes from that one operation are backed out, +** but the transaction is not rolled back. FAIL processing means that +** the operation in progress stops and returns an error code. But prior +** changes due to the same operation are not backed out and no rollback +** occurs. IGNORE means that the particular row that caused the constraint +** error is not inserted or updated. Processing continues and no error +** is returned. REPLACE means that preexisting database rows that caused +** a UNIQUE constraint violation are removed so that the new insert or +** update can proceed. Processing continues and no error is reported. +** UPDATE applies to insert operations only and means that the insert +** is omitted and the DO UPDATE clause of an upsert is run instead. +** +** RESTRICT, SETNULL, SETDFLT, and CASCADE actions apply only to foreign keys. +** RESTRICT is the same as ABORT for IMMEDIATE foreign keys and the +** same as ROLLBACK for DEFERRED keys. SETNULL means that the foreign +** key is set to NULL. SETDFLT means that the foreign key is set +** to its default value. CASCADE means that a DELETE or UPDATE of the +** referenced table row is propagated into the row that holds the +** foreign key. +** +** The OE_Default value is a place holder that means to use whatever +** conflict resolution algorthm is required from context. +** +** The following symbolic values are used to record which type +** of conflict resolution action to take. +*/ +#define OE_None 0 /* There is no constraint to check */ +#define OE_Rollback 1 /* Fail the operation and rollback the transaction */ +#define OE_Abort 2 /* Back out changes but do no rollback transaction */ +#define OE_Fail 3 /* Stop the operation but leave all prior changes */ +#define OE_Ignore 4 /* Ignore the error. Do not do the INSERT or UPDATE */ +#define OE_Replace 5 /* Delete existing record, then do INSERT or UPDATE */ +#define OE_Update 6 /* Process as a DO UPDATE in an upsert */ +#define OE_Restrict 7 /* OE_Abort for IMMEDIATE, OE_Rollback for DEFERRED */ +#define OE_SetNull 8 /* Set the foreign key value to NULL */ +#define OE_SetDflt 9 /* Set the foreign key value to its default */ +#define OE_Cascade 10 /* Cascade the changes */ +#define OE_Default 11 /* Do whatever the default action is */ + + +/* +** An instance of the following structure is passed as the first +** argument to sqlite3VdbeKeyCompare and is used to control the +** comparison of the two index keys. +** +** Note that aSortOrder[] and aColl[] have nField+1 slots. There +** are nField slots for the columns of an index then one extra slot +** for the rowid at the end. +*/ +struct KeyInfo { + u32 nRef; /* Number of references to this KeyInfo object */ + u8 enc; /* Text encoding - one of the SQLITE_UTF* values */ + u16 nKeyField; /* Number of key columns in the index */ + u16 nAllField; /* Total columns, including key plus others */ + sqlite3 *db; /* The database connection */ + u8 *aSortFlags; /* Sort order for each column. */ + CollSeq *aColl[1]; /* Collating sequence for each term of the key */ +}; + +/* +** Allowed bit values for entries in the KeyInfo.aSortFlags[] array. +*/ +#define KEYINFO_ORDER_DESC 0x01 /* DESC sort order */ +#define KEYINFO_ORDER_BIGNULL 0x02 /* NULL is larger than any other value */ + +/* +** This object holds a record which has been parsed out into individual +** fields, for the purposes of doing a comparison. +** +** A record is an object that contains one or more fields of data. +** Records are used to store the content of a table row and to store +** the key of an index. A blob encoding of a record is created by +** the OP_MakeRecord opcode of the VDBE and is disassembled by the +** OP_Column opcode. +** +** An instance of this object serves as a "key" for doing a search on +** an index b+tree. The goal of the search is to find the entry that +** is closed to the key described by this object. This object might hold +** just a prefix of the key. The number of fields is given by +** pKeyInfo->nField. +** +** The r1 and r2 fields are the values to return if this key is less than +** or greater than a key in the btree, respectively. These are normally +** -1 and +1 respectively, but might be inverted to +1 and -1 if the b-tree +** is in DESC order. +** +** The key comparison functions actually return default_rc when they find +** an equals comparison. default_rc can be -1, 0, or +1. If there are +** multiple entries in the b-tree with the same key (when only looking +** at the first pKeyInfo->nFields,) then default_rc can be set to -1 to +** cause the search to find the last match, or +1 to cause the search to +** find the first match. +** +** The key comparison functions will set eqSeen to true if they ever +** get and equal results when comparing this structure to a b-tree record. +** When default_rc!=0, the search might end up on the record immediately +** before the first match or immediately after the last match. The +** eqSeen field will indicate whether or not an exact match exists in the +** b-tree. +*/ +struct UnpackedRecord { + KeyInfo *pKeyInfo; /* Collation and sort-order information */ + Mem *aMem; /* Values */ + u16 nField; /* Number of entries in apMem[] */ + i8 default_rc; /* Comparison result if keys are equal */ + u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ + i8 r1; /* Value to return if (lhs < rhs) */ + i8 r2; /* Value to return if (lhs > rhs) */ + u8 eqSeen; /* True if an equality comparison has been seen */ +}; + + +/* +** Each SQL index is represented in memory by an +** instance of the following structure. +** +** The columns of the table that are to be indexed are described +** by the aiColumn[] field of this structure. For example, suppose +** we have the following table and index: +** +** CREATE TABLE Ex1(c1 int, c2 int, c3 text); +** CREATE INDEX Ex2 ON Ex1(c3,c1); +** +** In the Table structure describing Ex1, nCol==3 because there are +** three columns in the table. In the Index structure describing +** Ex2, nColumn==2 since 2 of the 3 columns of Ex1 are indexed. +** The value of aiColumn is {2, 0}. aiColumn[0]==2 because the +** first column to be indexed (c3) has an index of 2 in Ex1.aCol[]. +** The second column to be indexed (c1) has an index of 0 in +** Ex1.aCol[], hence Ex2.aiColumn[1]==0. +** +** The Index.onError field determines whether or not the indexed columns +** must be unique and what to do if they are not. When Index.onError=OE_None, +** it means this is not a unique index. Otherwise it is a unique index +** and the value of Index.onError indicate the which conflict resolution +** algorithm to employ whenever an attempt is made to insert a non-unique +** element. +** +** While parsing a CREATE TABLE or CREATE INDEX statement in order to +** generate VDBE code (as opposed to parsing one read from an sqlite_schema +** table as part of parsing an existing database schema), transient instances +** of this structure may be created. In this case the Index.tnum variable is +** used to store the address of a VDBE instruction, not a database page +** number (it cannot - the database page is not allocated until the VDBE +** program is executed). See convertToWithoutRowidTable() for details. +*/ +struct Index { + char *zName; /* Name of this index */ + i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ + LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ + Table *pTable; /* The SQL table being indexed */ + char *zColAff; /* String defining the affinity of each column */ + Index *pNext; /* The next index associated with the same table */ + Schema *pSchema; /* Schema containing this index */ + u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ + const char **azColl; /* Array of collation sequence names for index */ + Expr *pPartIdxWhere; /* WHERE clause for partial indices */ + ExprList *aColExpr; /* Column expressions */ + Pgno tnum; /* DB Page containing root of this index */ + LogEst szIdxRow; /* Estimated average row size in bytes */ + u16 nKeyCol; /* Number of columns forming the key */ + u16 nColumn; /* Number of columns stored in the index */ + u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + unsigned idxType:2; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK */ + unsigned bUnordered:1; /* Use this index for == or IN queries only */ + unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ + unsigned isResized:1; /* True if resizeIndexObject() has been called */ + unsigned isCovering:1; /* True if this is a covering index */ + unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ + unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ + unsigned bNoQuery:1; /* Do not use this index to optimize queries */ + unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ + unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ +#ifdef SQLITE_ENABLE_STAT4 + int nSample; /* Number of elements in aSample[] */ + int nSampleCol; /* Size of IndexSample.anEq[] and so on */ + tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ + IndexSample *aSample; /* Samples of the left-most key */ + tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ + tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ +#endif + Bitmask colNotIdxed; /* 0 for unindexed columns in pTab */ +}; + +/* +** Allowed values for Index.idxType +*/ +#define SQLITE_IDXTYPE_APPDEF 0 /* Created using CREATE INDEX */ +#define SQLITE_IDXTYPE_UNIQUE 1 /* Implements a UNIQUE constraint */ +#define SQLITE_IDXTYPE_PRIMARYKEY 2 /* Is the PRIMARY KEY for the table */ +#define SQLITE_IDXTYPE_IPK 3 /* INTEGER PRIMARY KEY index */ + +/* Return true if index X is a PRIMARY KEY index */ +#define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY) + +/* Return true if index X is a UNIQUE index */ +#define IsUniqueIndex(X) ((X)->onError!=OE_None) + +/* The Index.aiColumn[] values are normally positive integer. But +** there are some negative values that have special meaning: +*/ +#define XN_ROWID (-1) /* Indexed column is the rowid */ +#define XN_EXPR (-2) /* Indexed column is an expression */ + +/* +** Each sample stored in the sqlite_stat4 table is represented in memory +** using a structure of this type. See documentation at the top of the +** analyze.c source file for additional information. +*/ +struct IndexSample { + void *p; /* Pointer to sampled record */ + int n; /* Size of record in bytes */ + tRowcnt *anEq; /* Est. number of rows where the key equals this sample */ + tRowcnt *anLt; /* Est. number of rows where key is less than this sample */ + tRowcnt *anDLt; /* Est. number of distinct keys less than this sample */ +}; + +/* +** Possible values to use within the flags argument to sqlite3GetToken(). +*/ +#define SQLITE_TOKEN_QUOTED 0x1 /* Token is a quoted identifier. */ +#define SQLITE_TOKEN_KEYWORD 0x2 /* Token is a keyword. */ + +/* +** Each token coming out of the lexer is an instance of +** this structure. Tokens are also used as part of an expression. +** +** The memory that "z" points to is owned by other objects. Take care +** that the owner of the "z" string does not deallocate the string before +** the Token goes out of scope! Very often, the "z" points to some place +** in the middle of the Parse.zSql text. But it might also point to a +** static string. +*/ +struct Token { + const char *z; /* Text of the token. Not NULL-terminated! */ + unsigned int n; /* Number of characters in this token */ +}; + +/* +** An instance of this structure contains information needed to generate +** code for a SELECT that contains aggregate functions. +** +** If Expr.op==TK_AGG_COLUMN or TK_AGG_FUNCTION then Expr.pAggInfo is a +** pointer to this structure. The Expr.iAgg field is the index in +** AggInfo.aCol[] or AggInfo.aFunc[] of information needed to generate +** code for that node. +** +** AggInfo.pGroupBy and AggInfo.aFunc.pExpr point to fields within the +** original Select structure that describes the SELECT statement. These +** fields do not need to be freed when deallocating the AggInfo structure. +*/ +struct AggInfo { + u8 directMode; /* Direct rendering mode means take data directly + ** from source tables rather than from accumulators */ + u8 useSortingIdx; /* In direct mode, reference the sorting index rather + ** than the source table */ + int sortingIdx; /* Cursor number of the sorting index */ + int sortingIdxPTab; /* Cursor number of pseudo-table */ + int nSortingColumn; /* Number of columns in the sorting index */ + int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */ + ExprList *pGroupBy; /* The group by clause */ + struct AggInfo_col { /* For each column used in source tables */ + Table *pTab; /* Source table */ + Expr *pCExpr; /* The original expression */ + int iTable; /* Cursor number of the source table */ + int iMem; /* Memory location that acts as accumulator */ + i16 iColumn; /* Column number within the source table */ + i16 iSorterColumn; /* Column number in the sorting index */ + } *aCol; + int nColumn; /* Number of used entries in aCol[] */ + int nAccumulator; /* Number of columns that show through to the output. + ** Additional columns are used only as parameters to + ** aggregate functions */ + struct AggInfo_func { /* For each aggregate function */ + Expr *pFExpr; /* Expression encoding the function */ + FuncDef *pFunc; /* The aggregate function implementation */ + int iMem; /* Memory location that acts as accumulator */ + int iDistinct; /* Ephemeral table used to enforce DISTINCT */ + int iDistAddr; /* Address of OP_OpenEphemeral */ + } *aFunc; + int nFunc; /* Number of entries in aFunc[] */ + u32 selId; /* Select to which this AggInfo belongs */ +}; + +/* +** The datatype ynVar is a signed integer, either 16-bit or 32-bit. +** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater +** than 32767 we have to make it 32-bit. 16-bit is preferred because +** it uses less memory in the Expr object, which is a big memory user +** in systems with lots of prepared statements. And few applications +** need more than about 10 or 20 variables. But some extreme users want +** to have prepared statements with over 32766 variables, and for them +** the option is available (at compile-time). +*/ +#if SQLITE_MAX_VARIABLE_NUMBER<32767 +typedef i16 ynVar; +#else +typedef int ynVar; +#endif + +/* +** Each node of an expression in the parse tree is an instance +** of this structure. +** +** Expr.op is the opcode. The integer parser token codes are reused +** as opcodes here. For example, the parser defines TK_GE to be an integer +** code representing the ">=" operator. This same integer code is reused +** to represent the greater-than-or-equal-to operator in the expression +** tree. +** +** If the expression is an SQL literal (TK_INTEGER, TK_FLOAT, TK_BLOB, +** or TK_STRING), then Expr.token contains the text of the SQL literal. If +** the expression is a variable (TK_VARIABLE), then Expr.token contains the +** variable name. Finally, if the expression is an SQL function (TK_FUNCTION), +** then Expr.token contains the name of the function. +** +** Expr.pRight and Expr.pLeft are the left and right subexpressions of a +** binary operator. Either or both may be NULL. +** +** Expr.x.pList is a list of arguments if the expression is an SQL function, +** a CASE expression or an IN expression of the form " IN (, ...)". +** Expr.x.pSelect is used if the expression is a sub-select or an expression of +** the form " IN (SELECT ...)". If the EP_xIsSelect bit is set in the +** Expr.flags mask, then Expr.x.pSelect is valid. Otherwise, Expr.x.pList is +** valid. +** +** An expression of the form ID or ID.ID refers to a column in a table. +** For such expressions, Expr.op is set to TK_COLUMN and Expr.iTable is +** the integer cursor number of a VDBE cursor pointing to that table and +** Expr.iColumn is the column number for the specific column. If the +** expression is used as a result in an aggregate SELECT, then the +** value is also stored in the Expr.iAgg column in the aggregate so that +** it can be accessed after all aggregates are computed. +** +** If the expression is an unbound variable marker (a question mark +** character '?' in the original SQL) then the Expr.iTable holds the index +** number for that variable. +** +** If the expression is a subquery then Expr.iColumn holds an integer +** register number containing the result of the subquery. If the +** subquery gives a constant result, then iTable is -1. If the subquery +** gives a different answer at different times during statement processing +** then iTable is the address of a subroutine that computes the subquery. +** +** If the Expr is of type OP_Column, and the table it is selecting from +** is a disk table or the "old.*" pseudo-table, then pTab points to the +** corresponding table definition. +** +** ALLOCATION NOTES: +** +** Expr objects can use a lot of memory space in database schema. To +** help reduce memory requirements, sometimes an Expr object will be +** truncated. And to reduce the number of memory allocations, sometimes +** two or more Expr objects will be stored in a single memory allocation, +** together with Expr.zToken strings. +** +** If the EP_Reduced and EP_TokenOnly flags are set when +** an Expr object is truncated. When EP_Reduced is set, then all +** the child Expr objects in the Expr.pLeft and Expr.pRight subtrees +** are contained within the same memory allocation. Note, however, that +** the subtrees in Expr.x.pList or Expr.x.pSelect are always separately +** allocated, regardless of whether or not EP_Reduced is set. +*/ +struct Expr { + u8 op; /* Operation performed by this node */ + char affExpr; /* affinity, or RAISE type */ + u8 op2; /* TK_REGISTER/TK_TRUTH: original value of Expr.op + ** TK_COLUMN: the value of p5 for OP_Column + ** TK_AGG_FUNCTION: nesting depth + ** TK_FUNCTION: NC_SelfRef flag if needs OP_PureFunc */ +#ifdef SQLITE_DEBUG + u8 vvaFlags; /* Verification flags. */ +#endif + u32 flags; /* Various flags. EP_* See below */ + union { + char *zToken; /* Token value. Zero terminated and dequoted */ + int iValue; /* Non-negative integer value if EP_IntValue */ + } u; + + /* If the EP_TokenOnly flag is set in the Expr.flags mask, then no + ** space is allocated for the fields below this point. An attempt to + ** access them will result in a segfault or malfunction. + *********************************************************************/ + + Expr *pLeft; /* Left subnode */ + Expr *pRight; /* Right subnode */ + union { + ExprList *pList; /* op = IN, EXISTS, SELECT, CASE, FUNCTION, BETWEEN */ + Select *pSelect; /* EP_xIsSelect and op = IN, EXISTS, SELECT */ + } x; + + /* If the EP_Reduced flag is set in the Expr.flags mask, then no + ** space is allocated for the fields below this point. An attempt to + ** access them will result in a segfault or malfunction. + *********************************************************************/ + +#if SQLITE_MAX_EXPR_DEPTH>0 + int nHeight; /* Height of the tree headed by this node */ +#endif + int iTable; /* TK_COLUMN: cursor number of table holding column + ** TK_REGISTER: register number + ** TK_TRIGGER: 1 -> new, 0 -> old + ** EP_Unlikely: 134217728 times likelihood + ** TK_IN: ephemerial table holding RHS + ** TK_SELECT_COLUMN: Number of columns on the LHS + ** TK_SELECT: 1st register of result vector */ + ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. + ** TK_VARIABLE: variable number (always >= 1). + ** TK_SELECT_COLUMN: column of the result vector */ + i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ + int iRightJoinTable; /* If EP_FromJoin, the right table of the join */ + AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ + union { + Table *pTab; /* TK_COLUMN: Table containing column. Can be NULL + ** for a column of an index on an expression */ + Window *pWin; /* EP_WinFunc: Window/Filter defn for a function */ + struct { /* TK_IN, TK_SELECT, and TK_EXISTS */ + int iAddr; /* Subroutine entry address */ + int regReturn; /* Register used to hold return address */ + } sub; + } y; +}; + +/* +** The following are the meanings of bits in the Expr.flags field. +** Value restrictions: +** +** EP_Agg == NC_HasAgg == SF_HasAgg +** EP_Win == NC_HasWin +*/ +#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ +#define EP_Distinct 0x000002 /* Aggregate function with DISTINCT keyword */ +#define EP_HasFunc 0x000004 /* Contains one or more functions of any kind */ +#define EP_FixedCol 0x000008 /* TK_Column with a known fixed value */ +#define EP_Agg 0x000010 /* Contains one or more aggregate functions */ +#define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ +#define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ +#define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ +#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ +#define EP_Commuted 0x000200 /* Comparison operator has been commuted */ +#define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ +#define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ +#define EP_Skip 0x001000 /* Operator does not contribute to affinity */ +#define EP_Reduced 0x002000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ +#define EP_TokenOnly 0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ +#define EP_Win 0x008000 /* Contains window functions */ +#define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ +#define EP_IfNullRow 0x020000 /* The TK_IF_NULL_ROW opcode */ +#define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ +#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ +#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ +#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ + /* 0x400000 // Available */ +#define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ +#define EP_WinFunc 0x1000000 /* TK_FUNCTION with Expr.y.pWin set */ +#define EP_Subrtn 0x2000000 /* Uses Expr.y.sub. TK_IN, _SELECT, or _EXISTS */ +#define EP_Quoted 0x4000000 /* TK_ID was originally quoted */ +#define EP_Static 0x8000000 /* Held in memory not obtained from malloc() */ +#define EP_IsTrue 0x10000000 /* Always has boolean value of TRUE */ +#define EP_IsFalse 0x20000000 /* Always has boolean value of FALSE */ +#define EP_FromDDL 0x40000000 /* Originates from sqlite_schema */ + /* 0x80000000 // Available */ + +/* +** The EP_Propagate mask is a set of properties that automatically propagate +** upwards into parent nodes. +*/ +#define EP_Propagate (EP_Collate|EP_Subquery|EP_HasFunc) + +/* +** These macros can be used to test, set, or clear bits in the +** Expr.flags field. +*/ +#define ExprHasProperty(E,P) (((E)->flags&(P))!=0) +#define ExprHasAllProperty(E,P) (((E)->flags&(P))==(P)) +#define ExprSetProperty(E,P) (E)->flags|=(P) +#define ExprClearProperty(E,P) (E)->flags&=~(P) +#define ExprAlwaysTrue(E) (((E)->flags&(EP_FromJoin|EP_IsTrue))==EP_IsTrue) +#define ExprAlwaysFalse(E) (((E)->flags&(EP_FromJoin|EP_IsFalse))==EP_IsFalse) + + +/* Flags for use with Expr.vvaFlags +*/ +#define EP_NoReduce 0x01 /* Cannot EXPRDUP_REDUCE this Expr */ +#define EP_Immutable 0x02 /* Do not change this Expr node */ + +/* The ExprSetVVAProperty() macro is used for Verification, Validation, +** and Accreditation only. It works like ExprSetProperty() during VVA +** processes but is a no-op for delivery. +*/ +#ifdef SQLITE_DEBUG +# define ExprSetVVAProperty(E,P) (E)->vvaFlags|=(P) +# define ExprHasVVAProperty(E,P) (((E)->vvaFlags&(P))!=0) +# define ExprClearVVAProperties(E) (E)->vvaFlags = 0 +#else +# define ExprSetVVAProperty(E,P) +# define ExprHasVVAProperty(E,P) 0 +# define ExprClearVVAProperties(E) +#endif + +/* +** Macros to determine the number of bytes required by a normal Expr +** struct, an Expr struct with the EP_Reduced flag set in Expr.flags +** and an Expr struct with the EP_TokenOnly flag set. +*/ +#define EXPR_FULLSIZE sizeof(Expr) /* Full size */ +#define EXPR_REDUCEDSIZE offsetof(Expr,iTable) /* Common features */ +#define EXPR_TOKENONLYSIZE offsetof(Expr,pLeft) /* Fewer features */ + +/* +** Flags passed to the sqlite3ExprDup() function. See the header comment +** above sqlite3ExprDup() for details. +*/ +#define EXPRDUP_REDUCE 0x0001 /* Used reduced-size Expr nodes */ + +/* +** True if the expression passed as an argument was a function with +** an OVER() clause (a window function). +*/ +#ifdef SQLITE_OMIT_WINDOWFUNC +# define IsWindowFunc(p) 0 +#else +# define IsWindowFunc(p) ( \ + ExprHasProperty((p), EP_WinFunc) && p->y.pWin->eFrmType!=TK_FILTER \ + ) +#endif + +/* +** A list of expressions. Each expression may optionally have a +** name. An expr/name combination can be used in several ways, such +** as the list of "expr AS ID" fields following a "SELECT" or in the +** list of "ID = expr" items in an UPDATE. A list of expressions can +** also be used as the argument to a function, in which case the a.zName +** field is not used. +** +** In order to try to keep memory usage down, the Expr.a.zEName field +** is used for multiple purposes: +** +** eEName Usage +** ---------- ------------------------- +** ENAME_NAME (1) the AS of result set column +** (2) COLUMN= of an UPDATE +** +** ENAME_TAB DB.TABLE.NAME used to resolve names +** of subqueries +** +** ENAME_SPAN Text of the original result set +** expression. +*/ +struct ExprList { + int nExpr; /* Number of expressions on the list */ + int nAlloc; /* Number of a[] slots allocated */ + struct ExprList_item { /* For each expression in the list */ + Expr *pExpr; /* The parse tree for this expression */ + char *zEName; /* Token associated with this expression */ + u8 sortFlags; /* Mask of KEYINFO_ORDER_* flags */ + unsigned eEName :2; /* Meaning of zEName */ + unsigned done :1; /* A flag to indicate when processing is finished */ + unsigned reusable :1; /* Constant expression is reusable */ + unsigned bSorterRef :1; /* Defer evaluation until after sorting */ + unsigned bNulls: 1; /* True if explicit "NULLS FIRST/LAST" */ + union { + struct { + u16 iOrderByCol; /* For ORDER BY, column number in result set */ + u16 iAlias; /* Index into Parse.aAlias[] for zName */ + } x; + int iConstExprReg; /* Register in which Expr value is cached */ + } u; + } a[1]; /* One slot for each expression in the list */ +}; + +/* +** Allowed values for Expr.a.eEName +*/ +#define ENAME_NAME 0 /* The AS clause of a result set */ +#define ENAME_SPAN 1 /* Complete text of the result set expression */ +#define ENAME_TAB 2 /* "DB.TABLE.NAME" for the result set */ + +/* +** An instance of this structure can hold a simple list of identifiers, +** such as the list "a,b,c" in the following statements: +** +** INSERT INTO t(a,b,c) VALUES ...; +** CREATE INDEX idx ON t(a,b,c); +** CREATE TRIGGER trig BEFORE UPDATE ON t(a,b,c) ...; +** +** The IdList.a.idx field is used when the IdList represents the list of +** column names after a table name in an INSERT statement. In the statement +** +** INSERT INTO t(a,b,c) ... +** +** If "a" is the k-th column of table "t", then IdList.a[0].idx==k. +*/ +struct IdList { + struct IdList_item { + char *zName; /* Name of the identifier */ + int idx; /* Index in some Table.aCol[] of a column named zName */ + } *a; + int nId; /* Number of identifiers on the list */ +}; + +/* +** The SrcItem object represents a single term in the FROM clause of a query. +** The SrcList object is mostly an array of SrcItems. +*/ +struct SrcItem { + Schema *pSchema; /* Schema to which this item is fixed */ + char *zDatabase; /* Name of database holding this table */ + char *zName; /* Name of the table */ + char *zAlias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ + Table *pTab; /* An SQL table corresponding to zName */ + Select *pSelect; /* A SELECT statement used in place of a table name */ + int addrFillSub; /* Address of subroutine to manifest a subquery */ + int regReturn; /* Register holding return address of addrFillSub */ + int regResult; /* Registers holding results of a co-routine */ + struct { + u8 jointype; /* Type of join between this table and the previous */ + unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ + unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ + unsigned isTabFunc :1; /* True if table-valued-function syntax */ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ + unsigned fromDDL :1; /* Comes from sqlite_schema */ + unsigned isCte :1; /* This is a CTE */ + unsigned notCte :1; /* This item may not match a CTE */ + } fg; + int iCursor; /* The VDBE cursor number used to access this table */ + Expr *pOn; /* The ON clause of a join */ + IdList *pUsing; /* The USING clause of a join */ + Bitmask colUsed; /* Bit N (1<" clause */ + ExprList *pFuncArg; /* Arguments to table-valued-function */ + } u1; + union { + Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ + CteUse *pCteUse; /* CTE Usage info info fg.isCte is true */ + } u2; +}; + +/* +** The following structure describes the FROM clause of a SELECT statement. +** Each table or subquery in the FROM clause is a separate element of +** the SrcList.a[] array. +** +** With the addition of multiple database support, the following structure +** can also be used to describe a particular table such as the table that +** is modified by an INSERT, DELETE, or UPDATE statement. In standard SQL, +** such a table must be a simple name: ID. But in SQLite, the table can +** now be identified by a database name, a dot, then the table name: ID.ID. +** +** The jointype starts out showing the join type between the current table +** and the next table on the list. The parser builds the list this way. +** But sqlite3SrcListShiftJoinType() later shifts the jointypes so that each +** jointype expresses the join between the table and the previous table. +** +** In the colUsed field, the high-order bit (bit 63) is set if the table +** contains more than 63 columns and the 64-th or later column is used. +*/ +struct SrcList { + int nSrc; /* Number of tables or subqueries in the FROM clause */ + u32 nAlloc; /* Number of entries allocated in a[] below */ + SrcItem a[1]; /* One entry for each identifier on the list */ +}; + +/* +** Permitted values of the SrcList.a.jointype field +*/ +#define JT_INNER 0x0001 /* Any kind of inner or cross join */ +#define JT_CROSS 0x0002 /* Explicit use of the CROSS keyword */ +#define JT_NATURAL 0x0004 /* True for a "natural" join */ +#define JT_LEFT 0x0008 /* Left outer join */ +#define JT_RIGHT 0x0010 /* Right outer join */ +#define JT_OUTER 0x0020 /* The "OUTER" keyword is present */ +#define JT_ERROR 0x0040 /* unknown or unsupported join type */ + + +/* +** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin() +** and the WhereInfo.wctrlFlags member. +** +** Value constraints (enforced via assert()): +** WHERE_USE_LIMIT == SF_FixedLimit +*/ +#define WHERE_ORDERBY_NORMAL 0x0000 /* No-op */ +#define WHERE_ORDERBY_MIN 0x0001 /* ORDER BY processing for min() func */ +#define WHERE_ORDERBY_MAX 0x0002 /* ORDER BY processing for max() func */ +#define WHERE_ONEPASS_DESIRED 0x0004 /* Want to do one-pass UPDATE/DELETE */ +#define WHERE_ONEPASS_MULTIROW 0x0008 /* ONEPASS is ok with multiple rows */ +#define WHERE_DUPLICATES_OK 0x0010 /* Ok to return a row more than once */ +#define WHERE_OR_SUBCLAUSE 0x0020 /* Processing a sub-WHERE as part of + ** the OR optimization */ +#define WHERE_GROUPBY 0x0040 /* pOrderBy is really a GROUP BY */ +#define WHERE_DISTINCTBY 0x0080 /* pOrderby is really a DISTINCT clause */ +#define WHERE_WANT_DISTINCT 0x0100 /* All output needs to be distinct */ +#define WHERE_SORTBYGROUP 0x0200 /* Support sqlite3WhereIsSorted() */ +#define WHERE_AGG_DISTINCT 0x0400 /* Query is "SELECT agg(DISTINCT ...)" */ +#define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ + /* 0x1000 not currently used */ + /* 0x2000 not currently used */ +#define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ + /* 0x8000 not currently used */ + +/* Allowed return values from sqlite3WhereIsDistinct() +*/ +#define WHERE_DISTINCT_NOOP 0 /* DISTINCT keyword not used */ +#define WHERE_DISTINCT_UNIQUE 1 /* No duplicates */ +#define WHERE_DISTINCT_ORDERED 2 /* All duplicates are adjacent */ +#define WHERE_DISTINCT_UNORDERED 3 /* Duplicates are scattered */ + +/* +** A NameContext defines a context in which to resolve table and column +** names. The context consists of a list of tables (the pSrcList) field and +** a list of named expression (pEList). The named expression list may +** be NULL. The pSrc corresponds to the FROM clause of a SELECT or +** to the table being operated on by INSERT, UPDATE, or DELETE. The +** pEList corresponds to the result set of a SELECT and is NULL for +** other statements. +** +** NameContexts can be nested. When resolving names, the inner-most +** context is searched first. If no match is found, the next outer +** context is checked. If there is still no match, the next context +** is checked. This process continues until either a match is found +** or all contexts are check. When a match is found, the nRef member of +** the context containing the match is incremented. +** +** Each subquery gets a new NameContext. The pNext field points to the +** NameContext in the parent query. Thus the process of scanning the +** NameContext list corresponds to searching through successively outer +** subqueries looking for a match. +*/ +struct NameContext { + Parse *pParse; /* The parser */ + SrcList *pSrcList; /* One or more tables used to resolve names */ + union { + ExprList *pEList; /* Optional list of result-set columns */ + AggInfo *pAggInfo; /* Information about aggregates at this level */ + Upsert *pUpsert; /* ON CONFLICT clause information from an upsert */ + int iBaseReg; /* For TK_REGISTER when parsing RETURNING */ + } uNC; + NameContext *pNext; /* Next outer name context. NULL for outermost */ + int nRef; /* Number of names resolved by this context */ + int nNcErr; /* Number of errors encountered while resolving names */ + int ncFlags; /* Zero or more NC_* flags defined below */ + Select *pWinSelect; /* SELECT statement for any window functions */ +}; + +/* +** Allowed values for the NameContext, ncFlags field. +** +** Value constraints (all checked via assert()): +** NC_HasAgg == SF_HasAgg == EP_Agg +** NC_MinMaxAgg == SF_MinMaxAgg == SQLITE_FUNC_MINMAX +** NC_OrderAgg == SF_OrderByReqd == SQLITE_FUNC_ANYORDER +** NC_HasWin == EP_Win +** +*/ +#define NC_AllowAgg 0x000001 /* Aggregate functions are allowed here */ +#define NC_PartIdx 0x000002 /* True if resolving a partial index WHERE */ +#define NC_IsCheck 0x000004 /* True if resolving a CHECK constraint */ +#define NC_GenCol 0x000008 /* True for a GENERATED ALWAYS AS clause */ +#define NC_HasAgg 0x000010 /* One or more aggregate functions seen */ +#define NC_IdxExpr 0x000020 /* True if resolving columns of CREATE INDEX */ +#define NC_SelfRef 0x00002e /* Combo: PartIdx, isCheck, GenCol, and IdxExpr */ +#define NC_VarSelect 0x000040 /* A correlated subquery has been seen */ +#define NC_UEList 0x000080 /* True if uNC.pEList is used */ +#define NC_UAggInfo 0x000100 /* True if uNC.pAggInfo is used */ +#define NC_UUpsert 0x000200 /* True if uNC.pUpsert is used */ +#define NC_UBaseReg 0x000400 /* True if uNC.iBaseReg is used */ +#define NC_MinMaxAgg 0x001000 /* min/max aggregates seen. See note above */ +#define NC_Complex 0x002000 /* True if a function or subquery seen */ +#define NC_AllowWin 0x004000 /* Window functions are allowed here */ +#define NC_HasWin 0x008000 /* One or more window functions seen */ +#define NC_IsDDL 0x010000 /* Resolving names in a CREATE statement */ +#define NC_InAggFunc 0x020000 /* True if analyzing arguments to an agg func */ +#define NC_FromDDL 0x040000 /* SQL text comes from sqlite_schema */ +#define NC_NoSelect 0x080000 /* Do not descend into sub-selects */ +#define NC_OrderAgg 0x8000000 /* Has an aggregate other than count/min/max */ + +/* +** An instance of the following object describes a single ON CONFLICT +** clause in an upsert. +** +** The pUpsertTarget field is only set if the ON CONFLICT clause includes +** conflict-target clause. (In "ON CONFLICT(a,b)" the "(a,b)" is the +** conflict-target clause.) The pUpsertTargetWhere is the optional +** WHERE clause used to identify partial unique indexes. +** +** pUpsertSet is the list of column=expr terms of the UPDATE statement. +** The pUpsertSet field is NULL for a ON CONFLICT DO NOTHING. The +** pUpsertWhere is the WHERE clause for the UPDATE and is NULL if the +** WHERE clause is omitted. +*/ +struct Upsert { + ExprList *pUpsertTarget; /* Optional description of conflict target */ + Expr *pUpsertTargetWhere; /* WHERE clause for partial index targets */ + ExprList *pUpsertSet; /* The SET clause from an ON CONFLICT UPDATE */ + Expr *pUpsertWhere; /* WHERE clause for the ON CONFLICT UPDATE */ + Upsert *pNextUpsert; /* Next ON CONFLICT clause in the list */ + u8 isDoUpdate; /* True for DO UPDATE. False for DO NOTHING */ + /* Above this point is the parse tree for the ON CONFLICT clauses. + ** The next group of fields stores intermediate data. */ + void *pToFree; /* Free memory when deleting the Upsert object */ + /* All fields above are owned by the Upsert object and must be freed + ** when the Upsert is destroyed. The fields below are used to transfer + ** information from the INSERT processing down into the UPDATE processing + ** while generating code. The fields below are owned by the INSERT + ** statement and will be freed by INSERT processing. */ + Index *pUpsertIdx; /* UNIQUE constraint specified by pUpsertTarget */ + SrcList *pUpsertSrc; /* Table to be updated */ + int regData; /* First register holding array of VALUES */ + int iDataCur; /* Index of the data cursor */ + int iIdxCur; /* Index of the first index cursor */ +}; + +/* +** An instance of the following structure contains all information +** needed to generate code for a single SELECT statement. +** +** See the header comment on the computeLimitRegisters() routine for a +** detailed description of the meaning of the iLimit and iOffset fields. +** +** addrOpenEphm[] entries contain the address of OP_OpenEphemeral opcodes. +** These addresses must be stored so that we can go back and fill in +** the P4_KEYINFO and P2 parameters later. Neither the KeyInfo nor +** the number of columns in P2 can be computed at the same time +** as the OP_OpenEphm instruction is coded because not +** enough information about the compound query is known at that point. +** The KeyInfo for addrOpenTran[0] and [1] contains collating sequences +** for the result set. The KeyInfo for addrOpenEphm[2] contains collating +** sequences for the ORDER BY clause. +*/ +struct Select { + u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ + LogEst nSelectRow; /* Estimated number of result rows */ + u32 selFlags; /* Various SF_* values */ + int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ + u32 selId; /* Unique identifier number for this SELECT */ + int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ + ExprList *pEList; /* The fields of the result */ + SrcList *pSrc; /* The FROM clause */ + Expr *pWhere; /* The WHERE clause */ + ExprList *pGroupBy; /* The GROUP BY clause */ + Expr *pHaving; /* The HAVING clause */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Select *pPrior; /* Prior select in a compound select statement */ + Select *pNext; /* Next select to the left in a compound */ + Expr *pLimit; /* LIMIT expression. NULL means not used. */ + With *pWith; /* WITH clause attached to this select. Or NULL. */ +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin; /* List of window functions */ + Window *pWinDefn; /* List of named window definitions */ +#endif +}; + +/* +** Allowed values for Select.selFlags. The "SF" prefix stands for +** "Select Flag". +** +** Value constraints (all checked via assert()) +** SF_HasAgg == NC_HasAgg +** SF_MinMaxAgg == NC_MinMaxAgg == SQLITE_FUNC_MINMAX +** SF_OrderByReqd == NC_OrderAgg == SQLITE_FUNC_ANYORDER +** SF_FixedLimit == WHERE_USE_LIMIT +*/ +#define SF_Distinct 0x0000001 /* Output should be DISTINCT */ +#define SF_All 0x0000002 /* Includes the ALL keyword */ +#define SF_Resolved 0x0000004 /* Identifiers have been resolved */ +#define SF_Aggregate 0x0000008 /* Contains agg functions or a GROUP BY */ +#define SF_HasAgg 0x0000010 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x0000020 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x0000040 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x0000080 /* FROM subqueries have Table metadata */ +#define SF_Compound 0x0000100 /* Part of a compound query */ +#define SF_Values 0x0000200 /* Synthesized from VALUES clause */ +#define SF_MultiValue 0x0000400 /* Single VALUES term with multiple rows */ +#define SF_NestedFrom 0x0000800 /* Part of a parenthesized FROM clause */ +#define SF_MinMaxAgg 0x0001000 /* Aggregate containing min() or max() */ +#define SF_Recursive 0x0002000 /* The recursive part of a recursive CTE */ +#define SF_FixedLimit 0x0004000 /* nSelectRow set by a constant LIMIT */ +#define SF_MaybeConvert 0x0008000 /* Need convertCompoundSelectToSubquery() */ +#define SF_Converted 0x0010000 /* By convertCompoundSelectToSubquery() */ +#define SF_IncludeHidden 0x0020000 /* Include hidden columns in output */ +#define SF_ComplexResult 0x0040000 /* Result contains subquery or function */ +#define SF_WhereBegin 0x0080000 /* Really a WhereBegin() call. Debug Only */ +#define SF_WinRewrite 0x0100000 /* Window function rewrite accomplished */ +#define SF_View 0x0200000 /* SELECT statement is a view */ +#define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ +#define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ +#define SF_PushDown 0x1000000 /* SELECT has be modified by push-down opt */ +#define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ +#define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ +#define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ + +/* +** The results of a SELECT can be distributed in several ways, as defined +** by one of the following macros. The "SRT" prefix means "SELECT Result +** Type". +** +** SRT_Union Store results as a key in a temporary index +** identified by pDest->iSDParm. +** +** SRT_Except Remove results from the temporary index pDest->iSDParm. +** +** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result +** set is not empty. +** +** SRT_Discard Throw the results away. This is used by SELECT +** statements within triggers whose only purpose is +** the side-effects of functions. +** +** SRT_Output Generate a row of output (using the OP_ResultRow +** opcode) for each row in the result set. +** +** SRT_Mem Only valid if the result is a single column. +** Store the first column of the first result row +** in register pDest->iSDParm then abandon the rest +** of the query. This destination implies "LIMIT 1". +** +** SRT_Set The result must be a single column. Store each +** row of result as the key in table pDest->iSDParm. +** Apply the affinity pDest->affSdst before storing +** results. Used to implement "IN (SELECT ...)". +** +** SRT_EphemTab Create an temporary table pDest->iSDParm and store +** the result there. The cursor is left open after +** returning. This is like SRT_Table except that +** this destination uses OP_OpenEphemeral to create +** the table first. +** +** SRT_Coroutine Generate a co-routine that returns a new row of +** results each time it is invoked. The entry point +** of the co-routine is stored in register pDest->iSDParm +** and the result row is stored in pDest->nDest registers +** starting with pDest->iSdst. +** +** SRT_Table Store results in temporary table pDest->iSDParm. +** SRT_Fifo This is like SRT_EphemTab except that the table +** is assumed to already be open. SRT_Fifo has +** the additional property of being able to ignore +** the ORDER BY clause. +** +** SRT_DistFifo Store results in a temporary table pDest->iSDParm. +** But also use temporary table pDest->iSDParm+1 as +** a record of all prior results and ignore any duplicate +** rows. Name means: "Distinct Fifo". +** +** SRT_Queue Store results in priority queue pDest->iSDParm (really +** an index). Append a sequence number so that all entries +** are distinct. +** +** SRT_DistQueue Store results in priority queue pDest->iSDParm only if +** the same record has never been stored before. The +** index at pDest->iSDParm+1 hold all prior stores. +** +** SRT_Upfrom Store results in the temporary table already opened by +** pDest->iSDParm. If (pDest->iSDParm<0), then the temp +** table is an intkey table - in this case the first +** column returned by the SELECT is used as the integer +** key. If (pDest->iSDParm>0), then the table is an index +** table. (pDest->iSDParm) is the number of key columns in +** each index record in this case. +*/ +#define SRT_Union 1 /* Store result as keys in an index */ +#define SRT_Except 2 /* Remove result from a UNION index */ +#define SRT_Exists 3 /* Store 1 if the result is not empty */ +#define SRT_Discard 4 /* Do not save the results anywhere */ +#define SRT_DistFifo 5 /* Like SRT_Fifo, but unique results only */ +#define SRT_DistQueue 6 /* Like SRT_Queue, but unique results only */ + +/* The DISTINCT clause is ignored for all of the above. Not that +** IgnorableDistinct() implies IgnorableOrderby() */ +#define IgnorableDistinct(X) ((X->eDest)<=SRT_DistQueue) + +#define SRT_Queue 7 /* Store result in an queue */ +#define SRT_Fifo 8 /* Store result as data with an automatic rowid */ + +/* The ORDER BY clause is ignored for all of the above */ +#define IgnorableOrderby(X) ((X->eDest)<=SRT_Fifo) + +#define SRT_Output 9 /* Output each row of result */ +#define SRT_Mem 10 /* Store result in a memory cell */ +#define SRT_Set 11 /* Store results as keys in an index */ +#define SRT_EphemTab 12 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 13 /* Generate a single row of result */ +#define SRT_Table 14 /* Store result as data with an automatic rowid */ +#define SRT_Upfrom 15 /* Store result as data with rowid */ + +/* +** An instance of this object describes where to put of the results of +** a SELECT statement. +*/ +struct SelectDest { + u8 eDest; /* How to dispose of the results. One of SRT_* above. */ + int iSDParm; /* A parameter used by the eDest disposal method */ + int iSDParm2; /* A second parameter for the eDest disposal method */ + int iSdst; /* Base register where results are written */ + int nSdst; /* Number of registers allocated */ + char *zAffSdst; /* Affinity used when eDest==SRT_Set */ + ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ +}; + +/* +** During code generation of statements that do inserts into AUTOINCREMENT +** tables, the following information is attached to the Table.u.autoInc.p +** pointer of each autoincrement table to record some side information that +** the code generator needs. We have to keep per-table autoincrement +** information in case inserts are done within triggers. Triggers do not +** normally coordinate their activities, but we do need to coordinate the +** loading and saving of autoincrement information. +*/ +struct AutoincInfo { + AutoincInfo *pNext; /* Next info block in a list of them all */ + Table *pTab; /* Table this info block refers to */ + int iDb; /* Index in sqlite3.aDb[] of database holding pTab */ + int regCtr; /* Memory register holding the rowid counter */ +}; + +/* +** At least one instance of the following structure is created for each +** trigger that may be fired while parsing an INSERT, UPDATE or DELETE +** statement. All such objects are stored in the linked list headed at +** Parse.pTriggerPrg and deleted once statement compilation has been +** completed. +** +** A Vdbe sub-program that implements the body and WHEN clause of trigger +** TriggerPrg.pTrigger, assuming a default ON CONFLICT clause of +** TriggerPrg.orconf, is stored in the TriggerPrg.pProgram variable. +** The Parse.pTriggerPrg list never contains two entries with the same +** values for both pTrigger and orconf. +** +** The TriggerPrg.aColmask[0] variable is set to a mask of old.* columns +** accessed (or set to 0 for triggers fired as a result of INSERT +** statements). Similarly, the TriggerPrg.aColmask[1] variable is set to +** a mask of new.* columns used by the program. +*/ +struct TriggerPrg { + Trigger *pTrigger; /* Trigger this program was coded from */ + TriggerPrg *pNext; /* Next entry in Parse.pTriggerPrg list */ + SubProgram *pProgram; /* Program implementing pTrigger/orconf */ + int orconf; /* Default ON CONFLICT policy */ + u32 aColmask[2]; /* Masks of old.*, new.* columns accessed */ +}; + +/* +** The yDbMask datatype for the bitmask of all attached databases. +*/ +#if SQLITE_MAX_ATTACHED>30 + typedef unsigned char yDbMask[(SQLITE_MAX_ATTACHED+9)/8]; +# define DbMaskTest(M,I) (((M)[(I)/8]&(1<<((I)&7)))!=0) +# define DbMaskZero(M) memset((M),0,sizeof(M)) +# define DbMaskSet(M,I) (M)[(I)/8]|=(1<<((I)&7)) +# define DbMaskAllZero(M) sqlite3DbMaskAllZero(M) +# define DbMaskNonZero(M) (sqlite3DbMaskAllZero(M)==0) +#else + typedef unsigned int yDbMask; +# define DbMaskTest(M,I) (((M)&(((yDbMask)1)<<(I)))!=0) +# define DbMaskZero(M) (M)=0 +# define DbMaskSet(M,I) (M)|=(((yDbMask)1)<<(I)) +# define DbMaskAllZero(M) (M)==0 +# define DbMaskNonZero(M) (M)!=0 +#endif + +/* +** An instance of the ParseCleanup object specifies an operation that +** should be performed after parsing to deallocation resources obtained +** during the parse and which are no longer needed. +*/ +struct ParseCleanup { + ParseCleanup *pNext; /* Next cleanup task */ + void *pPtr; /* Pointer to object to deallocate */ + void (*xCleanup)(sqlite3*,void*); /* Deallocation routine */ +}; + +/* +** An SQL parser context. A copy of this structure is passed through +** the parser and down into all the parser action routine in order to +** carry around information that is global to the entire parse. +** +** The structure is divided into two parts. When the parser and code +** generate call themselves recursively, the first part of the structure +** is constant but the second part is reset at the beginning and end of +** each recursion. +** +** The nTableLock and aTableLock variables are only used if the shared-cache +** feature is enabled (if sqlite3Tsd()->useSharedData is true). They are +** used to store the set of table-locks required by the statement being +** compiled. Function sqlite3TableLock() is used to add entries to the +** list. +*/ +struct Parse { + sqlite3 *db; /* The main database structure */ + char *zErrMsg; /* An error message */ + Vdbe *pVdbe; /* An engine for executing database bytecode */ + int rc; /* Return code from execution */ + u8 colNamesSet; /* TRUE after OP_ColumnName has been issued to pVdbe */ + u8 checkSchema; /* Causes schema cookie check after an error */ + u8 nested; /* Number of nested calls to the parser/code generator */ + u8 nTempReg; /* Number of temporary registers in aTempReg[] */ + u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ + u8 mayAbort; /* True if statement may throw an ABORT exception */ + u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ + u8 okConstFactor; /* OK to factor out constants */ + u8 disableLookaside; /* Number of times lookaside has been disabled */ + u8 disableVtab; /* Disable all virtual tables for this parse */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ +#endif + int nRangeReg; /* Size of the temporary register block */ + int iRangeReg; /* First register in temporary register block */ + int nErr; /* Number of errors seen */ + int nTab; /* Number of previously allocated VDBE cursors */ + int nMem; /* Number of memory cells used so far */ + int szOpAlloc; /* Bytes of memory space allocated for Vdbe.aOp[] */ + int iSelfTab; /* Table associated with an index on expr, or negative + ** of the base register during check-constraint eval */ + int nLabel; /* The *negative* of the number of labels used */ + int nLabelAlloc; /* Number of slots in aLabel */ + int *aLabel; /* Space to hold the labels */ + ExprList *pConstExpr;/* Constant expressions */ + Token constraintName;/* Name of the constraint currently being parsed */ + yDbMask writeMask; /* Start a write transaction on these databases */ + yDbMask cookieMask; /* Bitmask of schema verified databases */ + int regRowid; /* Register holding rowid of CREATE TABLE entry */ + int regRoot; /* Register holding root page number for new objects */ + int nMaxArg; /* Max args passed to user function by sub-program */ + int nSelect; /* Number of SELECT stmts. Counter for Select.selId */ +#ifndef SQLITE_OMIT_SHARED_CACHE + int nTableLock; /* Number of locks in aTableLock */ + TableLock *aTableLock; /* Required table locks for shared-cache mode */ +#endif + AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ + Parse *pToplevel; /* Parse structure for main program (or NULL) */ + Table *pTriggerTab; /* Table triggers are being coded for */ + Parse *pParentParse; /* Parent parser if this parser is nested */ + union { + int addrCrTab; /* Address of OP_CreateBtree on CREATE TABLE */ + Returning *pReturning; /* The RETURNING clause */ + } u1; + u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ + u32 oldmask; /* Mask of old.* columns referenced */ + u32 newmask; /* Mask of new.* columns referenced */ + u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ + u8 bReturning; /* Coding a RETURNING trigger */ + u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ + u8 disableTriggers; /* True to disable triggers */ + + /************************************************************************** + ** Fields above must be initialized to zero. The fields that follow, + ** down to the beginning of the recursive section, do not need to be + ** initialized as they will be set before being used. The boundary is + ** determined by offsetof(Parse,aTempReg). + **************************************************************************/ + + int aTempReg[8]; /* Holding area for temporary registers */ + Token sNameToken; /* Token with unqualified schema object name */ + + /************************************************************************ + ** Above is constant between recursions. Below is reset before and after + ** each recursion. The boundary between these two regions is determined + ** using offsetof(Parse,sLastToken) so the sLastToken field must be the + ** first field in the recursive region. + ************************************************************************/ + + Token sLastToken; /* The last token parsed */ + ynVar nVar; /* Number of '?' variables seen in the SQL so far */ + u8 iPkSortOrder; /* ASC or DESC for INTEGER PRIMARY KEY */ + u8 explain; /* True if the EXPLAIN flag is found on the query */ + u8 eParseMode; /* PARSE_MODE_XXX constant */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int nVtabLock; /* Number of virtual tables to lock */ +#endif + int nHeight; /* Expression tree height of current sub-select */ +#ifndef SQLITE_OMIT_EXPLAIN + int addrExplain; /* Address of current OP_Explain opcode */ +#endif + VList *pVList; /* Mapping between variable names and numbers */ + Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ + const char *zTail; /* All SQL text past the last semicolon parsed */ + Table *pNewTable; /* A table being constructed by CREATE TABLE */ + Index *pNewIndex; /* An index being constructed by CREATE INDEX. + ** Also used to hold redundant UNIQUE constraints + ** during a RENAME COLUMN */ + Trigger *pNewTrigger; /* Trigger under construct by a CREATE TRIGGER */ + const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + Token sArg; /* Complete text of a module argument */ + Table **apVtabLock; /* Pointer to virtual tables needing locking */ +#endif + TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ + With *pWith; /* Current WITH clause, or NULL */ + ParseCleanup *pCleanup; /* List of cleanup operations to run after parse */ +#ifndef SQLITE_OMIT_ALTERTABLE + RenameToken *pRename; /* Tokens subject to renaming by ALTER TABLE */ +#endif +}; + +#define PARSE_MODE_NORMAL 0 +#define PARSE_MODE_DECLARE_VTAB 1 +#define PARSE_MODE_RENAME 2 +#define PARSE_MODE_UNMAP 3 + +/* +** Sizes and pointers of various parts of the Parse object. +*/ +#define PARSE_HDR_SZ offsetof(Parse,aTempReg) /* Recursive part w/o aColCache*/ +#define PARSE_RECURSE_SZ offsetof(Parse,sLastToken) /* Recursive part */ +#define PARSE_TAIL_SZ (sizeof(Parse)-PARSE_RECURSE_SZ) /* Non-recursive part */ +#define PARSE_TAIL(X) (((char*)(X))+PARSE_RECURSE_SZ) /* Pointer to tail */ + +/* +** Return true if currently inside an sqlite3_declare_vtab() call. +*/ +#ifdef SQLITE_OMIT_VIRTUALTABLE + #define IN_DECLARE_VTAB 0 +#else + #define IN_DECLARE_VTAB (pParse->eParseMode==PARSE_MODE_DECLARE_VTAB) +#endif + +#if defined(SQLITE_OMIT_ALTERTABLE) + #define IN_RENAME_OBJECT 0 +#else + #define IN_RENAME_OBJECT (pParse->eParseMode>=PARSE_MODE_RENAME) +#endif + +#if defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_OMIT_ALTERTABLE) + #define IN_SPECIAL_PARSE 0 +#else + #define IN_SPECIAL_PARSE (pParse->eParseMode!=PARSE_MODE_NORMAL) +#endif + +/* +** An instance of the following structure can be declared on a stack and used +** to save the Parse.zAuthContext value so that it can be restored later. +*/ +struct AuthContext { + const char *zAuthContext; /* Put saved Parse.zAuthContext here */ + Parse *pParse; /* The Parse structure */ +}; + +/* +** Bitfield flags for P5 value in various opcodes. +** +** Value constraints (enforced via assert()): +** OPFLAG_LENGTHARG == SQLITE_FUNC_LENGTH +** OPFLAG_TYPEOFARG == SQLITE_FUNC_TYPEOF +** OPFLAG_BULKCSR == BTREE_BULKLOAD +** OPFLAG_SEEKEQ == BTREE_SEEK_EQ +** OPFLAG_FORDELETE == BTREE_FORDELETE +** OPFLAG_SAVEPOSITION == BTREE_SAVEPOSITION +** OPFLAG_AUXDELETE == BTREE_AUXDELETE +*/ +#define OPFLAG_NCHANGE 0x01 /* OP_Insert: Set to update db->nChange */ + /* Also used in P2 (not P5) of OP_Delete */ +#define OPFLAG_NOCHNG 0x01 /* OP_VColumn nochange for UPDATE */ +#define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ +#define OPFLAG_LASTROWID 0x20 /* Set to update db->lastRowid */ +#define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ +#define OPFLAG_APPEND 0x08 /* This is likely to be an append */ +#define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ +#define OPFLAG_ISNOOP 0x40 /* OP_Delete does pre-update-hook only */ +#define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ +#define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ +#define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ +#define OPFLAG_SEEKEQ 0x02 /* OP_Open** cursor uses EQ seek only */ +#define OPFLAG_FORDELETE 0x08 /* OP_Open should use BTREE_FORDELETE */ +#define OPFLAG_P2ISREG 0x10 /* P2 to OP_Open** is a register number */ +#define OPFLAG_PERMUTE 0x01 /* OP_Compare: use the permutation */ +#define OPFLAG_SAVEPOSITION 0x02 /* OP_Delete/Insert: save cursor pos */ +#define OPFLAG_AUXDELETE 0x04 /* OP_Delete: index in a DELETE op */ +#define OPFLAG_NOCHNG_MAGIC 0x6d /* OP_MakeRecord: serialtype 10 is ok */ +#define OPFLAG_PREFORMAT 0x80 /* OP_Insert uses preformatted cell */ + +/* + * Each trigger present in the database schema is stored as an instance of + * struct Trigger. + * + * Pointers to instances of struct Trigger are stored in two ways. + * 1. In the "trigHash" hash table (part of the sqlite3* that represents the + * database). This allows Trigger structures to be retrieved by name. + * 2. All triggers associated with a single table form a linked list, using the + * pNext member of struct Trigger. A pointer to the first element of the + * linked list is stored as the "pTrigger" member of the associated + * struct Table. + * + * The "step_list" member points to the first element of a linked list + * containing the SQL statements specified as the trigger program. + */ +struct Trigger { + char *zName; /* The name of the trigger */ + char *table; /* The table or view to which the trigger applies */ + u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT */ + u8 tr_tm; /* One of TRIGGER_BEFORE, TRIGGER_AFTER */ + u8 bReturning; /* This trigger implements a RETURNING clause */ + Expr *pWhen; /* The WHEN clause of the expression (may be NULL) */ + IdList *pColumns; /* If this is an UPDATE OF trigger, + the is stored here */ + Schema *pSchema; /* Schema containing the trigger */ + Schema *pTabSchema; /* Schema containing the table */ + TriggerStep *step_list; /* Link list of trigger program steps */ + Trigger *pNext; /* Next trigger associated with the table */ +}; + +/* +** A trigger is either a BEFORE or an AFTER trigger. The following constants +** determine which. +** +** If there are multiple triggers, you might of some BEFORE and some AFTER. +** In that cases, the constants below can be ORed together. +*/ +#define TRIGGER_BEFORE 1 +#define TRIGGER_AFTER 2 + +/* + * An instance of struct TriggerStep is used to store a single SQL statement + * that is a part of a trigger-program. + * + * Instances of struct TriggerStep are stored in a singly linked list (linked + * using the "pNext" member) referenced by the "step_list" member of the + * associated struct Trigger instance. The first element of the linked list is + * the first step of the trigger-program. + * + * The "op" member indicates whether this is a "DELETE", "INSERT", "UPDATE" or + * "SELECT" statement. The meanings of the other members is determined by the + * value of "op" as follows: + * + * (op == TK_INSERT) + * orconf -> stores the ON CONFLICT algorithm + * pSelect -> If this is an INSERT INTO ... SELECT ... statement, then + * this stores a pointer to the SELECT statement. Otherwise NULL. + * zTarget -> Dequoted name of the table to insert into. + * pExprList -> If this is an INSERT INTO ... VALUES ... statement, then + * this stores values to be inserted. Otherwise NULL. + * pIdList -> If this is an INSERT INTO ... () VALUES ... + * statement, then this stores the column-names to be + * inserted into. + * + * (op == TK_DELETE) + * zTarget -> Dequoted name of the table to delete from. + * pWhere -> The WHERE clause of the DELETE statement if one is specified. + * Otherwise NULL. + * + * (op == TK_UPDATE) + * zTarget -> Dequoted name of the table to update. + * pWhere -> The WHERE clause of the UPDATE statement if one is specified. + * Otherwise NULL. + * pExprList -> A list of the columns to update and the expressions to update + * them to. See sqlite3Update() documentation of "pChanges" + * argument. + * + */ +struct TriggerStep { + u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT, + ** or TK_RETURNING */ + u8 orconf; /* OE_Rollback etc. */ + Trigger *pTrig; /* The trigger that this step is a part of */ + Select *pSelect; /* SELECT statement or RHS of INSERT INTO SELECT ... */ + char *zTarget; /* Target table for DELETE, UPDATE, INSERT */ + SrcList *pFrom; /* FROM clause for UPDATE statement (if any) */ + Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ + ExprList *pExprList; /* SET clause for UPDATE, or RETURNING clause */ + IdList *pIdList; /* Column names for INSERT */ + Upsert *pUpsert; /* Upsert clauses on an INSERT */ + char *zSpan; /* Original SQL text of this command */ + TriggerStep *pNext; /* Next in the link-list */ + TriggerStep *pLast; /* Last element in link-list. Valid for 1st elem only */ +}; + +/* +** Information about a RETURNING clause +*/ +struct Returning { + Parse *pParse; /* The parse that includes the RETURNING clause */ + ExprList *pReturnEL; /* List of expressions to return */ + Trigger retTrig; /* The transient trigger that implements RETURNING */ + TriggerStep retTStep; /* The trigger step */ + int iRetCur; /* Transient table holding RETURNING results */ + int nRetCol; /* Number of in pReturnEL after expansion */ + int iRetReg; /* Register array for holding a row of RETURNING */ +}; + +/* +** An objected used to accumulate the text of a string where we +** do not necessarily know how big the string will be in the end. +*/ +struct sqlite3_str { + sqlite3 *db; /* Optional database for lookaside. Can be NULL */ + char *zText; /* The string collected so far */ + u32 nAlloc; /* Amount of space allocated in zText */ + u32 mxAlloc; /* Maximum allowed allocation. 0 for no malloc usage */ + u32 nChar; /* Length of the string so far */ + u8 accError; /* SQLITE_NOMEM or SQLITE_TOOBIG */ + u8 printfFlags; /* SQLITE_PRINTF flags below */ +}; +#define SQLITE_PRINTF_INTERNAL 0x01 /* Internal-use-only converters allowed */ +#define SQLITE_PRINTF_SQLFUNC 0x02 /* SQL function arguments to VXPrintf */ +#define SQLITE_PRINTF_MALLOCED 0x04 /* True if xText is allocated space */ + +#define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) + + +/* +** A pointer to this structure is used to communicate information +** from sqlite3Init and OP_ParseSchema into the sqlite3InitCallback. +*/ +typedef struct { + sqlite3 *db; /* The database being initialized */ + char **pzErrMsg; /* Error message stored here */ + int iDb; /* 0 for main database. 1 for TEMP, 2.. for ATTACHed */ + int rc; /* Result code stored here */ + u32 mInitFlags; /* Flags controlling error messages */ + u32 nInitRow; /* Number of rows processed */ + Pgno mxPage; /* Maximum page number. 0 for no limit. */ +} InitData; + +/* +** Allowed values for mInitFlags +*/ +#define INITFLAG_AlterRename 0x0001 /* Reparse after a RENAME */ +#define INITFLAG_AlterDrop 0x0002 /* Reparse after a DROP COLUMN */ + +/* Tuning parameters are set using SQLITE_TESTCTRL_TUNE and are controlled +** on debug-builds of the CLI using ".testctrl tune ID VALUE". Tuning +** parameters are for temporary use during development, to help find +** optimial values for parameters in the query planner. The should not +** be used on trunk check-ins. They are a temporary mechanism available +** for transient development builds only. +** +** Tuning parameters are numbered starting with 1. +*/ +#define SQLITE_NTUNE 6 /* Should be zero for all trunk check-ins */ +#ifdef SQLITE_DEBUG +# define Tuning(X) (sqlite3Config.aTune[(X)-1]) +#else +# define Tuning(X) 0 +#endif + +/* +** Structure containing global configuration data for the SQLite library. +** +** This structure also contains some state information. +*/ +struct Sqlite3Config { + int bMemstat; /* True to enable memory status */ + u8 bCoreMutex; /* True to enable core mutexing */ + u8 bFullMutex; /* True to enable full mutexing */ + u8 bOpenUri; /* True to interpret filenames as URIs */ + u8 bUseCis; /* Use covering indices for full-scans */ + u8 bSmallMalloc; /* Avoid large memory allocations if true */ + u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */ + int mxStrlen; /* Maximum string length */ + int neverCorrupt; /* Database is always well-formed */ + int szLookaside; /* Default lookaside buffer size */ + int nLookaside; /* Default lookaside buffer count */ + int nStmtSpill; /* Stmt-journal spill-to-disk threshold */ + sqlite3_mem_methods m; /* Low-level memory allocation interface */ + sqlite3_mutex_methods mutex; /* Low-level mutex interface */ + sqlite3_pcache_methods2 pcache2; /* Low-level page-cache interface */ + void *pHeap; /* Heap storage space */ + int nHeap; /* Size of pHeap[] */ + int mnReq, mxReq; /* Min and max heap requests sizes */ + sqlite3_int64 szMmap; /* mmap() space per open file */ + sqlite3_int64 mxMmap; /* Maximum value for szMmap */ + void *pPage; /* Page cache memory */ + int szPage; /* Size of each page in pPage[] */ + int nPage; /* Number of pages in pPage[] */ + int mxParserStack; /* maximum depth of the parser stack */ + int sharedCacheEnabled; /* true if shared-cache mode enabled */ + u32 szPma; /* Maximum Sorter PMA size */ + /* The above might be initialized to non-zero. The following need to always + ** initially be zero, however. */ + int isInit; /* True after initialization has finished */ + int inProgress; /* True while initialization in progress */ + int isMutexInit; /* True after mutexes are initialized */ + int isMallocInit; /* True after malloc is initialized */ + int isPCacheInit; /* True after malloc is initialized */ + int nRefInitMutex; /* Number of users of pInitMutex */ + sqlite3_mutex *pInitMutex; /* Mutex used by sqlite3_initialize() */ + void (*xLog)(void*,int,const char*); /* Function for logging */ + void *pLogArg; /* First argument to xLog() */ +#ifdef SQLITE_ENABLE_SQLLOG + void(*xSqllog)(void*,sqlite3*,const char*, int); + void *pSqllogArg; +#endif +#ifdef SQLITE_VDBE_COVERAGE + /* The following callback (if not NULL) is invoked on every VDBE branch + ** operation. Set the callback using SQLITE_TESTCTRL_VDBE_COVERAGE. + */ + void (*xVdbeBranch)(void*,unsigned iSrcLine,u8 eThis,u8 eMx); /* Callback */ + void *pVdbeBranchArg; /* 1st argument */ +#endif +#ifndef SQLITE_OMIT_DESERIALIZE + sqlite3_int64 mxMemdbSize; /* Default max memdb size */ +#endif +#ifndef SQLITE_UNTESTABLE + int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ +#endif + int bLocaltimeFault; /* True to fail localtime() calls */ + int iOnceResetThreshold; /* When to reset OP_Once counters */ + u32 szSorterRef; /* Min size in bytes to use sorter-refs */ + unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */ + /* vvvv--- must be last ---vvv */ +#ifdef SQLITE_DEBUG + sqlite3_int64 aTune[SQLITE_NTUNE]; /* Tuning parameters */ +#endif +}; + +/* +** This macro is used inside of assert() statements to indicate that +** the assert is only valid on a well-formed database. Instead of: +** +** assert( X ); +** +** One writes: +** +** assert( X || CORRUPT_DB ); +** +** CORRUPT_DB is true during normal operation. CORRUPT_DB does not indicate +** that the database is definitely corrupt, only that it might be corrupt. +** For most test cases, CORRUPT_DB is set to false using a special +** sqlite3_test_control(). This enables assert() statements to prove +** things that are always true for well-formed databases. +*/ +#define CORRUPT_DB (sqlite3Config.neverCorrupt==0) + +/* +** Context pointer passed down through the tree-walk. +*/ +struct Walker { + Parse *pParse; /* Parser context. */ + int (*xExprCallback)(Walker*, Expr*); /* Callback for expressions */ + int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ + void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ + int walkerDepth; /* Number of subqueries */ + u16 eCode; /* A small processing code */ + union { /* Extra data for callback */ + NameContext *pNC; /* Naming context */ + int n; /* A counter */ + int iCur; /* A cursor number */ + SrcList *pSrcList; /* FROM clause */ + struct SrcCount *pSrcCount; /* Counting column references */ + struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ + int *aiCol; /* array of column indexes */ + struct IdxCover *pIdxCover; /* Check for index coverage */ + struct IdxExprTrans *pIdxTrans; /* Convert idxed expr to column */ + ExprList *pGroupBy; /* GROUP BY clause */ + Select *pSelect; /* HAVING to WHERE clause ctx */ + struct WindowRewrite *pRewrite; /* Window rewrite context */ + struct WhereConst *pConst; /* WHERE clause constants */ + struct RenameCtx *pRename; /* RENAME COLUMN context */ + struct Table *pTab; /* Table of generated column */ + SrcItem *pSrcItem; /* A single FROM clause item */ + DbFixer *pFix; + } u; +}; + +/* +** The following structure contains information used by the sqliteFix... +** routines as they walk the parse tree to make database references +** explicit. +*/ +struct DbFixer { + Parse *pParse; /* The parsing context. Error messages written here */ + Walker w; /* Walker object */ + Schema *pSchema; /* Fix items to this schema */ + u8 bTemp; /* True for TEMP schema entries */ + const char *zDb; /* Make sure all objects are contained in this database */ + const char *zType; /* Type of the container - used for error messages */ + const Token *pName; /* Name of the container - used for error messages */ +}; + +/* Forward declarations */ +SQLITE_PRIVATE int sqlite3WalkExpr(Walker*, Expr*); +SQLITE_PRIVATE int sqlite3WalkExprList(Walker*, ExprList*); +SQLITE_PRIVATE int sqlite3WalkSelect(Walker*, Select*); +SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker*, Select*); +SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker*, Select*); +SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker*, Expr*); +SQLITE_PRIVATE int sqlite3SelectWalkNoop(Walker*, Select*); +SQLITE_PRIVATE int sqlite3SelectWalkFail(Walker*, Select*); +SQLITE_PRIVATE int sqlite3WalkerDepthIncrease(Walker*,Select*); +SQLITE_PRIVATE void sqlite3WalkerDepthDecrease(Walker*,Select*); +SQLITE_PRIVATE void sqlite3WalkWinDefnDummyCallback(Walker*,Select*); + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3SelectWalkAssert2(Walker*, Select*); +#endif + +#ifndef SQLITE_OMIT_CTE +SQLITE_PRIVATE void sqlite3SelectPopWith(Walker*, Select*); +#else +# define sqlite3SelectPopWith 0 +#endif + +/* +** Return code from the parse-tree walking primitives and their +** callbacks. +*/ +#define WRC_Continue 0 /* Continue down into children */ +#define WRC_Prune 1 /* Omit children but continue walking siblings */ +#define WRC_Abort 2 /* Abandon the tree walk */ + +/* +** A single common table expression +*/ +struct Cte { + char *zName; /* Name of this CTE */ + ExprList *pCols; /* List of explicit column names, or NULL */ + Select *pSelect; /* The definition of this CTE */ + const char *zCteErr; /* Error message for circular references */ + CteUse *pUse; /* Usage information for this CTE */ + u8 eM10d; /* The MATERIALIZED flag */ +}; + +/* +** Allowed values for the materialized flag (eM10d): +*/ +#define M10d_Yes 0 /* AS MATERIALIZED */ +#define M10d_Any 1 /* Not specified. Query planner's choice */ +#define M10d_No 2 /* AS NOT MATERIALIZED */ + +/* +** An instance of the With object represents a WITH clause containing +** one or more CTEs (common table expressions). +*/ +struct With { + int nCte; /* Number of CTEs in the WITH clause */ + int bView; /* Belongs to the outermost Select of a view */ + With *pOuter; /* Containing WITH clause, or NULL */ + Cte a[1]; /* For each CTE in the WITH clause.... */ +}; + +/* +** The Cte object is not guaranteed to persist for the entire duration +** of code generation. (The query flattener or other parser tree +** edits might delete it.) The following object records information +** about each Common Table Expression that must be preserved for the +** duration of the parse. +** +** The CteUse objects are freed using sqlite3ParserAddCleanup() rather +** than sqlite3SelectDelete(), which is what enables them to persist +** until the end of code generation. +*/ +struct CteUse { + int nUse; /* Number of users of this CTE */ + int addrM9e; /* Start of subroutine to compute materialization */ + int regRtn; /* Return address register for addrM9e subroutine */ + int iCur; /* Ephemeral table holding the materialization */ + LogEst nRowEst; /* Estimated number of rows in the table */ + u8 eM10d; /* The MATERIALIZED flag */ +}; + + +#ifdef SQLITE_DEBUG +/* +** An instance of the TreeView object is used for printing the content of +** data structures on sqlite3DebugPrintf() using a tree-like view. +*/ +struct TreeView { + int iLevel; /* Which level of the tree we are on */ + u8 bLine[100]; /* Draw vertical in column i if bLine[i] is true */ +}; +#endif /* SQLITE_DEBUG */ + +/* +** This object is used in various ways, most (but not all) related to window +** functions. +** +** (1) A single instance of this structure is attached to the +** the Expr.y.pWin field for each window function in an expression tree. +** This object holds the information contained in the OVER clause, +** plus additional fields used during code generation. +** +** (2) All window functions in a single SELECT form a linked-list +** attached to Select.pWin. The Window.pFunc and Window.pExpr +** fields point back to the expression that is the window function. +** +** (3) The terms of the WINDOW clause of a SELECT are instances of this +** object on a linked list attached to Select.pWinDefn. +** +** (4) For an aggregate function with a FILTER clause, an instance +** of this object is stored in Expr.y.pWin with eFrmType set to +** TK_FILTER. In this case the only field used is Window.pFilter. +** +** The uses (1) and (2) are really the same Window object that just happens +** to be accessible in two different ways. Use case (3) are separate objects. +*/ +struct Window { + char *zName; /* Name of window (may be NULL) */ + char *zBase; /* Name of base window for chaining (may be NULL) */ + ExprList *pPartition; /* PARTITION BY clause */ + ExprList *pOrderBy; /* ORDER BY clause */ + u8 eFrmType; /* TK_RANGE, TK_GROUPS, TK_ROWS, or 0 */ + u8 eStart; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ + u8 eEnd; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ + u8 bImplicitFrame; /* True if frame was implicitly specified */ + u8 eExclude; /* TK_NO, TK_CURRENT, TK_TIES, TK_GROUP, or 0 */ + Expr *pStart; /* Expression for " PRECEDING" */ + Expr *pEnd; /* Expression for " FOLLOWING" */ + Window **ppThis; /* Pointer to this object in Select.pWin list */ + Window *pNextWin; /* Next window function belonging to this SELECT */ + Expr *pFilter; /* The FILTER expression */ + FuncDef *pFunc; /* The function */ + int iEphCsr; /* Partition buffer or Peer buffer */ + int regAccum; /* Accumulator */ + int regResult; /* Interim result */ + int csrApp; /* Function cursor (used by min/max) */ + int regApp; /* Function register (also used by min/max) */ + int regPart; /* Array of registers for PARTITION BY values */ + Expr *pOwner; /* Expression object this window is attached to */ + int nBufferCol; /* Number of columns in buffer table */ + int iArgCol; /* Offset of first argument for this function */ + int regOne; /* Register containing constant value 1 */ + int regStartRowid; + int regEndRowid; + u8 bExprArgs; /* Defer evaluation of window function arguments + ** due to the SQLITE_SUBTYPE flag */ +}; + +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3*, Window*); +SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window*); +SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p); +SQLITE_PRIVATE Window *sqlite3WindowAlloc(Parse*, int, int, Expr*, int , Expr*, u8); +SQLITE_PRIVATE void sqlite3WindowAttach(Parse*, Expr*, Window*); +SQLITE_PRIVATE void sqlite3WindowLink(Select *pSel, Window *pWin); +SQLITE_PRIVATE int sqlite3WindowCompare(const Parse*, const Window*, const Window*, int); +SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse*, Select*); +SQLITE_PRIVATE void sqlite3WindowCodeStep(Parse*, Select*, WhereInfo*, int, int); +SQLITE_PRIVATE int sqlite3WindowRewrite(Parse*, Select*); +SQLITE_PRIVATE void sqlite3WindowUpdate(Parse*, Window*, Window*, FuncDef*); +SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p); +SQLITE_PRIVATE Window *sqlite3WindowListDup(sqlite3 *db, Window *p); +SQLITE_PRIVATE void sqlite3WindowFunctions(void); +SQLITE_PRIVATE void sqlite3WindowChain(Parse*, Window*, Window*); +SQLITE_PRIVATE Window *sqlite3WindowAssemble(Parse*, Window*, ExprList*, ExprList*, Token*); +#else +# define sqlite3WindowDelete(a,b) +# define sqlite3WindowFunctions() +# define sqlite3WindowAttach(a,b,c) +#endif + +/* +** Assuming zIn points to the first byte of a UTF-8 character, +** advance zIn to point to the first byte of the next UTF-8 character. +*/ +#define SQLITE_SKIP_UTF8(zIn) { \ + if( (*(zIn++))>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){ zIn++; } \ + } \ +} + +/* +** The SQLITE_*_BKPT macros are substitutes for the error codes with +** the same name but without the _BKPT suffix. These macros invoke +** routines that report the line-number on which the error originated +** using sqlite3_log(). The routines also provide a convenient place +** to set a debugger breakpoint. +*/ +SQLITE_PRIVATE int sqlite3ReportError(int iErr, int lineno, const char *zType); +SQLITE_PRIVATE int sqlite3CorruptError(int); +SQLITE_PRIVATE int sqlite3MisuseError(int); +SQLITE_PRIVATE int sqlite3CantopenError(int); +#define SQLITE_CORRUPT_BKPT sqlite3CorruptError(__LINE__) +#define SQLITE_MISUSE_BKPT sqlite3MisuseError(__LINE__) +#define SQLITE_CANTOPEN_BKPT sqlite3CantopenError(__LINE__) +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NomemError(int); +SQLITE_PRIVATE int sqlite3IoerrnomemError(int); +# define SQLITE_NOMEM_BKPT sqlite3NomemError(__LINE__) +# define SQLITE_IOERR_NOMEM_BKPT sqlite3IoerrnomemError(__LINE__) +#else +# define SQLITE_NOMEM_BKPT SQLITE_NOMEM +# define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM +#endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO) +SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); +# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptPgnoError(__LINE__,(P)) +#else +# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptError(__LINE__) +#endif + +/* +** FTS3 and FTS4 both require virtual table support +*/ +#if defined(SQLITE_OMIT_VIRTUALTABLE) +# undef SQLITE_ENABLE_FTS3 +# undef SQLITE_ENABLE_FTS4 +#endif + +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also call +** the SQLITE_ENABLE_FTS4 macro to serve as an alias for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 1 +#endif + +/* +** The ctype.h header is needed for non-ASCII systems. It is also +** needed by FTS3 when FTS3 is included in the amalgamation. +*/ +#if !defined(SQLITE_ASCII) || \ + (defined(SQLITE_ENABLE_FTS3) && defined(SQLITE_AMALGAMATION)) +# include +#endif + +/* +** The following macros mimic the standard library functions toupper(), +** isspace(), isalnum(), isdigit() and isxdigit(), respectively. The +** sqlite versions only work for ASCII characters, regardless of locale. +*/ +#ifdef SQLITE_ASCII +# define sqlite3Toupper(x) ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20)) +# define sqlite3Isspace(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x01) +# define sqlite3Isalnum(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x06) +# define sqlite3Isalpha(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x02) +# define sqlite3Isdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x04) +# define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) +# define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) +# define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) +#else +# define sqlite3Toupper(x) toupper((unsigned char)(x)) +# define sqlite3Isspace(x) isspace((unsigned char)(x)) +# define sqlite3Isalnum(x) isalnum((unsigned char)(x)) +# define sqlite3Isalpha(x) isalpha((unsigned char)(x)) +# define sqlite3Isdigit(x) isdigit((unsigned char)(x)) +# define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) +# define sqlite3Tolower(x) tolower((unsigned char)(x)) +# define sqlite3Isquote(x) ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`') +#endif +SQLITE_PRIVATE int sqlite3IsIdChar(u8); + +/* +** Internal function prototypes +*/ +SQLITE_PRIVATE int sqlite3StrICmp(const char*,const char*); +SQLITE_PRIVATE int sqlite3Strlen30(const char*); +#define sqlite3Strlen30NN(C) (strlen(C)&0x3fffffff) +SQLITE_PRIVATE char *sqlite3ColumnType(Column*,char*); +#define sqlite3StrNICmp sqlite3_strnicmp + +SQLITE_PRIVATE int sqlite3MallocInit(void); +SQLITE_PRIVATE void sqlite3MallocEnd(void); +SQLITE_PRIVATE void *sqlite3Malloc(u64); +SQLITE_PRIVATE void *sqlite3MallocZero(u64); +SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3*, u64); +SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3*, u64); +SQLITE_PRIVATE void *sqlite3DbMallocRawNN(sqlite3*, u64); +SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3*,const char*); +SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3*,const char*, u64); +SQLITE_PRIVATE char *sqlite3DbSpanDup(sqlite3*,const char*,const char*); +SQLITE_PRIVATE void *sqlite3Realloc(void*, u64); +SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *, void *, u64); +SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *, void *, u64); +SQLITE_PRIVATE void sqlite3DbFree(sqlite3*, void*); +SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3*, void*); +SQLITE_PRIVATE int sqlite3MallocSize(const void*); +SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3*, const void*); +SQLITE_PRIVATE void *sqlite3PageMalloc(int); +SQLITE_PRIVATE void sqlite3PageFree(void*); +SQLITE_PRIVATE void sqlite3MemSetDefault(void); +#ifndef SQLITE_UNTESTABLE +SQLITE_PRIVATE void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); +#endif +SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); + +/* +** On systems with ample stack space and that support alloca(), make +** use of alloca() to obtain space for large automatic objects. By default, +** obtain space from malloc(). +** +** The alloca() routine never returns NULL. This will cause code paths +** that deal with sqlite3StackAlloc() failures to be unreachable. +*/ +#ifdef SQLITE_USE_ALLOCA +# define sqlite3StackAllocRaw(D,N) alloca(N) +# define sqlite3StackAllocZero(D,N) memset(alloca(N), 0, N) +# define sqlite3StackFree(D,P) +#else +# define sqlite3StackAllocRaw(D,N) sqlite3DbMallocRaw(D,N) +# define sqlite3StackAllocZero(D,N) sqlite3DbMallocZero(D,N) +# define sqlite3StackFree(D,P) sqlite3DbFree(D,P) +#endif + +/* Do not allow both MEMSYS5 and MEMSYS3 to be defined together. If they +** are, disable MEMSYS3 +*/ +#ifdef SQLITE_ENABLE_MEMSYS5 +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void); +#undef SQLITE_ENABLE_MEMSYS3 +#endif +#ifdef SQLITE_ENABLE_MEMSYS3 +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void); +#endif + + +#ifndef SQLITE_MUTEX_OMIT +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void); +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3NoopMutex(void); +SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int); +SQLITE_PRIVATE int sqlite3MutexInit(void); +SQLITE_PRIVATE int sqlite3MutexEnd(void); +#endif +#if !defined(SQLITE_MUTEX_OMIT) && !defined(SQLITE_MUTEX_NOOP) +SQLITE_PRIVATE void sqlite3MemoryBarrier(void); +#else +# define sqlite3MemoryBarrier() +#endif + +SQLITE_PRIVATE sqlite3_int64 sqlite3StatusValue(int); +SQLITE_PRIVATE void sqlite3StatusUp(int, int); +SQLITE_PRIVATE void sqlite3StatusDown(int, int); +SQLITE_PRIVATE void sqlite3StatusHighwater(int, int); +SQLITE_PRIVATE int sqlite3LookasideUsed(sqlite3*,int*); + +/* Access to mutexes used by sqlite3_status() */ +SQLITE_PRIVATE sqlite3_mutex *sqlite3Pcache1Mutex(void); +SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void); + +#if defined(SQLITE_ENABLE_MULTITHREADED_CHECKS) && !defined(SQLITE_MUTEX_OMIT) +SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex*); +#else +# define sqlite3MutexWarnOnContention(x) +#endif + +#ifndef SQLITE_OMIT_FLOATING_POINT +# define EXP754 (((u64)0x7ff)<<52) +# define MAN754 ((((u64)1)<<52)-1) +# define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) +SQLITE_PRIVATE int sqlite3IsNaN(double); +#else +# define IsNaN(X) 0 +# define sqlite3IsNaN(X) 0 +#endif + +/* +** An instance of the following structure holds information about SQL +** functions arguments that are the parameters to the printf() function. +*/ +struct PrintfArguments { + int nArg; /* Total number of arguments */ + int nUsed; /* Number of arguments used so far */ + sqlite3_value **apArg; /* The argument values */ +}; + +SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3*,const char*, ...); +SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3*,const char*, va_list); +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) +SQLITE_PRIVATE void sqlite3DebugPrintf(const char*, ...); +#endif +#if defined(SQLITE_TEST) +SQLITE_PRIVATE void *sqlite3TestTextToPtr(const char*); +#endif + +#if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); +SQLITE_PRIVATE void sqlite3TreeViewBareExprList(TreeView*, const ExprList*, const char*); +SQLITE_PRIVATE void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); +SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView*, const SrcList*); +SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); +SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView*, const With*, u8); +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView*, const Window*, u8); +SQLITE_PRIVATE void sqlite3TreeViewWinFunc(TreeView*, const Window*, u8); +#endif +#endif + + +SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); +SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); +SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3*,int); +SQLITE_PRIVATE void sqlite3Dequote(char*); +SQLITE_PRIVATE void sqlite3DequoteExpr(Expr*); +SQLITE_PRIVATE void sqlite3DequoteToken(Token*); +SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); +SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); +SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*, char **); +SQLITE_PRIVATE void sqlite3FinishCoding(Parse*); +SQLITE_PRIVATE int sqlite3GetTempReg(Parse*); +SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse*,int); +SQLITE_PRIVATE int sqlite3GetTempRange(Parse*,int); +SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse*,int,int); +SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse*,int,int); +#endif +SQLITE_PRIVATE Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); +SQLITE_PRIVATE Expr *sqlite3Expr(sqlite3*,int,const char*); +SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); +SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*); +SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); +SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse*,Expr*, Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, const Token*, int); +SQLITE_PRIVATE void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*); +SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); +SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); +SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse*, Expr*); +SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); +SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); +SQLITE_PRIVATE Select *sqlite3ExprListToValues(Parse*, int, ExprList*); +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int,int); +SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,const Token*,int); +SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*); +SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); +SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList*); +SQLITE_PRIVATE int sqlite3IndexHasDuplicateRootPage(Index*); +SQLITE_PRIVATE int sqlite3Init(sqlite3*, char**); +SQLITE_PRIVATE int sqlite3InitCallback(void*, int, char**, char**); +SQLITE_PRIVATE int sqlite3InitOne(sqlite3*, int, char**, u32); +SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int); +#ifndef SQLITE_OMIT_VIRTUALTABLE +SQLITE_PRIVATE Module *sqlite3PragmaVtabRegister(sqlite3*,const char *zName); +#endif +SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3*); +SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3*,int); +SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3*); +SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); +SQLITE_PRIVATE void sqlite3ColumnSetExpr(Parse*,Table*,Column*,Expr*); +SQLITE_PRIVATE Expr *sqlite3ColumnExpr(Table*,Column*); +SQLITE_PRIVATE void sqlite3ColumnSetColl(sqlite3*,Column*,const char*zColl); +SQLITE_PRIVATE const char *sqlite3ColumnColl(Column*); +SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); +SQLITE_PRIVATE void sqlite3GenerateColumnNames(Parse *pParse, Select *pSelect); +SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); +SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*,char); +SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*,char); +SQLITE_PRIVATE void sqlite3OpenSchemaTable(Parse *, int); +SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); +SQLITE_PRIVATE i16 sqlite3TableColumnToIndex(Index*, i16); +#ifdef SQLITE_OMIT_GENERATED_COLUMNS +# define sqlite3TableColumnToStorage(T,X) (X) /* No-op pass-through */ +# define sqlite3StorageColumnToTable(T,X) (X) /* No-op pass-through */ +#else +SQLITE_PRIVATE i16 sqlite3TableColumnToStorage(Table*, i16); +SQLITE_PRIVATE i16 sqlite3StorageColumnToTable(Table*, i16); +#endif +SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int); +#if SQLITE_ENABLE_HIDDEN_COLUMNS +SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table*, Column*); +#else +# define sqlite3ColumnPropertiesFromName(T,C) /* no-op */ +#endif +SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token,Token); +SQLITE_PRIVATE void sqlite3AddNotNull(Parse*, int); +SQLITE_PRIVATE void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int); +SQLITE_PRIVATE void sqlite3AddCheckConstraint(Parse*, Expr*, const char*, const char*); +SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse*,Expr*,const char*,const char*); +SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*); +SQLITE_PRIVATE void sqlite3AddGenerated(Parse*,Expr*,Token*); +SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u32,Select*); +SQLITE_PRIVATE void sqlite3AddReturning(Parse*,ExprList*); +SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, + sqlite3_vfs**,char**,char **); +#define sqlite3CodecQueryParameters(A,B,C) 0 +SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3*,const char*); + +#ifdef SQLITE_UNTESTABLE +# define sqlite3FaultSim(X) SQLITE_OK +#else +SQLITE_PRIVATE int sqlite3FaultSim(int); +#endif + +SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32); +SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec*, u32); +SQLITE_PRIVATE int sqlite3BitvecTestNotNull(Bitvec*, u32); +SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec*, u32); +SQLITE_PRIVATE void sqlite3BitvecClear(Bitvec*, u32, void*); +SQLITE_PRIVATE void sqlite3BitvecDestroy(Bitvec*); +SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec*); +#ifndef SQLITE_UNTESTABLE +SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int,int*); +#endif + +SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*); +SQLITE_PRIVATE void sqlite3RowSetDelete(void*); +SQLITE_PRIVATE void sqlite3RowSetClear(void*); +SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet*, i64); +SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, int iBatch, i64); +SQLITE_PRIVATE int sqlite3RowSetNext(RowSet*, i64*); + +SQLITE_PRIVATE void sqlite3CreateView(Parse*,Token*,Token*,Token*,ExprList*,Select*,int,int); + +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) +SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse*,Table*); +#else +# define sqlite3ViewGetColumnNames(A,B) 0 +#endif + +#if SQLITE_MAX_ATTACHED>30 +SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask); +#endif +SQLITE_PRIVATE void sqlite3DropTable(Parse*, SrcList*, int, int); +SQLITE_PRIVATE void sqlite3CodeDropTable(Parse*, Table*, int, int); +SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3*, Table*); +SQLITE_PRIVATE void sqlite3FreeIndex(sqlite3*, Index*); +#ifndef SQLITE_OMIT_AUTOINCREMENT +SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse); +SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse); +#else +# define sqlite3AutoincrementBegin(X) +# define sqlite3AutoincrementEnd(X) +#endif +SQLITE_PRIVATE void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +SQLITE_PRIVATE void sqlite3ComputeGeneratedColumns(Parse*, int, Table*); +#endif +SQLITE_PRIVATE void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); +SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse*, IdList*, Token*); +SQLITE_PRIVATE int sqlite3IdListIndex(IdList*,const char*); +SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge(Parse*, SrcList*, int, int); +SQLITE_PRIVATE SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2); +SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(Parse*, SrcList*, Token*, Token*); +SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, + Token*, Select*, Expr*, IdList*); +SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); +SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse*, SrcList*, ExprList*); +SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *, SrcItem *); +SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList*); +SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse*, SrcList*); +SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3*, IdList*); +SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3*, SrcList*); +SQLITE_PRIVATE Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**); +SQLITE_PRIVATE void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, + Expr*, int, int, u8); +SQLITE_PRIVATE void sqlite3DropIndex(Parse*, SrcList*, int); +SQLITE_PRIVATE int sqlite3Select(Parse*, Select*, SelectDest*); +SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, + Expr*,ExprList*,u32,Expr*); +SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); +SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int); +SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) +SQLITE_PRIVATE Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*); +#endif +SQLITE_PRIVATE void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*); +SQLITE_PRIVATE void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*, + Upsert*); +SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); +SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*); +SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereOrderByLimitOptLabel(WhereInfo*); +SQLITE_PRIVATE void sqlite3WhereMinMaxOptEarlyOut(Vdbe*,WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo*, int*); +#define ONEPASS_OFF 0 /* Use of ONEPASS not allowed */ +#define ONEPASS_SINGLE 1 /* ONEPASS valid for a single row update */ +#define ONEPASS_MULTI 2 /* ONEPASS is valid for multiple rows */ +SQLITE_PRIVATE int sqlite3WhereUsesDeferredSeek(WhereInfo*); +SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); +SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); +SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); +SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); +SQLITE_PRIVATE void sqlite3ExprCode(Parse*, Expr*, int); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn(Parse*, Table*, Column*, int); +#endif +SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, Expr*, int); +SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse*, Expr*, int); +SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce(Parse*, Expr*, int); +SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); +SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); +SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); +#define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ +#define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ +#define SQLITE_ECEL_REF 0x04 /* Use ExprList.u.x.iOrderByCol */ +#define SQLITE_ECEL_OMITREF 0x08 /* Omit if ExprList.u.x.iOrderByCol */ +SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse*, Expr*, int, int); +SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse*, Expr*, int, int); +SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int); +SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3*,const char*, const char*); +#define LOCATE_VIEW 0x01 +#define LOCATE_NOERR 0x02 +SQLITE_PRIVATE Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*); +SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,u32 flags,SrcItem *); +SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3*,const char*, const char*); +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); +SQLITE_PRIVATE void sqlite3Vacuum(Parse*,Token*,Expr*); +SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*, int, sqlite3_value*); +SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, const Token*); +SQLITE_PRIVATE int sqlite3ExprCompare(const Parse*,const Expr*,const Expr*, int); +SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr*,Expr*,int); +SQLITE_PRIVATE int sqlite3ExprListCompare(const ExprList*,const ExprList*, int); +SQLITE_PRIVATE int sqlite3ExprImpliesExpr(const Parse*,const Expr*,const Expr*, int); +SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr*,int); +SQLITE_PRIVATE void sqlite3AggInfoPersistWalkerInit(Walker*,Parse*); +SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); +SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); +SQLITE_PRIVATE int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx); +SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); +SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse*); +#ifndef SQLITE_UNTESTABLE +SQLITE_PRIVATE void sqlite3PrngSaveState(void); +SQLITE_PRIVATE void sqlite3PrngRestoreState(void); +#endif +SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3*,int); +SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse*, int); +SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); +SQLITE_PRIVATE void sqlite3BeginTransaction(Parse*, int); +SQLITE_PRIVATE void sqlite3EndTransaction(Parse*,int); +SQLITE_PRIVATE void sqlite3Savepoint(Parse*, int, Token*); +SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *); +SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3*); +SQLITE_PRIVATE u32 sqlite3IsTrueOrFalse(const char*); +SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr*); +SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr*); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); +SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); +SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); +SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); +SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); +#ifdef SQLITE_ENABLE_CURSOR_HINTS +SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); +#endif +SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr*, int*); +SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); +SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); +SQLITE_PRIVATE int sqlite3IsRowid(const char*); +SQLITE_PRIVATE void sqlite3GenerateRowDelete( + Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int); +SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); +SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); +SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse*,int); +SQLITE_PRIVATE int sqlite3ExprReferencesUpdatedColumn(Expr*,int*,int); +SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, + u8,u8,int,int*,int*,Upsert*); +#ifdef SQLITE_ENABLE_NULL_TRIM +SQLITE_PRIVATE void sqlite3SetMakeRecordP5(Vdbe*,Table*); +#else +# define sqlite3SetMakeRecordP5(A,B) +#endif +SQLITE_PRIVATE void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); +SQLITE_PRIVATE int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*); +SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse*, int, int); +SQLITE_PRIVATE void sqlite3MultiWrite(Parse*); +SQLITE_PRIVATE void sqlite3MayAbort(Parse*); +SQLITE_PRIVATE void sqlite3HaltConstraint(Parse*, int, int, char*, i8, u8); +SQLITE_PRIVATE void sqlite3UniqueConstraint(Parse*, int, Index*); +SQLITE_PRIVATE void sqlite3RowidConstraint(Parse*, int, Table*); +SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3*,const Expr*,int); +SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3*,const ExprList*,int); +SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3*,const SrcList*,int); +SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3*,const IdList*); +SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3*,const Select*,int); +SQLITE_PRIVATE FuncDef *sqlite3FunctionSearch(int,const char*); +SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs(FuncDef*,int); +SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); +SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void); +SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void); +SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*); +SQLITE_PRIVATE int sqlite3SafetyCheckOk(sqlite3*); +SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3*); +SQLITE_PRIVATE void sqlite3ChangeCookie(Parse*, int); +SQLITE_PRIVATE With *sqlite3WithDup(sqlite3 *db, With *p); + +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) +SQLITE_PRIVATE void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); +#endif + +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE void sqlite3BeginTrigger(Parse*, Token*,Token*,int,int,IdList*,SrcList*, + Expr*,int, int); +SQLITE_PRIVATE void sqlite3FinishTrigger(Parse*, TriggerStep*, Token*); +SQLITE_PRIVATE void sqlite3DropTrigger(Parse*, SrcList*, int); +SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse*, Trigger*); +SQLITE_PRIVATE Trigger *sqlite3TriggersExist(Parse *, Table*, int, ExprList*, int *pMask); +SQLITE_PRIVATE Trigger *sqlite3TriggerList(Parse *, Table *); +SQLITE_PRIVATE void sqlite3CodeRowTrigger(Parse*, Trigger *, int, ExprList*, int, Table *, + int, int, int); +SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect(Parse *, Trigger *, Table *, int, int, int); + void sqliteViewTriggers(Parse*, Table*, Expr*, int, ExprList*); +SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*, + const char*,const char*); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(Parse*,Token*, IdList*, + Select*,u8,Upsert*, + const char*,const char*); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(Parse*,Token*,SrcList*,ExprList*, + Expr*, u8, const char*,const char*); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(Parse*,Token*, Expr*, + const char*,const char*); +SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*); +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); +SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); +SQLITE_PRIVATE SrcList *sqlite3TriggerStepSrc(Parse*, TriggerStep*); +# define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) +# define sqlite3IsToplevel(p) ((p)->pToplevel==0) +#else +# define sqlite3TriggersExist(B,C,D,E,F) 0 +# define sqlite3DeleteTrigger(A,B) +# define sqlite3DropTriggerPtr(A,B) +# define sqlite3UnlinkAndDeleteTrigger(A,B,C) +# define sqlite3CodeRowTrigger(A,B,C,D,E,F,G,H,I) +# define sqlite3CodeRowTriggerDirect(A,B,C,D,E,F) +# define sqlite3TriggerList(X, Y) 0 +# define sqlite3ParseToplevel(p) p +# define sqlite3IsToplevel(p) 1 +# define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 +# define sqlite3TriggerStepSrc(A,B) 0 +#endif + +SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*); +SQLITE_PRIVATE int sqlite3ColumnIndex(Table *pTab, const char *zCol); +SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr*,int); +SQLITE_PRIVATE void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int); +SQLITE_PRIVATE void sqlite3DeferForeignKey(Parse*, int); +#ifndef SQLITE_OMIT_AUTHORIZATION +SQLITE_PRIVATE void sqlite3AuthRead(Parse*,Expr*,Schema*,SrcList*); +SQLITE_PRIVATE int sqlite3AuthCheck(Parse*,int, const char*, const char*, const char*); +SQLITE_PRIVATE void sqlite3AuthContextPush(Parse*, AuthContext*, const char*); +SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext*); +SQLITE_PRIVATE int sqlite3AuthReadCol(Parse*, const char *, const char *, int); +#else +# define sqlite3AuthRead(a,b,c,d) +# define sqlite3AuthCheck(a,b,c,d,e) SQLITE_OK +# define sqlite3AuthContextPush(a,b,c) +# define sqlite3AuthContextPop(a) ((void)(a)) +#endif +SQLITE_PRIVATE int sqlite3DbIsNamed(sqlite3 *db, int iDb, const char *zName); +SQLITE_PRIVATE void sqlite3Attach(Parse*, Expr*, Expr*, Expr*); +SQLITE_PRIVATE void sqlite3Detach(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*); +SQLITE_PRIVATE int sqlite3FixSrcList(DbFixer*, SrcList*); +SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); +SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); +SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); +SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64); +SQLITE_PRIVATE void sqlite3Int64ToText(i64,char*); +SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); +SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); +SQLITE_PRIVATE int sqlite3GetUInt32(const char*, u32*); +SQLITE_PRIVATE int sqlite3Atoi(const char*); +#ifndef SQLITE_OMIT_UTF16 +SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); +#endif +SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); +SQLITE_PRIVATE u32 sqlite3Utf8Read(const u8**); +SQLITE_PRIVATE LogEst sqlite3LogEst(u64); +SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst,LogEst); +#ifndef SQLITE_OMIT_VIRTUALTABLE +SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double); +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ + defined(SQLITE_ENABLE_STAT4) || \ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) +SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst); +#endif +SQLITE_PRIVATE VList *sqlite3VListAdd(sqlite3*,VList*,const char*,int,int); +SQLITE_PRIVATE const char *sqlite3VListNumToName(VList*,int); +SQLITE_PRIVATE int sqlite3VListNameToNum(VList*,const char*,int); + +/* +** Routines to read and write variable-length integers. These used to +** be defined locally, but now we use the varint routines in the util.c +** file. +*/ +SQLITE_PRIVATE int sqlite3PutVarint(unsigned char*, u64); +SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *, u64 *); +SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *, u32 *); +SQLITE_PRIVATE int sqlite3VarintLen(u64 v); + +/* +** The common case is for a varint to be a single byte. They following +** macros handle the common case without a procedure call, but then call +** the procedure for larger varints. +*/ +#define getVarint32(A,B) \ + (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite3GetVarint32((A),(u32 *)&(B))) +#define getVarint32NR(A,B) \ + B=(u32)*(A);if(B>=0x80)sqlite3GetVarint32((A),(u32*)&(B)) +#define putVarint32(A,B) \ + (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\ + sqlite3PutVarint((A),(B))) +#define getVarint sqlite3GetVarint +#define putVarint sqlite3PutVarint + + +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); +SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); +SQLITE_PRIVATE char sqlite3CompareAffinity(const Expr *pExpr, char aff2); +SQLITE_PRIVATE int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity); +SQLITE_PRIVATE char sqlite3TableColumnAffinity(const Table*,int); +SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr); +SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); +SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char*, i64*); +SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); +SQLITE_PRIVATE void sqlite3Error(sqlite3*,int); +SQLITE_PRIVATE void sqlite3ErrorClear(sqlite3*); +SQLITE_PRIVATE void sqlite3SystemError(sqlite3*,int); +SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); +SQLITE_PRIVATE u8 sqlite3HexToInt(int h); +SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); + +#if defined(SQLITE_NEED_ERR_NAME) +SQLITE_PRIVATE const char *sqlite3ErrName(int); +#endif + +#ifndef SQLITE_OMIT_DESERIALIZE +SQLITE_PRIVATE int sqlite3MemdbInit(void); +#endif + +SQLITE_PRIVATE const char *sqlite3ErrStr(int); +SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); +SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); +SQLITE_PRIVATE int sqlite3IsBinary(const CollSeq*); +SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); +SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8); +SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr); +SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, const Expr *pExpr); +SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse*,const Expr*,const Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(const Parse *pParse, Expr*, const Token*, int); +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(const Parse*,Expr*,const char*); +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr*); +SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); +SQLITE_PRIVATE int sqlite3WritableSchema(sqlite3*); +SQLITE_PRIVATE int sqlite3CheckObjectName(Parse*, const char*,const char*,const char*); +SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *, i64); +SQLITE_PRIVATE int sqlite3AddInt64(i64*,i64); +SQLITE_PRIVATE int sqlite3SubInt64(i64*,i64); +SQLITE_PRIVATE int sqlite3MulInt64(i64*,i64); +SQLITE_PRIVATE int sqlite3AbsInt32(int); +#ifdef SQLITE_ENABLE_8_3_NAMES +SQLITE_PRIVATE void sqlite3FileSuffix3(const char*, char*); +#else +# define sqlite3FileSuffix3(X,Y) +#endif +SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z,u8); + +SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); +SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value*, u8); +SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, + void(*)(void*)); +SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value*); +SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); +#ifndef SQLITE_UNTESTABLE +SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context*); +#endif +SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *); +#ifndef SQLITE_OMIT_UTF16 +SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *, const void*, int, u8); +#endif +SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 *, const Expr *, u8, u8, sqlite3_value **); +SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8); +#ifndef SQLITE_AMALGAMATION +SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; +SQLITE_PRIVATE const char sqlite3StrBINARY[]; +SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[]; +SQLITE_PRIVATE const char sqlite3StdTypeAffinity[]; +SQLITE_PRIVATE const char sqlite3StdTypeMap[]; +SQLITE_PRIVATE const char *sqlite3StdType[]; +SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; +SQLITE_PRIVATE const unsigned char *sqlite3aLTb; +SQLITE_PRIVATE const unsigned char *sqlite3aEQb; +SQLITE_PRIVATE const unsigned char *sqlite3aGTb; +SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[]; +SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config; +SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; +#ifndef SQLITE_OMIT_WSD +SQLITE_PRIVATE int sqlite3PendingByte; +#endif +#endif /* SQLITE_AMALGAMATION */ +#ifdef VDBE_PROFILE +SQLITE_PRIVATE sqlite3_uint64 sqlite3NProfileCnt; +#endif +SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3*, int, Pgno, Pgno); +SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*); +SQLITE_PRIVATE void sqlite3AlterFunctions(void); +SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); +SQLITE_PRIVATE void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); +SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *, int *); +SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); +SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*, int); +SQLITE_PRIVATE void sqlite3CodeRhsOfIN(Parse*, Expr*, int); +SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); +SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse*, SrcItem*); +SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); +SQLITE_PRIVATE int sqlite3MatchEName( + const struct ExprList_item*, + const char*, + const char*, + const char* +); +SQLITE_PRIVATE Bitmask sqlite3ExprColUsed(Expr*); +SQLITE_PRIVATE u8 sqlite3StrIHash(const char*); +SQLITE_PRIVATE int sqlite3ResolveExprNames(NameContext*, Expr*); +SQLITE_PRIVATE int sqlite3ResolveExprListNames(NameContext*, ExprList*); +SQLITE_PRIVATE void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); +SQLITE_PRIVATE int sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); +SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); +SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *, Table *, int, int); +SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *, Token *); +SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *, SrcList *); +SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse*, SrcList*, const Token*); +SQLITE_PRIVATE const void *sqlite3RenameTokenMap(Parse*, const void*, const Token*); +SQLITE_PRIVATE void sqlite3RenameTokenRemap(Parse*, const void *pTo, const void *pFrom); +SQLITE_PRIVATE void sqlite3RenameExprUnmap(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3RenameExprlistUnmap(Parse*, ExprList*); +SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); +SQLITE_PRIVATE char sqlite3AffinityType(const char*, Column*); +SQLITE_PRIVATE void sqlite3Analyze(Parse*, Token*, Token*); +SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler*); +SQLITE_PRIVATE int sqlite3FindDb(sqlite3*, Token*); +SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *, const char *); +SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3*,int iDB); +SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3*,Index*); +SQLITE_PRIVATE void sqlite3DefaultRowEst(Index*); +SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3*, int); +SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3*,Expr*,int*,char*); +SQLITE_PRIVATE void sqlite3SchemaClear(void *); +SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *, Btree *); +SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *); +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3*,int,int); +SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo*); +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoRef(KeyInfo*); +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse*, Index*); +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList(Parse*, ExprList*, int, int); +SQLITE_PRIVATE const char *sqlite3SelectOpName(int); +SQLITE_PRIVATE int sqlite3HasExplicitNulls(Parse*, ExprList*); + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo*); +#endif +SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, + void (*)(sqlite3_context*,int,sqlite3_value **), + void (*)(sqlite3_context*,int,sqlite3_value **), + void (*)(sqlite3_context*), + void (*)(sqlite3_context*), + void (*)(sqlite3_context*,int,sqlite3_value **), + FuncDestructor *pDestructor +); +SQLITE_PRIVATE void sqlite3NoopDestructor(void*); +SQLITE_PRIVATE void sqlite3OomFault(sqlite3*); +SQLITE_PRIVATE void sqlite3OomClear(sqlite3*); +SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); +SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); + +SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); +SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum*); +SQLITE_PRIVATE void sqlite3StrAccumSetError(StrAccum*, u8); +SQLITE_PRIVATE void sqlite3ResultStrAccum(sqlite3_context*,StrAccum*); +SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest*,int,int); +SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); + +SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *); +SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); + +#ifndef SQLITE_OMIT_SUBQUERY +SQLITE_PRIVATE int sqlite3ExprCheckIN(Parse*, Expr*); +#else +# define sqlite3ExprCheckIN(x,y) SQLITE_OK +#endif + +#ifdef SQLITE_ENABLE_STAT4 +SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue( + Parse*,Index*,UnpackedRecord**,Expr*,int,int,int*); +SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**); +SQLITE_PRIVATE void sqlite3Stat4ProbeFree(UnpackedRecord*); +SQLITE_PRIVATE int sqlite3Stat4Column(sqlite3*, const void*, int, int, sqlite3_value**); +SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3*, Index*, int); +#endif + +/* +** The interface to the LEMON-generated parser +*/ +#ifndef SQLITE_AMALGAMATION +SQLITE_PRIVATE void *sqlite3ParserAlloc(void*(*)(u64), Parse*); +SQLITE_PRIVATE void sqlite3ParserFree(void*, void(*)(void*)); +#endif +SQLITE_PRIVATE void sqlite3Parser(void*, int, Token); +SQLITE_PRIVATE int sqlite3ParserFallback(int); +#ifdef YYTRACKMAXSTACKDEPTH +SQLITE_PRIVATE int sqlite3ParserStackPeak(void*); +#endif + +SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3*); +#ifndef SQLITE_OMIT_LOAD_EXTENSION +SQLITE_PRIVATE void sqlite3CloseExtensions(sqlite3*); +#else +# define sqlite3CloseExtensions(X) +#endif + +#ifndef SQLITE_OMIT_SHARED_CACHE +SQLITE_PRIVATE void sqlite3TableLock(Parse *, int, Pgno, u8, const char *); +#else + #define sqlite3TableLock(v,w,x,y,z) +#endif + +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); +#endif + +#ifdef SQLITE_OMIT_VIRTUALTABLE +# define sqlite3VtabClear(D,T) +# define sqlite3VtabSync(X,Y) SQLITE_OK +# define sqlite3VtabRollback(X) +# define sqlite3VtabCommit(X) +# define sqlite3VtabInSync(db) 0 +# define sqlite3VtabLock(X) +# define sqlite3VtabUnlock(X) +# define sqlite3VtabModuleUnref(D,X) +# define sqlite3VtabUnlockList(X) +# define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK +# define sqlite3GetVTable(X,Y) ((VTable*)0) +#else +SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table*); +SQLITE_PRIVATE void sqlite3VtabDisconnect(sqlite3 *db, Table *p); +SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, Vdbe*); +SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db); +SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db); +SQLITE_PRIVATE void sqlite3VtabLock(VTable *); +SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *); +SQLITE_PRIVATE void sqlite3VtabModuleUnref(sqlite3*,Module*); +SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3*); +SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *, int, int); +SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*); +SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3*, Table*); +SQLITE_PRIVATE Module *sqlite3VtabCreateModule( + sqlite3*, + const char*, + const sqlite3_module*, + void*, + void(*)(void*) + ); +# define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) +#endif +SQLITE_PRIVATE int sqlite3ReadOnlyShadowTables(sqlite3 *db); +#ifndef SQLITE_OMIT_VIRTUALTABLE +SQLITE_PRIVATE int sqlite3ShadowTableName(sqlite3 *db, const char *zName); +SQLITE_PRIVATE int sqlite3IsShadowTableOf(sqlite3*,Table*,const char*); +#else +# define sqlite3ShadowTableName(A,B) 0 +# define sqlite3IsShadowTableOf(A,B,C) 0 +#endif +SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse*,Module*); +SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3*,Module*); +SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); +SQLITE_PRIVATE void sqlite3VtabBeginParse(Parse*, Token*, Token*, Token*, int); +SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse*, Token*); +SQLITE_PRIVATE void sqlite3VtabArgInit(Parse*); +SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse*, Token*); +SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3*, int, const char *, char **); +SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse*, Table*); +SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3*, int, const char *); +SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *, VTable *); +SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*); +SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); +SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); +SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); +SQLITE_PRIVATE void sqlite3ParserReset(Parse*); +SQLITE_PRIVATE void *sqlite3ParserAddCleanup(Parse*,void(*)(sqlite3*,void*),void*); +#ifdef SQLITE_ENABLE_NORMALIZE +SQLITE_PRIVATE char *sqlite3Normalize(Vdbe*, const char*); +#endif +SQLITE_PRIVATE int sqlite3Reprepare(Vdbe*); +SQLITE_PRIVATE void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); +SQLITE_PRIVATE CollSeq *sqlite3ExprCompareCollSeq(Parse*,const Expr*); +SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq(Parse *, const Expr*, const Expr*); +SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3*); +SQLITE_PRIVATE const char *sqlite3JournalModename(int); +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); +SQLITE_PRIVATE int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); +#endif +#ifndef SQLITE_OMIT_CTE +SQLITE_PRIVATE Cte *sqlite3CteNew(Parse*,Token*,ExprList*,Select*,u8); +SQLITE_PRIVATE void sqlite3CteDelete(sqlite3*,Cte*); +SQLITE_PRIVATE With *sqlite3WithAdd(Parse*,With*,Cte*); +SQLITE_PRIVATE void sqlite3WithDelete(sqlite3*,With*); +SQLITE_PRIVATE With *sqlite3WithPush(Parse*, With*, u8); +#else +# define sqlite3CteNew(P,T,E,S) ((void*)0) +# define sqlite3CteDelete(D,C) +# define sqlite3CteWithAdd(P,W,C) ((void*)0) +# define sqlite3WithDelete(x,y) +# define sqlite3WithPush(x,y,z) +#endif +#ifndef SQLITE_OMIT_UPSERT +SQLITE_PRIVATE Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*,Upsert*); +SQLITE_PRIVATE void sqlite3UpsertDelete(sqlite3*,Upsert*); +SQLITE_PRIVATE Upsert *sqlite3UpsertDup(sqlite3*,Upsert*); +SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*); +SQLITE_PRIVATE void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int); +SQLITE_PRIVATE Upsert *sqlite3UpsertOfIndex(Upsert*,Index*); +SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert*); +#else +#define sqlite3UpsertNew(u,v,w,x,y,z) ((Upsert*)0) +#define sqlite3UpsertDelete(x,y) +#define sqlite3UpsertDup(x,y) ((Upsert*)0) +#define sqlite3UpsertOfIndex(x,y) ((Upsert*)0) +#define sqlite3UpsertNextIsIPK(x) 0 +#endif + + +/* Declarations for functions in fkey.c. All of these are replaced by +** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign +** key functionality is available. If OMIT_TRIGGER is defined but +** OMIT_FOREIGN_KEY is not, only some of the functions are no-oped. In +** this case foreign keys are parsed, but no other functionality is +** provided (enforcement of FK constraints requires the triggers sub-system). +*/ +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) +SQLITE_PRIVATE void sqlite3FkCheck(Parse*, Table*, int, int, int*, int); +SQLITE_PRIVATE void sqlite3FkDropTable(Parse*, SrcList *, Table*); +SQLITE_PRIVATE void sqlite3FkActions(Parse*, Table*, ExprList*, int, int*, int); +SQLITE_PRIVATE int sqlite3FkRequired(Parse*, Table*, int*, int); +SQLITE_PRIVATE u32 sqlite3FkOldmask(Parse*, Table*); +SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *); +#else + #define sqlite3FkActions(a,b,c,d,e,f) + #define sqlite3FkCheck(a,b,c,d,e,f) + #define sqlite3FkDropTable(a,b,c) + #define sqlite3FkOldmask(a,b) 0 + #define sqlite3FkRequired(a,b,c,d) 0 + #define sqlite3FkReferences(a) 0 +#endif +#ifndef SQLITE_OMIT_FOREIGN_KEY +SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *, Table*); +SQLITE_PRIVATE int sqlite3FkLocateIndex(Parse*,Table*,FKey*,Index**,int**); +#else + #define sqlite3FkDelete(a,b) + #define sqlite3FkLocateIndex(a,b,c,d,e) +#endif + + +/* +** Available fault injectors. Should be numbered beginning with 0. +*/ +#define SQLITE_FAULTINJECTOR_MALLOC 0 +#define SQLITE_FAULTINJECTOR_COUNT 1 + +/* +** The interface to the code in fault.c used for identifying "benign" +** malloc failures. This is only present if SQLITE_UNTESTABLE +** is not defined. +*/ +#ifndef SQLITE_UNTESTABLE +SQLITE_PRIVATE void sqlite3BeginBenignMalloc(void); +SQLITE_PRIVATE void sqlite3EndBenignMalloc(void); +#else + #define sqlite3BeginBenignMalloc() + #define sqlite3EndBenignMalloc() +#endif + +/* +** Allowed return values from sqlite3FindInIndex() +*/ +#define IN_INDEX_ROWID 1 /* Search the rowid of the table */ +#define IN_INDEX_EPH 2 /* Search an ephemeral b-tree */ +#define IN_INDEX_INDEX_ASC 3 /* Existing index ASCENDING */ +#define IN_INDEX_INDEX_DESC 4 /* Existing index DESCENDING */ +#define IN_INDEX_NOOP 5 /* No table available. Use comparisons */ +/* +** Allowed flags for the 3rd parameter to sqlite3FindInIndex(). +*/ +#define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ +#define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ +#define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ +SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*, int*, int*); + +SQLITE_PRIVATE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); +SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *); +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *); +#endif + +SQLITE_PRIVATE int sqlite3JournalIsInMemory(sqlite3_file *p); +SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *); + +SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p); +#if SQLITE_MAX_EXPR_DEPTH>0 +SQLITE_PRIVATE int sqlite3SelectExprHeight(const Select *); +SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse*, int); +#else + #define sqlite3SelectExprHeight(x) 0 + #define sqlite3ExprCheckHeight(x,y) +#endif + +SQLITE_PRIVATE u32 sqlite3Get4byte(const u8*); +SQLITE_PRIVATE void sqlite3Put4byte(u8*, u32); + +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY +SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *, sqlite3 *); +SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db); +SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db); +#else + #define sqlite3ConnectionBlocked(x,y) + #define sqlite3ConnectionUnlocked(x) + #define sqlite3ConnectionClosed(x) +#endif + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3ParserTrace(FILE*, char *); +#endif +#if defined(YYCOVERAGE) +SQLITE_PRIVATE int sqlite3ParserCoverage(FILE*); +#endif + +/* +** If the SQLITE_ENABLE IOTRACE exists then the global variable +** sqlite3IoTrace is a pointer to a printf-like routine used to +** print I/O tracing messages. +*/ +#ifdef SQLITE_ENABLE_IOTRACE +# define IOTRACE(A) if( sqlite3IoTrace ){ sqlite3IoTrace A; } +SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe*); +SQLITE_API SQLITE_EXTERN void (SQLITE_CDECL *sqlite3IoTrace)(const char*,...); +#else +# define IOTRACE(A) +# define sqlite3VdbeIOTraceSql(X) +#endif + +/* +** These routines are available for the mem2.c debugging memory allocator +** only. They are used to verify that different "types" of memory +** allocations are properly tracked by the system. +** +** sqlite3MemdebugSetType() sets the "type" of an allocation to one of +** the MEMTYPE_* macros defined below. The type must be a bitmask with +** a single bit set. +** +** sqlite3MemdebugHasType() returns true if any of the bits in its second +** argument match the type set by the previous sqlite3MemdebugSetType(). +** sqlite3MemdebugHasType() is intended for use inside assert() statements. +** +** sqlite3MemdebugNoType() returns true if none of the bits in its second +** argument match the type set by the previous sqlite3MemdebugSetType(). +** +** Perhaps the most important point is the difference between MEMTYPE_HEAP +** and MEMTYPE_LOOKASIDE. If an allocation is MEMTYPE_LOOKASIDE, that means +** it might have been allocated by lookaside, except the allocation was +** too large or lookaside was already full. It is important to verify +** that allocations that might have been satisfied by lookaside are not +** passed back to non-lookaside free() routines. Asserts such as the +** example above are placed on the non-lookaside free() routines to verify +** this constraint. +** +** All of this is no-op for a production build. It only comes into +** play when the SQLITE_MEMDEBUG compile-time option is used. +*/ +#ifdef SQLITE_MEMDEBUG +SQLITE_PRIVATE void sqlite3MemdebugSetType(void*,u8); +SQLITE_PRIVATE int sqlite3MemdebugHasType(void*,u8); +SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); +#else +# define sqlite3MemdebugSetType(X,Y) /* no-op */ +# define sqlite3MemdebugHasType(X,Y) 1 +# define sqlite3MemdebugNoType(X,Y) 1 +#endif +#define MEMTYPE_HEAP 0x01 /* General heap allocations */ +#define MEMTYPE_LOOKASIDE 0x02 /* Heap that might have been lookaside */ +#define MEMTYPE_PCACHE 0x04 /* Page cache allocations */ + +/* +** Threading interface +*/ +#if SQLITE_MAX_WORKER_THREADS>0 +SQLITE_PRIVATE int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); +SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread*, void**); +#endif + +#if defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST) +SQLITE_PRIVATE int sqlite3DbpageRegister(sqlite3*); +#endif +#if defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST) +SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3*); +#endif + +SQLITE_PRIVATE int sqlite3ExprVectorSize(const Expr *pExpr); +SQLITE_PRIVATE int sqlite3ExprIsVector(const Expr *pExpr); +SQLITE_PRIVATE Expr *sqlite3VectorFieldSubexpr(Expr*, int); +SQLITE_PRIVATE Expr *sqlite3ExprForVectorField(Parse*,Expr*,int,int); +SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse*, Expr*); + +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt); +#endif + +#endif /* SQLITEINT_H */ + +/************** End of sqliteInt.h *******************************************/ +/************** Begin file os_common.h ***************************************/ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains macros and a little bit of code that is common to +** all of the platform-specific files (os_*.c) and is #included into those +** files. +** +** This file should be #included by the os_*.c files only. It is not a +** general purpose header file. +*/ +#ifndef _OS_COMMON_H_ +#define _OS_COMMON_H_ + +/* +** At least two bugs have slipped in because we changed the MEMORY_DEBUG +** macro to SQLITE_DEBUG and some older makefiles have not yet made the +** switch. The following code should catch this problem at compile-time. +*/ +#ifdef MEMORY_DEBUG +# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." +#endif + +/* +** Macros for performance tracing. Normally turned off. Only works +** on i486 hardware. +*/ +#ifdef SQLITE_PERFORMANCE_TRACE + +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +/************** Include hwtime.h in the middle of os_common.h ****************/ +/************** Begin file hwtime.h ******************************************/ +/* +** 2008 May 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 and x86_64 class CPUs. +*/ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H + +/* +** The following routine only works on pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if !defined(__STRICT_ANSI__) && \ + (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long val; + __asm__ __volatile__ ("rdtsc" : "=A" (val)); + return val; + } + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } + +#else + + /* + ** asm() is needed for hardware timing support. Without asm(), + ** disable the sqlite3Hwtime() routine. + ** + ** sqlite3Hwtime() is only used for some obscure debugging + ** and analysis configurations, not in any deliverable, so this + ** should not be a great loss. + */ +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(SQLITE_HWTIME_H) */ + +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in os_common.h ******************/ + +static sqlite_uint64 g_start; +static sqlite_uint64 g_elapsed; +#define TIMER_START g_start=sqlite3Hwtime() +#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start +#define TIMER_ELAPSED g_elapsed +#else +#define TIMER_START +#define TIMER_END +#define TIMER_ELAPSED ((sqlite_uint64)0) +#endif + +/* +** If we compile with the SQLITE_TEST macro set, then the following block +** of code will give us the ability to simulate a disk I/O error. This +** is used for testing the I/O recovery logic. +*/ +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_io_error_hit; +SQLITE_API extern int sqlite3_io_error_hardhit; +SQLITE_API extern int sqlite3_io_error_pending; +SQLITE_API extern int sqlite3_io_error_persist; +SQLITE_API extern int sqlite3_io_error_benign; +SQLITE_API extern int sqlite3_diskfull_pending; +SQLITE_API extern int sqlite3_diskfull; +#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) +#define SimulateIOError(CODE) \ + if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ + || sqlite3_io_error_pending-- == 1 ) \ + { local_ioerr(); CODE; } +static void local_ioerr(){ + IOTRACE(("IOERR\n")); + sqlite3_io_error_hit++; + if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; +} +#define SimulateDiskfullError(CODE) \ + if( sqlite3_diskfull_pending ){ \ + if( sqlite3_diskfull_pending == 1 ){ \ + local_ioerr(); \ + sqlite3_diskfull = 1; \ + sqlite3_io_error_hit = 1; \ + CODE; \ + }else{ \ + sqlite3_diskfull_pending--; \ + } \ + } +#else +#define SimulateIOErrorBenign(X) +#define SimulateIOError(A) +#define SimulateDiskfullError(A) +#endif /* defined(SQLITE_TEST) */ + +/* +** When testing, keep a count of the number of open files. +*/ +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_open_file_count; +#define OpenCounter(X) sqlite3_open_file_count+=(X) +#else +#define OpenCounter(X) +#endif /* defined(SQLITE_TEST) */ + +#endif /* !defined(_OS_COMMON_H_) */ + +/************** End of os_common.h *******************************************/ +/************** Begin file ctime.c *******************************************/ +/* +** 2010 February 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements routines used to report what compile-time options +** SQLite was built with. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* IMP: R-16824-07538 */ + +/* +** Include the configuration header output by 'configure' if we're using the +** autoconf-based build +*/ +#if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) +/* #include "config.h" */ +#define SQLITECONFIG_H 1 +#endif + +/* These macros are provided to "stringify" the value of the define +** for those options in which the value is meaningful. */ +#define CTIMEOPT_VAL_(opt) #opt +#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) + +/* Like CTIMEOPT_VAL, but especially for SQLITE_DEFAULT_LOOKASIDE. This +** option requires a separate macro because legal values contain a single +** comma. e.g. (-DSQLITE_DEFAULT_LOOKASIDE="100,100") */ +#define CTIMEOPT_VAL2_(opt1,opt2) #opt1 "," #opt2 +#define CTIMEOPT_VAL2(opt) CTIMEOPT_VAL2_(opt) +/* #include "sqliteInt.h" */ + +/* +** An array of names of all compile-time options. This array should +** be sorted A-Z. +** +** This array looks large, but in a typical installation actually uses +** only a handful of compile-time options, so most times this array is usually +** rather short and uses little memory space. +*/ +static const char * const sqlite3azCompileOpt[] = { + +/* +** BEGIN CODE GENERATED BY tool/mkctime.tcl +*/ +#ifdef SQLITE_32BIT_ROWID + "32BIT_ROWID", +#endif +#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC + "4_BYTE_ALIGNED_MALLOC", +#endif +#ifdef SQLITE_64BIT_STATS + "64BIT_STATS", +#endif +#ifdef SQLITE_ALLOW_COVERING_INDEX_SCAN +# if SQLITE_ALLOW_COVERING_INDEX_SCAN != 1 + "ALLOW_COVERING_INDEX_SCAN=" CTIMEOPT_VAL(SQLITE_ALLOW_COVERING_INDEX_SCAN), +# endif +#endif +#ifdef SQLITE_ALLOW_URI_AUTHORITY + "ALLOW_URI_AUTHORITY", +#endif +#ifdef SQLITE_ATOMIC_INTRINSICS + "ATOMIC_INTRINSICS=" CTIMEOPT_VAL(SQLITE_ATOMIC_INTRINSICS), +#endif +#ifdef SQLITE_BITMASK_TYPE + "BITMASK_TYPE=" CTIMEOPT_VAL(SQLITE_BITMASK_TYPE), +#endif +#ifdef SQLITE_BUG_COMPATIBLE_20160819 + "BUG_COMPATIBLE_20160819", +#endif +#ifdef SQLITE_CASE_SENSITIVE_LIKE + "CASE_SENSITIVE_LIKE", +#endif +#ifdef SQLITE_CHECK_PAGES + "CHECK_PAGES", +#endif +#if defined(__clang__) && defined(__clang_major__) + "COMPILER=clang-" CTIMEOPT_VAL(__clang_major__) "." + CTIMEOPT_VAL(__clang_minor__) "." + CTIMEOPT_VAL(__clang_patchlevel__), +#elif defined(_MSC_VER) + "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER), +#elif defined(__GNUC__) && defined(__VERSION__) + "COMPILER=gcc-" __VERSION__, +#endif +#ifdef SQLITE_COVERAGE_TEST + "COVERAGE_TEST", +#endif +#ifdef SQLITE_DEBUG + "DEBUG", +#endif +#ifdef SQLITE_DEFAULT_AUTOMATIC_INDEX + "DEFAULT_AUTOMATIC_INDEX", +#endif +#ifdef SQLITE_DEFAULT_AUTOVACUUM + "DEFAULT_AUTOVACUUM", +#endif +#ifdef SQLITE_DEFAULT_CACHE_SIZE + "DEFAULT_CACHE_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_CACHE_SIZE), +#endif +#ifdef SQLITE_DEFAULT_CKPTFULLFSYNC + "DEFAULT_CKPTFULLFSYNC", +#endif +#ifdef SQLITE_DEFAULT_FILE_FORMAT + "DEFAULT_FILE_FORMAT=" CTIMEOPT_VAL(SQLITE_DEFAULT_FILE_FORMAT), +#endif +#ifdef SQLITE_DEFAULT_FILE_PERMISSIONS + "DEFAULT_FILE_PERMISSIONS=" CTIMEOPT_VAL(SQLITE_DEFAULT_FILE_PERMISSIONS), +#endif +#ifdef SQLITE_DEFAULT_FOREIGN_KEYS + "DEFAULT_FOREIGN_KEYS", +#endif +#ifdef SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT + "DEFAULT_JOURNAL_SIZE_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT), +#endif +#ifdef SQLITE_DEFAULT_LOCKING_MODE + "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), +#endif +#ifdef SQLITE_DEFAULT_LOOKASIDE + "DEFAULT_LOOKASIDE=" CTIMEOPT_VAL2(SQLITE_DEFAULT_LOOKASIDE), +#endif +#ifdef SQLITE_DEFAULT_MEMSTATUS +# if SQLITE_DEFAULT_MEMSTATUS != 1 + "DEFAULT_MEMSTATUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_MEMSTATUS), +# endif +#endif +#ifdef SQLITE_DEFAULT_MMAP_SIZE + "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), +#endif +#ifdef SQLITE_DEFAULT_PAGE_SIZE + "DEFAULT_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_PAGE_SIZE), +#endif +#ifdef SQLITE_DEFAULT_PCACHE_INITSZ + "DEFAULT_PCACHE_INITSZ=" CTIMEOPT_VAL(SQLITE_DEFAULT_PCACHE_INITSZ), +#endif +#ifdef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS + "DEFAULT_PROXYDIR_PERMISSIONS=" CTIMEOPT_VAL(SQLITE_DEFAULT_PROXYDIR_PERMISSIONS), +#endif +#ifdef SQLITE_DEFAULT_RECURSIVE_TRIGGERS + "DEFAULT_RECURSIVE_TRIGGERS", +#endif +#ifdef SQLITE_DEFAULT_ROWEST + "DEFAULT_ROWEST=" CTIMEOPT_VAL(SQLITE_DEFAULT_ROWEST), +#endif +#ifdef SQLITE_DEFAULT_SECTOR_SIZE + "DEFAULT_SECTOR_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_SECTOR_SIZE), +#endif +#ifdef SQLITE_DEFAULT_SYNCHRONOUS + "DEFAULT_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_SYNCHRONOUS), +#endif +#ifdef SQLITE_DEFAULT_WAL_AUTOCHECKPOINT + "DEFAULT_WAL_AUTOCHECKPOINT=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_AUTOCHECKPOINT), +#endif +#ifdef SQLITE_DEFAULT_WAL_SYNCHRONOUS + "DEFAULT_WAL_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_SYNCHRONOUS), +#endif +#ifdef SQLITE_DEFAULT_WORKER_THREADS + "DEFAULT_WORKER_THREADS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WORKER_THREADS), +#endif +#ifdef SQLITE_DIRECT_OVERFLOW_READ + "DIRECT_OVERFLOW_READ", +#endif +#ifdef SQLITE_DISABLE_DIRSYNC + "DISABLE_DIRSYNC", +#endif +#ifdef SQLITE_DISABLE_FTS3_UNICODE + "DISABLE_FTS3_UNICODE", +#endif +#ifdef SQLITE_DISABLE_FTS4_DEFERRED + "DISABLE_FTS4_DEFERRED", +#endif +#ifdef SQLITE_DISABLE_INTRINSIC + "DISABLE_INTRINSIC", +#endif +#ifdef SQLITE_DISABLE_LFS + "DISABLE_LFS", +#endif +#ifdef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS + "DISABLE_PAGECACHE_OVERFLOW_STATS", +#endif +#ifdef SQLITE_DISABLE_SKIPAHEAD_DISTINCT + "DISABLE_SKIPAHEAD_DISTINCT", +#endif +#ifdef SQLITE_ENABLE_8_3_NAMES + "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), +#endif +#ifdef SQLITE_ENABLE_API_ARMOR + "ENABLE_API_ARMOR", +#endif +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + "ENABLE_ATOMIC_WRITE", +#endif +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + "ENABLE_BATCH_ATOMIC_WRITE", +#endif +#ifdef SQLITE_ENABLE_BYTECODE_VTAB + "ENABLE_BYTECODE_VTAB", +#endif +#ifdef SQLITE_ENABLE_CEROD + "ENABLE_CEROD=" CTIMEOPT_VAL(SQLITE_ENABLE_CEROD), +#endif +#ifdef SQLITE_ENABLE_COLUMN_METADATA + "ENABLE_COLUMN_METADATA", +#endif +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + "ENABLE_COLUMN_USED_MASK", +#endif +#ifdef SQLITE_ENABLE_COSTMULT + "ENABLE_COSTMULT", +#endif +#ifdef SQLITE_ENABLE_CURSOR_HINTS + "ENABLE_CURSOR_HINTS", +#endif +#ifdef SQLITE_ENABLE_DBPAGE_VTAB + "ENABLE_DBPAGE_VTAB", +#endif +#ifdef SQLITE_ENABLE_DBSTAT_VTAB + "ENABLE_DBSTAT_VTAB", +#endif +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + "ENABLE_EXPENSIVE_ASSERT", +#endif +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + "ENABLE_EXPLAIN_COMMENTS", +#endif +#ifdef SQLITE_ENABLE_FTS3 + "ENABLE_FTS3", +#endif +#ifdef SQLITE_ENABLE_FTS3_PARENTHESIS + "ENABLE_FTS3_PARENTHESIS", +#endif +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER + "ENABLE_FTS3_TOKENIZER", +#endif +#ifdef SQLITE_ENABLE_FTS4 + "ENABLE_FTS4", +#endif +#ifdef SQLITE_ENABLE_FTS5 + "ENABLE_FTS5", +#endif +#ifdef SQLITE_ENABLE_GEOPOLY + "ENABLE_GEOPOLY", +#endif +#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS + "ENABLE_HIDDEN_COLUMNS", +#endif +#ifdef SQLITE_ENABLE_ICU + "ENABLE_ICU", +#endif +#ifdef SQLITE_ENABLE_IOTRACE + "ENABLE_IOTRACE", +#endif +#ifdef SQLITE_ENABLE_JSON1 + "ENABLE_JSON1", +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + "ENABLE_LOAD_EXTENSION", +#endif +#ifdef SQLITE_ENABLE_LOCKING_STYLE + "ENABLE_LOCKING_STYLE=" CTIMEOPT_VAL(SQLITE_ENABLE_LOCKING_STYLE), +#endif +#ifdef SQLITE_ENABLE_MATH_FUNCTIONS + "ENABLE_MATH_FUNCTIONS", +#endif +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + "ENABLE_MEMORY_MANAGEMENT", +#endif +#ifdef SQLITE_ENABLE_MEMSYS3 + "ENABLE_MEMSYS3", +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + "ENABLE_MEMSYS5", +#endif +#ifdef SQLITE_ENABLE_MULTIPLEX + "ENABLE_MULTIPLEX", +#endif +#ifdef SQLITE_ENABLE_NORMALIZE + "ENABLE_NORMALIZE", +#endif +#ifdef SQLITE_ENABLE_NULL_TRIM + "ENABLE_NULL_TRIM", +#endif +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + "ENABLE_OFFSET_SQL_FUNC", +#endif +#ifdef SQLITE_ENABLE_OVERSIZE_CELL_CHECK + "ENABLE_OVERSIZE_CELL_CHECK", +#endif +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + "ENABLE_PREUPDATE_HOOK", +#endif +#ifdef SQLITE_ENABLE_QPSG + "ENABLE_QPSG", +#endif +#ifdef SQLITE_ENABLE_RBU + "ENABLE_RBU", +#endif +#ifdef SQLITE_ENABLE_RTREE + "ENABLE_RTREE", +#endif +#ifdef SQLITE_ENABLE_SELECTTRACE + "ENABLE_SELECTTRACE", +#endif +#ifdef SQLITE_ENABLE_SESSION + "ENABLE_SESSION", +#endif +#ifdef SQLITE_ENABLE_SNAPSHOT + "ENABLE_SNAPSHOT", +#endif +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + "ENABLE_SORTER_REFERENCES", +#endif +#ifdef SQLITE_ENABLE_SQLLOG + "ENABLE_SQLLOG", +#endif +#ifdef SQLITE_ENABLE_STAT4 + "ENABLE_STAT4", +#endif +#ifdef SQLITE_ENABLE_STMTVTAB + "ENABLE_STMTVTAB", +#endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + "ENABLE_STMT_SCANSTATUS", +#endif +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + "ENABLE_UNKNOWN_SQL_FUNCTION", +#endif +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + "ENABLE_UNLOCK_NOTIFY", +#endif +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + "ENABLE_UPDATE_DELETE_LIMIT", +#endif +#ifdef SQLITE_ENABLE_URI_00_ERROR + "ENABLE_URI_00_ERROR", +#endif +#ifdef SQLITE_ENABLE_VFSTRACE + "ENABLE_VFSTRACE", +#endif +#ifdef SQLITE_ENABLE_WHERETRACE + "ENABLE_WHERETRACE", +#endif +#ifdef SQLITE_ENABLE_ZIPVFS + "ENABLE_ZIPVFS", +#endif +#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS + "EXPLAIN_ESTIMATED_ROWS", +#endif +#ifdef SQLITE_EXTRA_IFNULLROW + "EXTRA_IFNULLROW", +#endif +#ifdef SQLITE_EXTRA_INIT + "EXTRA_INIT=" CTIMEOPT_VAL(SQLITE_EXTRA_INIT), +#endif +#ifdef SQLITE_EXTRA_SHUTDOWN + "EXTRA_SHUTDOWN=" CTIMEOPT_VAL(SQLITE_EXTRA_SHUTDOWN), +#endif +#ifdef SQLITE_FTS3_MAX_EXPR_DEPTH + "FTS3_MAX_EXPR_DEPTH=" CTIMEOPT_VAL(SQLITE_FTS3_MAX_EXPR_DEPTH), +#endif +#ifdef SQLITE_FTS5_ENABLE_TEST_MI + "FTS5_ENABLE_TEST_MI", +#endif +#ifdef SQLITE_FTS5_NO_WITHOUT_ROWID + "FTS5_NO_WITHOUT_ROWID", +#endif +#if HAVE_ISNAN || SQLITE_HAVE_ISNAN + "HAVE_ISNAN", +#endif +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX +# if SQLITE_HOMEGROWN_RECURSIVE_MUTEX != 1 + "HOMEGROWN_RECURSIVE_MUTEX=" CTIMEOPT_VAL(SQLITE_HOMEGROWN_RECURSIVE_MUTEX), +# endif +#endif +#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS + "IGNORE_AFP_LOCK_ERRORS", +#endif +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + "IGNORE_FLOCK_LOCK_ERRORS", +#endif +#ifdef SQLITE_INLINE_MEMCPY + "INLINE_MEMCPY", +#endif +#ifdef SQLITE_INT64_TYPE + "INT64_TYPE", +#endif +#ifdef SQLITE_INTEGRITY_CHECK_ERROR_MAX + "INTEGRITY_CHECK_ERROR_MAX=" CTIMEOPT_VAL(SQLITE_INTEGRITY_CHECK_ERROR_MAX), +#endif +#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS + "LIKE_DOESNT_MATCH_BLOBS", +#endif +#ifdef SQLITE_LOCK_TRACE + "LOCK_TRACE", +#endif +#ifdef SQLITE_LOG_CACHE_SPILL + "LOG_CACHE_SPILL", +#endif +#ifdef SQLITE_MALLOC_SOFT_LIMIT + "MALLOC_SOFT_LIMIT=" CTIMEOPT_VAL(SQLITE_MALLOC_SOFT_LIMIT), +#endif +#ifdef SQLITE_MAX_ATTACHED + "MAX_ATTACHED=" CTIMEOPT_VAL(SQLITE_MAX_ATTACHED), +#endif +#ifdef SQLITE_MAX_COLUMN + "MAX_COLUMN=" CTIMEOPT_VAL(SQLITE_MAX_COLUMN), +#endif +#ifdef SQLITE_MAX_COMPOUND_SELECT + "MAX_COMPOUND_SELECT=" CTIMEOPT_VAL(SQLITE_MAX_COMPOUND_SELECT), +#endif +#ifdef SQLITE_MAX_DEFAULT_PAGE_SIZE + "MAX_DEFAULT_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_DEFAULT_PAGE_SIZE), +#endif +#ifdef SQLITE_MAX_EXPR_DEPTH + "MAX_EXPR_DEPTH=" CTIMEOPT_VAL(SQLITE_MAX_EXPR_DEPTH), +#endif +#ifdef SQLITE_MAX_FUNCTION_ARG + "MAX_FUNCTION_ARG=" CTIMEOPT_VAL(SQLITE_MAX_FUNCTION_ARG), +#endif +#ifdef SQLITE_MAX_LENGTH + "MAX_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_LENGTH), +#endif +#ifdef SQLITE_MAX_LIKE_PATTERN_LENGTH + "MAX_LIKE_PATTERN_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_LIKE_PATTERN_LENGTH), +#endif +#ifdef SQLITE_MAX_MEMORY + "MAX_MEMORY=" CTIMEOPT_VAL(SQLITE_MAX_MEMORY), +#endif +#ifdef SQLITE_MAX_MMAP_SIZE + "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), +#endif +#ifdef SQLITE_MAX_MMAP_SIZE_ + "MAX_MMAP_SIZE_=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE_), +#endif +#ifdef SQLITE_MAX_PAGE_COUNT + "MAX_PAGE_COUNT=" CTIMEOPT_VAL(SQLITE_MAX_PAGE_COUNT), +#endif +#ifdef SQLITE_MAX_PAGE_SIZE + "MAX_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_PAGE_SIZE), +#endif +#ifdef SQLITE_MAX_SCHEMA_RETRY + "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), +#endif +#ifdef SQLITE_MAX_SQL_LENGTH + "MAX_SQL_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_SQL_LENGTH), +#endif +#ifdef SQLITE_MAX_TRIGGER_DEPTH + "MAX_TRIGGER_DEPTH=" CTIMEOPT_VAL(SQLITE_MAX_TRIGGER_DEPTH), +#endif +#ifdef SQLITE_MAX_VARIABLE_NUMBER + "MAX_VARIABLE_NUMBER=" CTIMEOPT_VAL(SQLITE_MAX_VARIABLE_NUMBER), +#endif +#ifdef SQLITE_MAX_VDBE_OP + "MAX_VDBE_OP=" CTIMEOPT_VAL(SQLITE_MAX_VDBE_OP), +#endif +#ifdef SQLITE_MAX_WORKER_THREADS + "MAX_WORKER_THREADS=" CTIMEOPT_VAL(SQLITE_MAX_WORKER_THREADS), +#endif +#ifdef SQLITE_MEMDEBUG + "MEMDEBUG", +#endif +#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT + "MIXED_ENDIAN_64BIT_FLOAT", +#endif +#ifdef SQLITE_MMAP_READWRITE + "MMAP_READWRITE", +#endif +#ifdef SQLITE_MUTEX_NOOP + "MUTEX_NOOP", +#endif +#ifdef SQLITE_MUTEX_OMIT + "MUTEX_OMIT", +#endif +#ifdef SQLITE_MUTEX_PTHREADS + "MUTEX_PTHREADS", +#endif +#ifdef SQLITE_MUTEX_W32 + "MUTEX_W32", +#endif +#ifdef SQLITE_NEED_ERR_NAME + "NEED_ERR_NAME", +#endif +#ifdef SQLITE_NO_SYNC + "NO_SYNC", +#endif +#ifdef SQLITE_OMIT_ALTERTABLE + "OMIT_ALTERTABLE", +#endif +#ifdef SQLITE_OMIT_ANALYZE + "OMIT_ANALYZE", +#endif +#ifdef SQLITE_OMIT_ATTACH + "OMIT_ATTACH", +#endif +#ifdef SQLITE_OMIT_AUTHORIZATION + "OMIT_AUTHORIZATION", +#endif +#ifdef SQLITE_OMIT_AUTOINCREMENT + "OMIT_AUTOINCREMENT", +#endif +#ifdef SQLITE_OMIT_AUTOINIT + "OMIT_AUTOINIT", +#endif +#ifdef SQLITE_OMIT_AUTOMATIC_INDEX + "OMIT_AUTOMATIC_INDEX", +#endif +#ifdef SQLITE_OMIT_AUTORESET + "OMIT_AUTORESET", +#endif +#ifdef SQLITE_OMIT_AUTOVACUUM + "OMIT_AUTOVACUUM", +#endif +#ifdef SQLITE_OMIT_BETWEEN_OPTIMIZATION + "OMIT_BETWEEN_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_BLOB_LITERAL + "OMIT_BLOB_LITERAL", +#endif +#ifdef SQLITE_OMIT_CAST + "OMIT_CAST", +#endif +#ifdef SQLITE_OMIT_CHECK + "OMIT_CHECK", +#endif +#ifdef SQLITE_OMIT_COMPLETE + "OMIT_COMPLETE", +#endif +#ifdef SQLITE_OMIT_COMPOUND_SELECT + "OMIT_COMPOUND_SELECT", +#endif +#ifdef SQLITE_OMIT_CONFLICT_CLAUSE + "OMIT_CONFLICT_CLAUSE", +#endif +#ifdef SQLITE_OMIT_CTE + "OMIT_CTE", +#endif +#if defined(SQLITE_OMIT_DATETIME_FUNCS) || defined(SQLITE_OMIT_FLOATING_POINT) + "OMIT_DATETIME_FUNCS", +#endif +#ifdef SQLITE_OMIT_DECLTYPE + "OMIT_DECLTYPE", +#endif +#ifdef SQLITE_OMIT_DEPRECATED + "OMIT_DEPRECATED", +#endif +#ifdef SQLITE_OMIT_DESERIALIZE + "OMIT_DESERIALIZE", +#endif +#ifdef SQLITE_OMIT_DISKIO + "OMIT_DISKIO", +#endif +#ifdef SQLITE_OMIT_EXPLAIN + "OMIT_EXPLAIN", +#endif +#ifdef SQLITE_OMIT_FLAG_PRAGMAS + "OMIT_FLAG_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_FLOATING_POINT + "OMIT_FLOATING_POINT", +#endif +#ifdef SQLITE_OMIT_FOREIGN_KEY + "OMIT_FOREIGN_KEY", +#endif +#ifdef SQLITE_OMIT_GET_TABLE + "OMIT_GET_TABLE", +#endif +#ifdef SQLITE_OMIT_HEX_INTEGER + "OMIT_HEX_INTEGER", +#endif +#ifdef SQLITE_OMIT_INCRBLOB + "OMIT_INCRBLOB", +#endif +#ifdef SQLITE_OMIT_INTEGRITY_CHECK + "OMIT_INTEGRITY_CHECK", +#endif +#ifdef SQLITE_OMIT_INTROSPECTION_PRAGMAS + "OMIT_INTROSPECTION_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_LIKE_OPTIMIZATION + "OMIT_LIKE_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_LOAD_EXTENSION + "OMIT_LOAD_EXTENSION", +#endif +#ifdef SQLITE_OMIT_LOCALTIME + "OMIT_LOCALTIME", +#endif +#ifdef SQLITE_OMIT_LOOKASIDE + "OMIT_LOOKASIDE", +#endif +#ifdef SQLITE_OMIT_MEMORYDB + "OMIT_MEMORYDB", +#endif +#ifdef SQLITE_OMIT_OR_OPTIMIZATION + "OMIT_OR_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_PAGER_PRAGMAS + "OMIT_PAGER_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_PARSER_TRACE + "OMIT_PARSER_TRACE", +#endif +#ifdef SQLITE_OMIT_POPEN + "OMIT_POPEN", +#endif +#ifdef SQLITE_OMIT_PRAGMA + "OMIT_PRAGMA", +#endif +#ifdef SQLITE_OMIT_PROGRESS_CALLBACK + "OMIT_PROGRESS_CALLBACK", +#endif +#ifdef SQLITE_OMIT_QUICKBALANCE + "OMIT_QUICKBALANCE", +#endif +#ifdef SQLITE_OMIT_REINDEX + "OMIT_REINDEX", +#endif +#ifdef SQLITE_OMIT_SCHEMA_PRAGMAS + "OMIT_SCHEMA_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS + "OMIT_SCHEMA_VERSION_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_SHARED_CACHE + "OMIT_SHARED_CACHE", +#endif +#ifdef SQLITE_OMIT_SHUTDOWN_DIRECTORIES + "OMIT_SHUTDOWN_DIRECTORIES", +#endif +#ifdef SQLITE_OMIT_SUBQUERY + "OMIT_SUBQUERY", +#endif +#ifdef SQLITE_OMIT_TCL_VARIABLE + "OMIT_TCL_VARIABLE", +#endif +#ifdef SQLITE_OMIT_TEMPDB + "OMIT_TEMPDB", +#endif +#ifdef SQLITE_OMIT_TEST_CONTROL + "OMIT_TEST_CONTROL", +#endif +#ifdef SQLITE_OMIT_TRACE +# if SQLITE_OMIT_TRACE != 1 + "OMIT_TRACE=" CTIMEOPT_VAL(SQLITE_OMIT_TRACE), +# endif +#endif +#ifdef SQLITE_OMIT_TRIGGER + "OMIT_TRIGGER", +#endif +#ifdef SQLITE_OMIT_TRUNCATE_OPTIMIZATION + "OMIT_TRUNCATE_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_UTF16 + "OMIT_UTF16", +#endif +#ifdef SQLITE_OMIT_VACUUM + "OMIT_VACUUM", +#endif +#ifdef SQLITE_OMIT_VIEW + "OMIT_VIEW", +#endif +#ifdef SQLITE_OMIT_VIRTUALTABLE + "OMIT_VIRTUALTABLE", +#endif +#ifdef SQLITE_OMIT_WAL + "OMIT_WAL", +#endif +#ifdef SQLITE_OMIT_WSD + "OMIT_WSD", +#endif +#ifdef SQLITE_OMIT_XFER_OPT + "OMIT_XFER_OPT", +#endif +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + "PCACHE_SEPARATE_HEADER", +#endif +#ifdef SQLITE_PERFORMANCE_TRACE + "PERFORMANCE_TRACE", +#endif +#ifdef SQLITE_POWERSAFE_OVERWRITE +# if SQLITE_POWERSAFE_OVERWRITE != 1 + "POWERSAFE_OVERWRITE=" CTIMEOPT_VAL(SQLITE_POWERSAFE_OVERWRITE), +# endif +#endif +#ifdef SQLITE_PREFER_PROXY_LOCKING + "PREFER_PROXY_LOCKING", +#endif +#ifdef SQLITE_PROXY_DEBUG + "PROXY_DEBUG", +#endif +#ifdef SQLITE_REVERSE_UNORDERED_SELECTS + "REVERSE_UNORDERED_SELECTS", +#endif +#ifdef SQLITE_RTREE_INT_ONLY + "RTREE_INT_ONLY", +#endif +#ifdef SQLITE_SECURE_DELETE + "SECURE_DELETE", +#endif +#ifdef SQLITE_SMALL_STACK + "SMALL_STACK", +#endif +#ifdef SQLITE_SORTER_PMASZ + "SORTER_PMASZ=" CTIMEOPT_VAL(SQLITE_SORTER_PMASZ), +#endif +#ifdef SQLITE_SOUNDEX + "SOUNDEX", +#endif +#ifdef SQLITE_STAT4_SAMPLES + "STAT4_SAMPLES=" CTIMEOPT_VAL(SQLITE_STAT4_SAMPLES), +#endif +#ifdef SQLITE_STMTJRNL_SPILL + "STMTJRNL_SPILL=" CTIMEOPT_VAL(SQLITE_STMTJRNL_SPILL), +#endif +#ifdef SQLITE_SUBSTR_COMPATIBILITY + "SUBSTR_COMPATIBILITY", +#endif +#if (!defined(SQLITE_WIN32_MALLOC) \ + && !defined(SQLITE_ZERO_MALLOC) \ + && !defined(SQLITE_MEMDEBUG) \ + ) || defined(SQLITE_SYSTEM_MALLOC) + "SYSTEM_MALLOC", +#endif +#ifdef SQLITE_TCL + "TCL", +#endif +#ifdef SQLITE_TEMP_STORE + "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), +#endif +#ifdef SQLITE_TEST + "TEST", +#endif +#if defined(SQLITE_THREADSAFE) + "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), +#elif defined(THREADSAFE) + "THREADSAFE=" CTIMEOPT_VAL(THREADSAFE), +#else + "THREADSAFE=1", +#endif +#ifdef SQLITE_UNLINK_AFTER_CLOSE + "UNLINK_AFTER_CLOSE", +#endif +#ifdef SQLITE_UNTESTABLE + "UNTESTABLE", +#endif +#ifdef SQLITE_USER_AUTHENTICATION + "USER_AUTHENTICATION", +#endif +#ifdef SQLITE_USE_ALLOCA + "USE_ALLOCA", +#endif +#ifdef SQLITE_USE_FCNTL_TRACE + "USE_FCNTL_TRACE", +#endif +#ifdef SQLITE_USE_URI + "USE_URI", +#endif +#ifdef SQLITE_VDBE_COVERAGE + "VDBE_COVERAGE", +#endif +#ifdef SQLITE_WIN32_MALLOC + "WIN32_MALLOC", +#endif +#ifdef SQLITE_ZERO_MALLOC + "ZERO_MALLOC", +#endif +/* +** END CODE GENERATED BY tool/mkctime.tcl +*/ +}; + +SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt){ + *pnOpt = sizeof(sqlite3azCompileOpt) / sizeof(sqlite3azCompileOpt[0]); + return (const char**)sqlite3azCompileOpt; +} + +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +/************** End of ctime.c ***********************************************/ +/************** Begin file global.c ******************************************/ +/* +** 2008 June 13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains definitions of global variables and constants. +*/ +/* #include "sqliteInt.h" */ + +/* An array to map all upper-case characters into their corresponding +** lower-case character. +** +** SQLite only considers US-ASCII (or EBCDIC) characters. We do not +** handle case conversions for the UTF character set since the tables +** involved are nearly as big or bigger than SQLite itself. +*/ +SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { +#ifdef SQLITE_ASCII + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121, + 122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107, + 108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125, + 126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161, + 162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179, + 180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197, + 198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233, + 234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251, + 252,253,254,255, +#endif +#ifdef SQLITE_EBCDIC + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, /* 0x */ + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, /* 1x */ + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, /* 2x */ + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, /* 3x */ + 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, /* 4x */ + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, /* 5x */ + 96, 97, 98, 99,100,101,102,103,104,105,106,107,108,109,110,111, /* 6x */ + 112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127, /* 7x */ + 128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, /* 8x */ + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159, /* 9x */ + 160,161,162,163,164,165,166,167,168,169,170,171,140,141,142,175, /* Ax */ + 176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191, /* Bx */ + 192,129,130,131,132,133,134,135,136,137,202,203,204,205,206,207, /* Cx */ + 208,145,146,147,148,149,150,151,152,153,218,219,220,221,222,223, /* Dx */ + 224,225,162,163,164,165,166,167,168,169,234,235,236,237,238,239, /* Ex */ + 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255, /* Fx */ +#endif +/* All of the upper-to-lower conversion data is above. The following +** 18 integers are completely unrelated. They are appended to the +** sqlite3UpperToLower[] array to avoid UBSAN warnings. Here's what is +** going on: +** +** The SQL comparison operators (<>, =, >, <=, <, and >=) are implemented +** by invoking sqlite3MemCompare(A,B) which compares values A and B and +** returns negative, zero, or positive if A is less then, equal to, or +** greater than B, respectively. Then the true false results is found by +** consulting sqlite3aLTb[opcode], sqlite3aEQb[opcode], or +** sqlite3aGTb[opcode] depending on whether the result of compare(A,B) +** is negative, zero, or positive, where opcode is the specific opcode. +** The only works because the comparison opcodes are consecutive and in +** this order: NE EQ GT LE LT GE. Various assert()s throughout the code +** ensure that is the case. +** +** These elements must be appended to another array. Otherwise the +** index (here shown as [256-OP_Ne]) would be out-of-bounds and thus +** be undefined behavior. That's goofy, but the C-standards people thought +** it was a good idea, so here we are. +*/ +/* NE EQ GT LE LT GE */ + 1, 0, 0, 1, 1, 0, /* aLTb[]: Use when compare(A,B) less than zero */ + 0, 1, 0, 1, 0, 1, /* aEQb[]: Use when compare(A,B) equals zero */ + 1, 0, 1, 0, 0, 1 /* aGTb[]: Use when compare(A,B) greater than zero*/ +}; +SQLITE_PRIVATE const unsigned char *sqlite3aLTb = &sqlite3UpperToLower[256-OP_Ne]; +SQLITE_PRIVATE const unsigned char *sqlite3aEQb = &sqlite3UpperToLower[256+6-OP_Ne]; +SQLITE_PRIVATE const unsigned char *sqlite3aGTb = &sqlite3UpperToLower[256+12-OP_Ne]; + +/* +** The following 256 byte lookup table is used to support SQLites built-in +** equivalents to the following standard library functions: +** +** isspace() 0x01 +** isalpha() 0x02 +** isdigit() 0x04 +** isalnum() 0x06 +** isxdigit() 0x08 +** toupper() 0x20 +** SQLite identifier character 0x40 +** Quote character 0x80 +** +** Bit 0x20 is set if the mapped character requires translation to upper +** case. i.e. if the character is a lower-case ASCII character. +** If x is a lower-case ASCII character, then its upper-case equivalent +** is (x - 0x20). Therefore toupper() can be implemented as: +** +** (x & ~(map[x]&0x20)) +** +** The equivalent of tolower() is implemented using the sqlite3UpperToLower[] +** array. tolower() is used more often than toupper() by SQLite. +** +** Bit 0x40 is set if the character is non-alphanumeric and can be used in an +** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any +** non-ASCII UTF character. Hence the test for whether or not a character is +** part of an identifier is 0x46. +*/ +SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */ + 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */ + 0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80, /* 20..27 !"#$%&' */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */ + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */ + 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */ + + 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ + 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ + 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ + 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ +}; + +/* EVIDENCE-OF: R-02982-34736 In order to maintain full backwards +** compatibility for legacy applications, the URI filename capability is +** disabled by default. +** +** EVIDENCE-OF: R-38799-08373 URI filenames can be enabled or disabled +** using the SQLITE_USE_URI=1 or SQLITE_USE_URI=0 compile-time options. +** +** EVIDENCE-OF: R-43642-56306 By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** SQLITE_USE_URI symbol defined. +*/ +#ifndef SQLITE_USE_URI +# define SQLITE_USE_URI 0 +#endif + +/* EVIDENCE-OF: R-38720-18127 The default setting is determined by the +** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if +** that compile-time option is omitted. +*/ +#if !defined(SQLITE_ALLOW_COVERING_INDEX_SCAN) +# define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 +#else +# if !SQLITE_ALLOW_COVERING_INDEX_SCAN +# error "Compile-time disabling of covering index scan using the\ + -DSQLITE_ALLOW_COVERING_INDEX_SCAN=0 option is deprecated.\ + Contact SQLite developers if this is a problem for you, and\ + delete this #error macro to continue with your build." +# endif +#endif + +/* The minimum PMA size is set to this value multiplied by the database +** page size in bytes. +*/ +#ifndef SQLITE_SORTER_PMASZ +# define SQLITE_SORTER_PMASZ 250 +#endif + +/* Statement journals spill to disk when their size exceeds the following +** threshold (in bytes). 0 means that statement journals are created and +** written to disk immediately (the default behavior for SQLite versions +** before 3.12.0). -1 means always keep the entire statement journal in +** memory. (The statement journal is also always held entirely in memory +** if journal_mode=MEMORY or if temp_store=MEMORY, regardless of this +** setting.) +*/ +#ifndef SQLITE_STMTJRNL_SPILL +# define SQLITE_STMTJRNL_SPILL (64*1024) +#endif + +/* +** The default lookaside-configuration, the format "SZ,N". SZ is the +** number of bytes in each lookaside slot (should be a multiple of 8) +** and N is the number of slots. The lookaside-configuration can be +** changed as start-time using sqlite3_config(SQLITE_CONFIG_LOOKASIDE) +** or at run-time for an individual database connection using +** sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE); +** +** With the two-size-lookaside enhancement, less lookaside is required. +** The default configuration of 1200,40 actually provides 30 1200-byte slots +** and 93 128-byte slots, which is more lookaside than is available +** using the older 1200,100 configuration without two-size-lookaside. +*/ +#ifndef SQLITE_DEFAULT_LOOKASIDE +# ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE +# define SQLITE_DEFAULT_LOOKASIDE 1200,100 /* 120KB of memory */ +# else +# define SQLITE_DEFAULT_LOOKASIDE 1200,40 /* 48KB of memory */ +# endif +#endif + + +/* The default maximum size of an in-memory database created using +** sqlite3_deserialize() +*/ +#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE +# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824 +#endif + +/* +** The following singleton contains the global configuration for +** the SQLite library. +*/ +SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { + SQLITE_DEFAULT_MEMSTATUS, /* bMemstat */ + 1, /* bCoreMutex */ + SQLITE_THREADSAFE==1, /* bFullMutex */ + SQLITE_USE_URI, /* bOpenUri */ + SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ + 0, /* bSmallMalloc */ + 1, /* bExtraSchemaChecks */ + 0x7ffffffe, /* mxStrlen */ + 0, /* neverCorrupt */ + SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ + SQLITE_STMTJRNL_SPILL, /* nStmtSpill */ + {0,0,0,0,0,0,0,0}, /* m */ + {0,0,0,0,0,0,0,0,0}, /* mutex */ + {0,0,0,0,0,0,0,0,0,0,0,0,0},/* pcache2 */ + (void*)0, /* pHeap */ + 0, /* nHeap */ + 0, 0, /* mnHeap, mxHeap */ + SQLITE_DEFAULT_MMAP_SIZE, /* szMmap */ + SQLITE_MAX_MMAP_SIZE, /* mxMmap */ + (void*)0, /* pPage */ + 0, /* szPage */ + SQLITE_DEFAULT_PCACHE_INITSZ, /* nPage */ + 0, /* mxParserStack */ + 0, /* sharedCacheEnabled */ + SQLITE_SORTER_PMASZ, /* szPma */ + /* All the rest should always be initialized to zero */ + 0, /* isInit */ + 0, /* inProgress */ + 0, /* isMutexInit */ + 0, /* isMallocInit */ + 0, /* isPCacheInit */ + 0, /* nRefInitMutex */ + 0, /* pInitMutex */ + 0, /* xLog */ + 0, /* pLogArg */ +#ifdef SQLITE_ENABLE_SQLLOG + 0, /* xSqllog */ + 0, /* pSqllogArg */ +#endif +#ifdef SQLITE_VDBE_COVERAGE + 0, /* xVdbeBranch */ + 0, /* pVbeBranchArg */ +#endif +#ifndef SQLITE_OMIT_DESERIALIZE + SQLITE_MEMDB_DEFAULT_MAXSIZE, /* mxMemdbSize */ +#endif +#ifndef SQLITE_UNTESTABLE + 0, /* xTestCallback */ +#endif + 0, /* bLocaltimeFault */ + 0x7ffffffe, /* iOnceResetThreshold */ + SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ + 0, /* iPrngSeed */ +}; + +/* +** Hash table for global functions - functions common to all +** database connections. After initialization, this table is +** read-only. +*/ +SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; + +#ifdef VDBE_PROFILE +/* +** The following performance counter can be used in place of +** sqlite3Hwtime() for profiling. This is a no-op on standard builds. +*/ +SQLITE_PRIVATE sqlite3_uint64 sqlite3NProfileCnt = 0; +#endif + +/* +** The value of the "pending" byte must be 0x40000000 (1 byte past the +** 1-gibabyte boundary) in a compatible database. SQLite never uses +** the database page that contains the pending byte. It never attempts +** to read or write that page. The pending byte page is set aside +** for use by the VFS layers as space for managing file locks. +** +** During testing, it is often desirable to move the pending byte to +** a different position in the file. This allows code that has to +** deal with the pending byte to run on files that are much smaller +** than 1 GiB. The sqlite3_test_control() interface can be used to +** move the pending byte. +** +** IMPORTANT: Changing the pending byte to any value other than +** 0x40000000 results in an incompatible database file format! +** Changing the pending byte during operation will result in undefined +** and incorrect behavior. +*/ +#ifndef SQLITE_OMIT_WSD +SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000; +#endif + +/* +** Tracing flags set by SQLITE_TESTCTRL_TRACEFLAGS. +*/ +SQLITE_PRIVATE u32 sqlite3SelectTrace = 0; +SQLITE_PRIVATE u32 sqlite3WhereTrace = 0; + +/* #include "opcodes.h" */ +/* +** Properties of opcodes. The OPFLG_INITIALIZER macro is +** created by mkopcodeh.awk during compilation. Data is obtained +** from the comments following the "case OP_xxxx:" statements in +** the vdbe.c file. +*/ +SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; + +/* +** Name of the default collating sequence +*/ +SQLITE_PRIVATE const char sqlite3StrBINARY[] = "BINARY"; + +/* +** Standard typenames. These names must match the COLTYPE_* definitions. +** Adjust the SQLITE_N_STDTYPE value if adding or removing entries. +** +** sqlite3StdType[] The actual names of the datatypes. +** +** sqlite3StdTypeLen[] The length (in bytes) of each entry +** in sqlite3StdType[]. +** +** sqlite3StdTypeAffinity[] The affinity associated with each entry +** in sqlite3StdType[]. +** +** sqlite3StdTypeMap[] The type value (as returned from +** sqlite3_column_type() or sqlite3_value_type()) +** for each entry in sqlite3StdType[]. +*/ +SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[] = { 3, 4, 3, 7, 4, 4 }; +SQLITE_PRIVATE const char sqlite3StdTypeAffinity[] = { + SQLITE_AFF_NUMERIC, + SQLITE_AFF_BLOB, + SQLITE_AFF_INTEGER, + SQLITE_AFF_INTEGER, + SQLITE_AFF_REAL, + SQLITE_AFF_TEXT +}; +SQLITE_PRIVATE const char sqlite3StdTypeMap[] = { + 0, + SQLITE_BLOB, + SQLITE_INTEGER, + SQLITE_INTEGER, + SQLITE_FLOAT, + SQLITE_TEXT +}; +SQLITE_PRIVATE const char *sqlite3StdType[] = { + "ANY", + "BLOB", + "INT", + "INTEGER", + "REAL", + "TEXT" +}; + +/************** End of global.c **********************************************/ +/************** Begin file status.c ******************************************/ +/* +** 2008 June 18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module implements the sqlite3_status() interface and related +** functionality. +*/ +/* #include "sqliteInt.h" */ +/************** Include vdbeInt.h in the middle of status.c ******************/ +/************** Begin file vdbeInt.h *****************************************/ +/* +** 2003 September 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for information that is private to the +** VDBE. This information used to all be at the top of the single +** source code file "vdbe.c". When that file became too big (over +** 6000 lines long) it was split up into several smaller files and +** this header information was factored out. +*/ +#ifndef SQLITE_VDBEINT_H +#define SQLITE_VDBEINT_H + +/* +** The maximum number of times that a statement will try to reparse +** itself before giving up and returning SQLITE_SCHEMA. +*/ +#ifndef SQLITE_MAX_SCHEMA_RETRY +# define SQLITE_MAX_SCHEMA_RETRY 50 +#endif + +/* +** VDBE_DISPLAY_P4 is true or false depending on whether or not the +** "explain" P4 display logic is enabled. +*/ +#if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ + || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) \ + || defined(SQLITE_ENABLE_BYTECODE_VTAB) +# define VDBE_DISPLAY_P4 1 +#else +# define VDBE_DISPLAY_P4 0 +#endif + +/* +** SQL is translated into a sequence of instructions to be +** executed by a virtual machine. Each instruction is an instance +** of the following structure. +*/ +typedef struct VdbeOp Op; + +/* +** Boolean values +*/ +typedef unsigned Bool; + +/* Opaque type used by code in vdbesort.c */ +typedef struct VdbeSorter VdbeSorter; + +/* Elements of the linked list at Vdbe.pAuxData */ +typedef struct AuxData AuxData; + +/* Types of VDBE cursors */ +#define CURTYPE_BTREE 0 +#define CURTYPE_SORTER 1 +#define CURTYPE_VTAB 2 +#define CURTYPE_PSEUDO 3 + +/* +** A VdbeCursor is an superclass (a wrapper) for various cursor objects: +** +** * A b-tree cursor +** - In the main database or in an ephemeral database +** - On either an index or a table +** * A sorter +** * A virtual table +** * A one-row "pseudotable" stored in a single register +*/ +typedef struct VdbeCursor VdbeCursor; +struct VdbeCursor { + u8 eCurType; /* One of the CURTYPE_* values above */ + i8 iDb; /* Index of cursor database in db->aDb[] (or -1) */ + u8 nullRow; /* True if pointing to a row with no data */ + u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ + u8 isTable; /* True for rowid tables. False for indexes */ +#ifdef SQLITE_DEBUG + u8 seekOp; /* Most recent seek operation on this cursor */ + u8 wrFlag; /* The wrFlag argument to sqlite3BtreeCursor() */ +#endif + Bool isEphemeral:1; /* True for an ephemeral table */ + Bool useRandomRowid:1; /* Generate new record numbers semi-randomly */ + Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ + Bool hasBeenDuped:1; /* This cursor was source or target of OP_OpenDup */ + u16 seekHit; /* See the OP_SeekHit and OP_IfNoHope opcodes */ + Btree *pBtx; /* Separate file holding temporary table */ + i64 seqCount; /* Sequence counter */ + u32 *aAltMap; /* Mapping from table to index column numbers */ + + /* Cached OP_Column parse information is only valid if cacheStatus matches + ** Vdbe.cacheCtr. Vdbe.cacheCtr will never take on the value of + ** CACHE_STALE (0) and so setting cacheStatus=CACHE_STALE guarantees that + ** the cache is out of date. */ + u32 cacheStatus; /* Cache is valid if this matches Vdbe.cacheCtr */ + int seekResult; /* Result of previous sqlite3BtreeMoveto() or 0 + ** if there have been no prior seeks on the cursor. */ + /* seekResult does not distinguish between "no seeks have ever occurred + ** on this cursor" and "the most recent seek was an exact match". + ** For CURTYPE_PSEUDO, seekResult is the register holding the record */ + + /* When a new VdbeCursor is allocated, only the fields above are zeroed. + ** The fields that follow are uninitialized, and must be individually + ** initialized prior to first use. */ + VdbeCursor *pAltCursor; /* Associated index cursor from which to read */ + union { + BtCursor *pCursor; /* CURTYPE_BTREE or _PSEUDO. Btree cursor */ + sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB. Vtab cursor */ + VdbeSorter *pSorter; /* CURTYPE_SORTER. Sorter object */ + } uc; + KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ + u32 iHdrOffset; /* Offset to next unparsed byte of the header */ + Pgno pgnoRoot; /* Root page of the open btree cursor */ + i16 nField; /* Number of fields in the header */ + u16 nHdrParsed; /* Number of header fields parsed so far */ + i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ + u32 *aOffset; /* Pointer to aType[nField] */ + const u8 *aRow; /* Data for the current row, if all on one page */ + u32 payloadSize; /* Total number of bytes in the record */ + u32 szRow; /* Byte available in aRow */ +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + u64 maskUsed; /* Mask of columns used by this cursor */ +#endif + + /* 2*nField extra array elements allocated for aType[], beyond the one + ** static element declared in the structure. nField total array slots for + ** aType[] and nField+1 array slots for aOffset[] */ + u32 aType[1]; /* Type values record decode. MUST BE LAST */ +}; + + +/* +** A value for VdbeCursor.cacheStatus that means the cache is always invalid. +*/ +#define CACHE_STALE 0 + +/* +** When a sub-program is executed (OP_Program), a structure of this type +** is allocated to store the current value of the program counter, as +** well as the current memory cell array and various other frame specific +** values stored in the Vdbe struct. When the sub-program is finished, +** these values are copied back to the Vdbe from the VdbeFrame structure, +** restoring the state of the VM to as it was before the sub-program +** began executing. +** +** The memory for a VdbeFrame object is allocated and managed by a memory +** cell in the parent (calling) frame. When the memory cell is deleted or +** overwritten, the VdbeFrame object is not freed immediately. Instead, it +** is linked into the Vdbe.pDelFrame list. The contents of the Vdbe.pDelFrame +** list is deleted when the VM is reset in VdbeHalt(). The reason for doing +** this instead of deleting the VdbeFrame immediately is to avoid recursive +** calls to sqlite3VdbeMemRelease() when the memory cells belonging to the +** child frame are released. +** +** The currently executing frame is stored in Vdbe.pFrame. Vdbe.pFrame is +** set to NULL if the currently executing frame is the main program. +*/ +typedef struct VdbeFrame VdbeFrame; +struct VdbeFrame { + Vdbe *v; /* VM this frame belongs to */ + VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ + Op *aOp; /* Program instructions for parent frame */ + i64 *anExec; /* Event counters from parent frame */ + Mem *aMem; /* Array of memory cells for parent frame */ + VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ + u8 *aOnce; /* Bitmask used by OP_Once */ + void *token; /* Copy of SubProgram.token */ + i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ + AuxData *pAuxData; /* Linked list of auxdata allocations */ +#if SQLITE_DEBUG + u32 iFrameMagic; /* magic number for sanity checking */ +#endif + int nCursor; /* Number of entries in apCsr */ + int pc; /* Program Counter in parent (calling) frame */ + int nOp; /* Size of aOp array */ + int nMem; /* Number of entries in aMem */ + int nChildMem; /* Number of memory cells for child frame */ + int nChildCsr; /* Number of cursors for child frame */ + i64 nChange; /* Statement changes (Vdbe.nChange) */ + i64 nDbChange; /* Value of db->nChange */ +}; + +/* Magic number for sanity checking on VdbeFrame objects */ +#define SQLITE_FRAME_MAGIC 0x879fb71e + +/* +** Return a pointer to the array of registers allocated for use +** by a VdbeFrame. +*/ +#define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) + +/* +** Internally, the vdbe manipulates nearly all SQL values as Mem +** structures. Each Mem struct may cache multiple representations (string, +** integer etc.) of the same value. +*/ +struct sqlite3_value { + union MemValue { + double r; /* Real value used when MEM_Real is set in flags */ + i64 i; /* Integer value used when MEM_Int is set in flags */ + int nZero; /* Extra zero bytes when MEM_Zero and MEM_Blob set */ + const char *zPType; /* Pointer type when MEM_Term|MEM_Subtype|MEM_Null */ + FuncDef *pDef; /* Used only when flags==MEM_Agg */ + } u; + u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ + u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ + u8 eSubtype; /* Subtype for this value */ + int n; /* Number of characters in string value, excluding '\0' */ + char *z; /* String or BLOB value */ + /* ShallowCopy only needs to copy the information above */ + char *zMalloc; /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */ + int szMalloc; /* Size of the zMalloc allocation */ + u32 uTemp; /* Transient storage for serial_type in OP_MakeRecord */ + sqlite3 *db; /* The associated database connection */ + void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ +#ifdef SQLITE_DEBUG + Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ + u16 mScopyFlags; /* flags value immediately after the shallow copy */ +#endif +}; + +/* +** Size of struct Mem not including the Mem.zMalloc member or anything that +** follows. +*/ +#define MEMCELLSIZE offsetof(Mem,zMalloc) + +/* One or more of the following flags are set to indicate the validOK +** representations of the value stored in the Mem struct. +** +** If the MEM_Null flag is set, then the value is an SQL NULL value. +** For a pointer type created using sqlite3_bind_pointer() or +** sqlite3_result_pointer() the MEM_Term and MEM_Subtype flags are also set. +** +** If the MEM_Str flag is set then Mem.z points at a string representation. +** Usually this is encoded in the same unicode encoding as the main +** database (see below for exceptions). If the MEM_Term flag is also +** set, then the string is nul terminated. The MEM_Int and MEM_Real +** flags may coexist with the MEM_Str flag. +*/ +#define MEM_Null 0x0001 /* Value is NULL (or a pointer) */ +#define MEM_Str 0x0002 /* Value is a string */ +#define MEM_Int 0x0004 /* Value is an integer */ +#define MEM_Real 0x0008 /* Value is a real number */ +#define MEM_Blob 0x0010 /* Value is a BLOB */ +#define MEM_IntReal 0x0020 /* MEM_Int that stringifies like MEM_Real */ +#define MEM_AffMask 0x003f /* Mask of affinity bits */ +#define MEM_FromBind 0x0040 /* Value originates from sqlite3_bind() */ +#define MEM_Undefined 0x0080 /* Value is undefined */ +#define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ +#define MEM_TypeMask 0xc1bf /* Mask of type bits */ + + +/* Whenever Mem contains a valid string or blob representation, one of +** the following flags must be set to determine the memory management +** policy for Mem.z. The MEM_Term flag tells us whether or not the +** string is \000 or \u0000 terminated +*/ +#define MEM_Term 0x0200 /* String in Mem.z is zero terminated */ +#define MEM_Dyn 0x0400 /* Need to call Mem.xDel() on Mem.z */ +#define MEM_Static 0x0800 /* Mem.z points to a static string */ +#define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ +#define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ +#define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ +#define MEM_Subtype 0x8000 /* Mem.eSubtype is valid */ +#ifdef SQLITE_OMIT_INCRBLOB + #undef MEM_Zero + #define MEM_Zero 0x0000 +#endif + +/* Return TRUE if Mem X contains dynamically allocated content - anything +** that needs to be deallocated to avoid a leak. +*/ +#define VdbeMemDynamic(X) \ + (((X)->flags&(MEM_Agg|MEM_Dyn))!=0) + +/* +** Clear any existing type flags from a Mem and replace them with f +*/ +#define MemSetTypeFlag(p, f) \ + ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) + +/* +** True if Mem X is a NULL-nochng type. +*/ +#define MemNullNochng(X) \ + (((X)->flags&MEM_TypeMask)==(MEM_Null|MEM_Zero) \ + && (X)->n==0 && (X)->u.nZero==0) + +/* +** Return true if a memory cell is not marked as invalid. This macro +** is for use inside assert() statements only. +*/ +#ifdef SQLITE_DEBUG +#define memIsValid(M) ((M)->flags & MEM_Undefined)==0 +#endif + +/* +** Each auxiliary data pointer stored by a user defined function +** implementation calling sqlite3_set_auxdata() is stored in an instance +** of this structure. All such structures associated with a single VM +** are stored in a linked list headed at Vdbe.pAuxData. All are destroyed +** when the VM is halted (if not before). +*/ +struct AuxData { + int iAuxOp; /* Instruction number of OP_Function opcode */ + int iAuxArg; /* Index of function argument. */ + void *pAux; /* Aux data pointer */ + void (*xDeleteAux)(void*); /* Destructor for the aux data */ + AuxData *pNextAux; /* Next element in list */ +}; + +/* +** The "context" argument for an installable function. A pointer to an +** instance of this structure is the first argument to the routines used +** implement the SQL functions. +** +** There is a typedef for this structure in sqlite.h. So all routines, +** even the public interface to SQLite, can use a pointer to this structure. +** But this file is the only place where the internal details of this +** structure are known. +** +** This structure is defined inside of vdbeInt.h because it uses substructures +** (Mem) which are only defined there. +*/ +struct sqlite3_context { + Mem *pOut; /* The return value is stored here */ + FuncDef *pFunc; /* Pointer to function information */ + Mem *pMem; /* Memory cell used to store aggregate context */ + Vdbe *pVdbe; /* The VM that owns this context */ + int iOp; /* Instruction number of OP_Function */ + int isError; /* Error code returned by the function. */ + u8 skipFlag; /* Skip accumulator loading if true */ + u8 argc; /* Number of arguments */ + sqlite3_value *argv[1]; /* Argument set */ +}; + +/* A bitfield type for use inside of structures. Always follow with :N where +** N is the number of bits. +*/ +typedef unsigned bft; /* Bit Field Type */ + +/* The ScanStatus object holds a single value for the +** sqlite3_stmt_scanstatus() interface. +*/ +typedef struct ScanStatus ScanStatus; +struct ScanStatus { + int addrExplain; /* OP_Explain for loop */ + int addrLoop; /* Address of "loops" counter */ + int addrVisit; /* Address of "rows visited" counter */ + int iSelectID; /* The "Select-ID" for this loop */ + LogEst nEst; /* Estimated output rows per loop */ + char *zName; /* Name of table or index */ +}; + +/* The DblquoteStr object holds the text of a double-quoted +** string for a prepared statement. A linked list of these objects +** is constructed during statement parsing and is held on Vdbe.pDblStr. +** When computing a normalized SQL statement for an SQL statement, that +** list is consulted for each double-quoted identifier to see if the +** identifier should really be a string literal. +*/ +typedef struct DblquoteStr DblquoteStr; +struct DblquoteStr { + DblquoteStr *pNextStr; /* Next string literal in the list */ + char z[8]; /* Dequoted value for the string */ +}; + +/* +** An instance of the virtual machine. This structure contains the complete +** state of the virtual machine. +** +** The "sqlite3_stmt" structure pointer that is returned by sqlite3_prepare() +** is really a pointer to an instance of this structure. +*/ +struct Vdbe { + sqlite3 *db; /* The database connection that owns this statement */ + Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ + Parse *pParse; /* Parsing context used to create this Vdbe */ + ynVar nVar; /* Number of entries in aVar[] */ + u32 iVdbeMagic; /* Magic number defining state of the SQL statement */ + int nMem; /* Number of memory locations currently allocated */ + int nCursor; /* Number of slots in apCsr[] */ + u32 cacheCtr; /* VdbeCursor row cache generation counter */ + int pc; /* The program counter */ + int rc; /* Value to return */ + i64 nChange; /* Number of db changes made since last reset */ + int iStatement; /* Statement number (or 0 if has no opened stmt) */ + i64 iCurrentTime; /* Value of julianday('now') for this statement */ + i64 nFkConstraint; /* Number of imm. FK constraints this VM */ + i64 nStmtDefCons; /* Number of def. constraints when stmt started */ + i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ + Mem *aMem; /* The memory locations */ + Mem **apArg; /* Arguments to currently executing user function */ + VdbeCursor **apCsr; /* One element of this array for each open cursor */ + Mem *aVar; /* Values for the OP_Variable opcode. */ + + /* When allocating a new Vdbe object, all of the fields below should be + ** initialized to zero or NULL */ + + Op *aOp; /* Space to hold the virtual machine's program */ + int nOp; /* Number of instructions in the program */ + int nOpAlloc; /* Slots allocated for aOp[] */ + Mem *aColName; /* Column names to return */ + Mem *pResultSet; /* Pointer to an array of results */ + char *zErrMsg; /* Error message written here */ + VList *pVList; /* Name of variables */ +#ifndef SQLITE_OMIT_TRACE + i64 startTime; /* Time when query started - used for profiling */ +#endif +#ifdef SQLITE_DEBUG + int rcApp; /* errcode set by sqlite3_result_error_code() */ + u32 nWrite; /* Number of write operations that have occurred */ +#endif + u16 nResColumn; /* Number of columns in one row of the result set */ + u8 errorAction; /* Recovery action to do in case of an error */ + u8 minWriteFileFormat; /* Minimum file format for writable database files */ + u8 prepFlags; /* SQLITE_PREPARE_* flags */ + u8 doingRerun; /* True if rerunning after an auto-reprepare */ + bft expired:2; /* 1: recompile VM immediately 2: when convenient */ + bft explain:2; /* True if EXPLAIN present on SQL command */ + bft changeCntOn:1; /* True to update the change-counter */ + bft runOnlyOnce:1; /* Automatically expire on reset */ + bft usesStmtJournal:1; /* True if uses a statement journal */ + bft readOnly:1; /* True for statements that do not write */ + bft bIsReader:1; /* True for statements that read */ + yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ + yDbMask lockMask; /* Subset of btreeMask that requires a lock */ + u32 aCounter[7]; /* Counters used by sqlite3_stmt_status() */ + char *zSql; /* Text of the SQL statement that generated this */ +#ifdef SQLITE_ENABLE_NORMALIZE + char *zNormSql; /* Normalization of the associated SQL statement */ + DblquoteStr *pDblStr; /* List of double-quoted string literals */ +#endif + void *pFree; /* Free this when deleting the vdbe */ + VdbeFrame *pFrame; /* Parent frame */ + VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ + int nFrame; /* Number of frames in pFrame list */ + u32 expmask; /* Binding to these vars invalidates VM */ + SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ + AuxData *pAuxData; /* Linked list of auxdata allocations */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + i64 *anExec; /* Number of times each op has been executed */ + int nScan; /* Entries in aScan[] */ + ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ +#endif +}; + +/* +** The following are allowed values for Vdbe.magic +*/ +#define VDBE_MAGIC_INIT 0x16bceaa5 /* Building a VDBE program */ +#define VDBE_MAGIC_RUN 0x2df20da3 /* VDBE is ready to execute */ +#define VDBE_MAGIC_HALT 0x319c2973 /* VDBE has completed execution */ +#define VDBE_MAGIC_RESET 0x48fa9f76 /* Reset and ready to run again */ +#define VDBE_MAGIC_DEAD 0x5606c3c8 /* The VDBE has been deallocated */ + +/* +** Structure used to store the context required by the +** sqlite3_preupdate_*() API functions. +*/ +struct PreUpdate { + Vdbe *v; + VdbeCursor *pCsr; /* Cursor to read old values from */ + int op; /* One of SQLITE_INSERT, UPDATE, DELETE */ + u8 *aRecord; /* old.* database record */ + KeyInfo keyinfo; + UnpackedRecord *pUnpacked; /* Unpacked version of aRecord[] */ + UnpackedRecord *pNewUnpacked; /* Unpacked version of new.* record */ + int iNewReg; /* Register for new.* values */ + int iBlobWrite; /* Value returned by preupdate_blobwrite() */ + i64 iKey1; /* First key value passed to hook */ + i64 iKey2; /* Second key value passed to hook */ + Mem *aNew; /* Array of new.* values */ + Table *pTab; /* Schema object being upated */ + Index *pPk; /* PK index if pTab is WITHOUT ROWID */ +}; + +/* +** Function prototypes +*/ +SQLITE_PRIVATE void sqlite3VdbeError(Vdbe*, const char *, ...); +SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); +void sqliteVdbePopStack(Vdbe*,int); +SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeFinishMoveto(VdbeCursor*); +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor**, u32*); +SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor*); +SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); +SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8); +SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32); +SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3*, AuxData**, int, int); + +int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); +SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(sqlite3*,VdbeCursor*,UnpackedRecord*,int*); +SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor*, i64*); +SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); +#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_BYTECODE_VTAB) +SQLITE_PRIVATE int sqlite3VdbeNextOpcode(Vdbe*,Mem*,int,int*,int*,Op**); +SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3*,Op*); +#endif +#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) +SQLITE_PRIVATE char *sqlite3VdbeDisplayComment(sqlite3*,const Op*,const char*); +#endif +#if !defined(SQLITE_OMIT_EXPLAIN) +SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); +#endif +SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *, int); +SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem*, const Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem*, const Mem*, int); +SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem*, Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemSetStr(Mem*, const char*, i64, u8, void(*)(void*)); +SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem*, i64); +#ifdef SQLITE_OMIT_FLOATING_POINT +# define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 +#else +SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem*, double); +#endif +SQLITE_PRIVATE void sqlite3VdbeMemSetPointer(Mem*, void*, const char*, void(*)(void*)); +SQLITE_PRIVATE void sqlite3VdbeMemInit(Mem*,sqlite3*,u16); +SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem*,int); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem*); +#endif +SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, u8, u8); +SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); +SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); +SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); +SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem*, int ifNull); +SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem*,u8,u8); +SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemFromBtreeZeroOffset(BtCursor*,u32,Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); +SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem*, Mem*, FuncDef*); +#endif +#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_BYTECODE_VTAB) +SQLITE_PRIVATE const char *sqlite3OpcodeName(int); +#endif +SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); +SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int n); +SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeFrameIsValid(VdbeFrame*); +#endif +SQLITE_PRIVATE void sqlite3VdbeFrameMemDel(void*); /* Destructor on Mem */ +SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); /* Actually deletes the Frame */ +SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( + Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int,int); +#endif +SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p); + +SQLITE_PRIVATE int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); +SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); +SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); +SQLITE_PRIVATE int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); +SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *); +SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *, int *); +SQLITE_PRIVATE int sqlite3VdbeSorterWrite(const VdbeCursor *, Mem *); +SQLITE_PRIVATE int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbeIncrWriteCounter(Vdbe*, VdbeCursor*); +SQLITE_PRIVATE void sqlite3VdbeAssertAbortable(Vdbe*); +#else +# define sqlite3VdbeIncrWriteCounter(V,C) +# define sqlite3VdbeAssertAbortable(V) +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) +SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe*); +#else +# define sqlite3VdbeEnter(X) +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe*); +#else +# define sqlite3VdbeLeave(X) +#endif + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe*,Mem*); +SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem*); +#endif + +#ifndef SQLITE_OMIT_FOREIGN_KEY +SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); +#else +# define sqlite3VdbeCheckFk(p,i) 0 +#endif + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr); +#endif +#ifndef SQLITE_OMIT_UTF16 +SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem*, u8); +SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem); +#endif + +#ifndef SQLITE_OMIT_INCRBLOB +SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); + #define ExpandBlob(P) (((P)->flags&MEM_Zero)?sqlite3VdbeMemExpandBlob(P):0) +#else + #define sqlite3VdbeMemExpandBlob(x) SQLITE_OK + #define ExpandBlob(P) SQLITE_OK +#endif + +#endif /* !defined(SQLITE_VDBEINT_H) */ + +/************** End of vdbeInt.h *********************************************/ +/************** Continuing where we left off in status.c *********************/ + +/* +** Variables in which to record status information. +*/ +#if SQLITE_PTRSIZE>4 +typedef sqlite3_int64 sqlite3StatValueType; +#else +typedef u32 sqlite3StatValueType; +#endif +typedef struct sqlite3StatType sqlite3StatType; +static SQLITE_WSD struct sqlite3StatType { + sqlite3StatValueType nowValue[10]; /* Current value */ + sqlite3StatValueType mxValue[10]; /* Maximum value */ +} sqlite3Stat = { {0,}, {0,} }; + +/* +** Elements of sqlite3Stat[] are protected by either the memory allocator +** mutex, or by the pcache1 mutex. The following array determines which. +*/ +static const char statMutex[] = { + 0, /* SQLITE_STATUS_MEMORY_USED */ + 1, /* SQLITE_STATUS_PAGECACHE_USED */ + 1, /* SQLITE_STATUS_PAGECACHE_OVERFLOW */ + 0, /* SQLITE_STATUS_SCRATCH_USED */ + 0, /* SQLITE_STATUS_SCRATCH_OVERFLOW */ + 0, /* SQLITE_STATUS_MALLOC_SIZE */ + 0, /* SQLITE_STATUS_PARSER_STACK */ + 1, /* SQLITE_STATUS_PAGECACHE_SIZE */ + 0, /* SQLITE_STATUS_SCRATCH_SIZE */ + 0, /* SQLITE_STATUS_MALLOC_COUNT */ +}; + + +/* The "wsdStat" macro will resolve to the status information +** state vector. If writable static data is unsupported on the target, +** we have to locate the state vector at run-time. In the more common +** case where writable static data is supported, wsdStat can refer directly +** to the "sqlite3Stat" state vector declared above. +*/ +#ifdef SQLITE_OMIT_WSD +# define wsdStatInit sqlite3StatType *x = &GLOBAL(sqlite3StatType,sqlite3Stat) +# define wsdStat x[0] +#else +# define wsdStatInit +# define wsdStat sqlite3Stat +#endif + +/* +** Return the current value of a status parameter. The caller must +** be holding the appropriate mutex. +*/ +SQLITE_PRIVATE sqlite3_int64 sqlite3StatusValue(int op){ + wsdStatInit; + assert( op>=0 && op=0 && op=0 && op=0 && opwsdStat.mxValue[op] ){ + wsdStat.mxValue[op] = wsdStat.nowValue[op]; + } +} +SQLITE_PRIVATE void sqlite3StatusDown(int op, int N){ + wsdStatInit; + assert( N>=0 ); + assert( op>=0 && op=0 && op=0 ); + newValue = (sqlite3StatValueType)X; + assert( op>=0 && op=0 && opwsdStat.mxValue[op] ){ + wsdStat.mxValue[op] = newValue; + } +} + +/* +** Query status information. +*/ +SQLITE_API int sqlite3_status64( + int op, + sqlite3_int64 *pCurrent, + sqlite3_int64 *pHighwater, + int resetFlag +){ + sqlite3_mutex *pMutex; + wsdStatInit; + if( op<0 || op>=ArraySize(wsdStat.nowValue) ){ + return SQLITE_MISUSE_BKPT; + } +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; +#endif + pMutex = statMutex[op] ? sqlite3Pcache1Mutex() : sqlite3MallocMutex(); + sqlite3_mutex_enter(pMutex); + *pCurrent = wsdStat.nowValue[op]; + *pHighwater = wsdStat.mxValue[op]; + if( resetFlag ){ + wsdStat.mxValue[op] = wsdStat.nowValue[op]; + } + sqlite3_mutex_leave(pMutex); + (void)pMutex; /* Prevent warning when SQLITE_THREADSAFE=0 */ + return SQLITE_OK; +} +SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ + sqlite3_int64 iCur = 0, iHwtr = 0; + int rc; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; +#endif + rc = sqlite3_status64(op, &iCur, &iHwtr, resetFlag); + if( rc==0 ){ + *pCurrent = (int)iCur; + *pHighwater = (int)iHwtr; + } + return rc; +} + +/* +** Return the number of LookasideSlot elements on the linked list +*/ +static u32 countLookasideSlots(LookasideSlot *p){ + u32 cnt = 0; + while( p ){ + p = p->pNext; + cnt++; + } + return cnt; +} + +/* +** Count the number of slots of lookaside memory that are outstanding +*/ +SQLITE_PRIVATE int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){ + u32 nInit = countLookasideSlots(db->lookaside.pInit); + u32 nFree = countLookasideSlots(db->lookaside.pFree); +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + nInit += countLookasideSlots(db->lookaside.pSmallInit); + nFree += countLookasideSlots(db->lookaside.pSmallFree); +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit; + return db->lookaside.nSlot - (nInit+nFree); +} + +/* +** Query status information for a single database connection +*/ +SQLITE_API int sqlite3_db_status( + sqlite3 *db, /* The database connection whose status is desired */ + int op, /* Status verb */ + int *pCurrent, /* Write current value here */ + int *pHighwater, /* Write high-water mark here */ + int resetFlag /* Reset high-water mark if true */ +){ + int rc = SQLITE_OK; /* Return code */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + switch( op ){ + case SQLITE_DBSTATUS_LOOKASIDE_USED: { + *pCurrent = sqlite3LookasideUsed(db, pHighwater); + if( resetFlag ){ + LookasideSlot *p = db->lookaside.pFree; + if( p ){ + while( p->pNext ) p = p->pNext; + p->pNext = db->lookaside.pInit; + db->lookaside.pInit = db->lookaside.pFree; + db->lookaside.pFree = 0; + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + p = db->lookaside.pSmallFree; + if( p ){ + while( p->pNext ) p = p->pNext; + p->pNext = db->lookaside.pSmallInit; + db->lookaside.pSmallInit = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = 0; + } +#endif + } + break; + } + + case SQLITE_DBSTATUS_LOOKASIDE_HIT: + case SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE: + case SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL: { + testcase( op==SQLITE_DBSTATUS_LOOKASIDE_HIT ); + testcase( op==SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE ); + testcase( op==SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL ); + assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)>=0 ); + assert( (op-SQLITE_DBSTATUS_LOOKASIDE_HIT)<3 ); + *pCurrent = 0; + *pHighwater = db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT]; + if( resetFlag ){ + db->lookaside.anStat[op - SQLITE_DBSTATUS_LOOKASIDE_HIT] = 0; + } + break; + } + + /* + ** Return an approximation for the amount of memory currently used + ** by all pagers associated with the given database connection. The + ** highwater mark is meaningless and is returned as zero. + */ + case SQLITE_DBSTATUS_CACHE_USED_SHARED: + case SQLITE_DBSTATUS_CACHE_USED: { + int totalUsed = 0; + int i; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + Pager *pPager = sqlite3BtreePager(pBt); + int nByte = sqlite3PagerMemUsed(pPager); + if( op==SQLITE_DBSTATUS_CACHE_USED_SHARED ){ + nByte = nByte / sqlite3BtreeConnectionCount(pBt); + } + totalUsed += nByte; + } + } + sqlite3BtreeLeaveAll(db); + *pCurrent = totalUsed; + *pHighwater = 0; + break; + } + + /* + ** *pCurrent gets an accurate estimate of the amount of memory used + ** to store the schema for all databases (main, temp, and any ATTACHed + ** databases. *pHighwater is set to zero. + */ + case SQLITE_DBSTATUS_SCHEMA_USED: { + int i; /* Used to iterate through schemas */ + int nByte = 0; /* Used to accumulate return value */ + + sqlite3BtreeEnterAll(db); + db->pnBytesFreed = &nByte; + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( ALWAYS(pSchema!=0) ){ + HashElem *p; + + nByte += sqlite3GlobalConfig.m.xRoundup(sizeof(HashElem)) * ( + pSchema->tblHash.count + + pSchema->trigHash.count + + pSchema->idxHash.count + + pSchema->fkeyHash.count + ); + nByte += sqlite3_msize(pSchema->tblHash.ht); + nByte += sqlite3_msize(pSchema->trigHash.ht); + nByte += sqlite3_msize(pSchema->idxHash.ht); + nByte += sqlite3_msize(pSchema->fkeyHash.ht); + + for(p=sqliteHashFirst(&pSchema->trigHash); p; p=sqliteHashNext(p)){ + sqlite3DeleteTrigger(db, (Trigger*)sqliteHashData(p)); + } + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + sqlite3DeleteTable(db, (Table *)sqliteHashData(p)); + } + } + } + db->pnBytesFreed = 0; + sqlite3BtreeLeaveAll(db); + + *pHighwater = 0; + *pCurrent = nByte; + break; + } + + /* + ** *pCurrent gets an accurate estimate of the amount of memory used + ** to store all prepared statements. + ** *pHighwater is set to zero. + */ + case SQLITE_DBSTATUS_STMT_USED: { + struct Vdbe *pVdbe; /* Used to iterate through VMs */ + int nByte = 0; /* Used to accumulate return value */ + + db->pnBytesFreed = &nByte; + for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){ + sqlite3VdbeClearObject(db, pVdbe); + sqlite3DbFree(db, pVdbe); + } + db->pnBytesFreed = 0; + + *pHighwater = 0; /* IMP: R-64479-57858 */ + *pCurrent = nByte; + + break; + } + + /* + ** Set *pCurrent to the total cache hits or misses encountered by all + ** pagers the database handle is connected to. *pHighwater is always set + ** to zero. + */ + case SQLITE_DBSTATUS_CACHE_SPILL: + op = SQLITE_DBSTATUS_CACHE_WRITE+1; + /* no break */ deliberate_fall_through + case SQLITE_DBSTATUS_CACHE_HIT: + case SQLITE_DBSTATUS_CACHE_MISS: + case SQLITE_DBSTATUS_CACHE_WRITE:{ + int i; + int nRet = 0; + assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); + assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 ); + + for(i=0; inDb; i++){ + if( db->aDb[i].pBt ){ + Pager *pPager = sqlite3BtreePager(db->aDb[i].pBt); + sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet); + } + } + *pHighwater = 0; /* IMP: R-42420-56072 */ + /* IMP: R-54100-20147 */ + /* IMP: R-29431-39229 */ + *pCurrent = nRet; + break; + } + + /* Set *pCurrent to non-zero if there are unresolved deferred foreign + ** key constraints. Set *pCurrent to zero if all foreign key constraints + ** have been satisfied. The *pHighwater is always set to zero. + */ + case SQLITE_DBSTATUS_DEFERRED_FKS: { + *pHighwater = 0; /* IMP: R-11967-56545 */ + *pCurrent = db->nDeferredImmCons>0 || db->nDeferredCons>0; + break; + } + + default: { + rc = SQLITE_ERROR; + } + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/************** End of status.c **********************************************/ +/************** Begin file date.c ********************************************/ +/* +** 2003 October 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement date and time +** functions for SQLite. +** +** There is only one exported symbol in this file - the function +** sqlite3RegisterDateTimeFunctions() found at the bottom of the file. +** All other code has file scope. +** +** SQLite processes all times and dates as julian day numbers. The +** dates and times are stored as the number of days since noon +** in Greenwich on November 24, 4714 B.C. according to the Gregorian +** calendar system. +** +** 1970-01-01 00:00:00 is JD 2440587.5 +** 2000-01-01 00:00:00 is JD 2451544.5 +** +** This implementation requires years to be expressed as a 4-digit number +** which means that only dates between 0000-01-01 and 9999-12-31 can +** be represented, even though julian day numbers allow a much wider +** range of dates. +** +** The Gregorian calendar system is used for all dates and times, +** even those that predate the Gregorian calendar. Historians usually +** use the julian calendar for dates prior to 1582-10-15 and for some +** dates afterwards, depending on locale. Beware of this difference. +** +** The conversion algorithms are implemented based on descriptions +** in the following text: +** +** Jean Meeus +** Astronomical Algorithms, 2nd Edition, 1998 +** ISBN 0-943396-61-1 +** Willmann-Bell, Inc +** Richmond, Virginia (USA) +*/ +/* #include "sqliteInt.h" */ +/* #include */ +/* #include */ +#include + +#ifndef SQLITE_OMIT_DATETIME_FUNCS + +/* +** The MSVC CRT on Windows CE may not have a localtime() function. +** So declare a substitute. The substitute function itself is +** defined in "os_win.c". +*/ +#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ + (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) +struct tm *__cdecl localtime(const time_t *); +#endif + +/* +** A structure for holding a single date and time. +*/ +typedef struct DateTime DateTime; +struct DateTime { + sqlite3_int64 iJD; /* The julian day number times 86400000 */ + int Y, M, D; /* Year, month, and day */ + int h, m; /* Hour and minutes */ + int tz; /* Timezone offset in minutes */ + double s; /* Seconds */ + char validJD; /* True (1) if iJD is valid */ + char rawS; /* Raw numeric value stored in s */ + char validYMD; /* True (1) if Y,M,D are valid */ + char validHMS; /* True (1) if h,m,s are valid */ + char validTZ; /* True (1) if tz is valid */ + char tzSet; /* Timezone was set explicitly */ + char isError; /* An overflow has occurred */ +}; + + +/* +** Convert zDate into one or more integers according to the conversion +** specifier zFormat. +** +** zFormat[] contains 4 characters for each integer converted, except for +** the last integer which is specified by three characters. The meaning +** of a four-character format specifiers ABCD is: +** +** A: number of digits to convert. Always "2" or "4". +** B: minimum value. Always "0" or "1". +** C: maximum value, decoded as: +** a: 12 +** b: 14 +** c: 24 +** d: 31 +** e: 59 +** f: 9999 +** D: the separator character, or \000 to indicate this is the +** last number to convert. +** +** Example: To translate an ISO-8601 date YYYY-MM-DD, the format would +** be "40f-21a-20c". The "40f-" indicates the 4-digit year followed by "-". +** The "21a-" indicates the 2-digit month followed by "-". The "20c" indicates +** the 2-digit day which is the last integer in the set. +** +** The function returns the number of successful conversions. +*/ +static int getDigits(const char *zDate, const char *zFormat, ...){ + /* The aMx[] array translates the 3rd character of each format + ** spec into a max size: a b c d e f */ + static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 }; + va_list ap; + int cnt = 0; + char nextC; + va_start(ap, zFormat); + do{ + char N = zFormat[0] - '0'; + char min = zFormat[1] - '0'; + int val = 0; + u16 max; + + assert( zFormat[2]>='a' && zFormat[2]<='f' ); + max = aMx[zFormat[2] - 'a']; + nextC = zFormat[3]; + val = 0; + while( N-- ){ + if( !sqlite3Isdigit(*zDate) ){ + goto end_getDigits; + } + val = val*10 + *zDate - '0'; + zDate++; + } + if( val<(int)min || val>(int)max || (nextC!=0 && nextC!=*zDate) ){ + goto end_getDigits; + } + *va_arg(ap,int*) = val; + zDate++; + cnt++; + zFormat += 4; + }while( nextC ); +end_getDigits: + va_end(ap); + return cnt; +} + +/* +** Parse a timezone extension on the end of a date-time. +** The extension is of the form: +** +** (+/-)HH:MM +** +** Or the "zulu" notation: +** +** Z +** +** If the parse is successful, write the number of minutes +** of change in p->tz and return 0. If a parser error occurs, +** return non-zero. +** +** A missing specifier is not considered an error. +*/ +static int parseTimezone(const char *zDate, DateTime *p){ + int sgn = 0; + int nHr, nMn; + int c; + while( sqlite3Isspace(*zDate) ){ zDate++; } + p->tz = 0; + c = *zDate; + if( c=='-' ){ + sgn = -1; + }else if( c=='+' ){ + sgn = +1; + }else if( c=='Z' || c=='z' ){ + zDate++; + goto zulu_time; + }else{ + return c!=0; + } + zDate++; + if( getDigits(zDate, "20b:20e", &nHr, &nMn)!=2 ){ + return 1; + } + zDate += 5; + p->tz = sgn*(nMn + nHr*60); +zulu_time: + while( sqlite3Isspace(*zDate) ){ zDate++; } + p->tzSet = 1; + return *zDate!=0; +} + +/* +** Parse times of the form HH:MM or HH:MM:SS or HH:MM:SS.FFFF. +** The HH, MM, and SS must each be exactly 2 digits. The +** fractional seconds FFFF can be one or more digits. +** +** Return 1 if there is a parsing error and 0 on success. +*/ +static int parseHhMmSs(const char *zDate, DateTime *p){ + int h, m, s; + double ms = 0.0; + if( getDigits(zDate, "20c:20e", &h, &m)!=2 ){ + return 1; + } + zDate += 5; + if( *zDate==':' ){ + zDate++; + if( getDigits(zDate, "20e", &s)!=1 ){ + return 1; + } + zDate += 2; + if( *zDate=='.' && sqlite3Isdigit(zDate[1]) ){ + double rScale = 1.0; + zDate++; + while( sqlite3Isdigit(*zDate) ){ + ms = ms*10.0 + *zDate - '0'; + rScale *= 10.0; + zDate++; + } + ms /= rScale; + } + }else{ + s = 0; + } + p->validJD = 0; + p->rawS = 0; + p->validHMS = 1; + p->h = h; + p->m = m; + p->s = s + ms; + if( parseTimezone(zDate, p) ) return 1; + p->validTZ = (p->tz!=0)?1:0; + return 0; +} + +/* +** Put the DateTime object into its error state. +*/ +static void datetimeError(DateTime *p){ + memset(p, 0, sizeof(*p)); + p->isError = 1; +} + +/* +** Convert from YYYY-MM-DD HH:MM:SS to julian day. We always assume +** that the YYYY-MM-DD is according to the Gregorian calendar. +** +** Reference: Meeus page 61 +*/ +static void computeJD(DateTime *p){ + int Y, M, D, A, B, X1, X2; + + if( p->validJD ) return; + if( p->validYMD ){ + Y = p->Y; + M = p->M; + D = p->D; + }else{ + Y = 2000; /* If no YMD specified, assume 2000-Jan-01 */ + M = 1; + D = 1; + } + if( Y<-4713 || Y>9999 || p->rawS ){ + datetimeError(p); + return; + } + if( M<=2 ){ + Y--; + M += 12; + } + A = Y/100; + B = 2 - A + (A/4); + X1 = 36525*(Y+4716)/100; + X2 = 306001*(M+1)/10000; + p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); + p->validJD = 1; + if( p->validHMS ){ + p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000); + if( p->validTZ ){ + p->iJD -= p->tz*60000; + p->validYMD = 0; + p->validHMS = 0; + p->validTZ = 0; + } + } +} + +/* +** Parse dates of the form +** +** YYYY-MM-DD HH:MM:SS.FFF +** YYYY-MM-DD HH:MM:SS +** YYYY-MM-DD HH:MM +** YYYY-MM-DD +** +** Write the result into the DateTime structure and return 0 +** on success and 1 if the input string is not a well-formed +** date. +*/ +static int parseYyyyMmDd(const char *zDate, DateTime *p){ + int Y, M, D, neg; + + if( zDate[0]=='-' ){ + zDate++; + neg = 1; + }else{ + neg = 0; + } + if( getDigits(zDate, "40f-21a-21d", &Y, &M, &D)!=3 ){ + return 1; + } + zDate += 10; + while( sqlite3Isspace(*zDate) || 'T'==*(u8*)zDate ){ zDate++; } + if( parseHhMmSs(zDate, p)==0 ){ + /* We got the time */ + }else if( *zDate==0 ){ + p->validHMS = 0; + }else{ + return 1; + } + p->validJD = 0; + p->validYMD = 1; + p->Y = neg ? -Y : Y; + p->M = M; + p->D = D; + if( p->validTZ ){ + computeJD(p); + } + return 0; +} + +/* +** Set the time to the current time reported by the VFS. +** +** Return the number of errors. +*/ +static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ + p->iJD = sqlite3StmtCurrentTime(context); + if( p->iJD>0 ){ + p->validJD = 1; + return 0; + }else{ + return 1; + } +} + +/* +** Input "r" is a numeric quantity which might be a julian day number, +** or the number of seconds since 1970. If the value if r is within +** range of a julian day number, install it as such and set validJD. +** If the value is a valid unix timestamp, put it in p->s and set p->rawS. +*/ +static void setRawDateNumber(DateTime *p, double r){ + p->s = r; + p->rawS = 1; + if( r>=0.0 && r<5373484.5 ){ + p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); + p->validJD = 1; + } +} + +/* +** Attempt to parse the given string into a julian day number. Return +** the number of errors. +** +** The following are acceptable forms for the input string: +** +** YYYY-MM-DD HH:MM:SS.FFF +/-HH:MM +** DDDD.DD +** now +** +** In the first form, the +/-HH:MM is always optional. The fractional +** seconds extension (the ".FFF") is optional. The seconds portion +** (":SS.FFF") is option. The year and date can be omitted as long +** as there is a time string. The time string can be omitted as long +** as there is a year and date. +*/ +static int parseDateOrTime( + sqlite3_context *context, + const char *zDate, + DateTime *p +){ + double r; + if( parseYyyyMmDd(zDate,p)==0 ){ + return 0; + }else if( parseHhMmSs(zDate, p)==0 ){ + return 0; + }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ + return setDateTimeToCurrent(context, p); + }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ + setRawDateNumber(p, r); + return 0; + } + return 1; +} + +/* The julian day number for 9999-12-31 23:59:59.999 is 5373484.4999999. +** Multiplying this by 86400000 gives 464269060799999 as the maximum value +** for DateTime.iJD. +** +** But some older compilers (ex: gcc 4.2.1 on older Macs) cannot deal with +** such a large integer literal, so we have to encode it. +*/ +#define INT_464269060799999 ((((i64)0x1a640)<<32)|0x1072fdff) + +/* +** Return TRUE if the given julian day number is within range. +** +** The input is the JulianDay times 86400000. +*/ +static int validJulianDay(sqlite3_int64 iJD){ + return iJD>=0 && iJD<=INT_464269060799999; +} + +/* +** Compute the Year, Month, and Day from the julian day number. +*/ +static void computeYMD(DateTime *p){ + int Z, A, B, C, D, E, X1; + if( p->validYMD ) return; + if( !p->validJD ){ + p->Y = 2000; + p->M = 1; + p->D = 1; + }else if( !validJulianDay(p->iJD) ){ + datetimeError(p); + return; + }else{ + Z = (int)((p->iJD + 43200000)/86400000); + A = (int)((Z - 1867216.25)/36524.25); + A = Z + 1 + A - (A/4); + B = A + 1524; + C = (int)((B - 122.1)/365.25); + D = (36525*(C&32767))/100; + E = (int)((B-D)/30.6001); + X1 = (int)(30.6001*E); + p->D = B - D - X1; + p->M = E<14 ? E-1 : E-13; + p->Y = p->M>2 ? C - 4716 : C - 4715; + } + p->validYMD = 1; +} + +/* +** Compute the Hour, Minute, and Seconds from the julian day number. +*/ +static void computeHMS(DateTime *p){ + int s; + if( p->validHMS ) return; + computeJD(p); + s = (int)((p->iJD + 43200000) % 86400000); + p->s = s/1000.0; + s = (int)p->s; + p->s -= s; + p->h = s/3600; + s -= p->h*3600; + p->m = s/60; + p->s += s - p->m*60; + p->rawS = 0; + p->validHMS = 1; +} + +/* +** Compute both YMD and HMS +*/ +static void computeYMD_HMS(DateTime *p){ + computeYMD(p); + computeHMS(p); +} + +/* +** Clear the YMD and HMS and the TZ +*/ +static void clearYMD_HMS_TZ(DateTime *p){ + p->validYMD = 0; + p->validHMS = 0; + p->validTZ = 0; +} + +#ifndef SQLITE_OMIT_LOCALTIME +/* +** On recent Windows platforms, the localtime_s() function is available +** as part of the "Secure CRT". It is essentially equivalent to +** localtime_r() available under most POSIX platforms, except that the +** order of the parameters is reversed. +** +** See http://msdn.microsoft.com/en-us/library/a442x3ye(VS.80).aspx. +** +** If the user has not indicated to use localtime_r() or localtime_s() +** already, check for an MSVC build environment that provides +** localtime_s(). +*/ +#if !HAVE_LOCALTIME_R && !HAVE_LOCALTIME_S \ + && defined(_MSC_VER) && defined(_CRT_INSECURE_DEPRECATE) +#undef HAVE_LOCALTIME_S +#define HAVE_LOCALTIME_S 1 +#endif + +/* +** The following routine implements the rough equivalent of localtime_r() +** using whatever operating-system specific localtime facility that +** is available. This routine returns 0 on success and +** non-zero on any kind of error. +** +** If the sqlite3GlobalConfig.bLocaltimeFault variable is true then this +** routine will always fail. +** +** EVIDENCE-OF: R-62172-00036 In this implementation, the standard C +** library function localtime_r() is used to assist in the calculation of +** local time. +*/ +static int osLocaltime(time_t *t, struct tm *pTm){ + int rc; +#if !HAVE_LOCALTIME_R && !HAVE_LOCALTIME_S + struct tm *pX; +#if SQLITE_THREADSAFE>0 + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); +#endif + sqlite3_mutex_enter(mutex); + pX = localtime(t); +#ifndef SQLITE_UNTESTABLE + if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0; +#endif + if( pX ) *pTm = *pX; + sqlite3_mutex_leave(mutex); + rc = pX==0; +#else +#ifndef SQLITE_UNTESTABLE + if( sqlite3GlobalConfig.bLocaltimeFault ) return 1; +#endif +#if HAVE_LOCALTIME_R + rc = localtime_r(t, pTm)==0; +#else + rc = localtime_s(pTm, t); +#endif /* HAVE_LOCALTIME_R */ +#endif /* HAVE_LOCALTIME_R || HAVE_LOCALTIME_S */ + return rc; +} +#endif /* SQLITE_OMIT_LOCALTIME */ + + +#ifndef SQLITE_OMIT_LOCALTIME +/* +** Compute the difference (in milliseconds) between localtime and UTC +** (a.k.a. GMT) for the time value p where p is in UTC. If no error occurs, +** return this value and set *pRc to SQLITE_OK. +** +** Or, if an error does occur, set *pRc to SQLITE_ERROR. The returned value +** is undefined in this case. +*/ +static sqlite3_int64 localtimeOffset( + DateTime *p, /* Date at which to calculate offset */ + sqlite3_context *pCtx, /* Write error here if one occurs */ + int *pRc /* OUT: Error code. SQLITE_OK or ERROR */ +){ + DateTime x, y; + time_t t; + struct tm sLocal; + + /* Initialize the contents of sLocal to avoid a compiler warning. */ + memset(&sLocal, 0, sizeof(sLocal)); + + x = *p; + computeYMD_HMS(&x); + if( x.Y<1971 || x.Y>=2038 ){ + /* EVIDENCE-OF: R-55269-29598 The localtime_r() C function normally only + ** works for years between 1970 and 2037. For dates outside this range, + ** SQLite attempts to map the year into an equivalent year within this + ** range, do the calculation, then map the year back. + */ + x.Y = 2000; + x.M = 1; + x.D = 1; + x.h = 0; + x.m = 0; + x.s = 0.0; + } else { + int s = (int)(x.s + 0.5); + x.s = s; + } + x.tz = 0; + x.validJD = 0; + computeJD(&x); + t = (time_t)(x.iJD/1000 - 21086676*(i64)10000); + if( osLocaltime(&t, &sLocal) ){ + sqlite3_result_error(pCtx, "local time unavailable", -1); + *pRc = SQLITE_ERROR; + return 0; + } + y.Y = sLocal.tm_year + 1900; + y.M = sLocal.tm_mon + 1; + y.D = sLocal.tm_mday; + y.h = sLocal.tm_hour; + y.m = sLocal.tm_min; + y.s = sLocal.tm_sec; + y.validYMD = 1; + y.validHMS = 1; + y.validJD = 0; + y.rawS = 0; + y.validTZ = 0; + y.isError = 0; + computeJD(&y); + *pRc = SQLITE_OK; + return y.iJD - x.iJD; +} +#endif /* SQLITE_OMIT_LOCALTIME */ + +/* +** The following table defines various date transformations of the form +** +** 'NNN days' +** +** Where NNN is an arbitrary floating-point number and "days" can be one +** of several units of time. +*/ +static const struct { + u8 eType; /* Transformation type code */ + u8 nName; /* Length of th name */ + char *zName; /* Name of the transformation */ + double rLimit; /* Maximum NNN value for this transform */ + double rXform; /* Constant used for this transform */ +} aXformType[] = { + { 0, 6, "second", 464269060800.0, 1000.0 }, + { 0, 6, "minute", 7737817680.0, 60000.0 }, + { 0, 4, "hour", 128963628.0, 3600000.0 }, + { 0, 3, "day", 5373485.0, 86400000.0 }, + { 1, 5, "month", 176546.0, 2592000000.0 }, + { 2, 4, "year", 14713.0, 31536000000.0 }, +}; + +/* +** Process a modifier to a date-time stamp. The modifiers are +** as follows: +** +** NNN days +** NNN hours +** NNN minutes +** NNN.NNNN seconds +** NNN months +** NNN years +** start of month +** start of year +** start of week +** start of day +** weekday N +** unixepoch +** localtime +** utc +** +** Return 0 on success and 1 if there is any kind of error. If the error +** is in a system call (i.e. localtime()), then an error message is written +** to context pCtx. If the error is an unrecognized modifier, no error is +** written to pCtx. +*/ +static int parseModifier( + sqlite3_context *pCtx, /* Function context */ + const char *z, /* The text of the modifier */ + int n, /* Length of zMod in bytes */ + DateTime *p /* The date/time value to be modified */ +){ + int rc = 1; + double r; + switch(sqlite3UpperToLower[(u8)z[0]] ){ +#ifndef SQLITE_OMIT_LOCALTIME + case 'l': { + /* localtime + ** + ** Assuming the current time value is UTC (a.k.a. GMT), shift it to + ** show local time. + */ + if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){ + computeJD(p); + p->iJD += localtimeOffset(p, pCtx, &rc); + clearYMD_HMS_TZ(p); + } + break; + } +#endif + case 'u': { + /* + ** unixepoch + ** + ** Treat the current value of p->s as the number of + ** seconds since 1970. Convert to a real julian day number. + */ + if( sqlite3_stricmp(z, "unixepoch")==0 && p->rawS ){ + r = p->s*1000.0 + 210866760000000.0; + if( r>=0.0 && r<464269060800000.0 ){ + clearYMD_HMS_TZ(p); + p->iJD = (sqlite3_int64)(r + 0.5); + p->validJD = 1; + p->rawS = 0; + rc = 0; + } + } +#ifndef SQLITE_OMIT_LOCALTIME + else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){ + if( p->tzSet==0 ){ + sqlite3_int64 c1; + computeJD(p); + c1 = localtimeOffset(p, pCtx, &rc); + if( rc==SQLITE_OK ){ + p->iJD -= c1; + clearYMD_HMS_TZ(p); + p->iJD += c1 - localtimeOffset(p, pCtx, &rc); + } + p->tzSet = 1; + }else{ + rc = SQLITE_OK; + } + } +#endif + break; + } + case 'w': { + /* + ** weekday N + ** + ** Move the date to the same time on the next occurrence of + ** weekday N where 0==Sunday, 1==Monday, and so forth. If the + ** date is already on the appropriate weekday, this is a no-op. + */ + if( sqlite3_strnicmp(z, "weekday ", 8)==0 + && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 + && (n=(int)r)==r && n>=0 && r<7 ){ + sqlite3_int64 Z; + computeYMD_HMS(p); + p->validTZ = 0; + p->validJD = 0; + computeJD(p); + Z = ((p->iJD + 129600000)/86400000) % 7; + if( Z>n ) Z -= 7; + p->iJD += (n - Z)*86400000; + clearYMD_HMS_TZ(p); + rc = 0; + } + break; + } + case 's': { + /* + ** start of TTTTT + ** + ** Move the date backwards to the beginning of the current day, + ** or month or year. + */ + if( sqlite3_strnicmp(z, "start of ", 9)!=0 ) break; + if( !p->validJD && !p->validYMD && !p->validHMS ) break; + z += 9; + computeYMD(p); + p->validHMS = 1; + p->h = p->m = 0; + p->s = 0.0; + p->rawS = 0; + p->validTZ = 0; + p->validJD = 0; + if( sqlite3_stricmp(z,"month")==0 ){ + p->D = 1; + rc = 0; + }else if( sqlite3_stricmp(z,"year")==0 ){ + p->M = 1; + p->D = 1; + rc = 0; + }else if( sqlite3_stricmp(z,"day")==0 ){ + rc = 0; + } + break; + } + case '+': + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + double rRounder; + int i; + for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} + if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ + rc = 1; + break; + } + if( z[n]==':' ){ + /* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the + ** specified number of hours, minutes, seconds, and fractional seconds + ** to the time. The ".FFF" may be omitted. The ":SS.FFF" may be + ** omitted. + */ + const char *z2 = z; + DateTime tx; + sqlite3_int64 day; + if( !sqlite3Isdigit(*z2) ) z2++; + memset(&tx, 0, sizeof(tx)); + if( parseHhMmSs(z2, &tx) ) break; + computeJD(&tx); + tx.iJD -= 43200000; + day = tx.iJD/86400000; + tx.iJD -= day*86400000; + if( z[0]=='-' ) tx.iJD = -tx.iJD; + computeJD(p); + clearYMD_HMS_TZ(p); + p->iJD += tx.iJD; + rc = 0; + break; + } + + /* If control reaches this point, it means the transformation is + ** one of the forms like "+NNN days". */ + z += n; + while( sqlite3Isspace(*z) ) z++; + n = sqlite3Strlen30(z); + if( n>10 || n<3 ) break; + if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--; + computeJD(p); + rc = 1; + rRounder = r<0 ? -0.5 : +0.5; + for(i=0; i-aXformType[i].rLimit && rM += (int)r; + x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; + p->Y += x; + p->M -= x*12; + p->validJD = 0; + r -= (int)r; + break; + } + case 2: { /* Special processing to add years */ + int y = (int)r; + computeYMD_HMS(p); + p->Y += y; + p->validJD = 0; + r -= (int)r; + break; + } + } + computeJD(p); + p->iJD += (sqlite3_int64)(r*aXformType[i].rXform + rRounder); + rc = 0; + break; + } + } + clearYMD_HMS_TZ(p); + break; + } + default: { + break; + } + } + return rc; +} + +/* +** Process time function arguments. argv[0] is a date-time stamp. +** argv[1] and following are modifiers. Parse them all and write +** the resulting time into the DateTime structure p. Return 0 +** on success and 1 if there are any errors. +** +** If there are zero parameters (if even argv[0] is undefined) +** then assume a default value of "now" for argv[0]. +*/ +static int isDate( + sqlite3_context *context, + int argc, + sqlite3_value **argv, + DateTime *p +){ + int i, n; + const unsigned char *z; + int eType; + memset(p, 0, sizeof(*p)); + if( argc==0 ){ + if( !sqlite3NotPureFunc(context) ) return 1; + return setDateTimeToCurrent(context, p); + } + if( (eType = sqlite3_value_type(argv[0]))==SQLITE_FLOAT + || eType==SQLITE_INTEGER ){ + setRawDateNumber(p, sqlite3_value_double(argv[0])); + }else{ + z = sqlite3_value_text(argv[0]); + if( !z || parseDateOrTime(context, (char*)z, p) ){ + return 1; + } + } + for(i=1; iisError || !validJulianDay(p->iJD) ) return 1; + return 0; +} + + +/* +** The following routines implement the various date and time functions +** of SQLite. +*/ + +/* +** julianday( TIMESTRING, MOD, MOD, ...) +** +** Return the julian day number of the date specified in the arguments +*/ +static void juliandayFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + computeJD(&x); + sqlite3_result_double(context, x.iJD/86400000.0); + } +} + +/* +** datetime( TIMESTRING, MOD, MOD, ...) +** +** Return YYYY-MM-DD HH:MM:SS +*/ +static void datetimeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char zBuf[100]; + computeYMD_HMS(&x); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%04d-%02d-%02d %02d:%02d:%02d", + x.Y, x.M, x.D, x.h, x.m, (int)(x.s)); + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + } +} + +/* +** time( TIMESTRING, MOD, MOD, ...) +** +** Return HH:MM:SS +*/ +static void timeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char zBuf[100]; + computeHMS(&x); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%02d:%02d:%02d", x.h, x.m, (int)x.s); + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + } +} + +/* +** date( TIMESTRING, MOD, MOD, ...) +** +** Return YYYY-MM-DD +*/ +static void dateFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char zBuf[100]; + computeYMD(&x); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%04d-%02d-%02d", x.Y, x.M, x.D); + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + } +} + +/* +** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) +** +** Return a string described by FORMAT. Conversions as follows: +** +** %d day of month +** %f ** fractional seconds SS.SSS +** %H hour 00-24 +** %j day of year 000-366 +** %J ** julian day number +** %m month 01-12 +** %M minute 00-59 +** %s seconds since 1970-01-01 +** %S seconds 00-59 +** %w day of week 0-6 sunday==0 +** %W week of year 00-53 +** %Y year 0000-9999 +** %% % +*/ +static void strftimeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + size_t i,j; + sqlite3 *db; + const char *zFmt; + sqlite3_str sRes; + + + if( argc==0 ) return; + zFmt = (const char*)sqlite3_value_text(argv[0]); + if( zFmt==0 || isDate(context, argc-1, argv+1, &x) ) return; + db = sqlite3_context_db_handle(context); + sqlite3StrAccumInit(&sRes, 0, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); + + computeJD(&x); + computeYMD_HMS(&x); + for(i=j=0; zFmt[i]; i++){ + if( zFmt[i]!='%' ) continue; + if( j59.999 ) s = 59.999; + sqlite3_str_appendf(&sRes, "%06.3f", s); + break; + } + case 'H': { + sqlite3_str_appendf(&sRes, "%02d", x.h); + break; + } + case 'W': /* Fall thru */ + case 'j': { + int nDay; /* Number of days since 1st day of year */ + DateTime y = x; + y.validJD = 0; + y.M = 1; + y.D = 1; + computeJD(&y); + nDay = (int)((x.iJD-y.iJD+43200000)/86400000); + if( zFmt[i]=='W' ){ + int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ + wd = (int)(((x.iJD+43200000)/86400000)%7); + sqlite3_str_appendf(&sRes,"%02d",(nDay+7-wd)/7); + }else{ + sqlite3_str_appendf(&sRes,"%03d",nDay+1); + } + break; + } + case 'J': { + sqlite3_str_appendf(&sRes,"%.16g",x.iJD/86400000.0); + break; + } + case 'm': { + sqlite3_str_appendf(&sRes,"%02d",x.M); + break; + } + case 'M': { + sqlite3_str_appendf(&sRes,"%02d",x.m); + break; + } + case 's': { + i64 iS = (i64)(x.iJD/1000 - 21086676*(i64)10000); + sqlite3_str_appendf(&sRes,"%lld",iS); + break; + } + case 'S': { + sqlite3_str_appendf(&sRes,"%02d",(int)x.s); + break; + } + case 'w': { + sqlite3_str_appendchar(&sRes, 1, + (char)(((x.iJD+129600000)/86400000) % 7) + '0'); + break; + } + case 'Y': { + sqlite3_str_appendf(&sRes,"%04d",x.Y); + break; + } + case '%': { + sqlite3_str_appendchar(&sRes, 1, '%'); + break; + } + default: { + sqlite3_str_reset(&sRes); + return; + } + } + } + if( jpMethods ){ + pId->pMethods->xClose(pId); + pId->pMethods = 0; + } +} +SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file *id, void *pBuf, int amt, i64 offset){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xRead(id, pBuf, amt, offset); +} +SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file *id, const void *pBuf, int amt, i64 offset){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xWrite(id, pBuf, amt, offset); +} +SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file *id, i64 size){ + return id->pMethods->xTruncate(id, size); +} +SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file *id, int flags){ + DO_OS_MALLOC_TEST(id); + return flags ? id->pMethods->xSync(id, flags) : SQLITE_OK; +} +SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xFileSize(id, pSize); +} +SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file *id, int lockType){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xLock(id, lockType); +} +SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file *id, int lockType){ + return id->pMethods->xUnlock(id, lockType); +} +SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xCheckReservedLock(id, pResOut); +} + +/* +** Use sqlite3OsFileControl() when we are doing something that might fail +** and we need to know about the failures. Use sqlite3OsFileControlHint() +** when simply tossing information over the wall to the VFS and we do not +** really care if the VFS receives and understands the information since it +** is only a hint and can be safely ignored. The sqlite3OsFileControlHint() +** routine has no return value since the return value would be meaningless. +*/ +SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ + if( id->pMethods==0 ) return SQLITE_NOTFOUND; +#ifdef SQLITE_TEST + if( op!=SQLITE_FCNTL_COMMIT_PHASETWO + && op!=SQLITE_FCNTL_LOCK_TIMEOUT + && op!=SQLITE_FCNTL_CKPT_DONE + && op!=SQLITE_FCNTL_CKPT_START + ){ + /* Faults are not injected into COMMIT_PHASETWO because, assuming SQLite + ** is using a regular VFS, it is called after the corresponding + ** transaction has been committed. Injecting a fault at this point + ** confuses the test scripts - the COMMIT comand returns SQLITE_NOMEM + ** but the transaction is committed anyway. + ** + ** The core must call OsFileControl() though, not OsFileControlHint(), + ** as if a custom VFS (e.g. zipvfs) returns an error here, it probably + ** means the commit really has failed and an error should be returned + ** to the user. + ** + ** The CKPT_DONE and CKPT_START file-controls are write-only signals + ** to the cksumvfs. Their return code is meaningless and is ignored + ** by the SQLite core, so there is no point in simulating OOMs for them. + */ + DO_OS_MALLOC_TEST(id); + } +#endif + return id->pMethods->xFileControl(id, op, pArg); +} +SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file *id, int op, void *pArg){ + if( id->pMethods ) (void)id->pMethods->xFileControl(id, op, pArg); +} + +SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id){ + int (*xSectorSize)(sqlite3_file*) = id->pMethods->xSectorSize; + return (xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE); +} +SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id){ + if( NEVER(id->pMethods==0) ) return 0; + return id->pMethods->xDeviceCharacteristics(id); +} +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int offset, int n, int flags){ + return id->pMethods->xShmLock(id, offset, n, flags); +} +SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id){ + id->pMethods->xShmBarrier(id); +} +SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int deleteFlag){ + return id->pMethods->xShmUnmap(id, deleteFlag); +} +SQLITE_PRIVATE int sqlite3OsShmMap( + sqlite3_file *id, /* Database file handle */ + int iPage, + int pgsz, + int bExtend, /* True to extend file if necessary */ + void volatile **pp /* OUT: Pointer to mapping */ +){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); +} +#endif /* SQLITE_OMIT_WAL */ + +#if SQLITE_MAX_MMAP_SIZE>0 +/* The real implementation of xFetch and xUnfetch */ +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + DO_OS_MALLOC_TEST(id); + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return id->pMethods->xUnfetch(id, iOff, p); +} +#else +/* No-op stubs to use when memory-mapped I/O is disabled */ +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + *pp = 0; + return SQLITE_OK; +} +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return SQLITE_OK; +} +#endif + +/* +** The next group of routines are convenience wrappers around the +** VFS methods. +*/ +SQLITE_PRIVATE int sqlite3OsOpen( + sqlite3_vfs *pVfs, + const char *zPath, + sqlite3_file *pFile, + int flags, + int *pFlagsOut +){ + int rc; + DO_OS_MALLOC_TEST(0); + /* 0x87f7f is a mask of SQLITE_OPEN_ flags that are valid to be passed + ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, + ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before + ** reaching the VFS. */ + rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut); + assert( rc==SQLITE_OK || pFile->pMethods==0 ); + return rc; +} +SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + DO_OS_MALLOC_TEST(0); + assert( dirSync==0 || dirSync==1 ); + return pVfs->xDelete!=0 ? pVfs->xDelete(pVfs, zPath, dirSync) : SQLITE_OK; +} +SQLITE_PRIVATE int sqlite3OsAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + DO_OS_MALLOC_TEST(0); + return pVfs->xAccess(pVfs, zPath, flags, pResOut); +} +SQLITE_PRIVATE int sqlite3OsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nPathOut, + char *zPathOut +){ + DO_OS_MALLOC_TEST(0); + zPathOut[0] = 0; + return pVfs->xFullPathname(pVfs, zPath, nPathOut, zPathOut); +} +#ifndef SQLITE_OMIT_LOAD_EXTENSION +SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + assert( zPath!=0 ); + assert( strlen(zPath)<=SQLITE_MAX_PATHLEN ); /* tag-20210611-1 */ + return pVfs->xDlOpen(pVfs, zPath); +} +SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + pVfs->xDlError(pVfs, nByte, zBufOut); +} +SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *pVfs, void *pHdle, const char *zSym))(void){ + return pVfs->xDlSym(pVfs, pHdle, zSym); +} +SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *pVfs, void *pHandle){ + pVfs->xDlClose(pVfs, pHandle); +} +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + if( sqlite3Config.iPrngSeed ){ + memset(zBufOut, 0, nByte); + if( ALWAYS(nByte>(signed)sizeof(unsigned)) ) nByte = sizeof(unsigned int); + memcpy(zBufOut, &sqlite3Config.iPrngSeed, nByte); + return SQLITE_OK; + }else{ + return pVfs->xRandomness(pVfs, nByte, zBufOut); + } + +} +SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ + return pVfs->xSleep(pVfs, nMicro); +} +SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs *pVfs){ + return pVfs->xGetLastError ? pVfs->xGetLastError(pVfs, 0, 0) : 0; +} +SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ + int rc; + /* IMPLEMENTATION-OF: R-49045-42493 SQLite will use the xCurrentTimeInt64() + ** method to get the current date and time if that method is available + ** (if iVersion is 2 or greater and the function pointer is not NULL) and + ** will fall back to xCurrentTime() if xCurrentTimeInt64() is + ** unavailable. + */ + if( pVfs->iVersion>=2 && pVfs->xCurrentTimeInt64 ){ + rc = pVfs->xCurrentTimeInt64(pVfs, pTimeOut); + }else{ + double r; + rc = pVfs->xCurrentTime(pVfs, &r); + *pTimeOut = (sqlite3_int64)(r*86400000.0); + } + return rc; +} + +SQLITE_PRIVATE int sqlite3OsOpenMalloc( + sqlite3_vfs *pVfs, + const char *zFile, + sqlite3_file **ppFile, + int flags, + int *pOutFlags +){ + int rc; + sqlite3_file *pFile; + pFile = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile); + if( pFile ){ + rc = sqlite3OsOpen(pVfs, zFile, pFile, flags, pOutFlags); + if( rc!=SQLITE_OK ){ + sqlite3_free(pFile); + }else{ + *ppFile = pFile; + } + }else{ + rc = SQLITE_NOMEM_BKPT; + } + return rc; +} +SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *pFile){ + assert( pFile ); + sqlite3OsClose(pFile); + sqlite3_free(pFile); +} + +/* +** This function is a wrapper around the OS specific implementation of +** sqlite3_os_init(). The purpose of the wrapper is to provide the +** ability to simulate a malloc failure, so that the handling of an +** error in sqlite3_os_init() by the upper layers can be tested. +*/ +SQLITE_PRIVATE int sqlite3OsInit(void){ + void *p = sqlite3_malloc(10); + if( p==0 ) return SQLITE_NOMEM_BKPT; + sqlite3_free(p); + return sqlite3_os_init(); +} + +/* +** The list of all registered VFS implementations. +*/ +static sqlite3_vfs * SQLITE_WSD vfsList = 0; +#define vfsList GLOBAL(sqlite3_vfs *, vfsList) + +/* +** Locate a VFS by name. If no name is given, simply return the +** first VFS on the list. +*/ +SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfs){ + sqlite3_vfs *pVfs = 0; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex; +#endif +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return 0; +#endif +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); +#endif + sqlite3_mutex_enter(mutex); + for(pVfs = vfsList; pVfs; pVfs=pVfs->pNext){ + if( zVfs==0 ) break; + if( strcmp(zVfs, pVfs->zName)==0 ) break; + } + sqlite3_mutex_leave(mutex); + return pVfs; +} + +/* +** Unlink a VFS from the linked list +*/ +static void vfsUnlink(sqlite3_vfs *pVfs){ + assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)) ); + if( pVfs==0 ){ + /* No-op */ + }else if( vfsList==pVfs ){ + vfsList = pVfs->pNext; + }else if( vfsList ){ + sqlite3_vfs *p = vfsList; + while( p->pNext && p->pNext!=pVfs ){ + p = p->pNext; + } + if( p->pNext==pVfs ){ + p->pNext = pVfs->pNext; + } + } +} + +/* +** Register a VFS with the system. It is harmless to register the same +** VFS multiple times. The new VFS becomes the default if makeDflt is +** true. +*/ +SQLITE_API int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ + MUTEX_LOGIC(sqlite3_mutex *mutex;) +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; +#endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( pVfs==0 ) return SQLITE_MISUSE_BKPT; +#endif + + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) + sqlite3_mutex_enter(mutex); + vfsUnlink(pVfs); + if( makeDflt || vfsList==0 ){ + pVfs->pNext = vfsList; + vfsList = pVfs; + }else{ + pVfs->pNext = vfsList->pNext; + vfsList->pNext = pVfs; + } + assert(vfsList); + sqlite3_mutex_leave(mutex); + return SQLITE_OK; +} + +/* +** Unregister a VFS so that it is no longer accessible. +*/ +SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ + MUTEX_LOGIC(sqlite3_mutex *mutex;) +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) + sqlite3_mutex_enter(mutex); + vfsUnlink(pVfs); + sqlite3_mutex_leave(mutex); + return SQLITE_OK; +} + +/************** End of os.c **************************************************/ +/************** Begin file fault.c *******************************************/ +/* +** 2008 Jan 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code to support the concept of "benign" +** malloc failures (when the xMalloc() or xRealloc() method of the +** sqlite3_mem_methods structure fails to allocate a block of memory +** and returns 0). +** +** Most malloc failures are non-benign. After they occur, SQLite +** abandons the current operation and returns an error code (usually +** SQLITE_NOMEM) to the user. However, sometimes a fault is not necessarily +** fatal. For example, if a malloc fails while resizing a hash table, this +** is completely recoverable simply by not carrying out the resize. The +** hash table will continue to function normally. So a malloc failure +** during a hash table resize is a benign fault. +*/ + +/* #include "sqliteInt.h" */ + +#ifndef SQLITE_UNTESTABLE + +/* +** Global variables. +*/ +typedef struct BenignMallocHooks BenignMallocHooks; +static SQLITE_WSD struct BenignMallocHooks { + void (*xBenignBegin)(void); + void (*xBenignEnd)(void); +} sqlite3Hooks = { 0, 0 }; + +/* The "wsdHooks" macro will resolve to the appropriate BenignMallocHooks +** structure. If writable static data is unsupported on the target, +** we have to locate the state vector at run-time. In the more common +** case where writable static data is supported, wsdHooks can refer directly +** to the "sqlite3Hooks" state vector declared above. +*/ +#ifdef SQLITE_OMIT_WSD +# define wsdHooksInit \ + BenignMallocHooks *x = &GLOBAL(BenignMallocHooks,sqlite3Hooks) +# define wsdHooks x[0] +#else +# define wsdHooksInit +# define wsdHooks sqlite3Hooks +#endif + + +/* +** Register hooks to call when sqlite3BeginBenignMalloc() and +** sqlite3EndBenignMalloc() are called, respectively. +*/ +SQLITE_PRIVATE void sqlite3BenignMallocHooks( + void (*xBenignBegin)(void), + void (*xBenignEnd)(void) +){ + wsdHooksInit; + wsdHooks.xBenignBegin = xBenignBegin; + wsdHooks.xBenignEnd = xBenignEnd; +} + +/* +** This (sqlite3EndBenignMalloc()) is called by SQLite code to indicate that +** subsequent malloc failures are benign. A call to sqlite3EndBenignMalloc() +** indicates that subsequent malloc failures are non-benign. +*/ +SQLITE_PRIVATE void sqlite3BeginBenignMalloc(void){ + wsdHooksInit; + if( wsdHooks.xBenignBegin ){ + wsdHooks.xBenignBegin(); + } +} +SQLITE_PRIVATE void sqlite3EndBenignMalloc(void){ + wsdHooksInit; + if( wsdHooks.xBenignEnd ){ + wsdHooks.xBenignEnd(); + } +} + +#endif /* #ifndef SQLITE_UNTESTABLE */ + +/************** End of fault.c ***********************************************/ +/************** Begin file mem0.c ********************************************/ +/* +** 2008 October 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains a no-op memory allocation drivers for use when +** SQLITE_ZERO_MALLOC is defined. The allocation drivers implemented +** here always fail. SQLite will not operate with these drivers. These +** are merely placeholders. Real drivers must be substituted using +** sqlite3_config() before SQLite will operate. +*/ +/* #include "sqliteInt.h" */ + +/* +** This version of the memory allocator is the default. It is +** used when no other memory allocator is specified using compile-time +** macros. +*/ +#ifdef SQLITE_ZERO_MALLOC + +/* +** No-op versions of all memory allocation routines +*/ +static void *sqlite3MemMalloc(int nByte){ return 0; } +static void sqlite3MemFree(void *pPrior){ return; } +static void *sqlite3MemRealloc(void *pPrior, int nByte){ return 0; } +static int sqlite3MemSize(void *pPrior){ return 0; } +static int sqlite3MemRoundup(int n){ return n; } +static int sqlite3MemInit(void *NotUsed){ return SQLITE_OK; } +static void sqlite3MemShutdown(void *NotUsed){ return; } + +/* +** This routine is the only routine in this file with external linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +SQLITE_PRIVATE void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +#endif /* SQLITE_ZERO_MALLOC */ + +/************** End of mem0.c ************************************************/ +/************** Begin file mem1.c ********************************************/ +/* +** 2007 August 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains low-level memory allocation drivers for when +** SQLite will use the standard C-library malloc/realloc/free interface +** to obtain the memory it needs. +** +** This file contains implementations of the low-level memory allocation +** routines specified in the sqlite3_mem_methods object. The content of +** this file is only used if SQLITE_SYSTEM_MALLOC is defined. The +** SQLITE_SYSTEM_MALLOC macro is defined automatically if neither the +** SQLITE_MEMDEBUG nor the SQLITE_WIN32_MALLOC macros are defined. The +** default configuration is to use memory allocation routines in this +** file. +** +** C-preprocessor macro summary: +** +** HAVE_MALLOC_USABLE_SIZE The configure script sets this symbol if +** the malloc_usable_size() interface exists +** on the target platform. Or, this symbol +** can be set manually, if desired. +** If an equivalent interface exists by +** a different name, using a separate -D +** option to rename it. +** +** SQLITE_WITHOUT_ZONEMALLOC Some older macs lack support for the zone +** memory allocator. Set this symbol to enable +** building on older macs. +** +** SQLITE_WITHOUT_MSIZE Set this symbol to disable the use of +** _msize() on windows systems. This might +** be necessary when compiling for Delphi, +** for example. +*/ +/* #include "sqliteInt.h" */ + +/* +** This version of the memory allocator is the default. It is +** used when no other memory allocator is specified using compile-time +** macros. +*/ +#ifdef SQLITE_SYSTEM_MALLOC +#if defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) + +/* +** Use the zone allocator available on apple products unless the +** SQLITE_WITHOUT_ZONEMALLOC symbol is defined. +*/ +#include +#include +#ifdef SQLITE_MIGHT_BE_SINGLE_CORE +#include +#endif /* SQLITE_MIGHT_BE_SINGLE_CORE */ +static malloc_zone_t* _sqliteZone_; +#define SQLITE_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) +#define SQLITE_FREE(x) malloc_zone_free(_sqliteZone_, (x)); +#define SQLITE_REALLOC(x,y) malloc_zone_realloc(_sqliteZone_, (x), (y)) +#define SQLITE_MALLOCSIZE(x) \ + (_sqliteZone_ ? _sqliteZone_->size(_sqliteZone_,x) : malloc_size(x)) + +#else /* if not __APPLE__ */ + +/* +** Use standard C library malloc and free on non-Apple systems. +** Also used by Apple systems if SQLITE_WITHOUT_ZONEMALLOC is defined. +*/ +#define SQLITE_MALLOC(x) malloc(x) +#define SQLITE_FREE(x) free(x) +#define SQLITE_REALLOC(x,y) realloc((x),(y)) + +/* +** The malloc.h header file is needed for malloc_usable_size() function +** on some systems (e.g. Linux). +*/ +#if HAVE_MALLOC_H && HAVE_MALLOC_USABLE_SIZE +# define SQLITE_USE_MALLOC_H 1 +# define SQLITE_USE_MALLOC_USABLE_SIZE 1 +/* +** The MSVCRT has malloc_usable_size(), but it is called _msize(). The +** use of _msize() is automatic, but can be disabled by compiling with +** -DSQLITE_WITHOUT_MSIZE. Using the _msize() function also requires +** the malloc.h header file. +*/ +#elif defined(_MSC_VER) && !defined(SQLITE_WITHOUT_MSIZE) +# define SQLITE_USE_MALLOC_H +# define SQLITE_USE_MSIZE +#endif + +/* +** Include the malloc.h header file, if necessary. Also set define macro +** SQLITE_MALLOCSIZE to the appropriate function name, which is _msize() +** for MSVC and malloc_usable_size() for most other systems (e.g. Linux). +** The memory size function can always be overridden manually by defining +** the macro SQLITE_MALLOCSIZE to the desired function name. +*/ +#if defined(SQLITE_USE_MALLOC_H) +# include +# if defined(SQLITE_USE_MALLOC_USABLE_SIZE) +# if !defined(SQLITE_MALLOCSIZE) +# define SQLITE_MALLOCSIZE(x) malloc_usable_size(x) +# endif +# elif defined(SQLITE_USE_MSIZE) +# if !defined(SQLITE_MALLOCSIZE) +# define SQLITE_MALLOCSIZE _msize +# endif +# endif +#endif /* defined(SQLITE_USE_MALLOC_H) */ + +#endif /* __APPLE__ or not __APPLE__ */ + +/* +** Like malloc(), but remember the size of the allocation +** so that we can find it later using sqlite3MemSize(). +** +** For this low-level routine, we are guaranteed that nByte>0 because +** cases of nByte<=0 will be intercepted and dealt with by higher level +** routines. +*/ +static void *sqlite3MemMalloc(int nByte){ +#ifdef SQLITE_MALLOCSIZE + void *p; + testcase( ROUND8(nByte)==nByte ); + p = SQLITE_MALLOC( nByte ); + if( p==0 ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); + } + return p; +#else + sqlite3_int64 *p; + assert( nByte>0 ); + testcase( ROUND8(nByte)!=nByte ); + p = SQLITE_MALLOC( nByte+8 ); + if( p ){ + p[0] = nByte; + p++; + }else{ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); + } + return (void *)p; +#endif +} + +/* +** Like free() but works for allocations obtained from sqlite3MemMalloc() +** or sqlite3MemRealloc(). +** +** For this low-level routine, we already know that pPrior!=0 since +** cases where pPrior==0 will have been intecepted and dealt with +** by higher-level routines. +*/ +static void sqlite3MemFree(void *pPrior){ +#ifdef SQLITE_MALLOCSIZE + SQLITE_FREE(pPrior); +#else + sqlite3_int64 *p = (sqlite3_int64*)pPrior; + assert( pPrior!=0 ); + p--; + SQLITE_FREE(p); +#endif +} + +/* +** Report the allocated size of a prior return from xMalloc() +** or xRealloc(). +*/ +static int sqlite3MemSize(void *pPrior){ +#ifdef SQLITE_MALLOCSIZE + assert( pPrior!=0 ); + return (int)SQLITE_MALLOCSIZE(pPrior); +#else + sqlite3_int64 *p; + assert( pPrior!=0 ); + p = (sqlite3_int64*)pPrior; + p--; + return (int)p[0]; +#endif +} + +/* +** Like realloc(). Resize an allocation previously obtained from +** sqlite3MemMalloc(). +** +** For this low-level interface, we know that pPrior!=0. Cases where +** pPrior==0 while have been intercepted by higher-level routine and +** redirected to xMalloc. Similarly, we know that nByte>0 because +** cases where nByte<=0 will have been intercepted by higher-level +** routines and redirected to xFree. +*/ +static void *sqlite3MemRealloc(void *pPrior, int nByte){ +#ifdef SQLITE_MALLOCSIZE + void *p = SQLITE_REALLOC(pPrior, nByte); + if( p==0 ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, + "failed memory resize %u to %u bytes", + SQLITE_MALLOCSIZE(pPrior), nByte); + } + return p; +#else + sqlite3_int64 *p = (sqlite3_int64*)pPrior; + assert( pPrior!=0 && nByte>0 ); + assert( nByte==ROUND8(nByte) ); /* EV: R-46199-30249 */ + p--; + p = SQLITE_REALLOC(p, nByte+8 ); + if( p ){ + p[0] = nByte; + p++; + }else{ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, + "failed memory resize %u to %u bytes", + sqlite3MemSize(pPrior), nByte); + } + return (void*)p; +#endif +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int sqlite3MemRoundup(int n){ + return ROUND8(n); +} + +/* +** Initialize this module. +*/ +static int sqlite3MemInit(void *NotUsed){ +#if defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) + int cpuCount; + size_t len; + if( _sqliteZone_ ){ + return SQLITE_OK; + } + len = sizeof(cpuCount); + /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ + sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); + if( cpuCount>1 ){ + /* defer MT decisions to system malloc */ + _sqliteZone_ = malloc_default_zone(); + }else{ + /* only 1 core, use our own zone to contention over global locks, + ** e.g. we have our own dedicated locks */ + _sqliteZone_ = malloc_create_zone(4096, 0); + malloc_set_zone_name(_sqliteZone_, "Sqlite_Heap"); + } +#endif /* defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) */ + UNUSED_PARAMETER(NotUsed); + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void sqlite3MemShutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + return; +} + +/* +** This routine is the only routine in this file with external linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +SQLITE_PRIVATE void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +#endif /* SQLITE_SYSTEM_MALLOC */ + +/************** End of mem1.c ************************************************/ +/************** Begin file mem2.c ********************************************/ +/* +** 2007 August 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains low-level memory allocation drivers for when +** SQLite will use the standard C-library malloc/realloc/free interface +** to obtain the memory it needs while adding lots of additional debugging +** information to each allocation in order to help detect and fix memory +** leaks and memory usage errors. +** +** This file contains implementations of the low-level memory allocation +** routines specified in the sqlite3_mem_methods object. +*/ +/* #include "sqliteInt.h" */ + +/* +** This version of the memory allocator is used only if the +** SQLITE_MEMDEBUG macro is defined +*/ +#ifdef SQLITE_MEMDEBUG + +/* +** The backtrace functionality is only available with GLIBC +*/ +#ifdef __GLIBC__ + extern int backtrace(void**,int); + extern void backtrace_symbols_fd(void*const*,int,int); +#else +# define backtrace(A,B) 1 +# define backtrace_symbols_fd(A,B,C) +#endif +/* #include */ + +/* +** Each memory allocation looks like this: +** +** ------------------------------------------------------------------------ +** | Title | backtrace pointers | MemBlockHdr | allocation | EndGuard | +** ------------------------------------------------------------------------ +** +** The application code sees only a pointer to the allocation. We have +** to back up from the allocation pointer to find the MemBlockHdr. The +** MemBlockHdr tells us the size of the allocation and the number of +** backtrace pointers. There is also a guard word at the end of the +** MemBlockHdr. +*/ +struct MemBlockHdr { + i64 iSize; /* Size of this allocation */ + struct MemBlockHdr *pNext, *pPrev; /* Linked list of all unfreed memory */ + char nBacktrace; /* Number of backtraces on this alloc */ + char nBacktraceSlots; /* Available backtrace slots */ + u8 nTitle; /* Bytes of title; includes '\0' */ + u8 eType; /* Allocation type code */ + int iForeGuard; /* Guard word for sanity */ +}; + +/* +** Guard words +*/ +#define FOREGUARD 0x80F5E153 +#define REARGUARD 0xE4676B53 + +/* +** Number of malloc size increments to track. +*/ +#define NCSIZE 1000 + +/* +** All of the static variables used by this module are collected +** into a single structure named "mem". This is to keep the +** static variables organized and to reduce namespace pollution +** when this module is combined with other in the amalgamation. +*/ +static struct { + + /* + ** Mutex to control access to the memory allocation subsystem. + */ + sqlite3_mutex *mutex; + + /* + ** Head and tail of a linked list of all outstanding allocations + */ + struct MemBlockHdr *pFirst; + struct MemBlockHdr *pLast; + + /* + ** The number of levels of backtrace to save in new allocations. + */ + int nBacktrace; + void (*xBacktrace)(int, int, void **); + + /* + ** Title text to insert in front of each block + */ + int nTitle; /* Bytes of zTitle to save. Includes '\0' and padding */ + char zTitle[100]; /* The title text */ + + /* + ** sqlite3MallocDisallow() increments the following counter. + ** sqlite3MallocAllow() decrements it. + */ + int disallow; /* Do not allow memory allocation */ + + /* + ** Gather statistics on the sizes of memory allocations. + ** nAlloc[i] is the number of allocation attempts of i*8 + ** bytes. i==NCSIZE is the number of allocation attempts for + ** sizes more than NCSIZE*8 bytes. + */ + int nAlloc[NCSIZE]; /* Total number of allocations */ + int nCurrent[NCSIZE]; /* Current number of allocations */ + int mxCurrent[NCSIZE]; /* Highwater mark for nCurrent */ + +} mem; + + +/* +** Adjust memory usage statistics +*/ +static void adjustStats(int iSize, int increment){ + int i = ROUND8(iSize)/8; + if( i>NCSIZE-1 ){ + i = NCSIZE - 1; + } + if( increment>0 ){ + mem.nAlloc[i]++; + mem.nCurrent[i]++; + if( mem.nCurrent[i]>mem.mxCurrent[i] ){ + mem.mxCurrent[i] = mem.nCurrent[i]; + } + }else{ + mem.nCurrent[i]--; + assert( mem.nCurrent[i]>=0 ); + } +} + +/* +** Given an allocation, find the MemBlockHdr for that allocation. +** +** This routine checks the guards at either end of the allocation and +** if they are incorrect it asserts. +*/ +static struct MemBlockHdr *sqlite3MemsysGetHeader(void *pAllocation){ + struct MemBlockHdr *p; + int *pInt; + u8 *pU8; + int nReserve; + + p = (struct MemBlockHdr*)pAllocation; + p--; + assert( p->iForeGuard==(int)FOREGUARD ); + nReserve = ROUND8(p->iSize); + pInt = (int*)pAllocation; + pU8 = (u8*)pAllocation; + assert( pInt[nReserve/sizeof(int)]==(int)REARGUARD ); + /* This checks any of the "extra" bytes allocated due + ** to rounding up to an 8 byte boundary to ensure + ** they haven't been overwritten. + */ + while( nReserve-- > p->iSize ) assert( pU8[nReserve]==0x65 ); + return p; +} + +/* +** Return the number of bytes currently allocated at address p. +*/ +static int sqlite3MemSize(void *p){ + struct MemBlockHdr *pHdr; + if( !p ){ + return 0; + } + pHdr = sqlite3MemsysGetHeader(p); + return (int)pHdr->iSize; +} + +/* +** Initialize the memory allocation subsystem. +*/ +static int sqlite3MemInit(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + assert( (sizeof(struct MemBlockHdr)&7) == 0 ); + if( !sqlite3GlobalConfig.bMemstat ){ + /* If memory status is enabled, then the malloc.c wrapper will already + ** hold the STATIC_MEM mutex when the routines here are invoked. */ + mem.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + return SQLITE_OK; +} + +/* +** Deinitialize the memory allocation subsystem. +*/ +static void sqlite3MemShutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + mem.mutex = 0; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int sqlite3MemRoundup(int n){ + return ROUND8(n); +} + +/* +** Fill a buffer with pseudo-random bytes. This is used to preset +** the content of a new memory allocation to unpredictable values and +** to clear the content of a freed allocation to unpredictable values. +*/ +static void randomFill(char *pBuf, int nByte){ + unsigned int x, y, r; + x = SQLITE_PTR_TO_INT(pBuf); + y = nByte | 1; + while( nByte >= 4 ){ + x = (x>>1) ^ (-(int)(x&1) & 0xd0000001); + y = y*1103515245 + 12345; + r = x ^ y; + *(int*)pBuf = r; + pBuf += 4; + nByte -= 4; + } + while( nByte-- > 0 ){ + x = (x>>1) ^ (-(int)(x&1) & 0xd0000001); + y = y*1103515245 + 12345; + r = x ^ y; + *(pBuf++) = r & 0xff; + } +} + +/* +** Allocate nByte bytes of memory. +*/ +static void *sqlite3MemMalloc(int nByte){ + struct MemBlockHdr *pHdr; + void **pBt; + char *z; + int *pInt; + void *p = 0; + int totalSize; + int nReserve; + sqlite3_mutex_enter(mem.mutex); + assert( mem.disallow==0 ); + nReserve = ROUND8(nByte); + totalSize = nReserve + sizeof(*pHdr) + sizeof(int) + + mem.nBacktrace*sizeof(void*) + mem.nTitle; + p = malloc(totalSize); + if( p ){ + z = p; + pBt = (void**)&z[mem.nTitle]; + pHdr = (struct MemBlockHdr*)&pBt[mem.nBacktrace]; + pHdr->pNext = 0; + pHdr->pPrev = mem.pLast; + if( mem.pLast ){ + mem.pLast->pNext = pHdr; + }else{ + mem.pFirst = pHdr; + } + mem.pLast = pHdr; + pHdr->iForeGuard = FOREGUARD; + pHdr->eType = MEMTYPE_HEAP; + pHdr->nBacktraceSlots = mem.nBacktrace; + pHdr->nTitle = mem.nTitle; + if( mem.nBacktrace ){ + void *aAddr[40]; + pHdr->nBacktrace = backtrace(aAddr, mem.nBacktrace+1)-1; + memcpy(pBt, &aAddr[1], pHdr->nBacktrace*sizeof(void*)); + assert(pBt[0]); + if( mem.xBacktrace ){ + mem.xBacktrace(nByte, pHdr->nBacktrace-1, &aAddr[1]); + } + }else{ + pHdr->nBacktrace = 0; + } + if( mem.nTitle ){ + memcpy(z, mem.zTitle, mem.nTitle); + } + pHdr->iSize = nByte; + adjustStats(nByte, +1); + pInt = (int*)&pHdr[1]; + pInt[nReserve/sizeof(int)] = REARGUARD; + randomFill((char*)pInt, nByte); + memset(((char*)pInt)+nByte, 0x65, nReserve-nByte); + p = (void*)pInt; + } + sqlite3_mutex_leave(mem.mutex); + return p; +} + +/* +** Free memory. +*/ +static void sqlite3MemFree(void *pPrior){ + struct MemBlockHdr *pHdr; + void **pBt; + char *z; + assert( sqlite3GlobalConfig.bMemstat || sqlite3GlobalConfig.bCoreMutex==0 + || mem.mutex!=0 ); + pHdr = sqlite3MemsysGetHeader(pPrior); + pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + sqlite3_mutex_enter(mem.mutex); + if( pHdr->pPrev ){ + assert( pHdr->pPrev->pNext==pHdr ); + pHdr->pPrev->pNext = pHdr->pNext; + }else{ + assert( mem.pFirst==pHdr ); + mem.pFirst = pHdr->pNext; + } + if( pHdr->pNext ){ + assert( pHdr->pNext->pPrev==pHdr ); + pHdr->pNext->pPrev = pHdr->pPrev; + }else{ + assert( mem.pLast==pHdr ); + mem.pLast = pHdr->pPrev; + } + z = (char*)pBt; + z -= pHdr->nTitle; + adjustStats((int)pHdr->iSize, -1); + randomFill(z, sizeof(void*)*pHdr->nBacktraceSlots + sizeof(*pHdr) + + (int)pHdr->iSize + sizeof(int) + pHdr->nTitle); + free(z); + sqlite3_mutex_leave(mem.mutex); +} + +/* +** Change the size of an existing memory allocation. +** +** For this debugging implementation, we *always* make a copy of the +** allocation into a new place in memory. In this way, if the +** higher level code is using pointer to the old allocation, it is +** much more likely to break and we are much more liking to find +** the error. +*/ +static void *sqlite3MemRealloc(void *pPrior, int nByte){ + struct MemBlockHdr *pOldHdr; + void *pNew; + assert( mem.disallow==0 ); + assert( (nByte & 7)==0 ); /* EV: R-46199-30249 */ + pOldHdr = sqlite3MemsysGetHeader(pPrior); + pNew = sqlite3MemMalloc(nByte); + if( pNew ){ + memcpy(pNew, pPrior, (int)(nByteiSize ? nByte : pOldHdr->iSize)); + if( nByte>pOldHdr->iSize ){ + randomFill(&((char*)pNew)[pOldHdr->iSize], nByte - (int)pOldHdr->iSize); + } + sqlite3MemFree(pPrior); + } + return pNew; +} + +/* +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. +*/ +SQLITE_PRIVATE void sqlite3MemSetDefault(void){ + static const sqlite3_mem_methods defaultMethods = { + sqlite3MemMalloc, + sqlite3MemFree, + sqlite3MemRealloc, + sqlite3MemSize, + sqlite3MemRoundup, + sqlite3MemInit, + sqlite3MemShutdown, + 0 + }; + sqlite3_config(SQLITE_CONFIG_MALLOC, &defaultMethods); +} + +/* +** Set the "type" of an allocation. +*/ +SQLITE_PRIVATE void sqlite3MemdebugSetType(void *p, u8 eType){ + if( p && sqlite3GlobalConfig.m.xFree==sqlite3MemFree ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); + pHdr->eType = eType; + } +} + +/* +** Return TRUE if the mask of type in eType matches the type of the +** allocation p. Also return true if p==NULL. +** +** This routine is designed for use within an assert() statement, to +** verify the type of an allocation. For example: +** +** assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); +*/ +SQLITE_PRIVATE int sqlite3MemdebugHasType(void *p, u8 eType){ + int rc = 1; + if( p && sqlite3GlobalConfig.m.xFree==sqlite3MemFree ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ + if( (pHdr->eType&eType)==0 ){ + rc = 0; + } + } + return rc; +} + +/* +** Return TRUE if the mask of type in eType matches no bits of the type of the +** allocation p. Also return true if p==NULL. +** +** This routine is designed for use within an assert() statement, to +** verify the type of an allocation. For example: +** +** assert( sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); +*/ +SQLITE_PRIVATE int sqlite3MemdebugNoType(void *p, u8 eType){ + int rc = 1; + if( p && sqlite3GlobalConfig.m.xFree==sqlite3MemFree ){ + struct MemBlockHdr *pHdr; + pHdr = sqlite3MemsysGetHeader(p); + assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ + if( (pHdr->eType&eType)!=0 ){ + rc = 0; + } + } + return rc; +} + +/* +** Set the number of backtrace levels kept for each allocation. +** A value of zero turns off backtracing. The number is always rounded +** up to a multiple of 2. +*/ +SQLITE_PRIVATE void sqlite3MemdebugBacktrace(int depth){ + if( depth<0 ){ depth = 0; } + if( depth>20 ){ depth = 20; } + depth = (depth+1)&0xfe; + mem.nBacktrace = depth; +} + +SQLITE_PRIVATE void sqlite3MemdebugBacktraceCallback(void (*xBacktrace)(int, int, void **)){ + mem.xBacktrace = xBacktrace; +} + +/* +** Set the title string for subsequent allocations. +*/ +SQLITE_PRIVATE void sqlite3MemdebugSettitle(const char *zTitle){ + unsigned int n = sqlite3Strlen30(zTitle) + 1; + sqlite3_mutex_enter(mem.mutex); + if( n>=sizeof(mem.zTitle) ) n = sizeof(mem.zTitle)-1; + memcpy(mem.zTitle, zTitle, n); + mem.zTitle[n] = 0; + mem.nTitle = ROUND8(n); + sqlite3_mutex_leave(mem.mutex); +} + +SQLITE_PRIVATE void sqlite3MemdebugSync(){ + struct MemBlockHdr *pHdr; + for(pHdr=mem.pFirst; pHdr; pHdr=pHdr->pNext){ + void **pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + mem.xBacktrace((int)pHdr->iSize, pHdr->nBacktrace-1, &pBt[1]); + } +} + +/* +** Open the file indicated and write a log of all unfreed memory +** allocations into that log. +*/ +SQLITE_PRIVATE void sqlite3MemdebugDump(const char *zFilename){ + FILE *out; + struct MemBlockHdr *pHdr; + void **pBt; + int i; + out = fopen(zFilename, "w"); + if( out==0 ){ + fprintf(stderr, "** Unable to output memory debug output log: %s **\n", + zFilename); + return; + } + for(pHdr=mem.pFirst; pHdr; pHdr=pHdr->pNext){ + char *z = (char*)pHdr; + z -= pHdr->nBacktraceSlots*sizeof(void*) + pHdr->nTitle; + fprintf(out, "**** %lld bytes at %p from %s ****\n", + pHdr->iSize, &pHdr[1], pHdr->nTitle ? z : "???"); + if( pHdr->nBacktrace ){ + fflush(out); + pBt = (void**)pHdr; + pBt -= pHdr->nBacktraceSlots; + backtrace_symbols_fd(pBt, pHdr->nBacktrace, fileno(out)); + fprintf(out, "\n"); + } + } + fprintf(out, "COUNTS:\n"); + for(i=0; i=1 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( size==mem3.aPool[i+size-1].u.hdr.prevSize ); + assert( size>=2 ); + if( size <= MX_SMALL ){ + memsys3UnlinkFromList(i, &mem3.aiSmall[size-2]); + }else{ + hash = size % N_HASH; + memsys3UnlinkFromList(i, &mem3.aiHash[hash]); + } +} + +/* +** Link the chunk at mem3.aPool[i] so that is on the list rooted +** at *pRoot. +*/ +static void memsys3LinkIntoList(u32 i, u32 *pRoot){ + assert( sqlite3_mutex_held(mem3.mutex) ); + mem3.aPool[i].u.list.next = *pRoot; + mem3.aPool[i].u.list.prev = 0; + if( *pRoot ){ + mem3.aPool[*pRoot].u.list.prev = i; + } + *pRoot = i; +} + +/* +** Link the chunk at index i into either the appropriate +** small chunk list, or into the large chunk hash table. +*/ +static void memsys3Link(u32 i){ + u32 size, hash; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( i>=1 ); + assert( (mem3.aPool[i-1].u.hdr.size4x & 1)==0 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( size==mem3.aPool[i+size-1].u.hdr.prevSize ); + assert( size>=2 ); + if( size <= MX_SMALL ){ + memsys3LinkIntoList(i, &mem3.aiSmall[size-2]); + }else{ + hash = size % N_HASH; + memsys3LinkIntoList(i, &mem3.aiHash[hash]); + } +} + +/* +** If the STATIC_MEM mutex is not already held, obtain it now. The mutex +** will already be held (obtained by code in malloc.c) if +** sqlite3GlobalConfig.bMemStat is true. +*/ +static void memsys3Enter(void){ + if( sqlite3GlobalConfig.bMemstat==0 && mem3.mutex==0 ){ + mem3.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + sqlite3_mutex_enter(mem3.mutex); +} +static void memsys3Leave(void){ + sqlite3_mutex_leave(mem3.mutex); +} + +/* +** Called when we are unable to satisfy an allocation of nBytes. +*/ +static void memsys3OutOfMemory(int nByte){ + if( !mem3.alarmBusy ){ + mem3.alarmBusy = 1; + assert( sqlite3_mutex_held(mem3.mutex) ); + sqlite3_mutex_leave(mem3.mutex); + sqlite3_release_memory(nByte); + sqlite3_mutex_enter(mem3.mutex); + mem3.alarmBusy = 0; + } +} + + +/* +** Chunk i is a free chunk that has been unlinked. Adjust its +** size parameters for check-out and return a pointer to the +** user portion of the chunk. +*/ +static void *memsys3Checkout(u32 i, u32 nBlock){ + u32 x; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( i>=1 ); + assert( mem3.aPool[i-1].u.hdr.size4x/4==nBlock ); + assert( mem3.aPool[i+nBlock-1].u.hdr.prevSize==nBlock ); + x = mem3.aPool[i-1].u.hdr.size4x; + mem3.aPool[i-1].u.hdr.size4x = nBlock*4 | 1 | (x&2); + mem3.aPool[i+nBlock-1].u.hdr.prevSize = nBlock; + mem3.aPool[i+nBlock-1].u.hdr.size4x |= 2; + return &mem3.aPool[i]; +} + +/* +** Carve a piece off of the end of the mem3.iKeyBlk free chunk. +** Return a pointer to the new allocation. Or, if the key chunk +** is not large enough, return 0. +*/ +static void *memsys3FromKeyBlk(u32 nBlock){ + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( mem3.szKeyBlk>=nBlock ); + if( nBlock>=mem3.szKeyBlk-1 ){ + /* Use the entire key chunk */ + void *p = memsys3Checkout(mem3.iKeyBlk, mem3.szKeyBlk); + mem3.iKeyBlk = 0; + mem3.szKeyBlk = 0; + mem3.mnKeyBlk = 0; + return p; + }else{ + /* Split the key block. Return the tail. */ + u32 newi, x; + newi = mem3.iKeyBlk + mem3.szKeyBlk - nBlock; + assert( newi > mem3.iKeyBlk+1 ); + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = nBlock; + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x |= 2; + mem3.aPool[newi-1].u.hdr.size4x = nBlock*4 + 1; + mem3.szKeyBlk -= nBlock; + mem3.aPool[newi-1].u.hdr.prevSize = mem3.szKeyBlk; + x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2; + mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x; + if( mem3.szKeyBlk < mem3.mnKeyBlk ){ + mem3.mnKeyBlk = mem3.szKeyBlk; + } + return (void*)&mem3.aPool[newi]; + } +} + +/* +** *pRoot is the head of a list of free chunks of the same size +** or same size hash. In other words, *pRoot is an entry in either +** mem3.aiSmall[] or mem3.aiHash[]. +** +** This routine examines all entries on the given list and tries +** to coalesce each entries with adjacent free chunks. +** +** If it sees a chunk that is larger than mem3.iKeyBlk, it replaces +** the current mem3.iKeyBlk with the new larger chunk. In order for +** this mem3.iKeyBlk replacement to work, the key chunk must be +** linked into the hash tables. That is not the normal state of +** affairs, of course. The calling routine must link the key +** chunk before invoking this routine, then must unlink the (possibly +** changed) key chunk once this routine has finished. +*/ +static void memsys3Merge(u32 *pRoot){ + u32 iNext, prev, size, i, x; + + assert( sqlite3_mutex_held(mem3.mutex) ); + for(i=*pRoot; i>0; i=iNext){ + iNext = mem3.aPool[i].u.list.next; + size = mem3.aPool[i-1].u.hdr.size4x; + assert( (size&1)==0 ); + if( (size&2)==0 ){ + memsys3UnlinkFromList(i, pRoot); + assert( i > mem3.aPool[i-1].u.hdr.prevSize ); + prev = i - mem3.aPool[i-1].u.hdr.prevSize; + if( prev==iNext ){ + iNext = mem3.aPool[prev].u.list.next; + } + memsys3Unlink(prev); + size = i + size/4 - prev; + x = mem3.aPool[prev-1].u.hdr.size4x & 2; + mem3.aPool[prev-1].u.hdr.size4x = size*4 | x; + mem3.aPool[prev+size-1].u.hdr.prevSize = size; + memsys3Link(prev); + i = prev; + }else{ + size /= 4; + } + if( size>mem3.szKeyBlk ){ + mem3.iKeyBlk = i; + mem3.szKeyBlk = size; + } + } +} + +/* +** Return a block of memory of at least nBytes in size. +** Return NULL if unable. +** +** This function assumes that the necessary mutexes, if any, are +** already held by the caller. Hence "Unsafe". +*/ +static void *memsys3MallocUnsafe(int nByte){ + u32 i; + u32 nBlock; + u32 toFree; + + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( sizeof(Mem3Block)==8 ); + if( nByte<=12 ){ + nBlock = 2; + }else{ + nBlock = (nByte + 11)/8; + } + assert( nBlock>=2 ); + + /* STEP 1: + ** Look for an entry of the correct size in either the small + ** chunk table or in the large chunk hash table. This is + ** successful most of the time (about 9 times out of 10). + */ + if( nBlock <= MX_SMALL ){ + i = mem3.aiSmall[nBlock-2]; + if( i>0 ){ + memsys3UnlinkFromList(i, &mem3.aiSmall[nBlock-2]); + return memsys3Checkout(i, nBlock); + } + }else{ + int hash = nBlock % N_HASH; + for(i=mem3.aiHash[hash]; i>0; i=mem3.aPool[i].u.list.next){ + if( mem3.aPool[i-1].u.hdr.size4x/4==nBlock ){ + memsys3UnlinkFromList(i, &mem3.aiHash[hash]); + return memsys3Checkout(i, nBlock); + } + } + } + + /* STEP 2: + ** Try to satisfy the allocation by carving a piece off of the end + ** of the key chunk. This step usually works if step 1 fails. + */ + if( mem3.szKeyBlk>=nBlock ){ + return memsys3FromKeyBlk(nBlock); + } + + + /* STEP 3: + ** Loop through the entire memory pool. Coalesce adjacent free + ** chunks. Recompute the key chunk as the largest free chunk. + ** Then try again to satisfy the allocation by carving a piece off + ** of the end of the key chunk. This step happens very + ** rarely (we hope!) + */ + for(toFree=nBlock*16; toFree<(mem3.nPool*16); toFree *= 2){ + memsys3OutOfMemory(toFree); + if( mem3.iKeyBlk ){ + memsys3Link(mem3.iKeyBlk); + mem3.iKeyBlk = 0; + mem3.szKeyBlk = 0; + } + for(i=0; i=nBlock ){ + return memsys3FromKeyBlk(nBlock); + } + } + } + + /* If none of the above worked, then we fail. */ + return 0; +} + +/* +** Free an outstanding memory allocation. +** +** This function assumes that the necessary mutexes, if any, are +** already held by the caller. Hence "Unsafe". +*/ +static void memsys3FreeUnsafe(void *pOld){ + Mem3Block *p = (Mem3Block*)pOld; + int i; + u32 size, x; + assert( sqlite3_mutex_held(mem3.mutex) ); + assert( p>mem3.aPool && p<&mem3.aPool[mem3.nPool] ); + i = p - mem3.aPool; + assert( (mem3.aPool[i-1].u.hdr.size4x&1)==1 ); + size = mem3.aPool[i-1].u.hdr.size4x/4; + assert( i+size<=mem3.nPool+1 ); + mem3.aPool[i-1].u.hdr.size4x &= ~1; + mem3.aPool[i+size-1].u.hdr.prevSize = size; + mem3.aPool[i+size-1].u.hdr.size4x &= ~2; + memsys3Link(i); + + /* Try to expand the key using the newly freed chunk */ + if( mem3.iKeyBlk ){ + while( (mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x&2)==0 ){ + size = mem3.aPool[mem3.iKeyBlk-1].u.hdr.prevSize; + mem3.iKeyBlk -= size; + mem3.szKeyBlk += size; + memsys3Unlink(mem3.iKeyBlk); + x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2; + mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x; + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = mem3.szKeyBlk; + } + x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2; + while( (mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x&1)==0 ){ + memsys3Unlink(mem3.iKeyBlk+mem3.szKeyBlk); + mem3.szKeyBlk += mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x/4; + mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x; + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = mem3.szKeyBlk; + } + } +} + +/* +** Return the size of an outstanding allocation, in bytes. The +** size returned omits the 8-byte header overhead. This only +** works for chunks that are currently checked out. +*/ +static int memsys3Size(void *p){ + Mem3Block *pBlock; + assert( p!=0 ); + pBlock = (Mem3Block*)p; + assert( (pBlock[-1].u.hdr.size4x&1)!=0 ); + return (pBlock[-1].u.hdr.size4x&~3)*2 - 4; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int memsys3Roundup(int n){ + if( n<=12 ){ + return 12; + }else{ + return ((n+11)&~7) - 4; + } +} + +/* +** Allocate nBytes of memory. +*/ +static void *memsys3Malloc(int nBytes){ + sqlite3_int64 *p; + assert( nBytes>0 ); /* malloc.c filters out 0 byte requests */ + memsys3Enter(); + p = memsys3MallocUnsafe(nBytes); + memsys3Leave(); + return (void*)p; +} + +/* +** Free memory. +*/ +static void memsys3Free(void *pPrior){ + assert( pPrior ); + memsys3Enter(); + memsys3FreeUnsafe(pPrior); + memsys3Leave(); +} + +/* +** Change the size of an existing memory allocation +*/ +static void *memsys3Realloc(void *pPrior, int nBytes){ + int nOld; + void *p; + if( pPrior==0 ){ + return sqlite3_malloc(nBytes); + } + if( nBytes<=0 ){ + sqlite3_free(pPrior); + return 0; + } + nOld = memsys3Size(pPrior); + if( nBytes<=nOld && nBytes>=nOld-128 ){ + return pPrior; + } + memsys3Enter(); + p = memsys3MallocUnsafe(nBytes); + if( p ){ + if( nOld>1)!=(size&1) ){ + fprintf(out, "%p tail checkout bit is incorrect\n", &mem3.aPool[i]); + assert( 0 ); + break; + } + if( size&1 ){ + fprintf(out, "%p %6d bytes checked out\n", &mem3.aPool[i], (size/4)*8-8); + }else{ + fprintf(out, "%p %6d bytes free%s\n", &mem3.aPool[i], (size/4)*8-8, + i==mem3.iKeyBlk ? " **key**" : ""); + } + } + for(i=0; i0; j=mem3.aPool[j].u.list.next){ + fprintf(out, " %p(%d)", &mem3.aPool[j], + (mem3.aPool[j-1].u.hdr.size4x/4)*8-8); + } + fprintf(out, "\n"); + } + for(i=0; i0; j=mem3.aPool[j].u.list.next){ + fprintf(out, " %p(%d)", &mem3.aPool[j], + (mem3.aPool[j-1].u.hdr.size4x/4)*8-8); + } + fprintf(out, "\n"); + } + fprintf(out, "key=%d\n", mem3.iKeyBlk); + fprintf(out, "nowUsed=%d\n", mem3.nPool*8 - mem3.szKeyBlk*8); + fprintf(out, "mxUsed=%d\n", mem3.nPool*8 - mem3.mnKeyBlk*8); + sqlite3_mutex_leave(mem3.mutex); + if( out==stdout ){ + fflush(stdout); + }else{ + fclose(out); + } +#else + UNUSED_PARAMETER(zFilename); +#endif +} + +/* +** This routine is the only routine in this file with external +** linkage. +** +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. The +** arguments specify the block of memory to manage. +** +** This routine is only called by sqlite3_config(), and therefore +** is not required to be threadsafe (it is not). +*/ +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ + static const sqlite3_mem_methods mempoolMethods = { + memsys3Malloc, + memsys3Free, + memsys3Realloc, + memsys3Size, + memsys3Roundup, + memsys3Init, + memsys3Shutdown, + 0 + }; + return &mempoolMethods; +} + +#endif /* SQLITE_ENABLE_MEMSYS3 */ + +/************** End of mem3.c ************************************************/ +/************** Begin file mem5.c ********************************************/ +/* +** 2007 October 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement a memory +** allocation subsystem for use by SQLite. +** +** This version of the memory allocation subsystem omits all +** use of malloc(). The application gives SQLite a block of memory +** before calling sqlite3_initialize() from which allocations +** are made and returned by the xMalloc() and xRealloc() +** implementations. Once sqlite3_initialize() has been called, +** the amount of memory available to SQLite is fixed and cannot +** be changed. +** +** This version of the memory allocation subsystem is included +** in the build only if SQLITE_ENABLE_MEMSYS5 is defined. +** +** This memory allocator uses the following algorithm: +** +** 1. All memory allocation sizes are rounded up to a power of 2. +** +** 2. If two adjacent free blocks are the halves of a larger block, +** then the two blocks are coalesced into the single larger block. +** +** 3. New memory is allocated from the first available free block. +** +** This algorithm is described in: J. M. Robson. "Bounds for Some Functions +** Concerning Dynamic Storage Allocation". Journal of the Association for +** Computing Machinery, Volume 21, Number 8, July 1974, pages 491-499. +** +** Let n be the size of the largest allocation divided by the minimum +** allocation size (after rounding all sizes up to a power of 2.) Let M +** be the maximum amount of memory ever outstanding at one time. Let +** N be the total amount of memory available for allocation. Robson +** proved that this memory allocator will never breakdown due to +** fragmentation as long as the following constraint holds: +** +** N >= M*(1 + log2(n)/2) - n + 1 +** +** The sqlite3_status() logic tracks the maximum values of n and M so +** that an application can, at any time, verify this constraint. +*/ +/* #include "sqliteInt.h" */ + +/* +** This version of the memory allocator is used only when +** SQLITE_ENABLE_MEMSYS5 is defined. +*/ +#ifdef SQLITE_ENABLE_MEMSYS5 + +/* +** A minimum allocation is an instance of the following structure. +** Larger allocations are an array of these structures where the +** size of the array is a power of 2. +** +** The size of this object must be a power of two. That fact is +** verified in memsys5Init(). +*/ +typedef struct Mem5Link Mem5Link; +struct Mem5Link { + int next; /* Index of next free chunk */ + int prev; /* Index of previous free chunk */ +}; + +/* +** Maximum size of any allocation is ((1<=0 && i=0 && iLogsize<=LOGMAX ); + assert( (mem5.aCtrl[i] & CTRL_LOGSIZE)==iLogsize ); + + next = MEM5LINK(i)->next; + prev = MEM5LINK(i)->prev; + if( prev<0 ){ + mem5.aiFreelist[iLogsize] = next; + }else{ + MEM5LINK(prev)->next = next; + } + if( next>=0 ){ + MEM5LINK(next)->prev = prev; + } +} + +/* +** Link the chunk at mem5.aPool[i] so that is on the iLogsize +** free list. +*/ +static void memsys5Link(int i, int iLogsize){ + int x; + assert( sqlite3_mutex_held(mem5.mutex) ); + assert( i>=0 && i=0 && iLogsize<=LOGMAX ); + assert( (mem5.aCtrl[i] & CTRL_LOGSIZE)==iLogsize ); + + x = MEM5LINK(i)->next = mem5.aiFreelist[iLogsize]; + MEM5LINK(i)->prev = -1; + if( x>=0 ){ + assert( xprev = i; + } + mem5.aiFreelist[iLogsize] = i; +} + +/* +** Obtain or release the mutex needed to access global data structures. +*/ +static void memsys5Enter(void){ + sqlite3_mutex_enter(mem5.mutex); +} +static void memsys5Leave(void){ + sqlite3_mutex_leave(mem5.mutex); +} + +/* +** Return the size of an outstanding allocation, in bytes. +** This only works for chunks that are currently checked out. +*/ +static int memsys5Size(void *p){ + int iSize, i; + assert( p!=0 ); + i = (int)(((u8 *)p-mem5.zPool)/mem5.szAtom); + assert( i>=0 && i0 ); + + /* No more than 1GiB per allocation */ + if( nByte > 0x40000000 ) return 0; + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* Keep track of the maximum allocation request. Even unfulfilled + ** requests are counted */ + if( (u32)nByte>mem5.maxRequest ){ + mem5.maxRequest = nByte; + } +#endif + + + /* Round nByte up to the next valid power of two */ + for(iFullSz=mem5.szAtom,iLogsize=0; iFullSzLOGMAX ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes", nByte); + return 0; + } + i = mem5.aiFreelist[iBin]; + memsys5Unlink(i, iBin); + while( iBin>iLogsize ){ + int newSize; + + iBin--; + newSize = 1 << iBin; + mem5.aCtrl[i+newSize] = CTRL_FREE | iBin; + memsys5Link(i+newSize, iBin); + } + mem5.aCtrl[i] = iLogsize; + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* Update allocator performance statistics. */ + mem5.nAlloc++; + mem5.totalAlloc += iFullSz; + mem5.totalExcess += iFullSz - nByte; + mem5.currentCount++; + mem5.currentOut += iFullSz; + if( mem5.maxCount=0 && iBlock0 ); + assert( mem5.currentOut>=(size*mem5.szAtom) ); + mem5.currentCount--; + mem5.currentOut -= size*mem5.szAtom; + assert( mem5.currentOut>0 || mem5.currentCount==0 ); + assert( mem5.currentCount>0 || mem5.currentOut==0 ); +#endif + + mem5.aCtrl[iBlock] = CTRL_FREE | iLogsize; + while( ALWAYS(iLogsize>iLogsize) & 1 ){ + iBuddy = iBlock - size; + assert( iBuddy>=0 ); + }else{ + iBuddy = iBlock + size; + if( iBuddy>=mem5.nBlock ) break; + } + if( mem5.aCtrl[iBuddy]!=(CTRL_FREE | iLogsize) ) break; + memsys5Unlink(iBuddy, iLogsize); + iLogsize++; + if( iBuddy0 ){ + memsys5Enter(); + p = memsys5MallocUnsafe(nBytes); + memsys5Leave(); + } + return (void*)p; +} + +/* +** Free memory. +** +** The outer layer memory allocator prevents this routine from +** being called with pPrior==0. +*/ +static void memsys5Free(void *pPrior){ + assert( pPrior!=0 ); + memsys5Enter(); + memsys5FreeUnsafe(pPrior); + memsys5Leave(); +} + +/* +** Change the size of an existing memory allocation. +** +** The outer layer memory allocator prevents this routine from +** being called with pPrior==0. +** +** nBytes is always a value obtained from a prior call to +** memsys5Round(). Hence nBytes is always a non-negative power +** of two. If nBytes==0 that means that an oversize allocation +** (an allocation larger than 0x40000000) was requested and this +** routine should return 0 without freeing pPrior. +*/ +static void *memsys5Realloc(void *pPrior, int nBytes){ + int nOld; + void *p; + assert( pPrior!=0 ); + assert( (nBytes&(nBytes-1))==0 ); /* EV: R-46199-30249 */ + assert( nBytes>=0 ); + if( nBytes==0 ){ + return 0; + } + nOld = memsys5Size(pPrior); + if( nBytes<=nOld ){ + return pPrior; + } + p = memsys5Malloc(nBytes); + if( p ){ + memcpy(p, pPrior, nOld); + memsys5Free(pPrior); + } + return p; +} + +/* +** Round up a request size to the next valid allocation size. If +** the allocation is too large to be handled by this allocation system, +** return 0. +** +** All allocations must be a power of two and must be expressed by a +** 32-bit signed integer. Hence the largest allocation is 0x40000000 +** or 1073741824 bytes. +*/ +static int memsys5Roundup(int n){ + int iFullSz; + if( n > 0x40000000 ) return 0; + for(iFullSz=mem5.szAtom; iFullSz 0 +** memsys5Log(2) -> 1 +** memsys5Log(4) -> 2 +** memsys5Log(5) -> 3 +** memsys5Log(8) -> 3 +** memsys5Log(9) -> 4 +*/ +static int memsys5Log(int iValue){ + int iLog; + for(iLog=0; (iLog<(int)((sizeof(int)*8)-1)) && (1<mem5.szAtom ){ + mem5.szAtom = mem5.szAtom << 1; + } + + mem5.nBlock = (nByte / (mem5.szAtom+sizeof(u8))); + mem5.zPool = zByte; + mem5.aCtrl = (u8 *)&mem5.zPool[mem5.nBlock*mem5.szAtom]; + + for(ii=0; ii<=LOGMAX; ii++){ + mem5.aiFreelist[ii] = -1; + } + + iOffset = 0; + for(ii=LOGMAX; ii>=0; ii--){ + int nAlloc = (1<mem5.nBlock); + } + + /* If a mutex is required for normal operation, allocate one */ + if( sqlite3GlobalConfig.bMemstat==0 ){ + mem5.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); + } + + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void memsys5Shutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + mem5.mutex = 0; + return; +} + +#ifdef SQLITE_TEST +/* +** Open the file indicated and write a log of all unfreed memory +** allocations into that log. +*/ +SQLITE_PRIVATE void sqlite3Memsys5Dump(const char *zFilename){ + FILE *out; + int i, j, n; + int nMinLog; + + if( zFilename==0 || zFilename[0]==0 ){ + out = stdout; + }else{ + out = fopen(zFilename, "w"); + if( out==0 ){ + fprintf(stderr, "** Unable to output memory debug output log: %s **\n", + zFilename); + return; + } + } + memsys5Enter(); + nMinLog = memsys5Log(mem5.szAtom); + for(i=0; i<=LOGMAX && i+nMinLog<32; i++){ + for(n=0, j=mem5.aiFreelist[i]; j>=0; j = MEM5LINK(j)->next, n++){} + fprintf(out, "freelist items of size %d: %d\n", mem5.szAtom << i, n); + } + fprintf(out, "mem5.nAlloc = %llu\n", mem5.nAlloc); + fprintf(out, "mem5.totalAlloc = %llu\n", mem5.totalAlloc); + fprintf(out, "mem5.totalExcess = %llu\n", mem5.totalExcess); + fprintf(out, "mem5.currentOut = %u\n", mem5.currentOut); + fprintf(out, "mem5.currentCount = %u\n", mem5.currentCount); + fprintf(out, "mem5.maxOut = %u\n", mem5.maxOut); + fprintf(out, "mem5.maxCount = %u\n", mem5.maxCount); + fprintf(out, "mem5.maxRequest = %u\n", mem5.maxRequest); + memsys5Leave(); + if( out==stdout ){ + fflush(stdout); + }else{ + fclose(out); + } +} +#endif + +/* +** This routine is the only routine in this file with external +** linkage. It returns a pointer to a static sqlite3_mem_methods +** struct populated with the memsys5 methods. +*/ +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void){ + static const sqlite3_mem_methods memsys5Methods = { + memsys5Malloc, + memsys5Free, + memsys5Realloc, + memsys5Size, + memsys5Roundup, + memsys5Init, + memsys5Shutdown, + 0 + }; + return &memsys5Methods; +} + +#endif /* SQLITE_ENABLE_MEMSYS5 */ + +/************** End of mem5.c ************************************************/ +/************** Begin file mutex.c *******************************************/ +/* +** 2007 August 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes. +** +** This file contains code that is common across all mutex implementations. +*/ +/* #include "sqliteInt.h" */ + +#if defined(SQLITE_DEBUG) && !defined(SQLITE_MUTEX_OMIT) +/* +** For debugging purposes, record when the mutex subsystem is initialized +** and uninitialized so that we can assert() if there is an attempt to +** allocate a mutex while the system is uninitialized. +*/ +static SQLITE_WSD int mutexIsInit = 0; +#endif /* SQLITE_DEBUG && !defined(SQLITE_MUTEX_OMIT) */ + + +#ifndef SQLITE_MUTEX_OMIT + +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +/* +** This block (enclosed by SQLITE_ENABLE_MULTITHREADED_CHECKS) contains +** the implementation of a wrapper around the system default mutex +** implementation (sqlite3DefaultMutex()). +** +** Most calls are passed directly through to the underlying default +** mutex implementation. Except, if a mutex is configured by calling +** sqlite3MutexWarnOnContention() on it, then if contention is ever +** encountered within xMutexEnter() a warning is emitted via sqlite3_log(). +** +** This type of mutex is used as the database handle mutex when testing +** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. +*/ + +/* +** Type for all mutexes used when SQLITE_ENABLE_MULTITHREADED_CHECKS +** is defined. Variable CheckMutex.mutex is a pointer to the real mutex +** allocated by the system mutex implementation. Variable iType is usually set +** to the type of mutex requested - SQLITE_MUTEX_RECURSIVE, SQLITE_MUTEX_FAST +** or one of the static mutex identifiers. Or, if this is a recursive mutex +** that has been configured using sqlite3MutexWarnOnContention(), it is +** set to SQLITE_MUTEX_WARNONCONTENTION. +*/ +typedef struct CheckMutex CheckMutex; +struct CheckMutex { + int iType; + sqlite3_mutex *mutex; +}; + +#define SQLITE_MUTEX_WARNONCONTENTION (-1) + +/* +** Pointer to real mutex methods object used by the CheckMutex +** implementation. Set by checkMutexInit(). +*/ +static SQLITE_WSD const sqlite3_mutex_methods *pGlobalMutexMethods; + +#ifdef SQLITE_DEBUG +static int checkMutexHeld(sqlite3_mutex *p){ + return pGlobalMutexMethods->xMutexHeld(((CheckMutex*)p)->mutex); +} +static int checkMutexNotheld(sqlite3_mutex *p){ + return pGlobalMutexMethods->xMutexNotheld(((CheckMutex*)p)->mutex); +} +#endif + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int checkMutexInit(void){ + pGlobalMutexMethods = sqlite3DefaultMutex(); + return SQLITE_OK; +} +static int checkMutexEnd(void){ + pGlobalMutexMethods = 0; + return SQLITE_OK; +} + +/* +** Allocate a mutex. +*/ +static sqlite3_mutex *checkMutexAlloc(int iType){ + static CheckMutex staticMutexes[] = { + {2, 0}, {3, 0}, {4, 0}, {5, 0}, + {6, 0}, {7, 0}, {8, 0}, {9, 0}, + {10, 0}, {11, 0}, {12, 0}, {13, 0} + }; + CheckMutex *p = 0; + + assert( SQLITE_MUTEX_RECURSIVE==1 && SQLITE_MUTEX_FAST==0 ); + if( iType<2 ){ + p = sqlite3MallocZero(sizeof(CheckMutex)); + if( p==0 ) return 0; + p->iType = iType; + }else{ +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + p = &staticMutexes[iType-2]; + } + + if( p->mutex==0 ){ + p->mutex = pGlobalMutexMethods->xMutexAlloc(iType); + if( p->mutex==0 ){ + if( iType<2 ){ + sqlite3_free(p); + } + p = 0; + } + } + + return (sqlite3_mutex*)p; +} + +/* +** Free a mutex. +*/ +static void checkMutexFree(sqlite3_mutex *p){ + assert( SQLITE_MUTEX_RECURSIVE<2 ); + assert( SQLITE_MUTEX_FAST<2 ); + assert( SQLITE_MUTEX_WARNONCONTENTION<2 ); + +#if SQLITE_ENABLE_API_ARMOR + if( ((CheckMutex*)p)->iType<2 ) +#endif + { + CheckMutex *pCheck = (CheckMutex*)p; + pGlobalMutexMethods->xMutexFree(pCheck->mutex); + sqlite3_free(pCheck); + } +#ifdef SQLITE_ENABLE_API_ARMOR + else{ + (void)SQLITE_MISUSE_BKPT; + } +#endif +} + +/* +** Enter the mutex. +*/ +static void checkMutexEnter(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + if( pCheck->iType==SQLITE_MUTEX_WARNONCONTENTION ){ + if( SQLITE_OK==pGlobalMutexMethods->xMutexTry(pCheck->mutex) ){ + return; + } + sqlite3_log(SQLITE_MISUSE, + "illegal multi-threaded access to database connection" + ); + } + pGlobalMutexMethods->xMutexEnter(pCheck->mutex); +} + +/* +** Enter the mutex (do not block). +*/ +static int checkMutexTry(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + return pGlobalMutexMethods->xMutexTry(pCheck->mutex); +} + +/* +** Leave the mutex. +*/ +static void checkMutexLeave(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + pGlobalMutexMethods->xMutexLeave(pCheck->mutex); +} + +sqlite3_mutex_methods const *multiThreadedCheckMutex(void){ + static const sqlite3_mutex_methods sMutex = { + checkMutexInit, + checkMutexEnd, + checkMutexAlloc, + checkMutexFree, + checkMutexEnter, + checkMutexTry, + checkMutexLeave, +#ifdef SQLITE_DEBUG + checkMutexHeld, + checkMutexNotheld +#else + 0, + 0 +#endif + }; + return &sMutex; +} + +/* +** Mark the SQLITE_MUTEX_RECURSIVE mutex passed as the only argument as +** one on which there should be no contention. +*/ +SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex *p){ + if( sqlite3GlobalConfig.mutex.xMutexAlloc==checkMutexAlloc ){ + CheckMutex *pCheck = (CheckMutex*)p; + assert( pCheck->iType==SQLITE_MUTEX_RECURSIVE ); + pCheck->iType = SQLITE_MUTEX_WARNONCONTENTION; + } +} +#endif /* ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS */ + +/* +** Initialize the mutex system. +*/ +SQLITE_PRIVATE int sqlite3MutexInit(void){ + int rc = SQLITE_OK; + if( !sqlite3GlobalConfig.mutex.xMutexAlloc ){ + /* If the xMutexAlloc method has not been set, then the user did not + ** install a mutex implementation via sqlite3_config() prior to + ** sqlite3_initialize() being called. This block copies pointers to + ** the default implementation into the sqlite3GlobalConfig structure. + */ + sqlite3_mutex_methods const *pFrom; + sqlite3_mutex_methods *pTo = &sqlite3GlobalConfig.mutex; + + if( sqlite3GlobalConfig.bCoreMutex ){ +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS + pFrom = multiThreadedCheckMutex(); +#else + pFrom = sqlite3DefaultMutex(); +#endif + }else{ + pFrom = sqlite3NoopMutex(); + } + pTo->xMutexInit = pFrom->xMutexInit; + pTo->xMutexEnd = pFrom->xMutexEnd; + pTo->xMutexFree = pFrom->xMutexFree; + pTo->xMutexEnter = pFrom->xMutexEnter; + pTo->xMutexTry = pFrom->xMutexTry; + pTo->xMutexLeave = pFrom->xMutexLeave; + pTo->xMutexHeld = pFrom->xMutexHeld; + pTo->xMutexNotheld = pFrom->xMutexNotheld; + sqlite3MemoryBarrier(); + pTo->xMutexAlloc = pFrom->xMutexAlloc; + } + assert( sqlite3GlobalConfig.mutex.xMutexInit ); + rc = sqlite3GlobalConfig.mutex.xMutexInit(); + +#ifdef SQLITE_DEBUG + GLOBAL(int, mutexIsInit) = 1; +#endif + + sqlite3MemoryBarrier(); + return rc; +} + +/* +** Shutdown the mutex system. This call frees resources allocated by +** sqlite3MutexInit(). +*/ +SQLITE_PRIVATE int sqlite3MutexEnd(void){ + int rc = SQLITE_OK; + if( sqlite3GlobalConfig.mutex.xMutexEnd ){ + rc = sqlite3GlobalConfig.mutex.xMutexEnd(); + } + +#ifdef SQLITE_DEBUG + GLOBAL(int, mutexIsInit) = 0; +#endif + + return rc; +} + +/* +** Retrieve a pointer to a static mutex or allocate a new dynamic one. +*/ +SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int id){ +#ifndef SQLITE_OMIT_AUTOINIT + if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; + if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; +#endif + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); + return sqlite3GlobalConfig.mutex.xMutexAlloc(id); +} + +SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int id){ + if( !sqlite3GlobalConfig.bCoreMutex ){ + return 0; + } + assert( GLOBAL(int, mutexIsInit) ); + assert( sqlite3GlobalConfig.mutex.xMutexAlloc ); + return sqlite3GlobalConfig.mutex.xMutexAlloc(id); +} + +/* +** Free a dynamic mutex. +*/ +SQLITE_API void sqlite3_mutex_free(sqlite3_mutex *p){ + if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexFree ); + sqlite3GlobalConfig.mutex.xMutexFree(p); + } +} + +/* +** Obtain the mutex p. If some other thread already has the mutex, block +** until it can be obtained. +*/ +SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex *p){ + if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexEnter ); + sqlite3GlobalConfig.mutex.xMutexEnter(p); + } +} + +/* +** Obtain the mutex p. If successful, return SQLITE_OK. Otherwise, if another +** thread holds the mutex and it cannot be obtained, return SQLITE_BUSY. +*/ +SQLITE_API int sqlite3_mutex_try(sqlite3_mutex *p){ + int rc = SQLITE_OK; + if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexTry ); + return sqlite3GlobalConfig.mutex.xMutexTry(p); + } + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was previously +** entered by the same thread. The behavior is undefined if the mutex +** is not currently entered. If a NULL pointer is passed as an argument +** this function is a no-op. +*/ +SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex *p){ + if( p ){ + assert( sqlite3GlobalConfig.mutex.xMutexLeave ); + sqlite3GlobalConfig.mutex.xMutexLeave(p); + } +} + +#ifndef NDEBUG +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use inside assert() statements. +*/ +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); + return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); +} +SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex *p){ + assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); + return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); +} +#endif + +#endif /* !defined(SQLITE_MUTEX_OMIT) */ + +/************** End of mutex.c ***********************************************/ +/************** Begin file mutex_noop.c **************************************/ +/* +** 2008 October 07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes. +** +** This implementation in this file does not provide any mutual +** exclusion and is thus suitable for use only in applications +** that use SQLite in a single thread. The routines defined +** here are place-holders. Applications can substitute working +** mutex routines at start-time using the +** +** sqlite3_config(SQLITE_CONFIG_MUTEX,...) +** +** interface. +** +** If compiled with SQLITE_DEBUG, then additional logic is inserted +** that does error checking on mutexes to make sure they are being +** called correctly. +*/ +/* #include "sqliteInt.h" */ + +#ifndef SQLITE_MUTEX_OMIT + +#ifndef SQLITE_DEBUG +/* +** Stub routines for all mutex methods. +** +** This routines provide no mutual exclusion or error checking. +*/ +static int noopMutexInit(void){ return SQLITE_OK; } +static int noopMutexEnd(void){ return SQLITE_OK; } +static sqlite3_mutex *noopMutexAlloc(int id){ + UNUSED_PARAMETER(id); + return (sqlite3_mutex*)8; +} +static void noopMutexFree(sqlite3_mutex *p){ UNUSED_PARAMETER(p); return; } +static void noopMutexEnter(sqlite3_mutex *p){ UNUSED_PARAMETER(p); return; } +static int noopMutexTry(sqlite3_mutex *p){ + UNUSED_PARAMETER(p); + return SQLITE_OK; +} +static void noopMutexLeave(sqlite3_mutex *p){ UNUSED_PARAMETER(p); return; } + +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3NoopMutex(void){ + static const sqlite3_mutex_methods sMutex = { + noopMutexInit, + noopMutexEnd, + noopMutexAlloc, + noopMutexFree, + noopMutexEnter, + noopMutexTry, + noopMutexLeave, + + 0, + 0, + }; + + return &sMutex; +} +#endif /* !SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* +** In this implementation, error checking is provided for testing +** and debugging purposes. The mutexes still do not provide any +** mutual exclusion. +*/ + +/* +** The mutex object +*/ +typedef struct sqlite3_debug_mutex { + int id; /* The mutex type */ + int cnt; /* Number of entries without a matching leave */ +} sqlite3_debug_mutex; + +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use inside assert() statements. +*/ +static int debugMutexHeld(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + return p==0 || p->cnt>0; +} +static int debugMutexNotheld(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + return p==0 || p->cnt==0; +} + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int debugMutexInit(void){ return SQLITE_OK; } +static int debugMutexEnd(void){ return SQLITE_OK; } + +/* +** The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. If it returns NULL +** that means that a mutex could not be allocated. +*/ +static sqlite3_mutex *debugMutexAlloc(int id){ + static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_VFS3 - 1]; + sqlite3_debug_mutex *pNew = 0; + switch( id ){ + case SQLITE_MUTEX_FAST: + case SQLITE_MUTEX_RECURSIVE: { + pNew = sqlite3Malloc(sizeof(*pNew)); + if( pNew ){ + pNew->id = id; + pNew->cnt = 0; + } + break; + } + default: { +#ifdef SQLITE_ENABLE_API_ARMOR + if( id-2<0 || id-2>=ArraySize(aStatic) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + pNew = &aStatic[id-2]; + pNew->id = id; + break; + } + } + return (sqlite3_mutex*)pNew; +} + +/* +** This routine deallocates a previously allocated mutex. +*/ +static void debugMutexFree(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( p->cnt==0 ); + if( p->id==SQLITE_MUTEX_RECURSIVE || p->id==SQLITE_MUTEX_FAST ){ + sqlite3_free(p); + }else{ +#ifdef SQLITE_ENABLE_API_ARMOR + (void)SQLITE_MISUSE_BKPT; +#endif + } +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void debugMutexEnter(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(pX) ); + p->cnt++; +} +static int debugMutexTry(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(pX) ); + p->cnt++; + return SQLITE_OK; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void debugMutexLeave(sqlite3_mutex *pX){ + sqlite3_debug_mutex *p = (sqlite3_debug_mutex*)pX; + assert( debugMutexHeld(pX) ); + p->cnt--; + assert( p->id==SQLITE_MUTEX_RECURSIVE || debugMutexNotheld(pX) ); +} + +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3NoopMutex(void){ + static const sqlite3_mutex_methods sMutex = { + debugMutexInit, + debugMutexEnd, + debugMutexAlloc, + debugMutexFree, + debugMutexEnter, + debugMutexTry, + debugMutexLeave, + + debugMutexHeld, + debugMutexNotheld + }; + + return &sMutex; +} +#endif /* SQLITE_DEBUG */ + +/* +** If compiled with SQLITE_MUTEX_NOOP, then the no-op mutex implementation +** is used regardless of the run-time threadsafety setting. +*/ +#ifdef SQLITE_MUTEX_NOOP +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ + return sqlite3NoopMutex(); +} +#endif /* defined(SQLITE_MUTEX_NOOP) */ +#endif /* !defined(SQLITE_MUTEX_OMIT) */ + +/************** End of mutex_noop.c ******************************************/ +/************** Begin file mutex_unix.c **************************************/ +/* +** 2007 August 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes for pthreads +*/ +/* #include "sqliteInt.h" */ + +/* +** The code in this file is only used if we are compiling threadsafe +** under unix with pthreads. +** +** Note that this implementation requires a version of pthreads that +** supports recursive mutexes. +*/ +#ifdef SQLITE_MUTEX_PTHREADS + +#include + +/* +** The sqlite3_mutex.id, sqlite3_mutex.nRef, and sqlite3_mutex.owner fields +** are necessary under two condidtions: (1) Debug builds and (2) using +** home-grown mutexes. Encapsulate these conditions into a single #define. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_HOMEGROWN_RECURSIVE_MUTEX) +# define SQLITE_MUTEX_NREF 1 +#else +# define SQLITE_MUTEX_NREF 0 +#endif + +/* +** Each recursive mutex is an instance of the following structure. +*/ +struct sqlite3_mutex { + pthread_mutex_t mutex; /* Mutex controlling the lock */ +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + int id; /* Mutex type */ +#endif +#if SQLITE_MUTEX_NREF + volatile int nRef; /* Number of entrances */ + volatile pthread_t owner; /* Thread that is within this mutex */ + int trace; /* True to trace changes */ +#endif +}; +#if SQLITE_MUTEX_NREF +# define SQLITE3_MUTEX_INITIALIZER(id) \ + {PTHREAD_MUTEX_INITIALIZER,id,0,(pthread_t)0,0} +#elif defined(SQLITE_ENABLE_API_ARMOR) +# define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER, id } +#else +#define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER } +#endif + +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use only inside assert() statements. On some platforms, +** there might be race conditions that can cause these routines to +** deliver incorrect results. In particular, if pthread_equal() is +** not an atomic operation, then these routines might delivery +** incorrect results. On most platforms, pthread_equal() is a +** comparison of two integers and is therefore atomic. But we are +** told that HPUX is not such a platform. If so, then these routines +** will not always work correctly on HPUX. +** +** On those platforms where pthread_equal() is not atomic, SQLite +** should be compiled without -DSQLITE_DEBUG and with -DNDEBUG to +** make sure no assert() statements are evaluated and hence these +** routines are never called. +*/ +#if !defined(NDEBUG) || defined(SQLITE_DEBUG) +static int pthreadMutexHeld(sqlite3_mutex *p){ + return (p->nRef!=0 && pthread_equal(p->owner, pthread_self())); +} +static int pthreadMutexNotheld(sqlite3_mutex *p){ + return p->nRef==0 || pthread_equal(p->owner, pthread_self())==0; +} +#endif + +/* +** Try to provide a memory barrier operation, needed for initialization +** and also for the implementation of xShmBarrier in the VFS in cases +** where SQLite is compiled without mutexes. +*/ +SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) && GCC_VERSION>=4001000 + __sync_synchronize(); +#endif +} + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int pthreadMutexInit(void){ return SQLITE_OK; } +static int pthreadMutexEnd(void){ return SQLITE_OK; } + +/* +** The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. If it returns NULL +** that means that a mutex could not be allocated. SQLite +** will unwind its stack and return an error. The argument +** to sqlite3_mutex_alloc() is one of these integer constants: +** +**
      +**
    • SQLITE_MUTEX_FAST +**
    • SQLITE_MUTEX_RECURSIVE +**
    • SQLITE_MUTEX_STATIC_MAIN +**
    • SQLITE_MUTEX_STATIC_MEM +**
    • SQLITE_MUTEX_STATIC_OPEN +**
    • SQLITE_MUTEX_STATIC_PRNG +**
    • SQLITE_MUTEX_STATIC_LRU +**
    • SQLITE_MUTEX_STATIC_PMEM +**
    • SQLITE_MUTEX_STATIC_APP1 +**
    • SQLITE_MUTEX_STATIC_APP2 +**
    • SQLITE_MUTEX_STATIC_APP3 +**
    • SQLITE_MUTEX_STATIC_VFS1 +**
    • SQLITE_MUTEX_STATIC_VFS2 +**
    • SQLITE_MUTEX_STATIC_VFS3 +**
    +** +** The first two constants cause sqlite3_mutex_alloc() to create +** a new mutex. The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. But SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** The other allowed parameters to sqlite3_mutex_alloc() each return +** a pointer to a static preexisting mutex. Six static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. But for the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +*/ +static sqlite3_mutex *pthreadMutexAlloc(int iType){ + static sqlite3_mutex staticMutexes[] = { + SQLITE3_MUTEX_INITIALIZER(2), + SQLITE3_MUTEX_INITIALIZER(3), + SQLITE3_MUTEX_INITIALIZER(4), + SQLITE3_MUTEX_INITIALIZER(5), + SQLITE3_MUTEX_INITIALIZER(6), + SQLITE3_MUTEX_INITIALIZER(7), + SQLITE3_MUTEX_INITIALIZER(8), + SQLITE3_MUTEX_INITIALIZER(9), + SQLITE3_MUTEX_INITIALIZER(10), + SQLITE3_MUTEX_INITIALIZER(11), + SQLITE3_MUTEX_INITIALIZER(12), + SQLITE3_MUTEX_INITIALIZER(13) + }; + sqlite3_mutex *p; + switch( iType ){ + case SQLITE_MUTEX_RECURSIVE: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, we will have to + ** build our own. See below. */ + pthread_mutex_init(&p->mutex, 0); +#else + /* Use a recursive mutex if it is available */ + pthread_mutexattr_t recursiveAttr; + pthread_mutexattr_init(&recursiveAttr); + pthread_mutexattr_settype(&recursiveAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&p->mutex, &recursiveAttr); + pthread_mutexattr_destroy(&recursiveAttr); +#endif +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + p->id = SQLITE_MUTEX_RECURSIVE; +#endif + } + break; + } + case SQLITE_MUTEX_FAST: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ + pthread_mutex_init(&p->mutex, 0); +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + p->id = SQLITE_MUTEX_FAST; +#endif + } + break; + } + default: { +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + p = &staticMutexes[iType-2]; + break; + } + } +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + assert( p==0 || p->id==iType ); +#endif + return p; +} + + +/* +** This routine deallocates a previously +** allocated mutex. SQLite is careful to deallocate every +** mutex that it allocates. +*/ +static void pthreadMutexFree(sqlite3_mutex *p){ + assert( p->nRef==0 ); +#if SQLITE_ENABLE_API_ARMOR + if( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ) +#endif + { + pthread_mutex_destroy(&p->mutex); + sqlite3_free(p); + } +#ifdef SQLITE_ENABLE_API_ARMOR + else{ + (void)SQLITE_MISUSE_BKPT; + } +#endif +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void pthreadMutexEnter(sqlite3_mutex *p){ + assert( p->id==SQLITE_MUTEX_RECURSIVE || pthreadMutexNotheld(p) ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, then we have to grow + ** our own. This implementation assumes that pthread_equal() + ** is atomic - that it cannot be deceived into thinking self + ** and p->owner are equal if p->owner changes between two values + ** that are not equal to self while the comparison is taking place. + ** This implementation also assumes a coherent cache - that + ** separate processes cannot read different values from the same + ** address at the same time. If either of these two conditions + ** are not met, then the mutexes will fail and problems will result. + */ + { + pthread_t self = pthread_self(); + if( p->nRef>0 && pthread_equal(p->owner, self) ){ + p->nRef++; + }else{ + pthread_mutex_lock(&p->mutex); + assert( p->nRef==0 ); + p->owner = self; + p->nRef = 1; + } + } +#else + /* Use the built-in recursive mutexes if they are available. + */ + pthread_mutex_lock(&p->mutex); +#if SQLITE_MUTEX_NREF + assert( p->nRef>0 || p->owner==0 ); + p->owner = pthread_self(); + p->nRef++; +#endif +#endif + +#ifdef SQLITE_DEBUG + if( p->trace ){ + printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif +} +static int pthreadMutexTry(sqlite3_mutex *p){ + int rc; + assert( p->id==SQLITE_MUTEX_RECURSIVE || pthreadMutexNotheld(p) ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + /* If recursive mutexes are not available, then we have to grow + ** our own. This implementation assumes that pthread_equal() + ** is atomic - that it cannot be deceived into thinking self + ** and p->owner are equal if p->owner changes between two values + ** that are not equal to self while the comparison is taking place. + ** This implementation also assumes a coherent cache - that + ** separate processes cannot read different values from the same + ** address at the same time. If either of these two conditions + ** are not met, then the mutexes will fail and problems will result. + */ + { + pthread_t self = pthread_self(); + if( p->nRef>0 && pthread_equal(p->owner, self) ){ + p->nRef++; + rc = SQLITE_OK; + }else if( pthread_mutex_trylock(&p->mutex)==0 ){ + assert( p->nRef==0 ); + p->owner = self; + p->nRef = 1; + rc = SQLITE_OK; + }else{ + rc = SQLITE_BUSY; + } + } +#else + /* Use the built-in recursive mutexes if they are available. + */ + if( pthread_mutex_trylock(&p->mutex)==0 ){ +#if SQLITE_MUTEX_NREF + p->owner = pthread_self(); + p->nRef++; +#endif + rc = SQLITE_OK; + }else{ + rc = SQLITE_BUSY; + } +#endif + +#ifdef SQLITE_DEBUG + if( rc==SQLITE_OK && p->trace ){ + printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void pthreadMutexLeave(sqlite3_mutex *p){ + assert( pthreadMutexHeld(p) ); +#if SQLITE_MUTEX_NREF + p->nRef--; + if( p->nRef==0 ) p->owner = 0; +#endif + assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); + +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX + if( p->nRef==0 ){ + pthread_mutex_unlock(&p->mutex); + } +#else + pthread_mutex_unlock(&p->mutex); +#endif + +#ifdef SQLITE_DEBUG + if( p->trace ){ + printf("leave mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + } +#endif +} + +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ + static const sqlite3_mutex_methods sMutex = { + pthreadMutexInit, + pthreadMutexEnd, + pthreadMutexAlloc, + pthreadMutexFree, + pthreadMutexEnter, + pthreadMutexTry, + pthreadMutexLeave, +#ifdef SQLITE_DEBUG + pthreadMutexHeld, + pthreadMutexNotheld +#else + 0, + 0 +#endif + }; + + return &sMutex; +} + +#endif /* SQLITE_MUTEX_PTHREADS */ + +/************** End of mutex_unix.c ******************************************/ +/************** Begin file mutex_w32.c ***************************************/ +/* +** 2007 August 14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C functions that implement mutexes for Win32. +*/ +/* #include "sqliteInt.h" */ + +#if SQLITE_OS_WIN +/* +** Include code that is common to all os_*.c files +*/ +/* #include "os_common.h" */ + +/* +** Include the header file for the Windows VFS. +*/ +/************** Include os_win.h in the middle of mutex_w32.c ****************/ +/************** Begin file os_win.h ******************************************/ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Windows. +*/ +#ifndef SQLITE_OS_WIN_H +#define SQLITE_OS_WIN_H + +/* +** Include the primary Windows SDK header file. +*/ +#include "windows.h" + +#ifdef __CYGWIN__ +# include +# include /* amalgamator: dontcache */ +#endif + +/* +** Determine if we are dealing with Windows NT. +** +** We ought to be able to determine if we are compiling for Windows 9x or +** Windows NT using the _WIN32_WINNT macro as follows: +** +** #if defined(_WIN32_WINNT) +** # define SQLITE_OS_WINNT 1 +** #else +** # define SQLITE_OS_WINNT 0 +** #endif +** +** However, Visual Studio 2005 does not set _WIN32_WINNT by default, as +** it ought to, so the above test does not work. We'll just assume that +** everything is Windows NT unless the programmer explicitly says otherwise +** by setting SQLITE_OS_WINNT to 0. +*/ +#if SQLITE_OS_WIN && !defined(SQLITE_OS_WINNT) +# define SQLITE_OS_WINNT 1 +#endif + +/* +** Determine if we are dealing with Windows CE - which has a much reduced +** API. +*/ +#if defined(_WIN32_WCE) +# define SQLITE_OS_WINCE 1 +#else +# define SQLITE_OS_WINCE 0 +#endif + +/* +** Determine if we are dealing with WinRT, which provides only a subset of +** the full Win32 API. +*/ +#if !defined(SQLITE_OS_WINRT) +# define SQLITE_OS_WINRT 0 +#endif + +/* +** For WinCE, some API function parameters do not appear to be declared as +** volatile. +*/ +#if SQLITE_OS_WINCE +# define SQLITE_WIN32_VOLATILE +#else +# define SQLITE_WIN32_VOLATILE volatile +#endif + +/* +** For some Windows sub-platforms, the _beginthreadex() / _endthreadex() +** functions are not available (e.g. those not using MSVC, Cygwin, etc). +*/ +#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ + SQLITE_THREADSAFE>0 && !defined(__CYGWIN__) +# define SQLITE_OS_WIN_THREADS 1 +#else +# define SQLITE_OS_WIN_THREADS 0 +#endif + +#endif /* SQLITE_OS_WIN_H */ + +/************** End of os_win.h **********************************************/ +/************** Continuing where we left off in mutex_w32.c ******************/ +#endif + +/* +** The code in this file is only used if we are compiling multithreaded +** on a Win32 system. +*/ +#ifdef SQLITE_MUTEX_W32 + +/* +** Each recursive mutex is an instance of the following structure. +*/ +struct sqlite3_mutex { + CRITICAL_SECTION mutex; /* Mutex controlling the lock */ + int id; /* Mutex type */ +#ifdef SQLITE_DEBUG + volatile int nRef; /* Number of enterances */ + volatile DWORD owner; /* Thread holding this mutex */ + volatile LONG trace; /* True to trace changes */ +#endif +}; + +/* +** These are the initializer values used when declaring a "static" mutex +** on Win32. It should be noted that all mutexes require initialization +** on the Win32 platform. +*/ +#define SQLITE_W32_MUTEX_INITIALIZER { 0 } + +#ifdef SQLITE_DEBUG +#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id, \ + 0L, (DWORD)0, 0 } +#else +#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id } +#endif + +#ifdef SQLITE_DEBUG +/* +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are +** intended for use only inside assert() statements. +*/ +static int winMutexHeld(sqlite3_mutex *p){ + return p->nRef!=0 && p->owner==GetCurrentThreadId(); +} + +static int winMutexNotheld2(sqlite3_mutex *p, DWORD tid){ + return p->nRef==0 || p->owner!=tid; +} + +static int winMutexNotheld(sqlite3_mutex *p){ + DWORD tid = GetCurrentThreadId(); + return winMutexNotheld2(p, tid); +} +#endif + +/* +** Try to provide a memory barrier operation, needed for initialization +** and also for the xShmBarrier method of the VFS in cases when SQLite is +** compiled without mutexes (SQLITE_THREADSAFE=0). +*/ +SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ +#if defined(SQLITE_MEMORY_BARRIER) + SQLITE_MEMORY_BARRIER; +#elif defined(__GNUC__) + __sync_synchronize(); +#elif MSVC_VERSION>=1300 + _ReadWriteBarrier(); +#elif defined(MemoryBarrier) + MemoryBarrier(); +#endif +} + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static sqlite3_mutex winMutex_staticMutexes[] = { + SQLITE3_MUTEX_INITIALIZER(2), + SQLITE3_MUTEX_INITIALIZER(3), + SQLITE3_MUTEX_INITIALIZER(4), + SQLITE3_MUTEX_INITIALIZER(5), + SQLITE3_MUTEX_INITIALIZER(6), + SQLITE3_MUTEX_INITIALIZER(7), + SQLITE3_MUTEX_INITIALIZER(8), + SQLITE3_MUTEX_INITIALIZER(9), + SQLITE3_MUTEX_INITIALIZER(10), + SQLITE3_MUTEX_INITIALIZER(11), + SQLITE3_MUTEX_INITIALIZER(12), + SQLITE3_MUTEX_INITIALIZER(13) +}; + +static int winMutex_isInit = 0; +static int winMutex_isNt = -1; /* <0 means "need to query" */ + +/* As the winMutexInit() and winMutexEnd() functions are called as part +** of the sqlite3_initialize() and sqlite3_shutdown() processing, the +** "interlocked" magic used here is probably not strictly necessary. +*/ +static LONG SQLITE_WIN32_VOLATILE winMutex_lock = 0; + +SQLITE_API int sqlite3_win32_is_nt(void); /* os_win.c */ +SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ + +static int winMutexInit(void){ + /* The first to increment to 1 does actual initialization */ + if( InterlockedCompareExchange(&winMutex_lock, 1, 0)==0 ){ + int i; + for(i=0; i +**
  • SQLITE_MUTEX_FAST +**
  • SQLITE_MUTEX_RECURSIVE +**
  • SQLITE_MUTEX_STATIC_MAIN +**
  • SQLITE_MUTEX_STATIC_MEM +**
  • SQLITE_MUTEX_STATIC_OPEN +**
  • SQLITE_MUTEX_STATIC_PRNG +**
  • SQLITE_MUTEX_STATIC_LRU +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 +**
  • SQLITE_MUTEX_STATIC_VFS1 +**
  • SQLITE_MUTEX_STATIC_VFS2 +**
  • SQLITE_MUTEX_STATIC_VFS3 +** +** +** The first two constants cause sqlite3_mutex_alloc() to create +** a new mutex. The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. But SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** The other allowed parameters to sqlite3_mutex_alloc() each return +** a pointer to a static preexisting mutex. Six static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. But for the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +*/ +static sqlite3_mutex *winMutexAlloc(int iType){ + sqlite3_mutex *p; + + switch( iType ){ + case SQLITE_MUTEX_FAST: + case SQLITE_MUTEX_RECURSIVE: { + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ + p->id = iType; +#ifdef SQLITE_DEBUG +#ifdef SQLITE_WIN32_MUTEX_TRACE_DYNAMIC + p->trace = 1; +#endif +#endif +#if SQLITE_OS_WINRT + InitializeCriticalSectionEx(&p->mutex, 0, 0); +#else + InitializeCriticalSection(&p->mutex); +#endif + } + break; + } + default: { +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(winMutex_staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + p = &winMutex_staticMutexes[iType-2]; +#ifdef SQLITE_DEBUG +#ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC + InterlockedCompareExchange(&p->trace, 1, 0); +#endif +#endif + break; + } + } + assert( p==0 || p->id==iType ); + return p; +} + + +/* +** This routine deallocates a previously +** allocated mutex. SQLite is careful to deallocate every +** mutex that it allocates. +*/ +static void winMutexFree(sqlite3_mutex *p){ + assert( p ); + assert( p->nRef==0 && p->owner==0 ); + if( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ){ + DeleteCriticalSection(&p->mutex); + sqlite3_free(p); + }else{ +#ifdef SQLITE_ENABLE_API_ARMOR + (void)SQLITE_MISUSE_BKPT; +#endif + } +} + +/* +** The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK +** upon successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can +** be entered multiple times by the same thread. In such cases the, +** mutex must be exited an equal number of times before another thread +** can enter. If the same thread tries to enter any other kind of mutex +** more than once, the behavior is undefined. +*/ +static void winMutexEnter(sqlite3_mutex *p){ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); +#endif +#ifdef SQLITE_DEBUG + assert( p ); + assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); +#else + assert( p ); +#endif + assert( winMutex_isInit==1 ); + EnterCriticalSection(&p->mutex); +#ifdef SQLITE_DEBUG + assert( p->nRef>0 || p->owner==0 ); + p->owner = tid; + p->nRef++; + if( p->trace ){ + OSTRACE(("ENTER-MUTEX tid=%lu, mutex(%d)=%p (%d), nRef=%d\n", + tid, p->id, p, p->trace, p->nRef)); + } +#endif +} + +static int winMutexTry(sqlite3_mutex *p){ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); +#endif + int rc = SQLITE_BUSY; + assert( p ); + assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); + /* + ** The sqlite3_mutex_try() routine is very rarely used, and when it + ** is used it is merely an optimization. So it is OK for it to always + ** fail. + ** + ** The TryEnterCriticalSection() interface is only available on WinNT. + ** And some windows compilers complain if you try to use it without + ** first doing some #defines that prevent SQLite from building on Win98. + ** For that reason, we will omit this optimization for now. See + ** ticket #2685. + */ +#if defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x0400 + assert( winMutex_isInit==1 ); + assert( winMutex_isNt>=-1 && winMutex_isNt<=1 ); + if( winMutex_isNt<0 ){ + winMutex_isNt = sqlite3_win32_is_nt(); + } + assert( winMutex_isNt==0 || winMutex_isNt==1 ); + if( winMutex_isNt && TryEnterCriticalSection(&p->mutex) ){ +#ifdef SQLITE_DEBUG + p->owner = tid; + p->nRef++; +#endif + rc = SQLITE_OK; + } +#else + UNUSED_PARAMETER(p); +#endif +#ifdef SQLITE_DEBUG + if( p->trace ){ + OSTRACE(("TRY-MUTEX tid=%lu, mutex(%d)=%p (%d), owner=%lu, nRef=%d, rc=%s\n", + tid, p->id, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); + } +#endif + return rc; +} + +/* +** The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered or +** is not currently allocated. SQLite will never do either. +*/ +static void winMutexLeave(sqlite3_mutex *p){ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); +#endif + assert( p ); +#ifdef SQLITE_DEBUG + assert( p->nRef>0 ); + assert( p->owner==tid ); + p->nRef--; + if( p->nRef==0 ) p->owner = 0; + assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); +#endif + assert( winMutex_isInit==1 ); + LeaveCriticalSection(&p->mutex); +#ifdef SQLITE_DEBUG + if( p->trace ){ + OSTRACE(("LEAVE-MUTEX tid=%lu, mutex(%d)=%p (%d), nRef=%d\n", + tid, p->id, p, p->trace, p->nRef)); + } +#endif +} + +SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ + static const sqlite3_mutex_methods sMutex = { + winMutexInit, + winMutexEnd, + winMutexAlloc, + winMutexFree, + winMutexEnter, + winMutexTry, + winMutexLeave, +#ifdef SQLITE_DEBUG + winMutexHeld, + winMutexNotheld +#else + 0, + 0 +#endif + }; + return &sMutex; +} + +#endif /* SQLITE_MUTEX_W32 */ + +/************** End of mutex_w32.c *******************************************/ +/************** Begin file malloc.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Memory allocation functions used throughout sqlite. +*/ +/* #include "sqliteInt.h" */ +/* #include */ + +/* +** Attempt to release up to n bytes of non-essential memory currently +** held by SQLite. An example of non-essential memory is memory used to +** cache database pages that are not currently in use. +*/ +SQLITE_API int sqlite3_release_memory(int n){ +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + return sqlite3PcacheReleaseMemory(n); +#else + /* IMPLEMENTATION-OF: R-34391-24921 The sqlite3_release_memory() routine + ** is a no-op returning zero if SQLite is not compiled with + ** SQLITE_ENABLE_MEMORY_MANAGEMENT. */ + UNUSED_PARAMETER(n); + return 0; +#endif +} + +/* +** Default value of the hard heap limit. 0 means "no limit". +*/ +#ifndef SQLITE_MAX_MEMORY +# define SQLITE_MAX_MEMORY 0 +#endif + +/* +** State information local to the memory allocation subsystem. +*/ +static SQLITE_WSD struct Mem0Global { + sqlite3_mutex *mutex; /* Mutex to serialize access */ + sqlite3_int64 alarmThreshold; /* The soft heap limit */ + sqlite3_int64 hardLimit; /* The hard upper bound on memory */ + + /* + ** True if heap is nearly "full" where "full" is defined by the + ** sqlite3_soft_heap_limit() setting. + */ + int nearlyFull; +} mem0 = { 0, SQLITE_MAX_MEMORY, SQLITE_MAX_MEMORY, 0 }; + +#define mem0 GLOBAL(struct Mem0Global, mem0) + +/* +** Return the memory allocator mutex. sqlite3_status() needs it. +*/ +SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void){ + return mem0.mutex; +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Deprecated external interface. It used to set an alarm callback +** that was invoked when memory usage grew too large. Now it is a +** no-op. +*/ +SQLITE_API int sqlite3_memory_alarm( + void(*xCallback)(void *pArg, sqlite3_int64 used,int N), + void *pArg, + sqlite3_int64 iThreshold +){ + (void)xCallback; + (void)pArg; + (void)iThreshold; + return SQLITE_OK; +} +#endif + +/* +** Set the soft heap-size limit for the library. An argument of +** zero disables the limit. A negative argument is a no-op used to +** obtain the return value. +** +** The return value is the value of the heap limit just before this +** interface was called. +** +** If the hard heap limit is enabled, then the soft heap limit cannot +** be disabled nor raised above the hard heap limit. +*/ +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ + sqlite3_int64 priorLimit; + sqlite3_int64 excess; + sqlite3_int64 nUsed; +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return -1; +#endif + sqlite3_mutex_enter(mem0.mutex); + priorLimit = mem0.alarmThreshold; + if( n<0 ){ + sqlite3_mutex_leave(mem0.mutex); + return priorLimit; + } + if( mem0.hardLimit>0 && (n>mem0.hardLimit || n==0) ){ + n = mem0.hardLimit; + } + mem0.alarmThreshold = n; + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + AtomicStore(&mem0.nearlyFull, n>0 && n<=nUsed); + sqlite3_mutex_leave(mem0.mutex); + excess = sqlite3_memory_used() - n; + if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); + return priorLimit; +} +SQLITE_API void sqlite3_soft_heap_limit(int n){ + if( n<0 ) n = 0; + sqlite3_soft_heap_limit64(n); +} + +/* +** Set the hard heap-size limit for the library. An argument of zero +** disables the hard heap limit. A negative argument is a no-op used +** to obtain the return value without affecting the hard heap limit. +** +** The return value is the value of the hard heap limit just prior to +** calling this interface. +** +** Setting the hard heap limit will also activate the soft heap limit +** and constrain the soft heap limit to be no more than the hard heap +** limit. +*/ +SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 n){ + sqlite3_int64 priorLimit; +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return -1; +#endif + sqlite3_mutex_enter(mem0.mutex); + priorLimit = mem0.hardLimit; + if( n>=0 ){ + mem0.hardLimit = n; + if( n0 ); + + /* In Firefox (circa 2017-02-08), xRoundup() is remapped to an internal + ** implementation of malloc_good_size(), which must be called in debug + ** mode and specifically when the DMD "Dark Matter Detector" is enabled + ** or else a crash results. Hence, do not attempt to optimize out the + ** following xRoundup() call. */ + nFull = sqlite3GlobalConfig.m.xRoundup(n); + + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); + if( mem0.alarmThreshold>0 ){ + sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed >= mem0.alarmThreshold - nFull ){ + AtomicStore(&mem0.nearlyFull, 1); + sqlite3MallocAlarm(nFull); + if( mem0.hardLimit ){ + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed >= mem0.hardLimit - nFull ){ + *pp = 0; + return; + } + } + }else{ + AtomicStore(&mem0.nearlyFull, 0); + } + } + p = sqlite3GlobalConfig.m.xMalloc(nFull); +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + if( p==0 && mem0.alarmThreshold>0 ){ + sqlite3MallocAlarm(nFull); + p = sqlite3GlobalConfig.m.xMalloc(nFull); + } +#endif + if( p ){ + nFull = sqlite3MallocSize(p); + sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nFull); + sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1); + } + *pp = p; +} + +/* +** Allocate memory. This routine is like sqlite3_malloc() except that it +** assumes the memory subsystem has already been initialized. +*/ +SQLITE_PRIVATE void *sqlite3Malloc(u64 n){ + void *p; + if( n==0 || n>=0x7fffff00 ){ + /* A memory allocation of a number of bytes which is near the maximum + ** signed integer value might cause an integer overflow inside of the + ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving + ** 255 bytes of overhead. SQLite itself will never use anything near + ** this amount. The only way to reach the limit is with sqlite3_malloc() */ + p = 0; + }else if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_mutex_enter(mem0.mutex); + mallocWithAlarm((int)n, &p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + p = sqlite3GlobalConfig.m.xMalloc((int)n); + } + assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-11148-40995 */ + return p; +} + +/* +** This version of the memory allocation is for use by the application. +** First make sure the memory subsystem is initialized, then do the +** allocation. +*/ +SQLITE_API void *sqlite3_malloc(int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return n<=0 ? 0 : sqlite3Malloc(n); +} +SQLITE_API void *sqlite3_malloc64(sqlite3_uint64 n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return sqlite3Malloc(n); +} + +/* +** TRUE if p is a lookaside memory allocation from db +*/ +#ifndef SQLITE_OMIT_LOOKASIDE +static int isLookaside(sqlite3 *db, const void *p){ + return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd); +} +#else +#define isLookaside(A,B) 0 +#endif + +/* +** Return the size of a memory allocation previously obtained from +** sqlite3Malloc() or sqlite3_malloc(). +*/ +SQLITE_PRIVATE int sqlite3MallocSize(const void *p){ + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + return sqlite3GlobalConfig.m.xSize((void*)p); +} +static int lookasideMallocSize(sqlite3 *db, const void *p){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + return plookaside.pMiddle ? db->lookaside.szTrue : LOOKASIDE_SMALL; +#else + return db->lookaside.szTrue; +#endif +} +SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, const void *p){ + assert( p!=0 ); +#ifdef SQLITE_DEBUG + if( db==0 || !isLookaside(db,p) ){ + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + }else{ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + } + } +#endif + if( db ){ + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + assert( sqlite3_mutex_held(db->mutex) ); + return LOOKASIDE_SMALL; + } +#endif + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + assert( sqlite3_mutex_held(db->mutex) ); + return db->lookaside.szTrue; + } + } + } + return sqlite3GlobalConfig.m.xSize((void*)p); +} +SQLITE_API sqlite3_uint64 sqlite3_msize(void *p){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + return p ? sqlite3GlobalConfig.m.xSize(p) : 0; +} + +/* +** Free memory previously obtained from sqlite3Malloc(). +*/ +SQLITE_API void sqlite3_free(void *p){ + if( p==0 ) return; /* IMP: R-49053-54554 */ + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusDown(SQLITE_STATUS_MEMORY_USED, sqlite3MallocSize(p)); + sqlite3StatusDown(SQLITE_STATUS_MALLOC_COUNT, 1); + sqlite3GlobalConfig.m.xFree(p); + sqlite3_mutex_leave(mem0.mutex); + }else{ + sqlite3GlobalConfig.m.xFree(p); + } +} + +/* +** Add the size of memory allocation "p" to the count in +** *db->pnBytesFreed. +*/ +static SQLITE_NOINLINE void measureAllocationSize(sqlite3 *db, void *p){ + *db->pnBytesFreed += sqlite3DbMallocSize(db,p); +} + +/* +** Free memory that might be associated with a particular database +** connection. Calling sqlite3DbFree(D,X) for X==0 is a harmless no-op. +** The sqlite3DbFreeNN(D,X) version requires that X be non-NULL. +*/ +SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ + assert( db==0 || sqlite3_mutex_held(db->mutex) ); + assert( p!=0 ); + if( db ){ + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; +#ifdef SQLITE_DEBUG + memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = pBuf; + return; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; +#ifdef SQLITE_DEBUG + memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + return; + } + } + } + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + sqlite3_free(p); +} +SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ + assert( db==0 || sqlite3_mutex_held(db->mutex) ); + if( p ) sqlite3DbFreeNN(db, p); +} + +/* +** Change the size of an existing memory allocation +*/ +SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ + int nOld, nNew, nDiff; + void *pNew; + assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(pOld, (u8)~MEMTYPE_HEAP) ); + if( pOld==0 ){ + return sqlite3Malloc(nBytes); /* IMP: R-04300-56712 */ + } + if( nBytes==0 ){ + sqlite3_free(pOld); /* IMP: R-26507-47431 */ + return 0; + } + if( nBytes>=0x7fffff00 ){ + /* The 0x7ffff00 limit term is explained in comments on sqlite3Malloc() */ + return 0; + } + nOld = sqlite3MallocSize(pOld); + /* IMPLEMENTATION-OF: R-46199-30249 SQLite guarantees that the second + ** argument to xRealloc is always a value returned by a prior call to + ** xRoundup. */ + nNew = sqlite3GlobalConfig.m.xRoundup((int)nBytes); + if( nOld==nNew ){ + pNew = pOld; + }else if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_int64 nUsed; + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); + nDiff = nNew - nOld; + if( nDiff>0 && (nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)) >= + mem0.alarmThreshold-nDiff ){ + sqlite3MallocAlarm(nDiff); + if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){ + sqlite3_mutex_leave(mem0.mutex); + return 0; + } + } + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + if( pNew==0 && mem0.alarmThreshold>0 ){ + sqlite3MallocAlarm((int)nBytes); + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); + } +#endif + if( pNew ){ + nNew = sqlite3MallocSize(pNew); + sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nNew-nOld); + } + sqlite3_mutex_leave(mem0.mutex); + }else{ + pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); + } + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-11148-40995 */ + return pNew; +} + +/* +** The public interface to sqlite3Realloc. Make sure that the memory +** subsystem is initialized prior to invoking sqliteRealloc. +*/ +SQLITE_API void *sqlite3_realloc(void *pOld, int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + if( n<0 ) n = 0; /* IMP: R-26507-47431 */ + return sqlite3Realloc(pOld, n); +} +SQLITE_API void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return sqlite3Realloc(pOld, n); +} + + +/* +** Allocate and zero memory. +*/ +SQLITE_PRIVATE void *sqlite3MallocZero(u64 n){ + void *p = sqlite3Malloc(n); + if( p ){ + memset(p, 0, (size_t)n); + } + return p; +} + +/* +** Allocate and zero memory. If the allocation fails, make +** the mallocFailed flag in the connection pointer. +*/ +SQLITE_PRIVATE void *sqlite3DbMallocZero(sqlite3 *db, u64 n){ + void *p; + testcase( db==0 ); + p = sqlite3DbMallocRaw(db, n); + if( p ) memset(p, 0, (size_t)n); + return p; +} + + +/* Finish the work of sqlite3DbMallocRawNN for the unusual and +** slower case when the allocation cannot be fulfilled using lookaside. +*/ +static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){ + void *p; + assert( db!=0 ); + p = sqlite3Malloc(n); + if( !p ) sqlite3OomFault(db); + sqlite3MemdebugSetType(p, + (db->lookaside.bDisable==0) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); + return p; +} + +/* +** Allocate memory, either lookaside (if possible) or heap. +** If the allocation fails, set the mallocFailed flag in +** the connection pointer. +** +** If db!=0 and db->mallocFailed is true (indicating a prior malloc +** failure on the same database connection) then always return 0. +** Hence for a particular database connection, once malloc starts +** failing, it fails consistently until mallocFailed is reset. +** This is an important assumption. There are many places in the +** code that do things like this: +** +** int *a = (int*)sqlite3DbMallocRaw(db, 100); +** int *b = (int*)sqlite3DbMallocRaw(db, 200); +** if( b ) a[10] = 9; +** +** In other words, if a subsequent malloc (ex: "b") worked, it is assumed +** that all prior mallocs (ex: "a") worked too. +** +** The sqlite3MallocRawNN() variant guarantees that the "db" parameter is +** not a NULL pointer. +*/ +SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){ + void *p; + if( db ) return sqlite3DbMallocRawNN(db, n); + p = sqlite3Malloc(n); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + return p; +} +SQLITE_PRIVATE void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ +#ifndef SQLITE_OMIT_LOOKASIDE + LookasideSlot *pBuf; + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( db->pnBytesFreed==0 ); + if( n>db->lookaside.sz ){ + if( !db->lookaside.bDisable ){ + db->lookaside.anStat[1]++; + }else if( db->mallocFailed ){ + return 0; + } + return dbMallocRawFinish(db, n); + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( n<=LOOKASIDE_SMALL ){ + if( (pBuf = db->lookaside.pSmallFree)!=0 ){ + db->lookaside.pSmallFree = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + }else if( (pBuf = db->lookaside.pSmallInit)!=0 ){ + db->lookaside.pSmallInit = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + } + } +#endif + if( (pBuf = db->lookaside.pFree)!=0 ){ + db->lookaside.pFree = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + }else if( (pBuf = db->lookaside.pInit)!=0 ){ + db->lookaside.pInit = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + }else{ + db->lookaside.anStat[2]++; + } +#else + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( db->pnBytesFreed==0 ); + if( db->mallocFailed ){ + return 0; + } +#endif + return dbMallocRawFinish(db, n); +} + +/* Forward declaration */ +static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n); + +/* +** Resize the block of memory pointed to by p to n bytes. If the +** resize fails, set the mallocFailed flag in the connection object. +*/ +SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ + assert( db!=0 ); + if( p==0 ) return sqlite3DbMallocRawNN(db, n); + assert( sqlite3_mutex_held(db->mutex) ); + if( ((uptr)p)<(uptr)db->lookaside.pEnd ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)db->lookaside.pMiddle ){ + if( n<=LOOKASIDE_SMALL ) return p; + }else +#endif + if( ((uptr)p)>=(uptr)db->lookaside.pStart ){ + if( n<=db->lookaside.szTrue ) return p; + } + } + return dbReallocFinish(db, p, n); +} +static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ + void *pNew = 0; + assert( db!=0 ); + assert( p!=0 ); + if( db->mallocFailed==0 ){ + if( isLookaside(db, p) ){ + pNew = sqlite3DbMallocRawNN(db, n); + if( pNew ){ + memcpy(pNew, p, lookasideMallocSize(db, p)); + sqlite3DbFree(db, p); + } + }else{ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + pNew = sqlite3Realloc(p, n); + if( !pNew ){ + sqlite3OomFault(db); + } + sqlite3MemdebugSetType(pNew, + (db->lookaside.bDisable==0 ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); + } + } + return pNew; +} + +/* +** Attempt to reallocate p. If the reallocation fails, then free p +** and set the mallocFailed flag in the database connection. +*/ +SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, u64 n){ + void *pNew; + pNew = sqlite3DbRealloc(db, p, n); + if( !pNew ){ + sqlite3DbFree(db, p); + } + return pNew; +} + +/* +** Make a copy of a string in memory obtained from sqliteMalloc(). These +** functions call sqlite3MallocRaw() directly instead of sqliteMalloc(). This +** is because when memory debugging is turned on, these two functions are +** called via macros that record the current file and line number in the +** ThreadData structure. +*/ +SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3 *db, const char *z){ + char *zNew; + size_t n; + if( z==0 ){ + return 0; + } + n = strlen(z) + 1; + zNew = sqlite3DbMallocRaw(db, n); + if( zNew ){ + memcpy(zNew, z, n); + } + return zNew; +} +SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ + char *zNew; + assert( db!=0 ); + assert( z!=0 || n==0 ); + assert( (n&0x7fffffff)==n ); + zNew = z ? sqlite3DbMallocRawNN(db, n+1) : 0; + if( zNew ){ + memcpy(zNew, z, (size_t)n); + zNew[n] = 0; + } + return zNew; +} + +/* +** The text between zStart and zEnd represents a phrase within a larger +** SQL statement. Make a copy of this phrase in space obtained form +** sqlite3DbMalloc(). Omit leading and trailing whitespace. +*/ +SQLITE_PRIVATE char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){ + int n; + while( sqlite3Isspace(zStart[0]) ) zStart++; + n = (int)(zEnd - zStart); + while( ALWAYS(n>0) && sqlite3Isspace(zStart[n-1]) ) n--; + return sqlite3DbStrNDup(db, zStart, n); +} + +/* +** Free any prior content in *pz and replace it with a copy of zNew. +*/ +SQLITE_PRIVATE void sqlite3SetString(char **pz, sqlite3 *db, const char *zNew){ + sqlite3DbFree(db, *pz); + *pz = sqlite3DbStrDup(db, zNew); +} + +/* +** Call this routine to record the fact that an OOM (out-of-memory) error +** has happened. This routine will set db->mallocFailed, and also +** temporarily disable the lookaside memory allocator and interrupt +** any running VDBEs. +*/ +SQLITE_PRIVATE void sqlite3OomFault(sqlite3 *db){ + if( db->mallocFailed==0 && db->bBenignMalloc==0 ){ + db->mallocFailed = 1; + if( db->nVdbeExec>0 ){ + AtomicStore(&db->u1.isInterrupted, 1); + } + DisableLookaside; + if( db->pParse ){ + db->pParse->rc = SQLITE_NOMEM_BKPT; + } + } +} + +/* +** This routine reactivates the memory allocator and clears the +** db->mallocFailed flag as necessary. +** +** The memory allocator is not restarted if there are running +** VDBEs. +*/ +SQLITE_PRIVATE void sqlite3OomClear(sqlite3 *db){ + if( db->mallocFailed && db->nVdbeExec==0 ){ + db->mallocFailed = 0; + AtomicStore(&db->u1.isInterrupted, 0); + assert( db->lookaside.bDisable>0 ); + EnableLookaside; + } +} + +/* +** Take actions at the end of an API call to deal with error codes. +*/ +static SQLITE_NOINLINE int apiHandleError(sqlite3 *db, int rc){ + if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomClear(db); + sqlite3Error(db, SQLITE_NOMEM); + return SQLITE_NOMEM_BKPT; + } + return rc & db->errMask; +} + +/* +** This function must be called before exiting any API function (i.e. +** returning control to the user) that has called sqlite3_malloc or +** sqlite3_realloc. +** +** The returned value is normally a copy of the second argument to this +** function. However, if a malloc() failure has occurred since the previous +** invocation SQLITE_NOMEM is returned instead. +** +** If an OOM as occurred, then the connection error-code (the value +** returned by sqlite3_errcode()) is set to SQLITE_NOMEM. +*/ +SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ + /* If the db handle must hold the connection handle mutex here. + ** Otherwise the read (and possible write) of db->mallocFailed + ** is unsafe, as is the call to sqlite3Error(). + */ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + if( db->mallocFailed || rc ){ + return apiHandleError(db, rc); + } + return rc & db->errMask; +} + +/************** End of malloc.c **********************************************/ +/************** Begin file printf.c ******************************************/ +/* +** The "printf" code that follows dates from the 1980's. It is in +** the public domain. +** +************************************************************************** +** +** This file contains code for a set of "printf"-like routines. These +** routines format strings much like the printf() from the standard C +** library, though the implementation here has enhancements to support +** SQLite. +*/ +/* #include "sqliteInt.h" */ + +/* +** Conversion types fall into various categories as defined by the +** following enumeration. +*/ +#define etRADIX 0 /* non-decimal integer types. %x %o */ +#define etFLOAT 1 /* Floating point. %f */ +#define etEXP 2 /* Exponentional notation. %e and %E */ +#define etGENERIC 3 /* Floating or exponential, depending on exponent. %g */ +#define etSIZE 4 /* Return number of characters processed so far. %n */ +#define etSTRING 5 /* Strings. %s */ +#define etDYNSTRING 6 /* Dynamically allocated strings. %z */ +#define etPERCENT 7 /* Percent symbol. %% */ +#define etCHARX 8 /* Characters. %c */ +/* The rest are extensions, not normally found in printf() */ +#define etSQLESCAPE 9 /* Strings with '\'' doubled. %q */ +#define etSQLESCAPE2 10 /* Strings with '\'' doubled and enclosed in '', + NULL pointers replaced by SQL NULL. %Q */ +#define etTOKEN 11 /* a pointer to a Token structure */ +#define etSRCITEM 12 /* a pointer to a SrcItem */ +#define etPOINTER 13 /* The %p conversion */ +#define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ +#define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etDECIMAL 16 /* %d or %u, but not %x, %o */ + +#define etINVALID 17 /* Any unrecognized conversion type */ + + +/* +** An "etByte" is an 8-bit unsigned value. +*/ +typedef unsigned char etByte; + +/* +** Each builtin conversion character (ex: the 'd' in "%d") is described +** by an instance of the following structure +*/ +typedef struct et_info { /* Information about each format field */ + char fmttype; /* The format field code letter */ + etByte base; /* The base for radix conversion */ + etByte flags; /* One or more of FLAG_ constants below */ + etByte type; /* Conversion paradigm */ + etByte charset; /* Offset into aDigits[] of the digits string */ + etByte prefix; /* Offset into aPrefix[] of the prefix string */ +} et_info; + +/* +** Allowed values for et_info.flags +*/ +#define FLAG_SIGNED 1 /* True if the value to convert is signed */ +#define FLAG_STRING 4 /* Allow infinite precision */ + + +/* +** The following table is searched linearly, so it is good to put the +** most frequently used conversion types first. +*/ +static const char aDigits[] = "0123456789ABCDEF0123456789abcdef"; +static const char aPrefix[] = "-x0\000X0"; +static const et_info fmtinfo[] = { + { 'd', 10, 1, etDECIMAL, 0, 0 }, + { 's', 0, 4, etSTRING, 0, 0 }, + { 'g', 0, 1, etGENERIC, 30, 0 }, + { 'z', 0, 4, etDYNSTRING, 0, 0 }, + { 'q', 0, 4, etSQLESCAPE, 0, 0 }, + { 'Q', 0, 4, etSQLESCAPE2, 0, 0 }, + { 'w', 0, 4, etSQLESCAPE3, 0, 0 }, + { 'c', 0, 0, etCHARX, 0, 0 }, + { 'o', 8, 0, etRADIX, 0, 2 }, + { 'u', 10, 0, etDECIMAL, 0, 0 }, + { 'x', 16, 0, etRADIX, 16, 1 }, + { 'X', 16, 0, etRADIX, 0, 4 }, +#ifndef SQLITE_OMIT_FLOATING_POINT + { 'f', 0, 1, etFLOAT, 0, 0 }, + { 'e', 0, 1, etEXP, 30, 0 }, + { 'E', 0, 1, etEXP, 14, 0 }, + { 'G', 0, 1, etGENERIC, 14, 0 }, +#endif + { 'i', 10, 1, etDECIMAL, 0, 0 }, + { 'n', 0, 0, etSIZE, 0, 0 }, + { '%', 0, 0, etPERCENT, 0, 0 }, + { 'p', 16, 0, etPOINTER, 0, 1 }, + + /* All the rest are undocumented and are for internal use only */ + { 'T', 0, 0, etTOKEN, 0, 0 }, + { 'S', 0, 0, etSRCITEM, 0, 0 }, + { 'r', 10, 1, etORDINAL, 0, 0 }, +}; + +/* Notes: +** +** %S Takes a pointer to SrcItem. Shows name or database.name +** %!S Like %S but prefer the zName over the zAlias +*/ + +/* Floating point constants used for rounding */ +static const double arRound[] = { + 5.0e-01, 5.0e-02, 5.0e-03, 5.0e-04, 5.0e-05, + 5.0e-06, 5.0e-07, 5.0e-08, 5.0e-09, 5.0e-10, +}; + +/* +** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point +** conversions will work. +*/ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** "*val" is a double such that 0.1 <= *val < 10.0 +** Return the ascii code for the leading digit of *val, then +** multiply "*val" by 10.0 to renormalize. +** +** Example: +** input: *val = 3.14159 +** output: *val = 1.4159 function return = '3' +** +** The counter *cnt is incremented each time. After counter exceeds +** 16 (the number of significant digits in a 64-bit float) '0' is +** always returned. +*/ +static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ + int digit; + LONGDOUBLE_TYPE d; + if( (*cnt)<=0 ) return '0'; + (*cnt)--; + digit = (int)*val; + d = digit; + digit += '0'; + *val = (*val - d)*10.0; + return (char)digit; +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + +/* +** Set the StrAccum object to an error mode. +*/ +SQLITE_PRIVATE void sqlite3StrAccumSetError(StrAccum *p, u8 eError){ + assert( eError==SQLITE_NOMEM || eError==SQLITE_TOOBIG ); + p->accError = eError; + if( p->mxAlloc ) sqlite3_str_reset(p); + if( eError==SQLITE_TOOBIG ) sqlite3ErrorToParser(p->db, eError); +} + +/* +** Extra argument values from a PrintfArguments object +*/ +static sqlite3_int64 getIntArg(PrintfArguments *p){ + if( p->nArg<=p->nUsed ) return 0; + return sqlite3_value_int64(p->apArg[p->nUsed++]); +} +static double getDoubleArg(PrintfArguments *p){ + if( p->nArg<=p->nUsed ) return 0.0; + return sqlite3_value_double(p->apArg[p->nUsed++]); +} +static char *getTextArg(PrintfArguments *p){ + if( p->nArg<=p->nUsed ) return 0; + return (char*)sqlite3_value_text(p->apArg[p->nUsed++]); +} + +/* +** Allocate memory for a temporary buffer needed for printf rendering. +** +** If the requested size of the temp buffer is larger than the size +** of the output buffer in pAccum, then cause an SQLITE_TOOBIG error. +** Do the size check before the memory allocation to prevent rogue +** SQL from requesting large allocations using the precision or width +** field of the printf() function. +*/ +static char *printfTempBuf(sqlite3_str *pAccum, sqlite3_int64 n){ + char *z; + if( pAccum->accError ) return 0; + if( n>pAccum->nAlloc && n>pAccum->mxAlloc ){ + sqlite3StrAccumSetError(pAccum, SQLITE_TOOBIG); + return 0; + } + z = sqlite3DbMallocRaw(pAccum->db, n); + if( z==0 ){ + sqlite3StrAccumSetError(pAccum, SQLITE_NOMEM); + } + return z; +} + +/* +** On machines with a small stack size, you can redefine the +** SQLITE_PRINT_BUF_SIZE to be something smaller, if desired. +*/ +#ifndef SQLITE_PRINT_BUF_SIZE +# define SQLITE_PRINT_BUF_SIZE 70 +#endif +#define etBUFSIZE SQLITE_PRINT_BUF_SIZE /* Size of the output buffer */ + +/* +** Hard limit on the precision of floating-point conversions. +*/ +#ifndef SQLITE_PRINTF_PRECISION_LIMIT +# define SQLITE_FP_PRECISION_LIMIT 100000000 +#endif + +/* +** Render a string given by "fmt" into the StrAccum object. +*/ +SQLITE_API void sqlite3_str_vappendf( + sqlite3_str *pAccum, /* Accumulate results here */ + const char *fmt, /* Format string */ + va_list ap /* arguments */ +){ + int c; /* Next character in the format string */ + char *bufpt; /* Pointer to the conversion buffer */ + int precision; /* Precision of the current field */ + int length; /* Length of the field */ + int idx; /* A general purpose loop counter */ + int width; /* Width of the current field */ + etByte flag_leftjustify; /* True if "-" flag is present */ + etByte flag_prefix; /* '+' or ' ' or 0 for prefix */ + etByte flag_alternateform; /* True if "#" flag is present */ + etByte flag_altform2; /* True if "!" flag is present */ + etByte flag_zeropad; /* True if field width constant starts with zero */ + etByte flag_long; /* 1 for the "l" flag, 2 for "ll", 0 by default */ + etByte done; /* Loop termination flag */ + etByte cThousand; /* Thousands separator for %d and %u */ + etByte xtype = etINVALID; /* Conversion paradigm */ + u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ + char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ + sqlite_uint64 longvalue; /* Value for integer types */ + LONGDOUBLE_TYPE realvalue; /* Value for real types */ + const et_info *infop; /* Pointer to the appropriate info structure */ + char *zOut; /* Rendering buffer */ + int nOut; /* Size of the rendering buffer */ + char *zExtra = 0; /* Malloced memory used by some conversion */ +#ifndef SQLITE_OMIT_FLOATING_POINT + int exp, e2; /* exponent of real numbers */ + int nsd; /* Number of significant digits returned */ + double rounder; /* Used for rounding floating point values */ + etByte flag_dp; /* True if decimal point should be shown */ + etByte flag_rtz; /* True if trailing zeros should be removed */ +#endif + PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ + char buf[etBUFSIZE]; /* Conversion buffer */ + + /* pAccum never starts out with an empty buffer that was obtained from + ** malloc(). This precondition is required by the mprintf("%z...") + ** optimization. */ + assert( pAccum->nChar>0 || (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 ); + + bufpt = 0; + if( (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC)!=0 ){ + pArgList = va_arg(ap, PrintfArguments*); + bArgList = 1; + }else{ + bArgList = 0; + } + for(; (c=(*fmt))!=0; ++fmt){ + if( c!='%' ){ + bufpt = (char *)fmt; +#if HAVE_STRCHRNUL + fmt = strchrnul(fmt, '%'); +#else + do{ fmt++; }while( *fmt && *fmt != '%' ); +#endif + sqlite3_str_append(pAccum, bufpt, (int)(fmt - bufpt)); + if( *fmt==0 ) break; + } + if( (c=(*++fmt))==0 ){ + sqlite3_str_append(pAccum, "%", 1); + break; + } + /* Find out what flags are present */ + flag_leftjustify = flag_prefix = cThousand = + flag_alternateform = flag_altform2 = flag_zeropad = 0; + done = 0; + width = 0; + flag_long = 0; + precision = -1; + do{ + switch( c ){ + case '-': flag_leftjustify = 1; break; + case '+': flag_prefix = '+'; break; + case ' ': flag_prefix = ' '; break; + case '#': flag_alternateform = 1; break; + case '!': flag_altform2 = 1; break; + case '0': flag_zeropad = 1; break; + case ',': cThousand = ','; break; + default: done = 1; break; + case 'l': { + flag_long = 1; + c = *++fmt; + if( c=='l' ){ + c = *++fmt; + flag_long = 2; + } + done = 1; + break; + } + case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': { + unsigned wx = c - '0'; + while( (c = *++fmt)>='0' && c<='9' ){ + wx = wx*10 + c - '0'; + } + testcase( wx>0x7fffffff ); + width = wx & 0x7fffffff; +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + if( c!='.' && c!='l' ){ + done = 1; + }else{ + fmt--; + } + break; + } + case '*': { + if( bArgList ){ + width = (int)getIntArg(pArgList); + }else{ + width = va_arg(ap,int); + } + if( width<0 ){ + flag_leftjustify = 1; + width = width >= -2147483647 ? -width : 0; + } +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + if( (c = fmt[1])!='.' && c!='l' ){ + c = *++fmt; + done = 1; + } + break; + } + case '.': { + c = *++fmt; + if( c=='*' ){ + if( bArgList ){ + precision = (int)getIntArg(pArgList); + }else{ + precision = va_arg(ap,int); + } + if( precision<0 ){ + precision = precision >= -2147483647 ? -precision : -1; + } + c = *++fmt; + }else{ + unsigned px = 0; + while( c>='0' && c<='9' ){ + px = px*10 + c - '0'; + c = *++fmt; + } + testcase( px>0x7fffffff ); + precision = px & 0x7fffffff; + } +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( precision>SQLITE_PRINTF_PRECISION_LIMIT ){ + precision = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + if( c=='l' ){ + --fmt; + }else{ + done = 1; + } + break; + } + } + }while( !done && (c=(*++fmt))!=0 ); + + /* Fetch the info entry for the field */ + infop = &fmtinfo[0]; + xtype = etINVALID; + for(idx=0; idxtype; + break; + } + } + + /* + ** At this point, variables are initialized as follows: + ** + ** flag_alternateform TRUE if a '#' is present. + ** flag_altform2 TRUE if a '!' is present. + ** flag_prefix '+' or ' ' or zero + ** flag_leftjustify TRUE if a '-' is present or if the + ** field width was negative. + ** flag_zeropad TRUE if the width began with 0. + ** flag_long 1 for "l", 2 for "ll" + ** width The specified field width. This is + ** always non-negative. Zero is the default. + ** precision The specified precision. The default + ** is -1. + ** xtype The class of the conversion. + ** infop Pointer to the appropriate info struct. + */ + assert( width>=0 ); + assert( precision>=(-1) ); + switch( xtype ){ + case etPOINTER: + flag_long = sizeof(char*)==sizeof(i64) ? 2 : + sizeof(char*)==sizeof(long int) ? 1 : 0; + /* no break */ deliberate_fall_through + case etORDINAL: + case etRADIX: + cThousand = 0; + /* no break */ deliberate_fall_through + case etDECIMAL: + if( infop->flags & FLAG_SIGNED ){ + i64 v; + if( bArgList ){ + v = getIntArg(pArgList); + }else if( flag_long ){ + if( flag_long==2 ){ + v = va_arg(ap,i64) ; + }else{ + v = va_arg(ap,long int); + } + }else{ + v = va_arg(ap,int); + } + if( v<0 ){ + testcase( v==SMALLEST_INT64 ); + testcase( v==(-1) ); + longvalue = ~v; + longvalue++; + prefix = '-'; + }else{ + longvalue = v; + prefix = flag_prefix; + } + }else{ + if( bArgList ){ + longvalue = (u64)getIntArg(pArgList); + }else if( flag_long ){ + if( flag_long==2 ){ + longvalue = va_arg(ap,u64); + }else{ + longvalue = va_arg(ap,unsigned long int); + } + }else{ + longvalue = va_arg(ap,unsigned int); + } + prefix = 0; + } + if( longvalue==0 ) flag_alternateform = 0; + if( flag_zeropad && precision=4 || (longvalue/10)%10==1 ){ + x = 0; + } + *(--bufpt) = zOrd[x*2+1]; + *(--bufpt) = zOrd[x*2]; + } + { + const char *cset = &aDigits[infop->charset]; + u8 base = infop->base; + do{ /* Convert to ascii */ + *(--bufpt) = cset[longvalue%base]; + longvalue = longvalue/base; + }while( longvalue>0 ); + } + length = (int)(&zOut[nOut-1]-bufpt); + while( precision>length ){ + *(--bufpt) = '0'; /* Zero pad */ + length++; + } + if( cThousand ){ + int nn = (length - 1)/3; /* Number of "," to insert */ + int ix = (length - 1)%3 + 1; + bufpt -= nn; + for(idx=0; nn>0; idx++){ + bufpt[idx] = bufpt[idx+nn]; + ix--; + if( ix==0 ){ + bufpt[++idx] = cThousand; + nn--; + ix = 3; + } + } + } + if( prefix ) *(--bufpt) = prefix; /* Add sign */ + if( flag_alternateform && infop->prefix ){ /* Add "0" or "0x" */ + const char *pre; + char x; + pre = &aPrefix[infop->prefix]; + for(; (x=(*pre))!=0; pre++) *(--bufpt) = x; + } + length = (int)(&zOut[nOut-1]-bufpt); + break; + case etFLOAT: + case etEXP: + case etGENERIC: + if( bArgList ){ + realvalue = getDoubleArg(pArgList); + }else{ + realvalue = va_arg(ap,double); + } +#ifdef SQLITE_OMIT_FLOATING_POINT + length = 0; +#else + if( precision<0 ) precision = 6; /* Set default precision */ +#ifdef SQLITE_FP_PRECISION_LIMIT + if( precision>SQLITE_FP_PRECISION_LIMIT ){ + precision = SQLITE_FP_PRECISION_LIMIT; + } +#endif + if( realvalue<0.0 ){ + realvalue = -realvalue; + prefix = '-'; + }else{ + prefix = flag_prefix; + } + if( xtype==etGENERIC && precision>0 ) precision--; + testcase( precision>0xfff ); + idx = precision & 0xfff; + rounder = arRound[idx%10]; + while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } + if( xtype==etFLOAT ){ + double rx = (double)realvalue; + sqlite3_uint64 u; + int ex; + memcpy(&u, &rx, sizeof(u)); + ex = -1023 + (int)((u>>52)&0x7ff); + if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; + realvalue += rounder; + } + /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ + exp = 0; + if( sqlite3IsNaN((double)realvalue) ){ + bufpt = "NaN"; + length = 3; + break; + } + if( realvalue>0.0 ){ + LONGDOUBLE_TYPE scale = 1.0; + while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} + while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } + while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } + realvalue /= scale; + while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } + while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } + if( exp>350 ){ + bufpt = buf; + buf[0] = prefix; + memcpy(buf+(prefix!=0),"Inf",4); + length = 3+(prefix!=0); + break; + } + } + bufpt = buf; + /* + ** If the field type is etGENERIC, then convert to either etEXP + ** or etFLOAT, as appropriate. + */ + if( xtype!=etFLOAT ){ + realvalue += rounder; + if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } + } + if( xtype==etGENERIC ){ + flag_rtz = !flag_alternateform; + if( exp<-4 || exp>precision ){ + xtype = etEXP; + }else{ + precision = precision - exp; + xtype = etFLOAT; + } + }else{ + flag_rtz = flag_altform2; + } + if( xtype==etEXP ){ + e2 = 0; + }else{ + e2 = exp; + } + { + i64 szBufNeeded; /* Size of a temporary buffer needed */ + szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; + if( szBufNeeded > etBUFSIZE ){ + bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); + if( bufpt==0 ) return; + } + } + zOut = bufpt; + nsd = 16 + flag_altform2*10; + flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; + /* The sign in front of the number */ + if( prefix ){ + *(bufpt++) = prefix; + } + /* Digits prior to the decimal point */ + if( e2<0 ){ + *(bufpt++) = '0'; + }else{ + for(; e2>=0; e2--){ + *(bufpt++) = et_getdigit(&realvalue,&nsd); + } + } + /* The decimal point */ + if( flag_dp ){ + *(bufpt++) = '.'; + } + /* "0" digits after the decimal point but before the first + ** significant digit of the number */ + for(e2++; e2<0; precision--, e2++){ + assert( precision>0 ); + *(bufpt++) = '0'; + } + /* Significant digits after the decimal point */ + while( (precision--)>0 ){ + *(bufpt++) = et_getdigit(&realvalue,&nsd); + } + /* Remove trailing zeros and the "." if no digits follow the "." */ + if( flag_rtz && flag_dp ){ + while( bufpt[-1]=='0' ) *(--bufpt) = 0; + assert( bufpt>zOut ); + if( bufpt[-1]=='.' ){ + if( flag_altform2 ){ + *(bufpt++) = '0'; + }else{ + *(--bufpt) = 0; + } + } + } + /* Add the "eNNN" suffix */ + if( xtype==etEXP ){ + *(bufpt++) = aDigits[infop->charset]; + if( exp<0 ){ + *(bufpt++) = '-'; exp = -exp; + }else{ + *(bufpt++) = '+'; + } + if( exp>=100 ){ + *(bufpt++) = (char)((exp/100)+'0'); /* 100's digit */ + exp %= 100; + } + *(bufpt++) = (char)(exp/10+'0'); /* 10's digit */ + *(bufpt++) = (char)(exp%10+'0'); /* 1's digit */ + } + *bufpt = 0; + + /* The converted number is in buf[] and zero terminated. Output it. + ** Note that the number is in the usual order, not reversed as with + ** integer conversions. */ + length = (int)(bufpt-zOut); + bufpt = zOut; + + /* Special case: Add leading zeros if the flag_zeropad flag is + ** set and we are not left justified */ + if( flag_zeropad && !flag_leftjustify && length < width){ + int i; + int nPad = width - length; + for(i=width; i>=nPad; i--){ + bufpt[i] = bufpt[i-nPad]; + } + i = prefix!=0; + while( nPad-- ) bufpt[i++] = '0'; + length = width; + } +#endif /* !defined(SQLITE_OMIT_FLOATING_POINT) */ + break; + case etSIZE: + if( !bArgList ){ + *(va_arg(ap,int*)) = pAccum->nChar; + } + length = width = 0; + break; + case etPERCENT: + buf[0] = '%'; + bufpt = buf; + length = 1; + break; + case etCHARX: + if( bArgList ){ + bufpt = getTextArg(pArgList); + length = 1; + if( bufpt ){ + buf[0] = c = *(bufpt++); + if( (c&0xc0)==0xc0 ){ + while( length<4 && (bufpt[0]&0xc0)==0x80 ){ + buf[length++] = *(bufpt++); + } + } + }else{ + buf[0] = 0; + } + }else{ + unsigned int ch = va_arg(ap,unsigned int); + if( ch<0x00080 ){ + buf[0] = ch & 0xff; + length = 1; + }else if( ch<0x00800 ){ + buf[0] = 0xc0 + (u8)((ch>>6)&0x1f); + buf[1] = 0x80 + (u8)(ch & 0x3f); + length = 2; + }else if( ch<0x10000 ){ + buf[0] = 0xe0 + (u8)((ch>>12)&0x0f); + buf[1] = 0x80 + (u8)((ch>>6) & 0x3f); + buf[2] = 0x80 + (u8)(ch & 0x3f); + length = 3; + }else{ + buf[0] = 0xf0 + (u8)((ch>>18) & 0x07); + buf[1] = 0x80 + (u8)((ch>>12) & 0x3f); + buf[2] = 0x80 + (u8)((ch>>6) & 0x3f); + buf[3] = 0x80 + (u8)(ch & 0x3f); + length = 4; + } + } + if( precision>1 ){ + width -= precision-1; + if( width>1 && !flag_leftjustify ){ + sqlite3_str_appendchar(pAccum, width-1, ' '); + width = 0; + } + while( precision-- > 1 ){ + sqlite3_str_append(pAccum, buf, length); + } + } + bufpt = buf; + flag_altform2 = 1; + goto adjust_width_for_utf8; + case etSTRING: + case etDYNSTRING: + if( bArgList ){ + bufpt = getTextArg(pArgList); + xtype = etSTRING; + }else{ + bufpt = va_arg(ap,char*); + } + if( bufpt==0 ){ + bufpt = ""; + }else if( xtype==etDYNSTRING ){ + if( pAccum->nChar==0 + && pAccum->mxAlloc + && width==0 + && precision<0 + && pAccum->accError==0 + ){ + /* Special optimization for sqlite3_mprintf("%z..."): + ** Extend an existing memory allocation rather than creating + ** a new one. */ + assert( (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 ); + pAccum->zText = bufpt; + pAccum->nAlloc = sqlite3DbMallocSize(pAccum->db, bufpt); + pAccum->nChar = 0x7fffffff & (int)strlen(bufpt); + pAccum->printfFlags |= SQLITE_PRINTF_MALLOCED; + length = 0; + break; + } + zExtra = bufpt; + } + if( precision>=0 ){ + if( flag_altform2 ){ + /* Set length to the number of bytes needed in order to display + ** precision characters */ + unsigned char *z = (unsigned char*)bufpt; + while( precision-- > 0 && z[0] ){ + SQLITE_SKIP_UTF8(z); + } + length = (int)(z - (unsigned char*)bufpt); + }else{ + for(length=0; length0 ){ + /* Adjust width to account for extra bytes in UTF-8 characters */ + int ii = length - 1; + while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++; + } + break; + case etSQLESCAPE: /* %q: Escape ' characters */ + case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */ + case etSQLESCAPE3: { /* %w: Escape " characters */ + int i, j, k, n, isnull; + int needQuote; + char ch; + char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */ + char *escarg; + + if( bArgList ){ + escarg = getTextArg(pArgList); + }else{ + escarg = va_arg(ap,char*); + } + isnull = escarg==0; + if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); + /* For %q, %Q, and %w, the precision is the number of bytes (or + ** characters if the ! flags is present) to use from the input. + ** Because of the extra quoting characters inserted, the number + ** of output characters may be larger than the precision. + */ + k = precision; + for(i=n=0; k!=0 && (ch=escarg[i])!=0; i++, k--){ + if( ch==q ) n++; + if( flag_altform2 && (ch&0xc0)==0xc0 ){ + while( (escarg[i+1]&0xc0)==0x80 ){ i++; } + } + } + needQuote = !isnull && xtype==etSQLESCAPE2; + n += i + 3; + if( n>etBUFSIZE ){ + bufpt = zExtra = printfTempBuf(pAccum, n); + if( bufpt==0 ) return; + }else{ + bufpt = buf; + } + j = 0; + if( needQuote ) bufpt[j++] = q; + k = i; + for(i=0; iprintfFlags & SQLITE_PRINTF_INTERNAL)==0 ) return; + pToken = va_arg(ap, Token*); + assert( bArgList==0 ); + if( pToken && pToken->n ){ + sqlite3_str_append(pAccum, (const char*)pToken->z, pToken->n); + } + length = width = 0; + break; + } + case etSRCITEM: { + SrcItem *pItem; + if( (pAccum->printfFlags & SQLITE_PRINTF_INTERNAL)==0 ) return; + pItem = va_arg(ap, SrcItem*); + assert( bArgList==0 ); + if( pItem->zAlias && !flag_altform2 ){ + sqlite3_str_appendall(pAccum, pItem->zAlias); + }else if( pItem->zName ){ + if( pItem->zDatabase ){ + sqlite3_str_appendall(pAccum, pItem->zDatabase); + sqlite3_str_append(pAccum, ".", 1); + } + sqlite3_str_appendall(pAccum, pItem->zName); + }else if( pItem->zAlias ){ + sqlite3_str_appendall(pAccum, pItem->zAlias); + }else if( ALWAYS(pItem->pSelect) ){ + sqlite3_str_appendf(pAccum, "SUBQUERY %u", pItem->pSelect->selId); + } + length = width = 0; + break; + } + default: { + assert( xtype==etINVALID ); + return; + } + }/* End switch over the format type */ + /* + ** The text of the conversion is pointed to by "bufpt" and is + ** "length" characters long. The field width is "width". Do + ** the output. Both length and width are in bytes, not characters, + ** at this point. If the "!" flag was present on string conversions + ** indicating that width and precision should be expressed in characters, + ** then the values have been translated prior to reaching this point. + */ + width -= length; + if( width>0 ){ + if( !flag_leftjustify ) sqlite3_str_appendchar(pAccum, width, ' '); + sqlite3_str_append(pAccum, bufpt, length); + if( flag_leftjustify ) sqlite3_str_appendchar(pAccum, width, ' '); + }else{ + sqlite3_str_append(pAccum, bufpt, length); + } + + if( zExtra ){ + sqlite3DbFree(pAccum->db, zExtra); + zExtra = 0; + } + }/* End for loop over the format string */ +} /* End of function */ + +/* +** Enlarge the memory allocation on a StrAccum object so that it is +** able to accept at least N more bytes of text. +** +** Return the number of bytes of text that StrAccum is able to accept +** after the attempted enlargement. The value returned might be zero. +*/ +static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ + char *zNew; + assert( p->nChar+(i64)N >= p->nAlloc ); /* Only called if really needed */ + if( p->accError ){ + testcase(p->accError==SQLITE_TOOBIG); + testcase(p->accError==SQLITE_NOMEM); + return 0; + } + if( p->mxAlloc==0 ){ + sqlite3StrAccumSetError(p, SQLITE_TOOBIG); + return p->nAlloc - p->nChar - 1; + }else{ + char *zOld = isMalloced(p) ? p->zText : 0; + i64 szNew = p->nChar; + szNew += (sqlite3_int64)N + 1; + if( szNew+p->nChar<=p->mxAlloc ){ + /* Force exponential buffer size growth as long as it does not overflow, + ** to avoid having to call this routine too often */ + szNew += p->nChar; + } + if( szNew > p->mxAlloc ){ + sqlite3_str_reset(p); + sqlite3StrAccumSetError(p, SQLITE_TOOBIG); + return 0; + }else{ + p->nAlloc = (int)szNew; + } + if( p->db ){ + zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); + }else{ + zNew = sqlite3Realloc(zOld, p->nAlloc); + } + if( zNew ){ + assert( p->zText!=0 || p->nChar==0 ); + if( !isMalloced(p) && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); + p->zText = zNew; + p->nAlloc = sqlite3DbMallocSize(p->db, zNew); + p->printfFlags |= SQLITE_PRINTF_MALLOCED; + }else{ + sqlite3_str_reset(p); + sqlite3StrAccumSetError(p, SQLITE_NOMEM); + return 0; + } + } + return N; +} + +/* +** Append N copies of character c to the given string buffer. +*/ +SQLITE_API void sqlite3_str_appendchar(sqlite3_str *p, int N, char c){ + testcase( p->nChar + (i64)N > 0x7fffffff ); + if( p->nChar+(i64)N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ){ + return; + } + while( (N--)>0 ) p->zText[p->nChar++] = c; +} + +/* +** The StrAccum "p" is not large enough to accept N new bytes of z[]. +** So enlarge if first, then do the append. +** +** This is a helper routine to sqlite3_str_append() that does special-case +** work (enlarging the buffer) using tail recursion, so that the +** sqlite3_str_append() routine can use fast calling semantics. +*/ +static void SQLITE_NOINLINE enlargeAndAppend(StrAccum *p, const char *z, int N){ + N = sqlite3StrAccumEnlarge(p, N); + if( N>0 ){ + memcpy(&p->zText[p->nChar], z, N); + p->nChar += N; + } +} + +/* +** Append N bytes of text from z to the StrAccum object. Increase the +** size of the memory allocation for StrAccum if necessary. +*/ +SQLITE_API void sqlite3_str_append(sqlite3_str *p, const char *z, int N){ + assert( z!=0 || N==0 ); + assert( p->zText!=0 || p->nChar==0 || p->accError ); + assert( N>=0 ); + assert( p->accError==0 || p->nAlloc==0 || p->mxAlloc==0 ); + if( p->nChar+N >= p->nAlloc ){ + enlargeAndAppend(p,z,N); + }else if( N ){ + assert( p->zText ); + p->nChar += N; + memcpy(&p->zText[p->nChar-N], z, N); + } +} + +/* +** Append the complete text of zero-terminated string z[] to the p string. +*/ +SQLITE_API void sqlite3_str_appendall(sqlite3_str *p, const char *z){ + sqlite3_str_append(p, z, sqlite3Strlen30(z)); +} + + +/* +** Finish off a string by making sure it is zero-terminated. +** Return a pointer to the resulting string. Return a NULL +** pointer if any kind of error was encountered. +*/ +static SQLITE_NOINLINE char *strAccumFinishRealloc(StrAccum *p){ + char *zText; + assert( p->mxAlloc>0 && !isMalloced(p) ); + zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + if( zText ){ + memcpy(zText, p->zText, p->nChar+1); + p->printfFlags |= SQLITE_PRINTF_MALLOCED; + }else{ + sqlite3StrAccumSetError(p, SQLITE_NOMEM); + } + p->zText = zText; + return zText; +} +SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ + if( p->zText ){ + p->zText[p->nChar] = 0; + if( p->mxAlloc>0 && !isMalloced(p) ){ + return strAccumFinishRealloc(p); + } + } + return p->zText; +} + +/* +** Use the content of the StrAccum passed as the second argument +** as the result of an SQL function. +*/ +SQLITE_PRIVATE void sqlite3ResultStrAccum(sqlite3_context *pCtx, StrAccum *p){ + if( p->accError ){ + sqlite3_result_error_code(pCtx, p->accError); + sqlite3_str_reset(p); + }else if( isMalloced(p) ){ + sqlite3_result_text(pCtx, p->zText, p->nChar, SQLITE_DYNAMIC); + }else{ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + sqlite3_str_reset(p); + } +} + +/* +** This singleton is an sqlite3_str object that is returned if +** sqlite3_malloc() fails to provide space for a real one. This +** sqlite3_str object accepts no new text and always returns +** an SQLITE_NOMEM error. +*/ +static sqlite3_str sqlite3OomStr = { + 0, 0, 0, 0, 0, SQLITE_NOMEM, 0 +}; + +/* Finalize a string created using sqlite3_str_new(). +*/ +SQLITE_API char *sqlite3_str_finish(sqlite3_str *p){ + char *z; + if( p!=0 && p!=&sqlite3OomStr ){ + z = sqlite3StrAccumFinish(p); + sqlite3_free(p); + }else{ + z = 0; + } + return z; +} + +/* Return any error code associated with p */ +SQLITE_API int sqlite3_str_errcode(sqlite3_str *p){ + return p ? p->accError : SQLITE_NOMEM; +} + +/* Return the current length of p in bytes */ +SQLITE_API int sqlite3_str_length(sqlite3_str *p){ + return p ? p->nChar : 0; +} + +/* Return the current value for p */ +SQLITE_API char *sqlite3_str_value(sqlite3_str *p){ + if( p==0 || p->nChar==0 ) return 0; + p->zText[p->nChar] = 0; + return p->zText; +} + +/* +** Reset an StrAccum string. Reclaim all malloced memory. +*/ +SQLITE_API void sqlite3_str_reset(StrAccum *p){ + if( isMalloced(p) ){ + sqlite3DbFree(p->db, p->zText); + p->printfFlags &= ~SQLITE_PRINTF_MALLOCED; + } + p->nAlloc = 0; + p->nChar = 0; + p->zText = 0; +} + +/* +** Initialize a string accumulator. +** +** p: The accumulator to be initialized. +** db: Pointer to a database connection. May be NULL. Lookaside +** memory is used if not NULL. db->mallocFailed is set appropriately +** when not NULL. +** zBase: An initial buffer. May be NULL in which case the initial buffer +** is malloced. +** n: Size of zBase in bytes. If total space requirements never exceed +** n then no memory allocations ever occur. +** mx: Maximum number of bytes to accumulate. If mx==0 then no memory +** allocations will ever occur. +*/ +SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum *p, sqlite3 *db, char *zBase, int n, int mx){ + p->zText = zBase; + p->db = db; + p->nAlloc = n; + p->mxAlloc = mx; + p->nChar = 0; + p->accError = 0; + p->printfFlags = 0; +} + +/* Allocate and initialize a new dynamic string object */ +SQLITE_API sqlite3_str *sqlite3_str_new(sqlite3 *db){ + sqlite3_str *p = sqlite3_malloc64(sizeof(*p)); + if( p ){ + sqlite3StrAccumInit(p, 0, 0, 0, + db ? db->aLimit[SQLITE_LIMIT_LENGTH] : SQLITE_MAX_LENGTH); + }else{ + p = &sqlite3OomStr; + } + return p; +} + +/* +** Print into memory obtained from sqliteMalloc(). Use the internal +** %-conversion extensions. +*/ +SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list ap){ + char *z; + char zBase[SQLITE_PRINT_BUF_SIZE]; + StrAccum acc; + assert( db!=0 ); + sqlite3StrAccumInit(&acc, db, zBase, sizeof(zBase), + db->aLimit[SQLITE_LIMIT_LENGTH]); + acc.printfFlags = SQLITE_PRINTF_INTERNAL; + sqlite3_str_vappendf(&acc, zFormat, ap); + z = sqlite3StrAccumFinish(&acc); + if( acc.accError==SQLITE_NOMEM ){ + sqlite3OomFault(db); + } + return z; +} + +/* +** Print into memory obtained from sqliteMalloc(). Use the internal +** %-conversion extensions. +*/ +SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + return z; +} + +/* +** Print into memory obtained from sqlite3_malloc(). Omit the internal +** %-conversion extensions. +*/ +SQLITE_API char *sqlite3_vmprintf(const char *zFormat, va_list ap){ + char *z; + char zBase[SQLITE_PRINT_BUF_SIZE]; + StrAccum acc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zFormat==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + sqlite3StrAccumInit(&acc, 0, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); + sqlite3_str_vappendf(&acc, zFormat, ap); + z = sqlite3StrAccumFinish(&acc); + return z; +} + +/* +** Print into memory obtained from sqlite3_malloc()(). Omit the internal +** %-conversion extensions. +*/ +SQLITE_API char *sqlite3_mprintf(const char *zFormat, ...){ + va_list ap; + char *z; +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + return z; +} + +/* +** sqlite3_snprintf() works like snprintf() except that it ignores the +** current locale settings. This is important for SQLite because we +** are not able to use a "," as the decimal point in place of "." as +** specified by some locales. +** +** Oops: The first two arguments of sqlite3_snprintf() are backwards +** from the snprintf() standard. Unfortunately, it is too late to change +** this without breaking compatibility, so we just have to live with the +** mistake. +** +** sqlite3_vsnprintf() is the varargs version. +*/ +SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ + StrAccum acc; + if( n<=0 ) return zBuf; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf ) zBuf[0] = 0; + return zBuf; + } +#endif + sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); + sqlite3_str_vappendf(&acc, zFormat, ap); + zBuf[acc.nChar] = 0; + return zBuf; +} +SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ + char *z; + va_list ap; + va_start(ap,zFormat); + z = sqlite3_vsnprintf(n, zBuf, zFormat, ap); + va_end(ap); + return z; +} + +/* +** This is the routine that actually formats the sqlite3_log() message. +** We house it in a separate routine from sqlite3_log() to avoid using +** stack space on small-stack systems when logging is disabled. +** +** sqlite3_log() must render into a static buffer. It cannot dynamically +** allocate memory because it might be called while the memory allocator +** mutex is held. +** +** sqlite3_str_vappendf() might ask for *temporary* memory allocations for +** certain format characters (%q) or for very large precisions or widths. +** Care must be taken that any sqlite3_log() calls that occur while the +** memory mutex is held do not use these mechanisms. +*/ +static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ + StrAccum acc; /* String accumulator */ + char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ + + sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); + sqlite3_str_vappendf(&acc, zFormat, ap); + sqlite3GlobalConfig.xLog(sqlite3GlobalConfig.pLogArg, iErrCode, + sqlite3StrAccumFinish(&acc)); +} + +/* +** Format and write a message to the log if logging is enabled. +*/ +SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...){ + va_list ap; /* Vararg list */ + if( sqlite3GlobalConfig.xLog ){ + va_start(ap, zFormat); + renderLogMsg(iErrCode, zFormat, ap); + va_end(ap); + } +} + +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) +/* +** A version of printf() that understands %lld. Used for debugging. +** The printf() built into some versions of windows does not understand %lld +** and segfaults if you give it a long long int. +*/ +SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ + va_list ap; + StrAccum acc; + char zBuf[SQLITE_PRINT_BUF_SIZE*10]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + va_start(ap,zFormat); + sqlite3_str_vappendf(&acc, zFormat, ap); + va_end(ap); + sqlite3StrAccumFinish(&acc); +#ifdef SQLITE_OS_TRACE_PROC + { + extern void SQLITE_OS_TRACE_PROC(const char *zBuf, int nBuf); + SQLITE_OS_TRACE_PROC(zBuf, sizeof(zBuf)); + } +#else + fprintf(stdout,"%s", zBuf); + fflush(stdout); +#endif +} +#endif + + +/* +** variable-argument wrapper around sqlite3_str_vappendf(). The bFlags argument +** can contain the bit SQLITE_PRINTF_INTERNAL enable internal formats. +*/ +SQLITE_API void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){ + va_list ap; + va_start(ap,zFormat); + sqlite3_str_vappendf(p, zFormat, ap); + va_end(ap); +} + +/************** End of printf.c **********************************************/ +/************** Begin file treeview.c ****************************************/ +/* +** 2015-06-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains C code to implement the TreeView debugging routines. +** These routines print a parse tree to standard output for debugging and +** analysis. +** +** The interfaces in this file is only available when compiling +** with SQLITE_DEBUG. +*/ +/* #include "sqliteInt.h" */ +#ifdef SQLITE_DEBUG + +/* +** Add a new subitem to the tree. The moreToFollow flag indicates that this +** is not the last item in the tree. +*/ +static TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ + if( p==0 ){ + p = sqlite3_malloc64( sizeof(*p) ); + if( p==0 ) return 0; + memset(p, 0, sizeof(*p)); + }else{ + p->iLevel++; + } + assert( moreToFollow==0 || moreToFollow==1 ); + if( p->iLevelbLine) ) p->bLine[p->iLevel] = moreToFollow; + return p; +} + +/* +** Finished with one layer of the tree +*/ +static void sqlite3TreeViewPop(TreeView *p){ + if( p==0 ) return; + p->iLevel--; + if( p->iLevel<0 ) sqlite3_free(p); +} + +/* +** Generate a single line of output for the tree, with a prefix that contains +** all the appropriate tree lines +*/ +static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ + va_list ap; + int i; + StrAccum acc; + char zBuf[500]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + if( p ){ + for(i=0; iiLevel && ibLine)-1; i++){ + sqlite3_str_append(&acc, p->bLine[i] ? "| " : " ", 4); + } + sqlite3_str_append(&acc, p->bLine[i] ? "|-- " : "'-- ", 4); + } + if( zFormat!=0 ){ + va_start(ap, zFormat); + sqlite3_str_vappendf(&acc, zFormat, ap); + va_end(ap); + assert( acc.nChar>0 || acc.accError ); + sqlite3_str_append(&acc, "\n", 1); + } + sqlite3StrAccumFinish(&acc); + fprintf(stdout,"%s", zBuf); + fflush(stdout); +} + +/* +** Shorthand for starting a new tree item that consists of a single label +*/ +static void sqlite3TreeViewItem(TreeView *p, const char *zLabel,u8 moreFollows){ + p = sqlite3TreeViewPush(p, moreFollows); + sqlite3TreeViewLine(p, "%s", zLabel); +} + +/* +** Generate a human-readable description of a WITH clause. +*/ +SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 moreToFollow){ + int i; + if( pWith==0 ) return; + if( pWith->nCte==0 ) return; + if( pWith->pOuter ){ + sqlite3TreeViewLine(pView, "WITH (0x%p, pOuter=0x%p)",pWith,pWith->pOuter); + }else{ + sqlite3TreeViewLine(pView, "WITH (0x%p)", pWith); + } + if( pWith->nCte>0 ){ + pView = sqlite3TreeViewPush(pView, 1); + for(i=0; inCte; i++){ + StrAccum x; + char zLine[1000]; + const struct Cte *pCte = &pWith->a[i]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + sqlite3_str_appendf(&x, "%s", pCte->zName); + if( pCte->pCols && pCte->pCols->nExpr>0 ){ + char cSep = '('; + int j; + for(j=0; jpCols->nExpr; j++){ + sqlite3_str_appendf(&x, "%c%s", cSep, pCte->pCols->a[j].zEName); + cSep = ','; + } + sqlite3_str_appendf(&x, ")"); + } + if( pCte->pUse ){ + sqlite3_str_appendf(&x, " (pUse=0x%p, nUse=%d)", pCte->pUse, + pCte->pUse->nUse); + } + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, inCte-1); + sqlite3TreeViewSelect(pView, pCte->pSelect, 0); + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); + } +} + +/* +** Generate a human-readable description of a SrcList object. +*/ +SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ + int i; + for(i=0; inSrc; i++){ + const SrcItem *pItem = &pSrc->a[i]; + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + x.printfFlags |= SQLITE_PRINTF_INTERNAL; + sqlite3_str_appendf(&x, "{%d:*} %!S", pItem->iCursor, pItem); + if( pItem->pTab ){ + sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx", + pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, pItem->colUsed); + } + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3_str_appendf(&x, " LEFT-JOIN"); + }else if( pItem->fg.jointype & JT_CROSS ){ + sqlite3_str_appendf(&x, " CROSS-JOIN"); + } + if( pItem->fg.fromDDL ){ + sqlite3_str_appendf(&x, " DDL"); + } + if( pItem->fg.isCte ){ + sqlite3_str_appendf(&x, " CteUse=0x%p", pItem->u2.pCteUse); + } + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, inSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + if( pItem->fg.isTabFunc ){ + sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); + } + sqlite3TreeViewPop(pView); + } +} + +/* +** Generate a human-readable description of a Select object. +*/ +SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ + int n = 0; + int cnt = 0; + if( p==0 ){ + sqlite3TreeViewLine(pView, "nil-SELECT"); + return; + } + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( p->pWith ){ + sqlite3TreeViewWith(pView, p->pWith, 1); + cnt = 1; + sqlite3TreeViewPush(pView, 1); + } + do{ + if( p->selFlags & SF_WhereBegin ){ + sqlite3TreeViewLine(pView, "sqlite3WhereBegin()"); + }else{ + sqlite3TreeViewLine(pView, + "SELECT%s%s (%u/%p) selFlags=0x%x nSelectRow=%d", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), + p->selId, p, p->selFlags, + (int)p->nSelectRow + ); + } + if( cnt++ ) sqlite3TreeViewPop(pView); + if( p->pPrior ){ + n = 1000; + }else{ + n = 0; + if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pWhere ) n++; + if( p->pGroupBy ) n++; + if( p->pHaving ) n++; + if( p->pOrderBy ) n++; + if( p->pLimit ) n++; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ) n++; + if( p->pWinDefn ) n++; +#endif + } + if( p->pEList ){ + sqlite3TreeViewExprList(pView, p->pEList, n>0, "result-set"); + } + n--; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ){ + Window *pX; + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "window-functions"); + for(pX=p->pWin; pX; pX=pX->pNextWin){ + sqlite3TreeViewWinFunc(pView, pX, pX->pNextWin!=0); + } + sqlite3TreeViewPop(pView); + } +#endif + if( p->pSrc && p->pSrc->nSrc ){ + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "FROM"); + sqlite3TreeViewSrcList(pView, p->pSrc); + sqlite3TreeViewPop(pView); + } + if( p->pWhere ){ + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); + sqlite3TreeViewExpr(pView, p->pWhere, 0); + sqlite3TreeViewPop(pView); + } + if( p->pGroupBy ){ + sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); + } + if( p->pHaving ){ + sqlite3TreeViewItem(pView, "HAVING", (n--)>0); + sqlite3TreeViewExpr(pView, p->pHaving, 0); + sqlite3TreeViewPop(pView); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWinDefn ){ + Window *pX; + sqlite3TreeViewItem(pView, "WINDOW", (n--)>0); + for(pX=p->pWinDefn; pX; pX=pX->pNextWin){ + sqlite3TreeViewWindow(pView, pX, pX->pNextWin!=0); + } + sqlite3TreeViewPop(pView); + } +#endif + if( p->pOrderBy ){ + sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); + } + if( p->pLimit ){ + sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); + sqlite3TreeViewExpr(pView, p->pLimit->pLeft, p->pLimit->pRight!=0); + if( p->pLimit->pRight ){ + sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); + sqlite3TreeViewExpr(pView, p->pLimit->pRight, 0); + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); + } + if( p->pPrior ){ + const char *zOp = "UNION"; + switch( p->op ){ + case TK_ALL: zOp = "UNION ALL"; break; + case TK_INTERSECT: zOp = "INTERSECT"; break; + case TK_EXCEPT: zOp = "EXCEPT"; break; + } + sqlite3TreeViewItem(pView, zOp, 1); + } + p = p->pPrior; + }while( p!=0 ); + sqlite3TreeViewPop(pView); +} + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a description of starting or stopping bounds +*/ +SQLITE_PRIVATE void sqlite3TreeViewBound( + TreeView *pView, /* View context */ + u8 eBound, /* UNBOUNDED, CURRENT, PRECEDING, FOLLOWING */ + Expr *pExpr, /* Value for PRECEDING or FOLLOWING */ + u8 moreToFollow /* True if more to follow */ +){ + switch( eBound ){ + case TK_UNBOUNDED: { + sqlite3TreeViewItem(pView, "UNBOUNDED", moreToFollow); + sqlite3TreeViewPop(pView); + break; + } + case TK_CURRENT: { + sqlite3TreeViewItem(pView, "CURRENT", moreToFollow); + sqlite3TreeViewPop(pView); + break; + } + case TK_PRECEDING: { + sqlite3TreeViewItem(pView, "PRECEDING", moreToFollow); + sqlite3TreeViewExpr(pView, pExpr, 0); + sqlite3TreeViewPop(pView); + break; + } + case TK_FOLLOWING: { + sqlite3TreeViewItem(pView, "FOLLOWING", moreToFollow); + sqlite3TreeViewExpr(pView, pExpr, 0); + sqlite3TreeViewPop(pView); + break; + } + } +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a human-readable explanation for a Window object +*/ +SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u8 more){ + int nElement = 0; + if( pWin->pFilter ){ + sqlite3TreeViewItem(pView, "FILTER", 1); + sqlite3TreeViewExpr(pView, pWin->pFilter, 0); + sqlite3TreeViewPop(pView); + } + pView = sqlite3TreeViewPush(pView, more); + if( pWin->zName ){ + sqlite3TreeViewLine(pView, "OVER %s (%p)", pWin->zName, pWin); + }else{ + sqlite3TreeViewLine(pView, "OVER (%p)", pWin); + } + if( pWin->zBase ) nElement++; + if( pWin->pOrderBy ) nElement++; + if( pWin->eFrmType ) nElement++; + if( pWin->eExclude ) nElement++; + if( pWin->zBase ){ + sqlite3TreeViewPush(pView, (--nElement)>0); + sqlite3TreeViewLine(pView, "window: %s", pWin->zBase); + sqlite3TreeViewPop(pView); + } + if( pWin->pPartition ){ + sqlite3TreeViewExprList(pView, pWin->pPartition, nElement>0,"PARTITION-BY"); + } + if( pWin->pOrderBy ){ + sqlite3TreeViewExprList(pView, pWin->pOrderBy, (--nElement)>0, "ORDER-BY"); + } + if( pWin->eFrmType ){ + char zBuf[30]; + const char *zFrmType = "ROWS"; + if( pWin->eFrmType==TK_RANGE ) zFrmType = "RANGE"; + if( pWin->eFrmType==TK_GROUPS ) zFrmType = "GROUPS"; + sqlite3_snprintf(sizeof(zBuf),zBuf,"%s%s",zFrmType, + pWin->bImplicitFrame ? " (implied)" : ""); + sqlite3TreeViewItem(pView, zBuf, (--nElement)>0); + sqlite3TreeViewBound(pView, pWin->eStart, pWin->pStart, 1); + sqlite3TreeViewBound(pView, pWin->eEnd, pWin->pEnd, 0); + sqlite3TreeViewPop(pView); + } + if( pWin->eExclude ){ + char zBuf[30]; + const char *zExclude; + switch( pWin->eExclude ){ + case TK_NO: zExclude = "NO OTHERS"; break; + case TK_CURRENT: zExclude = "CURRENT ROW"; break; + case TK_GROUP: zExclude = "GROUP"; break; + case TK_TIES: zExclude = "TIES"; break; + default: + sqlite3_snprintf(sizeof(zBuf),zBuf,"invalid(%d)", pWin->eExclude); + zExclude = zBuf; + break; + } + sqlite3TreeViewPush(pView, 0); + sqlite3TreeViewLine(pView, "EXCLUDE %s", zExclude); + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a human-readable explanation for a Window Function object +*/ +SQLITE_PRIVATE void sqlite3TreeViewWinFunc(TreeView *pView, const Window *pWin, u8 more){ + pView = sqlite3TreeViewPush(pView, more); + sqlite3TreeViewLine(pView, "WINFUNC %s(%d)", + pWin->pFunc->zName, pWin->pFunc->nArg); + sqlite3TreeViewWindow(pView, pWin, 0); + sqlite3TreeViewPop(pView); +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** Generate a human-readable explanation of an expression tree. +*/ +SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ + const char *zBinOp = 0; /* Binary operator */ + const char *zUniOp = 0; /* Unary operator */ + char zFlgs[200]; + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( pExpr==0 ){ + sqlite3TreeViewLine(pView, "nil"); + sqlite3TreeViewPop(pView); + return; + } + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ + StrAccum x; + sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); + sqlite3_str_appendf(&x, " fg.af=%x.%c", + pExpr->flags, pExpr->affExpr ? pExpr->affExpr : 'n'); + if( ExprHasProperty(pExpr, EP_FromJoin) ){ + sqlite3_str_appendf(&x, " iRJT=%d", pExpr->iRightJoinTable); + } + if( ExprHasProperty(pExpr, EP_FromDDL) ){ + sqlite3_str_appendf(&x, " DDL"); + } + if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ + sqlite3_str_appendf(&x, " IMMUTABLE"); + } + sqlite3StrAccumFinish(&x); + }else{ + zFlgs[0] = 0; + } + switch( pExpr->op ){ + case TK_AGG_COLUMN: { + sqlite3TreeViewLine(pView, "AGG{%d:%d}%s", + pExpr->iTable, pExpr->iColumn, zFlgs); + break; + } + case TK_COLUMN: { + if( pExpr->iTable<0 ){ + /* This only happens when coding check constraints */ + char zOp2[16]; + if( pExpr->op2 ){ + sqlite3_snprintf(sizeof(zOp2),zOp2," op2=0x%02x",pExpr->op2); + }else{ + zOp2[0] = 0; + } + sqlite3TreeViewLine(pView, "COLUMN(%d)%s%s", + pExpr->iColumn, zFlgs, zOp2); + }else{ + sqlite3TreeViewLine(pView, "{%d:%d} pTab=%p%s", + pExpr->iTable, pExpr->iColumn, + pExpr->y.pTab, zFlgs); + } + if( ExprHasProperty(pExpr, EP_FixedCol) ){ + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + } + break; + } + case TK_INTEGER: { + if( pExpr->flags & EP_IntValue ){ + sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue); + }else{ + sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken); + } + break; + } +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_STRING: { + sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); + break; + } + case TK_NULL: { + sqlite3TreeViewLine(pView,"NULL"); + break; + } + case TK_TRUEFALSE: { + sqlite3TreeViewLine(pView,"%s%s", + sqlite3ExprTruthValue(pExpr) ? "TRUE" : "FALSE", zFlgs); + break; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); + break; + } +#endif + case TK_VARIABLE: { + sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", + pExpr->u.zToken, pExpr->iColumn); + break; + } + case TK_REGISTER: { + sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); + break; + } + case TK_ID: { + sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken); + break; + } +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } +#endif /* SQLITE_OMIT_CAST */ + case TK_LT: zBinOp = "LT"; break; + case TK_LE: zBinOp = "LE"; break; + case TK_GT: zBinOp = "GT"; break; + case TK_GE: zBinOp = "GE"; break; + case TK_NE: zBinOp = "NE"; break; + case TK_EQ: zBinOp = "EQ"; break; + case TK_IS: zBinOp = "IS"; break; + case TK_ISNOT: zBinOp = "ISNOT"; break; + case TK_AND: zBinOp = "AND"; break; + case TK_OR: zBinOp = "OR"; break; + case TK_PLUS: zBinOp = "ADD"; break; + case TK_STAR: zBinOp = "MUL"; break; + case TK_MINUS: zBinOp = "SUB"; break; + case TK_REM: zBinOp = "REM"; break; + case TK_BITAND: zBinOp = "BITAND"; break; + case TK_BITOR: zBinOp = "BITOR"; break; + case TK_SLASH: zBinOp = "DIV"; break; + case TK_LSHIFT: zBinOp = "LSHIFT"; break; + case TK_RSHIFT: zBinOp = "RSHIFT"; break; + case TK_CONCAT: zBinOp = "CONCAT"; break; + case TK_DOT: zBinOp = "DOT"; break; + case TK_LIMIT: zBinOp = "LIMIT"; break; + + case TK_UMINUS: zUniOp = "UMINUS"; break; + case TK_UPLUS: zUniOp = "UPLUS"; break; + case TK_BITNOT: zUniOp = "BITNOT"; break; + case TK_NOT: zUniOp = "NOT"; break; + case TK_ISNULL: zUniOp = "ISNULL"; break; + case TK_NOTNULL: zUniOp = "NOTNULL"; break; + + case TK_TRUTH: { + int x; + const char *azOp[] = { + "IS-FALSE", "IS-TRUE", "IS-NOT-FALSE", "IS-NOT-TRUE" + }; + assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); + assert( pExpr->pRight ); + assert( sqlite3ExprSkipCollate(pExpr->pRight)->op==TK_TRUEFALSE ); + x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight); + zUniOp = azOp[x]; + break; + } + + case TK_SPAN: { + sqlite3TreeViewLine(pView, "SPAN %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + + case TK_COLLATE: { + /* COLLATE operators without the EP_Collate flag are intended to + ** emulate collation associated with a table column. These show + ** up in the treeview output as "SOFT-COLLATE". Explicit COLLATE + ** operators that appear in the original SQL always have the + ** EP_Collate bit set and appear in treeview output as just "COLLATE" */ + sqlite3TreeViewLine(pView, "%sCOLLATE %Q%s", + !ExprHasProperty(pExpr, EP_Collate) ? "SOFT-" : "", + pExpr->u.zToken, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + + case TK_AGG_FUNCTION: + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + Window *pWin; + if( ExprHasProperty(pExpr, EP_TokenOnly) ){ + pFarg = 0; + pWin = 0; + }else{ + pFarg = pExpr->x.pList; +#ifndef SQLITE_OMIT_WINDOWFUNC + pWin = ExprHasProperty(pExpr, EP_WinFunc) ? pExpr->y.pWin : 0; +#else + pWin = 0; +#endif + } + if( pExpr->op==TK_AGG_FUNCTION ){ + sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q%s agg=%d[%d]/%p", + pExpr->op2, pExpr->u.zToken, zFlgs, + pExpr->pAggInfo ? pExpr->pAggInfo->selId : 0, + pExpr->iAgg, pExpr->pAggInfo); + }else if( pExpr->op2!=0 ){ + const char *zOp2; + char zBuf[8]; + sqlite3_snprintf(sizeof(zBuf),zBuf,"0x%02x",pExpr->op2); + zOp2 = zBuf; + if( pExpr->op2==NC_IsCheck ) zOp2 = "NC_IsCheck"; + if( pExpr->op2==NC_IdxExpr ) zOp2 = "NC_IdxExpr"; + if( pExpr->op2==NC_PartIdx ) zOp2 = "NC_PartIdx"; + if( pExpr->op2==NC_GenCol ) zOp2 = "NC_GenCol"; + sqlite3TreeViewLine(pView, "FUNCTION %Q%s op2=%s", + pExpr->u.zToken, zFlgs, zOp2); + }else{ + sqlite3TreeViewLine(pView, "FUNCTION %Q%s", pExpr->u.zToken, zFlgs); + } + if( pFarg ){ + sqlite3TreeViewExprList(pView, pFarg, pWin!=0, 0); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + sqlite3TreeViewWindow(pView, pWin, 0); + } +#endif + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: { + sqlite3TreeViewLine(pView, "EXISTS-expr flags=0x%x", pExpr->flags); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_SELECT: { + sqlite3TreeViewLine(pView, "subquery-expr flags=0x%x", pExpr->flags); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + break; + } + case TK_IN: { + sqlite3TreeViewLine(pView, "IN flags=0x%x", pExpr->flags); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); + }else{ + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + } + break; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + Expr *pX = pExpr->pLeft; + Expr *pY = pExpr->x.pList->a[0].pExpr; + Expr *pZ = pExpr->x.pList->a[1].pExpr; + sqlite3TreeViewLine(pView, "BETWEEN"); + sqlite3TreeViewExpr(pView, pX, 1); + sqlite3TreeViewExpr(pView, pY, 1); + sqlite3TreeViewExpr(pView, pZ, 0); + break; + } + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + */ + sqlite3TreeViewLine(pView, "%s(%d)", + pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); + break; + } + case TK_CASE: { + sqlite3TreeViewLine(pView, "CASE"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + const char *zType = "unk"; + switch( pExpr->affExpr ){ + case OE_Rollback: zType = "rollback"; break; + case OE_Abort: zType = "abort"; break; + case OE_Fail: zType = "fail"; break; + case OE_Ignore: zType = "ignore"; break; + } + sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); + break; + } +#endif + case TK_MATCH: { + sqlite3TreeViewLine(pView, "MATCH {%d:%d}%s", + pExpr->iTable, pExpr->iColumn, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); + break; + } + case TK_VECTOR: { + char *z = sqlite3_mprintf("VECTOR%s",zFlgs); + sqlite3TreeViewBareExprList(pView, pExpr->x.pList, z); + sqlite3_free(z); + break; + } + case TK_SELECT_COLUMN: { + sqlite3TreeViewLine(pView, "SELECT-COLUMN %d of [0..%d]%s", + pExpr->iColumn, pExpr->iTable-1, + pExpr->pRight==pExpr->pLeft ? " (SELECT-owner)" : ""); + sqlite3TreeViewSelect(pView, pExpr->pLeft->x.pSelect, 0); + break; + } + case TK_IF_NULL_ROW: { + sqlite3TreeViewLine(pView, "IF-NULL-ROW %d", pExpr->iTable); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + case TK_ERROR: { + Expr tmp; + sqlite3TreeViewLine(pView, "ERROR"); + tmp = *pExpr; + tmp.op = pExpr->op2; + sqlite3TreeViewExpr(pView, &tmp, 0); + break; + } + case TK_ROW: { + if( pExpr->iColumn<=0 ){ + sqlite3TreeViewLine(pView, "First FROM table rowid"); + }else{ + sqlite3TreeViewLine(pView, "First FROM table column %d", + pExpr->iColumn-1); + } + break; + } + default: { + sqlite3TreeViewLine(pView, "op=%d", pExpr->op); + break; + } + } + if( zBinOp ){ + sqlite3TreeViewLine(pView, "%s%s", zBinOp, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); + }else if( zUniOp ){ + sqlite3TreeViewLine(pView, "%s%s", zUniOp, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + } + sqlite3TreeViewPop(pView); +} + + +/* +** Generate a human-readable explanation of an expression list. +*/ +SQLITE_PRIVATE void sqlite3TreeViewBareExprList( + TreeView *pView, + const ExprList *pList, + const char *zLabel +){ + if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; + if( pList==0 ){ + sqlite3TreeViewLine(pView, "%s (empty)", zLabel); + }else{ + int i; + sqlite3TreeViewLine(pView, "%s", zLabel); + for(i=0; inExpr; i++){ + int j = pList->a[i].u.x.iOrderByCol; + char *zName = pList->a[i].zEName; + int moreToFollow = inExpr - 1; + if( pList->a[i].eEName!=ENAME_NAME ) zName = 0; + if( j || zName ){ + sqlite3TreeViewPush(pView, moreToFollow); + moreToFollow = 0; + sqlite3TreeViewLine(pView, 0); + if( zName ){ + fprintf(stdout, "AS %s ", zName); + } + if( j ){ + fprintf(stdout, "iOrderByCol=%d", j); + } + fprintf(stdout, "\n"); + fflush(stdout); + } + sqlite3TreeViewExpr(pView, pList->a[i].pExpr, moreToFollow); + if( j || zName ){ + sqlite3TreeViewPop(pView); + } + } + } +} +SQLITE_PRIVATE void sqlite3TreeViewExprList( + TreeView *pView, + const ExprList *pList, + u8 moreToFollow, + const char *zLabel +){ + pView = sqlite3TreeViewPush(pView, moreToFollow); + sqlite3TreeViewBareExprList(pView, pList, zLabel); + sqlite3TreeViewPop(pView); +} + +#endif /* SQLITE_DEBUG */ + +/************** End of treeview.c ********************************************/ +/************** Begin file random.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code to implement a pseudo-random number +** generator (PRNG) for SQLite. +** +** Random numbers are used by some of the database backends in order +** to generate random integer keys for tables or random filenames. +*/ +/* #include "sqliteInt.h" */ + + +/* All threads share a single random number generator. +** This structure is the current state of the generator. +*/ +static SQLITE_WSD struct sqlite3PrngType { + unsigned char isInit; /* True if initialized */ + unsigned char i, j; /* State variables */ + unsigned char s[256]; /* State variables */ +} sqlite3Prng; + +/* +** Return N random bytes. +*/ +SQLITE_API void sqlite3_randomness(int N, void *pBuf){ + unsigned char t; + unsigned char *zBuf = pBuf; + + /* The "wsdPrng" macro will resolve to the pseudo-random number generator + ** state vector. If writable static data is unsupported on the target, + ** we have to locate the state vector at run-time. In the more common + ** case where writable static data is supported, wsdPrng can refer directly + ** to the "sqlite3Prng" state vector declared above. + */ +#ifdef SQLITE_OMIT_WSD + struct sqlite3PrngType *p = &GLOBAL(struct sqlite3PrngType, sqlite3Prng); +# define wsdPrng p[0] +#else +# define wsdPrng sqlite3Prng +#endif + +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex; +#endif + +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return; +#endif + +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); +#endif + + sqlite3_mutex_enter(mutex); + if( N<=0 || pBuf==0 ){ + wsdPrng.isInit = 0; + sqlite3_mutex_leave(mutex); + return; + } + + /* Initialize the state of the random number generator once, + ** the first time this routine is called. The seed value does + ** not need to contain a lot of randomness since we are not + ** trying to do secure encryption or anything like that... + ** + ** Nothing in this file or anywhere else in SQLite does any kind of + ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random + ** number generator) not as an encryption device. + */ + if( !wsdPrng.isInit ){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + int i; + char k[256]; + wsdPrng.j = 0; + wsdPrng.i = 0; + if( NEVER(pVfs==0) ){ + memset(k, 0, sizeof(k)); + }else{ + sqlite3OsRandomness(pVfs, 256, k); + } + for(i=0; i<256; i++){ + wsdPrng.s[i] = (u8)i; + } + for(i=0; i<256; i++){ + wsdPrng.j += wsdPrng.s[i] + k[i]; + t = wsdPrng.s[wsdPrng.j]; + wsdPrng.s[wsdPrng.j] = wsdPrng.s[i]; + wsdPrng.s[i] = t; + } + wsdPrng.isInit = 1; + } + + assert( N>0 ); + do{ + wsdPrng.i++; + t = wsdPrng.s[wsdPrng.i]; + wsdPrng.j += t; + wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; + wsdPrng.s[wsdPrng.j] = t; + t += wsdPrng.s[wsdPrng.i]; + *(zBuf++) = wsdPrng.s[t]; + }while( --N ); + sqlite3_mutex_leave(mutex); +} + +#ifndef SQLITE_UNTESTABLE +/* +** For testing purposes, we sometimes want to preserve the state of +** PRNG and restore the PRNG to its saved state at a later time, or +** to reset the PRNG to its initial state. These routines accomplish +** those tasks. +** +** The sqlite3_test_control() interface calls these routines to +** control the PRNG. +*/ +static SQLITE_WSD struct sqlite3PrngType sqlite3SavedPrng; +SQLITE_PRIVATE void sqlite3PrngSaveState(void){ + memcpy( + &GLOBAL(struct sqlite3PrngType, sqlite3SavedPrng), + &GLOBAL(struct sqlite3PrngType, sqlite3Prng), + sizeof(sqlite3Prng) + ); +} +SQLITE_PRIVATE void sqlite3PrngRestoreState(void){ + memcpy( + &GLOBAL(struct sqlite3PrngType, sqlite3Prng), + &GLOBAL(struct sqlite3PrngType, sqlite3SavedPrng), + sizeof(sqlite3Prng) + ); +} +#endif /* SQLITE_UNTESTABLE */ + +/************** End of random.c **********************************************/ +/************** Begin file threads.c *****************************************/ +/* +** 2012 July 21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file presents a simple cross-platform threading interface for +** use internally by SQLite. +** +** A "thread" can be created using sqlite3ThreadCreate(). This thread +** runs independently of its creator until it is joined using +** sqlite3ThreadJoin(), at which point it terminates. +** +** Threads do not have to be real. It could be that the work of the +** "thread" is done by the main thread at either the sqlite3ThreadCreate() +** or sqlite3ThreadJoin() call. This is, in fact, what happens in +** single threaded systems. Nothing in SQLite requires multiple threads. +** This interface exists so that applications that want to take advantage +** of multiple cores can do so, while also allowing applications to stay +** single-threaded if desired. +*/ +/* #include "sqliteInt.h" */ +#if SQLITE_OS_WIN +/* # include "os_win.h" */ +#endif + +#if SQLITE_MAX_WORKER_THREADS>0 + +/********************************* Unix Pthreads ****************************/ +#if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) && SQLITE_THREADSAFE>0 + +#define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ +/* #include */ + +/* A running thread */ +struct SQLiteThread { + pthread_t tid; /* Thread ID */ + int done; /* Set to true when thread finishes */ + void *pOut; /* Result returned by the thread */ + void *(*xTask)(void*); /* The thread routine */ + void *pIn; /* Argument to the thread */ +}; + +/* Create a new thread */ +SQLITE_PRIVATE int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + int rc; + + assert( ppThread!=0 ); + assert( xTask!=0 ); + /* This routine is never used in single-threaded mode */ + assert( sqlite3GlobalConfig.bCoreMutex!=0 ); + + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM_BKPT; + memset(p, 0, sizeof(*p)); + p->xTask = xTask; + p->pIn = pIn; + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** for testing purposes. */ + if( sqlite3FaultSim(200) ){ + rc = 1; + }else{ + rc = pthread_create(&p->tid, 0, xTask, pIn); + } + if( rc ){ + p->done = 1; + p->pOut = xTask(pIn); + } + *ppThread = p; + return SQLITE_OK; +} + +/* Get the results of the thread */ +SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + int rc; + + assert( ppOut!=0 ); + if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; + if( p->done ){ + *ppOut = p->pOut; + rc = SQLITE_OK; + }else{ + rc = pthread_join(p->tid, ppOut) ? SQLITE_ERROR : SQLITE_OK; + } + sqlite3_free(p); + return rc; +} + +#endif /* SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) */ +/******************************** End Unix Pthreads *************************/ + + +/********************************* Win32 Threads ****************************/ +#if SQLITE_OS_WIN_THREADS + +#define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ +#include + +/* A running thread */ +struct SQLiteThread { + void *tid; /* The thread handle */ + unsigned id; /* The thread identifier */ + void *(*xTask)(void*); /* The routine to run as a thread */ + void *pIn; /* Argument to xTask */ + void *pResult; /* Result of xTask */ +}; + +/* Thread procedure Win32 compatibility shim */ +static unsigned __stdcall sqlite3ThreadProc( + void *pArg /* IN: Pointer to the SQLiteThread structure */ +){ + SQLiteThread *p = (SQLiteThread *)pArg; + + assert( p!=0 ); +#if 0 + /* + ** This assert appears to trigger spuriously on certain + ** versions of Windows, possibly due to _beginthreadex() + ** and/or CreateThread() not fully setting their thread + ** ID parameter before starting the thread. + */ + assert( p->id==GetCurrentThreadId() ); +#endif + assert( p->xTask!=0 ); + p->pResult = p->xTask(p->pIn); + + _endthreadex(0); + return 0; /* NOT REACHED */ +} + +/* Create a new thread */ +SQLITE_PRIVATE int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM_BKPT; + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** (via the sqlite3FaultSim() term of the conditional) for testing + ** purposes. */ + if( sqlite3GlobalConfig.bCoreMutex==0 || sqlite3FaultSim(200) ){ + memset(p, 0, sizeof(*p)); + }else{ + p->xTask = xTask; + p->pIn = pIn; + p->tid = (void*)_beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id); + if( p->tid==0 ){ + memset(p, 0, sizeof(*p)); + } + } + if( p->xTask==0 ){ + p->id = GetCurrentThreadId(); + p->pResult = xTask(pIn); + } + *ppThread = p; + return SQLITE_OK; +} + +SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject); /* os_win.c */ + +/* Get the results of the thread */ +SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + DWORD rc; + BOOL bRc; + + assert( ppOut!=0 ); + if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; + if( p->xTask==0 ){ + /* assert( p->id==GetCurrentThreadId() ); */ + rc = WAIT_OBJECT_0; + assert( p->tid==0 ); + }else{ + assert( p->id!=0 && p->id!=GetCurrentThreadId() ); + rc = sqlite3Win32Wait((HANDLE)p->tid); + assert( rc!=WAIT_IO_COMPLETION ); + bRc = CloseHandle((HANDLE)p->tid); + assert( bRc ); + } + if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; + sqlite3_free(p); + return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR; +} + +#endif /* SQLITE_OS_WIN_THREADS */ +/******************************** End Win32 Threads *************************/ + + +/********************************* Single-Threaded **************************/ +#ifndef SQLITE_THREADS_IMPLEMENTED +/* +** This implementation does not actually create a new thread. It does the +** work of the thread in the main thread, when either the thread is created +** or when it is joined +*/ + +/* A running thread */ +struct SQLiteThread { + void *(*xTask)(void*); /* The routine to run as a thread */ + void *pIn; /* Argument to xTask */ + void *pResult; /* Result of xTask */ +}; + +/* Create a new thread */ +SQLITE_PRIVATE int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM_BKPT; + if( (SQLITE_PTR_TO_INT(p)/17)&1 ){ + p->xTask = xTask; + p->pIn = pIn; + }else{ + p->xTask = 0; + p->pResult = xTask(pIn); + } + *ppThread = p; + return SQLITE_OK; +} + +/* Get the results of the thread */ +SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + + assert( ppOut!=0 ); + if( NEVER(p==0) ) return SQLITE_NOMEM_BKPT; + if( p->xTask ){ + *ppOut = p->xTask(p->pIn); + }else{ + *ppOut = p->pResult; + } + sqlite3_free(p); + +#if defined(SQLITE_TEST) + { + void *pTstAlloc = sqlite3Malloc(10); + if (!pTstAlloc) return SQLITE_NOMEM_BKPT; + sqlite3_free(pTstAlloc); + } +#endif + + return SQLITE_OK; +} + +#endif /* !defined(SQLITE_THREADS_IMPLEMENTED) */ +/****************************** End Single-Threaded *************************/ +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ + +/************** End of threads.c *********************************************/ +/************** Begin file utf.c *********************************************/ +/* +** 2004 April 13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains routines used to translate between UTF-8, +** UTF-16, UTF-16BE, and UTF-16LE. +** +** Notes on UTF-8: +** +** Byte-0 Byte-1 Byte-2 Byte-3 Value +** 0xxxxxxx 00000000 00000000 0xxxxxxx +** 110yyyyy 10xxxxxx 00000000 00000yyy yyxxxxxx +** 1110zzzz 10yyyyyy 10xxxxxx 00000000 zzzzyyyy yyxxxxxx +** 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx 000uuuuu zzzzyyyy yyxxxxxx +** +** +** Notes on UTF-16: (with wwww+1==uuuuu) +** +** Word-0 Word-1 Value +** 110110ww wwzzzzyy 110111yy yyxxxxxx 000uuuuu zzzzyyyy yyxxxxxx +** zzzzyyyy yyxxxxxx 00000000 zzzzyyyy yyxxxxxx +** +** +** BOM or Byte Order Mark: +** 0xff 0xfe little-endian utf-16 follows +** 0xfe 0xff big-endian utf-16 follows +** +*/ +/* #include "sqliteInt.h" */ +/* #include */ +/* #include "vdbeInt.h" */ + +#if !defined(SQLITE_AMALGAMATION) && SQLITE_BYTEORDER==0 +/* +** The following constant value is used by the SQLITE_BIGENDIAN and +** SQLITE_LITTLEENDIAN macros. +*/ +SQLITE_PRIVATE const int sqlite3one = 1; +#endif /* SQLITE_AMALGAMATION && SQLITE_BYTEORDER==0 */ + +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. +*/ +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + + +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (u8)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ +} + +#define WRITE_UTF16LE(zOut, c) { \ + if( c<=0xFFFF ){ \ + *zOut++ = (u8)(c&0x00FF); \ + *zOut++ = (u8)((c>>8)&0x00FF); \ + }else{ \ + *zOut++ = (u8)(((c>>10)&0x003F) + (((c-0x10000)>>10)&0x00C0)); \ + *zOut++ = (u8)(0x00D8 + (((c-0x10000)>>18)&0x03)); \ + *zOut++ = (u8)(c&0x00FF); \ + *zOut++ = (u8)(0x00DC + ((c>>8)&0x03)); \ + } \ +} + +#define WRITE_UTF16BE(zOut, c) { \ + if( c<=0xFFFF ){ \ + *zOut++ = (u8)((c>>8)&0x00FF); \ + *zOut++ = (u8)(c&0x00FF); \ + }else{ \ + *zOut++ = (u8)(0x00D8 + (((c-0x10000)>>18)&0x03)); \ + *zOut++ = (u8)(((c>>10)&0x003F) + (((c-0x10000)>>10)&0x00C0)); \ + *zOut++ = (u8)(0x00DC + ((c>>8)&0x03)); \ + *zOut++ = (u8)(c&0x00FF); \ + } \ +} + +/* +** Translate a single UTF-8 character. Return the unicode value. +** +** During translation, assume that the byte that zTerm points +** is a 0x00. +** +** Write a pointer to the next unread byte back into *pzNext. +** +** Notes On Invalid UTF-8: +** +** * This routine never allows a 7-bit character (0x00 through 0x7f) to +** be encoded as a multi-byte character. Any multi-byte character that +** attempts to encode a value between 0x00 and 0x7f is rendered as 0xfffd. +** +** * This routine never allows a UTF16 surrogate value to be encoded. +** If a multi-byte character attempts to encode a value between +** 0xd800 and 0xe000 then it is rendered as 0xfffd. +** +** * Bytes in the range of 0x80 through 0xbf which occur as the first +** byte of a character are interpreted as single-byte characters +** and rendered as themselves even though they are technically +** invalid characters. +** +** * This routine accepts over-length UTF8 encodings +** for unicode values 0x80 and greater. It does not change over-length +** encodings to 0xfffd as some systems recommend. +*/ +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } +SQLITE_PRIVATE u32 sqlite3Utf8Read( + const unsigned char **pz /* Pointer to string from which to read char */ +){ + unsigned int c; + + /* Same as READ_UTF8() above but without the zTerm parameter. + ** For this routine, we assume the UTF8 string is always zero-terminated. + */ + c = *((*pz)++); + if( c>=0xc0 ){ + c = sqlite3Utf8Trans1[c-0xc0]; + while( (*(*pz) & 0xc0)==0x80 ){ + c = (c<<6) + (0x3f & *((*pz)++)); + } + if( c<0x80 + || (c&0xFFFFF800)==0xD800 + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } + } + return c; +} + + + + +/* +** If the TRANSLATE_TRACE macro is defined, the value of each Mem is +** printed on stderr on the way into and out of sqlite3VdbeMemTranslate(). +*/ +/* #define TRANSLATE_TRACE 1 */ + +#ifndef SQLITE_OMIT_UTF16 +/* +** This routine transforms the internal text encoding used by pMem to +** desiredEnc. It is an error if the string is already of the desired +** encoding, or if *pMem does not contain a string value. +*/ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ + sqlite3_int64 len; /* Maximum length of output string in bytes */ + unsigned char *zOut; /* Output buffer */ + unsigned char *zIn; /* Input iterator */ + unsigned char *zTerm; /* End of input */ + unsigned char *z; /* Output iterator */ + unsigned int c; + + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( pMem->flags&MEM_Str ); + assert( pMem->enc!=desiredEnc ); + assert( pMem->enc!=0 ); + assert( pMem->n>=0 ); + +#if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) + { + StrAccum acc; + char zBuf[1000]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3VdbeMemPrettyPrint(pMem, &acc); + fprintf(stderr, "INPUT: %s\n", sqlite3StrAccumFinish(&acc)); + } +#endif + + /* If the translation is between UTF-16 little and big endian, then + ** all that is required is to swap the byte order. This case is handled + ** differently from the others. + */ + if( pMem->enc!=SQLITE_UTF8 && desiredEnc!=SQLITE_UTF8 ){ + u8 temp; + int rc; + rc = sqlite3VdbeMemMakeWriteable(pMem); + if( rc!=SQLITE_OK ){ + assert( rc==SQLITE_NOMEM ); + return SQLITE_NOMEM_BKPT; + } + zIn = (u8*)pMem->z; + zTerm = &zIn[pMem->n&~1]; + while( zInenc = desiredEnc; + goto translate_out; + } + + /* Set len to the maximum number of bytes required in the output buffer. */ + if( desiredEnc==SQLITE_UTF8 ){ + /* When converting from UTF-16, the maximum growth results from + ** translating a 2-byte character to a 4-byte UTF-8 character. + ** A single byte is required for the output string + ** nul-terminator. + */ + pMem->n &= ~1; + len = 2 * (sqlite3_int64)pMem->n + 1; + }else{ + /* When converting from UTF-8 to UTF-16 the maximum growth is caused + ** when a 1-byte UTF-8 character is translated into a 2-byte UTF-16 + ** character. Two bytes are required in the output buffer for the + ** nul-terminator. + */ + len = 2 * (sqlite3_int64)pMem->n + 2; + } + + /* Set zIn to point at the start of the input buffer and zTerm to point 1 + ** byte past the end. + ** + ** Variable zOut is set to point at the output buffer, space obtained + ** from sqlite3_malloc(). + */ + zIn = (u8*)pMem->z; + zTerm = &zIn[pMem->n]; + zOut = sqlite3DbMallocRaw(pMem->db, len); + if( !zOut ){ + return SQLITE_NOMEM_BKPT; + } + z = zOut; + + if( pMem->enc==SQLITE_UTF8 ){ + if( desiredEnc==SQLITE_UTF16LE ){ + /* UTF-8 -> UTF-16 Little-endian */ + while( zIn UTF-16 Big-endian */ + while( zInn = (int)(z - zOut); + *z++ = 0; + }else{ + assert( desiredEnc==SQLITE_UTF8 ); + if( pMem->enc==SQLITE_UTF16LE ){ + /* UTF-16 Little-endian -> UTF-8 */ + while( zIn=0xd800 && c<0xe000 ){ +#ifdef SQLITE_REPLACE_INVALID_UTF + if( c>=0xdc00 || zIn>=zTerm ){ + c = 0xfffd; + }else{ + int c2 = *(zIn++); + c2 += (*(zIn++))<<8; + if( c2<0xdc00 || c2>=0xe000 ){ + zIn -= 2; + c = 0xfffd; + }else{ + c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; + } + } +#else + if( zIn UTF-8 */ + while( zIn=0xd800 && c<0xe000 ){ +#ifdef SQLITE_REPLACE_INVALID_UTF + if( c>=0xdc00 || zIn>=zTerm ){ + c = 0xfffd; + }else{ + int c2 = (*(zIn++))<<8; + c2 += *(zIn++); + if( c2<0xdc00 || c2>=0xe000 ){ + zIn -= 2; + c = 0xfffd; + }else{ + c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; + } + } +#else + if( zInn = (int)(z - zOut); + } + *z = 0; + assert( (pMem->n+(desiredEnc==SQLITE_UTF8?1:2))<=len ); + + c = MEM_Str|MEM_Term|(pMem->flags&(MEM_AffMask|MEM_Subtype)); + sqlite3VdbeMemRelease(pMem); + pMem->flags = c; + pMem->enc = desiredEnc; + pMem->z = (char*)zOut; + pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->z); + +translate_out: +#if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) + { + StrAccum acc; + char zBuf[1000]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3VdbeMemPrettyPrint(pMem, &acc); + fprintf(stderr, "OUTPUT: %s\n", sqlite3StrAccumFinish(&acc)); + } +#endif + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_UTF16 */ + +#ifndef SQLITE_OMIT_UTF16 +/* +** This routine checks for a byte-order mark at the beginning of the +** UTF-16 string stored in *pMem. If one is present, it is removed and +** the encoding of the Mem adjusted. This routine does not do any +** byte-swapping, it just sets Mem.enc appropriately. +** +** The allocation (static, dynamic etc.) and encoding of the Mem may be +** changed by this function. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem){ + int rc = SQLITE_OK; + u8 bom = 0; + + assert( pMem->n>=0 ); + if( pMem->n>1 ){ + u8 b1 = *(u8 *)pMem->z; + u8 b2 = *(((u8 *)pMem->z) + 1); + if( b1==0xFE && b2==0xFF ){ + bom = SQLITE_UTF16BE; + } + if( b1==0xFF && b2==0xFE ){ + bom = SQLITE_UTF16LE; + } + } + + if( bom ){ + rc = sqlite3VdbeMemMakeWriteable(pMem); + if( rc==SQLITE_OK ){ + pMem->n -= 2; + memmove(pMem->z, &pMem->z[2], pMem->n); + pMem->z[pMem->n] = '\0'; + pMem->z[pMem->n+1] = '\0'; + pMem->flags |= MEM_Term; + pMem->enc = bom; + } + } + return rc; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** pZ is a UTF-8 encoded unicode string. If nByte is less than zero, +** return the number of unicode characters in pZ up to (but not including) +** the first 0x00 byte. If nByte is not less than zero, return the +** number of unicode characters in the first nByte of pZ (or up to +** the first 0x00, whichever comes first). +*/ +SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *zIn, int nByte){ + int r = 0; + const u8 *z = (const u8*)zIn; + const u8 *zTerm; + if( nByte>=0 ){ + zTerm = &z[nByte]; + }else{ + zTerm = (const u8*)(-1); + } + assert( z<=zTerm ); + while( *z!=0 && zmallocFailed ){ + sqlite3VdbeMemRelease(&m); + m.z = 0; + } + assert( (m.flags & MEM_Term)!=0 || db->mallocFailed ); + assert( (m.flags & MEM_Str)!=0 || db->mallocFailed ); + assert( m.z || db->mallocFailed ); + return m.z; +} + +/* +** zIn is a UTF-16 encoded unicode string at least nChar characters long. +** Return the number of bytes in the first nChar unicode characters +** in pZ. nChar must be non-negative. +*/ +SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *zIn, int nChar){ + int c; + unsigned char const *z = zIn; + int n = 0; + + if( SQLITE_UTF16NATIVE==SQLITE_UTF16LE ) z++; + while( n=0xd8 && c<0xdc && z[0]>=0xdc && z[0]<0xe0 ) z += 2; + n++; + } + return (int)(z-(unsigned char const *)zIn) + - (SQLITE_UTF16NATIVE==SQLITE_UTF16LE); +} + +#if defined(SQLITE_TEST) +/* +** This routine is called from the TCL test function "translate_selftest". +** It checks that the primitives for serializing and deserializing +** characters in each encoding are inverses of each other. +*/ +SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ + unsigned int i, t; + unsigned char zBuf[20]; + unsigned char *z; + int n; + unsigned int c; + + for(i=0; i<0x00110000; i++){ + z = zBuf; + WRITE_UTF8(z, i); + n = (int)(z-zBuf); + assert( n>0 && n<=4 ); + z[0] = 0; + z = zBuf; + c = sqlite3Utf8Read((const u8**)&z); + t = i; + if( i>=0xD800 && i<=0xDFFF ) t = 0xFFFD; + if( (i&0xFFFFFFFE)==0xFFFE ) t = 0xFFFD; + assert( c==t ); + assert( (z-zBuf)==n ); + } +} +#endif /* SQLITE_TEST */ +#endif /* SQLITE_OMIT_UTF16 */ + +/************** End of utf.c *************************************************/ +/************** Begin file util.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Utility functions used throughout sqlite. +** +** This file contains functions for allocating memory, comparing +** strings, and stuff like that. +** +*/ +/* #include "sqliteInt.h" */ +/* #include */ +#ifndef SQLITE_OMIT_FLOATING_POINT +#include +#endif + +/* +** Routine needed to support the testcase() macro. +*/ +#ifdef SQLITE_COVERAGE_TEST +SQLITE_PRIVATE void sqlite3Coverage(int x){ + static unsigned dummy = 0; + dummy += (unsigned)x; +} +#endif + +/* +** Calls to sqlite3FaultSim() are used to simulate a failure during testing, +** or to bypass normal error detection during testing in order to let +** execute proceed futher downstream. +** +** In deployment, sqlite3FaultSim() *always* return SQLITE_OK (0). The +** sqlite3FaultSim() function only returns non-zero during testing. +** +** During testing, if the test harness has set a fault-sim callback using +** a call to sqlite3_test_control(SQLITE_TESTCTRL_FAULT_INSTALL), then +** each call to sqlite3FaultSim() is relayed to that application-supplied +** callback and the integer return value form the application-supplied +** callback is returned by sqlite3FaultSim(). +** +** The integer argument to sqlite3FaultSim() is a code to identify which +** sqlite3FaultSim() instance is being invoked. Each call to sqlite3FaultSim() +** should have a unique code. To prevent legacy testing applications from +** breaking, the codes should not be changed or reused. +*/ +#ifndef SQLITE_UNTESTABLE +SQLITE_PRIVATE int sqlite3FaultSim(int iTest){ + int (*xCallback)(int) = sqlite3GlobalConfig.xTestCallback; + return xCallback ? xCallback(iTest) : SQLITE_OK; +} +#endif + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Return true if the floating point value is Not a Number (NaN). +** +** Use the math library isnan() function if compiled with SQLITE_HAVE_ISNAN. +** Otherwise, we have our own implementation that works on most systems. +*/ +SQLITE_PRIVATE int sqlite3IsNaN(double x){ + int rc; /* The value return */ +#if !SQLITE_HAVE_ISNAN && !HAVE_ISNAN + u64 y; + memcpy(&y,&x,sizeof(y)); + rc = IsNaN(y); +#else + rc = isnan(x); +#endif /* HAVE_ISNAN */ + testcase( rc ); + return rc; +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + +/* +** Compute a string length that is limited to what can be stored in +** lower 30 bits of a 32-bit signed integer. +** +** The value returned will never be negative. Nor will it ever be greater +** than the actual length of the string. For very long strings (greater +** than 1GiB) the value returned might be less than the true string length. +*/ +SQLITE_PRIVATE int sqlite3Strlen30(const char *z){ + if( z==0 ) return 0; + return 0x3fffffff & (int)strlen(z); +} + +/* +** Return the declared type of a column. Or return zDflt if the column +** has no declared type. +** +** The column type is an extra string stored after the zero-terminator on +** the column name if and only if the COLFLAG_HASTYPE flag is set. +*/ +SQLITE_PRIVATE char *sqlite3ColumnType(Column *pCol, char *zDflt){ + if( pCol->colFlags & COLFLAG_HASTYPE ){ + return pCol->zCnName + strlen(pCol->zCnName) + 1; + }else if( pCol->eCType ){ + assert( pCol->eCType<=SQLITE_N_STDTYPE ); + return (char*)sqlite3StdType[pCol->eCType-1]; + }else{ + return zDflt; + } +} + +/* +** Helper function for sqlite3Error() - called rarely. Broken out into +** a separate routine to avoid unnecessary register saves on entry to +** sqlite3Error(). +*/ +static SQLITE_NOINLINE void sqlite3ErrorFinish(sqlite3 *db, int err_code){ + if( db->pErr ) sqlite3ValueSetNull(db->pErr); + sqlite3SystemError(db, err_code); +} + +/* +** Set the current error code to err_code and clear any prior error message. +** Also set iSysErrno (by calling sqlite3System) if the err_code indicates +** that would be appropriate. +*/ +SQLITE_PRIVATE void sqlite3Error(sqlite3 *db, int err_code){ + assert( db!=0 ); + db->errCode = err_code; + if( err_code || db->pErr ) sqlite3ErrorFinish(db, err_code); +} + +/* +** The equivalent of sqlite3Error(db, SQLITE_OK). Clear the error state +** and error message. +*/ +SQLITE_PRIVATE void sqlite3ErrorClear(sqlite3 *db){ + assert( db!=0 ); + db->errCode = SQLITE_OK; + if( db->pErr ) sqlite3ValueSetNull(db->pErr); +} + +/* +** Load the sqlite3.iSysErrno field if that is an appropriate thing +** to do based on the SQLite error code in rc. +*/ +SQLITE_PRIVATE void sqlite3SystemError(sqlite3 *db, int rc){ + if( rc==SQLITE_IOERR_NOMEM ) return; + rc &= 0xff; + if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){ + db->iSysErrno = sqlite3OsGetLastError(db->pVfs); + } +} + +/* +** Set the most recent error code and error string for the sqlite +** handle "db". The error code is set to "err_code". +** +** If it is not NULL, string zFormat specifies the format of the +** error string in the style of the printf functions: The following +** format characters are allowed: +** +** %s Insert a string +** %z A string that should be freed after use +** %d Insert an integer +** %T Insert a token +** %S Insert the first element of a SrcList +** +** zFormat and any string tokens that follow it are assumed to be +** encoded in UTF-8. +** +** To clear the most recent error for sqlite handle "db", sqlite3Error +** should be called with err_code set to SQLITE_OK and zFormat set +** to NULL. +*/ +SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *zFormat, ...){ + assert( db!=0 ); + db->errCode = err_code; + sqlite3SystemError(db, err_code); + if( zFormat==0 ){ + sqlite3Error(db, err_code); + }else if( db->pErr || (db->pErr = sqlite3ValueNew(db))!=0 ){ + char *z; + va_list ap; + va_start(ap, zFormat); + z = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + sqlite3ValueSetStr(db->pErr, -1, z, SQLITE_UTF8, SQLITE_DYNAMIC); + } +} + +/* +** Add an error message to pParse->zErrMsg and increment pParse->nErr. +** The following formatting characters are allowed: +** +** %s Insert a string +** %z A string that should be freed after use +** %d Insert an integer +** %T Insert a token +** %S Insert the first element of a SrcList +** +** This function should be used to report any error that occurs while +** compiling an SQL statement (i.e. within sqlite3_prepare()). The +** last thing the sqlite3_prepare() function does is copy the error +** stored by this function into the database handle using sqlite3Error(). +** Functions sqlite3Error() or sqlite3ErrorWithMsg() should be used +** during statement execution (sqlite3_step() etc.). +*/ +SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ + char *zMsg; + va_list ap; + sqlite3 *db = pParse->db; + va_start(ap, zFormat); + zMsg = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + if( db->suppressErr ){ + sqlite3DbFree(db, zMsg); + }else{ + pParse->nErr++; + sqlite3DbFree(db, pParse->zErrMsg); + pParse->zErrMsg = zMsg; + pParse->rc = SQLITE_ERROR; + pParse->pWith = 0; + } +} + +/* +** If database connection db is currently parsing SQL, then transfer +** error code errCode to that parser if the parser has not already +** encountered some other kind of error. +*/ +SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3 *db, int errCode){ + Parse *pParse; + if( db==0 || (pParse = db->pParse)==0 ) return errCode; + pParse->rc = errCode; + pParse->nErr++; + return errCode; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** The input string must be zero-terminated. A new zero-terminator +** is added to the dequoted string. +** +** The return value is -1 if no dequoting occurs or the length of the +** dequoted string, exclusive of the zero terminator, if dequoting does +** occur. +** +** 2002-02-14: This routine is extended to remove MS-Access style +** brackets from around identifiers. For example: "[a-b-c]" becomes +** "a-b-c". +*/ +SQLITE_PRIVATE void sqlite3Dequote(char *z){ + char quote; + int i, j; + if( z==0 ) return; + quote = z[0]; + if( !sqlite3Isquote(quote) ) return; + if( quote=='[' ) quote = ']'; + for(i=1, j=0;; i++){ + assert( z[i] ); + if( z[i]==quote ){ + if( z[i+1]==quote ){ + z[j++] = quote; + i++; + }else{ + break; + } + }else{ + z[j++] = z[i]; + } + } + z[j] = 0; +} +SQLITE_PRIVATE void sqlite3DequoteExpr(Expr *p){ + assert( sqlite3Isquote(p->u.zToken[0]) ); + p->flags |= p->u.zToken[0]=='"' ? EP_Quoted|EP_DblQuoted : EP_Quoted; + sqlite3Dequote(p->u.zToken); +} + +/* +** If the input token p is quoted, try to adjust the token to remove +** the quotes. This is not always possible: +** +** "abc" -> abc +** "ab""cd" -> (not possible because of the interior "") +** +** Remove the quotes if possible. This is a optimization. The overall +** system should still return the correct answer even if this routine +** is always a no-op. +*/ +SQLITE_PRIVATE void sqlite3DequoteToken(Token *p){ + unsigned int i; + if( p->n<2 ) return; + if( !sqlite3Isquote(p->z[0]) ) return; + for(i=1; in-1; i++){ + if( sqlite3Isquote(p->z[i]) ) return; + } + p->n -= 2; + p->z++; +} + +/* +** Generate a Token object from a string +*/ +SQLITE_PRIVATE void sqlite3TokenInit(Token *p, char *z){ + p->z = z; + p->n = sqlite3Strlen30(z); +} + +/* Convenient short-hand */ +#define UpperToLower sqlite3UpperToLower + +/* +** Some systems have stricmp(). Others have strcasecmp(). Because +** there is no consistency, we will define our own. +** +** IMPLEMENTATION-OF: R-30243-02494 The sqlite3_stricmp() and +** sqlite3_strnicmp() APIs allow applications and extensions to compare +** the contents of two buffers containing UTF-8 strings in a +** case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. +*/ +SQLITE_API int sqlite3_stricmp(const char *zLeft, const char *zRight){ + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } + return sqlite3StrICmp(zLeft, zRight); +} +SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ + unsigned char *a, *b; + int c, x; + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + for(;;){ + c = *a; + x = *b; + if( c==x ){ + if( c==0 ) break; + }else{ + c = (int)UpperToLower[c] - (int)UpperToLower[x]; + if( c ) break; + } + a++; + b++; + } + return c; +} +SQLITE_API int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ + register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } + a = (unsigned char *)zLeft; + b = (unsigned char *)zRight; + while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } + return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; +} + +/* +** Compute an 8-bit hash on a string that is insensitive to case differences +*/ +SQLITE_PRIVATE u8 sqlite3StrIHash(const char *z){ + u8 h = 0; + if( z==0 ) return 0; + while( z[0] ){ + h += UpperToLower[(unsigned char)z[0]]; + z++; + } + return h; +} + +/* +** Compute 10 to the E-th power. Examples: E==1 results in 10. +** E==2 results in 100. E==50 results in 1.0e50. +** +** This routine only works for values of E between 1 and 341. +*/ +static LONGDOUBLE_TYPE sqlite3Pow10(int E){ +#if defined(_MSC_VER) + static const LONGDOUBLE_TYPE x[] = { + 1.0e+001L, + 1.0e+002L, + 1.0e+004L, + 1.0e+008L, + 1.0e+016L, + 1.0e+032L, + 1.0e+064L, + 1.0e+128L, + 1.0e+256L + }; + LONGDOUBLE_TYPE r = 1.0; + int i; + assert( E>=0 && E<=307 ); + for(i=0; E!=0; i++, E >>=1){ + if( E & 1 ) r *= x[i]; + } + return r; +#else + LONGDOUBLE_TYPE x = 10.0; + LONGDOUBLE_TYPE r = 1.0; + while(1){ + if( E & 1 ) r *= x; + E >>= 1; + if( E==0 ) break; + x *= x; + } + return r; +#endif +} + +/* +** The string z[] is an text representation of a real number. +** Convert this string to a double and write it into *pResult. +** +** The string z[] is length bytes in length (bytes, not characters) and +** uses the encoding enc. The string is not necessarily zero-terminated. +** +** Return TRUE if the result is a valid real number (or integer) and FALSE +** if the string is empty or contains extraneous text. More specifically +** return +** 1 => The input string is a pure integer +** 2 or more => The input has a decimal point or eNNN clause +** 0 or less => The input string is not a valid number +** -1 => Not a valid number, but has a valid prefix which +** includes a decimal point and/or an eNNN clause +** +** Valid numbers are in one of these formats: +** +** [+-]digits[E[+-]digits] +** [+-]digits.[digits][E[+-]digits] +** [+-].digits[E[+-]digits] +** +** Leading and trailing whitespace is ignored for the purpose of determining +** validity. +** +** If some prefix of the input string is a valid number, this routine +** returns FALSE but it still converts the prefix and writes the result +** into *pResult. +*/ +#if defined(_MSC_VER) +#pragma warning(disable : 4756) +#endif +SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ +#ifndef SQLITE_OMIT_FLOATING_POINT + int incr; + const char *zEnd; + /* sign * significand * (10 ^ (esign * exponent)) */ + int sign = 1; /* sign of significand */ + i64 s = 0; /* significand */ + int d = 0; /* adjust exponent for shifting decimal point */ + int esign = 1; /* sign of exponent */ + int e = 0; /* exponent */ + int eValid = 1; /* True exponent is either not used or is well-formed */ + double result; + int nDigit = 0; /* Number of digits processed */ + int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ + + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + *pResult = 0.0; /* Default return value, in case of an error */ + if( length==0 ) return 0; + + if( enc==SQLITE_UTF8 ){ + incr = 1; + zEnd = z + length; + }else{ + int i; + incr = 2; + length &= ~1; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + testcase( enc==SQLITE_UTF16LE ); + testcase( enc==SQLITE_UTF16BE ); + for(i=3-enc; i=zEnd ) return 0; + + /* get sign of significand */ + if( *z=='-' ){ + sign = -1; + z+=incr; + }else if( *z=='+' ){ + z+=incr; + } + + /* copy max significant digits to significand */ + while( z=((LARGEST_INT64-9)/10) ){ + /* skip non-significant significand digits + ** (increase exponent by d to shift decimal left) */ + while( z=zEnd ) goto do_atof_calc; + + /* if decimal point is present */ + if( *z=='.' ){ + z+=incr; + eType++; + /* copy digits from after decimal to significand + ** (decrease exponent by d to shift decimal right) */ + while( z=zEnd ) goto do_atof_calc; + + /* if exponent is present */ + if( *z=='e' || *z=='E' ){ + z+=incr; + eValid = 0; + eType++; + + /* This branch is needed to avoid a (harmless) buffer overread. The + ** special comment alerts the mutation tester that the correct answer + ** is obtained even if the branch is omitted */ + if( z>=zEnd ) goto do_atof_calc; /*PREVENTS-HARMLESS-OVERREAD*/ + + /* get sign of exponent */ + if( *z=='-' ){ + esign = -1; + z+=incr; + }else if( *z=='+' ){ + z+=incr; + } + /* copy digits to exponent */ + while( z0 ){ /*OPTIMIZATION-IF-TRUE*/ + if( esign>0 ){ + if( s>=(LARGEST_INT64/10) ) break; /*OPTIMIZATION-IF-FALSE*/ + s *= 10; + }else{ + if( s%10!=0 ) break; /*OPTIMIZATION-IF-FALSE*/ + s /= 10; + } + e--; + } + + /* adjust the sign of significand */ + s = sign<0 ? -s : s; + + if( e==0 ){ /*OPTIMIZATION-IF-TRUE*/ + result = (double)s; + }else{ + /* attempt to handle extremely small/large numbers better */ + if( e>307 ){ /*OPTIMIZATION-IF-TRUE*/ + if( e<342 ){ /*OPTIMIZATION-IF-TRUE*/ + LONGDOUBLE_TYPE scale = sqlite3Pow10(e-308); + if( esign<0 ){ + result = s / scale; + result /= 1.0e+308; + }else{ + result = s * scale; + result *= 1.0e+308; + } + }else{ assert( e>=342 ); + if( esign<0 ){ + result = 0.0*s; + }else{ +#ifdef INFINITY + result = INFINITY*s; +#else + result = 1e308*1e308*s; /* Infinity */ +#endif + } + } + }else{ + LONGDOUBLE_TYPE scale = sqlite3Pow10(e); + if( esign<0 ){ + result = s / scale; + }else{ + result = s * scale; + } + } + } + } + + /* store the result */ + *pResult = result; + + /* return true if number and no extra non-whitespace chracters after */ + if( z==zEnd && nDigit>0 && eValid && eType>0 ){ + return eType; + }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ + return -1; + }else{ + return 0; + } +#else + return !sqlite3Atoi64(z, pResult, length, enc); +#endif /* SQLITE_OMIT_FLOATING_POINT */ +} +#if defined(_MSC_VER) +#pragma warning(default : 4756) +#endif + +/* +** Render an signed 64-bit integer as text. Store the result in zOut[]. +** +** The caller must ensure that zOut[] is at least 21 bytes in size. +*/ +SQLITE_PRIVATE void sqlite3Int64ToText(i64 v, char *zOut){ + int i; + u64 x; + char zTemp[22]; + if( v<0 ){ + x = (v==SMALLEST_INT64) ? ((u64)1)<<63 : (u64)-v; + }else{ + x = v; + } + i = sizeof(zTemp)-2; + zTemp[sizeof(zTemp)-1] = 0; + do{ + zTemp[i--] = (x%10) + '0'; + x = x/10; + }while( x ); + if( v<0 ) zTemp[i--] = '-'; + memcpy(zOut, &zTemp[i+1], sizeof(zTemp)-1-i); +} + +/* +** Compare the 19-character string zNum against the text representation +** value 2^63: 9223372036854775808. Return negative, zero, or positive +** if zNum is less than, equal to, or greater than the string. +** Note that zNum must contain exactly 19 characters. +** +** Unlike memcmp() this routine is guaranteed to return the difference +** in the values of the last digit if the only difference is in the +** last digit. So, for example, +** +** compare2pow63("9223372036854775800", 1) +** +** will return -8. +*/ +static int compare2pow63(const char *zNum, int incr){ + int c = 0; + int i; + /* 012345678901234567 */ + const char *pow63 = "922337203685477580"; + for(i=0; c==0 && i<18; i++){ + c = (zNum[i*incr]-pow63[i])*10; + } + if( c==0 ){ + c = zNum[18*incr] - '8'; + testcase( c==(-1) ); + testcase( c==0 ); + testcase( c==(+1) ); + } + return c; +} + +/* +** Convert zNum to a 64-bit signed integer. zNum must be decimal. This +** routine does *not* accept hexadecimal notation. +** +** Returns: +** +** -1 Not even a prefix of the input text looks like an integer +** 0 Successful transformation. Fits in a 64-bit signed integer. +** 1 Excess non-space text after the integer value +** 2 Integer too large for a 64-bit signed integer or is malformed +** 3 Special case of 9223372036854775808 +** +** length is the number of bytes in the string (bytes, not characters). +** The string is not necessarily zero-terminated. The encoding is +** given by enc. +*/ +SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ + int incr; + u64 u = 0; + int neg = 0; /* assume positive */ + int i; + int c = 0; + int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ + int rc; /* Baseline return code */ + const char *zStart; + const char *zEnd = zNum + length; + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + if( enc==SQLITE_UTF8 ){ + incr = 1; + }else{ + incr = 2; + length &= ~1; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + for(i=3-enc; i='0' && c<='9'; i+=incr){ + u = u*10 + c - '0'; + } + testcase( i==18*incr ); + testcase( i==19*incr ); + testcase( i==20*incr ); + if( u>LARGEST_INT64 ){ + /* This test and assignment is needed only to suppress UB warnings + ** from clang and -fsanitize=undefined. This test and assignment make + ** the code a little larger and slower, and no harm comes from omitting + ** them, but we must appaise the undefined-behavior pharisees. */ + *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; + }else if( neg ){ + *pNum = -(i64)u; + }else{ + *pNum = (i64)u; + } + rc = 0; + if( i==0 && zStart==zNum ){ /* No digits */ + rc = -1; + }else if( nonNum ){ /* UTF16 with high-order bytes non-zero */ + rc = 1; + }else if( &zNum[i]19*incr ? 1 : compare2pow63(zNum, incr); + if( c<0 ){ + /* zNum is less than 9223372036854775808 so it fits */ + assert( u<=LARGEST_INT64 ); + return rc; + }else{ + *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; + if( c>0 ){ + /* zNum is greater than 9223372036854775808 so it overflows */ + return 2; + }else{ + /* zNum is exactly 9223372036854775808. Fits if negative. The + ** special case 2 overflow if positive */ + assert( u-1==LARGEST_INT64 ); + return neg ? rc : 3; + } + } + } +} + +/* +** Transform a UTF-8 integer literal, in either decimal or hexadecimal, +** into a 64-bit signed integer. This routine accepts hexadecimal literals, +** whereas sqlite3Atoi64() does not. +** +** Returns: +** +** 0 Successful transformation. Fits in a 64-bit signed integer. +** 1 Excess text after the integer value +** 2 Integer too large for a 64-bit signed integer or is malformed +** 3 Special case of 9223372036854775808 +*/ +SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' + && (z[1]=='x' || z[1]=='X') + ){ + u64 u = 0; + int i, k; + for(i=2; z[i]=='0'; i++){} + for(k=i; sqlite3Isxdigit(z[k]); k++){ + u = u*16 + sqlite3HexToInt(z[k]); + } + memcpy(pOut, &u, 8); + return (z[k]==0 && k-i<=16) ? 0 : 2; + }else +#endif /* SQLITE_OMIT_HEX_INTEGER */ + { + return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8); + } +} + +/* +** If zNum represents an integer that will fit in 32-bits, then set +** *pValue to that integer and return true. Otherwise return false. +** +** This routine accepts both decimal and hexadecimal notation for integers. +** +** Any non-numeric characters that following zNum are ignored. +** This is different from sqlite3Atoi64() which requires the +** input number to be zero-terminated. +*/ +SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ + sqlite_int64 v = 0; + int i, c; + int neg = 0; + if( zNum[0]=='-' ){ + neg = 1; + zNum++; + }else if( zNum[0]=='+' ){ + zNum++; + } +#ifndef SQLITE_OMIT_HEX_INTEGER + else if( zNum[0]=='0' + && (zNum[1]=='x' || zNum[1]=='X') + && sqlite3Isxdigit(zNum[2]) + ){ + u32 u = 0; + zNum += 2; + while( zNum[0]=='0' ) zNum++; + for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){ + u = u*16 + sqlite3HexToInt(zNum[i]); + } + if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){ + memcpy(pValue, &u, 4); + return 1; + }else{ + return 0; + } + } +#endif + if( !sqlite3Isdigit(zNum[0]) ) return 0; + while( zNum[0]=='0' ) zNum++; + for(i=0; i<11 && (c = zNum[i] - '0')>=0 && c<=9; i++){ + v = v*10 + c; + } + + /* The longest decimal representation of a 32 bit integer is 10 digits: + ** + ** 1234567890 + ** 2^31 -> 2147483648 + */ + testcase( i==10 ); + if( i>10 ){ + return 0; + } + testcase( v-neg==2147483647 ); + if( v-neg>2147483647 ){ + return 0; + } + if( neg ){ + v = -v; + } + *pValue = (int)v; + return 1; +} + +/* +** Return a 32-bit integer value extracted from a string. If the +** string is not an integer, just return 0. +*/ +SQLITE_PRIVATE int sqlite3Atoi(const char *z){ + int x = 0; + sqlite3GetInt32(z, &x); + return x; +} + +/* +** Try to convert z into an unsigned 32-bit integer. Return true on +** success and false if there is an error. +** +** Only decimal notation is accepted. +*/ +SQLITE_PRIVATE int sqlite3GetUInt32(const char *z, u32 *pI){ + u64 v = 0; + int i; + for(i=0; sqlite3Isdigit(z[i]); i++){ + v = v*10 + z[i] - '0'; + if( v>4294967296LL ){ *pI = 0; return 0; } + } + if( i==0 || z[i]!=0 ){ *pI = 0; return 0; } + *pI = (u32)v; + return 1; +} + +/* +** The variable-length integer encoding is as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** C = xxxxxxxx 8 bits of data +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** 28 bits - BBBA +** 35 bits - BBBBA +** 42 bits - BBBBBA +** 49 bits - BBBBBBA +** 56 bits - BBBBBBBA +** 64 bits - BBBBBBBBC +*/ + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data write will be between 1 and 9 bytes. The number +** of bytes written is returned. +** +** A variable-length integer consists of the lower 7 bits of each byte +** for all bytes that have the 8th bit set and one byte with the 8th +** bit clear. Except, if we get to the 9th byte, it stores the full +** 8 bits and is the last byte. +*/ +static int SQLITE_NOINLINE putVarint64(unsigned char *p, u64 v){ + int i, j, n; + u8 buf[10]; + if( v & (((u64)0xff000000)<<32) ){ + p[8] = (u8)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + } + return 9; + } + n = 0; + do{ + buf[n++] = (u8)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[0] &= 0x7f; + assert( n<=9 ); + for(i=0, j=n-1; j>=0; j--, i++){ + p[i] = buf[j]; + } + return n; +} +SQLITE_PRIVATE int sqlite3PutVarint(unsigned char *p, u64 v){ + if( v<=0x7f ){ + p[0] = v&0x7f; + return 1; + } + if( v<=0x3fff ){ + p[0] = ((v>>7)&0x7f)|0x80; + p[1] = v&0x7f; + return 2; + } + return putVarint64(p,v); +} + +/* +** Bitmasks used by sqlite3GetVarint(). These precomputed constants +** are defined here rather than simply putting the constant expressions +** inline in order to work around bugs in the RVT compiler. +** +** SLOT_2_0 A mask for (0x7f<<14) | 0x7f +** +** SLOT_4_2_0 A mask for (0x7f<<28) | SLOT_2_0 +*/ +#define SLOT_2_0 0x001fc07f +#define SLOT_4_2_0 0xf01fc07f + + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read. The value is stored in *v. +*/ +SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ + u32 a,b,s; + + if( ((signed char*)p)[0]>=0 ){ + *v = *p; + return 1; + } + if( ((signed char*)p)[1]>=0 ){ + *v = ((u32)(p[0]&0x7f)<<7) | p[1]; + return 2; + } + + /* Verify that constants are precomputed correctly */ + assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); + assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); + + a = ((u32)p[0])<<14; + b = p[1]; + p += 2; + a |= *p; + /* a: p0<<14 | p2 (unmasked) */ + if (!(a&0x80)) + { + a &= SLOT_2_0; + b &= 0x7f; + b = b<<7; + a |= b; + *v = a; + return 3; + } + + /* CSE1 from below */ + a &= SLOT_2_0; + p++; + b = b<<14; + b |= *p; + /* b: p1<<14 | p3 (unmasked) */ + if (!(b&0x80)) + { + b &= SLOT_2_0; + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + *v = a; + return 4; + } + + /* a: p0<<14 | p2 (masked) */ + /* b: p1<<14 | p3 (unmasked) */ + /* 1:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + /* moved CSE1 up */ + /* a &= (0x7f<<14)|(0x7f); */ + b &= SLOT_2_0; + s = a; + /* s: p0<<14 | p2 (masked) */ + + p++; + a = a<<14; + a |= *p; + /* a: p0<<28 | p2<<14 | p4 (unmasked) */ + if (!(a&0x80)) + { + /* we can skip these cause they were (effectively) done above + ** while calculating s */ + /* a &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + /* b &= (0x7f<<14)|(0x7f); */ + b = b<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 5; + } + + /* 2:save off p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + s = s<<7; + s |= b; + /* s: p0<<21 | p1<<14 | p2<<7 | p3 (masked) */ + + p++; + b = b<<14; + b |= *p; + /* b: p1<<28 | p3<<14 | p5 (unmasked) */ + if (!(b&0x80)) + { + /* we can skip this cause it was (effectively) done above in calc'ing s */ + /* b &= (0x7f<<28)|(0x7f<<14)|(0x7f); */ + a &= SLOT_2_0; + a = a<<7; + a |= b; + s = s>>18; + *v = ((u64)s)<<32 | a; + return 6; + } + + p++; + a = a<<14; + a |= *p; + /* a: p2<<28 | p4<<14 | p6 (unmasked) */ + if (!(a&0x80)) + { + a &= SLOT_4_2_0; + b &= SLOT_2_0; + b = b<<7; + a |= b; + s = s>>11; + *v = ((u64)s)<<32 | a; + return 7; + } + + /* CSE2 from below */ + a &= SLOT_2_0; + p++; + b = b<<14; + b |= *p; + /* b: p3<<28 | p5<<14 | p7 (unmasked) */ + if (!(b&0x80)) + { + b &= SLOT_4_2_0; + /* moved CSE2 up */ + /* a &= (0x7f<<14)|(0x7f); */ + a = a<<7; + a |= b; + s = s>>4; + *v = ((u64)s)<<32 | a; + return 8; + } + + p++; + a = a<<15; + a |= *p; + /* a: p4<<29 | p6<<15 | p8 (unmasked) */ + + /* moved CSE2 up */ + /* a &= (0x7f<<29)|(0x7f<<15)|(0xff); */ + b &= SLOT_2_0; + b = b<<8; + a |= b; + + s = s<<4; + b = p[-4]; + b &= 0x7f; + b = b>>3; + s |= b; + + *v = ((u64)s)<<32 | a; + + return 9; +} + +/* +** Read a 32-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read. The value is stored in *v. +** +** If the varint stored in p[0] is larger than can fit in a 32-bit unsigned +** integer, then set *v to 0xffffffff. +** +** A MACRO version, getVarint32, is provided which inlines the +** single-byte case. All code should use the MACRO version as +** this function assumes the single-byte case has already been handled. +*/ +SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ + u32 a,b; + + /* The 1-byte case. Overwhelmingly the most common. Handled inline + ** by the getVarin32() macro */ + a = *p; + /* a: p0 (unmasked) */ +#ifndef getVarint32 + if (!(a&0x80)) + { + /* Values between 0 and 127 */ + *v = a; + return 1; + } +#endif + + /* The 2-byte case */ + p++; + b = *p; + /* b: p1 (unmasked) */ + if (!(b&0x80)) + { + /* Values between 128 and 16383 */ + a &= 0x7f; + a = a<<7; + *v = a | b; + return 2; + } + + /* The 3-byte case */ + p++; + a = a<<14; + a |= *p; + /* a: p0<<14 | p2 (unmasked) */ + if (!(a&0x80)) + { + /* Values between 16384 and 2097151 */ + a &= (0x7f<<14)|(0x7f); + b &= 0x7f; + b = b<<7; + *v = a | b; + return 3; + } + + /* A 32-bit varint is used to store size information in btrees. + ** Objects are rarely larger than 2MiB limit of a 3-byte varint. + ** A 3-byte varint is sufficient, for example, to record the size + ** of a 1048569-byte BLOB or string. + ** + ** We only unroll the first 1-, 2-, and 3- byte cases. The very + ** rare larger cases can be handled by the slower 64-bit varint + ** routine. + */ +#if 1 + { + u64 v64; + u8 n; + + n = sqlite3GetVarint(p-2, &v64); + assert( n>3 && n<=9 ); + if( (v64 & SQLITE_MAX_U32)!=v64 ){ + *v = 0xffffffff; + }else{ + *v = (u32)v64; + } + return n; + } + +#else + /* For following code (kept for historical record only) shows an + ** unrolling for the 3- and 4-byte varint cases. This code is + ** slightly faster, but it is also larger and much harder to test. + */ + p++; + b = b<<14; + b |= *p; + /* b: p1<<14 | p3 (unmasked) */ + if (!(b&0x80)) + { + /* Values between 2097152 and 268435455 */ + b &= (0x7f<<14)|(0x7f); + a &= (0x7f<<14)|(0x7f); + a = a<<7; + *v = a | b; + return 4; + } + + p++; + a = a<<14; + a |= *p; + /* a: p0<<28 | p2<<14 | p4 (unmasked) */ + if (!(a&0x80)) + { + /* Values between 268435456 and 34359738367 */ + a &= SLOT_4_2_0; + b &= SLOT_4_2_0; + b = b<<7; + *v = a | b; + return 5; + } + + /* We can only reach this point when reading a corrupt database + ** file. In that case we are not in any hurry. Use the (relatively + ** slow) general-purpose sqlite3GetVarint() routine to extract the + ** value. */ + { + u64 v64; + u8 n; + + p -= 4; + n = sqlite3GetVarint(p, &v64); + assert( n>5 && n<=9 ); + *v = (u32)v64; + return n; + } +#endif +} + +/* +** Return the number of bytes that will be needed to store the given +** 64-bit integer. +*/ +SQLITE_PRIVATE int sqlite3VarintLen(u64 v){ + int i; + for(i=1; (v >>= 7)!=0; i++){ assert( i<10 ); } + return i; +} + + +/* +** Read or write a four-byte big-endian integer value. +*/ +SQLITE_PRIVATE u32 sqlite3Get4byte(const u8 *p){ +#if SQLITE_BYTEORDER==4321 + u32 x; + memcpy(&x,p,4); + return x; +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + u32 x; + memcpy(&x,p,4); + return __builtin_bswap32(x); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + u32 x; + memcpy(&x,p,4); + return _byteswap_ulong(x); +#else + testcase( p[0]&0x80 ); + return ((unsigned)p[0]<<24) | (p[1]<<16) | (p[2]<<8) | p[3]; +#endif +} +SQLITE_PRIVATE void sqlite3Put4byte(unsigned char *p, u32 v){ +#if SQLITE_BYTEORDER==4321 + memcpy(p,&v,4); +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + u32 x = __builtin_bswap32(v); + memcpy(p,&x,4); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + u32 x = _byteswap_ulong(v); + memcpy(p,&x,4); +#else + p[0] = (u8)(v>>24); + p[1] = (u8)(v>>16); + p[2] = (u8)(v>>8); + p[3] = (u8)v; +#endif +} + + + +/* +** Translate a single byte of Hex into an integer. +** This routine only works if h really is a valid hexadecimal +** character: 0..9a..fA..F +*/ +SQLITE_PRIVATE u8 sqlite3HexToInt(int h){ + assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); +#ifdef SQLITE_ASCII + h += 9*(1&(h>>6)); +#endif +#ifdef SQLITE_EBCDIC + h += 9*(1&~(h>>4)); +#endif + return (u8)(h & 0xf); +} + +#if !defined(SQLITE_OMIT_BLOB_LITERAL) +/* +** Convert a BLOB literal of the form "x'hhhhhh'" into its binary +** value. Return a pointer to its binary value. Space to hold the +** binary value has been obtained from malloc and must be freed by +** the calling routine. +*/ +SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ + char *zBlob; + int i; + + zBlob = (char *)sqlite3DbMallocRawNN(db, n/2 + 1); + n--; + if( zBlob ){ + for(i=0; ieOpenState; + if( eOpenState!=SQLITE_STATE_OPEN ){ + if( sqlite3SafetyCheckSickOrOk(db) ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + logBadConnection("unopened"); + } + return 0; + }else{ + return 1; + } +} +SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ + u8 eOpenState; + eOpenState = db->eOpenState; + if( eOpenState!=SQLITE_STATE_SICK && + eOpenState!=SQLITE_STATE_OPEN && + eOpenState!=SQLITE_STATE_BUSY ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + logBadConnection("invalid"); + return 0; + }else{ + return 1; + } +} + +/* +** Attempt to add, substract, or multiply the 64-bit signed value iB against +** the other 64-bit signed integer at *pA and store the result in *pA. +** Return 0 on success. Or if the operation would have resulted in an +** overflow, leave *pA unchanged and return 1. +*/ +SQLITE_PRIVATE int sqlite3AddInt64(i64 *pA, i64 iB){ +#if GCC_VERSION>=5004000 && !defined(__INTEL_COMPILER) + return __builtin_add_overflow(*pA, iB, pA); +#else + i64 iA = *pA; + testcase( iA==0 ); testcase( iA==1 ); + testcase( iB==-1 ); testcase( iB==0 ); + if( iB>=0 ){ + testcase( iA>0 && LARGEST_INT64 - iA == iB ); + testcase( iA>0 && LARGEST_INT64 - iA == iB - 1 ); + if( iA>0 && LARGEST_INT64 - iA < iB ) return 1; + }else{ + testcase( iA<0 && -(iA + LARGEST_INT64) == iB + 1 ); + testcase( iA<0 && -(iA + LARGEST_INT64) == iB + 2 ); + if( iA<0 && -(iA + LARGEST_INT64) > iB + 1 ) return 1; + } + *pA += iB; + return 0; +#endif +} +SQLITE_PRIVATE int sqlite3SubInt64(i64 *pA, i64 iB){ +#if GCC_VERSION>=5004000 && !defined(__INTEL_COMPILER) + return __builtin_sub_overflow(*pA, iB, pA); +#else + testcase( iB==SMALLEST_INT64+1 ); + if( iB==SMALLEST_INT64 ){ + testcase( (*pA)==(-1) ); testcase( (*pA)==0 ); + if( (*pA)>=0 ) return 1; + *pA -= iB; + return 0; + }else{ + return sqlite3AddInt64(pA, -iB); + } +#endif +} +SQLITE_PRIVATE int sqlite3MulInt64(i64 *pA, i64 iB){ +#if GCC_VERSION>=5004000 && !defined(__INTEL_COMPILER) + return __builtin_mul_overflow(*pA, iB, pA); +#else + i64 iA = *pA; + if( iB>0 ){ + if( iA>LARGEST_INT64/iB ) return 1; + if( iA0 ){ + if( iBLARGEST_INT64/-iB ) return 1; + } + } + *pA = iA*iB; + return 0; +#endif +} + +/* +** Compute the absolute value of a 32-bit signed integer, of possible. Or +** if the integer has a value of -2147483648, return +2147483647 +*/ +SQLITE_PRIVATE int sqlite3AbsInt32(int x){ + if( x>=0 ) return x; + if( x==(int)0x80000000 ) return 0x7fffffff; + return -x; +} + +#ifdef SQLITE_ENABLE_8_3_NAMES +/* +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than +** three characters, then shorten the suffix on z[] to be the last three +** characters of the original suffix. +** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. +** +** Examples: +** +** test.db-journal => test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa +*/ +SQLITE_PRIVATE void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ +#if SQLITE_ENABLE_8_3_NAMES<2 + if( sqlite3_uri_boolean(zBaseFilename, "8_3_names", 0) ) +#endif + { + int i, sz; + sz = sqlite3Strlen30(z); + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); + } +} +#endif + +/* +** Find (an approximate) sum of two LogEst values. This computation is +** not a simple "+" operator because LogEst is stored as a logarithmic +** value. +** +*/ +SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst a, LogEst b){ + static const unsigned char x[] = { + 10, 10, /* 0,1 */ + 9, 9, /* 2,3 */ + 8, 8, /* 4,5 */ + 7, 7, 7, /* 6,7,8 */ + 6, 6, 6, /* 9,10,11 */ + 5, 5, 5, /* 12-14 */ + 4, 4, 4, 4, /* 15-18 */ + 3, 3, 3, 3, 3, 3, /* 19-24 */ + 2, 2, 2, 2, 2, 2, 2, /* 25-31 */ + }; + if( a>=b ){ + if( a>b+49 ) return a; + if( a>b+31 ) return a+1; + return a+x[a-b]; + }else{ + if( b>a+49 ) return b; + if( b>a+31 ) return b+1; + return b+x[b-a]; + } +} + +/* +** Convert an integer into a LogEst. In other words, compute an +** approximation for 10*log2(x). +*/ +SQLITE_PRIVATE LogEst sqlite3LogEst(u64 x){ + static LogEst a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; + LogEst y = 40; + if( x<8 ){ + if( x<2 ) return 0; + while( x<8 ){ y -= 10; x <<= 1; } + }else{ +#if GCC_VERSION>=5004000 + int i = 60 - __builtin_clzll(x); + y += i*10; + x >>= i; +#else + while( x>255 ){ y += 40; x >>= 4; } /*OPTIMIZATION-IF-TRUE*/ + while( x>15 ){ y += 10; x >>= 1; } +#endif + } + return a[x&7] + y - 10; +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Convert a double into a LogEst +** In other words, compute an approximation for 10*log2(x). +*/ +SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double x){ + u64 a; + LogEst e; + assert( sizeof(x)==8 && sizeof(a)==8 ); + if( x<=1 ) return 0; + if( x<=2000000000 ) return sqlite3LogEst((u64)x); + memcpy(&a, &x, 8); + e = (a>>52) - 1022; + return e*10; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ + defined(SQLITE_ENABLE_STAT4) || \ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) +/* +** Convert a LogEst into an integer. +** +** Note that this routine is only used when one or more of various +** non-standard compile-time options is enabled. +*/ +SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ + u64 n; + n = x%10; + x /= 10; + if( n>=5 ) n -= 2; + else if( n>=1 ) n -= 1; +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ + defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) + if( x>60 ) return (u64)LARGEST_INT64; +#else + /* If only SQLITE_ENABLE_STAT4 is on, then the largest input + ** possible to this routine is 310, resulting in a maximum x of 31 */ + assert( x<=60 ); +#endif + return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x); +} +#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */ + +/* +** Add a new name/number pair to a VList. This might require that the +** VList object be reallocated, so return the new VList. If an OOM +** error occurs, the original VList returned and the +** db->mallocFailed flag is set. +** +** A VList is really just an array of integers. To destroy a VList, +** simply pass it to sqlite3DbFree(). +** +** The first integer is the number of integers allocated for the whole +** VList. The second integer is the number of integers actually used. +** Each name/number pair is encoded by subsequent groups of 3 or more +** integers. +** +** Each name/number pair starts with two integers which are the numeric +** value for the pair and the size of the name/number pair, respectively. +** The text name overlays one or more following integers. The text name +** is always zero-terminated. +** +** Conceptually: +** +** struct VList { +** int nAlloc; // Number of allocated slots +** int nUsed; // Number of used slots +** struct VListEntry { +** int iValue; // Value for this entry +** int nSlot; // Slots used by this entry +** // ... variable name goes here +** } a[0]; +** } +** +** During code generation, pointers to the variable names within the +** VList are taken. When that happens, nAlloc is set to zero as an +** indication that the VList may never again be enlarged, since the +** accompanying realloc() would invalidate the pointers. +*/ +SQLITE_PRIVATE VList *sqlite3VListAdd( + sqlite3 *db, /* The database connection used for malloc() */ + VList *pIn, /* The input VList. Might be NULL */ + const char *zName, /* Name of symbol to add */ + int nName, /* Bytes of text in zName */ + int iVal /* Value to associate with zName */ +){ + int nInt; /* number of sizeof(int) objects needed for zName */ + char *z; /* Pointer to where zName will be stored */ + int i; /* Index in pIn[] where zName is stored */ + + nInt = nName/4 + 3; + assert( pIn==0 || pIn[0]>=3 ); /* Verify ok to add new elements */ + if( pIn==0 || pIn[1]+nInt > pIn[0] ){ + /* Enlarge the allocation */ + sqlite3_int64 nAlloc = (pIn ? 2*(sqlite3_int64)pIn[0] : 10) + nInt; + VList *pOut = sqlite3DbRealloc(db, pIn, nAlloc*sizeof(int)); + if( pOut==0 ) return pIn; + if( pIn==0 ) pOut[1] = 2; + pIn = pOut; + pIn[0] = nAlloc; + } + i = pIn[1]; + pIn[i] = iVal; + pIn[i+1] = nInt; + z = (char*)&pIn[i+2]; + pIn[1] = i+nInt; + assert( pIn[1]<=pIn[0] ); + memcpy(z, zName, nName); + z[nName] = 0; + return pIn; +} + +/* +** Return a pointer to the name of a variable in the given VList that +** has the value iVal. Or return a NULL if there is no such variable in +** the list +*/ +SQLITE_PRIVATE const char *sqlite3VListNumToName(VList *pIn, int iVal){ + int i, mx; + if( pIn==0 ) return 0; + mx = pIn[1]; + i = 2; + do{ + if( pIn[i]==iVal ) return (char*)&pIn[i+2]; + i += pIn[i+1]; + }while( i */ + +/* Turn bulk memory into a hash table object by initializing the +** fields of the Hash structure. +** +** "pNew" is a pointer to the hash table that is to be initialized. +*/ +SQLITE_PRIVATE void sqlite3HashInit(Hash *pNew){ + assert( pNew!=0 ); + pNew->first = 0; + pNew->count = 0; + pNew->htsize = 0; + pNew->ht = 0; +} + +/* Remove all entries from a hash table. Reclaim all memory. +** Call this routine to delete a hash table or to reset a hash table +** to the empty state. +*/ +SQLITE_PRIVATE void sqlite3HashClear(Hash *pH){ + HashElem *elem; /* For looping over all elements of the table */ + + assert( pH!=0 ); + elem = pH->first; + pH->first = 0; + sqlite3_free(pH->ht); + pH->ht = 0; + pH->htsize = 0; + while( elem ){ + HashElem *next_elem = elem->next; + sqlite3_free(elem); + elem = next_elem; + } + pH->count = 0; +} + +/* +** The hashing function. +*/ +static unsigned int strHash(const char *z){ + unsigned int h = 0; + unsigned char c; + while( (c = (unsigned char)*z++)!=0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Knuth multiplicative hashing. (Sorting & Searching, p. 510). + ** 0x9e3779b1 is 2654435761 which is the closest prime number to + ** (2**32)*golden_ratio, where golden_ratio = (sqrt(5) - 1)/2. */ + h += sqlite3UpperToLower[c]; + h *= 0x9e3779b1; + } + return h; +} + + +/* Link pNew element into the hash table pH. If pEntry!=0 then also +** insert pNew into the pEntry hash bucket. +*/ +static void insertElement( + Hash *pH, /* The complete hash table */ + struct _ht *pEntry, /* The entry into which pNew is inserted */ + HashElem *pNew /* The element to be inserted */ +){ + HashElem *pHead; /* First element already in pEntry */ + if( pEntry ){ + pHead = pEntry->count ? pEntry->chain : 0; + pEntry->count++; + pEntry->chain = pNew; + }else{ + pHead = 0; + } + if( pHead ){ + pNew->next = pHead; + pNew->prev = pHead->prev; + if( pHead->prev ){ pHead->prev->next = pNew; } + else { pH->first = pNew; } + pHead->prev = pNew; + }else{ + pNew->next = pH->first; + if( pH->first ){ pH->first->prev = pNew; } + pNew->prev = 0; + pH->first = pNew; + } +} + + +/* Resize the hash table so that it cantains "new_size" buckets. +** +** The hash table might fail to resize if sqlite3_malloc() fails or +** if the new size is the same as the prior size. +** Return TRUE if the resize occurs and false if not. +*/ +static int rehash(Hash *pH, unsigned int new_size){ + struct _ht *new_ht; /* The new hash table */ + HashElem *elem, *next_elem; /* For looping over existing elements */ + +#if SQLITE_MALLOC_SOFT_LIMIT>0 + if( new_size*sizeof(struct _ht)>SQLITE_MALLOC_SOFT_LIMIT ){ + new_size = SQLITE_MALLOC_SOFT_LIMIT/sizeof(struct _ht); + } + if( new_size==pH->htsize ) return 0; +#endif + + /* The inability to allocates space for a larger hash table is + ** a performance hit but it is not a fatal error. So mark the + ** allocation as a benign. Use sqlite3Malloc()/memset(0) instead of + ** sqlite3MallocZero() to make the allocation, as sqlite3MallocZero() + ** only zeroes the requested number of bytes whereas this module will + ** use the actual amount of space allocated for the hash table (which + ** may be larger than the requested amount). + */ + sqlite3BeginBenignMalloc(); + new_ht = (struct _ht *)sqlite3Malloc( new_size*sizeof(struct _ht) ); + sqlite3EndBenignMalloc(); + + if( new_ht==0 ) return 0; + sqlite3_free(pH->ht); + pH->ht = new_ht; + pH->htsize = new_size = sqlite3MallocSize(new_ht)/sizeof(struct _ht); + memset(new_ht, 0, new_size*sizeof(struct _ht)); + for(elem=pH->first, pH->first=0; elem; elem = next_elem){ + unsigned int h = strHash(elem->pKey) % new_size; + next_elem = elem->next; + insertElement(pH, &new_ht[h], elem); + } + return 1; +} + +/* This function (for internal use only) locates an element in an +** hash table that matches the given key. If no element is found, +** a pointer to a static null element with HashElem.data==0 is returned. +** If pH is not NULL, then the hash for this key is written to *pH. +*/ +static HashElem *findElementWithHash( + const Hash *pH, /* The pH to be searched */ + const char *pKey, /* The key we are searching for */ + unsigned int *pHash /* Write the hash value here */ +){ + HashElem *elem; /* Used to loop thru the element list */ + unsigned int count; /* Number of elements left to test */ + unsigned int h; /* The computed hash */ + static HashElem nullElement = { 0, 0, 0, 0 }; + + if( pH->ht ){ /*OPTIMIZATION-IF-TRUE*/ + struct _ht *pEntry; + h = strHash(pKey) % pH->htsize; + pEntry = &pH->ht[h]; + elem = pEntry->chain; + count = pEntry->count; + }else{ + h = 0; + elem = pH->first; + count = pH->count; + } + if( pHash ) *pHash = h; + while( count-- ){ + assert( elem!=0 ); + if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ + return elem; + } + elem = elem->next; + } + return &nullElement; +} + +/* Remove a single entry from the hash table given a pointer to that +** element and a hash on the element's key. +*/ +static void removeElementGivenHash( + Hash *pH, /* The pH containing "elem" */ + HashElem* elem, /* The element to be removed from the pH */ + unsigned int h /* Hash value for the element */ +){ + struct _ht *pEntry; + if( elem->prev ){ + elem->prev->next = elem->next; + }else{ + pH->first = elem->next; + } + if( elem->next ){ + elem->next->prev = elem->prev; + } + if( pH->ht ){ + pEntry = &pH->ht[h]; + if( pEntry->chain==elem ){ + pEntry->chain = elem->next; + } + assert( pEntry->count>0 ); + pEntry->count--; + } + sqlite3_free( elem ); + pH->count--; + if( pH->count==0 ){ + assert( pH->first==0 ); + assert( pH->count==0 ); + sqlite3HashClear(pH); + } +} + +/* Attempt to locate an element of the hash table pH with a key +** that matches pKey. Return the data for this element if it is +** found, or NULL if there is no match. +*/ +SQLITE_PRIVATE void *sqlite3HashFind(const Hash *pH, const char *pKey){ + assert( pH!=0 ); + assert( pKey!=0 ); + return findElementWithHash(pH, pKey, 0)->data; +} + +/* Insert an element into the hash table pH. The key is pKey +** and the data is "data". +** +** If no element exists with a matching key, then a new +** element is created and NULL is returned. +** +** If another element already exists with the same key, then the +** new data replaces the old data and the old data is returned. +** The key is not copied in this instance. If a malloc fails, then +** the new data is returned and the hash table is unchanged. +** +** If the "data" parameter to this function is NULL, then the +** element corresponding to "key" is removed from the hash table. +*/ +SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ + unsigned int h; /* the hash of the key modulo hash table size */ + HashElem *elem; /* Used to loop thru the element list */ + HashElem *new_elem; /* New element added to the pH */ + + assert( pH!=0 ); + assert( pKey!=0 ); + elem = findElementWithHash(pH,pKey,&h); + if( elem->data ){ + void *old_data = elem->data; + if( data==0 ){ + removeElementGivenHash(pH,elem,h); + }else{ + elem->data = data; + elem->pKey = pKey; + } + return old_data; + } + if( data==0 ) return 0; + new_elem = (HashElem*)sqlite3Malloc( sizeof(HashElem) ); + if( new_elem==0 ) return data; + new_elem->pKey = pKey; + new_elem->data = data; + pH->count++; + if( pH->count>=10 && pH->count > 2*pH->htsize ){ + if( rehash(pH, pH->count*2) ){ + assert( pH->htsize>0 ); + h = strHash(pKey) % pH->htsize; + } + } + insertElement(pH, pH->ht ? &pH->ht[h] : 0, new_elem); + return 0; +} + +/************** End of hash.c ************************************************/ +/************** Begin file opcodes.c *****************************************/ +/* Automatically generated. Do not edit */ +/* See the tool/mkopcodec.tcl script for details. */ +#if !defined(SQLITE_OMIT_EXPLAIN) \ + || defined(VDBE_PROFILE) \ + || defined(SQLITE_DEBUG) +#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) || defined(SQLITE_DEBUG) +# define OpHelp(X) "\0" X +#else +# define OpHelp(X) +#endif +SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ + static const char *const azName[] = { + /* 0 */ "Savepoint" OpHelp(""), + /* 1 */ "AutoCommit" OpHelp(""), + /* 2 */ "Transaction" OpHelp(""), + /* 3 */ "SorterNext" OpHelp(""), + /* 4 */ "Prev" OpHelp(""), + /* 5 */ "Next" OpHelp(""), + /* 6 */ "Checkpoint" OpHelp(""), + /* 7 */ "JournalMode" OpHelp(""), + /* 8 */ "Vacuum" OpHelp(""), + /* 9 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 10 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 11 */ "Goto" OpHelp(""), + /* 12 */ "Gosub" OpHelp(""), + /* 13 */ "InitCoroutine" OpHelp(""), + /* 14 */ "Yield" OpHelp(""), + /* 15 */ "MustBeInt" OpHelp(""), + /* 16 */ "Jump" OpHelp(""), + /* 17 */ "Once" OpHelp(""), + /* 18 */ "If" OpHelp(""), + /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), + /* 20 */ "IfNot" OpHelp(""), + /* 21 */ "IsNullOrType" OpHelp("if typeof(r[P1]) IN (P3,5) goto P2"), + /* 22 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), + /* 23 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 24 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 25 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 26 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 27 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), + /* 28 */ "IfNoHope" OpHelp("key=r[P3@P4]"), + /* 29 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 30 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 31 */ "Found" OpHelp("key=r[P3@P4]"), + /* 32 */ "SeekRowid" OpHelp("intkey=r[P3]"), + /* 33 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 34 */ "Last" OpHelp(""), + /* 35 */ "IfSmaller" OpHelp(""), + /* 36 */ "SorterSort" OpHelp(""), + /* 37 */ "Sort" OpHelp(""), + /* 38 */ "Rewind" OpHelp(""), + /* 39 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 40 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 41 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 42 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 43 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), + /* 44 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), + /* 45 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 46 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 47 */ "Program" OpHelp(""), + /* 48 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 49 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 50 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), + /* 51 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), + /* 52 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), + /* 53 */ "Eq" OpHelp("IF r[P3]==r[P1]"), + /* 54 */ "Gt" OpHelp("IF r[P3]>r[P1]"), + /* 55 */ "Le" OpHelp("IF r[P3]<=r[P1]"), + /* 56 */ "Lt" OpHelp("IF r[P3]=r[P1]"), + /* 58 */ "ElseEq" OpHelp(""), + /* 59 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 60 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 61 */ "IncrVacuum" OpHelp(""), + /* 62 */ "VNext" OpHelp(""), + /* 63 */ "Init" OpHelp("Start at P2"), + /* 64 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), + /* 65 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), + /* 66 */ "Return" OpHelp(""), + /* 67 */ "EndCoroutine" OpHelp(""), + /* 68 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 69 */ "Halt" OpHelp(""), + /* 70 */ "Integer" OpHelp("r[P2]=P1"), + /* 71 */ "Int64" OpHelp("r[P2]=P4"), + /* 72 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 73 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 74 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 75 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 76 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 77 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 78 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 79 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 80 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 81 */ "ChngCntRow" OpHelp("output=r[P1]"), + /* 82 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 83 */ "CollSeq" OpHelp(""), + /* 84 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 85 */ "RealAffinity" OpHelp(""), + /* 86 */ "Cast" OpHelp("affinity(r[P1])"), + /* 87 */ "Permutation" OpHelp(""), + /* 88 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 89 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), + /* 90 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), + /* 91 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), + /* 92 */ "Column" OpHelp("r[P3]=PX"), + /* 93 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), + /* 94 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 95 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 96 */ "Count" OpHelp("r[P2]=count()"), + /* 97 */ "ReadCookie" OpHelp(""), + /* 98 */ "SetCookie" OpHelp(""), + /* 99 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 100 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 101 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 102 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 103 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 104 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), + /* 106 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 107 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 108 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 109 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 110 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 111 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), + /* 112 */ "OpenDup" OpHelp(""), + /* 113 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), + /* 114 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 115 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 116 */ "String8" OpHelp("r[P2]='P4'"), + /* 117 */ "SorterOpen" OpHelp(""), + /* 118 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 119 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 120 */ "Close" OpHelp(""), + /* 121 */ "ColumnsUsed" OpHelp(""), + /* 122 */ "SeekScan" OpHelp("Scan-ahead up to P1 rows"), + /* 123 */ "SeekHit" OpHelp("set P2<=seekHit<=P3"), + /* 124 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 125 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 126 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 127 */ "RowCell" OpHelp(""), + /* 128 */ "Delete" OpHelp(""), + /* 129 */ "ResetCount" OpHelp(""), + /* 130 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 131 */ "SorterData" OpHelp("r[P2]=data"), + /* 132 */ "RowData" OpHelp("r[P2]=data"), + /* 133 */ "Rowid" OpHelp("r[P2]=rowid"), + /* 134 */ "NullRow" OpHelp(""), + /* 135 */ "SeekEnd" OpHelp(""), + /* 136 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 137 */ "SorterInsert" OpHelp("key=r[P2]"), + /* 138 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 139 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), + /* 140 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 141 */ "FinishSeek" OpHelp(""), + /* 142 */ "Destroy" OpHelp(""), + /* 143 */ "Clear" OpHelp(""), + /* 144 */ "ResetSorter" OpHelp(""), + /* 145 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), + /* 146 */ "SqlExec" OpHelp(""), + /* 147 */ "ParseSchema" OpHelp(""), + /* 148 */ "LoadAnalysis" OpHelp(""), + /* 149 */ "DropTable" OpHelp(""), + /* 150 */ "DropIndex" OpHelp(""), + /* 151 */ "DropTrigger" OpHelp(""), + /* 152 */ "Real" OpHelp("r[P2]=P4"), + /* 153 */ "IntegrityCk" OpHelp(""), + /* 154 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 155 */ "Param" OpHelp(""), + /* 156 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 157 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 158 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 159 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 160 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 161 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 162 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 163 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 164 */ "Expire" OpHelp(""), + /* 165 */ "CursorLock" OpHelp(""), + /* 166 */ "CursorUnlock" OpHelp(""), + /* 167 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 168 */ "VBegin" OpHelp(""), + /* 169 */ "VCreate" OpHelp(""), + /* 170 */ "VDestroy" OpHelp(""), + /* 171 */ "VOpen" OpHelp(""), + /* 172 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 173 */ "VRename" OpHelp(""), + /* 174 */ "Pagecount" OpHelp(""), + /* 175 */ "MaxPgcnt" OpHelp(""), + /* 176 */ "Trace" OpHelp(""), + /* 177 */ "CursorHint" OpHelp(""), + /* 178 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), + /* 179 */ "Noop" OpHelp(""), + /* 180 */ "Explain" OpHelp(""), + /* 181 */ "Abortable" OpHelp(""), + }; + return azName[i]; +} +#endif + +/************** End of opcodes.c *********************************************/ +/************** Begin file os_unix.c *****************************************/ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains the VFS implementation for unix-like operating systems +** include Linux, MacOSX, *BSD, QNX, VxWorks, AIX, HPUX, and others. +** +** There are actually several different VFS implementations in this file. +** The differences are in the way that file locking is done. The default +** implementation uses Posix Advisory Locks. Alternative implementations +** use flock(), dot-files, various proprietary locking schemas, or simply +** skip locking all together. +** +** This source file is organized into divisions where the logic for various +** subfunctions is contained within the appropriate division. PLEASE +** KEEP THE STRUCTURE OF THIS FILE INTACT. New code should be placed +** in the correct division and should be clearly labeled. +** +** The layout of divisions is as follows: +** +** * General-purpose declarations and utility functions. +** * Unique file ID logic used by VxWorks. +** * Various locking primitive implementations (all except proxy locking): +** + for Posix Advisory Locks +** + for no-op locks +** + for dot-file locks +** + for flock() locking +** + for named semaphore locks (VxWorks only) +** + for AFP filesystem locks (MacOSX only) +** * sqlite3_file methods not associated with locking. +** * Definitions of sqlite3_io_methods objects for all locking +** methods plus "finder" functions for each locking method. +** * sqlite3_vfs method implementations. +** * Locking primitives for the proxy uber-locking-method. (MacOSX only) +** * Definitions of sqlite3_vfs objects for all locking methods +** plus implementations of sqlite3_os_init() and sqlite3_os_end(). +*/ +/* #include "sqliteInt.h" */ +#if SQLITE_OS_UNIX /* This file is used on unix only */ + +/* +** There are various methods for file locking used for concurrency +** control: +** +** 1. POSIX locking (the default), +** 2. No locking, +** 3. Dot-file locking, +** 4. flock() locking, +** 5. AFP locking (OSX only), +** 6. Named POSIX semaphores (VXWorks only), +** 7. proxy locking. (OSX only) +** +** Styles 4, 5, and 7 are only available of SQLITE_ENABLE_LOCKING_STYLE +** is defined to 1. The SQLITE_ENABLE_LOCKING_STYLE also enables automatic +** selection of the appropriate locking style based on the filesystem +** where the database is located. +*/ +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif + +/* Use pread() and pwrite() if they are available */ +#if defined(__APPLE__) +# define HAVE_PREAD 1 +# define HAVE_PWRITE 1 +#endif +#if defined(HAVE_PREAD64) && defined(HAVE_PWRITE64) +# undef USE_PREAD +# define USE_PREAD64 1 +#elif defined(HAVE_PREAD) && defined(HAVE_PWRITE) +# undef USE_PREAD64 +# define USE_PREAD 1 +#endif + +/* +** standard include files. +*/ +#include +#include +#include +#include +#include +/* #include */ +#include +#include +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +# include +#endif + +#if SQLITE_ENABLE_LOCKING_STYLE +/* # include */ +# include +# include +#endif /* SQLITE_ENABLE_LOCKING_STYLE */ + +/* +** Try to determine if gethostuuid() is available based on standard +** macros. This might sometimes compute the wrong value for some +** obscure platforms. For those cases, simply compile with one of +** the following: +** +** -DHAVE_GETHOSTUUID=0 +** -DHAVE_GETHOSTUUID=1 +** +** None if this matters except when building on Apple products with +** -DSQLITE_ENABLE_LOCKING_STYLE. +*/ +#ifndef HAVE_GETHOSTUUID +# define HAVE_GETHOSTUUID 0 +# if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ + (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) +# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ + && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0))\ + && (!defined(TARGET_OS_MACCATALYST) || (TARGET_OS_MACCATALYST==0)) +# undef HAVE_GETHOSTUUID +# define HAVE_GETHOSTUUID 1 +# else +# warning "gethostuuid() is disabled." +# endif +# endif +#endif + + +#if OS_VXWORKS +/* # include */ +# include +# include +#endif /* OS_VXWORKS */ + +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE +# include +#endif + +#ifdef HAVE_UTIME +# include +#endif + +/* +** Allowed values of unixFile.fsFlags +*/ +#define SQLITE_FSFLAGS_IS_MSDOS 0x1 + +/* +** If we are to be thread-safe, include the pthreads header. +*/ +#if SQLITE_THREADSAFE +/* # include */ +#endif + +/* +** Default permissions when creating a new file +*/ +#ifndef SQLITE_DEFAULT_FILE_PERMISSIONS +# define SQLITE_DEFAULT_FILE_PERMISSIONS 0644 +#endif + +/* +** Default permissions when creating auto proxy dir +*/ +#ifndef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS +# define SQLITE_DEFAULT_PROXYDIR_PERMISSIONS 0755 +#endif + +/* +** Maximum supported path-length. +*/ +#define MAX_PATHNAME 512 + +/* +** Maximum supported symbolic links +*/ +#define SQLITE_MAX_SYMLINKS 100 + +/* Always cast the getpid() return type for compatibility with +** kernel modules in VxWorks. */ +#define osGetpid(X) (pid_t)getpid() + +/* +** Only set the lastErrno if the error code is a real error and not +** a normal expected return code of SQLITE_BUSY or SQLITE_OK +*/ +#define IS_LOCK_ERROR(x) ((x != SQLITE_OK) && (x != SQLITE_BUSY)) + +/* Forward references */ +typedef struct unixShm unixShm; /* Connection shared memory */ +typedef struct unixShmNode unixShmNode; /* Shared memory instance */ +typedef struct unixInodeInfo unixInodeInfo; /* An i-node */ +typedef struct UnixUnusedFd UnixUnusedFd; /* An unused file descriptor */ + +/* +** Sometimes, after a file handle is closed by SQLite, the file descriptor +** cannot be closed immediately. In these cases, instances of the following +** structure are used to store the file descriptor while waiting for an +** opportunity to either close or reuse it. +*/ +struct UnixUnusedFd { + int fd; /* File descriptor to close */ + int flags; /* Flags this file descriptor was opened with */ + UnixUnusedFd *pNext; /* Next unused file descriptor on same file */ +}; + +/* +** The unixFile structure is subclass of sqlite3_file specific to the unix +** VFS implementations. +*/ +typedef struct unixFile unixFile; +struct unixFile { + sqlite3_io_methods const *pMethod; /* Always the first entry */ + sqlite3_vfs *pVfs; /* The VFS that created this unixFile */ + unixInodeInfo *pInode; /* Info about locks on this inode */ + int h; /* The file descriptor */ + unsigned char eFileLock; /* The type of lock held on this fd */ + unsigned short int ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ + int lastErrno; /* The unix errno from last I/O error */ + void *lockingContext; /* Locking style specific state */ + UnixUnusedFd *pPreallocatedUnused; /* Pre-allocated UnixUnusedFd */ + const char *zPath; /* Name of the file */ + unixShm *pShm; /* Shared memory segment information */ + int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ +#if SQLITE_MAX_MMAP_SIZE>0 + int nFetchOut; /* Number of outstanding xFetch refs */ + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ +#endif + int sectorSize; /* Device sector size */ + int deviceCharacteristics; /* Precomputed device characteristics */ +#if SQLITE_ENABLE_LOCKING_STYLE + int openFlags; /* The flags specified at open() */ +#endif +#if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) + unsigned fsFlags; /* cached details from statfs() */ +#endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + unsigned iBusyTimeout; /* Wait this many millisec on locks */ +#endif +#if OS_VXWORKS + struct vxworksFileId *pId; /* Unique file ID */ +#endif +#ifdef SQLITE_DEBUG + /* The next group of variables are used to track whether or not the + ** transaction counter in bytes 24-27 of database files are updated + ** whenever any part of the database changes. An assertion fault will + ** occur if a file is updated without also updating the transaction + ** counter. This test is made to avoid new problems similar to the + ** one described by ticket #3584. + */ + unsigned char transCntrChng; /* True if the transaction counter changed */ + unsigned char dbUpdate; /* True if any part of database file changed */ + unsigned char inNormalWrite; /* True if in a normal write operation */ + +#endif + +#ifdef SQLITE_TEST + /* In test mode, increase the size of this structure a bit so that + ** it is larger than the struct CrashFile defined in test6.c. + */ + char aPadding[32]; +#endif +}; + +/* This variable holds the process id (pid) from when the xRandomness() +** method was called. If xOpen() is called from a different process id, +** indicating that a fork() has occurred, the PRNG will be reset. +*/ +static pid_t randomnessPid = 0; + +/* +** Allowed values for the unixFile.ctrlFlags bitmask: +*/ +#define UNIXFILE_EXCL 0x01 /* Connections from one process only */ +#define UNIXFILE_RDONLY 0x02 /* Connection is read only */ +#define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ +#ifndef SQLITE_DISABLE_DIRSYNC +# define UNIXFILE_DIRSYNC 0x08 /* Directory sync needed */ +#else +# define UNIXFILE_DIRSYNC 0x00 +#endif +#define UNIXFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ +#define UNIXFILE_DELETE 0x20 /* Delete on close */ +#define UNIXFILE_URI 0x40 /* Filename might have query parameters */ +#define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ + +/* +** Include code that is common to all os_*.c files +*/ +/* #include "os_common.h" */ + +/* +** Define various macros that are missing from some systems. +*/ +#ifndef O_LARGEFILE +# define O_LARGEFILE 0 +#endif +#ifdef SQLITE_DISABLE_LFS +# undef O_LARGEFILE +# define O_LARGEFILE 0 +#endif +#ifndef O_NOFOLLOW +# define O_NOFOLLOW 0 +#endif +#ifndef O_BINARY +# define O_BINARY 0 +#endif + +/* +** The threadid macro resolves to the thread-id or to 0. Used for +** testing and debugging only. +*/ +#if SQLITE_THREADSAFE +#define threadid pthread_self() +#else +#define threadid 0 +#endif + +/* +** HAVE_MREMAP defaults to true on Linux and false everywhere else. +*/ +#if !defined(HAVE_MREMAP) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_MREMAP 1 +# else +# define HAVE_MREMAP 0 +# endif +#endif + +/* +** Explicitly call the 64-bit version of lseek() on Android. Otherwise, lseek() +** is the 32-bit version, even if _FILE_OFFSET_BITS=64 is defined. +*/ +#ifdef __ANDROID__ +# define lseek lseek64 +#endif + +#ifdef __linux__ +/* +** Linux-specific IOCTL magic numbers used for controlling F2FS +*/ +#define F2FS_IOCTL_MAGIC 0xf5 +#define F2FS_IOC_START_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 1) +#define F2FS_IOC_COMMIT_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 2) +#define F2FS_IOC_START_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 3) +#define F2FS_IOC_ABORT_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 5) +#define F2FS_IOC_GET_FEATURES _IOR(F2FS_IOCTL_MAGIC, 12, u32) +#define F2FS_FEATURE_ATOMIC_WRITE 0x0004 +#endif /* __linux__ */ + + +/* +** Different Unix systems declare open() in different ways. Same use +** open(const char*,int,mode_t). Others use open(const char*,int,...). +** The difference is important when using a pointer to the function. +** +** The safest way to deal with the problem is to always use this wrapper +** which always has the same well-defined interface. +*/ +static int posixOpen(const char *zFile, int flags, int mode){ + return open(zFile, flags, mode); +} + +/* Forward reference */ +static int openDirectory(const char*, int*); +static int unixGetpagesize(void); + +/* +** Many system calls are accessed through pointer-to-functions so that +** they may be overridden at runtime to facilitate fault injection during +** testing and sandboxing. The following array holds the names and pointers +** to all overrideable system calls. +*/ +static struct unix_syscall { + const char *zName; /* Name of the system call */ + sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ + sqlite3_syscall_ptr pDefault; /* Default value */ +} aSyscall[] = { + { "open", (sqlite3_syscall_ptr)posixOpen, 0 }, +#define osOpen ((int(*)(const char*,int,int))aSyscall[0].pCurrent) + + { "close", (sqlite3_syscall_ptr)close, 0 }, +#define osClose ((int(*)(int))aSyscall[1].pCurrent) + + { "access", (sqlite3_syscall_ptr)access, 0 }, +#define osAccess ((int(*)(const char*,int))aSyscall[2].pCurrent) + + { "getcwd", (sqlite3_syscall_ptr)getcwd, 0 }, +#define osGetcwd ((char*(*)(char*,size_t))aSyscall[3].pCurrent) + + { "stat", (sqlite3_syscall_ptr)stat, 0 }, +#define osStat ((int(*)(const char*,struct stat*))aSyscall[4].pCurrent) + +/* +** The DJGPP compiler environment looks mostly like Unix, but it +** lacks the fcntl() system call. So redefine fcntl() to be something +** that always succeeds. This means that locking does not occur under +** DJGPP. But it is DOS - what did you expect? +*/ +#ifdef __DJGPP__ + { "fstat", 0, 0 }, +#define osFstat(a,b,c) 0 +#else + { "fstat", (sqlite3_syscall_ptr)fstat, 0 }, +#define osFstat ((int(*)(int,struct stat*))aSyscall[5].pCurrent) +#endif + + { "ftruncate", (sqlite3_syscall_ptr)ftruncate, 0 }, +#define osFtruncate ((int(*)(int,off_t))aSyscall[6].pCurrent) + + { "fcntl", (sqlite3_syscall_ptr)fcntl, 0 }, +#define osFcntl ((int(*)(int,int,...))aSyscall[7].pCurrent) + + { "read", (sqlite3_syscall_ptr)read, 0 }, +#define osRead ((ssize_t(*)(int,void*,size_t))aSyscall[8].pCurrent) + +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE + { "pread", (sqlite3_syscall_ptr)pread, 0 }, +#else + { "pread", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPread ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[9].pCurrent) + +#if defined(USE_PREAD64) + { "pread64", (sqlite3_syscall_ptr)pread64, 0 }, +#else + { "pread64", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPread64 ((ssize_t(*)(int,void*,size_t,off64_t))aSyscall[10].pCurrent) + + { "write", (sqlite3_syscall_ptr)write, 0 }, +#define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) + +#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE + { "pwrite", (sqlite3_syscall_ptr)pwrite, 0 }, +#else + { "pwrite", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPwrite ((ssize_t(*)(int,const void*,size_t,off_t))\ + aSyscall[12].pCurrent) + +#if defined(USE_PREAD64) + { "pwrite64", (sqlite3_syscall_ptr)pwrite64, 0 }, +#else + { "pwrite64", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off64_t))\ + aSyscall[13].pCurrent) + + { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, +#define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) + +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + { "fallocate", (sqlite3_syscall_ptr)posix_fallocate, 0 }, +#else + { "fallocate", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osFallocate ((int(*)(int,off_t,off_t))aSyscall[15].pCurrent) + + { "unlink", (sqlite3_syscall_ptr)unlink, 0 }, +#define osUnlink ((int(*)(const char*))aSyscall[16].pCurrent) + + { "openDirectory", (sqlite3_syscall_ptr)openDirectory, 0 }, +#define osOpenDirectory ((int(*)(const char*,int*))aSyscall[17].pCurrent) + + { "mkdir", (sqlite3_syscall_ptr)mkdir, 0 }, +#define osMkdir ((int(*)(const char*,mode_t))aSyscall[18].pCurrent) + + { "rmdir", (sqlite3_syscall_ptr)rmdir, 0 }, +#define osRmdir ((int(*)(const char*))aSyscall[19].pCurrent) + +#if defined(HAVE_FCHOWN) + { "fchown", (sqlite3_syscall_ptr)fchown, 0 }, +#else + { "fchown", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + +#if defined(HAVE_FCHOWN) + { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, +#else + { "geteuid", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) + +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#else + { "mmap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[22].pCurrent) + +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#else + { "munmap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMunmap ((int(*)(void*,size_t))aSyscall[23].pCurrent) + +#if HAVE_MREMAP && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) + { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, +#else + { "mremap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[24].pCurrent) + +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, +#else + { "getpagesize", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osGetpagesize ((int(*)(void))aSyscall[25].pCurrent) + +#if defined(HAVE_READLINK) + { "readlink", (sqlite3_syscall_ptr)readlink, 0 }, +#else + { "readlink", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osReadlink ((ssize_t(*)(const char*,char*,size_t))aSyscall[26].pCurrent) + +#if defined(HAVE_LSTAT) + { "lstat", (sqlite3_syscall_ptr)lstat, 0 }, +#else + { "lstat", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) + +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +# ifdef __ANDROID__ + { "ioctl", (sqlite3_syscall_ptr)(int(*)(int, int, ...))ioctl, 0 }, +#define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent) +# else + { "ioctl", (sqlite3_syscall_ptr)ioctl, 0 }, +#define osIoctl ((int(*)(int,unsigned long,...))aSyscall[28].pCurrent) +# endif +#else + { "ioctl", (sqlite3_syscall_ptr)0, 0 }, +#endif + +}; /* End of the overrideable system calls */ + + +/* +** On some systems, calls to fchown() will trigger a message in a security +** log if they come from non-root processes. So avoid calling fchown() if +** we are not running as root. +*/ +static int robustFchown(int fd, uid_t uid, gid_t gid){ +#if defined(HAVE_FCHOWN) + return osGeteuid() ? 0 : osFchown(fd,uid,gid); +#else + return 0; +#endif +} + +/* +** This is the xSetSystemCall() method of sqlite3_vfs for all of the +** "unix" VFSes. Return SQLITE_OK opon successfully updating the +** system call pointer, or SQLITE_NOTFOUND if there is no configurable +** system call named zName. +*/ +static int unixSetSystemCall( + sqlite3_vfs *pNotUsed, /* The VFS pointer. Not used */ + const char *zName, /* Name of system call to override */ + sqlite3_syscall_ptr pNewFunc /* Pointer to new system call value */ +){ + unsigned int i; + int rc = SQLITE_NOTFOUND; + + UNUSED_PARAMETER(pNotUsed); + if( zName==0 ){ + /* If no zName is given, restore all system calls to their default + ** settings and return NULL + */ + rc = SQLITE_OK; + for(i=0; i=SQLITE_MINIMUM_FILE_DESCRIPTOR ) break; + osClose(fd); + sqlite3_log(SQLITE_WARNING, + "attempt to open \"%s\" as file descriptor %d", z, fd); + fd = -1; + if( osOpen("/dev/null", O_RDONLY, m)<0 ) break; + } + if( fd>=0 ){ + if( m!=0 ){ + struct stat statbuf; + if( osFstat(fd, &statbuf)==0 + && statbuf.st_size==0 + && (statbuf.st_mode&0777)!=m + ){ + osFchmod(fd, m); + } + } +#if defined(FD_CLOEXEC) && (!defined(O_CLOEXEC) || O_CLOEXEC==0) + osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); +#endif + } + return fd; +} + +/* +** Helper functions to obtain and relinquish the global mutex. The +** global mutex is used to protect the unixInodeInfo and +** vxworksFileId objects used by this file, all of which may be +** shared by multiple threads. +** +** Function unixMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() +** statements. e.g. +** +** unixEnterMutex() +** assert( unixMutexHeld() ); +** unixEnterLeave() +** +** To prevent deadlock, the global unixBigLock must must be acquired +** before the unixInodeInfo.pLockMutex mutex, if both are held. It is +** OK to get the pLockMutex without holding unixBigLock first, but if +** that happens, the unixBigLock mutex must not be acquired until after +** pLockMutex is released. +** +** OK: enter(unixBigLock), enter(pLockInfo) +** OK: enter(unixBigLock) +** OK: enter(pLockInfo) +** ERROR: enter(pLockInfo), enter(unixBigLock) +*/ +static sqlite3_mutex *unixBigLock = 0; +static void unixEnterMutex(void){ + assert( sqlite3_mutex_notheld(unixBigLock) ); /* Not a recursive mutex */ + sqlite3_mutex_enter(unixBigLock); +} +static void unixLeaveMutex(void){ + assert( sqlite3_mutex_held(unixBigLock) ); + sqlite3_mutex_leave(unixBigLock); +} +#ifdef SQLITE_DEBUG +static int unixMutexHeld(void) { + return sqlite3_mutex_held(unixBigLock); +} +#endif + + +#ifdef SQLITE_HAVE_OS_TRACE +/* +** Helper function for printing out trace information from debugging +** binaries. This returns the string representation of the supplied +** integer lock-type. +*/ +static const char *azFileLock(int eFileLock){ + switch( eFileLock ){ + case NO_LOCK: return "NONE"; + case SHARED_LOCK: return "SHARED"; + case RESERVED_LOCK: return "RESERVED"; + case PENDING_LOCK: return "PENDING"; + case EXCLUSIVE_LOCK: return "EXCLUSIVE"; + } + return "ERROR"; +} +#endif + +#ifdef SQLITE_LOCK_TRACE +/* +** Print out information about all locking operations. +** +** This routine is used for troubleshooting locks on multithreaded +** platforms. Enable by compiling with the -DSQLITE_LOCK_TRACE +** command-line option on the compiler. This code is normally +** turned off. +*/ +static int lockTrace(int fd, int op, struct flock *p){ + char *zOpName, *zType; + int s; + int savedErrno; + if( op==F_GETLK ){ + zOpName = "GETLK"; + }else if( op==F_SETLK ){ + zOpName = "SETLK"; + }else{ + s = osFcntl(fd, op, p); + sqlite3DebugPrintf("fcntl unknown %d %d %d\n", fd, op, s); + return s; + } + if( p->l_type==F_RDLCK ){ + zType = "RDLCK"; + }else if( p->l_type==F_WRLCK ){ + zType = "WRLCK"; + }else if( p->l_type==F_UNLCK ){ + zType = "UNLCK"; + }else{ + assert( 0 ); + } + assert( p->l_whence==SEEK_SET ); + s = osFcntl(fd, op, p); + savedErrno = errno; + sqlite3DebugPrintf("fcntl %d %d %s %s %d %d %d %d\n", + threadid, fd, zOpName, zType, (int)p->l_start, (int)p->l_len, + (int)p->l_pid, s); + if( s==(-1) && op==F_SETLK && (p->l_type==F_RDLCK || p->l_type==F_WRLCK) ){ + struct flock l2; + l2 = *p; + osFcntl(fd, F_GETLK, &l2); + if( l2.l_type==F_RDLCK ){ + zType = "RDLCK"; + }else if( l2.l_type==F_WRLCK ){ + zType = "WRLCK"; + }else if( l2.l_type==F_UNLCK ){ + zType = "UNLCK"; + }else{ + assert( 0 ); + } + sqlite3DebugPrintf("fcntl-failure-reason: %s %d %d %d\n", + zType, (int)l2.l_start, (int)l2.l_len, (int)l2.l_pid); + } + errno = savedErrno; + return s; +} +#undef osFcntl +#define osFcntl lockTrace +#endif /* SQLITE_LOCK_TRACE */ + +/* +** Retry ftruncate() calls that fail due to EINTR +** +** All calls to ftruncate() within this file should be made through +** this wrapper. On the Android platform, bypassing the logic below +** could lead to a corrupt database. +*/ +static int robust_ftruncate(int h, sqlite3_int64 sz){ + int rc; +#ifdef __ANDROID__ + /* On Android, ftruncate() always uses 32-bit offsets, even if + ** _FILE_OFFSET_BITS=64 is defined. This means it is unsafe to attempt to + ** truncate a file to any size larger than 2GiB. Silently ignore any + ** such attempts. */ + if( sz>(sqlite3_int64)0x7FFFFFFF ){ + rc = SQLITE_OK; + }else +#endif + do{ rc = osFtruncate(h,sz); }while( rc<0 && errno==EINTR ); + return rc; +} + +/* +** This routine translates a standard POSIX errno code into something +** useful to the clients of the sqlite3 functions. Specifically, it is +** intended to translate a variety of "try again" errors into SQLITE_BUSY +** and a variety of "please close the file descriptor NOW" errors into +** SQLITE_IOERR +** +** Errors during initialization of locks, or file system support for locks, +** should handle ENOLCK, ENOTSUP, EOPNOTSUPP separately. +*/ +static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { + assert( (sqliteIOErr == SQLITE_IOERR_LOCK) || + (sqliteIOErr == SQLITE_IOERR_UNLOCK) || + (sqliteIOErr == SQLITE_IOERR_RDLOCK) || + (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ); + switch (posixError) { + case EACCES: + case EAGAIN: + case ETIMEDOUT: + case EBUSY: + case EINTR: + case ENOLCK: + /* random NFS retry error, unless during file system support + * introspection, in which it actually means what it says */ + return SQLITE_BUSY; + + case EPERM: + return SQLITE_PERM; + + default: + return sqliteIOErr; + } +} + + +/****************************************************************************** +****************** Begin Unique File ID Utility Used By VxWorks *************** +** +** On most versions of unix, we can get a unique ID for a file by concatenating +** the device number and the inode number. But this does not work on VxWorks. +** On VxWorks, a unique file id must be based on the canonical filename. +** +** A pointer to an instance of the following structure can be used as a +** unique file ID in VxWorks. Each instance of this structure contains +** a copy of the canonical filename. There is also a reference count. +** The structure is reclaimed when the number of pointers to it drops to +** zero. +** +** There are never very many files open at one time and lookups are not +** a performance-critical path, so it is sufficient to put these +** structures on a linked list. +*/ +struct vxworksFileId { + struct vxworksFileId *pNext; /* Next in a list of them all */ + int nRef; /* Number of references to this one */ + int nName; /* Length of the zCanonicalName[] string */ + char *zCanonicalName; /* Canonical filename */ +}; + +#if OS_VXWORKS +/* +** All unique filenames are held on a linked list headed by this +** variable: +*/ +static struct vxworksFileId *vxworksFileList = 0; + +/* +** Simplify a filename into its canonical form +** by making the following changes: +** +** * removing any trailing and duplicate / +** * convert /./ into just / +** * convert /A/../ where A is any simple name into just / +** +** Changes are made in-place. Return the new name length. +** +** The original filename is in z[0..n-1]. Return the number of +** characters in the simplified name. +*/ +static int vxworksSimplifyName(char *z, int n){ + int i, j; + while( n>1 && z[n-1]=='/' ){ n--; } + for(i=j=0; i0 && z[j-1]!='/' ){ j--; } + if( j>0 ){ j--; } + i += 2; + continue; + } + } + z[j++] = z[i]; + } + z[j] = 0; + return j; +} + +/* +** Find a unique file ID for the given absolute pathname. Return +** a pointer to the vxworksFileId object. This pointer is the unique +** file ID. +** +** The nRef field of the vxworksFileId object is incremented before +** the object is returned. A new vxworksFileId object is created +** and added to the global list if necessary. +** +** If a memory allocation error occurs, return NULL. +*/ +static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ + struct vxworksFileId *pNew; /* search key and new file ID */ + struct vxworksFileId *pCandidate; /* For looping over existing file IDs */ + int n; /* Length of zAbsoluteName string */ + + assert( zAbsoluteName[0]=='/' ); + n = (int)strlen(zAbsoluteName); + pNew = sqlite3_malloc64( sizeof(*pNew) + (n+1) ); + if( pNew==0 ) return 0; + pNew->zCanonicalName = (char*)&pNew[1]; + memcpy(pNew->zCanonicalName, zAbsoluteName, n+1); + n = vxworksSimplifyName(pNew->zCanonicalName, n); + + /* Search for an existing entry that matching the canonical name. + ** If found, increment the reference count and return a pointer to + ** the existing file ID. + */ + unixEnterMutex(); + for(pCandidate=vxworksFileList; pCandidate; pCandidate=pCandidate->pNext){ + if( pCandidate->nName==n + && memcmp(pCandidate->zCanonicalName, pNew->zCanonicalName, n)==0 + ){ + sqlite3_free(pNew); + pCandidate->nRef++; + unixLeaveMutex(); + return pCandidate; + } + } + + /* No match was found. We will make a new file ID */ + pNew->nRef = 1; + pNew->nName = n; + pNew->pNext = vxworksFileList; + vxworksFileList = pNew; + unixLeaveMutex(); + return pNew; +} + +/* +** Decrement the reference count on a vxworksFileId object. Free +** the object when the reference count reaches zero. +*/ +static void vxworksReleaseFileId(struct vxworksFileId *pId){ + unixEnterMutex(); + assert( pId->nRef>0 ); + pId->nRef--; + if( pId->nRef==0 ){ + struct vxworksFileId **pp; + for(pp=&vxworksFileList; *pp && *pp!=pId; pp = &((*pp)->pNext)){} + assert( *pp==pId ); + *pp = pId->pNext; + sqlite3_free(pId); + } + unixLeaveMutex(); +} +#endif /* OS_VXWORKS */ +/*************** End of Unique File ID Utility Used By VxWorks **************** +******************************************************************************/ + + +/****************************************************************************** +*************************** Posix Advisory Locking **************************** +** +** POSIX advisory locks are broken by design. ANSI STD 1003.1 (1996) +** section 6.5.2.2 lines 483 through 490 specify that when a process +** sets or clears a lock, that operation overrides any prior locks set +** by the same process. It does not explicitly say so, but this implies +** that it overrides locks set by the same process using a different +** file descriptor. Consider this test case: +** +** int fd1 = open("./file1", O_RDWR|O_CREAT, 0644); +** int fd2 = open("./file2", O_RDWR|O_CREAT, 0644); +** +** Suppose ./file1 and ./file2 are really the same file (because +** one is a hard or symbolic link to the other) then if you set +** an exclusive lock on fd1, then try to get an exclusive lock +** on fd2, it works. I would have expected the second lock to +** fail since there was already a lock on the file due to fd1. +** But not so. Since both locks came from the same process, the +** second overrides the first, even though they were on different +** file descriptors opened on different file names. +** +** This means that we cannot use POSIX locks to synchronize file access +** among competing threads of the same process. POSIX locks will work fine +** to synchronize access for threads in separate processes, but not +** threads within the same process. +** +** To work around the problem, SQLite has to manage file locks internally +** on its own. Whenever a new database is opened, we have to find the +** specific inode of the database file (the inode is determined by the +** st_dev and st_ino fields of the stat structure that fstat() fills in) +** and check for locks already existing on that inode. When locks are +** created or removed, we have to look at our own internal record of the +** locks to see if another thread has previously set a lock on that same +** inode. +** +** (Aside: The use of inode numbers as unique IDs does not work on VxWorks. +** For VxWorks, we have to use the alternative unique ID system based on +** canonical filename and implemented in the previous division.) +** +** The sqlite3_file structure for POSIX is no longer just an integer file +** descriptor. It is now a structure that holds the integer file +** descriptor and a pointer to a structure that describes the internal +** locks on the corresponding inode. There is one locking structure +** per inode, so if the same inode is opened twice, both unixFile structures +** point to the same locking structure. The locking structure keeps +** a reference count (so we will know when to delete it) and a "cnt" +** field that tells us its internal lock status. cnt==0 means the +** file is unlocked. cnt==-1 means the file has an exclusive lock. +** cnt>0 means there are cnt shared locks on the file. +** +** Any attempt to lock or unlock a file first checks the locking +** structure. The fcntl() system call is only invoked to set a +** POSIX lock if the internal lock structure transitions between +** a locked and an unlocked state. +** +** But wait: there are yet more problems with POSIX advisory locks. +** +** If you close a file descriptor that points to a file that has locks, +** all locks on that file that are owned by the current process are +** released. To work around this problem, each unixInodeInfo object +** maintains a count of the number of pending locks on tha inode. +** When an attempt is made to close an unixFile, if there are +** other unixFile open on the same inode that are holding locks, the call +** to close() the file descriptor is deferred until all of the locks clear. +** The unixInodeInfo structure keeps a list of file descriptors that need to +** be closed and that list is walked (and cleared) when the last lock +** clears. +** +** Yet another problem: LinuxThreads do not play well with posix locks. +** +** Many older versions of linux use the LinuxThreads library which is +** not posix compliant. Under LinuxThreads, a lock created by thread +** A cannot be modified or overridden by a different thread B. +** Only thread A can modify the lock. Locking behavior is correct +** if the appliation uses the newer Native Posix Thread Library (NPTL) +** on linux - with NPTL a lock created by thread A can override locks +** in thread B. But there is no way to know at compile-time which +** threading library is being used. So there is no way to know at +** compile-time whether or not thread A can override locks on thread B. +** One has to do a run-time check to discover the behavior of the +** current process. +** +** SQLite used to support LinuxThreads. But support for LinuxThreads +** was dropped beginning with version 3.7.0. SQLite will still work with +** LinuxThreads provided that (1) there is no more than one connection +** per database file in the same process and (2) database connections +** do not move across threads. +*/ + +/* +** An instance of the following structure serves as the key used +** to locate a particular unixInodeInfo object. +*/ +struct unixFileId { + dev_t dev; /* Device number */ +#if OS_VXWORKS + struct vxworksFileId *pId; /* Unique file ID for vxworks. */ +#else + /* We are told that some versions of Android contain a bug that + ** sizes ino_t at only 32-bits instead of 64-bits. (See + ** https://android-review.googlesource.com/#/c/115351/3/dist/sqlite3.c) + ** To work around this, always allocate 64-bits for the inode number. + ** On small machines that only have 32-bit inodes, this wastes 4 bytes, + ** but that should not be a big deal. */ + /* WAS: ino_t ino; */ + u64 ino; /* Inode number */ +#endif +}; + +/* +** An instance of the following structure is allocated for each open +** inode. +** +** A single inode can have multiple file descriptors, so each unixFile +** structure contains a pointer to an instance of this object and this +** object keeps a count of the number of unixFile pointing to it. +** +** Mutex rules: +** +** (1) Only the pLockMutex mutex must be held in order to read or write +** any of the locking fields: +** nShared, nLock, eFileLock, bProcessLock, pUnused +** +** (2) When nRef>0, then the following fields are unchanging and can +** be read (but not written) without holding any mutex: +** fileId, pLockMutex +** +** (3) With the exceptions above, all the fields may only be read +** or written while holding the global unixBigLock mutex. +** +** Deadlock prevention: The global unixBigLock mutex may not +** be acquired while holding the pLockMutex mutex. If both unixBigLock +** and pLockMutex are needed, then unixBigLock must be acquired first. +*/ +struct unixInodeInfo { + struct unixFileId fileId; /* The lookup key */ + sqlite3_mutex *pLockMutex; /* Hold this mutex for... */ + int nShared; /* Number of SHARED locks held */ + int nLock; /* Number of outstanding file locks */ + unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ + unsigned char bProcessLock; /* An exclusive process lock is held */ + UnixUnusedFd *pUnused; /* Unused file descriptors to close */ + int nRef; /* Number of pointers to this structure */ + unixShmNode *pShmNode; /* Shared memory associated with this inode */ + unixInodeInfo *pNext; /* List of all unixInodeInfo objects */ + unixInodeInfo *pPrev; /* .... doubly linked */ +#if SQLITE_ENABLE_LOCKING_STYLE + unsigned long long sharedByte; /* for AFP simulated shared lock */ +#endif +#if OS_VXWORKS + sem_t *pSem; /* Named POSIX semaphore */ + char aSemName[MAX_PATHNAME+2]; /* Name of that semaphore */ +#endif +}; + +/* +** A lists of all unixInodeInfo objects. +** +** Must hold unixBigLock in order to read or write this variable. +*/ +static unixInodeInfo *inodeList = 0; /* All unixInodeInfo objects */ + +#ifdef SQLITE_DEBUG +/* +** True if the inode mutex (on the unixFile.pFileMutex field) is held, or not. +** This routine is used only within assert() to help verify correct mutex +** usage. +*/ +int unixFileMutexHeld(unixFile *pFile){ + assert( pFile->pInode ); + return sqlite3_mutex_held(pFile->pInode->pLockMutex); +} +int unixFileMutexNotheld(unixFile *pFile){ + assert( pFile->pInode ); + return sqlite3_mutex_notheld(pFile->pInode->pLockMutex); +} +#endif + +/* +** +** This function - unixLogErrorAtLine(), is only ever called via the macro +** unixLogError(). +** +** It is invoked after an error occurs in an OS function and errno has been +** set. It logs a message using sqlite3_log() containing the current value of +** errno and, if possible, the human-readable equivalent from strerror() or +** strerror_r(). +** +** The first argument passed to the macro should be the error code that +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** The two subsequent arguments should be the name of the OS function that +** failed (e.g. "unlink", "open") and the associated file-system path, +** if any. +*/ +#define unixLogError(a,b,c) unixLogErrorAtLine(a,b,c,__LINE__) +static int unixLogErrorAtLine( + int errcode, /* SQLite error code */ + const char *zFunc, /* Name of OS function that failed */ + const char *zPath, /* File path associated with error */ + int iLine /* Source line number where error occurred */ +){ + char *zErr; /* Message from strerror() or equivalent */ + int iErrno = errno; /* Saved syscall error number */ + + /* If this is not a threadsafe build (SQLITE_THREADSAFE==0), then use + ** the strerror() function to obtain the human-readable error message + ** equivalent to errno. Otherwise, use strerror_r(). + */ +#if SQLITE_THREADSAFE && defined(HAVE_STRERROR_R) + char aErr[80]; + memset(aErr, 0, sizeof(aErr)); + zErr = aErr; + + /* If STRERROR_R_CHAR_P (set by autoconf scripts) or __USE_GNU is defined, + ** assume that the system provides the GNU version of strerror_r() that + ** returns a pointer to a buffer containing the error message. That pointer + ** may point to aErr[], or it may point to some static storage somewhere. + ** Otherwise, assume that the system provides the POSIX version of + ** strerror_r(), which always writes an error message into aErr[]. + ** + ** If the code incorrectly assumes that it is the POSIX version that is + ** available, the error message will often be an empty string. Not a + ** huge problem. Incorrectly concluding that the GNU version is available + ** could lead to a segfault though. + */ +#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) + zErr = +# endif + strerror_r(iErrno, aErr, sizeof(aErr)-1); + +#elif SQLITE_THREADSAFE + /* This is a threadsafe build, but strerror_r() is not available. */ + zErr = ""; +#else + /* Non-threadsafe build, use strerror(). */ + zErr = strerror(iErrno); +#endif + + if( zPath==0 ) zPath = ""; + sqlite3_log(errcode, + "os_unix.c:%d: (%d) %s(%s) - %s", + iLine, iErrno, zFunc, zPath, zErr + ); + + return errcode; +} + +/* +** Close a file descriptor. +** +** We assume that close() almost always works, since it is only in a +** very sick application or on a very sick platform that it might fail. +** If it does fail, simply leak the file descriptor, but do log the +** error. +** +** Note that it is not safe to retry close() after EINTR since the +** file descriptor might have already been reused by another thread. +** So we don't even try to recover from an EINTR. Just log the error +** and move on. +*/ +static void robust_close(unixFile *pFile, int h, int lineno){ + if( osClose(h) ){ + unixLogErrorAtLine(SQLITE_IOERR_CLOSE, "close", + pFile ? pFile->zPath : 0, lineno); + } +} + +/* +** Set the pFile->lastErrno. Do this in a subroutine as that provides +** a convenient place to set a breakpoint. +*/ +static void storeLastErrno(unixFile *pFile, int error){ + pFile->lastErrno = error; +} + +/* +** Close all file descriptors accumuated in the unixInodeInfo->pUnused list. +*/ +static void closePendingFds(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + UnixUnusedFd *p; + UnixUnusedFd *pNext; + assert( unixFileMutexHeld(pFile) ); + for(p=pInode->pUnused; p; p=pNext){ + pNext = p->pNext; + robust_close(pFile, p->fd, __LINE__); + sqlite3_free(p); + } + pInode->pUnused = 0; +} + +/* +** Release a unixInodeInfo structure previously allocated by findInodeInfo(). +** +** The global mutex must be held when this routine is called, but the mutex +** on the inode being deleted must NOT be held. +*/ +static void releaseInodeInfo(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + assert( unixMutexHeld() ); + assert( unixFileMutexNotheld(pFile) ); + if( ALWAYS(pInode) ){ + pInode->nRef--; + if( pInode->nRef==0 ){ + assert( pInode->pShmNode==0 ); + sqlite3_mutex_enter(pInode->pLockMutex); + closePendingFds(pFile); + sqlite3_mutex_leave(pInode->pLockMutex); + if( pInode->pPrev ){ + assert( pInode->pPrev->pNext==pInode ); + pInode->pPrev->pNext = pInode->pNext; + }else{ + assert( inodeList==pInode ); + inodeList = pInode->pNext; + } + if( pInode->pNext ){ + assert( pInode->pNext->pPrev==pInode ); + pInode->pNext->pPrev = pInode->pPrev; + } + sqlite3_mutex_free(pInode->pLockMutex); + sqlite3_free(pInode); + } + } +} + +/* +** Given a file descriptor, locate the unixInodeInfo object that +** describes that file descriptor. Create a new one if necessary. The +** return value might be uninitialized if an error occurs. +** +** The global mutex must held when calling this routine. +** +** Return an appropriate error code. +*/ +static int findInodeInfo( + unixFile *pFile, /* Unix file with file desc used in the key */ + unixInodeInfo **ppInode /* Return the unixInodeInfo object here */ +){ + int rc; /* System call return code */ + int fd; /* The file descriptor for pFile */ + struct unixFileId fileId; /* Lookup key for the unixInodeInfo */ + struct stat statbuf; /* Low-level file information */ + unixInodeInfo *pInode = 0; /* Candidate unixInodeInfo object */ + + assert( unixMutexHeld() ); + + /* Get low-level information about the file that we can used to + ** create a unique name for the file. + */ + fd = pFile->h; + rc = osFstat(fd, &statbuf); + if( rc!=0 ){ + storeLastErrno(pFile, errno); +#if defined(EOVERFLOW) && defined(SQLITE_DISABLE_LFS) + if( pFile->lastErrno==EOVERFLOW ) return SQLITE_NOLFS; +#endif + return SQLITE_IOERR; + } + +#ifdef __APPLE__ + /* On OS X on an msdos filesystem, the inode number is reported + ** incorrectly for zero-size files. See ticket #3260. To work + ** around this problem (we consider it a bug in OS X, not SQLite) + ** we always increase the file size to 1 by writing a single byte + ** prior to accessing the inode number. The one byte written is + ** an ASCII 'S' character which also happens to be the first byte + ** in the header of every SQLite database. In this way, if there + ** is a race condition such that another thread has already populated + ** the first page of the database, no damage is done. + */ + if( statbuf.st_size==0 && (pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS)!=0 ){ + do{ rc = osWrite(fd, "S", 1); }while( rc<0 && errno==EINTR ); + if( rc!=1 ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR; + } + rc = osFstat(fd, &statbuf); + if( rc!=0 ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR; + } + } +#endif + + memset(&fileId, 0, sizeof(fileId)); + fileId.dev = statbuf.st_dev; +#if OS_VXWORKS + fileId.pId = pFile->pId; +#else + fileId.ino = (u64)statbuf.st_ino; +#endif + assert( unixMutexHeld() ); + pInode = inodeList; + while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ + pInode = pInode->pNext; + } + if( pInode==0 ){ + pInode = sqlite3_malloc64( sizeof(*pInode) ); + if( pInode==0 ){ + return SQLITE_NOMEM_BKPT; + } + memset(pInode, 0, sizeof(*pInode)); + memcpy(&pInode->fileId, &fileId, sizeof(fileId)); + if( sqlite3GlobalConfig.bCoreMutex ){ + pInode->pLockMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pInode->pLockMutex==0 ){ + sqlite3_free(pInode); + return SQLITE_NOMEM_BKPT; + } + } + pInode->nRef = 1; + assert( unixMutexHeld() ); + pInode->pNext = inodeList; + pInode->pPrev = 0; + if( inodeList ) inodeList->pPrev = pInode; + inodeList = pInode; + }else{ + pInode->nRef++; + } + *ppInode = pInode; + return SQLITE_OK; +} + +/* +** Return TRUE if pFile has been renamed or unlinked since it was first opened. +*/ +static int fileHasMoved(unixFile *pFile){ +#if OS_VXWORKS + return pFile->pInode!=0 && pFile->pId!=pFile->pInode->fileId.pId; +#else + struct stat buf; + return pFile->pInode!=0 && + (osStat(pFile->zPath, &buf)!=0 + || (u64)buf.st_ino!=pFile->pInode->fileId.ino); +#endif +} + + +/* +** Check a unixFile that is a database. Verify the following: +** +** (1) There is exactly one hard link on the file +** (2) The file is not a symbolic link +** (3) The file has not been renamed or unlinked +** +** Issue sqlite3_log(SQLITE_WARNING,...) messages if anything is not right. +*/ +static void verifyDbFile(unixFile *pFile){ + struct stat buf; + int rc; + + /* These verifications occurs for the main database only */ + if( pFile->ctrlFlags & UNIXFILE_NOLOCK ) return; + + rc = osFstat(pFile->h, &buf); + if( rc!=0 ){ + sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); + return; + } + if( buf.st_nlink==0 ){ + sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); + return; + } + if( buf.st_nlink>1 ){ + sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); + return; + } + if( fileHasMoved(pFile) ){ + sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); + return; + } +} + + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + assert( pFile->eFileLock<=SHARED_LOCK ); + sqlite3_mutex_enter(pFile->pInode->pLockMutex); + + /* Check if a thread in this process holds such a lock */ + if( pFile->pInode->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. + */ +#ifndef __DJGPP__ + if( !reserved && !pFile->pInode->bProcessLock ){ + struct flock lock; + lock.l_whence = SEEK_SET; + lock.l_start = RESERVED_BYTE; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pFile->h, F_GETLK, &lock) ){ + rc = SQLITE_IOERR_CHECKRESERVEDLOCK; + storeLastErrno(pFile, errno); + } else if( lock.l_type!=F_UNLCK ){ + reserved = 1; + } + } +#endif + + sqlite3_mutex_leave(pFile->pInode->pLockMutex); + OSTRACE(("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved)); + + *pResOut = reserved; + return rc; +} + +/* Forward declaration*/ +static int unixSleep(sqlite3_vfs*,int); + +/* +** Set a posix-advisory-lock. +** +** There are two versions of this routine. If compiled with +** SQLITE_ENABLE_SETLK_TIMEOUT then the routine has an extra parameter +** which is a pointer to a unixFile. If the unixFile->iBusyTimeout +** value is set, then it is the number of milliseconds to wait before +** failing the lock. The iBusyTimeout value is always reset back to +** zero on each call. +** +** If SQLITE_ENABLE_SETLK_TIMEOUT is not defined, then do a non-blocking +** attempt to set the lock. +*/ +#ifndef SQLITE_ENABLE_SETLK_TIMEOUT +# define osSetPosixAdvisoryLock(h,x,t) osFcntl(h,F_SETLK,x) +#else +static int osSetPosixAdvisoryLock( + int h, /* The file descriptor on which to take the lock */ + struct flock *pLock, /* The description of the lock */ + unixFile *pFile /* Structure holding timeout value */ +){ + int tm = pFile->iBusyTimeout; + int rc = osFcntl(h,F_SETLK,pLock); + while( rc<0 && tm>0 ){ + /* On systems that support some kind of blocking file lock with a timeout, + ** make appropriate changes here to invoke that blocking file lock. On + ** generic posix, however, there is no such API. So we simply try the + ** lock once every millisecond until either the timeout expires, or until + ** the lock is obtained. */ + unixSleep(0,1000); + rc = osFcntl(h,F_SETLK,pLock); + tm--; + } + return rc; +} +#endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ + + +/* +** Attempt to set a system-lock on the file pFile. The lock is +** described by pLock. +** +** If the pFile was opened read/write from unix-excl, then the only lock +** ever obtained is an exclusive lock, and it is obtained exactly once +** the first time any lock is attempted. All subsequent system locking +** operations become no-ops. Locking operations still happen internally, +** in order to coordinate access between separate database connections +** within this process, but all of that is handled in memory and the +** operating system does not participate. +** +** This function is a pass-through to fcntl(F_SETLK) if pFile is using +** any VFS other than "unix-excl" or if pFile is opened on "unix-excl" +** and is read-only. +** +** Zero is returned if the call completes successfully, or -1 if a call +** to fcntl() fails. In this case, errno is set appropriately (by fcntl()). +*/ +static int unixFileLock(unixFile *pFile, struct flock *pLock){ + int rc; + unixInodeInfo *pInode = pFile->pInode; + assert( pInode!=0 ); + assert( sqlite3_mutex_held(pInode->pLockMutex) ); + if( (pFile->ctrlFlags & (UNIXFILE_EXCL|UNIXFILE_RDONLY))==UNIXFILE_EXCL ){ + if( pInode->bProcessLock==0 ){ + struct flock lock; + assert( pInode->nLock==0 ); + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + lock.l_type = F_WRLCK; + rc = osSetPosixAdvisoryLock(pFile->h, &lock, pFile); + if( rc<0 ) return rc; + pInode->bProcessLock = 1; + pInode->nLock++; + }else{ + rc = 0; + } + }else{ + rc = osSetPosixAdvisoryLock(pFile->h, pLock, pFile); + } + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int unixLock(sqlite3_file *id, int eFileLock){ + /* The following describes the implementation of the various locks and + ** lock transitions in terms of the POSIX advisory shared and exclusive + ** lock primitives (called read-locks and write-locks below, to avoid + ** confusion with SQLite lock names). The algorithms are complicated + ** slightly in order to be compatible with Windows95 systems simultaneously + ** accessing the same database file, in case that is ever required. + ** + ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved + ** byte', each single bytes at well known offsets, and the 'shared byte + ** range', a range of 510 bytes at a well known offset. + ** + ** To obtain a SHARED lock, a read-lock is obtained on the 'pending + ** byte'. If this is successful, 'shared byte range' is read-locked + ** and the lock on the 'pending byte' released. (Legacy note: When + ** SQLite was first developed, Windows95 systems were still very common, + ** and Widnows95 lacks a shared-lock capability. So on Windows95, a + ** single randomly selected by from the 'shared byte range' is locked. + ** Windows95 is now pretty much extinct, but this work-around for the + ** lack of shared-locks on Windows95 lives on, for backwards + ** compatibility.) + ** + ** A process may only obtain a RESERVED lock after it has a SHARED lock. + ** A RESERVED lock is implemented by grabbing a write-lock on the + ** 'reserved byte'. + ** + ** A process may only obtain a PENDING lock after it has obtained a + ** SHARED lock. A PENDING lock is implemented by obtaining a write-lock + ** on the 'pending byte'. This ensures that no new SHARED locks can be + ** obtained, but existing SHARED locks are allowed to persist. A process + ** does not have to obtain a RESERVED lock on the way to a PENDING lock. + ** This property is used by the algorithm for rolling back a journal file + ** after a crash. + ** + ** An EXCLUSIVE lock, obtained after a PENDING lock is held, is + ** implemented by obtaining a write-lock on the entire 'shared byte + ** range'. Since all other locks require a read-lock on one of the bytes + ** within this range, this ensures that no other locks are held on the + ** database. + */ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode; + struct flock lock; + int tErrno = 0; + + assert( pFile ); + OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (unix)\n", pFile->h, + azFileLock(eFileLock), azFileLock(pFile->eFileLock), + azFileLock(pFile->pInode->eFileLock), pFile->pInode->nShared, + osGetpid(0))); + + /* If there is already a lock of this type or more restrictive on the + ** unixFile, do nothing. Don't use the end_lock: exit path, as + ** unixEnterMutex() hasn't been called yet. + */ + if( pFile->eFileLock>=eFileLock ){ + OSTRACE(("LOCK %d %s ok (already held) (unix)\n", pFile->h, + azFileLock(eFileLock))); + return SQLITE_OK; + } + + /* Make sure the locking sequence is correct. + ** (1) We never move from unlocked to anything higher than shared lock. + ** (2) SQLite never explicitly requests a pendig lock. + ** (3) A shared lock is always held when a reserve lock is requested. + */ + assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); + assert( eFileLock!=PENDING_LOCK ); + assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); + + /* This mutex is needed because pFile->pInode is shared across threads + */ + pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + + /* If some thread using this PID has a lock via a different unixFile* + ** handle that precludes the requested lock, return BUSY. + */ + if( (pFile->eFileLock!=pInode->eFileLock && + (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) + ){ + rc = SQLITE_BUSY; + goto end_lock; + } + + /* If a SHARED lock is requested, and some thread using this PID already + ** has a SHARED or RESERVED lock, then increment reference counts and + ** return SQLITE_OK. + */ + if( eFileLock==SHARED_LOCK && + (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ + assert( eFileLock==SHARED_LOCK ); + assert( pFile->eFileLock==0 ); + assert( pInode->nShared>0 ); + pFile->eFileLock = SHARED_LOCK; + pInode->nShared++; + pInode->nLock++; + goto end_lock; + } + + + /* A PENDING lock is needed before acquiring a SHARED lock and before + ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will + ** be released. + */ + lock.l_len = 1L; + lock.l_whence = SEEK_SET; + if( eFileLock==SHARED_LOCK + || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLocknShared==0 ); + assert( pInode->eFileLock==0 ); + assert( rc==SQLITE_OK ); + + /* Now get the read-lock */ + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + if( unixFileLock(pFile, &lock) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + } + + /* Drop the temporary PENDING lock */ + lock.l_start = PENDING_BYTE; + lock.l_len = 1L; + lock.l_type = F_UNLCK; + if( unixFileLock(pFile, &lock) && rc==SQLITE_OK ){ + /* This could happen with a network mount */ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + } + + if( rc ){ + if( rc!=SQLITE_BUSY ){ + storeLastErrno(pFile, tErrno); + } + goto end_lock; + }else{ + pFile->eFileLock = SHARED_LOCK; + pInode->nLock++; + pInode->nShared = 1; + } + }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + }else{ + /* The request was for a RESERVED or EXCLUSIVE lock. It is + ** assumed that there is a SHARED or greater lock on the file + ** already. + */ + assert( 0!=pFile->eFileLock ); + lock.l_type = F_WRLCK; + + assert( eFileLock==RESERVED_LOCK || eFileLock==EXCLUSIVE_LOCK ); + if( eFileLock==RESERVED_LOCK ){ + lock.l_start = RESERVED_BYTE; + lock.l_len = 1L; + }else{ + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + } + + if( unixFileLock(pFile, &lock) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( rc!=SQLITE_BUSY ){ + storeLastErrno(pFile, tErrno); + } + } + } + + +#ifdef SQLITE_DEBUG + /* Set up the transaction-counter change checking flags when + ** transitioning from a SHARED to a RESERVED lock. The change + ** from SHARED to RESERVED marks the beginning of a normal + ** write operation (not a hot journal rollback). + */ + if( rc==SQLITE_OK + && pFile->eFileLock<=SHARED_LOCK + && eFileLock==RESERVED_LOCK + ){ + pFile->transCntrChng = 0; + pFile->dbUpdate = 0; + pFile->inNormalWrite = 1; + } +#endif + + + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + pInode->eFileLock = eFileLock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; + pInode->eFileLock = PENDING_LOCK; + } + +end_lock: + sqlite3_mutex_leave(pInode->pLockMutex); + OSTRACE(("LOCK %d %s %s (unix)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); + return rc; +} + +/* +** Add the file descriptor used by file handle pFile to the corresponding +** pUnused list. +*/ +static void setPendingFd(unixFile *pFile){ + unixInodeInfo *pInode = pFile->pInode; + UnixUnusedFd *p = pFile->pPreallocatedUnused; + assert( unixFileMutexHeld(pFile) ); + p->pNext = pInode->pUnused; + pInode->pUnused = p; + pFile->h = -1; + pFile->pPreallocatedUnused = 0; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** If handleNFSUnlock is true, then on downgrading an EXCLUSIVE_LOCK to SHARED +** the byte range is divided into 2 parts and the first part is unlocked then +** set to a read lock, then the other part is simply unlocked. This works +** around a bug in BSD NFS lockd (also seen on MacOSX 10.3+) that fails to +** remove the write lock on a region when a read lock is set. +*/ +static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode; + struct flock lock; + int rc = SQLITE_OK; + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (unix)\n", pFile->h, eFileLock, + pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, + osGetpid(0))); + + assert( eFileLock<=SHARED_LOCK ); + if( pFile->eFileLock<=eFileLock ){ + return SQLITE_OK; + } + pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + assert( pInode->nShared!=0 ); + if( pFile->eFileLock>SHARED_LOCK ){ + assert( pInode->eFileLock==pFile->eFileLock ); + +#ifdef SQLITE_DEBUG + /* When reducing a lock such that other processes can start + ** reading the database file again, make sure that the + ** transaction counter was updated if any part of the database + ** file changed. If the transaction counter is not updated, + ** other connections to the same file might not realize that + ** the file has changed and hence might not know to flush their + ** cache. The use of a stale cache can lead to database corruption. + */ + pFile->inNormalWrite = 0; +#endif + + /* downgrading to a shared lock on NFS involves clearing the write lock + ** before establishing the readlock - to avoid a race condition we downgrade + ** the lock in 2 blocks, so that part of the range will be covered by a + ** write lock until the rest is covered by a read lock: + ** 1: [WWWWW] + ** 2: [....W] + ** 3: [RRRRW] + ** 4: [RRRR.] + */ + if( eFileLock==SHARED_LOCK ){ +#if !defined(__APPLE__) || !SQLITE_ENABLE_LOCKING_STYLE + (void)handleNFSUnlock; + assert( handleNFSUnlock==0 ); +#endif +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + if( handleNFSUnlock ){ + int tErrno; /* Error code from system call errors */ + off_t divSize = SHARED_SIZE - 1; + + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, tErrno); + goto end_unlock; + } + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } + goto end_unlock; + } + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST+divSize; + lock.l_len = SHARED_SIZE-divSize; + if( unixFileLock(pFile, &lock)==(-1) ){ + tErrno = errno; + rc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, tErrno); + goto end_unlock; + } + }else +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ + { + lock.l_type = F_RDLCK; + lock.l_whence = SEEK_SET; + lock.l_start = SHARED_FIRST; + lock.l_len = SHARED_SIZE; + if( unixFileLock(pFile, &lock) ){ + /* In theory, the call to unixFileLock() cannot fail because another + ** process is holding an incompatible lock. If it does, this + ** indicates that the other process is not following the locking + ** protocol. If this happens, return SQLITE_IOERR_RDLOCK. Returning + ** SQLITE_BUSY would confuse the upper layer (in practice it causes + ** an assert to fail). */ + rc = SQLITE_IOERR_RDLOCK; + storeLastErrno(pFile, errno); + goto end_unlock; + } + } + } + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = PENDING_BYTE; + lock.l_len = 2L; assert( PENDING_BYTE+1==RESERVED_BYTE ); + if( unixFileLock(pFile, &lock)==0 ){ + pInode->eFileLock = SHARED_LOCK; + }else{ + rc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, errno); + goto end_unlock; + } + } + if( eFileLock==NO_LOCK ){ + /* Decrement the shared lock counter. Release the lock using an + ** OS call only when all threads in this same process have released + ** the lock. + */ + pInode->nShared--; + if( pInode->nShared==0 ){ + lock.l_type = F_UNLCK; + lock.l_whence = SEEK_SET; + lock.l_start = lock.l_len = 0L; + if( unixFileLock(pFile, &lock)==0 ){ + pInode->eFileLock = NO_LOCK; + }else{ + rc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, errno); + pInode->eFileLock = NO_LOCK; + pFile->eFileLock = NO_LOCK; + } + } + + /* Decrement the count of locks against this same file. When the + ** count reaches zero, close any other file descriptors whose close + ** was deferred because of outstanding locks. + */ + pInode->nLock--; + assert( pInode->nLock>=0 ); + if( pInode->nLock==0 ) closePendingFds(pFile); + } + +end_unlock: + sqlite3_mutex_leave(pInode->pLockMutex); + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + } + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int unixUnlock(sqlite3_file *id, int eFileLock){ +#if SQLITE_MAX_MMAP_SIZE>0 + assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 ); +#endif + return posixUnlock(id, eFileLock, 0); +} + +#if SQLITE_MAX_MMAP_SIZE>0 +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); +#endif + +/* +** This function performs the parts of the "close file" operation +** common to all locking schemes. It closes the directory and file +** handles, if they are valid, and sets all fields of the unixFile +** structure to 0. +** +** It is *not* necessary to hold the mutex when this routine is called, +** even on VxWorks. A mutex will be acquired on VxWorks by the +** vxworksReleaseFileId() routine. +*/ +static int closeUnixFile(sqlite3_file *id){ + unixFile *pFile = (unixFile*)id; +#if SQLITE_MAX_MMAP_SIZE>0 + unixUnmapfile(pFile); +#endif + if( pFile->h>=0 ){ + robust_close(pFile, pFile->h, __LINE__); + pFile->h = -1; + } +#if OS_VXWORKS + if( pFile->pId ){ + if( pFile->ctrlFlags & UNIXFILE_DELETE ){ + osUnlink(pFile->pId->zCanonicalName); + } + vxworksReleaseFileId(pFile->pId); + pFile->pId = 0; + } +#endif +#ifdef SQLITE_UNLINK_AFTER_CLOSE + if( pFile->ctrlFlags & UNIXFILE_DELETE ){ + osUnlink(pFile->zPath); + sqlite3_free(*(char**)&pFile->zPath); + pFile->zPath = 0; + } +#endif + OSTRACE(("CLOSE %-3d\n", pFile->h)); + OpenCounter(-1); + sqlite3_free(pFile->pPreallocatedUnused); + memset(pFile, 0, sizeof(unixFile)); + return SQLITE_OK; +} + +/* +** Close a file. +*/ +static int unixClose(sqlite3_file *id){ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile *)id; + unixInodeInfo *pInode = pFile->pInode; + + assert( pInode!=0 ); + verifyDbFile(pFile); + unixUnlock(id, NO_LOCK); + assert( unixFileMutexNotheld(pFile) ); + unixEnterMutex(); + + /* unixFile.pInode is always valid here. Otherwise, a different close + ** routine (e.g. nolockClose()) would be called instead. + */ + assert( pFile->pInode->nLock>0 || pFile->pInode->bProcessLock==0 ); + sqlite3_mutex_enter(pInode->pLockMutex); + if( pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->pUnused list. It will be automatically closed + ** when the last lock is cleared. + */ + setPendingFd(pFile); + } + sqlite3_mutex_leave(pInode->pLockMutex); + releaseInodeInfo(pFile); + assert( pFile->pShm==0 ); + rc = closeUnixFile(id); + unixLeaveMutex(); + return rc; +} + +/************** End of the posix advisory lock implementation ***************** +******************************************************************************/ + +/****************************************************************************** +****************************** No-op Locking ********************************** +** +** Of the various locking implementations available, this is by far the +** simplest: locking is ignored. No attempt is made to lock the database +** file for reading or writing. +** +** This locking mode is appropriate for use on read-only databases +** (ex: databases that are burned into CD-ROM, for example.) It can +** also be used if the application employs some external mechanism to +** prevent simultaneous access of the same database by two or more +** database connections. But there is a serious risk of database +** corruption if this locking mode is used in situations where multiple +** database connections are accessing the same database file at the same +** time and one or more of those connections are writing. +*/ + +static int nolockCheckReservedLock(sqlite3_file *NotUsed, int *pResOut){ + UNUSED_PARAMETER(NotUsed); + *pResOut = 0; + return SQLITE_OK; +} +static int nolockLock(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; +} +static int nolockUnlock(sqlite3_file *NotUsed, int NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return SQLITE_OK; +} + +/* +** Close the file. +*/ +static int nolockClose(sqlite3_file *id) { + return closeUnixFile(id); +} + +/******************* End of the no-op lock implementation ********************* +******************************************************************************/ + +/****************************************************************************** +************************* Begin dot-file Locking ****************************** +** +** The dotfile locking implementation uses the existence of separate lock +** files (really a directory) to control access to the database. This works +** on just about every filesystem imaginable. But there are serious downsides: +** +** (1) There is zero concurrency. A single reader blocks all other +** connections from reading or writing the database. +** +** (2) An application crash or power loss can leave stale lock files +** sitting around that need to be cleared manually. +** +** Nevertheless, a dotlock is an appropriate locking mode for use if no +** other locking strategy is available. +** +** Dotfile locking works by creating a subdirectory in the same directory as +** the database and with the same name but with a ".lock" extension added. +** The existence of a lock directory implies an EXCLUSIVE lock. All other +** lock types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. +*/ + +/* +** The file suffix added to the data base filename in order to create the +** lock directory. +*/ +#define DOTLOCK_SUFFIX ".lock" + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +** +** In dotfile locking, either a lock exists or it does not. So in this +** variation of CheckReservedLock(), *pResOut is set to true if any lock +** is held on the file and false if the file is unlocked. +*/ +static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + reserved = osAccess((const char*)pFile->lockingContext, 0)==0; + OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +** +** With dotfile locking, we really only support state (4): EXCLUSIVE. +** But we track the other locking levels internally. +*/ +static int dotlockLock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + char *zLockFile = (char *)pFile->lockingContext; + int rc = SQLITE_OK; + + + /* If we have any lock, then the lock file already exists. All we have + ** to do is adjust our internal record of the lock level. + */ + if( pFile->eFileLock > NO_LOCK ){ + pFile->eFileLock = eFileLock; + /* Always update the timestamp on the old file */ +#ifdef HAVE_UTIME + utime(zLockFile, NULL); +#else + utimes(zLockFile, NULL); +#endif + return SQLITE_OK; + } + + /* grab an exclusive lock */ + rc = osMkdir(zLockFile, 0777); + if( rc<0 ){ + /* failed to open/create the lock directory */ + int tErrno = errno; + if( EEXIST == tErrno ){ + rc = SQLITE_BUSY; + } else { + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( rc!=SQLITE_BUSY ){ + storeLastErrno(pFile, tErrno); + } + } + return rc; + } + + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** When the locking level reaches NO_LOCK, delete the lock file. +*/ +static int dotlockUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + char *zLockFile = (char *)pFile->lockingContext; + int rc; + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, eFileLock, + pFile->eFileLock, osGetpid(0))); + assert( eFileLock<=SHARED_LOCK ); + + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; + } + + /* To downgrade to shared, simply update our internal notion of the + ** lock state. No need to mess with the file on disk. + */ + if( eFileLock==SHARED_LOCK ){ + pFile->eFileLock = SHARED_LOCK; + return SQLITE_OK; + } + + /* To fully unlock the database, delete the lock file */ + assert( eFileLock==NO_LOCK ); + rc = osRmdir(zLockFile); + if( rc<0 ){ + int tErrno = errno; + if( tErrno==ENOENT ){ + rc = SQLITE_OK; + }else{ + rc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, tErrno); + } + return rc; + } + pFile->eFileLock = NO_LOCK; + return SQLITE_OK; +} + +/* +** Close a file. Make sure the lock has been released before closing. +*/ +static int dotlockClose(sqlite3_file *id) { + unixFile *pFile = (unixFile*)id; + assert( id!=0 ); + dotlockUnlock(id, NO_LOCK); + sqlite3_free(pFile->lockingContext); + return closeUnixFile(id); +} +/****************** End of the dot-file lock implementation ******************* +******************************************************************************/ + +/****************************************************************************** +************************** Begin flock Locking ******************************** +** +** Use the flock() system call to do file locking. +** +** flock() locking is like dot-file locking in that the various +** fine-grain locking levels supported by SQLite are collapsed into +** a single exclusive lock. In other words, SHARED, RESERVED, and +** PENDING locks are the same thing as an EXCLUSIVE lock. SQLite +** still works when you do this, but concurrency is reduced since +** only a single process can be reading the database at a time. +** +** Omit this section if SQLITE_ENABLE_LOCKING_STYLE is turned off +*/ +#if SQLITE_ENABLE_LOCKING_STYLE + +/* +** Retry flock() calls that fail with EINTR +*/ +#ifdef EINTR +static int robust_flock(int fd, int op){ + int rc; + do{ rc = flock(fd,op); }while( rc<0 && errno==EINTR ); + return rc; +} +#else +# define robust_flock(a,b) flock(a,b) +#endif + + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + + /* Check if a thread in this process holds such a lock */ + if( pFile->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. */ + if( !reserved ){ + /* attempt to get the lock */ + int lrc = robust_flock(pFile->h, LOCK_EX | LOCK_NB); + if( !lrc ){ + /* got the lock, unlock it */ + lrc = robust_flock(pFile->h, LOCK_UN); + if ( lrc ) { + int tErrno = errno; + /* unlock failed with an error */ + lrc = SQLITE_IOERR_UNLOCK; + storeLastErrno(pFile, tErrno); + rc = lrc; + } + } else { + int tErrno = errno; + reserved = 1; + /* someone else might have it reserved */ + lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(lrc) ){ + storeLastErrno(pFile, tErrno); + rc = lrc; + } + } + } + OSTRACE(("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved)); + +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + if( (rc & 0xff) == SQLITE_IOERR ){ + rc = SQLITE_OK; + reserved=1; + } +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** flock() only really support EXCLUSIVE locks. We track intermediate +** lock states in the sqlite3_file structure, but all locks SHARED or +** above are really EXCLUSIVE locks and exclude all other processes from +** access the file. +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int flockLock(sqlite3_file *id, int eFileLock) { + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + + assert( pFile ); + + /* if we already have a lock, it is exclusive. + ** Just adjust level and punt on outta here. */ + if (pFile->eFileLock > NO_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; + } + + /* grab an exclusive lock */ + + if (robust_flock(pFile->h, LOCK_EX | LOCK_NB)) { + int tErrno = errno; + /* didn't get, must be busy */ + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } + } else { + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + } + OSTRACE(("LOCK %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + if( (rc & 0xff) == SQLITE_IOERR ){ + rc = SQLITE_BUSY; + } +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + return rc; +} + + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int flockUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (flock)\n", pFile->h, eFileLock, + pFile->eFileLock, osGetpid(0))); + assert( eFileLock<=SHARED_LOCK ); + + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; + } + + /* shared can just be set because we always have an exclusive */ + if (eFileLock==SHARED_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; + } + + /* no, really, unlock. */ + if( robust_flock(pFile->h, LOCK_UN) ){ +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + return SQLITE_OK; +#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ + return SQLITE_IOERR_UNLOCK; + }else{ + pFile->eFileLock = NO_LOCK; + return SQLITE_OK; + } +} + +/* +** Close a file. +*/ +static int flockClose(sqlite3_file *id) { + assert( id!=0 ); + flockUnlock(id, NO_LOCK); + return closeUnixFile(id); +} + +#endif /* SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORK */ + +/******************* End of the flock lock implementation ********************* +******************************************************************************/ + +/****************************************************************************** +************************ Begin Named Semaphore Locking ************************ +** +** Named semaphore locking is only supported on VxWorks. +** +** Semaphore locking is like dot-lock and flock in that it really only +** supports EXCLUSIVE locking. Only a single process can read or write +** the database file at a time. This reduces potential concurrency, but +** makes the lock implementation much easier. +*/ +#if OS_VXWORKS + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int semXCheckReservedLock(sqlite3_file *id, int *pResOut) { + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + + /* Check if a thread in this process holds such a lock */ + if( pFile->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. */ + if( !reserved ){ + sem_t *pSem = pFile->pInode->pSem; + + if( sem_trywait(pSem)==-1 ){ + int tErrno = errno; + if( EAGAIN != tErrno ){ + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_CHECKRESERVEDLOCK); + storeLastErrno(pFile, tErrno); + } else { + /* someone else has the lock when we are in NO_LOCK */ + reserved = (pFile->eFileLock < SHARED_LOCK); + } + }else{ + /* we could have it if we want it */ + sem_post(pSem); + } + } + OSTRACE(("TEST WR-LOCK %d %d %d (sem)\n", pFile->h, rc, reserved)); + + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** Semaphore locks only really support EXCLUSIVE locks. We track intermediate +** lock states in the sqlite3_file structure, but all locks SHARED or +** above are really EXCLUSIVE locks and exclude all other processes from +** access the file. +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int semXLock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + sem_t *pSem = pFile->pInode->pSem; + int rc = SQLITE_OK; + + /* if we already have a lock, it is exclusive. + ** Just adjust level and punt on outta here. */ + if (pFile->eFileLock > NO_LOCK) { + pFile->eFileLock = eFileLock; + rc = SQLITE_OK; + goto sem_end_lock; + } + + /* lock semaphore now but bail out when already locked. */ + if( sem_trywait(pSem)==-1 ){ + rc = SQLITE_BUSY; + goto sem_end_lock; + } + + /* got it, set the type and return ok */ + pFile->eFileLock = eFileLock; + + sem_end_lock: + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int semXUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + sem_t *pSem = pFile->pInode->pSem; + + assert( pFile ); + assert( pSem ); + OSTRACE(("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, eFileLock, + pFile->eFileLock, osGetpid(0))); + assert( eFileLock<=SHARED_LOCK ); + + /* no-op if possible */ + if( pFile->eFileLock==eFileLock ){ + return SQLITE_OK; + } + + /* shared can just be set because we always have an exclusive */ + if (eFileLock==SHARED_LOCK) { + pFile->eFileLock = eFileLock; + return SQLITE_OK; + } + + /* no, really unlock. */ + if ( sem_post(pSem)==-1 ) { + int rc, tErrno = errno; + rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } + return rc; + } + pFile->eFileLock = NO_LOCK; + return SQLITE_OK; +} + +/* + ** Close a file. + */ +static int semXClose(sqlite3_file *id) { + if( id ){ + unixFile *pFile = (unixFile*)id; + semXUnlock(id, NO_LOCK); + assert( pFile ); + assert( unixFileMutexNotheld(pFile) ); + unixEnterMutex(); + releaseInodeInfo(pFile); + unixLeaveMutex(); + closeUnixFile(id); + } + return SQLITE_OK; +} + +#endif /* OS_VXWORKS */ +/* +** Named semaphore locking is only available on VxWorks. +** +*************** End of the named semaphore lock implementation **************** +******************************************************************************/ + + +/****************************************************************************** +*************************** Begin AFP Locking ********************************* +** +** AFP is the Apple Filing Protocol. AFP is a network filesystem found +** on Apple Macintosh computers - both OS9 and OSX. +** +** Third-party implementations of AFP are available. But this code here +** only works on OSX. +*/ + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* +** The afpLockingContext structure contains all afp lock specific state +*/ +typedef struct afpLockingContext afpLockingContext; +struct afpLockingContext { + int reserved; + const char *dbPath; /* Name of the open file */ +}; + +struct ByteRangeLockPB2 +{ + unsigned long long offset; /* offset to first byte to lock */ + unsigned long long length; /* nbr of bytes to lock */ + unsigned long long retRangeStart; /* nbr of 1st byte locked if successful */ + unsigned char unLockFlag; /* 1 = unlock, 0 = lock */ + unsigned char startEndFlag; /* 1=rel to end of fork, 0=rel to start */ + int fd; /* file desc to assoc this lock with */ +}; + +#define afpfsByteRangeLock2FSCTL _IOWR('z', 23, struct ByteRangeLockPB2) + +/* +** This is a utility for setting or clearing a bit-range lock on an +** AFP filesystem. +** +** Return SQLITE_OK on success, SQLITE_BUSY on failure. +*/ +static int afpSetLock( + const char *path, /* Name of the file to be locked or unlocked */ + unixFile *pFile, /* Open file descriptor on path */ + unsigned long long offset, /* First byte to be locked */ + unsigned long long length, /* Number of bytes to lock */ + int setLockFlag /* True to set lock. False to clear lock */ +){ + struct ByteRangeLockPB2 pb; + int err; + + pb.unLockFlag = setLockFlag ? 0 : 1; + pb.startEndFlag = 0; + pb.offset = offset; + pb.length = length; + pb.fd = pFile->h; + + OSTRACE(("AFPSETLOCK [%s] for %d%s in range %llx:%llx\n", + (setLockFlag?"ON":"OFF"), pFile->h, (pb.fd==-1?"[testval-1]":""), + offset, length)); + err = fsctl(path, afpfsByteRangeLock2FSCTL, &pb, 0); + if ( err==-1 ) { + int rc; + int tErrno = errno; + OSTRACE(("AFPSETLOCK failed to fsctl() '%s' %d %s\n", + path, tErrno, strerror(tErrno))); +#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS + rc = SQLITE_BUSY; +#else + rc = sqliteErrorFromPosixError(tErrno, + setLockFlag ? SQLITE_IOERR_LOCK : SQLITE_IOERR_UNLOCK); +#endif /* SQLITE_IGNORE_AFP_LOCK_ERRORS */ + if( IS_LOCK_ERROR(rc) ){ + storeLastErrno(pFile, tErrno); + } + return rc; + } else { + return SQLITE_OK; + } +} + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ + int rc = SQLITE_OK; + int reserved = 0; + unixFile *pFile = (unixFile*)id; + afpLockingContext *context; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + context = (afpLockingContext *) pFile->lockingContext; + if( context->reserved ){ + *pResOut = 1; + return SQLITE_OK; + } + sqlite3_mutex_enter(pFile->pInode->pLockMutex); + /* Check if a thread in this process holds such a lock */ + if( pFile->pInode->eFileLock>SHARED_LOCK ){ + reserved = 1; + } + + /* Otherwise see if some other process holds it. + */ + if( !reserved ){ + /* lock the RESERVED byte */ + int lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + if( SQLITE_OK==lrc ){ + /* if we succeeded in taking the reserved lock, unlock it to restore + ** the original state */ + lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); + } else { + /* if we failed to get the lock then someone else must have it */ + reserved = 1; + } + if( IS_LOCK_ERROR(lrc) ){ + rc=lrc; + } + } + + sqlite3_mutex_leave(pFile->pInode->pLockMutex); + OSTRACE(("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved)); + + *pResOut = reserved; + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int afpLock(sqlite3_file *id, int eFileLock){ + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode = pFile->pInode; + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + + assert( pFile ); + OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (afp)\n", pFile->h, + azFileLock(eFileLock), azFileLock(pFile->eFileLock), + azFileLock(pInode->eFileLock), pInode->nShared , osGetpid(0))); + + /* If there is already a lock of this type or more restrictive on the + ** unixFile, do nothing. Don't use the afp_end_lock: exit path, as + ** unixEnterMutex() hasn't been called yet. + */ + if( pFile->eFileLock>=eFileLock ){ + OSTRACE(("LOCK %d %s ok (already held) (afp)\n", pFile->h, + azFileLock(eFileLock))); + return SQLITE_OK; + } + + /* Make sure the locking sequence is correct + ** (1) We never move from unlocked to anything higher than shared lock. + ** (2) SQLite never explicitly requests a pendig lock. + ** (3) A shared lock is always held when a reserve lock is requested. + */ + assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); + assert( eFileLock!=PENDING_LOCK ); + assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); + + /* This mutex is needed because pFile->pInode is shared across threads + */ + pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + + /* If some thread using this PID has a lock via a different unixFile* + ** handle that precludes the requested lock, return BUSY. + */ + if( (pFile->eFileLock!=pInode->eFileLock && + (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) + ){ + rc = SQLITE_BUSY; + goto afp_end_lock; + } + + /* If a SHARED lock is requested, and some thread using this PID already + ** has a SHARED or RESERVED lock, then increment reference counts and + ** return SQLITE_OK. + */ + if( eFileLock==SHARED_LOCK && + (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ + assert( eFileLock==SHARED_LOCK ); + assert( pFile->eFileLock==0 ); + assert( pInode->nShared>0 ); + pFile->eFileLock = SHARED_LOCK; + pInode->nShared++; + pInode->nLock++; + goto afp_end_lock; + } + + /* A PENDING lock is needed before acquiring a SHARED lock and before + ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will + ** be released. + */ + if( eFileLock==SHARED_LOCK + || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLockdbPath, pFile, PENDING_BYTE, 1, 1); + if (failed) { + rc = failed; + goto afp_end_lock; + } + } + + /* If control gets to this point, then actually go ahead and make + ** operating system calls for the specified lock. + */ + if( eFileLock==SHARED_LOCK ){ + int lrc1, lrc2, lrc1Errno = 0; + long lk, mask; + + assert( pInode->nShared==0 ); + assert( pInode->eFileLock==0 ); + + mask = (sizeof(long)==8) ? LARGEST_INT64 : 0x7fffffff; + /* Now get the read-lock SHARED_LOCK */ + /* note that the quality of the randomness doesn't matter that much */ + lk = random(); + pInode->sharedByte = (lk & mask)%(SHARED_SIZE - 1); + lrc1 = afpSetLock(context->dbPath, pFile, + SHARED_FIRST+pInode->sharedByte, 1, 1); + if( IS_LOCK_ERROR(lrc1) ){ + lrc1Errno = pFile->lastErrno; + } + /* Drop the temporary PENDING lock */ + lrc2 = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); + + if( IS_LOCK_ERROR(lrc1) ) { + storeLastErrno(pFile, lrc1Errno); + rc = lrc1; + goto afp_end_lock; + } else if( IS_LOCK_ERROR(lrc2) ){ + rc = lrc2; + goto afp_end_lock; + } else if( lrc1 != SQLITE_OK ) { + rc = lrc1; + } else { + pFile->eFileLock = SHARED_LOCK; + pInode->nLock++; + pInode->nShared = 1; + } + }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + }else{ + /* The request was for a RESERVED or EXCLUSIVE lock. It is + ** assumed that there is a SHARED or greater lock on the file + ** already. + */ + int failed = 0; + assert( 0!=pFile->eFileLock ); + if (eFileLock >= RESERVED_LOCK && pFile->eFileLock < RESERVED_LOCK) { + /* Acquire a RESERVED lock */ + failed = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + if( !failed ){ + context->reserved = 1; + } + } + if (!failed && eFileLock == EXCLUSIVE_LOCK) { + /* Acquire an EXCLUSIVE lock */ + + /* Remove the shared lock before trying the range. we'll need to + ** reestablish the shared lock if we can't get the afpUnlock + */ + if( !(failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST + + pInode->sharedByte, 1, 0)) ){ + int failed2 = SQLITE_OK; + /* now attemmpt to get the exclusive lock range */ + failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, + SHARED_SIZE, 1); + if( failed && (failed2 = afpSetLock(context->dbPath, pFile, + SHARED_FIRST + pInode->sharedByte, 1, 1)) ){ + /* Can't reestablish the shared lock. Sqlite can't deal, this is + ** a critical I/O error + */ + rc = ((failed & 0xff) == SQLITE_IOERR) ? failed2 : + SQLITE_IOERR_LOCK; + goto afp_end_lock; + } + }else{ + rc = failed; + } + } + if( failed ){ + rc = failed; + } + } + + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + pInode->eFileLock = eFileLock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; + pInode->eFileLock = PENDING_LOCK; + } + +afp_end_lock: + sqlite3_mutex_leave(pInode->pLockMutex); + OSTRACE(("LOCK %d %s %s (afp)\n", pFile->h, azFileLock(eFileLock), + rc==SQLITE_OK ? "ok" : "failed")); + return rc; +} + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int afpUnlock(sqlite3_file *id, int eFileLock) { + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + unixInodeInfo *pInode; + afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + int skipShared = 0; +#ifdef SQLITE_TEST + int h = pFile->h; +#endif + + assert( pFile ); + OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, eFileLock, + pFile->eFileLock, pFile->pInode->eFileLock, pFile->pInode->nShared, + osGetpid(0))); + + assert( eFileLock<=SHARED_LOCK ); + if( pFile->eFileLock<=eFileLock ){ + return SQLITE_OK; + } + pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + assert( pInode->nShared!=0 ); + if( pFile->eFileLock>SHARED_LOCK ){ + assert( pInode->eFileLock==pFile->eFileLock ); + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + +#ifdef SQLITE_DEBUG + /* When reducing a lock such that other processes can start + ** reading the database file again, make sure that the + ** transaction counter was updated if any part of the database + ** file changed. If the transaction counter is not updated, + ** other connections to the same file might not realize that + ** the file has changed and hence might not know to flush their + ** cache. The use of a stale cache can lead to database corruption. + */ + assert( pFile->inNormalWrite==0 + || pFile->dbUpdate==0 + || pFile->transCntrChng==1 ); + pFile->inNormalWrite = 0; +#endif + + if( pFile->eFileLock==EXCLUSIVE_LOCK ){ + rc = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 0); + if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1) ){ + /* only re-establish the shared lock if necessary */ + int sharedLockByte = SHARED_FIRST+pInode->sharedByte; + rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 1); + } else { + skipShared = 1; + } + } + if( rc==SQLITE_OK && pFile->eFileLock>=PENDING_LOCK ){ + rc = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); + } + if( rc==SQLITE_OK && pFile->eFileLock>=RESERVED_LOCK && context->reserved ){ + rc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); + if( !rc ){ + context->reserved = 0; + } + } + if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1)){ + pInode->eFileLock = SHARED_LOCK; + } + } + if( rc==SQLITE_OK && eFileLock==NO_LOCK ){ + + /* Decrement the shared lock counter. Release the lock using an + ** OS call only when all threads in this same process have released + ** the lock. + */ + unsigned long long sharedLockByte = SHARED_FIRST+pInode->sharedByte; + pInode->nShared--; + if( pInode->nShared==0 ){ + SimulateIOErrorBenign(1); + SimulateIOError( h=(-1) ) + SimulateIOErrorBenign(0); + if( !skipShared ){ + rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 0); + } + if( !rc ){ + pInode->eFileLock = NO_LOCK; + pFile->eFileLock = NO_LOCK; + } + } + if( rc==SQLITE_OK ){ + pInode->nLock--; + assert( pInode->nLock>=0 ); + if( pInode->nLock==0 ) closePendingFds(pFile); + } + } + + sqlite3_mutex_leave(pInode->pLockMutex); + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + } + return rc; +} + +/* +** Close a file & cleanup AFP specific locking context +*/ +static int afpClose(sqlite3_file *id) { + int rc = SQLITE_OK; + unixFile *pFile = (unixFile*)id; + assert( id!=0 ); + afpUnlock(id, NO_LOCK); + assert( unixFileMutexNotheld(pFile) ); + unixEnterMutex(); + if( pFile->pInode ){ + unixInodeInfo *pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + if( pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + setPendingFd(pFile); + } + sqlite3_mutex_leave(pInode->pLockMutex); + } + releaseInodeInfo(pFile); + sqlite3_free(pFile->lockingContext); + rc = closeUnixFile(id); + unixLeaveMutex(); + return rc; +} + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The code above is the AFP lock implementation. The code is specific +** to MacOSX and does not work on other unix platforms. No alternative +** is available. If you don't compile for a mac, then the "unix-afp" +** VFS is not available. +** +********************* End of the AFP lock implementation ********************** +******************************************************************************/ + +/****************************************************************************** +*************************** Begin NFS Locking ********************************/ + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* + ** Lower the locking level on file descriptor pFile to eFileLock. eFileLock + ** must be either NO_LOCK or SHARED_LOCK. + ** + ** If the locking level of the file descriptor is already at or below + ** the requested locking level, this routine is a no-op. + */ +static int nfsUnlock(sqlite3_file *id, int eFileLock){ + return posixUnlock(id, eFileLock, 1); +} + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The code above is the NFS lock implementation. The code is specific +** to MacOSX and does not work on other unix platforms. No alternative +** is available. +** +********************* End of the NFS lock implementation ********************** +******************************************************************************/ + +/****************************************************************************** +**************** Non-locking sqlite3_file methods ***************************** +** +** The next division contains implementations for all methods of the +** sqlite3_file object other than the locking methods. The locking +** methods were defined in divisions above (one locking method per +** division). Those methods that are common to all locking modes +** are gather together into this division. +*/ + +/* +** Seek to the offset passed as the second argument, then read cnt +** bytes into pBuf. Return the number of bytes actually read. +** +** NB: If you define USE_PREAD or USE_PREAD64, then it might also +** be necessary to define _XOPEN_SOURCE to be 500. This varies from +** one system to another. Since SQLite does not define USE_PREAD +** in any form by default, we will not attempt to define _XOPEN_SOURCE. +** See tickets #2741 and #2681. +** +** To avoid stomping the errno value on a failed read the lastErrno value +** is set before returning. +*/ +static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ + int got; + int prior = 0; +#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) + i64 newOffset; +#endif + TIMER_START; + assert( cnt==(cnt&0x1ffff) ); + assert( id->h>2 ); + do{ +#if defined(USE_PREAD) + got = osPread(id->h, pBuf, cnt, offset); + SimulateIOError( got = -1 ); +#elif defined(USE_PREAD64) + got = osPread64(id->h, pBuf, cnt, offset); + SimulateIOError( got = -1 ); +#else + newOffset = lseek(id->h, offset, SEEK_SET); + SimulateIOError( newOffset = -1 ); + if( newOffset<0 ){ + storeLastErrno((unixFile*)id, errno); + return -1; + } + got = osRead(id->h, pBuf, cnt); +#endif + if( got==cnt ) break; + if( got<0 ){ + if( errno==EINTR ){ got = 1; continue; } + prior = 0; + storeLastErrno((unixFile*)id, errno); + break; + }else if( got>0 ){ + cnt -= got; + offset += got; + prior += got; + pBuf = (void*)(got + (char*)pBuf); + } + }while( got>0 ); + TIMER_END; + OSTRACE(("READ %-3d %5d %7lld %llu\n", + id->h, got+prior, offset-prior, TIMER_ELAPSED)); + return got+prior; +} + +/* +** Read data from a file into a buffer. Return SQLITE_OK if all +** bytes were read successfully and SQLITE_IOERR if anything goes +** wrong. +*/ +static int unixRead( + sqlite3_file *id, + void *pBuf, + int amt, + sqlite3_int64 offset +){ + unixFile *pFile = (unixFile *)id; + int got; + assert( id ); + assert( offset>=0 ); + assert( amt>0 ); + + /* If this is a database file (not a journal, super-journal or temp + ** file), the bytes in the locking range should never be read or written. */ +#if 0 + assert( pFile->pPreallocatedUnused==0 + || offset>=PENDING_BYTE+512 + || offset+amt<=PENDING_BYTE + ); +#endif + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + + got = seekAndRead(pFile, offset, pBuf, amt); + if( got==amt ){ + return SQLITE_OK; + }else if( got<0 ){ + /* pFile->lastErrno has been set by seekAndRead(). + ** Usually we return SQLITE_IOERR_READ here, though for some + ** kinds of errors we return SQLITE_IOERR_CORRUPTFS. The + ** SQLITE_IOERR_CORRUPTFS will be converted into SQLITE_CORRUPT + ** prior to returning to the application by the sqlite3ApiExit() + ** routine. + */ + switch( pFile->lastErrno ){ + case ERANGE: + case EIO: +#ifdef ENXIO + case ENXIO: +#endif +#ifdef EDEVERR + case EDEVERR: +#endif + return SQLITE_IOERR_CORRUPTFS; + } + return SQLITE_IOERR_READ; + }else{ + storeLastErrno(pFile, 0); /* not a system error */ + /* Unread parts of the buffer must be zero-filled */ + memset(&((char*)pBuf)[got], 0, amt-got); + return SQLITE_IOERR_SHORT_READ; + } +} + +/* +** Attempt to seek the file-descriptor passed as the first argument to +** absolute offset iOff, then attempt to write nBuf bytes of data from +** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, +** return the actual number of bytes written (which may be less than +** nBuf). +*/ +static int seekAndWriteFd( + int fd, /* File descriptor to write to */ + i64 iOff, /* File offset to begin writing at */ + const void *pBuf, /* Copy data from this buffer to the file */ + int nBuf, /* Size of buffer pBuf in bytes */ + int *piErrno /* OUT: Error number if error occurs */ +){ + int rc = 0; /* Value returned by system call */ + + assert( nBuf==(nBuf&0x1ffff) ); + assert( fd>2 ); + assert( piErrno!=0 ); + nBuf &= 0x1ffff; + TIMER_START; + +#if defined(USE_PREAD) + do{ rc = (int)osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); +#elif defined(USE_PREAD64) + do{ rc = (int)osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); +#else + do{ + i64 iSeek = lseek(fd, iOff, SEEK_SET); + SimulateIOError( iSeek = -1 ); + if( iSeek<0 ){ + rc = -1; + break; + } + rc = osWrite(fd, pBuf, nBuf); + }while( rc<0 && errno==EINTR ); +#endif + + TIMER_END; + OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); + + if( rc<0 ) *piErrno = errno; + return rc; +} + + +/* +** Seek to the offset in id->offset then read cnt bytes into pBuf. +** Return the number of bytes actually read. Update the offset. +** +** To avoid stomping the errno value on a failed write the lastErrno value +** is set before returning. +*/ +static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ + return seekAndWriteFd(id->h, offset, pBuf, cnt, &id->lastErrno); +} + + +/* +** Write data from a buffer into a file. Return SQLITE_OK on success +** or some other error code on failure. +*/ +static int unixWrite( + sqlite3_file *id, + const void *pBuf, + int amt, + sqlite3_int64 offset +){ + unixFile *pFile = (unixFile*)id; + int wrote = 0; + assert( id ); + assert( amt>0 ); + + /* If this is a database file (not a journal, super-journal or temp + ** file), the bytes in the locking range should never be read or written. */ +#if 0 + assert( pFile->pPreallocatedUnused==0 + || offset>=PENDING_BYTE+512 + || offset+amt<=PENDING_BYTE + ); +#endif + +#ifdef SQLITE_DEBUG + /* If we are doing a normal write to a database file (as opposed to + ** doing a hot-journal rollback or a write to some file other than a + ** normal database file) then record the fact that the database + ** has changed. If the transaction counter is modified, record that + ** fact too. + */ + if( pFile->inNormalWrite ){ + pFile->dbUpdate = 1; /* The database has been modified */ + if( offset<=24 && offset+amt>=27 ){ + int rc; + char oldCntr[4]; + SimulateIOErrorBenign(1); + rc = seekAndRead(pFile, 24, oldCntr, 4); + SimulateIOErrorBenign(0); + if( rc!=4 || memcmp(oldCntr, &((char*)pBuf)[24-offset], 4)!=0 ){ + pFile->transCntrChng = 1; /* The transaction counter has changed */ + } + } + } +#endif + +#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + + while( (wrote = seekAndWrite(pFile, offset, pBuf, amt))0 ){ + amt -= wrote; + offset += wrote; + pBuf = &((char*)pBuf)[wrote]; + } + SimulateIOError(( wrote=(-1), amt=1 )); + SimulateDiskfullError(( wrote=0, amt=1 )); + + if( amt>wrote ){ + if( wrote<0 && pFile->lastErrno!=ENOSPC ){ + /* lastErrno set by seekAndWrite */ + return SQLITE_IOERR_WRITE; + }else{ + storeLastErrno(pFile, 0); /* not a system error */ + return SQLITE_FULL; + } + } + + return SQLITE_OK; +} + +#ifdef SQLITE_TEST +/* +** Count the number of fullsyncs and normal syncs. This is used to test +** that syncs and fullsyncs are occurring at the right times. +*/ +SQLITE_API int sqlite3_sync_count = 0; +SQLITE_API int sqlite3_fullsync_count = 0; +#endif + +/* +** We do not trust systems to provide a working fdatasync(). Some do. +** Others do no. To be safe, we will stick with the (slightly slower) +** fsync(). If you know that your system does support fdatasync() correctly, +** then simply compile with -Dfdatasync=fdatasync or -DHAVE_FDATASYNC +*/ +#if !defined(fdatasync) && !HAVE_FDATASYNC +# define fdatasync fsync +#endif + +/* +** Define HAVE_FULLFSYNC to 0 or 1 depending on whether or not +** the F_FULLFSYNC macro is defined. F_FULLFSYNC is currently +** only available on Mac OS X. But that could change. +*/ +#ifdef F_FULLFSYNC +# define HAVE_FULLFSYNC 1 +#else +# define HAVE_FULLFSYNC 0 +#endif + + +/* +** The fsync() system call does not work as advertised on many +** unix systems. The following procedure is an attempt to make +** it work better. +** +** The SQLITE_NO_SYNC macro disables all fsync()s. This is useful +** for testing when we want to run through the test suite quickly. +** You are strongly advised *not* to deploy with SQLITE_NO_SYNC +** enabled, however, since with SQLITE_NO_SYNC enabled, an OS crash +** or power failure will likely corrupt the database file. +** +** SQLite sets the dataOnly flag if the size of the file is unchanged. +** The idea behind dataOnly is that it should only write the file content +** to disk, not the inode. We only set dataOnly if the file size is +** unchanged since the file size is part of the inode. However, +** Ted Ts'o tells us that fdatasync() will also write the inode if the +** file size has changed. The only real difference between fdatasync() +** and fsync(), Ted tells us, is that fdatasync() will not flush the +** inode if the mtime or owner or other inode attributes have changed. +** We only care about the file size, not the other file attributes, so +** as far as SQLite is concerned, an fdatasync() is always adequate. +** So, we always use fdatasync() if it is available, regardless of +** the value of the dataOnly flag. +*/ +static int full_fsync(int fd, int fullSync, int dataOnly){ + int rc; + + /* The following "ifdef/elif/else/" block has the same structure as + ** the one below. It is replicated here solely to avoid cluttering + ** up the real code with the UNUSED_PARAMETER() macros. + */ +#ifdef SQLITE_NO_SYNC + UNUSED_PARAMETER(fd); + UNUSED_PARAMETER(fullSync); + UNUSED_PARAMETER(dataOnly); +#elif HAVE_FULLFSYNC + UNUSED_PARAMETER(dataOnly); +#else + UNUSED_PARAMETER(fullSync); + UNUSED_PARAMETER(dataOnly); +#endif + + /* Record the number of times that we do a normal fsync() and + ** FULLSYNC. This is used during testing to verify that this procedure + ** gets called with the correct arguments. + */ +#ifdef SQLITE_TEST + if( fullSync ) sqlite3_fullsync_count++; + sqlite3_sync_count++; +#endif + + /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a + ** no-op. But go ahead and call fstat() to validate the file + ** descriptor as we need a method to provoke a failure during + ** coverate testing. + */ +#ifdef SQLITE_NO_SYNC + { + struct stat buf; + rc = osFstat(fd, &buf); + } +#elif HAVE_FULLFSYNC + if( fullSync ){ + rc = osFcntl(fd, F_FULLFSYNC, 0); + }else{ + rc = 1; + } + /* If the FULLFSYNC failed, fall back to attempting an fsync(). + ** It shouldn't be possible for fullfsync to fail on the local + ** file system (on OSX), so failure indicates that FULLFSYNC + ** isn't supported for this file system. So, attempt an fsync + ** and (for now) ignore the overhead of a superfluous fcntl call. + ** It'd be better to detect fullfsync support once and avoid + ** the fcntl call every time sync is called. + */ + if( rc ) rc = fsync(fd); + +#elif defined(__APPLE__) + /* fdatasync() on HFS+ doesn't yet flush the file size if it changed correctly + ** so currently we default to the macro that redefines fdatasync to fsync + */ + rc = fsync(fd); +#else + rc = fdatasync(fd); +#if OS_VXWORKS + if( rc==-1 && errno==ENOTSUP ){ + rc = fsync(fd); + } +#endif /* OS_VXWORKS */ +#endif /* ifdef SQLITE_NO_SYNC elif HAVE_FULLFSYNC */ + + if( OS_VXWORKS && rc!= -1 ){ + rc = 0; + } + return rc; +} + +/* +** Open a file descriptor to the directory containing file zFilename. +** If successful, *pFd is set to the opened file descriptor and +** SQLITE_OK is returned. If an error occurs, either SQLITE_NOMEM +** or SQLITE_CANTOPEN is returned and *pFd is set to an undefined +** value. +** +** The directory file descriptor is used for only one thing - to +** fsync() a directory to make sure file creation and deletion events +** are flushed to disk. Such fsyncs are not needed on newer +** journaling filesystems, but are required on older filesystems. +** +** This routine can be overridden using the xSetSysCall interface. +** The ability to override this routine was added in support of the +** chromium sandbox. Opening a directory is a security risk (we are +** told) so making it overrideable allows the chromium sandbox to +** replace this routine with a harmless no-op. To make this routine +** a no-op, replace it with a stub that returns SQLITE_OK but leaves +** *pFd set to a negative number. +** +** If SQLITE_OK is returned, the caller is responsible for closing +** the file descriptor *pFd using close(). +*/ +static int openDirectory(const char *zFilename, int *pFd){ + int ii; + int fd = -1; + char zDirname[MAX_PATHNAME+1]; + + sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); + for(ii=(int)strlen(zDirname); ii>0 && zDirname[ii]!='/'; ii--); + if( ii>0 ){ + zDirname[ii] = '\0'; + }else{ + if( zDirname[0]!='/' ) zDirname[0] = '.'; + zDirname[1] = 0; + } + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); + if( fd>=0 ){ + OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); + } + *pFd = fd; + if( fd>=0 ) return SQLITE_OK; + return unixLogError(SQLITE_CANTOPEN_BKPT, "openDirectory", zDirname); +} + +/* +** Make sure all writes to a particular file are committed to disk. +** +** If dataOnly==0 then both the file itself and its metadata (file +** size, access time, etc) are synced. If dataOnly!=0 then only the +** file data is synced. +** +** Under Unix, also make sure that the directory entry for the file +** has been created by fsync-ing the directory that contains the file. +** If we do not do this and we encounter a power failure, the directory +** entry for the journal might not exist after we reboot. The next +** SQLite to access the file will not know that the journal exists (because +** the directory entry for the journal was never created) and the transaction +** will not roll back - possibly leading to database corruption. +*/ +static int unixSync(sqlite3_file *id, int flags){ + int rc; + unixFile *pFile = (unixFile*)id; + + int isDataOnly = (flags&SQLITE_SYNC_DATAONLY); + int isFullsync = (flags&0x0F)==SQLITE_SYNC_FULL; + + /* Check that one of SQLITE_SYNC_NORMAL or FULL was passed */ + assert((flags&0x0F)==SQLITE_SYNC_NORMAL + || (flags&0x0F)==SQLITE_SYNC_FULL + ); + + /* Unix cannot, but some systems may return SQLITE_FULL from here. This + ** line is to test that doing so does not cause any problems. + */ + SimulateDiskfullError( return SQLITE_FULL ); + + assert( pFile ); + OSTRACE(("SYNC %-3d\n", pFile->h)); + rc = full_fsync(pFile->h, isFullsync, isDataOnly); + SimulateIOError( rc=1 ); + if( rc ){ + storeLastErrno(pFile, errno); + return unixLogError(SQLITE_IOERR_FSYNC, "full_fsync", pFile->zPath); + } + + /* Also fsync the directory containing the file if the DIRSYNC flag + ** is set. This is a one-time occurrence. Many systems (examples: AIX) + ** are unable to fsync a directory, so ignore errors on the fsync. + */ + if( pFile->ctrlFlags & UNIXFILE_DIRSYNC ){ + int dirfd; + OSTRACE(("DIRSYNC %s (have_fullfsync=%d fullsync=%d)\n", pFile->zPath, + HAVE_FULLFSYNC, isFullsync)); + rc = osOpenDirectory(pFile->zPath, &dirfd); + if( rc==SQLITE_OK ){ + full_fsync(dirfd, 0, 0); + robust_close(pFile, dirfd, __LINE__); + }else{ + assert( rc==SQLITE_CANTOPEN ); + rc = SQLITE_OK; + } + pFile->ctrlFlags &= ~UNIXFILE_DIRSYNC; + } + return rc; +} + +/* +** Truncate an open file to a specified size +*/ +static int unixTruncate(sqlite3_file *id, i64 nByte){ + unixFile *pFile = (unixFile *)id; + int rc; + assert( pFile ); + SimulateIOError( return SQLITE_IOERR_TRUNCATE ); + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk>0 ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + + rc = robust_ftruncate(pFile->h, nByte); + if( rc ){ + storeLastErrno(pFile, errno); + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + }else{ +#ifdef SQLITE_DEBUG + /* If we are doing a normal write to a database file (as opposed to + ** doing a hot-journal rollback or a write to some file other than a + ** normal database file) and we truncate the file to zero length, + ** that effectively updates the change counter. This might happen + ** when restoring a database using the backup API from a zero-length + ** source. + */ + if( pFile->inNormalWrite && nByte==0 ){ + pFile->transCntrChng = 1; + } +#endif + +#if SQLITE_MAX_MMAP_SIZE>0 + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( nBytemmapSize ){ + pFile->mmapSize = nByte; + } +#endif + + return SQLITE_OK; + } +} + +/* +** Determine the current size of a file in bytes +*/ +static int unixFileSize(sqlite3_file *id, i64 *pSize){ + int rc; + struct stat buf; + assert( id ); + rc = osFstat(((unixFile*)id)->h, &buf); + SimulateIOError( rc=1 ); + if( rc!=0 ){ + storeLastErrno((unixFile*)id, errno); + return SQLITE_IOERR_FSTAT; + } + *pSize = buf.st_size; + + /* When opening a zero-size database, the findInodeInfo() procedure + ** writes a single byte into that file in order to work around a bug + ** in the OS-X msdos filesystem. In order to avoid problems with upper + ** layers, we need to report this file size as zero even though it is + ** really 1. Ticket #3260. + */ + if( *pSize==1 ) *pSize = 0; + + + return SQLITE_OK; +} + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) +/* +** Handler for proxy-locking file-control verbs. Defined below in the +** proxying locking division. +*/ +static int proxyFileControl(sqlite3_file*,int,void*); +#endif + +/* +** This function is called to handle the SQLITE_FCNTL_SIZE_HINT +** file-control operation. Enlarge the database to nBytes in size +** (rounded up to the next chunk-size). If the database is already +** nBytes or larger, this routine is a no-op. +*/ +static int fcntlSizeHint(unixFile *pFile, i64 nByte){ + if( pFile->szChunk>0 ){ + i64 nSize; /* Required file size */ + struct stat buf; /* Used to hold return values of fstat() */ + + if( osFstat(pFile->h, &buf) ){ + return SQLITE_IOERR_FSTAT; + } + + nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; + if( nSize>(i64)buf.st_size ){ + +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + /* The code below is handling the return value of osFallocate() + ** correctly. posix_fallocate() is defined to "returns zero on success, + ** or an error number on failure". See the manpage for details. */ + int err; + do{ + err = osFallocate(pFile->h, buf.st_size, nSize-buf.st_size); + }while( err==EINTR ); + if( err && err!=EINVAL ) return SQLITE_IOERR_WRITE; +#else + /* If the OS does not have posix_fallocate(), fake it. Write a + ** single byte to the last byte in each block that falls entirely + ** within the extended region. Then, if required, a single byte + ** at offset (nSize-1), to set the size of the file correctly. + ** This is a similar technique to that used by glibc on systems + ** that do not have a real fallocate() call. + */ + int nBlk = buf.st_blksize; /* File-system block size */ + int nWrite = 0; /* Number of bytes written by seekAndWrite */ + i64 iWrite; /* Next offset to write to */ + + iWrite = (buf.st_size/nBlk)*nBlk + nBlk - 1; + assert( iWrite>=buf.st_size ); + assert( ((iWrite+1)%nBlk)==0 ); + for(/*no-op*/; iWrite=nSize ) iWrite = nSize - 1; + nWrite = seekAndWrite(pFile, iWrite, "", 1); + if( nWrite!=1 ) return SQLITE_IOERR_WRITE; + } +#endif + } + } + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->mmapSizeMax>0 && nByte>pFile->mmapSize ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + storeLastErrno(pFile, errno); + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } +#endif + + return SQLITE_OK; +} + +/* +** If *pArg is initially negative then this is a query. Set *pArg to +** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. +** +** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. +*/ +static void unixModeBit(unixFile *pFile, unsigned char mask, int *pArg){ + if( *pArg<0 ){ + *pArg = (pFile->ctrlFlags & mask)!=0; + }else if( (*pArg)==0 ){ + pFile->ctrlFlags &= ~mask; + }else{ + pFile->ctrlFlags |= mask; + } +} + +/* Forward declaration */ +static int unixGetTempname(int nBuf, char *zBuf); +static int unixFcntlExternalReader(unixFile*, int*); + +/* +** Information and control of an open file handle. +*/ +static int unixFileControl(sqlite3_file *id, int op, void *pArg){ + unixFile *pFile = (unixFile*)id; + switch( op ){ +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_START_ATOMIC_WRITE); + return rc ? SQLITE_IOERR_BEGIN_ATOMIC : SQLITE_OK; + } + case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_COMMIT_ATOMIC_WRITE); + return rc ? SQLITE_IOERR_COMMIT_ATOMIC : SQLITE_OK; + } + case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_ABORT_VOLATILE_WRITE); + return rc ? SQLITE_IOERR_ROLLBACK_ATOMIC : SQLITE_OK; + } +#endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + + case SQLITE_FCNTL_LOCKSTATE: { + *(int*)pArg = pFile->eFileLock; + return SQLITE_OK; + } + case SQLITE_FCNTL_LAST_ERRNO: { + *(int*)pArg = pFile->lastErrno; + return SQLITE_OK; + } + case SQLITE_FCNTL_CHUNK_SIZE: { + pFile->szChunk = *(int *)pArg; + return SQLITE_OK; + } + case SQLITE_FCNTL_SIZE_HINT: { + int rc; + SimulateIOErrorBenign(1); + rc = fcntlSizeHint(pFile, *(i64 *)pArg); + SimulateIOErrorBenign(0); + return rc; + } + case SQLITE_FCNTL_PERSIST_WAL: { + unixModeBit(pFile, UNIXFILE_PERSIST_WAL, (int*)pArg); + return SQLITE_OK; + } + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { + unixModeBit(pFile, UNIXFILE_PSOW, (int*)pArg); + return SQLITE_OK; + } + case SQLITE_FCNTL_VFSNAME: { + *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); + return SQLITE_OK; + } + case SQLITE_FCNTL_TEMPFILENAME: { + char *zTFile = sqlite3_malloc64( pFile->pVfs->mxPathname ); + if( zTFile ){ + unixGetTempname(pFile->pVfs->mxPathname, zTFile); + *(char**)pArg = zTFile; + } + return SQLITE_OK; + } + case SQLITE_FCNTL_HAS_MOVED: { + *(int*)pArg = fileHasMoved(pFile); + return SQLITE_OK; + } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + case SQLITE_FCNTL_LOCK_TIMEOUT: { + int iOld = pFile->iBusyTimeout; + pFile->iBusyTimeout = *(int*)pArg; + *(int*)pArg = iOld; + return SQLITE_OK; + } +#endif +#if SQLITE_MAX_MMAP_SIZE>0 + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + int rc = SQLITE_OK; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + + /* The value of newLimit may be eventually cast to (size_t) and passed + ** to mmap(). Restrict its value to 2GB if (size_t) is not at least a + ** 64-bit type. */ + if( newLimit>0 && sizeof(size_t)<8 ){ + newLimit = (newLimit & 0x7FFFFFFF); + } + + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ + pFile->mmapSizeMax = newLimit; + if( pFile->mmapSize>0 ){ + unixUnmapfile(pFile); + rc = unixMapfile(pFile, -1); + } + } + return rc; + } +#endif +#ifdef SQLITE_DEBUG + /* The pager calls this method to signal that it has done + ** a rollback and that the database is therefore unchanged and + ** it hence it is OK for the transaction change counter to be + ** unchanged. + */ + case SQLITE_FCNTL_DB_UNCHANGED: { + ((unixFile*)id)->dbUpdate = 0; + return SQLITE_OK; + } +#endif +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + case SQLITE_FCNTL_SET_LOCKPROXYFILE: + case SQLITE_FCNTL_GET_LOCKPROXYFILE: { + return proxyFileControl(id,op,pArg); + } +#endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ + + case SQLITE_FCNTL_EXTERNAL_READER: { + return unixFcntlExternalReader((unixFile*)id, (int*)pArg); + } + } + return SQLITE_NOTFOUND; +} + +/* +** If pFd->sectorSize is non-zero when this function is called, it is a +** no-op. Otherwise, the values of pFd->sectorSize and +** pFd->deviceCharacteristics are set according to the file-system +** characteristics. +** +** There are two versions of this function. One for QNX and one for all +** other systems. +*/ +#ifndef __QNXNTO__ +static void setDeviceCharacteristics(unixFile *pFd){ + assert( pFd->deviceCharacteristics==0 || pFd->sectorSize!=0 ); + if( pFd->sectorSize==0 ){ +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + int res; + u32 f = 0; + + /* Check for support for F2FS atomic batch writes. */ + res = osIoctl(pFd->h, F2FS_IOC_GET_FEATURES, &f); + if( res==0 && (f & F2FS_FEATURE_ATOMIC_WRITE) ){ + pFd->deviceCharacteristics = SQLITE_IOCAP_BATCH_ATOMIC; + } +#endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + + /* Set the POWERSAFE_OVERWRITE flag if requested. */ + if( pFd->ctrlFlags & UNIXFILE_PSOW ){ + pFd->deviceCharacteristics |= SQLITE_IOCAP_POWERSAFE_OVERWRITE; + } + + pFd->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + } +} +#else +#include +#include +static void setDeviceCharacteristics(unixFile *pFile){ + if( pFile->sectorSize == 0 ){ + struct statvfs fsInfo; + + /* Set defaults for non-supported filesystems */ + pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + pFile->deviceCharacteristics = 0; + if( fstatvfs(pFile->h, &fsInfo) == -1 ) { + return; + } + + if( !strcmp(fsInfo.f_basetype, "tmp") ) { + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC4K | /* All ram filesystem writes are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( strstr(fsInfo.f_basetype, "etfs") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* etfs cluster size writes are atomic */ + (pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) | + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( !strcmp(fsInfo.f_basetype, "qnx6") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC | /* All filesystem writes are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( !strcmp(fsInfo.f_basetype, "qnx4") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* full bitset of atomics from max sector size and smaller */ + ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else if( strstr(fsInfo.f_basetype, "dos") ){ + pFile->sectorSize = fsInfo.f_bsize; + pFile->deviceCharacteristics = + /* full bitset of atomics from max sector size and smaller */ + ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind + ** so it is ordered */ + 0; + }else{ + pFile->deviceCharacteristics = + SQLITE_IOCAP_ATOMIC512 | /* blocks are atomic */ + SQLITE_IOCAP_SAFE_APPEND | /* growing the file does not occur until + ** the write succeeds */ + 0; + } + } + /* Last chance verification. If the sector size isn't a multiple of 512 + ** then it isn't valid.*/ + if( pFile->sectorSize % 512 != 0 ){ + pFile->deviceCharacteristics = 0; + pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + } +} +#endif + +/* +** Return the sector size in bytes of the underlying block device for +** the specified file. This is almost always 512 bytes, but may be +** larger for some devices. +** +** SQLite code assumes this function cannot fail. It also assumes that +** if two files are created in the same file-system directory (i.e. +** a database and its journal file) that the sector size will be the +** same for both. +*/ +static int unixSectorSize(sqlite3_file *id){ + unixFile *pFd = (unixFile*)id; + setDeviceCharacteristics(pFd); + return pFd->sectorSize; +} + +/* +** Return the device characteristics for the file. +** +** This VFS is set up to return SQLITE_IOCAP_POWERSAFE_OVERWRITE by default. +** However, that choice is controversial since technically the underlying +** file system does not always provide powersafe overwrites. (In other +** words, after a power-loss event, parts of the file that were never +** written might end up being altered.) However, non-PSOW behavior is very, +** very rare. And asserting PSOW makes a large reduction in the amount +** of required I/O for journaling, since a lot of padding is eliminated. +** Hence, while POWERSAFE_OVERWRITE is on by default, there is a file-control +** available to turn it off and URI query parameter available to turn it off. +*/ +static int unixDeviceCharacteristics(sqlite3_file *id){ + unixFile *pFd = (unixFile*)id; + setDeviceCharacteristics(pFd); + return pFd->deviceCharacteristics; +} + +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + +/* +** Return the system page size. +** +** This function should not be called directly by other code in this file. +** Instead, it should be called via macro osGetpagesize(). +*/ +static int unixGetpagesize(void){ +#if OS_VXWORKS + return 1024; +#elif defined(_BSD_SOURCE) + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} + +#endif /* !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 */ + +#ifndef SQLITE_OMIT_WAL + +/* +** Object used to represent an shared memory buffer. +** +** When multiple threads all reference the same wal-index, each thread +** has its own unixShm object, but they all point to a single instance +** of this unixShmNode object. In other words, each wal-index is opened +** only once per process. +** +** Each unixShmNode object is connected to a single unixInodeInfo object. +** We could coalesce this object into unixInodeInfo, but that would mean +** every open file that does not use shared memory (in other words, most +** open files) would have to carry around this extra information. So +** the unixInodeInfo object contains a pointer to this unixShmNode object +** and the unixShmNode object is created only when needed. +** +** unixMutexHeld() must be true when creating or destroying +** this object or while reading or writing the following fields: +** +** nRef +** +** The following fields are read-only after the object is created: +** +** hShm +** zFilename +** +** Either unixShmNode.pShmMutex must be held or unixShmNode.nRef==0 and +** unixMutexHeld() is true when reading or writing any other field +** in this structure. +*/ +struct unixShmNode { + unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ + sqlite3_mutex *pShmMutex; /* Mutex to access this object */ + char *zFilename; /* Name of the mmapped file */ + int hShm; /* Open file descriptor */ + int szRegion; /* Size of shared-memory regions */ + u16 nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ + char **apRegion; /* Array of mapped shared-memory regions */ + int nRef; /* Number of unixShm objects pointing to this */ + unixShm *pFirst; /* All unixShm objects pointing to this */ + int aLock[SQLITE_SHM_NLOCK]; /* # shared locks on slot, -1==excl lock */ +#ifdef SQLITE_DEBUG + u8 exclMask; /* Mask of exclusive locks held */ + u8 sharedMask; /* Mask of shared locks held */ + u8 nextShmId; /* Next available unixShm.id value */ +#endif +}; + +/* +** Structure used internally by this VFS to record the state of an +** open shared memory connection. +** +** The following fields are initialized when this object is created and +** are read-only thereafter: +** +** unixShm.pShmNode +** unixShm.id +** +** All other fields are read/write. The unixShm.pShmNode->pShmMutex must +** be held while accessing any read/write fields. +*/ +struct unixShm { + unixShmNode *pShmNode; /* The underlying unixShmNode object */ + unixShm *pNext; /* Next unixShm with the same unixShmNode */ + u8 hasMutex; /* True if holding the unixShmNode->pShmMutex */ + u8 id; /* Id of this connection within its unixShmNode */ + u16 sharedMask; /* Mask of shared locks held */ + u16 exclMask; /* Mask of exclusive locks held */ +}; + +/* +** Constants used for locking +*/ +#define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ +#define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ + +/* +** Use F_GETLK to check whether or not there are any readers with open +** wal-mode transactions in other processes on database file pFile. If +** no error occurs, return SQLITE_OK and set (*piOut) to 1 if there are +** such transactions, or 0 otherwise. If an error occurs, return an +** SQLite error code. The final value of *piOut is undefined in this +** case. +*/ +static int unixFcntlExternalReader(unixFile *pFile, int *piOut){ + int rc = SQLITE_OK; + *piOut = 0; + if( pFile->pShm){ + unixShmNode *pShmNode = pFile->pShm->pShmNode; + struct flock f; + + memset(&f, 0, sizeof(f)); + f.l_type = F_WRLCK; + f.l_whence = SEEK_SET; + f.l_start = UNIX_SHM_BASE + 3; + f.l_len = SQLITE_SHM_NLOCK - 3; + + sqlite3_mutex_enter(pShmNode->pShmMutex); + if( osFcntl(pShmNode->hShm, F_GETLK, &f)<0 ){ + rc = SQLITE_IOERR_LOCK; + }else{ + *piOut = (f.l_type!=F_UNLCK); + } + sqlite3_mutex_leave(pShmNode->pShmMutex); + } + + return rc; +} + + +/* +** Apply posix advisory locks for all bytes from ofst through ofst+n-1. +** +** Locks block if the mask is exactly UNIX_SHM_C and are non-blocking +** otherwise. +*/ +static int unixShmSystemLock( + unixFile *pFile, /* Open connection to the WAL file */ + int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ + int ofst, /* First byte of the locking range */ + int n /* Number of bytes to lock */ +){ + unixShmNode *pShmNode; /* Apply locks to this open shared-memory segment */ + struct flock f; /* The posix advisory locking structure */ + int rc = SQLITE_OK; /* Result code form fcntl() */ + + /* Access to the unixShmNode object is serialized by the caller */ + pShmNode = pFile->pInode->pShmNode; + assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); + assert( pShmNode->nRef>0 || unixMutexHeld() ); + + /* Shared locks never span more than one byte */ + assert( n==1 || lockType!=F_RDLCK ); + + /* Locks are within range */ + assert( n>=1 && n<=SQLITE_SHM_NLOCK ); + + if( pShmNode->hShm>=0 ){ + int res; + /* Initialize the locking parameters */ + f.l_type = lockType; + f.l_whence = SEEK_SET; + f.l_start = ofst; + f.l_len = n; + res = osSetPosixAdvisoryLock(pShmNode->hShm, &f, pFile); + if( res==-1 ){ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + rc = (pFile->iBusyTimeout ? SQLITE_BUSY_TIMEOUT : SQLITE_BUSY); +#else + rc = SQLITE_BUSY; +#endif + } + } + + /* Update the global lock state and do debug tracing */ +#ifdef SQLITE_DEBUG + { u16 mask; + OSTRACE(("SHM-LOCK ")); + mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; + pShmNode->sharedMask &= ~mask; + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock %d ok", ofst)); + pShmNode->exclMask &= ~mask; + pShmNode->sharedMask |= mask; + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d ok", ofst)); + pShmNode->exclMask |= mask; + pShmNode->sharedMask &= ~mask; + } + }else{ + if( lockType==F_UNLCK ){ + OSTRACE(("unlock %d failed", ofst)); + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock failed")); + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock %d failed", ofst)); + } + } + OSTRACE((" - afterwards %03x,%03x\n", + pShmNode->sharedMask, pShmNode->exclMask)); + } +#endif + + return rc; +} + +/* +** Return the minimum number of 32KB shm regions that should be mapped at +** a time, assuming that each mapping must be an integer multiple of the +** current system page-size. +** +** Usually, this is 1. The exception seems to be systems that are configured +** to use 64KB pages - in this case each mapping must cover at least two +** shm regions. +*/ +static int unixShmRegionPerMap(void){ + int shmsz = 32*1024; /* SHM region size */ + int pgsz = osGetpagesize(); /* System page size */ + assert( ((pgsz-1)&pgsz)==0 ); /* Page size must be a power of 2 */ + if( pgszpInode->pShmNode; + assert( unixMutexHeld() ); + if( p && ALWAYS(p->nRef==0) ){ + int nShmPerMap = unixShmRegionPerMap(); + int i; + assert( p->pInode==pFd->pInode ); + sqlite3_mutex_free(p->pShmMutex); + for(i=0; inRegion; i+=nShmPerMap){ + if( p->hShm>=0 ){ + osMunmap(p->apRegion[i], p->szRegion); + }else{ + sqlite3_free(p->apRegion[i]); + } + } + sqlite3_free(p->apRegion); + if( p->hShm>=0 ){ + robust_close(pFd, p->hShm, __LINE__); + p->hShm = -1; + } + p->pInode->pShmNode = 0; + sqlite3_free(p); + } +} + +/* +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +*/ +static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ + struct flock lock; + int rc = SQLITE_OK; + + /* Use F_GETLK to determine the locks other processes are holding + ** on the DMS byte. If it indicates that another process is holding + ** a SHARED lock, then this process may also take a SHARED lock + ** and proceed with opening the *-shm file. + ** + ** Or, if no other process is holding any lock, then this process + ** is the first to open it. In this case take an EXCLUSIVE lock on the + ** DMS byte and truncate the *-shm file to zero bytes in size. Then + ** downgrade to a SHARED lock on the DMS byte. + ** + ** If another process is holding an EXCLUSIVE lock on the DMS byte, + ** return SQLITE_BUSY to the caller (it will try again). An earlier + ** version of this code attempted the SHARED lock at this point. But + ** this introduced a subtle race condition: if the process holding + ** EXCLUSIVE failed just before truncating the *-shm file, then this + ** process might open and use the *-shm file without truncating it. + ** And if the *-shm file has been corrupted by a power failure or + ** system crash, the database itself may also become corrupt. */ + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pShmNode->hShm, F_GETLK, &lock)!=0 ) { + rc = SQLITE_IOERR_LOCK; + }else if( lock.l_type==F_UNLCK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + rc = SQLITE_READONLY_CANTINIT; + }else{ + rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); + /* The first connection to attach must truncate the -shm file. We + ** truncate to 3 bytes (an arbitrary small number, less than the + ** -shm header size) rather than 0 as a system debugging aid, to + ** help detect if a -shm file truncation is legitimate or is the work + ** or a rogue process. */ + if( rc==SQLITE_OK && robust_ftruncate(pShmNode->hShm, 3) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename); + } + } + }else if( lock.l_type==F_WRLCK ){ + rc = SQLITE_BUSY; + } + + if( rc==SQLITE_OK ){ + assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); + rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); + } + return rc; +} + +/* +** Open a shared-memory area associated with open database file pDbFd. +** This particular implementation uses mmapped files. +** +** The file used to implement shared-memory is in the same directory +** as the open database file and has the same name as the open database +** file with the "-shm" suffix added. For example, if the database file +** is "/home/user1/config.db" then the file that is created and mmapped +** for shared memory will be called "/home/user1/config.db-shm". +** +** Another approach to is to use files in /dev/shm or /dev/tmp or an +** some other tmpfs mount. But if a file in a different directory +** from the database file is used, then differing access permissions +** or a chroot() might cause two different processes on the same +** database to end up using different files for shared memory - +** meaning that their memory would not really be shared - resulting +** in database corruption. Nevertheless, this tmpfs file usage +** can be enabled at compile-time using -DSQLITE_SHM_DIRECTORY="/dev/shm" +** or the equivalent. The use of the SQLITE_SHM_DIRECTORY compile-time +** option results in an incompatible build of SQLite; builds of SQLite +** that with differing SQLITE_SHM_DIRECTORY settings attempt to use the +** same database file at the same time, database corruption will likely +** result. The SQLITE_SHM_DIRECTORY compile-time option is considered +** "unsupported" and may go away in a future SQLite release. +** +** When opening a new shared-memory file, if no other instances of that +** file are currently open, in this process or in other processes, then +** the file must be truncated to zero length or have its header cleared. +** +** If the original database file (pDbFd) is using the "unix-excl" VFS +** that means that an exclusive lock is held on the database file and +** that no other processes are able to read or write the database. In +** that case, we do not really need shared memory. No shared memory +** file is created. The shared memory will be simulated with heap memory. +*/ +static int unixOpenSharedMemory(unixFile *pDbFd){ + struct unixShm *p = 0; /* The connection to be opened */ + struct unixShmNode *pShmNode; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ + unixInodeInfo *pInode; /* The inode of fd */ + char *zShm; /* Name of the file used for SHM */ + int nShmFilename; /* Size of the SHM filename in bytes */ + + /* Allocate space for the new unixShm object. */ + p = sqlite3_malloc64( sizeof(*p) ); + if( p==0 ) return SQLITE_NOMEM_BKPT; + memset(p, 0, sizeof(*p)); + assert( pDbFd->pShm==0 ); + + /* Check to see if a unixShmNode object already exists. Reuse an existing + ** one if present. Create a new one if necessary. + */ + assert( unixFileMutexNotheld(pDbFd) ); + unixEnterMutex(); + pInode = pDbFd->pInode; + pShmNode = pInode->pShmNode; + if( pShmNode==0 ){ + struct stat sStat; /* fstat() info for database file */ +#ifndef SQLITE_SHM_DIRECTORY + const char *zBasePath = pDbFd->zPath; +#endif + + /* Call fstat() to figure out the permissions on the database file. If + ** a new *-shm file is created, an attempt will be made to create it + ** with the same permissions. + */ + if( osFstat(pDbFd->h, &sStat) ){ + rc = SQLITE_IOERR_FSTAT; + goto shm_open_err; + } + +#ifdef SQLITE_SHM_DIRECTORY + nShmFilename = sizeof(SQLITE_SHM_DIRECTORY) + 31; +#else + nShmFilename = 6 + (int)strlen(zBasePath); +#endif + pShmNode = sqlite3_malloc64( sizeof(*pShmNode) + nShmFilename ); + if( pShmNode==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } + memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); + zShm = pShmNode->zFilename = (char*)&pShmNode[1]; +#ifdef SQLITE_SHM_DIRECTORY + sqlite3_snprintf(nShmFilename, zShm, + SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", + (u32)sStat.st_ino, (u32)sStat.st_dev); +#else + sqlite3_snprintf(nShmFilename, zShm, "%s-shm", zBasePath); + sqlite3FileSuffix3(pDbFd->zPath, zShm); +#endif + pShmNode->hShm = -1; + pDbFd->pInode->pShmNode = pShmNode; + pShmNode->pInode = pDbFd->pInode; + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->pShmMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->pShmMutex==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } + } + + if( pInode->bProcessLock==0 ){ + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + pShmNode->hShm = robust_open(zShm, O_RDWR|O_CREAT|O_NOFOLLOW, + (sStat.st_mode&0777)); + } + if( pShmNode->hShm<0 ){ + pShmNode->hShm = robust_open(zShm, O_RDONLY|O_NOFOLLOW, + (sStat.st_mode&0777)); + if( pShmNode->hShm<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShm); + goto shm_open_err; + } + pShmNode->isReadonly = 1; + } + + /* If this process is running as root, make sure that the SHM file + ** is owned by the same user that owns the original database. Otherwise, + ** the original owner will not be able to connect. + */ + robustFchown(pShmNode->hShm, sStat.st_uid, sStat.st_gid); + + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; + } + } + + /* Make the new connection a child of the unixShmNode */ + p->pShmNode = pShmNode; +#ifdef SQLITE_DEBUG + p->id = pShmNode->nextShmId++; +#endif + pShmNode->nRef++; + pDbFd->pShm = p; + unixLeaveMutex(); + + /* The reference count on pShmNode has already been incremented under + ** the cover of the unixEnterMutex() mutex and the pointer from the + ** new (struct unixShm) object to the pShmNode has been set. All that is + ** left to do is to link the new object into the linked list starting + ** at pShmNode->pFirst. This must be done while holding the + ** pShmNode->pShmMutex. + */ + sqlite3_mutex_enter(pShmNode->pShmMutex); + p->pNext = pShmNode->pFirst; + pShmNode->pFirst = p; + sqlite3_mutex_leave(pShmNode->pShmMutex); + return rc; + + /* Jump here on any error */ +shm_open_err: + unixShmPurge(pDbFd); /* This call frees pShmNode if required */ + sqlite3_free(p); + unixLeaveMutex(); + return rc; +} + +/* +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. +** +** If an error occurs, an error code is returned and *pp is set to NULL. +** +** Otherwise, if the bExtend parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** bExtend is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. +*/ +static int unixShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int bExtend, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + unixFile *pDbFd = (unixFile*)fd; + unixShm *p; + unixShmNode *pShmNode; + int rc = SQLITE_OK; + int nShmPerMap = unixShmRegionPerMap(); + int nReqRegion; + + /* If the shared-memory file has not yet been opened, open it now. */ + if( pDbFd->pShm==0 ){ + rc = unixOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; + } + + p = pDbFd->pShm; + pShmNode = p->pShmNode; + sqlite3_mutex_enter(pShmNode->pShmMutex); + if( pShmNode->isUnlocked ){ + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + assert( pShmNode->pInode==pDbFd->pInode ); + assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); + + /* Minimum number of regions required to be mapped. */ + nReqRegion = ((iRegion+nShmPerMap) / nShmPerMap) * nShmPerMap; + + if( pShmNode->nRegionszRegion = szRegion; + + if( pShmNode->hShm>=0 ){ + /* The requested region is not mapped into this processes address space. + ** Check to see if it has been allocated (i.e. if the wal-index file is + ** large enough to contain the requested region). + */ + if( osFstat(pShmNode->hShm, &sStat) ){ + rc = SQLITE_IOERR_SHMSIZE; + goto shmpage_out; + } + + if( sStat.st_sizehShm, iPg*pgsz + pgsz-1,"",1,&x)!=1 ){ + const char *zFile = pShmNode->zFilename; + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); + goto shmpage_out; + } + } + } + } + } + + /* Map the requested memory region into this processes address space. */ + apNew = (char **)sqlite3_realloc( + pShmNode->apRegion, nReqRegion*sizeof(char *) + ); + if( !apNew ){ + rc = SQLITE_IOERR_NOMEM_BKPT; + goto shmpage_out; + } + pShmNode->apRegion = apNew; + while( pShmNode->nRegionhShm>=0 ){ + pMem = osMmap(0, nMap, + pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, + MAP_SHARED, pShmNode->hShm, szRegion*(i64)pShmNode->nRegion + ); + if( pMem==MAP_FAILED ){ + rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); + goto shmpage_out; + } + }else{ + pMem = sqlite3_malloc64(nMap); + if( pMem==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shmpage_out; + } + memset(pMem, 0, nMap); + } + + for(i=0; iapRegion[pShmNode->nRegion+i] = &((char*)pMem)[szRegion*i]; + } + pShmNode->nRegion += nShmPerMap; + } + } + +shmpage_out: + if( pShmNode->nRegion>iRegion ){ + *pp = pShmNode->apRegion[iRegion]; + }else{ + *pp = 0; + } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; + sqlite3_mutex_leave(pShmNode->pShmMutex); + return rc; +} + +/* +** Check that the pShmNode->aLock[] array comports with the locking bitmasks +** held by each client. Return true if it does, or false otherwise. This +** is to be used in an assert(). e.g. +** +** assert( assertLockingArrayOk(pShmNode) ); +*/ +#ifdef SQLITE_DEBUG +static int assertLockingArrayOk(unixShmNode *pShmNode){ + unixShm *pX; + int aLock[SQLITE_SHM_NLOCK]; + assert( sqlite3_mutex_held(pShmNode->pShmMutex) ); + + memset(aLock, 0, sizeof(aLock)); + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + int i; + for(i=0; iexclMask & (1<sharedMask & (1<=0 ); + aLock[i]++; + } + } + } + + assert( 0==memcmp(pShmNode->aLock, aLock, sizeof(aLock)) ); + return (memcmp(pShmNode->aLock, aLock, sizeof(aLock))==0); +} +#endif + +/* +** Change the lock state for a shared-memory segment. +** +** Note that the relationship between SHAREd and EXCLUSIVE locks is a little +** different here than in posix. In xShmLock(), one can go from unlocked +** to shared and back or from unlocked to exclusive and back. But one may +** not go from shared to exclusive or from exclusive to shared. +*/ +static int unixShmLock( + sqlite3_file *fd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + unixFile *pDbFd = (unixFile*)fd; /* Connection holding shared memory */ + unixShm *p = pDbFd->pShm; /* The shared memory being locked */ + unixShmNode *pShmNode = p->pShmNode; /* The underlying file iNode */ + int rc = SQLITE_OK; /* Result code */ + u16 mask; /* Mask of locks to take or release */ + int *aLock = pShmNode->aLock; + + assert( pShmNode==pDbFd->pInode->pShmNode ); + assert( pShmNode->pInode==pDbFd->pInode ); + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); + assert( n>=1 ); + assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); + assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); + assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); + + /* Check that, if this to be a blocking lock, no locks that occur later + ** in the following list than the lock being obtained are already held: + ** + ** 1. Checkpointer lock (ofst==1). + ** 2. Write lock (ofst==0). + ** 3. Read locks (ofst>=3 && ofstiBusyTimeout==0 || ( + (ofst!=2) /* not RECOVER */ + && (ofst!=1 || (p->exclMask|p->sharedMask)==0) + && (ofst!=0 || (p->exclMask|p->sharedMask)<3) + && (ofst<3 || (p->exclMask|p->sharedMask)<(1<1 || mask==(1<pShmMutex); + assert( assertLockingArrayOk(pShmNode) ); + if( flags & SQLITE_SHM_UNLOCK ){ + if( (p->exclMask|p->sharedMask) & mask ){ + int ii; + int bUnlock = 1; + + for(ii=ofst; ii((p->sharedMask & (1<sharedMask & (1<1 ); + aLock[ofst]--; + } + + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~mask; + p->sharedMask &= ~mask; + } + } + }else if( flags & SQLITE_SHM_SHARED ){ + assert( n==1 ); + assert( (p->exclMask & (1<sharedMask & mask)==0 ){ + if( aLock[ofst]<0 ){ + rc = SQLITE_BUSY; + }else if( aLock[ofst]==0 ){ + rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); + } + + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; + aLock[ofst]++; + } + } + }else{ + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. */ + int ii; + for(ii=ofst; iisharedMask & mask)==0 ); + if( ALWAYS((p->exclMask & (1<sharedMask & mask)==0 ); + p->exclMask |= mask; + for(ii=ofst; iipShmMutex); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", + p->id, osGetpid(0), p->sharedMask, p->exclMask)); + return rc; +} + +/* +** Implement a memory barrier or memory fence on shared memory. +** +** All loads and stores begun before the barrier must complete before +** any load or store begun after the barrier. +*/ +static void unixShmBarrier( + sqlite3_file *fd /* Database file holding the shared memory */ +){ + UNUSED_PARAMETER(fd); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + assert( fd->pMethods->xLock==nolockLock + || unixFileMutexNotheld((unixFile*)fd) + ); + unixEnterMutex(); /* Also mutex, for redundancy */ + unixLeaveMutex(); +} + +/* +** Close a connection to shared-memory. Delete the underlying +** storage if deleteFlag is true. +** +** If there is no shared memory associated with the connection then this +** routine is a harmless no-op. +*/ +static int unixShmUnmap( + sqlite3_file *fd, /* The underlying database file */ + int deleteFlag /* Delete shared-memory if true */ +){ + unixShm *p; /* The connection to be closed */ + unixShmNode *pShmNode; /* The underlying shared-memory file */ + unixShm **pp; /* For looping over sibling connections */ + unixFile *pDbFd; /* The underlying database file */ + + pDbFd = (unixFile*)fd; + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + pShmNode = p->pShmNode; + + assert( pShmNode==pDbFd->pInode->pShmNode ); + assert( pShmNode->pInode==pDbFd->pInode ); + + /* Remove connection p from the set of connections associated + ** with pShmNode */ + sqlite3_mutex_enter(pShmNode->pShmMutex); + for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} + *pp = p->pNext; + + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; + sqlite3_mutex_leave(pShmNode->pShmMutex); + + /* If pShmNode->nRef has reached 0, then close the underlying + ** shared-memory file, too */ + assert( unixFileMutexNotheld(pDbFd) ); + unixEnterMutex(); + assert( pShmNode->nRef>0 ); + pShmNode->nRef--; + if( pShmNode->nRef==0 ){ + if( deleteFlag && pShmNode->hShm>=0 ){ + osUnlink(pShmNode->zFilename); + } + unixShmPurge(pDbFd); + } + unixLeaveMutex(); + + return SQLITE_OK; +} + + +#else +# define unixShmMap 0 +# define unixShmLock 0 +# define unixShmBarrier 0 +# define unixShmUnmap 0 +#endif /* #ifndef SQLITE_OMIT_WAL */ + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** If it is currently memory mapped, unmap file pFd. +*/ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); + if( pFd->pMapRegion ){ + osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapSizeActual = 0; + } +} + +/* +** Attempt to set the size of the memory mapping maintained by file +** descriptor pFd to nNew bytes. Any existing mapping is discarded. +** +** If successful, this function sets the following variables: +** +** unixFile.pMapRegion +** unixFile.mmapSize +** unixFile.mmapSizeActual +** +** If unsuccessful, an error message is logged via sqlite3_log() and +** the three variables above are zeroed. In this case SQLite should +** continue accessing the database using the xRead() and xWrite() +** methods. +*/ +static void unixRemapfile( + unixFile *pFd, /* File descriptor object */ + i64 nNew /* Required mapping size */ +){ + const char *zErr = "mmap"; + int h = pFd->h; /* File descriptor open on db file */ + u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ + i64 nOrig = pFd->mmapSizeActual; /* Size of pOrig region in bytes */ + u8 *pNew = 0; /* Location of new mapping */ + int flags = PROT_READ; /* Flags to pass to mmap() */ + + assert( pFd->nFetchOut==0 ); + assert( nNew>pFd->mmapSize ); + assert( nNew<=pFd->mmapSizeMax ); + assert( nNew>0 ); + assert( pFd->mmapSizeActual>=pFd->mmapSize ); + assert( MAP_FAILED!=0 ); + +#ifdef SQLITE_MMAP_READWRITE + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; +#endif + + if( pOrig ){ +#if HAVE_MREMAP + i64 nReuse = pFd->mmapSize; +#else + const int szSyspage = osGetpagesize(); + i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); +#endif + u8 *pReq = &pOrig[nReuse]; + + /* Unmap any pages of the existing mapping that cannot be reused. */ + if( nReuse!=nOrig ){ + osMunmap(pReq, nOrig-nReuse); + } + +#if HAVE_MREMAP + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; +#else + pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=pReq ){ + osMunmap(pNew, nNew - nReuse); + pNew = 0; + }else{ + pNew = pOrig; + } + } +#endif + + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ + osMunmap(pOrig, nReuse); + } + } + + /* If pNew is still NULL, try to create an entirely new mapping. */ + if( pNew==0 ){ + pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); + } + + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, zErr, pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapSizeMax = 0; + } + pFd->pMapRegion = (void *)pNew; + pFd->mmapSize = pFd->mmapSizeActual = nNew; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int unixMapfile(unixFile *pFd, i64 nMap){ + assert( nMap>=0 || pFd->nFetchOut==0 ); + assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + struct stat statbuf; /* Low-level file information */ + if( osFstat(pFd->h, &statbuf) ){ + return SQLITE_IOERR_FSTAT; + } + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + + assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); + if( nMap!=pFd->mmapSize ){ + unixRemapfile(pFd, nMap); + } + + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ +#endif + *pp = 0; + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + UNUSED_PARAMETER(iOff); + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); +#else + UNUSED_PARAMETER(fd); + UNUSED_PARAMETER(p); + UNUSED_PARAMETER(iOff); +#endif + return SQLITE_OK; +} + +/* +** Here ends the implementation of all sqlite3_file methods. +** +********************** End sqlite3_file Methods ******************************* +******************************************************************************/ + +/* +** This division contains definitions of sqlite3_io_methods objects that +** implement various file locking strategies. It also contains definitions +** of "finder" functions. A finder-function is used to locate the appropriate +** sqlite3_io_methods object for a particular database file. The pAppData +** field of the sqlite3_vfs VFS objects are initialized to be pointers to +** the correct finder-function for that VFS. +** +** Most finder functions return a pointer to a fixed sqlite3_io_methods +** object. The only interesting finder-function is autolockIoFinder, which +** looks at the filesystem type and tries to guess the best locking +** strategy from that. +** +** For finder-function F, two objects are created: +** +** (1) The real finder-function named "FImpt()". +** +** (2) A constant pointer to this function named just "F". +** +** +** A pointer to the F pointer is used as the pAppData value for VFS +** objects. We have to do this instead of letting pAppData point +** directly at the finder-function since C90 rules prevent a void* +** from be cast into a function pointer. +** +** +** Each instance of this macro generates two objects: +** +** * A constant sqlite3_io_methods object call METHOD that has locking +** methods CLOSE, LOCK, UNLOCK, CKRESLOCK. +** +** * An I/O method finder function called FINDER that returns a pointer +** to the METHOD object in the previous bullet. +*/ +#define IOMETHODS(FINDER,METHOD,VERSION,CLOSE,LOCK,UNLOCK,CKLOCK,SHMMAP) \ +static const sqlite3_io_methods METHOD = { \ + VERSION, /* iVersion */ \ + CLOSE, /* xClose */ \ + unixRead, /* xRead */ \ + unixWrite, /* xWrite */ \ + unixTruncate, /* xTruncate */ \ + unixSync, /* xSync */ \ + unixFileSize, /* xFileSize */ \ + LOCK, /* xLock */ \ + UNLOCK, /* xUnlock */ \ + CKLOCK, /* xCheckReservedLock */ \ + unixFileControl, /* xFileControl */ \ + unixSectorSize, /* xSectorSize */ \ + unixDeviceCharacteristics, /* xDeviceCapabilities */ \ + SHMMAP, /* xShmMap */ \ + unixShmLock, /* xShmLock */ \ + unixShmBarrier, /* xShmBarrier */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ +}; \ +static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ + UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ + return &METHOD; \ +} \ +static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ + = FINDER##Impl; + +/* +** Here are all of the sqlite3_io_methods objects for each of the +** locking strategies. Functions that return pointers to these methods +** are also created. +*/ +IOMETHODS( + posixIoFinder, /* Finder function name */ + posixIoMethods, /* sqlite3_io_methods object name */ + 3, /* shared memory and mmap are enabled */ + unixClose, /* xClose method */ + unixLock, /* xLock method */ + unixUnlock, /* xUnlock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + unixShmMap /* xShmMap method */ +) +IOMETHODS( + nolockIoFinder, /* Finder function name */ + nolockIoMethods, /* sqlite3_io_methods object name */ + 3, /* shared memory and mmap are enabled */ + nolockClose, /* xClose method */ + nolockLock, /* xLock method */ + nolockUnlock, /* xUnlock method */ + nolockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +IOMETHODS( + dotlockIoFinder, /* Finder function name */ + dotlockIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + dotlockClose, /* xClose method */ + dotlockLock, /* xLock method */ + dotlockUnlock, /* xUnlock method */ + dotlockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) + +#if SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + flockIoFinder, /* Finder function name */ + flockIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + flockClose, /* xClose method */ + flockLock, /* xLock method */ + flockUnlock, /* xUnlock method */ + flockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +#endif + +#if OS_VXWORKS +IOMETHODS( + semIoFinder, /* Finder function name */ + semIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + semXClose, /* xClose method */ + semXLock, /* xLock method */ + semXUnlock, /* xUnlock method */ + semXCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +#endif + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + afpIoFinder, /* Finder function name */ + afpIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + afpClose, /* xClose method */ + afpLock, /* xLock method */ + afpUnlock, /* xUnlock method */ + afpCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +#endif + +/* +** The proxy locking method is a "super-method" in the sense that it +** opens secondary file descriptors for the conch and lock files and +** it uses proxy, dot-file, AFP, and flock() locking methods on those +** secondary files. For this reason, the division that implements +** proxy locking is located much further down in the file. But we need +** to go ahead and define the sqlite3_io_methods and finder function +** for proxy locking here. So we forward declare the I/O methods. +*/ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +static int proxyClose(sqlite3_file*); +static int proxyLock(sqlite3_file*, int); +static int proxyUnlock(sqlite3_file*, int); +static int proxyCheckReservedLock(sqlite3_file*, int*); +IOMETHODS( + proxyIoFinder, /* Finder function name */ + proxyIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + proxyClose, /* xClose method */ + proxyLock, /* xLock method */ + proxyUnlock, /* xUnlock method */ + proxyCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +#endif + +/* nfs lockd on OSX 10.3+ doesn't clear write locks when a read lock is set */ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +IOMETHODS( + nfsIoFinder, /* Finder function name */ + nfsIoMethods, /* sqlite3_io_methods object name */ + 1, /* shared memory is disabled */ + unixClose, /* xClose method */ + unixLock, /* xLock method */ + nfsUnlock, /* xUnlock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ +) +#endif + +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE +/* +** This "finder" function attempts to determine the best locking strategy +** for the database file "filePath". It then returns the sqlite3_io_methods +** object that implements that strategy. +** +** This is for MacOSX only. +*/ +static const sqlite3_io_methods *autolockIoFinderImpl( + const char *filePath, /* name of the database file */ + unixFile *pNew /* open file object for the database file */ +){ + static const struct Mapping { + const char *zFilesystem; /* Filesystem type name */ + const sqlite3_io_methods *pMethods; /* Appropriate locking method */ + } aMap[] = { + { "hfs", &posixIoMethods }, + { "ufs", &posixIoMethods }, + { "afpfs", &afpIoMethods }, + { "smbfs", &afpIoMethods }, + { "webdav", &nolockIoMethods }, + { 0, 0 } + }; + int i; + struct statfs fsInfo; + struct flock lockInfo; + + if( !filePath ){ + /* If filePath==NULL that means we are dealing with a transient file + ** that does not need to be locked. */ + return &nolockIoMethods; + } + if( statfs(filePath, &fsInfo) != -1 ){ + if( fsInfo.f_flags & MNT_RDONLY ){ + return &nolockIoMethods; + } + for(i=0; aMap[i].zFilesystem; i++){ + if( strcmp(fsInfo.f_fstypename, aMap[i].zFilesystem)==0 ){ + return aMap[i].pMethods; + } + } + } + + /* Default case. Handles, amongst others, "nfs". + ** Test byte-range lock using fcntl(). If the call succeeds, + ** assume that the file-system supports POSIX style locks. + */ + lockInfo.l_len = 1; + lockInfo.l_start = 0; + lockInfo.l_whence = SEEK_SET; + lockInfo.l_type = F_RDLCK; + if( osFcntl(pNew->h, F_GETLK, &lockInfo)!=-1 ) { + if( strcmp(fsInfo.f_fstypename, "nfs")==0 ){ + return &nfsIoMethods; + } else { + return &posixIoMethods; + } + }else{ + return &dotlockIoMethods; + } +} +static const sqlite3_io_methods + *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ + +#if OS_VXWORKS +/* +** This "finder" function for VxWorks checks to see if posix advisory +** locking works. If it does, then that is what is used. If it does not +** work, then fallback to named semaphore locking. +*/ +static const sqlite3_io_methods *vxworksIoFinderImpl( + const char *filePath, /* name of the database file */ + unixFile *pNew /* the open file object */ +){ + struct flock lockInfo; + + if( !filePath ){ + /* If filePath==NULL that means we are dealing with a transient file + ** that does not need to be locked. */ + return &nolockIoMethods; + } + + /* Test if fcntl() is supported and use POSIX style locks. + ** Otherwise fall back to the named semaphore method. + */ + lockInfo.l_len = 1; + lockInfo.l_start = 0; + lockInfo.l_whence = SEEK_SET; + lockInfo.l_type = F_RDLCK; + if( osFcntl(pNew->h, F_GETLK, &lockInfo)!=-1 ) { + return &posixIoMethods; + }else{ + return &semIoMethods; + } +} +static const sqlite3_io_methods + *(*const vxworksIoFinder)(const char*,unixFile*) = vxworksIoFinderImpl; + +#endif /* OS_VXWORKS */ + +/* +** An abstract type for a pointer to an IO method finder function: +*/ +typedef const sqlite3_io_methods *(*finder_type)(const char*,unixFile*); + + +/**************************************************************************** +**************************** sqlite3_vfs methods **************************** +** +** This division contains the implementation of methods on the +** sqlite3_vfs object. +*/ + +/* +** Initialize the contents of the unixFile structure pointed to by pId. +*/ +static int fillInUnixFile( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + int h, /* Open file descriptor of file being opened */ + sqlite3_file *pId, /* Write to the unixFile structure here */ + const char *zFilename, /* Name of the file being opened */ + int ctrlFlags /* Zero or more UNIXFILE_* values */ +){ + const sqlite3_io_methods *pLockingStyle; + unixFile *pNew = (unixFile *)pId; + int rc = SQLITE_OK; + + assert( pNew->pInode==NULL ); + + /* No locking occurs in temporary files */ + assert( zFilename!=0 || (ctrlFlags & UNIXFILE_NOLOCK)!=0 ); + + OSTRACE(("OPEN %-3d %s\n", h, zFilename)); + pNew->h = h; + pNew->pVfs = pVfs; + pNew->zPath = zFilename; + pNew->ctrlFlags = (u8)ctrlFlags; +#if SQLITE_MAX_MMAP_SIZE>0 + pNew->mmapSizeMax = sqlite3GlobalConfig.szMmap; +#endif + if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), + "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + pNew->ctrlFlags |= UNIXFILE_PSOW; + } + if( strcmp(pVfs->zName,"unix-excl")==0 ){ + pNew->ctrlFlags |= UNIXFILE_EXCL; + } + +#if OS_VXWORKS + pNew->pId = vxworksFindFileId(zFilename); + if( pNew->pId==0 ){ + ctrlFlags |= UNIXFILE_NOLOCK; + rc = SQLITE_NOMEM_BKPT; + } +#endif + + if( ctrlFlags & UNIXFILE_NOLOCK ){ + pLockingStyle = &nolockIoMethods; + }else{ + pLockingStyle = (**(finder_type*)pVfs->pAppData)(zFilename, pNew); +#if SQLITE_ENABLE_LOCKING_STYLE + /* Cache zFilename in the locking context (AFP and dotlock override) for + ** proxyLock activation is possible (remote proxy is based on db name) + ** zFilename remains valid until file is closed, to support */ + pNew->lockingContext = (void*)zFilename; +#endif + } + + if( pLockingStyle == &posixIoMethods +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + || pLockingStyle == &nfsIoMethods +#endif + ){ + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( rc!=SQLITE_OK ){ + /* If an error occurred in findInodeInfo(), close the file descriptor + ** immediately, before releasing the mutex. findInodeInfo() may fail + ** in two scenarios: + ** + ** (a) A call to fstat() failed. + ** (b) A malloc failed. + ** + ** Scenario (b) may only occur if the process is holding no other + ** file descriptors open on the same file. If there were other file + ** descriptors on this file, then no malloc would be required by + ** findInodeInfo(). If this is the case, it is quite safe to close + ** handle h - as it is guaranteed that no posix locks will be released + ** by doing so. + ** + ** If scenario (a) caused the error then things are not so safe. The + ** implicit assumption here is that if fstat() fails, things are in + ** such bad shape that dropping a lock or two doesn't matter much. + */ + robust_close(pNew, h, __LINE__); + h = -1; + } + unixLeaveMutex(); + } + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + else if( pLockingStyle == &afpIoMethods ){ + /* AFP locking uses the file path so it needs to be included in + ** the afpLockingContext. + */ + afpLockingContext *pCtx; + pNew->lockingContext = pCtx = sqlite3_malloc64( sizeof(*pCtx) ); + if( pCtx==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + /* NB: zFilename exists and remains valid until the file is closed + ** according to requirement F11141. So we do not need to make a + ** copy of the filename. */ + pCtx->dbPath = zFilename; + pCtx->reserved = 0; + srandomdev(); + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew->lockingContext); + robust_close(pNew, h, __LINE__); + h = -1; + } + unixLeaveMutex(); + } + } +#endif + + else if( pLockingStyle == &dotlockIoMethods ){ + /* Dotfile locking uses the file path so it needs to be included in + ** the dotlockLockingContext + */ + char *zLockFile; + int nFilename; + assert( zFilename!=0 ); + nFilename = (int)strlen(zFilename) + 6; + zLockFile = (char *)sqlite3_malloc64(nFilename); + if( zLockFile==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + sqlite3_snprintf(nFilename, zLockFile, "%s" DOTLOCK_SUFFIX, zFilename); + } + pNew->lockingContext = zLockFile; + } + +#if OS_VXWORKS + else if( pLockingStyle == &semIoMethods ){ + /* Named semaphore locking uses the file path so it needs to be + ** included in the semLockingContext + */ + unixEnterMutex(); + rc = findInodeInfo(pNew, &pNew->pInode); + if( (rc==SQLITE_OK) && (pNew->pInode->pSem==NULL) ){ + char *zSemName = pNew->pInode->aSemName; + int n; + sqlite3_snprintf(MAX_PATHNAME, zSemName, "/%s.sem", + pNew->pId->zCanonicalName); + for( n=1; zSemName[n]; n++ ) + if( zSemName[n]=='/' ) zSemName[n] = '_'; + pNew->pInode->pSem = sem_open(zSemName, O_CREAT, 0666, 1); + if( pNew->pInode->pSem == SEM_FAILED ){ + rc = SQLITE_NOMEM_BKPT; + pNew->pInode->aSemName[0] = '\0'; + } + } + unixLeaveMutex(); + } +#endif + + storeLastErrno(pNew, 0); +#if OS_VXWORKS + if( rc!=SQLITE_OK ){ + if( h>=0 ) robust_close(pNew, h, __LINE__); + h = -1; + osUnlink(zFilename); + pNew->ctrlFlags |= UNIXFILE_DELETE; + } +#endif + if( rc!=SQLITE_OK ){ + if( h>=0 ) robust_close(pNew, h, __LINE__); + }else{ + pId->pMethods = pLockingStyle; + OpenCounter(+1); + verifyDbFile(pNew); + } + return rc; +} + +/* +** Return the name of a directory in which to put temporary files. +** If no suitable temporary file directory can be found, return NULL. +*/ +static const char *unixTempFileDir(void){ + static const char *azDirs[] = { + 0, + 0, + "/var/tmp", + "/usr/tmp", + "/tmp", + "." + }; + unsigned int i = 0; + struct stat buf; + const char *zDir = sqlite3_temp_directory; + + if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); + if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); + while(1){ + if( zDir!=0 + && osStat(zDir, &buf)==0 + && S_ISDIR(buf.st_mode) + && osAccess(zDir, 03)==0 + ){ + return zDir; + } + if( i>=sizeof(azDirs)/sizeof(azDirs[0]) ) break; + zDir = azDirs[i++]; + } + return 0; +} + +/* +** Create a temporary file name in zBuf. zBuf must be allocated +** by the calling process and must be big enough to hold at least +** pVfs->mxPathname bytes. +*/ +static int unixGetTempname(int nBuf, char *zBuf){ + const char *zDir; + int iLimit = 0; + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. + */ + zBuf[0] = 0; + SimulateIOError( return SQLITE_IOERR ); + + zDir = unixTempFileDir(); + if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; + do{ + u64 r; + sqlite3_randomness(sizeof(r), &r); + assert( nBuf>2 ); + zBuf[nBuf-2] = 0; + sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", + zDir, r, 0); + if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ) return SQLITE_ERROR; + }while( osAccess(zBuf,0)==0 ); + return SQLITE_OK; +} + +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) +/* +** Routine to transform a unixFile into a proxy-locking unixFile. +** Implementation in the proxy-lock division, but used by unixOpen() +** if SQLITE_PREFER_PROXY_LOCKING is defined. +*/ +static int proxyTransformUnixFile(unixFile*, const char*); +#endif + +/* +** Search for an unused file descriptor that was opened on the database +** file (not a journal or super-journal file) identified by pathname +** zPath with SQLITE_OPEN_XXX flags matching those passed as the second +** argument to this function. +** +** Such a file descriptor may exist if a database connection was closed +** but the associated file descriptor could not be closed because some +** other file descriptor open on the same file is holding a file-lock. +** Refer to comments in the unixClose() function and the lengthy comment +** describing "Posix Advisory Locking" at the start of this file for +** further details. Also, ticket #4018. +** +** If a suitable file descriptor is found, then it is returned. If no +** such file descriptor is located, -1 is returned. +*/ +static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ + UnixUnusedFd *pUnused = 0; + + /* Do not search for an unused file descriptor on vxworks. Not because + ** vxworks would not benefit from the change (it might, we're not sure), + ** but because no way to test it is currently available. It is better + ** not to risk breaking vxworks support for the sake of such an obscure + ** feature. */ +#if !OS_VXWORKS + struct stat sStat; /* Results of stat() call */ + + unixEnterMutex(); + + /* A stat() call may fail for various reasons. If this happens, it is + ** almost certain that an open() call on the same path will also fail. + ** For this reason, if an error occurs in the stat() call here, it is + ** ignored and -1 is returned. The caller will try to open a new file + ** descriptor on the same path, fail, and return an error to SQLite. + ** + ** Even if a subsequent open() call does succeed, the consequences of + ** not searching for a reusable file descriptor are not dire. */ + if( inodeList!=0 && 0==osStat(zPath, &sStat) ){ + unixInodeInfo *pInode; + + pInode = inodeList; + while( pInode && (pInode->fileId.dev!=sStat.st_dev + || pInode->fileId.ino!=(u64)sStat.st_ino) ){ + pInode = pInode->pNext; + } + if( pInode ){ + UnixUnusedFd **pp; + assert( sqlite3_mutex_notheld(pInode->pLockMutex) ); + sqlite3_mutex_enter(pInode->pLockMutex); + flags &= (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); + for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); + pUnused = *pp; + if( pUnused ){ + *pp = pUnused->pNext; + } + sqlite3_mutex_leave(pInode->pLockMutex); + } + } + unixLeaveMutex(); +#endif /* if !OS_VXWORKS */ + return pUnused; +} + +/* +** Find the mode, uid and gid of file zFile. +*/ +static int getFileMode( + const char *zFile, /* File name */ + mode_t *pMode, /* OUT: Permissions of zFile */ + uid_t *pUid, /* OUT: uid of zFile. */ + gid_t *pGid /* OUT: gid of zFile. */ +){ + struct stat sStat; /* Output of stat() on database file */ + int rc = SQLITE_OK; + if( 0==osStat(zFile, &sStat) ){ + *pMode = sStat.st_mode & 0777; + *pUid = sStat.st_uid; + *pGid = sStat.st_gid; + }else{ + rc = SQLITE_IOERR_FSTAT; + } + return rc; +} + +/* +** This function is called by unixOpen() to determine the unix permissions +** to create new files with. If no error occurs, then SQLITE_OK is returned +** and a value suitable for passing as the third argument to open(2) is +** written to *pMode. If an IO error occurs, an SQLite error code is +** returned and the value of *pMode is not modified. +** +** In most cases, this routine sets *pMode to 0, which will become +** an indication to robust_open() to create the file using +** SQLITE_DEFAULT_FILE_PERMISSIONS adjusted by the umask. +** But if the file being opened is a WAL or regular journal file, then +** this function queries the file-system for the permissions on the +** corresponding database file and sets *pMode to this value. Whenever +** possible, WAL and journal files are created using the same permissions +** as the associated database file. +** +** If the SQLITE_ENABLE_8_3_NAMES option is enabled, then the +** original filename is unavailable. But 8_3_NAMES is only used for +** FAT filesystems and permissions do not matter there, so just use +** the default permissions. In 8_3_NAMES mode, leave *pMode set to zero. +*/ +static int findCreateFileMode( + const char *zPath, /* Path of file (possibly) being created */ + int flags, /* Flags passed as 4th argument to xOpen() */ + mode_t *pMode, /* OUT: Permissions to open file with */ + uid_t *pUid, /* OUT: uid to set on the file */ + gid_t *pGid /* OUT: gid to set on the file */ +){ + int rc = SQLITE_OK; /* Return Code */ + *pMode = 0; + *pUid = 0; + *pGid = 0; + if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + char zDb[MAX_PATHNAME+1]; /* Database file path */ + int nDb; /* Number of valid bytes in zDb */ + + /* zPath is a path to a WAL or journal file. The following block derives + ** the path to the associated database file from zPath. This block handles + ** the following naming conventions: + ** + ** "-journal" + ** "-wal" + ** "-journalNN" + ** "-walNN" + ** + ** where NN is a decimal number. The NN naming schemes are + ** used by the test_multiplex.c module. + */ + nDb = sqlite3Strlen30(zPath) - 1; + while( zPath[nDb]!='-' ){ + /* In normal operation, the journal file name will always contain + ** a '-' character. However in 8+3 filename mode, or if a corrupt + ** rollback journal specifies a super-journal with a goofy name, then + ** the '-' might be missing. */ + if( nDb==0 || zPath[nDb]=='.' ) return SQLITE_OK; + nDb--; + } + memcpy(zDb, zPath, nDb); + zDb[nDb] = '\0'; + + rc = getFileMode(zDb, pMode, pUid, pGid); + }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ + *pMode = 0600; + }else if( flags & SQLITE_OPEN_URI ){ + /* If this is a main database file and the file was opened using a URI + ** filename, check for the "modeof" parameter. If present, interpret + ** its value as a filename and try to copy the mode, uid and gid from + ** that file. */ + const char *z = sqlite3_uri_parameter(zPath, "modeof"); + if( z ){ + rc = getFileMode(z, pMode, pUid, pGid); + } + } + return rc; +} + +/* +** Open the file zPath. +** +** Previously, the SQLite OS layer used three functions in place of this +** one: +** +** sqlite3OsOpenReadWrite(); +** sqlite3OsOpenReadOnly(); +** sqlite3OsOpenExclusive(); +** +** These calls correspond to the following combinations of flags: +** +** ReadWrite() -> (READWRITE | CREATE) +** ReadOnly() -> (READONLY) +** OpenExclusive() -> (READWRITE | CREATE | EXCLUSIVE) +** +** The old OpenExclusive() accepted a boolean argument - "delFlag". If +** true, the file was configured to be automatically deleted when the +** file handle closed. To achieve the same effect using this new +** interface, add the DELETEONCLOSE flag to those specified above for +** OpenExclusive(). +*/ +static int unixOpen( + sqlite3_vfs *pVfs, /* The VFS for which this is the xOpen method */ + const char *zPath, /* Pathname of file to be opened */ + sqlite3_file *pFile, /* The file descriptor to be filled in */ + int flags, /* Input flags to control the opening */ + int *pOutFlags /* Output flags returned to SQLite core */ +){ + unixFile *p = (unixFile *)pFile; + int fd = -1; /* File descriptor returned by open() */ + int openFlags = 0; /* Flags to pass to open() */ + int eType = flags&0x0FFF00; /* Type of file to open */ + int noLock; /* True to omit locking primitives */ + int rc = SQLITE_OK; /* Function Return Code */ + int ctrlFlags = 0; /* UNIXFILE_* flags */ + + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); + int isReadonly = (flags & SQLITE_OPEN_READONLY); + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); +#if SQLITE_ENABLE_LOCKING_STYLE + int isAutoProxy = (flags & SQLITE_OPEN_AUTOPROXY); +#endif +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE + struct statfs fsInfo; +#endif + + /* If creating a super- or main-file journal, this function will open + ** a file-descriptor on the directory too. The first time unixSync() + ** is called the directory file descriptor will be fsync()ed and close()d. + */ + int isNewJrnl = (isCreate && ( + eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); + + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char zTmpname[MAX_PATHNAME+2]; + const char *zName = zPath; + + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); + + /* The main DB, main journal, WAL file and super-journal are never + ** automatically deleted. Nor are they ever temporary files. */ + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_SUPER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); + + /* Detect a pid change and reset the PRNG. There is a race condition + ** here such that two or more threads all trying to open databases at + ** the same instant might all reset the PRNG. But multiple resets + ** are harmless. + */ + if( randomnessPid!=osGetpid(0) ){ + randomnessPid = osGetpid(0); + sqlite3_randomness(0,0); + } + memset(p, 0, sizeof(unixFile)); + + if( eType==SQLITE_OPEN_MAIN_DB ){ + UnixUnusedFd *pUnused; + pUnused = findReusableFd(zName, flags); + if( pUnused ){ + fd = pUnused->fd; + }else{ + pUnused = sqlite3_malloc64(sizeof(*pUnused)); + if( !pUnused ){ + return SQLITE_NOMEM_BKPT; + } + } + p->pPreallocatedUnused = pUnused; + + /* Database filenames are double-zero terminated if they are not + ** URIs with parameters. Hence, they can always be passed into + ** sqlite3_uri_parameter(). */ + assert( (flags & SQLITE_OPEN_URI) || zName[strlen(zName)+1]==0 ); + + }else if( !zName ){ + /* If zName is NULL, the upper layer is requesting a temp file. */ + assert(isDelete && !isNewJrnl); + rc = unixGetTempname(pVfs->mxPathname, zTmpname); + if( rc!=SQLITE_OK ){ + return rc; + } + zName = zTmpname; + + /* Generated temporary filenames are always double-zero terminated + ** for use by sqlite3_uri_parameter(). */ + assert( zName[strlen(zName)+1]==0 ); + } + + /* Determine the value of the flags parameter passed to POSIX function + ** open(). These must be calculated even if open() is not called, as + ** they may be stored as part of the file handle and used by the + ** 'conch file' locking functions later on. */ + if( isReadonly ) openFlags |= O_RDONLY; + if( isReadWrite ) openFlags |= O_RDWR; + if( isCreate ) openFlags |= O_CREAT; + if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); + openFlags |= (O_LARGEFILE|O_BINARY|O_NOFOLLOW); + + if( fd<0 ){ + mode_t openMode; /* Permissions to create file with */ + uid_t uid; /* Userid for the file */ + gid_t gid; /* Groupid for the file */ + rc = findCreateFileMode(zName, flags, &openMode, &uid, &gid); + if( rc!=SQLITE_OK ){ + assert( !p->pPreallocatedUnused ); + assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); + return rc; + } + fd = robust_open(zName, openFlags, openMode); + OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); + assert( !isExclusive || (openFlags & O_CREAT)!=0 ); + if( fd<0 ){ + if( isNewJrnl && errno==EACCES && osAccess(zName, F_OK) ){ + /* If unable to create a journal because the directory is not + ** writable, change the error code to indicate that. */ + rc = SQLITE_READONLY_DIRECTORY; + }else if( errno==EEXIST ){ + rc = SQLITE_CANTOPEN_EXISTS; + }else if( errno!=EISDIR && isReadWrite ){ + /* Failed to open the file for read/write access. Try read-only. */ + flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); + openFlags &= ~(O_RDWR|O_CREAT); + flags |= SQLITE_OPEN_READONLY; + openFlags |= O_RDONLY; + isReadonly = 1; + fd = robust_open(zName, openFlags, openMode); + } + } + if( fd<0 ){ + int rc2 = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); + if( rc==SQLITE_OK ) rc = rc2; + goto open_finished; + } + + /* The owner of the rollback journal or WAL file should always be the + ** same as the owner of the database file. Try to ensure that this is + ** the case. The chown() system call will be a no-op if the current + ** process lacks root privileges, be we should at least try. Without + ** this step, if a root process opens a database file, it can leave + ** behinds a journal/WAL that is owned by root and hence make the + ** database inaccessible to unprivileged processes. + ** + ** If openMode==0, then that means uid and gid are not set correctly + ** (probably because SQLite is configured to use 8+3 filename mode) and + ** in that case we do not want to attempt the chown(). + */ + if( openMode && (flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL))!=0 ){ + robustFchown(fd, uid, gid); + } + } + assert( fd>=0 ); + if( pOutFlags ){ + *pOutFlags = flags; + } + + if( p->pPreallocatedUnused ){ + p->pPreallocatedUnused->fd = fd; + p->pPreallocatedUnused->flags = + flags & (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); + } + + if( isDelete ){ +#if OS_VXWORKS + zPath = zName; +#elif defined(SQLITE_UNLINK_AFTER_CLOSE) + zPath = sqlite3_mprintf("%s", zName); + if( zPath==0 ){ + robust_close(p, fd, __LINE__); + return SQLITE_NOMEM_BKPT; + } +#else + osUnlink(zName); +#endif + } +#if SQLITE_ENABLE_LOCKING_STYLE + else{ + p->openFlags = openFlags; + } +#endif + +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE + if( fstatfs(fd, &fsInfo) == -1 ){ + storeLastErrno(p, errno); + robust_close(p, fd, __LINE__); + return SQLITE_IOERR_ACCESS; + } + if (0 == strncmp("msdos", fsInfo.f_fstypename, 5)) { + ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; + } + if (0 == strncmp("exfat", fsInfo.f_fstypename, 5)) { + ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; + } +#endif + + /* Set up appropriate ctrlFlags */ + if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; + if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; + noLock = eType!=SQLITE_OPEN_MAIN_DB; + if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; + if( isNewJrnl ) ctrlFlags |= UNIXFILE_DIRSYNC; + if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; + +#if SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_PREFER_PROXY_LOCKING + isAutoProxy = 1; +#endif + if( isAutoProxy && (zPath!=NULL) && (!noLock) && pVfs->xOpen ){ + char *envforce = getenv("SQLITE_FORCE_PROXY_LOCKING"); + int useProxy = 0; + + /* SQLITE_FORCE_PROXY_LOCKING==1 means force always use proxy, 0 means + ** never use proxy, NULL means use proxy for non-local files only. */ + if( envforce!=NULL ){ + useProxy = atoi(envforce)>0; + }else{ + useProxy = !(fsInfo.f_flags&MNT_LOCAL); + } + if( useProxy ){ + rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); + if( rc==SQLITE_OK ){ + rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); + if( rc!=SQLITE_OK ){ + /* Use unixClose to clean up the resources added in fillInUnixFile + ** and clear all the structure's references. Specifically, + ** pFile->pMethods will be NULL so sqlite3OsClose will be a no-op + */ + unixClose(pFile); + return rc; + } + } + goto open_finished; + } + } +#endif + + assert( zPath==0 || zPath[0]=='/' + || eType==SQLITE_OPEN_SUPER_JOURNAL || eType==SQLITE_OPEN_MAIN_JOURNAL + ); + rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); + +open_finished: + if( rc!=SQLITE_OK ){ + sqlite3_free(p->pPreallocatedUnused); + } + return rc; +} + + +/* +** Delete the file at zPath. If the dirSync argument is true, fsync() +** the directory after deleting the file. +*/ +static int unixDelete( + sqlite3_vfs *NotUsed, /* VFS containing this as the xDelete method */ + const char *zPath, /* Name of file to be deleted */ + int dirSync /* If true, fsync() directory after deleting file */ +){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(NotUsed); + SimulateIOError(return SQLITE_IOERR_DELETE); + if( osUnlink(zPath)==(-1) ){ + if( errno==ENOENT +#if OS_VXWORKS + || osAccess(zPath,0)!=0 +#endif + ){ + rc = SQLITE_IOERR_DELETE_NOENT; + }else{ + rc = unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); + } + return rc; + } +#ifndef SQLITE_DISABLE_DIRSYNC + if( (dirSync & 1)!=0 ){ + int fd; + rc = osOpenDirectory(zPath, &fd); + if( rc==SQLITE_OK ){ + if( full_fsync(fd,0,0) ){ + rc = unixLogError(SQLITE_IOERR_DIR_FSYNC, "fsync", zPath); + } + robust_close(0, fd, __LINE__); + }else{ + assert( rc==SQLITE_CANTOPEN ); + rc = SQLITE_OK; + } + } +#endif + return rc; +} + +/* +** Test the existence of or access permissions of file zPath. The +** test performed depends on the value of flags: +** +** SQLITE_ACCESS_EXISTS: Return 1 if the file exists +** SQLITE_ACCESS_READWRITE: Return 1 if the file is read and writable. +** SQLITE_ACCESS_READONLY: Return 1 if the file is readable. +** +** Otherwise return 0. +*/ +static int unixAccess( + sqlite3_vfs *NotUsed, /* The VFS containing this xAccess method */ + const char *zPath, /* Path of the file to examine */ + int flags, /* What do we want to learn about the zPath file? */ + int *pResOut /* Write result boolean here */ +){ + UNUSED_PARAMETER(NotUsed); + SimulateIOError( return SQLITE_IOERR_ACCESS; ); + assert( pResOut!=0 ); + + /* The spec says there are three possible values for flags. But only + ** two of them are actually used */ + assert( flags==SQLITE_ACCESS_EXISTS || flags==SQLITE_ACCESS_READWRITE ); + + if( flags==SQLITE_ACCESS_EXISTS ){ + struct stat buf; + *pResOut = 0==osStat(zPath, &buf) && + (!S_ISREG(buf.st_mode) || buf.st_size>0); + }else{ + *pResOut = osAccess(zPath, W_OK|R_OK)==0; + } + return SQLITE_OK; +} + +/* +** If the last component of the pathname in z[0]..z[j-1] is something +** other than ".." then back it out and return true. If the last +** component is empty or if it is ".." then return false. +*/ +static int unixBackupDir(const char *z, int *pJ){ + int j = *pJ; + int i; + if( j<=0 ) return 0; + for(i=j-1; i>0 && z[i-1]!='/'; i--){} + if( i==0 ) return 0; + if( z[i]=='.' && i==j-2 && z[i+1]=='.' ) return 0; + *pJ = i-1; + return 1; +} + +/* +** Convert a relative pathname into a full pathname. Also +** simplify the pathname as follows: +** +** Remove all instances of /./ +** Remove all isntances of /X/../ for any X +*/ +static int mkFullPathname( + const char *zPath, /* Input path */ + char *zOut, /* Output buffer */ + int nOut /* Allocated size of buffer zOut */ +){ + int nPath = sqlite3Strlen30(zPath); + int iOff = 0; + int i, j; + if( zPath[0]!='/' ){ + if( osGetcwd(zOut, nOut-2)==0 ){ + return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); + } + iOff = sqlite3Strlen30(zOut); + zOut[iOff++] = '/'; + } + if( (iOff+nPath+1)>nOut ){ + /* SQLite assumes that xFullPathname() nul-terminates the output buffer + ** even if it returns an error. */ + zOut[iOff] = '\0'; + return SQLITE_CANTOPEN_BKPT; + } + sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); + + /* Remove duplicate '/' characters. Except, two // at the beginning + ** of a pathname is allowed since this is important on windows. */ + for(i=j=1; zOut[i]; i++){ + zOut[j++] = zOut[i]; + while( zOut[i]=='/' && zOut[i+1]=='/' ) i++; + } + zOut[j] = 0; + + assert( zOut[0]=='/' ); + for(i=j=0; zOut[i]; i++){ + if( zOut[i]=='/' ){ + /* Skip over internal "/." directory components */ + if( zOut[i+1]=='.' && zOut[i+2]=='/' ){ + i += 1; + continue; + } + + /* If this is a "/.." directory component then back out the + ** previous term of the directory if it is something other than "..". + */ + if( zOut[i+1]=='.' + && zOut[i+2]=='.' + && zOut[i+3]=='/' + && unixBackupDir(zOut, &j) + ){ + i += 2; + continue; + } + } + if( ALWAYS(j>=0) ) zOut[j] = zOut[i]; + j++; + } + if( NEVER(j==0) ) zOut[j++] = '/'; + zOut[j] = 0; + return SQLITE_OK; +} + +/* +** Turn a relative pathname into a full pathname. The relative path +** is stored as a nul-terminated string in the buffer pointed to by +** zPath. +** +** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes +** (in this case, MAX_PATHNAME bytes). The full-path is written to +** this buffer before returning. +*/ +static int unixFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zPath, /* Possibly relative input path */ + int nOut, /* Size of output buffer in bytes */ + char *zOut /* Output buffer */ +){ +#if !defined(HAVE_READLINK) || !defined(HAVE_LSTAT) + return mkFullPathname(zPath, zOut, nOut); +#else + int rc = SQLITE_OK; + int nByte; + int nLink = 0; /* Number of symbolic links followed so far */ + const char *zIn = zPath; /* Input path for each iteration of loop */ + char *zDel = 0; + + assert( pVfs->mxPathname==MAX_PATHNAME ); + UNUSED_PARAMETER(pVfs); + + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. This function could fail if, for example, the + ** current working directory has been unlinked. + */ + SimulateIOError( return SQLITE_ERROR ); + + do { + + /* Call stat() on path zIn. Set bLink to true if the path is a symbolic + ** link, or false otherwise. */ + int bLink = 0; + struct stat buf; + if( osLstat(zIn, &buf)!=0 ){ + if( errno!=ENOENT ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "lstat", zIn); + } + }else{ + bLink = S_ISLNK(buf.st_mode); + } + + if( bLink ){ + nLink++; + if( zDel==0 ){ + zDel = sqlite3_malloc(nOut); + if( zDel==0 ) rc = SQLITE_NOMEM_BKPT; + }else if( nLink>=SQLITE_MAX_SYMLINKS ){ + rc = SQLITE_CANTOPEN_BKPT; + } + + if( rc==SQLITE_OK ){ + nByte = osReadlink(zIn, zDel, nOut-1); + if( nByte<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zIn); + }else{ + if( zDel[0]!='/' ){ + int n; + for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); + if( nByte+n+1>nOut ){ + rc = SQLITE_CANTOPEN_BKPT; + }else{ + memmove(&zDel[n], zDel, nByte+1); + memcpy(zDel, zIn, n); + nByte += n; + } + } + zDel[nByte] = '\0'; + } + } + + zIn = zDel; + } + + assert( rc!=SQLITE_OK || zIn!=zOut || zIn[0]=='/' ); + if( rc==SQLITE_OK && zIn!=zOut ){ + rc = mkFullPathname(zIn, zOut, nOut); + } + if( bLink==0 ) break; + zIn = zOut; + }while( rc==SQLITE_OK ); + + sqlite3_free(zDel); + if( rc==SQLITE_OK && nLink ) rc = SQLITE_OK_SYMLINK; + return rc; +#endif /* HAVE_READLINK && HAVE_LSTAT */ +} + + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. +*/ +#include +static void *unixDlOpen(sqlite3_vfs *NotUsed, const char *zFilename){ + UNUSED_PARAMETER(NotUsed); + return dlopen(zFilename, RTLD_NOW | RTLD_GLOBAL); +} + +/* +** SQLite calls this function immediately after a call to unixDlSym() or +** unixDlOpen() fails (returns a null pointer). If a more detailed error +** message is available, it is written to zBufOut. If no error message +** is available, zBufOut is left unmodified and SQLite uses a default +** error message. +*/ +static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){ + const char *zErr; + UNUSED_PARAMETER(NotUsed); + unixEnterMutex(); + zErr = dlerror(); + if( zErr ){ + sqlite3_snprintf(nBuf, zBufOut, "%s", zErr); + } + unixLeaveMutex(); +} +static void (*unixDlSym(sqlite3_vfs *NotUsed, void *p, const char*zSym))(void){ + /* + ** GCC with -pedantic-errors says that C90 does not allow a void* to be + ** cast into a pointer to a function. And yet the library dlsym() routine + ** returns a void* which is really a pointer to a function. So how do we + ** use dlsym() with -pedantic-errors? + ** + ** Variable x below is defined to be a pointer to a function taking + ** parameters void* and const char* and returning a pointer to a function. + ** We initialize x by assigning it a pointer to the dlsym() function. + ** (That assignment requires a cast.) Then we call the function that + ** x points to. + ** + ** This work-around is unlikely to work correctly on any system where + ** you really cannot cast a function pointer into void*. But then, on the + ** other hand, dlsym() will not work on such a system either, so we have + ** not really lost anything. + */ + void (*(*x)(void*,const char*))(void); + UNUSED_PARAMETER(NotUsed); + x = (void(*(*)(void*,const char*))(void))dlsym; + return (*x)(p, zSym); +} +static void unixDlClose(sqlite3_vfs *NotUsed, void *pHandle){ + UNUSED_PARAMETER(NotUsed); + dlclose(pHandle); +} +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define unixDlOpen 0 + #define unixDlError 0 + #define unixDlSym 0 + #define unixDlClose 0 +#endif + +/* +** Write nBuf bytes of random data to the supplied buffer zBuf. +*/ +static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ + UNUSED_PARAMETER(NotUsed); + assert((size_t)nBuf>=(sizeof(time_t)+sizeof(int))); + + /* We have to initialize zBuf to prevent valgrind from reporting + ** errors. The reports issued by valgrind are incorrect - we would + ** prefer that the randomness be increased by making use of the + ** uninitialized space in zBuf - but valgrind errors tend to worry + ** some users. Rather than argue, it seems easier just to initialize + ** the whole array and silence valgrind, even if that means less randomness + ** in the random seed. + ** + ** When testing, initializing zBuf[] to zero is all we do. That means + ** that we always use the same random number sequence. This makes the + ** tests repeatable. + */ + memset(zBuf, 0, nBuf); + randomnessPid = osGetpid(0); +#if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) + { + int fd, got; + fd = robust_open("/dev/urandom", O_RDONLY, 0); + if( fd<0 ){ + time_t t; + time(&t); + memcpy(zBuf, &t, sizeof(t)); + memcpy(&zBuf[sizeof(t)], &randomnessPid, sizeof(randomnessPid)); + assert( sizeof(t)+sizeof(randomnessPid)<=(size_t)nBuf ); + nBuf = sizeof(t) + sizeof(randomnessPid); + }else{ + do{ got = osRead(fd, zBuf, nBuf); }while( got<0 && errno==EINTR ); + robust_close(0, fd, __LINE__); + } + } +#endif + return nBuf; +} + + +/* +** Sleep for a little while. Return the amount of time slept. +** The argument is the number of microseconds we want to sleep. +** The return value is the number of microseconds of sleep actually +** requested from the underlying operating system, a number which +** might be greater than or equal to the argument, but not less +** than the argument. +*/ +static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ +#if OS_VXWORKS + struct timespec sp; + + sp.tv_sec = microseconds / 1000000; + sp.tv_nsec = (microseconds % 1000000) * 1000; + nanosleep(&sp, NULL); + UNUSED_PARAMETER(NotUsed); + return microseconds; +#elif defined(HAVE_USLEEP) && HAVE_USLEEP + if( microseconds>=1000000 ) sleep(microseconds/1000000); + if( microseconds%1000000 ) usleep(microseconds%1000000); + UNUSED_PARAMETER(NotUsed); + return microseconds; +#else + int seconds = (microseconds+999999)/1000000; + sleep(seconds); + UNUSED_PARAMETER(NotUsed); + return seconds*1000000; +#endif +} + +/* +** The following variable, if set to a non-zero value, is interpreted as +** the number of seconds since 1970 and is used to set the result of +** sqlite3OsCurrentTime() during testing. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ +#endif + +/* +** Find the current time (in Universal Coordinated Time). Write into *piNow +** the current time and date as a Julian Day number times 86_400_000. In +** other words, write into *piNow the number of milliseconds since the Julian +** epoch of noon in Greenwich on November 24, 4714 B.C according to the +** proleptic Gregorian calendar. +** +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** cannot be found. +*/ +static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; + int rc = SQLITE_OK; +#if defined(NO_GETTOD) + time_t t; + time(&t); + *piNow = ((sqlite3_int64)t)*1000 + unixEpoch; +#elif OS_VXWORKS + struct timespec sNow; + clock_gettime(CLOCK_REALTIME, &sNow); + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_nsec/1000000; +#else + struct timeval sNow; + (void)gettimeofday(&sNow, 0); /* Cannot fail given valid arguments */ + *piNow = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; +#endif + +#ifdef SQLITE_TEST + if( sqlite3_current_time ){ + *piNow = 1000*(sqlite3_int64)sqlite3_current_time + unixEpoch; + } +#endif + UNUSED_PARAMETER(NotUsed); + return rc; +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Find the current time (in Universal Coordinated Time). Write the +** current time and date as a Julian Day number into *prNow and +** return 0. Return 1 if the time and date cannot be found. +*/ +static int unixCurrentTime(sqlite3_vfs *NotUsed, double *prNow){ + sqlite3_int64 i = 0; + int rc; + UNUSED_PARAMETER(NotUsed); + rc = unixCurrentTimeInt64(0, &i); + *prNow = i/86400000.0; + return rc; +} +#else +# define unixCurrentTime 0 +#endif + +/* +** The xGetLastError() method is designed to return a better +** low-level error message when operating-system problems come up +** during SQLite operation. Only the integer return code is currently +** used. +*/ +static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ + UNUSED_PARAMETER(NotUsed); + UNUSED_PARAMETER(NotUsed2); + UNUSED_PARAMETER(NotUsed3); + return errno; +} + + +/* +************************ End of sqlite3_vfs methods *************************** +******************************************************************************/ + +/****************************************************************************** +************************** Begin Proxy Locking ******************************** +** +** Proxy locking is a "uber-locking-method" in this sense: It uses the +** other locking methods on secondary lock files. Proxy locking is a +** meta-layer over top of the primitive locking implemented above. For +** this reason, the division that implements of proxy locking is deferred +** until late in the file (here) after all of the other I/O methods have +** been defined - so that the primitive locking methods are available +** as services to help with the implementation of proxy locking. +** +**** +** +** The default locking schemes in SQLite use byte-range locks on the +** database file to coordinate safe, concurrent access by multiple readers +** and writers [http://sqlite.org/lockingv3.html]. The five file locking +** states (UNLOCKED, PENDING, SHARED, RESERVED, EXCLUSIVE) are implemented +** as POSIX read & write locks over fixed set of locations (via fsctl), +** on AFP and SMB only exclusive byte-range locks are available via fsctl +** with _IOWR('z', 23, struct ByteRangeLockPB2) to track the same 5 states. +** To simulate a F_RDLCK on the shared range, on AFP a randomly selected +** address in the shared range is taken for a SHARED lock, the entire +** shared range is taken for an EXCLUSIVE lock): +** +** PENDING_BYTE 0x40000000 +** RESERVED_BYTE 0x40000001 +** SHARED_RANGE 0x40000002 -> 0x40000200 +** +** This works well on the local file system, but shows a nearly 100x +** slowdown in read performance on AFP because the AFP client disables +** the read cache when byte-range locks are present. Enabling the read +** cache exposes a cache coherency problem that is present on all OS X +** supported network file systems. NFS and AFP both observe the +** close-to-open semantics for ensuring cache coherency +** [http://nfs.sourceforge.net/#faq_a8], which does not effectively +** address the requirements for concurrent database access by multiple +** readers and writers +** [http://www.nabble.com/SQLite-on-NFS-cache-coherency-td15655701.html]. +** +** To address the performance and cache coherency issues, proxy file locking +** changes the way database access is controlled by limiting access to a +** single host at a time and moving file locks off of the database file +** and onto a proxy file on the local file system. +** +** +** Using proxy locks +** ----------------- +** +** C APIs +** +** sqlite3_file_control(db, dbname, SQLITE_FCNTL_SET_LOCKPROXYFILE, +** | ":auto:"); +** sqlite3_file_control(db, dbname, SQLITE_FCNTL_GET_LOCKPROXYFILE, +** &); +** +** +** SQL pragmas +** +** PRAGMA [database.]lock_proxy_file= | :auto: +** PRAGMA [database.]lock_proxy_file +** +** Specifying ":auto:" means that if there is a conch file with a matching +** host ID in it, the proxy path in the conch file will be used, otherwise +** a proxy path based on the user's temp dir +** (via confstr(_CS_DARWIN_USER_TEMP_DIR,...)) will be used and the +** actual proxy file name is generated from the name and path of the +** database file. For example: +** +** For database path "/Users/me/foo.db" +** The lock path will be "/sqliteplocks/_Users_me_foo.db:auto:") +** +** Once a lock proxy is configured for a database connection, it can not +** be removed, however it may be switched to a different proxy path via +** the above APIs (assuming the conch file is not being held by another +** connection or process). +** +** +** How proxy locking works +** ----------------------- +** +** Proxy file locking relies primarily on two new supporting files: +** +** * conch file to limit access to the database file to a single host +** at a time +** +** * proxy file to act as a proxy for the advisory locks normally +** taken on the database +** +** The conch file - to use a proxy file, sqlite must first "hold the conch" +** by taking an sqlite-style shared lock on the conch file, reading the +** contents and comparing the host's unique host ID (see below) and lock +** proxy path against the values stored in the conch. The conch file is +** stored in the same directory as the database file and the file name +** is patterned after the database file name as ".-conch". +** If the conch file does not exist, or its contents do not match the +** host ID and/or proxy path, then the lock is escalated to an exclusive +** lock and the conch file contents is updated with the host ID and proxy +** path and the lock is downgraded to a shared lock again. If the conch +** is held by another process (with a shared lock), the exclusive lock +** will fail and SQLITE_BUSY is returned. +** +** The proxy file - a single-byte file used for all advisory file locks +** normally taken on the database file. This allows for safe sharing +** of the database file for multiple readers and writers on the same +** host (the conch ensures that they all use the same local lock file). +** +** Requesting the lock proxy does not immediately take the conch, it is +** only taken when the first request to lock database file is made. +** This matches the semantics of the traditional locking behavior, where +** opening a connection to a database file does not take a lock on it. +** The shared lock and an open file descriptor are maintained until +** the connection to the database is closed. +** +** The proxy file and the lock file are never deleted so they only need +** to be created the first time they are used. +** +** Configuration options +** --------------------- +** +** SQLITE_PREFER_PROXY_LOCKING +** +** Database files accessed on non-local file systems are +** automatically configured for proxy locking, lock files are +** named automatically using the same logic as +** PRAGMA lock_proxy_file=":auto:" +** +** SQLITE_PROXY_DEBUG +** +** Enables the logging of error messages during host id file +** retrieval and creation +** +** LOCKPROXYDIR +** +** Overrides the default directory used for lock proxy files that +** are named automatically via the ":auto:" setting +** +** SQLITE_DEFAULT_PROXYDIR_PERMISSIONS +** +** Permissions to use when creating a directory for storing the +** lock proxy files, only used when LOCKPROXYDIR is not set. +** +** +** As mentioned above, when compiled with SQLITE_PREFER_PROXY_LOCKING, +** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will +** force proxy locking to be used for every database file opened, and 0 +** will force automatic proxy locking to be disabled for all database +** files (explicitly calling the SQLITE_FCNTL_SET_LOCKPROXYFILE pragma or +** sqlite_file_control API is not affected by SQLITE_FORCE_PROXY_LOCKING). +*/ + +/* +** Proxy locking is only available on MacOSX +*/ +#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE + +/* +** The proxyLockingContext has the path and file structures for the remote +** and local proxy files in it +*/ +typedef struct proxyLockingContext proxyLockingContext; +struct proxyLockingContext { + unixFile *conchFile; /* Open conch file */ + char *conchFilePath; /* Name of the conch file */ + unixFile *lockProxy; /* Open proxy lock file */ + char *lockProxyPath; /* Name of the proxy lock file */ + char *dbPath; /* Name of the open file */ + int conchHeld; /* 1 if the conch is held, -1 if lockless */ + int nFails; /* Number of conch taking failures */ + void *oldLockingContext; /* Original lockingcontext to restore on close */ + sqlite3_io_methods const *pOldMethod; /* Original I/O methods for close */ +}; + +/* +** The proxy lock file path for the database at dbPath is written into lPath, +** which must point to valid, writable memory large enough for a maxLen length +** file path. +*/ +static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ + int len; + int dbLen; + int i; + +#ifdef LOCKPROXYDIR + len = strlcpy(lPath, LOCKPROXYDIR, maxLen); +#else +# ifdef _CS_DARWIN_USER_TEMP_DIR + { + if( !confstr(_CS_DARWIN_USER_TEMP_DIR, lPath, maxLen) ){ + OSTRACE(("GETLOCKPATH failed %s errno=%d pid=%d\n", + lPath, errno, osGetpid(0))); + return SQLITE_IOERR_LOCK; + } + len = strlcat(lPath, "sqliteplocks", maxLen); + } +# else + len = strlcpy(lPath, "/tmp/", maxLen); +# endif +#endif + + if( lPath[len-1]!='/' ){ + len = strlcat(lPath, "/", maxLen); + } + + /* transform the db path to a unique cache name */ + dbLen = (int)strlen(dbPath); + for( i=0; i 0) ){ + /* only mkdir if leaf dir != "." or "/" or ".." */ + if( i-start>2 || (i-start==1 && buf[start] != '.' && buf[start] != '/') + || (i-start==2 && buf[start] != '.' && buf[start+1] != '.') ){ + buf[i]='\0'; + if( osMkdir(buf, SQLITE_DEFAULT_PROXYDIR_PERMISSIONS) ){ + int err=errno; + if( err!=EEXIST ) { + OSTRACE(("CREATELOCKPATH FAILED creating %s, " + "'%s' proxy lock path=%s pid=%d\n", + buf, strerror(err), lockPath, osGetpid(0))); + return err; + } + } + } + start=i+1; + } + buf[i] = lockPath[i]; + } + OSTRACE(("CREATELOCKPATH proxy lock path=%s pid=%d\n",lockPath,osGetpid(0))); + return 0; +} + +/* +** Create a new VFS file descriptor (stored in memory obtained from +** sqlite3_malloc) and open the file named "path" in the file descriptor. +** +** The caller is responsible not only for closing the file descriptor +** but also for freeing the memory associated with the file descriptor. +*/ +static int proxyCreateUnixFile( + const char *path, /* path for the new unixFile */ + unixFile **ppFile, /* unixFile created and returned by ref */ + int islockfile /* if non zero missing dirs will be created */ +) { + int fd = -1; + unixFile *pNew; + int rc = SQLITE_OK; + int openFlags = O_RDWR | O_CREAT | O_NOFOLLOW; + sqlite3_vfs dummyVfs; + int terrno = 0; + UnixUnusedFd *pUnused = NULL; + + /* 1. first try to open/create the file + ** 2. if that fails, and this is a lock file (not-conch), try creating + ** the parent directories and then try again. + ** 3. if that fails, try to open the file read-only + ** otherwise return BUSY (if lock file) or CANTOPEN for the conch file + */ + pUnused = findReusableFd(path, openFlags); + if( pUnused ){ + fd = pUnused->fd; + }else{ + pUnused = sqlite3_malloc64(sizeof(*pUnused)); + if( !pUnused ){ + return SQLITE_NOMEM_BKPT; + } + } + if( fd<0 ){ + fd = robust_open(path, openFlags, 0); + terrno = errno; + if( fd<0 && errno==ENOENT && islockfile ){ + if( proxyCreateLockPath(path) == SQLITE_OK ){ + fd = robust_open(path, openFlags, 0); + } + } + } + if( fd<0 ){ + openFlags = O_RDONLY | O_NOFOLLOW; + fd = robust_open(path, openFlags, 0); + terrno = errno; + } + if( fd<0 ){ + if( islockfile ){ + return SQLITE_BUSY; + } + switch (terrno) { + case EACCES: + return SQLITE_PERM; + case EIO: + return SQLITE_IOERR_LOCK; /* even though it is the conch */ + default: + return SQLITE_CANTOPEN_BKPT; + } + } + + pNew = (unixFile *)sqlite3_malloc64(sizeof(*pNew)); + if( pNew==NULL ){ + rc = SQLITE_NOMEM_BKPT; + goto end_create_proxy; + } + memset(pNew, 0, sizeof(unixFile)); + pNew->openFlags = openFlags; + memset(&dummyVfs, 0, sizeof(dummyVfs)); + dummyVfs.pAppData = (void*)&autolockIoFinder; + dummyVfs.zName = "dummy"; + pUnused->fd = fd; + pUnused->flags = openFlags; + pNew->pPreallocatedUnused = pUnused; + + rc = fillInUnixFile(&dummyVfs, fd, (sqlite3_file*)pNew, path, 0); + if( rc==SQLITE_OK ){ + *ppFile = pNew; + return SQLITE_OK; + } +end_create_proxy: + robust_close(pNew, fd, __LINE__); + sqlite3_free(pNew); + sqlite3_free(pUnused); + return rc; +} + +#ifdef SQLITE_TEST +/* simulate multiple hosts by creating unique hostid file paths */ +SQLITE_API int sqlite3_hostid_num = 0; +#endif + +#define PROXY_HOSTIDLEN 16 /* conch file host id length */ + +#if HAVE_GETHOSTUUID +/* Not always defined in the headers as it ought to be */ +extern int gethostuuid(uuid_t id, const struct timespec *wait); +#endif + +/* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN +** bytes of writable memory. +*/ +static int proxyGetHostID(unsigned char *pHostID, int *pError){ + assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); + memset(pHostID, 0, PROXY_HOSTIDLEN); +#if HAVE_GETHOSTUUID + { + struct timespec timeout = {1, 0}; /* 1 sec timeout */ + if( gethostuuid(pHostID, &timeout) ){ + int err = errno; + if( pError ){ + *pError = err; + } + return SQLITE_IOERR; + } + } +#else + UNUSED_PARAMETER(pError); +#endif +#ifdef SQLITE_TEST + /* simulate multiple hosts by creating unique hostid file paths */ + if( sqlite3_hostid_num != 0){ + pHostID[0] = (char)(pHostID[0] + (char)(sqlite3_hostid_num & 0xFF)); + } +#endif + + return SQLITE_OK; +} + +/* The conch file contains the header, host id and lock file path + */ +#define PROXY_CONCHVERSION 2 /* 1-byte header, 16-byte host id, path */ +#define PROXY_HEADERLEN 1 /* conch file header length */ +#define PROXY_PATHINDEX (PROXY_HEADERLEN+PROXY_HOSTIDLEN) +#define PROXY_MAXCONCHLEN (PROXY_HEADERLEN+PROXY_HOSTIDLEN+MAXPATHLEN) + +/* +** Takes an open conch file, copies the contents to a new path and then moves +** it back. The newly created file's file descriptor is assigned to the +** conch file structure and finally the original conch file descriptor is +** closed. Returns zero if successful. +*/ +static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *conchFile = pCtx->conchFile; + char tPath[MAXPATHLEN]; + char buf[PROXY_MAXCONCHLEN]; + char *cPath = pCtx->conchFilePath; + size_t readLen = 0; + size_t pathLen = 0; + char errmsg[64] = ""; + int fd = -1; + int rc = -1; + UNUSED_PARAMETER(myHostID); + + /* create a new path by replace the trailing '-conch' with '-break' */ + pathLen = strlcpy(tPath, cPath, MAXPATHLEN); + if( pathLen>MAXPATHLEN || pathLen<6 || + (strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){ + sqlite3_snprintf(sizeof(errmsg),errmsg,"path error (len %d)",(int)pathLen); + goto end_breaklock; + } + /* read the conch content */ + readLen = osPread(conchFile->h, buf, PROXY_MAXCONCHLEN, 0); + if( readLenh, __LINE__); + conchFile->h = fd; + conchFile->openFlags = O_RDWR | O_CREAT; + +end_breaklock: + if( rc ){ + if( fd>=0 ){ + osUnlink(tPath); + robust_close(pFile, fd, __LINE__); + } + fprintf(stderr, "failed to break stale lock on %s, %s\n", cPath, errmsg); + } + return rc; +} + +/* Take the requested lock on the conch file and break a stale lock if the +** host id matches. +*/ +static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *conchFile = pCtx->conchFile; + int rc = SQLITE_OK; + int nTries = 0; + struct timespec conchModTime; + + memset(&conchModTime, 0, sizeof(conchModTime)); + do { + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); + nTries ++; + if( rc==SQLITE_BUSY ){ + /* If the lock failed (busy): + * 1st try: get the mod time of the conch, wait 0.5s and try again. + * 2nd try: fail if the mod time changed or host id is different, wait + * 10 sec and try again + * 3rd try: break the lock unless the mod time has changed. + */ + struct stat buf; + if( osFstat(conchFile->h, &buf) ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR_LOCK; + } + + if( nTries==1 ){ + conchModTime = buf.st_mtimespec; + unixSleep(0,500000); /* wait 0.5 sec and try the lock again*/ + continue; + } + + assert( nTries>1 ); + if( conchModTime.tv_sec != buf.st_mtimespec.tv_sec || + conchModTime.tv_nsec != buf.st_mtimespec.tv_nsec ){ + return SQLITE_BUSY; + } + + if( nTries==2 ){ + char tBuf[PROXY_MAXCONCHLEN]; + int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); + if( len<0 ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR_LOCK; + } + if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ + /* don't break the lock if the host id doesn't match */ + if( 0!=memcmp(&tBuf[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN) ){ + return SQLITE_BUSY; + } + }else{ + /* don't break the lock on short read or a version mismatch */ + return SQLITE_BUSY; + } + unixSleep(0,10000000); /* wait 10 sec and try the lock again */ + continue; + } + + assert( nTries==3 ); + if( 0==proxyBreakConchLock(pFile, myHostID) ){ + rc = SQLITE_OK; + if( lockType==EXCLUSIVE_LOCK ){ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, SHARED_LOCK); + } + if( !rc ){ + rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); + } + } + } + } while( rc==SQLITE_BUSY && nTries<3 ); + + return rc; +} + +/* Takes the conch by taking a shared lock and read the contents conch, if +** lockPath is non-NULL, the host ID and lock file path must match. A NULL +** lockPath means that the lockPath in the conch file will be used if the +** host IDs match, or a new lock path will be generated automatically +** and written to the conch file. +*/ +static int proxyTakeConch(unixFile *pFile){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + + if( pCtx->conchHeld!=0 ){ + return SQLITE_OK; + }else{ + unixFile *conchFile = pCtx->conchFile; + uuid_t myHostID; + int pError = 0; + char readBuf[PROXY_MAXCONCHLEN]; + char lockPath[MAXPATHLEN]; + char *tempLockPath = NULL; + int rc = SQLITE_OK; + int createConch = 0; + int hostIdMatch = 0; + int readLen = 0; + int tryOldLockPath = 0; + int forceNewLockPath = 0; + + OSTRACE(("TAKECONCH %d for %s pid=%d\n", conchFile->h, + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), + osGetpid(0))); + + rc = proxyGetHostID(myHostID, &pError); + if( (rc&0xff)==SQLITE_IOERR ){ + storeLastErrno(pFile, pError); + goto end_takeconch; + } + rc = proxyConchLock(pFile, myHostID, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + goto end_takeconch; + } + /* read the existing conch file */ + readLen = seekAndRead((unixFile*)conchFile, 0, readBuf, PROXY_MAXCONCHLEN); + if( readLen<0 ){ + /* I/O error: lastErrno set by seekAndRead */ + storeLastErrno(pFile, conchFile->lastErrno); + rc = SQLITE_IOERR_READ; + goto end_takeconch; + }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || + readBuf[0]!=(char)PROXY_CONCHVERSION ){ + /* a short read or version format mismatch means we need to create a new + ** conch file. + */ + createConch = 1; + } + /* if the host id matches and the lock path already exists in the conch + ** we'll try to use the path there, if we can't open that path, we'll + ** retry with a new auto-generated path + */ + do { /* in case we need to try again for an :auto: named lock file */ + + if( !createConch && !forceNewLockPath ){ + hostIdMatch = !memcmp(&readBuf[PROXY_HEADERLEN], myHostID, + PROXY_HOSTIDLEN); + /* if the conch has data compare the contents */ + if( !pCtx->lockProxyPath ){ + /* for auto-named local lock file, just check the host ID and we'll + ** use the local lock file path that's already in there + */ + if( hostIdMatch ){ + size_t pathLen = (readLen - PROXY_PATHINDEX); + + if( pathLen>=MAXPATHLEN ){ + pathLen=MAXPATHLEN-1; + } + memcpy(lockPath, &readBuf[PROXY_PATHINDEX], pathLen); + lockPath[pathLen] = 0; + tempLockPath = lockPath; + tryOldLockPath = 1; + /* create a copy of the lock path if the conch is taken */ + goto end_takeconch; + } + }else if( hostIdMatch + && !strncmp(pCtx->lockProxyPath, &readBuf[PROXY_PATHINDEX], + readLen-PROXY_PATHINDEX) + ){ + /* conch host and lock path match */ + goto end_takeconch; + } + } + + /* if the conch isn't writable and doesn't match, we can't take it */ + if( (conchFile->openFlags&O_RDWR) == 0 ){ + rc = SQLITE_BUSY; + goto end_takeconch; + } + + /* either the conch didn't match or we need to create a new one */ + if( !pCtx->lockProxyPath ){ + proxyGetLockPath(pCtx->dbPath, lockPath, MAXPATHLEN); + tempLockPath = lockPath; + /* create a copy of the lock path _only_ if the conch is taken */ + } + + /* update conch with host and path (this will fail if other process + ** has a shared lock already), if the host id matches, use the big + ** stick. + */ + futimes(conchFile->h, NULL); + if( hostIdMatch && !createConch ){ + if( conchFile->pInode && conchFile->pInode->nShared>1 ){ + /* We are trying for an exclusive lock but another thread in this + ** same process is still holding a shared lock. */ + rc = SQLITE_BUSY; + } else { + rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); + } + }else{ + rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); + } + if( rc==SQLITE_OK ){ + char writeBuffer[PROXY_MAXCONCHLEN]; + int writeSize = 0; + + writeBuffer[0] = (char)PROXY_CONCHVERSION; + memcpy(&writeBuffer[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); + if( pCtx->lockProxyPath!=NULL ){ + strlcpy(&writeBuffer[PROXY_PATHINDEX], pCtx->lockProxyPath, + MAXPATHLEN); + }else{ + strlcpy(&writeBuffer[PROXY_PATHINDEX], tempLockPath, MAXPATHLEN); + } + writeSize = PROXY_PATHINDEX + strlen(&writeBuffer[PROXY_PATHINDEX]); + robust_ftruncate(conchFile->h, writeSize); + rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); + full_fsync(conchFile->h,0,0); + /* If we created a new conch file (not just updated the contents of a + ** valid conch file), try to match the permissions of the database + */ + if( rc==SQLITE_OK && createConch ){ + struct stat buf; + int err = osFstat(pFile->h, &buf); + if( err==0 ){ + mode_t cmode = buf.st_mode&(S_IRUSR|S_IWUSR | S_IRGRP|S_IWGRP | + S_IROTH|S_IWOTH); + /* try to match the database file R/W permissions, ignore failure */ +#ifndef SQLITE_PROXY_DEBUG + osFchmod(conchFile->h, cmode); +#else + do{ + rc = osFchmod(conchFile->h, cmode); + }while( rc==(-1) && errno==EINTR ); + if( rc!=0 ){ + int code = errno; + fprintf(stderr, "fchmod %o FAILED with %d %s\n", + cmode, code, strerror(code)); + } else { + fprintf(stderr, "fchmod %o SUCCEDED\n",cmode); + } + }else{ + int code = errno; + fprintf(stderr, "STAT FAILED[%d] with %d %s\n", + err, code, strerror(code)); +#endif + } + } + } + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, SHARED_LOCK); + + end_takeconch: + OSTRACE(("TRANSPROXY: CLOSE %d\n", pFile->h)); + if( rc==SQLITE_OK && pFile->openFlags ){ + int fd; + if( pFile->h>=0 ){ + robust_close(pFile, pFile->h, __LINE__); + } + pFile->h = -1; + fd = robust_open(pCtx->dbPath, pFile->openFlags, 0); + OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); + if( fd>=0 ){ + pFile->h = fd; + }else{ + rc=SQLITE_CANTOPEN_BKPT; /* SQLITE_BUSY? proxyTakeConch called + during locking */ + } + } + if( rc==SQLITE_OK && !pCtx->lockProxy ){ + char *path = tempLockPath ? tempLockPath : pCtx->lockProxyPath; + rc = proxyCreateUnixFile(path, &pCtx->lockProxy, 1); + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && tryOldLockPath ){ + /* we couldn't create the proxy lock file with the old lock file path + ** so try again via auto-naming + */ + forceNewLockPath = 1; + tryOldLockPath = 0; + continue; /* go back to the do {} while start point, try again */ + } + } + if( rc==SQLITE_OK ){ + /* Need to make a copy of path if we extracted the value + ** from the conch file or the path was allocated on the stack + */ + if( tempLockPath ){ + pCtx->lockProxyPath = sqlite3DbStrDup(0, tempLockPath); + if( !pCtx->lockProxyPath ){ + rc = SQLITE_NOMEM_BKPT; + } + } + } + if( rc==SQLITE_OK ){ + pCtx->conchHeld = 1; + + if( pCtx->lockProxy->pMethod == &afpIoMethods ){ + afpLockingContext *afpCtx; + afpCtx = (afpLockingContext *)pCtx->lockProxy->lockingContext; + afpCtx->dbPath = pCtx->lockProxyPath; + } + } else { + conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + } + OSTRACE(("TAKECONCH %d %s\n", conchFile->h, + rc==SQLITE_OK?"ok":"failed")); + return rc; + } while (1); /* in case we need to retry the :auto: lock file - + ** we should never get here except via the 'continue' call. */ + } +} + +/* +** If pFile holds a lock on a conch file, then release that lock. +*/ +static int proxyReleaseConch(unixFile *pFile){ + int rc = SQLITE_OK; /* Subroutine return code */ + proxyLockingContext *pCtx; /* The locking context for the proxy lock */ + unixFile *conchFile; /* Name of the conch file */ + + pCtx = (proxyLockingContext *)pFile->lockingContext; + conchFile = pCtx->conchFile; + OSTRACE(("RELEASECONCH %d for %s pid=%d\n", conchFile->h, + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), + osGetpid(0))); + if( pCtx->conchHeld>0 ){ + rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); + } + pCtx->conchHeld = 0; + OSTRACE(("RELEASECONCH %d %s\n", conchFile->h, + (rc==SQLITE_OK ? "ok" : "failed"))); + return rc; +} + +/* +** Given the name of a database file, compute the name of its conch file. +** Store the conch filename in memory obtained from sqlite3_malloc64(). +** Make *pConchPath point to the new name. Return SQLITE_OK on success +** or SQLITE_NOMEM if unable to obtain memory. +** +** The caller is responsible for ensuring that the allocated memory +** space is eventually freed. +** +** *pConchPath is set to NULL if a memory allocation error occurs. +*/ +static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ + int i; /* Loop counter */ + int len = (int)strlen(dbPath); /* Length of database filename - dbPath */ + char *conchPath; /* buffer in which to construct conch name */ + + /* Allocate space for the conch filename and initialize the name to + ** the name of the original database file. */ + *pConchPath = conchPath = (char *)sqlite3_malloc64(len + 8); + if( conchPath==0 ){ + return SQLITE_NOMEM_BKPT; + } + memcpy(conchPath, dbPath, len+1); + + /* now insert a "." before the last / character */ + for( i=(len-1); i>=0; i-- ){ + if( conchPath[i]=='/' ){ + i++; + break; + } + } + conchPath[i]='.'; + while ( ilockingContext; + char *oldPath = pCtx->lockProxyPath; + int rc = SQLITE_OK; + + if( pFile->eFileLock!=NO_LOCK ){ + return SQLITE_BUSY; + } + + /* nothing to do if the path is NULL, :auto: or matches the existing path */ + if( !path || path[0]=='\0' || !strcmp(path, ":auto:") || + (oldPath && !strncmp(oldPath, path, MAXPATHLEN)) ){ + return SQLITE_OK; + }else{ + unixFile *lockProxy = pCtx->lockProxy; + pCtx->lockProxy=NULL; + pCtx->conchHeld = 0; + if( lockProxy!=NULL ){ + rc=lockProxy->pMethod->xClose((sqlite3_file *)lockProxy); + if( rc ) return rc; + sqlite3_free(lockProxy); + } + sqlite3_free(oldPath); + pCtx->lockProxyPath = sqlite3DbStrDup(0, path); + } + + return rc; +} + +/* +** pFile is a file that has been opened by a prior xOpen call. dbPath +** is a string buffer at least MAXPATHLEN+1 characters in size. +** +** This routine find the filename associated with pFile and writes it +** int dbPath. +*/ +static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ +#if defined(__APPLE__) + if( pFile->pMethod == &afpIoMethods ){ + /* afp style keeps a reference to the db path in the filePath field + ** of the struct */ + assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); + strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, + MAXPATHLEN); + } else +#endif + if( pFile->pMethod == &dotlockIoMethods ){ + /* dot lock style uses the locking context to store the dot lock + ** file path */ + int len = strlen((char *)pFile->lockingContext) - strlen(DOTLOCK_SUFFIX); + memcpy(dbPath, (char *)pFile->lockingContext, len + 1); + }else{ + /* all other styles use the locking context to store the db file path */ + assert( strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); + strlcpy(dbPath, (char *)pFile->lockingContext, MAXPATHLEN); + } + return SQLITE_OK; +} + +/* +** Takes an already filled in unix file and alters it so all file locking +** will be performed on the local proxy lock file. The following fields +** are preserved in the locking context so that they can be restored and +** the unix structure properly cleaned up at close time: +** ->lockingContext +** ->pMethod +*/ +static int proxyTransformUnixFile(unixFile *pFile, const char *path) { + proxyLockingContext *pCtx; + char dbPath[MAXPATHLEN+1]; /* Name of the database file */ + char *lockPath=NULL; + int rc = SQLITE_OK; + + if( pFile->eFileLock!=NO_LOCK ){ + return SQLITE_BUSY; + } + proxyGetDbPathForUnixFile(pFile, dbPath); + if( !path || path[0]=='\0' || !strcmp(path, ":auto:") ){ + lockPath=NULL; + }else{ + lockPath=(char *)path; + } + + OSTRACE(("TRANSPROXY %d for %s pid=%d\n", pFile->h, + (lockPath ? lockPath : ":auto:"), osGetpid(0))); + + pCtx = sqlite3_malloc64( sizeof(*pCtx) ); + if( pCtx==0 ){ + return SQLITE_NOMEM_BKPT; + } + memset(pCtx, 0, sizeof(*pCtx)); + + rc = proxyCreateConchPathname(dbPath, &pCtx->conchFilePath); + if( rc==SQLITE_OK ){ + rc = proxyCreateUnixFile(pCtx->conchFilePath, &pCtx->conchFile, 0); + if( rc==SQLITE_CANTOPEN && ((pFile->openFlags&O_RDWR) == 0) ){ + /* if (a) the open flags are not O_RDWR, (b) the conch isn't there, and + ** (c) the file system is read-only, then enable no-locking access. + ** Ugh, since O_RDONLY==0x0000 we test for !O_RDWR since unixOpen asserts + ** that openFlags will have only one of O_RDONLY or O_RDWR. + */ + struct statfs fsInfo; + struct stat conchInfo; + int goLockless = 0; + + if( osStat(pCtx->conchFilePath, &conchInfo) == -1 ) { + int err = errno; + if( (err==ENOENT) && (statfs(dbPath, &fsInfo) != -1) ){ + goLockless = (fsInfo.f_flags&MNT_RDONLY) == MNT_RDONLY; + } + } + if( goLockless ){ + pCtx->conchHeld = -1; /* read only FS/ lockless */ + rc = SQLITE_OK; + } + } + } + if( rc==SQLITE_OK && lockPath ){ + pCtx->lockProxyPath = sqlite3DbStrDup(0, lockPath); + } + + if( rc==SQLITE_OK ){ + pCtx->dbPath = sqlite3DbStrDup(0, dbPath); + if( pCtx->dbPath==NULL ){ + rc = SQLITE_NOMEM_BKPT; + } + } + if( rc==SQLITE_OK ){ + /* all memory is allocated, proxys are created and assigned, + ** switch the locking context and pMethod then return. + */ + pCtx->oldLockingContext = pFile->lockingContext; + pFile->lockingContext = pCtx; + pCtx->pOldMethod = pFile->pMethod; + pFile->pMethod = &proxyIoMethods; + }else{ + if( pCtx->conchFile ){ + pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); + sqlite3_free(pCtx->conchFile); + } + sqlite3DbFree(0, pCtx->lockProxyPath); + sqlite3_free(pCtx->conchFilePath); + sqlite3_free(pCtx); + } + OSTRACE(("TRANSPROXY %d %s\n", pFile->h, + (rc==SQLITE_OK ? "ok" : "failed"))); + return rc; +} + + +/* +** This routine handles sqlite3_file_control() calls that are specific +** to proxy locking. +*/ +static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ + switch( op ){ + case SQLITE_FCNTL_GET_LOCKPROXYFILE: { + unixFile *pFile = (unixFile*)id; + if( pFile->pMethod == &proxyIoMethods ){ + proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; + proxyTakeConch(pFile); + if( pCtx->lockProxyPath ){ + *(const char **)pArg = pCtx->lockProxyPath; + }else{ + *(const char **)pArg = ":auto: (not held)"; + } + } else { + *(const char **)pArg = NULL; + } + return SQLITE_OK; + } + case SQLITE_FCNTL_SET_LOCKPROXYFILE: { + unixFile *pFile = (unixFile*)id; + int rc = SQLITE_OK; + int isProxyStyle = (pFile->pMethod == &proxyIoMethods); + if( pArg==NULL || (const char *)pArg==0 ){ + if( isProxyStyle ){ + /* turn off proxy locking - not supported. If support is added for + ** switching proxy locking mode off then it will need to fail if + ** the journal mode is WAL mode. + */ + rc = SQLITE_ERROR /*SQLITE_PROTOCOL? SQLITE_MISUSE?*/; + }else{ + /* turn off proxy locking - already off - NOOP */ + rc = SQLITE_OK; + } + }else{ + const char *proxyPath = (const char *)pArg; + if( isProxyStyle ){ + proxyLockingContext *pCtx = + (proxyLockingContext*)pFile->lockingContext; + if( !strcmp(pArg, ":auto:") + || (pCtx->lockProxyPath && + !strncmp(pCtx->lockProxyPath, proxyPath, MAXPATHLEN)) + ){ + rc = SQLITE_OK; + }else{ + rc = switchLockProxyPath(pFile, proxyPath); + } + }else{ + /* turn on proxy file locking */ + rc = proxyTransformUnixFile(pFile, proxyPath); + } + } + return rc; + } + default: { + assert( 0 ); /* The call assures that only valid opcodes are sent */ + } + } + /*NOTREACHED*/ assert(0); + return SQLITE_ERROR; +} + +/* +** Within this division (the proxying locking implementation) the procedures +** above this point are all utilities. The lock-related methods of the +** proxy-locking sqlite3_io_method object follow. +*/ + + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, set *pResOut +** to a non-zero value otherwise *pResOut is set to zero. The return value +** is set to SQLITE_OK unless an I/O error occurs during lock checking. +*/ +static int proxyCheckReservedLock(sqlite3_file *id, int *pResOut) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + return proxy->pMethod->xCheckReservedLock((sqlite3_file*)proxy, pResOut); + }else{ /* conchHeld < 0 is lockless */ + pResOut=0; + } + } + return rc; +} + +/* +** Lock the file with the lock specified by parameter eFileLock - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. Use the sqlite3OsUnlock() +** routine to lower a locking level. +*/ +static int proxyLock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xLock((sqlite3_file*)proxy, eFileLock); + pFile->eFileLock = proxy->eFileLock; + }else{ + /* conchHeld < 0 is lockless */ + } + } + return rc; +} + + +/* +** Lower the locking level on file descriptor pFile to eFileLock. eFileLock +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +*/ +static int proxyUnlock(sqlite3_file *id, int eFileLock) { + unixFile *pFile = (unixFile*)id; + int rc = proxyTakeConch(pFile); + if( rc==SQLITE_OK ){ + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld>0 ){ + unixFile *proxy = pCtx->lockProxy; + rc = proxy->pMethod->xUnlock((sqlite3_file*)proxy, eFileLock); + pFile->eFileLock = proxy->eFileLock; + }else{ + /* conchHeld < 0 is lockless */ + } + } + return rc; +} + +/* +** Close a file that uses proxy locks. +*/ +static int proxyClose(sqlite3_file *id) { + if( ALWAYS(id) ){ + unixFile *pFile = (unixFile*)id; + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + unixFile *lockProxy = pCtx->lockProxy; + unixFile *conchFile = pCtx->conchFile; + int rc = SQLITE_OK; + + if( lockProxy ){ + rc = lockProxy->pMethod->xUnlock((sqlite3_file*)lockProxy, NO_LOCK); + if( rc ) return rc; + rc = lockProxy->pMethod->xClose((sqlite3_file*)lockProxy); + if( rc ) return rc; + sqlite3_free(lockProxy); + pCtx->lockProxy = 0; + } + if( conchFile ){ + if( pCtx->conchHeld ){ + rc = proxyReleaseConch(pFile); + if( rc ) return rc; + } + rc = conchFile->pMethod->xClose((sqlite3_file*)conchFile); + if( rc ) return rc; + sqlite3_free(conchFile); + } + sqlite3DbFree(0, pCtx->lockProxyPath); + sqlite3_free(pCtx->conchFilePath); + sqlite3DbFree(0, pCtx->dbPath); + /* restore the original locking context and pMethod then close it */ + pFile->lockingContext = pCtx->oldLockingContext; + pFile->pMethod = pCtx->pOldMethod; + sqlite3_free(pCtx); + return pFile->pMethod->xClose(id); + } + return SQLITE_OK; +} + + + +#endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ +/* +** The proxy locking style is intended for use with AFP filesystems. +** And since AFP is only supported on MacOSX, the proxy locking is also +** restricted to MacOSX. +** +** +******************* End of the proxy lock implementation ********************** +******************************************************************************/ + +/* +** Initialize the operating system interface. +** +** This routine registers all VFS implementations for unix-like operating +** systems. This routine, and the sqlite3_os_end() routine that follows, +** should be the only routines in this file that are visible from other +** files. +** +** This routine is called once during SQLite initialization and by a +** single thread. The memory allocation and mutex subsystems have not +** necessarily been initialized when this routine is called, and so they +** should not be used. +*/ +SQLITE_API int sqlite3_os_init(void){ + /* + ** The following macro defines an initializer for an sqlite3_vfs object. + ** The name of the VFS is NAME. The pAppData is a pointer to a pointer + ** to the "finder" function. (pAppData is a pointer to a pointer because + ** silly C90 rules prohibit a void* from being cast to a function pointer + ** and so we have to go through the intermediate pointer to avoid problems + ** when compiling with -pedantic-errors on GCC.) + ** + ** The FINDER parameter to this macro is the name of the pointer to the + ** finder-function. The finder-function returns a pointer to the + ** sqlite_io_methods object that implements the desired locking + ** behaviors. See the division above that contains the IOMETHODS + ** macro for addition information on finder-functions. + ** + ** Most finders simply return a pointer to a fixed sqlite3_io_methods + ** object. But the "autolockIoFinder" available on MacOSX does a little + ** more than that; it looks at the filesystem type that hosts the + ** database file and tries to choose an locking method appropriate for + ** that filesystem time. + */ + #define UNIXVFS(VFSNAME, FINDER) { \ + 3, /* iVersion */ \ + sizeof(unixFile), /* szOsFile */ \ + MAX_PATHNAME, /* mxPathname */ \ + 0, /* pNext */ \ + VFSNAME, /* zName */ \ + (void*)&FINDER, /* pAppData */ \ + unixOpen, /* xOpen */ \ + unixDelete, /* xDelete */ \ + unixAccess, /* xAccess */ \ + unixFullPathname, /* xFullPathname */ \ + unixDlOpen, /* xDlOpen */ \ + unixDlError, /* xDlError */ \ + unixDlSym, /* xDlSym */ \ + unixDlClose, /* xDlClose */ \ + unixRandomness, /* xRandomness */ \ + unixSleep, /* xSleep */ \ + unixCurrentTime, /* xCurrentTime */ \ + unixGetLastError, /* xGetLastError */ \ + unixCurrentTimeInt64, /* xCurrentTimeInt64 */ \ + unixSetSystemCall, /* xSetSystemCall */ \ + unixGetSystemCall, /* xGetSystemCall */ \ + unixNextSystemCall, /* xNextSystemCall */ \ + } + + /* + ** All default VFSes for unix are contained in the following array. + ** + ** Note that the sqlite3_vfs.pNext field of the VFS object is modified + ** by the SQLite core when the VFS is registered. So the following + ** array cannot be const. + */ + static sqlite3_vfs aVfs[] = { +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + UNIXVFS("unix", autolockIoFinder ), +#elif OS_VXWORKS + UNIXVFS("unix", vxworksIoFinder ), +#else + UNIXVFS("unix", posixIoFinder ), +#endif + UNIXVFS("unix-none", nolockIoFinder ), + UNIXVFS("unix-dotfile", dotlockIoFinder ), + UNIXVFS("unix-excl", posixIoFinder ), +#if OS_VXWORKS + UNIXVFS("unix-namedsem", semIoFinder ), +#endif +#if SQLITE_ENABLE_LOCKING_STYLE || OS_VXWORKS + UNIXVFS("unix-posix", posixIoFinder ), +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + UNIXVFS("unix-flock", flockIoFinder ), +#endif +#if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) + UNIXVFS("unix-afp", afpIoFinder ), + UNIXVFS("unix-nfs", nfsIoFinder ), + UNIXVFS("unix-proxy", proxyIoFinder ), +#endif + }; + unsigned int i; /* Loop counter */ + + /* Double-check that the aSyscall[] array has been constructed + ** correctly. See ticket [bb3a86e890c8e96ab] */ + assert( ArraySize(aSyscall)==29 ); + + /* Register all VFSes defined in the aVfs[] array */ + for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ + sqlite3_vfs_register(&aVfs[i], i==0); + } + unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); + +#ifndef SQLITE_OMIT_WAL + /* Validate lock assumptions */ + assert( SQLITE_SHM_NLOCK==8 ); /* Number of available locks */ + assert( UNIX_SHM_BASE==120 ); /* Start of locking area */ + /* Locks: + ** WRITE UNIX_SHM_BASE 120 + ** CKPT UNIX_SHM_BASE+1 121 + ** RECOVER UNIX_SHM_BASE+2 122 + ** READ-0 UNIX_SHM_BASE+3 123 + ** READ-1 UNIX_SHM_BASE+4 124 + ** READ-2 UNIX_SHM_BASE+5 125 + ** READ-3 UNIX_SHM_BASE+6 126 + ** READ-4 UNIX_SHM_BASE+7 127 + ** DMS UNIX_SHM_BASE+8 128 + */ + assert( UNIX_SHM_DMS==128 ); /* Byte offset of the deadman-switch */ +#endif + + return SQLITE_OK; +} + +/* +** Shutdown the operating system interface. +** +** Some operating systems might need to do some cleanup in this routine, +** to release dynamically allocated objects. But not on unix. +** This routine is a no-op for unix. +*/ +SQLITE_API int sqlite3_os_end(void){ + unixBigLock = 0; + return SQLITE_OK; +} + +#endif /* SQLITE_OS_UNIX */ + +/************** End of os_unix.c *********************************************/ +/************** Begin file os_win.c ******************************************/ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code that is specific to Windows. +*/ +/* #include "sqliteInt.h" */ +#if SQLITE_OS_WIN /* This file is used for Windows only */ + +/* +** Include code that is common to all os_*.c files +*/ +/* #include "os_common.h" */ + +/* +** Include the header file for the Windows VFS. +*/ +/* #include "os_win.h" */ + +/* +** Compiling and using WAL mode requires several APIs that are only +** available in Windows platforms based on the NT kernel. +*/ +#if !SQLITE_OS_WINNT && !defined(SQLITE_OMIT_WAL) +# error "WAL mode requires support from the Windows NT kernel, compile\ + with SQLITE_OMIT_WAL." +#endif + +#if !SQLITE_OS_WINNT && SQLITE_MAX_MMAP_SIZE>0 +# error "Memory mapped files require support from the Windows NT kernel,\ + compile with SQLITE_MAX_MMAP_SIZE=0." +#endif + +/* +** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(SQLITE_WIN32_NO_ANSI) +# define SQLITE_WIN32_HAS_ANSI +#endif + +/* +** Are most of the Win32 Unicode APIs available (i.e. with certain exceptions +** based on the sub-platform)? +*/ +#if (SQLITE_OS_WINCE || SQLITE_OS_WINNT || SQLITE_OS_WINRT) && \ + !defined(SQLITE_WIN32_NO_WIDE) +# define SQLITE_WIN32_HAS_WIDE +#endif + +/* +** Make sure at least one set of Win32 APIs is available. +*/ +#if !defined(SQLITE_WIN32_HAS_ANSI) && !defined(SQLITE_WIN32_HAS_WIDE) +# error "At least one of SQLITE_WIN32_HAS_ANSI and SQLITE_WIN32_HAS_WIDE\ + must be defined." +#endif + +/* +** Define the required Windows SDK version constants if they are not +** already available. +*/ +#ifndef NTDDI_WIN8 +# define NTDDI_WIN8 0x06020000 +#endif + +#ifndef NTDDI_WINBLUE +# define NTDDI_WINBLUE 0x06030000 +#endif + +#ifndef NTDDI_WINTHRESHOLD +# define NTDDI_WINTHRESHOLD 0x06040000 +#endif + +/* +** Check to see if the GetVersionEx[AW] functions are deprecated on the +** target system. GetVersionEx was first deprecated in Win8.1. +*/ +#ifndef SQLITE_WIN32_GETVERSIONEX +# if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WINBLUE +# define SQLITE_WIN32_GETVERSIONEX 0 /* GetVersionEx() is deprecated */ +# else +# define SQLITE_WIN32_GETVERSIONEX 1 /* GetVersionEx() is current */ +# endif +#endif + +/* +** Check to see if the CreateFileMappingA function is supported on the +** target system. It is unavailable when using "mincore.lib" on Win10. +** When compiling for Windows 10, always assume "mincore.lib" is in use. +*/ +#ifndef SQLITE_WIN32_CREATEFILEMAPPINGA +# if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WINTHRESHOLD +# define SQLITE_WIN32_CREATEFILEMAPPINGA 0 +# else +# define SQLITE_WIN32_CREATEFILEMAPPINGA 1 +# endif +#endif + +/* +** This constant should already be defined (in the "WinDef.h" SDK file). +*/ +#ifndef MAX_PATH +# define MAX_PATH (260) +#endif + +/* +** Maximum pathname length (in chars) for Win32. This should normally be +** MAX_PATH. +*/ +#ifndef SQLITE_WIN32_MAX_PATH_CHARS +# define SQLITE_WIN32_MAX_PATH_CHARS (MAX_PATH) +#endif + +/* +** This constant should already be defined (in the "WinNT.h" SDK file). +*/ +#ifndef UNICODE_STRING_MAX_CHARS +# define UNICODE_STRING_MAX_CHARS (32767) +#endif + +/* +** Maximum pathname length (in chars) for WinNT. This should normally be +** UNICODE_STRING_MAX_CHARS. +*/ +#ifndef SQLITE_WINNT_MAX_PATH_CHARS +# define SQLITE_WINNT_MAX_PATH_CHARS (UNICODE_STRING_MAX_CHARS) +#endif + +/* +** Maximum pathname length (in bytes) for Win32. The MAX_PATH macro is in +** characters, so we allocate 4 bytes per character assuming worst-case of +** 4-bytes-per-character for UTF8. +*/ +#ifndef SQLITE_WIN32_MAX_PATH_BYTES +# define SQLITE_WIN32_MAX_PATH_BYTES (SQLITE_WIN32_MAX_PATH_CHARS*4) +#endif + +/* +** Maximum pathname length (in bytes) for WinNT. This should normally be +** UNICODE_STRING_MAX_CHARS * sizeof(WCHAR). +*/ +#ifndef SQLITE_WINNT_MAX_PATH_BYTES +# define SQLITE_WINNT_MAX_PATH_BYTES \ + (sizeof(WCHAR) * SQLITE_WINNT_MAX_PATH_CHARS) +#endif + +/* +** Maximum error message length (in chars) for WinRT. +*/ +#ifndef SQLITE_WIN32_MAX_ERRMSG_CHARS +# define SQLITE_WIN32_MAX_ERRMSG_CHARS (1024) +#endif + +/* +** Returns non-zero if the character should be treated as a directory +** separator. +*/ +#ifndef winIsDirSep +# define winIsDirSep(a) (((a) == '/') || ((a) == '\\')) +#endif + +/* +** This macro is used when a local variable is set to a value that is +** [sometimes] not used by the code (e.g. via conditional compilation). +*/ +#ifndef UNUSED_VARIABLE_VALUE +# define UNUSED_VARIABLE_VALUE(x) (void)(x) +#endif + +/* +** Returns the character that should be used as the directory separator. +*/ +#ifndef winGetDirSep +# define winGetDirSep() '\\' +#endif + +/* +** Do we need to manually define the Win32 file mapping APIs for use with WAL +** mode or memory mapped files (e.g. these APIs are available in the Windows +** CE SDK; however, they are not present in the header file)? +*/ +#if SQLITE_WIN32_FILEMAPPING_API && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) +/* +** Two of the file mapping APIs are different under WinRT. Figure out which +** set we need. +*/ +#if SQLITE_OS_WINRT +WINBASEAPI HANDLE WINAPI CreateFileMappingFromApp(HANDLE, \ + LPSECURITY_ATTRIBUTES, ULONG, ULONG64, LPCWSTR); + +WINBASEAPI LPVOID WINAPI MapViewOfFileFromApp(HANDLE, ULONG, ULONG64, SIZE_T); +#else +#if defined(SQLITE_WIN32_HAS_ANSI) +WINBASEAPI HANDLE WINAPI CreateFileMappingA(HANDLE, LPSECURITY_ATTRIBUTES, \ + DWORD, DWORD, DWORD, LPCSTR); +#endif /* defined(SQLITE_WIN32_HAS_ANSI) */ + +#if defined(SQLITE_WIN32_HAS_WIDE) +WINBASEAPI HANDLE WINAPI CreateFileMappingW(HANDLE, LPSECURITY_ATTRIBUTES, \ + DWORD, DWORD, DWORD, LPCWSTR); +#endif /* defined(SQLITE_WIN32_HAS_WIDE) */ + +WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); +#endif /* SQLITE_OS_WINRT */ + +/* +** These file mapping APIs are common to both Win32 and WinRT. +*/ + +WINBASEAPI BOOL WINAPI FlushViewOfFile(LPCVOID, SIZE_T); +WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); +#endif /* SQLITE_WIN32_FILEMAPPING_API */ + +/* +** Some Microsoft compilers lack this definition. +*/ +#ifndef INVALID_FILE_ATTRIBUTES +# define INVALID_FILE_ATTRIBUTES ((DWORD)-1) +#endif + +#ifndef FILE_FLAG_MASK +# define FILE_FLAG_MASK (0xFF3C0000) +#endif + +#ifndef FILE_ATTRIBUTE_MASK +# define FILE_ATTRIBUTE_MASK (0x0003FFF7) +#endif + +#ifndef SQLITE_OMIT_WAL +/* Forward references to structures used for WAL */ +typedef struct winShm winShm; /* A connection to shared-memory */ +typedef struct winShmNode winShmNode; /* A region of shared-memory */ +#endif + +/* +** WinCE lacks native support for file locking so we have to fake it +** with some code of our own. +*/ +#if SQLITE_OS_WINCE +typedef struct winceLock { + int nReaders; /* Number of reader locks obtained */ + BOOL bPending; /* Indicates a pending lock has been obtained */ + BOOL bReserved; /* Indicates a reserved lock has been obtained */ + BOOL bExclusive; /* Indicates an exclusive lock has been obtained */ +} winceLock; +#endif + +/* +** The winFile structure is a subclass of sqlite3_file* specific to the win32 +** portability layer. +*/ +typedef struct winFile winFile; +struct winFile { + const sqlite3_io_methods *pMethod; /*** Must be first ***/ + sqlite3_vfs *pVfs; /* The VFS used to open this file */ + HANDLE h; /* Handle for accessing the file */ + u8 locktype; /* Type of lock currently held on this file */ + short sharedLockByte; /* Randomly chosen byte used as a shared lock */ + u8 ctrlFlags; /* Flags. See WINFILE_* below */ + DWORD lastErrno; /* The Windows errno from the last I/O error */ +#ifndef SQLITE_OMIT_WAL + winShm *pShm; /* Instance of shared memory on this file */ +#endif + const char *zPath; /* Full pathname of this file */ + int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ +#if SQLITE_OS_WINCE + LPWSTR zDeleteOnClose; /* Name of file to delete when closing */ + HANDLE hMutex; /* Mutex used to control access to shared lock */ + HANDLE hShared; /* Shared memory segment used for locking */ + winceLock local; /* Locks obtained by this instance of winFile */ + winceLock *shared; /* Global shared lock memory for the file */ +#endif +#if SQLITE_MAX_MMAP_SIZE>0 + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Size of mapped region */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ +#endif +}; + +/* +** The winVfsAppData structure is used for the pAppData member for all of the +** Win32 VFS variants. +*/ +typedef struct winVfsAppData winVfsAppData; +struct winVfsAppData { + const sqlite3_io_methods *pMethod; /* The file I/O methods to use. */ + void *pAppData; /* The extra pAppData, if any. */ + BOOL bNoLock; /* Non-zero if locking is disabled. */ +}; + +/* +** Allowed values for winFile.ctrlFlags +*/ +#define WINFILE_RDONLY 0x02 /* Connection is read only */ +#define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ +#define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ + +/* + * The size of the buffer used by sqlite3_win32_write_debug(). + */ +#ifndef SQLITE_WIN32_DBG_BUF_SIZE +# define SQLITE_WIN32_DBG_BUF_SIZE ((int)(4096-sizeof(DWORD))) +#endif + +/* + * If compiled with SQLITE_WIN32_MALLOC on Windows, we will use the + * various Win32 API heap functions instead of our own. + */ +#ifdef SQLITE_WIN32_MALLOC + +/* + * If this is non-zero, an isolated heap will be created by the native Win32 + * allocator subsystem; otherwise, the default process heap will be used. This + * setting has no effect when compiling for WinRT. By default, this is enabled + * and an isolated heap will be created to store all allocated data. + * + ****************************************************************************** + * WARNING: It is important to note that when this setting is non-zero and the + * winMemShutdown function is called (e.g. by the sqlite3_shutdown + * function), all data that was allocated using the isolated heap will + * be freed immediately and any attempt to access any of that freed + * data will almost certainly result in an immediate access violation. + ****************************************************************************** + */ +#ifndef SQLITE_WIN32_HEAP_CREATE +# define SQLITE_WIN32_HEAP_CREATE (TRUE) +#endif + +/* + * This is the maximum possible initial size of the Win32-specific heap, in + * bytes. + */ +#ifndef SQLITE_WIN32_HEAP_MAX_INIT_SIZE +# define SQLITE_WIN32_HEAP_MAX_INIT_SIZE (4294967295U) +#endif + +/* + * This is the extra space for the initial size of the Win32-specific heap, + * in bytes. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_INIT_EXTRA +# define SQLITE_WIN32_HEAP_INIT_EXTRA (4194304) +#endif + +/* + * Calculate the maximum legal cache size, in pages, based on the maximum + * possible initial heap size and the default page size, setting aside the + * needed extra space. + */ +#ifndef SQLITE_WIN32_MAX_CACHE_SIZE +# define SQLITE_WIN32_MAX_CACHE_SIZE (((SQLITE_WIN32_HEAP_MAX_INIT_SIZE) - \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) / \ + (SQLITE_DEFAULT_PAGE_SIZE)) +#endif + +/* + * This is cache size used in the calculation of the initial size of the + * Win32-specific heap. It cannot be negative. + */ +#ifndef SQLITE_WIN32_CACHE_SIZE +# if SQLITE_DEFAULT_CACHE_SIZE>=0 +# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) +# else +# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) +# endif +#endif + +/* + * Make sure that the calculated cache size, in pages, cannot cause the + * initial size of the Win32-specific heap to exceed the maximum amount + * of memory that can be specified in the call to HeapCreate. + */ +#if SQLITE_WIN32_CACHE_SIZE>SQLITE_WIN32_MAX_CACHE_SIZE +# undef SQLITE_WIN32_CACHE_SIZE +# define SQLITE_WIN32_CACHE_SIZE (2000) +#endif + +/* + * The initial size of the Win32-specific heap. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_INIT_SIZE +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ + (SQLITE_DEFAULT_PAGE_SIZE) + \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) +#endif + +/* + * The maximum size of the Win32-specific heap. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_MAX_SIZE +# define SQLITE_WIN32_HEAP_MAX_SIZE (0) +#endif + +/* + * The extra flags to use in calls to the Win32 heap APIs. This value may be + * zero for the default behavior. + */ +#ifndef SQLITE_WIN32_HEAP_FLAGS +# define SQLITE_WIN32_HEAP_FLAGS (0) +#endif + + +/* +** The winMemData structure stores information required by the Win32-specific +** sqlite3_mem_methods implementation. +*/ +typedef struct winMemData winMemData; +struct winMemData { +#ifndef NDEBUG + u32 magic1; /* Magic number to detect structure corruption. */ +#endif + HANDLE hHeap; /* The handle to our heap. */ + BOOL bOwned; /* Do we own the heap (i.e. destroy it on shutdown)? */ +#ifndef NDEBUG + u32 magic2; /* Magic number to detect structure corruption. */ +#endif +}; + +#ifndef NDEBUG +#define WINMEM_MAGIC1 0x42b2830b +#define WINMEM_MAGIC2 0xbd4d7cf4 +#endif + +static struct winMemData win_mem_data = { +#ifndef NDEBUG + WINMEM_MAGIC1, +#endif + NULL, FALSE +#ifndef NDEBUG + ,WINMEM_MAGIC2 +#endif +}; + +#ifndef NDEBUG +#define winMemAssertMagic1() assert( win_mem_data.magic1==WINMEM_MAGIC1 ) +#define winMemAssertMagic2() assert( win_mem_data.magic2==WINMEM_MAGIC2 ) +#define winMemAssertMagic() winMemAssertMagic1(); winMemAssertMagic2(); +#else +#define winMemAssertMagic() +#endif + +#define winMemGetDataPtr() &win_mem_data +#define winMemGetHeap() win_mem_data.hHeap +#define winMemGetOwned() win_mem_data.bOwned + +static void *winMemMalloc(int nBytes); +static void winMemFree(void *pPrior); +static void *winMemRealloc(void *pPrior, int nBytes); +static int winMemSize(void *p); +static int winMemRoundup(int n); +static int winMemInit(void *pAppData); +static void winMemShutdown(void *pAppData); + +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetWin32(void); +#endif /* SQLITE_WIN32_MALLOC */ + +/* +** The following variable is (normally) set once and never changes +** thereafter. It records whether the operating system is Win9x +** or WinNT. +** +** 0: Operating system unknown. +** 1: Operating system is Win9x. +** 2: Operating system is WinNT. +** +** In order to facilitate testing on a WinNT system, the test fixture +** can manually set this value to 1 to emulate Win98 behavior. +*/ +#ifdef SQLITE_TEST +SQLITE_API LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; +#else +static LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; +#endif + +#ifndef SYSCALL +# define SYSCALL sqlite3_syscall_ptr +#endif + +/* +** This function is not available on Windows CE or WinRT. + */ + +#if SQLITE_OS_WINCE || SQLITE_OS_WINRT +# define osAreFileApisANSI() 1 +#endif + +/* +** Many system calls are accessed through pointer-to-functions so that +** they may be overridden at runtime to facilitate fault injection during +** testing and sandboxing. The following array holds the names and pointers +** to all overrideable system calls. +*/ +static struct win_syscall { + const char *zName; /* Name of the system call */ + sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ + sqlite3_syscall_ptr pDefault; /* Default value */ +} aSyscall[] = { +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "AreFileApisANSI", (SYSCALL)AreFileApisANSI, 0 }, +#else + { "AreFileApisANSI", (SYSCALL)0, 0 }, +#endif + +#ifndef osAreFileApisANSI +#define osAreFileApisANSI ((BOOL(WINAPI*)(VOID))aSyscall[0].pCurrent) +#endif + +#if SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) + { "CharLowerW", (SYSCALL)CharLowerW, 0 }, +#else + { "CharLowerW", (SYSCALL)0, 0 }, +#endif + +#define osCharLowerW ((LPWSTR(WINAPI*)(LPWSTR))aSyscall[1].pCurrent) + +#if SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_WIDE) + { "CharUpperW", (SYSCALL)CharUpperW, 0 }, +#else + { "CharUpperW", (SYSCALL)0, 0 }, +#endif + +#define osCharUpperW ((LPWSTR(WINAPI*)(LPWSTR))aSyscall[2].pCurrent) + + { "CloseHandle", (SYSCALL)CloseHandle, 0 }, + +#define osCloseHandle ((BOOL(WINAPI*)(HANDLE))aSyscall[3].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "CreateFileA", (SYSCALL)CreateFileA, 0 }, +#else + { "CreateFileA", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileA ((HANDLE(WINAPI*)(LPCSTR,DWORD,DWORD, \ + LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[4].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "CreateFileW", (SYSCALL)CreateFileW, 0 }, +#else + { "CreateFileW", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileW ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD, \ + LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) && \ + SQLITE_WIN32_CREATEFILEMAPPINGA + { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, +#else + { "CreateFileMappingA", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingA ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ + DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) + +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) + { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, +#else + { "CreateFileMappingW", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingW ((HANDLE(WINAPI*)(HANDLE,LPSECURITY_ATTRIBUTES, \ + DWORD,DWORD,DWORD,LPCWSTR))aSyscall[7].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "CreateMutexW", (SYSCALL)CreateMutexW, 0 }, +#else + { "CreateMutexW", (SYSCALL)0, 0 }, +#endif + +#define osCreateMutexW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,BOOL, \ + LPCWSTR))aSyscall[8].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "DeleteFileA", (SYSCALL)DeleteFileA, 0 }, +#else + { "DeleteFileA", (SYSCALL)0, 0 }, +#endif + +#define osDeleteFileA ((BOOL(WINAPI*)(LPCSTR))aSyscall[9].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "DeleteFileW", (SYSCALL)DeleteFileW, 0 }, +#else + { "DeleteFileW", (SYSCALL)0, 0 }, +#endif + +#define osDeleteFileW ((BOOL(WINAPI*)(LPCWSTR))aSyscall[10].pCurrent) + +#if SQLITE_OS_WINCE + { "FileTimeToLocalFileTime", (SYSCALL)FileTimeToLocalFileTime, 0 }, +#else + { "FileTimeToLocalFileTime", (SYSCALL)0, 0 }, +#endif + +#define osFileTimeToLocalFileTime ((BOOL(WINAPI*)(CONST FILETIME*, \ + LPFILETIME))aSyscall[11].pCurrent) + +#if SQLITE_OS_WINCE + { "FileTimeToSystemTime", (SYSCALL)FileTimeToSystemTime, 0 }, +#else + { "FileTimeToSystemTime", (SYSCALL)0, 0 }, +#endif + +#define osFileTimeToSystemTime ((BOOL(WINAPI*)(CONST FILETIME*, \ + LPSYSTEMTIME))aSyscall[12].pCurrent) + + { "FlushFileBuffers", (SYSCALL)FlushFileBuffers, 0 }, + +#define osFlushFileBuffers ((BOOL(WINAPI*)(HANDLE))aSyscall[13].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "FormatMessageA", (SYSCALL)FormatMessageA, 0 }, +#else + { "FormatMessageA", (SYSCALL)0, 0 }, +#endif + +#define osFormatMessageA ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPSTR, \ + DWORD,va_list*))aSyscall[14].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "FormatMessageW", (SYSCALL)FormatMessageW, 0 }, +#else + { "FormatMessageW", (SYSCALL)0, 0 }, +#endif + +#define osFormatMessageW ((DWORD(WINAPI*)(DWORD,LPCVOID,DWORD,DWORD,LPWSTR, \ + DWORD,va_list*))aSyscall[15].pCurrent) + +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "FreeLibrary", (SYSCALL)FreeLibrary, 0 }, +#else + { "FreeLibrary", (SYSCALL)0, 0 }, +#endif + +#define osFreeLibrary ((BOOL(WINAPI*)(HMODULE))aSyscall[16].pCurrent) + + { "GetCurrentProcessId", (SYSCALL)GetCurrentProcessId, 0 }, + +#define osGetCurrentProcessId ((DWORD(WINAPI*)(VOID))aSyscall[17].pCurrent) + +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) + { "GetDiskFreeSpaceA", (SYSCALL)GetDiskFreeSpaceA, 0 }, +#else + { "GetDiskFreeSpaceA", (SYSCALL)0, 0 }, +#endif + +#define osGetDiskFreeSpaceA ((BOOL(WINAPI*)(LPCSTR,LPDWORD,LPDWORD,LPDWORD, \ + LPDWORD))aSyscall[18].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetDiskFreeSpaceW", (SYSCALL)GetDiskFreeSpaceW, 0 }, +#else + { "GetDiskFreeSpaceW", (SYSCALL)0, 0 }, +#endif + +#define osGetDiskFreeSpaceW ((BOOL(WINAPI*)(LPCWSTR,LPDWORD,LPDWORD,LPDWORD, \ + LPDWORD))aSyscall[19].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "GetFileAttributesA", (SYSCALL)GetFileAttributesA, 0 }, +#else + { "GetFileAttributesA", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesA ((DWORD(WINAPI*)(LPCSTR))aSyscall[20].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetFileAttributesW", (SYSCALL)GetFileAttributesW, 0 }, +#else + { "GetFileAttributesW", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesW ((DWORD(WINAPI*)(LPCWSTR))aSyscall[21].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "GetFileAttributesExW", (SYSCALL)GetFileAttributesExW, 0 }, +#else + { "GetFileAttributesExW", (SYSCALL)0, 0 }, +#endif + +#define osGetFileAttributesExW ((BOOL(WINAPI*)(LPCWSTR,GET_FILEEX_INFO_LEVELS, \ + LPVOID))aSyscall[22].pCurrent) + +#if !SQLITE_OS_WINRT + { "GetFileSize", (SYSCALL)GetFileSize, 0 }, +#else + { "GetFileSize", (SYSCALL)0, 0 }, +#endif + +#define osGetFileSize ((DWORD(WINAPI*)(HANDLE,LPDWORD))aSyscall[23].pCurrent) + +#if !SQLITE_OS_WINCE && defined(SQLITE_WIN32_HAS_ANSI) + { "GetFullPathNameA", (SYSCALL)GetFullPathNameA, 0 }, +#else + { "GetFullPathNameA", (SYSCALL)0, 0 }, +#endif + +#define osGetFullPathNameA ((DWORD(WINAPI*)(LPCSTR,DWORD,LPSTR, \ + LPSTR*))aSyscall[24].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetFullPathNameW", (SYSCALL)GetFullPathNameW, 0 }, +#else + { "GetFullPathNameW", (SYSCALL)0, 0 }, +#endif + +#define osGetFullPathNameW ((DWORD(WINAPI*)(LPCWSTR,DWORD,LPWSTR, \ + LPWSTR*))aSyscall[25].pCurrent) + + { "GetLastError", (SYSCALL)GetLastError, 0 }, + +#define osGetLastError ((DWORD(WINAPI*)(VOID))aSyscall[26].pCurrent) + +#if !defined(SQLITE_OMIT_LOAD_EXTENSION) +#if SQLITE_OS_WINCE + /* The GetProcAddressA() routine is only available on Windows CE. */ + { "GetProcAddressA", (SYSCALL)GetProcAddressA, 0 }, +#else + /* All other Windows platforms expect GetProcAddress() to take + ** an ANSI string regardless of the _UNICODE setting */ + { "GetProcAddressA", (SYSCALL)GetProcAddress, 0 }, +#endif +#else + { "GetProcAddressA", (SYSCALL)0, 0 }, +#endif + +#define osGetProcAddressA ((FARPROC(WINAPI*)(HMODULE, \ + LPCSTR))aSyscall[27].pCurrent) + +#if !SQLITE_OS_WINRT + { "GetSystemInfo", (SYSCALL)GetSystemInfo, 0 }, +#else + { "GetSystemInfo", (SYSCALL)0, 0 }, +#endif + +#define osGetSystemInfo ((VOID(WINAPI*)(LPSYSTEM_INFO))aSyscall[28].pCurrent) + + { "GetSystemTime", (SYSCALL)GetSystemTime, 0 }, + +#define osGetSystemTime ((VOID(WINAPI*)(LPSYSTEMTIME))aSyscall[29].pCurrent) + +#if !SQLITE_OS_WINCE + { "GetSystemTimeAsFileTime", (SYSCALL)GetSystemTimeAsFileTime, 0 }, +#else + { "GetSystemTimeAsFileTime", (SYSCALL)0, 0 }, +#endif + +#define osGetSystemTimeAsFileTime ((VOID(WINAPI*)( \ + LPFILETIME))aSyscall[30].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "GetTempPathA", (SYSCALL)GetTempPathA, 0 }, +#else + { "GetTempPathA", (SYSCALL)0, 0 }, +#endif + +#define osGetTempPathA ((DWORD(WINAPI*)(DWORD,LPSTR))aSyscall[31].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) + { "GetTempPathW", (SYSCALL)GetTempPathW, 0 }, +#else + { "GetTempPathW", (SYSCALL)0, 0 }, +#endif + +#define osGetTempPathW ((DWORD(WINAPI*)(DWORD,LPWSTR))aSyscall[32].pCurrent) + +#if !SQLITE_OS_WINRT + { "GetTickCount", (SYSCALL)GetTickCount, 0 }, +#else + { "GetTickCount", (SYSCALL)0, 0 }, +#endif + +#define osGetTickCount ((DWORD(WINAPI*)(VOID))aSyscall[33].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_GETVERSIONEX + { "GetVersionExA", (SYSCALL)GetVersionExA, 0 }, +#else + { "GetVersionExA", (SYSCALL)0, 0 }, +#endif + +#define osGetVersionExA ((BOOL(WINAPI*)( \ + LPOSVERSIONINFOA))aSyscall[34].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + SQLITE_WIN32_GETVERSIONEX + { "GetVersionExW", (SYSCALL)GetVersionExW, 0 }, +#else + { "GetVersionExW", (SYSCALL)0, 0 }, +#endif + +#define osGetVersionExW ((BOOL(WINAPI*)( \ + LPOSVERSIONINFOW))aSyscall[35].pCurrent) + + { "HeapAlloc", (SYSCALL)HeapAlloc, 0 }, + +#define osHeapAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD, \ + SIZE_T))aSyscall[36].pCurrent) + +#if !SQLITE_OS_WINRT + { "HeapCreate", (SYSCALL)HeapCreate, 0 }, +#else + { "HeapCreate", (SYSCALL)0, 0 }, +#endif + +#define osHeapCreate ((HANDLE(WINAPI*)(DWORD,SIZE_T, \ + SIZE_T))aSyscall[37].pCurrent) + +#if !SQLITE_OS_WINRT + { "HeapDestroy", (SYSCALL)HeapDestroy, 0 }, +#else + { "HeapDestroy", (SYSCALL)0, 0 }, +#endif + +#define osHeapDestroy ((BOOL(WINAPI*)(HANDLE))aSyscall[38].pCurrent) + + { "HeapFree", (SYSCALL)HeapFree, 0 }, + +#define osHeapFree ((BOOL(WINAPI*)(HANDLE,DWORD,LPVOID))aSyscall[39].pCurrent) + + { "HeapReAlloc", (SYSCALL)HeapReAlloc, 0 }, + +#define osHeapReAlloc ((LPVOID(WINAPI*)(HANDLE,DWORD,LPVOID, \ + SIZE_T))aSyscall[40].pCurrent) + + { "HeapSize", (SYSCALL)HeapSize, 0 }, + +#define osHeapSize ((SIZE_T(WINAPI*)(HANDLE,DWORD, \ + LPCVOID))aSyscall[41].pCurrent) + +#if !SQLITE_OS_WINRT + { "HeapValidate", (SYSCALL)HeapValidate, 0 }, +#else + { "HeapValidate", (SYSCALL)0, 0 }, +#endif + +#define osHeapValidate ((BOOL(WINAPI*)(HANDLE,DWORD, \ + LPCVOID))aSyscall[42].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "HeapCompact", (SYSCALL)HeapCompact, 0 }, +#else + { "HeapCompact", (SYSCALL)0, 0 }, +#endif + +#define osHeapCompact ((UINT(WINAPI*)(HANDLE,DWORD))aSyscall[43].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) && !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "LoadLibraryA", (SYSCALL)LoadLibraryA, 0 }, +#else + { "LoadLibraryA", (SYSCALL)0, 0 }, +#endif + +#define osLoadLibraryA ((HMODULE(WINAPI*)(LPCSTR))aSyscall[44].pCurrent) + +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "LoadLibraryW", (SYSCALL)LoadLibraryW, 0 }, +#else + { "LoadLibraryW", (SYSCALL)0, 0 }, +#endif + +#define osLoadLibraryW ((HMODULE(WINAPI*)(LPCWSTR))aSyscall[45].pCurrent) + +#if !SQLITE_OS_WINRT + { "LocalFree", (SYSCALL)LocalFree, 0 }, +#else + { "LocalFree", (SYSCALL)0, 0 }, +#endif + +#define osLocalFree ((HLOCAL(WINAPI*)(HLOCAL))aSyscall[46].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "LockFile", (SYSCALL)LockFile, 0 }, +#else + { "LockFile", (SYSCALL)0, 0 }, +#endif + +#ifndef osLockFile +#define osLockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + DWORD))aSyscall[47].pCurrent) +#endif + +#if !SQLITE_OS_WINCE + { "LockFileEx", (SYSCALL)LockFileEx, 0 }, +#else + { "LockFileEx", (SYSCALL)0, 0 }, +#endif + +#ifndef osLockFileEx +#define osLockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD,DWORD, \ + LPOVERLAPPED))aSyscall[48].pCurrent) +#endif + +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) + { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, +#else + { "MapViewOfFile", (SYSCALL)0, 0 }, +#endif + +#define osMapViewOfFile ((LPVOID(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + SIZE_T))aSyscall[49].pCurrent) + + { "MultiByteToWideChar", (SYSCALL)MultiByteToWideChar, 0 }, + +#define osMultiByteToWideChar ((int(WINAPI*)(UINT,DWORD,LPCSTR,int,LPWSTR, \ + int))aSyscall[50].pCurrent) + + { "QueryPerformanceCounter", (SYSCALL)QueryPerformanceCounter, 0 }, + +#define osQueryPerformanceCounter ((BOOL(WINAPI*)( \ + LARGE_INTEGER*))aSyscall[51].pCurrent) + + { "ReadFile", (SYSCALL)ReadFile, 0 }, + +#define osReadFile ((BOOL(WINAPI*)(HANDLE,LPVOID,DWORD,LPDWORD, \ + LPOVERLAPPED))aSyscall[52].pCurrent) + + { "SetEndOfFile", (SYSCALL)SetEndOfFile, 0 }, + +#define osSetEndOfFile ((BOOL(WINAPI*)(HANDLE))aSyscall[53].pCurrent) + +#if !SQLITE_OS_WINRT + { "SetFilePointer", (SYSCALL)SetFilePointer, 0 }, +#else + { "SetFilePointer", (SYSCALL)0, 0 }, +#endif + +#define osSetFilePointer ((DWORD(WINAPI*)(HANDLE,LONG,PLONG, \ + DWORD))aSyscall[54].pCurrent) + +#if !SQLITE_OS_WINRT + { "Sleep", (SYSCALL)Sleep, 0 }, +#else + { "Sleep", (SYSCALL)0, 0 }, +#endif + +#define osSleep ((VOID(WINAPI*)(DWORD))aSyscall[55].pCurrent) + + { "SystemTimeToFileTime", (SYSCALL)SystemTimeToFileTime, 0 }, + +#define osSystemTimeToFileTime ((BOOL(WINAPI*)(CONST SYSTEMTIME*, \ + LPFILETIME))aSyscall[56].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT + { "UnlockFile", (SYSCALL)UnlockFile, 0 }, +#else + { "UnlockFile", (SYSCALL)0, 0 }, +#endif + +#ifndef osUnlockFile +#define osUnlockFile ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + DWORD))aSyscall[57].pCurrent) +#endif + +#if !SQLITE_OS_WINCE + { "UnlockFileEx", (SYSCALL)UnlockFileEx, 0 }, +#else + { "UnlockFileEx", (SYSCALL)0, 0 }, +#endif + +#define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ + LPOVERLAPPED))aSyscall[58].pCurrent) + +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 + { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, +#else + { "UnmapViewOfFile", (SYSCALL)0, 0 }, +#endif + +#define osUnmapViewOfFile ((BOOL(WINAPI*)(LPCVOID))aSyscall[59].pCurrent) + + { "WideCharToMultiByte", (SYSCALL)WideCharToMultiByte, 0 }, + +#define osWideCharToMultiByte ((int(WINAPI*)(UINT,DWORD,LPCWSTR,int,LPSTR,int, \ + LPCSTR,LPBOOL))aSyscall[60].pCurrent) + + { "WriteFile", (SYSCALL)WriteFile, 0 }, + +#define osWriteFile ((BOOL(WINAPI*)(HANDLE,LPCVOID,DWORD,LPDWORD, \ + LPOVERLAPPED))aSyscall[61].pCurrent) + +#if SQLITE_OS_WINRT + { "CreateEventExW", (SYSCALL)CreateEventExW, 0 }, +#else + { "CreateEventExW", (SYSCALL)0, 0 }, +#endif + +#define osCreateEventExW ((HANDLE(WINAPI*)(LPSECURITY_ATTRIBUTES,LPCWSTR, \ + DWORD,DWORD))aSyscall[62].pCurrent) + +#if !SQLITE_OS_WINRT + { "WaitForSingleObject", (SYSCALL)WaitForSingleObject, 0 }, +#else + { "WaitForSingleObject", (SYSCALL)0, 0 }, +#endif + +#define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ + DWORD))aSyscall[63].pCurrent) + +#if !SQLITE_OS_WINCE + { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, +#else + { "WaitForSingleObjectEx", (SYSCALL)0, 0 }, +#endif + +#define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ + BOOL))aSyscall[64].pCurrent) + +#if SQLITE_OS_WINRT + { "SetFilePointerEx", (SYSCALL)SetFilePointerEx, 0 }, +#else + { "SetFilePointerEx", (SYSCALL)0, 0 }, +#endif + +#define osSetFilePointerEx ((BOOL(WINAPI*)(HANDLE,LARGE_INTEGER, \ + PLARGE_INTEGER,DWORD))aSyscall[65].pCurrent) + +#if SQLITE_OS_WINRT + { "GetFileInformationByHandleEx", (SYSCALL)GetFileInformationByHandleEx, 0 }, +#else + { "GetFileInformationByHandleEx", (SYSCALL)0, 0 }, +#endif + +#define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ + FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) + +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) + { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, +#else + { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, +#endif + +#define osMapViewOfFileFromApp ((LPVOID(WINAPI*)(HANDLE,ULONG,ULONG64, \ + SIZE_T))aSyscall[67].pCurrent) + +#if SQLITE_OS_WINRT + { "CreateFile2", (SYSCALL)CreateFile2, 0 }, +#else + { "CreateFile2", (SYSCALL)0, 0 }, +#endif + +#define osCreateFile2 ((HANDLE(WINAPI*)(LPCWSTR,DWORD,DWORD,DWORD, \ + LPCREATEFILE2_EXTENDED_PARAMETERS))aSyscall[68].pCurrent) + +#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_LOAD_EXTENSION) + { "LoadPackagedLibrary", (SYSCALL)LoadPackagedLibrary, 0 }, +#else + { "LoadPackagedLibrary", (SYSCALL)0, 0 }, +#endif + +#define osLoadPackagedLibrary ((HMODULE(WINAPI*)(LPCWSTR, \ + DWORD))aSyscall[69].pCurrent) + +#if SQLITE_OS_WINRT + { "GetTickCount64", (SYSCALL)GetTickCount64, 0 }, +#else + { "GetTickCount64", (SYSCALL)0, 0 }, +#endif + +#define osGetTickCount64 ((ULONGLONG(WINAPI*)(VOID))aSyscall[70].pCurrent) + +#if SQLITE_OS_WINRT + { "GetNativeSystemInfo", (SYSCALL)GetNativeSystemInfo, 0 }, +#else + { "GetNativeSystemInfo", (SYSCALL)0, 0 }, +#endif + +#define osGetNativeSystemInfo ((VOID(WINAPI*)( \ + LPSYSTEM_INFO))aSyscall[71].pCurrent) + +#if defined(SQLITE_WIN32_HAS_ANSI) + { "OutputDebugStringA", (SYSCALL)OutputDebugStringA, 0 }, +#else + { "OutputDebugStringA", (SYSCALL)0, 0 }, +#endif + +#define osOutputDebugStringA ((VOID(WINAPI*)(LPCSTR))aSyscall[72].pCurrent) + +#if defined(SQLITE_WIN32_HAS_WIDE) + { "OutputDebugStringW", (SYSCALL)OutputDebugStringW, 0 }, +#else + { "OutputDebugStringW", (SYSCALL)0, 0 }, +#endif + +#define osOutputDebugStringW ((VOID(WINAPI*)(LPCWSTR))aSyscall[73].pCurrent) + + { "GetProcessHeap", (SYSCALL)GetProcessHeap, 0 }, + +#define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) + +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) + { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, +#else + { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, +#endif + +#define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ + LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[75].pCurrent) + +/* +** NOTE: On some sub-platforms, the InterlockedCompareExchange "function" +** is really just a macro that uses a compiler intrinsic (e.g. x64). +** So do not try to make this is into a redefinable interface. +*/ +#if defined(InterlockedCompareExchange) + { "InterlockedCompareExchange", (SYSCALL)0, 0 }, + +#define osInterlockedCompareExchange InterlockedCompareExchange +#else + { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, + +#define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG \ + SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[76].pCurrent) +#endif /* defined(InterlockedCompareExchange) */ + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + { "UuidCreate", (SYSCALL)UuidCreate, 0 }, +#else + { "UuidCreate", (SYSCALL)0, 0 }, +#endif + +#define osUuidCreate ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[77].pCurrent) + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + { "UuidCreateSequential", (SYSCALL)UuidCreateSequential, 0 }, +#else + { "UuidCreateSequential", (SYSCALL)0, 0 }, +#endif + +#define osUuidCreateSequential \ + ((RPC_STATUS(RPC_ENTRY*)(UUID*))aSyscall[78].pCurrent) + +#if !defined(SQLITE_NO_SYNC) && SQLITE_MAX_MMAP_SIZE>0 + { "FlushViewOfFile", (SYSCALL)FlushViewOfFile, 0 }, +#else + { "FlushViewOfFile", (SYSCALL)0, 0 }, +#endif + +#define osFlushViewOfFile \ + ((BOOL(WINAPI*)(LPCVOID,SIZE_T))aSyscall[79].pCurrent) + +}; /* End of the overrideable system calls */ + +/* +** This is the xSetSystemCall() method of sqlite3_vfs for all of the +** "win32" VFSes. Return SQLITE_OK opon successfully updating the +** system call pointer, or SQLITE_NOTFOUND if there is no configurable +** system call named zName. +*/ +static int winSetSystemCall( + sqlite3_vfs *pNotUsed, /* The VFS pointer. Not used */ + const char *zName, /* Name of system call to override */ + sqlite3_syscall_ptr pNewFunc /* Pointer to new system call value */ +){ + unsigned int i; + int rc = SQLITE_NOTFOUND; + + UNUSED_PARAMETER(pNotUsed); + if( zName==0 ){ + /* If no zName is given, restore all system calls to their default + ** settings and return NULL + */ + rc = SQLITE_OK; + for(i=0; i0 ){ + memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); + memcpy(zDbgBuf, zBuf, nMin); + osOutputDebugStringA(zDbgBuf); + }else{ + osOutputDebugStringA(zBuf); + } +#elif defined(SQLITE_WIN32_HAS_WIDE) + memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); + if ( osMultiByteToWideChar( + osAreFileApisANSI() ? CP_ACP : CP_OEMCP, 0, zBuf, + nMin, (LPWSTR)zDbgBuf, SQLITE_WIN32_DBG_BUF_SIZE/sizeof(WCHAR))<=0 ){ + return; + } + osOutputDebugStringW((LPCWSTR)zDbgBuf); +#else + if( nMin>0 ){ + memset(zDbgBuf, 0, SQLITE_WIN32_DBG_BUF_SIZE); + memcpy(zDbgBuf, zBuf, nMin); + fprintf(stderr, "%s", zDbgBuf); + }else{ + fprintf(stderr, "%s", zBuf); + } +#endif +} + +/* +** The following routine suspends the current thread for at least ms +** milliseconds. This is equivalent to the Win32 Sleep() interface. +*/ +#if SQLITE_OS_WINRT +static HANDLE sleepObj = NULL; +#endif + +SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds){ +#if SQLITE_OS_WINRT + if ( sleepObj==NULL ){ + sleepObj = osCreateEventExW(NULL, NULL, CREATE_EVENT_MANUAL_RESET, + SYNCHRONIZE); + } + assert( sleepObj!=NULL ); + osWaitForSingleObjectEx(sleepObj, milliseconds, FALSE); +#else + osSleep(milliseconds); +#endif +} + +#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ + SQLITE_THREADSAFE>0 +SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject){ + DWORD rc; + while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + TRUE))==WAIT_IO_COMPLETION ){} + return rc; +} +#endif + +/* +** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, +** or WinCE. Return false (zero) for Win95, Win98, or WinME. +** +** Here is an interesting observation: Win95, Win98, and WinME lack +** the LockFileEx() API. But we can still statically link against that +** API as long as we don't call it when running Win95/98/ME. A call to +** this routine is used to determine if the host is Win95/98/ME or +** WinNT/2K/XP so that we will know whether or not we can safely call +** the LockFileEx() API. +*/ + +#if !SQLITE_WIN32_GETVERSIONEX +# define osIsNT() (1) +#elif SQLITE_OS_WINCE || SQLITE_OS_WINRT || !defined(SQLITE_WIN32_HAS_ANSI) +# define osIsNT() (1) +#elif !defined(SQLITE_WIN32_HAS_WIDE) +# define osIsNT() (0) +#else +# define osIsNT() ((sqlite3_os_type==2) || sqlite3_win32_is_nt()) +#endif + +/* +** This function determines if the machine is running a version of Windows +** based on the NT kernel. +*/ +SQLITE_API int sqlite3_win32_is_nt(void){ +#if SQLITE_OS_WINRT + /* + ** NOTE: The WinRT sub-platform is always assumed to be based on the NT + ** kernel. + */ + return 1; +#elif SQLITE_WIN32_GETVERSIONEX + if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ +#if defined(SQLITE_WIN32_HAS_ANSI) + OSVERSIONINFOA sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExA(&sInfo); + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); +#elif defined(SQLITE_WIN32_HAS_WIDE) + OSVERSIONINFOW sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExW(&sInfo); + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); +#endif + } + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; +#elif SQLITE_TEST + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; +#else + /* + ** NOTE: All sub-platforms where the GetVersionEx[AW] functions are + ** deprecated are always assumed to be based on the NT kernel. + */ + return 1; +#endif +} + +#ifdef SQLITE_WIN32_MALLOC +/* +** Allocate nBytes of memory. +*/ +static void *winMemMalloc(int nBytes){ + HANDLE hHeap; + void *p; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + assert( nBytes>=0 ); + p = osHeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); + if( !p ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapAlloc %u bytes (%lu), heap=%p", + nBytes, osGetLastError(), (void*)hHeap); + } + return p; +} + +/* +** Free memory. +*/ +static void winMemFree(void *pPrior){ + HANDLE hHeap; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); +#endif + if( !pPrior ) return; /* Passing NULL to HeapFree is undefined. */ + if( !osHeapFree(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapFree block %p (%lu), heap=%p", + pPrior, osGetLastError(), (void*)hHeap); + } +} + +/* +** Change the size of an existing memory allocation +*/ +static void *winMemRealloc(void *pPrior, int nBytes){ + HANDLE hHeap; + void *p; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior) ); +#endif + assert( nBytes>=0 ); + if( !pPrior ){ + p = osHeapAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, (SIZE_T)nBytes); + }else{ + p = osHeapReAlloc(hHeap, SQLITE_WIN32_HEAP_FLAGS, pPrior, (SIZE_T)nBytes); + } + if( !p ){ + sqlite3_log(SQLITE_NOMEM, "failed to %s %u bytes (%lu), heap=%p", + pPrior ? "HeapReAlloc" : "HeapAlloc", nBytes, osGetLastError(), + (void*)hHeap); + } + return p; +} + +/* +** Return the size of an outstanding allocation, in bytes. +*/ +static int winMemSize(void *p){ + HANDLE hHeap; + SIZE_T n; + + winMemAssertMagic(); + hHeap = winMemGetHeap(); + assert( hHeap!=0 ); + assert( hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(hHeap, SQLITE_WIN32_HEAP_FLAGS, p) ); +#endif + if( !p ) return 0; + n = osHeapSize(hHeap, SQLITE_WIN32_HEAP_FLAGS, p); + if( n==(SIZE_T)-1 ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapSize block %p (%lu), heap=%p", + p, osGetLastError(), (void*)hHeap); + return 0; + } + return (int)n; +} + +/* +** Round up a request size to the next valid allocation size. +*/ +static int winMemRoundup(int n){ + return n; +} + +/* +** Initialize this module. +*/ +static int winMemInit(void *pAppData){ + winMemData *pWinMemData = (winMemData *)pAppData; + + if( !pWinMemData ) return SQLITE_ERROR; + assert( pWinMemData->magic1==WINMEM_MAGIC1 ); + assert( pWinMemData->magic2==WINMEM_MAGIC2 ); + +#if !SQLITE_OS_WINRT && SQLITE_WIN32_HEAP_CREATE + if( !pWinMemData->hHeap ){ + DWORD dwInitialSize = SQLITE_WIN32_HEAP_INIT_SIZE; + DWORD dwMaximumSize = (DWORD)sqlite3GlobalConfig.nHeap; + if( dwMaximumSize==0 ){ + dwMaximumSize = SQLITE_WIN32_HEAP_MAX_SIZE; + }else if( dwInitialSize>dwMaximumSize ){ + dwInitialSize = dwMaximumSize; + } + pWinMemData->hHeap = osHeapCreate(SQLITE_WIN32_HEAP_FLAGS, + dwInitialSize, dwMaximumSize); + if( !pWinMemData->hHeap ){ + sqlite3_log(SQLITE_NOMEM, + "failed to HeapCreate (%lu), flags=%u, initSize=%lu, maxSize=%lu", + osGetLastError(), SQLITE_WIN32_HEAP_FLAGS, dwInitialSize, + dwMaximumSize); + return SQLITE_NOMEM_BKPT; + } + pWinMemData->bOwned = TRUE; + assert( pWinMemData->bOwned ); + } +#else + pWinMemData->hHeap = osGetProcessHeap(); + if( !pWinMemData->hHeap ){ + sqlite3_log(SQLITE_NOMEM, + "failed to GetProcessHeap (%lu)", osGetLastError()); + return SQLITE_NOMEM_BKPT; + } + pWinMemData->bOwned = FALSE; + assert( !pWinMemData->bOwned ); +#endif + assert( pWinMemData->hHeap!=0 ); + assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + return SQLITE_OK; +} + +/* +** Deinitialize this module. +*/ +static void winMemShutdown(void *pAppData){ + winMemData *pWinMemData = (winMemData *)pAppData; + + if( !pWinMemData ) return; + assert( pWinMemData->magic1==WINMEM_MAGIC1 ); + assert( pWinMemData->magic2==WINMEM_MAGIC2 ); + + if( pWinMemData->hHeap ){ + assert( pWinMemData->hHeap!=INVALID_HANDLE_VALUE ); +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_MALLOC_VALIDATE) + assert( osHeapValidate(pWinMemData->hHeap, SQLITE_WIN32_HEAP_FLAGS, NULL) ); +#endif + if( pWinMemData->bOwned ){ + if( !osHeapDestroy(pWinMemData->hHeap) ){ + sqlite3_log(SQLITE_NOMEM, "failed to HeapDestroy (%lu), heap=%p", + osGetLastError(), (void*)pWinMemData->hHeap); + } + pWinMemData->bOwned = FALSE; + } + pWinMemData->hHeap = NULL; + } +} + +/* +** Populate the low-level memory allocation function pointers in +** sqlite3GlobalConfig.m with pointers to the routines in this file. The +** arguments specify the block of memory to manage. +** +** This routine is only called by sqlite3_config(), and therefore +** is not required to be threadsafe (it is not). +*/ +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetWin32(void){ + static const sqlite3_mem_methods winMemMethods = { + winMemMalloc, + winMemFree, + winMemRealloc, + winMemSize, + winMemRoundup, + winMemInit, + winMemShutdown, + &win_mem_data + }; + return &winMemMethods; +} + +SQLITE_PRIVATE void sqlite3MemSetDefault(void){ + sqlite3_config(SQLITE_CONFIG_MALLOC, sqlite3MemGetWin32()); +} +#endif /* SQLITE_WIN32_MALLOC */ + +/* +** Convert a UTF-8 string to Microsoft Unicode. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static LPWSTR winUtf8ToUnicode(const char *zText){ + int nChar; + LPWSTR zWideText; + + nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideText = sqlite3MallocZero( nChar*sizeof(WCHAR) ); + if( zWideText==0 ){ + return 0; + } + nChar = osMultiByteToWideChar(CP_UTF8, 0, zText, -1, zWideText, + nChar); + if( nChar==0 ){ + sqlite3_free(zWideText); + zWideText = 0; + } + return zWideText; +} + +/* +** Convert a Microsoft Unicode string to UTF-8. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static char *winUnicodeToUtf8(LPCWSTR zWideText){ + int nByte; + char *zText; + + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, 0, 0, 0, 0); + if( nByte == 0 ){ + return 0; + } + zText = sqlite3MallocZero( nByte ); + if( zText==0 ){ + return 0; + } + nByte = osWideCharToMultiByte(CP_UTF8, 0, zWideText, -1, zText, nByte, + 0, 0); + if( nByte == 0 ){ + sqlite3_free(zText); + zText = 0; + } + return zText; +} + +/* +** Convert an ANSI string to Microsoft Unicode, using the ANSI or OEM +** code page. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static LPWSTR winMbcsToUnicode(const char *zText, int useAnsi){ + int nByte; + LPWSTR zMbcsText; + int codepage = useAnsi ? CP_ACP : CP_OEMCP; + + nByte = osMultiByteToWideChar(codepage, 0, zText, -1, NULL, + 0)*sizeof(WCHAR); + if( nByte==0 ){ + return 0; + } + zMbcsText = sqlite3MallocZero( nByte*sizeof(WCHAR) ); + if( zMbcsText==0 ){ + return 0; + } + nByte = osMultiByteToWideChar(codepage, 0, zText, -1, zMbcsText, + nByte); + if( nByte==0 ){ + sqlite3_free(zMbcsText); + zMbcsText = 0; + } + return zMbcsText; +} + +/* +** Convert a Microsoft Unicode string to a multi-byte character string, +** using the ANSI or OEM code page. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static char *winUnicodeToMbcs(LPCWSTR zWideText, int useAnsi){ + int nByte; + char *zText; + int codepage = useAnsi ? CP_ACP : CP_OEMCP; + + nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, 0, 0, 0, 0); + if( nByte == 0 ){ + return 0; + } + zText = sqlite3MallocZero( nByte ); + if( zText==0 ){ + return 0; + } + nByte = osWideCharToMultiByte(codepage, 0, zWideText, -1, zText, + nByte, 0, 0); + if( nByte == 0 ){ + sqlite3_free(zText); + zText = 0; + } + return zText; +} + +/* +** Convert a multi-byte character string to UTF-8. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static char *winMbcsToUtf8(const char *zText, int useAnsi){ + char *zTextUtf8; + LPWSTR zTmpWide; + + zTmpWide = winMbcsToUnicode(zText, useAnsi); + if( zTmpWide==0 ){ + return 0; + } + zTextUtf8 = winUnicodeToUtf8(zTmpWide); + sqlite3_free(zTmpWide); + return zTextUtf8; +} + +/* +** Convert a UTF-8 string to a multi-byte character string. +** +** Space to hold the returned string is obtained from sqlite3_malloc(). +*/ +static char *winUtf8ToMbcs(const char *zText, int useAnsi){ + char *zTextMbcs; + LPWSTR zTmpWide; + + zTmpWide = winUtf8ToUnicode(zText); + if( zTmpWide==0 ){ + return 0; + } + zTextMbcs = winUnicodeToMbcs(zTmpWide, useAnsi); + sqlite3_free(zTmpWide); + return zTextMbcs; +} + +/* +** This is a public wrapper for the winUtf8ToUnicode() function. +*/ +SQLITE_API LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToUnicode(zText); +} + +/* +** This is a public wrapper for the winUnicodeToUtf8() function. +*/ +SQLITE_API char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zWideText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUnicodeToUtf8(zWideText); +} + +/* +** This is a public wrapper for the winMbcsToUtf8() function. +*/ +SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winMbcsToUtf8(zText, osAreFileApisANSI()); +} + +/* +** This is a public wrapper for the winMbcsToUtf8() function. +*/ +SQLITE_API char *sqlite3_win32_mbcs_to_utf8_v2(const char *zText, int useAnsi){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winMbcsToUtf8(zText, useAnsi); +} + +/* +** This is a public wrapper for the winUtf8ToMbcs() function. +*/ +SQLITE_API char *sqlite3_win32_utf8_to_mbcs(const char *zText){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToMbcs(zText, osAreFileApisANSI()); +} + +/* +** This is a public wrapper for the winUtf8ToMbcs() function. +*/ +SQLITE_API char *sqlite3_win32_utf8_to_mbcs_v2(const char *zText, int useAnsi){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !zText ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return winUtf8ToMbcs(zText, useAnsi); +} + +/* +** This function is the same as sqlite3_win32_set_directory (below); however, +** it accepts a UTF-8 string. +*/ +SQLITE_API int sqlite3_win32_set_directory8( + unsigned long type, /* Identifier for directory being set or reset */ + const char *zValue /* New value for directory being set or reset */ +){ + char **ppDirectory = 0; +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + if( type==SQLITE_WIN32_DATA_DIRECTORY_TYPE ){ + ppDirectory = &sqlite3_data_directory; + }else if( type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE ){ + ppDirectory = &sqlite3_temp_directory; + } + assert( !ppDirectory || type==SQLITE_WIN32_DATA_DIRECTORY_TYPE + || type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE + ); + assert( !ppDirectory || sqlite3MemdebugHasType(*ppDirectory, MEMTYPE_HEAP) ); + if( ppDirectory ){ + char *zCopy = 0; + if( zValue && zValue[0] ){ + zCopy = sqlite3_mprintf("%s", zValue); + if ( zCopy==0 ){ + return SQLITE_NOMEM_BKPT; + } + } + sqlite3_free(*ppDirectory); + *ppDirectory = zCopy; + return SQLITE_OK; + } + return SQLITE_ERROR; +} + +/* +** This function is the same as sqlite3_win32_set_directory (below); however, +** it accepts a UTF-16 string. +*/ +SQLITE_API int sqlite3_win32_set_directory16( + unsigned long type, /* Identifier for directory being set or reset */ + const void *zValue /* New value for directory being set or reset */ +){ + int rc; + char *zUtf8 = 0; + if( zValue ){ + zUtf8 = sqlite3_win32_unicode_to_utf8(zValue); + if( zUtf8==0 ) return SQLITE_NOMEM_BKPT; + } + rc = sqlite3_win32_set_directory8(type, zUtf8); + if( zUtf8 ) sqlite3_free(zUtf8); + return rc; +} + +/* +** This function sets the data directory or the temporary directory based on +** the provided arguments. The type argument must be 1 in order to set the +** data directory or 2 in order to set the temporary directory. The zValue +** argument is the name of the directory to use. The return value will be +** SQLITE_OK if successful. +*/ +SQLITE_API int sqlite3_win32_set_directory( + unsigned long type, /* Identifier for directory being set or reset */ + void *zValue /* New value for directory being set or reset */ +){ + return sqlite3_win32_set_directory16(type, zValue); +} + +/* +** The return value of winGetLastErrorMsg +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). +*/ +static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ + /* FormatMessage returns 0 on failure. Otherwise it + ** returns the number of TCHARs written to the output + ** buffer, excluding the terminating null char. + */ + DWORD dwLen = 0; + char *zOut = 0; + + if( osIsNT() ){ +#if SQLITE_OS_WINRT + WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG_CHARS+1]; + dwLen = osFormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + zTempWide, + SQLITE_WIN32_MAX_ERRMSG_CHARS, + 0); +#else + LPWSTR zTempWide = NULL; + dwLen = osFormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + (LPWSTR) &zTempWide, + 0, + 0); +#endif + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + sqlite3BeginBenignMalloc(); + zOut = winUnicodeToUtf8(zTempWide); + sqlite3EndBenignMalloc(); +#if !SQLITE_OS_WINRT + /* free the system buffer allocated by FormatMessage */ + osLocalFree(zTempWide); +#endif + } + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + char *zTemp = NULL; + dwLen = osFormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + lastErrno, + 0, + (LPSTR) &zTemp, + 0, + 0); + if( dwLen > 0 ){ + /* allocate a buffer and convert to UTF8 */ + sqlite3BeginBenignMalloc(); + zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); + sqlite3EndBenignMalloc(); + /* free the system buffer allocated by FormatMessage */ + osLocalFree(zTemp); + } + } +#endif + if( 0 == dwLen ){ + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%lx (%lu)", lastErrno, lastErrno); + }else{ + /* copy a maximum of nBuf chars to output buffer */ + sqlite3_snprintf(nBuf, zBuf, "%s", zOut); + /* free the UTF8 buffer */ + sqlite3_free(zOut); + } + return 0; +} + +/* +** +** This function - winLogErrorAtLine() - is only ever called via the macro +** winLogError(). +** +** This routine is invoked after an error occurs in an OS function. +** It logs a message using sqlite3_log() containing the current value of +** error code and, if possible, the human-readable equivalent from +** FormatMessage. +** +** The first argument passed to the macro should be the error code that +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** The two subsequent arguments should be the name of the OS function that +** failed and the associated file-system path, if any. +*/ +#define winLogError(a,b,c,d) winLogErrorAtLine(a,b,c,d,__LINE__) +static int winLogErrorAtLine( + int errcode, /* SQLite error code */ + DWORD lastErrno, /* Win32 last error */ + const char *zFunc, /* Name of OS function that failed */ + const char *zPath, /* File path associated with error */ + int iLine /* Source line number where error occurred */ +){ + char zMsg[500]; /* Human readable error text */ + int i; /* Loop counter */ + + zMsg[0] = 0; + winGetLastErrorMsg(lastErrno, sizeof(zMsg), zMsg); + assert( errcode!=SQLITE_OK ); + if( zPath==0 ) zPath = ""; + for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} + zMsg[i] = 0; + sqlite3_log(errcode, + "os_win.c:%d: (%lu) %s(%s) - %s", + iLine, lastErrno, zFunc, zPath, zMsg + ); + + return errcode; +} + +/* +** The number of times that a ReadFile(), WriteFile(), and DeleteFile() +** will be retried following a locking error - probably caused by +** antivirus software. Also the initial delay before the first retry. +** The delay increases linearly with each retry. +*/ +#ifndef SQLITE_WIN32_IOERR_RETRY +# define SQLITE_WIN32_IOERR_RETRY 10 +#endif +#ifndef SQLITE_WIN32_IOERR_RETRY_DELAY +# define SQLITE_WIN32_IOERR_RETRY_DELAY 25 +#endif +static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; +static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; + +/* +** The "winIoerrCanRetry1" macro is used to determine if a particular I/O +** error code obtained via GetLastError() is eligible to be retried. It +** must accept the error code DWORD as its only argument and should return +** non-zero if the error code is transient in nature and the operation +** responsible for generating the original error might succeed upon being +** retried. The argument to this macro should be a variable. +** +** Additionally, a macro named "winIoerrCanRetry2" may be defined. If it +** is defined, it will be consulted only when the macro "winIoerrCanRetry1" +** returns zero. The "winIoerrCanRetry2" macro is completely optional and +** may be used to include additional error codes in the set that should +** result in the failing I/O operation being retried by the caller. If +** defined, the "winIoerrCanRetry2" macro must exhibit external semantics +** identical to those of the "winIoerrCanRetry1" macro. +*/ +#if !defined(winIoerrCanRetry1) +#define winIoerrCanRetry1(a) (((a)==ERROR_ACCESS_DENIED) || \ + ((a)==ERROR_SHARING_VIOLATION) || \ + ((a)==ERROR_LOCK_VIOLATION) || \ + ((a)==ERROR_DEV_NOT_EXIST) || \ + ((a)==ERROR_NETNAME_DELETED) || \ + ((a)==ERROR_SEM_TIMEOUT) || \ + ((a)==ERROR_NETWORK_UNREACHABLE)) +#endif + +/* +** If a ReadFile() or WriteFile() error occurs, invoke this routine +** to see if it should be retried. Return TRUE to retry. Return FALSE +** to give up with an error. +*/ +static int winRetryIoerr(int *pnRetry, DWORD *pError){ + DWORD e = osGetLastError(); + if( *pnRetry>=winIoerrRetry ){ + if( pError ){ + *pError = e; + } + return 0; + } + if( winIoerrCanRetry1(e) ){ + sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } +#if defined(winIoerrCanRetry2) + else if( winIoerrCanRetry2(e) ){ + sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); + ++*pnRetry; + return 1; + } +#endif + if( pError ){ + *pError = e; + } + return 0; +} + +/* +** Log a I/O error retry episode. +*/ +static void winLogIoerr(int nRetry, int lineno){ + if( nRetry ){ + sqlite3_log(SQLITE_NOTICE, + "delayed %dms for lock/sharing conflict at line %d", + winIoerrRetryDelay*nRetry*(nRetry+1)/2, lineno + ); + } +} + +/* +** This #if does not rely on the SQLITE_OS_WINCE define because the +** corresponding section in "date.c" cannot use it. +*/ +#if !defined(SQLITE_OMIT_LOCALTIME) && defined(_WIN32_WCE) && \ + (!defined(SQLITE_MSVC_LOCALTIME_API) || !SQLITE_MSVC_LOCALTIME_API) +/* +** The MSVC CRT on Windows CE may not have a localtime() function. +** So define a substitute. +*/ +/* # include */ +struct tm *__cdecl localtime(const time_t *t) +{ + static struct tm y; + FILETIME uTm, lTm; + SYSTEMTIME pTm; + sqlite3_int64 t64; + t64 = *t; + t64 = (t64 + 11644473600)*10000000; + uTm.dwLowDateTime = (DWORD)(t64 & 0xFFFFFFFF); + uTm.dwHighDateTime= (DWORD)(t64 >> 32); + osFileTimeToLocalFileTime(&uTm,&lTm); + osFileTimeToSystemTime(&lTm,&pTm); + y.tm_year = pTm.wYear - 1900; + y.tm_mon = pTm.wMonth - 1; + y.tm_wday = pTm.wDayOfWeek; + y.tm_mday = pTm.wDay; + y.tm_hour = pTm.wHour; + y.tm_min = pTm.wMinute; + y.tm_sec = pTm.wSecond; + return &y; +} +#endif + +#if SQLITE_OS_WINCE +/************************************************************************* +** This section contains code for WinCE only. +*/ +#define HANDLE_TO_WINFILE(a) (winFile*)&((char*)a)[-(int)offsetof(winFile,h)] + +/* +** Acquire a lock on the handle h +*/ +static void winceMutexAcquire(HANDLE h){ + DWORD dwErr; + do { + dwErr = osWaitForSingleObject(h, INFINITE); + } while (dwErr != WAIT_OBJECT_0 && dwErr != WAIT_ABANDONED); +} +/* +** Release a lock acquired by winceMutexAcquire() +*/ +#define winceMutexRelease(h) ReleaseMutex(h) + +/* +** Create the mutex and shared memory used for locking in the file +** descriptor pFile +*/ +static int winceCreateLock(const char *zFilename, winFile *pFile){ + LPWSTR zTok; + LPWSTR zName; + DWORD lastErrno; + BOOL bLogged = FALSE; + BOOL bInit = TRUE; + + zName = winUtf8ToUnicode(zFilename); + if( zName==0 ){ + /* out of memory */ + return SQLITE_IOERR_NOMEM_BKPT; + } + + /* Initialize the local lockdata */ + memset(&pFile->local, 0, sizeof(pFile->local)); + + /* Replace the backslashes from the filename and lowercase it + ** to derive a mutex name. */ + zTok = osCharLowerW(zName); + for (;*zTok;zTok++){ + if (*zTok == '\\') *zTok = '_'; + } + + /* Create/open the named mutex */ + pFile->hMutex = osCreateMutexW(NULL, FALSE, zName); + if (!pFile->hMutex){ + pFile->lastErrno = osGetLastError(); + sqlite3_free(zName); + return winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock1", zFilename); + } + + /* Acquire the mutex before continuing */ + winceMutexAcquire(pFile->hMutex); + + /* Since the names of named mutexes, semaphores, file mappings etc are + ** case-sensitive, take advantage of that by uppercasing the mutex name + ** and using that as the shared filemapping name. + */ + osCharUpperW(zName); + pFile->hShared = osCreateFileMappingW(INVALID_HANDLE_VALUE, NULL, + PAGE_READWRITE, 0, sizeof(winceLock), + zName); + + /* Set a flag that indicates we're the first to create the memory so it + ** must be zero-initialized */ + lastErrno = osGetLastError(); + if (lastErrno == ERROR_ALREADY_EXISTS){ + bInit = FALSE; + } + + sqlite3_free(zName); + + /* If we succeeded in making the shared memory handle, map it. */ + if( pFile->hShared ){ + pFile->shared = (winceLock*)osMapViewOfFile(pFile->hShared, + FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, sizeof(winceLock)); + /* If mapping failed, close the shared memory handle and erase it */ + if( !pFile->shared ){ + pFile->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock2", zFilename); + bLogged = TRUE; + osCloseHandle(pFile->hShared); + pFile->hShared = NULL; + } + } + + /* If shared memory could not be created, then close the mutex and fail */ + if( pFile->hShared==NULL ){ + if( !bLogged ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR, pFile->lastErrno, + "winceCreateLock3", zFilename); + bLogged = TRUE; + } + winceMutexRelease(pFile->hMutex); + osCloseHandle(pFile->hMutex); + pFile->hMutex = NULL; + return SQLITE_IOERR; + } + + /* Initialize the shared memory if we're supposed to */ + if( bInit ){ + memset(pFile->shared, 0, sizeof(winceLock)); + } + + winceMutexRelease(pFile->hMutex); + return SQLITE_OK; +} + +/* +** Destroy the part of winFile that deals with wince locks +*/ +static void winceDestroyLock(winFile *pFile){ + if (pFile->hMutex){ + /* Acquire the mutex */ + winceMutexAcquire(pFile->hMutex); + + /* The following blocks should probably assert in debug mode, but they + are to cleanup in case any locks remained open */ + if (pFile->local.nReaders){ + pFile->shared->nReaders --; + } + if (pFile->local.bReserved){ + pFile->shared->bReserved = FALSE; + } + if (pFile->local.bPending){ + pFile->shared->bPending = FALSE; + } + if (pFile->local.bExclusive){ + pFile->shared->bExclusive = FALSE; + } + + /* De-reference and close our copy of the shared memory handle */ + osUnmapViewOfFile(pFile->shared); + osCloseHandle(pFile->hShared); + + /* Done with the mutex */ + winceMutexRelease(pFile->hMutex); + osCloseHandle(pFile->hMutex); + pFile->hMutex = NULL; + } +} + +/* +** An implementation of the LockFile() API of Windows for CE +*/ +static BOOL winceLockFile( + LPHANDLE phFile, + DWORD dwFileOffsetLow, + DWORD dwFileOffsetHigh, + DWORD nNumberOfBytesToLockLow, + DWORD nNumberOfBytesToLockHigh +){ + winFile *pFile = HANDLE_TO_WINFILE(phFile); + BOOL bReturn = FALSE; + + UNUSED_PARAMETER(dwFileOffsetHigh); + UNUSED_PARAMETER(nNumberOfBytesToLockHigh); + + if (!pFile->hMutex) return TRUE; + winceMutexAcquire(pFile->hMutex); + + /* Wanting an exclusive lock? */ + if (dwFileOffsetLow == (DWORD)SHARED_FIRST + && nNumberOfBytesToLockLow == (DWORD)SHARED_SIZE){ + if (pFile->shared->nReaders == 0 && pFile->shared->bExclusive == 0){ + pFile->shared->bExclusive = TRUE; + pFile->local.bExclusive = TRUE; + bReturn = TRUE; + } + } + + /* Want a read-only lock? */ + else if (dwFileOffsetLow == (DWORD)SHARED_FIRST && + nNumberOfBytesToLockLow == 1){ + if (pFile->shared->bExclusive == 0){ + pFile->local.nReaders ++; + if (pFile->local.nReaders == 1){ + pFile->shared->nReaders ++; + } + bReturn = TRUE; + } + } + + /* Want a pending lock? */ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToLockLow == 1){ + /* If no pending lock has been acquired, then acquire it */ + if (pFile->shared->bPending == 0) { + pFile->shared->bPending = TRUE; + pFile->local.bPending = TRUE; + bReturn = TRUE; + } + } + + /* Want a reserved lock? */ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToLockLow == 1){ + if (pFile->shared->bReserved == 0) { + pFile->shared->bReserved = TRUE; + pFile->local.bReserved = TRUE; + bReturn = TRUE; + } + } + + winceMutexRelease(pFile->hMutex); + return bReturn; +} + +/* +** An implementation of the UnlockFile API of Windows for CE +*/ +static BOOL winceUnlockFile( + LPHANDLE phFile, + DWORD dwFileOffsetLow, + DWORD dwFileOffsetHigh, + DWORD nNumberOfBytesToUnlockLow, + DWORD nNumberOfBytesToUnlockHigh +){ + winFile *pFile = HANDLE_TO_WINFILE(phFile); + BOOL bReturn = FALSE; + + UNUSED_PARAMETER(dwFileOffsetHigh); + UNUSED_PARAMETER(nNumberOfBytesToUnlockHigh); + + if (!pFile->hMutex) return TRUE; + winceMutexAcquire(pFile->hMutex); + + /* Releasing a reader lock or an exclusive lock */ + if (dwFileOffsetLow == (DWORD)SHARED_FIRST){ + /* Did we have an exclusive lock? */ + if (pFile->local.bExclusive){ + assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE); + pFile->local.bExclusive = FALSE; + pFile->shared->bExclusive = FALSE; + bReturn = TRUE; + } + + /* Did we just have a reader lock? */ + else if (pFile->local.nReaders){ + assert(nNumberOfBytesToUnlockLow == (DWORD)SHARED_SIZE + || nNumberOfBytesToUnlockLow == 1); + pFile->local.nReaders --; + if (pFile->local.nReaders == 0) + { + pFile->shared->nReaders --; + } + bReturn = TRUE; + } + } + + /* Releasing a pending lock */ + else if (dwFileOffsetLow == (DWORD)PENDING_BYTE + && nNumberOfBytesToUnlockLow == 1){ + if (pFile->local.bPending){ + pFile->local.bPending = FALSE; + pFile->shared->bPending = FALSE; + bReturn = TRUE; + } + } + /* Releasing a reserved lock */ + else if (dwFileOffsetLow == (DWORD)RESERVED_BYTE + && nNumberOfBytesToUnlockLow == 1){ + if (pFile->local.bReserved) { + pFile->local.bReserved = FALSE; + pFile->shared->bReserved = FALSE; + bReturn = TRUE; + } + } + + winceMutexRelease(pFile->hMutex); + return bReturn; +} +/* +** End of the special code for wince +*****************************************************************************/ +#endif /* SQLITE_OS_WINCE */ + +/* +** Lock a file region. +*/ +static BOOL winLockFile( + LPHANDLE phFile, + DWORD flags, + DWORD offsetLow, + DWORD offsetHigh, + DWORD numBytesLow, + DWORD numBytesHigh +){ +#if SQLITE_OS_WINCE + /* + ** NOTE: Windows CE is handled differently here due its lack of the Win32 + ** API LockFile. + */ + return winceLockFile(phFile, offsetLow, offsetHigh, + numBytesLow, numBytesHigh); +#else + if( osIsNT() ){ + OVERLAPPED ovlp; + memset(&ovlp, 0, sizeof(OVERLAPPED)); + ovlp.Offset = offsetLow; + ovlp.OffsetHigh = offsetHigh; + return osLockFileEx(*phFile, flags, 0, numBytesLow, numBytesHigh, &ovlp); + }else{ + return osLockFile(*phFile, offsetLow, offsetHigh, numBytesLow, + numBytesHigh); + } +#endif +} + +/* +** Unlock a file region. + */ +static BOOL winUnlockFile( + LPHANDLE phFile, + DWORD offsetLow, + DWORD offsetHigh, + DWORD numBytesLow, + DWORD numBytesHigh +){ +#if SQLITE_OS_WINCE + /* + ** NOTE: Windows CE is handled differently here due its lack of the Win32 + ** API UnlockFile. + */ + return winceUnlockFile(phFile, offsetLow, offsetHigh, + numBytesLow, numBytesHigh); +#else + if( osIsNT() ){ + OVERLAPPED ovlp; + memset(&ovlp, 0, sizeof(OVERLAPPED)); + ovlp.Offset = offsetLow; + ovlp.OffsetHigh = offsetHigh; + return osUnlockFileEx(*phFile, 0, numBytesLow, numBytesHigh, &ovlp); + }else{ + return osUnlockFile(*phFile, offsetLow, offsetHigh, numBytesLow, + numBytesHigh); + } +#endif +} + +/***************************************************************************** +** The next group of routines implement the I/O methods specified +** by the sqlite3_io_methods object. +******************************************************************************/ + +/* +** Some Microsoft compilers lack this definition. +*/ +#ifndef INVALID_SET_FILE_POINTER +# define INVALID_SET_FILE_POINTER ((DWORD)-1) +#endif + +/* +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. +** Otherwise, set pFile->lastErrno and return non-zero. +*/ +static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ +#if !SQLITE_OS_WINRT + LONG upperBits; /* Most sig. 32 bits of new offset */ + LONG lowerBits; /* Least sig. 32 bits of new offset */ + DWORD dwRet; /* Value returned by SetFilePointer() */ + DWORD lastErrno; /* Value returned by GetLastError() */ + + OSTRACE(("SEEK file=%p, offset=%lld\n", pFile->h, iOffset)); + + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); + lowerBits = (LONG)(iOffset & 0xffffffff); + + /* API oddity: If successful, SetFilePointer() returns a dword + ** containing the lower 32-bits of the new file-offset. Or, if it fails, + ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, + ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine + ** whether an error has actually occurred, it is also necessary to call + ** GetLastError(). + */ + dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); + + if( (dwRet==INVALID_SET_FILE_POINTER + && ((lastErrno = osGetLastError())!=NO_ERROR)) ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, + "winSeekFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); + return 1; + } + + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); + return 0; +#else + /* + ** Same as above, except that this implementation works for WinRT. + */ + + LARGE_INTEGER x; /* The new offset */ + BOOL bRet; /* Value returned by SetFilePointerEx() */ + + x.QuadPart = iOffset; + bRet = osSetFilePointerEx(pFile->h, x, 0, FILE_BEGIN); + + if(!bRet){ + pFile->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, + "winSeekFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); + return 1; + } + + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); + return 0; +#endif +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* Forward references to VFS helper methods used for memory mapped files */ +static int winMapfile(winFile*, sqlite3_int64); +static int winUnmapfile(winFile*); +#endif + +/* +** Close a file. +** +** It is reported that an attempt to close a handle might sometimes +** fail. This is a very unreasonable result, but Windows is notorious +** for being unreasonable so I do not doubt that it might happen. If +** the close fails, we pause for 100 milliseconds and try again. As +** many as MX_CLOSE_ATTEMPT attempts to close the handle are made before +** giving up and returning an error. +*/ +#define MX_CLOSE_ATTEMPT 3 +static int winClose(sqlite3_file *id){ + int rc, cnt = 0; + winFile *pFile = (winFile*)id; + + assert( id!=0 ); +#ifndef SQLITE_OMIT_WAL + assert( pFile->pShm==0 ); +#endif + assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + OSTRACE(("CLOSE pid=%lu, pFile=%p, file=%p\n", + osGetCurrentProcessId(), pFile, pFile->h)); + +#if SQLITE_MAX_MMAP_SIZE>0 + winUnmapfile(pFile); +#endif + + do{ + rc = osCloseHandle(pFile->h); + /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ + }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (sqlite3_win32_sleep(100), 1) ); +#if SQLITE_OS_WINCE +#define WINCE_DELETION_ATTEMPTS 3 + { + winVfsAppData *pAppData = (winVfsAppData*)pFile->pVfs->pAppData; + if( pAppData==NULL || !pAppData->bNoLock ){ + winceDestroyLock(pFile); + } + } + if( pFile->zDeleteOnClose ){ + int cnt = 0; + while( + osDeleteFileW(pFile->zDeleteOnClose)==0 + && osGetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff + && cnt++ < WINCE_DELETION_ATTEMPTS + ){ + sqlite3_win32_sleep(100); /* Wait a little before trying again */ + } + sqlite3_free(pFile->zDeleteOnClose); + } +#endif + if( rc ){ + pFile->h = NULL; + } + OpenCounter(-1); + OSTRACE(("CLOSE pid=%lu, pFile=%p, file=%p, rc=%s\n", + osGetCurrentProcessId(), pFile, pFile->h, rc ? "ok" : "failed")); + return rc ? SQLITE_OK + : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), + "winClose", pFile->zPath); +} + +/* +** Read data from a file into a buffer. Return SQLITE_OK if all +** bytes were read successfully and SQLITE_IOERR if anything goes +** wrong. +*/ +static int winRead( + sqlite3_file *id, /* File to read from */ + void *pBuf, /* Write content into this buffer */ + int amt, /* Number of bytes to read */ + sqlite3_int64 offset /* Begin reading at this offset */ +){ +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) + OVERLAPPED overlapped; /* The offset for ReadFile. */ +#endif + winFile *pFile = (winFile*)id; /* file handle */ + DWORD nRead; /* Number of bytes actually read from file */ + int nRetry = 0; /* Number of retrys */ + + assert( id!=0 ); + assert( amt>0 ); + assert( offset>=0 ); + SimulateIOError(return SQLITE_IOERR_READ); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, buffer=%p, amount=%d, " + "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile, + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + OSTRACE(("READ-MMAP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) + if( winSeekFile(pFile, offset) ){ + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_FULL\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_FULL; + } + while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ +#else + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); + while( !osReadFile(pFile->h, pBuf, amt, &nRead, &overlapped) && + osGetLastError()!=ERROR_HANDLE_EOF ){ +#endif + DWORD lastErrno; + if( winRetryIoerr(&nRetry, &lastErrno) ) continue; + pFile->lastErrno = lastErrno; + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_READ\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, + "winRead", pFile->zPath); + } + winLogIoerr(nRetry, __LINE__); + if( nRead<(DWORD)amt ){ + /* Unread parts of the buffer must be zero-filled */ + memset(&((char*)pBuf)[nRead], 0, amt-nRead); + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_SHORT_READ\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_IOERR_SHORT_READ; + } + + OSTRACE(("READ pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; +} + +/* +** Write data from a buffer into a file. Return SQLITE_OK on success +** or some other error code on failure. +*/ +static int winWrite( + sqlite3_file *id, /* File to write into */ + const void *pBuf, /* The bytes to be written */ + int amt, /* Number of bytes to write */ + sqlite3_int64 offset /* Offset into the file to begin writing at */ +){ + int rc = 0; /* True if error has occurred, else false */ + winFile *pFile = (winFile*)id; /* File handle */ + int nRetry = 0; /* Number of retries */ + + assert( amt>0 ); + assert( pFile ); + SimulateIOError(return SQLITE_IOERR_WRITE); + SimulateDiskfullError(return SQLITE_FULL); + + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, buffer=%p, amount=%d, " + "offset=%lld, lock=%d\n", osGetCurrentProcessId(), pFile, + pFile->h, pBuf, amt, offset, pFile->locktype)); + +#if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + OSTRACE(("WRITE-MMAP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } +#endif + +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) + rc = winSeekFile(pFile, offset); + if( rc==0 ){ +#else + { +#endif +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) + OVERLAPPED overlapped; /* The offset for WriteFile. */ +#endif + u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ + int nRem = amt; /* Number of bytes yet to be written */ + DWORD nWrite; /* Bytes written by each WriteFile() call */ + DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ + +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) + memset(&overlapped, 0, sizeof(OVERLAPPED)); + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif + + while( nRem>0 ){ +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ +#else + if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ +#endif + if( winRetryIoerr(&nRetry, &lastErrno) ) continue; + break; + } + assert( nWrite==0 || nWrite<=(DWORD)nRem ); + if( nWrite==0 || nWrite>(DWORD)nRem ){ + lastErrno = osGetLastError(); + break; + } +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) + offset += nWrite; + overlapped.Offset = (LONG)(offset & 0xffffffff); + overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); +#endif + aRem += nWrite; + nRem -= nWrite; + } + if( nRem>0 ){ + pFile->lastErrno = lastErrno; + rc = 1; + } + } + + if( rc ){ + if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) + || ( pFile->lastErrno==ERROR_DISK_FULL )){ + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_FULL\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return winLogError(SQLITE_FULL, pFile->lastErrno, + "winWrite1", pFile->zPath); + } + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_WRITE\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, + "winWrite2", pFile->zPath); + }else{ + winLogIoerr(nRetry, __LINE__); + } + OSTRACE(("WRITE pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; +} + +/* +** Truncate an open file to a specified size +*/ +static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ + winFile *pFile = (winFile*)id; /* File handle object */ + int rc = SQLITE_OK; /* Return code for this function */ + DWORD lastErrno; +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_int64 oldMmapSize; + if( pFile->nFetchOut>0 ){ + /* File truncation is a no-op if there are outstanding memory mapped + ** pages. This is because truncating the file means temporarily unmapping + ** the file, and that might delete memory out from under existing cursors. + ** + ** This can result in incremental vacuum not truncating the file, + ** if there is an active read cursor when the incremental vacuum occurs. + ** No real harm comes of this - the database file is not corrupted, + ** though some folks might complain that the file is bigger than it + ** needs to be. + ** + ** The only feasible work-around is to defer the truncation until after + ** all references to memory-mapped content are closed. That is doable, + ** but involves adding a few branches in the common write code path which + ** could slow down normal operations slightly. Hence, we have decided for + ** now to simply make trancations a no-op if there are pending reads. We + ** can maybe revisit this decision in the future. + */ + return SQLITE_OK; + } +#endif + + assert( pFile ); + SimulateIOError(return SQLITE_IOERR_TRUNCATE); + OSTRACE(("TRUNCATE pid=%lu, pFile=%p, file=%p, size=%lld, lock=%d\n", + osGetCurrentProcessId(), pFile, pFile->h, nByte, pFile->locktype)); + + /* If the user has configured a chunk-size for this file, truncate the + ** file so that it consists of an integer number of chunks (i.e. the + ** actual file size after the operation may be larger than the requested + ** size). + */ + if( pFile->szChunk>0 ){ + nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; + } + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->pMapRegion ){ + oldMmapSize = pFile->mmapSize; + }else{ + oldMmapSize = 0; + } + winUnmapfile(pFile); +#endif + + /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ + if( winSeekFile(pFile, nByte) ){ + rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) && + ((lastErrno = osGetLastError())!=ERROR_USER_MAPPED_FILE) ){ + pFile->lastErrno = lastErrno; + rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, + "winTruncate2", pFile->zPath); + } + +#if SQLITE_MAX_MMAP_SIZE>0 + if( rc==SQLITE_OK && oldMmapSize>0 ){ + if( oldMmapSize>nByte ){ + winMapfile(pFile, -1); + }else{ + winMapfile(pFile, oldMmapSize); + } + } +#endif + + OSTRACE(("TRUNCATE pid=%lu, pFile=%p, file=%p, rc=%s\n", + osGetCurrentProcessId(), pFile, pFile->h, sqlite3ErrName(rc))); + return rc; +} + +#ifdef SQLITE_TEST +/* +** Count the number of fullsyncs and normal syncs. This is used to test +** that syncs and fullsyncs are occuring at the right times. +*/ +SQLITE_API int sqlite3_sync_count = 0; +SQLITE_API int sqlite3_fullsync_count = 0; +#endif + +/* +** Make sure all writes to a particular file are committed to disk. +*/ +static int winSync(sqlite3_file *id, int flags){ +#ifndef SQLITE_NO_SYNC + /* + ** Used only when SQLITE_NO_SYNC is not defined. + */ + BOOL rc; +#endif +#if !defined(NDEBUG) || !defined(SQLITE_NO_SYNC) || \ + defined(SQLITE_HAVE_OS_TRACE) + /* + ** Used when SQLITE_NO_SYNC is not defined and by the assert() and/or + ** OSTRACE() macros. + */ + winFile *pFile = (winFile*)id; +#else + UNUSED_PARAMETER(id); +#endif + + assert( pFile ); + /* Check that one of SQLITE_SYNC_NORMAL or FULL was passed */ + assert((flags&0x0F)==SQLITE_SYNC_NORMAL + || (flags&0x0F)==SQLITE_SYNC_FULL + ); + + /* Unix cannot, but some systems may return SQLITE_FULL from here. This + ** line is to test that doing so does not cause any problems. + */ + SimulateDiskfullError( return SQLITE_FULL ); + + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, flags=%x, lock=%d\n", + osGetCurrentProcessId(), pFile, pFile->h, flags, + pFile->locktype)); + +#ifndef SQLITE_TEST + UNUSED_PARAMETER(flags); +#else + if( (flags&0x0F)==SQLITE_SYNC_FULL ){ + sqlite3_fullsync_count++; + } + sqlite3_sync_count++; +#endif + + /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a + ** no-op + */ +#ifdef SQLITE_NO_SYNC + OSTRACE(("SYNC-NOP pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; +#else +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->pMapRegion ){ + if( osFlushViewOfFile(pFile->pMapRegion, 0) ){ + OSTRACE(("SYNC-MMAP pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_OK\n", osGetCurrentProcessId(), + pFile, pFile->pMapRegion)); + }else{ + pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC-MMAP pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), + pFile, pFile->pMapRegion)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winSync1", pFile->zPath); + } + } +#endif + rc = osFlushFileBuffers(pFile->h); + SimulateIOError( rc=FALSE ); + if( rc ){ + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return SQLITE_OK; + }else{ + pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC pid=%lu, pFile=%p, file=%p, rc=SQLITE_IOERR_FSYNC\n", + osGetCurrentProcessId(), pFile, pFile->h)); + return winLogError(SQLITE_IOERR_FSYNC, pFile->lastErrno, + "winSync2", pFile->zPath); + } +#endif +} + +/* +** Determine the current size of a file in bytes +*/ +static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ + winFile *pFile = (winFile*)id; + int rc = SQLITE_OK; + + assert( id!=0 ); + assert( pSize!=0 ); + SimulateIOError(return SQLITE_IOERR_FSTAT); + OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); + +#if SQLITE_OS_WINRT + { + FILE_STANDARD_INFO info; + if( osGetFileInformationByHandleEx(pFile->h, FileStandardInfo, + &info, sizeof(info)) ){ + *pSize = info.EndOfFile.QuadPart; + }else{ + pFile->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_FSTAT, pFile->lastErrno, + "winFileSize", pFile->zPath); + } + } +#else + { + DWORD upperBits; + DWORD lowerBits; + DWORD lastErrno; + + lowerBits = osGetFileSize(pFile->h, &upperBits); + *pSize = (((sqlite3_int64)upperBits)<<32) + lowerBits; + if( (lowerBits == INVALID_FILE_SIZE) + && ((lastErrno = osGetLastError())!=NO_ERROR) ){ + pFile->lastErrno = lastErrno; + rc = winLogError(SQLITE_IOERR_FSTAT, pFile->lastErrno, + "winFileSize", pFile->zPath); + } + } +#endif + OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", + pFile->h, pSize, *pSize, sqlite3ErrName(rc))); + return rc; +} + +/* +** LOCKFILE_FAIL_IMMEDIATELY is undefined on some Windows systems. +*/ +#ifndef LOCKFILE_FAIL_IMMEDIATELY +# define LOCKFILE_FAIL_IMMEDIATELY 1 +#endif + +#ifndef LOCKFILE_EXCLUSIVE_LOCK +# define LOCKFILE_EXCLUSIVE_LOCK 2 +#endif + +/* +** Historically, SQLite has used both the LockFile and LockFileEx functions. +** When the LockFile function was used, it was always expected to fail +** immediately if the lock could not be obtained. Also, it always expected to +** obtain an exclusive lock. These flags are used with the LockFileEx function +** and reflect those expectations; therefore, they should not be changed. +*/ +#ifndef SQLITE_LOCKFILE_FLAGS +# define SQLITE_LOCKFILE_FLAGS (LOCKFILE_FAIL_IMMEDIATELY | \ + LOCKFILE_EXCLUSIVE_LOCK) +#endif + +/* +** Currently, SQLite never calls the LockFileEx function without wanting the +** call to fail immediately if the lock cannot be obtained. +*/ +#ifndef SQLITE_LOCKFILEEX_FLAGS +# define SQLITE_LOCKFILEEX_FLAGS (LOCKFILE_FAIL_IMMEDIATELY) +#endif + +/* +** Acquire a reader lock. +** Different API routines are called depending on whether or not this +** is Win9x or WinNT. +*/ +static int winGetReadLock(winFile *pFile){ + int res; + OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); + if( osIsNT() ){ +#if SQLITE_OS_WINCE + /* + ** NOTE: Windows CE is handled differently here due its lack of the Win32 + ** API LockFileEx. + */ + res = winceLockFile(&pFile->h, SHARED_FIRST, 0, 1, 0); +#else + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS, SHARED_FIRST, 0, + SHARED_SIZE, 0); +#endif + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + int lk; + sqlite3_randomness(sizeof(lk), &lk); + pFile->sharedLockByte = (short)((lk & 0x7fffffff)%(SHARED_SIZE - 1)); + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, + SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); + } +#endif + if( res == 0 ){ + pFile->lastErrno = osGetLastError(); + /* No need to log a failure to lock */ + } + OSTRACE(("READ-LOCK file=%p, result=%d\n", pFile->h, res)); + return res; +} + +/* +** Undo a readlock +*/ +static int winUnlockReadLock(winFile *pFile){ + int res; + DWORD lastErrno; + OSTRACE(("READ-UNLOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); + if( osIsNT() ){ + res = winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + res = winUnlockFile(&pFile->h, SHARED_FIRST+pFile->sharedLockByte, 0, 1, 0); + } +#endif + if( res==0 && ((lastErrno = osGetLastError())!=ERROR_NOT_LOCKED) ){ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_IOERR_UNLOCK, pFile->lastErrno, + "winUnlockReadLock", pFile->zPath); + } + OSTRACE(("READ-UNLOCK file=%p, result=%d\n", pFile->h, res)); + return res; +} + +/* +** Lock the file with the lock specified by parameter locktype - one +** of the following: +** +** (1) SHARED_LOCK +** (2) RESERVED_LOCK +** (3) PENDING_LOCK +** (4) EXCLUSIVE_LOCK +** +** Sometimes when requesting one lock state, additional lock states +** are inserted in between. The locking might fail on one of the later +** transitions leaving the lock state different from what it started but +** still short of its goal. The following chart shows the allowed +** transitions and the inserted intermediate states: +** +** UNLOCKED -> SHARED +** SHARED -> RESERVED +** SHARED -> (PENDING) -> EXCLUSIVE +** RESERVED -> (PENDING) -> EXCLUSIVE +** PENDING -> EXCLUSIVE +** +** This routine will only increase a lock. The winUnlock() routine +** erases all locks at once and returns us immediately to locking level 0. +** It is not possible to lower the locking level one step at a time. You +** must go straight to locking level 0. +*/ +static int winLock(sqlite3_file *id, int locktype){ + int rc = SQLITE_OK; /* Return code from subroutines */ + int res = 1; /* Result of a Windows lock call */ + int newLocktype; /* Set pFile->locktype to this value before exiting */ + int gotPendingLock = 0;/* True if we acquired a PENDING lock this time */ + winFile *pFile = (winFile*)id; + DWORD lastErrno = NO_ERROR; + + assert( id!=0 ); + OSTRACE(("LOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); + + /* If there is already a lock of this type or more restrictive on the + ** OsFile, do nothing. Don't use the end_lock: exit path, as + ** sqlite3OsEnterMutex() hasn't been called yet. + */ + if( pFile->locktype>=locktype ){ + OSTRACE(("LOCK-HELD file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + + /* Do not allow any kind of write-lock on a read-only database + */ + if( (pFile->ctrlFlags & WINFILE_RDONLY)!=0 && locktype>=RESERVED_LOCK ){ + return SQLITE_IOERR_LOCK; + } + + /* Make sure the locking sequence is correct + */ + assert( pFile->locktype!=NO_LOCK || locktype==SHARED_LOCK ); + assert( locktype!=PENDING_LOCK ); + assert( locktype!=RESERVED_LOCK || pFile->locktype==SHARED_LOCK ); + + /* Lock the PENDING_LOCK byte if we need to acquire a PENDING lock or + ** a SHARED lock. If we are acquiring a SHARED lock, the acquisition of + ** the PENDING_LOCK byte is temporary. + */ + newLocktype = pFile->locktype; + if( pFile->locktype==NO_LOCK + || (locktype==EXCLUSIVE_LOCK && pFile->locktype<=RESERVED_LOCK) + ){ + int cnt = 3; + while( cnt-->0 && (res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, + PENDING_BYTE, 0, 1, 0))==0 ){ + /* Try 3 times to get the pending lock. This is needed to work + ** around problems caused by indexing and/or anti-virus software on + ** Windows systems. + ** If you are using this code as a model for alternative VFSes, do not + ** copy this retry logic. It is a hack intended for Windows only. + */ + lastErrno = osGetLastError(); + OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, result=%d\n", + pFile->h, cnt, res)); + if( lastErrno==ERROR_INVALID_HANDLE ){ + pFile->lastErrno = lastErrno; + rc = SQLITE_IOERR_LOCK; + OSTRACE(("LOCK-FAIL file=%p, count=%d, rc=%s\n", + pFile->h, cnt, sqlite3ErrName(rc))); + return rc; + } + if( cnt ) sqlite3_win32_sleep(1); + } + gotPendingLock = res; + if( !res ){ + lastErrno = osGetLastError(); + } + } + + /* Acquire a shared lock + */ + if( locktype==SHARED_LOCK && res ){ + assert( pFile->locktype==NO_LOCK ); + res = winGetReadLock(pFile); + if( res ){ + newLocktype = SHARED_LOCK; + }else{ + lastErrno = osGetLastError(); + } + } + + /* Acquire a RESERVED lock + */ + if( locktype==RESERVED_LOCK && res ){ + assert( pFile->locktype==SHARED_LOCK ); + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, RESERVED_BYTE, 0, 1, 0); + if( res ){ + newLocktype = RESERVED_LOCK; + }else{ + lastErrno = osGetLastError(); + } + } + + /* Acquire a PENDING lock + */ + if( locktype==EXCLUSIVE_LOCK && res ){ + newLocktype = PENDING_LOCK; + gotPendingLock = 0; + } + + /* Acquire an EXCLUSIVE lock + */ + if( locktype==EXCLUSIVE_LOCK && res ){ + assert( pFile->locktype>=SHARED_LOCK ); + res = winUnlockReadLock(pFile); + res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, + SHARED_SIZE, 0); + if( res ){ + newLocktype = EXCLUSIVE_LOCK; + }else{ + lastErrno = osGetLastError(); + winGetReadLock(pFile); + } + } + + /* If we are holding a PENDING lock that ought to be released, then + ** release it now. + */ + if( gotPendingLock && locktype==SHARED_LOCK ){ + winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); + } + + /* Update the state of the lock has held in the file descriptor then + ** return the appropriate result code. + */ + if( res ){ + rc = SQLITE_OK; + }else{ + pFile->lastErrno = lastErrno; + rc = SQLITE_BUSY; + OSTRACE(("LOCK-FAIL file=%p, wanted=%d, got=%d\n", + pFile->h, locktype, newLocktype)); + } + pFile->locktype = (u8)newLocktype; + OSTRACE(("LOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); + return rc; +} + +/* +** This routine checks if there is a RESERVED lock held on the specified +** file by this or any other process. If such a lock is held, return +** non-zero, otherwise zero. +*/ +static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ + int res; + winFile *pFile = (winFile*)id; + + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p\n", pFile->h, pResOut)); + + assert( id!=0 ); + if( pFile->locktype>=RESERVED_LOCK ){ + res = 1; + OSTRACE(("TEST-WR-LOCK file=%p, result=%d (local)\n", pFile->h, res)); + }else{ + res = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE,0,1,0); + if( res ){ + winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); + } + res = !res; + OSTRACE(("TEST-WR-LOCK file=%p, result=%d (remote)\n", pFile->h, res)); + } + *pResOut = res; + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + pFile->h, pResOut, *pResOut)); + return SQLITE_OK; +} + +/* +** Lower the locking level on file descriptor id to locktype. locktype +** must be either NO_LOCK or SHARED_LOCK. +** +** If the locking level of the file descriptor is already at or below +** the requested locking level, this routine is a no-op. +** +** It is not possible for this routine to fail if the second argument +** is NO_LOCK. If the second argument is SHARED_LOCK then this routine +** might return SQLITE_IOERR; +*/ +static int winUnlock(sqlite3_file *id, int locktype){ + int type; + winFile *pFile = (winFile*)id; + int rc = SQLITE_OK; + assert( pFile!=0 ); + assert( locktype<=SHARED_LOCK ); + OSTRACE(("UNLOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); + type = pFile->locktype; + if( type>=EXCLUSIVE_LOCK ){ + winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); + if( locktype==SHARED_LOCK && !winGetReadLock(pFile) ){ + /* This should never happen. We should always be able to + ** reacquire the read lock */ + rc = winLogError(SQLITE_IOERR_UNLOCK, osGetLastError(), + "winUnlock", pFile->zPath); + } + } + if( type>=RESERVED_LOCK ){ + winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); + } + if( locktype==NO_LOCK && type>=SHARED_LOCK ){ + winUnlockReadLock(pFile); + } + if( type>=PENDING_LOCK ){ + winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); + } + pFile->locktype = (u8)locktype; + OSTRACE(("UNLOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); + return rc; +} + +/****************************************************************************** +****************************** No-op Locking ********************************** +** +** Of the various locking implementations available, this is by far the +** simplest: locking is ignored. No attempt is made to lock the database +** file for reading or writing. +** +** This locking mode is appropriate for use on read-only databases +** (ex: databases that are burned into CD-ROM, for example.) It can +** also be used if the application employs some external mechanism to +** prevent simultaneous access of the same database by two or more +** database connections. But there is a serious risk of database +** corruption if this locking mode is used in situations where multiple +** database connections are accessing the same database file at the same +** time and one or more of those connections are writing. +*/ + +static int winNolockLock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +static int winNolockCheckReservedLock(sqlite3_file *id, int *pResOut){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(pResOut); + return SQLITE_OK; +} + +static int winNolockUnlock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +/******************* End of the no-op lock implementation ********************* +******************************************************************************/ + +/* +** If *pArg is initially negative then this is a query. Set *pArg to +** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. +** +** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. +*/ +static void winModeBit(winFile *pFile, unsigned char mask, int *pArg){ + if( *pArg<0 ){ + *pArg = (pFile->ctrlFlags & mask)!=0; + }else if( (*pArg)==0 ){ + pFile->ctrlFlags &= ~mask; + }else{ + pFile->ctrlFlags |= mask; + } +} + +/* Forward references to VFS helper methods used for temporary files */ +static int winGetTempname(sqlite3_vfs *, char **); +static int winIsDir(const void *); +static BOOL winIsLongPathPrefix(const char *); +static BOOL winIsDriveLetterAndColon(const char *); + +/* +** Control and query of the open file handle. +*/ +static int winFileControl(sqlite3_file *id, int op, void *pArg){ + winFile *pFile = (winFile*)id; + OSTRACE(("FCNTL file=%p, op=%d, pArg=%p\n", pFile->h, op, pArg)); + switch( op ){ + case SQLITE_FCNTL_LOCKSTATE: { + *(int*)pArg = pFile->locktype; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_LAST_ERRNO: { + *(int*)pArg = (int)pFile->lastErrno; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_CHUNK_SIZE: { + pFile->szChunk = *(int *)pArg; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_SIZE_HINT: { + if( pFile->szChunk>0 ){ + sqlite3_int64 oldSz; + int rc = winFileSize(id, &oldSz); + if( rc==SQLITE_OK ){ + sqlite3_int64 newSz = *(sqlite3_int64*)pArg; + if( newSz>oldSz ){ + SimulateIOErrorBenign(1); + rc = winTruncate(id, newSz); + SimulateIOErrorBenign(0); + } + } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + return rc; + } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_PERSIST_WAL: { + winModeBit(pFile, WINFILE_PERSIST_WAL, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { + winModeBit(pFile, WINFILE_PSOW, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_VFSNAME: { + *(char**)pArg = sqlite3_mprintf("%s", pFile->pVfs->zName); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_WIN32_AV_RETRY: { + int *a = (int*)pArg; + if( a[0]>0 ){ + winIoerrRetry = a[0]; + }else{ + a[0] = winIoerrRetry; + } + if( a[1]>0 ){ + winIoerrRetryDelay = a[1]; + }else{ + a[1] = winIoerrRetryDelay; + } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } + case SQLITE_FCNTL_WIN32_GET_HANDLE: { + LPHANDLE phFile = (LPHANDLE)pArg; + *phFile = pFile->h; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } +#ifdef SQLITE_TEST + case SQLITE_FCNTL_WIN32_SET_HANDLE: { + LPHANDLE phFile = (LPHANDLE)pArg; + HANDLE hOldFile = pFile->h; + pFile->h = *phFile; + *phFile = hOldFile; + OSTRACE(("FCNTL oldFile=%p, newFile=%p, rc=SQLITE_OK\n", + hOldFile, pFile->h)); + return SQLITE_OK; + } +#endif + case SQLITE_FCNTL_TEMPFILENAME: { + char *zTFile = 0; + int rc = winGetTempname(pFile->pVfs, &zTFile); + if( rc==SQLITE_OK ){ + *(char**)pArg = zTFile; + } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + return rc; + } +#if SQLITE_MAX_MMAP_SIZE>0 + case SQLITE_FCNTL_MMAP_SIZE: { + i64 newLimit = *(i64*)pArg; + int rc = SQLITE_OK; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + + /* The value of newLimit may be eventually cast to (SIZE_T) and passed + ** to MapViewOfFile(). Restrict its value to 2GB if (SIZE_T) is not at + ** least a 64-bit type. */ + if( newLimit>0 && sizeof(SIZE_T)<8 ){ + newLimit = (newLimit & 0x7FFFFFFF); + } + + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ + pFile->mmapSizeMax = newLimit; + if( pFile->mmapSize>0 ){ + winUnmapfile(pFile); + rc = winMapfile(pFile, -1); + } + } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); + return rc; + } +#endif + } + OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); + return SQLITE_NOTFOUND; +} + +/* +** Return the sector size in bytes of the underlying block device for +** the specified file. This is almost always 512 bytes, but may be +** larger for some devices. +** +** SQLite code assumes this function cannot fail. It also assumes that +** if two files are created in the same file-system directory (i.e. +** a database and its journal file) that the sector size will be the +** same for both. +*/ +static int winSectorSize(sqlite3_file *id){ + (void)id; + return SQLITE_DEFAULT_SECTOR_SIZE; +} + +/* +** Return a vector of device characteristics. +*/ +static int winDeviceCharacteristics(sqlite3_file *id){ + winFile *p = (winFile*)id; + return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | + ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); +} + +/* +** Windows will only let you create file view mappings +** on allocation size granularity boundaries. +** During sqlite3_os_init() we do a GetSystemInfo() +** to get the granularity size. +*/ +static SYSTEM_INFO winSysInfo; + +#ifndef SQLITE_OMIT_WAL + +/* +** Helper functions to obtain and relinquish the global mutex. The +** global mutex is used to protect the winLockInfo objects used by +** this file, all of which may be shared by multiple threads. +** +** Function winShmMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() +** statements. e.g. +** +** winShmEnterMutex() +** assert( winShmMutexHeld() ); +** winShmLeaveMutex() +*/ +static sqlite3_mutex *winBigLock = 0; +static void winShmEnterMutex(void){ + sqlite3_mutex_enter(winBigLock); +} +static void winShmLeaveMutex(void){ + sqlite3_mutex_leave(winBigLock); +} +#ifndef NDEBUG +static int winShmMutexHeld(void) { + return sqlite3_mutex_held(winBigLock); +} +#endif + +/* +** Object used to represent a single file opened and mmapped to provide +** shared memory. When multiple threads all reference the same +** log-summary, each thread has its own winFile object, but they all +** point to a single instance of this object. In other words, each +** log-summary is opened only once per process. +** +** winShmMutexHeld() must be true when creating or destroying +** this object or while reading or writing the following fields: +** +** nRef +** pNext +** +** The following fields are read-only after the object is created: +** +** fid +** zFilename +** +** Either winShmNode.mutex must be held or winShmNode.nRef==0 and +** winShmMutexHeld() is true when reading or writing any other field +** in this structure. +** +*/ +struct winShmNode { + sqlite3_mutex *mutex; /* Mutex to access this object */ + char *zFilename; /* Name of the file */ + winFile hFile; /* File handle from winOpen */ + + int szRegion; /* Size of shared-memory regions */ + int nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ + + struct ShmRegion { + HANDLE hMap; /* File handle from CreateFileMapping */ + void *pMap; + } *aRegion; + DWORD lastErrno; /* The Windows errno from the last I/O error */ + + int nRef; /* Number of winShm objects pointing to this */ + winShm *pFirst; /* All winShm objects pointing to this */ + winShmNode *pNext; /* Next in list of all winShmNode objects */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) + u8 nextShmId; /* Next available winShm.id value */ +#endif +}; + +/* +** A global array of all winShmNode objects. +** +** The winShmMutexHeld() must be true while reading or writing this list. +*/ +static winShmNode *winShmNodeList = 0; + +/* +** Structure used internally by this VFS to record the state of an +** open shared memory connection. +** +** The following fields are initialized when this object is created and +** are read-only thereafter: +** +** winShm.pShmNode +** winShm.id +** +** All other fields are read/write. The winShm.pShmNode->mutex must be held +** while accessing any read/write fields. +*/ +struct winShm { + winShmNode *pShmNode; /* The underlying winShmNode object */ + winShm *pNext; /* Next winShm with the same winShmNode */ + u8 hasMutex; /* True if holding the winShmNode mutex */ + u16 sharedMask; /* Mask of shared locks held */ + u16 exclMask; /* Mask of exclusive locks held */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) + u8 id; /* Id of this connection with its winShmNode */ +#endif +}; + +/* +** Constants used for locking +*/ +#define WIN_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ +#define WIN_SHM_DMS (WIN_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ + +/* +** Apply advisory locks for all n bytes beginning at ofst. +*/ +#define WINSHM_UNLCK 1 +#define WINSHM_RDLCK 2 +#define WINSHM_WRLCK 3 +static int winShmSystemLock( + winShmNode *pFile, /* Apply locks to this open shared-memory segment */ + int lockType, /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ + int ofst, /* Offset to first byte to be locked/unlocked */ + int nByte /* Number of bytes to lock or unlock */ +){ + int rc = 0; /* Result code form Lock/UnlockFileEx() */ + + /* Access to the winShmNode object is serialized by the caller */ + assert( pFile->nRef==0 || sqlite3_mutex_held(pFile->mutex) ); + + OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", + pFile->hFile.h, lockType, ofst, nByte)); + + /* Release/Acquire the system-level lock */ + if( lockType==WINSHM_UNLCK ){ + rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); + }else{ + /* Initialize the locking parameters */ + DWORD dwFlags = LOCKFILE_FAIL_IMMEDIATELY; + if( lockType == WINSHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; + rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); + } + + if( rc!= 0 ){ + rc = SQLITE_OK; + }else{ + pFile->lastErrno = osGetLastError(); + rc = SQLITE_BUSY; + } + + OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", + pFile->hFile.h, (lockType == WINSHM_UNLCK) ? "winUnlockFile" : + "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); + + return rc; +} + +/* Forward references to VFS methods */ +static int winOpen(sqlite3_vfs*,const char*,sqlite3_file*,int,int*); +static int winDelete(sqlite3_vfs *,const char*,int); + +/* +** Purge the winShmNodeList list of all entries with winShmNode.nRef==0. +** +** This is not a VFS shared-memory method; it is a utility function called +** by VFS shared-memory methods. +*/ +static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ + winShmNode **pp; + winShmNode *p; + assert( winShmMutexHeld() ); + OSTRACE(("SHM-PURGE pid=%lu, deleteFlag=%d\n", + osGetCurrentProcessId(), deleteFlag)); + pp = &winShmNodeList; + while( (p = *pp)!=0 ){ + if( p->nRef==0 ){ + int i; + if( p->mutex ){ sqlite3_mutex_free(p->mutex); } + for(i=0; inRegion; i++){ + BOOL bRc = osUnmapViewOfFile(p->aRegion[i].pMap); + OSTRACE(("SHM-PURGE-UNMAP pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); + UNUSED_VARIABLE_VALUE(bRc); + bRc = osCloseHandle(p->aRegion[i].hMap); + OSTRACE(("SHM-PURGE-CLOSE pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); + UNUSED_VARIABLE_VALUE(bRc); + } + if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){ + SimulateIOErrorBenign(1); + winClose((sqlite3_file *)&p->hFile); + SimulateIOErrorBenign(0); + } + if( deleteFlag ){ + SimulateIOErrorBenign(1); + sqlite3BeginBenignMalloc(); + winDelete(pVfs, p->zFilename, 0); + sqlite3EndBenignMalloc(); + SimulateIOErrorBenign(0); + } + *pp = p->pNext; + sqlite3_free(p->aRegion); + sqlite3_free(p); + }else{ + pp = &p->pNext; + } + } +} + +/* +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +*/ +static int winLockSharedMemory(winShmNode *pShmNode){ + int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); + + if( rc==SQLITE_OK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return SQLITE_READONLY_CANTINIT; + }else if( winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), + "winLockSharedMemory", pShmNode->zFilename); + } + } + + if( rc==SQLITE_OK ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + } + + return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); +} + +/* +** Open the shared-memory area associated with database file pDbFd. +** +** When opening a new shared-memory file, if no other instances of that +** file are currently open, in this process or in other processes, then +** the file must be truncated to zero length or have its header cleared. +*/ +static int winOpenSharedMemory(winFile *pDbFd){ + struct winShm *p; /* The connection to be opened */ + winShmNode *pShmNode = 0; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ + winShmNode *pNew; /* Newly allocated winShmNode */ + int nName; /* Size of zName in bytes */ + + assert( pDbFd->pShm==0 ); /* Not previously opened */ + + /* Allocate space for the new sqlite3_shm object. Also speculatively + ** allocate space for a new winShmNode and filename. + */ + p = sqlite3MallocZero( sizeof(*p) ); + if( p==0 ) return SQLITE_IOERR_NOMEM_BKPT; + nName = sqlite3Strlen30(pDbFd->zPath); + pNew = sqlite3MallocZero( sizeof(*pShmNode) + nName + 17 ); + if( pNew==0 ){ + sqlite3_free(p); + return SQLITE_IOERR_NOMEM_BKPT; + } + pNew->zFilename = (char*)&pNew[1]; + sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); + sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); + + /* Look to see if there is an existing winShmNode that can be used. + ** If no matching winShmNode currently exists, create a new one. + */ + winShmEnterMutex(); + for(pShmNode = winShmNodeList; pShmNode; pShmNode=pShmNode->pNext){ + /* TBD need to come up with better match here. Perhaps + ** use FILE_ID_BOTH_DIR_INFO Structure. + */ + if( sqlite3StrICmp(pShmNode->zFilename, pNew->zFilename)==0 ) break; + } + if( pShmNode ){ + sqlite3_free(pNew); + }else{ + int inFlags = SQLITE_OPEN_WAL; + int outFlags = 0; + + pShmNode = pNew; + pNew = 0; + ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE; + pShmNode->pNext = winShmNodeList; + winShmNodeList = pShmNode; + + if( sqlite3GlobalConfig.bCoreMutex ){ + pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->mutex==0 ){ + rc = SQLITE_IOERR_NOMEM_BKPT; + goto shm_open_err; + } + } + + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + inFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + }else{ + inFlags |= SQLITE_OPEN_READONLY; + } + rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, + (sqlite3_file*)&pShmNode->hFile, + inFlags, &outFlags); + if( rc!=SQLITE_OK ){ + rc = winLogError(rc, osGetLastError(), "winOpenShm", + pShmNode->zFilename); + goto shm_open_err; + } + if( outFlags==SQLITE_OPEN_READONLY ) pShmNode->isReadonly = 1; + + rc = winLockSharedMemory(pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; + } + + /* Make the new connection a child of the winShmNode */ + p->pShmNode = pShmNode; +#if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) + p->id = pShmNode->nextShmId++; +#endif + pShmNode->nRef++; + pDbFd->pShm = p; + winShmLeaveMutex(); + + /* The reference count on pShmNode has already been incremented under + ** the cover of the winShmEnterMutex() mutex and the pointer from the + ** new (struct winShm) object to the pShmNode has been set. All that is + ** left to do is to link the new object into the linked list starting + ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex + ** mutex. + */ + sqlite3_mutex_enter(pShmNode->mutex); + p->pNext = pShmNode->pFirst; + pShmNode->pFirst = p; + sqlite3_mutex_leave(pShmNode->mutex); + return rc; + + /* Jump here on any error */ +shm_open_err: + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ + sqlite3_free(p); + sqlite3_free(pNew); + winShmLeaveMutex(); + return rc; +} + +/* +** Close a connection to shared-memory. Delete the underlying +** storage if deleteFlag is true. +*/ +static int winShmUnmap( + sqlite3_file *fd, /* Database holding shared memory */ + int deleteFlag /* Delete after closing if true */ +){ + winFile *pDbFd; /* Database holding shared-memory */ + winShm *p; /* The connection to be closed */ + winShmNode *pShmNode; /* The underlying shared-memory file */ + winShm **pp; /* For looping over sibling connections */ + + pDbFd = (winFile*)fd; + p = pDbFd->pShm; + if( p==0 ) return SQLITE_OK; + pShmNode = p->pShmNode; + + /* Remove connection p from the set of connections associated + ** with pShmNode */ + sqlite3_mutex_enter(pShmNode->mutex); + for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} + *pp = p->pNext; + + /* Free the connection p */ + sqlite3_free(p); + pDbFd->pShm = 0; + sqlite3_mutex_leave(pShmNode->mutex); + + /* If pShmNode->nRef has reached 0, then close the underlying + ** shared-memory file, too */ + winShmEnterMutex(); + assert( pShmNode->nRef>0 ); + pShmNode->nRef--; + if( pShmNode->nRef==0 ){ + winShmPurge(pDbFd->pVfs, deleteFlag); + } + winShmLeaveMutex(); + + return SQLITE_OK; +} + +/* +** Change the lock state for a shared-memory segment. +*/ +static int winShmLock( + sqlite3_file *fd, /* Database file holding the shared memory */ + int ofst, /* First lock to acquire or release */ + int n, /* Number of locks to acquire or release */ + int flags /* What to do with the lock */ +){ + winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ + winShm *p = pDbFd->pShm; /* The shared memory being locked */ + winShm *pX; /* For looping over all siblings */ + winShmNode *pShmNode = p->pShmNode; + int rc = SQLITE_OK; /* Result code */ + u16 mask; /* Mask of locks to take or release */ + + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); + assert( n>=1 ); + assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) + || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); + assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); + + mask = (u16)((1U<<(ofst+n)) - (1U<1 || mask==(1<mutex); + if( flags & SQLITE_SHM_UNLOCK ){ + u16 allMask = 0; /* Mask of locks held by siblings */ + + /* See if any siblings hold this same lock */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( pX==p ) continue; + assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); + allMask |= pX->sharedMask; + } + + /* Unlock the system-level locks */ + if( (mask & allMask)==0 ){ + rc = winShmSystemLock(pShmNode, WINSHM_UNLCK, ofst+WIN_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~mask; + p->sharedMask &= ~mask; + } + }else if( flags & SQLITE_SHM_SHARED ){ + u16 allShared = 0; /* Union of locks held by connections other than "p" */ + + /* Find out which shared locks are already held by sibling connections. + ** If any sibling already holds an exclusive lock, go ahead and return + ** SQLITE_BUSY. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + allShared |= pX->sharedMask; + } + + /* Get shared locks at the system level, if necessary */ + if( rc==SQLITE_OK ){ + if( (allShared & mask)==0 ){ + rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, ofst+WIN_SHM_BASE, n); + }else{ + rc = SQLITE_OK; + } + } + + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; + } + }else{ + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. + */ + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ + rc = SQLITE_BUSY; + break; + } + } + + /* Get the exclusive locks at the system level. Then if successful + ** also mark the local connection as being locked. + */ + if( rc==SQLITE_OK ){ + rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, ofst+WIN_SHM_BASE, n); + if( rc==SQLITE_OK ){ + assert( (p->sharedMask & mask)==0 ); + p->exclMask |= mask; + } + } + } + sqlite3_mutex_leave(pShmNode->mutex); + OSTRACE(("SHM-LOCK pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", + osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + sqlite3ErrName(rc))); + return rc; +} + +/* +** Implement a memory barrier or memory fence on shared memory. +** +** All loads and stores begun before the barrier must complete before +** any load or store begun after the barrier. +*/ +static void winShmBarrier( + sqlite3_file *fd /* Database holding the shared memory */ +){ + UNUSED_PARAMETER(fd); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + winShmEnterMutex(); /* Also mutex, for redundancy */ + winShmLeaveMutex(); +} + +/* +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. +** +** If an error occurs, an error code is returned and *pp is set to NULL. +** +** Otherwise, if the isWrite parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** isWrite is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. +*/ +static int winShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int isWrite, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + winFile *pDbFd = (winFile*)fd; + winShm *pShm = pDbFd->pShm; + winShmNode *pShmNode; + DWORD protect = PAGE_READWRITE; + DWORD flags = FILE_MAP_WRITE | FILE_MAP_READ; + int rc = SQLITE_OK; + + if( !pShm ){ + rc = winOpenSharedMemory(pDbFd); + if( rc!=SQLITE_OK ) return rc; + pShm = pDbFd->pShm; + assert( pShm!=0 ); + } + pShmNode = pShm->pShmNode; + + sqlite3_mutex_enter(pShmNode->mutex); + if( pShmNode->isUnlocked ){ + rc = winLockSharedMemory(pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); + + if( pShmNode->nRegion<=iRegion ){ + struct ShmRegion *apNew; /* New aRegion[] array */ + int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ + sqlite3_int64 sz; /* Current size of wal-index file */ + + pShmNode->szRegion = szRegion; + + /* The requested region is not mapped into this processes address space. + ** Check to see if it has been allocated (i.e. if the wal-index file is + ** large enough to contain the requested region). + */ + rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); + if( rc!=SQLITE_OK ){ + rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), + "winShmMap1", pDbFd->zPath); + goto shmpage_out; + } + + if( szhFile, nByte); + if( rc!=SQLITE_OK ){ + rc = winLogError(SQLITE_IOERR_SHMSIZE, osGetLastError(), + "winShmMap2", pDbFd->zPath); + goto shmpage_out; + } + } + + /* Map the requested memory region into this processes address space. */ + apNew = (struct ShmRegion *)sqlite3_realloc64( + pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) + ); + if( !apNew ){ + rc = SQLITE_IOERR_NOMEM_BKPT; + goto shmpage_out; + } + pShmNode->aRegion = apNew; + + if( pShmNode->isReadonly ){ + protect = PAGE_READONLY; + flags = FILE_MAP_READ; + } + + while( pShmNode->nRegion<=iRegion ){ + HANDLE hMap = NULL; /* file-mapping handle */ + void *pMap = 0; /* Mapped memory region */ + +#if SQLITE_OS_WINRT + hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, + NULL, protect, nByte, NULL + ); +#elif defined(SQLITE_WIN32_HAS_WIDE) + hMap = osCreateFileMappingW(pShmNode->hFile.h, + NULL, protect, 0, nByte, NULL + ); +#elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA + hMap = osCreateFileMappingA(pShmNode->hFile.h, + NULL, protect, 0, nByte, NULL + ); +#endif + OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, nByte, + hMap ? "ok" : "failed")); + if( hMap ){ + int iOffset = pShmNode->nRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; +#if SQLITE_OS_WINRT + pMap = osMapViewOfFileFromApp(hMap, flags, + iOffset - iOffsetShift, szRegion + iOffsetShift + ); +#else + pMap = osMapViewOfFile(hMap, flags, + 0, iOffset - iOffsetShift, szRegion + iOffsetShift + ); +#endif + OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, iOffset, + szRegion, pMap ? "ok" : "failed")); + } + if( !pMap ){ + pShmNode->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_SHMMAP, pShmNode->lastErrno, + "winShmMap3", pDbFd->zPath); + if( hMap ) osCloseHandle(hMap); + goto shmpage_out; + } + + pShmNode->aRegion[pShmNode->nRegion].pMap = pMap; + pShmNode->aRegion[pShmNode->nRegion].hMap = hMap; + pShmNode->nRegion++; + } + } + +shmpage_out: + if( pShmNode->nRegion>iRegion ){ + int iOffset = iRegion*szRegion; + int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; + char *p = (char *)pShmNode->aRegion[iRegion].pMap; + *pp = (void *)&p[iOffsetShift]; + }else{ + *pp = 0; + } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; + sqlite3_mutex_leave(pShmNode->mutex); + return rc; +} + +#else +# define winShmMap 0 +# define winShmLock 0 +# define winShmBarrier 0 +# define winShmUnmap 0 +#endif /* #ifndef SQLITE_OMIT_WAL */ + +/* +** Cleans up the mapped region of the specified file, if any. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +static int winUnmapfile(winFile *pFile){ + assert( pFile!=0 ); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, pMapRegion=%p, " + "mmapSize=%lld, mmapSizeMax=%lld\n", + osGetCurrentProcessId(), pFile, pFile->hMap, pFile->pMapRegion, + pFile->mmapSize, pFile->mmapSizeMax)); + if( pFile->pMapRegion ){ + if( !osUnmapViewOfFile(pFile->pMapRegion) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), pFile, + pFile->pMapRegion)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmapfile1", pFile->zPath); + } + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + } + if( pFile->hMap!=NULL ){ + if( !osCloseHandle(pFile->hMap) ){ + pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFile, pFile->hMap)); + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, + "winUnmapfile2", pFile->zPath); + } + pFile->hMap = NULL; + } + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile)); + return SQLITE_OK; +} + +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ +static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ + sqlite3_int64 nMap = nByte; + int rc; + + assert( nMap>=0 || pFd->nFetchOut==0 ); + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, size=%lld\n", + osGetCurrentProcessId(), pFd, nByte)); + + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ + rc = winFileSize((sqlite3_file*)pFd, &nMap); + if( rc ){ + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_IOERR_FSTAT\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_IOERR_FSTAT; + } + } + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; + } + nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); + + if( nMap==0 && pFd->mmapSize>0 ){ + winUnmapfile(pFd); + } + if( nMap!=pFd->mmapSize ){ + void *pNew = 0; + DWORD protect = PAGE_READONLY; + DWORD flags = FILE_MAP_READ; + + winUnmapfile(pFd); +#ifdef SQLITE_MMAP_READWRITE + if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ + protect = PAGE_READWRITE; + flags |= FILE_MAP_WRITE; + } +#endif +#if SQLITE_OS_WINRT + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) + pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA + pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); +#endif + if( pFd->hMap==NULL ){ + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile1", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + OSTRACE(("MAP-FILE-CREATE pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return SQLITE_OK; + } + assert( (nMap % winSysInfo.dwPageSize)==0 ); + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, (SIZE_T)nMap); +#else + pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); +#endif + if( pNew==NULL ){ + osCloseHandle(pFd->hMap); + pFd->hMap = NULL; + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile2", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + OSTRACE(("MAP-FILE-MAP pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return SQLITE_OK; + } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + } + + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFd)); + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling winUnfetch(). +*/ +static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ +#endif + *pp = 0; + + OSTRACE(("FETCH pid=%lu, pFile=%p, offset=%lld, amount=%d, pp=%p\n", + osGetCurrentProcessId(), fd, iOff, nAmt, pp)); + +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = winMapfile(pFd, -1); + if( rc!=SQLITE_OK ){ + OSTRACE(("FETCH pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return rc; + } + } + if( pFd->mmapSize >= iOff+nAmt ){ + assert( pFd->pMapRegion!=0 ); + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } +#endif + + OSTRACE(("FETCH pid=%lu, pFile=%p, pp=%p, *pp=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd, pp, *pp)); + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to winFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the winFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 + winFile *pFd = (winFile*)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, offset=%lld, p=%p\n", + osGetCurrentProcessId(), pFd, iOff, p)); + + if( p ){ + pFd->nFetchOut--; + }else{ + /* FIXME: If Windows truly always prevents truncating or deleting a + ** file while a mapping is held, then the following winUnmapfile() call + ** is unnecessary can be omitted - potentially improving + ** performance. */ + winUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); +#endif + + OSTRACE(("UNFETCH pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd)); + return SQLITE_OK; +} + +/* +** Here ends the implementation of all sqlite3_file methods. +** +********************** End sqlite3_file Methods ******************************* +******************************************************************************/ + +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for win32. +*/ +static const sqlite3_io_methods winIoMethod = { + 3, /* iVersion */ + winClose, /* xClose */ + winRead, /* xRead */ + winWrite, /* xWrite */ + winTruncate, /* xTruncate */ + winSync, /* xSync */ + winFileSize, /* xFileSize */ + winLock, /* xLock */ + winUnlock, /* xUnlock */ + winCheckReservedLock, /* xCheckReservedLock */ + winFileControl, /* xFileControl */ + winSectorSize, /* xSectorSize */ + winDeviceCharacteristics, /* xDeviceCharacteristics */ + winShmMap, /* xShmMap */ + winShmLock, /* xShmLock */ + winShmBarrier, /* xShmBarrier */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ +}; + +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for win32 without performing any locking. +*/ +static const sqlite3_io_methods winIoNolockMethod = { + 3, /* iVersion */ + winClose, /* xClose */ + winRead, /* xRead */ + winWrite, /* xWrite */ + winTruncate, /* xTruncate */ + winSync, /* xSync */ + winFileSize, /* xFileSize */ + winNolockLock, /* xLock */ + winNolockUnlock, /* xUnlock */ + winNolockCheckReservedLock, /* xCheckReservedLock */ + winFileControl, /* xFileControl */ + winSectorSize, /* xSectorSize */ + winDeviceCharacteristics, /* xDeviceCharacteristics */ + winShmMap, /* xShmMap */ + winShmLock, /* xShmLock */ + winShmBarrier, /* xShmBarrier */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ +}; + +static winVfsAppData winAppData = { + &winIoMethod, /* pMethod */ + 0, /* pAppData */ + 0 /* bNoLock */ +}; + +static winVfsAppData winNolockAppData = { + &winIoNolockMethod, /* pMethod */ + 0, /* pAppData */ + 1 /* bNoLock */ +}; + +/**************************************************************************** +**************************** sqlite3_vfs methods **************************** +** +** This division contains the implementation of methods on the +** sqlite3_vfs object. +*/ + +#if defined(__CYGWIN__) +/* +** Convert a filename from whatever the underlying operating system +** supports for filenames into UTF-8. Space to hold the result is +** obtained from malloc and must be freed by the calling function. +*/ +static char *winConvertToUtf8Filename(const void *zFilename){ + char *zConverted = 0; + if( osIsNT() ){ + zConverted = winUnicodeToUtf8(zFilename); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + zConverted = winMbcsToUtf8(zFilename, osAreFileApisANSI()); + } +#endif + /* caller will handle out of memory */ + return zConverted; +} +#endif + +/* +** Convert a UTF-8 filename into whatever form the underlying +** operating system wants filenames in. Space to hold the result +** is obtained from malloc and must be freed by the calling +** function. +*/ +static void *winConvertFromUtf8Filename(const char *zFilename){ + void *zConverted = 0; + if( osIsNT() ){ + zConverted = winUtf8ToUnicode(zFilename); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + zConverted = winUtf8ToMbcs(zFilename, osAreFileApisANSI()); + } +#endif + /* caller will handle out of memory */ + return zConverted; +} + +/* +** This function returns non-zero if the specified UTF-8 string buffer +** ends with a directory separator character or one was successfully +** added to it. +*/ +static int winMakeEndInDirSep(int nBuf, char *zBuf){ + if( zBuf ){ + int nLen = sqlite3Strlen30(zBuf); + if( nLen>0 ){ + if( winIsDirSep(zBuf[nLen-1]) ){ + return 1; + }else if( nLen+1mxPathname; nBuf = nMax + 2; + zBuf = sqlite3MallocZero( nBuf ); + if( !zBuf ){ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); + return SQLITE_IOERR_NOMEM_BKPT; + } + + /* Figure out the effective temporary directory. First, check if one + ** has been explicitly set by the application; otherwise, use the one + ** configured by the operating system. + */ + nDir = nMax - (nPre + 15); + assert( nDir>0 ); + if( sqlite3_temp_directory ){ + int nDirLen = sqlite3Strlen30(sqlite3_temp_directory); + if( nDirLen>0 ){ + if( !winIsDirSep(sqlite3_temp_directory[nDirLen-1]) ){ + nDirLen++; + } + if( nDirLen>nDir ){ + sqlite3_free(zBuf); + OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); + return winLogError(SQLITE_ERROR, 0, "winGetTempname1", 0); + } + sqlite3_snprintf(nMax, zBuf, "%s", sqlite3_temp_directory); + } + } +#if defined(__CYGWIN__) + else{ + static const char *azDirs[] = { + 0, /* getenv("SQLITE_TMPDIR") */ + 0, /* getenv("TMPDIR") */ + 0, /* getenv("TMP") */ + 0, /* getenv("TEMP") */ + 0, /* getenv("USERPROFILE") */ + "/var/tmp", + "/usr/tmp", + "/tmp", + ".", + 0 /* List terminator */ + }; + unsigned int i; + const char *zDir = 0; + + if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); + if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); + if( !azDirs[2] ) azDirs[2] = getenv("TMP"); + if( !azDirs[3] ) azDirs[3] = getenv("TEMP"); + if( !azDirs[4] ) azDirs[4] = getenv("USERPROFILE"); + for(i=0; i/etilqs_XXXXXXXXXXXXXXX\0\0" + ** + ** If not, return SQLITE_ERROR. The number 17 is used here in order to + ** account for the space used by the 15 character random suffix and the + ** two trailing NUL characters. The final directory separator character + ** has already added if it was not already present. + */ + nLen = sqlite3Strlen30(zBuf); + if( (nLen + nPre + 17) > nBuf ){ + sqlite3_free(zBuf); + OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); + return winLogError(SQLITE_ERROR, 0, "winGetTempname5", 0); + } + + sqlite3_snprintf(nBuf-16-nLen, zBuf+nLen, SQLITE_TEMP_FILE_PREFIX); + + j = sqlite3Strlen30(zBuf); + sqlite3_randomness(15, &zBuf[j]); + for(i=0; i<15; i++, j++){ + zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; + } + zBuf[j] = 0; + zBuf[j+1] = 0; + *pzBuf = zBuf; + + OSTRACE(("TEMP-FILENAME name=%s, rc=SQLITE_OK\n", zBuf)); + return SQLITE_OK; +} + +/* +** Return TRUE if the named file is really a directory. Return false if +** it is something other than a directory, or if there is any kind of memory +** allocation failure. +*/ +static int winIsDir(const void *zConverted){ + DWORD attr; + int rc = 0; + DWORD lastErrno; + + if( osIsNT() ){ + int cnt = 0; + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, + GetFileExInfoStandard, + &sAttrData)) && winRetryIoerr(&cnt, &lastErrno) ){} + if( !rc ){ + return 0; /* Invalid name? */ + } + attr = sAttrData.dwFileAttributes; +#if SQLITE_OS_WINCE==0 + }else{ + attr = osGetFileAttributesA((char*)zConverted); +#endif + } + return (attr!=INVALID_FILE_ATTRIBUTES) && (attr&FILE_ATTRIBUTE_DIRECTORY); +} + +/* forward reference */ +static int winAccess( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to check */ + int flags, /* Type of test to make on this file */ + int *pResOut /* OUT: Result */ +); + +/* +** Open a file. +*/ +static int winOpen( + sqlite3_vfs *pVfs, /* Used to get maximum path length and AppData */ + const char *zName, /* Name of the file (UTF-8) */ + sqlite3_file *id, /* Write the SQLite file handle here */ + int flags, /* Open mode flags */ + int *pOutFlags /* Status return flags */ +){ + HANDLE h; + DWORD lastErrno = 0; + DWORD dwDesiredAccess; + DWORD dwShareMode; + DWORD dwCreationDisposition; + DWORD dwFlagsAndAttributes = 0; +#if SQLITE_OS_WINCE + int isTemp = 0; +#endif + winVfsAppData *pAppData; + winFile *pFile = (winFile*)id; + void *zConverted; /* Filename in OS encoding */ + const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ + int cnt = 0; + + /* If argument zPath is a NULL pointer, this function is required to open + ** a temporary file. Use this buffer to store the file name in. + */ + char *zTmpname = 0; /* For temporary filename, if necessary. */ + + int rc = SQLITE_OK; /* Function Return Code */ +#if !defined(NDEBUG) || SQLITE_OS_WINCE + int eType = flags&0xFFFFFF00; /* Type of file to open */ +#endif + + int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); + int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); + int isCreate = (flags & SQLITE_OPEN_CREATE); + int isReadonly = (flags & SQLITE_OPEN_READONLY); + int isReadWrite = (flags & SQLITE_OPEN_READWRITE); + +#ifndef NDEBUG + int isOpenJournal = (isCreate && ( + eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL + || eType==SQLITE_OPEN_WAL + )); +#endif + + OSTRACE(("OPEN name=%s, pFile=%p, flags=%x, pOutFlags=%p\n", + zUtf8Name, id, flags, pOutFlags)); + + /* Check the following statements are true: + ** + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (b) if CREATE is set, then READWRITE must also be set, and + ** (c) if EXCLUSIVE is set, then CREATE must also be set. + ** (d) if DELETEONCLOSE is set, then CREATE must also be set. + */ + assert((isReadonly==0 || isReadWrite==0) && (isReadWrite || isReadonly)); + assert(isCreate==0 || isReadWrite); + assert(isExclusive==0 || isCreate); + assert(isDelete==0 || isCreate); + + /* The main DB, main journal, WAL file and super-journal are never + ** automatically deleted. Nor are they ever temporary files. */ + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_SUPER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); + + /* Assert that the upper layer has set one of the "file-type" flags. */ + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL + ); + + assert( pFile!=0 ); + memset(pFile, 0, sizeof(winFile)); + pFile->h = INVALID_HANDLE_VALUE; + +#if SQLITE_OS_WINRT + if( !zUtf8Name && !sqlite3_temp_directory ){ + sqlite3_log(SQLITE_ERROR, + "sqlite3_temp_directory variable should be set for WinRT"); + } +#endif + + /* If the second argument to this function is NULL, generate a + ** temporary file name to use + */ + if( !zUtf8Name ){ + assert( isDelete && !isOpenJournal ); + rc = winGetTempname(pVfs, &zTmpname); + if( rc!=SQLITE_OK ){ + OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); + return rc; + } + zUtf8Name = zTmpname; + } + + /* Database filenames are double-zero terminated if they are not + ** URIs with parameters. Hence, they can always be passed into + ** sqlite3_uri_parameter(). + */ + assert( (eType!=SQLITE_OPEN_MAIN_DB) || (flags & SQLITE_OPEN_URI) || + zUtf8Name[sqlite3Strlen30(zUtf8Name)+1]==0 ); + + /* Convert the filename to the system encoding. */ + zConverted = winConvertFromUtf8Filename(zUtf8Name); + if( zConverted==0 ){ + sqlite3_free(zTmpname); + OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); + return SQLITE_IOERR_NOMEM_BKPT; + } + + if( winIsDir(zConverted) ){ + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8Name)); + return SQLITE_CANTOPEN_ISDIR; + } + + if( isReadWrite ){ + dwDesiredAccess = GENERIC_READ | GENERIC_WRITE; + }else{ + dwDesiredAccess = GENERIC_READ; + } + + /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is + ** created. SQLite doesn't use it to indicate "exclusive access" + ** as it is usually understood. + */ + if( isExclusive ){ + /* Creates a new file, only if it does not already exist. */ + /* If the file exists, it fails. */ + dwCreationDisposition = CREATE_NEW; + }else if( isCreate ){ + /* Open existing file, or create if it doesn't exist */ + dwCreationDisposition = OPEN_ALWAYS; + }else{ + /* Opens a file, only if it exists. */ + dwCreationDisposition = OPEN_EXISTING; + } + + if( 0==sqlite3_uri_boolean(zName, "exclusive", 0) ){ + dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; + }else{ + dwShareMode = 0; + } + + if( isDelete ){ +#if SQLITE_OS_WINCE + dwFlagsAndAttributes = FILE_ATTRIBUTE_HIDDEN; + isTemp = 1; +#else + dwFlagsAndAttributes = FILE_ATTRIBUTE_TEMPORARY + | FILE_ATTRIBUTE_HIDDEN + | FILE_FLAG_DELETE_ON_CLOSE; +#endif + }else{ + dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL; + } + /* Reports from the internet are that performance is always + ** better if FILE_FLAG_RANDOM_ACCESS is used. Ticket #2699. */ +#if SQLITE_OS_WINCE + dwFlagsAndAttributes |= FILE_FLAG_RANDOM_ACCESS; +#endif + + if( osIsNT() ){ +#if SQLITE_OS_WINRT + CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; + extendedParameters.dwSize = sizeof(CREATEFILE2_EXTENDED_PARAMETERS); + extendedParameters.dwFileAttributes = + dwFlagsAndAttributes & FILE_ATTRIBUTE_MASK; + extendedParameters.dwFileFlags = dwFlagsAndAttributes & FILE_FLAG_MASK; + extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; + extendedParameters.lpSecurityAttributes = NULL; + extendedParameters.hTemplateFile = NULL; + do{ + h = osCreateFile2((LPCWSTR)zConverted, + dwDesiredAccess, + dwShareMode, + dwCreationDisposition, + &extendedParameters); + if( h!=INVALID_HANDLE_VALUE ) break; + if( isReadWrite ){ + int rc2, isRO = 0; + sqlite3BeginBenignMalloc(); + rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + sqlite3EndBenignMalloc(); + if( rc2==SQLITE_OK && isRO ) break; + } + }while( winRetryIoerr(&cnt, &lastErrno) ); +#else + do{ + h = osCreateFileW((LPCWSTR)zConverted, + dwDesiredAccess, + dwShareMode, NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL); + if( h!=INVALID_HANDLE_VALUE ) break; + if( isReadWrite ){ + int rc2, isRO = 0; + sqlite3BeginBenignMalloc(); + rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + sqlite3EndBenignMalloc(); + if( rc2==SQLITE_OK && isRO ) break; + } + }while( winRetryIoerr(&cnt, &lastErrno) ); +#endif + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + do{ + h = osCreateFileA((LPCSTR)zConverted, + dwDesiredAccess, + dwShareMode, NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL); + if( h!=INVALID_HANDLE_VALUE ) break; + if( isReadWrite ){ + int rc2, isRO = 0; + sqlite3BeginBenignMalloc(); + rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO); + sqlite3EndBenignMalloc(); + if( rc2==SQLITE_OK && isRO ) break; + } + }while( winRetryIoerr(&cnt, &lastErrno) ); + } +#endif + winLogIoerr(cnt, __LINE__); + + OSTRACE(("OPEN file=%p, name=%s, access=%lx, rc=%s\n", h, zUtf8Name, + dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + + if( h==INVALID_HANDLE_VALUE ){ + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + if( isReadWrite && !isExclusive ){ + return winOpen(pVfs, zName, id, + ((flags|SQLITE_OPEN_READONLY) & + ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), + pOutFlags); + }else{ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_CANTOPEN, pFile->lastErrno, "winOpen", zUtf8Name); + return SQLITE_CANTOPEN_BKPT; + } + } + + if( pOutFlags ){ + if( isReadWrite ){ + *pOutFlags = SQLITE_OPEN_READWRITE; + }else{ + *pOutFlags = SQLITE_OPEN_READONLY; + } + } + + OSTRACE(("OPEN file=%p, name=%s, access=%lx, pOutFlags=%p, *pOutFlags=%d, " + "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? + *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + + pAppData = (winVfsAppData*)pVfs->pAppData; + +#if SQLITE_OS_WINCE + { + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB + && ((pAppData==NULL) || !pAppData->bNoLock) + && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK + ){ + osCloseHandle(h); + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); + return rc; + } + } + if( isTemp ){ + pFile->zDeleteOnClose = zConverted; + }else +#endif + { + sqlite3_free(zConverted); + } + + sqlite3_free(zTmpname); + id->pMethods = pAppData ? pAppData->pMethod : &winIoMethod; + pFile->pVfs = pVfs; + pFile->h = h; + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } + if( (flags & SQLITE_OPEN_MAIN_DB) + && sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) + ){ + pFile->ctrlFlags |= WINFILE_PSOW; + } + pFile->lastErrno = NO_ERROR; + pFile->zPath = zName; +#if SQLITE_MAX_MMAP_SIZE>0 + pFile->hMap = NULL; + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + pFile->mmapSizeMax = sqlite3GlobalConfig.szMmap; +#endif + + OpenCounter(+1); + return rc; +} + +/* +** Delete the named file. +** +** Note that Windows does not allow a file to be deleted if some other +** process has it open. Sometimes a virus scanner or indexing program +** will open a journal file shortly after it is created in order to do +** whatever it does. While this other process is holding the +** file open, we will be unable to delete it. To work around this +** problem, we delay 100 milliseconds and try to delete again. Up +** to MX_DELETION_ATTEMPTs deletion attempts are run before giving +** up and returning an error. +*/ +static int winDelete( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to delete */ + int syncDir /* Not used on win32 */ +){ + int cnt = 0; + int rc; + DWORD attr; + DWORD lastErrno = 0; + void *zConverted; + UNUSED_PARAMETER(pVfs); + UNUSED_PARAMETER(syncDir); + + SimulateIOError(return SQLITE_IOERR_DELETE); + OSTRACE(("DELETE name=%s, syncDir=%d\n", zFilename, syncDir)); + + zConverted = winConvertFromUtf8Filename(zFilename); + if( zConverted==0 ){ + OSTRACE(("DELETE name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); + return SQLITE_IOERR_NOMEM_BKPT; + } + if( osIsNT() ){ + do { +#if SQLITE_OS_WINRT + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + if ( osGetFileAttributesExW(zConverted, GetFileExInfoStandard, + &sAttrData) ){ + attr = sAttrData.dwFileAttributes; + }else{ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } + break; + } +#else + attr = osGetFileAttributesW(zConverted); +#endif + if ( attr==INVALID_FILE_ATTRIBUTES ){ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } + break; + } + if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ + rc = SQLITE_ERROR; /* Files only. */ + break; + } + if ( osDeleteFileW(zConverted) ){ + rc = SQLITE_OK; /* Deleted OK. */ + break; + } + if ( !winRetryIoerr(&cnt, &lastErrno) ){ + rc = SQLITE_ERROR; /* No more retries. */ + break; + } + } while(1); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + do { + attr = osGetFileAttributesA(zConverted); + if ( attr==INVALID_FILE_ATTRIBUTES ){ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND + || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } + break; + } + if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ + rc = SQLITE_ERROR; /* Files only. */ + break; + } + if ( osDeleteFileA(zConverted) ){ + rc = SQLITE_OK; /* Deleted OK. */ + break; + } + if ( !winRetryIoerr(&cnt, &lastErrno) ){ + rc = SQLITE_ERROR; /* No more retries. */ + break; + } + } while(1); + } +#endif + if( rc && rc!=SQLITE_IOERR_DELETE_NOENT ){ + rc = winLogError(SQLITE_IOERR_DELETE, lastErrno, "winDelete", zFilename); + }else{ + winLogIoerr(cnt, __LINE__); + } + sqlite3_free(zConverted); + OSTRACE(("DELETE name=%s, rc=%s\n", zFilename, sqlite3ErrName(rc))); + return rc; +} + +/* +** Check the existence and status of a file. +*/ +static int winAccess( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to check */ + int flags, /* Type of test to make on this file */ + int *pResOut /* OUT: Result */ +){ + DWORD attr; + int rc = 0; + DWORD lastErrno = 0; + void *zConverted; + UNUSED_PARAMETER(pVfs); + + SimulateIOError( return SQLITE_IOERR_ACCESS; ); + OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", + zFilename, flags, pResOut)); + + zConverted = winConvertFromUtf8Filename(zFilename); + if( zConverted==0 ){ + OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); + return SQLITE_IOERR_NOMEM_BKPT; + } + if( osIsNT() ){ + int cnt = 0; + WIN32_FILE_ATTRIBUTE_DATA sAttrData; + memset(&sAttrData, 0, sizeof(sAttrData)); + while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, + GetFileExInfoStandard, + &sAttrData)) && winRetryIoerr(&cnt, &lastErrno) ){} + if( rc ){ + /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file + ** as if it does not exist. + */ + if( flags==SQLITE_ACCESS_EXISTS + && sAttrData.nFileSizeHigh==0 + && sAttrData.nFileSizeLow==0 ){ + attr = INVALID_FILE_ATTRIBUTES; + }else{ + attr = sAttrData.dwFileAttributes; + } + }else{ + winLogIoerr(cnt, __LINE__); + if( lastErrno!=ERROR_FILE_NOT_FOUND && lastErrno!=ERROR_PATH_NOT_FOUND ){ + sqlite3_free(zConverted); + return winLogError(SQLITE_IOERR_ACCESS, lastErrno, "winAccess", + zFilename); + }else{ + attr = INVALID_FILE_ATTRIBUTES; + } + } + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + attr = osGetFileAttributesA((char*)zConverted); + } +#endif + sqlite3_free(zConverted); + switch( flags ){ + case SQLITE_ACCESS_READ: + case SQLITE_ACCESS_EXISTS: + rc = attr!=INVALID_FILE_ATTRIBUTES; + break; + case SQLITE_ACCESS_READWRITE: + rc = attr!=INVALID_FILE_ATTRIBUTES && + (attr & FILE_ATTRIBUTE_READONLY)==0; + break; + default: + assert(!"Invalid flags argument"); + } + *pResOut = rc; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); + return SQLITE_OK; +} + +/* +** Returns non-zero if the specified path name starts with the "long path" +** prefix. +*/ +static BOOL winIsLongPathPrefix( + const char *zPathname +){ + return ( zPathname[0]=='\\' && zPathname[1]=='\\' + && zPathname[2]=='?' && zPathname[3]=='\\' ); +} + +/* +** Returns non-zero if the specified path name starts with a drive letter +** followed by a colon character. +*/ +static BOOL winIsDriveLetterAndColon( + const char *zPathname +){ + return ( sqlite3Isalpha(zPathname[0]) && zPathname[1]==':' ); +} + +/* +** Returns non-zero if the specified path name should be used verbatim. If +** non-zero is returned from this function, the calling function must simply +** use the provided path name verbatim -OR- resolve it into a full path name +** using the GetFullPathName Win32 API function (if available). +*/ +static BOOL winIsVerbatimPathname( + const char *zPathname +){ + /* + ** If the path name starts with a forward slash or a backslash, it is either + ** a legal UNC name, a volume relative path, or an absolute path name in the + ** "Unix" format on Windows. There is no easy way to differentiate between + ** the final two cases; therefore, we return the safer return value of TRUE + ** so that callers of this function will simply use it verbatim. + */ + if ( winIsDirSep(zPathname[0]) ){ + return TRUE; + } + + /* + ** If the path name starts with a letter and a colon it is either a volume + ** relative path or an absolute path. Callers of this function must not + ** attempt to treat it as a relative path name (i.e. they should simply use + ** it verbatim). + */ + if ( winIsDriveLetterAndColon(zPathname) ){ + return TRUE; + } + + /* + ** If we get to this point, the path name should almost certainly be a purely + ** relative one (i.e. not a UNC name, not absolute, and not volume relative). + */ + return FALSE; +} + +/* +** Turn a relative pathname into a full pathname. Write the full +** pathname into zOut[]. zOut[] will be at least pVfs->mxPathname +** bytes in size. +*/ +static int winFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) + DWORD nByte; + void *zConverted; + char *zOut; +#endif + + /* If this path name begins with "/X:" or "\\?\", where "X" is any + ** alphabetic character, discard the initial "/" from the pathname. + */ + if( zRelative[0]=='/' && (winIsDriveLetterAndColon(zRelative+1) + || winIsLongPathPrefix(zRelative+1)) ){ + zRelative++; + } + +#if defined(__CYGWIN__) + SimulateIOError( return SQLITE_ERROR ); + UNUSED_PARAMETER(nFull); + assert( nFull>=pVfs->mxPathname ); + if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ + /* + ** NOTE: We are dealing with a relative path name and the data + ** directory has been set. Therefore, use it as the basis + ** for converting the relative path name to an absolute + ** one by prepending the data directory and a slash. + */ + char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); + if( !zOut ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + if( cygwin_conv_path( + (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A) | + CCP_RELATIVE, zRelative, zOut, pVfs->mxPathname+1)<0 ){ + sqlite3_free(zOut); + return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, + "winFullPathname1", zRelative); + }else{ + char *zUtf8 = winConvertToUtf8Filename(zOut); + if( !zUtf8 ){ + sqlite3_free(zOut); + return SQLITE_IOERR_NOMEM_BKPT; + } + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", + sqlite3_data_directory, winGetDirSep(), zUtf8); + sqlite3_free(zUtf8); + sqlite3_free(zOut); + } + }else{ + char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); + if( !zOut ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + if( cygwin_conv_path( + (osIsNT() ? CCP_POSIX_TO_WIN_W : CCP_POSIX_TO_WIN_A), + zRelative, zOut, pVfs->mxPathname+1)<0 ){ + sqlite3_free(zOut); + return winLogError(SQLITE_CANTOPEN_CONVPATH, (DWORD)errno, + "winFullPathname2", zRelative); + }else{ + char *zUtf8 = winConvertToUtf8Filename(zOut); + if( !zUtf8 ){ + sqlite3_free(zOut); + return SQLITE_IOERR_NOMEM_BKPT; + } + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zUtf8); + sqlite3_free(zUtf8); + sqlite3_free(zOut); + } + } + return SQLITE_OK; +#endif + +#if (SQLITE_OS_WINCE || SQLITE_OS_WINRT) && !defined(__CYGWIN__) + SimulateIOError( return SQLITE_ERROR ); + /* WinCE has no concept of a relative pathname, or so I am told. */ + /* WinRT has no way to convert a relative path to an absolute one. */ + if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ + /* + ** NOTE: We are dealing with a relative path name and the data + ** directory has been set. Therefore, use it as the basis + ** for converting the relative path name to an absolute + ** one by prepending the data directory and a backslash. + */ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", + sqlite3_data_directory, winGetDirSep(), zRelative); + }else{ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zRelative); + } + return SQLITE_OK; +#endif + +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) + /* It's odd to simulate an io-error here, but really this is just + ** using the io-error infrastructure to test that SQLite handles this + ** function failing. This function could fail if, for example, the + ** current working directory has been unlinked. + */ + SimulateIOError( return SQLITE_ERROR ); + if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ + /* + ** NOTE: We are dealing with a relative path name and the data + ** directory has been set. Therefore, use it as the basis + ** for converting the relative path name to an absolute + ** one by prepending the data directory and a backslash. + */ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%c%s", + sqlite3_data_directory, winGetDirSep(), zRelative); + return SQLITE_OK; + } + zConverted = winConvertFromUtf8Filename(zRelative); + if( zConverted==0 ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + if( osIsNT() ){ + LPWSTR zTemp; + nByte = osGetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname1", zRelative); + } + nByte += 3; + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); + if( zTemp==0 ){ + sqlite3_free(zConverted); + return SQLITE_IOERR_NOMEM_BKPT; + } + nByte = osGetFullPathNameW((LPCWSTR)zConverted, nByte, zTemp, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + sqlite3_free(zTemp); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname2", zRelative); + } + sqlite3_free(zConverted); + zOut = winUnicodeToUtf8(zTemp); + sqlite3_free(zTemp); + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + char *zTemp; + nByte = osGetFullPathNameA((char*)zConverted, 0, 0, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname3", zRelative); + } + nByte += 3; + zTemp = sqlite3MallocZero( nByte*sizeof(zTemp[0]) ); + if( zTemp==0 ){ + sqlite3_free(zConverted); + return SQLITE_IOERR_NOMEM_BKPT; + } + nByte = osGetFullPathNameA((char*)zConverted, nByte, zTemp, 0); + if( nByte==0 ){ + sqlite3_free(zConverted); + sqlite3_free(zTemp); + return winLogError(SQLITE_CANTOPEN_FULLPATH, osGetLastError(), + "winFullPathname4", zRelative); + } + sqlite3_free(zConverted); + zOut = winMbcsToUtf8(zTemp, osAreFileApisANSI()); + sqlite3_free(zTemp); + } +#endif + if( zOut ){ + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zOut); + sqlite3_free(zOut); + return SQLITE_OK; + }else{ + return SQLITE_IOERR_NOMEM_BKPT; + } +#endif +} + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** Interfaces for opening a shared library, finding entry points +** within the shared library, and closing the shared library. +*/ +static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ + HANDLE h; +#if defined(__CYGWIN__) + int nFull = pVfs->mxPathname+1; + char *zFull = sqlite3MallocZero( nFull ); + void *zConverted = 0; + if( zFull==0 ){ + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); + return 0; + } + if( winFullPathname(pVfs, zFilename, nFull, zFull)!=SQLITE_OK ){ + sqlite3_free(zFull); + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); + return 0; + } + zConverted = winConvertFromUtf8Filename(zFull); + sqlite3_free(zFull); +#else + void *zConverted = winConvertFromUtf8Filename(zFilename); + UNUSED_PARAMETER(pVfs); +#endif + if( zConverted==0 ){ + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); + return 0; + } + if( osIsNT() ){ +#if SQLITE_OS_WINRT + h = osLoadPackagedLibrary((LPCWSTR)zConverted, 0); +#else + h = osLoadLibraryW((LPCWSTR)zConverted); +#endif + } +#ifdef SQLITE_WIN32_HAS_ANSI + else{ + h = osLoadLibraryA((char*)zConverted); + } +#endif + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)h)); + sqlite3_free(zConverted); + return (void*)h; +} +static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ + UNUSED_PARAMETER(pVfs); + winGetLastErrorMsg(osGetLastError(), nBuf, zBufOut); +} +static void (*winDlSym(sqlite3_vfs *pVfs,void *pH,const char *zSym))(void){ + FARPROC proc; + UNUSED_PARAMETER(pVfs); + proc = osGetProcAddressA((HANDLE)pH, zSym); + OSTRACE(("DLSYM handle=%p, symbol=%s, address=%p\n", + (void*)pH, zSym, (void*)proc)); + return (void(*)(void))proc; +} +static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ + UNUSED_PARAMETER(pVfs); + osFreeLibrary((HANDLE)pHandle); + OSTRACE(("DLCLOSE handle=%p\n", (void*)pHandle)); +} +#else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ + #define winDlOpen 0 + #define winDlError 0 + #define winDlSym 0 + #define winDlClose 0 +#endif + +/* State information for the randomness gatherer. */ +typedef struct EntropyGatherer EntropyGatherer; +struct EntropyGatherer { + unsigned char *a; /* Gather entropy into this buffer */ + int na; /* Size of a[] in bytes */ + int i; /* XOR next input into a[i] */ + int nXor; /* Number of XOR operations done */ +}; + +#if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) +/* Mix sz bytes of entropy into p. */ +static void xorMemory(EntropyGatherer *p, unsigned char *x, int sz){ + int j, k; + for(j=0, k=p->i; ja[k++] ^= x[j]; + if( k>=p->na ) k = 0; + } + p->i = k; + p->nXor += sz; +} +#endif /* !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) */ + +/* +** Write up to nBuf bytes of randomness into zBuf. +*/ +static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ +#if defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) + UNUSED_PARAMETER(pVfs); + memset(zBuf, 0, nBuf); + return nBuf; +#else + EntropyGatherer e; + UNUSED_PARAMETER(pVfs); + memset(zBuf, 0, nBuf); + e.a = (unsigned char*)zBuf; + e.na = nBuf; + e.nXor = 0; + e.i = 0; + { + SYSTEMTIME x; + osGetSystemTime(&x); + xorMemory(&e, (unsigned char*)&x, sizeof(SYSTEMTIME)); + } + { + DWORD pid = osGetCurrentProcessId(); + xorMemory(&e, (unsigned char*)&pid, sizeof(DWORD)); + } +#if SQLITE_OS_WINRT + { + ULONGLONG cnt = osGetTickCount64(); + xorMemory(&e, (unsigned char*)&cnt, sizeof(ULONGLONG)); + } +#else + { + DWORD cnt = osGetTickCount(); + xorMemory(&e, (unsigned char*)&cnt, sizeof(DWORD)); + } +#endif /* SQLITE_OS_WINRT */ + { + LARGE_INTEGER i; + osQueryPerformanceCounter(&i); + xorMemory(&e, (unsigned char*)&i, sizeof(LARGE_INTEGER)); + } +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID + { + UUID id; + memset(&id, 0, sizeof(UUID)); + osUuidCreate(&id); + xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); + memset(&id, 0, sizeof(UUID)); + osUuidCreateSequential(&id); + xorMemory(&e, (unsigned char*)&id, sizeof(UUID)); + } +#endif /* !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_WIN32_USE_UUID */ + return e.nXor>nBuf ? nBuf : e.nXor; +#endif /* defined(SQLITE_TEST) || defined(SQLITE_OMIT_RANDOMNESS) */ +} + + +/* +** Sleep for a little while. Return the amount of time slept. +*/ +static int winSleep(sqlite3_vfs *pVfs, int microsec){ + sqlite3_win32_sleep((microsec+999)/1000); + UNUSED_PARAMETER(pVfs); + return ((microsec+999)/1000)*1000; +} + +/* +** The following variable, if set to a non-zero value, is interpreted as +** the number of seconds since 1970 and is used to set the result of +** sqlite3OsCurrentTime() during testing. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ +#endif + +/* +** Find the current time (in Universal Coordinated Time). Write into *piNow +** the current time and date as a Julian Day number times 86_400_000. In +** other words, write into *piNow the number of milliseconds since the Julian +** epoch of noon in Greenwich on November 24, 4714 B.C according to the +** proleptic Gregorian calendar. +** +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** cannot be found. +*/ +static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ + /* FILETIME structure is a 64-bit value representing the number of + 100-nanosecond intervals since January 1, 1601 (= JD 2305813.5). + */ + FILETIME ft; + static const sqlite3_int64 winFiletimeEpoch = 23058135*(sqlite3_int64)8640000; +#ifdef SQLITE_TEST + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; +#endif + /* 2^32 - to avoid use of LL and warnings in gcc */ + static const sqlite3_int64 max32BitValue = + (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + + (sqlite3_int64)294967296; + +#if SQLITE_OS_WINCE + SYSTEMTIME time; + osGetSystemTime(&time); + /* if SystemTimeToFileTime() fails, it returns zero. */ + if (!osSystemTimeToFileTime(&time,&ft)){ + return SQLITE_ERROR; + } +#else + osGetSystemTimeAsFileTime( &ft ); +#endif + + *piNow = winFiletimeEpoch + + ((((sqlite3_int64)ft.dwHighDateTime)*max32BitValue) + + (sqlite3_int64)ft.dwLowDateTime)/(sqlite3_int64)10000; + +#ifdef SQLITE_TEST + if( sqlite3_current_time ){ + *piNow = 1000*(sqlite3_int64)sqlite3_current_time + unixEpoch; + } +#endif + UNUSED_PARAMETER(pVfs); + return SQLITE_OK; +} + +/* +** Find the current time (in Universal Coordinated Time). Write the +** current time and date as a Julian Day number into *prNow and +** return 0. Return 1 if the time and date cannot be found. +*/ +static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ + int rc; + sqlite3_int64 i; + rc = winCurrentTimeInt64(pVfs, &i); + if( !rc ){ + *prNow = i/86400000.0; + } + return rc; +} + +/* +** The idea is that this function works like a combination of +** GetLastError() and FormatMessage() on Windows (or errno and +** strerror_r() on Unix). After an error is returned by an OS +** function, SQLite calls this function with zBuf pointing to +** a buffer of nBuf bytes. The OS layer should populate the +** buffer with a nul-terminated UTF-8 encoded error message +** describing the last IO error to have occurred within the calling +** thread. +** +** If the error message is too large for the supplied buffer, +** it should be truncated. The return value of xGetLastError +** is zero if the error message fits in the buffer, or non-zero +** otherwise (if the message was truncated). If non-zero is returned, +** then it is not necessary to include the nul-terminator character +** in the output buffer. +** +** Not supplying an error message will have no adverse effect +** on SQLite. It is fine to have an implementation that never +** returns an error message: +** +** int xGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ +** assert(zBuf[0]=='\0'); +** return 0; +** } +** +** However if an error message is supplied, it will be incorporated +** by sqlite into the error message available to the user using +** sqlite3_errmsg(), possibly making IO errors easier to debug. +*/ +static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ + DWORD e = osGetLastError(); + UNUSED_PARAMETER(pVfs); + if( nBuf>0 ) winGetLastErrorMsg(e, nBuf, zBuf); + return e; +} + +/* +** Initialize and deinitialize the operating system interface. +*/ +SQLITE_API int sqlite3_os_init(void){ + static sqlite3_vfs winVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#if defined(SQLITE_WIN32_HAS_WIDE) + static sqlite3_vfs winLongPathVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-longpath", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#endif + static sqlite3_vfs winNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#if defined(SQLITE_WIN32_HAS_WIDE) + static sqlite3_vfs winLongPathNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-longpath-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#endif + + /* Double-check that the aSyscall[] array has been constructed + ** correctly. See ticket [bb3a86e890c8e96ab] */ + assert( ArraySize(aSyscall)==80 ); + + /* get memory map allocation granularity */ + memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); +#if SQLITE_OS_WINRT + osGetNativeSystemInfo(&winSysInfo); +#else + osGetSystemInfo(&winSysInfo); +#endif + assert( winSysInfo.dwAllocationGranularity>0 ); + assert( winSysInfo.dwPageSize>0 ); + + sqlite3_vfs_register(&winVfs, 1); + +#if defined(SQLITE_WIN32_HAS_WIDE) + sqlite3_vfs_register(&winLongPathVfs, 0); +#endif + + sqlite3_vfs_register(&winNolockVfs, 0); + +#if defined(SQLITE_WIN32_HAS_WIDE) + sqlite3_vfs_register(&winLongPathNolockVfs, 0); +#endif + +#ifndef SQLITE_OMIT_WAL + winBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#endif + + return SQLITE_OK; +} + +SQLITE_API int sqlite3_os_end(void){ +#if SQLITE_OS_WINRT + if( sleepObj!=NULL ){ + osCloseHandle(sleepObj); + sleepObj = NULL; + } +#endif + +#ifndef SQLITE_OMIT_WAL + winBigLock = 0; +#endif + + return SQLITE_OK; +} + +#endif /* SQLITE_OS_WIN */ + +/************** End of os_win.c **********************************************/ +/************** Begin file memdb.c *******************************************/ +/* +** 2016-09-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements an in-memory VFS. A database is held as a contiguous +** block of memory. +** +** This file also implements interface sqlite3_serialize() and +** sqlite3_deserialize(). +*/ +/* #include "sqliteInt.h" */ +#ifndef SQLITE_OMIT_DESERIALIZE + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs MemVfs; +typedef struct MemFile MemFile; +typedef struct MemStore MemStore; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) + +/* Storage for a memdb file. +** +** An memdb object can be shared or separate. Shared memdb objects can be +** used by more than one database connection. Mutexes are used by shared +** memdb objects to coordinate access. Separate memdb objects are only +** connected to a single database connection and do not require additional +** mutexes. +** +** Shared memdb objects have .zFName!=0 and .pMutex!=0. They are created +** using "file:/name?vfs=memdb". The first character of the name must be +** "/" or else the object will be a separate memdb object. All shared +** memdb objects are stored in memdb_g.apMemStore[] in an arbitrary order. +** +** Separate memdb objects are created using a name that does not begin +** with "/" or using sqlite3_deserialize(). +** +** Access rules for shared MemStore objects: +** +** * .zFName is initialized when the object is created and afterwards +** is unchanged until the object is destroyed. So it can be accessed +** at any time as long as we know the object is not being destroyed, +** which means while either the SQLITE_MUTEX_STATIC_VFS1 or +** .pMutex is held or the object is not part of memdb_g.apMemStore[]. +** +** * Can .pMutex can only be changed while holding the +** SQLITE_MUTEX_STATIC_VFS1 mutex or while the object is not part +** of memdb_g.apMemStore[]. +** +** * Other fields can only be changed while holding the .pMutex mutex +** or when the .nRef is less than zero and the object is not part of +** memdb_g.apMemStore[]. +** +** * The .aData pointer has the added requirement that it can can only +** be changed (for resizing) when nMmap is zero. +** +*/ +struct MemStore { + sqlite3_int64 sz; /* Size of the file */ + sqlite3_int64 szAlloc; /* Space allocated to aData */ + sqlite3_int64 szMax; /* Maximum allowed size of the file */ + unsigned char *aData; /* content of the file */ + sqlite3_mutex *pMutex; /* Used by shared stores only */ + int nMmap; /* Number of memory mapped pages */ + unsigned mFlags; /* Flags */ + int nRdLock; /* Number of readers */ + int nWrLock; /* Number of writers. (Always 0 or 1) */ + int nRef; /* Number of users of this MemStore */ + char *zFName; /* The filename for shared stores */ +}; + +/* An open file */ +struct MemFile { + sqlite3_file base; /* IO methods */ + MemStore *pStore; /* The storage */ + int eLock; /* Most recent lock against this file */ +}; + +/* +** File-scope variables for holding the memdb files that are accessible +** to multiple database connections in separate threads. +** +** Must hold SQLITE_MUTEX_STATIC_VFS1 to access any part of this object. +*/ +static struct MemFS { + int nMemStore; /* Number of shared MemStore objects */ + MemStore **apMemStore; /* Array of all shared MemStore objects */ +} memdb_g; + +/* +** Methods for MemFile +*/ +static int memdbClose(sqlite3_file*); +static int memdbRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int memdbWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int memdbTruncate(sqlite3_file*, sqlite3_int64 size); +static int memdbSync(sqlite3_file*, int flags); +static int memdbFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int memdbLock(sqlite3_file*, int); +/* static int memdbCheckReservedLock(sqlite3_file*, int *pResOut);// not used */ +static int memdbFileControl(sqlite3_file*, int op, void *pArg); +/* static int memdbSectorSize(sqlite3_file*); // not used */ +static int memdbDeviceCharacteristics(sqlite3_file*); +static int memdbFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int memdbUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for MemVfs +*/ +static int memdbOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +/* static int memdbDelete(sqlite3_vfs*, const char *zName, int syncDir); */ +static int memdbAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int memdbFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *memdbDlOpen(sqlite3_vfs*, const char *zFilename); +static void memdbDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*memdbDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void memdbDlClose(sqlite3_vfs*, void*); +static int memdbRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int memdbSleep(sqlite3_vfs*, int microseconds); +/* static int memdbCurrentTime(sqlite3_vfs*, double*); */ +static int memdbGetLastError(sqlite3_vfs*, int, char *); +static int memdbCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs memdb_vfs = { + 2, /* iVersion */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "memdb", /* zName */ + 0, /* pAppData (set when registered) */ + memdbOpen, /* xOpen */ + 0, /* memdbDelete, */ /* xDelete */ + memdbAccess, /* xAccess */ + memdbFullPathname, /* xFullPathname */ + memdbDlOpen, /* xDlOpen */ + memdbDlError, /* xDlError */ + memdbDlSym, /* xDlSym */ + memdbDlClose, /* xDlClose */ + memdbRandomness, /* xRandomness */ + memdbSleep, /* xSleep */ + 0, /* memdbCurrentTime, */ /* xCurrentTime */ + memdbGetLastError, /* xGetLastError */ + memdbCurrentTimeInt64, /* xCurrentTimeInt64 */ + 0, /* xSetSystemCall */ + 0, /* xGetSystemCall */ + 0, /* xNextSystemCall */ +}; + +static const sqlite3_io_methods memdb_io_methods = { + 3, /* iVersion */ + memdbClose, /* xClose */ + memdbRead, /* xRead */ + memdbWrite, /* xWrite */ + memdbTruncate, /* xTruncate */ + memdbSync, /* xSync */ + memdbFileSize, /* xFileSize */ + memdbLock, /* xLock */ + memdbLock, /* xUnlock - same as xLock in this case */ + 0, /* memdbCheckReservedLock, */ /* xCheckReservedLock */ + memdbFileControl, /* xFileControl */ + 0, /* memdbSectorSize,*/ /* xSectorSize */ + memdbDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + memdbFetch, /* xFetch */ + memdbUnfetch /* xUnfetch */ +}; + +/* +** Enter/leave the mutex on a MemStore +*/ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0 +static void memdbEnter(MemStore *p){ + UNUSED_PARAMETER(p); +} +static void memdbLeave(MemStore *p){ + UNUSED_PARAMETER(p); +} +#else +static void memdbEnter(MemStore *p){ + sqlite3_mutex_enter(p->pMutex); +} +static void memdbLeave(MemStore *p){ + sqlite3_mutex_leave(p->pMutex); +} +#endif + + + +/* +** Close an memdb-file. +** Free the underlying MemStore object when its refcount drops to zero +** or less. +*/ +static int memdbClose(sqlite3_file *pFile){ + MemStore *p = ((MemFile*)pFile)->pStore; + if( p->zFName ){ + int i; +#ifndef SQLITE_MUTEX_OMIT + sqlite3_mutex *pVfsMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#endif + sqlite3_mutex_enter(pVfsMutex); + for(i=0; ALWAYS(inRef==1 ){ + memdb_g.apMemStore[i] = memdb_g.apMemStore[--memdb_g.nMemStore]; + if( memdb_g.nMemStore==0 ){ + sqlite3_free(memdb_g.apMemStore); + memdb_g.apMemStore = 0; + } + } + break; + } + } + sqlite3_mutex_leave(pVfsMutex); + }else{ + memdbEnter(p); + } + p->nRef--; + if( p->nRef<=0 ){ + if( p->mFlags & SQLITE_DESERIALIZE_FREEONCLOSE ){ + sqlite3_free(p->aData); + } + memdbLeave(p); + sqlite3_mutex_free(p->pMutex); + sqlite3_free(p); + }else{ + memdbLeave(p); + } + return SQLITE_OK; +} + +/* +** Read data from an memdb-file. +*/ +static int memdbRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + MemStore *p = ((MemFile*)pFile)->pStore; + memdbEnter(p); + if( iOfst+iAmt>p->sz ){ + memset(zBuf, 0, iAmt); + if( iOfstsz ) memcpy(zBuf, p->aData+iOfst, p->sz - iOfst); + memdbLeave(p); + return SQLITE_IOERR_SHORT_READ; + } + memcpy(zBuf, p->aData+iOfst, iAmt); + memdbLeave(p); + return SQLITE_OK; +} + +/* +** Try to enlarge the memory allocation to hold at least sz bytes +*/ +static int memdbEnlarge(MemStore *p, sqlite3_int64 newSz){ + unsigned char *pNew; + if( (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)==0 || p->nMmap>0 ){ + return SQLITE_FULL; + } + if( newSz>p->szMax ){ + return SQLITE_FULL; + } + newSz *= 2; + if( newSz>p->szMax ) newSz = p->szMax; + pNew = sqlite3Realloc(p->aData, newSz); + if( pNew==0 ) return SQLITE_IOERR_NOMEM; + p->aData = pNew; + p->szAlloc = newSz; + return SQLITE_OK; +} + +/* +** Write data to an memdb-file. +*/ +static int memdbWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + MemStore *p = ((MemFile*)pFile)->pStore; + memdbEnter(p); + if( NEVER(p->mFlags & SQLITE_DESERIALIZE_READONLY) ){ + /* Can't happen: memdbLock() will return SQLITE_READONLY before + ** reaching this point */ + memdbLeave(p); + return SQLITE_IOERR_WRITE; + } + if( iOfst+iAmt>p->sz ){ + int rc; + if( iOfst+iAmt>p->szAlloc + && (rc = memdbEnlarge(p, iOfst+iAmt))!=SQLITE_OK + ){ + memdbLeave(p); + return rc; + } + if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz); + p->sz = iOfst+iAmt; + } + memcpy(p->aData+iOfst, z, iAmt); + memdbLeave(p); + return SQLITE_OK; +} + +/* +** Truncate an memdb-file. +** +** In rollback mode (which is always the case for memdb, as it does not +** support WAL mode) the truncate() method is only used to reduce +** the size of a file, never to increase the size. +*/ +static int memdbTruncate(sqlite3_file *pFile, sqlite_int64 size){ + MemStore *p = ((MemFile*)pFile)->pStore; + int rc = SQLITE_OK; + memdbEnter(p); + if( NEVER(size>p->sz) ){ + rc = SQLITE_FULL; + }else{ + p->sz = size; + } + memdbLeave(p); + return rc; +} + +/* +** Sync an memdb-file. +*/ +static int memdbSync(sqlite3_file *pFile, int flags){ + UNUSED_PARAMETER(pFile); + UNUSED_PARAMETER(flags); + return SQLITE_OK; +} + +/* +** Return the current file-size of an memdb-file. +*/ +static int memdbFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + MemStore *p = ((MemFile*)pFile)->pStore; + memdbEnter(p); + *pSize = p->sz; + memdbLeave(p); + return SQLITE_OK; +} + +/* +** Lock an memdb-file. +*/ +static int memdbLock(sqlite3_file *pFile, int eLock){ + MemFile *pThis = (MemFile*)pFile; + MemStore *p = pThis->pStore; + int rc = SQLITE_OK; + if( eLock==pThis->eLock ) return SQLITE_OK; + memdbEnter(p); + if( eLock>SQLITE_LOCK_SHARED ){ + if( p->mFlags & SQLITE_DESERIALIZE_READONLY ){ + rc = SQLITE_READONLY; + }else if( pThis->eLock<=SQLITE_LOCK_SHARED ){ + if( p->nWrLock ){ + rc = SQLITE_BUSY; + }else{ + p->nWrLock = 1; + } + } + }else if( eLock==SQLITE_LOCK_SHARED ){ + if( pThis->eLock > SQLITE_LOCK_SHARED ){ + assert( p->nWrLock==1 ); + p->nWrLock = 0; + }else if( p->nWrLock ){ + rc = SQLITE_BUSY; + }else{ + p->nRdLock++; + } + }else{ + assert( eLock==SQLITE_LOCK_NONE ); + if( pThis->eLock>SQLITE_LOCK_SHARED ){ + assert( p->nWrLock==1 ); + p->nWrLock = 0; + } + assert( p->nRdLock>0 ); + p->nRdLock--; + } + if( rc==SQLITE_OK ) pThis->eLock = eLock; + memdbLeave(p); + return rc; +} + +#if 0 +/* +** This interface is only used for crash recovery, which does not +** occur on an in-memory database. +*/ +static int memdbCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + *pResOut = 0; + return SQLITE_OK; +} +#endif + + +/* +** File control method. For custom operations on an memdb-file. +*/ +static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){ + MemStore *p = ((MemFile*)pFile)->pStore; + int rc = SQLITE_NOTFOUND; + memdbEnter(p); + if( op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz); + rc = SQLITE_OK; + } + if( op==SQLITE_FCNTL_SIZE_LIMIT ){ + sqlite3_int64 iLimit = *(sqlite3_int64*)pArg; + if( iLimitsz ){ + if( iLimit<0 ){ + iLimit = p->szMax; + }else{ + iLimit = p->sz; + } + } + p->szMax = iLimit; + *(sqlite3_int64*)pArg = iLimit; + rc = SQLITE_OK; + } + memdbLeave(p); + return rc; +} + +#if 0 /* Not used because of SQLITE_IOCAP_POWERSAFE_OVERWRITE */ +/* +** Return the sector-size in bytes for an memdb-file. +*/ +static int memdbSectorSize(sqlite3_file *pFile){ + return 1024; +} +#endif + +/* +** Return the device characteristic flags supported by an memdb-file. +*/ +static int memdbDeviceCharacteristics(sqlite3_file *pFile){ + UNUSED_PARAMETER(pFile); + return SQLITE_IOCAP_ATOMIC | + SQLITE_IOCAP_POWERSAFE_OVERWRITE | + SQLITE_IOCAP_SAFE_APPEND | + SQLITE_IOCAP_SEQUENTIAL; +} + +/* Fetch a page of a memory-mapped file */ +static int memdbFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + MemStore *p = ((MemFile*)pFile)->pStore; + memdbEnter(p); + if( iOfst+iAmt>p->sz ){ + *pp = 0; + }else{ + p->nMmap++; + *pp = (void*)(p->aData + iOfst); + } + memdbLeave(p); + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int memdbUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + MemStore *p = ((MemFile*)pFile)->pStore; + UNUSED_PARAMETER(iOfst); + UNUSED_PARAMETER(pPage); + memdbEnter(p); + p->nMmap--; + memdbLeave(p); + return SQLITE_OK; +} + +/* +** Open an mem file handle. +*/ +static int memdbOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFd, + int flags, + int *pOutFlags +){ + MemFile *pFile = (MemFile*)pFd; + MemStore *p = 0; + int szName; + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ + return ORIGVFS(pVfs)->xOpen(ORIGVFS(pVfs), zName, pFd, flags, pOutFlags); + } + memset(pFile, 0, sizeof(*pFile)); + szName = sqlite3Strlen30(zName); + if( szName>1 && zName[0]=='/' ){ + int i; +#ifndef SQLITE_MUTEX_OMIT + sqlite3_mutex *pVfsMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#endif + sqlite3_mutex_enter(pVfsMutex); + for(i=0; izFName,zName)==0 ){ + p = memdb_g.apMemStore[i]; + break; + } + } + if( p==0 ){ + MemStore **apNew; + p = sqlite3Malloc( sizeof(*p) + szName + 3 ); + if( p==0 ){ + sqlite3_mutex_leave(pVfsMutex); + return SQLITE_NOMEM; + } + apNew = sqlite3Realloc(memdb_g.apMemStore, + sizeof(apNew[0])*(memdb_g.nMemStore+1) ); + if( apNew==0 ){ + sqlite3_free(p); + sqlite3_mutex_leave(pVfsMutex); + return SQLITE_NOMEM; + } + apNew[memdb_g.nMemStore++] = p; + memdb_g.apMemStore = apNew; + memset(p, 0, sizeof(*p)); + p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE|SQLITE_DESERIALIZE_FREEONCLOSE; + p->szMax = sqlite3GlobalConfig.mxMemdbSize; + p->zFName = (char*)&p[1]; + memcpy(p->zFName, zName, szName+1); + p->pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( p->pMutex==0 ){ + memdb_g.nMemStore--; + sqlite3_free(p); + sqlite3_mutex_leave(pVfsMutex); + return SQLITE_NOMEM; + } + p->nRef = 1; + memdbEnter(p); + }else{ + memdbEnter(p); + p->nRef++; + } + sqlite3_mutex_leave(pVfsMutex); + }else{ + p = sqlite3Malloc( sizeof(*p) ); + if( p==0 ){ + return SQLITE_NOMEM; + } + memset(p, 0, sizeof(*p)); + p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE; + p->szMax = sqlite3GlobalConfig.mxMemdbSize; + } + pFile->pStore = p; + assert( pOutFlags!=0 ); /* True because flags==SQLITE_OPEN_MAIN_DB */ + *pOutFlags = flags | SQLITE_OPEN_MEMORY; + pFd->pMethods = &memdb_io_methods; + memdbLeave(p); + return SQLITE_OK; +} + +#if 0 /* Only used to delete rollback journals, super-journals, and WAL + ** files, none of which exist in memdb. So this routine is never used */ +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int memdbDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return SQLITE_IOERR_DELETE; +} +#endif + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +** +** With memdb, no files ever exist on disk. So always return false. +*/ +static int memdbAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + UNUSED_PARAMETER(pVfs); + UNUSED_PARAMETER(zPath); + UNUSED_PARAMETER(flags); + *pResOut = 0; + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int memdbFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + UNUSED_PARAMETER(pVfs); + sqlite3_snprintf(nOut, zOut, "%s", zPath); + return SQLITE_OK; +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *memdbDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void memdbDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*memdbDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void memdbDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int memdbRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int memdbSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} + +#if 0 /* Never used. Modern cores only call xCurrentTimeInt64() */ +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int memdbCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +#endif + +static int memdbGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int memdbCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} + +/* +** Translate a database connection pointer and schema name into a +** MemFile pointer. +*/ +static MemFile *memdbFromDbSchema(sqlite3 *db, const char *zSchema){ + MemFile *p = 0; + MemStore *pStore; + int rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p); + if( rc ) return 0; + if( p->base.pMethods!=&memdb_io_methods ) return 0; + pStore = p->pStore; + memdbEnter(pStore); + if( pStore->zFName!=0 ) p = 0; + memdbLeave(pStore); + return p; +} + +/* +** Return the serialization of a database +*/ +SQLITE_API unsigned char *sqlite3_serialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which database within the connection */ + sqlite3_int64 *piSize, /* Write size here, if not NULL */ + unsigned int mFlags /* Maybe SQLITE_SERIALIZE_NOCOPY */ +){ + MemFile *p; + int iDb; + Btree *pBt; + sqlite3_int64 sz; + int szPage = 0; + sqlite3_stmt *pStmt = 0; + unsigned char *pOut; + char *zSql; + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + if( zSchema==0 ) zSchema = db->aDb[0].zDbSName; + p = memdbFromDbSchema(db, zSchema); + iDb = sqlite3FindDbName(db, zSchema); + if( piSize ) *piSize = -1; + if( iDb<0 ) return 0; + if( p ){ + MemStore *pStore = p->pStore; + assert( pStore->pMutex==0 ); + if( piSize ) *piSize = pStore->sz; + if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ + pOut = pStore->aData; + }else{ + pOut = sqlite3_malloc64( pStore->sz ); + if( pOut ) memcpy(pOut, pStore->aData, pStore->sz); + } + return pOut; + } + pBt = db->aDb[iDb].pBt; + if( pBt==0 ) return 0; + szPage = sqlite3BtreeGetPageSize(pBt); + zSql = sqlite3_mprintf("PRAGMA \"%w\".page_count", zSchema); + rc = zSql ? sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) : SQLITE_NOMEM; + sqlite3_free(zSql); + if( rc ) return 0; + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + pOut = 0; + }else{ + sz = sqlite3_column_int64(pStmt, 0)*szPage; + if( piSize ) *piSize = sz; + if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ + pOut = 0; + }else{ + pOut = sqlite3_malloc64( sz ); + if( pOut ){ + int nPage = sqlite3_column_int(pStmt, 0); + Pager *pPager = sqlite3BtreePager(pBt); + int pgno; + for(pgno=1; pgno<=nPage; pgno++){ + DbPage *pPage = 0; + unsigned char *pTo = pOut + szPage*(sqlite3_int64)(pgno-1); + rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pPage, 0); + if( rc==SQLITE_OK ){ + memcpy(pTo, sqlite3PagerGetData(pPage), szPage); + }else{ + memset(pTo, 0, szPage); + } + sqlite3PagerUnref(pPage); + } + } + } + } + sqlite3_finalize(pStmt); + return pOut; +} + +/* Convert zSchema to a MemDB and initialize its content. +*/ +SQLITE_API int sqlite3_deserialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which DB to reopen with the deserialization */ + unsigned char *pData, /* The serialized database content */ + sqlite3_int64 szDb, /* Number bytes in the deserialization */ + sqlite3_int64 szBuf, /* Total size of buffer pData[] */ + unsigned mFlags /* Zero or more SQLITE_DESERIALIZE_* flags */ +){ + MemFile *p; + char *zSql; + sqlite3_stmt *pStmt = 0; + int rc; + int iDb; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( szDb<0 ) return SQLITE_MISUSE_BKPT; + if( szBuf<0 ) return SQLITE_MISUSE_BKPT; +#endif + + sqlite3_mutex_enter(db->mutex); + if( zSchema==0 ) zSchema = db->aDb[0].zDbSName; + iDb = sqlite3FindDbName(db, zSchema); + testcase( iDb==1 ); + if( iDb<2 && iDb!=0 ){ + rc = SQLITE_ERROR; + goto end_deserialize; + } + zSql = sqlite3_mprintf("ATTACH x AS %Q", zSchema); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + if( rc ) goto end_deserialize; + db->init.iDb = (u8)iDb; + db->init.reopenMemdb = 1; + rc = sqlite3_step(pStmt); + db->init.reopenMemdb = 0; + if( rc!=SQLITE_DONE ){ + rc = SQLITE_ERROR; + goto end_deserialize; + } + p = memdbFromDbSchema(db, zSchema); + if( p==0 ){ + rc = SQLITE_ERROR; + }else{ + MemStore *pStore = p->pStore; + pStore->aData = pData; + pData = 0; + pStore->sz = szDb; + pStore->szAlloc = szBuf; + pStore->szMax = szBuf; + if( pStore->szMaxszMax = sqlite3GlobalConfig.mxMemdbSize; + } + pStore->mFlags = mFlags; + rc = SQLITE_OK; + } + +end_deserialize: + sqlite3_finalize(pStmt); + if( pData && (mFlags & SQLITE_DESERIALIZE_FREEONCLOSE)!=0 ){ + sqlite3_free(pData); + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +SQLITE_PRIVATE int sqlite3MemdbInit(void){ + sqlite3_vfs *pLower = sqlite3_vfs_find(0); + unsigned int sz; + if( NEVER(pLower==0) ) return SQLITE_ERROR; + sz = pLower->szOsFile; + memdb_vfs.pAppData = pLower; + /* The following conditional can only be true when compiled for + ** Windows x86 and SQLITE_MAX_MMAP_SIZE=0. We always leave + ** it in, to be safe, but it is marked as NO_TEST since there + ** is no way to reach it under most builds. */ + if( szBITVEC_NBIT and iDivisor==0 then Bitvec.u.aHash[] is +** a hash table that will hold up to BITVEC_MXHASH distinct values. +** +** Otherwise, the value i is redirected into one of BITVEC_NPTR +** sub-bitmaps pointed to by Bitvec.u.apSub[]. Each subbitmap +** handles up to iDivisor separate values of i. apSub[0] holds +** values between 1 and iDivisor. apSub[1] holds values between +** iDivisor+1 and 2*iDivisor. apSub[N] holds values between +** N*iDivisor+1 and (N+1)*iDivisor. Each subbitmap is normalized +** to hold deal with values between 1 and iDivisor. +*/ +struct Bitvec { + u32 iSize; /* Maximum bit index. Max iSize is 4,294,967,296. */ + u32 nSet; /* Number of bits that are set - only valid for aHash + ** element. Max is BITVEC_NINT. For BITVEC_SZ of 512, + ** this would be 125. */ + u32 iDivisor; /* Number of bits handled by each apSub[] entry. */ + /* Should >=0 for apSub element. */ + /* Max iDivisor is max(u32) / BITVEC_NPTR + 1. */ + /* For a BITVEC_SZ of 512, this would be 34,359,739. */ + union { + BITVEC_TELEM aBitmap[BITVEC_NELEM]; /* Bitmap representation */ + u32 aHash[BITVEC_NINT]; /* Hash table representation */ + Bitvec *apSub[BITVEC_NPTR]; /* Recursive representation */ + } u; +}; + +/* +** Create a new bitmap object able to handle bits between 0 and iSize, +** inclusive. Return a pointer to the new object. Return NULL if +** malloc fails. +*/ +SQLITE_PRIVATE Bitvec *sqlite3BitvecCreate(u32 iSize){ + Bitvec *p; + assert( sizeof(*p)==BITVEC_SZ ); + p = sqlite3MallocZero( sizeof(*p) ); + if( p ){ + p->iSize = iSize; + } + return p; +} + +/* +** Check to see if the i-th bit is set. Return true or false. +** If p is NULL (if the bitmap has not been created) or if +** i is out of range, then return false. +*/ +SQLITE_PRIVATE int sqlite3BitvecTestNotNull(Bitvec *p, u32 i){ + assert( p!=0 ); + i--; + if( i>=p->iSize ) return 0; + while( p->iDivisor ){ + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + p = p->u.apSub[bin]; + if (!p) { + return 0; + } + } + if( p->iSize<=BITVEC_NBIT ){ + return (p->u.aBitmap[i/BITVEC_SZELEM] & (1<<(i&(BITVEC_SZELEM-1))))!=0; + } else{ + u32 h = BITVEC_HASH(i++); + while( p->u.aHash[h] ){ + if( p->u.aHash[h]==i ) return 1; + h = (h+1) % BITVEC_NINT; + } + return 0; + } +} +SQLITE_PRIVATE int sqlite3BitvecTest(Bitvec *p, u32 i){ + return p!=0 && sqlite3BitvecTestNotNull(p,i); +} + +/* +** Set the i-th bit. Return 0 on success and an error code if +** anything goes wrong. +** +** This routine might cause sub-bitmaps to be allocated. Failing +** to get the memory needed to hold the sub-bitmap is the only +** that can go wrong with an insert, assuming p and i are valid. +** +** The calling function must ensure that p is a valid Bitvec object +** and that the value for "i" is within range of the Bitvec object. +** Otherwise the behavior is undefined. +*/ +SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec *p, u32 i){ + u32 h; + if( p==0 ) return SQLITE_OK; + assert( i>0 ); + assert( i<=p->iSize ); + i--; + while((p->iSize > BITVEC_NBIT) && p->iDivisor) { + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + if( p->u.apSub[bin]==0 ){ + p->u.apSub[bin] = sqlite3BitvecCreate( p->iDivisor ); + if( p->u.apSub[bin]==0 ) return SQLITE_NOMEM_BKPT; + } + p = p->u.apSub[bin]; + } + if( p->iSize<=BITVEC_NBIT ){ + p->u.aBitmap[i/BITVEC_SZELEM] |= 1 << (i&(BITVEC_SZELEM-1)); + return SQLITE_OK; + } + h = BITVEC_HASH(i++); + /* if there wasn't a hash collision, and this doesn't */ + /* completely fill the hash, then just add it without */ + /* worring about sub-dividing and re-hashing. */ + if( !p->u.aHash[h] ){ + if (p->nSet<(BITVEC_NINT-1)) { + goto bitvec_set_end; + } else { + goto bitvec_set_rehash; + } + } + /* there was a collision, check to see if it's already */ + /* in hash, if not, try to find a spot for it */ + do { + if( p->u.aHash[h]==i ) return SQLITE_OK; + h++; + if( h>=BITVEC_NINT ) h = 0; + } while( p->u.aHash[h] ); + /* we didn't find it in the hash. h points to the first */ + /* available free spot. check to see if this is going to */ + /* make our hash too "full". */ +bitvec_set_rehash: + if( p->nSet>=BITVEC_MXHASH ){ + unsigned int j; + int rc; + u32 *aiValues = sqlite3StackAllocRaw(0, sizeof(p->u.aHash)); + if( aiValues==0 ){ + return SQLITE_NOMEM_BKPT; + }else{ + memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); + memset(p->u.apSub, 0, sizeof(p->u.apSub)); + p->iDivisor = (p->iSize + BITVEC_NPTR - 1)/BITVEC_NPTR; + rc = sqlite3BitvecSet(p, i); + for(j=0; jnSet++; + p->u.aHash[h] = i; + return SQLITE_OK; +} + +/* +** Clear the i-th bit. +** +** pBuf must be a pointer to at least BITVEC_SZ bytes of temporary storage +** that BitvecClear can use to rebuilt its hash table. +*/ +SQLITE_PRIVATE void sqlite3BitvecClear(Bitvec *p, u32 i, void *pBuf){ + if( p==0 ) return; + assert( i>0 ); + i--; + while( p->iDivisor ){ + u32 bin = i/p->iDivisor; + i = i%p->iDivisor; + p = p->u.apSub[bin]; + if (!p) { + return; + } + } + if( p->iSize<=BITVEC_NBIT ){ + p->u.aBitmap[i/BITVEC_SZELEM] &= ~(1 << (i&(BITVEC_SZELEM-1))); + }else{ + unsigned int j; + u32 *aiValues = pBuf; + memcpy(aiValues, p->u.aHash, sizeof(p->u.aHash)); + memset(p->u.aHash, 0, sizeof(p->u.aHash)); + p->nSet = 0; + for(j=0; jnSet++; + while( p->u.aHash[h] ){ + h++; + if( h>=BITVEC_NINT ) h = 0; + } + p->u.aHash[h] = aiValues[j]; + } + } + } +} + +/* +** Destroy a bitmap object. Reclaim all memory used. +*/ +SQLITE_PRIVATE void sqlite3BitvecDestroy(Bitvec *p){ + if( p==0 ) return; + if( p->iDivisor ){ + unsigned int i; + for(i=0; iu.apSub[i]); + } + } + sqlite3_free(p); +} + +/* +** Return the value of the iSize parameter specified when Bitvec *p +** was created. +*/ +SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ + return p->iSize; +} + +#ifndef SQLITE_UNTESTABLE +/* +** Let V[] be an array of unsigned characters sufficient to hold +** up to N bits. Let I be an integer between 0 and N. 0<=I>3] |= (1<<(I&7)) +#define CLEARBIT(V,I) V[I>>3] &= ~(1<<(I&7)) +#define TESTBIT(V,I) (V[I>>3]&(1<<(I&7)))!=0 + +/* +** This routine runs an extensive test of the Bitvec code. +** +** The input is an array of integers that acts as a program +** to test the Bitvec. The integers are opcodes followed +** by 0, 1, or 3 operands, depending on the opcode. Another +** opcode follows immediately after the last operand. +** +** There are 6 opcodes numbered from 0 through 5. 0 is the +** "halt" opcode and causes the test to end. +** +** 0 Halt and return the number of errors +** 1 N S X Set N bits beginning with S and incrementing by X +** 2 N S X Clear N bits beginning with S and incrementing by X +** 3 N Set N randomly chosen bits +** 4 N Clear N randomly chosen bits +** 5 N S X Set N bits from S increment X in array only, not in bitvec +** +** The opcodes 1 through 4 perform set and clear operations are performed +** on both a Bitvec object and on a linear array of bits obtained from malloc. +** Opcode 5 works on the linear array only, not on the Bitvec. +** Opcode 5 is used to deliberately induce a fault in order to +** confirm that error detection works. +** +** At the conclusion of the test the linear array is compared +** against the Bitvec object. If there are any differences, +** an error is returned. If they are the same, zero is returned. +** +** If a memory allocation error occurs, return -1. +*/ +SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int sz, int *aOp){ + Bitvec *pBitvec = 0; + unsigned char *pV = 0; + int rc = -1; + int i, nx, pc, op; + void *pTmpSpace; + + /* Allocate the Bitvec to be tested and a linear array of + ** bits to act as the reference */ + pBitvec = sqlite3BitvecCreate( sz ); + pV = sqlite3MallocZero( (sz+7)/8 + 1 ); + pTmpSpace = sqlite3_malloc64(BITVEC_SZ); + if( pBitvec==0 || pV==0 || pTmpSpace==0 ) goto bitvec_end; + + /* NULL pBitvec tests */ + sqlite3BitvecSet(0, 1); + sqlite3BitvecClear(0, 1, pTmpSpace); + + /* Run the program */ + pc = 0; + while( (op = aOp[pc])!=0 ){ + switch( op ){ + case 1: + case 2: + case 5: { + nx = 4; + i = aOp[pc+2] - 1; + aOp[pc+2] += aOp[pc+3]; + break; + } + case 3: + case 4: + default: { + nx = 2; + sqlite3_randomness(sizeof(i), &i); + break; + } + } + if( (--aOp[pc+1]) > 0 ) nx = 0; + pc += nx; + i = (i & 0x7fffffff)%sz; + if( (op & 1)!=0 ){ + SETBIT(pV, (i+1)); + if( op!=5 ){ + if( sqlite3BitvecSet(pBitvec, i+1) ) goto bitvec_end; + } + }else{ + CLEARBIT(pV, (i+1)); + sqlite3BitvecClear(pBitvec, i+1, pTmpSpace); + } + } + + /* Test to make sure the linear array exactly matches the + ** Bitvec object. Start with the assumption that they do + ** match (rc==0). Change rc to non-zero if a discrepancy + ** is found. + */ + rc = sqlite3BitvecTest(0,0) + sqlite3BitvecTest(pBitvec, sz+1) + + sqlite3BitvecTest(pBitvec, 0) + + (sqlite3BitvecSize(pBitvec) - sz); + for(i=1; i<=sz; i++){ + if( (TESTBIT(pV,i))!=sqlite3BitvecTest(pBitvec,i) ){ + rc = i; + break; + } + } + + /* Free allocated structure */ +bitvec_end: + sqlite3_free(pTmpSpace); + sqlite3_free(pV); + sqlite3BitvecDestroy(pBitvec); + return rc; +} +#endif /* SQLITE_UNTESTABLE */ + +/************** End of bitvec.c **********************************************/ +/************** Begin file pcache.c ******************************************/ +/* +** 2008 August 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements that page cache. +*/ +/* #include "sqliteInt.h" */ + +/* +** A complete page cache is an instance of this structure. Every +** entry in the cache holds a single page of the database file. The +** btree layer only operates on the cached copy of the database pages. +** +** A page cache entry is "clean" if it exactly matches what is currently +** on disk. A page is "dirty" if it has been modified and needs to be +** persisted to disk. +** +** pDirty, pDirtyTail, pSynced: +** All dirty pages are linked into the doubly linked list using +** PgHdr.pDirtyNext and pDirtyPrev. The list is maintained in LRU order +** such that p was added to the list more recently than p->pDirtyNext. +** PCache.pDirty points to the first (newest) element in the list and +** pDirtyTail to the last (oldest). +** +** The PCache.pSynced variable is used to optimize searching for a dirty +** page to eject from the cache mid-transaction. It is better to eject +** a page that does not require a journal sync than one that does. +** Therefore, pSynced is maintained so that it *almost* always points +** to either the oldest page in the pDirty/pDirtyTail list that has a +** clear PGHDR_NEED_SYNC flag or to a page that is older than this one +** (so that the right page to eject can be found by following pDirtyPrev +** pointers). +*/ +struct PCache { + PgHdr *pDirty, *pDirtyTail; /* List of dirty pages in LRU order */ + PgHdr *pSynced; /* Last synced page in dirty page list */ + int nRefSum; /* Sum of ref counts over all pages */ + int szCache; /* Configured cache size */ + int szSpill; /* Size before spilling occurs */ + int szPage; /* Size of every page in this cache */ + int szExtra; /* Size of extra space for each page */ + u8 bPurgeable; /* True if pages are on backing store */ + u8 eCreate; /* eCreate value for for xFetch() */ + int (*xStress)(void*,PgHdr*); /* Call to try make a page clean */ + void *pStress; /* Argument to xStress */ + sqlite3_pcache *pCache; /* Pluggable cache module */ +}; + +/********************************** Test and Debug Logic **********************/ +/* +** Debug tracing macros. Enable by by changing the "0" to "1" and +** recompiling. +** +** When sqlite3PcacheTrace is 1, single line trace messages are issued. +** When sqlite3PcacheTrace is 2, a dump of the pcache showing all cache entries +** is displayed for many operations, resulting in a lot of output. +*/ +#if defined(SQLITE_DEBUG) && 0 + int sqlite3PcacheTrace = 2; /* 0: off 1: simple 2: cache dumps */ + int sqlite3PcacheMxDump = 9999; /* Max cache entries for pcacheDump() */ +# define pcacheTrace(X) if(sqlite3PcacheTrace){sqlite3DebugPrintf X;} + void pcacheDump(PCache *pCache){ + int N; + int i, j; + sqlite3_pcache_page *pLower; + PgHdr *pPg; + unsigned char *a; + + if( sqlite3PcacheTrace<2 ) return; + if( pCache->pCache==0 ) return; + N = sqlite3PcachePagecount(pCache); + if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump; + for(i=1; i<=N; i++){ + pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); + if( pLower==0 ) continue; + pPg = (PgHdr*)pLower->pExtra; + printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); + a = (unsigned char *)pLower->pBuf; + for(j=0; j<12; j++) printf("%02x", a[j]); + printf("\n"); + if( pPg->pPage==0 ){ + sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); + } + } + } + #else +# define pcacheTrace(X) +# define pcacheDump(X) +#endif + +/* +** Check invariants on a PgHdr entry. Return true if everything is OK. +** Return false if any invariant is violated. +** +** This routine is for use inside of assert() statements only. For +** example: +** +** assert( sqlite3PcachePageSanity(pPg) ); +*/ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr *pPg){ + PCache *pCache; + assert( pPg!=0 ); + assert( pPg->pgno>0 || pPg->pPager==0 ); /* Page number is 1 or more */ + pCache = pPg->pCache; + assert( pCache!=0 ); /* Every page has an associated PCache */ + if( pPg->flags & PGHDR_CLEAN ){ + assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ + assert( pCache->pDirty!=pPg ); /* CLEAN pages not on dirty list */ + assert( pCache->pDirtyTail!=pPg ); + } + /* WRITEABLE pages must also be DIRTY */ + if( pPg->flags & PGHDR_WRITEABLE ){ + assert( pPg->flags & PGHDR_DIRTY ); /* WRITEABLE implies DIRTY */ + } + /* NEED_SYNC can be set independently of WRITEABLE. This can happen, + ** for example, when using the sqlite3PagerDontWrite() optimization: + ** (1) Page X is journalled, and gets WRITEABLE and NEED_SEEK. + ** (2) Page X moved to freelist, WRITEABLE is cleared + ** (3) Page X reused, WRITEABLE is set again + ** If NEED_SYNC had been cleared in step 2, then it would not be reset + ** in step 3, and page might be written into the database without first + ** syncing the rollback journal, which might cause corruption on a power + ** loss. + ** + ** Another example is when the database page size is smaller than the + ** disk sector size. When any page of a sector is journalled, all pages + ** in that sector are marked NEED_SYNC even if they are still CLEAN, just + ** in case they are later modified, since all pages in the same sector + ** must be journalled and synced before any of those pages can be safely + ** written. + */ + return 1; +} +#endif /* SQLITE_DEBUG */ + + +/********************************** Linked List Management ********************/ + +/* Allowed values for second argument to pcacheManageDirtyList() */ +#define PCACHE_DIRTYLIST_REMOVE 1 /* Remove pPage from dirty list */ +#define PCACHE_DIRTYLIST_ADD 2 /* Add pPage to the dirty list */ +#define PCACHE_DIRTYLIST_FRONT 3 /* Move pPage to the front of the list */ + +/* +** Manage pPage's participation on the dirty list. Bits of the addRemove +** argument determines what operation to do. The 0x01 bit means first +** remove pPage from the dirty list. The 0x02 means add pPage back to +** the dirty list. Doing both moves pPage to the front of the dirty list. +*/ +static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ + PCache *p = pPage->pCache; + + pcacheTrace(("%p.DIRTYLIST.%s %d\n", p, + addRemove==1 ? "REMOVE" : addRemove==2 ? "ADD" : "FRONT", + pPage->pgno)); + if( addRemove & PCACHE_DIRTYLIST_REMOVE ){ + assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); + assert( pPage->pDirtyPrev || pPage==p->pDirty ); + + /* Update the PCache1.pSynced variable if necessary. */ + if( p->pSynced==pPage ){ + p->pSynced = pPage->pDirtyPrev; + } + + if( pPage->pDirtyNext ){ + pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; + }else{ + assert( pPage==p->pDirtyTail ); + p->pDirtyTail = pPage->pDirtyPrev; + } + if( pPage->pDirtyPrev ){ + pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; + }else{ + /* If there are now no dirty pages in the cache, set eCreate to 2. + ** This is an optimization that allows sqlite3PcacheFetch() to skip + ** searching for a dirty page to eject from the cache when it might + ** otherwise have to. */ + assert( pPage==p->pDirty ); + p->pDirty = pPage->pDirtyNext; + assert( p->bPurgeable || p->eCreate==2 ); + if( p->pDirty==0 ){ /*OPTIMIZATION-IF-TRUE*/ + assert( p->bPurgeable==0 || p->eCreate==1 ); + p->eCreate = 2; + } + } + } + if( addRemove & PCACHE_DIRTYLIST_ADD ){ + pPage->pDirtyPrev = 0; + pPage->pDirtyNext = p->pDirty; + if( pPage->pDirtyNext ){ + assert( pPage->pDirtyNext->pDirtyPrev==0 ); + pPage->pDirtyNext->pDirtyPrev = pPage; + }else{ + p->pDirtyTail = pPage; + if( p->bPurgeable ){ + assert( p->eCreate==2 ); + p->eCreate = 1; + } + } + p->pDirty = pPage; + + /* If pSynced is NULL and this page has a clear NEED_SYNC flag, set + ** pSynced to point to it. Checking the NEED_SYNC flag is an + ** optimization, as if pSynced points to a page with the NEED_SYNC + ** flag set sqlite3PcacheFetchStress() searches through all newer + ** entries of the dirty-list for a page with NEED_SYNC clear anyway. */ + if( !p->pSynced + && 0==(pPage->flags&PGHDR_NEED_SYNC) /*OPTIMIZATION-IF-FALSE*/ + ){ + p->pSynced = pPage; + } + } + pcacheDump(p); +} + +/* +** Wrapper around the pluggable caches xUnpin method. If the cache is +** being used for an in-memory database, this function is a no-op. +*/ +static void pcacheUnpin(PgHdr *p){ + if( p->pCache->bPurgeable ){ + pcacheTrace(("%p.UNPIN %d\n", p->pCache, p->pgno)); + sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0); + pcacheDump(p->pCache); + } +} + +/* +** Compute the number of pages of cache requested. p->szCache is the +** cache size requested by the "PRAGMA cache_size" statement. +*/ +static int numberOfCachePages(PCache *p){ + if( p->szCache>=0 ){ + /* IMPLEMENTATION-OF: R-42059-47211 If the argument N is positive then the + ** suggested cache size is set to N. */ + return p->szCache; + }else{ + i64 n; + /* IMPLEMANTATION-OF: R-59858-46238 If the argument N is negative, then the + ** number of cache pages is adjusted to be a number of pages that would + ** use approximately abs(N*1024) bytes of memory based on the current + ** page size. */ + n = ((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); + if( n>1000000000 ) n = 1000000000; + return (int)n; + } +} + +/*************************************************** General Interfaces ****** +** +** Initialize and shutdown the page cache subsystem. Neither of these +** functions are threadsafe. +*/ +SQLITE_PRIVATE int sqlite3PcacheInitialize(void){ + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ + /* IMPLEMENTATION-OF: R-26801-64137 If the xInit() method is NULL, then the + ** built-in default page cache is used instead of the application defined + ** page cache. */ + sqlite3PCacheSetDefault(); + assert( sqlite3GlobalConfig.pcache2.xInit!=0 ); + } + return sqlite3GlobalConfig.pcache2.xInit(sqlite3GlobalConfig.pcache2.pArg); +} +SQLITE_PRIVATE void sqlite3PcacheShutdown(void){ + if( sqlite3GlobalConfig.pcache2.xShutdown ){ + /* IMPLEMENTATION-OF: R-26000-56589 The xShutdown() method may be NULL. */ + sqlite3GlobalConfig.pcache2.xShutdown(sqlite3GlobalConfig.pcache2.pArg); + } +} + +/* +** Return the size in bytes of a PCache object. +*/ +SQLITE_PRIVATE int sqlite3PcacheSize(void){ return sizeof(PCache); } + +/* +** Create a new PCache object. Storage space to hold the object +** has already been allocated and is passed in as the p pointer. +** The caller discovers how much space needs to be allocated by +** calling sqlite3PcacheSize(). +** +** szExtra is some extra space allocated for each page. The first +** 8 bytes of the extra space will be zeroed as the page is allocated, +** but remaining content will be uninitialized. Though it is opaque +** to this module, the extra space really ends up being the MemPage +** structure in the pager. +*/ +SQLITE_PRIVATE int sqlite3PcacheOpen( + int szPage, /* Size of every page */ + int szExtra, /* Extra space associated with each page */ + int bPurgeable, /* True if pages are on backing store */ + int (*xStress)(void*,PgHdr*),/* Call to try to make pages clean */ + void *pStress, /* Argument to xStress */ + PCache *p /* Preallocated space for the PCache */ +){ + memset(p, 0, sizeof(PCache)); + p->szPage = 1; + p->szExtra = szExtra; + assert( szExtra>=8 ); /* First 8 bytes will be zeroed */ + p->bPurgeable = bPurgeable; + p->eCreate = 2; + p->xStress = xStress; + p->pStress = pStress; + p->szCache = 100; + p->szSpill = 1; + pcacheTrace(("%p.OPEN szPage %d bPurgeable %d\n",p,szPage,bPurgeable)); + return sqlite3PcacheSetPageSize(p, szPage); +} + +/* +** Change the page size for PCache object. The caller must ensure that there +** are no outstanding page references when this function is called. +*/ +SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ + assert( pCache->nRefSum==0 && pCache->pDirty==0 ); + if( pCache->szPage ){ + sqlite3_pcache *pNew; + pNew = sqlite3GlobalConfig.pcache2.xCreate( + szPage, pCache->szExtra + ROUND8(sizeof(PgHdr)), + pCache->bPurgeable + ); + if( pNew==0 ) return SQLITE_NOMEM_BKPT; + sqlite3GlobalConfig.pcache2.xCachesize(pNew, numberOfCachePages(pCache)); + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); + } + pCache->pCache = pNew; + pCache->szPage = szPage; + pcacheTrace(("%p.PAGESIZE %d\n",pCache,szPage)); + } + return SQLITE_OK; +} + +/* +** Try to obtain a page from the cache. +** +** This routine returns a pointer to an sqlite3_pcache_page object if +** such an object is already in cache, or if a new one is created. +** This routine returns a NULL pointer if the object was not in cache +** and could not be created. +** +** The createFlags should be 0 to check for existing pages and should +** be 3 (not 1, but 3) to try to create a new page. +** +** If the createFlag is 0, then NULL is always returned if the page +** is not already in the cache. If createFlag is 1, then a new page +** is created only if that can be done without spilling dirty pages +** and without exceeding the cache size limit. +** +** The caller needs to invoke sqlite3PcacheFetchFinish() to properly +** initialize the sqlite3_pcache_page object and convert it into a +** PgHdr object. The sqlite3PcacheFetch() and sqlite3PcacheFetchFinish() +** routines are split this way for performance reasons. When separated +** they can both (usually) operate without having to push values to +** the stack on entry and pop them back off on exit, which saves a +** lot of pushing and popping. +*/ +SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number to obtain */ + int createFlag /* If true, create page if it does not exist already */ +){ + int eCreate; + sqlite3_pcache_page *pRes; + + assert( pCache!=0 ); + assert( pCache->pCache!=0 ); + assert( createFlag==3 || createFlag==0 ); + assert( pCache->eCreate==((pCache->bPurgeable && pCache->pDirty) ? 1 : 2) ); + + /* eCreate defines what to do if the page does not exist. + ** 0 Do not allocate a new page. (createFlag==0) + ** 1 Allocate a new page if doing so is inexpensive. + ** (createFlag==1 AND bPurgeable AND pDirty) + ** 2 Allocate a new page even it doing so is difficult. + ** (createFlag==1 AND !(bPurgeable AND pDirty) + */ + eCreate = createFlag & pCache->eCreate; + assert( eCreate==0 || eCreate==1 || eCreate==2 ); + assert( createFlag==0 || pCache->eCreate==eCreate ); + assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); + pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); + pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno, + createFlag?" create":"",pRes)); + return pRes; +} + +/* +** If the sqlite3PcacheFetch() routine is unable to allocate a new +** page because no clean pages are available for reuse and the cache +** size limit has been reached, then this routine can be invoked to +** try harder to allocate a page. This routine might invoke the stress +** callback to spill dirty pages to the journal. It will then try to +** allocate the new page and will only fail to allocate a new page on +** an OOM error. +** +** This routine should be invoked only after sqlite3PcacheFetch() fails. +*/ +SQLITE_PRIVATE int sqlite3PcacheFetchStress( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number to obtain */ + sqlite3_pcache_page **ppPage /* Write result here */ +){ + PgHdr *pPg; + if( pCache->eCreate==2 ) return 0; + + if( sqlite3PcachePagecount(pCache)>pCache->szSpill ){ + /* Find a dirty page to write-out and recycle. First try to find a + ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC + ** cleared), but if that is not possible settle for any other + ** unreferenced dirty page. + ** + ** If the LRU page in the dirty list that has a clear PGHDR_NEED_SYNC + ** flag is currently referenced, then the following may leave pSynced + ** set incorrectly (pointing to other than the LRU page with NEED_SYNC + ** cleared). This is Ok, as pSynced is just an optimization. */ + for(pPg=pCache->pSynced; + pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); + pPg=pPg->pDirtyPrev + ); + pCache->pSynced = pPg; + if( !pPg ){ + for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); + } + if( pPg ){ + int rc; +#ifdef SQLITE_LOG_CACHE_SPILL + sqlite3_log(SQLITE_FULL, + "spill page %d making room for %d - cache used: %d/%d", + pPg->pgno, pgno, + sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache), + numberOfCachePages(pCache)); +#endif + pcacheTrace(("%p.SPILL %d\n",pCache,pPg->pgno)); + rc = pCache->xStress(pCache->pStress, pPg); + pcacheDump(pCache); + if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ + return rc; + } + } + } + *ppPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); + return *ppPage==0 ? SQLITE_NOMEM_BKPT : SQLITE_OK; +} + +/* +** This is a helper routine for sqlite3PcacheFetchFinish() +** +** In the uncommon case where the page being fetched has not been +** initialized, this routine is invoked to do the initialization. +** This routine is broken out into a separate function since it +** requires extra stack manipulation that can be avoided in the common +** case. +*/ +static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number obtained */ + sqlite3_pcache_page *pPage /* Page obtained by prior PcacheFetch() call */ +){ + PgHdr *pPgHdr; + assert( pPage!=0 ); + pPgHdr = (PgHdr*)pPage->pExtra; + assert( pPgHdr->pPage==0 ); + memset(&pPgHdr->pDirty, 0, sizeof(PgHdr) - offsetof(PgHdr,pDirty)); + pPgHdr->pPage = pPage; + pPgHdr->pData = pPage->pBuf; + pPgHdr->pExtra = (void *)&pPgHdr[1]; + memset(pPgHdr->pExtra, 0, 8); + pPgHdr->pCache = pCache; + pPgHdr->pgno = pgno; + pPgHdr->flags = PGHDR_CLEAN; + return sqlite3PcacheFetchFinish(pCache,pgno,pPage); +} + +/* +** This routine converts the sqlite3_pcache_page object returned by +** sqlite3PcacheFetch() into an initialized PgHdr object. This routine +** must be called after sqlite3PcacheFetch() in order to get a usable +** result. +*/ +SQLITE_PRIVATE PgHdr *sqlite3PcacheFetchFinish( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number obtained */ + sqlite3_pcache_page *pPage /* Page obtained by prior PcacheFetch() call */ +){ + PgHdr *pPgHdr; + + assert( pPage!=0 ); + pPgHdr = (PgHdr *)pPage->pExtra; + + if( !pPgHdr->pPage ){ + return pcacheFetchFinishWithInit(pCache, pgno, pPage); + } + pCache->nRefSum++; + pPgHdr->nRef++; + assert( sqlite3PcachePageSanity(pPgHdr) ); + return pPgHdr; +} + +/* +** Decrement the reference count on a page. If the page is clean and the +** reference count drops to 0, then it is made eligible for recycling. +*/ +SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ + assert( p->nRef>0 ); + p->pCache->nRefSum--; + if( (--p->nRef)==0 ){ + if( p->flags&PGHDR_CLEAN ){ + pcacheUnpin(p); + }else{ + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + } + } +} + +/* +** Increase the reference count of a supplied page by 1. +*/ +SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr *p){ + assert(p->nRef>0); + assert( sqlite3PcachePageSanity(p) ); + p->nRef++; + p->pCache->nRefSum++; +} + +/* +** Drop a page from the cache. There must be exactly one reference to the +** page. This function deletes that reference, so after it returns the +** page pointed to by p is invalid. +*/ +SQLITE_PRIVATE void sqlite3PcacheDrop(PgHdr *p){ + assert( p->nRef==1 ); + assert( sqlite3PcachePageSanity(p) ); + if( p->flags&PGHDR_DIRTY ){ + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); + } + p->pCache->nRefSum--; + sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 1); +} + +/* +** Make sure the page is marked as dirty. If it isn't dirty already, +** make it so. +*/ +SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ + assert( p->nRef>0 ); + assert( sqlite3PcachePageSanity(p) ); + if( p->flags & (PGHDR_CLEAN|PGHDR_DONT_WRITE) ){ /*OPTIMIZATION-IF-FALSE*/ + p->flags &= ~PGHDR_DONT_WRITE; + if( p->flags & PGHDR_CLEAN ){ + p->flags ^= (PGHDR_DIRTY|PGHDR_CLEAN); + pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno)); + assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); + } + assert( sqlite3PcachePageSanity(p) ); + } +} + +/* +** Make sure the page is marked as clean. If it isn't clean already, +** make it so. +*/ +SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr *p){ + assert( sqlite3PcachePageSanity(p) ); + assert( (p->flags & PGHDR_DIRTY)!=0 ); + assert( (p->flags & PGHDR_CLEAN)==0 ); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); + p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + p->flags |= PGHDR_CLEAN; + pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno)); + assert( sqlite3PcachePageSanity(p) ); + if( p->nRef==0 ){ + pcacheUnpin(p); + } +} + +/* +** Make every page in the cache clean. +*/ +SQLITE_PRIVATE void sqlite3PcacheCleanAll(PCache *pCache){ + PgHdr *p; + pcacheTrace(("%p.CLEAN-ALL\n",pCache)); + while( (p = pCache->pDirty)!=0 ){ + sqlite3PcacheMakeClean(p); + } +} + +/* +** Clear the PGHDR_NEED_SYNC and PGHDR_WRITEABLE flag from all dirty pages. +*/ +SQLITE_PRIVATE void sqlite3PcacheClearWritable(PCache *pCache){ + PgHdr *p; + pcacheTrace(("%p.CLEAR-WRITEABLE\n",pCache)); + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~(PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + } + pCache->pSynced = pCache->pDirtyTail; +} + +/* +** Clear the PGHDR_NEED_SYNC flag from all dirty pages. +*/ +SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *pCache){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + p->flags &= ~PGHDR_NEED_SYNC; + } + pCache->pSynced = pCache->pDirtyTail; +} + +/* +** Change the page number of page p to newPgno. +*/ +SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ + PCache *pCache = p->pCache; + assert( p->nRef>0 ); + assert( newPgno>0 ); + assert( sqlite3PcachePageSanity(p) ); + pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); + sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); + p->pgno = newPgno; + if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + } +} + +/* +** Drop every cache entry whose page number is greater than "pgno". The +** caller must ensure that there are no outstanding references to any pages +** other than page 1 with a page number greater than pgno. +** +** If there is a reference to page 1 and the pgno parameter passed to this +** function is 0, then the data area associated with page 1 is zeroed, but +** the page object is not dropped. +*/ +SQLITE_PRIVATE void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ + if( pCache->pCache ){ + PgHdr *p; + PgHdr *pNext; + pcacheTrace(("%p.TRUNCATE %d\n",pCache,pgno)); + for(p=pCache->pDirty; p; p=pNext){ + pNext = p->pDirtyNext; + /* This routine never gets call with a positive pgno except right + ** after sqlite3PcacheCleanAll(). So if there are dirty pages, + ** it must be that pgno==0. + */ + assert( p->pgno>0 ); + if( p->pgno>pgno ){ + assert( p->flags&PGHDR_DIRTY ); + sqlite3PcacheMakeClean(p); + } + } + if( pgno==0 && pCache->nRefSum ){ + sqlite3_pcache_page *pPage1; + pPage1 = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache,1,0); + if( ALWAYS(pPage1) ){ /* Page 1 is always available in cache, because + ** pCache->nRefSum>0 */ + memset(pPage1->pBuf, 0, pCache->szPage); + pgno = 1; + } + } + sqlite3GlobalConfig.pcache2.xTruncate(pCache->pCache, pgno+1); + } +} + +/* +** Close a cache. +*/ +SQLITE_PRIVATE void sqlite3PcacheClose(PCache *pCache){ + assert( pCache->pCache!=0 ); + pcacheTrace(("%p.CLOSE\n",pCache)); + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); +} + +/* +** Discard the contents of the cache. +*/ +SQLITE_PRIVATE void sqlite3PcacheClear(PCache *pCache){ + sqlite3PcacheTruncate(pCache, 0); +} + +/* +** Merge two lists of pages connected by pDirty and in pgno order. +** Do not bother fixing the pDirtyPrev pointers. +*/ +static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ + PgHdr result, *pTail; + pTail = &result; + assert( pA!=0 && pB!=0 ); + for(;;){ + if( pA->pgnopgno ){ + pTail->pDirty = pA; + pTail = pA; + pA = pA->pDirty; + if( pA==0 ){ + pTail->pDirty = pB; + break; + } + }else{ + pTail->pDirty = pB; + pTail = pB; + pB = pB->pDirty; + if( pB==0 ){ + pTail->pDirty = pA; + break; + } + } + } + return result.pDirty; +} + +/* +** Sort the list of pages in accending order by pgno. Pages are +** connected by pDirty pointers. The pDirtyPrev pointers are +** corrupted by this sort. +** +** Since there cannot be more than 2^31 distinct pages in a database, +** there cannot be more than 31 buckets required by the merge sorter. +** One extra bucket is added to catch overflow in case something +** ever changes to make the previous sentence incorrect. +*/ +#define N_SORT_BUCKET 32 +static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ + PgHdr *a[N_SORT_BUCKET], *p; + int i; + memset(a, 0, sizeof(a)); + while( pIn ){ + p = pIn; + pIn = p->pDirty; + p->pDirty = 0; + for(i=0; ALWAYS(ipDirty; p; p=p->pDirtyNext){ + p->pDirty = p->pDirtyNext; + } + return pcacheSortDirtyList(pCache->pDirty); +} + +/* +** Return the total number of references to all pages held by the cache. +** +** This is not the total number of pages referenced, but the sum of the +** reference count for all pages. +*/ +SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache *pCache){ + return pCache->nRefSum; +} + +/* +** Return the number of references to the page supplied as an argument. +*/ +SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr *p){ + return p->nRef; +} + +/* +** Return the total number of pages in the cache. +*/ +SQLITE_PRIVATE int sqlite3PcachePagecount(PCache *pCache){ + assert( pCache->pCache!=0 ); + return sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache); +} + +#ifdef SQLITE_TEST +/* +** Get the suggested cache-size value. +*/ +SQLITE_PRIVATE int sqlite3PcacheGetCachesize(PCache *pCache){ + return numberOfCachePages(pCache); +} +#endif + +/* +** Set the suggested cache-size value. +*/ +SQLITE_PRIVATE void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ + assert( pCache->pCache!=0 ); + pCache->szCache = mxPage; + sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache, + numberOfCachePages(pCache)); +} + +/* +** Set the suggested cache-spill value. Make no changes if if the +** argument is zero. Return the effective cache-spill size, which will +** be the larger of the szSpill and szCache. +*/ +SQLITE_PRIVATE int sqlite3PcacheSetSpillsize(PCache *p, int mxPage){ + int res; + assert( p->pCache!=0 ); + if( mxPage ){ + if( mxPage<0 ){ + mxPage = (int)((-1024*(i64)mxPage)/(p->szPage+p->szExtra)); + } + p->szSpill = mxPage; + } + res = numberOfCachePages(p); + if( resszSpill ) res = p->szSpill; + return res; +} + +/* +** Free up as much memory as possible from the page cache. +*/ +SQLITE_PRIVATE void sqlite3PcacheShrink(PCache *pCache){ + assert( pCache->pCache!=0 ); + sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); +} + +/* +** Return the size of the header added by this middleware layer +** in the page-cache hierarchy. +*/ +SQLITE_PRIVATE int sqlite3HeaderSizePcache(void){ return ROUND8(sizeof(PgHdr)); } + +/* +** Return the number of dirty pages currently in the cache, as a percentage +** of the configured cache size. +*/ +SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache *pCache){ + PgHdr *pDirty; + int nDirty = 0; + int nCache = numberOfCachePages(pCache); + for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext) nDirty++; + return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; +} + +#ifdef SQLITE_DIRECT_OVERFLOW_READ +/* +** Return true if there are one or more dirty pages in the cache. Else false. +*/ +SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache){ + return (pCache->pDirty!=0); +} +#endif + +#if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) +/* +** For all dirty pages currently in the cache, invoke the specified +** callback. This is only used if the SQLITE_CHECK_PAGES macro is +** defined. +*/ +SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)){ + PgHdr *pDirty; + for(pDirty=pCache->pDirty; pDirty; pDirty=pDirty->pDirtyNext){ + xIter(pDirty); + } +} +#endif + +/************** End of pcache.c **********************************************/ +/************** Begin file pcache1.c *****************************************/ +/* +** 2008 November 05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements the default page cache implementation (the +** sqlite3_pcache interface). It also contains part of the implementation +** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. +** If the default page cache implementation is overridden, then neither of +** these two features are available. +** +** A Page cache line looks like this: +** +** ------------------------------------------------------------- +** | database page content | PgHdr1 | MemPage | PgHdr | +** ------------------------------------------------------------- +** +** The database page content is up front (so that buffer overreads tend to +** flow harmlessly into the PgHdr1, MemPage, and PgHdr extensions). MemPage +** is the extension added by the btree.c module containing information such +** as the database page number and how that database page is used. PgHdr +** is added by the pcache.c layer and contains information used to keep track +** of which pages are "dirty". PgHdr1 is an extension added by this +** module (pcache1.c). The PgHdr1 header is a subclass of sqlite3_pcache_page. +** PgHdr1 contains information needed to look up a page by its page number. +** The superclass sqlite3_pcache_page.pBuf points to the start of the +** database page content and sqlite3_pcache_page.pExtra points to PgHdr. +** +** The size of the extension (MemPage+PgHdr+PgHdr1) can be determined at +** runtime using sqlite3_config(SQLITE_CONFIG_PCACHE_HDRSZ, &size). The +** sizes of the extensions sum to 272 bytes on x64 for 3.8.10, but this +** size can vary according to architecture, compile-time options, and +** SQLite library version number. +** +** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained +** using a separate memory allocation from the database page content. This +** seeks to overcome the "clownshoe" problem (also called "internal +** fragmentation" in academic literature) of allocating a few bytes more +** than a power of two with the memory allocator rounding up to the next +** power of two, and leaving the rounded-up space unused. +** +** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates +** with this module. Information is passed back and forth as PgHdr1 pointers. +** +** The pcache.c and pager.c modules deal pointers to PgHdr objects. +** The btree.c module deals with pointers to MemPage objects. +** +** SOURCE OF PAGE CACHE MEMORY: +** +** Memory for a page might come from any of three sources: +** +** (1) The general-purpose memory allocator - sqlite3Malloc() +** (2) Global page-cache memory provided using sqlite3_config() with +** SQLITE_CONFIG_PAGECACHE. +** (3) PCache-local bulk allocation. +** +** The third case is a chunk of heap memory (defaulting to 100 pages worth) +** that is allocated when the page cache is created. The size of the local +** bulk allocation can be adjusted using +** +** sqlite3_config(SQLITE_CONFIG_PAGECACHE, (void*)0, 0, N). +** +** If N is positive, then N pages worth of memory are allocated using a single +** sqlite3Malloc() call and that memory is used for the first N pages allocated. +** Or if N is negative, then -1024*N bytes of memory are allocated and used +** for as many pages as can be accomodated. +** +** Only one of (2) or (3) can be used. Once the memory available to (2) or +** (3) is exhausted, subsequent allocations fail over to the general-purpose +** memory allocator (1). +** +** Earlier versions of SQLite used only methods (1) and (2). But experiments +** show that method (3) with N==100 provides about a 5% performance boost for +** common workloads. +*/ +/* #include "sqliteInt.h" */ + +typedef struct PCache1 PCache1; +typedef struct PgHdr1 PgHdr1; +typedef struct PgFreeslot PgFreeslot; +typedef struct PGroup PGroup; + +/* +** Each cache entry is represented by an instance of the following +** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of +** PgHdr1.pCache->szPage bytes is allocated directly before this structure +** in memory. +** +** Note: Variables isBulkLocal and isAnchor were once type "u8". That works, +** but causes a 2-byte gap in the structure for most architectures (since +** pointers must be either 4 or 8-byte aligned). As this structure is located +** in memory directly after the associated page data, if the database is +** corrupt, code at the b-tree layer may overread the page buffer and +** read part of this structure before the corruption is detected. This +** can cause a valgrind error if the unitialized gap is accessed. Using u16 +** ensures there is no such gap, and therefore no bytes of unitialized memory +** in the structure. +*/ +struct PgHdr1 { + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u16 isBulkLocal; /* This page from bulk local storage */ + u16 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + /* NB: pLruPrev is only valid if pLruNext!=0 */ +}; + +/* +** A page is pinned if it is not on the LRU list. To be "pinned" means +** that the page is in active use and must not be deallocated. +*/ +#define PAGE_IS_PINNED(p) ((p)->pLruNext==0) +#define PAGE_IS_UNPINNED(p) ((p)->pLruNext!=0) + +/* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set +** of one or more PCaches that are able to recycle each other's unpinned +** pages when they are under memory pressure. A PGroup is an instance of +** the following object. +** +** This page cache implementation works in one of two modes: +** +** (1) Every PCache is the sole member of its own PGroup. There is +** one PGroup per PCache. +** +** (2) There is a single global PGroup that all PCaches are a member +** of. +** +** Mode 1 uses more memory (since PCache instances are not able to rob +** unused pages from other PCaches) but it also operates without a mutex, +** and is therefore often faster. Mode 2 requires a mutex in order to be +** threadsafe, but recycles pages more efficiently. +** +** For mode (1), PGroup.mutex is NULL. For mode (2) there is only a single +** PGroup which is the pcache1.grp global variable and its mutex is +** SQLITE_MUTEX_STATIC_LRU. +*/ +struct PGroup { + sqlite3_mutex *mutex; /* MUTEX_STATIC_LRU or NULL */ + unsigned int nMaxPage; /* Sum of nMax for purgeable caches */ + unsigned int nMinPage; /* Sum of nMin for purgeable caches */ + unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ + unsigned int nPurgeable; /* Number of purgeable pages allocated */ + PgHdr1 lru; /* The beginning and end of the LRU list */ +}; + +/* Each page cache is an instance of the following object. Every +** open database file (including each in-memory database and each +** temporary or transient database) has a single page cache which +** is an instance of this object. +** +** Pointers to structures of this type are cast and returned as +** opaque sqlite3_pcache* handles. +*/ +struct PCache1 { + /* Cache configuration parameters. Page size (szPage) and the purgeable + ** flag (bPurgeable) and the pnPurgeable pointer are all set when the + ** cache is created and are never changed thereafter. nMax may be + ** modified at any time by a call to the pcache1Cachesize() method. + ** The PGroup mutex must be held when accessing nMax. + */ + PGroup *pGroup; /* PGroup this cache belongs to */ + unsigned int *pnPurgeable; /* Pointer to pGroup->nPurgeable */ + int szPage; /* Size of database content section */ + int szExtra; /* sizeof(MemPage)+sizeof(PgHdr) */ + int szAlloc; /* Total size of one pcache line */ + int bPurgeable; /* True if cache is purgeable */ + unsigned int nMin; /* Minimum number of pages reserved */ + unsigned int nMax; /* Configured "cache_size" value */ + unsigned int n90pct; /* nMax*9/10 */ + unsigned int iMaxKey; /* Largest key seen since xTruncate() */ + unsigned int nPurgeableDummy; /* pnPurgeable points here when not used*/ + + /* Hash table of all pages. The following variables may only be accessed + ** when the accessor is holding the PGroup mutex. + */ + unsigned int nRecyclable; /* Number of pages in the LRU list */ + unsigned int nPage; /* Total number of pages in apHash */ + unsigned int nHash; /* Number of slots in apHash[] */ + PgHdr1 **apHash; /* Hash table for fast lookup by key */ + PgHdr1 *pFree; /* List of unused pcache-local pages */ + void *pBulk; /* Bulk memory used by pcache-local */ +}; + +/* +** Free slots in the allocator used to divide up the global page cache +** buffer provided using the SQLITE_CONFIG_PAGECACHE mechanism. +*/ +struct PgFreeslot { + PgFreeslot *pNext; /* Next free slot */ +}; + +/* +** Global data used by this cache. +*/ +static SQLITE_WSD struct PCacheGlobal { + PGroup grp; /* The global PGroup for mode (2) */ + + /* Variables related to SQLITE_CONFIG_PAGECACHE settings. The + ** szSlot, nSlot, pStart, pEnd, nReserve, and isInit values are all + ** fixed at sqlite3_initialize() time and do not require mutex protection. + ** The nFreeSlot and pFree values do require mutex protection. + */ + int isInit; /* True if initialized */ + int separateCache; /* Use a new PGroup for each PCache */ + int nInitPage; /* Initial bulk allocation size */ + int szSlot; /* Size of each free slot */ + int nSlot; /* The number of pcache slots */ + int nReserve; /* Try to keep nFreeSlot above this */ + void *pStart, *pEnd; /* Bounds of global page cache memory */ + /* Above requires no mutex. Use mutex below for variable that follow. */ + sqlite3_mutex *mutex; /* Mutex for accessing the following: */ + PgFreeslot *pFree; /* Free page blocks */ + int nFreeSlot; /* Number of unused pcache slots */ + /* The following value requires a mutex to change. We skip the mutex on + ** reading because (1) most platforms read a 32-bit integer atomically and + ** (2) even if an incorrect value is read, no great harm is done since this + ** is really just an optimization. */ + int bUnderPressure; /* True if low on PAGECACHE memory */ +} pcache1_g; + +/* +** All code in this file should access the global structure above via the +** alias "pcache1". This ensures that the WSD emulation is used when +** compiling for systems that do not support real WSD. +*/ +#define pcache1 (GLOBAL(struct PCacheGlobal, pcache1_g)) + +/* +** Macros to enter and leave the PCache LRU mutex. +*/ +#if !defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 +# define pcache1EnterMutex(X) assert((X)->mutex==0) +# define pcache1LeaveMutex(X) assert((X)->mutex==0) +# define PCACHE1_MIGHT_USE_GROUP_MUTEX 0 +#else +# define pcache1EnterMutex(X) sqlite3_mutex_enter((X)->mutex) +# define pcache1LeaveMutex(X) sqlite3_mutex_leave((X)->mutex) +# define PCACHE1_MIGHT_USE_GROUP_MUTEX 1 +#endif + +/******************************************************************************/ +/******** Page Allocation/SQLITE_CONFIG_PCACHE Related Functions **************/ + + +/* +** This function is called during initialization if a static buffer is +** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE +** verb to sqlite3_config(). Parameter pBuf points to an allocation large +** enough to contain 'n' buffers of 'sz' bytes each. +** +** This routine is called from sqlite3_initialize() and so it is guaranteed +** to be serialized already. There is no need for further mutexing. +*/ +SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ + if( pcache1.isInit ){ + PgFreeslot *p; + if( pBuf==0 ) sz = n = 0; + if( n==0 ) sz = 0; + sz = ROUNDDOWN8(sz); + pcache1.szSlot = sz; + pcache1.nSlot = pcache1.nFreeSlot = n; + pcache1.nReserve = n>90 ? 10 : (n/10 + 1); + pcache1.pStart = pBuf; + pcache1.pFree = 0; + pcache1.bUnderPressure = 0; + while( n-- ){ + p = (PgFreeslot*)pBuf; + p->pNext = pcache1.pFree; + pcache1.pFree = p; + pBuf = (void*)&((char*)pBuf)[sz]; + } + pcache1.pEnd = pBuf; + } +} + +/* +** Try to initialize the pCache->pFree and pCache->pBulk fields. Return +** true if pCache->pFree ends up containing one or more free pages. +*/ +static int pcache1InitBulk(PCache1 *pCache){ + i64 szBulk; + char *zBulk; + if( pcache1.nInitPage==0 ) return 0; + /* Do not bother with a bulk allocation if the cache size very small */ + if( pCache->nMax<3 ) return 0; + sqlite3BeginBenignMalloc(); + if( pcache1.nInitPage>0 ){ + szBulk = pCache->szAlloc * (i64)pcache1.nInitPage; + }else{ + szBulk = -1024 * (i64)pcache1.nInitPage; + } + if( szBulk > pCache->szAlloc*(i64)pCache->nMax ){ + szBulk = pCache->szAlloc*(i64)pCache->nMax; + } + zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); + sqlite3EndBenignMalloc(); + if( zBulk ){ + int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; + do{ + PgHdr1 *pX = (PgHdr1*)&zBulk[pCache->szPage]; + pX->page.pBuf = zBulk; + pX->page.pExtra = &pX[1]; + pX->isBulkLocal = 1; + pX->isAnchor = 0; + pX->pNext = pCache->pFree; + pX->pLruPrev = 0; /* Initializing this saves a valgrind error */ + pCache->pFree = pX; + zBulk += pCache->szAlloc; + }while( --nBulk ); + } + return pCache->pFree!=0; +} + +/* +** Malloc function used within this file to allocate space from the buffer +** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no +** such buffer exists or there is no space left in it, this function falls +** back to sqlite3Malloc(). +** +** Multiple threads can run this routine at the same time. Global variables +** in pcache1 need to be protected via mutex. +*/ +static void *pcache1Alloc(int nByte){ + void *p = 0; + assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); + if( nByte<=pcache1.szSlot ){ + sqlite3_mutex_enter(pcache1.mutex); + p = (PgHdr1 *)pcache1.pFree; + if( p ){ + pcache1.pFree = pcache1.pFree->pNext; + pcache1.nFreeSlot--; + pcache1.bUnderPressure = pcache1.nFreeSlot=0 ); + sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_USED, 1); + } + sqlite3_mutex_leave(pcache1.mutex); + } + if( p==0 ){ + /* Memory is not available in the SQLITE_CONFIG_PAGECACHE pool. Get + ** it from sqlite3Malloc instead. + */ + p = sqlite3Malloc(nByte); +#ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS + if( p ){ + int sz = sqlite3MallocSize(p); + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); + sqlite3_mutex_leave(pcache1.mutex); + } +#endif + sqlite3MemdebugSetType(p, MEMTYPE_PCACHE); + } + return p; +} + +/* +** Free an allocated buffer obtained from pcache1Alloc(). +*/ +static void pcache1Free(void *p){ + if( p==0 ) return; + if( SQLITE_WITHIN(p, pcache1.pStart, pcache1.pEnd) ){ + PgFreeslot *pSlot; + sqlite3_mutex_enter(pcache1.mutex); + sqlite3StatusDown(SQLITE_STATUS_PAGECACHE_USED, 1); + pSlot = (PgFreeslot*)p; + pSlot->pNext = pcache1.pFree; + pcache1.pFree = pSlot; + pcache1.nFreeSlot++; + pcache1.bUnderPressure = pcache1.nFreeSlot=pcache1.pStart && ppGroup->mutex) ); + if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ + assert( pCache->pFree!=0 ); + p = pCache->pFree; + pCache->pFree = p->pNext; + p->pNext = 0; + }else{ +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + /* The group mutex must be released before pcache1Alloc() is called. This + ** is because it might call sqlite3_release_memory(), which assumes that + ** this mutex is not held. */ + assert( pcache1.separateCache==0 ); + assert( pCache->pGroup==&pcache1.grp ); + pcache1LeaveMutex(pCache->pGroup); +#endif + if( benignMalloc ){ sqlite3BeginBenignMalloc(); } +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + pPg = pcache1Alloc(pCache->szPage); + p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); + if( !pPg || !p ){ + pcache1Free(pPg); + sqlite3_free(p); + pPg = 0; + } +#else + pPg = pcache1Alloc(pCache->szAlloc); +#endif + if( benignMalloc ){ sqlite3EndBenignMalloc(); } +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + pcache1EnterMutex(pCache->pGroup); +#endif + if( pPg==0 ) return 0; +#ifndef SQLITE_PCACHE_SEPARATE_HEADER + p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; +#endif + p->page.pBuf = pPg; + p->page.pExtra = &p[1]; + p->isBulkLocal = 0; + p->isAnchor = 0; + p->pLruPrev = 0; /* Initializing this saves a valgrind error */ + } + (*pCache->pnPurgeable)++; + return p; +} + +/* +** Free a page object allocated by pcache1AllocPage(). +*/ +static void pcache1FreePage(PgHdr1 *p){ + PCache1 *pCache; + assert( p!=0 ); + pCache = p->pCache; + assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); + if( p->isBulkLocal ){ + p->pNext = pCache->pFree; + pCache->pFree = p; + }else{ + pcache1Free(p->page.pBuf); +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + sqlite3_free(p); +#endif + } + (*pCache->pnPurgeable)--; +} + +/* +** Malloc function used by SQLite to obtain space from the buffer configured +** using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no such buffer +** exists, this function falls back to sqlite3Malloc(). +*/ +SQLITE_PRIVATE void *sqlite3PageMalloc(int sz){ + assert( sz<=65536+8 ); /* These allocations are never very large */ + return pcache1Alloc(sz); +} + +/* +** Free an allocated buffer obtained from sqlite3PageMalloc(). +*/ +SQLITE_PRIVATE void sqlite3PageFree(void *p){ + pcache1Free(p); +} + + +/* +** Return true if it desirable to avoid allocating a new page cache +** entry. +** +** If memory was allocated specifically to the page cache using +** SQLITE_CONFIG_PAGECACHE but that memory has all been used, then +** it is desirable to avoid allocating a new page cache entry because +** presumably SQLITE_CONFIG_PAGECACHE was suppose to be sufficient +** for all page cache needs and we should not need to spill the +** allocation onto the heap. +** +** Or, the heap is used for all page cache memory but the heap is +** under memory pressure, then again it is desirable to avoid +** allocating a new page cache entry in order to avoid stressing +** the heap even further. +*/ +static int pcache1UnderMemoryPressure(PCache1 *pCache){ + if( pcache1.nSlot && (pCache->szPage+pCache->szExtra)<=pcache1.szSlot ){ + return pcache1.bUnderPressure; + }else{ + return sqlite3HeapNearlyFull(); + } +} + +/******************************************************************************/ +/******** General Implementation Functions ************************************/ + +/* +** This function is used to resize the hash table used by the cache passed +** as the first argument. +** +** The PCache mutex must be held when this function is called. +*/ +static void pcache1ResizeHash(PCache1 *p){ + PgHdr1 **apNew; + unsigned int nNew; + unsigned int i; + + assert( sqlite3_mutex_held(p->pGroup->mutex) ); + + nNew = p->nHash*2; + if( nNew<256 ){ + nNew = 256; + } + + pcache1LeaveMutex(p->pGroup); + if( p->nHash ){ sqlite3BeginBenignMalloc(); } + apNew = (PgHdr1 **)sqlite3MallocZero(sizeof(PgHdr1 *)*nNew); + if( p->nHash ){ sqlite3EndBenignMalloc(); } + pcache1EnterMutex(p->pGroup); + if( apNew ){ + for(i=0; inHash; i++){ + PgHdr1 *pPage; + PgHdr1 *pNext = p->apHash[i]; + while( (pPage = pNext)!=0 ){ + unsigned int h = pPage->iKey % nNew; + pNext = pPage->pNext; + pPage->pNext = apNew[h]; + apNew[h] = pPage; + } + } + sqlite3_free(p->apHash); + p->apHash = apNew; + p->nHash = nNew; + } +} + +/* +** This function is used internally to remove the page pPage from the +** PGroup LRU list, if is part of it. If pPage is not part of the PGroup +** LRU list, then this function is a no-op. +** +** The PGroup mutex must be held when this function is called. +*/ +static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ + assert( pPage!=0 ); + assert( PAGE_IS_UNPINNED(pPage) ); + assert( pPage->pLruNext ); + assert( pPage->pLruPrev ); + assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) ); + pPage->pLruPrev->pLruNext = pPage->pLruNext; + pPage->pLruNext->pLruPrev = pPage->pLruPrev; + pPage->pLruNext = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */ + assert( pPage->isAnchor==0 ); + assert( pPage->pCache->pGroup->lru.isAnchor==1 ); + pPage->pCache->nRecyclable--; + return pPage; +} + + +/* +** Remove the page supplied as an argument from the hash table +** (PCache1.apHash structure) that it is currently stored in. +** Also free the page if freePage is true. +** +** The PGroup mutex must be held when this function is called. +*/ +static void pcache1RemoveFromHash(PgHdr1 *pPage, int freeFlag){ + unsigned int h; + PCache1 *pCache = pPage->pCache; + PgHdr1 **pp; + + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + h = pPage->iKey % pCache->nHash; + for(pp=&pCache->apHash[h]; (*pp)!=pPage; pp=&(*pp)->pNext); + *pp = (*pp)->pNext; + + pCache->nPage--; + if( freeFlag ) pcache1FreePage(pPage); +} + +/* +** If there are currently more than nMaxPage pages allocated, try +** to recycle pages to reduce the number allocated to nMaxPage. +*/ +static void pcache1EnforceMaxPage(PCache1 *pCache){ + PGroup *pGroup = pCache->pGroup; + PgHdr1 *p; + assert( sqlite3_mutex_held(pGroup->mutex) ); + while( pGroup->nPurgeable>pGroup->nMaxPage + && (p=pGroup->lru.pLruPrev)->isAnchor==0 + ){ + assert( p->pCache->pGroup==pGroup ); + assert( PAGE_IS_UNPINNED(p) ); + pcache1PinPage(p); + pcache1RemoveFromHash(p, 1); + } + if( pCache->nPage==0 && pCache->pBulk ){ + sqlite3_free(pCache->pBulk); + pCache->pBulk = pCache->pFree = 0; + } +} + +/* +** Discard all pages from cache pCache with a page number (key value) +** greater than or equal to iLimit. Any pinned pages that meet this +** criteria are unpinned before they are discarded. +** +** The PCache mutex must be held when this function is called. +*/ +static void pcache1TruncateUnsafe( + PCache1 *pCache, /* The cache to truncate */ + unsigned int iLimit /* Drop pages with this pgno or larger */ +){ + TESTONLY( int nPage = 0; ) /* To assert pCache->nPage is correct */ + unsigned int h, iStop; + assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + assert( pCache->iMaxKey >= iLimit ); + assert( pCache->nHash > 0 ); + if( pCache->iMaxKey - iLimit < pCache->nHash ){ + /* If we are just shaving the last few pages off the end of the + ** cache, then there is no point in scanning the entire hash table. + ** Only scan those hash slots that might contain pages that need to + ** be removed. */ + h = iLimit % pCache->nHash; + iStop = pCache->iMaxKey % pCache->nHash; + TESTONLY( nPage = -10; ) /* Disable the pCache->nPage validity check */ + }else{ + /* This is the general case where many pages are being removed. + ** It is necessary to scan the entire hash table */ + h = pCache->nHash/2; + iStop = h - 1; + } + for(;;){ + PgHdr1 **pp; + PgHdr1 *pPage; + assert( hnHash ); + pp = &pCache->apHash[h]; + while( (pPage = *pp)!=0 ){ + if( pPage->iKey>=iLimit ){ + pCache->nPage--; + *pp = pPage->pNext; + if( PAGE_IS_UNPINNED(pPage) ) pcache1PinPage(pPage); + pcache1FreePage(pPage); + }else{ + pp = &pPage->pNext; + TESTONLY( if( nPage>=0 ) nPage++; ) + } + } + if( h==iStop ) break; + h = (h+1) % pCache->nHash; + } + assert( nPage<0 || pCache->nPage==(unsigned)nPage ); +} + +/******************************************************************************/ +/******** sqlite3_pcache Methods **********************************************/ + +/* +** Implementation of the sqlite3_pcache.xInit method. +*/ +static int pcache1Init(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + assert( pcache1.isInit==0 ); + memset(&pcache1, 0, sizeof(pcache1)); + + + /* + ** The pcache1.separateCache variable is true if each PCache has its own + ** private PGroup (mode-1). pcache1.separateCache is false if the single + ** PGroup in pcache1.grp is used for all page caches (mode-2). + ** + ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT + ** + ** * Use a unified cache in single-threaded applications that have + ** configured a start-time buffer for use as page-cache memory using + ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL + ** pBuf argument. + ** + ** * Otherwise use separate caches (mode-1) + */ +#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) + pcache1.separateCache = 0; +#elif SQLITE_THREADSAFE + pcache1.separateCache = sqlite3GlobalConfig.pPage==0 + || sqlite3GlobalConfig.bCoreMutex>0; +#else + pcache1.separateCache = sqlite3GlobalConfig.pPage==0; +#endif + +#if SQLITE_THREADSAFE + if( sqlite3GlobalConfig.bCoreMutex ){ + pcache1.grp.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_LRU); + pcache1.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PMEM); + } +#endif + if( pcache1.separateCache + && sqlite3GlobalConfig.nPage!=0 + && sqlite3GlobalConfig.pPage==0 + ){ + pcache1.nInitPage = sqlite3GlobalConfig.nPage; + }else{ + pcache1.nInitPage = 0; + } + pcache1.grp.mxPinned = 10; + pcache1.isInit = 1; + return SQLITE_OK; +} + +/* +** Implementation of the sqlite3_pcache.xShutdown method. +** Note that the static mutex allocated in xInit does +** not need to be freed. +*/ +static void pcache1Shutdown(void *NotUsed){ + UNUSED_PARAMETER(NotUsed); + assert( pcache1.isInit!=0 ); + memset(&pcache1, 0, sizeof(pcache1)); +} + +/* forward declaration */ +static void pcache1Destroy(sqlite3_pcache *p); + +/* +** Implementation of the sqlite3_pcache.xCreate method. +** +** Allocate a new cache. +*/ +static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ + PCache1 *pCache; /* The newly created page cache */ + PGroup *pGroup; /* The group the new page cache will belong to */ + int sz; /* Bytes of memory required to allocate the new cache */ + + assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); + assert( szExtra < 300 ); + + sz = sizeof(PCache1) + sizeof(PGroup)*pcache1.separateCache; + pCache = (PCache1 *)sqlite3MallocZero(sz); + if( pCache ){ + if( pcache1.separateCache ){ + pGroup = (PGroup*)&pCache[1]; + pGroup->mxPinned = 10; + }else{ + pGroup = &pcache1.grp; + } + pcache1EnterMutex(pGroup); + if( pGroup->lru.isAnchor==0 ){ + pGroup->lru.isAnchor = 1; + pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; + } + pCache->pGroup = pGroup; + pCache->szPage = szPage; + pCache->szExtra = szExtra; + pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); + pCache->bPurgeable = (bPurgeable ? 1 : 0); + pcache1ResizeHash(pCache); + if( bPurgeable ){ + pCache->nMin = 10; + pGroup->nMinPage += pCache->nMin; + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pCache->pnPurgeable = &pGroup->nPurgeable; + }else{ + pCache->pnPurgeable = &pCache->nPurgeableDummy; + } + pcache1LeaveMutex(pGroup); + if( pCache->nHash==0 ){ + pcache1Destroy((sqlite3_pcache*)pCache); + pCache = 0; + } + } + return (sqlite3_pcache *)pCache; +} + +/* +** Implementation of the sqlite3_pcache.xCachesize method. +** +** Configure the cache_size limit for a cache. +*/ +static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ + PCache1 *pCache = (PCache1 *)p; + u32 n; + assert( nMax>=0 ); + if( pCache->bPurgeable ){ + PGroup *pGroup = pCache->pGroup; + pcache1EnterMutex(pGroup); + n = (u32)nMax; + if( n > 0x7fff0000 - pGroup->nMaxPage + pCache->nMax ){ + n = 0x7fff0000 - pGroup->nMaxPage + pCache->nMax; + } + pGroup->nMaxPage += (n - pCache->nMax); + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pCache->nMax = n; + pCache->n90pct = pCache->nMax*9/10; + pcache1EnforceMaxPage(pCache); + pcache1LeaveMutex(pGroup); + } +} + +/* +** Implementation of the sqlite3_pcache.xShrink method. +** +** Free up as much memory as possible. +*/ +static void pcache1Shrink(sqlite3_pcache *p){ + PCache1 *pCache = (PCache1*)p; + if( pCache->bPurgeable ){ + PGroup *pGroup = pCache->pGroup; + unsigned int savedMaxPage; + pcache1EnterMutex(pGroup); + savedMaxPage = pGroup->nMaxPage; + pGroup->nMaxPage = 0; + pcache1EnforceMaxPage(pCache); + pGroup->nMaxPage = savedMaxPage; + pcache1LeaveMutex(pGroup); + } +} + +/* +** Implementation of the sqlite3_pcache.xPagecount method. +*/ +static int pcache1Pagecount(sqlite3_pcache *p){ + int n; + PCache1 *pCache = (PCache1*)p; + pcache1EnterMutex(pCache->pGroup); + n = pCache->nPage; + pcache1LeaveMutex(pCache->pGroup); + return n; +} + + +/* +** Implement steps 3, 4, and 5 of the pcache1Fetch() algorithm described +** in the header of the pcache1Fetch() procedure. +** +** This steps are broken out into a separate procedure because they are +** usually not needed, and by avoiding the stack initialization required +** for these steps, the main pcache1Fetch() procedure can run faster. +*/ +static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( + PCache1 *pCache, + unsigned int iKey, + int createFlag +){ + unsigned int nPinned; + PGroup *pGroup = pCache->pGroup; + PgHdr1 *pPage = 0; + + /* Step 3: Abort if createFlag is 1 but the cache is nearly full */ + assert( pCache->nPage >= pCache->nRecyclable ); + nPinned = pCache->nPage - pCache->nRecyclable; + assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage ); + assert( pCache->n90pct == pCache->nMax*9/10 ); + if( createFlag==1 && ( + nPinned>=pGroup->mxPinned + || nPinned>=pCache->n90pct + || (pcache1UnderMemoryPressure(pCache) && pCache->nRecyclablenPage>=pCache->nHash ) pcache1ResizeHash(pCache); + assert( pCache->nHash>0 && pCache->apHash ); + + /* Step 4. Try to recycle a page. */ + if( pCache->bPurgeable + && !pGroup->lru.pLruPrev->isAnchor + && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache)) + ){ + PCache1 *pOther; + pPage = pGroup->lru.pLruPrev; + assert( PAGE_IS_UNPINNED(pPage) ); + pcache1RemoveFromHash(pPage, 0); + pcache1PinPage(pPage); + pOther = pPage->pCache; + if( pOther->szAlloc != pCache->szAlloc ){ + pcache1FreePage(pPage); + pPage = 0; + }else{ + pGroup->nPurgeable -= (pOther->bPurgeable - pCache->bPurgeable); + } + } + + /* Step 5. If a usable page buffer has still not been found, + ** attempt to allocate a new one. + */ + if( !pPage ){ + pPage = pcache1AllocPage(pCache, createFlag==1); + } + + if( pPage ){ + unsigned int h = iKey % pCache->nHash; + pCache->nPage++; + pPage->iKey = iKey; + pPage->pNext = pCache->apHash[h]; + pPage->pCache = pCache; + pPage->pLruNext = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */ + *(void **)pPage->page.pExtra = 0; + pCache->apHash[h] = pPage; + if( iKey>pCache->iMaxKey ){ + pCache->iMaxKey = iKey; + } + } + return pPage; +} + +/* +** Implementation of the sqlite3_pcache.xFetch method. +** +** Fetch a page by key value. +** +** Whether or not a new page may be allocated by this function depends on +** the value of the createFlag argument. 0 means do not allocate a new +** page. 1 means allocate a new page if space is easily available. 2 +** means to try really hard to allocate a new page. +** +** For a non-purgeable cache (a cache used as the storage for an in-memory +** database) there is really no difference between createFlag 1 and 2. So +** the calling function (pcache.c) will never have a createFlag of 1 on +** a non-purgeable cache. +** +** There are three different approaches to obtaining space for a page, +** depending on the value of parameter createFlag (which may be 0, 1 or 2). +** +** 1. Regardless of the value of createFlag, the cache is searched for a +** copy of the requested page. If one is found, it is returned. +** +** 2. If createFlag==0 and the page is not already in the cache, NULL is +** returned. +** +** 3. If createFlag is 1, and the page is not already in the cache, then +** return NULL (do not allocate a new page) if any of the following +** conditions are true: +** +** (a) the number of pages pinned by the cache is greater than +** PCache1.nMax, or +** +** (b) the number of pages pinned by the cache is greater than +** the sum of nMax for all purgeable caches, less the sum of +** nMin for all other purgeable caches, or +** +** 4. If none of the first three conditions apply and the cache is marked +** as purgeable, and if one of the following is true: +** +** (a) The number of pages allocated for the cache is already +** PCache1.nMax, or +** +** (b) The number of pages allocated for all purgeable caches is +** already equal to or greater than the sum of nMax for all +** purgeable caches, +** +** (c) The system is under memory pressure and wants to avoid +** unnecessary pages cache entry allocations +** +** then attempt to recycle a page from the LRU list. If it is the right +** size, return the recycled buffer. Otherwise, free the buffer and +** proceed to step 5. +** +** 5. Otherwise, allocate and return a new page buffer. +** +** There are two versions of this routine. pcache1FetchWithMutex() is +** the general case. pcache1FetchNoMutex() is a faster implementation for +** the common case where pGroup->mutex is NULL. The pcache1Fetch() wrapper +** invokes the appropriate routine. +*/ +static PgHdr1 *pcache1FetchNoMutex( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = 0; + + /* Step 1: Search the hash table for an existing entry. */ + pPage = pCache->apHash[iKey % pCache->nHash]; + while( pPage && pPage->iKey!=iKey ){ pPage = pPage->pNext; } + + /* Step 2: If the page was found in the hash table, then return it. + ** If the page was not in the hash table and createFlag is 0, abort. + ** Otherwise (page not in hash and createFlag!=0) continue with + ** subsequent steps to try to create the page. */ + if( pPage ){ + if( PAGE_IS_UNPINNED(pPage) ){ + return pcache1PinPage(pPage); + }else{ + return pPage; + } + }else if( createFlag ){ + /* Steps 3, 4, and 5 implemented by this subroutine */ + return pcache1FetchStage2(pCache, iKey, createFlag); + }else{ + return 0; + } +} +#if PCACHE1_MIGHT_USE_GROUP_MUTEX +static PgHdr1 *pcache1FetchWithMutex( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage; + + pcache1EnterMutex(pCache->pGroup); + pPage = pcache1FetchNoMutex(p, iKey, createFlag); + assert( pPage==0 || pCache->iMaxKey>=iKey ); + pcache1LeaveMutex(pCache->pGroup); + return pPage; +} +#endif +static sqlite3_pcache_page *pcache1Fetch( + sqlite3_pcache *p, + unsigned int iKey, + int createFlag +){ +#if PCACHE1_MIGHT_USE_GROUP_MUTEX || defined(SQLITE_DEBUG) + PCache1 *pCache = (PCache1 *)p; +#endif + + assert( offsetof(PgHdr1,page)==0 ); + assert( pCache->bPurgeable || createFlag!=1 ); + assert( pCache->bPurgeable || pCache->nMin==0 ); + assert( pCache->bPurgeable==0 || pCache->nMin==10 ); + assert( pCache->nMin==0 || pCache->bPurgeable ); + assert( pCache->nHash>0 ); +#if PCACHE1_MIGHT_USE_GROUP_MUTEX + if( pCache->pGroup->mutex ){ + return (sqlite3_pcache_page*)pcache1FetchWithMutex(p, iKey, createFlag); + }else +#endif + { + return (sqlite3_pcache_page*)pcache1FetchNoMutex(p, iKey, createFlag); + } +} + + +/* +** Implementation of the sqlite3_pcache.xUnpin method. +** +** Mark a page as unpinned (eligible for asynchronous recycling). +*/ +static void pcache1Unpin( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + int reuseUnlikely +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = (PgHdr1 *)pPg; + PGroup *pGroup = pCache->pGroup; + + assert( pPage->pCache==pCache ); + pcache1EnterMutex(pGroup); + + /* It is an error to call this function if the page is already + ** part of the PGroup LRU list. + */ + assert( pPage->pLruNext==0 ); + assert( PAGE_IS_PINNED(pPage) ); + + if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){ + pcache1RemoveFromHash(pPage, 1); + }else{ + /* Add the page to the PGroup LRU list. */ + PgHdr1 **ppFirst = &pGroup->lru.pLruNext; + pPage->pLruPrev = &pGroup->lru; + (pPage->pLruNext = *ppFirst)->pLruPrev = pPage; + *ppFirst = pPage; + pCache->nRecyclable++; + } + + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xRekey method. +*/ +static void pcache1Rekey( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + unsigned int iOld, + unsigned int iNew +){ + PCache1 *pCache = (PCache1 *)p; + PgHdr1 *pPage = (PgHdr1 *)pPg; + PgHdr1 **pp; + unsigned int h; + assert( pPage->iKey==iOld ); + assert( pPage->pCache==pCache ); + + pcache1EnterMutex(pCache->pGroup); + + h = iOld%pCache->nHash; + pp = &pCache->apHash[h]; + while( (*pp)!=pPage ){ + pp = &(*pp)->pNext; + } + *pp = pPage->pNext; + + h = iNew%pCache->nHash; + pPage->iKey = iNew; + pPage->pNext = pCache->apHash[h]; + pCache->apHash[h] = pPage; + if( iNew>pCache->iMaxKey ){ + pCache->iMaxKey = iNew; + } + + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xTruncate method. +** +** Discard all unpinned pages in the cache with a page number equal to +** or greater than parameter iLimit. Any pinned pages with a page number +** equal to or greater than iLimit are implicitly unpinned. +*/ +static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ + PCache1 *pCache = (PCache1 *)p; + pcache1EnterMutex(pCache->pGroup); + if( iLimit<=pCache->iMaxKey ){ + pcache1TruncateUnsafe(pCache, iLimit); + pCache->iMaxKey = iLimit-1; + } + pcache1LeaveMutex(pCache->pGroup); +} + +/* +** Implementation of the sqlite3_pcache.xDestroy method. +** +** Destroy a cache allocated using pcache1Create(). +*/ +static void pcache1Destroy(sqlite3_pcache *p){ + PCache1 *pCache = (PCache1 *)p; + PGroup *pGroup = pCache->pGroup; + assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); + pcache1EnterMutex(pGroup); + if( pCache->nPage ) pcache1TruncateUnsafe(pCache, 0); + assert( pGroup->nMaxPage >= pCache->nMax ); + pGroup->nMaxPage -= pCache->nMax; + assert( pGroup->nMinPage >= pCache->nMin ); + pGroup->nMinPage -= pCache->nMin; + pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pcache1EnforceMaxPage(pCache); + pcache1LeaveMutex(pGroup); + sqlite3_free(pCache->pBulk); + sqlite3_free(pCache->apHash); + sqlite3_free(pCache); +} + +/* +** This function is called during initialization (sqlite3_initialize()) to +** install the default pluggable cache module, assuming the user has not +** already provided an alternative. +*/ +SQLITE_PRIVATE void sqlite3PCacheSetDefault(void){ + static const sqlite3_pcache_methods2 defaultMethods = { + 1, /* iVersion */ + 0, /* pArg */ + pcache1Init, /* xInit */ + pcache1Shutdown, /* xShutdown */ + pcache1Create, /* xCreate */ + pcache1Cachesize, /* xCachesize */ + pcache1Pagecount, /* xPagecount */ + pcache1Fetch, /* xFetch */ + pcache1Unpin, /* xUnpin */ + pcache1Rekey, /* xRekey */ + pcache1Truncate, /* xTruncate */ + pcache1Destroy, /* xDestroy */ + pcache1Shrink /* xShrink */ + }; + sqlite3_config(SQLITE_CONFIG_PCACHE2, &defaultMethods); +} + +/* +** Return the size of the header on each page of this PCACHE implementation. +*/ +SQLITE_PRIVATE int sqlite3HeaderSizePcache1(void){ return ROUND8(sizeof(PgHdr1)); } + +/* +** Return the global mutex used by this PCACHE implementation. The +** sqlite3_status() routine needs access to this mutex. +*/ +SQLITE_PRIVATE sqlite3_mutex *sqlite3Pcache1Mutex(void){ + return pcache1.mutex; +} + +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT +/* +** This function is called to free superfluous dynamically allocated memory +** held by the pager system. Memory in use by any SQLite pager allocated +** by the current thread may be sqlite3_free()ed. +** +** nReq is the number of bytes of memory required. Once this much has +** been released, the function returns. The return value is the total number +** of bytes of memory released. +*/ +SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ + int nFree = 0; + assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); + assert( sqlite3_mutex_notheld(pcache1.mutex) ); + if( sqlite3GlobalConfig.pPage==0 ){ + PgHdr1 *p; + pcache1EnterMutex(&pcache1.grp); + while( (nReq<0 || nFreeisAnchor==0 + ){ + nFree += pcache1MemSize(p->page.pBuf); +#ifdef SQLITE_PCACHE_SEPARATE_HEADER + nFree += sqlite3MemSize(p); +#endif + assert( PAGE_IS_UNPINNED(p) ); + pcache1PinPage(p); + pcache1RemoveFromHash(p, 1); + } + pcache1LeaveMutex(&pcache1.grp); + } + return nFree; +} +#endif /* SQLITE_ENABLE_MEMORY_MANAGEMENT */ + +#ifdef SQLITE_TEST +/* +** This function is used by test procedures to inspect the internal state +** of the global cache. +*/ +SQLITE_PRIVATE void sqlite3PcacheStats( + int *pnCurrent, /* OUT: Total number of pages cached */ + int *pnMax, /* OUT: Global maximum cache size */ + int *pnMin, /* OUT: Sum of PCache1.nMin for purgeable caches */ + int *pnRecyclable /* OUT: Total number of pages available for recycling */ +){ + PgHdr1 *p; + int nRecyclable = 0; + for(p=pcache1.grp.lru.pLruNext; p && !p->isAnchor; p=p->pLruNext){ + assert( PAGE_IS_UNPINNED(p) ); + nRecyclable++; + } + *pnCurrent = pcache1.grp.nPurgeable; + *pnMax = (int)pcache1.grp.nMaxPage; + *pnMin = (int)pcache1.grp.nMinPage; + *pnRecyclable = nRecyclable; +} +#endif + +/************** End of pcache1.c *********************************************/ +/************** Begin file rowset.c ******************************************/ +/* +** 2008 December 3 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This module implements an object we call a "RowSet". +** +** The RowSet object is a collection of rowids. Rowids +** are inserted into the RowSet in an arbitrary order. Inserts +** can be intermixed with tests to see if a given rowid has been +** previously inserted into the RowSet. +** +** After all inserts are finished, it is possible to extract the +** elements of the RowSet in sorted order. Once this extraction +** process has started, no new elements may be inserted. +** +** Hence, the primitive operations for a RowSet are: +** +** CREATE +** INSERT +** TEST +** SMALLEST +** DESTROY +** +** The CREATE and DESTROY primitives are the constructor and destructor, +** obviously. The INSERT primitive adds a new element to the RowSet. +** TEST checks to see if an element is already in the RowSet. SMALLEST +** extracts the least value from the RowSet. +** +** The INSERT primitive might allocate additional memory. Memory is +** allocated in chunks so most INSERTs do no allocation. There is an +** upper bound on the size of allocated memory. No memory is freed +** until DESTROY. +** +** The TEST primitive includes a "batch" number. The TEST primitive +** will only see elements that were inserted before the last change +** in the batch number. In other words, if an INSERT occurs between +** two TESTs where the TESTs have the same batch nubmer, then the +** value added by the INSERT will not be visible to the second TEST. +** The initial batch number is zero, so if the very first TEST contains +** a non-zero batch number, it will see all prior INSERTs. +** +** No INSERTs may occurs after a SMALLEST. An assertion will fail if +** that is attempted. +** +** The cost of an INSERT is roughly constant. (Sometimes new memory +** has to be allocated on an INSERT.) The cost of a TEST with a new +** batch number is O(NlogN) where N is the number of elements in the RowSet. +** The cost of a TEST using the same batch number is O(logN). The cost +** of the first SMALLEST is O(NlogN). Second and subsequent SMALLEST +** primitives are constant time. The cost of DESTROY is O(N). +** +** TEST and SMALLEST may not be used by the same RowSet. This used to +** be possible, but the feature was not used, so it was removed in order +** to simplify the code. +*/ +/* #include "sqliteInt.h" */ + + +/* +** Target size for allocation chunks. +*/ +#define ROWSET_ALLOCATION_SIZE 1024 + +/* +** The number of rowset entries per allocation chunk. +*/ +#define ROWSET_ENTRY_PER_CHUNK \ + ((ROWSET_ALLOCATION_SIZE-8)/sizeof(struct RowSetEntry)) + +/* +** Each entry in a RowSet is an instance of the following object. +** +** This same object is reused to store a linked list of trees of RowSetEntry +** objects. In that alternative use, pRight points to the next entry +** in the list, pLeft points to the tree, and v is unused. The +** RowSet.pForest value points to the head of this forest list. +*/ +struct RowSetEntry { + i64 v; /* ROWID value for this entry */ + struct RowSetEntry *pRight; /* Right subtree (larger entries) or list */ + struct RowSetEntry *pLeft; /* Left subtree (smaller entries) */ +}; + +/* +** RowSetEntry objects are allocated in large chunks (instances of the +** following structure) to reduce memory allocation overhead. The +** chunks are kept on a linked list so that they can be deallocated +** when the RowSet is destroyed. +*/ +struct RowSetChunk { + struct RowSetChunk *pNextChunk; /* Next chunk on list of them all */ + struct RowSetEntry aEntry[ROWSET_ENTRY_PER_CHUNK]; /* Allocated entries */ +}; + +/* +** A RowSet in an instance of the following structure. +** +** A typedef of this structure if found in sqliteInt.h. +*/ +struct RowSet { + struct RowSetChunk *pChunk; /* List of all chunk allocations */ + sqlite3 *db; /* The database connection */ + struct RowSetEntry *pEntry; /* List of entries using pRight */ + struct RowSetEntry *pLast; /* Last entry on the pEntry list */ + struct RowSetEntry *pFresh; /* Source of new entry objects */ + struct RowSetEntry *pForest; /* List of binary trees of entries */ + u16 nFresh; /* Number of objects on pFresh */ + u16 rsFlags; /* Various flags */ + int iBatch; /* Current insert batch */ +}; + +/* +** Allowed values for RowSet.rsFlags +*/ +#define ROWSET_SORTED 0x01 /* True if RowSet.pEntry is sorted */ +#define ROWSET_NEXT 0x02 /* True if sqlite3RowSetNext() has been called */ + +/* +** Allocate a RowSet object. Return NULL if a memory allocation +** error occurs. +*/ +SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db){ + RowSet *p = sqlite3DbMallocRawNN(db, sizeof(*p)); + if( p ){ + int N = sqlite3DbMallocSize(db, p); + p->pChunk = 0; + p->db = db; + p->pEntry = 0; + p->pLast = 0; + p->pForest = 0; + p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); + p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); + p->rsFlags = ROWSET_SORTED; + p->iBatch = 0; + } + return p; +} + +/* +** Deallocate all chunks from a RowSet. This frees all memory that +** the RowSet has allocated over its lifetime. This routine is +** the destructor for the RowSet. +*/ +SQLITE_PRIVATE void sqlite3RowSetClear(void *pArg){ + RowSet *p = (RowSet*)pArg; + struct RowSetChunk *pChunk, *pNextChunk; + for(pChunk=p->pChunk; pChunk; pChunk = pNextChunk){ + pNextChunk = pChunk->pNextChunk; + sqlite3DbFree(p->db, pChunk); + } + p->pChunk = 0; + p->nFresh = 0; + p->pEntry = 0; + p->pLast = 0; + p->pForest = 0; + p->rsFlags = ROWSET_SORTED; +} + +/* +** Deallocate all chunks from a RowSet. This frees all memory that +** the RowSet has allocated over its lifetime. This routine is +** the destructor for the RowSet. +*/ +SQLITE_PRIVATE void sqlite3RowSetDelete(void *pArg){ + sqlite3RowSetClear(pArg); + sqlite3DbFree(((RowSet*)pArg)->db, pArg); +} + +/* +** Allocate a new RowSetEntry object that is associated with the +** given RowSet. Return a pointer to the new and completely uninitialized +** object. +** +** In an OOM situation, the RowSet.db->mallocFailed flag is set and this +** routine returns NULL. +*/ +static struct RowSetEntry *rowSetEntryAlloc(RowSet *p){ + assert( p!=0 ); + if( p->nFresh==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* We could allocate a fresh RowSetEntry each time one is needed, but it + ** is more efficient to pull a preallocated entry from the pool */ + struct RowSetChunk *pNew; + pNew = sqlite3DbMallocRawNN(p->db, sizeof(*pNew)); + if( pNew==0 ){ + return 0; + } + pNew->pNextChunk = p->pChunk; + p->pChunk = pNew; + p->pFresh = pNew->aEntry; + p->nFresh = ROWSET_ENTRY_PER_CHUNK; + } + p->nFresh--; + return p->pFresh++; +} + +/* +** Insert a new value into a RowSet. +** +** The mallocFailed flag of the database connection is set if a +** memory allocation fails. +*/ +SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet *p, i64 rowid){ + struct RowSetEntry *pEntry; /* The new entry */ + struct RowSetEntry *pLast; /* The last prior entry */ + + /* This routine is never called after sqlite3RowSetNext() */ + assert( p!=0 && (p->rsFlags & ROWSET_NEXT)==0 ); + + pEntry = rowSetEntryAlloc(p); + if( pEntry==0 ) return; + pEntry->v = rowid; + pEntry->pRight = 0; + pLast = p->pLast; + if( pLast ){ + if( rowid<=pLast->v ){ /*OPTIMIZATION-IF-FALSE*/ + /* Avoid unnecessary sorts by preserving the ROWSET_SORTED flags + ** where possible */ + p->rsFlags &= ~ROWSET_SORTED; + } + pLast->pRight = pEntry; + }else{ + p->pEntry = pEntry; + } + p->pLast = pEntry; +} + +/* +** Merge two lists of RowSetEntry objects. Remove duplicates. +** +** The input lists are connected via pRight pointers and are +** assumed to each already be in sorted order. +*/ +static struct RowSetEntry *rowSetEntryMerge( + struct RowSetEntry *pA, /* First sorted list to be merged */ + struct RowSetEntry *pB /* Second sorted list to be merged */ +){ + struct RowSetEntry head; + struct RowSetEntry *pTail; + + pTail = &head; + assert( pA!=0 && pB!=0 ); + for(;;){ + assert( pA->pRight==0 || pA->v<=pA->pRight->v ); + assert( pB->pRight==0 || pB->v<=pB->pRight->v ); + if( pA->v<=pB->v ){ + if( pA->vv ) pTail = pTail->pRight = pA; + pA = pA->pRight; + if( pA==0 ){ + pTail->pRight = pB; + break; + } + }else{ + pTail = pTail->pRight = pB; + pB = pB->pRight; + if( pB==0 ){ + pTail->pRight = pA; + break; + } + } + } + return head.pRight; +} + +/* +** Sort all elements on the list of RowSetEntry objects into order of +** increasing v. +*/ +static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ + unsigned int i; + struct RowSetEntry *pNext, *aBucket[40]; + + memset(aBucket, 0, sizeof(aBucket)); + while( pIn ){ + pNext = pIn->pRight; + pIn->pRight = 0; + for(i=0; aBucket[i]; i++){ + pIn = rowSetEntryMerge(aBucket[i], pIn); + aBucket[i] = 0; + } + aBucket[i] = pIn; + pIn = pNext; + } + pIn = aBucket[0]; + for(i=1; ipLeft ){ + struct RowSetEntry *p; + rowSetTreeToList(pIn->pLeft, ppFirst, &p); + p->pRight = pIn; + }else{ + *ppFirst = pIn; + } + if( pIn->pRight ){ + rowSetTreeToList(pIn->pRight, &pIn->pRight, ppLast); + }else{ + *ppLast = pIn; + } + assert( (*ppLast)->pRight==0 ); +} + + +/* +** Convert a sorted list of elements (connected by pRight) into a binary +** tree with depth of iDepth. A depth of 1 means the tree contains a single +** node taken from the head of *ppList. A depth of 2 means a tree with +** three nodes. And so forth. +** +** Use as many entries from the input list as required and update the +** *ppList to point to the unused elements of the list. If the input +** list contains too few elements, then construct an incomplete tree +** and leave *ppList set to NULL. +** +** Return a pointer to the root of the constructed binary tree. +*/ +static struct RowSetEntry *rowSetNDeepTree( + struct RowSetEntry **ppList, + int iDepth +){ + struct RowSetEntry *p; /* Root of the new tree */ + struct RowSetEntry *pLeft; /* Left subtree */ + if( *ppList==0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Prevent unnecessary deep recursion when we run out of entries */ + return 0; + } + if( iDepth>1 ){ /*OPTIMIZATION-IF-TRUE*/ + /* This branch causes a *balanced* tree to be generated. A valid tree + ** is still generated without this branch, but the tree is wildly + ** unbalanced and inefficient. */ + pLeft = rowSetNDeepTree(ppList, iDepth-1); + p = *ppList; + if( p==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* It is safe to always return here, but the resulting tree + ** would be unbalanced */ + return pLeft; + } + p->pLeft = pLeft; + *ppList = p->pRight; + p->pRight = rowSetNDeepTree(ppList, iDepth-1); + }else{ + p = *ppList; + *ppList = p->pRight; + p->pLeft = p->pRight = 0; + } + return p; +} + +/* +** Convert a sorted list of elements into a binary tree. Make the tree +** as deep as it needs to be in order to contain the entire list. +*/ +static struct RowSetEntry *rowSetListToTree(struct RowSetEntry *pList){ + int iDepth; /* Depth of the tree so far */ + struct RowSetEntry *p; /* Current tree root */ + struct RowSetEntry *pLeft; /* Left subtree */ + + assert( pList!=0 ); + p = pList; + pList = p->pRight; + p->pLeft = p->pRight = 0; + for(iDepth=1; pList; iDepth++){ + pLeft = p; + p = pList; + pList = p->pRight; + p->pLeft = pLeft; + p->pRight = rowSetNDeepTree(&pList, iDepth); + } + return p; +} + +/* +** Extract the smallest element from the RowSet. +** Write the element into *pRowid. Return 1 on success. Return +** 0 if the RowSet is already empty. +** +** After this routine has been called, the sqlite3RowSetInsert() +** routine may not be called again. +** +** This routine may not be called after sqlite3RowSetTest() has +** been used. Older versions of RowSet allowed that, but as the +** capability was not used by the code generator, it was removed +** for code economy. +*/ +SQLITE_PRIVATE int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ + assert( p!=0 ); + assert( p->pForest==0 ); /* Cannot be used with sqlite3RowSetText() */ + + /* Merge the forest into a single sorted list on first call */ + if( (p->rsFlags & ROWSET_NEXT)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + if( (p->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + p->pEntry = rowSetEntrySort(p->pEntry); + } + p->rsFlags |= ROWSET_SORTED|ROWSET_NEXT; + } + + /* Return the next entry on the list */ + if( p->pEntry ){ + *pRowid = p->pEntry->v; + p->pEntry = p->pEntry->pRight; + if( p->pEntry==0 ){ /*OPTIMIZATION-IF-TRUE*/ + /* Free memory immediately, rather than waiting on sqlite3_finalize() */ + sqlite3RowSetClear(p); + } + return 1; + }else{ + return 0; + } +} + +/* +** Check to see if element iRowid was inserted into the rowset as +** part of any insert batch prior to iBatch. Return 1 or 0. +** +** If this is the first test of a new batch and if there exist entries +** on pRowSet->pEntry, then sort those entries into the forest at +** pRowSet->pForest so that they can be tested. +*/ +SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ + struct RowSetEntry *p, *pTree; + + /* This routine is never called after sqlite3RowSetNext() */ + assert( pRowSet!=0 && (pRowSet->rsFlags & ROWSET_NEXT)==0 ); + + /* Sort entries into the forest on the first test of a new batch. + ** To save unnecessary work, only do this when the batch number changes. + */ + if( iBatch!=pRowSet->iBatch ){ /*OPTIMIZATION-IF-FALSE*/ + p = pRowSet->pEntry; + if( p ){ + struct RowSetEntry **ppPrevTree = &pRowSet->pForest; + if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + /* Only sort the current set of entries if they need it */ + p = rowSetEntrySort(p); + } + for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ + ppPrevTree = &pTree->pRight; + if( pTree->pLeft==0 ){ + pTree->pLeft = rowSetListToTree(p); + break; + }else{ + struct RowSetEntry *pAux, *pTail; + rowSetTreeToList(pTree->pLeft, &pAux, &pTail); + pTree->pLeft = 0; + p = rowSetEntryMerge(pAux, p); + } + } + if( pTree==0 ){ + *ppPrevTree = pTree = rowSetEntryAlloc(pRowSet); + if( pTree ){ + pTree->v = 0; + pTree->pRight = 0; + pTree->pLeft = rowSetListToTree(p); + } + } + pRowSet->pEntry = 0; + pRowSet->pLast = 0; + pRowSet->rsFlags |= ROWSET_SORTED; + } + pRowSet->iBatch = iBatch; + } + + /* Test to see if the iRowid value appears anywhere in the forest. + ** Return 1 if it does and 0 if not. + */ + for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ + p = pTree->pLeft; + while( p ){ + if( p->vpRight; + }else if( p->v>iRowid ){ + p = p->pLeft; + }else{ + return 1; + } + } + } + return 0; +} + +/************** End of rowset.c **********************************************/ +/************** Begin file pager.c *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the implementation of the page cache subsystem or "pager". +** +** The pager is used to access a database disk file. It implements +** atomic commit and rollback through the use of a journal file that +** is separate from the database file. The pager also implements file +** locking to prevent two processes from writing the same database +** file simultaneously, or one process from reading the database while +** another is writing. +*/ +#ifndef SQLITE_OMIT_DISKIO +/* #include "sqliteInt.h" */ +/************** Include wal.h in the middle of pager.c ***********************/ +/************** Begin file wal.h *********************************************/ +/* +** 2010 February 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface to the write-ahead logging +** system. Refer to the comments below and the header comment attached to +** the implementation of each function in log.c for further details. +*/ + +#ifndef SQLITE_WAL_H +#define SQLITE_WAL_H + +/* #include "sqliteInt.h" */ + +/* Macros for extracting appropriate sync flags for either transaction +** commits (WAL_SYNC_FLAGS(X)) or for checkpoint ops (CKPT_SYNC_FLAGS(X)): +*/ +#define WAL_SYNC_FLAGS(X) ((X)&0x03) +#define CKPT_SYNC_FLAGS(X) (((X)>>2)&0x03) + +#ifdef SQLITE_OMIT_WAL +# define sqlite3WalOpen(x,y,z) 0 +# define sqlite3WalLimit(x,y) +# define sqlite3WalClose(v,w,x,y,z) 0 +# define sqlite3WalBeginReadTransaction(y,z) 0 +# define sqlite3WalEndReadTransaction(z) +# define sqlite3WalDbsize(y) 0 +# define sqlite3WalBeginWriteTransaction(y) 0 +# define sqlite3WalEndWriteTransaction(x) 0 +# define sqlite3WalUndo(x,y,z) 0 +# define sqlite3WalSavepoint(y,z) +# define sqlite3WalSavepointUndo(y,z) 0 +# define sqlite3WalFrames(u,v,w,x,y,z) 0 +# define sqlite3WalCheckpoint(q,r,s,t,u,v,w,x,y,z) 0 +# define sqlite3WalCallback(z) 0 +# define sqlite3WalExclusiveMode(y,z) 0 +# define sqlite3WalHeapMemory(z) 0 +# define sqlite3WalFramesize(z) 0 +# define sqlite3WalFindFrame(x,y,z) 0 +# define sqlite3WalFile(x) 0 +#else + +#define WAL_SAVEPOINT_NDATA 4 + +/* Connection to a write-ahead log (WAL) file. +** There is one object of this type for each pager. +*/ +typedef struct Wal Wal; + +/* Open and close a connection to a write-ahead log. */ +SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); +SQLITE_PRIVATE int sqlite3WalClose(Wal *pWal, sqlite3*, int sync_flags, int, u8 *); + +/* Set the limiting size of a WAL file. */ +SQLITE_PRIVATE void sqlite3WalLimit(Wal*, i64); + +/* Used by readers to open (lock) and close (unlock) a snapshot. A +** snapshot is like a read-transaction. It is the state of the database +** at an instant in time. sqlite3WalOpenSnapshot gets a read lock and +** preserves the current state even if the other threads or processes +** write to or checkpoint the WAL. sqlite3WalCloseSnapshot() closes the +** transaction and releases the lock. +*/ +SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *); +SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal); + +/* Read a page from the write-ahead log, if it is present. */ +SQLITE_PRIVATE int sqlite3WalFindFrame(Wal *, Pgno, u32 *); +SQLITE_PRIVATE int sqlite3WalReadFrame(Wal *, u32, int, u8 *); + +/* If the WAL is not empty, return the size of the database. */ +SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal); + +/* Obtain or release the WRITER lock. */ +SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal); +SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal); + +/* Undo any frames written (but not committed) to the log */ +SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx); + +/* Return an integer that records the current (uncommitted) write +** position in the WAL */ +SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData); + +/* Move the write position of the WAL back to iFrame. Called in +** response to a ROLLBACK TO command. */ +SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData); + +/* Write a frame or frames to the log. */ +SQLITE_PRIVATE int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int); + +/* Copy pages from the log to the database file */ +SQLITE_PRIVATE int sqlite3WalCheckpoint( + Wal *pWal, /* Write-ahead log connection */ + sqlite3 *db, /* Check this handle's interrupt flag */ + int eMode, /* One of PASSIVE, FULL and RESTART */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags to sync db file with (or 0) */ + int nBuf, /* Size of buffer nBuf */ + u8 *zBuf, /* Temporary buffer to use */ + int *pnLog, /* OUT: Number of frames in WAL */ + int *pnCkpt /* OUT: Number of backfilled frames in WAL */ +); + +/* Return the value to pass to a sqlite3_wal_hook callback, the +** number of frames in the WAL at the point of the last commit since +** sqlite3WalCallback() was called. If no commits have occurred since +** the last call, then return 0. +*/ +SQLITE_PRIVATE int sqlite3WalCallback(Wal *pWal); + +/* Tell the wal layer that an EXCLUSIVE lock has been obtained (or released) +** by the pager layer on the database file. +*/ +SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op); + +/* Return true if the argument is non-NULL and the WAL module is using +** heap-memory for the wal-index. Otherwise, if the argument is NULL or the +** WAL module is using shared-memory, return false. +*/ +SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); + +#ifdef SQLITE_ENABLE_SNAPSHOT +SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot); +SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal); +SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE void sqlite3WalSnapshotUnlock(Wal *pWal); +#endif + +#ifdef SQLITE_ENABLE_ZIPVFS +/* If the WAL file is not empty, return the number of bytes of content +** stored in each frame (i.e. the db page-size when the WAL was created). +*/ +SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal); +#endif + +/* Return the sqlite3_file object for the WAL file */ +SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal); + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +SQLITE_PRIVATE int sqlite3WalWriteLock(Wal *pWal, int bLock); +SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db); +#endif + +#endif /* ifndef SQLITE_OMIT_WAL */ +#endif /* SQLITE_WAL_H */ + +/************** End of wal.h *************************************************/ +/************** Continuing where we left off in pager.c **********************/ + + +/******************* NOTES ON THE DESIGN OF THE PAGER ************************ +** +** This comment block describes invariants that hold when using a rollback +** journal. These invariants do not apply for journal_mode=WAL, +** journal_mode=MEMORY, or journal_mode=OFF. +** +** Within this comment block, a page is deemed to have been synced +** automatically as soon as it is written when PRAGMA synchronous=OFF. +** Otherwise, the page is not synced until the xSync method of the VFS +** is called successfully on the file containing the page. +** +** Definition: A page of the database file is said to be "overwriteable" if +** one or more of the following are true about the page: +** +** (a) The original content of the page as it was at the beginning of +** the transaction has been written into the rollback journal and +** synced. +** +** (b) The page was a freelist leaf page at the start of the transaction. +** +** (c) The page number is greater than the largest page that existed in +** the database file at the start of the transaction. +** +** (1) A page of the database file is never overwritten unless one of the +** following are true: +** +** (a) The page and all other pages on the same sector are overwriteable. +** +** (b) The atomic page write optimization is enabled, and the entire +** transaction other than the update of the transaction sequence +** number consists of a single page change. +** +** (2) The content of a page written into the rollback journal exactly matches +** both the content in the database when the rollback journal was written +** and the content in the database at the beginning of the current +** transaction. +** +** (3) Writes to the database file are an integer multiple of the page size +** in length and are aligned on a page boundary. +** +** (4) Reads from the database file are either aligned on a page boundary and +** an integer multiple of the page size in length or are taken from the +** first 100 bytes of the database file. +** +** (5) All writes to the database file are synced prior to the rollback journal +** being deleted, truncated, or zeroed. +** +** (6) If a super-journal file is used, then all writes to the database file +** are synced prior to the super-journal being deleted. +** +** Definition: Two databases (or the same database at two points it time) +** are said to be "logically equivalent" if they give the same answer to +** all queries. Note in particular the content of freelist leaf +** pages can be changed arbitrarily without affecting the logical equivalence +** of the database. +** +** (7) At any time, if any subset, including the empty set and the total set, +** of the unsynced changes to a rollback journal are removed and the +** journal is rolled back, the resulting database file will be logically +** equivalent to the database file at the beginning of the transaction. +** +** (8) When a transaction is rolled back, the xTruncate method of the VFS +** is called to restore the database file to the same size it was at +** the beginning of the transaction. (In some VFSes, the xTruncate +** method is a no-op, but that does not change the fact the SQLite will +** invoke it.) +** +** (9) Whenever the database file is modified, at least one bit in the range +** of bytes from 24 through 39 inclusive will be changed prior to releasing +** the EXCLUSIVE lock, thus signaling other connections on the same +** database to flush their caches. +** +** (10) The pattern of bits in bytes 24 through 39 shall not repeat in less +** than one billion transactions. +** +** (11) A database file is well-formed at the beginning and at the conclusion +** of every transaction. +** +** (12) An EXCLUSIVE lock is held on the database file when writing to +** the database file. +** +** (13) A SHARED lock is held on the database file while reading any +** content out of the database file. +** +******************************************************************************/ + +/* +** Macros for troubleshooting. Normally turned off +*/ +#if 0 +int sqlite3PagerTrace=1; /* True to enable tracing */ +#define sqlite3DebugPrintf printf +#define PAGERTRACE(X) if( sqlite3PagerTrace ){ sqlite3DebugPrintf X; } +#else +#define PAGERTRACE(X) +#endif + +/* +** The following two macros are used within the PAGERTRACE() macros above +** to print out file-descriptors. +** +** PAGERID() takes a pointer to a Pager struct as its argument. The +** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file +** struct as its argument. +*/ +#define PAGERID(p) (SQLITE_PTR_TO_INT(p->fd)) +#define FILEHANDLEID(fd) (SQLITE_PTR_TO_INT(fd)) + +/* +** The Pager.eState variable stores the current 'state' of a pager. A +** pager may be in any one of the seven states shown in the following +** state diagram. +** +** OPEN <------+------+ +** | | | +** V | | +** +---------> READER-------+ | +** | | | +** | V | +** |<-------WRITER_LOCKED------> ERROR +** | | ^ +** | V | +** |<------WRITER_CACHEMOD-------->| +** | | | +** | V | +** |<-------WRITER_DBMOD---------->| +** | | | +** | V | +** +<------WRITER_FINISHED-------->+ +** +** +** List of state transitions and the C [function] that performs each: +** +** OPEN -> READER [sqlite3PagerSharedLock] +** READER -> OPEN [pager_unlock] +** +** READER -> WRITER_LOCKED [sqlite3PagerBegin] +** WRITER_LOCKED -> WRITER_CACHEMOD [pager_open_journal] +** WRITER_CACHEMOD -> WRITER_DBMOD [syncJournal] +** WRITER_DBMOD -> WRITER_FINISHED [sqlite3PagerCommitPhaseOne] +** WRITER_*** -> READER [pager_end_transaction] +** +** WRITER_*** -> ERROR [pager_error] +** ERROR -> OPEN [pager_unlock] +** +** +** OPEN: +** +** The pager starts up in this state. Nothing is guaranteed in this +** state - the file may or may not be locked and the database size is +** unknown. The database may not be read or written. +** +** * No read or write transaction is active. +** * Any lock, or no lock at all, may be held on the database file. +** * The dbSize, dbOrigSize and dbFileSize variables may not be trusted. +** +** READER: +** +** In this state all the requirements for reading the database in +** rollback (non-WAL) mode are met. Unless the pager is (or recently +** was) in exclusive-locking mode, a user-level read transaction is +** open. The database size is known in this state. +** +** A connection running with locking_mode=normal enters this state when +** it opens a read-transaction on the database and returns to state +** OPEN after the read-transaction is completed. However a connection +** running in locking_mode=exclusive (including temp databases) remains in +** this state even after the read-transaction is closed. The only way +** a locking_mode=exclusive connection can transition from READER to OPEN +** is via the ERROR state (see below). +** +** * A read transaction may be active (but a write-transaction cannot). +** * A SHARED or greater lock is held on the database file. +** * The dbSize variable may be trusted (even if a user-level read +** transaction is not active). The dbOrigSize and dbFileSize variables +** may not be trusted at this point. +** * If the database is a WAL database, then the WAL connection is open. +** * Even if a read-transaction is not open, it is guaranteed that +** there is no hot-journal in the file-system. +** +** WRITER_LOCKED: +** +** The pager moves to this state from READER when a write-transaction +** is first opened on the database. In WRITER_LOCKED state, all locks +** required to start a write-transaction are held, but no actual +** modifications to the cache or database have taken place. +** +** In rollback mode, a RESERVED or (if the transaction was opened with +** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when +** moving to this state, but the journal file is not written to or opened +** to in this state. If the transaction is committed or rolled back while +** in WRITER_LOCKED state, all that is required is to unlock the database +** file. +** +** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file. +** If the connection is running with locking_mode=exclusive, an attempt +** is made to obtain an EXCLUSIVE lock on the database file. +** +** * A write transaction is active. +** * If the connection is open in rollback-mode, a RESERVED or greater +** lock is held on the database file. +** * If the connection is open in WAL-mode, a WAL write transaction +** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully +** called). +** * The dbSize, dbOrigSize and dbFileSize variables are all valid. +** * The contents of the pager cache have not been modified. +** * The journal file may or may not be open. +** * Nothing (not even the first header) has been written to the journal. +** +** WRITER_CACHEMOD: +** +** A pager moves from WRITER_LOCKED state to this state when a page is +** first modified by the upper layer. In rollback mode the journal file +** is opened (if it is not already open) and a header written to the +** start of it. The database file on disk has not been modified. +** +** * A write transaction is active. +** * A RESERVED or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** to it, but the header has not been synced to disk. +** * The contents of the page cache have been modified. +** +** WRITER_DBMOD: +** +** The pager transitions from WRITER_CACHEMOD into WRITER_DBMOD state +** when it modifies the contents of the database file. WAL connections +** never enter this state (since they do not modify the database file, +** just the log file). +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * The journal file is open and the first header has been written +** and synced to disk. +** * The contents of the page cache have been modified (and possibly +** written to disk). +** +** WRITER_FINISHED: +** +** It is not possible for a WAL connection to enter this state. +** +** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD +** state after the entire transaction has been successfully written into the +** database file. In this state the transaction may be committed simply +** by finalizing the journal file. Once in WRITER_FINISHED state, it is +** not possible to modify the database further. At this point, the upper +** layer must either commit or rollback the transaction. +** +** * A write transaction is active. +** * An EXCLUSIVE or greater lock is held on the database file. +** * All writing and syncing of journal and database data has finished. +** If no error occurred, all that remains is to finalize the journal to +** commit the transaction. If an error did occur, the caller will need +** to rollback the transaction. +** +** ERROR: +** +** The ERROR state is entered when an IO or disk-full error (including +** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it +** difficult to be sure that the in-memory pager state (cache contents, +** db size etc.) are consistent with the contents of the file-system. +** +** Temporary pager files may enter the ERROR state, but in-memory pagers +** cannot. +** +** For example, if an IO error occurs while performing a rollback, +** the contents of the page-cache may be left in an inconsistent state. +** At this point it would be dangerous to change back to READER state +** (as usually happens after a rollback). Any subsequent readers might +** report database corruption (due to the inconsistent cache), and if +** they upgrade to writers, they may inadvertently corrupt the database +** file. To avoid this hazard, the pager switches into the ERROR state +** instead of READER following such an error. +** +** Once it has entered the ERROR state, any attempt to use the pager +** to read or write data returns an error. Eventually, once all +** outstanding transactions have been abandoned, the pager is able to +** transition back to OPEN state, discarding the contents of the +** page-cache and any other in-memory state at the same time. Everything +** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** when a read-transaction is next opened on the pager (transitioning +** the pager into READER state). At that point the system has recovered +** from the error. +** +** Specifically, the pager jumps into the ERROR state if: +** +** 1. An error occurs while attempting a rollback. This happens in +** function sqlite3PagerRollback(). +** +** 2. An error occurs while attempting to finalize a journal file +** following a commit in function sqlite3PagerCommitPhaseTwo(). +** +** 3. An error occurs while attempting to write to the journal or +** database file in function pagerStress() in order to free up +** memory. +** +** In other cases, the error is returned to the b-tree layer. The b-tree +** layer then attempts a rollback operation. If the error condition +** persists, the pager enters the ERROR state via condition (1) above. +** +** Condition (3) is necessary because it can be triggered by a read-only +** statement executed within a transaction. In this case, if the error +** code were simply returned to the user, the b-tree layer would not +** automatically attempt a rollback, as it assumes that an error in a +** read-only statement cannot leave the pager in an internally inconsistent +** state. +** +** * The Pager.errCode variable is set to something other than SQLITE_OK. +** * There are one or more outstanding references to pages (after the +** last reference is dropped the pager should move back to OPEN state). +** * The pager is not an in-memory pager. +** +** +** Notes: +** +** * A pager is never in WRITER_DBMOD or WRITER_FINISHED state if the +** connection is open in WAL mode. A WAL connection is always in one +** of the first four states. +** +** * Normally, a connection open in exclusive mode is never in PAGER_OPEN +** state. There are two exceptions: immediately after exclusive-mode has +** been turned on (and before any read or write transactions are +** executed), and when the pager is leaving the "error state". +** +** * See also: assert_pager_state(). +*/ +#define PAGER_OPEN 0 +#define PAGER_READER 1 +#define PAGER_WRITER_LOCKED 2 +#define PAGER_WRITER_CACHEMOD 3 +#define PAGER_WRITER_DBMOD 4 +#define PAGER_WRITER_FINISHED 5 +#define PAGER_ERROR 6 + +/* +** The Pager.eLock variable is almost always set to one of the +** following locking-states, according to the lock currently held on +** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** This variable is kept up to date as locks are taken and released by +** the pagerLockDb() and pagerUnlockDb() wrappers. +** +** If the VFS xLock() or xUnlock() returns an error other than SQLITE_BUSY +** (i.e. one of the SQLITE_IOERR subtypes), it is not clear whether or not +** the operation was successful. In these circumstances pagerLockDb() and +** pagerUnlockDb() take a conservative approach - eLock is always updated +** when unlocking the file, and only updated when locking the file if the +** VFS call is successful. This way, the Pager.eLock variable may be set +** to a less exclusive (lower) value than the lock that is actually held +** at the system level, but it is never set to a more exclusive value. +** +** This is usually safe. If an xUnlock fails or appears to fail, there may +** be a few redundant xLock() calls or a lock may be held for longer than +** required, but nothing really goes wrong. +** +** The exception is when the database file is unlocked as the pager moves +** from ERROR to OPEN state. At this point there may be a hot-journal file +** in the file-system that needs to be rolled back (as part of an OPEN->SHARED +** transition, by the same pager or any other). If the call to xUnlock() +** fails at this point and the pager is left holding an EXCLUSIVE lock, this +** can confuse the call to xCheckReservedLock() call made later as part +** of hot-journal detection. +** +** xCheckReservedLock() is defined as returning true "if there is a RESERVED +** lock held by this process or any others". So xCheckReservedLock may +** return true because the caller itself is holding an EXCLUSIVE lock (but +** doesn't know it because of a previous error in xUnlock). If this happens +** a hot-journal may be mistaken for a journal being created by an active +** transaction in another process, causing SQLite to read from the database +** without rolling it back. +** +** To work around this, if a call to xUnlock() fails when unlocking the +** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It +** is only changed back to a real locking state after a successful call +** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition +** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK +** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE +** lock on the database file before attempting to roll it back. See function +** PagerSharedLock() for more detail. +** +** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in +** PAGER_OPEN state. +*/ +#define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) + +/* +** The maximum allowed sector size. 64KiB. If the xSectorsize() method +** returns a value larger than this, then MAX_SECTOR_SIZE is used instead. +** This could conceivably cause corruption following a power failure on +** such a system. This is currently an undocumented limit. +*/ +#define MAX_SECTOR_SIZE 0x10000 + + +/* +** An instance of the following structure is allocated for each active +** savepoint and statement transaction in the system. All such structures +** are stored in the Pager.aSavepoint[] array, which is allocated and +** resized using sqlite3Realloc(). +** +** When a savepoint is created, the PagerSavepoint.iHdrOffset field is +** set to 0. If a journal-header is written into the main journal while +** the savepoint is active, then iHdrOffset is set to the byte offset +** immediately following the last journal record written into the main +** journal before the journal-header. This is required during savepoint +** rollback (see pagerPlaybackSavepoint()). +*/ +typedef struct PagerSavepoint PagerSavepoint; +struct PagerSavepoint { + i64 iOffset; /* Starting offset in main journal */ + i64 iHdrOffset; /* See above */ + Bitvec *pInSavepoint; /* Set of pages in this savepoint */ + Pgno nOrig; /* Original number of pages in file */ + Pgno iSubRec; /* Index of first record in sub-journal */ + int bTruncateOnRelease; /* If stmt journal may be truncated on RELEASE */ +#ifndef SQLITE_OMIT_WAL + u32 aWalData[WAL_SAVEPOINT_NDATA]; /* WAL savepoint context */ +#endif +}; + +/* +** Bits of the Pager.doNotSpill flag. See further description below. +*/ +#define SPILLFLAG_OFF 0x01 /* Never spill cache. Set via pragma */ +#define SPILLFLAG_ROLLBACK 0x02 /* Current rolling back, so do not spill */ +#define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ + +/* +** An open page cache is an instance of struct Pager. A description of +** some of the more important member variables follows: +** +** eState +** +** The current 'state' of the pager object. See the comment and state +** diagram above for a description of the pager state. +** +** eLock +** +** For a real on-disk database, the current lock held on the database file - +** NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. +** +** For a temporary or in-memory database (neither of which require any +** locks), this variable is always set to EXCLUSIVE_LOCK. Since such +** databases always have Pager.exclusiveMode==1, this tricks the pager +** logic into thinking that it already has all the locks it will ever +** need (and no reason to release them). +** +** In some (obscure) circumstances, this variable may also be set to +** UNKNOWN_LOCK. See the comment above the #define of UNKNOWN_LOCK for +** details. +** +** changeCountDone +** +** This boolean variable is used to make sure that the change-counter +** (the 4-byte header field at byte offset 24 of the database file) is +** not updated more often than necessary. +** +** It is set to true when the change-counter field is updated, which +** can only happen if an exclusive lock is held on the database file. +** It is cleared (set to false) whenever an exclusive lock is +** relinquished on the database file. Each time a transaction is committed, +** The changeCountDone flag is inspected. If it is true, the work of +** updating the change-counter is omitted for the current transaction. +** +** This mechanism means that when running in exclusive mode, a connection +** need only update the change-counter once, for the first transaction +** committed. +** +** setSuper +** +** When PagerCommitPhaseOne() is called to commit a transaction, it may +** (or may not) specify a super-journal name to be written into the +** journal file before it is synced to disk. +** +** Whether or not a journal file contains a super-journal pointer affects +** the way in which the journal file is finalized after the transaction is +** committed or rolled back when running in "journal_mode=PERSIST" mode. +** If a journal file does not contain a super-journal pointer, it is +** finalized by overwriting the first journal header with zeroes. If +** it does contain a super-journal pointer the journal file is finalized +** by truncating it to zero bytes, just as if the connection were +** running in "journal_mode=truncate" mode. +** +** Journal files that contain super-journal pointers cannot be finalized +** simply by overwriting the first journal-header with zeroes, as the +** super-journal pointer could interfere with hot-journal rollback of any +** subsequently interrupted transaction that reuses the journal file. +** +** The flag is cleared as soon as the journal file is finalized (either +** by PagerCommitPhaseTwo or PagerRollback). If an IO error prevents the +** journal file from being successfully finalized, the setSuper flag +** is cleared anyway (and the pager will move to ERROR state). +** +** doNotSpill +** +** This variables control the behavior of cache-spills (calls made by +** the pcache module to the pagerStress() routine to write cached data +** to the file-system in order to free up memory). +** +** When bits SPILLFLAG_OFF or SPILLFLAG_ROLLBACK of doNotSpill are set, +** writing to the database from pagerStress() is disabled altogether. +** The SPILLFLAG_ROLLBACK case is done in a very obscure case that +** comes up during savepoint rollback that requires the pcache module +** to allocate a new page to prevent the journal file from being written +** while it is being traversed by code in pager_playback(). The SPILLFLAG_OFF +** case is a user preference. +** +** If the SPILLFLAG_NOSYNC bit is set, writing to the database from +** pagerStress() is permitted, but syncing the journal file is not. +** This flag is set by sqlite3PagerWrite() when the file-system sector-size +** is larger than the database page-size in order to prevent a journal sync +** from happening in between the journalling of two pages on the same sector. +** +** subjInMemory +** +** This is a boolean variable. If true, then any required sub-journal +** is opened as an in-memory journal file. If false, then in-memory +** sub-journals are only used for in-memory pager files. +** +** This variable is updated by the upper layer each time a new +** write-transaction is opened. +** +** dbSize, dbOrigSize, dbFileSize +** +** Variable dbSize is set to the number of pages in the database file. +** It is valid in PAGER_READER and higher states (all states except for +** OPEN and ERROR). +** +** dbSize is set based on the size of the database file, which may be +** larger than the size of the database (the value stored at offset +** 28 of the database header by the btree). If the size of the file +** is not an integer multiple of the page-size, the value stored in +** dbSize is rounded down (i.e. a 5KB file with 2K page-size has dbSize==2). +** Except, any file that is greater than 0 bytes in size is considered +** to have at least one page. (i.e. a 1KB file with 2K page-size leads +** to dbSize==1). +** +** During a write-transaction, if pages with page-numbers greater than +** dbSize are modified in the cache, dbSize is updated accordingly. +** Similarly, if the database is truncated using PagerTruncateImage(), +** dbSize is updated. +** +** Variables dbOrigSize and dbFileSize are valid in states +** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize +** variable at the start of the transaction. It is used during rollback, +** and to determine whether or not pages need to be journalled before +** being modified. +** +** Throughout a write-transaction, dbFileSize contains the size of +** the file on disk in pages. It is set to a copy of dbSize when the +** write-transaction is first opened, and updated when VFS calls are made +** to write or truncate the database file on disk. +** +** The only reason the dbFileSize variable is required is to suppress +** unnecessary calls to xTruncate() after committing a transaction. If, +** when a transaction is committed, the dbFileSize variable indicates +** that the database file is larger than the database image (Pager.dbSize), +** pager_truncate() is called. The pager_truncate() call uses xFilesize() +** to measure the database file on disk, and then truncates it if required. +** dbFileSize is not used when rolling back a transaction. In this case +** pager_truncate() is called unconditionally (which means there may be +** a call to xFilesize() that is not strictly required). In either case, +** pager_truncate() may cause the file to become smaller or larger. +** +** dbHintSize +** +** The dbHintSize variable is used to limit the number of calls made to +** the VFS xFileControl(FCNTL_SIZE_HINT) method. +** +** dbHintSize is set to a copy of the dbSize variable when a +** write-transaction is opened (at the same time as dbFileSize and +** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called, +** dbHintSize is increased to the number of pages that correspond to the +** size-hint passed to the method call. See pager_write_pagelist() for +** details. +** +** errCode +** +** The Pager.errCode variable is only ever used in PAGER_ERROR state. It +** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode +** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX +** sub-codes. +** +** syncFlags, walSyncFlags +** +** syncFlags is either SQLITE_SYNC_NORMAL (0x02) or SQLITE_SYNC_FULL (0x03). +** syncFlags is used for rollback mode. walSyncFlags is used for WAL mode +** and contains the flags used to sync the checkpoint operations in the +** lower two bits, and sync flags used for transaction commits in the WAL +** file in bits 0x04 and 0x08. In other words, to get the correct sync flags +** for checkpoint operations, use (walSyncFlags&0x03) and to get the correct +** sync flags for transaction commit, use ((walSyncFlags>>2)&0x03). Note +** that with synchronous=NORMAL in WAL mode, transaction commit is not synced +** meaning that the 0x04 and 0x08 bits are both zero. +*/ +struct Pager { + sqlite3_vfs *pVfs; /* OS functions to use for IO */ + u8 exclusiveMode; /* Boolean. True if locking_mode==EXCLUSIVE */ + u8 journalMode; /* One of the PAGER_JOURNALMODE_* values */ + u8 useJournal; /* Use a rollback journal on this file */ + u8 noSync; /* Do not sync the journal if true */ + u8 fullSync; /* Do extra syncs of the journal for robustness */ + u8 extraSync; /* sync directory after journal delete */ + u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ + u8 walSyncFlags; /* See description above */ + u8 tempFile; /* zFilename is a temporary or immutable file */ + u8 noLock; /* Do not lock (except in WAL mode) */ + u8 readOnly; /* True for a read-only database */ + u8 memDb; /* True to inhibit all file I/O */ + + /************************************************************************** + ** The following block contains those class members that change during + ** routine operation. Class members not in this block are either fixed + ** when the pager is first created or else only change when there is a + ** significant mode change (such as changing the page_size, locking_mode, + ** or the journal_mode). From another view, these class members describe + ** the "state" of the pager, while other class members describe the + ** "configuration" of the pager. + */ + u8 eState; /* Pager state (OPEN, READER, WRITER_LOCKED..) */ + u8 eLock; /* Current lock held on database file */ + u8 changeCountDone; /* Set after incrementing the change-counter */ + u8 setSuper; /* Super-jrnl name is written into jrnl */ + u8 doNotSpill; /* Do not spill the cache when non-zero */ + u8 subjInMemory; /* True to use in-memory sub-journals */ + u8 bUseFetch; /* True to use xFetch() */ + u8 hasHeldSharedLock; /* True if a shared lock has ever been held */ + Pgno dbSize; /* Number of pages in the database */ + Pgno dbOrigSize; /* dbSize before the current transaction */ + Pgno dbFileSize; /* Number of pages in the database file */ + Pgno dbHintSize; /* Value passed to FCNTL_SIZE_HINT call */ + int errCode; /* One of several kinds of errors */ + int nRec; /* Pages journalled since last j-header written */ + u32 cksumInit; /* Quasi-random value added to every checksum */ + u32 nSubRec; /* Number of records written to sub-journal */ + Bitvec *pInJournal; /* One bit for each page in the database file */ + sqlite3_file *fd; /* File descriptor for database */ + sqlite3_file *jfd; /* File descriptor for main journal */ + sqlite3_file *sjfd; /* File descriptor for sub-journal */ + i64 journalOff; /* Current write offset in the journal file */ + i64 journalHdr; /* Byte offset to previous journal header */ + sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ + PagerSavepoint *aSavepoint; /* Array of active savepoints */ + int nSavepoint; /* Number of elements in aSavepoint[] */ + u32 iDataVersion; /* Changes whenever database content changes */ + char dbFileVers[16]; /* Changes whenever database file changes */ + + int nMmapOut; /* Number of mmap pages currently outstanding */ + sqlite3_int64 szMmap; /* Desired maximum mmap size */ + PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ + /* + ** End of the routinely-changing class members + ***************************************************************************/ + + u16 nExtra; /* Add this many bytes to each in-memory page */ + i16 nReserve; /* Number of unused bytes at end of each page */ + u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ + u32 sectorSize; /* Assumed sector size during rollback */ + Pgno mxPgno; /* Maximum allowed size of the database */ + i64 pageSize; /* Number of bytes in a page */ + i64 journalSizeLimit; /* Size limit for persistent journal files */ + char *zFilename; /* Name of the database file */ + char *zJournal; /* Name of the journal file */ + int (*xBusyHandler)(void*); /* Function to call when busy */ + void *pBusyHandlerArg; /* Context argument for xBusyHandler */ + int aStat[4]; /* Total cache hits, misses, writes, spills */ +#ifdef SQLITE_TEST + int nRead; /* Database pages read */ +#endif + void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ + int (*xGet)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */ + char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ + PCache *pPCache; /* Pointer to page cache object */ +#ifndef SQLITE_OMIT_WAL + Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ + char *zWal; /* File name for write-ahead log */ +#endif +}; + +/* +** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains +** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS +** or CACHE_WRITE to sqlite3_db_status(). +*/ +#define PAGER_STAT_HIT 0 +#define PAGER_STAT_MISS 1 +#define PAGER_STAT_WRITE 2 +#define PAGER_STAT_SPILL 3 + +/* +** The following global variables hold counters used for +** testing purposes only. These variables do not exist in +** a non-testing build. These variables are not thread-safe. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_pager_readdb_count = 0; /* Number of full pages read from DB */ +SQLITE_API int sqlite3_pager_writedb_count = 0; /* Number of full pages written to DB */ +SQLITE_API int sqlite3_pager_writej_count = 0; /* Number of pages written to journal */ +# define PAGER_INCR(v) v++ +#else +# define PAGER_INCR(v) +#endif + + + +/* +** Journal files begin with the following magic string. The data +** was obtained from /dev/random. It is used only as a sanity check. +** +** Since version 2.8.0, the journal format contains additional sanity +** checking information. If the power fails while the journal is being +** written, semi-random garbage data might appear in the journal +** file after power is restored. If an attempt is then made +** to roll the journal back, the database could be corrupted. The additional +** sanity checking data is an attempt to discover the garbage in the +** journal and ignore it. +** +** The sanity checking information for the new journal format consists +** of a 32-bit checksum on each page of data. The checksum covers both +** the page number and the pPager->pageSize bytes of data for the page. +** This cksum is initialized to a 32-bit random value that appears in the +** journal file right after the header. The random initializer is important, +** because garbage data that appears at the end of a journal is likely +** data that was once in other files that have now been deleted. If the +** garbage data came from an obsolete journal file, the checksums might +** be correct. But by initializing the checksum to random value which +** is different for every journal, we minimize that risk. +*/ +static const unsigned char aJournalMagic[] = { + 0xd9, 0xd5, 0x05, 0xf9, 0x20, 0xa1, 0x63, 0xd7, +}; + +/* +** The size of the of each page record in the journal is given by +** the following macro. +*/ +#define JOURNAL_PG_SZ(pPager) ((pPager->pageSize) + 8) + +/* +** The journal header size for this pager. This is usually the same +** size as a single disk sector. See also setSectorSize(). +*/ +#define JOURNAL_HDR_SZ(pPager) (pPager->sectorSize) + +/* +** The macro MEMDB is true if we are dealing with an in-memory database. +** We do this as a macro so that if the SQLITE_OMIT_MEMORYDB macro is set, +** the value of MEMDB will be a constant and the compiler will optimize +** out code that would never execute. +*/ +#ifdef SQLITE_OMIT_MEMORYDB +# define MEMDB 0 +#else +# define MEMDB pPager->memDb +#endif + +/* +** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch +** interfaces to access the database using memory-mapped I/O. +*/ +#if SQLITE_MAX_MMAP_SIZE>0 +# define USEFETCH(x) ((x)->bUseFetch) +#else +# define USEFETCH(x) 0 +#endif + +/* +** The argument to this macro is a file descriptor (type sqlite3_file*). +** Return 0 if it is not open, or non-zero (but not 1) if it is. +** +** This is so that expressions can be written as: +** +** if( isOpen(pPager->jfd) ){ ... +** +** instead of +** +** if( pPager->jfd->pMethods ){ ... +*/ +#define isOpen(pFd) ((pFd)->pMethods!=0) + +#ifdef SQLITE_DIRECT_OVERFLOW_READ +/* +** Return true if page pgno can be read directly from the database file +** by the b-tree layer. This is the case if: +** +** * the database file is open, +** * there are no dirty pages in the cache, and +** * the desired page is not currently in the wal file. +*/ +SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ + if( pPager->fd->pMethods==0 ) return 0; + if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; +#ifndef SQLITE_OMIT_WAL + if( pPager->pWal ){ + u32 iRead = 0; + int rc; + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); + return (rc==SQLITE_OK && iRead==0); + } +#endif + return 1; +} +#endif + +#ifndef SQLITE_OMIT_WAL +# define pagerUseWal(x) ((x)->pWal!=0) +#else +# define pagerUseWal(x) 0 +# define pagerRollbackWal(x) 0 +# define pagerWalFrames(v,w,x,y) 0 +# define pagerOpenWalIfPresent(z) SQLITE_OK +# define pagerBeginReadTransaction(z) SQLITE_OK +#endif + +#ifndef NDEBUG +/* +** Usage: +** +** assert( assert_pager_state(pPager) ); +** +** This function runs many asserts to try to find inconsistencies in +** the internal state of the Pager object. +*/ +static int assert_pager_state(Pager *p){ + Pager *pPager = p; + + /* State must be valid. */ + assert( p->eState==PAGER_OPEN + || p->eState==PAGER_READER + || p->eState==PAGER_WRITER_LOCKED + || p->eState==PAGER_WRITER_CACHEMOD + || p->eState==PAGER_WRITER_DBMOD + || p->eState==PAGER_WRITER_FINISHED + || p->eState==PAGER_ERROR + ); + + /* Regardless of the current state, a temp-file connection always behaves + ** as if it has an exclusive lock on the database file. It never updates + ** the change-counter field, so the changeCountDone flag is always set. + */ + assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK ); + assert( p->tempFile==0 || pPager->changeCountDone ); + + /* If the useJournal flag is clear, the journal-mode must be "OFF". + ** And if the journal-mode is "OFF", the journal file must not be open. + */ + assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal ); + assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) ); + + /* Check that MEMDB implies noSync. And an in-memory journal. Since + ** this means an in-memory pager performs no IO at all, it cannot encounter + ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing + ** a journal file. (although the in-memory journal implementation may + ** return SQLITE_IOERR_NOMEM while the journal file is being written). It + ** is therefore not possible for an in-memory pager to enter the ERROR + ** state. + */ + if( MEMDB ){ + assert( !isOpen(p->fd) ); + assert( p->noSync ); + assert( p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_MEMORY + ); + assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN ); + assert( pagerUseWal(p)==0 ); + } + + /* If changeCountDone is set, a RESERVED lock or greater must be held + ** on the file. + */ + assert( pPager->changeCountDone==0 || pPager->eLock>=RESERVED_LOCK ); + assert( p->eLock!=PENDING_LOCK ); + + switch( p->eState ){ + case PAGER_OPEN: + assert( !MEMDB ); + assert( pPager->errCode==SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 || pPager->tempFile ); + break; + + case PAGER_READER: + assert( pPager->errCode==SQLITE_OK ); + assert( p->eLock!=UNKNOWN_LOCK ); + assert( p->eLock>=SHARED_LOCK ); + break; + + case PAGER_WRITER_LOCKED: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + assert( p->eLock>=RESERVED_LOCK ); + } + assert( pPager->dbSize==pPager->dbOrigSize ); + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + assert( pPager->setSuper==0 ); + break; + + case PAGER_WRITER_CACHEMOD: + assert( p->eLock!=UNKNOWN_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + if( !pagerUseWal(pPager) ){ + /* It is possible that if journal_mode=wal here that neither the + ** journal file nor the WAL file are open. This happens during + ** a rollback transaction that switches from journal_mode=off + ** to journal_mode=wal. + */ + assert( p->eLock>=RESERVED_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + ); + } + assert( pPager->dbOrigSize==pPager->dbFileSize ); + assert( pPager->dbOrigSize==pPager->dbHintSize ); + break; + + case PAGER_WRITER_DBMOD: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( p->eLock>=EXCLUSIVE_LOCK ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) + ); + assert( pPager->dbOrigSize<=pPager->dbHintSize ); + break; + + case PAGER_WRITER_FINISHED: + assert( p->eLock==EXCLUSIVE_LOCK ); + assert( pPager->errCode==SQLITE_OK ); + assert( !pagerUseWal(pPager) ); + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) + ); + break; + + case PAGER_ERROR: + /* There must be at least one outstanding reference to the pager if + ** in ERROR state. Otherwise the pager should have already dropped + ** back to OPEN state. + */ + assert( pPager->errCode!=SQLITE_OK ); + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 || pPager->tempFile ); + break; + } + + return 1; +} +#endif /* ifndef NDEBUG */ + +#ifdef SQLITE_DEBUG +/* +** Return a pointer to a human readable string in a static buffer +** containing the state of the Pager object passed as an argument. This +** is intended to be used within debuggers. For example, as an alternative +** to "print *pPager" in gdb: +** +** (gdb) printf "%s", print_pager_state(pPager) +** +** This routine has external linkage in order to suppress compiler warnings +** about an unused function. It is enclosed within SQLITE_DEBUG and so does +** not appear in normal builds. +*/ +char *print_pager_state(Pager *p){ + static char zRet[1024]; + + sqlite3_snprintf(1024, zRet, + "Filename: %s\n" + "State: %s errCode=%d\n" + "Lock: %s\n" + "Locking mode: locking_mode=%s\n" + "Journal mode: journal_mode=%s\n" + "Backing store: tempFile=%d memDb=%d useJournal=%d\n" + "Journal: journalOff=%lld journalHdr=%lld\n" + "Size: dbsize=%d dbOrigSize=%d dbFileSize=%d\n" + , p->zFilename + , p->eState==PAGER_OPEN ? "OPEN" : + p->eState==PAGER_READER ? "READER" : + p->eState==PAGER_WRITER_LOCKED ? "WRITER_LOCKED" : + p->eState==PAGER_WRITER_CACHEMOD ? "WRITER_CACHEMOD" : + p->eState==PAGER_WRITER_DBMOD ? "WRITER_DBMOD" : + p->eState==PAGER_WRITER_FINISHED ? "WRITER_FINISHED" : + p->eState==PAGER_ERROR ? "ERROR" : "?error?" + , (int)p->errCode + , p->eLock==NO_LOCK ? "NO_LOCK" : + p->eLock==RESERVED_LOCK ? "RESERVED" : + p->eLock==EXCLUSIVE_LOCK ? "EXCLUSIVE" : + p->eLock==SHARED_LOCK ? "SHARED" : + p->eLock==UNKNOWN_LOCK ? "UNKNOWN" : "?error?" + , p->exclusiveMode ? "exclusive" : "normal" + , p->journalMode==PAGER_JOURNALMODE_MEMORY ? "memory" : + p->journalMode==PAGER_JOURNALMODE_OFF ? "off" : + p->journalMode==PAGER_JOURNALMODE_DELETE ? "delete" : + p->journalMode==PAGER_JOURNALMODE_PERSIST ? "persist" : + p->journalMode==PAGER_JOURNALMODE_TRUNCATE ? "truncate" : + p->journalMode==PAGER_JOURNALMODE_WAL ? "wal" : "?error?" + , (int)p->tempFile, (int)p->memDb, (int)p->useJournal + , p->journalOff, p->journalHdr + , (int)p->dbSize, (int)p->dbOrigSize, (int)p->dbFileSize + ); + + return zRet; +} +#endif + +/* Forward references to the various page getters */ +static int getPageNormal(Pager*,Pgno,DbPage**,int); +static int getPageError(Pager*,Pgno,DbPage**,int); +#if SQLITE_MAX_MMAP_SIZE>0 +static int getPageMMap(Pager*,Pgno,DbPage**,int); +#endif + +/* +** Set the Pager.xGet method for the appropriate routine used to fetch +** content from the pager. +*/ +static void setGetterMethod(Pager *pPager){ + if( pPager->errCode ){ + pPager->xGet = getPageError; +#if SQLITE_MAX_MMAP_SIZE>0 + }else if( USEFETCH(pPager) ){ + pPager->xGet = getPageMMap; +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + }else{ + pPager->xGet = getPageNormal; + } +} + +/* +** Return true if it is necessary to write page *pPg into the sub-journal. +** A page needs to be written into the sub-journal if there exists one +** or more open savepoints for which: +** +** * The page-number is less than or equal to PagerSavepoint.nOrig, and +** * The bit corresponding to the page-number is not set in +** PagerSavepoint.pInSavepoint. +*/ +static int subjRequiresPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + PagerSavepoint *p; + Pgno pgno = pPg->pgno; + int i; + for(i=0; inSavepoint; i++){ + p = &pPager->aSavepoint[i]; + if( p->nOrig>=pgno && 0==sqlite3BitvecTestNotNull(p->pInSavepoint, pgno) ){ + for(i=i+1; inSavepoint; i++){ + pPager->aSavepoint[i].bTruncateOnRelease = 0; + } + return 1; + } + } + return 0; +} + +#ifdef SQLITE_DEBUG +/* +** Return true if the page is already in the journal file. +*/ +static int pageInJournal(Pager *pPager, PgHdr *pPg){ + return sqlite3BitvecTest(pPager->pInJournal, pPg->pgno); +} +#endif + +/* +** Read a 32-bit integer from the given file descriptor. Store the integer +** that is read in *pRes. Return SQLITE_OK if everything worked, or an +** error code is something goes wrong. +** +** All values are stored on disk as big-endian. +*/ +static int read32bits(sqlite3_file *fd, i64 offset, u32 *pRes){ + unsigned char ac[4]; + int rc = sqlite3OsRead(fd, ac, sizeof(ac), offset); + if( rc==SQLITE_OK ){ + *pRes = sqlite3Get4byte(ac); + } + return rc; +} + +/* +** Write a 32-bit integer into a string buffer in big-endian byte order. +*/ +#define put32bits(A,B) sqlite3Put4byte((u8*)A,B) + + +/* +** Write a 32-bit integer into the given file descriptor. Return SQLITE_OK +** on success or an error code is something goes wrong. +*/ +static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ + char ac[4]; + put32bits(ac, val); + return sqlite3OsWrite(fd, ac, 4, offset); +} + +/* +** Unlock the database file to level eLock, which must be either NO_LOCK +** or SHARED_LOCK. Regardless of whether or not the call to xUnlock() +** succeeds, set the Pager.eLock variable to match the (attempted) new lock. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it. See the comment above the #define of +** UNKNOWN_LOCK for an explanation of this. +*/ +static int pagerUnlockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( !pPager->exclusiveMode || pPager->eLock==eLock ); + assert( eLock==NO_LOCK || eLock==SHARED_LOCK ); + assert( eLock!=NO_LOCK || pagerUseWal(pPager)==0 ); + if( isOpen(pPager->fd) ){ + assert( pPager->eLock>=eLock ); + rc = pPager->noLock ? SQLITE_OK : sqlite3OsUnlock(pPager->fd, eLock); + if( pPager->eLock!=UNKNOWN_LOCK ){ + pPager->eLock = (u8)eLock; + } + IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) + } + pPager->changeCountDone = pPager->tempFile; /* ticket fb3b3024ea238d5c */ + return rc; +} + +/* +** Lock the database file to level eLock, which must be either SHARED_LOCK, +** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the +** Pager.eLock variable to the new locking state. +** +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. +** See the comment above the #define of UNKNOWN_LOCK for an explanation +** of this. +*/ +static int pagerLockDb(Pager *pPager, int eLock){ + int rc = SQLITE_OK; + + assert( eLock==SHARED_LOCK || eLock==RESERVED_LOCK || eLock==EXCLUSIVE_LOCK ); + if( pPager->eLockeLock==UNKNOWN_LOCK ){ + rc = pPager->noLock ? SQLITE_OK : sqlite3OsLock(pPager->fd, eLock); + if( rc==SQLITE_OK && (pPager->eLock!=UNKNOWN_LOCK||eLock==EXCLUSIVE_LOCK) ){ + pPager->eLock = (u8)eLock; + IOTRACE(("LOCK %p %d\n", pPager, eLock)) + } + } + return rc; +} + +/* +** This function determines whether or not the atomic-write or +** atomic-batch-write optimizations can be used with this pager. The +** atomic-write optimization can be used if: +** +** (a) the value returned by OsDeviceCharacteristics() indicates that +** a database page may be written atomically, and +** (b) the value returned by OsSectorSize() is less than or equal +** to the page size. +** +** If it can be used, then the value returned is the size of the journal +** file when it contains rollback data for exactly one page. +** +** The atomic-batch-write optimization can be used if OsDeviceCharacteristics() +** returns a value with the SQLITE_IOCAP_BATCH_ATOMIC bit set. -1 is +** returned in this case. +** +** If neither optimization can be used, 0 is returned. +*/ +static int jrnlBufferSize(Pager *pPager){ + assert( !MEMDB ); + +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + int dc; /* Device characteristics */ + + assert( isOpen(pPager->fd) ); + dc = sqlite3OsDeviceCharacteristics(pPager->fd); +#else + UNUSED_PARAMETER(pPager); +#endif + +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( pPager->dbSize>0 && (dc&SQLITE_IOCAP_BATCH_ATOMIC) ){ + return -1; + } +#endif + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + { + int nSector = pPager->sectorSize; + int szPage = pPager->pageSize; + + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + if( 0==(dc&(SQLITE_IOCAP_ATOMIC|(szPage>>8)) || nSector>szPage) ){ + return 0; + } + } + + return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); +#endif + + return 0; +} + +/* +** If SQLITE_CHECK_PAGES is defined then we do some sanity checking +** on the cache using a hash function. This is used for testing +** and debugging only. +*/ +#ifdef SQLITE_CHECK_PAGES +/* +** Return a 32-bit hash of the page data for pPage. +*/ +static u32 pager_datahash(int nByte, unsigned char *pData){ + u32 hash = 0; + int i; + for(i=0; ipPager->pageSize, (unsigned char *)pPage->pData); +} +static void pager_set_pagehash(PgHdr *pPage){ + pPage->pageHash = pager_pagehash(pPage); +} + +/* +** The CHECK_PAGE macro takes a PgHdr* as an argument. If SQLITE_CHECK_PAGES +** is defined, and NDEBUG is not defined, an assert() statement checks +** that the page is either dirty or still matches the calculated page-hash. +*/ +#define CHECK_PAGE(x) checkPage(x) +static void checkPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + assert( pPager->eState!=PAGER_ERROR ); + assert( (pPg->flags&PGHDR_DIRTY) || pPg->pageHash==pager_pagehash(pPg) ); +} + +#else +#define pager_datahash(X,Y) 0 +#define pager_pagehash(X) 0 +#define pager_set_pagehash(X) +#define CHECK_PAGE(x) +#endif /* SQLITE_CHECK_PAGES */ + +/* +** When this is called the journal file for pager pPager must be open. +** This function attempts to read a super-journal file name from the +** end of the file and, if successful, copies it into memory supplied +** by the caller. See comments above writeSuperJournal() for the format +** used to store a super-journal file name at the end of a journal file. +** +** zSuper must point to a buffer of at least nSuper bytes allocated by +** the caller. This should be sqlite3_vfs.mxPathname+1 (to ensure there is +** enough space to write the super-journal name). If the super-journal +** name in the journal is longer than nSuper bytes (including a +** nul-terminator), then this is handled as if no super-journal name +** were present in the journal. +** +** If a super-journal file name is present at the end of the journal +** file, then it is copied into the buffer pointed to by zSuper. A +** nul-terminator byte is appended to the buffer following the +** super-journal file name. +** +** If it is determined that no super-journal file name is present +** zSuper[0] is set to 0 and SQLITE_OK returned. +** +** If an error occurs while reading from the journal file, an SQLite +** error code is returned. +*/ +static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u32 nSuper){ + int rc; /* Return code */ + u32 len; /* Length in bytes of super-journal name */ + i64 szJ; /* Total size in bytes of journal file pJrnl */ + u32 cksum; /* MJ checksum value read from journal */ + u32 u; /* Unsigned loop counter */ + unsigned char aMagic[8]; /* A buffer to hold the magic header */ + zSuper[0] = '\0'; + + if( SQLITE_OK!=(rc = sqlite3OsFileSize(pJrnl, &szJ)) + || szJ<16 + || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len)) + || len>=nSuper + || len>szJ-16 + || len==0 + || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum)) + || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8)) + || memcmp(aMagic, aJournalMagic, 8) + || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, zSuper, len, szJ-16-len)) + ){ + return rc; + } + + /* See if the checksum matches the super-journal name */ + for(u=0; ujournalOff, assuming a sector +** size of pPager->sectorSize bytes. +** +** i.e for a sector size of 512: +** +** Pager.journalOff Return value +** --------------------------------------- +** 0 0 +** 512 512 +** 100 512 +** 2000 2048 +** +*/ +static i64 journalHdrOffset(Pager *pPager){ + i64 offset = 0; + i64 c = pPager->journalOff; + if( c ){ + offset = ((c-1)/JOURNAL_HDR_SZ(pPager) + 1) * JOURNAL_HDR_SZ(pPager); + } + assert( offset%JOURNAL_HDR_SZ(pPager)==0 ); + assert( offset>=c ); + assert( (offset-c)jfd) ); + assert( !sqlite3JournalIsInMemory(pPager->jfd) ); + if( pPager->journalOff ){ + const i64 iLimit = pPager->journalSizeLimit; /* Local cache of jsl */ + + IOTRACE(("JZEROHDR %p\n", pPager)) + if( doTruncate || iLimit==0 ){ + rc = sqlite3OsTruncate(pPager->jfd, 0); + }else{ + static const char zeroHdr[28] = {0}; + rc = sqlite3OsWrite(pPager->jfd, zeroHdr, sizeof(zeroHdr), 0); + } + if( rc==SQLITE_OK && !pPager->noSync ){ + rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags); + } + + /* At this point the transaction is committed but the write lock + ** is still held on the file. If there is a size limit configured for + ** the persistent journal and the journal file currently consumes more + ** space than that limit allows for, truncate it now. There is no need + ** to sync the file following this operation. + */ + if( rc==SQLITE_OK && iLimit>0 ){ + i64 sz; + rc = sqlite3OsFileSize(pPager->jfd, &sz); + if( rc==SQLITE_OK && sz>iLimit ){ + rc = sqlite3OsTruncate(pPager->jfd, iLimit); + } + } + } + return rc; +} + +/* +** The journal file must be open when this routine is called. A journal +** header (JOURNAL_HDR_SZ bytes) is written into the journal file at the +** current location. +** +** The format for the journal header is as follows: +** - 8 bytes: Magic identifying journal format. +** - 4 bytes: Number of records in journal, or -1 no-sync mode is on. +** - 4 bytes: Random number used for page hash. +** - 4 bytes: Initial database page count. +** - 4 bytes: Sector size used by the process that wrote this journal. +** - 4 bytes: Database page size. +** +** Followed by (JOURNAL_HDR_SZ - 28) bytes of unused space. +*/ +static int writeJournalHdr(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + char *zHeader = pPager->pTmpSpace; /* Temporary space used to build header */ + u32 nHeader = (u32)pPager->pageSize;/* Size of buffer pointed to by zHeader */ + u32 nWrite; /* Bytes of header sector written */ + int ii; /* Loop counter */ + + assert( isOpen(pPager->jfd) ); /* Journal file must be open. */ + + if( nHeader>JOURNAL_HDR_SZ(pPager) ){ + nHeader = JOURNAL_HDR_SZ(pPager); + } + + /* If there are active savepoints and any of them were created + ** since the most recent journal header was written, update the + ** PagerSavepoint.iHdrOffset fields now. + */ + for(ii=0; iinSavepoint; ii++){ + if( pPager->aSavepoint[ii].iHdrOffset==0 ){ + pPager->aSavepoint[ii].iHdrOffset = pPager->journalOff; + } + } + + pPager->journalHdr = pPager->journalOff = journalHdrOffset(pPager); + + /* + ** Write the nRec Field - the number of page records that follow this + ** journal header. Normally, zero is written to this value at this time. + ** After the records are added to the journal (and the journal synced, + ** if in full-sync mode), the zero is overwritten with the true number + ** of records (see syncJournal()). + ** + ** A faster alternative is to write 0xFFFFFFFF to the nRec field. When + ** reading the journal this value tells SQLite to assume that the + ** rest of the journal file contains valid page records. This assumption + ** is dangerous, as if a failure occurred whilst writing to the journal + ** file it may contain some garbage data. There are two scenarios + ** where this risk can be ignored: + ** + ** * When the pager is in no-sync mode. Corruption can follow a + ** power failure in this case anyway. + ** + ** * When the SQLITE_IOCAP_SAFE_APPEND flag is set. This guarantees + ** that garbage data is never appended to the journal file. + */ + assert( isOpen(pPager->fd) || pPager->noSync ); + if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) + || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) + ){ + memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); + put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff); + }else{ + memset(zHeader, 0, sizeof(aJournalMagic)+4); + } + + /* The random check-hash initializer */ + sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); + /* The initial database size */ + put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize); + /* The assumed sector size for this process */ + put32bits(&zHeader[sizeof(aJournalMagic)+12], pPager->sectorSize); + + /* The page size */ + put32bits(&zHeader[sizeof(aJournalMagic)+16], pPager->pageSize); + + /* Initializing the tail of the buffer is not necessary. Everything + ** works find if the following memset() is omitted. But initializing + ** the memory prevents valgrind from complaining, so we are willing to + ** take the performance hit. + */ + memset(&zHeader[sizeof(aJournalMagic)+20], 0, + nHeader-(sizeof(aJournalMagic)+20)); + + /* In theory, it is only necessary to write the 28 bytes that the + ** journal header consumes to the journal file here. Then increment the + ** Pager.journalOff variable by JOURNAL_HDR_SZ so that the next + ** record is written to the following sector (leaving a gap in the file + ** that will be implicitly filled in by the OS). + ** + ** However it has been discovered that on some systems this pattern can + ** be significantly slower than contiguously writing data to the file, + ** even if that means explicitly writing data to the block of + ** (JOURNAL_HDR_SZ - 28) bytes that will not be used. So that is what + ** is done. + ** + ** The loop is required here in case the sector-size is larger than the + ** database page size. Since the zHeader buffer is only Pager.pageSize + ** bytes in size, more than one call to sqlite3OsWrite() may be required + ** to populate the entire journal header sector. + */ + for(nWrite=0; rc==SQLITE_OK&&nWritejournalHdr, nHeader)) + rc = sqlite3OsWrite(pPager->jfd, zHeader, nHeader, pPager->journalOff); + assert( pPager->journalHdr <= pPager->journalOff ); + pPager->journalOff += nHeader; + } + + return rc; +} + +/* +** The journal file must be open when this is called. A journal header file +** (JOURNAL_HDR_SZ bytes) is read from the current location in the journal +** file. The current location in the journal file is given by +** pPager->journalOff. See comments above function writeJournalHdr() for +** a description of the journal header format. +** +** If the header is read successfully, *pNRec is set to the number of +** page records following this header and *pDbSize is set to the size of the +** database before the transaction began, in pages. Also, pPager->cksumInit +** is set to the value read from the journal header. SQLITE_OK is returned +** in this case. +** +** If the journal header file appears to be corrupted, SQLITE_DONE is +** returned and *pNRec and *PDbSize are undefined. If JOURNAL_HDR_SZ bytes +** cannot be read from the journal file an error code is returned. +*/ +static int readJournalHdr( + Pager *pPager, /* Pager object */ + int isHot, + i64 journalSize, /* Size of the open journal file in bytes */ + u32 *pNRec, /* OUT: Value read from the nRec field */ + u32 *pDbSize /* OUT: Value of original database size field */ +){ + int rc; /* Return code */ + unsigned char aMagic[8]; /* A buffer to hold the magic header */ + i64 iHdrOff; /* Offset of journal header being read */ + + assert( isOpen(pPager->jfd) ); /* Journal file must be open. */ + + /* Advance Pager.journalOff to the start of the next sector. If the + ** journal file is too small for there to be a header stored at this + ** point, return SQLITE_DONE. + */ + pPager->journalOff = journalHdrOffset(pPager); + if( pPager->journalOff+JOURNAL_HDR_SZ(pPager) > journalSize ){ + return SQLITE_DONE; + } + iHdrOff = pPager->journalOff; + + /* Read in the first 8 bytes of the journal header. If they do not match + ** the magic string found at the start of each journal header, return + ** SQLITE_DONE. If an IO error occurs, return an error code. Otherwise, + ** proceed. + */ + if( isHot || iHdrOff!=pPager->journalHdr ){ + rc = sqlite3OsRead(pPager->jfd, aMagic, sizeof(aMagic), iHdrOff); + if( rc ){ + return rc; + } + if( memcmp(aMagic, aJournalMagic, sizeof(aMagic))!=0 ){ + return SQLITE_DONE; + } + } + + /* Read the first three 32-bit fields of the journal header: The nRec + ** field, the checksum-initializer and the database size at the start + ** of the transaction. Return an error code if anything goes wrong. + */ + if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+8, pNRec)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+12, &pPager->cksumInit)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+16, pDbSize)) + ){ + return rc; + } + + if( pPager->journalOff==0 ){ + u32 iPageSize; /* Page-size field of journal header */ + u32 iSectorSize; /* Sector-size field of journal header */ + + /* Read the page-size and sector-size journal header fields. */ + if( SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+20, &iSectorSize)) + || SQLITE_OK!=(rc = read32bits(pPager->jfd, iHdrOff+24, &iPageSize)) + ){ + return rc; + } + + /* Versions of SQLite prior to 3.5.8 set the page-size field of the + ** journal header to zero. In this case, assume that the Pager.pageSize + ** variable is already set to the correct page size. + */ + if( iPageSize==0 ){ + iPageSize = pPager->pageSize; + } + + /* Check that the values read from the page-size and sector-size fields + ** are within range. To be 'in range', both values need to be a power + ** of two greater than or equal to 512 or 32, and not greater than their + ** respective compile time maximum limits. + */ + if( iPageSize<512 || iSectorSize<32 + || iPageSize>SQLITE_MAX_PAGE_SIZE || iSectorSize>MAX_SECTOR_SIZE + || ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0 + ){ + /* If the either the page-size or sector-size in the journal-header is + ** invalid, then the process that wrote the journal-header must have + ** crashed before the header was synced. In this case stop reading + ** the journal file here. + */ + return SQLITE_DONE; + } + + /* Update the page-size to match the value read from the journal. + ** Use a testcase() macro to make sure that malloc failure within + ** PagerSetPagesize() is tested. + */ + rc = sqlite3PagerSetPagesize(pPager, &iPageSize, -1); + testcase( rc!=SQLITE_OK ); + + /* Update the assumed sector-size to match the value used by + ** the process that created this journal. If this journal was + ** created by a process other than this one, then this routine + ** is being called from within pager_playback(). The local value + ** of Pager.sectorSize is restored at the end of that routine. + */ + pPager->sectorSize = iSectorSize; + } + + pPager->journalOff += JOURNAL_HDR_SZ(pPager); + return rc; +} + + +/* +** Write the supplied super-journal name into the journal file for pager +** pPager at the current location. The super-journal name must be the last +** thing written to a journal file. If the pager is in full-sync mode, the +** journal file descriptor is advanced to the next sector boundary before +** anything is written. The format is: +** +** + 4 bytes: PAGER_MJ_PGNO. +** + N bytes: super-journal filename in utf-8. +** + 4 bytes: N (length of super-journal name in bytes, no nul-terminator). +** + 4 bytes: super-journal name checksum. +** + 8 bytes: aJournalMagic[]. +** +** The super-journal page checksum is the sum of the bytes in thesuper-journal +** name, where each byte is interpreted as a signed 8-bit integer. +** +** If zSuper is a NULL pointer (occurs for a single database transaction), +** this call is a no-op. +*/ +static int writeSuperJournal(Pager *pPager, const char *zSuper){ + int rc; /* Return code */ + int nSuper; /* Length of string zSuper */ + i64 iHdrOff; /* Offset of header in journal file */ + i64 jrnlSize; /* Size of journal file on disk */ + u32 cksum = 0; /* Checksum of string zSuper */ + + assert( pPager->setSuper==0 ); + assert( !pagerUseWal(pPager) ); + + if( !zSuper + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + || !isOpen(pPager->jfd) + ){ + return SQLITE_OK; + } + pPager->setSuper = 1; + assert( pPager->journalHdr <= pPager->journalOff ); + + /* Calculate the length in bytes and the checksum of zSuper */ + for(nSuper=0; zSuper[nSuper]; nSuper++){ + cksum += zSuper[nSuper]; + } + + /* If in full-sync mode, advance to the next disk sector before writing + ** the super-journal name. This is in case the previous page written to + ** the journal has already been synced. + */ + if( pPager->fullSync ){ + pPager->journalOff = journalHdrOffset(pPager); + } + iHdrOff = pPager->journalOff; + + /* Write the super-journal data to the end of the journal file. If + ** an error occurs, return the error code to the caller. + */ + if( (0 != (rc = write32bits(pPager->jfd, iHdrOff, PAGER_MJ_PGNO(pPager)))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, zSuper, nSuper, iHdrOff+4))) + || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nSuper, nSuper))) + || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nSuper+4, cksum))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, + iHdrOff+4+nSuper+8))) + ){ + return rc; + } + pPager->journalOff += (nSuper+20); + + /* If the pager is in peristent-journal mode, then the physical + ** journal-file may extend past the end of the super-journal name + ** and 8 bytes of magic data just written to the file. This is + ** dangerous because the code to rollback a hot-journal file + ** will not be able to find the super-journal name to determine + ** whether or not the journal is hot. + ** + ** Easiest thing to do in this scenario is to truncate the journal + ** file to the required size. + */ + if( SQLITE_OK==(rc = sqlite3OsFileSize(pPager->jfd, &jrnlSize)) + && jrnlSize>pPager->journalOff + ){ + rc = sqlite3OsTruncate(pPager->jfd, pPager->journalOff); + } + return rc; +} + +/* +** Discard the entire contents of the in-memory page-cache. +*/ +static void pager_reset(Pager *pPager){ + pPager->iDataVersion++; + sqlite3BackupRestart(pPager->pBackup); + sqlite3PcacheClear(pPager->pPCache); +} + +/* +** Return the pPager->iDataVersion value +*/ +SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager *pPager){ + return pPager->iDataVersion; +} + +/* +** Free all structures in the Pager.aSavepoint[] array and set both +** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal +** if it is open and the pager is not in exclusive mode. +*/ +static void releaseAllSavepoints(Pager *pPager){ + int ii; /* Iterator for looping through Pager.aSavepoint */ + for(ii=0; iinSavepoint; ii++){ + sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); + } + if( !pPager->exclusiveMode || sqlite3JournalIsInMemory(pPager->sjfd) ){ + sqlite3OsClose(pPager->sjfd); + } + sqlite3_free(pPager->aSavepoint); + pPager->aSavepoint = 0; + pPager->nSavepoint = 0; + pPager->nSubRec = 0; +} + +/* +** Set the bit number pgno in the PagerSavepoint.pInSavepoint +** bitvecs of all open savepoints. Return SQLITE_OK if successful +** or SQLITE_NOMEM if a malloc failure occurs. +*/ +static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ + int ii; /* Loop counter */ + int rc = SQLITE_OK; /* Result code */ + + for(ii=0; iinSavepoint; ii++){ + PagerSavepoint *p = &pPager->aSavepoint[ii]; + if( pgno<=p->nOrig ){ + rc |= sqlite3BitvecSet(p->pInSavepoint, pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + } + } + return rc; +} + +/* +** This function is a no-op if the pager is in exclusive mode and not +** in the ERROR state. Otherwise, it switches the pager to PAGER_OPEN +** state. +** +** If the pager is not in exclusive-access mode, the database file is +** completely unlocked. If the file is unlocked and the file-system does +** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is +** closed (if it is open). +** +** If the pager is in ERROR state when this function is called, the +** contents of the pager cache are discarded before switching back to +** the OPEN state. Regardless of whether the pager is in exclusive-mode +** or not, any journal file left in the file-system will be treated +** as a hot-journal and rolled back the next time a read-transaction +** is opened (by this or by any other connection). +*/ +static void pager_unlock(Pager *pPager){ + + assert( pPager->eState==PAGER_READER + || pPager->eState==PAGER_OPEN + || pPager->eState==PAGER_ERROR + ); + + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + releaseAllSavepoints(pPager); + + if( pagerUseWal(pPager) ){ + assert( !isOpen(pPager->jfd) ); + sqlite3WalEndReadTransaction(pPager->pWal); + pPager->eState = PAGER_OPEN; + }else if( !pPager->exclusiveMode ){ + int rc; /* Error code returned by pagerUnlockDb() */ + int iDc = isOpen(pPager->fd)?sqlite3OsDeviceCharacteristics(pPager->fd):0; + + /* If the operating system support deletion of open files, then + ** close the journal file when dropping the database lock. Otherwise + ** another connection with journal_mode=delete might delete the file + ** out from under us. + */ + assert( (PAGER_JOURNALMODE_MEMORY & 5)!=1 ); + assert( (PAGER_JOURNALMODE_OFF & 5)!=1 ); + assert( (PAGER_JOURNALMODE_WAL & 5)!=1 ); + assert( (PAGER_JOURNALMODE_DELETE & 5)!=1 ); + assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); + assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); + if( 0==(iDc & SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN) + || 1!=(pPager->journalMode & 5) + ){ + sqlite3OsClose(pPager->jfd); + } + + /* If the pager is in the ERROR state and the call to unlock the database + ** file fails, set the current lock to UNKNOWN_LOCK. See the comment + ** above the #define for UNKNOWN_LOCK for an explanation of why this + ** is necessary. + */ + rc = pagerUnlockDb(pPager, NO_LOCK); + if( rc!=SQLITE_OK && pPager->eState==PAGER_ERROR ){ + pPager->eLock = UNKNOWN_LOCK; + } + + /* The pager state may be changed from PAGER_ERROR to PAGER_OPEN here + ** without clearing the error code. This is intentional - the error + ** code is cleared and the cache reset in the block below. + */ + assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); + pPager->eState = PAGER_OPEN; + } + + /* If Pager.errCode is set, the contents of the pager cache cannot be + ** trusted. Now that there are no outstanding references to the pager, + ** it can safely move back to PAGER_OPEN state. This happens in both + ** normal and exclusive-locking mode. + */ + assert( pPager->errCode==SQLITE_OK || !MEMDB ); + if( pPager->errCode ){ + if( pPager->tempFile==0 ){ + pager_reset(pPager); + pPager->changeCountDone = 0; + pPager->eState = PAGER_OPEN; + }else{ + pPager->eState = (isOpen(pPager->jfd) ? PAGER_OPEN : PAGER_READER); + } + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + pPager->errCode = SQLITE_OK; + setGetterMethod(pPager); + } + + pPager->journalOff = 0; + pPager->journalHdr = 0; + pPager->setSuper = 0; +} + +/* +** This function is called whenever an IOERR or FULL error that requires +** the pager to transition into the ERROR state may ahve occurred. +** The first argument is a pointer to the pager structure, the second +** the error-code about to be returned by a pager API function. The +** value returned is a copy of the second argument to this function. +** +** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the +** IOERR sub-codes, the pager enters the ERROR state and the error code +** is stored in Pager.errCode. While the pager remains in the ERROR state, +** all major API calls on the Pager will immediately return Pager.errCode. +** +** The ERROR state indicates that the contents of the pager-cache +** cannot be trusted. This state can be cleared by completely discarding +** the contents of the pager-cache. If a transaction was active when +** the persistent error occurred, then the rollback journal may need +** to be replayed to restore the contents of the database file (as if +** it were a hot-journal). +*/ +static int pager_error(Pager *pPager, int rc){ + int rc2 = rc & 0xff; + assert( rc==SQLITE_OK || !MEMDB ); + assert( + pPager->errCode==SQLITE_FULL || + pPager->errCode==SQLITE_OK || + (pPager->errCode & 0xff)==SQLITE_IOERR + ); + if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){ + pPager->errCode = rc; + pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); + } + return rc; +} + +static int pager_truncate(Pager *pPager, Pgno nPage); + +/* +** The write transaction open on pPager is being committed (bCommit==1) +** or rolled back (bCommit==0). +** +** Return TRUE if and only if all dirty pages should be flushed to disk. +** +** Rules: +** +** * For non-TEMP databases, always sync to disk. This is necessary +** for transactions to be durable. +** +** * Sync TEMP database only on a COMMIT (not a ROLLBACK) when the backing +** file has been created already (via a spill on pagerStress()) and +** when the number of dirty pages in memory exceeds 25% of the total +** cache size. +*/ +static int pagerFlushOnCommit(Pager *pPager, int bCommit){ + if( pPager->tempFile==0 ) return 1; + if( !bCommit ) return 0; + if( !isOpen(pPager->fd) ) return 0; + return (sqlite3PCachePercentDirty(pPager->pPCache)>=25); +} + +/* +** This routine ends a transaction. A transaction is usually ended by +** either a COMMIT or a ROLLBACK operation. This routine may be called +** after rollback of a hot-journal, or if an error occurs while opening +** the journal file or writing the very first journal-header of a +** database transaction. +** +** This routine is never called in PAGER_ERROR state. If it is called +** in PAGER_NONE or PAGER_SHARED state and the lock held is less +** exclusive than a RESERVED lock, it is a no-op. +** +** Otherwise, any active savepoints are released. +** +** If the journal file is open, then it is "finalized". Once a journal +** file has been finalized it is not possible to use it to roll back a +** transaction. Nor will it be considered to be a hot-journal by this +** or any other database connection. Exactly how a journal is finalized +** depends on whether or not the pager is running in exclusive mode and +** the current journal-mode (Pager.journalMode value), as follows: +** +** journalMode==MEMORY +** Journal file descriptor is simply closed. This destroys an +** in-memory journal. +** +** journalMode==TRUNCATE +** Journal file is truncated to zero bytes in size. +** +** journalMode==PERSIST +** The first 28 bytes of the journal file are zeroed. This invalidates +** the first journal header in the file, and hence the entire journal +** file. An invalid journal file cannot be rolled back. +** +** journalMode==DELETE +** The journal file is closed and deleted using sqlite3OsDelete(). +** +** If the pager is running in exclusive mode, this method of finalizing +** the journal file is never used. Instead, if the journalMode is +** DELETE and the pager is in exclusive mode, the method described under +** journalMode==PERSIST is used instead. +** +** After the journal is finalized, the pager moves to PAGER_READER state. +** If running in non-exclusive rollback mode, the lock on the file is +** downgraded to a SHARED_LOCK. +** +** SQLITE_OK is returned if no error occurs. If an error occurs during +** any of the IO operations to finalize the journal file or unlock the +** database then the IO error code is returned to the user. If the +** operation to finalize the journal file fails, then the code still +** tries to unlock the database file if not in exclusive mode. If the +** unlock operation fails as well, then the first error code related +** to the first error encountered (the journal finalization one) is +** returned. +*/ +static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ + int rc = SQLITE_OK; /* Error code from journal finalization operation */ + int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ + + /* Do nothing if the pager does not have an open write transaction + ** or at least a RESERVED lock. This function may be called when there + ** is no write-transaction active but a RESERVED or greater lock is + ** held under two circumstances: + ** + ** 1. After a successful hot-journal rollback, it is called with + ** eState==PAGER_NONE and eLock==EXCLUSIVE_LOCK. + ** + ** 2. If a connection with locking_mode=exclusive holding an EXCLUSIVE + ** lock switches back to locking_mode=normal and then executes a + ** read-transaction, this function is called with eState==PAGER_READER + ** and eLock==EXCLUSIVE_LOCK when the read-transaction is closed. + */ + assert( assert_pager_state(pPager) ); + assert( pPager->eState!=PAGER_ERROR ); + if( pPager->eStateeLockjfd) || pPager->pInJournal==0 + || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_BATCH_ATOMIC) + ); + if( isOpen(pPager->jfd) ){ + assert( !pagerUseWal(pPager) ); + + /* Finalize the journal file. */ + if( sqlite3JournalIsInMemory(pPager->jfd) ){ + /* assert( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ); */ + sqlite3OsClose(pPager->jfd); + }else if( pPager->journalMode==PAGER_JOURNALMODE_TRUNCATE ){ + if( pPager->journalOff==0 ){ + rc = SQLITE_OK; + }else{ + rc = sqlite3OsTruncate(pPager->jfd, 0); + if( rc==SQLITE_OK && pPager->fullSync ){ + /* Make sure the new file size is written into the inode right away. + ** Otherwise the journal might resurrect following a power loss and + ** cause the last transaction to roll back. See + ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773 + */ + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + } + } + pPager->journalOff = 0; + }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST + || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) + ){ + rc = zeroJournalHdr(pPager, hasSuper||pPager->tempFile); + pPager->journalOff = 0; + }else{ + /* This branch may be executed with Pager.journalMode==MEMORY if + ** a hot-journal was just rolled back. In this case the journal + ** file should be closed and deleted. If this connection writes to + ** the database file, it will do so using an in-memory journal. + */ + int bDelete = !pPager->tempFile; + assert( sqlite3JournalIsInMemory(pPager->jfd)==0 ); + assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + || pPager->journalMode==PAGER_JOURNALMODE_WAL + ); + sqlite3OsClose(pPager->jfd); + if( bDelete ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zJournal, pPager->extraSync); + } + } + } + +#ifdef SQLITE_CHECK_PAGES + sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); + if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ + PgHdr *p = sqlite3PagerLookup(pPager, 1); + if( p ){ + p->pageHash = 0; + sqlite3PagerUnrefNotNull(p); + } + } +#endif + + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + pPager->nRec = 0; + if( rc==SQLITE_OK ){ + if( MEMDB || pagerFlushOnCommit(pPager, bCommit) ){ + sqlite3PcacheCleanAll(pPager->pPCache); + }else{ + sqlite3PcacheClearWritable(pPager->pPCache); + } + sqlite3PcacheTruncate(pPager->pPCache, pPager->dbSize); + } + + if( pagerUseWal(pPager) ){ + /* Drop the WAL write-lock, if any. Also, if the connection was in + ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE + ** lock held on the database file. + */ + rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); + assert( rc2==SQLITE_OK ); + }else if( rc==SQLITE_OK && bCommit && pPager->dbFileSize>pPager->dbSize ){ + /* This branch is taken when committing a transaction in rollback-journal + ** mode if the database file on disk is larger than the database image. + ** At this point the journal has been finalized and the transaction + ** successfully committed, but the EXCLUSIVE lock is still held on the + ** file. So it is safe to truncate the database file to its minimum + ** required size. */ + assert( pPager->eLock==EXCLUSIVE_LOCK ); + rc = pager_truncate(pPager, pPager->dbSize); + } + + if( rc==SQLITE_OK && bCommit ){ + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_COMMIT_PHASETWO, 0); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + } + + if( !pPager->exclusiveMode + && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) + ){ + rc2 = pagerUnlockDb(pPager, SHARED_LOCK); + } + pPager->eState = PAGER_READER; + pPager->setSuper = 0; + + return (rc==SQLITE_OK?rc2:rc); +} + +/* +** Execute a rollback if a transaction is active and unlock the +** database file. +** +** If the pager has already entered the ERROR state, do not attempt +** the rollback at this time. Instead, pager_unlock() is called. The +** call to pager_unlock() will discard all in-memory pages, unlock +** the database file and move the pager back to OPEN state. If this +** means that there is a hot-journal left in the file-system, the next +** connection to obtain a shared lock on the pager (which may be this one) +** will roll it back. +** +** If the pager has not already entered the ERROR state, but an IO or +** malloc error occurs during a rollback, then this will itself cause +** the pager to enter the ERROR state. Which will be cleared by the +** call to pager_unlock(), as described above. +*/ +static void pagerUnlockAndRollback(Pager *pPager){ + if( pPager->eState!=PAGER_ERROR && pPager->eState!=PAGER_OPEN ){ + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_LOCKED ){ + sqlite3BeginBenignMalloc(); + sqlite3PagerRollback(pPager); + sqlite3EndBenignMalloc(); + }else if( !pPager->exclusiveMode ){ + assert( pPager->eState==PAGER_READER ); + pager_end_transaction(pPager, 0, 0); + } + } + pager_unlock(pPager); +} + +/* +** Parameter aData must point to a buffer of pPager->pageSize bytes +** of data. Compute and return a checksum based ont the contents of the +** page of data and the current value of pPager->cksumInit. +** +** This is not a real checksum. It is really just the sum of the +** random initial value (pPager->cksumInit) and every 200th byte +** of the page data, starting with byte offset (pPager->pageSize%200). +** Each byte is interpreted as an 8-bit unsigned integer. +** +** Changing the formula used to compute this checksum results in an +** incompatible journal file format. +** +** If journal corruption occurs due to a power failure, the most likely +** scenario is that one end or the other of the record will be changed. +** It is much less likely that the two ends of the journal record will be +** correct and the middle be corrupt. Thus, this "checksum" scheme, +** though fast and simple, catches the mostly likely kind of corruption. +*/ +static u32 pager_cksum(Pager *pPager, const u8 *aData){ + u32 cksum = pPager->cksumInit; /* Checksum value to return */ + int i = pPager->pageSize-200; /* Loop counter */ + while( i>0 ){ + cksum += aData[i]; + i -= 200; + } + return cksum; +} + +/* +** Read a single page from either the journal file (if isMainJrnl==1) or +** from the sub-journal (if isMainJrnl==0) and playback that page. +** The page begins at offset *pOffset into the file. The *pOffset +** value is increased to the start of the next page in the journal. +** +** The main rollback journal uses checksums - the statement journal does +** not. +** +** If the page number of the page record read from the (sub-)journal file +** is greater than the current value of Pager.dbSize, then playback is +** skipped and SQLITE_OK is returned. +** +** If pDone is not NULL, then it is a record of pages that have already +** been played back. If the page at *pOffset has already been played back +** (if the corresponding pDone bit is set) then skip the playback. +** Make sure the pDone bit corresponding to the *pOffset page is set +** prior to returning. +** +** If the page record is successfully read from the (sub-)journal file +** and played back, then SQLITE_OK is returned. If an IO error occurs +** while reading the record from the (sub-)journal file or while writing +** to the database file, then the IO error code is returned. If data +** is successfully read from the (sub-)journal file but appears to be +** corrupted, SQLITE_DONE is returned. Data is considered corrupted in +** two circumstances: +** +** * If the record page-number is illegal (0 or PAGER_MJ_PGNO), or +** * If the record is being rolled back from the main journal file +** and the checksum field does not match the record content. +** +** Neither of these two scenarios are possible during a savepoint rollback. +** +** If this is a savepoint rollback, then memory may have to be dynamically +** allocated by this function. If this is the case and an allocation fails, +** SQLITE_NOMEM is returned. +*/ +static int pager_playback_one_page( + Pager *pPager, /* The pager being played back */ + i64 *pOffset, /* Offset of record to playback */ + Bitvec *pDone, /* Bitvec of pages already played back */ + int isMainJrnl, /* 1 -> main journal. 0 -> sub-journal. */ + int isSavepnt /* True for a savepoint rollback */ +){ + int rc; + PgHdr *pPg; /* An existing page in the cache */ + Pgno pgno; /* The page number of a page in journal */ + u32 cksum; /* Checksum used for sanity checking */ + char *aData; /* Temporary storage for the page */ + sqlite3_file *jfd; /* The file descriptor for the journal file */ + int isSynced; /* True if journal page is synced */ + + assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */ + assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */ + assert( isMainJrnl || pDone ); /* pDone always used on sub-journals */ + assert( isSavepnt || pDone==0 ); /* pDone never used on non-savepoint */ + + aData = pPager->pTmpSpace; + assert( aData ); /* Temp storage must have already been allocated */ + assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); + + /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction + ** or savepoint rollback done at the request of the caller) or this is + ** a hot-journal rollback. If it is a hot-journal rollback, the pager + ** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback + ** only reads from the main journal, not the sub-journal. + */ + assert( pPager->eState>=PAGER_WRITER_CACHEMOD + || (pPager->eState==PAGER_OPEN && pPager->eLock==EXCLUSIVE_LOCK) + ); + assert( pPager->eState>=PAGER_WRITER_CACHEMOD || isMainJrnl ); + + /* Read the page number and page data from the journal or sub-journal + ** file. Return an error code to the caller if an IO error occurs. + */ + jfd = isMainJrnl ? pPager->jfd : pPager->sjfd; + rc = read32bits(jfd, *pOffset, &pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsRead(jfd, (u8*)aData, pPager->pageSize, (*pOffset)+4); + if( rc!=SQLITE_OK ) return rc; + *pOffset += pPager->pageSize + 4 + isMainJrnl*4; + + /* Sanity checking on the page. This is more important that I originally + ** thought. If a power failure occurs while the journal is being written, + ** it could cause invalid data to be written into the journal. We need to + ** detect this invalid data (with high probability) and ignore it. + */ + if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){ + assert( !isSavepnt ); + return SQLITE_DONE; + } + if( pgno>(Pgno)pPager->dbSize || sqlite3BitvecTest(pDone, pgno) ){ + return SQLITE_OK; + } + if( isMainJrnl ){ + rc = read32bits(jfd, (*pOffset)-4, &cksum); + if( rc ) return rc; + if( !isSavepnt && pager_cksum(pPager, (u8*)aData)!=cksum ){ + return SQLITE_DONE; + } + } + + /* If this page has already been played back before during the current + ** rollback, then don't bother to play it back again. + */ + if( pDone && (rc = sqlite3BitvecSet(pDone, pgno))!=SQLITE_OK ){ + return rc; + } + + /* When playing back page 1, restore the nReserve setting + */ + if( pgno==1 && pPager->nReserve!=((u8*)aData)[20] ){ + pPager->nReserve = ((u8*)aData)[20]; + } + + /* If the pager is in CACHEMOD state, then there must be a copy of this + ** page in the pager cache. In this case just update the pager cache, + ** not the database file. The page is left marked dirty in this case. + ** + ** An exception to the above rule: If the database is in no-sync mode + ** and a page is moved during an incremental vacuum then the page may + ** not be in the pager cache. Later: if a malloc() or IO error occurs + ** during a Movepage() call, then the page may not be in the cache + ** either. So the condition described in the above paragraph is not + ** assert()able. + ** + ** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the + ** pager cache if it exists and the main file. The page is then marked + ** not dirty. Since this code is only executed in PAGER_OPEN state for + ** a hot-journal rollback, it is guaranteed that the page-cache is empty + ** if the pager is in OPEN state. + ** + ** Ticket #1171: The statement journal might contain page content that is + ** different from the page content at the start of the transaction. + ** This occurs when a page is changed prior to the start of a statement + ** then changed again within the statement. When rolling back such a + ** statement we must not write to the original database unless we know + ** for certain that original page contents are synced into the main rollback + ** journal. Otherwise, a power loss might leave modified data in the + ** database file without an entry in the rollback journal that can + ** restore the database to its original form. Two conditions must be + ** met before writing to the database files. (1) the database must be + ** locked. (2) we know that the original page content is fully synced + ** in the main journal either because the page is not in cache or else + ** the page is marked as needSync==0. + ** + ** 2008-04-14: When attempting to vacuum a corrupt database file, it + ** is possible to fail a statement on a database that does not yet exist. + ** Do not attempt to write if database file has never been opened. + */ + if( pagerUseWal(pPager) ){ + pPg = 0; + }else{ + pPg = sqlite3PagerLookup(pPager, pgno); + } + assert( pPg || !MEMDB ); + assert( pPager->eState!=PAGER_OPEN || pPg==0 || pPager->tempFile ); + PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", + PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), + (isMainJrnl?"main-journal":"sub-journal") + )); + if( isMainJrnl ){ + isSynced = pPager->noSync || (*pOffset <= pPager->journalHdr); + }else{ + isSynced = (pPg==0 || 0==(pPg->flags & PGHDR_NEED_SYNC)); + } + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + && isSynced + ){ + i64 ofst = (pgno-1)*(i64)pPager->pageSize; + testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); + assert( !pagerUseWal(pPager) ); + + /* Write the data read from the journal back into the database file. + ** This is usually safe even for an encrypted database - as the data + ** was encrypted before it was written to the journal file. The exception + ** is if the data was just read from an in-memory sub-journal. In that + ** case it must be encrypted here before it is copied into the database + ** file. */ + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); + + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; + } + if( pPager->pBackup ){ + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); + } + }else if( !isMainJrnl && pPg==0 ){ + /* If this is a rollback of a savepoint and data was not written to + ** the database and the page is not in-memory, there is a potential + ** problem. When the page is next fetched by the b-tree layer, it + ** will be read from the database file, which may or may not be + ** current. + ** + ** There are a couple of different ways this can happen. All are quite + ** obscure. When running in synchronous mode, this can only happen + ** if the page is on the free-list at the start of the transaction, then + ** populated, then moved using sqlite3PagerMovepage(). + ** + ** The solution is to add an in-memory page to the cache containing + ** the data just read from the sub-journal. Mark the page as dirty + ** and if the pager requires a journal-sync, then mark the page as + ** requiring a journal-sync before it is written. + */ + assert( isSavepnt ); + assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)==0 ); + pPager->doNotSpill |= SPILLFLAG_ROLLBACK; + rc = sqlite3PagerGet(pPager, pgno, &pPg, 1); + assert( (pPager->doNotSpill & SPILLFLAG_ROLLBACK)!=0 ); + pPager->doNotSpill &= ~SPILLFLAG_ROLLBACK; + if( rc!=SQLITE_OK ) return rc; + sqlite3PcacheMakeDirty(pPg); + } + if( pPg ){ + /* No page should ever be explicitly rolled back that is in use, except + ** for page 1 which is held in use in order to keep the lock on the + ** database active. However such a page may be rolled back as a result + ** of an internal error resulting in an automatic call to + ** sqlite3PagerRollback(). + */ + void *pData; + pData = pPg->pData; + memcpy(pData, (u8*)aData, pPager->pageSize); + pPager->xReiniter(pPg); + /* It used to be that sqlite3PcacheMakeClean(pPg) was called here. But + ** that call was dangerous and had no detectable benefit since the cache + ** is normally cleaned by sqlite3PcacheCleanAll() after rollback and so + ** has been removed. */ + pager_set_pagehash(pPg); + + /* If this was page 1, then restore the value of Pager.dbFileVers. + ** Do this before any decoding. */ + if( pgno==1 ){ + memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); + } + sqlite3PcacheRelease(pPg); + } + return rc; +} + +/* +** Parameter zSuper is the name of a super-journal file. A single journal +** file that referred to the super-journal file has just been rolled back. +** This routine checks if it is possible to delete the super-journal file, +** and does so if it is. +** +** Argument zSuper may point to Pager.pTmpSpace. So that buffer is not +** available for use within this function. +** +** When a super-journal file is created, it is populated with the names +** of all of its child journals, one after another, formatted as utf-8 +** encoded text. The end of each child journal file is marked with a +** nul-terminator byte (0x00). i.e. the entire contents of a super-journal +** file for a transaction involving two databases might be: +** +** "/home/bill/a.db-journal\x00/home/bill/b.db-journal\x00" +** +** A super-journal file may only be deleted once all of its child +** journals have been rolled back. +** +** This function reads the contents of the super-journal file into +** memory and loops through each of the child journal names. For +** each child journal, it checks if: +** +** * if the child journal exists, and if so +** * if the child journal contains a reference to super-journal +** file zSuper +** +** If a child journal can be found that matches both of the criteria +** above, this function returns without doing anything. Otherwise, if +** no such child journal can be found, file zSuper is deleted from +** the file-system using sqlite3OsDelete(). +** +** If an IO error within this function, an error code is returned. This +** function allocates memory by calling sqlite3Malloc(). If an allocation +** fails, SQLITE_NOMEM is returned. Otherwise, if no IO or malloc errors +** occur, SQLITE_OK is returned. +** +** TODO: This function allocates a single block of memory to load +** the entire contents of the super-journal file. This could be +** a couple of kilobytes or so - potentially larger than the page +** size. +*/ +static int pager_delsuper(Pager *pPager, const char *zSuper){ + sqlite3_vfs *pVfs = pPager->pVfs; + int rc; /* Return code */ + sqlite3_file *pSuper; /* Malloc'd super-journal file descriptor */ + sqlite3_file *pJournal; /* Malloc'd child-journal file descriptor */ + char *zSuperJournal = 0; /* Contents of super-journal file */ + i64 nSuperJournal; /* Size of super-journal file */ + char *zJournal; /* Pointer to one journal within MJ file */ + char *zSuperPtr; /* Space to hold super-journal filename */ + char *zFree = 0; /* Free this buffer */ + int nSuperPtr; /* Amount of space allocated to zSuperPtr[] */ + + /* Allocate space for both the pJournal and pSuper file descriptors. + ** If successful, open the super-journal file for reading. + */ + pSuper = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); + if( !pSuper ){ + rc = SQLITE_NOMEM_BKPT; + pJournal = 0; + }else{ + const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_SUPER_JOURNAL); + rc = sqlite3OsOpen(pVfs, zSuper, pSuper, flags, 0); + pJournal = (sqlite3_file *)(((u8 *)pSuper) + pVfs->szOsFile); + } + if( rc!=SQLITE_OK ) goto delsuper_out; + + /* Load the entire super-journal file into space obtained from + ** sqlite3_malloc() and pointed to by zSuperJournal. Also obtain + ** sufficient space (in zSuperPtr) to hold the names of super-journal + ** files extracted from regular rollback-journals. + */ + rc = sqlite3OsFileSize(pSuper, &nSuperJournal); + if( rc!=SQLITE_OK ) goto delsuper_out; + nSuperPtr = pVfs->mxPathname+1; + zFree = sqlite3Malloc(4 + nSuperJournal + nSuperPtr + 2); + if( !zFree ){ + rc = SQLITE_NOMEM_BKPT; + goto delsuper_out; + } + zFree[0] = zFree[1] = zFree[2] = zFree[3] = 0; + zSuperJournal = &zFree[4]; + zSuperPtr = &zSuperJournal[nSuperJournal+2]; + rc = sqlite3OsRead(pSuper, zSuperJournal, (int)nSuperJournal, 0); + if( rc!=SQLITE_OK ) goto delsuper_out; + zSuperJournal[nSuperJournal] = 0; + zSuperJournal[nSuperJournal+1] = 0; + + zJournal = zSuperJournal; + while( (zJournal-zSuperJournal)pageSize bytes). +** If the file on disk is currently larger than nPage pages, then use the VFS +** xTruncate() method to truncate it. +** +** Or, it might be the case that the file on disk is smaller than +** nPage pages. Some operating system implementations can get confused if +** you try to truncate a file to some size that is larger than it +** currently is, so detect this case and write a single zero byte to +** the end of the new file instead. +** +** If successful, return SQLITE_OK. If an IO error occurs while modifying +** the database file, return the error code to the caller. +*/ +static int pager_truncate(Pager *pPager, Pgno nPage){ + int rc = SQLITE_OK; + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState!=PAGER_READER ); + + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ + i64 currentSize, newSize; + int szPage = pPager->pageSize; + assert( pPager->eLock==EXCLUSIVE_LOCK ); + /* TODO: Is it safe to use Pager.dbFileSize here? */ + rc = sqlite3OsFileSize(pPager->fd, ¤tSize); + newSize = szPage*(i64)nPage; + if( rc==SQLITE_OK && currentSize!=newSize ){ + if( currentSize>newSize ){ + rc = sqlite3OsTruncate(pPager->fd, newSize); + }else if( (currentSize+szPage)<=newSize ){ + char *pTmp = pPager->pTmpSpace; + memset(pTmp, 0, szPage); + testcase( (newSize-szPage) == currentSize ); + testcase( (newSize-szPage) > currentSize ); + rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, newSize-szPage); + } + if( rc==SQLITE_OK ){ + pPager->dbFileSize = nPage; + } + } + } + return rc; +} + +/* +** Return a sanitized version of the sector-size of OS file pFile. The +** return value is guaranteed to lie between 32 and MAX_SECTOR_SIZE. +*/ +SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *pFile){ + int iRet = sqlite3OsSectorSize(pFile); + if( iRet<32 ){ + iRet = 512; + }else if( iRet>MAX_SECTOR_SIZE ){ + assert( MAX_SECTOR_SIZE>=512 ); + iRet = MAX_SECTOR_SIZE; + } + return iRet; +} + +/* +** Set the value of the Pager.sectorSize variable for the given +** pager based on the value returned by the xSectorSize method +** of the open database file. The sector size will be used +** to determine the size and alignment of journal header and +** super-journal pointers within created journal files. +** +** For temporary files the effective sector size is always 512 bytes. +** +** Otherwise, for non-temporary files, the effective sector size is +** the value returned by the xSectorSize() method rounded up to 32 if +** it is less than 32, or rounded down to MAX_SECTOR_SIZE if it +** is greater than MAX_SECTOR_SIZE. +** +** If the file has the SQLITE_IOCAP_POWERSAFE_OVERWRITE property, then set +** the effective sector size to its minimum value (512). The purpose of +** pPager->sectorSize is to define the "blast radius" of bytes that +** might change if a crash occurs while writing to a single byte in +** that range. But with POWERSAFE_OVERWRITE, the blast radius is zero +** (that is what POWERSAFE_OVERWRITE means), so we minimize the sector +** size. For backwards compatibility of the rollback journal file format, +** we cannot reduce the effective sector size below 512. +*/ +static void setSectorSize(Pager *pPager){ + assert( isOpen(pPager->fd) || pPager->tempFile ); + + if( pPager->tempFile + || (sqlite3OsDeviceCharacteristics(pPager->fd) & + SQLITE_IOCAP_POWERSAFE_OVERWRITE)!=0 + ){ + /* Sector size doesn't matter for temporary files. Also, the file + ** may not have been opened yet, in which case the OsSectorSize() + ** call will segfault. */ + pPager->sectorSize = 512; + }else{ + pPager->sectorSize = sqlite3SectorSize(pPager->fd); + } +} + +/* +** Playback the journal and thus restore the database file to +** the state it was in before we started making changes. +** +** The journal file format is as follows: +** +** (1) 8 byte prefix. A copy of aJournalMagic[]. +** (2) 4 byte big-endian integer which is the number of valid page records +** in the journal. If this value is 0xffffffff, then compute the +** number of page records from the journal size. +** (3) 4 byte big-endian integer which is the initial value for the +** sanity checksum. +** (4) 4 byte integer which is the number of pages to truncate the +** database to during a rollback. +** (5) 4 byte big-endian integer which is the sector size. The header +** is this many bytes in size. +** (6) 4 byte big-endian integer which is the page size. +** (7) zero padding out to the next sector size. +** (8) Zero or more pages instances, each as follows: +** + 4 byte page number. +** + pPager->pageSize bytes of data. +** + 4 byte checksum +** +** When we speak of the journal header, we mean the first 7 items above. +** Each entry in the journal is an instance of the 8th item. +** +** Call the value from the second bullet "nRec". nRec is the number of +** valid page entries in the journal. In most cases, you can compute the +** value of nRec from the size of the journal file. But if a power +** failure occurred while the journal was being written, it could be the +** case that the size of the journal file had already been increased but +** the extra entries had not yet made it safely to disk. In such a case, +** the value of nRec computed from the file size would be too large. For +** that reason, we always use the nRec value in the header. +** +** If the nRec value is 0xffffffff it means that nRec should be computed +** from the file size. This value is used when the user selects the +** no-sync option for the journal. A power failure could lead to corruption +** in this case. But for things like temporary table (which will be +** deleted when the power is restored) we don't care. +** +** If the file opened as the journal file is not a well-formed +** journal file then all pages up to the first corrupted page are rolled +** back (or no pages if the journal header is corrupted). The journal file +** is then deleted and SQLITE_OK returned, just as if no corruption had +** been encountered. +** +** If an I/O or malloc() error occurs, the journal-file is not deleted +** and an error code is returned. +** +** The isHot parameter indicates that we are trying to rollback a journal +** that might be a hot journal. Or, it could be that the journal is +** preserved because of JOURNALMODE_PERSIST or JOURNALMODE_TRUNCATE. +** If the journal really is hot, reset the pager cache prior rolling +** back any content. If the journal is merely persistent, no reset is +** needed. +*/ +static int pager_playback(Pager *pPager, int isHot){ + sqlite3_vfs *pVfs = pPager->pVfs; + i64 szJ; /* Size of the journal file in bytes */ + u32 nRec; /* Number of Records in the journal */ + u32 u; /* Unsigned loop counter */ + Pgno mxPg = 0; /* Size of the original file in pages */ + int rc; /* Result code of a subroutine */ + int res = 1; /* Value returned by sqlite3OsAccess() */ + char *zSuper = 0; /* Name of super-journal file if any */ + int needPagerReset; /* True to reset page prior to first page rollback */ + int nPlayback = 0; /* Total number of pages restored from journal */ + u32 savedPageSize = pPager->pageSize; + + /* Figure out how many records are in the journal. Abort early if + ** the journal is empty. + */ + assert( isOpen(pPager->jfd) ); + rc = sqlite3OsFileSize(pPager->jfd, &szJ); + if( rc!=SQLITE_OK ){ + goto end_playback; + } + + /* Read the super-journal name from the journal, if it is present. + ** If a super-journal file name is specified, but the file is not + ** present on disk, then the journal is not hot and does not need to be + ** played back. + ** + ** TODO: Technically the following is an error because it assumes that + ** buffer Pager.pTmpSpace is (mxPathname+1) bytes or larger. i.e. that + ** (pPager->pageSize >= pPager->pVfs->mxPathname+1). Using os_unix.c, + ** mxPathname is 512, which is the same as the minimum allowable value + ** for pageSize. + */ + zSuper = pPager->pTmpSpace; + rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); + if( rc==SQLITE_OK && zSuper[0] ){ + rc = sqlite3OsAccess(pVfs, zSuper, SQLITE_ACCESS_EXISTS, &res); + } + zSuper = 0; + if( rc!=SQLITE_OK || !res ){ + goto end_playback; + } + pPager->journalOff = 0; + needPagerReset = isHot; + + /* This loop terminates either when a readJournalHdr() or + ** pager_playback_one_page() call returns SQLITE_DONE or an IO error + ** occurs. + */ + while( 1 ){ + /* Read the next journal header from the journal file. If there are + ** not enough bytes left in the journal file for a complete header, or + ** it is corrupted, then a process must have failed while writing it. + ** This indicates nothing more needs to be rolled back. + */ + rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + goto end_playback; + } + + /* If nRec is 0xffffffff, then this journal was created by a process + ** working in no-sync mode. This means that the rest of the journal + ** file consists of pages, there are no more journal headers. Compute + ** the value of nRec based on this assumption. + */ + if( nRec==0xffffffff ){ + assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ); + nRec = (int)((szJ - JOURNAL_HDR_SZ(pPager))/JOURNAL_PG_SZ(pPager)); + } + + /* If nRec is 0 and this rollback is of a transaction created by this + ** process and if this is the final header in the journal, then it means + ** that this part of the journal was being filled but has not yet been + ** synced to disk. Compute the number of pages based on the remaining + ** size of the file. + ** + ** The third term of the test was added to fix ticket #2565. + ** When rolling back a hot journal, nRec==0 always means that the next + ** chunk of the journal contains zero pages to be rolled back. But + ** when doing a ROLLBACK and the nRec==0 chunk is the last chunk in + ** the journal, it means that the journal might contain additional + ** pages that need to be rolled back and that the number of pages + ** should be computed based on the journal file size. + */ + if( nRec==0 && !isHot && + pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff ){ + nRec = (int)((szJ - pPager->journalOff) / JOURNAL_PG_SZ(pPager)); + } + + /* If this is the first header read from the journal, truncate the + ** database file back to its original size. + */ + if( pPager->journalOff==JOURNAL_HDR_SZ(pPager) ){ + rc = pager_truncate(pPager, mxPg); + if( rc!=SQLITE_OK ){ + goto end_playback; + } + pPager->dbSize = mxPg; + } + + /* Copy original pages out of the journal and back into the + ** database file and/or page cache. + */ + for(u=0; ujournalOff,0,1,0); + if( rc==SQLITE_OK ){ + nPlayback++; + }else{ + if( rc==SQLITE_DONE ){ + pPager->journalOff = szJ; + break; + }else if( rc==SQLITE_IOERR_SHORT_READ ){ + /* If the journal has been truncated, simply stop reading and + ** processing the journal. This might happen if the journal was + ** not completely written and synced prior to a crash. In that + ** case, the database should have never been written in the + ** first place so it is OK to simply abandon the rollback. */ + rc = SQLITE_OK; + goto end_playback; + }else{ + /* If we are unable to rollback, quit and return the error + ** code. This will cause the pager to enter the error state + ** so that no further harm will be done. Perhaps the next + ** process to come along will be able to rollback the database. + */ + goto end_playback; + } + } + } + } + /*NOTREACHED*/ + assert( 0 ); + +end_playback: + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSetPagesize(pPager, &savedPageSize, -1); + } + /* Following a rollback, the database file should be back in its original + ** state prior to the start of the transaction, so invoke the + ** SQLITE_FCNTL_DB_UNCHANGED file-control method to disable the + ** assertion that the transaction counter was modified. + */ +#ifdef SQLITE_DEBUG + sqlite3OsFileControlHint(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0); +#endif + + /* If this playback is happening automatically as a result of an IO or + ** malloc error that occurred after the change-counter was updated but + ** before the transaction was committed, then the change-counter + ** modification may just have been reverted. If this happens in exclusive + ** mode, then subsequent transactions performed by the connection will not + ** update the change-counter at all. This may lead to cache inconsistency + ** problems for other processes at some point in the future. So, just + ** in case this has happened, clear the changeCountDone flag now. + */ + pPager->changeCountDone = pPager->tempFile; + + if( rc==SQLITE_OK ){ + /* Leave 4 bytes of space before the super-journal filename in memory. + ** This is because it may end up being passed to sqlite3OsOpen(), in + ** which case it requires 4 0x00 bytes in memory immediately before + ** the filename. */ + zSuper = &pPager->pTmpSpace[4]; + rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); + testcase( rc!=SQLITE_OK ); + } + if( rc==SQLITE_OK + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + ){ + rc = sqlite3PagerSync(pPager, 0); + } + if( rc==SQLITE_OK ){ + rc = pager_end_transaction(pPager, zSuper[0]!='\0', 0); + testcase( rc!=SQLITE_OK ); + } + if( rc==SQLITE_OK && zSuper[0] && res ){ + /* If there was a super-journal and this routine will return success, + ** see if it is possible to delete the super-journal. + */ + assert( zSuper==&pPager->pTmpSpace[4] ); + memset(&zSuper[-4], 0, 4); + rc = pager_delsuper(pPager, zSuper); + testcase( rc!=SQLITE_OK ); + } + if( isHot && nPlayback ){ + sqlite3_log(SQLITE_NOTICE_RECOVER_ROLLBACK, "recovered %d pages from %s", + nPlayback, pPager->zJournal); + } + + /* The Pager.sectorSize variable may have been updated while rolling + ** back a journal created by a process with a different sector size + ** value. Reset it to the correct value for this process. + */ + setSectorSize(pPager); + return rc; +} + + +/* +** Read the content for page pPg out of the database file (or out of +** the WAL if that is where the most recent copy if found) into +** pPg->pData. A shared lock or greater must be held on the database +** file before this function is called. +** +** If page 1 is read, then the value of Pager.dbFileVers[] is set to +** the value read from the database file. +** +** If an IO error occurs, then the IO error is returned to the caller. +** Otherwise, SQLITE_OK is returned. +*/ +static int readDbPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ + int rc = SQLITE_OK; /* Return code */ + +#ifndef SQLITE_OMIT_WAL + u32 iFrame = 0; /* Frame of WAL containing pgno */ + + assert( pPager->eState>=PAGER_READER && !MEMDB ); + assert( isOpen(pPager->fd) ); + + if( pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + if( rc ) return rc; + } + if( iFrame ){ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame,pPager->pageSize,pPg->pData); + }else +#endif + { + i64 iOffset = (pPg->pgno-1)*(i64)pPager->pageSize; + rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + } + + if( pPg->pgno==1 ){ + if( rc ){ + /* If the read is unsuccessful, set the dbFileVers[] to something + ** that will never be a valid file version. dbFileVers[] is a copy + ** of bytes 24..39 of the database. Bytes 28..31 should always be + ** zero or the size of the database in page. Bytes 32..35 and 35..39 + ** should be page numbers which are never 0xffffffff. So filling + ** pPager->dbFileVers[] with all 0xff bytes should suffice. + ** + ** For an encrypted database, the situation is more complex: bytes + ** 24..39 of the database are white noise. But the probability of + ** white noise equaling 16 bytes of 0xff is vanishingly small so + ** we should still be ok. + */ + memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); + }else{ + u8 *dbFileVers = &((u8*)pPg->pData)[24]; + memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); + } + } + PAGER_INCR(sqlite3_pager_readdb_count); + PAGER_INCR(pPager->nRead); + IOTRACE(("PGIN %p %d\n", pPager, pPg->pgno)); + PAGERTRACE(("FETCH %d page %d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, pager_pagehash(pPg))); + + return rc; +} + +/* +** Update the value of the change-counter at offsets 24 and 92 in +** the header and the sqlite version number at offset 96. +** +** This is an unconditional update. See also the pager_incr_changecounter() +** routine which only updates the change-counter if the update is actually +** needed, as determined by the pPager->changeCountDone state variable. +*/ +static void pager_write_changecounter(PgHdr *pPg){ + u32 change_counter; + + /* Increment the value just read and write it back to byte 24. */ + change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1; + put32bits(((char*)pPg->pData)+24, change_counter); + + /* Also store the SQLite version number in bytes 96..99 and in + ** bytes 92..95 store the change counter for which the version number + ** is valid. */ + put32bits(((char*)pPg->pData)+92, change_counter); + put32bits(((char*)pPg->pData)+96, SQLITE_VERSION_NUMBER); +} + +#ifndef SQLITE_OMIT_WAL +/* +** This function is invoked once for each page that has already been +** written into the log file when a WAL transaction is rolled back. +** Parameter iPg is the page number of said page. The pCtx argument +** is actually a pointer to the Pager structure. +** +** If page iPg is present in the cache, and has no outstanding references, +** it is discarded. Otherwise, if there are one or more outstanding +** references, the page content is reloaded from the database. If the +** attempt to reload content from the database is required and fails, +** return an SQLite error code. Otherwise, SQLITE_OK. +*/ +static int pagerUndoCallback(void *pCtx, Pgno iPg){ + int rc = SQLITE_OK; + Pager *pPager = (Pager *)pCtx; + PgHdr *pPg; + + assert( pagerUseWal(pPager) ); + pPg = sqlite3PagerLookup(pPager, iPg); + if( pPg ){ + if( sqlite3PcachePageRefcount(pPg)==1 ){ + sqlite3PcacheDrop(pPg); + }else{ + rc = readDbPage(pPg); + if( rc==SQLITE_OK ){ + pPager->xReiniter(pPg); + } + sqlite3PagerUnrefNotNull(pPg); + } + } + + /* Normally, if a transaction is rolled back, any backup processes are + ** updated as data is copied out of the rollback journal and into the + ** database. This is not generally possible with a WAL database, as + ** rollback involves simply truncating the log file. Therefore, if one + ** or more frames have already been written to the log (and therefore + ** also copied into the backup databases) as part of this transaction, + ** the backups must be restarted. + */ + sqlite3BackupRestart(pPager->pBackup); + + return rc; +} + +/* +** This function is called to rollback a transaction on a WAL database. +*/ +static int pagerRollbackWal(Pager *pPager){ + int rc; /* Return Code */ + PgHdr *pList; /* List of dirty pages to revert */ + + /* For all pages in the cache that are currently dirty or have already + ** been written (but not committed) to the log file, do one of the + ** following: + ** + ** + Discard the cached page (if refcount==0), or + ** + Reload page content from the database (if refcount>0). + */ + pPager->dbSize = pPager->dbOrigSize; + rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager); + pList = sqlite3PcacheDirtyList(pPager->pPCache); + while( pList && rc==SQLITE_OK ){ + PgHdr *pNext = pList->pDirty; + rc = pagerUndoCallback((void *)pPager, pList->pgno); + pList = pNext; + } + + return rc; +} + +/* +** This function is a wrapper around sqlite3WalFrames(). As well as logging +** the contents of the list of pages headed by pList (connected by pDirty), +** this function notifies any active backup processes that the pages have +** changed. +** +** The list of pages passed into this routine is always sorted by page number. +** Hence, if page 1 appears anywhere on the list, it will be the first page. +*/ +static int pagerWalFrames( + Pager *pPager, /* Pager object */ + PgHdr *pList, /* List of frames to log */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit /* True if this is a commit */ +){ + int rc; /* Return code */ + int nList; /* Number of pages in pList */ + PgHdr *p; /* For looping over pages */ + + assert( pPager->pWal ); + assert( pList ); +#ifdef SQLITE_DEBUG + /* Verify that the page list is in accending order */ + for(p=pList; p && p->pDirty; p=p->pDirty){ + assert( p->pgno < p->pDirty->pgno ); + } +#endif + + assert( pList->pDirty==0 || isCommit ); + if( isCommit ){ + /* If a WAL transaction is being committed, there is no point in writing + ** any pages with page numbers greater than nTruncate into the WAL file. + ** They will never be read by any client. So remove them from the pDirty + ** list here. */ + PgHdr **ppNext = &pList; + nList = 0; + for(p=pList; (*ppNext = p)!=0; p=p->pDirty){ + if( p->pgno<=nTruncate ){ + ppNext = &p->pDirty; + nList++; + } + } + assert( pList ); + }else{ + nList = 1; + } + pPager->aStat[PAGER_STAT_WRITE] += nList; + + if( pList->pgno==1 ) pager_write_changecounter(pList); + rc = sqlite3WalFrames(pPager->pWal, + pPager->pageSize, pList, nTruncate, isCommit, pPager->walSyncFlags + ); + if( rc==SQLITE_OK && pPager->pBackup ){ + for(p=pList; p; p=p->pDirty){ + sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); + } + } + +#ifdef SQLITE_CHECK_PAGES + pList = sqlite3PcacheDirtyList(pPager->pPCache); + for(p=pList; p; p=p->pDirty){ + pager_set_pagehash(p); + } +#endif + + return rc; +} + +/* +** Begin a read transaction on the WAL. +** +** This routine used to be called "pagerOpenSnapshot()" because it essentially +** makes a snapshot of the database at the current point in time and preserves +** that snapshot for use by the reader in spite of concurrently changes by +** other writers or checkpointers. +*/ +static int pagerBeginReadTransaction(Pager *pPager){ + int rc; /* Return code */ + int changed = 0; /* True if cache must be reset */ + + assert( pagerUseWal(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); + + /* sqlite3WalEndReadTransaction() was not called for the previous + ** transaction in locking_mode=EXCLUSIVE. So call it now. If we + ** are in locking_mode=NORMAL and EndRead() was previously called, + ** the duplicate call is harmless. + */ + sqlite3WalEndReadTransaction(pPager->pWal); + + rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); + if( rc!=SQLITE_OK || changed ){ + pager_reset(pPager); + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); + } + + return rc; +} +#endif + +/* +** This function is called as part of the transition from PAGER_OPEN +** to PAGER_READER state to determine the size of the database file +** in pages (assuming the page size currently stored in Pager.pageSize). +** +** If no error occurs, SQLITE_OK is returned and the size of the database +** in pages is stored in *pnPage. Otherwise, an error code (perhaps +** SQLITE_IOERR_FSTAT) is returned and *pnPage is left unmodified. +*/ +static int pagerPagecount(Pager *pPager, Pgno *pnPage){ + Pgno nPage; /* Value to return via *pnPage */ + + /* Query the WAL sub-system for the database size. The WalDbsize() + ** function returns zero if the WAL is not open (i.e. Pager.pWal==0), or + ** if the database size is not available. The database size is not + ** available from the WAL sub-system if the log file is empty or + ** contains no valid committed transactions. + */ + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK ); + assert( isOpen(pPager->fd) ); + assert( pPager->tempFile==0 ); + nPage = sqlite3WalDbsize(pPager->pWal); + + /* If the number of pages in the database is not available from the + ** WAL sub-system, determine the page count based on the size of + ** the database file. If the size of the database file is not an + ** integer multiple of the page-size, round up the result. + */ + if( nPage==0 && ALWAYS(isOpen(pPager->fd)) ){ + i64 n = 0; /* Size of db file in bytes */ + int rc = sqlite3OsFileSize(pPager->fd, &n); + if( rc!=SQLITE_OK ){ + return rc; + } + nPage = (Pgno)((n+pPager->pageSize-1) / pPager->pageSize); + } + + /* If the current number of pages in the file is greater than the + ** configured maximum pager number, increase the allowed limit so + ** that the file can be read. + */ + if( nPage>pPager->mxPgno ){ + pPager->mxPgno = (Pgno)nPage; + } + + *pnPage = nPage; + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_WAL +/* +** Check if the *-wal file that corresponds to the database opened by pPager +** exists if the database is not empy, or verify that the *-wal file does +** not exist (by deleting it) if the database file is empty. +** +** If the database is not empty and the *-wal file exists, open the pager +** in WAL mode. If the database is empty or if no *-wal file exists and +** if no error occurs, make sure Pager.journalMode is not set to +** PAGER_JOURNALMODE_WAL. +** +** Return SQLITE_OK or an error code. +** +** The caller must hold a SHARED lock on the database file to call this +** function. Because an EXCLUSIVE lock on the db file is required to delete +** a WAL on a none-empty database, this ensures there is no race condition +** between the xAccess() below and an xDelete() being executed by some +** other connection. +*/ +static int pagerOpenWalIfPresent(Pager *pPager){ + int rc = SQLITE_OK; + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->eLock>=SHARED_LOCK ); + + if( !pPager->tempFile ){ + int isWal; /* True if WAL file exists */ + rc = sqlite3OsAccess( + pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &isWal + ); + if( rc==SQLITE_OK ){ + if( isWal ){ + Pgno nPage; /* Size of the database file */ + + rc = pagerPagecount(pPager, &nPage); + if( rc ) return rc; + if( nPage==0 ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); + }else{ + testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); + rc = sqlite3PagerOpenWal(pPager, 0); + } + }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + pPager->journalMode = PAGER_JOURNALMODE_DELETE; + } + } + } + return rc; +} +#endif + +/* +** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback +** the entire super-journal file. The case pSavepoint==NULL occurs when +** a ROLLBACK TO command is invoked on a SAVEPOINT that is a transaction +** savepoint. +** +** When pSavepoint is not NULL (meaning a non-transaction savepoint is +** being rolled back), then the rollback consists of up to three stages, +** performed in the order specified: +** +** * Pages are played back from the main journal starting at byte +** offset PagerSavepoint.iOffset and continuing to +** PagerSavepoint.iHdrOffset, or to the end of the main journal +** file if PagerSavepoint.iHdrOffset is zero. +** +** * If PagerSavepoint.iHdrOffset is not zero, then pages are played +** back starting from the journal header immediately following +** PagerSavepoint.iHdrOffset to the end of the main journal file. +** +** * Pages are then played back from the sub-journal file, starting +** with the PagerSavepoint.iSubRec and continuing to the end of +** the journal file. +** +** Throughout the rollback process, each time a page is rolled back, the +** corresponding bit is set in a bitvec structure (variable pDone in the +** implementation below). This is used to ensure that a page is only +** rolled back the first time it is encountered in either journal. +** +** If pSavepoint is NULL, then pages are only played back from the main +** journal file. There is no need for a bitvec in this case. +** +** In either case, before playback commences the Pager.dbSize variable +** is reset to the value that it held at the start of the savepoint +** (or transaction). No page with a page-number greater than this value +** is played back. If one is encountered it is simply skipped. +*/ +static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ + i64 szJ; /* Effective size of the main journal */ + i64 iHdrOff; /* End of first segment of main-journal records */ + int rc = SQLITE_OK; /* Return code */ + Bitvec *pDone = 0; /* Bitvec to ensure pages played back only once */ + + assert( pPager->eState!=PAGER_ERROR ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + + /* Allocate a bitvec to use to store the set of pages rolled back */ + if( pSavepoint ){ + pDone = sqlite3BitvecCreate(pSavepoint->nOrig); + if( !pDone ){ + return SQLITE_NOMEM_BKPT; + } + } + + /* Set the database size back to the value it was before the savepoint + ** being reverted was opened. + */ + pPager->dbSize = pSavepoint ? pSavepoint->nOrig : pPager->dbOrigSize; + pPager->changeCountDone = pPager->tempFile; + + if( !pSavepoint && pagerUseWal(pPager) ){ + return pagerRollbackWal(pPager); + } + + /* Use pPager->journalOff as the effective size of the main rollback + ** journal. The actual file might be larger than this in + ** PAGER_JOURNALMODE_TRUNCATE or PAGER_JOURNALMODE_PERSIST. But anything + ** past pPager->journalOff is off-limits to us. + */ + szJ = pPager->journalOff; + assert( pagerUseWal(pPager)==0 || szJ==0 ); + + /* Begin by rolling back records from the main journal starting at + ** PagerSavepoint.iOffset and continuing to the next journal header. + ** There might be records in the main journal that have a page number + ** greater than the current database size (pPager->dbSize) but those + ** will be skipped automatically. Pages are added to pDone as they + ** are played back. + */ + if( pSavepoint && !pagerUseWal(pPager) ){ + iHdrOff = pSavepoint->iHdrOffset ? pSavepoint->iHdrOffset : szJ; + pPager->journalOff = pSavepoint->iOffset; + while( rc==SQLITE_OK && pPager->journalOffjournalOff, pDone, 1, 1); + } + assert( rc!=SQLITE_DONE ); + }else{ + pPager->journalOff = 0; + } + + /* Continue rolling back records out of the main journal starting at + ** the first journal header seen and continuing until the effective end + ** of the main journal file. Continue to skip out-of-range pages and + ** continue adding pages rolled back to pDone. + */ + while( rc==SQLITE_OK && pPager->journalOffjournalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff" + ** test is related to ticket #2565. See the discussion in the + ** pager_playback() function for additional information. + */ + if( nJRec==0 + && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff + ){ + nJRec = (u32)((szJ - pPager->journalOff)/JOURNAL_PG_SZ(pPager)); + } + for(ii=0; rc==SQLITE_OK && iijournalOffjournalOff, pDone, 1, 1); + } + assert( rc!=SQLITE_DONE ); + } + assert( rc!=SQLITE_OK || pPager->journalOff>=szJ ); + + /* Finally, rollback pages from the sub-journal. Page that were + ** previously rolled back out of the main journal (and are hence in pDone) + ** will be skipped. Out-of-range pages are also skipped. + */ + if( pSavepoint ){ + u32 ii; /* Loop counter */ + i64 offset = (i64)pSavepoint->iSubRec*(4+pPager->pageSize); + + if( pagerUseWal(pPager) ){ + rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->aWalData); + } + for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && iinSubRec; ii++){ + assert( offset==(i64)ii*(4+pPager->pageSize) ); + rc = pager_playback_one_page(pPager, &offset, pDone, 0, 1); + } + assert( rc!=SQLITE_DONE ); + } + + sqlite3BitvecDestroy(pDone); + if( rc==SQLITE_OK ){ + pPager->journalOff = szJ; + } + + return rc; +} + +/* +** Change the maximum number of in-memory pages that are allowed +** before attempting to recycle clean and unused pages. +*/ +SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ + sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); +} + +/* +** Change the maximum number of in-memory pages that are allowed +** before attempting to spill pages to journal. +*/ +SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager *pPager, int mxPage){ + return sqlite3PcacheSetSpillsize(pPager->pPCache, mxPage); +} + +/* +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. +*/ +static void pagerFixMaplimit(Pager *pPager){ +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) && fd->pMethods->iVersion>=3 ){ + sqlite3_int64 sz; + sz = pPager->szMmap; + pPager->bUseFetch = (sz>0); + setGetterMethod(pPager); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz); + } +#endif +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 szMmap){ + pPager->szMmap = szMmap; + pagerFixMaplimit(pPager); +} + +/* +** Free as much memory as possible from the pager. +*/ +SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ + sqlite3PcacheShrink(pPager->pPCache); +} + +/* +** Adjust settings of the pager to those specified in the pgFlags parameter. +** +** The "level" in pgFlags & PAGER_SYNCHRONOUS_MASK sets the robustness +** of the database to damage due to OS crashes or power failures by +** changing the number of syncs()s when writing the journals. +** There are four levels: +** +** OFF sqlite3OsSync() is never called. This is the default +** for temporary and transient files. +** +** NORMAL The journal is synced once before writes begin on the +** database. This is normally adequate protection, but +** it is theoretically possible, though very unlikely, +** that an inopertune power failure could leave the journal +** in a state which would cause damage to the database +** when it is rolled back. +** +** FULL The journal is synced twice before writes begin on the +** database (with some additional information - the nRec field +** of the journal header - being written in between the two +** syncs). If we assume that writing a +** single disk sector is atomic, then this mode provides +** assurance that the journal will not be corrupted to the +** point of causing damage to the database during rollback. +** +** EXTRA This is like FULL except that is also syncs the directory +** that contains the rollback journal after the rollback +** journal is unlinked. +** +** The above is for a rollback-journal mode. For WAL mode, OFF continues +** to mean that no syncs ever occur. NORMAL means that the WAL is synced +** prior to the start of checkpoint and that the database file is synced +** at the conclusion of the checkpoint if the entire content of the WAL +** was written back into the database. But no sync operations occur for +** an ordinary commit in NORMAL mode with WAL. FULL means that the WAL +** file is synced following each commit operation, in addition to the +** syncs associated with NORMAL. There is no difference between FULL +** and EXTRA for WAL mode. +** +** Do not confuse synchronous=FULL with SQLITE_SYNC_FULL. The +** SQLITE_SYNC_FULL macro means to use the MacOSX-style full-fsync +** using fcntl(F_FULLFSYNC). SQLITE_SYNC_NORMAL means to do an +** ordinary fsync() call. There is no difference between SQLITE_SYNC_FULL +** and SQLITE_SYNC_NORMAL on platforms other than MacOSX. But the +** synchronous=FULL versus synchronous=NORMAL setting determines when +** the xSync primitive is called and is relevant to all platforms. +** +** Numeric values associated with these states are OFF==1, NORMAL=2, +** and FULL=3. +*/ +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +SQLITE_PRIVATE void sqlite3PagerSetFlags( + Pager *pPager, /* The pager to set safety level for */ + unsigned pgFlags /* Various flags */ +){ + unsigned level = pgFlags & PAGER_SYNCHRONOUS_MASK; + if( pPager->tempFile ){ + pPager->noSync = 1; + pPager->fullSync = 0; + pPager->extraSync = 0; + }else{ + pPager->noSync = level==PAGER_SYNCHRONOUS_OFF ?1:0; + pPager->fullSync = level>=PAGER_SYNCHRONOUS_FULL ?1:0; + pPager->extraSync = level==PAGER_SYNCHRONOUS_EXTRA ?1:0; + } + if( pPager->noSync ){ + pPager->syncFlags = 0; + }else if( pgFlags & PAGER_FULLFSYNC ){ + pPager->syncFlags = SQLITE_SYNC_FULL; + }else{ + pPager->syncFlags = SQLITE_SYNC_NORMAL; + } + pPager->walSyncFlags = (pPager->syncFlags<<2); + if( pPager->fullSync ){ + pPager->walSyncFlags |= pPager->syncFlags; + } + if( (pgFlags & PAGER_CKPT_FULLFSYNC) && !pPager->noSync ){ + pPager->walSyncFlags |= (SQLITE_SYNC_FULL<<2); + } + if( pgFlags & PAGER_CACHESPILL ){ + pPager->doNotSpill &= ~SPILLFLAG_OFF; + }else{ + pPager->doNotSpill |= SPILLFLAG_OFF; + } +} +#endif + +/* +** The following global variable is incremented whenever the library +** attempts to open a temporary file. This information is used for +** testing and analysis only. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_opentemp_count = 0; +#endif + +/* +** Open a temporary file. +** +** Write the file descriptor into *pFile. Return SQLITE_OK on success +** or some other error code if we fail. The OS will automatically +** delete the temporary file when it is closed. +** +** The flags passed to the VFS layer xOpen() call are those specified +** by parameter vfsFlags ORed with the following: +** +** SQLITE_OPEN_READWRITE +** SQLITE_OPEN_CREATE +** SQLITE_OPEN_EXCLUSIVE +** SQLITE_OPEN_DELETEONCLOSE +*/ +static int pagerOpentemp( + Pager *pPager, /* The pager object */ + sqlite3_file *pFile, /* Write the file descriptor here */ + int vfsFlags /* Flags passed through to the VFS */ +){ + int rc; /* Return code */ + +#ifdef SQLITE_TEST + sqlite3_opentemp_count++; /* Used for testing and analysis only */ +#endif + + vfsFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; + rc = sqlite3OsOpen(pPager->pVfs, 0, pFile, vfsFlags, 0); + assert( rc!=SQLITE_OK || isOpen(pFile) ); + return rc; +} + +/* +** Set the busy handler function. +** +** The pager invokes the busy-handler if sqlite3OsLock() returns +** SQLITE_BUSY when trying to upgrade from no-lock to a SHARED lock, +** or when trying to upgrade from a RESERVED lock to an EXCLUSIVE +** lock. It does *not* invoke the busy handler when upgrading from +** SHARED to RESERVED, or when upgrading from SHARED to EXCLUSIVE +** (which occurs during hot-journal rollback). Summary: +** +** Transition | Invokes xBusyHandler +** -------------------------------------------------------- +** NO_LOCK -> SHARED_LOCK | Yes +** SHARED_LOCK -> RESERVED_LOCK | No +** SHARED_LOCK -> EXCLUSIVE_LOCK | No +** RESERVED_LOCK -> EXCLUSIVE_LOCK | Yes +** +** If the busy-handler callback returns non-zero, the lock is +** retried. If it returns zero, then the SQLITE_BUSY error is +** returned to the caller of the pager API function. +*/ +SQLITE_PRIVATE void sqlite3PagerSetBusyHandler( + Pager *pPager, /* Pager object */ + int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ + void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ +){ + void **ap; + pPager->xBusyHandler = xBusyHandler; + pPager->pBusyHandlerArg = pBusyHandlerArg; + ap = (void **)&pPager->xBusyHandler; + assert( ((int(*)(void *))(ap[0]))==xBusyHandler ); + assert( ap[1]==pBusyHandlerArg ); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_BUSYHANDLER, (void *)ap); +} + +/* +** Change the page size used by the Pager object. The new page size +** is passed in *pPageSize. +** +** If the pager is in the error state when this function is called, it +** is a no-op. The value returned is the error state error code (i.e. +** one of SQLITE_IOERR, an SQLITE_IOERR_xxx sub-code or SQLITE_FULL). +** +** Otherwise, if all of the following are true: +** +** * the new page size (value of *pPageSize) is valid (a power +** of two between 512 and SQLITE_MAX_PAGE_SIZE, inclusive), and +** +** * there are no outstanding page references, and +** +** * the database is either not an in-memory database or it is +** an in-memory database that currently consists of zero pages. +** +** then the pager object page size is set to *pPageSize. +** +** If the page size is changed, then this function uses sqlite3PagerMalloc() +** to obtain a new Pager.pTmpSpace buffer. If this allocation attempt +** fails, SQLITE_NOMEM is returned and the page size remains unchanged. +** In all other cases, SQLITE_OK is returned. +** +** If the page size is not changed, either because one of the enumerated +** conditions above is not true, the pager was in error state when this +** function was called, or because the memory allocation attempt failed, +** then *pPageSize is set to the old, retained page size before returning. +*/ +SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ + int rc = SQLITE_OK; + + /* It is not possible to do a full assert_pager_state() here, as this + ** function may be called from within PagerOpen(), before the state + ** of the Pager object is internally consistent. + ** + ** At one point this function returned an error if the pager was in + ** PAGER_ERROR state. But since PAGER_ERROR state guarantees that + ** there is at least one outstanding page reference, this function + ** is a no-op for that case anyhow. + */ + + u32 pageSize = *pPageSize; + assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); + if( (pPager->memDb==0 || pPager->dbSize==0) + && sqlite3PcacheRefCount(pPager->pPCache)==0 + && pageSize && pageSize!=(u32)pPager->pageSize + ){ + char *pNew = NULL; /* New temp space */ + i64 nByte = 0; + + if( pPager->eState>PAGER_OPEN && isOpen(pPager->fd) ){ + rc = sqlite3OsFileSize(pPager->fd, &nByte); + } + if( rc==SQLITE_OK ){ + /* 8 bytes of zeroed overrun space is sufficient so that the b-tree + * cell header parser will never run off the end of the allocation */ + pNew = (char *)sqlite3PageMalloc(pageSize+8); + if( !pNew ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + memset(pNew+pageSize, 0, 8); + } + } + + if( rc==SQLITE_OK ){ + pager_reset(pPager); + rc = sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + } + if( rc==SQLITE_OK ){ + sqlite3PageFree(pPager->pTmpSpace); + pPager->pTmpSpace = pNew; + pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); + pPager->pageSize = pageSize; + }else{ + sqlite3PageFree(pNew); + } + } + + *pPageSize = pPager->pageSize; + if( rc==SQLITE_OK ){ + if( nReserve<0 ) nReserve = pPager->nReserve; + assert( nReserve>=0 && nReserve<1000 ); + pPager->nReserve = (i16)nReserve; + pagerFixMaplimit(pPager); + } + return rc; +} + +/* +** Return a pointer to the "temporary page" buffer held internally +** by the pager. This is a buffer that is big enough to hold the +** entire content of a database page. This buffer is used internally +** during rollback and will be overwritten whenever a rollback +** occurs. But other modules are free to use it too, as long as +** no rollbacks are happening. +*/ +SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager *pPager){ + return pPager->pTmpSpace; +} + +/* +** Attempt to set the maximum database page count if mxPage is positive. +** Make no changes if mxPage is zero or negative. And never reduce the +** maximum page count below the current size of the database. +** +** Regardless of mxPage, return the current maximum page count. +*/ +SQLITE_PRIVATE Pgno sqlite3PagerMaxPageCount(Pager *pPager, Pgno mxPage){ + if( mxPage>0 ){ + pPager->mxPgno = mxPage; + } + assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */ + /* assert( pPager->mxPgno>=pPager->dbSize ); */ + /* OP_MaxPgcnt ensures that the parameter passed to this function is not + ** less than the total number of valid pages in the database. But this + ** may be less than Pager.dbSize, and so the assert() above is not valid */ + return pPager->mxPgno; +} + +/* +** The following set of routines are used to disable the simulated +** I/O error mechanism. These routines are used to avoid simulated +** errors in places where we do not care about errors. +** +** Unless -DSQLITE_TEST=1 is used, these routines are all no-ops +** and generate no code. +*/ +#ifdef SQLITE_TEST +SQLITE_API extern int sqlite3_io_error_pending; +SQLITE_API extern int sqlite3_io_error_hit; +static int saved_cnt; +void disable_simulated_io_errors(void){ + saved_cnt = sqlite3_io_error_pending; + sqlite3_io_error_pending = -1; +} +void enable_simulated_io_errors(void){ + sqlite3_io_error_pending = saved_cnt; +} +#else +# define disable_simulated_io_errors() +# define enable_simulated_io_errors() +#endif + +/* +** Read the first N bytes from the beginning of the file into memory +** that pDest points to. +** +** If the pager was opened on a transient file (zFilename==""), or +** opened on a file less than N bytes in size, the output buffer is +** zeroed and SQLITE_OK returned. The rationale for this is that this +** function is used to read database headers, and a new transient or +** zero sized database has a header than consists entirely of zeroes. +** +** If any IO error apart from SQLITE_IOERR_SHORT_READ is encountered, +** the error code is returned to the caller and the contents of the +** output buffer undefined. +*/ +SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned char *pDest){ + int rc = SQLITE_OK; + memset(pDest, 0, N); + assert( isOpen(pPager->fd) || pPager->tempFile ); + + /* This routine is only called by btree immediately after creating + ** the Pager object. There has not been an opportunity to transition + ** to WAL mode yet. + */ + assert( !pagerUseWal(pPager) ); + + if( isOpen(pPager->fd) ){ + IOTRACE(("DBHDR %p 0 %d\n", pPager, N)) + rc = sqlite3OsRead(pPager->fd, pDest, N, 0); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + } + return rc; +} + +/* +** This function may only be called when a read-transaction is open on +** the pager. It returns the total number of pages in the database. +** +** However, if the file is between 1 and bytes in size, then +** this is considered a 1 page file. +*/ +SQLITE_PRIVATE void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ + assert( pPager->eState>=PAGER_READER ); + assert( pPager->eState!=PAGER_WRITER_FINISHED ); + *pnPage = (int)pPager->dbSize; +} + + +/* +** Try to obtain a lock of type locktype on the database file. If +** a similar or greater lock is already held, this function is a no-op +** (returning SQLITE_OK immediately). +** +** Otherwise, attempt to obtain the lock using sqlite3OsLock(). Invoke +** the busy callback if the lock is currently not available. Repeat +** until the busy callback returns false or until the attempt to +** obtain the lock succeeds. +** +** Return SQLITE_OK on success and an error code if we cannot obtain +** the lock. If the lock is obtained successfully, set the Pager.state +** variable to locktype before returning. +*/ +static int pager_wait_on_lock(Pager *pPager, int locktype){ + int rc; /* Return code */ + + /* Check that this is either a no-op (because the requested lock is + ** already held), or one of the transitions that the busy-handler + ** may be invoked during, according to the comment above + ** sqlite3PagerSetBusyhandler(). + */ + assert( (pPager->eLock>=locktype) + || (pPager->eLock==NO_LOCK && locktype==SHARED_LOCK) + || (pPager->eLock==RESERVED_LOCK && locktype==EXCLUSIVE_LOCK) + ); + + do { + rc = pagerLockDb(pPager, locktype); + }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); + return rc; +} + +/* +** Function assertTruncateConstraint(pPager) checks that one of the +** following is true for all dirty pages currently in the page-cache: +** +** a) The page number is less than or equal to the size of the +** current database image, in pages, OR +** +** b) if the page content were written at this time, it would not +** be necessary to write the current content out to the sub-journal +** (as determined by function subjRequiresPage()). +** +** If the condition asserted by this function were not true, and the +** dirty page were to be discarded from the cache via the pagerStress() +** routine, pagerStress() would not write the current page content to +** the database file. If a savepoint transaction were rolled back after +** this happened, the correct behavior would be to restore the current +** content of the page. However, since this content is not present in either +** the database file or the portion of the rollback journal and +** sub-journal rolled back the content could not be restored and the +** database image would become corrupt. It is therefore fortunate that +** this circumstance cannot arise. +*/ +#if defined(SQLITE_DEBUG) +static void assertTruncateConstraintCb(PgHdr *pPg){ + assert( pPg->flags&PGHDR_DIRTY ); + assert( !subjRequiresPage(pPg) || pPg->pgno<=pPg->pPager->dbSize ); +} +static void assertTruncateConstraint(Pager *pPager){ + sqlite3PcacheIterateDirty(pPager->pPCache, assertTruncateConstraintCb); +} +#else +# define assertTruncateConstraint(pPager) +#endif + +/* +** Truncate the in-memory database file image to nPage pages. This +** function does not actually modify the database file on disk. It +** just sets the internal state of the pager object so that the +** truncation will be done when the current transaction is committed. +** +** This function is only called right before committing a transaction. +** Once this function has been called, the transaction must either be +** rolled back or committed. It is not safe to call this function and +** then continue writing to the database. +*/ +SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ + assert( pPager->dbSize>=nPage || CORRUPT_DB ); + testcase( pPager->dbSizeeState>=PAGER_WRITER_CACHEMOD ); + pPager->dbSize = nPage; + + /* At one point the code here called assertTruncateConstraint() to + ** ensure that all pages being truncated away by this operation are, + ** if one or more savepoints are open, present in the savepoint + ** journal so that they can be restored if the savepoint is rolled + ** back. This is no longer necessary as this function is now only + ** called right before committing a transaction. So although the + ** Pager object may still have open savepoints (Pager.nSavepoint!=0), + ** they cannot be rolled back. So the assertTruncateConstraint() call + ** is no longer correct. */ +} + + +/* +** This function is called before attempting a hot-journal rollback. It +** syncs the journal file to disk, then sets pPager->journalHdr to the +** size of the journal file so that the pager_playback() routine knows +** that the entire journal file has been synced. +** +** Syncing a hot-journal to disk before attempting to roll it back ensures +** that if a power-failure occurs during the rollback, the process that +** attempts rollback following system recovery sees the same journal +** content as this process. +** +** If everything goes as planned, SQLITE_OK is returned. Otherwise, +** an SQLite error code. +*/ +static int pagerSyncHotJournal(Pager *pPager){ + int rc = SQLITE_OK; + if( !pPager->noSync ){ + rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_NORMAL); + } + if( rc==SQLITE_OK ){ + rc = sqlite3OsFileSize(pPager->jfd, &pPager->journalHdr); + } + return rc; +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Obtain a reference to a memory mapped page object for page number pgno. +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ + PgHdr *p; /* Memory mapped page to return */ + + if( pPager->pMmapFreelist ){ + *ppPage = p = pPager->pMmapFreelist; + pPager->pMmapFreelist = p->pDirty; + p->pDirty = 0; + assert( pPager->nExtra>=8 ); + memset(p->pExtra, 0, 8); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); + return SQLITE_NOMEM_BKPT; + } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } + + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pgno = pgno; + p->pData = pData; + pPager->nMmapOut++; + + return SQLITE_OK; +} +#endif + +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ +static void pagerReleaseMapPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + pPager->nMmapOut--; + pPg->pDirty = pPager->pMmapFreelist; + pPager->pMmapFreelist = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); +} + +/* +** Free all PgHdr objects stored in the Pager.pMmapFreelist list. +*/ +static void pagerFreeMapHdrs(Pager *pPager){ + PgHdr *p; + PgHdr *pNext; + for(p=pPager->pMmapFreelist; p; p=pNext){ + pNext = p->pDirty; + sqlite3_free(p); + } +} + +/* Verify that the database file has not be deleted or renamed out from +** under the pager. Return SQLITE_OK if the database is still where it ought +** to be on disk. Return non-zero (SQLITE_READONLY_DBMOVED or some other error +** code from sqlite3OsAccess()) if the database has gone missing. +*/ +static int databaseIsUnmoved(Pager *pPager){ + int bHasMoved = 0; + int rc; + + if( pPager->tempFile ) return SQLITE_OK; + if( pPager->dbSize==0 ) return SQLITE_OK; + assert( pPager->zFilename && pPager->zFilename[0] ); + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_HAS_MOVED, &bHasMoved); + if( rc==SQLITE_NOTFOUND ){ + /* If the HAS_MOVED file-control is unimplemented, assume that the file + ** has not been moved. That is the historical behavior of SQLite: prior to + ** version 3.8.3, it never checked */ + rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bHasMoved ){ + rc = SQLITE_READONLY_DBMOVED; + } + return rc; +} + + +/* +** Shutdown the page cache. Free all memory and close all files. +** +** If a transaction was in progress when this routine is called, that +** transaction is rolled back. All outstanding pages are invalidated +** and their memory is freed. Any attempt to use a page associated +** with this page cache after this function returns will likely +** result in a coredump. +** +** This function always succeeds. If a transaction is active an attempt +** is made to roll it back. If an error occurs during the rollback +** a hot journal may be left in the filesystem but no error is returned +** to the caller. +*/ +SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ + u8 *pTmp = (u8*)pPager->pTmpSpace; + assert( db || pagerUseWal(pPager)==0 ); + assert( assert_pager_state(pPager) ); + disable_simulated_io_errors(); + sqlite3BeginBenignMalloc(); + pagerFreeMapHdrs(pPager); + /* pPager->errCode = 0; */ + pPager->exclusiveMode = 0; +#ifndef SQLITE_OMIT_WAL + { + u8 *a = 0; + assert( db || pPager->pWal==0 ); + if( db && 0==(db->flags & SQLITE_NoCkptOnClose) + && SQLITE_OK==databaseIsUnmoved(pPager) + ){ + a = pTmp; + } + sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a); + pPager->pWal = 0; + } +#endif + pager_reset(pPager); + if( MEMDB ){ + pager_unlock(pPager); + }else{ + /* If it is open, sync the journal file before calling UnlockAndRollback. + ** If this is not done, then an unsynced portion of the open journal + ** file may be played back into the database. If a power failure occurs + ** while this is happening, the database could become corrupt. + ** + ** If an error occurs while trying to sync the journal, shift the pager + ** into the ERROR state. This causes UnlockAndRollback to unlock the + ** database and close the journal file without attempting to roll it + ** back or finalize it. The next database user will have to do hot-journal + ** rollback before accessing the database file. + */ + if( isOpen(pPager->jfd) ){ + pager_error(pPager, pagerSyncHotJournal(pPager)); + } + pagerUnlockAndRollback(pPager); + } + sqlite3EndBenignMalloc(); + enable_simulated_io_errors(); + PAGERTRACE(("CLOSE %d\n", PAGERID(pPager))); + IOTRACE(("CLOSE %p\n", pPager)) + sqlite3OsClose(pPager->jfd); + sqlite3OsClose(pPager->fd); + sqlite3PageFree(pTmp); + sqlite3PcacheClose(pPager->pPCache); + assert( !pPager->aSavepoint && !pPager->pInJournal ); + assert( !isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ); + + sqlite3_free(pPager); + return SQLITE_OK; +} + +#if !defined(NDEBUG) || defined(SQLITE_TEST) +/* +** Return the page number for page pPg. +*/ +SQLITE_PRIVATE Pgno sqlite3PagerPagenumber(DbPage *pPg){ + return pPg->pgno; +} +#endif + +/* +** Increment the reference count for page pPg. +*/ +SQLITE_PRIVATE void sqlite3PagerRef(DbPage *pPg){ + sqlite3PcacheRef(pPg); +} + +/* +** Sync the journal. In other words, make sure all the pages that have +** been written to the journal have actually reached the surface of the +** disk and can be restored in the event of a hot-journal rollback. +** +** If the Pager.noSync flag is set, then this function is a no-op. +** Otherwise, the actions required depend on the journal-mode and the +** device characteristics of the file-system, as follows: +** +** * If the journal file is an in-memory journal file, no action need +** be taken. +** +** * Otherwise, if the device does not support the SAFE_APPEND property, +** then the nRec field of the most recently written journal header +** is updated to contain the number of journal records that have +** been written following it. If the pager is operating in full-sync +** mode, then the journal file is synced before this field is updated. +** +** * If the device does not support the SEQUENTIAL property, then +** journal file is synced. +** +** Or, in pseudo-code: +** +** if( NOT ){ +** if( NOT SAFE_APPEND ){ +** if( ) xSync(); +** +** } +** if( NOT SEQUENTIAL ) xSync(); +** } +** +** If successful, this routine clears the PGHDR_NEED_SYNC flag of every +** page currently held in memory before returning SQLITE_OK. If an IO +** error is encountered, then the IO error code is returned to the caller. +*/ +static int syncJournal(Pager *pPager, int newHdr){ + int rc; /* Return code */ + + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + assert( !pagerUseWal(pPager) ); + + rc = sqlite3PagerExclusiveLock(pPager); + if( rc!=SQLITE_OK ) return rc; + + if( !pPager->noSync ){ + assert( !pPager->tempFile ); + if( isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ + const int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); + assert( isOpen(pPager->jfd) ); + + if( 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + /* This block deals with an obscure problem. If the last connection + ** that wrote to this database was operating in persistent-journal + ** mode, then the journal file may at this point actually be larger + ** than Pager.journalOff bytes. If the next thing in the journal + ** file happens to be a journal-header (written as part of the + ** previous connection's transaction), and a crash or power-failure + ** occurs after nRec is updated but before this connection writes + ** anything else to the journal file (or commits/rolls back its + ** transaction), then SQLite may become confused when doing the + ** hot-journal rollback following recovery. It may roll back all + ** of this connections data, then proceed to rolling back the old, + ** out-of-date data that follows it. Database corruption. + ** + ** To work around this, if the journal file does appear to contain + ** a valid header following Pager.journalOff, then write a 0x00 + ** byte to the start of it to prevent it from being recognized. + ** + ** Variable iNextHdrOffset is set to the offset at which this + ** problematic header will occur, if it exists. aMagic is used + ** as a temporary buffer to inspect the first couple of bytes of + ** the potential journal header. + */ + i64 iNextHdrOffset; + u8 aMagic[8]; + u8 zHeader[sizeof(aJournalMagic)+4]; + + memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); + put32bits(&zHeader[sizeof(aJournalMagic)], pPager->nRec); + + iNextHdrOffset = journalHdrOffset(pPager); + rc = sqlite3OsRead(pPager->jfd, aMagic, 8, iNextHdrOffset); + if( rc==SQLITE_OK && 0==memcmp(aMagic, aJournalMagic, 8) ){ + static const u8 zerobyte = 0; + rc = sqlite3OsWrite(pPager->jfd, &zerobyte, 1, iNextHdrOffset); + } + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ + return rc; + } + + /* Write the nRec value into the journal file header. If in + ** full-synchronous mode, sync the journal first. This ensures that + ** all data has really hit the disk before nRec is updated to mark + ** it as a candidate for rollback. + ** + ** This is not required if the persistent media supports the + ** SAFE_APPEND property. Because in this case it is not possible + ** for garbage data to be appended to the file, the nRec field + ** is populated with 0xFFFFFFFF when the journal header is written + ** and never needs to be updated. + */ + if( pPager->fullSync && 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ + PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); + IOTRACE(("JSYNC %p\n", pPager)) + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + if( rc!=SQLITE_OK ) return rc; + } + IOTRACE(("JHDR %p %lld\n", pPager, pPager->journalHdr)); + rc = sqlite3OsWrite( + pPager->jfd, zHeader, sizeof(zHeader), pPager->journalHdr + ); + if( rc!=SQLITE_OK ) return rc; + } + if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ + PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); + IOTRACE(("JSYNC %p\n", pPager)) + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags| + (pPager->syncFlags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) + ); + if( rc!=SQLITE_OK ) return rc; + } + + pPager->journalHdr = pPager->journalOff; + if( newHdr && 0==(iDc&SQLITE_IOCAP_SAFE_APPEND) ){ + pPager->nRec = 0; + rc = writeJournalHdr(pPager); + if( rc!=SQLITE_OK ) return rc; + } + }else{ + pPager->journalHdr = pPager->journalOff; + } + } + + /* Unless the pager is in noSync mode, the journal file was just + ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on + ** all pages. + */ + sqlite3PcacheClearSyncFlags(pPager->pPCache); + pPager->eState = PAGER_WRITER_DBMOD; + assert( assert_pager_state(pPager) ); + return SQLITE_OK; +} + +/* +** The argument is the first in a linked list of dirty pages connected +** by the PgHdr.pDirty pointer. This function writes each one of the +** in-memory pages in the list to the database file. The argument may +** be NULL, representing an empty list. In this case this function is +** a no-op. +** +** The pager must hold at least a RESERVED lock when this function +** is called. Before writing anything to the database file, this lock +** is upgraded to an EXCLUSIVE lock. If the lock cannot be obtained, +** SQLITE_BUSY is returned and no data is written to the database file. +** +** If the pager is a temp-file pager and the actual file-system file +** is not yet open, it is created and opened before any data is +** written out. +** +** Once the lock has been upgraded and, if necessary, the file opened, +** the pages are written out to the database file in list order. Writing +** a page is skipped if it meets either of the following criteria: +** +** * The page number is greater than Pager.dbSize, or +** * The PGHDR_DONT_WRITE flag is set on the page. +** +** If writing out a page causes the database file to grow, Pager.dbFileSize +** is updated accordingly. If page 1 is written out, then the value cached +** in Pager.dbFileVers[] is updated to match the new value stored in +** the database file. +** +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs, an IO error code is returned. Or, if the EXCLUSIVE lock cannot +** be obtained, SQLITE_BUSY is returned. +*/ +static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ + int rc = SQLITE_OK; /* Return code */ + + /* This function is only called for rollback pagers in WRITER_DBMOD state. */ + assert( !pagerUseWal(pPager) ); + assert( pPager->tempFile || pPager->eState==PAGER_WRITER_DBMOD ); + assert( pPager->eLock==EXCLUSIVE_LOCK ); + assert( isOpen(pPager->fd) || pList->pDirty==0 ); + + /* If the file is a temp-file has not yet been opened, open it now. It + ** is not possible for rc to be other than SQLITE_OK if this branch + ** is taken, as pager_wait_on_lock() is a no-op for temp-files. + */ + if( !isOpen(pPager->fd) ){ + assert( pPager->tempFile && rc==SQLITE_OK ); + rc = pagerOpentemp(pPager, pPager->fd, pPager->vfsFlags); + } + + /* Before the first write, give the VFS a hint of what the final + ** file size will be. + */ + assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); + if( rc==SQLITE_OK + && pPager->dbHintSizedbSize + && (pList->pDirty || pList->pgno>pPager->dbHintSize) + ){ + sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); + pPager->dbHintSize = pPager->dbSize; + } + + while( rc==SQLITE_OK && pList ){ + Pgno pgno = pList->pgno; + + /* If there are dirty pages in the page cache with page numbers greater + ** than Pager.dbSize, this means sqlite3PagerTruncateImage() was called to + ** make the file smaller (presumably by auto-vacuum code). Do not write + ** any such pages to the file. + ** + ** Also, do not write out any page that has the PGHDR_DONT_WRITE flag + ** set (set by sqlite3PagerDontWrite()). + */ + if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ + i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ + char *pData; /* Data to write */ + + assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); + if( pList->pgno==1 ) pager_write_changecounter(pList); + + pData = pList->pData; + + /* Write out the page data. */ + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); + + /* If page 1 was just written, update Pager.dbFileVers to match + ** the value now stored in the database file. If writing this + ** page caused the database file to grow, update dbFileSize. + */ + if( pgno==1 ){ + memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers)); + } + if( pgno>pPager->dbFileSize ){ + pPager->dbFileSize = pgno; + } + pPager->aStat[PAGER_STAT_WRITE]++; + + /* Update any backup objects copying the contents of this pager. */ + sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)pList->pData); + + PAGERTRACE(("STORE %d page %d hash(%08x)\n", + PAGERID(pPager), pgno, pager_pagehash(pList))); + IOTRACE(("PGOUT %p %d\n", pPager, pgno)); + PAGER_INCR(sqlite3_pager_writedb_count); + }else{ + PAGERTRACE(("NOSTORE %d page %d\n", PAGERID(pPager), pgno)); + } + pager_set_pagehash(pList); + pList = pList->pDirty; + } + + return rc; +} + +/* +** Ensure that the sub-journal file is open. If it is already open, this +** function is a no-op. +** +** SQLITE_OK is returned if everything goes according to plan. An +** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() +** fails. +*/ +static int openSubJournal(Pager *pPager){ + int rc = SQLITE_OK; + if( !isOpen(pPager->sjfd) ){ + const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE + | SQLITE_OPEN_DELETEONCLOSE; + int nStmtSpill = sqlite3Config.nStmtSpill; + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ + nStmtSpill = -1; + } + rc = sqlite3JournalOpen(pPager->pVfs, 0, pPager->sjfd, flags, nStmtSpill); + } + return rc; +} + +/* +** Append a record of the current state of page pPg to the sub-journal. +** +** If successful, set the bit corresponding to pPg->pgno in the bitvecs +** for all open savepoints before returning. +** +** This function returns SQLITE_OK if everything is successful, an IO +** error code if the attempt to write to the sub-journal fails, or +** SQLITE_NOMEM if a malloc fails while setting a bit in a savepoint +** bitvec. +*/ +static int subjournalPage(PgHdr *pPg){ + int rc = SQLITE_OK; + Pager *pPager = pPg->pPager; + if( pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + + /* Open the sub-journal, if it has not already been opened */ + assert( pPager->useJournal ); + assert( isOpen(pPager->jfd) || pagerUseWal(pPager) ); + assert( isOpen(pPager->sjfd) || pPager->nSubRec==0 ); + assert( pagerUseWal(pPager) + || pageInJournal(pPager, pPg) + || pPg->pgno>pPager->dbOrigSize + ); + rc = openSubJournal(pPager); + + /* If the sub-journal was opened successfully (or was already open), + ** write the journal record into the file. */ + if( rc==SQLITE_OK ){ + void *pData = pPg->pData; + i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); + char *pData2; + pData2 = pData; + PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); + rc = write32bits(pPager->sjfd, offset, pPg->pgno); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4); + } + } + } + if( rc==SQLITE_OK ){ + pPager->nSubRec++; + assert( pPager->nSavepoint>0 ); + rc = addToSavepointBitvecs(pPager, pPg->pgno); + } + return rc; +} +static int subjournalPageIfRequired(PgHdr *pPg){ + if( subjRequiresPage(pPg) ){ + return subjournalPage(pPg); + }else{ + return SQLITE_OK; + } +} + +/* +** This function is called by the pcache layer when it has reached some +** soft memory limit. The first argument is a pointer to a Pager object +** (cast as a void*). The pager is always 'purgeable' (not an in-memory +** database). The second argument is a reference to a page that is +** currently dirty but has no outstanding references. The page +** is always associated with the Pager object passed as the first +** argument. +** +** The job of this function is to make pPg clean by writing its contents +** out to the database file, if possible. This may involve syncing the +** journal file. +** +** If successful, sqlite3PcacheMakeClean() is called on the page and +** SQLITE_OK returned. If an IO error occurs while trying to make the +** page clean, the IO error code is returned. If the page cannot be +** made clean for some other reason, but no error occurs, then SQLITE_OK +** is returned by sqlite3PcacheMakeClean() is not called. +*/ +static int pagerStress(void *p, PgHdr *pPg){ + Pager *pPager = (Pager *)p; + int rc = SQLITE_OK; + + assert( pPg->pPager==pPager ); + assert( pPg->flags&PGHDR_DIRTY ); + + /* The doNotSpill NOSYNC bit is set during times when doing a sync of + ** journal (and adding a new header) is not allowed. This occurs + ** during calls to sqlite3PagerWrite() while trying to journal multiple + ** pages belonging to the same sector. + ** + ** The doNotSpill ROLLBACK and OFF bits inhibits all cache spilling + ** regardless of whether or not a sync is required. This is set during + ** a rollback or by user request, respectively. + ** + ** Spilling is also prohibited when in an error state since that could + ** lead to database corruption. In the current implementation it + ** is impossible for sqlite3PcacheFetch() to be called with createFlag==3 + ** while in the error state, hence it is impossible for this routine to + ** be called in the error state. Nevertheless, we include a NEVER() + ** test for the error state as a safeguard against future changes. + */ + if( NEVER(pPager->errCode) ) return SQLITE_OK; + testcase( pPager->doNotSpill & SPILLFLAG_ROLLBACK ); + testcase( pPager->doNotSpill & SPILLFLAG_OFF ); + testcase( pPager->doNotSpill & SPILLFLAG_NOSYNC ); + if( pPager->doNotSpill + && ((pPager->doNotSpill & (SPILLFLAG_ROLLBACK|SPILLFLAG_OFF))!=0 + || (pPg->flags & PGHDR_NEED_SYNC)!=0) + ){ + return SQLITE_OK; + } + + pPager->aStat[PAGER_STAT_SPILL]++; + pPg->pDirty = 0; + if( pagerUseWal(pPager) ){ + /* Write a single frame for this page to the log. */ + rc = subjournalPageIfRequired(pPg); + if( rc==SQLITE_OK ){ + rc = pagerWalFrames(pPager, pPg, 0, 0); + } + }else{ + +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( pPager->tempFile==0 ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ) return pager_error(pPager, rc); + } +#endif + + /* Sync the journal file if required. */ + if( pPg->flags&PGHDR_NEED_SYNC + || pPager->eState==PAGER_WRITER_CACHEMOD + ){ + rc = syncJournal(pPager, 1); + } + + /* Write the contents of the page out to the database file. */ + if( rc==SQLITE_OK ){ + assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); + rc = pager_write_pagelist(pPager, pPg); + } + } + + /* Mark the page as clean. */ + if( rc==SQLITE_OK ){ + PAGERTRACE(("STRESS %d page %d\n", PAGERID(pPager), pPg->pgno)); + sqlite3PcacheMakeClean(pPg); + } + + return pager_error(pPager, rc); +} + +/* +** Flush all unreferenced dirty pages to disk. +*/ +SQLITE_PRIVATE int sqlite3PagerFlush(Pager *pPager){ + int rc = pPager->errCode; + if( !MEMDB ){ + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + assert( assert_pager_state(pPager) ); + while( rc==SQLITE_OK && pList ){ + PgHdr *pNext = pList->pDirty; + if( pList->nRef==0 ){ + rc = pagerStress((void*)pPager, pList); + } + pList = pNext; + } + } + + return rc; +} + +/* +** Allocate and initialize a new Pager object and put a pointer to it +** in *ppPager. The pager should eventually be freed by passing it +** to sqlite3PagerClose(). +** +** The zFilename argument is the path to the database file to open. +** If zFilename is NULL then a randomly-named temporary file is created +** and used as the file to be cached. Temporary files are be deleted +** automatically when they are closed. If zFilename is ":memory:" then +** all information is held in cache. It is never written to disk. +** This can be used to implement an in-memory database. +** +** The nExtra parameter specifies the number of bytes of space allocated +** along with each page reference. This space is available to the user +** via the sqlite3PagerGetExtra() API. When a new page is allocated, the +** first 8 bytes of this space are zeroed but the remainder is uninitialized. +** (The extra space is used by btree as the MemPage object.) +** +** The flags argument is used to specify properties that affect the +** operation of the pager. It should be passed some bitwise combination +** of the PAGER_* flags. +** +** The vfsFlags parameter is a bitmask to pass to the flags parameter +** of the xOpen() method of the supplied VFS when opening files. +** +** If the pager object is allocated and the specified file opened +** successfully, SQLITE_OK is returned and *ppPager set to point to +** the new pager object. If an error occurs, *ppPager is set to NULL +** and error code returned. This function may return SQLITE_NOMEM +** (sqlite3Malloc() is used to allocate memory), SQLITE_CANTOPEN or +** various SQLITE_IO_XXX errors. +*/ +SQLITE_PRIVATE int sqlite3PagerOpen( + sqlite3_vfs *pVfs, /* The virtual file system to use */ + Pager **ppPager, /* OUT: Return the Pager structure here */ + const char *zFilename, /* Name of the database file to open */ + int nExtra, /* Extra bytes append to each in-memory page */ + int flags, /* flags controlling this file */ + int vfsFlags, /* flags passed through to sqlite3_vfs.xOpen() */ + void (*xReinit)(DbPage*) /* Function to reinitialize pages */ +){ + u8 *pPtr; + Pager *pPager = 0; /* Pager object to allocate and return */ + int rc = SQLITE_OK; /* Return code */ + int tempFile = 0; /* True for temp files (incl. in-memory files) */ + int memDb = 0; /* True if this is an in-memory file */ +#ifndef SQLITE_OMIT_DESERIALIZE + int memJM = 0; /* Memory journal mode */ +#else +# define memJM 0 +#endif + int readOnly = 0; /* True if this is a read-only file */ + int journalFileSize; /* Bytes to allocate for each journal fd */ + char *zPathname = 0; /* Full path to database file */ + int nPathname = 0; /* Number of bytes in zPathname */ + int useJournal = (flags & PAGER_OMIT_JOURNAL)==0; /* False to omit journal */ + int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ + u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ + const char *zUri = 0; /* URI args to copy */ + int nUriByte = 1; /* Number of bytes of URI args at *zUri */ + int nUri = 0; /* Number of URI parameters */ + + /* Figure out how much space is required for each journal file-handle + ** (there are two of them, the main journal and the sub-journal). */ + journalFileSize = ROUND8(sqlite3JournalSize(pVfs)); + + /* Set the output variable to NULL in case an error occurs. */ + *ppPager = 0; + +#ifndef SQLITE_OMIT_MEMORYDB + if( flags & PAGER_MEMORY ){ + memDb = 1; + if( zFilename && zFilename[0] ){ + zPathname = sqlite3DbStrDup(0, zFilename); + if( zPathname==0 ) return SQLITE_NOMEM_BKPT; + nPathname = sqlite3Strlen30(zPathname); + zFilename = 0; + } + } +#endif + + /* Compute and store the full pathname in an allocated buffer pointed + ** to by zPathname, length nPathname. Or, if this is a temporary file, + ** leave both nPathname and zPathname set to 0. + */ + if( zFilename && zFilename[0] ){ + const char *z; + nPathname = pVfs->mxPathname+1; + zPathname = sqlite3DbMallocRaw(0, nPathname*2); + if( zPathname==0 ){ + return SQLITE_NOMEM_BKPT; + } + zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ + rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK_SYMLINK ){ + if( vfsFlags & SQLITE_OPEN_NOFOLLOW ){ + rc = SQLITE_CANTOPEN_SYMLINK; + }else{ + rc = SQLITE_OK; + } + } + } + nPathname = sqlite3Strlen30(zPathname); + z = zUri = &zFilename[sqlite3Strlen30(zFilename)+1]; + while( *z ){ + z += strlen(z)+1; + z += strlen(z)+1; + nUri++; + } + nUriByte = (int)(&z[1] - zUri); + assert( nUriByte>=1 ); + if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ + /* This branch is taken when the journal path required by + ** the database being opened will be more than pVfs->mxPathname + ** bytes in length. This means the database cannot be opened, + ** as it will not be possible to open the journal file or even + ** check for a hot-journal before reading. + */ + rc = SQLITE_CANTOPEN_BKPT; + } + if( rc!=SQLITE_OK ){ + sqlite3DbFree(0, zPathname); + return rc; + } + } + + /* Allocate memory for the Pager structure, PCache object, the + ** three file descriptors, the database file name and the journal + ** file name. The layout in memory is as follows: + ** + ** Pager object (sizeof(Pager) bytes) + ** PCache object (sqlite3PcacheSize() bytes) + ** Database file handle (pVfs->szOsFile bytes) + ** Sub-journal file handle (journalFileSize bytes) + ** Main journal file handle (journalFileSize bytes) + ** Ptr back to the Pager (sizeof(Pager*) bytes) + ** \0\0\0\0 database prefix (4 bytes) + ** Database file name (nPathname+1 bytes) + ** URI query parameters (nUriByte bytes) + ** Journal filename (nPathname+8+1 bytes) + ** WAL filename (nPathname+4+1 bytes) + ** \0\0\0 terminator (3 bytes) + ** + ** Some 3rd-party software, over which we have no control, depends on + ** the specific order of the filenames and the \0 separators between them + ** so that it can (for example) find the database filename given the WAL + ** filename without using the sqlite3_filename_database() API. This is a + ** misuse of SQLite and a bug in the 3rd-party software, but the 3rd-party + ** software is in widespread use, so we try to avoid changing the filename + ** order and formatting if possible. In particular, the details of the + ** filename format expected by 3rd-party software should be as follows: + ** + ** - Main Database Path + ** - \0 + ** - Multiple URI components consisting of: + ** - Key + ** - \0 + ** - Value + ** - \0 + ** - \0 + ** - Journal Path + ** - \0 + ** - WAL Path (zWALName) + ** - \0 + ** + ** The sqlite3_create_filename() interface and the databaseFilename() utility + ** that is used by sqlite3_filename_database() and kin also depend on the + ** specific formatting and order of the various filenames, so if the format + ** changes here, be sure to change it there as well. + */ + pPtr = (u8 *)sqlite3MallocZero( + ROUND8(sizeof(*pPager)) + /* Pager structure */ + ROUND8(pcacheSize) + /* PCache object */ + ROUND8(pVfs->szOsFile) + /* The main db file */ + journalFileSize * 2 + /* The two journal files */ + sizeof(pPager) + /* Space to hold a pointer */ + 4 + /* Database prefix */ + nPathname + 1 + /* database filename */ + nUriByte + /* query parameters */ + nPathname + 8 + 1 + /* Journal filename */ +#ifndef SQLITE_OMIT_WAL + nPathname + 4 + 1 + /* WAL filename */ +#endif + 3 /* Terminator */ + ); + assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); + if( !pPtr ){ + sqlite3DbFree(0, zPathname); + return SQLITE_NOMEM_BKPT; + } + pPager = (Pager*)pPtr; pPtr += ROUND8(sizeof(*pPager)); + pPager->pPCache = (PCache*)pPtr; pPtr += ROUND8(pcacheSize); + pPager->fd = (sqlite3_file*)pPtr; pPtr += ROUND8(pVfs->szOsFile); + pPager->sjfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; + pPager->jfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; + assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); + memcpy(pPtr, &pPager, sizeof(pPager)); pPtr += sizeof(pPager); + + /* Fill in the Pager.zFilename and pPager.zQueryParam fields */ + pPtr += 4; /* Skip zero prefix */ + pPager->zFilename = (char*)pPtr; + if( nPathname>0 ){ + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname + 1; + if( zUri ){ + memcpy(pPtr, zUri, nUriByte); pPtr += nUriByte; + }else{ + pPtr++; + } + } + + + /* Fill in Pager.zJournal */ + if( nPathname>0 ){ + pPager->zJournal = (char*)pPtr; + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname; + memcpy(pPtr, "-journal",8); pPtr += 8 + 1; +#ifdef SQLITE_ENABLE_8_3_NAMES + sqlite3FileSuffix3(zFilename,pPager->zJournal); + pPtr = (u8*)(pPager->zJournal + sqlite3Strlen30(pPager->zJournal)+1); +#endif + }else{ + pPager->zJournal = 0; + } + +#ifndef SQLITE_OMIT_WAL + /* Fill in Pager.zWal */ + if( nPathname>0 ){ + pPager->zWal = (char*)pPtr; + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname; + memcpy(pPtr, "-wal", 4); pPtr += 4 + 1; +#ifdef SQLITE_ENABLE_8_3_NAMES + sqlite3FileSuffix3(zFilename, pPager->zWal); + pPtr = (u8*)(pPager->zWal + sqlite3Strlen30(pPager->zWal)+1); +#endif + }else{ + pPager->zWal = 0; + } +#endif + (void)pPtr; /* Suppress warning about unused pPtr value */ + + if( nPathname ) sqlite3DbFree(0, zPathname); + pPager->pVfs = pVfs; + pPager->vfsFlags = vfsFlags; + + /* Open the pager file. + */ + if( zFilename && zFilename[0] ){ + int fout = 0; /* VFS flags returned by xOpen() */ + rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); + assert( !memDb ); +#ifndef SQLITE_OMIT_DESERIALIZE + memJM = (fout&SQLITE_OPEN_MEMORY)!=0; +#endif + readOnly = (fout&SQLITE_OPEN_READONLY)!=0; + + /* If the file was successfully opened for read/write access, + ** choose a default page size in case we have to create the + ** database file. The default page size is the maximum of: + ** + ** + SQLITE_DEFAULT_PAGE_SIZE, + ** + The value returned by sqlite3OsSectorSize() + ** + The largest page size that can be written atomically. + */ + if( rc==SQLITE_OK ){ + int iDc = sqlite3OsDeviceCharacteristics(pPager->fd); + if( !readOnly ){ + setSectorSize(pPager); + assert(SQLITE_DEFAULT_PAGE_SIZE<=SQLITE_MAX_DEFAULT_PAGE_SIZE); + if( szPageDfltsectorSize ){ + if( pPager->sectorSize>SQLITE_MAX_DEFAULT_PAGE_SIZE ){ + szPageDflt = SQLITE_MAX_DEFAULT_PAGE_SIZE; + }else{ + szPageDflt = (u32)pPager->sectorSize; + } + } +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + { + int ii; + assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); + assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); + assert(SQLITE_MAX_DEFAULT_PAGE_SIZE<=65536); + for(ii=szPageDflt; ii<=SQLITE_MAX_DEFAULT_PAGE_SIZE; ii=ii*2){ + if( iDc&(SQLITE_IOCAP_ATOMIC|(ii>>8)) ){ + szPageDflt = ii; + } + } + } +#endif + } + pPager->noLock = sqlite3_uri_boolean(pPager->zFilename, "nolock", 0); + if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 + || sqlite3_uri_boolean(pPager->zFilename, "immutable", 0) ){ + vfsFlags |= SQLITE_OPEN_READONLY; + goto act_like_temp_file; + } + } + }else{ + /* If a temporary file is requested, it is not opened immediately. + ** In this case we accept the default page size and delay actually + ** opening the file until the first call to OsWrite(). + ** + ** This branch is also run for an in-memory database. An in-memory + ** database is the same as a temp-file that is never written out to + ** disk and uses an in-memory rollback journal. + ** + ** This branch also runs for files marked as immutable. + */ +act_like_temp_file: + tempFile = 1; + pPager->eState = PAGER_READER; /* Pretend we already have a lock */ + pPager->eLock = EXCLUSIVE_LOCK; /* Pretend we are in EXCLUSIVE mode */ + pPager->noLock = 1; /* Do no locking */ + readOnly = (vfsFlags&SQLITE_OPEN_READONLY); + } + + /* The following call to PagerSetPagesize() serves to set the value of + ** Pager.pageSize and to allocate the Pager.pTmpSpace buffer. + */ + if( rc==SQLITE_OK ){ + assert( pPager->memDb==0 ); + rc = sqlite3PagerSetPagesize(pPager, &szPageDflt, -1); + testcase( rc!=SQLITE_OK ); + } + + /* Initialize the PCache object. */ + if( rc==SQLITE_OK ){ + nExtra = ROUND8(nExtra); + assert( nExtra>=8 && nExtra<1000 ); + rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); + } + + /* If an error occurred above, free the Pager structure and close the file. + */ + if( rc!=SQLITE_OK ){ + sqlite3OsClose(pPager->fd); + sqlite3PageFree(pPager->pTmpSpace); + sqlite3_free(pPager); + return rc; + } + + PAGERTRACE(("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename)); + IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename)) + + pPager->useJournal = (u8)useJournal; + /* pPager->stmtOpen = 0; */ + /* pPager->stmtInUse = 0; */ + /* pPager->nRef = 0; */ + /* pPager->stmtSize = 0; */ + /* pPager->stmtJSize = 0; */ + /* pPager->nPage = 0; */ + pPager->mxPgno = SQLITE_MAX_PAGE_COUNT; + /* pPager->state = PAGER_UNLOCK; */ + /* pPager->errMask = 0; */ + pPager->tempFile = (u8)tempFile; + assert( tempFile==PAGER_LOCKINGMODE_NORMAL + || tempFile==PAGER_LOCKINGMODE_EXCLUSIVE ); + assert( PAGER_LOCKINGMODE_EXCLUSIVE==1 ); + pPager->exclusiveMode = (u8)tempFile; + pPager->changeCountDone = pPager->tempFile; + pPager->memDb = (u8)memDb; + pPager->readOnly = (u8)readOnly; + assert( useJournal || pPager->tempFile ); + pPager->noSync = pPager->tempFile; + if( pPager->noSync ){ + assert( pPager->fullSync==0 ); + assert( pPager->extraSync==0 ); + assert( pPager->syncFlags==0 ); + assert( pPager->walSyncFlags==0 ); + }else{ + pPager->fullSync = 1; + pPager->extraSync = 0; + pPager->syncFlags = SQLITE_SYNC_NORMAL; + pPager->walSyncFlags = SQLITE_SYNC_NORMAL | (SQLITE_SYNC_NORMAL<<2); + } + /* pPager->pFirst = 0; */ + /* pPager->pFirstSynced = 0; */ + /* pPager->pLast = 0; */ + pPager->nExtra = (u16)nExtra; + pPager->journalSizeLimit = SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT; + assert( isOpen(pPager->fd) || tempFile ); + setSectorSize(pPager); + if( !useJournal ){ + pPager->journalMode = PAGER_JOURNALMODE_OFF; + }else if( memDb || memJM ){ + pPager->journalMode = PAGER_JOURNALMODE_MEMORY; + } + /* pPager->xBusyHandler = 0; */ + /* pPager->pBusyHandlerArg = 0; */ + pPager->xReiniter = xReinit; + setGetterMethod(pPager); + /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + /* pPager->szMmap = SQLITE_DEFAULT_MMAP_SIZE // will be set by btree.c */ + + *ppPager = pPager; + return SQLITE_OK; +} + +/* +** Return the sqlite3_file for the main database given the name +** of the corresonding WAL or Journal name as passed into +** xOpen. +*/ +SQLITE_API sqlite3_file *sqlite3_database_file_object(const char *zName){ + Pager *pPager; + while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ + zName--; + } + pPager = *(Pager**)(zName - 4 - sizeof(Pager*)); + return pPager->fd; +} + + +/* +** This function is called after transitioning from PAGER_UNLOCK to +** PAGER_SHARED state. It tests if there is a hot journal present in +** the file-system for the given pager. A hot journal is one that +** needs to be played back. According to this function, a hot-journal +** file exists if the following criteria are met: +** +** * The journal file exists in the file system, and +** * No process holds a RESERVED or greater lock on the database file, and +** * The database file itself is greater than 0 bytes in size, and +** * The first byte of the journal file exists and is not 0x00. +** +** If the current size of the database file is 0 but a journal file +** exists, that is probably an old journal left over from a prior +** database with the same name. In this case the journal file is +** just deleted using OsDelete, *pExists is set to 0 and SQLITE_OK +** is returned. +** +** This routine does not check if there is a super-journal filename +** at the end of the file. If there is, and that super-journal file +** does not exist, then the journal file is not really hot. In this +** case this routine will return a false-positive. The pager_playback() +** routine will discover that the journal file is not really hot and +** will not roll it back. +** +** If a hot-journal file is found to exist, *pExists is set to 1 and +** SQLITE_OK returned. If no hot-journal file is present, *pExists is +** set to 0 and SQLITE_OK returned. If an IO error occurs while trying +** to determine whether or not a hot-journal file exists, the IO error +** code is returned and the value of *pExists is undefined. +*/ +static int hasHotJournal(Pager *pPager, int *pExists){ + sqlite3_vfs * const pVfs = pPager->pVfs; + int rc = SQLITE_OK; /* Return code */ + int exists = 1; /* True if a journal file is present */ + int jrnlOpen = !!isOpen(pPager->jfd); + + assert( pPager->useJournal ); + assert( isOpen(pPager->fd) ); + assert( pPager->eState==PAGER_OPEN ); + + assert( jrnlOpen==0 || ( sqlite3OsDeviceCharacteristics(pPager->jfd) & + SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN + )); + + *pExists = 0; + if( !jrnlOpen ){ + rc = sqlite3OsAccess(pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &exists); + } + if( rc==SQLITE_OK && exists ){ + int locked = 0; /* True if some process holds a RESERVED lock */ + + /* Race condition here: Another process might have been holding the + ** the RESERVED lock and have a journal open at the sqlite3OsAccess() + ** call above, but then delete the journal and drop the lock before + ** we get to the following sqlite3OsCheckReservedLock() call. If that + ** is the case, this routine might think there is a hot journal when + ** in fact there is none. This results in a false-positive which will + ** be dealt with by the playback routine. Ticket #3883. + */ + rc = sqlite3OsCheckReservedLock(pPager->fd, &locked); + if( rc==SQLITE_OK && !locked ){ + Pgno nPage; /* Number of pages in database file */ + + assert( pPager->tempFile==0 ); + rc = pagerPagecount(pPager, &nPage); + if( rc==SQLITE_OK ){ + /* If the database is zero pages in size, that means that either (1) the + ** journal is a remnant from a prior database with the same name where + ** the database file but not the journal was deleted, or (2) the initial + ** transaction that populates a new database is being rolled back. + ** In either case, the journal file can be deleted. However, take care + ** not to delete the journal file if it is already open due to + ** journal_mode=PERSIST. + */ + if( nPage==0 && !jrnlOpen ){ + sqlite3BeginBenignMalloc(); + if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ + sqlite3OsDelete(pVfs, pPager->zJournal, 0); + if( !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); + } + sqlite3EndBenignMalloc(); + }else{ + /* The journal file exists and no other connection has a reserved + ** or greater lock on the database file. Now check that there is + ** at least one non-zero bytes at the start of the journal file. + ** If there is, then we consider this journal to be hot. If not, + ** it can be ignored. + */ + if( !jrnlOpen ){ + int f = SQLITE_OPEN_READONLY|SQLITE_OPEN_MAIN_JOURNAL; + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &f); + } + if( rc==SQLITE_OK ){ + u8 first = 0; + rc = sqlite3OsRead(pPager->jfd, (void *)&first, 1, 0); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + if( !jrnlOpen ){ + sqlite3OsClose(pPager->jfd); + } + *pExists = (first!=0); + }else if( rc==SQLITE_CANTOPEN ){ + /* If we cannot open the rollback journal file in order to see if + ** it has a zero header, that might be due to an I/O error, or + ** it might be due to the race condition described above and in + ** ticket #3883. Either way, assume that the journal is hot. + ** This might be a false positive. But if it is, then the + ** automatic journal playback and recovery mechanism will deal + ** with it under an EXCLUSIVE lock where we do not need to + ** worry so much with race conditions. + */ + *pExists = 1; + rc = SQLITE_OK; + } + } + } + } + } + + return rc; +} + +/* +** This function is called to obtain a shared lock on the database file. +** It is illegal to call sqlite3PagerGet() until after this function +** has been successfully called. If a shared-lock is already held when +** this function is called, it is a no-op. +** +** The following operations are also performed by this function. +** +** 1) If the pager is currently in PAGER_OPEN state (no lock held +** on the database file), then an attempt is made to obtain a +** SHARED lock on the database file. Immediately after obtaining +** the SHARED lock, the file-system is checked for a hot-journal, +** which is played back if present. Following any hot-journal +** rollback, the contents of the cache are validated by checking +** the 'change-counter' field of the database file header and +** discarded if they are found to be invalid. +** +** 2) If the pager is running in exclusive-mode, and there are currently +** no outstanding references to any pages, and is in the error state, +** then an attempt is made to clear the error state by discarding +** the contents of the page cache and rolling back any open journal +** file. +** +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs while locking the database, checking for a hot-journal file or +** rolling back a journal file, the IO error code is returned. +*/ +SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + + /* This routine is only called from b-tree and only when there are no + ** outstanding pages. This implies that the pager state should either + ** be OPEN or READER. READER is only possible if the pager is or was in + ** exclusive access mode. */ + assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pPager->eState==PAGER_READER ); + assert( pPager->errCode==SQLITE_OK ); + + if( !pagerUseWal(pPager) && pPager->eState==PAGER_OPEN ){ + int bHotJournal = 1; /* True if there exists a hot journal-file */ + + assert( !MEMDB ); + assert( pPager->tempFile==0 || pPager->eLock==EXCLUSIVE_LOCK ); + + rc = pager_wait_on_lock(pPager, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + assert( pPager->eLock==NO_LOCK || pPager->eLock==UNKNOWN_LOCK ); + goto failed; + } + + /* If a journal file exists, and there is no RESERVED lock on the + ** database file, then it either needs to be played back or deleted. + */ + if( pPager->eLock<=SHARED_LOCK ){ + rc = hasHotJournal(pPager, &bHotJournal); + } + if( rc!=SQLITE_OK ){ + goto failed; + } + if( bHotJournal ){ + if( pPager->readOnly ){ + rc = SQLITE_READONLY_ROLLBACK; + goto failed; + } + + /* Get an EXCLUSIVE lock on the database file. At this point it is + ** important that a RESERVED lock is not obtained on the way to the + ** EXCLUSIVE lock. If it were, another process might open the + ** database file, detect the RESERVED lock, and conclude that the + ** database is safe to read while this process is still rolling the + ** hot-journal back. + ** + ** Because the intermediate RESERVED lock is not requested, any + ** other process attempting to access the database file will get to + ** this point in the code and fail to obtain its own EXCLUSIVE lock + ** on the database file. + ** + ** Unless the pager is in locking_mode=exclusive mode, the lock is + ** downgraded to SHARED_LOCK before this function returns. + */ + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + goto failed; + } + + /* If it is not already open and the file exists on disk, open the + ** journal for read/write access. Write access is required because + ** in exclusive-access mode the file descriptor will be kept open + ** and possibly used for a transaction later on. Also, write-access + ** is usually required to finalize the journal in journal_mode=persist + ** mode (and also for journal_mode=truncate on some systems). + ** + ** If the journal does not exist, it usually means that some + ** other connection managed to get in and roll it back before + ** this connection obtained the exclusive lock above. Or, it + ** may mean that the pager was in the error-state when this + ** function was called and the journal file does not exist. + */ + if( !isOpen(pPager->jfd) ){ + sqlite3_vfs * const pVfs = pPager->pVfs; + int bExists; /* True if journal file exists */ + rc = sqlite3OsAccess( + pVfs, pPager->zJournal, SQLITE_ACCESS_EXISTS, &bExists); + if( rc==SQLITE_OK && bExists ){ + int fout = 0; + int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_JOURNAL; + assert( !pPager->tempFile ); + rc = sqlite3OsOpen(pVfs, pPager->zJournal, pPager->jfd, f, &fout); + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + if( rc==SQLITE_OK && fout&SQLITE_OPEN_READONLY ){ + rc = SQLITE_CANTOPEN_BKPT; + sqlite3OsClose(pPager->jfd); + } + } + } + + /* Playback and delete the journal. Drop the database write + ** lock and reacquire the read lock. Purge the cache before + ** playing back the hot-journal so that we don't end up with + ** an inconsistent cache. Sync the hot journal before playing + ** it back since the process that crashed and left the hot journal + ** probably did not sync it and we are required to always sync + ** the journal before playing it back. + */ + if( isOpen(pPager->jfd) ){ + assert( rc==SQLITE_OK ); + rc = pagerSyncHotJournal(pPager); + if( rc==SQLITE_OK ){ + rc = pager_playback(pPager, !pPager->tempFile); + pPager->eState = PAGER_OPEN; + } + }else if( !pPager->exclusiveMode ){ + pagerUnlockDb(pPager, SHARED_LOCK); + } + + if( rc!=SQLITE_OK ){ + /* This branch is taken if an error occurs while trying to open + ** or roll back a hot-journal while holding an EXCLUSIVE lock. The + ** pager_unlock() routine will be called before returning to unlock + ** the file. If the unlock attempt fails, then Pager.eLock must be + ** set to UNKNOWN_LOCK (see the comment above the #define for + ** UNKNOWN_LOCK above for an explanation). + ** + ** In order to get pager_unlock() to do this, set Pager.eState to + ** PAGER_ERROR now. This is not actually counted as a transition + ** to ERROR state in the state diagram at the top of this file, + ** since we know that the same call to pager_unlock() will very + ** shortly transition the pager object to the OPEN state. Calling + ** assert_pager_state() would fail now, as it should not be possible + ** to be in ERROR state when there are zero outstanding page + ** references. + */ + pager_error(pPager, rc); + goto failed; + } + + assert( pPager->eState==PAGER_OPEN ); + assert( (pPager->eLock==SHARED_LOCK) + || (pPager->exclusiveMode && pPager->eLock>SHARED_LOCK) + ); + } + + if( !pPager->tempFile && pPager->hasHeldSharedLock ){ + /* The shared-lock has just been acquired then check to + ** see if the database has been modified. If the database has changed, + ** flush the cache. The hasHeldSharedLock flag prevents this from + ** occurring on the very first access to a file, in order to save a + ** single unnecessary sqlite3OsRead() call at the start-up. + ** + ** Database changes are detected by looking at 15 bytes beginning + ** at offset 24 into the file. The first 4 of these 16 bytes are + ** a 32-bit counter that is incremented with each change. The + ** other bytes change randomly with each file change when + ** a codec is in use. + ** + ** There is a vanishingly small chance that a change will not be + ** detected. The chance of an undetected change is so small that + ** it can be neglected. + */ + char dbFileVers[sizeof(pPager->dbFileVers)]; + + IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_IOERR_SHORT_READ ){ + goto failed; + } + memset(dbFileVers, 0, sizeof(dbFileVers)); + } + + if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ + pager_reset(pPager); + + /* Unmap the database file. It is possible that external processes + ** may have truncated the database file and then extended it back + ** to its original size while this process was not holding a lock. + ** In this case there may exist a Pager.pMap mapping that appears + ** to be the right size but is not actually valid. Avoid this + ** possibility by unmapping the db here. */ + if( USEFETCH(pPager) ){ + sqlite3OsUnfetch(pPager->fd, 0, 0); + } + } + } + + /* If there is a WAL file in the file-system, open this database in WAL + ** mode. Otherwise, the following function call is a no-op. + */ + rc = pagerOpenWalIfPresent(pPager); +#ifndef SQLITE_OMIT_WAL + assert( pPager->pWal==0 || rc==SQLITE_OK ); +#endif + } + + if( pagerUseWal(pPager) ){ + assert( rc==SQLITE_OK ); + rc = pagerBeginReadTransaction(pPager); + } + + if( pPager->tempFile==0 && pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ + rc = pagerPagecount(pPager, &pPager->dbSize); + } + + failed: + if( rc!=SQLITE_OK ){ + assert( !MEMDB ); + pager_unlock(pPager); + assert( pPager->eState==PAGER_OPEN ); + }else{ + pPager->eState = PAGER_READER; + pPager->hasHeldSharedLock = 1; + } + return rc; +} + +/* +** If the reference count has reached zero, rollback any active +** transaction and unlock the pager. +** +** Except, in locking_mode=EXCLUSIVE when there is nothing to in +** the rollback journal, the unlock is not performed and there is +** nothing to rollback, so this routine is a no-op. +*/ +static void pagerUnlockIfUnused(Pager *pPager){ + if( sqlite3PcacheRefCount(pPager->pPCache)==0 ){ + assert( pPager->nMmapOut==0 ); /* because page1 is never memory mapped */ + pagerUnlockAndRollback(pPager); + } +} + +/* +** The page getter methods each try to acquire a reference to a +** page with page number pgno. If the requested reference is +** successfully obtained, it is copied to *ppPage and SQLITE_OK returned. +** +** There are different implementations of the getter method depending +** on the current state of the pager. +** +** getPageNormal() -- The normal getter +** getPageError() -- Used if the pager is in an error state +** getPageMmap() -- Used if memory-mapped I/O is enabled +** +** If the requested page is already in the cache, it is returned. +** Otherwise, a new page object is allocated and populated with data +** read from the database file. In some cases, the pcache module may +** choose not to allocate a new page object and may reuse an existing +** object with no outstanding references. +** +** The extra data appended to a page is always initialized to zeros the +** first time a page is loaded into memory. If the page requested is +** already in the cache when this function is called, then the extra +** data is left as it was when the page object was last used. +** +** If the database image is smaller than the requested page or if +** the flags parameter contains the PAGER_GET_NOCONTENT bit and the +** requested page is not already stored in the cache, then no +** actual disk read occurs. In this case the memory image of the +** page is initialized to all zeros. +** +** If PAGER_GET_NOCONTENT is true, it means that we do not care about +** the contents of the page. This occurs in two scenarios: +** +** a) When reading a free-list leaf page from the database, and +** +** b) When a savepoint is being rolled back and we need to load +** a new page into the cache to be filled with the data read +** from the savepoint journal. +** +** If PAGER_GET_NOCONTENT is true, then the data returned is zeroed instead +** of being read from the database. Additionally, the bits corresponding +** to pgno in Pager.pInJournal (bitvec of pages already written to the +** journal file) and the PagerSavepoint.pInSavepoint bitvecs of any open +** savepoints are set. This means if the page is made writable at any +** point in the future, using a call to sqlite3PagerWrite(), its contents +** will not be journaled. This saves IO. +** +** The acquisition might fail for several reasons. In all cases, +** an appropriate error code is returned and *ppPage is set to NULL. +** +** See also sqlite3PagerLookup(). Both this routine and Lookup() attempt +** to find a page in the in-memory cache first. If the page is not already +** in memory, this routine goes to disk to read it in whereas Lookup() +** just returns 0. This routine acquires a read-lock the first time it +** has to go to disk, and could also playback an old journal if necessary. +** Since Lookup() never goes to disk, it never has to deal with locks +** or journal files. +*/ +static int getPageNormal( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + int rc = SQLITE_OK; + PgHdr *pPg; + u8 noContent; /* True if PAGER_GET_NOCONTENT is set */ + sqlite3_pcache_page *pBase; + + assert( pPager->errCode==SQLITE_OK ); + assert( pPager->eState>=PAGER_READER ); + assert( assert_pager_state(pPager) ); + assert( pPager->hasHeldSharedLock==1 ); + + if( pgno==0 ) return SQLITE_CORRUPT_BKPT; + pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); + if( pBase==0 ){ + pPg = 0; + rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + if( pBase==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto pager_acquire_err; + } + } + pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); + assert( pPg==(*ppPage) ); + assert( pPg->pgno==pgno ); + assert( pPg->pPager==pPager || pPg->pPager==0 ); + + noContent = (flags & PAGER_GET_NOCONTENT)!=0; + if( pPg->pPager && !noContent ){ + /* In this case the pcache already contains an initialized copy of + ** the page. Return without further ado. */ + assert( pgno!=PAGER_MJ_PGNO(pPager) ); + pPager->aStat[PAGER_STAT_HIT]++; + return SQLITE_OK; + + }else{ + /* The pager cache has created a new page. Its content needs to + ** be initialized. But first some error checks: + ** + ** (*) obsolete. Was: maximum page number is 2^31 + ** (2) Never try to fetch the locking page + */ + if( pgno==PAGER_MJ_PGNO(pPager) ){ + rc = SQLITE_CORRUPT_BKPT; + goto pager_acquire_err; + } + + pPg->pPager = pPager; + + assert( !isOpen(pPager->fd) || !MEMDB ); + if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ + rc = SQLITE_FULL; + goto pager_acquire_err; + } + if( noContent ){ + /* Failure to set the bits in the InJournal bit-vectors is benign. + ** It merely means that we might do some extra work to journal a + ** page that does not need to be journaled. Nevertheless, be sure + ** to test the case where a malloc error occurs while trying to set + ** a bit in a bit vector. + */ + sqlite3BeginBenignMalloc(); + if( pgno<=pPager->dbOrigSize ){ + TESTONLY( rc = ) sqlite3BitvecSet(pPager->pInJournal, pgno); + testcase( rc==SQLITE_NOMEM ); + } + TESTONLY( rc = ) addToSavepointBitvecs(pPager, pgno); + testcase( rc==SQLITE_NOMEM ); + sqlite3EndBenignMalloc(); + } + memset(pPg->pData, 0, pPager->pageSize); + IOTRACE(("ZERO %p %d\n", pPager, pgno)); + }else{ + assert( pPg->pPager==pPager ); + pPager->aStat[PAGER_STAT_MISS]++; + rc = readDbPage(pPg); + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } + } + pager_set_pagehash(pPg); + } + return SQLITE_OK; + +pager_acquire_err: + assert( rc!=SQLITE_OK ); + if( pPg ){ + sqlite3PcacheDrop(pPg); + } + pagerUnlockIfUnused(pPager); + *ppPage = 0; + return rc; +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* The page getter for when memory-mapped I/O is enabled */ +static int getPageMMap( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + int rc = SQLITE_OK; + PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = (pgno>1 + && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) + ); + + assert( USEFETCH(pPager) ); + + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" + ** test in the previous statement, and avoid testing pgno==0 in the + ** common case where pgno is large. */ + if( pgno<=1 && pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } + assert( pPager->eState>=PAGER_READER ); + assert( assert_pager_state(pPager) ); + assert( pPager->hasHeldSharedLock==1 ); + assert( pPager->errCode==SQLITE_OK ); + + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ){ + *ppPage = 0; + return rc; + } + } + if( bMmapOk && iFrame==0 ){ + void *pData = 0; + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + if( rc==SQLITE_OK && pData ){ + if( pPager->eState>PAGER_READER || pPager->tempFile ){ + pPg = sqlite3PagerLookup(pPager, pgno); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + } + } + if( rc!=SQLITE_OK ){ + *ppPage = 0; + return rc; + } + } + return getPageNormal(pPager, pgno, ppPage, flags); +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* The page getter method for when the pager is an error state */ +static int getPageError( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + UNUSED_PARAMETER(pgno); + UNUSED_PARAMETER(flags); + assert( pPager->errCode!=SQLITE_OK ); + *ppPage = 0; + return pPager->errCode; +} + + +/* Dispatch all page fetch requests to the appropriate getter method. +*/ +SQLITE_PRIVATE int sqlite3PagerGet( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + return pPager->xGet(pPager, pgno, ppPage, flags); +} + +/* +** Acquire a page if it is already in the in-memory cache. Do +** not read the page from disk. Return a pointer to the page, +** or 0 if the page is not in cache. +** +** See also sqlite3PagerGet(). The difference between this routine +** and sqlite3PagerGet() is that _get() will go to the disk and read +** in the page if the page is not already in cache. This routine +** returns NULL if the page is not in cache or if a disk I/O error +** has ever happened. +*/ +SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ + sqlite3_pcache_page *pPage; + assert( pPager!=0 ); + assert( pgno!=0 ); + assert( pPager->pPCache!=0 ); + pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); + assert( pPage==0 || pPager->hasHeldSharedLock ); + if( pPage==0 ) return 0; + return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); +} + +/* +** Release a page reference. +** +** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be +** used if we know that the page being released is not the last page. +** The btree layer always holds page1 open until the end, so these first +** to routines can be used to release any page other than BtShared.pPage1. +** +** Use sqlite3PagerUnrefPageOne() to release page1. This latter routine +** checks the total number of outstanding pages and if the number of +** pages reaches zero it drops the database lock. +*/ +SQLITE_PRIVATE void sqlite3PagerUnrefNotNull(DbPage *pPg){ + TESTONLY( Pager *pPager = pPg->pPager; ) + assert( pPg!=0 ); + if( pPg->flags & PGHDR_MMAP ){ + assert( pPg->pgno!=1 ); /* Page1 is never memory mapped */ + pagerReleaseMapPage(pPg); + }else{ + sqlite3PcacheRelease(pPg); + } + /* Do not use this routine to release the last reference to page1 */ + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); +} +SQLITE_PRIVATE void sqlite3PagerUnref(DbPage *pPg){ + if( pPg ) sqlite3PagerUnrefNotNull(pPg); +} +SQLITE_PRIVATE void sqlite3PagerUnrefPageOne(DbPage *pPg){ + Pager *pPager; + assert( pPg!=0 ); + assert( pPg->pgno==1 ); + assert( (pPg->flags & PGHDR_MMAP)==0 ); /* Page1 is never memory mapped */ + pPager = pPg->pPager; + sqlite3PcacheRelease(pPg); + pagerUnlockIfUnused(pPager); +} + +/* +** This function is called at the start of every write transaction. +** There must already be a RESERVED or EXCLUSIVE lock on the database +** file when this routine is called. +** +** Open the journal file for pager pPager and write a journal header +** to the start of it. If there are active savepoints, open the sub-journal +** as well. This function is only used when the journal file is being +** opened to write a rollback log for a transaction. It is not used +** when opening a hot journal file to roll it back. +** +** If the journal file is already open (as it may be in exclusive mode), +** then this function just writes a journal header to the start of the +** already open file. +** +** Whether or not the journal file is opened by this function, the +** Pager.pInJournal bitvec structure is allocated. +** +** Return SQLITE_OK if everything is successful. Otherwise, return +** SQLITE_NOMEM if the attempt to allocate Pager.pInJournal fails, or +** an IO error code if opening or writing the journal file fails. +*/ +static int pager_open_journal(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + sqlite3_vfs * const pVfs = pPager->pVfs; /* Local cache of vfs pointer */ + + assert( pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + assert( pPager->pInJournal==0 ); + + /* If already in the error state, this function is a no-op. But on + ** the other hand, this routine is never called if we are already in + ** an error state. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; + + if( !pagerUseWal(pPager) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + pPager->pInJournal = sqlite3BitvecCreate(pPager->dbSize); + if( pPager->pInJournal==0 ){ + return SQLITE_NOMEM_BKPT; + } + + /* Open the journal file if it is not already open. */ + if( !isOpen(pPager->jfd) ){ + if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ + sqlite3MemJournalOpen(pPager->jfd); + }else{ + int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + int nSpill; + + if( pPager->tempFile ){ + flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + nSpill = sqlite3Config.nStmtSpill; + }else{ + flags |= SQLITE_OPEN_MAIN_JOURNAL; + nSpill = jrnlBufferSize(pPager); + } + + /* Verify that the database still has the same name as it did when + ** it was originally opened. */ + rc = databaseIsUnmoved(pPager); + if( rc==SQLITE_OK ){ + rc = sqlite3JournalOpen ( + pVfs, pPager->zJournal, pPager->jfd, flags, nSpill + ); + } + } + assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); + } + + + /* Write the first journal header to the journal file and open + ** the sub-journal if necessary. + */ + if( rc==SQLITE_OK ){ + /* TODO: Check if all of these are really required. */ + pPager->nRec = 0; + pPager->journalOff = 0; + pPager->setSuper = 0; + pPager->journalHdr = 0; + rc = writeJournalHdr(pPager); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3BitvecDestroy(pPager->pInJournal); + pPager->pInJournal = 0; + }else{ + assert( pPager->eState==PAGER_WRITER_LOCKED ); + pPager->eState = PAGER_WRITER_CACHEMOD; + } + + return rc; +} + +/* +** Begin a write-transaction on the specified pager object. If a +** write-transaction has already been opened, this function is a no-op. +** +** If the exFlag argument is false, then acquire at least a RESERVED +** lock on the database file. If exFlag is true, then acquire at least +** an EXCLUSIVE lock. If such a lock is already held, no locking +** functions need be called. +** +** If the subjInMemory argument is non-zero, then any sub-journal opened +** within this transaction will be opened as an in-memory file. This +** has no effect if the sub-journal is already opened (as it may be when +** running in exclusive mode) or if the transaction does not require a +** sub-journal. If the subjInMemory argument is zero, then any required +** sub-journal is implemented in-memory if pPager is an in-memory database, +** or using a temporary file otherwise. +*/ +SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ + int rc = SQLITE_OK; + + if( pPager->errCode ) return pPager->errCode; + assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; + + if( pPager->eState==PAGER_READER ){ + assert( pPager->pInJournal==0 ); + + if( pagerUseWal(pPager) ){ + /* If the pager is configured to use locking_mode=exclusive, and an + ** exclusive lock on the database is not already held, obtain it now. + */ + if( pPager->exclusiveMode && sqlite3WalExclusiveMode(pPager->pWal, -1) ){ + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + return rc; + } + (void)sqlite3WalExclusiveMode(pPager->pWal, 1); + } + + /* Grab the write lock on the log file. If successful, upgrade to + ** PAGER_RESERVED state. Otherwise, return an error code to the caller. + ** The busy-handler is not invoked if another connection already + ** holds the write-lock. If possible, the upper layer will call it. + */ + rc = sqlite3WalBeginWriteTransaction(pPager->pWal); + }else{ + /* Obtain a RESERVED lock on the database file. If the exFlag parameter + ** is true, then immediately upgrade this to an EXCLUSIVE lock. The + ** busy-handler callback can be used when upgrading to the EXCLUSIVE + ** lock, but not when obtaining the RESERVED lock. + */ + rc = pagerLockDb(pPager, RESERVED_LOCK); + if( rc==SQLITE_OK && exFlag ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + } + + if( rc==SQLITE_OK ){ + /* Change to WRITER_LOCKED state. + ** + ** WAL mode sets Pager.eState to PAGER_WRITER_LOCKED or CACHEMOD + ** when it has an open transaction, but never to DBMOD or FINISHED. + ** This is because in those states the code to roll back savepoint + ** transactions may copy data from the sub-journal into the database + ** file as well as into the page cache. Which would be incorrect in + ** WAL mode. + */ + pPager->eState = PAGER_WRITER_LOCKED; + pPager->dbHintSize = pPager->dbSize; + pPager->dbFileSize = pPager->dbSize; + pPager->dbOrigSize = pPager->dbSize; + pPager->journalOff = 0; + } + + assert( rc==SQLITE_OK || pPager->eState==PAGER_READER ); + assert( rc!=SQLITE_OK || pPager->eState==PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + } + + PAGERTRACE(("TRANSACTION %d\n", PAGERID(pPager))); + return rc; +} + +/* +** Write page pPg onto the end of the rollback journal. +*/ +static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + int rc; + u32 cksum; + char *pData2; + i64 iOff = pPager->journalOff; + + /* We should never write to the journal file the page that + ** contains the database locks. The following assert verifies + ** that we do not. */ + assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); + + assert( pPager->journalHdr<=pPager->journalOff ); + pData2 = pPg->pData; + cksum = pager_cksum(pPager, (u8*)pData2); + + /* Even if an IO or diskfull error occurs while journalling the + ** page in the block above, set the need-sync flag for the page. + ** Otherwise, when the transaction is rolled back, the logic in + ** playback_one_page() will think that the page needs to be restored + ** in the database file. And if an IO error occurs while doing so, + ** then corruption may follow. + */ + pPg->flags |= PGHDR_NEED_SYNC; + + rc = write32bits(pPager->jfd, iOff, pPg->pgno); + if( rc!=SQLITE_OK ) return rc; + rc = sqlite3OsWrite(pPager->jfd, pData2, pPager->pageSize, iOff+4); + if( rc!=SQLITE_OK ) return rc; + rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); + if( rc!=SQLITE_OK ) return rc; + + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + pPager->journalOff, pPager->pageSize)); + PAGER_INCR(sqlite3_pager_writej_count); + PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); + + pPager->journalOff += 8 + pPager->pageSize; + pPager->nRec++; + assert( pPager->pInJournal!=0 ); + rc = sqlite3BitvecSet(pPager->pInJournal, pPg->pgno); + testcase( rc==SQLITE_NOMEM ); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + rc |= addToSavepointBitvecs(pPager, pPg->pgno); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + return rc; +} + +/* +** Mark a single data page as writeable. The page is written into the +** main journal or sub-journal as required. If the page is written into +** one of the journals, the corresponding bit is set in the +** Pager.pInJournal bitvec and the PagerSavepoint.pInSavepoint bitvecs +** of any open savepoints as appropriate. +*/ +static int pager_write(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + int rc = SQLITE_OK; + + /* This routine is not called unless a write-transaction has already + ** been started. The journal file may or may not be open at this point. + ** It is never called in the ERROR state. + */ + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + assert( pPager->errCode==0 ); + assert( pPager->readOnly==0 ); + CHECK_PAGE(pPg); + + /* The journal file needs to be opened. Higher level routines have already + ** obtained the necessary locks to begin the write-transaction, but the + ** rollback journal might not yet be open. Open it now if this is the case. + ** + ** This is done before calling sqlite3PcacheMakeDirty() on the page. + ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then + ** an error might occur and the pager would end up in WRITER_LOCKED state + ** with pages marked as dirty in the cache. + */ + if( pPager->eState==PAGER_WRITER_LOCKED ){ + rc = pager_open_journal(pPager); + if( rc!=SQLITE_OK ) return rc; + } + assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); + assert( assert_pager_state(pPager) ); + + /* Mark the page that is about to be modified as dirty. */ + sqlite3PcacheMakeDirty(pPg); + + /* If a rollback journal is in use, them make sure the page that is about + ** to change is in the rollback journal, or if the page is a new page off + ** then end of the file, make sure it is marked as PGHDR_NEED_SYNC. + */ + assert( (pPager->pInJournal!=0) == isOpen(pPager->jfd) ); + if( pPager->pInJournal!=0 + && sqlite3BitvecTestNotNull(pPager->pInJournal, pPg->pgno)==0 + ){ + assert( pagerUseWal(pPager)==0 ); + if( pPg->pgno<=pPager->dbOrigSize ){ + rc = pagerAddPageToRollbackJournal(pPg); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + if( pPager->eState!=PAGER_WRITER_DBMOD ){ + pPg->flags |= PGHDR_NEED_SYNC; + } + PAGERTRACE(("APPEND %d page %d needSync=%d\n", + PAGERID(pPager), pPg->pgno, + ((pPg->flags&PGHDR_NEED_SYNC)?1:0))); + } + } + + /* The PGHDR_DIRTY bit is set above when the page was added to the dirty-list + ** and before writing the page into the rollback journal. Wait until now, + ** after the page has been successfully journalled, before setting the + ** PGHDR_WRITEABLE bit that indicates that the page can be safely modified. + */ + pPg->flags |= PGHDR_WRITEABLE; + + /* If the statement journal is open and the page is not in it, + ** then write the page into the statement journal. + */ + if( pPager->nSavepoint>0 ){ + rc = subjournalPageIfRequired(pPg); + } + + /* Update the database size and return. */ + if( pPager->dbSizepgno ){ + pPager->dbSize = pPg->pgno; + } + return rc; +} + +/* +** This is a variant of sqlite3PagerWrite() that runs when the sector size +** is larger than the page size. SQLite makes the (reasonable) assumption that +** all bytes of a sector are written together by hardware. Hence, all bytes of +** a sector need to be journalled in case of a power loss in the middle of +** a write. +** +** Usually, the sector size is less than or equal to the page size, in which +** case pages can be individually written. This routine only runs in the +** exceptional case where the page size is smaller than the sector size. +*/ +static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ + int rc = SQLITE_OK; /* Return code */ + Pgno nPageCount; /* Total number of pages in database file */ + Pgno pg1; /* First page of the sector pPg is located on. */ + int nPage = 0; /* Number of pages starting at pg1 to journal */ + int ii; /* Loop counter */ + int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ + Pager *pPager = pPg->pPager; /* The pager that owns pPg */ + Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + + /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow + ** a journal header to be written between the pages journaled by + ** this function. + */ + assert( !MEMDB ); + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)==0 ); + pPager->doNotSpill |= SPILLFLAG_NOSYNC; + + /* This trick assumes that both the page-size and sector-size are + ** an integer power of 2. It sets variable pg1 to the identifier + ** of the first page of the sector pPg is located on. + */ + pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; + + nPageCount = pPager->dbSize; + if( pPg->pgno>nPageCount ){ + nPage = (pPg->pgno - pg1)+1; + }else if( (pg1+nPagePerSector-1)>nPageCount ){ + nPage = nPageCount+1-pg1; + }else{ + nPage = nPagePerSector; + } + assert(nPage>0); + assert(pg1<=pPg->pgno); + assert((pg1+nPage)>pPg->pgno); + + for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ + if( pg!=PAGER_MJ_PGNO(pPager) ){ + rc = sqlite3PagerGet(pPager, pg, &pPage, 0); + if( rc==SQLITE_OK ){ + rc = pager_write(pPage); + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + }else if( (pPage = sqlite3PagerLookup(pPager, pg))!=0 ){ + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + + /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages + ** starting at pg1, then it needs to be set for all of them. Because + ** writing to any of these nPage pages may damage the others, the + ** journal file must contain sync()ed copies of all of them + ** before any of them can be written out to the database file. + */ + if( rc==SQLITE_OK && needSync ){ + assert( !MEMDB ); + for(ii=0; iiflags |= PGHDR_NEED_SYNC; + sqlite3PagerUnrefNotNull(pPage); + } + } + } + + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)!=0 ); + pPager->doNotSpill &= ~SPILLFLAG_NOSYNC; + return rc; +} + +/* +** Mark a data page as writeable. This routine must be called before +** making changes to a page. The caller must check the return value +** of this function and be careful not to change any page data unless +** this routine returns SQLITE_OK. +** +** The difference between this function and pager_write() is that this +** function also deals with the special case where 2 or more pages +** fit on a single disk sector. In this case all co-resident pages +** must have been written to the journal file before returning. +** +** If an error occurs, SQLITE_NOMEM or an IO error code is returned +** as appropriate. Otherwise, SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3PagerWrite(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + assert( (pPg->flags & PGHDR_MMAP)==0 ); + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ + if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); + return SQLITE_OK; + }else if( pPager->errCode ){ + return pPager->errCode; + }else if( pPager->sectorSize > (u32)pPager->pageSize ){ + assert( pPager->tempFile==0 ); + return pagerWriteLargeSector(pPg); + }else{ + return pager_write(pPg); + } +} + +/* +** Return TRUE if the page given in the argument was previously passed +** to sqlite3PagerWrite(). In other words, return TRUE if it is ok +** to change the content of the page. +*/ +#ifndef NDEBUG +SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){ + return pPg->flags & PGHDR_WRITEABLE; +} +#endif + +/* +** A call to this routine tells the pager that it is not necessary to +** write the information on page pPg back to the disk, even though +** that page might be marked as dirty. This happens, for example, when +** the page has been added as a leaf of the freelist and so its +** content no longer matters. +** +** The overlying software layer calls this routine when all of the data +** on the given page is unused. The pager marks the page as clean so +** that it does not get written to disk. +** +** Tests show that this optimization can quadruple the speed of large +** DELETE operations. +** +** This optimization cannot be used with a temp-file, as the page may +** have been dirty at the start of the transaction. In that case, if +** memory pressure forces page pPg out of the cache, the data does need +** to be written out to disk so that it may be read back in if the +** current transaction is rolled back. +*/ +SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + if( !pPager->tempFile && (pPg->flags&PGHDR_DIRTY) && pPager->nSavepoint==0 ){ + PAGERTRACE(("DONT_WRITE page %d of %d\n", pPg->pgno, PAGERID(pPager))); + IOTRACE(("CLEAN %p %d\n", pPager, pPg->pgno)) + pPg->flags |= PGHDR_DONT_WRITE; + pPg->flags &= ~PGHDR_WRITEABLE; + testcase( pPg->flags & PGHDR_NEED_SYNC ); + pager_set_pagehash(pPg); + } +} + +/* +** This routine is called to increment the value of the database file +** change-counter, stored as a 4-byte big-endian integer starting at +** byte offset 24 of the pager file. The secondary change counter at +** 92 is also updated, as is the SQLite version number at offset 96. +** +** But this only happens if the pPager->changeCountDone flag is false. +** To avoid excess churning of page 1, the update only happens once. +** See also the pager_write_changecounter() routine that does an +** unconditional update of the change counters. +** +** If the isDirectMode flag is zero, then this is done by calling +** sqlite3PagerWrite() on page 1, then modifying the contents of the +** page data. In this case the file will be updated when the current +** transaction is committed. +** +** The isDirectMode flag may only be non-zero if the library was compiled +** with the SQLITE_ENABLE_ATOMIC_WRITE macro defined. In this case, +** if isDirect is non-zero, then the database file is updated directly +** by writing an updated version of page 1 using a call to the +** sqlite3OsWrite() function. +*/ +static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ + int rc = SQLITE_OK; + + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + + /* Declare and initialize constant integer 'isDirect'. If the + ** atomic-write optimization is enabled in this build, then isDirect + ** is initialized to the value passed as the isDirectMode parameter + ** to this function. Otherwise, it is always set to zero. + ** + ** The idea is that if the atomic-write optimization is not + ** enabled at compile time, the compiler can omit the tests of + ** 'isDirect' below, as well as the block enclosed in the + ** "if( isDirect )" condition. + */ +#ifndef SQLITE_ENABLE_ATOMIC_WRITE +# define DIRECT_MODE 0 + assert( isDirectMode==0 ); + UNUSED_PARAMETER(isDirectMode); +#else +# define DIRECT_MODE isDirectMode +#endif + + if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){ + PgHdr *pPgHdr; /* Reference to page 1 */ + + assert( !pPager->tempFile && isOpen(pPager->fd) ); + + /* Open page 1 of the file for writing. */ + rc = sqlite3PagerGet(pPager, 1, &pPgHdr, 0); + assert( pPgHdr==0 || rc==SQLITE_OK ); + + /* If page one was fetched successfully, and this function is not + ** operating in direct-mode, make page 1 writable. When not in + ** direct mode, page 1 is always held in cache and hence the PagerGet() + ** above is always successful - hence the ALWAYS on rc==SQLITE_OK. + */ + if( !DIRECT_MODE && ALWAYS(rc==SQLITE_OK) ){ + rc = sqlite3PagerWrite(pPgHdr); + } + + if( rc==SQLITE_OK ){ + /* Actually do the update of the change counter */ + pager_write_changecounter(pPgHdr); + + /* If running in direct mode, write the contents of page 1 to the file. */ + if( DIRECT_MODE ){ + const void *zBuf; + assert( pPager->dbFileSize>0 ); + zBuf = pPgHdr->pData; + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); + pPager->aStat[PAGER_STAT_WRITE]++; + } + if( rc==SQLITE_OK ){ + /* Update the pager's copy of the change-counter. Otherwise, the + ** next time a read transaction is opened the cache will be + ** flushed (as the change-counter values will not match). */ + const void *pCopy = (const void *)&((const char *)zBuf)[24]; + memcpy(&pPager->dbFileVers, pCopy, sizeof(pPager->dbFileVers)); + pPager->changeCountDone = 1; + } + }else{ + pPager->changeCountDone = 1; + } + } + + /* Release the page reference. */ + sqlite3PagerUnref(pPgHdr); + } + return rc; +} + +/* +** Sync the database file to disk. This is a no-op for in-memory databases +** or pages with the Pager.noSync flag set. +** +** If successful, or if called on a pager for which it is a no-op, this +** function returns SQLITE_OK. Otherwise, an IO error code is returned. +*/ +SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zSuper){ + int rc = SQLITE_OK; + void *pArg = (void*)zSuper; + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC, pArg); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc==SQLITE_OK && !pPager->noSync ){ + assert( !MEMDB ); + rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); + } + return rc; +} + +/* +** This function may only be called while a write-transaction is active in +** rollback. If the connection is in WAL mode, this call is a no-op. +** Otherwise, if the connection does not already have an EXCLUSIVE lock on +** the database file, an attempt is made to obtain one. +** +** If the EXCLUSIVE lock is already held or the attempt to obtain it is +** successful, or the connection is in WAL mode, SQLITE_OK is returned. +** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is +** returned. +*/ +SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ + int rc = pPager->errCode; + assert( assert_pager_state(pPager) ); + if( rc==SQLITE_OK ){ + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED + ); + assert( assert_pager_state(pPager) ); + if( 0==pagerUseWal(pPager) ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } + } + return rc; +} + +/* +** Sync the database file for the pager pPager. zSuper points to the name +** of a super-journal file that should be written into the individual +** journal file. zSuper may be NULL, which is interpreted as no +** super-journal (a single database transaction). +** +** This routine ensures that: +** +** * The database file change-counter is updated, +** * the journal is synced (unless the atomic-write optimization is used), +** * all dirty pages are written to the database file, +** * the database file is truncated (if required), and +** * the database file synced. +** +** The only thing that remains to commit the transaction is to finalize +** (delete, truncate or zero the first part of) the journal file (or +** delete the super-journal file if specified). +** +** Note that if zSuper==NULL, this does not overwrite a previous value +** passed to an sqlite3PagerCommitPhaseOne() call. +** +** If the final parameter - noSync - is true, then the database file itself +** is not synced. The caller must call sqlite3PagerSync() directly to +** sync the database file before calling CommitPhaseTwo() to delete the +** journal file in this case. +*/ +SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( + Pager *pPager, /* Pager object */ + const char *zSuper, /* If not NULL, the super-journal name */ + int noSync /* True to omit the xSync on the db file */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_ERROR + ); + assert( assert_pager_state(pPager) ); + + /* If a prior error occurred, report that error again. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; + + /* Provide the ability to easily simulate an I/O error during testing */ + if( sqlite3FaultSim(400) ) return SQLITE_IOERR; + + PAGERTRACE(("DATABASE SYNC: File=%s zSuper=%s nSize=%d\n", + pPager->zFilename, zSuper, pPager->dbSize)); + + /* If no database changes have been made, return early. */ + if( pPager->eStatetempFile ); + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( 0==pagerFlushOnCommit(pPager, 1) ){ + /* If this is an in-memory db, or no pages have been written to, or this + ** function has already been called, it is mostly a no-op. However, any + ** backup in progress needs to be restarted. */ + sqlite3BackupRestart(pPager->pBackup); + }else{ + PgHdr *pList; + if( pagerUseWal(pPager) ){ + PgHdr *pPageOne = 0; + pList = sqlite3PcacheDirtyList(pPager->pPCache); + if( pList==0 ){ + /* Must have at least one page for the WAL commit flag. + ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */ + rc = sqlite3PagerGet(pPager, 1, &pPageOne, 0); + pList = pPageOne; + pList->pDirty = 0; + } + assert( rc==SQLITE_OK ); + if( ALWAYS(pList) ){ + rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1); + } + sqlite3PagerUnref(pPageOne); + if( rc==SQLITE_OK ){ + sqlite3PcacheCleanAll(pPager->pPCache); + } + }else{ + /* The bBatch boolean is true if the batch-atomic-write commit method + ** should be used. No rollback journal is created if batch-atomic-write + ** is enabled. + */ +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + sqlite3_file *fd = pPager->fd; + int bBatch = zSuper==0 /* An SQLITE_IOCAP_BATCH_ATOMIC commit */ + && (sqlite3OsDeviceCharacteristics(fd) & SQLITE_IOCAP_BATCH_ATOMIC) + && !pPager->noSync + && sqlite3JournalIsInMemory(pPager->jfd); +#else +# define bBatch 0 +#endif + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + /* The following block updates the change-counter. Exactly how it + ** does this depends on whether or not the atomic-update optimization + ** was enabled at compile time, and if this transaction meets the + ** runtime criteria to use the operation: + ** + ** * The file-system supports the atomic-write property for + ** blocks of size page-size, and + ** * This commit is not part of a multi-file transaction, and + ** * Exactly one page has been modified and store in the journal file. + ** + ** If the optimization was not enabled at compile time, then the + ** pager_incr_changecounter() function is called to update the change + ** counter in 'indirect-mode'. If the optimization is compiled in but + ** is not applicable to this transaction, call sqlite3JournalCreate() + ** to make sure the journal file has actually been created, then call + ** pager_incr_changecounter() to update the change-counter in indirect + ** mode. + ** + ** Otherwise, if the optimization is both enabled and applicable, + ** then call pager_incr_changecounter() to update the change-counter + ** in 'direct' mode. In this case the journal file will never be + ** created for this transaction. + */ + if( bBatch==0 ){ + PgHdr *pPg; + assert( isOpen(pPager->jfd) + || pPager->journalMode==PAGER_JOURNALMODE_OFF + || pPager->journalMode==PAGER_JOURNALMODE_WAL + ); + if( !zSuper && isOpen(pPager->jfd) + && pPager->journalOff==jrnlBufferSize(pPager) + && pPager->dbSize>=pPager->dbOrigSize + && (!(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) + ){ + /* Update the db file change counter via the direct-write method. The + ** following call will modify the in-memory representation of page 1 + ** to include the updated change counter and then write page 1 + ** directly to the database file. Because of the atomic-write + ** property of the host file-system, this is safe. + */ + rc = pager_incr_changecounter(pPager, 1); + }else{ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc==SQLITE_OK ){ + rc = pager_incr_changecounter(pPager, 0); + } + } + } +#else /* SQLITE_ENABLE_ATOMIC_WRITE */ +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( zSuper ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + assert( bBatch==0 ); + } +#endif + rc = pager_incr_changecounter(pPager, 0); +#endif /* !SQLITE_ENABLE_ATOMIC_WRITE */ + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + /* Write the super-journal name into the journal file. If a + ** super-journal file name has already been written to the journal file, + ** or if zSuper is NULL (no super-journal), then this call is a no-op. + */ + rc = writeSuperJournal(pPager, zSuper); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + /* Sync the journal file and write all dirty pages to the database. + ** If the atomic-update optimization is being used, this sync will not + ** create the journal file or perform any real IO. + ** + ** Because the change-counter page was just modified, unless the + ** atomic-update optimization is used it is almost certain that the + ** journal requires a sync here. However, in locking_mode=exclusive + ** on a system under memory pressure it is just possible that this is + ** not the case. In this case it is likely enough that the redundant + ** xSync() call will be changed to a no-op by the OS anyhow. + */ + rc = syncJournal(pPager, 0); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + pList = sqlite3PcacheDirtyList(pPager->pPCache); +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( bBatch ){ + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0); + if( rc==SQLITE_OK ){ + rc = pager_write_pagelist(pPager, pList); + if( rc==SQLITE_OK ){ + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0); + } + if( rc!=SQLITE_OK ){ + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0); + } + } + + if( (rc&0xFF)==SQLITE_IOERR && rc!=SQLITE_IOERR_NOMEM ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ){ + sqlite3OsClose(pPager->jfd); + goto commit_phase_one_exit; + } + bBatch = 0; + }else{ + sqlite3OsClose(pPager->jfd); + } + } +#endif /* SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + + if( bBatch==0 ){ + rc = pager_write_pagelist(pPager, pList); + } + if( rc!=SQLITE_OK ){ + assert( rc!=SQLITE_IOERR_BLOCKED ); + goto commit_phase_one_exit; + } + sqlite3PcacheCleanAll(pPager->pPCache); + + /* If the file on disk is smaller than the database image, use + ** pager_truncate to grow the file here. This can happen if the database + ** image was extended as part of the current transaction and then the + ** last page in the db image moved to the free-list. In this case the + ** last page is never written out to disk, leaving the database file + ** undersized. Fix this now if it is the case. */ + if( pPager->dbSize>pPager->dbFileSize ){ + Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); + assert( pPager->eState==PAGER_WRITER_DBMOD ); + rc = pager_truncate(pPager, nNew); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + } + + /* Finally, sync the database file. */ + if( !noSync ){ + rc = sqlite3PagerSync(pPager, zSuper); + } + IOTRACE(("DBSYNC %p\n", pPager)) + } + } + +commit_phase_one_exit: + if( rc==SQLITE_OK && !pagerUseWal(pPager) ){ + pPager->eState = PAGER_WRITER_FINISHED; + } + return rc; +} + + +/* +** When this function is called, the database file has been completely +** updated to reflect the changes made by the current transaction and +** synced to disk. The journal file still exists in the file-system +** though, and if a failure occurs at this point it will eventually +** be used as a hot-journal and the current transaction rolled back. +** +** This function finalizes the journal file, either by deleting, +** truncating or partially zeroing it, so that it cannot be used +** for hot-journal rollback. Once this is done the transaction is +** irrevocably committed. +** +** If an error occurs, an IO error code is returned and the pager +** moves into the error state. Otherwise, SQLITE_OK is returned. +*/ +SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + + /* This routine should not be called if a prior error has occurred. + ** But if (due to a coding error elsewhere in the system) it does get + ** called, just return the same error code without doing anything. */ + if( NEVER(pPager->errCode) ) return pPager->errCode; + pPager->iDataVersion++; + + assert( pPager->eState==PAGER_WRITER_LOCKED + || pPager->eState==PAGER_WRITER_FINISHED + || (pagerUseWal(pPager) && pPager->eState==PAGER_WRITER_CACHEMOD) + ); + assert( assert_pager_state(pPager) ); + + /* An optimization. If the database was not actually modified during + ** this transaction, the pager is running in exclusive-mode and is + ** using persistent journals, then this function is a no-op. + ** + ** The start of the journal file currently contains a single journal + ** header with the nRec field set to 0. If such a journal is used as + ** a hot-journal during hot-journal rollback, 0 changes will be made + ** to the database file. So there is no need to zero the journal + ** header. Since the pager is in exclusive mode, there is no need + ** to drop any locks either. + */ + if( pPager->eState==PAGER_WRITER_LOCKED + && pPager->exclusiveMode + && pPager->journalMode==PAGER_JOURNALMODE_PERSIST + ){ + assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) || !pPager->journalOff ); + pPager->eState = PAGER_READER; + return SQLITE_OK; + } + + PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); + rc = pager_end_transaction(pPager, pPager->setSuper, 1); + return pager_error(pPager, rc); +} + +/* +** If a write transaction is open, then all changes made within the +** transaction are reverted and the current write-transaction is closed. +** The pager falls back to PAGER_READER state if successful, or PAGER_ERROR +** state if an error occurs. +** +** If the pager is already in PAGER_ERROR state when this function is called, +** it returns Pager.errCode immediately. No work is performed in this case. +** +** Otherwise, in rollback mode, this function performs two functions: +** +** 1) It rolls back the journal file, restoring all database file and +** in-memory cache pages to the state they were in when the transaction +** was opened, and +** +** 2) It finalizes the journal file, so that it is not used for hot +** rollback at any point in the future. +** +** Finalization of the journal file (task 2) is only performed if the +** rollback is successful. +** +** In WAL mode, all cache-entries containing data modified within the +** current transaction are either expelled from the cache or reverted to +** their pre-transaction state by re-reading data from the database or +** WAL files. The WAL transaction is then closed. +*/ +SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){ + int rc = SQLITE_OK; /* Return code */ + PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); + + /* PagerRollback() is a no-op if called in READER or OPEN state. If + ** the pager is already in the ERROR state, the rollback is not + ** attempted here. Instead, the error code is returned to the caller. + */ + assert( assert_pager_state(pPager) ); + if( pPager->eState==PAGER_ERROR ) return pPager->errCode; + if( pPager->eState<=PAGER_READER ) return SQLITE_OK; + + if( pagerUseWal(pPager) ){ + int rc2; + rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); + rc2 = pager_end_transaction(pPager, pPager->setSuper, 0); + if( rc==SQLITE_OK ) rc = rc2; + }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ + int eState = pPager->eState; + rc = pager_end_transaction(pPager, 0, 0); + if( !MEMDB && eState>PAGER_WRITER_LOCKED ){ + /* This can happen using journal_mode=off. Move the pager to the error + ** state to indicate that the contents of the cache may not be trusted. + ** Any active readers will get SQLITE_ABORT. + */ + pPager->errCode = SQLITE_ABORT; + pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); + return rc; + } + }else{ + rc = pager_playback(pPager, 0); + } + + assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); + assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT + || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR + || rc==SQLITE_CANTOPEN + ); + + /* If an error occurs during a ROLLBACK, we can no longer trust the pager + ** cache. So call pager_error() on the way out to make any error persistent. + */ + return pager_error(pPager, rc); +} + +/* +** Return TRUE if the database file is opened read-only. Return FALSE +** if the database is (in theory) writable. +*/ +SQLITE_PRIVATE u8 sqlite3PagerIsreadonly(Pager *pPager){ + return pPager->readOnly; +} + +#ifdef SQLITE_DEBUG +/* +** Return the sum of the reference counts for all pages held by pPager. +*/ +SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ + return sqlite3PcacheRefCount(pPager->pPCache); +} +#endif + +/* +** Return the approximate number of bytes of memory currently +** used by the pager and its associated cache. +*/ +SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager *pPager){ + int perPageSize = pPager->pageSize + pPager->nExtra + + (int)(sizeof(PgHdr) + 5*sizeof(void*)); + return perPageSize*sqlite3PcachePagecount(pPager->pPCache) + + sqlite3MallocSize(pPager) + + pPager->pageSize; +} + +/* +** Return the number of references to the specified page. +*/ +SQLITE_PRIVATE int sqlite3PagerPageRefcount(DbPage *pPage){ + return sqlite3PcachePageRefcount(pPage); +} + +#ifdef SQLITE_TEST +/* +** This routine is used for testing and analysis only. +*/ +SQLITE_PRIVATE int *sqlite3PagerStats(Pager *pPager){ + static int a[11]; + a[0] = sqlite3PcacheRefCount(pPager->pPCache); + a[1] = sqlite3PcachePagecount(pPager->pPCache); + a[2] = sqlite3PcacheGetCachesize(pPager->pPCache); + a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; + a[4] = pPager->eState; + a[5] = pPager->errCode; + a[6] = pPager->aStat[PAGER_STAT_HIT]; + a[7] = pPager->aStat[PAGER_STAT_MISS]; + a[8] = 0; /* Used to be pPager->nOvfl */ + a[9] = pPager->nRead; + a[10] = pPager->aStat[PAGER_STAT_WRITE]; + return a; +} +#endif + +/* +** Parameter eStat must be one of SQLITE_DBSTATUS_CACHE_HIT, _MISS, _WRITE, +** or _WRITE+1. The SQLITE_DBSTATUS_CACHE_WRITE+1 case is a translation +** of SQLITE_DBSTATUS_CACHE_SPILL. The _SPILL case is not contiguous because +** it was added later. +** +** Before returning, *pnVal is incremented by the +** current cache hit or miss count, according to the value of eStat. If the +** reset parameter is non-zero, the cache hit or miss count is zeroed before +** returning. +*/ +SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ + + assert( eStat==SQLITE_DBSTATUS_CACHE_HIT + || eStat==SQLITE_DBSTATUS_CACHE_MISS + || eStat==SQLITE_DBSTATUS_CACHE_WRITE + || eStat==SQLITE_DBSTATUS_CACHE_WRITE+1 + ); + + assert( SQLITE_DBSTATUS_CACHE_HIT+1==SQLITE_DBSTATUS_CACHE_MISS ); + assert( SQLITE_DBSTATUS_CACHE_HIT+2==SQLITE_DBSTATUS_CACHE_WRITE ); + assert( PAGER_STAT_HIT==0 && PAGER_STAT_MISS==1 + && PAGER_STAT_WRITE==2 && PAGER_STAT_SPILL==3 ); + + eStat -= SQLITE_DBSTATUS_CACHE_HIT; + *pnVal += pPager->aStat[eStat]; + if( reset ){ + pPager->aStat[eStat] = 0; + } +} + +/* +** Return true if this is an in-memory or temp-file backed pager. +*/ +SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ + return pPager->tempFile; +} + +/* +** Check that there are at least nSavepoint savepoints open. If there are +** currently less than nSavepoints open, then open one or more savepoints +** to make up the difference. If the number of savepoints is already +** equal to nSavepoint, then this function is a no-op. +** +** If a memory allocation fails, SQLITE_NOMEM is returned. If an error +** occurs while opening the sub-journal file, then an IO error code is +** returned. Otherwise, SQLITE_OK. +*/ +static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ + int rc = SQLITE_OK; /* Return code */ + int nCurrent = pPager->nSavepoint; /* Current number of savepoints */ + int ii; /* Iterator variable */ + PagerSavepoint *aNew; /* New Pager.aSavepoint array */ + + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + assert( nSavepoint>nCurrent && pPager->useJournal ); + + /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM + ** if the allocation fails. Otherwise, zero the new portion in case a + ** malloc failure occurs while populating it in the for(...) loop below. + */ + aNew = (PagerSavepoint *)sqlite3Realloc( + pPager->aSavepoint, sizeof(PagerSavepoint)*nSavepoint + ); + if( !aNew ){ + return SQLITE_NOMEM_BKPT; + } + memset(&aNew[nCurrent], 0, (nSavepoint-nCurrent) * sizeof(PagerSavepoint)); + pPager->aSavepoint = aNew; + + /* Populate the PagerSavepoint structures just allocated. */ + for(ii=nCurrent; iidbSize; + if( isOpen(pPager->jfd) && pPager->journalOff>0 ){ + aNew[ii].iOffset = pPager->journalOff; + }else{ + aNew[ii].iOffset = JOURNAL_HDR_SZ(pPager); + } + aNew[ii].iSubRec = pPager->nSubRec; + aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); + aNew[ii].bTruncateOnRelease = 1; + if( !aNew[ii].pInSavepoint ){ + return SQLITE_NOMEM_BKPT; + } + if( pagerUseWal(pPager) ){ + sqlite3WalSavepoint(pPager->pWal, aNew[ii].aWalData); + } + pPager->nSavepoint = ii+1; + } + assert( pPager->nSavepoint==nSavepoint ); + assertTruncateConstraint(pPager); + return rc; +} +SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ + assert( pPager->eState>=PAGER_WRITER_LOCKED ); + assert( assert_pager_state(pPager) ); + + if( nSavepoint>pPager->nSavepoint && pPager->useJournal ){ + return pagerOpenSavepoint(pPager, nSavepoint); + }else{ + return SQLITE_OK; + } +} + + +/* +** This function is called to rollback or release (commit) a savepoint. +** The savepoint to release or rollback need not be the most recently +** created savepoint. +** +** Parameter op is always either SAVEPOINT_ROLLBACK or SAVEPOINT_RELEASE. +** If it is SAVEPOINT_RELEASE, then release and destroy the savepoint with +** index iSavepoint. If it is SAVEPOINT_ROLLBACK, then rollback all changes +** that have occurred since the specified savepoint was created. +** +** The savepoint to rollback or release is identified by parameter +** iSavepoint. A value of 0 means to operate on the outermost savepoint +** (the first created). A value of (Pager.nSavepoint-1) means operate +** on the most recently created savepoint. If iSavepoint is greater than +** (Pager.nSavepoint-1), then this function is a no-op. +** +** If a negative value is passed to this function, then the current +** transaction is rolled back. This is different to calling +** sqlite3PagerRollback() because this function does not terminate +** the transaction or unlock the database, it just restores the +** contents of the database to its original state. +** +** In any case, all savepoints with an index greater than iSavepoint +** are destroyed. If this is a release operation (op==SAVEPOINT_RELEASE), +** then savepoint iSavepoint is also destroyed. +** +** This function may return SQLITE_NOMEM if a memory allocation fails, +** or an IO error code if an IO error occurs while rolling back a +** savepoint. If no errors occur, SQLITE_OK is returned. +*/ +SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ + int rc = pPager->errCode; + +#ifdef SQLITE_ENABLE_ZIPVFS + if( op==SAVEPOINT_RELEASE ) rc = SQLITE_OK; +#endif + + assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); + assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); + + if( rc==SQLITE_OK && iSavepointnSavepoint ){ + int ii; /* Iterator variable */ + int nNew; /* Number of remaining savepoints after this op. */ + + /* Figure out how many savepoints will still be active after this + ** operation. Store this value in nNew. Then free resources associated + ** with any savepoints that are destroyed by this operation. + */ + nNew = iSavepoint + (( op==SAVEPOINT_RELEASE ) ? 0 : 1); + for(ii=nNew; iinSavepoint; ii++){ + sqlite3BitvecDestroy(pPager->aSavepoint[ii].pInSavepoint); + } + pPager->nSavepoint = nNew; + + /* Truncate the sub-journal so that it only includes the parts + ** that are still in use. */ + if( op==SAVEPOINT_RELEASE ){ + PagerSavepoint *pRel = &pPager->aSavepoint[nNew]; + if( pRel->bTruncateOnRelease && isOpen(pPager->sjfd) ){ + /* Only truncate if it is an in-memory sub-journal. */ + if( sqlite3JournalIsInMemory(pPager->sjfd) ){ + i64 sz = (pPager->pageSize+4)*(i64)pRel->iSubRec; + rc = sqlite3OsTruncate(pPager->sjfd, sz); + assert( rc==SQLITE_OK ); + } + pPager->nSubRec = pRel->iSubRec; + } + } + /* Else this is a rollback operation, playback the specified savepoint. + ** If this is a temp-file, it is possible that the journal file has + ** not yet been opened. In this case there have been no changes to + ** the database file, so the playback operation can be skipped. + */ + else if( pagerUseWal(pPager) || isOpen(pPager->jfd) ){ + PagerSavepoint *pSavepoint = (nNew==0)?0:&pPager->aSavepoint[nNew-1]; + rc = pagerPlaybackSavepoint(pPager, pSavepoint); + assert(rc!=SQLITE_DONE); + } + +#ifdef SQLITE_ENABLE_ZIPVFS + /* If the cache has been modified but the savepoint cannot be rolled + ** back journal_mode=off, put the pager in the error state. This way, + ** if the VFS used by this pager includes ZipVFS, the entire transaction + ** can be rolled back at the ZipVFS level. */ + else if( + pPager->journalMode==PAGER_JOURNALMODE_OFF + && pPager->eState>=PAGER_WRITER_CACHEMOD + ){ + pPager->errCode = SQLITE_ABORT; + pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); + } +#endif + } + + return rc; +} + +/* +** Return the full pathname of the database file. +** +** Except, if the pager is in-memory only, then return an empty string if +** nullIfMemDb is true. This routine is called with nullIfMemDb==1 when +** used to report the filename to the user, for compatibility with legacy +** behavior. But when the Btree needs to know the filename for matching to +** shared cache, it uses nullIfMemDb==0 so that in-memory databases can +** participate in shared-cache. +** +** The return value to this routine is always safe to use with +** sqlite3_uri_parameter() and sqlite3_filename_database() and friends. +*/ +SQLITE_PRIVATE const char *sqlite3PagerFilename(const Pager *pPager, int nullIfMemDb){ + static const char zFake[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + return (nullIfMemDb && pPager->memDb) ? &zFake[4] : pPager->zFilename; +} + +/* +** Return the VFS structure for the pager. +*/ +SQLITE_PRIVATE sqlite3_vfs *sqlite3PagerVfs(Pager *pPager){ + return pPager->pVfs; +} + +/* +** Return the file handle for the database file associated +** with the pager. This might return NULL if the file has +** not yet been opened. +*/ +SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ + return pPager->fd; +} + +/* +** Return the file handle for the journal file (if it exists). +** This will be either the rollback journal or the WAL file. +*/ +SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ +#if SQLITE_OMIT_WAL + return pPager->jfd; +#else + return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; +#endif +} + +/* +** Return the full pathname of the journal file. +*/ +SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ + return pPager->zJournal; +} + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Move the page pPg to location pgno in the file. +** +** There must be no references to the page previously located at +** pgno (which we call pPgOld) though that page is allowed to be +** in cache. If the page previously located at pgno is not already +** in the rollback journal, it is not put there by by this routine. +** +** References to the page pPg remain valid. Updating any +** meta-data associated with pPg (i.e. data stored in the nExtra bytes +** allocated along with the page) is the responsibility of the caller. +** +** A transaction must be active when this routine is called. It used to be +** required that a statement transaction was not active, but this restriction +** has been removed (CREATE INDEX needs to move a page when a statement +** transaction is active). +** +** If the fourth argument, isCommit, is non-zero, then this page is being +** moved as part of a database reorganization just before the transaction +** is being committed. In this case, it is guaranteed that the database page +** pPg refers to will not be written to again within this transaction. +** +** This function may return SQLITE_NOMEM or an IO error code if an error +** occurs. Otherwise, it returns SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ + PgHdr *pPgOld; /* The page being overwritten. */ + Pgno needSyncPgno = 0; /* Old value of pPg->pgno, if sync is required */ + int rc; /* Return code */ + Pgno origPgno; /* The original page number */ + + assert( pPg->nRef>0 ); + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + ); + assert( assert_pager_state(pPager) ); + + /* In order to be able to rollback, an in-memory database must journal + ** the page we are moving from. + */ + assert( pPager->tempFile || !MEMDB ); + if( pPager->tempFile ){ + rc = sqlite3PagerWrite(pPg); + if( rc ) return rc; + } + + /* If the page being moved is dirty and has not been saved by the latest + ** savepoint, then save the current contents of the page into the + ** sub-journal now. This is required to handle the following scenario: + ** + ** BEGIN; + ** + ** SAVEPOINT one; + ** + ** ROLLBACK TO one; + ** + ** If page X were not written to the sub-journal here, it would not + ** be possible to restore its contents when the "ROLLBACK TO one" + ** statement were is processed. + ** + ** subjournalPage() may need to allocate space to store pPg->pgno into + ** one or more savepoint bitvecs. This is the reason this function + ** may return SQLITE_NOMEM. + */ + if( (pPg->flags & PGHDR_DIRTY)!=0 + && SQLITE_OK!=(rc = subjournalPageIfRequired(pPg)) + ){ + return rc; + } + + PAGERTRACE(("MOVE %d page %d (needSync=%d) moves to %d\n", + PAGERID(pPager), pPg->pgno, (pPg->flags&PGHDR_NEED_SYNC)?1:0, pgno)); + IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno)) + + /* If the journal needs to be sync()ed before page pPg->pgno can + ** be written to, store pPg->pgno in local variable needSyncPgno. + ** + ** If the isCommit flag is set, there is no need to remember that + ** the journal needs to be sync()ed before database page pPg->pgno + ** can be written to. The caller has already promised not to write to it. + */ + if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ + needSyncPgno = pPg->pgno; + assert( pPager->journalMode==PAGER_JOURNALMODE_OFF || + pageInJournal(pPager, pPg) || pPg->pgno>pPager->dbOrigSize ); + assert( pPg->flags&PGHDR_DIRTY ); + } + + /* If the cache contains a page with page-number pgno, remove it + ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for + ** page pgno before the 'move' operation, it needs to be retained + ** for the page moved there. + */ + pPg->flags &= ~PGHDR_NEED_SYNC; + pPgOld = sqlite3PagerLookup(pPager, pgno); + assert( !pPgOld || pPgOld->nRef==1 || CORRUPT_DB ); + if( pPgOld ){ + if( pPgOld->nRef>1 ){ + sqlite3PagerUnrefNotNull(pPgOld); + return SQLITE_CORRUPT_BKPT; + } + pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); + if( pPager->tempFile ){ + /* Do not discard pages from an in-memory database since we might + ** need to rollback later. Just move the page out of the way. */ + sqlite3PcacheMove(pPgOld, pPager->dbSize+1); + }else{ + sqlite3PcacheDrop(pPgOld); + } + } + + origPgno = pPg->pgno; + sqlite3PcacheMove(pPg, pgno); + sqlite3PcacheMakeDirty(pPg); + + /* For an in-memory database, make sure the original page continues + ** to exist, in case the transaction needs to roll back. Use pPgOld + ** as the original page since it has already been allocated. + */ + if( pPager->tempFile && pPgOld ){ + sqlite3PcacheMove(pPgOld, origPgno); + sqlite3PagerUnrefNotNull(pPgOld); + } + + if( needSyncPgno ){ + /* If needSyncPgno is non-zero, then the journal file needs to be + ** sync()ed before any data is written to database file page needSyncPgno. + ** Currently, no such page exists in the page-cache and the + ** "is journaled" bitvec flag has been set. This needs to be remedied by + ** loading the page into the pager-cache and setting the PGHDR_NEED_SYNC + ** flag. + ** + ** If the attempt to load the page into the page-cache fails, (due + ** to a malloc() or IO failure), clear the bit in the pInJournal[] + ** array. Otherwise, if the page is loaded and written again in + ** this transaction, it may be written to the database file before + ** it is synced into the journal file. This way, it may end up in + ** the journal file twice, but that is not a problem. + */ + PgHdr *pPgHdr; + rc = sqlite3PagerGet(pPager, needSyncPgno, &pPgHdr, 0); + if( rc!=SQLITE_OK ){ + if( needSyncPgno<=pPager->dbOrigSize ){ + assert( pPager->pTmpSpace!=0 ); + sqlite3BitvecClear(pPager->pInJournal, needSyncPgno, pPager->pTmpSpace); + } + return rc; + } + pPgHdr->flags |= PGHDR_NEED_SYNC; + sqlite3PcacheMakeDirty(pPgHdr); + sqlite3PagerUnrefNotNull(pPgHdr); + } + + return SQLITE_OK; +} +#endif + +/* +** The page handle passed as the first argument refers to a dirty page +** with a page number other than iNew. This function changes the page's +** page number to iNew and sets the value of the PgHdr.flags field to +** the value passed as the third parameter. +*/ +SQLITE_PRIVATE void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){ + assert( pPg->pgno!=iNew ); + pPg->flags = flags; + sqlite3PcacheMove(pPg, iNew); +} + +/* +** Return a pointer to the data for the specified page. +*/ +SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *pPg){ + assert( pPg->nRef>0 || pPg->pPager->memDb ); + return pPg->pData; +} + +/* +** Return a pointer to the Pager.nExtra bytes of "extra" space +** allocated along with the specified page. +*/ +SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *pPg){ + return pPg->pExtra; +} + +/* +** Get/set the locking-mode for this pager. Parameter eMode must be one +** of PAGER_LOCKINGMODE_QUERY, PAGER_LOCKINGMODE_NORMAL or +** PAGER_LOCKINGMODE_EXCLUSIVE. If the parameter is not _QUERY, then +** the locking-mode is set to the value specified. +** +** The returned value is either PAGER_LOCKINGMODE_NORMAL or +** PAGER_LOCKINGMODE_EXCLUSIVE, indicating the current (possibly updated) +** locking-mode. +*/ +SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *pPager, int eMode){ + assert( eMode==PAGER_LOCKINGMODE_QUERY + || eMode==PAGER_LOCKINGMODE_NORMAL + || eMode==PAGER_LOCKINGMODE_EXCLUSIVE ); + assert( PAGER_LOCKINGMODE_QUERY<0 ); + assert( PAGER_LOCKINGMODE_NORMAL>=0 && PAGER_LOCKINGMODE_EXCLUSIVE>=0 ); + assert( pPager->exclusiveMode || 0==sqlite3WalHeapMemory(pPager->pWal) ); + if( eMode>=0 && !pPager->tempFile && !sqlite3WalHeapMemory(pPager->pWal) ){ + pPager->exclusiveMode = (u8)eMode; + } + return (int)pPager->exclusiveMode; +} + +/* +** Set the journal-mode for this pager. Parameter eMode must be one of: +** +** PAGER_JOURNALMODE_DELETE +** PAGER_JOURNALMODE_TRUNCATE +** PAGER_JOURNALMODE_PERSIST +** PAGER_JOURNALMODE_OFF +** PAGER_JOURNALMODE_MEMORY +** PAGER_JOURNALMODE_WAL +** +** The journalmode is set to the value specified if the change is allowed. +** The change may be disallowed for the following reasons: +** +** * An in-memory database can only have its journal_mode set to _OFF +** or _MEMORY. +** +** * Temporary databases cannot have _WAL journalmode. +** +** The returned indicate the current (possibly updated) journal-mode. +*/ +SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ + u8 eOld = pPager->journalMode; /* Prior journalmode */ + + /* The eMode parameter is always valid */ + assert( eMode==PAGER_JOURNALMODE_DELETE + || eMode==PAGER_JOURNALMODE_TRUNCATE + || eMode==PAGER_JOURNALMODE_PERSIST + || eMode==PAGER_JOURNALMODE_OFF + || eMode==PAGER_JOURNALMODE_WAL + || eMode==PAGER_JOURNALMODE_MEMORY ); + + /* This routine is only called from the OP_JournalMode opcode, and + ** the logic there will never allow a temporary file to be changed + ** to WAL mode. + */ + assert( pPager->tempFile==0 || eMode!=PAGER_JOURNALMODE_WAL ); + + /* Do allow the journalmode of an in-memory database to be set to + ** anything other than MEMORY or OFF + */ + if( MEMDB ){ + assert( eOld==PAGER_JOURNALMODE_MEMORY || eOld==PAGER_JOURNALMODE_OFF ); + if( eMode!=PAGER_JOURNALMODE_MEMORY && eMode!=PAGER_JOURNALMODE_OFF ){ + eMode = eOld; + } + } + + if( eMode!=eOld ){ + + /* Change the journal mode. */ + assert( pPager->eState!=PAGER_ERROR ); + pPager->journalMode = (u8)eMode; + + /* When transistioning from TRUNCATE or PERSIST to any other journal + ** mode except WAL, unless the pager is in locking_mode=exclusive mode, + ** delete the journal file. + */ + assert( (PAGER_JOURNALMODE_TRUNCATE & 5)==1 ); + assert( (PAGER_JOURNALMODE_PERSIST & 5)==1 ); + assert( (PAGER_JOURNALMODE_DELETE & 5)==0 ); + assert( (PAGER_JOURNALMODE_MEMORY & 5)==4 ); + assert( (PAGER_JOURNALMODE_OFF & 5)==0 ); + assert( (PAGER_JOURNALMODE_WAL & 5)==5 ); + + assert( isOpen(pPager->fd) || pPager->exclusiveMode ); + if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){ + + /* In this case we would like to delete the journal file. If it is + ** not possible, then that is not a problem. Deleting the journal file + ** here is an optimization only. + ** + ** Before deleting the journal file, obtain a RESERVED lock on the + ** database file. This ensures that the journal file is not deleted + ** while it is in use by some other client. + */ + sqlite3OsClose(pPager->jfd); + if( pPager->eLock>=RESERVED_LOCK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + }else{ + int rc = SQLITE_OK; + int state = pPager->eState; + assert( state==PAGER_OPEN || state==PAGER_READER ); + if( state==PAGER_OPEN ){ + rc = sqlite3PagerSharedLock(pPager); + } + if( pPager->eState==PAGER_READER ){ + assert( rc==SQLITE_OK ); + rc = pagerLockDb(pPager, RESERVED_LOCK); + } + if( rc==SQLITE_OK ){ + sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); + } + if( rc==SQLITE_OK && state==PAGER_READER ){ + pagerUnlockDb(pPager, SHARED_LOCK); + }else if( state==PAGER_OPEN ){ + pager_unlock(pPager); + } + assert( state==pPager->eState ); + } + }else if( eMode==PAGER_JOURNALMODE_OFF ){ + sqlite3OsClose(pPager->jfd); + } + } + + /* Return the new journal mode */ + return (int)pPager->journalMode; +} + +/* +** Return the current journal mode. +*/ +SQLITE_PRIVATE int sqlite3PagerGetJournalMode(Pager *pPager){ + return (int)pPager->journalMode; +} + +/* +** Return TRUE if the pager is in a state where it is OK to change the +** journalmode. Journalmode changes can only happen when the database +** is unmodified. +*/ +SQLITE_PRIVATE int sqlite3PagerOkToChangeJournalMode(Pager *pPager){ + assert( assert_pager_state(pPager) ); + if( pPager->eState>=PAGER_WRITER_CACHEMOD ) return 0; + if( NEVER(isOpen(pPager->jfd) && pPager->journalOff>0) ) return 0; + return 1; +} + +/* +** Get/set the size-limit used for persistent journal files. +** +** Setting the size limit to -1 means no limit is enforced. +** An attempt to set a limit smaller than -1 is a no-op. +*/ +SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *pPager, i64 iLimit){ + if( iLimit>=-1 ){ + pPager->journalSizeLimit = iLimit; + sqlite3WalLimit(pPager->pWal, iLimit); + } + return pPager->journalSizeLimit; +} + +/* +** Return a pointer to the pPager->pBackup variable. The backup module +** in backup.c maintains the content of this variable. This module +** uses it opaquely as an argument to sqlite3BackupRestart() and +** sqlite3BackupUpdate() only. +*/ +SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ + return &pPager->pBackup; +} + +#ifndef SQLITE_OMIT_VACUUM +/* +** Unless this is an in-memory or temporary database, clear the pager cache. +*/ +SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *pPager){ + assert( MEMDB==0 || pPager->tempFile ); + if( pPager->tempFile==0 ) pager_reset(pPager); +} +#endif + + +#ifndef SQLITE_OMIT_WAL +/* +** This function is called when the user invokes "PRAGMA wal_checkpoint", +** "PRAGMA wal_blocking_checkpoint" or calls the sqlite3_wal_checkpoint() +** or wal_blocking_checkpoint() API functions. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +*/ +SQLITE_PRIVATE int sqlite3PagerCheckpoint( + Pager *pPager, /* Checkpoint on this pager */ + sqlite3 *db, /* Db handle used to check for interrupts */ + int eMode, /* Type of checkpoint */ + int *pnLog, /* OUT: Final number of frames in log */ + int *pnCkpt /* OUT: Final number of checkpointed frames */ +){ + int rc = SQLITE_OK; + if( pPager->pWal ){ + rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, + (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), + pPager->pBusyHandlerArg, + pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, + pnLog, pnCkpt + ); + } + return rc; +} + +SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager){ + return sqlite3WalCallback(pPager->pWal); +} + +/* +** Return true if the underlying VFS for the given pager supports the +** primitives necessary for write-ahead logging. +*/ +SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ + const sqlite3_io_methods *pMethods = pPager->fd->pMethods; + if( pPager->noLock ) return 0; + return pPager->exclusiveMode || (pMethods->iVersion>=2 && pMethods->xShmMap); +} + +/* +** Attempt to take an exclusive lock on the database file. If a PENDING lock +** is obtained instead, immediately release it. +*/ +static int pagerExclusiveLock(Pager *pPager){ + int rc; /* Return code */ + + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); + if( rc!=SQLITE_OK ){ + /* If the attempt to grab the exclusive lock failed, release the + ** pending lock that may have been obtained instead. */ + pagerUnlockDb(pPager, SHARED_LOCK); + } + + return rc; +} + +/* +** Call sqlite3WalOpen() to open the WAL handle. If the pager is in +** exclusive-locking mode when this function is called, take an EXCLUSIVE +** lock on the database file and use heap-memory to store the wal-index +** in. Otherwise, use the normal shared-memory. +*/ +static int pagerOpenWal(Pager *pPager){ + int rc = SQLITE_OK; + + assert( pPager->pWal==0 && pPager->tempFile==0 ); + assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + + /* If the pager is already in exclusive-mode, the WAL module will use + ** heap-memory for the wal-index instead of the VFS shared-memory + ** implementation. Take the exclusive lock now, before opening the WAL + ** file, to make sure this is safe. + */ + if( pPager->exclusiveMode ){ + rc = pagerExclusiveLock(pPager); + } + + /* Open the connection to the log file. If this operation fails, + ** (e.g. due to malloc() failure), return an error code. + */ + if( rc==SQLITE_OK ){ + rc = sqlite3WalOpen(pPager->pVfs, + pPager->fd, pPager->zWal, pPager->exclusiveMode, + pPager->journalSizeLimit, &pPager->pWal + ); + } + pagerFixMaplimit(pPager); + + return rc; +} + + +/* +** The caller must be holding a SHARED lock on the database file to call +** this function. +** +** If the pager passed as the first argument is open on a real database +** file (not a temp file or an in-memory database), and the WAL file +** is not already open, make an attempt to open it now. If successful, +** return SQLITE_OK. If an error occurs or the VFS used by the pager does +** not support the xShmXXX() methods, return an error code. *pbOpen is +** not modified in either case. +** +** If the pager is open on a temp-file (or in-memory database), or if +** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK +** without doing anything. +*/ +SQLITE_PRIVATE int sqlite3PagerOpenWal( + Pager *pPager, /* Pager object */ + int *pbOpen /* OUT: Set to true if call is a no-op */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( assert_pager_state(pPager) ); + assert( pPager->eState==PAGER_OPEN || pbOpen ); + assert( pPager->eState==PAGER_READER || !pbOpen ); + assert( pbOpen==0 || *pbOpen==0 ); + assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); + + if( !pPager->tempFile && !pPager->pWal ){ + if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; + + /* Close any rollback journal previously open */ + sqlite3OsClose(pPager->jfd); + + rc = pagerOpenWal(pPager); + if( rc==SQLITE_OK ){ + pPager->journalMode = PAGER_JOURNALMODE_WAL; + pPager->eState = PAGER_OPEN; + } + }else{ + *pbOpen = 1; + } + + return rc; +} + +/* +** This function is called to close the connection to the log file prior +** to switching from WAL to rollback mode. +** +** Before closing the log file, this function attempts to take an +** EXCLUSIVE lock on the database file. If this cannot be obtained, an +** error (SQLITE_BUSY) is returned and the log connection is not closed. +** If successful, the EXCLUSIVE lock is not released before returning. +*/ +SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ + int rc = SQLITE_OK; + + assert( pPager->journalMode==PAGER_JOURNALMODE_WAL ); + + /* If the log file is not already open, but does exist in the file-system, + ** it may need to be checkpointed before the connection can switch to + ** rollback mode. Open it now so this can happen. + */ + if( !pPager->pWal ){ + int logexists = 0; + rc = pagerLockDb(pPager, SHARED_LOCK); + if( rc==SQLITE_OK ){ + rc = sqlite3OsAccess( + pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &logexists + ); + } + if( rc==SQLITE_OK && logexists ){ + rc = pagerOpenWal(pPager); + } + } + + /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on + ** the database file, the log and log-summary files will be deleted. + */ + if( rc==SQLITE_OK && pPager->pWal ){ + rc = pagerExclusiveLock(pPager); + if( rc==SQLITE_OK ){ + rc = sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, + pPager->pageSize, (u8*)pPager->pTmpSpace); + pPager->pWal = 0; + pagerFixMaplimit(pPager); + if( rc && !pPager->exclusiveMode ) pagerUnlockDb(pPager, SHARED_LOCK); + } + } + return rc; +} + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +/* +** If pager pPager is a wal-mode database not in exclusive locking mode, +** invoke the sqlite3WalWriteLock() function on the associated Wal object +** with the same db and bLock parameters as were passed to this function. +** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. +*/ +SQLITE_PRIVATE int sqlite3PagerWalWriteLock(Pager *pPager, int bLock){ + int rc = SQLITE_OK; + if( pagerUseWal(pPager) && pPager->exclusiveMode==0 ){ + rc = sqlite3WalWriteLock(pPager->pWal, bLock); + } + return rc; +} + +/* +** Set the database handle used by the wal layer to determine if +** blocking locks are required. +*/ +SQLITE_PRIVATE void sqlite3PagerWalDb(Pager *pPager, sqlite3 *db){ + if( pagerUseWal(pPager) ){ + sqlite3WalDb(pPager->pWal, db); + } +} +#endif + +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** If this is a WAL database, obtain a snapshot handle for the snapshot +** currently open. Otherwise, return an error. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot){ + int rc = SQLITE_ERROR; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotGet(pPager->pWal, ppSnapshot); + } + return rc; +} + +/* +** If this is a WAL database, store a pointer to pSnapshot. Next time a +** read transaction is opened, attempt to read from the snapshot it +** identifies. If this is not a WAL database, return an error. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotOpen( + Pager *pPager, + sqlite3_snapshot *pSnapshot +){ + int rc = SQLITE_OK; + if( pPager->pWal ){ + sqlite3WalSnapshotOpen(pPager->pWal, pSnapshot); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** If this is a WAL database, call sqlite3WalSnapshotRecover(). If this +** is not a WAL database, return an error. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotRecover(Pager *pPager){ + int rc; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotRecover(pPager->pWal); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** The caller currently has a read transaction open on the database. +** If this is not a WAL database, SQLITE_ERROR is returned. Otherwise, +** this function takes a SHARED lock on the CHECKPOINTER slot and then +** checks if the snapshot passed as the second argument is still +** available. If so, SQLITE_OK is returned. +** +** If the snapshot is not available, SQLITE_ERROR is returned. Or, if +** the CHECKPOINTER lock cannot be obtained, SQLITE_BUSY. If any error +** occurs (any value other than SQLITE_OK is returned), the CHECKPOINTER +** lock is released before returning. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot){ + int rc; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotCheck(pPager->pWal, pSnapshot); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** Release a lock obtained by an earlier successful call to +** sqlite3PagerSnapshotCheck(). +*/ +SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager){ + assert( pPager->pWal ); + sqlite3WalSnapshotUnlock(pPager->pWal); +} + +#endif /* SQLITE_ENABLE_SNAPSHOT */ +#endif /* !SQLITE_OMIT_WAL */ + +#ifdef SQLITE_ENABLE_ZIPVFS +/* +** A read-lock must be held on the pager when this function is called. If +** the pager is in WAL mode and the WAL file currently contains one or more +** frames, return the size in bytes of the page images stored within the +** WAL frames. Otherwise, if this is not a WAL database or the WAL file +** is empty, return 0. +*/ +SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ + assert( pPager->eState>=PAGER_READER ); + return sqlite3WalFramesize(pPager->pWal); +} +#endif + +#endif /* SQLITE_OMIT_DISKIO */ + +/************** End of pager.c ***********************************************/ +/************** Begin file wal.c *********************************************/ +/* +** 2010 February 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of a write-ahead log (WAL) used in +** "journal_mode=WAL" mode. +** +** WRITE-AHEAD LOG (WAL) FILE FORMAT +** +** A WAL file consists of a header followed by zero or more "frames". +** Each frame records the revised content of a single page from the +** database file. All changes to the database are recorded by writing +** frames into the WAL. Transactions commit when a frame is written that +** contains a commit marker. A single WAL can and usually does record +** multiple transactions. Periodically, the content of the WAL is +** transferred back into the database file in an operation called a +** "checkpoint". +** +** A single WAL file can be used multiple times. In other words, the +** WAL can fill up with frames and then be checkpointed and then new +** frames can overwrite the old ones. A WAL always grows from beginning +** toward the end. Checksums and counters attached to each frame are +** used to determine which frames within the WAL are valid and which +** are leftovers from prior checkpoints. +** +** The WAL header is 32 bytes in size and consists of the following eight +** big-endian 32-bit unsigned integer values: +** +** 0: Magic number. 0x377f0682 or 0x377f0683 +** 4: File format version. Currently 3007000 +** 8: Database page size. Example: 1024 +** 12: Checkpoint sequence number +** 16: Salt-1, random integer incremented with each checkpoint +** 20: Salt-2, a different random integer changing with each ckpt +** 24: Checksum-1 (first part of checksum for first 24 bytes of header). +** 28: Checksum-2 (second part of checksum for first 24 bytes of header). +** +** Immediately following the wal-header are zero or more frames. Each +** frame consists of a 24-byte frame-header followed by a bytes +** of page data. The frame-header is six big-endian 32-bit unsigned +** integer values, as follows: +** +** 0: Page number. +** 4: For commit records, the size of the database image in pages +** after the commit. For all other records, zero. +** 8: Salt-1 (copied from the header) +** 12: Salt-2 (copied from the header) +** 16: Checksum-1. +** 20: Checksum-2. +** +** A frame is considered valid if and only if the following conditions are +** true: +** +** (1) The salt-1 and salt-2 values in the frame-header match +** salt values in the wal-header +** +** (2) The checksum values in the final 8 bytes of the frame-header +** exactly match the checksum computed consecutively on the +** WAL header and the first 8 bytes and the content of all frames +** up to and including the current frame. +** +** The checksum is computed using 32-bit big-endian integers if the +** magic number in the first 4 bytes of the WAL is 0x377f0683 and it +** is computed using little-endian if the magic number is 0x377f0682. +** The checksum values are always stored in the frame header in a +** big-endian format regardless of which byte order is used to compute +** the checksum. The checksum is computed by interpreting the input as +** an even number of unsigned 32-bit integers: x[0] through x[N]. The +** algorithm used for the checksum is as follows: +** +** for i from 0 to n-1 step 2: +** s0 += x[i] + s1; +** s1 += x[i+1] + s0; +** endfor +** +** Note that s0 and s1 are both weighted checksums using fibonacci weights +** in reverse order (the largest fibonacci weight occurs on the first element +** of the sequence being summed.) The s1 value spans all 32-bit +** terms of the sequence whereas s0 omits the final term. +** +** On a checkpoint, the WAL is first VFS.xSync-ed, then valid content of the +** WAL is transferred into the database, then the database is VFS.xSync-ed. +** The VFS.xSync operations serve as write barriers - all writes launched +** before the xSync must complete before any write that launches after the +** xSync begins. +** +** After each checkpoint, the salt-1 value is incremented and the salt-2 +** value is randomized. This prevents old and new frames in the WAL from +** being considered valid at the same time and being checkpointing together +** following a crash. +** +** READER ALGORITHM +** +** To read a page from the database (call it page number P), a reader +** first checks the WAL to see if it contains page P. If so, then the +** last valid instance of page P that is a followed by a commit frame +** or is a commit frame itself becomes the value read. If the WAL +** contains no copies of page P that are valid and which are a commit +** frame or are followed by a commit frame, then page P is read from +** the database file. +** +** To start a read transaction, the reader records the index of the last +** valid frame in the WAL. The reader uses this recorded "mxFrame" value +** for all subsequent read operations. New transactions can be appended +** to the WAL, but as long as the reader uses its original mxFrame value +** and ignores the newly appended content, it will see a consistent snapshot +** of the database from a single point in time. This technique allows +** multiple concurrent readers to view different versions of the database +** content simultaneously. +** +** The reader algorithm in the previous paragraphs works correctly, but +** because frames for page P can appear anywhere within the WAL, the +** reader has to scan the entire WAL looking for page P frames. If the +** WAL is large (multiple megabytes is typical) that scan can be slow, +** and read performance suffers. To overcome this problem, a separate +** data structure called the wal-index is maintained to expedite the +** search for frames of a particular page. +** +** WAL-INDEX FORMAT +** +** Conceptually, the wal-index is shared memory, though VFS implementations +** might choose to implement the wal-index using a mmapped file. Because +** the wal-index is shared memory, SQLite does not support journal_mode=WAL +** on a network filesystem. All users of the database must be able to +** share memory. +** +** In the default unix and windows implementation, the wal-index is a mmapped +** file whose name is the database name with a "-shm" suffix added. For that +** reason, the wal-index is sometimes called the "shm" file. +** +** The wal-index is transient. After a crash, the wal-index can (and should +** be) reconstructed from the original WAL file. In fact, the VFS is required +** to either truncate or zero the header of the wal-index when the last +** connection to it closes. Because the wal-index is transient, it can +** use an architecture-specific format; it does not have to be cross-platform. +** Hence, unlike the database and WAL file formats which store all values +** as big endian, the wal-index can store multi-byte values in the native +** byte order of the host computer. +** +** The purpose of the wal-index is to answer this question quickly: Given +** a page number P and a maximum frame index M, return the index of the +** last frame in the wal before frame M for page P in the WAL, or return +** NULL if there are no frames for page P in the WAL prior to M. +** +** The wal-index consists of a header region, followed by an one or +** more index blocks. +** +** The wal-index header contains the total number of frames within the WAL +** in the mxFrame field. +** +** Each index block except for the first contains information on +** HASHTABLE_NPAGE frames. The first index block contains information on +** HASHTABLE_NPAGE_ONE frames. The values of HASHTABLE_NPAGE_ONE and +** HASHTABLE_NPAGE are selected so that together the wal-index header and +** first index block are the same size as all other index blocks in the +** wal-index. The values are: +** +** HASHTABLE_NPAGE 4096 +** HASHTABLE_NPAGE_ONE 4062 +** +** Each index block contains two sections, a page-mapping that contains the +** database page number associated with each wal frame, and a hash-table +** that allows readers to query an index block for a specific page number. +** The page-mapping is an array of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE +** for the first index block) 32-bit page numbers. The first entry in the +** first index-block contains the database page number corresponding to the +** first frame in the WAL file. The first entry in the second index block +** in the WAL file corresponds to the (HASHTABLE_NPAGE_ONE+1)th frame in +** the log, and so on. +** +** The last index block in a wal-index usually contains less than the full +** complement of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE) page-numbers, +** depending on the contents of the WAL file. This does not change the +** allocated size of the page-mapping array - the page-mapping array merely +** contains unused entries. +** +** Even without using the hash table, the last frame for page P +** can be found by scanning the page-mapping sections of each index block +** starting with the last index block and moving toward the first, and +** within each index block, starting at the end and moving toward the +** beginning. The first entry that equals P corresponds to the frame +** holding the content for that page. +** +** The hash table consists of HASHTABLE_NSLOT 16-bit unsigned integers. +** HASHTABLE_NSLOT = 2*HASHTABLE_NPAGE, and there is one entry in the +** hash table for each page number in the mapping section, so the hash +** table is never more than half full. The expected number of collisions +** prior to finding a match is 1. Each entry of the hash table is an +** 1-based index of an entry in the mapping section of the same +** index block. Let K be the 1-based index of the largest entry in +** the mapping section. (For index blocks other than the last, K will +** always be exactly HASHTABLE_NPAGE (4096) and for the last index block +** K will be (mxFrame%HASHTABLE_NPAGE).) Unused slots of the hash table +** contain a value of 0. +** +** To look for page P in the hash table, first compute a hash iKey on +** P as follows: +** +** iKey = (P * 383) % HASHTABLE_NSLOT +** +** Then start scanning entries of the hash table, starting with iKey +** (wrapping around to the beginning when the end of the hash table is +** reached) until an unused hash slot is found. Let the first unused slot +** be at index iUnused. (iUnused might be less than iKey if there was +** wrap-around.) Because the hash table is never more than half full, +** the search is guaranteed to eventually hit an unused entry. Let +** iMax be the value between iKey and iUnused, closest to iUnused, +** where aHash[iMax]==P. If there is no iMax entry (if there exists +** no hash slot such that aHash[i]==p) then page P is not in the +** current index block. Otherwise the iMax-th mapping entry of the +** current index block corresponds to the last entry that references +** page P. +** +** A hash search begins with the last index block and moves toward the +** first index block, looking for entries corresponding to page P. On +** average, only two or three slots in each index block need to be +** examined in order to either find the last entry for page P, or to +** establish that no such entry exists in the block. Each index block +** holds over 4000 entries. So two or three index blocks are sufficient +** to cover a typical 10 megabyte WAL file, assuming 1K pages. 8 or 10 +** comparisons (on average) suffice to either locate a frame in the +** WAL or to establish that the frame does not exist in the WAL. This +** is much faster than scanning the entire 10MB WAL. +** +** Note that entries are added in order of increasing K. Hence, one +** reader might be using some value K0 and a second reader that started +** at a later time (after additional transactions were added to the WAL +** and to the wal-index) might be using a different value K1, where K1>K0. +** Both readers can use the same hash table and mapping section to get +** the correct result. There may be entries in the hash table with +** K>K0 but to the first reader, those entries will appear to be unused +** slots in the hash table and so the first reader will get an answer as +** if no values greater than K0 had ever been inserted into the hash table +** in the first place - which is what reader one wants. Meanwhile, the +** second reader using K1 will see additional values that were inserted +** later, which is exactly what reader two wants. +** +** When a rollback occurs, the value of K is decreased. Hash table entries +** that correspond to frames greater than the new K value are removed +** from the hash table at this point. +*/ +#ifndef SQLITE_OMIT_WAL + +/* #include "wal.h" */ + +/* +** Trace output macros +*/ +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3WalTrace = 0; +# define WALTRACE(X) if(sqlite3WalTrace) sqlite3DebugPrintf X +#else +# define WALTRACE(X) +#endif + +/* +** The maximum (and only) versions of the wal and wal-index formats +** that may be interpreted by this version of SQLite. +** +** If a client begins recovering a WAL file and finds that (a) the checksum +** values in the wal-header are correct and (b) the version field is not +** WAL_MAX_VERSION, recovery fails and SQLite returns SQLITE_CANTOPEN. +** +** Similarly, if a client successfully reads a wal-index header (i.e. the +** checksum test is successful) and finds that the version field is not +** WALINDEX_MAX_VERSION, then no read-transaction is opened and SQLite +** returns SQLITE_CANTOPEN. +*/ +#define WAL_MAX_VERSION 3007000 +#define WALINDEX_MAX_VERSION 3007000 + +/* +** Index numbers for various locking bytes. WAL_NREADER is the number +** of available reader locks and should be at least 3. The default +** is SQLITE_SHM_NLOCK==8 and WAL_NREADER==5. +** +** Technically, the various VFSes are free to implement these locks however +** they see fit. However, compatibility is encouraged so that VFSes can +** interoperate. The standard implemention used on both unix and windows +** is for the index number to indicate a byte offset into the +** WalCkptInfo.aLock[] array in the wal-index header. In other words, all +** locks are on the shm file. The WALINDEX_LOCK_OFFSET constant (which +** should be 120) is the location in the shm file for the first locking +** byte. +*/ +#define WAL_WRITE_LOCK 0 +#define WAL_ALL_BUT_WRITE 1 +#define WAL_CKPT_LOCK 1 +#define WAL_RECOVER_LOCK 2 +#define WAL_READ_LOCK(I) (3+(I)) +#define WAL_NREADER (SQLITE_SHM_NLOCK-3) + + +/* Object declarations */ +typedef struct WalIndexHdr WalIndexHdr; +typedef struct WalIterator WalIterator; +typedef struct WalCkptInfo WalCkptInfo; + + +/* +** The following object holds a copy of the wal-index header content. +** +** The actual header in the wal-index consists of two copies of this +** object followed by one instance of the WalCkptInfo object. +** For all versions of SQLite through 3.10.0 and probably beyond, +** the locking bytes (WalCkptInfo.aLock) start at offset 120 and +** the total header size is 136 bytes. +** +** The szPage value can be any power of 2 between 512 and 32768, inclusive. +** Or it can be 1 to represent a 65536-byte page. The latter case was +** added in 3.7.1 when support for 64K pages was added. +*/ +struct WalIndexHdr { + u32 iVersion; /* Wal-index version */ + u32 unused; /* Unused (padding) field */ + u32 iChange; /* Counter incremented each transaction */ + u8 isInit; /* 1 when initialized */ + u8 bigEndCksum; /* True if checksums in WAL are big-endian */ + u16 szPage; /* Database page size in bytes. 1==64K */ + u32 mxFrame; /* Index of last valid frame in the WAL */ + u32 nPage; /* Size of database in pages */ + u32 aFrameCksum[2]; /* Checksum of last frame in log */ + u32 aSalt[2]; /* Two salt values copied from WAL header */ + u32 aCksum[2]; /* Checksum over all prior fields */ +}; + +/* +** A copy of the following object occurs in the wal-index immediately +** following the second copy of the WalIndexHdr. This object stores +** information used by checkpoint. +** +** nBackfill is the number of frames in the WAL that have been written +** back into the database. (We call the act of moving content from WAL to +** database "backfilling".) The nBackfill number is never greater than +** WalIndexHdr.mxFrame. nBackfill can only be increased by threads +** holding the WAL_CKPT_LOCK lock (which includes a recovery thread). +** However, a WAL_WRITE_LOCK thread can move the value of nBackfill from +** mxFrame back to zero when the WAL is reset. +** +** nBackfillAttempted is the largest value of nBackfill that a checkpoint +** has attempted to achieve. Normally nBackfill==nBackfillAtempted, however +** the nBackfillAttempted is set before any backfilling is done and the +** nBackfill is only set after all backfilling completes. So if a checkpoint +** crashes, nBackfillAttempted might be larger than nBackfill. The +** WalIndexHdr.mxFrame must never be less than nBackfillAttempted. +** +** The aLock[] field is a set of bytes used for locking. These bytes should +** never be read or written. +** +** There is one entry in aReadMark[] for each reader lock. If a reader +** holds read-lock K, then the value in aReadMark[K] is no greater than +** the mxFrame for that reader. The value READMARK_NOT_USED (0xffffffff) +** for any aReadMark[] means that entry is unused. aReadMark[0] is +** a special case; its value is never used and it exists as a place-holder +** to avoid having to offset aReadMark[] indexs by one. Readers holding +** WAL_READ_LOCK(0) always ignore the entire WAL and read all content +** directly from the database. +** +** The value of aReadMark[K] may only be changed by a thread that +** is holding an exclusive lock on WAL_READ_LOCK(K). Thus, the value of +** aReadMark[K] cannot changed while there is a reader is using that mark +** since the reader will be holding a shared lock on WAL_READ_LOCK(K). +** +** The checkpointer may only transfer frames from WAL to database where +** the frame numbers are less than or equal to every aReadMark[] that is +** in use (that is, every aReadMark[j] for which there is a corresponding +** WAL_READ_LOCK(j)). New readers (usually) pick the aReadMark[] with the +** largest value and will increase an unused aReadMark[] to mxFrame if there +** is not already an aReadMark[] equal to mxFrame. The exception to the +** previous sentence is when nBackfill equals mxFrame (meaning that everything +** in the WAL has been backfilled into the database) then new readers +** will choose aReadMark[0] which has value 0 and hence such reader will +** get all their all content directly from the database file and ignore +** the WAL. +** +** Writers normally append new frames to the end of the WAL. However, +** if nBackfill equals mxFrame (meaning that all WAL content has been +** written back into the database) and if no readers are using the WAL +** (in other words, if there are no WAL_READ_LOCK(i) where i>0) then +** the writer will first "reset" the WAL back to the beginning and start +** writing new content beginning at frame 1. +** +** We assume that 32-bit loads are atomic and so no locks are needed in +** order to read from any aReadMark[] entries. +*/ +struct WalCkptInfo { + u32 nBackfill; /* Number of WAL frames backfilled into DB */ + u32 aReadMark[WAL_NREADER]; /* Reader marks */ + u8 aLock[SQLITE_SHM_NLOCK]; /* Reserved space for locks */ + u32 nBackfillAttempted; /* WAL frames perhaps written, or maybe not */ + u32 notUsed0; /* Available for future enhancements */ +}; +#define READMARK_NOT_USED 0xffffffff + +/* +** This is a schematic view of the complete 136-byte header of the +** wal-index file (also known as the -shm file): +** +** +-----------------------------+ +** 0: | iVersion | \ +** +-----------------------------+ | +** 4: | (unused padding) | | +** +-----------------------------+ | +** 8: | iChange | | +** +-------+-------+-------------+ | +** 12: | bInit | bBig | szPage | | +** +-------+-------+-------------+ | +** 16: | mxFrame | | First copy of the +** +-----------------------------+ | WalIndexHdr object +** 20: | nPage | | +** +-----------------------------+ | +** 24: | aFrameCksum | | +** | | | +** +-----------------------------+ | +** 32: | aSalt | | +** | | | +** +-----------------------------+ | +** 40: | aCksum | | +** | | / +** +-----------------------------+ +** 48: | iVersion | \ +** +-----------------------------+ | +** 52: | (unused padding) | | +** +-----------------------------+ | +** 56: | iChange | | +** +-------+-------+-------------+ | +** 60: | bInit | bBig | szPage | | +** +-------+-------+-------------+ | Second copy of the +** 64: | mxFrame | | WalIndexHdr +** +-----------------------------+ | +** 68: | nPage | | +** +-----------------------------+ | +** 72: | aFrameCksum | | +** | | | +** +-----------------------------+ | +** 80: | aSalt | | +** | | | +** +-----------------------------+ | +** 88: | aCksum | | +** | | / +** +-----------------------------+ +** 96: | nBackfill | +** +-----------------------------+ +** 100: | 5 read marks | +** | | +** | | +** | | +** | | +** +-------+-------+------+------+ +** 120: | Write | Ckpt | Rcvr | Rd0 | \ +** +-------+-------+------+------+ ) 8 lock bytes +** | Read1 | Read2 | Rd3 | Rd4 | / +** +-------+-------+------+------+ +** 128: | nBackfillAttempted | +** +-----------------------------+ +** 132: | (unused padding) | +** +-----------------------------+ +*/ + +/* A block of WALINDEX_LOCK_RESERVED bytes beginning at +** WALINDEX_LOCK_OFFSET is reserved for locks. Since some systems +** only support mandatory file-locks, we do not read or write data +** from the region of the file on which locks are applied. +*/ +#define WALINDEX_LOCK_OFFSET (sizeof(WalIndexHdr)*2+offsetof(WalCkptInfo,aLock)) +#define WALINDEX_HDR_SIZE (sizeof(WalIndexHdr)*2+sizeof(WalCkptInfo)) + +/* Size of header before each frame in wal */ +#define WAL_FRAME_HDRSIZE 24 + +/* Size of write ahead log header, including checksum. */ +#define WAL_HDRSIZE 32 + +/* WAL magic value. Either this value, or the same value with the least +** significant bit also set (WAL_MAGIC | 0x00000001) is stored in 32-bit +** big-endian format in the first 4 bytes of a WAL file. +** +** If the LSB is set, then the checksums for each frame within the WAL +** file are calculated by treating all data as an array of 32-bit +** big-endian words. Otherwise, they are calculated by interpreting +** all data as 32-bit little-endian words. +*/ +#define WAL_MAGIC 0x377f0682 + +/* +** Return the offset of frame iFrame in the write-ahead log file, +** assuming a database page size of szPage bytes. The offset returned +** is to the start of the write-ahead log frame-header. +*/ +#define walFrameOffset(iFrame, szPage) ( \ + WAL_HDRSIZE + ((iFrame)-1)*(i64)((szPage)+WAL_FRAME_HDRSIZE) \ +) + +/* +** An open write-ahead log file is represented by an instance of the +** following object. +*/ +struct Wal { + sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ + sqlite3_file *pDbFd; /* File handle for the database file */ + sqlite3_file *pWalFd; /* File handle for WAL file */ + u32 iCallback; /* Value to pass to log callback (or 0) */ + i64 mxWalSize; /* Truncate WAL to this size upon reset */ + int nWiData; /* Size of array apWiData */ + int szFirstBlock; /* Size of first block written to WAL file */ + volatile u32 **apWiData; /* Pointer to wal-index content in memory */ + u32 szPage; /* Database page size */ + i16 readLock; /* Which read lock is being held. -1 for none */ + u8 syncFlags; /* Flags to use to sync header writes */ + u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ + u8 writeLock; /* True if in a write transaction */ + u8 ckptLock; /* True if holding a checkpoint lock */ + u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ + u8 truncateOnCommit; /* True to truncate WAL file on commit */ + u8 syncHeader; /* Fsync the WAL header if true */ + u8 padToSectorBoundary; /* Pad transactions out to the next sector */ + u8 bShmUnreliable; /* SHM content is read-only and unreliable */ + WalIndexHdr hdr; /* Wal-index header for current transaction */ + u32 minFrame; /* Ignore wal frames before this one */ + u32 iReCksum; /* On commit, recalculate checksums from here */ + const char *zWalName; /* Name of WAL file */ + u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ +#ifdef SQLITE_DEBUG + u8 lockError; /* True if a locking error has occurred */ +#endif +#ifdef SQLITE_ENABLE_SNAPSHOT + WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ +#endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3 *db; +#endif +}; + +/* +** Candidate values for Wal.exclusiveMode. +*/ +#define WAL_NORMAL_MODE 0 +#define WAL_EXCLUSIVE_MODE 1 +#define WAL_HEAPMEMORY_MODE 2 + +/* +** Possible values for WAL.readOnly +*/ +#define WAL_RDWR 0 /* Normal read/write connection */ +#define WAL_RDONLY 1 /* The WAL file is readonly */ +#define WAL_SHM_RDONLY 2 /* The SHM file is readonly */ + +/* +** Each page of the wal-index mapping contains a hash-table made up of +** an array of HASHTABLE_NSLOT elements of the following type. +*/ +typedef u16 ht_slot; + +/* +** This structure is used to implement an iterator that loops through +** all frames in the WAL in database page order. Where two or more frames +** correspond to the same database page, the iterator visits only the +** frame most recently written to the WAL (in other words, the frame with +** the largest index). +** +** The internals of this structure are only accessed by: +** +** walIteratorInit() - Create a new iterator, +** walIteratorNext() - Step an iterator, +** walIteratorFree() - Free an iterator. +** +** This functionality is used by the checkpoint code (see walCheckpoint()). +*/ +struct WalIterator { + u32 iPrior; /* Last result returned from the iterator */ + int nSegment; /* Number of entries in aSegment[] */ + struct WalSegment { + int iNext; /* Next slot in aIndex[] not yet returned */ + ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ + u32 *aPgno; /* Array of page numbers. */ + int nEntry; /* Nr. of entries in aPgno[] and aIndex[] */ + int iZero; /* Frame number associated with aPgno[0] */ + } aSegment[1]; /* One for every 32KB page in the wal-index */ +}; + +/* +** Define the parameters of the hash tables in the wal-index file. There +** is a hash-table following every HASHTABLE_NPAGE page numbers in the +** wal-index. +** +** Changing any of these constants will alter the wal-index format and +** create incompatibilities. +*/ +#define HASHTABLE_NPAGE 4096 /* Must be power of 2 */ +#define HASHTABLE_HASH_1 383 /* Should be prime */ +#define HASHTABLE_NSLOT (HASHTABLE_NPAGE*2) /* Must be a power of 2 */ + +/* +** The block of page numbers associated with the first hash-table in a +** wal-index is smaller than usual. This is so that there is a complete +** hash-table on each aligned 32KB page of the wal-index. +*/ +#define HASHTABLE_NPAGE_ONE (HASHTABLE_NPAGE - (WALINDEX_HDR_SIZE/sizeof(u32))) + +/* The wal-index is divided into pages of WALINDEX_PGSZ bytes each. */ +#define WALINDEX_PGSZ ( \ + sizeof(ht_slot)*HASHTABLE_NSLOT + HASHTABLE_NPAGE*sizeof(u32) \ +) + +/* +** Obtain a pointer to the iPage'th page of the wal-index. The wal-index +** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are +** numbered from zero. +** +** If the wal-index is currently smaller the iPage pages then the size +** of the wal-index might be increased, but only if it is safe to do +** so. It is safe to enlarge the wal-index if pWal->writeLock is true +** or pWal->exclusiveMode==WAL_HEAPMEMORY_MODE. +** +** If this call is successful, *ppPage is set to point to the wal-index +** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs, +** then an SQLite error code is returned and *ppPage is set to 0. +*/ +static SQLITE_NOINLINE int walIndexPageRealloc( + Wal *pWal, /* The WAL context */ + int iPage, /* The page we seek */ + volatile u32 **ppPage /* Write the page pointer here */ +){ + int rc = SQLITE_OK; + + /* Enlarge the pWal->apWiData[] array if required */ + if( pWal->nWiData<=iPage ){ + sqlite3_int64 nByte = sizeof(u32*)*(iPage+1); + volatile u32 **apNew; + apNew = (volatile u32 **)sqlite3Realloc((void *)pWal->apWiData, nByte); + if( !apNew ){ + *ppPage = 0; + return SQLITE_NOMEM_BKPT; + } + memset((void*)&apNew[pWal->nWiData], 0, + sizeof(u32*)*(iPage+1-pWal->nWiData)); + pWal->apWiData = apNew; + pWal->nWiData = iPage+1; + } + + /* Request a pointer to the required page from the VFS */ + assert( pWal->apWiData[iPage]==0 ); + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, + pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] + ); + assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 ); + testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); + if( rc==SQLITE_OK ){ + if( iPage>0 && sqlite3FaultSim(600) ) rc = SQLITE_NOMEM; + }else if( (rc&0xff)==SQLITE_READONLY ){ + pWal->readOnly |= WAL_SHM_RDONLY; + if( rc==SQLITE_READONLY ){ + rc = SQLITE_OK; + } + } + } + + *ppPage = pWal->apWiData[iPage]; + assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); + return rc; +} +static int walIndexPage( + Wal *pWal, /* The WAL context */ + int iPage, /* The page we seek */ + volatile u32 **ppPage /* Write the page pointer here */ +){ + if( pWal->nWiData<=iPage || (*ppPage = pWal->apWiData[iPage])==0 ){ + return walIndexPageRealloc(pWal, iPage, ppPage); + } + return SQLITE_OK; +} + +/* +** Return a pointer to the WalCkptInfo structure in the wal-index. +*/ +static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + return (volatile WalCkptInfo*)&(pWal->apWiData[0][sizeof(WalIndexHdr)/2]); +} + +/* +** Return a pointer to the WalIndexHdr structure in the wal-index. +*/ +static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + return (volatile WalIndexHdr*)pWal->apWiData[0]; +} + +/* +** The argument to this macro must be of type u32. On a little-endian +** architecture, it returns the u32 value that results from interpreting +** the 4 bytes as a big-endian value. On a big-endian architecture, it +** returns the value that would be produced by interpreting the 4 bytes +** of the input value as a little-endian integer. +*/ +#define BYTESWAP32(x) ( \ + (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ + + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ +) + +/* +** Generate or extend an 8 byte checksum based on the data in +** array aByte[] and the initial values of aIn[0] and aIn[1] (or +** initial values of 0 and 0 if aIn==NULL). +** +** The checksum is written back into aOut[] before returning. +** +** nByte must be a positive multiple of 8. +*/ +static void walChecksumBytes( + int nativeCksum, /* True for native byte-order, false for non-native */ + u8 *a, /* Content to be checksummed */ + int nByte, /* Bytes of content in a[]. Must be a multiple of 8. */ + const u32 *aIn, /* Initial checksum value input */ + u32 *aOut /* OUT: Final checksum value output */ +){ + u32 s1, s2; + u32 *aData = (u32 *)a; + u32 *aEnd = (u32 *)&a[nByte]; + + if( aIn ){ + s1 = aIn[0]; + s2 = aIn[1]; + }else{ + s1 = s2 = 0; + } + + assert( nByte>=8 ); + assert( (nByte&0x00000007)==0 ); + assert( nByte<=65536 ); + + if( nativeCksum ){ + do { + s1 += *aData++ + s2; + s2 += *aData++ + s1; + }while( aDataexclusiveMode!=WAL_HEAPMEMORY_MODE ){ + sqlite3OsShmBarrier(pWal->pDbFd); + } +} + +/* +** Add the SQLITE_NO_TSAN as part of the return-type of a function +** definition as a hint that the function contains constructs that +** might give false-positive TSAN warnings. +** +** See tag-20200519-1. +*/ +#if defined(__clang__) && !defined(SQLITE_NO_TSAN) +# define SQLITE_NO_TSAN __attribute__((no_sanitize_thread)) +#else +# define SQLITE_NO_TSAN +#endif + +/* +** Write the header information in pWal->hdr into the wal-index. +** +** The checksum on pWal->hdr is updated before it is written. +*/ +static SQLITE_NO_TSAN void walIndexWriteHdr(Wal *pWal){ + volatile WalIndexHdr *aHdr = walIndexHdr(pWal); + const int nCksum = offsetof(WalIndexHdr, aCksum); + + assert( pWal->writeLock ); + pWal->hdr.isInit = 1; + pWal->hdr.iVersion = WALINDEX_MAX_VERSION; + walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); + /* Possible TSAN false-positive. See tag-20200519-1 */ + memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); + walShmBarrier(pWal); + memcpy((void*)&aHdr[0], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); +} + +/* +** This function encodes a single frame header and writes it to a buffer +** supplied by the caller. A frame-header is made up of a series of +** 4-byte big-endian integers, as follows: +** +** 0: Page number. +** 4: For commit records, the size of the database image in pages +** after the commit. For all other records, zero. +** 8: Salt-1 (copied from the wal-header) +** 12: Salt-2 (copied from the wal-header) +** 16: Checksum-1. +** 20: Checksum-2. +*/ +static void walEncodeFrame( + Wal *pWal, /* The write-ahead log */ + u32 iPage, /* Database page number for frame */ + u32 nTruncate, /* New db size (or 0 for non-commit frames) */ + u8 *aData, /* Pointer to page data */ + u8 *aFrame /* OUT: Write encoded frame here */ +){ + int nativeCksum; /* True for native byte-order checksums */ + u32 *aCksum = pWal->hdr.aFrameCksum; + assert( WAL_FRAME_HDRSIZE==24 ); + sqlite3Put4byte(&aFrame[0], iPage); + sqlite3Put4byte(&aFrame[4], nTruncate); + if( pWal->iReCksum==0 ){ + memcpy(&aFrame[8], pWal->hdr.aSalt, 8); + + nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); + walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); + + sqlite3Put4byte(&aFrame[16], aCksum[0]); + sqlite3Put4byte(&aFrame[20], aCksum[1]); + }else{ + memset(&aFrame[8], 0, 16); + } +} + +/* +** Check to see if the frame with header in aFrame[] and content +** in aData[] is valid. If it is a valid frame, fill *piPage and +** *pnTruncate and return true. Return if the frame is not valid. +*/ +static int walDecodeFrame( + Wal *pWal, /* The write-ahead log */ + u32 *piPage, /* OUT: Database page number for frame */ + u32 *pnTruncate, /* OUT: New db size (or 0 if not commit) */ + u8 *aData, /* Pointer to page data (for checksum) */ + u8 *aFrame /* Frame data */ +){ + int nativeCksum; /* True for native byte-order checksums */ + u32 *aCksum = pWal->hdr.aFrameCksum; + u32 pgno; /* Page number of the frame */ + assert( WAL_FRAME_HDRSIZE==24 ); + + /* A frame is only valid if the salt values in the frame-header + ** match the salt values in the wal-header. + */ + if( memcmp(&pWal->hdr.aSalt, &aFrame[8], 8)!=0 ){ + return 0; + } + + /* A frame is only valid if the page number is creater than zero. + */ + pgno = sqlite3Get4byte(&aFrame[0]); + if( pgno==0 ){ + return 0; + } + + /* A frame is only valid if a checksum of the WAL header, + ** all prior frams, the first 16 bytes of this frame-header, + ** and the frame-data matches the checksum in the last 8 + ** bytes of this frame-header. + */ + nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); + walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); + walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); + if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) + || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) + ){ + /* Checksum failed. */ + return 0; + } + + /* If we reach this point, the frame is valid. Return the page number + ** and the new database size. + */ + *piPage = pgno; + *pnTruncate = sqlite3Get4byte(&aFrame[4]); + return 1; +} + + +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) +/* +** Names of locks. This routine is used to provide debugging output and is not +** a part of an ordinary build. +*/ +static const char *walLockName(int lockIdx){ + if( lockIdx==WAL_WRITE_LOCK ){ + return "WRITE-LOCK"; + }else if( lockIdx==WAL_CKPT_LOCK ){ + return "CKPT-LOCK"; + }else if( lockIdx==WAL_RECOVER_LOCK ){ + return "RECOVER-LOCK"; + }else{ + static char zName[15]; + sqlite3_snprintf(sizeof(zName), zName, "READ-LOCK[%d]", + lockIdx-WAL_READ_LOCK(0)); + return zName; + } +} +#endif /*defined(SQLITE_TEST) || defined(SQLITE_DEBUG) */ + + +/* +** Set or release locks on the WAL. Locks are either shared or exclusive. +** A lock cannot be moved directly between shared and exclusive - it must go +** through the unlocked state first. +** +** In locking_mode=EXCLUSIVE, all of these routines become no-ops. +*/ +static int walLockShared(Wal *pWal, int lockIdx){ + int rc; + if( pWal->exclusiveMode ) return SQLITE_OK; + rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, + SQLITE_SHM_LOCK | SQLITE_SHM_SHARED); + WALTRACE(("WAL%p: acquire SHARED-%s %s\n", pWal, + walLockName(lockIdx), rc ? "failed" : "ok")); + VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) + return rc; +} +static void walUnlockShared(Wal *pWal, int lockIdx){ + if( pWal->exclusiveMode ) return; + (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, + SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED); + WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx))); +} +static int walLockExclusive(Wal *pWal, int lockIdx, int n){ + int rc; + if( pWal->exclusiveMode ) return SQLITE_OK; + rc = sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, + SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); + WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, + walLockName(lockIdx), n, rc ? "failed" : "ok")); + VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) + return rc; +} +static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ + if( pWal->exclusiveMode ) return; + (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, + SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE); + WALTRACE(("WAL%p: release EXCLUSIVE-%s cnt=%d\n", pWal, + walLockName(lockIdx), n)); +} + +/* +** Compute a hash on a page number. The resulting hash value must land +** between 0 and (HASHTABLE_NSLOT-1). The walHashNext() function advances +** the hash to the next value in the event of a collision. +*/ +static int walHash(u32 iPage){ + assert( iPage>0 ); + assert( (HASHTABLE_NSLOT & (HASHTABLE_NSLOT-1))==0 ); + return (iPage*HASHTABLE_HASH_1) & (HASHTABLE_NSLOT-1); +} +static int walNextHash(int iPriorHash){ + return (iPriorHash+1)&(HASHTABLE_NSLOT-1); +} + +/* +** An instance of the WalHashLoc object is used to describe the location +** of a page hash table in the wal-index. This becomes the return value +** from walHashGet(). +*/ +typedef struct WalHashLoc WalHashLoc; +struct WalHashLoc { + volatile ht_slot *aHash; /* Start of the wal-index hash table */ + volatile u32 *aPgno; /* aPgno[1] is the page of first frame indexed */ + u32 iZero; /* One less than the frame number of first indexed*/ +}; + +/* +** Return pointers to the hash table and page number array stored on +** page iHash of the wal-index. The wal-index is broken into 32KB pages +** numbered starting from 0. +** +** Set output variable pLoc->aHash to point to the start of the hash table +** in the wal-index file. Set pLoc->iZero to one less than the frame +** number of the first frame indexed by this hash table. If a +** slot in the hash table is set to N, it refers to frame number +** (pLoc->iZero+N) in the log. +** +** Finally, set pLoc->aPgno so that pLoc->aPgno[1] is the page number of the +** first frame indexed by the hash table, frame (pLoc->iZero+1). +*/ +static int walHashGet( + Wal *pWal, /* WAL handle */ + int iHash, /* Find the iHash'th table */ + WalHashLoc *pLoc /* OUT: Hash table location */ +){ + int rc; /* Return code */ + + rc = walIndexPage(pWal, iHash, &pLoc->aPgno); + assert( rc==SQLITE_OK || iHash>0 ); + + if( rc==SQLITE_OK ){ + pLoc->aHash = (volatile ht_slot *)&pLoc->aPgno[HASHTABLE_NPAGE]; + if( iHash==0 ){ + pLoc->aPgno = &pLoc->aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; + pLoc->iZero = 0; + }else{ + pLoc->iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; + } + pLoc->aPgno = &pLoc->aPgno[-1]; + } + return rc; +} + +/* +** Return the number of the wal-index page that contains the hash-table +** and page-number array that contain entries corresponding to WAL frame +** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages +** are numbered starting from 0. +*/ +static int walFramePage(u32 iFrame){ + int iHash = (iFrame+HASHTABLE_NPAGE-HASHTABLE_NPAGE_ONE-1) / HASHTABLE_NPAGE; + assert( (iHash==0 || iFrame>HASHTABLE_NPAGE_ONE) + && (iHash>=1 || iFrame<=HASHTABLE_NPAGE_ONE) + && (iHash<=1 || iFrame>(HASHTABLE_NPAGE_ONE+HASHTABLE_NPAGE)) + && (iHash>=2 || iFrame<=HASHTABLE_NPAGE_ONE+HASHTABLE_NPAGE) + && (iHash<=2 || iFrame>(HASHTABLE_NPAGE_ONE+2*HASHTABLE_NPAGE)) + ); + assert( iHash>=0 ); + return iHash; +} + +/* +** Return the page number associated with frame iFrame in this WAL. +*/ +static u32 walFramePgno(Wal *pWal, u32 iFrame){ + int iHash = walFramePage(iFrame); + if( iHash==0 ){ + return pWal->apWiData[0][WALINDEX_HDR_SIZE/sizeof(u32) + iFrame - 1]; + } + return pWal->apWiData[iHash][(iFrame-1-HASHTABLE_NPAGE_ONE)%HASHTABLE_NPAGE]; +} + +/* +** Remove entries from the hash table that point to WAL slots greater +** than pWal->hdr.mxFrame. +** +** This function is called whenever pWal->hdr.mxFrame is decreased due +** to a rollback or savepoint. +** +** At most only the hash table containing pWal->hdr.mxFrame needs to be +** updated. Any later hash tables will be automatically cleared when +** pWal->hdr.mxFrame advances to the point where those hash tables are +** actually needed. +*/ +static void walCleanupHash(Wal *pWal){ + WalHashLoc sLoc; /* Hash table location */ + int iLimit = 0; /* Zero values greater than this */ + int nByte; /* Number of bytes to zero in aPgno[] */ + int i; /* Used to iterate through aHash[] */ + + assert( pWal->writeLock ); + testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE-1 ); + testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE ); + testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE+1 ); + + if( pWal->hdr.mxFrame==0 ) return; + + /* Obtain pointers to the hash-table and page-number array containing + ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed + ** that the page said hash-table and array reside on is already mapped.(1) + */ + assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); + assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); + i = walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &sLoc); + if( NEVER(i) ) return; /* Defense-in-depth, in case (1) above is wrong */ + + /* Zero all hash-table entries that correspond to frame numbers greater + ** than pWal->hdr.mxFrame. + */ + iLimit = pWal->hdr.mxFrame - sLoc.iZero; + assert( iLimit>0 ); + for(i=0; iiLimit ){ + sLoc.aHash[i] = 0; + } + } + + /* Zero the entries in the aPgno array that correspond to frames with + ** frame numbers greater than pWal->hdr.mxFrame. + */ + nByte = (int)((char *)sLoc.aHash - (char *)&sLoc.aPgno[iLimit+1]); + memset((void *)&sLoc.aPgno[iLimit+1], 0, nByte); + +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + /* Verify that the every entry in the mapping region is still reachable + ** via the hash table even after the cleanup. + */ + if( iLimit ){ + int j; /* Loop counter */ + int iKey; /* Hash key */ + for(j=1; j<=iLimit; j++){ + for(iKey=walHash(sLoc.aPgno[j]);sLoc.aHash[iKey];iKey=walNextHash(iKey)){ + if( sLoc.aHash[iKey]==j ) break; + } + assert( sLoc.aHash[iKey]==j ); + } + } +#endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ +} + + +/* +** Set an entry in the wal-index that will map database page number +** pPage into WAL frame iFrame. +*/ +static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ + int rc; /* Return code */ + WalHashLoc sLoc; /* Wal-index hash table location */ + + rc = walHashGet(pWal, walFramePage(iFrame), &sLoc); + + /* Assuming the wal-index file was successfully mapped, populate the + ** page number array and hash table entry. + */ + if( rc==SQLITE_OK ){ + int iKey; /* Hash table key */ + int idx; /* Value to write to hash-table slot */ + int nCollide; /* Number of hash collisions */ + + idx = iFrame - sLoc.iZero; + assert( idx <= HASHTABLE_NSLOT/2 + 1 ); + + /* If this is the first entry to be added to this hash-table, zero the + ** entire hash table and aPgno[] array before proceeding. + */ + if( idx==1 ){ + int nByte = (int)((u8 *)&sLoc.aHash[HASHTABLE_NSLOT] + - (u8 *)&sLoc.aPgno[1]); + memset((void*)&sLoc.aPgno[1], 0, nByte); + } + + /* If the entry in aPgno[] is already set, then the previous writer + ** must have exited unexpectedly in the middle of a transaction (after + ** writing one or more dirty pages to the WAL to free up memory). + ** Remove the remnants of that writers uncommitted transaction from + ** the hash-table before writing any new entries. + */ + if( sLoc.aPgno[idx] ){ + walCleanupHash(pWal); + assert( !sLoc.aPgno[idx] ); + } + + /* Write the aPgno[] array entry and the hash-table slot. */ + nCollide = idx; + for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ + if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; + } + sLoc.aPgno[idx] = iPage; + AtomicStore(&sLoc.aHash[iKey], (ht_slot)idx); + +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + /* Verify that the number of entries in the hash table exactly equals + ** the number of entries in the mapping region. + */ + { + int i; /* Loop counter */ + int nEntry = 0; /* Number of entries in the hash table */ + for(i=0; ickptLock==1 || pWal->ckptLock==0 ); + assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 ); + assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); + assert( pWal->writeLock ); + iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; + rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + if( rc ){ + return rc; + } + + WALTRACE(("WAL%p: recovery begin...\n", pWal)); + + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + + rc = sqlite3OsFileSize(pWal->pWalFd, &nSize); + if( rc!=SQLITE_OK ){ + goto recovery_error; + } + + if( nSize>WAL_HDRSIZE ){ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u32 *aPrivate = 0; /* Heap copy of *-shm hash being populated */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + int szPage; /* Page size according to the log */ + u32 magic; /* Magic value read from WAL header */ + u32 version; /* Magic value read from WAL header */ + int isValid; /* True if this frame is valid */ + u32 iPg; /* Current 32KB wal-index page */ + u32 iLastFrame; /* Last frame in wal, based on nSize alone */ + + /* Read in the WAL header. */ + rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + goto recovery_error; + } + + /* If the database page size is not a power of two, or is greater than + ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid + ** data. Similarly, if the 'magic' value is invalid, ignore the whole + ** WAL file. + */ + magic = sqlite3Get4byte(&aBuf[0]); + szPage = sqlite3Get4byte(&aBuf[8]); + if( (magic&0xFFFFFFFE)!=WAL_MAGIC + || szPage&(szPage-1) + || szPage>SQLITE_MAX_PAGE_SIZE + || szPage<512 + ){ + goto finished; + } + pWal->hdr.bigEndCksum = (u8)(magic&0x00000001); + pWal->szPage = szPage; + pWal->nCkpt = sqlite3Get4byte(&aBuf[12]); + memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); + + /* Verify that the WAL header checksum is correct */ + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum + ); + if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) + || pWal->hdr.aFrameCksum[1]!=sqlite3Get4byte(&aBuf[28]) + ){ + goto finished; + } + + /* Verify that the version number on the WAL format is one that + ** are able to understand */ + version = sqlite3Get4byte(&aBuf[4]); + if( version!=WAL_MAX_VERSION ){ + rc = SQLITE_CANTOPEN_BKPT; + goto finished; + } + + /* Malloc a buffer to read frames into. */ + szFrame = szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ); + if( !aFrame ){ + rc = SQLITE_NOMEM_BKPT; + goto recovery_error; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + aPrivate = (u32*)&aData[szPage]; + + /* Read all frames from the log file. */ + iLastFrame = (nSize - WAL_HDRSIZE) / szFrame; + for(iPg=0; iPg<=(u32)walFramePage(iLastFrame); iPg++){ + u32 *aShare; + u32 iFrame; /* Index of last frame read */ + u32 iLast = MIN(iLastFrame, HASHTABLE_NPAGE_ONE+iPg*HASHTABLE_NPAGE); + u32 iFirst = 1 + (iPg==0?0:HASHTABLE_NPAGE_ONE+(iPg-1)*HASHTABLE_NPAGE); + u32 nHdr, nHdr32; + rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare); + if( rc ) break; + pWal->apWiData[iPg] = aPrivate; + + for(iFrame=iFirst; iFrame<=iLast; iFrame++){ + i64 iOffset = walFrameOffset(iFrame, szPage); + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + + /* Read and decode the next log frame. */ + rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); + if( !isValid ) break; + rc = walIndexAppend(pWal, iFrame, pgno); + if( NEVER(rc!=SQLITE_OK) ) break; + + /* If nTruncate is non-zero, this is a commit record. */ + if( nTruncate ){ + pWal->hdr.mxFrame = iFrame; + pWal->hdr.nPage = nTruncate; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; + aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + } + } + pWal->apWiData[iPg] = aShare; + nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0); + nHdr32 = nHdr / sizeof(u32); +#ifndef SQLITE_SAFER_WALINDEX_RECOVERY + /* Memcpy() should work fine here, on all reasonable implementations. + ** Technically, memcpy() might change the destination to some + ** intermediate value before setting to the final value, and that might + ** cause a concurrent reader to malfunction. Memcpy() is allowed to + ** do that, according to the spec, but no memcpy() implementation that + ** we know of actually does that, which is why we say that memcpy() + ** is safe for this. Memcpy() is certainly a lot faster. + */ + memcpy(&aShare[nHdr32], &aPrivate[nHdr32], WALINDEX_PGSZ-nHdr); +#else + /* In the event that some platform is found for which memcpy() + ** changes the destination to some intermediate value before + ** setting the final value, this alternative copy routine is + ** provided. + */ + { + int i; + for(i=nHdr32; ihdr.aFrameCksum[0] = aFrameCksum[0]; + pWal->hdr.aFrameCksum[1] = aFrameCksum[1]; + walIndexWriteHdr(pWal); + + /* Reset the checkpoint-header. This is safe because this thread is + ** currently holding locks that exclude all other writers and + ** checkpointers. Then set the values of read-mark slots 1 through N. + */ + pInfo = walCkptInfo(pWal); + pInfo->nBackfill = 0; + pInfo->nBackfillAttempted = pWal->hdr.mxFrame; + pInfo->aReadMark[0] = 0; + for(i=1; ihdr.mxFrame ){ + pInfo->aReadMark[i] = pWal->hdr.mxFrame; + }else{ + pInfo->aReadMark[i] = READMARK_NOT_USED; + } + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc!=SQLITE_BUSY ){ + goto recovery_error; + } + } + + /* If more than one frame was recovered from the log file, report an + ** event via sqlite3_log(). This is to help with identifying performance + ** problems caused by applications routinely shutting down without + ** checkpointing the log file. + */ + if( pWal->hdr.nPage ){ + sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, + "recovered %d frames from WAL file %s", + pWal->hdr.mxFrame, pWal->zWalName + ); + } + } + +recovery_error: + WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); + walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + return rc; +} + +/* +** Close an open wal-index. +*/ +static void walIndexClose(Wal *pWal, int isDelete){ + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bShmUnreliable ){ + int i; + for(i=0; inWiData; i++){ + sqlite3_free((void *)pWal->apWiData[i]); + pWal->apWiData[i] = 0; + } + } + if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ + sqlite3OsShmUnmap(pWal->pDbFd, isDelete); + } +} + +/* +** Open a connection to the WAL file zWalName. The database file must +** already be opened on connection pDbFd. The buffer that zWalName points +** to must remain valid for the lifetime of the returned Wal* handle. +** +** A SHARED lock should be held on the database file when this function +** is called. The purpose of this SHARED lock is to prevent any other +** client from unlinking the WAL or wal-index file. If another process +** were to do this just after this client opened one of these files, the +** system would be badly broken. +** +** If the log file is successfully opened, SQLITE_OK is returned and +** *ppWal is set to point to a new WAL handle. If an error occurs, +** an SQLite error code is returned and *ppWal is left unmodified. +*/ +SQLITE_PRIVATE int sqlite3WalOpen( + sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ + sqlite3_file *pDbFd, /* The open database file */ + const char *zWalName, /* Name of the WAL file */ + int bNoShm, /* True to run in heap-memory mode */ + i64 mxWalSize, /* Truncate WAL to this size on reset */ + Wal **ppWal /* OUT: Allocated Wal handle */ +){ + int rc; /* Return Code */ + Wal *pRet; /* Object to allocate and return */ + int flags; /* Flags passed to OsOpen() */ + + assert( zWalName && zWalName[0] ); + assert( pDbFd ); + + /* Verify the values of various constants. Any changes to the values + ** of these constants would result in an incompatible on-disk format + ** for the -shm file. Any change that causes one of these asserts to + ** fail is a backward compatibility problem, even if the change otherwise + ** works. + ** + ** This table also serves as a helpful cross-reference when trying to + ** interpret hex dumps of the -shm file. + */ + assert( 48 == sizeof(WalIndexHdr) ); + assert( 40 == sizeof(WalCkptInfo) ); + assert( 120 == WALINDEX_LOCK_OFFSET ); + assert( 136 == WALINDEX_HDR_SIZE ); + assert( 4096 == HASHTABLE_NPAGE ); + assert( 4062 == HASHTABLE_NPAGE_ONE ); + assert( 8192 == HASHTABLE_NSLOT ); + assert( 383 == HASHTABLE_HASH_1 ); + assert( 32768 == WALINDEX_PGSZ ); + assert( 8 == SQLITE_SHM_NLOCK ); + assert( 5 == WAL_NREADER ); + assert( 24 == WAL_FRAME_HDRSIZE ); + assert( 32 == WAL_HDRSIZE ); + assert( 120 == WALINDEX_LOCK_OFFSET + WAL_WRITE_LOCK ); + assert( 121 == WALINDEX_LOCK_OFFSET + WAL_CKPT_LOCK ); + assert( 122 == WALINDEX_LOCK_OFFSET + WAL_RECOVER_LOCK ); + assert( 123 == WALINDEX_LOCK_OFFSET + WAL_READ_LOCK(0) ); + assert( 124 == WALINDEX_LOCK_OFFSET + WAL_READ_LOCK(1) ); + assert( 125 == WALINDEX_LOCK_OFFSET + WAL_READ_LOCK(2) ); + assert( 126 == WALINDEX_LOCK_OFFSET + WAL_READ_LOCK(3) ); + assert( 127 == WALINDEX_LOCK_OFFSET + WAL_READ_LOCK(4) ); + + /* In the amalgamation, the os_unix.c and os_win.c source files come before + ** this source file. Verify that the #defines of the locking byte offsets + ** in os_unix.c and os_win.c agree with the WALINDEX_LOCK_OFFSET value. + ** For that matter, if the lock offset ever changes from its initial design + ** value of 120, we need to know that so there is an assert() to check it. + */ +#ifdef WIN_SHM_BASE + assert( WIN_SHM_BASE==WALINDEX_LOCK_OFFSET ); +#endif +#ifdef UNIX_SHM_BASE + assert( UNIX_SHM_BASE==WALINDEX_LOCK_OFFSET ); +#endif + + + /* Allocate an instance of struct Wal to return. */ + *ppWal = 0; + pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile); + if( !pRet ){ + return SQLITE_NOMEM_BKPT; + } + + pRet->pVfs = pVfs; + pRet->pWalFd = (sqlite3_file *)&pRet[1]; + pRet->pDbFd = pDbFd; + pRet->readLock = -1; + pRet->mxWalSize = mxWalSize; + pRet->zWalName = zWalName; + pRet->syncHeader = 1; + pRet->padToSectorBoundary = 1; + pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); + + /* Open file handle on the write-ahead log file. */ + flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); + if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ + pRet->readOnly = WAL_RDONLY; + } + + if( rc!=SQLITE_OK ){ + walIndexClose(pRet, 0); + sqlite3OsClose(pRet->pWalFd); + sqlite3_free(pRet); + }else{ + int iDC = sqlite3OsDeviceCharacteristics(pDbFd); + if( iDC & SQLITE_IOCAP_SEQUENTIAL ){ pRet->syncHeader = 0; } + if( iDC & SQLITE_IOCAP_POWERSAFE_OVERWRITE ){ + pRet->padToSectorBoundary = 0; + } + *ppWal = pRet; + WALTRACE(("WAL%d: opened\n", pRet)); + } + return rc; +} + +/* +** Change the size to which the WAL file is trucated on each reset. +*/ +SQLITE_PRIVATE void sqlite3WalLimit(Wal *pWal, i64 iLimit){ + if( pWal ) pWal->mxWalSize = iLimit; +} + +/* +** Find the smallest page number out of all pages held in the WAL that +** has not been returned by any prior invocation of this method on the +** same WalIterator object. Write into *piFrame the frame index where +** that page was last written into the WAL. Write into *piPage the page +** number. +** +** Return 0 on success. If there are no pages in the WAL with a page +** number larger than *piPage, then return 1. +*/ +static int walIteratorNext( + WalIterator *p, /* Iterator */ + u32 *piPage, /* OUT: The page number of the next page */ + u32 *piFrame /* OUT: Wal frame index of next page */ +){ + u32 iMin; /* Result pgno must be greater than iMin */ + u32 iRet = 0xFFFFFFFF; /* 0xffffffff is never a valid page number */ + int i; /* For looping through segments */ + + iMin = p->iPrior; + assert( iMin<0xffffffff ); + for(i=p->nSegment-1; i>=0; i--){ + struct WalSegment *pSegment = &p->aSegment[i]; + while( pSegment->iNextnEntry ){ + u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]]; + if( iPg>iMin ){ + if( iPgiZero + pSegment->aIndex[pSegment->iNext]; + } + break; + } + pSegment->iNext++; + } + } + + *piPage = p->iPrior = iRet; + return (iRet==0xFFFFFFFF); +} + +/* +** This function merges two sorted lists into a single sorted list. +** +** aLeft[] and aRight[] are arrays of indices. The sort key is +** aContent[aLeft[]] and aContent[aRight[]]. Upon entry, the following +** is guaranteed for all J0 && nRight>0 ); + while( iRight=nRight || aContent[aLeft[iLeft]]=nLeft || aContent[aLeft[iLeft]]>dbpage ); + assert( iRight>=nRight || aContent[aRight[iRight]]>dbpage ); + } + + *paRight = aLeft; + *pnRight = iOut; + memcpy(aLeft, aTmp, sizeof(aTmp[0])*iOut); +} + +/* +** Sort the elements in list aList using aContent[] as the sort key. +** Remove elements with duplicate keys, preferring to keep the +** larger aList[] values. +** +** The aList[] entries are indices into aContent[]. The values in +** aList[] are to be sorted so that for all J0 ); + assert( HASHTABLE_NPAGE==(1<<(ArraySize(aSub)-1)) ); + + for(iList=0; iListaList && p->nList<=(1<aList==&aList[iList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); + } + aSub[iSub].aList = aMerge; + aSub[iSub].nList = nMerge; + } + + for(iSub++; iSubnList<=(1<aList==&aList[nList&~((2<aList, p->nList, &aMerge, &nMerge, aBuffer); + } + } + assert( aMerge==aList ); + *pnList = nMerge; + +#ifdef SQLITE_DEBUG + { + int i; + for(i=1; i<*pnList; i++){ + assert( aContent[aList[i]] > aContent[aList[i-1]] ); + } + } +#endif +} + +/* +** Free an iterator allocated by walIteratorInit(). +*/ +static void walIteratorFree(WalIterator *p){ + sqlite3_free(p); +} + +/* +** Construct a WalInterator object that can be used to loop over all +** pages in the WAL following frame nBackfill in ascending order. Frames +** nBackfill or earlier may be included - excluding them is an optimization +** only. The caller must hold the checkpoint lock. +** +** On success, make *pp point to the newly allocated WalInterator object +** return SQLITE_OK. Otherwise, return an error code. If this routine +** returns an error, the value of *pp is undefined. +** +** The calling routine should invoke walIteratorFree() to destroy the +** WalIterator object when it has finished with it. +*/ +static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ + WalIterator *p; /* Return value */ + int nSegment; /* Number of segments to merge */ + u32 iLast; /* Last frame in log */ + sqlite3_int64 nByte; /* Number of bytes to allocate */ + int i; /* Iterator variable */ + ht_slot *aTmp; /* Temp space used by merge-sort */ + int rc = SQLITE_OK; /* Return Code */ + + /* This routine only runs while holding the checkpoint lock. And + ** it only runs if there is actually content in the log (mxFrame>0). + */ + assert( pWal->ckptLock && pWal->hdr.mxFrame>0 ); + iLast = pWal->hdr.mxFrame; + + /* Allocate space for the WalIterator object. */ + nSegment = walFramePage(iLast) + 1; + nByte = sizeof(WalIterator) + + (nSegment-1)*sizeof(struct WalSegment) + + iLast*sizeof(ht_slot); + p = (WalIterator *)sqlite3_malloc64(nByte); + if( !p ){ + return SQLITE_NOMEM_BKPT; + } + memset(p, 0, nByte); + p->nSegment = nSegment; + + /* Allocate temporary space used by the merge-sort routine. This block + ** of memory will be freed before this function returns. + */ + aTmp = (ht_slot *)sqlite3_malloc64( + sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) + ); + if( !aTmp ){ + rc = SQLITE_NOMEM_BKPT; + } + + for(i=walFramePage(nBackfill+1); rc==SQLITE_OK && iaSegment[p->nSegment])[sLoc.iZero]; + sLoc.iZero++; + + for(j=0; jaSegment[i].iZero = sLoc.iZero; + p->aSegment[i].nEntry = nEntry; + p->aSegment[i].aIndex = aIndex; + p->aSegment[i].aPgno = (u32 *)sLoc.aPgno; + } + } + sqlite3_free(aTmp); + + if( rc!=SQLITE_OK ){ + walIteratorFree(p); + p = 0; + } + *pp = p; + return rc; +} + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +/* +** Attempt to enable blocking locks. Blocking locks are enabled only if (a) +** they are supported by the VFS, and (b) the database handle is configured +** with a busy-timeout. Return 1 if blocking locks are successfully enabled, +** or 0 otherwise. +*/ +static int walEnableBlocking(Wal *pWal){ + int res = 0; + if( pWal->db ){ + int tmout = pWal->db->busyTimeout; + if( tmout ){ + int rc; + rc = sqlite3OsFileControl( + pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout + ); + res = (rc==SQLITE_OK); + } + } + return res; +} + +/* +** Disable blocking locks. +*/ +static void walDisableBlocking(Wal *pWal){ + int tmout = 0; + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout); +} + +/* +** If parameter bLock is true, attempt to enable blocking locks, take +** the WRITER lock, and then disable blocking locks. If blocking locks +** cannot be enabled, no attempt to obtain the WRITER lock is made. Return +** an SQLite error code if an error occurs, or SQLITE_OK otherwise. It is not +** an error if blocking locks can not be enabled. +** +** If the bLock parameter is false and the WRITER lock is held, release it. +*/ +SQLITE_PRIVATE int sqlite3WalWriteLock(Wal *pWal, int bLock){ + int rc = SQLITE_OK; + assert( pWal->readLock<0 || bLock==0 ); + if( bLock ){ + assert( pWal->db ); + if( walEnableBlocking(pWal) ){ + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + } + walDisableBlocking(pWal); + } + }else if( pWal->writeLock ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + } + return rc; +} + +/* +** Set the database handle used to determine if blocking locks are required. +*/ +SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db){ + pWal->db = db; +} + +/* +** Take an exclusive WRITE lock. Blocking if so configured. +*/ +static int walLockWriter(Wal *pWal){ + int rc; + walEnableBlocking(pWal); + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); + walDisableBlocking(pWal); + return rc; +} +#else +# define walEnableBlocking(x) 0 +# define walDisableBlocking(x) +# define walLockWriter(pWal) walLockExclusive((pWal), WAL_WRITE_LOCK, 1) +# define sqlite3WalDb(pWal, db) +#endif /* ifdef SQLITE_ENABLE_SETLK_TIMEOUT */ + + +/* +** Attempt to obtain the exclusive WAL lock defined by parameters lockIdx and +** n. If the attempt fails and parameter xBusy is not NULL, then it is a +** busy-handler function. Invoke it and retry the lock until either the +** lock is successfully obtained or the busy-handler returns 0. +*/ +static int walBusyLock( + Wal *pWal, /* WAL connection */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int lockIdx, /* Offset of first byte to lock */ + int n /* Number of bytes to lock */ +){ + int rc; + do { + rc = walLockExclusive(pWal, lockIdx, n); + }while( xBusy && rc==SQLITE_BUSY && xBusy(pBusyArg) ); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ){ + walDisableBlocking(pWal); + rc = SQLITE_BUSY; + } +#endif + return rc; +} + +/* +** The cache of the wal-index header must be valid to call this function. +** Return the page-size in bytes used by the database. +*/ +static int walPagesize(Wal *pWal){ + return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); +} + +/* +** The following is guaranteed when this function is called: +** +** a) the WRITER lock is held, +** b) the entire log file has been checkpointed, and +** c) any existing readers are reading exclusively from the database +** file - there are no readers that may attempt to read a frame from +** the log file. +** +** This function updates the shared-memory structures so that the next +** client to write to the database (which may be this one) does so by +** writing frames into the start of the log file. +** +** The value of parameter salt1 is used as the aSalt[1] value in the +** new wal-index header. It should be passed a pseudo-random value (i.e. +** one obtained from sqlite3_randomness()). +*/ +static void walRestartHdr(Wal *pWal, u32 salt1){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + int i; /* Loop counter */ + u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + pWal->nCkpt++; + pWal->hdr.mxFrame = 0; + sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); + memcpy(&pWal->hdr.aSalt[1], &salt1, 4); + walIndexWriteHdr(pWal); + AtomicStore(&pInfo->nBackfill, 0); + pInfo->nBackfillAttempted = 0; + pInfo->aReadMark[1] = 0; + for(i=2; iaReadMark[i] = READMARK_NOT_USED; + assert( pInfo->aReadMark[0]==0 ); +} + +/* +** Copy as much content as we can from the WAL back into the database file +** in response to an sqlite3_wal_checkpoint() request or the equivalent. +** +** The amount of information copies from WAL to database might be limited +** by active readers. This routine will never overwrite a database page +** that a concurrent reader might be using. +** +** All I/O barrier operations (a.k.a fsyncs) occur in this routine when +** SQLite is in WAL-mode in synchronous=NORMAL. That means that if +** checkpoints are always run by a background thread or background +** process, foreground threads will never block on a lengthy fsync call. +** +** Fsync is called on the WAL before writing content out of the WAL and +** into the database. This ensures that if the new content is persistent +** in the WAL and can be recovered following a power-loss or hard reset. +** +** Fsync is also called on the database file if (and only if) the entire +** WAL content is copied into the database file. This second fsync makes +** it safe to delete the WAL since the new content will persist in the +** database file. +** +** This routine uses and updates the nBackfill field of the wal-index header. +** This is the only routine that will increase the value of nBackfill. +** (A WAL reset or recovery will revert nBackfill to zero, but not increase +** its value.) +** +** The caller must be holding sufficient locks to ensure that no other +** checkpoint is running (in any other thread or process) at the same +** time. +*/ +static int walCheckpoint( + Wal *pWal, /* Wal connection */ + sqlite3 *db, /* Check for interrupts on this handle */ + int eMode, /* One of PASSIVE, FULL or RESTART */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags for OsSync() (or 0) */ + u8 *zBuf /* Temporary buffer to use */ +){ + int rc = SQLITE_OK; /* Return code */ + int szPage; /* Database page-size */ + WalIterator *pIter = 0; /* Wal iterator context */ + u32 iDbpage = 0; /* Next database page to write */ + u32 iFrame = 0; /* Wal frame containing data for iDbpage */ + u32 mxSafeFrame; /* Max frame that can be backfilled */ + u32 mxPage; /* Max database page to write */ + int i; /* Loop counter */ + volatile WalCkptInfo *pInfo; /* The checkpoint status information */ + + szPage = walPagesize(pWal); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + pInfo = walCkptInfo(pWal); + if( pInfo->nBackfillhdr.mxFrame ){ + + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + + /* Compute in mxSafeFrame the index of the last frame of the WAL that is + ** safe to write into the database. Frames beyond mxSafeFrame might + ** overwrite database pages that are in use by active readers and thus + ** cannot be backfilled from the WAL. + */ + mxSafeFrame = pWal->hdr.mxFrame; + mxPage = pWal->hdr.nPage; + for(i=1; iaReadMark+i); + if( mxSafeFrame>y ){ + assert( y<=pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); + if( rc==SQLITE_OK ){ + u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + AtomicStore(pInfo->aReadMark+i, iMark); + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc==SQLITE_BUSY ){ + mxSafeFrame = y; + xBusy = 0; + }else{ + goto walcheckpoint_out; + } + } + } + + /* Allocate the iterator */ + if( pInfo->nBackfillnBackfill, &pIter); + assert( rc==SQLITE_OK || pIter==0 ); + } + + if( pIter + && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK + ){ + u32 nBackfill = pInfo->nBackfill; + + pInfo->nBackfillAttempted = mxSafeFrame; + + /* Sync the WAL to disk */ + rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); + + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ + if( rc==SQLITE_OK ){ + i64 nReq = ((i64)mxPage * szPage); + i64 nSize; /* Current size of database file */ + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizehdr.mxFrame*szPage)pDbFd, SQLITE_FCNTL_SIZE_HINT,&nReq); + } + } + + } + + /* Iterate through the contents of the WAL, copying data to the db file */ + while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ + i64 iOffset; + assert( walFramePgno(pWal, iFrame)==iDbpage ); + if( AtomicLoad(&db->u1.isInterrupted) ){ + rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; + break; + } + if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){ + continue; + } + iOffset = walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ + rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + iOffset = (iDbpage-1)*(i64)szPage; + testcase( IS_BIG_INT(iOffset) ); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + if( rc!=SQLITE_OK ) break; + } + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0); + + /* If work was actually accomplished... */ + if( rc==SQLITE_OK ){ + if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ + i64 szDb = pWal->hdr.nPage*(i64)szPage; + testcase( IS_BIG_INT(szDb) ); + rc = sqlite3OsTruncate(pWal->pDbFd, szDb); + if( rc==SQLITE_OK ){ + rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); + } + } + if( rc==SQLITE_OK ){ + AtomicStore(&pInfo->nBackfill, mxSafeFrame); + } + } + + /* Release the reader lock held while backfilling */ + walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1); + } + + if( rc==SQLITE_BUSY ){ + /* Reset the return code so as not to report a checkpoint failure + ** just because there are active readers. */ + rc = SQLITE_OK; + } + } + + /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the + ** entire wal file has been copied into the database file, then block + ** until all readers have finished using the wal file. This ensures that + ** the next process to write to the database restarts the wal file. + */ + if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + assert( pWal->writeLock ); + if( pInfo->nBackfillhdr.mxFrame ){ + rc = SQLITE_BUSY; + }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ + u32 salt1; + sqlite3_randomness(4, &salt1); + assert( pInfo->nBackfill==pWal->hdr.mxFrame ); + rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc==SQLITE_OK ){ + if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ + /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as + ** SQLITE_CHECKPOINT_RESTART with the addition that it also + ** truncates the log file to zero bytes just prior to a + ** successful return. + ** + ** In theory, it might be safe to do this without updating the + ** wal-index header in shared memory, as all subsequent reader or + ** writer clients should see that the entire log file has been + ** checkpointed and behave accordingly. This seems unsafe though, + ** as it would leave the system in a state where the contents of + ** the wal-index header do not match the contents of the + ** file-system. To avoid this, update the wal-index header to + ** indicate that the log file contains zero valid frames. */ + walRestartHdr(pWal, salt1); + rc = sqlite3OsTruncate(pWal->pWalFd, 0); + } + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + } + } + } + + walcheckpoint_out: + walIteratorFree(pIter); + return rc; +} + +/* +** If the WAL file is currently larger than nMax bytes in size, truncate +** it to exactly nMax bytes. If an error occurs while doing so, ignore it. +*/ +static void walLimitSize(Wal *pWal, i64 nMax){ + i64 sz; + int rx; + sqlite3BeginBenignMalloc(); + rx = sqlite3OsFileSize(pWal->pWalFd, &sz); + if( rx==SQLITE_OK && (sz > nMax ) ){ + rx = sqlite3OsTruncate(pWal->pWalFd, nMax); + } + sqlite3EndBenignMalloc(); + if( rx ){ + sqlite3_log(rx, "cannot limit WAL size: %s", pWal->zWalName); + } +} + +/* +** Close a connection to a log file. +*/ +SQLITE_PRIVATE int sqlite3WalClose( + Wal *pWal, /* Wal to close */ + sqlite3 *db, /* For interrupt flag */ + int sync_flags, /* Flags to pass to OsSync() (or 0) */ + int nBuf, + u8 *zBuf /* Buffer of at least nBuf bytes */ +){ + int rc = SQLITE_OK; + if( pWal ){ + int isDelete = 0; /* True to unlink wal and wal-index files */ + + /* If an EXCLUSIVE lock can be obtained on the database file (using the + ** ordinary, rollback-mode locking methods, this guarantees that the + ** connection associated with this log file is the only connection to + ** the database. In this case checkpoint the database and unlink both + ** the wal and wal-index files. + ** + ** The EXCLUSIVE lock is not released before returning. + */ + if( zBuf!=0 + && SQLITE_OK==(rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE)) + ){ + if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; + } + rc = sqlite3WalCheckpoint(pWal, db, + SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 + ); + if( rc==SQLITE_OK ){ + int bPersist = -1; + sqlite3OsFileControlHint( + pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersist + ); + if( bPersist!=1 ){ + /* Try to delete the WAL file if the checkpoint completed and + ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal + ** mode (!bPersist) */ + isDelete = 1; + }else if( pWal->mxWalSize>=0 ){ + /* Try to truncate the WAL file to zero bytes if the checkpoint + ** completed and fsynced (rc==SQLITE_OK) and we are in persistent + ** WAL mode (bPersist) and if the PRAGMA journal_size_limit is a + ** non-negative value (pWal->mxWalSize>=0). Note that we truncate + ** to zero bytes as truncating to the journal_size_limit might + ** leave a corrupt WAL file on disk. */ + walLimitSize(pWal, 0); + } + } + } + + walIndexClose(pWal, isDelete); + sqlite3OsClose(pWal->pWalFd); + if( isDelete ){ + sqlite3BeginBenignMalloc(); + sqlite3OsDelete(pWal->pVfs, pWal->zWalName, 0); + sqlite3EndBenignMalloc(); + } + WALTRACE(("WAL%p: closed\n", pWal)); + sqlite3_free((void *)pWal->apWiData); + sqlite3_free(pWal); + } + return rc; +} + +/* +** Try to read the wal-index header. Return 0 on success and 1 if +** there is a problem. +** +** The wal-index is in shared memory. Another thread or process might +** be writing the header at the same time this procedure is trying to +** read it, which might result in inconsistency. A dirty read is detected +** by verifying that both copies of the header are the same and also by +** a checksum on the header. +** +** If and only if the read is consistent and the header is different from +** pWal->hdr, then pWal->hdr is updated to the content of the new header +** and *pChanged is set to 1. +** +** If the checksum cannot be verified return non-zero. If the header +** is read successfully and the checksum verified, return zero. +*/ +static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){ + u32 aCksum[2]; /* Checksum on the header content */ + WalIndexHdr h1, h2; /* Two copies of the header content */ + WalIndexHdr volatile *aHdr; /* Header in shared memory */ + + /* The first page of the wal-index must be mapped at this point. */ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + + /* Read the header. This might happen concurrently with a write to the + ** same area of shared memory on a different CPU in a SMP, + ** meaning it is possible that an inconsistent snapshot is read + ** from the file. If this happens, return non-zero. + ** + ** tag-20200519-1: + ** There are two copies of the header at the beginning of the wal-index. + ** When reading, read [0] first then [1]. Writes are in the reverse order. + ** Memory barriers are used to prevent the compiler or the hardware from + ** reordering the reads and writes. TSAN and similar tools can sometimes + ** give false-positive warnings about these accesses because the tools do not + ** account for the double-read and the memory barrier. The use of mutexes + ** here would be problematic as the memory being accessed is potentially + ** shared among multiple processes and not all mutex implementions work + ** reliably in that environment. + */ + aHdr = walIndexHdr(pWal); + memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); /* Possible TSAN false-positive */ + walShmBarrier(pWal); + memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); + + if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ + return 1; /* Dirty read */ + } + if( h1.isInit==0 ){ + return 1; /* Malformed header - probably all zeros */ + } + walChecksumBytes(1, (u8*)&h1, sizeof(h1)-sizeof(h1.aCksum), 0, aCksum); + if( aCksum[0]!=h1.aCksum[0] || aCksum[1]!=h1.aCksum[1] ){ + return 1; /* Checksum does not match */ + } + + if( memcmp(&pWal->hdr, &h1, sizeof(WalIndexHdr)) ){ + *pChanged = 1; + memcpy(&pWal->hdr, &h1, sizeof(WalIndexHdr)); + pWal->szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); + testcase( pWal->szPage<=32768 ); + testcase( pWal->szPage>=65536 ); + } + + /* The header was successfully read. Return zero. */ + return 0; +} + +/* +** This is the value that walTryBeginRead returns when it needs to +** be retried. +*/ +#define WAL_RETRY (-1) + +/* +** Read the wal-index header from the wal-index and into pWal->hdr. +** If the wal-header appears to be corrupt, try to reconstruct the +** wal-index from the WAL before returning. +** +** Set *pChanged to 1 if the wal-index header value in pWal->hdr is +** changed by this operation. If pWal->hdr is unchanged, set *pChanged +** to 0. +** +** If the wal-index header is successfully read, return SQLITE_OK. +** Otherwise an SQLite error code. +*/ +static int walIndexReadHdr(Wal *pWal, int *pChanged){ + int rc; /* Return code */ + int badHdr; /* True if a header read failed */ + volatile u32 *page0; /* Chunk of wal-index containing header */ + + /* Ensure that page 0 of the wal-index (the page that contains the + ** wal-index header) is mapped. Return early if an error occurs here. + */ + assert( pChanged ); + rc = walIndexPage(pWal, 0, &page0); + if( rc!=SQLITE_OK ){ + assert( rc!=SQLITE_READONLY ); /* READONLY changed to OK in walIndexPage */ + if( rc==SQLITE_READONLY_CANTINIT ){ + /* The SQLITE_READONLY_CANTINIT return means that the shared-memory + ** was openable but is not writable, and this thread is unable to + ** confirm that another write-capable connection has the shared-memory + ** open, and hence the content of the shared-memory is unreliable, + ** since the shared-memory might be inconsistent with the WAL file + ** and there is no writer on hand to fix it. */ + assert( page0==0 ); + assert( pWal->writeLock==0 ); + assert( pWal->readOnly & WAL_SHM_RDONLY ); + pWal->bShmUnreliable = 1; + pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; + *pChanged = 1; + }else{ + return rc; /* Any other non-OK return is just an error */ + } + }else{ + /* page0 can be NULL if the SHM is zero bytes in size and pWal->writeLock + ** is zero, which prevents the SHM from growing */ + testcase( page0!=0 ); + } + assert( page0!=0 || pWal->writeLock==0 ); + + /* If the first page of the wal-index has been mapped, try to read the + ** wal-index header immediately, without holding any lock. This usually + ** works, but may fail if the wal-index header is corrupt or currently + ** being modified by another thread or process. + */ + badHdr = (page0 ? walIndexTryHdr(pWal, pChanged) : 1); + + /* If the first attempt failed, it might have been due to a race + ** with a writer. So get a WRITE lock and try again. + */ + if( badHdr ){ + if( pWal->bShmUnreliable==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ + if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ + walUnlockShared(pWal, WAL_WRITE_LOCK); + rc = SQLITE_READONLY_RECOVERY; + } + }else{ + int bWriteLock = pWal->writeLock; + if( bWriteLock || SQLITE_OK==(rc = walLockWriter(pWal)) ){ + pWal->writeLock = 1; + if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ + badHdr = walIndexTryHdr(pWal, pChanged); + if( badHdr ){ + /* If the wal-index header is still malformed even while holding + ** a WRITE lock, it can only mean that the header is corrupted and + ** needs to be reconstructed. So run recovery to do exactly that. + */ + rc = walIndexRecover(pWal); + *pChanged = 1; + } + } + if( bWriteLock==0 ){ + pWal->writeLock = 0; + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + } + } + } + } + + /* If the header is read successfully, check the version number to make + ** sure the wal-index was not constructed with some future format that + ** this version of SQLite cannot understand. + */ + if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){ + rc = SQLITE_CANTOPEN_BKPT; + } + if( pWal->bShmUnreliable ){ + if( rc!=SQLITE_OK ){ + walIndexClose(pWal, 0); + pWal->bShmUnreliable = 0; + assert( pWal->nWiData>0 && pWal->apWiData[0]==0 ); + /* walIndexRecover() might have returned SHORT_READ if a concurrent + ** writer truncated the WAL out from under it. If that happens, it + ** indicates that a writer has fixed the SHM file for us, so retry */ + if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY; + } + pWal->exclusiveMode = WAL_NORMAL_MODE; + } + + return rc; +} + +/* +** Open a transaction in a connection where the shared-memory is read-only +** and where we cannot verify that there is a separate write-capable connection +** on hand to keep the shared-memory up-to-date with the WAL file. +** +** This can happen, for example, when the shared-memory is implemented by +** memory-mapping a *-shm file, where a prior writer has shut down and +** left the *-shm file on disk, and now the present connection is trying +** to use that database but lacks write permission on the *-shm file. +** Other scenarios are also possible, depending on the VFS implementation. +** +** Precondition: +** +** The *-wal file has been read and an appropriate wal-index has been +** constructed in pWal->apWiData[] using heap memory instead of shared +** memory. +** +** If this function returns SQLITE_OK, then the read transaction has +** been successfully opened. In this case output variable (*pChanged) +** is set to true before returning if the caller should discard the +** contents of the page cache before proceeding. Or, if it returns +** WAL_RETRY, then the heap memory wal-index has been discarded and +** the caller should retry opening the read transaction from the +** beginning (including attempting to map the *-shm file). +** +** If an error occurs, an SQLite error code is returned. +*/ +static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ + i64 szWal; /* Size of wal file on disk in bytes */ + i64 iOffset; /* Current offset when reading wal file */ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + volatile void *pDummy; /* Dummy argument for xShmMap */ + int rc; /* Return code */ + u32 aSaveCksum[2]; /* Saved copy of pWal->hdr.aFrameCksum */ + + assert( pWal->bShmUnreliable ); + assert( pWal->readOnly & WAL_SHM_RDONLY ); + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + + /* Take WAL_READ_LOCK(0). This has the effect of preventing any + ** writers from running a checkpoint, but does not stop them + ** from running recovery. */ + rc = walLockShared(pWal, WAL_READ_LOCK(0)); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_BUSY ) rc = WAL_RETRY; + goto begin_unreliable_shm_out; + } + pWal->readLock = 0; + + /* Check to see if a separate writer has attached to the shared-memory area, + ** thus making the shared-memory "reliable" again. Do this by invoking + ** the xShmMap() routine of the VFS and looking to see if the return + ** is SQLITE_READONLY instead of SQLITE_READONLY_CANTINIT. + ** + ** If the shared-memory is now "reliable" return WAL_RETRY, which will + ** cause the heap-memory WAL-index to be discarded and the actual + ** shared memory to be used in its place. + ** + ** This step is important because, even though this connection is holding + ** the WAL_READ_LOCK(0) which prevents a checkpoint, a writer might + ** have already checkpointed the WAL file and, while the current + ** is active, wrap the WAL and start overwriting frames that this + ** process wants to use. + ** + ** Once sqlite3OsShmMap() has been called for an sqlite3_file and has + ** returned any SQLITE_READONLY value, it must return only SQLITE_READONLY + ** or SQLITE_READONLY_CANTINIT or some error for all subsequent invocations, + ** even if some external agent does a "chmod" to make the shared-memory + ** writable by us, until sqlite3OsShmUnmap() has been called. + ** This is a requirement on the VFS implementation. + */ + rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); + assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */ + if( rc!=SQLITE_READONLY_CANTINIT ){ + rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc); + goto begin_unreliable_shm_out; + } + + /* We reach this point only if the real shared-memory is still unreliable. + ** Assume the in-memory WAL-index substitute is correct and load it + ** into pWal->hdr. + */ + memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + /* Make sure some writer hasn't come in and changed the WAL file out + ** from under us, then disconnected, while we were not looking. + */ + rc = sqlite3OsFileSize(pWal->pWalFd, &szWal); + if( rc!=SQLITE_OK ){ + goto begin_unreliable_shm_out; + } + if( szWalhdr.mxFrame==0 ? SQLITE_OK : WAL_RETRY); + goto begin_unreliable_shm_out; + } + + /* Check the salt keys at the start of the wal file still match. */ + rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + goto begin_unreliable_shm_out; + } + if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){ + /* Some writer has wrapped the WAL file while we were not looking. + ** Return WAL_RETRY which will cause the in-memory WAL-index to be + ** rebuilt. */ + rc = WAL_RETRY; + goto begin_unreliable_shm_out; + } + + /* Allocate a buffer to read frames into */ + szFrame = pWal->hdr.szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc64(szFrame); + if( aFrame==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto begin_unreliable_shm_out; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + + /* Check to see if a complete transaction has been appended to the + ** wal file since the heap-memory wal-index was created. If so, the + ** heap-memory wal-index is discarded and WAL_RETRY returned to + ** the caller. */ + aSaveCksum[0] = pWal->hdr.aFrameCksum[0]; + aSaveCksum[1] = pWal->hdr.aFrameCksum[1]; + for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->hdr.szPage); + iOffset+szFrame<=szWal; + iOffset+=szFrame + ){ + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + + /* Read and decode the next log frame. */ + rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break; + + /* If nTruncate is non-zero, then a complete transaction has been + ** appended to this wal file. Set rc to WAL_RETRY and break out of + ** the loop. */ + if( nTruncate ){ + rc = WAL_RETRY; + break; + } + } + pWal->hdr.aFrameCksum[0] = aSaveCksum[0]; + pWal->hdr.aFrameCksum[1] = aSaveCksum[1]; + + begin_unreliable_shm_out: + sqlite3_free(aFrame); + if( rc!=SQLITE_OK ){ + int i; + for(i=0; inWiData; i++){ + sqlite3_free((void*)pWal->apWiData[i]); + pWal->apWiData[i] = 0; + } + pWal->bShmUnreliable = 0; + sqlite3WalEndReadTransaction(pWal); + *pChanged = 1; + } + return rc; +} + +/* +** Attempt to start a read transaction. This might fail due to a race or +** other transient condition. When that happens, it returns WAL_RETRY to +** indicate to the caller that it is safe to retry immediately. +** +** On success return SQLITE_OK. On a permanent failure (such an +** I/O error or an SQLITE_BUSY because another process is running +** recovery) return a positive error code. +** +** The useWal parameter is true to force the use of the WAL and disable +** the case where the WAL is bypassed because it has been completely +** checkpointed. If useWal==0 then this routine calls walIndexReadHdr() +** to make a copy of the wal-index header into pWal->hdr. If the +** wal-index header has changed, *pChanged is set to 1 (as an indication +** to the caller that the local page cache is obsolete and needs to be +** flushed.) When useWal==1, the wal-index header is assumed to already +** be loaded and the pChanged parameter is unused. +** +** The caller must set the cnt parameter to the number of prior calls to +** this routine during the current read attempt that returned WAL_RETRY. +** This routine will start taking more aggressive measures to clear the +** race conditions after multiple WAL_RETRY returns, and after an excessive +** number of errors will ultimately return SQLITE_PROTOCOL. The +** SQLITE_PROTOCOL return indicates that some other process has gone rogue +** and is not honoring the locking protocol. There is a vanishingly small +** chance that SQLITE_PROTOCOL could be returned because of a run of really +** bad luck when there is lots of contention for the wal-index, but that +** possibility is so small that it can be safely neglected, we believe. +** +** On success, this routine obtains a read lock on +** WAL_READ_LOCK(pWal->readLock). The pWal->readLock integer is +** in the range 0 <= pWal->readLock < WAL_NREADER. If pWal->readLock==(-1) +** that means the Wal does not hold any read lock. The reader must not +** access any database page that is modified by a WAL frame up to and +** including frame number aReadMark[pWal->readLock]. The reader will +** use WAL frames up to and including pWal->hdr.mxFrame if pWal->readLock>0 +** Or if pWal->readLock==0, then the reader will ignore the WAL +** completely and get all content directly from the database file. +** If the useWal parameter is 1 then the WAL will never be ignored and +** this routine will always set pWal->readLock>0 on success. +** When the read transaction is completed, the caller must release the +** lock on WAL_READ_LOCK(pWal->readLock) and set pWal->readLock to -1. +** +** This routine uses the nBackfill and aReadMark[] fields of the header +** to select a particular WAL_READ_LOCK() that strives to let the +** checkpoint process do as much work as possible. This routine might +** update values of the aReadMark[] array in the header, but if it does +** so it takes care to hold an exclusive lock on the corresponding +** WAL_READ_LOCK() while changing values. +*/ +static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ + volatile WalCkptInfo *pInfo; /* Checkpoint information in wal-index */ + u32 mxReadMark; /* Largest aReadMark[] value */ + int mxI; /* Index of largest aReadMark[] value */ + int i; /* Loop counter */ + int rc = SQLITE_OK; /* Return code */ + u32 mxFrame; /* Wal frame to lock to */ + + assert( pWal->readLock<0 ); /* Not currently locked */ + + /* useWal may only be set for read/write connections */ + assert( (pWal->readOnly & WAL_SHM_RDONLY)==0 || useWal==0 ); + + /* Take steps to avoid spinning forever if there is a protocol error. + ** + ** Circumstances that cause a RETRY should only last for the briefest + ** instances of time. No I/O or other system calls are done while the + ** locks are held, so the locks should not be held for very long. But + ** if we are unlucky, another process that is holding a lock might get + ** paged out or take a page-fault that is time-consuming to resolve, + ** during the few nanoseconds that it is holding the lock. In that case, + ** it might take longer than normal for the lock to free. + ** + ** After 5 RETRYs, we begin calling sqlite3OsSleep(). The first few + ** calls to sqlite3OsSleep() have a delay of 1 microsecond. Really this + ** is more of a scheduler yield than an actual delay. But on the 10th + ** an subsequent retries, the delays start becoming longer and longer, + ** so that on the 100th (and last) RETRY we delay for 323 milliseconds. + ** The total delay time before giving up is less than 10 seconds. + */ + if( cnt>5 ){ + int nDelay = 1; /* Pause time in microseconds */ + if( cnt>100 ){ + VVA_ONLY( pWal->lockError = 1; ) + return SQLITE_PROTOCOL; + } + if( cnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39; + sqlite3OsSleep(pWal->pVfs, nDelay); + } + + if( !useWal ){ + assert( rc==SQLITE_OK ); + if( pWal->bShmUnreliable==0 ){ + rc = walIndexReadHdr(pWal, pChanged); + } + if( rc==SQLITE_BUSY ){ + /* If there is not a recovery running in another thread or process + ** then convert BUSY errors to WAL_RETRY. If recovery is known to + ** be running, convert BUSY to BUSY_RECOVERY. There is a race here + ** which might cause WAL_RETRY to be returned even if BUSY_RECOVERY + ** would be technically correct. But the race is benign since with + ** WAL_RETRY this routine will be called again and will probably be + ** right on the second iteration. + */ + if( pWal->apWiData[0]==0 ){ + /* This branch is taken when the xShmMap() method returns SQLITE_BUSY. + ** We assume this is a transient condition, so return WAL_RETRY. The + ** xShmMap() implementation used by the default unix and win32 VFS + ** modules may return SQLITE_BUSY due to a race condition in the + ** code that determines whether or not the shared-memory region + ** must be zeroed before the requested page is returned. + */ + rc = WAL_RETRY; + }else if( SQLITE_OK==(rc = walLockShared(pWal, WAL_RECOVER_LOCK)) ){ + walUnlockShared(pWal, WAL_RECOVER_LOCK); + rc = WAL_RETRY; + }else if( rc==SQLITE_BUSY ){ + rc = SQLITE_BUSY_RECOVERY; + } + } + if( rc!=SQLITE_OK ){ + return rc; + } + else if( pWal->bShmUnreliable ){ + return walBeginShmUnreliable(pWal, pChanged); + } + } + + assert( pWal->nWiData>0 ); + assert( pWal->apWiData[0]!=0 ); + pInfo = walCkptInfo(pWal); + if( !useWal && AtomicLoad(&pInfo->nBackfill)==pWal->hdr.mxFrame +#ifdef SQLITE_ENABLE_SNAPSHOT + && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0) +#endif + ){ + /* The WAL has been completely backfilled (or it is empty). + ** and can be safely ignored. + */ + rc = walLockShared(pWal, WAL_READ_LOCK(0)); + walShmBarrier(pWal); + if( rc==SQLITE_OK ){ + if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ + /* It is not safe to allow the reader to continue here if frames + ** may have been appended to the log before READ_LOCK(0) was obtained. + ** When holding READ_LOCK(0), the reader ignores the entire log file, + ** which implies that the database file contains a trustworthy + ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from + ** happening, this is usually correct. + ** + ** However, if frames have been appended to the log (or if the log + ** is wrapped and written for that matter) before the READ_LOCK(0) + ** is obtained, that is not necessarily true. A checkpointer may + ** have started to backfill the appended frames but crashed before + ** it finished. Leaving a corrupt image in the database file. + */ + walUnlockShared(pWal, WAL_READ_LOCK(0)); + return WAL_RETRY; + } + pWal->readLock = 0; + return SQLITE_OK; + }else if( rc!=SQLITE_BUSY ){ + return rc; + } + } + + /* If we get this far, it means that the reader will want to use + ** the WAL to get at content from recent commits. The job now is + ** to select one of the aReadMark[] entries that is closest to + ** but not exceeding pWal->hdr.mxFrame and lock that entry. + */ + mxReadMark = 0; + mxI = 0; + mxFrame = pWal->hdr.mxFrame; +#ifdef SQLITE_ENABLE_SNAPSHOT + if( pWal->pSnapshot && pWal->pSnapshot->mxFramepSnapshot->mxFrame; + } +#endif + for(i=1; iaReadMark+i); + if( mxReadMark<=thisMark && thisMark<=mxFrame ){ + assert( thisMark!=READMARK_NOT_USED ); + mxReadMark = thisMark; + mxI = i; + } + } + if( (pWal->readOnly & WAL_SHM_RDONLY)==0 + && (mxReadMarkaReadMark+i,mxFrame); + mxReadMark = mxFrame; + mxI = i; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + break; + }else if( rc!=SQLITE_BUSY ){ + return rc; + } + } + } + if( mxI==0 ){ + assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT; + } + + rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); + if( rc ){ + return rc==SQLITE_BUSY ? WAL_RETRY : rc; + } + /* Now that the read-lock has been obtained, check that neither the + ** value in the aReadMark[] array or the contents of the wal-index + ** header have changed. + ** + ** It is necessary to check that the wal-index header did not change + ** between the time it was read and when the shared-lock was obtained + ** on WAL_READ_LOCK(mxI) was obtained to account for the possibility + ** that the log file may have been wrapped by a writer, or that frames + ** that occur later in the log than pWal->hdr.mxFrame may have been + ** copied into the database by a checkpointer. If either of these things + ** happened, then reading the database with the current value of + ** pWal->hdr.mxFrame risks reading a corrupted snapshot. So, retry + ** instead. + ** + ** Before checking that the live wal-index header has not changed + ** since it was read, set Wal.minFrame to the first frame in the wal + ** file that has not yet been checkpointed. This client will not need + ** to read any frames earlier than minFrame from the wal file - they + ** can be safely read directly from the database file. + ** + ** Because a ShmBarrier() call is made between taking the copy of + ** nBackfill and checking that the wal-header in shared-memory still + ** matches the one cached in pWal->hdr, it is guaranteed that the + ** checkpointer that set nBackfill was not working with a wal-index + ** header newer than that cached in pWal->hdr. If it were, that could + ** cause a problem. The checkpointer could omit to checkpoint + ** a version of page X that lies before pWal->minFrame (call that version + ** A) on the basis that there is a newer version (version B) of the same + ** page later in the wal file. But if version B happens to like past + ** frame pWal->hdr.mxFrame - then the client would incorrectly assume + ** that it can read version A from the database file. However, since + ** we can guarantee that the checkpointer that set nBackfill could not + ** see any pages past pWal->hdr.mxFrame, this problem does not come up. + */ + pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; + walShmBarrier(pWal); + if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark + || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) + ){ + walUnlockShared(pWal, WAL_READ_LOCK(mxI)); + return WAL_RETRY; + }else{ + assert( mxReadMark<=pWal->hdr.mxFrame ); + pWal->readLock = (i16)mxI; + } + return rc; +} + +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** Attempt to reduce the value of the WalCkptInfo.nBackfillAttempted +** variable so that older snapshots can be accessed. To do this, loop +** through all wal frames from nBackfillAttempted to (nBackfill+1), +** comparing their content to the corresponding page with the database +** file, if any. Set nBackfillAttempted to the frame number of the +** first frame for which the wal file content matches the db file. +** +** This is only really safe if the file-system is such that any page +** writes made by earlier checkpointers were atomic operations, which +** is not always true. It is also possible that nBackfillAttempted +** may be left set to a value larger than expected, if a wal frame +** contains content that duplicate of an earlier version of the same +** page. +** +** SQLITE_OK is returned if successful, or an SQLite error code if an +** error occurs. It is not an error if nBackfillAttempted cannot be +** decreased at all. +*/ +SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){ + int rc; + + assert( pWal->readLock>=0 ); + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + if( rc==SQLITE_OK ){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + int szPage = (int)pWal->szPage; + i64 szDb; /* Size of db file in bytes */ + + rc = sqlite3OsFileSize(pWal->pDbFd, &szDb); + if( rc==SQLITE_OK ){ + void *pBuf1 = sqlite3_malloc(szPage); + void *pBuf2 = sqlite3_malloc(szPage); + if( pBuf1==0 || pBuf2==0 ){ + rc = SQLITE_NOMEM; + }else{ + u32 i = pInfo->nBackfillAttempted; + for(i=pInfo->nBackfillAttempted; i>AtomicLoad(&pInfo->nBackfill); i--){ + WalHashLoc sLoc; /* Hash table location */ + u32 pgno; /* Page number in db file */ + i64 iDbOff; /* Offset of db file entry */ + i64 iWalOff; /* Offset of wal file entry */ + + rc = walHashGet(pWal, walFramePage(i), &sLoc); + if( rc!=SQLITE_OK ) break; + pgno = sLoc.aPgno[i-sLoc.iZero]; + iDbOff = (i64)(pgno-1) * szPage; + + if( iDbOff+szPage<=szDb ){ + iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE; + rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff); + + if( rc==SQLITE_OK ){ + rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff); + } + + if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){ + break; + } + } + + pInfo->nBackfillAttempted = i-1; + } + } + + sqlite3_free(pBuf1); + sqlite3_free(pBuf2); + } + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); + } + + return rc; +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + +/* +** Begin a read transaction on the database. +** +** This routine used to be called sqlite3OpenSnapshot() and with good reason: +** it takes a snapshot of the state of the WAL and wal-index for the current +** instant in time. The current thread will continue to use this snapshot. +** Other threads might append new content to the WAL and wal-index but +** that extra content is ignored by the current thread. +** +** If the database contents have changes since the previous read +** transaction, then *pChanged is set to 1 before returning. The +** Pager layer will use this to know that its cache is stale and +** needs to be flushed. +*/ +SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ + int rc; /* Return code */ + int cnt = 0; /* Number of TryBeginRead attempts */ +#ifdef SQLITE_ENABLE_SNAPSHOT + int bChanged = 0; + WalIndexHdr *pSnapshot = pWal->pSnapshot; +#endif + + assert( pWal->ckptLock==0 ); + +#ifdef SQLITE_ENABLE_SNAPSHOT + if( pSnapshot ){ + if( memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ + bChanged = 1; + } + + /* It is possible that there is a checkpointer thread running + ** concurrent with this code. If this is the case, it may be that the + ** checkpointer has already determined that it will checkpoint + ** snapshot X, where X is later in the wal file than pSnapshot, but + ** has not yet set the pInfo->nBackfillAttempted variable to indicate + ** its intent. To avoid the race condition this leads to, ensure that + ** there is no checkpointer process by taking a shared CKPT lock + ** before checking pInfo->nBackfillAttempted. */ + (void)walEnableBlocking(pWal); + rc = walLockShared(pWal, WAL_CKPT_LOCK); + walDisableBlocking(pWal); + + if( rc!=SQLITE_OK ){ + return rc; + } + pWal->ckptLock = 1; + } +#endif + + do{ + rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); + }while( rc==WAL_RETRY ); + testcase( (rc&0xff)==SQLITE_BUSY ); + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); + +#ifdef SQLITE_ENABLE_SNAPSHOT + if( rc==SQLITE_OK ){ + if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ + /* At this point the client has a lock on an aReadMark[] slot holding + ** a value equal to or smaller than pSnapshot->mxFrame, but pWal->hdr + ** is populated with the wal-index header corresponding to the head + ** of the wal file. Verify that pSnapshot is still valid before + ** continuing. Reasons why pSnapshot might no longer be valid: + ** + ** (1) The WAL file has been reset since the snapshot was taken. + ** In this case, the salt will have changed. + ** + ** (2) A checkpoint as been attempted that wrote frames past + ** pSnapshot->mxFrame into the database file. Note that the + ** checkpoint need not have completed for this to cause problems. + */ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + + assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 ); + assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame ); + + /* Check that the wal file has not been wrapped. Assuming that it has + ** not, also check that no checkpointer has attempted to checkpoint any + ** frames beyond pSnapshot->mxFrame. If either of these conditions are + ** true, return SQLITE_ERROR_SNAPSHOT. Otherwise, overwrite pWal->hdr + ** with *pSnapshot and set *pChanged as appropriate for opening the + ** snapshot. */ + if( !memcmp(pSnapshot->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) + && pSnapshot->mxFrame>=pInfo->nBackfillAttempted + ){ + assert( pWal->readLock>0 ); + memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr)); + *pChanged = bChanged; + }else{ + rc = SQLITE_ERROR_SNAPSHOT; + } + + /* A client using a non-current snapshot may not ignore any frames + ** from the start of the wal file. This is because, for a system + ** where (minFrame < iSnapshot < maxFrame), a checkpointer may + ** have omitted to checkpoint a frame earlier than minFrame in + ** the file because there exists a frame after iSnapshot that + ** is the same database page. */ + pWal->minFrame = 1; + + if( rc!=SQLITE_OK ){ + sqlite3WalEndReadTransaction(pWal); + } + } + } + + /* Release the shared CKPT lock obtained above. */ + if( pWal->ckptLock ){ + assert( pSnapshot ); + walUnlockShared(pWal, WAL_CKPT_LOCK); + pWal->ckptLock = 0; + } +#endif + return rc; +} + +/* +** Finish with a read transaction. All this does is release the +** read-lock. +*/ +SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ + sqlite3WalEndWriteTransaction(pWal); + if( pWal->readLock>=0 ){ + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + pWal->readLock = -1; + } +} + +/* +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. +** +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. +*/ +SQLITE_PRIVATE int sqlite3WalFindFrame( + Wal *pWal, /* WAL handle */ + Pgno pgno, /* Database page number to read data for */ + u32 *piRead /* OUT: Frame number (or zero) */ +){ + u32 iRead = 0; /* If !=0, WAL frame to return data from */ + u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ + int iHash; /* Used to loop through N hash tables */ + int iMinHash; + + /* This routine is only be called from within a read transaction. */ + assert( pWal->readLock>=0 || pWal->lockError ); + + /* If the "last page" field of the wal-index header snapshot is 0, then + ** no data will be read from the wal under any circumstances. Return early + ** in this case as an optimization. Likewise, if pWal->readLock==0, + ** then the WAL is ignored by the reader so return early, as if the + ** WAL were empty. + */ + if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ + *piRead = 0; + return SQLITE_OK; + } + + /* Search the hash table or tables for an entry matching page number + ** pgno. Each iteration of the following for() loop searches one + ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames). + ** + ** This code might run concurrently to the code in walIndexAppend() + ** that adds entries to the wal-index (and possibly to this hash + ** table). This means the value just read from the hash + ** slot (aHash[iKey]) may have been added before or after the + ** current read transaction was opened. Values added after the + ** read transaction was opened may have been written incorrectly - + ** i.e. these slots may contain garbage data. However, we assume + ** that any slots written before the current read transaction was + ** opened remain unmodified. + ** + ** For the reasons above, the if(...) condition featured in the inner + ** loop of the following block is more stringent that would be required + ** if we had exclusive access to the hash-table: + ** + ** (aPgno[iFrame]==pgno): + ** This condition filters out normal hash-table collisions. + ** + ** (iFrame<=iLast): + ** This condition filters out entries that were added to the hash + ** table after the current read-transaction had started. + */ + iMinHash = walFramePage(pWal->minFrame); + for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){ + WalHashLoc sLoc; /* Hash table location */ + int iKey; /* Hash slot index */ + int nCollide; /* Number of hash collisions remaining */ + int rc; /* Error code */ + u32 iH; + + rc = walHashGet(pWal, iHash, &sLoc); + if( rc!=SQLITE_OK ){ + return rc; + } + nCollide = HASHTABLE_NSLOT; + iKey = walHash(pgno); + while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){ + u32 iFrame = iH + sLoc.iZero; + if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH]==pgno ){ + assert( iFrame>iRead || CORRUPT_DB ); + iRead = iFrame; + } + if( (nCollide--)==0 ){ + return SQLITE_CORRUPT_BKPT; + } + iKey = walNextHash(iKey); + } + if( iRead ) break; + } + +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + /* If expensive assert() statements are available, do a linear search + ** of the wal-index file content. Make sure the results agree with the + ** result obtained using the hash indexes above. */ + { + u32 iRead2 = 0; + u32 iTest; + assert( pWal->bShmUnreliable || pWal->minFrame>0 ); + for(iTest=iLast; iTest>=pWal->minFrame && iTest>0; iTest--){ + if( walFramePgno(pWal, iTest)==pgno ){ + iRead2 = iTest; + break; + } + } + assert( iRead==iRead2 ); + } +#endif + + *piRead = iRead; + return SQLITE_OK; +} + +/* +** Read the contents of frame iRead from the wal file into buffer pOut +** (which is nOut bytes in size). Return SQLITE_OK if successful, or an +** error code otherwise. +*/ +SQLITE_PRIVATE int sqlite3WalReadFrame( + Wal *pWal, /* WAL handle */ + u32 iRead, /* Frame to read */ + int nOut, /* Size of buffer pOut in bytes */ + u8 *pOut /* Buffer to write page data to */ +){ + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); +} + +/* +** Return the size of the database in pages (or zero, if unknown). +*/ +SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal){ + if( pWal && ALWAYS(pWal->readLock>=0) ){ + return pWal->hdr.nPage; + } + return 0; +} + + +/* +** This function starts a write transaction on the WAL. +** +** A read transaction must have already been started by a prior call +** to sqlite3WalBeginReadTransaction(). +** +** If another thread or process has written into the database since +** the read transaction was started, then it is not possible for this +** thread to write as doing so would cause a fork. So this routine +** returns SQLITE_BUSY in that case and no write transaction is started. +** +** There can only be a single writer active at a time. +*/ +SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ + int rc; + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* If the write-lock is already held, then it was obtained before the + ** read-transaction was even opened, making this call a no-op. + ** Return early. */ + if( pWal->writeLock ){ + assert( !memcmp(&pWal->hdr,(void *)walIndexHdr(pWal),sizeof(WalIndexHdr)) ); + return SQLITE_OK; + } +#endif + + /* Cannot start a write transaction without first holding a read + ** transaction. */ + assert( pWal->readLock>=0 ); + assert( pWal->writeLock==0 && pWal->iReCksum==0 ); + + if( pWal->readOnly ){ + return SQLITE_READONLY; + } + + /* Only one writer allowed at a time. Get the write lock. Return + ** SQLITE_BUSY if unable. + */ + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); + if( rc ){ + return rc; + } + pWal->writeLock = 1; + + /* If another connection has written to the database file since the + ** time the read transaction on this connection was started, then + ** the write is disallowed. + */ + if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + rc = SQLITE_BUSY_SNAPSHOT; + } + + return rc; +} + +/* +** End a write transaction. The commit has already been done. This +** routine merely releases the lock. +*/ +SQLITE_PRIVATE int sqlite3WalEndWriteTransaction(Wal *pWal){ + if( pWal->writeLock ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + pWal->iReCksum = 0; + pWal->truncateOnCommit = 0; + } + return SQLITE_OK; +} + +/* +** If any data has been written (but not committed) to the log file, this +** function moves the write-pointer back to the start of the transaction. +** +** Additionally, the callback function is invoked for each frame written +** to the WAL since the start of the transaction. If the callback returns +** other than SQLITE_OK, it is not invoked again and the error code is +** returned to the caller. +** +** Otherwise, if the callback function does not return an error, this +** function returns SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ + int rc = SQLITE_OK; + if( ALWAYS(pWal->writeLock) ){ + Pgno iMax = pWal->hdr.mxFrame; + Pgno iFrame; + + /* Restore the clients cache of the wal-index header to the state it + ** was in before the client began writing to the database. + */ + memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + for(iFrame=pWal->hdr.mxFrame+1; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + iFrame++ + ){ + /* This call cannot fail. Unless the page for which the page number + ** is passed as the second argument is (a) in the cache and + ** (b) has an outstanding reference, then xUndo is either a no-op + ** (if (a) is false) or simply expels the page from the cache (if (b) + ** is false). + ** + ** If the upper layer is doing a rollback, it is guaranteed that there + ** are no outstanding references to any page other than page 1. And + ** page 1 is never written to the log until the transaction is + ** committed. As a result, the call to xUndo may not fail. + */ + assert( walFramePgno(pWal, iFrame)!=1 ); + rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); + } + if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); + } + return rc; +} + +/* +** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32 +** values. This function populates the array with values required to +** "rollback" the write position of the WAL handle back to the current +** point in the event of a savepoint rollback (via WalSavepointUndo()). +*/ +SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ + assert( pWal->writeLock ); + aWalData[0] = pWal->hdr.mxFrame; + aWalData[1] = pWal->hdr.aFrameCksum[0]; + aWalData[2] = pWal->hdr.aFrameCksum[1]; + aWalData[3] = pWal->nCkpt; +} + +/* +** Move the write position of the WAL back to the point identified by +** the values in the aWalData[] array. aWalData must point to an array +** of WAL_SAVEPOINT_NDATA u32 values that has been previously populated +** by a call to WalSavepoint(). +*/ +SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ + int rc = SQLITE_OK; + + assert( pWal->writeLock ); + assert( aWalData[3]!=pWal->nCkpt || aWalData[0]<=pWal->hdr.mxFrame ); + + if( aWalData[3]!=pWal->nCkpt ){ + /* This savepoint was opened immediately after the write-transaction + ** was started. Right after that, the writer decided to wrap around + ** to the start of the log. Update the savepoint values to match. + */ + aWalData[0] = 0; + aWalData[3] = pWal->nCkpt; + } + + if( aWalData[0]hdr.mxFrame ){ + pWal->hdr.mxFrame = aWalData[0]; + pWal->hdr.aFrameCksum[0] = aWalData[1]; + pWal->hdr.aFrameCksum[1] = aWalData[2]; + walCleanupHash(pWal); + } + + return rc; +} + +/* +** This function is called just before writing a set of frames to the log +** file (see sqlite3WalFrames()). It checks to see if, instead of appending +** to the current log file, it is possible to overwrite the start of the +** existing log file with the new frames (i.e. "reset" the log). If so, +** it sets pWal->hdr.mxFrame to 0. Otherwise, pWal->hdr.mxFrame is left +** unchanged. +** +** SQLITE_OK is returned if no error is encountered (regardless of whether +** or not pWal->hdr.mxFrame is modified). An SQLite error code is returned +** if an error occurs. +*/ +static int walRestartLog(Wal *pWal){ + int rc = SQLITE_OK; + int cnt; + + if( pWal->readLock==0 ){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + assert( pInfo->nBackfill==pWal->hdr.mxFrame ); + if( pInfo->nBackfill>0 ){ + u32 salt1; + sqlite3_randomness(4, &salt1); + rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc==SQLITE_OK ){ + /* If all readers are using WAL_READ_LOCK(0) (in other words if no + ** readers are currently using the WAL), then the transactions + ** frames will overwrite the start of the existing log. Update the + ** wal-index header to reflect this. + ** + ** In theory it would be Ok to update the cache of the header only + ** at this point. But updating the actual wal-index header is also + ** safe and means there is no special case for sqlite3WalUndo() + ** to handle if this transaction is rolled back. */ + walRestartHdr(pWal, salt1); + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + }else if( rc!=SQLITE_BUSY ){ + return rc; + } + } + walUnlockShared(pWal, WAL_READ_LOCK(0)); + pWal->readLock = -1; + cnt = 0; + do{ + int notUsed; + rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); + }while( rc==WAL_RETRY ); + assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ + testcase( (rc&0xff)==SQLITE_IOERR ); + testcase( rc==SQLITE_PROTOCOL ); + testcase( rc==SQLITE_OK ); + } + return rc; +} + +/* +** Information about the current state of the WAL file and where +** the next fsync should occur - passed from sqlite3WalFrames() into +** walWriteToLog(). +*/ +typedef struct WalWriter { + Wal *pWal; /* The complete WAL information */ + sqlite3_file *pFd; /* The WAL file to which we write */ + sqlite3_int64 iSyncPoint; /* Fsync at this offset */ + int syncFlags; /* Flags for the fsync */ + int szPage; /* Size of one page */ +} WalWriter; + +/* +** Write iAmt bytes of content into the WAL file beginning at iOffset. +** Do a sync when crossing the p->iSyncPoint boundary. +** +** In other words, if iSyncPoint is in between iOffset and iOffset+iAmt, +** first write the part before iSyncPoint, then sync, then write the +** rest. +*/ +static int walWriteToLog( + WalWriter *p, /* WAL to write to */ + void *pContent, /* Content to be written */ + int iAmt, /* Number of bytes to write */ + sqlite3_int64 iOffset /* Start writing at this offset */ +){ + int rc; + if( iOffsetiSyncPoint && iOffset+iAmt>=p->iSyncPoint ){ + int iFirstAmt = (int)(p->iSyncPoint - iOffset); + rc = sqlite3OsWrite(p->pFd, pContent, iFirstAmt, iOffset); + if( rc ) return rc; + iOffset += iFirstAmt; + iAmt -= iFirstAmt; + pContent = (void*)(iFirstAmt + (char*)pContent); + assert( WAL_SYNC_FLAGS(p->syncFlags)!=0 ); + rc = sqlite3OsSync(p->pFd, WAL_SYNC_FLAGS(p->syncFlags)); + if( iAmt==0 || rc ) return rc; + } + rc = sqlite3OsWrite(p->pFd, pContent, iAmt, iOffset); + return rc; +} + +/* +** Write out a single frame of the WAL +*/ +static int walWriteOneFrame( + WalWriter *p, /* Where to write the frame */ + PgHdr *pPage, /* The page of the frame to be written */ + int nTruncate, /* The commit flag. Usually 0. >0 for commit */ + sqlite3_int64 iOffset /* Byte offset at which to write */ +){ + int rc; /* Result code from subfunctions */ + void *pData; /* Data actually written */ + u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ + pData = pPage->pData; + walEncodeFrame(p->pWal, pPage->pgno, nTruncate, pData, aFrame); + rc = walWriteToLog(p, aFrame, sizeof(aFrame), iOffset); + if( rc ) return rc; + /* Write the page data */ + rc = walWriteToLog(p, pData, p->szPage, iOffset+sizeof(aFrame)); + return rc; +} + +/* +** This function is called as part of committing a transaction within which +** one or more frames have been overwritten. It updates the checksums for +** all frames written to the wal file by the current transaction starting +** with the earliest to have been overwritten. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int walRewriteChecksums(Wal *pWal, u32 iLast){ + const int szPage = pWal->szPage;/* Database page size */ + int rc = SQLITE_OK; /* Return code */ + u8 *aBuf; /* Buffer to load data from wal file into */ + u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-headers in */ + u32 iRead; /* Next frame to read from wal file */ + i64 iCksumOff; + + aBuf = sqlite3_malloc(szPage + WAL_FRAME_HDRSIZE); + if( aBuf==0 ) return SQLITE_NOMEM_BKPT; + + /* Find the checksum values to use as input for the recalculating the + ** first checksum. If the first frame is frame 1 (implying that the current + ** transaction restarted the wal file), these values must be read from the + ** wal-file header. Otherwise, read them from the frame header of the + ** previous frame. */ + assert( pWal->iReCksum>0 ); + if( pWal->iReCksum==1 ){ + iCksumOff = 24; + }else{ + iCksumOff = walFrameOffset(pWal->iReCksum-1, szPage) + 16; + } + rc = sqlite3OsRead(pWal->pWalFd, aBuf, sizeof(u32)*2, iCksumOff); + pWal->hdr.aFrameCksum[0] = sqlite3Get4byte(aBuf); + pWal->hdr.aFrameCksum[1] = sqlite3Get4byte(&aBuf[sizeof(u32)]); + + iRead = pWal->iReCksum; + pWal->iReCksum = 0; + for(; rc==SQLITE_OK && iRead<=iLast; iRead++){ + i64 iOff = walFrameOffset(iRead, szPage); + rc = sqlite3OsRead(pWal->pWalFd, aBuf, szPage+WAL_FRAME_HDRSIZE, iOff); + if( rc==SQLITE_OK ){ + u32 iPgno, nDbSize; + iPgno = sqlite3Get4byte(aBuf); + nDbSize = sqlite3Get4byte(&aBuf[4]); + + walEncodeFrame(pWal, iPgno, nDbSize, &aBuf[WAL_FRAME_HDRSIZE], aFrame); + rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOff); + } + } + + sqlite3_free(aBuf); + return rc; +} + +/* +** Write a set of frames to the log. The caller must hold the write-lock +** on the log file (obtained using sqlite3WalBeginWriteTransaction()). +*/ +SQLITE_PRIVATE int sqlite3WalFrames( + Wal *pWal, /* Wal handle to write to */ + int szPage, /* Database page-size in bytes */ + PgHdr *pList, /* List of dirty pages to write */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit, /* True if this is a commit */ + int sync_flags /* Flags to pass to OsSync() (or 0) */ +){ + int rc; /* Used to catch return codes */ + u32 iFrame; /* Next frame address */ + PgHdr *p; /* Iterator to run through pList with. */ + PgHdr *pLast = 0; /* Last frame in list */ + int nExtra = 0; /* Number of extra copies of last page */ + int szFrame; /* The size of a single frame */ + i64 iOffset; /* Next byte to write in WAL file */ + WalWriter w; /* The writer */ + u32 iFirst = 0; /* First frame that may be overwritten */ + WalIndexHdr *pLive; /* Pointer to shared header */ + + assert( pList ); + assert( pWal->writeLock ); + + /* If this frame set completes a transaction, then nTruncate>0. If + ** nTruncate==0 then this frame set does not complete the transaction. */ + assert( (isCommit!=0)==(nTruncate!=0) ); + +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) + { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){} + WALTRACE(("WAL%p: frame write begin. %d frames. mxFrame=%d. %s\n", + pWal, cnt, pWal->hdr.mxFrame, isCommit ? "Commit" : "Spill")); + } +#endif + + pLive = (WalIndexHdr*)walIndexHdr(pWal); + if( memcmp(&pWal->hdr, (void *)pLive, sizeof(WalIndexHdr))!=0 ){ + iFirst = pLive->mxFrame+1; + } + + /* See if it is possible to write these frames into the start of the + ** log file, instead of appending to it at pWal->hdr.mxFrame. + */ + if( SQLITE_OK!=(rc = walRestartLog(pWal)) ){ + return rc; + } + + /* If this is the first frame written into the log, write the WAL + ** header to the start of the WAL file. See comments at the top of + ** this source file for a description of the WAL header format. + */ + iFrame = pWal->hdr.mxFrame; + if( iFrame==0 ){ + u8 aWalHdr[WAL_HDRSIZE]; /* Buffer to assemble wal-header in */ + u32 aCksum[2]; /* Checksum for wal-header */ + + sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN)); + sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); + sqlite3Put4byte(&aWalHdr[8], szPage); + sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); + if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt); + memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); + walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); + sqlite3Put4byte(&aWalHdr[24], aCksum[0]); + sqlite3Put4byte(&aWalHdr[28], aCksum[1]); + + pWal->szPage = szPage; + pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN; + pWal->hdr.aFrameCksum[0] = aCksum[0]; + pWal->hdr.aFrameCksum[1] = aCksum[1]; + pWal->truncateOnCommit = 1; + + rc = sqlite3OsWrite(pWal->pWalFd, aWalHdr, sizeof(aWalHdr), 0); + WALTRACE(("WAL%p: wal-header write %s\n", pWal, rc ? "failed" : "ok")); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Sync the header (unless SQLITE_IOCAP_SEQUENTIAL is true or unless + ** all syncing is turned off by PRAGMA synchronous=OFF). Otherwise + ** an out-of-order write following a WAL restart could result in + ** database corruption. See the ticket: + ** + ** https://sqlite.org/src/info/ff5be73dee + */ + if( pWal->syncHeader ){ + rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); + if( rc ) return rc; + } + } + assert( (int)pWal->szPage==szPage ); + + /* Setup information needed to write frames into the WAL */ + w.pWal = pWal; + w.pFd = pWal->pWalFd; + w.iSyncPoint = 0; + w.syncFlags = sync_flags; + w.szPage = szPage; + iOffset = walFrameOffset(iFrame+1, szPage); + szFrame = szPage + WAL_FRAME_HDRSIZE; + + /* Write all frames into the log file exactly once */ + for(p=pList; p; p=p->pDirty){ + int nDbSize; /* 0 normally. Positive == commit flag */ + + /* Check if this page has already been written into the wal file by + ** the current transaction. If so, overwrite the existing frame and + ** set Wal.writeLock to WAL_WRITELOCK_RECKSUM - indicating that + ** checksums must be recomputed when the transaction is committed. */ + if( iFirst && (p->pDirty || isCommit==0) ){ + u32 iWrite = 0; + VVA_ONLY(rc =) sqlite3WalFindFrame(pWal, p->pgno, &iWrite); + assert( rc==SQLITE_OK || iWrite==0 ); + if( iWrite>=iFirst ){ + i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; + void *pData; + if( pWal->iReCksum==0 || iWriteiReCksum ){ + pWal->iReCksum = iWrite; + } + pData = p->pData; + rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOff); + if( rc ) return rc; + p->flags &= ~PGHDR_WAL_APPEND; + continue; + } + } + + iFrame++; + assert( iOffset==walFrameOffset(iFrame, szPage) ); + nDbSize = (isCommit && p->pDirty==0) ? nTruncate : 0; + rc = walWriteOneFrame(&w, p, nDbSize, iOffset); + if( rc ) return rc; + pLast = p; + iOffset += szFrame; + p->flags |= PGHDR_WAL_APPEND; + } + + /* Recalculate checksums within the wal file if required. */ + if( isCommit && pWal->iReCksum ){ + rc = walRewriteChecksums(pWal, iFrame); + if( rc ) return rc; + } + + /* If this is the end of a transaction, then we might need to pad + ** the transaction and/or sync the WAL file. + ** + ** Padding and syncing only occur if this set of frames complete a + ** transaction and if PRAGMA synchronous=FULL. If synchronous==NORMAL + ** or synchronous==OFF, then no padding or syncing are needed. + ** + ** If SQLITE_IOCAP_POWERSAFE_OVERWRITE is defined, then padding is not + ** needed and only the sync is done. If padding is needed, then the + ** final frame is repeated (with its commit mark) until the next sector + ** boundary is crossed. Only the part of the WAL prior to the last + ** sector boundary is synced; the part of the last frame that extends + ** past the sector boundary is written after the sync. + */ + if( isCommit && WAL_SYNC_FLAGS(sync_flags)!=0 ){ + int bSync = 1; + if( pWal->padToSectorBoundary ){ + int sectorSize = sqlite3SectorSize(pWal->pWalFd); + w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; + bSync = (w.iSyncPoint==iOffset); + testcase( bSync ); + while( iOffsettruncateOnCommit && pWal->mxWalSize>=0 ){ + i64 sz = pWal->mxWalSize; + if( walFrameOffset(iFrame+nExtra+1, szPage)>pWal->mxWalSize ){ + sz = walFrameOffset(iFrame+nExtra+1, szPage); + } + walLimitSize(pWal, sz); + pWal->truncateOnCommit = 0; + } + + /* Append data to the wal-index. It is not necessary to lock the + ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index + ** guarantees that there are no other writers, and no data that may + ** be in use by existing readers is being overwritten. + */ + iFrame = pWal->hdr.mxFrame; + for(p=pList; p && rc==SQLITE_OK; p=p->pDirty){ + if( (p->flags & PGHDR_WAL_APPEND)==0 ) continue; + iFrame++; + rc = walIndexAppend(pWal, iFrame, p->pgno); + } + assert( pLast!=0 || nExtra==0 ); + while( rc==SQLITE_OK && nExtra>0 ){ + iFrame++; + nExtra--; + rc = walIndexAppend(pWal, iFrame, pLast->pgno); + } + + if( rc==SQLITE_OK ){ + /* Update the private copy of the header. */ + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + pWal->hdr.mxFrame = iFrame; + if( isCommit ){ + pWal->hdr.iChange++; + pWal->hdr.nPage = nTruncate; + } + /* If this is a commit, update the wal-index header too. */ + if( isCommit ){ + walIndexWriteHdr(pWal); + pWal->iCallback = iFrame; + } + } + + WALTRACE(("WAL%p: frame write %s\n", pWal, rc ? "failed" : "ok")); + return rc; +} + +/* +** This routine is called to implement sqlite3_wal_checkpoint() and +** related interfaces. +** +** Obtain a CHECKPOINT lock and then backfill as much information as +** we can from WAL into the database. +** +** If parameter xBusy is not NULL, it is a pointer to a busy-handler +** callback. In this case this function runs a blocking checkpoint. +*/ +SQLITE_PRIVATE int sqlite3WalCheckpoint( + Wal *pWal, /* Wal connection */ + sqlite3 *db, /* Check this handle's interrupt flag */ + int eMode, /* PASSIVE, FULL, RESTART, or TRUNCATE */ + int (*xBusy)(void*), /* Function to call when busy */ + void *pBusyArg, /* Context argument for xBusyHandler */ + int sync_flags, /* Flags to sync db file with (or 0) */ + int nBuf, /* Size of temporary buffer */ + u8 *zBuf, /* Temporary buffer to use */ + int *pnLog, /* OUT: Number of frames in WAL */ + int *pnCkpt /* OUT: Number of backfilled frames in WAL */ +){ + int rc; /* Return code */ + int isChanged = 0; /* True if a new wal-index header is loaded */ + int eMode2 = eMode; /* Mode to pass to walCheckpoint() */ + int (*xBusy2)(void*) = xBusy; /* Busy handler for eMode2 */ + + assert( pWal->ckptLock==0 ); + assert( pWal->writeLock==0 ); + + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + + if( pWal->readOnly ) return SQLITE_READONLY; + WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + + /* Enable blocking locks, if possible. If blocking locks are successfully + ** enabled, set xBusy2=0 so that the busy-handler is never invoked. */ + sqlite3WalDb(pWal, db); + (void)walEnableBlocking(pWal); + + /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive + ** "checkpoint" lock on the database file. + ** EVIDENCE-OF: R-10421-19736 If any other process is running a + ** checkpoint operation at the same time, the lock cannot be obtained and + ** SQLITE_BUSY is returned. + ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + ** it will not be invoked in this case. + */ + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + testcase( rc==SQLITE_BUSY ); + testcase( rc!=SQLITE_OK && xBusy2!=0 ); + if( rc==SQLITE_OK ){ + pWal->ckptLock = 1; + + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and + ** TRUNCATE modes also obtain the exclusive "writer" lock on the database + ** file. + ** + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. + */ + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + }else if( rc==SQLITE_BUSY ){ + eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; + rc = SQLITE_OK; + } + } + } + + + /* Read the wal-index header. */ + if( rc==SQLITE_OK ){ + walDisableBlocking(pWal); + rc = walIndexReadHdr(pWal, &isChanged); + (void)walEnableBlocking(pWal); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } + } + + /* Copy data from the log to the database file. */ + if( rc==SQLITE_OK ){ + + if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); + } + + /* If no error occurred, set the output variables. */ + if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ + if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; + if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); + } + } + + if( isChanged ){ + /* If a new wal-index header was loaded before the checkpoint was + ** performed, then the pager-cache associated with pWal is now + ** out of date. So zero the cached wal-index header to ensure that + ** next time the pager opens a snapshot on this database it knows that + ** the cache needs to be reset. + */ + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + } + + walDisableBlocking(pWal); + sqlite3WalDb(pWal, 0); + + /* Release the locks. */ + sqlite3WalEndWriteTransaction(pWal); + if( pWal->ckptLock ){ + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); + pWal->ckptLock = 0; + } + WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; +#endif + return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc); +} + +/* Return the value to pass to a sqlite3_wal_hook callback, the +** number of frames in the WAL at the point of the last commit since +** sqlite3WalCallback() was called. If no commits have occurred since +** the last call, then return 0. +*/ +SQLITE_PRIVATE int sqlite3WalCallback(Wal *pWal){ + u32 ret = 0; + if( pWal ){ + ret = pWal->iCallback; + pWal->iCallback = 0; + } + return (int)ret; +} + +/* +** This function is called to change the WAL subsystem into or out +** of locking_mode=EXCLUSIVE. +** +** If op is zero, then attempt to change from locking_mode=EXCLUSIVE +** into locking_mode=NORMAL. This means that we must acquire a lock +** on the pWal->readLock byte. If the WAL is already in locking_mode=NORMAL +** or if the acquisition of the lock fails, then return 0. If the +** transition out of exclusive-mode is successful, return 1. This +** operation must occur while the pager is still holding the exclusive +** lock on the main database file. +** +** If op is one, then change from locking_mode=NORMAL into +** locking_mode=EXCLUSIVE. This means that the pWal->readLock must +** be released. Return 1 if the transition is made and 0 if the +** WAL is already in exclusive-locking mode - meaning that this +** routine is a no-op. The pager must already hold the exclusive lock +** on the main database file before invoking this operation. +** +** If op is negative, then do a dry-run of the op==1 case but do +** not actually change anything. The pager uses this to see if it +** should acquire the database exclusive lock prior to invoking +** the op==1 case. +*/ +SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){ + int rc; + assert( pWal->writeLock==0 ); + assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 ); + + /* pWal->readLock is usually set, but might be -1 if there was a + ** prior error while attempting to acquire are read-lock. This cannot + ** happen if the connection is actually in exclusive mode (as no xShmLock + ** locks are taken in this case). Nor should the pager attempt to + ** upgrade to exclusive-mode following such an error. + */ + assert( pWal->readLock>=0 || pWal->lockError ); + assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) ); + + if( op==0 ){ + if( pWal->exclusiveMode!=WAL_NORMAL_MODE ){ + pWal->exclusiveMode = WAL_NORMAL_MODE; + if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){ + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; + } + rc = pWal->exclusiveMode==WAL_NORMAL_MODE; + }else{ + /* Already in locking_mode=NORMAL */ + rc = 0; + } + }else if( op>0 ){ + assert( pWal->exclusiveMode==WAL_NORMAL_MODE ); + assert( pWal->readLock>=0 ); + walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; + rc = 1; + }else{ + rc = pWal->exclusiveMode==WAL_NORMAL_MODE; + } + return rc; +} + +/* +** Return true if the argument is non-NULL and the WAL module is using +** heap-memory for the wal-index. Otherwise, if the argument is NULL or the +** WAL module is using shared-memory, return false. +*/ +SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){ + return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); +} + +#ifdef SQLITE_ENABLE_SNAPSHOT +/* Create a snapshot object. The content of a snapshot is opaque to +** every other subsystem, so the WAL module can put whatever it needs +** in the object. +*/ +SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot){ + int rc = SQLITE_OK; + WalIndexHdr *pRet; + static const u32 aZero[4] = { 0, 0, 0, 0 }; + + assert( pWal->readLock>=0 && pWal->writeLock==0 ); + + if( memcmp(&pWal->hdr.aFrameCksum[0],aZero,16)==0 ){ + *ppSnapshot = 0; + return SQLITE_ERROR; + } + pRet = (WalIndexHdr*)sqlite3_malloc(sizeof(WalIndexHdr)); + if( pRet==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + memcpy(pRet, &pWal->hdr, sizeof(WalIndexHdr)); + *ppSnapshot = (sqlite3_snapshot*)pRet; + } + + return rc; +} + +/* Try to open on pSnapshot when the next read-transaction starts +*/ +SQLITE_PRIVATE void sqlite3WalSnapshotOpen( + Wal *pWal, + sqlite3_snapshot *pSnapshot +){ + pWal->pSnapshot = (WalIndexHdr*)pSnapshot; +} + +/* +** Return a +ve value if snapshot p1 is newer than p2. A -ve value if +** p1 is older than p2 and zero if p1 and p2 are the same snapshot. +*/ +SQLITE_API int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){ + WalIndexHdr *pHdr1 = (WalIndexHdr*)p1; + WalIndexHdr *pHdr2 = (WalIndexHdr*)p2; + + /* aSalt[0] is a copy of the value stored in the wal file header. It + ** is incremented each time the wal file is restarted. */ + if( pHdr1->aSalt[0]aSalt[0] ) return -1; + if( pHdr1->aSalt[0]>pHdr2->aSalt[0] ) return +1; + if( pHdr1->mxFramemxFrame ) return -1; + if( pHdr1->mxFrame>pHdr2->mxFrame ) return +1; + return 0; +} + +/* +** The caller currently has a read transaction open on the database. +** This function takes a SHARED lock on the CHECKPOINTER slot and then +** checks if the snapshot passed as the second argument is still +** available. If so, SQLITE_OK is returned. +** +** If the snapshot is not available, SQLITE_ERROR is returned. Or, if +** the CHECKPOINTER lock cannot be obtained, SQLITE_BUSY. If any error +** occurs (any value other than SQLITE_OK is returned), the CHECKPOINTER +** lock is released before returning. +*/ +SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot){ + int rc; + rc = walLockShared(pWal, WAL_CKPT_LOCK); + if( rc==SQLITE_OK ){ + WalIndexHdr *pNew = (WalIndexHdr*)pSnapshot; + if( memcmp(pNew->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) + || pNew->mxFramenBackfillAttempted + ){ + rc = SQLITE_ERROR_SNAPSHOT; + walUnlockShared(pWal, WAL_CKPT_LOCK); + } + } + return rc; +} + +/* +** Release a lock obtained by an earlier successful call to +** sqlite3WalSnapshotCheck(). +*/ +SQLITE_PRIVATE void sqlite3WalSnapshotUnlock(Wal *pWal){ + assert( pWal ); + walUnlockShared(pWal, WAL_CKPT_LOCK); +} + + +#endif /* SQLITE_ENABLE_SNAPSHOT */ + +#ifdef SQLITE_ENABLE_ZIPVFS +/* +** If the argument is not NULL, it points to a Wal object that holds a +** read-lock. This function returns the database page-size if it is known, +** or zero if it is not (or if pWal is NULL). +*/ +SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal){ + assert( pWal==0 || pWal->readLock>=0 ); + return (pWal ? pWal->szPage : 0); +} +#endif + +/* Return the sqlite3_file object for the WAL file +*/ +SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ + return pWal->pWalFd; +} + +#endif /* #ifndef SQLITE_OMIT_WAL */ + +/************** End of wal.c *************************************************/ +/************** Begin file btmutex.c *****************************************/ +/* +** 2007 August 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code used to implement mutexes on Btree objects. +** This code really belongs in btree.c. But btree.c is getting too +** big and we want to break it down some. This packaged seemed like +** a good breakout. +*/ +/************** Include btreeInt.h in the middle of btmutex.c ****************/ +/************** Begin file btreeInt.h ****************************************/ +/* +** 2004 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements an external (disk-based) database using BTrees. +** For a detailed discussion of BTrees, refer to +** +** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3: +** "Sorting And Searching", pages 473-480. Addison-Wesley +** Publishing Company, Reading, Massachusetts. +** +** The basic idea is that each page of the file contains N database +** entries and N+1 pointers to subpages. +** +** ---------------------------------------------------------------- +** | Ptr(0) | Key(0) | Ptr(1) | Key(1) | ... | Key(N-1) | Ptr(N) | +** ---------------------------------------------------------------- +** +** All of the keys on the page that Ptr(0) points to have values less +** than Key(0). All of the keys on page Ptr(1) and its subpages have +** values greater than Key(0) and less than Key(1). All of the keys +** on Ptr(N) and its subpages have values greater than Key(N-1). And +** so forth. +** +** Finding a particular key requires reading O(log(M)) pages from the +** disk where M is the number of entries in the tree. +** +** In this implementation, a single file can hold one or more separate +** BTrees. Each BTree is identified by the index of its root page. The +** key and data for any entry are combined to form the "payload". A +** fixed amount of payload can be carried directly on the database +** page. If the payload is larger than the preset amount then surplus +** bytes are stored on overflow pages. The payload for an entry +** and the preceding pointer are combined to form a "Cell". Each +** page has a small header which contains the Ptr(N) pointer and other +** information such as the size of key and data. +** +** FORMAT DETAILS +** +** The file is divided into pages. The first page is called page 1, +** the second is page 2, and so forth. A page number of zero indicates +** "no such page". The page size can be any power of 2 between 512 and 65536. +** Each page can be either a btree page, a freelist page, an overflow +** page, or a pointer-map page. +** +** The first page is always a btree page. The first 100 bytes of the first +** page contain a special header (the "file header") that describes the file. +** The format of the file header is as follows: +** +** OFFSET SIZE DESCRIPTION +** 0 16 Header string: "SQLite format 3\000" +** 16 2 Page size in bytes. (1 means 65536) +** 18 1 File format write version +** 19 1 File format read version +** 20 1 Bytes of unused space at the end of each page +** 21 1 Max embedded payload fraction (must be 64) +** 22 1 Min embedded payload fraction (must be 32) +** 23 1 Min leaf payload fraction (must be 32) +** 24 4 File change counter +** 28 4 Reserved for future use +** 32 4 First freelist page +** 36 4 Number of freelist pages in the file +** 40 60 15 4-byte meta values passed to higher layers +** +** 40 4 Schema cookie +** 44 4 File format of schema layer +** 48 4 Size of page cache +** 52 4 Largest root-page (auto/incr_vacuum) +** 56 4 1=UTF-8 2=UTF16le 3=UTF16be +** 60 4 User version +** 64 4 Incremental vacuum mode +** 68 4 Application-ID +** 72 20 unused +** 92 4 The version-valid-for number +** 96 4 SQLITE_VERSION_NUMBER +** +** All of the integer values are big-endian (most significant byte first). +** +** The file change counter is incremented when the database is changed +** This counter allows other processes to know when the file has changed +** and thus when they need to flush their cache. +** +** The max embedded payload fraction is the amount of the total usable +** space in a page that can be consumed by a single cell for standard +** B-tree (non-LEAFDATA) tables. A value of 255 means 100%. The default +** is to limit the maximum cell size so that at least 4 cells will fit +** on one page. Thus the default max embedded payload fraction is 64. +** +** If the payload for a cell is larger than the max payload, then extra +** payload is spilled to overflow pages. Once an overflow page is allocated, +** as many bytes as possible are moved into the overflow pages without letting +** the cell size drop below the min embedded payload fraction. +** +** The min leaf payload fraction is like the min embedded payload fraction +** except that it applies to leaf nodes in a LEAFDATA tree. The maximum +** payload fraction for a LEAFDATA tree is always 100% (or 255) and it +** not specified in the header. +** +** Each btree pages is divided into three sections: The header, the +** cell pointer array, and the cell content area. Page 1 also has a 100-byte +** file header that occurs before the page header. +** +** |----------------| +** | file header | 100 bytes. Page 1 only. +** |----------------| +** | page header | 8 bytes for leaves. 12 bytes for interior nodes +** |----------------| +** | cell pointer | | 2 bytes per cell. Sorted order. +** | array | | Grows downward +** | | v +** |----------------| +** | unallocated | +** | space | +** |----------------| ^ Grows upwards +** | cell content | | Arbitrary order interspersed with freeblocks. +** | area | | and free space fragments. +** |----------------| +** +** The page headers looks like this: +** +** OFFSET SIZE DESCRIPTION +** 0 1 Flags. 1: intkey, 2: zerodata, 4: leafdata, 8: leaf +** 1 2 byte offset to the first freeblock +** 3 2 number of cells on this page +** 5 2 first byte of the cell content area +** 7 1 number of fragmented free bytes +** 8 4 Right child (the Ptr(N) value). Omitted on leaves. +** +** The flags define the format of this btree page. The leaf flag means that +** this page has no children. The zerodata flag means that this page carries +** only keys and no data. The intkey flag means that the key is an integer +** which is stored in the key size entry of the cell header rather than in +** the payload area. +** +** The cell pointer array begins on the first byte after the page header. +** The cell pointer array contains zero or more 2-byte numbers which are +** offsets from the beginning of the page to the cell content in the cell +** content area. The cell pointers occur in sorted order. The system strives +** to keep free space after the last cell pointer so that new cells can +** be easily added without having to defragment the page. +** +** Cell content is stored at the very end of the page and grows toward the +** beginning of the page. +** +** Unused space within the cell content area is collected into a linked list of +** freeblocks. Each freeblock is at least 4 bytes in size. The byte offset +** to the first freeblock is given in the header. Freeblocks occur in +** increasing order. Because a freeblock must be at least 4 bytes in size, +** any group of 3 or fewer unused bytes in the cell content area cannot +** exist on the freeblock chain. A group of 3 or fewer free bytes is called +** a fragment. The total number of bytes in all fragments is recorded. +** in the page header at offset 7. +** +** SIZE DESCRIPTION +** 2 Byte offset of the next freeblock +** 2 Bytes in this freeblock +** +** Cells are of variable length. Cells are stored in the cell content area at +** the end of the page. Pointers to the cells are in the cell pointer array +** that immediately follows the page header. Cells is not necessarily +** contiguous or in order, but cell pointers are contiguous and in order. +** +** Cell content makes use of variable length integers. A variable +** length integer is 1 to 9 bytes where the lower 7 bits of each +** byte are used. The integer consists of all bytes that have bit 8 set and +** the first byte with bit 8 clear. The most significant byte of the integer +** appears first. A variable-length integer may not be more than 9 bytes long. +** As a special case, all 8 bytes of the 9th byte are used as data. This +** allows a 64-bit integer to be encoded in 9 bytes. +** +** 0x00 becomes 0x00000000 +** 0x7f becomes 0x0000007f +** 0x81 0x00 becomes 0x00000080 +** 0x82 0x00 becomes 0x00000100 +** 0x80 0x7f becomes 0x0000007f +** 0x8a 0x91 0xd1 0xac 0x78 becomes 0x12345678 +** 0x81 0x81 0x81 0x81 0x01 becomes 0x10204081 +** +** Variable length integers are used for rowids and to hold the number of +** bytes of key and data in a btree cell. +** +** The content of a cell looks like this: +** +** SIZE DESCRIPTION +** 4 Page number of the left child. Omitted if leaf flag is set. +** var Number of bytes of data. Omitted if the zerodata flag is set. +** var Number of bytes of key. Or the key itself if intkey flag is set. +** * Payload +** 4 First page of the overflow chain. Omitted if no overflow +** +** Overflow pages form a linked list. Each page except the last is completely +** filled with data (pagesize - 4 bytes). The last page can have as little +** as 1 byte of data. +** +** SIZE DESCRIPTION +** 4 Page number of next overflow page +** * Data +** +** Freelist pages come in two subtypes: trunk pages and leaf pages. The +** file header points to the first in a linked list of trunk page. Each trunk +** page points to multiple leaf pages. The content of a leaf page is +** unspecified. A trunk page looks like this: +** +** SIZE DESCRIPTION +** 4 Page number of next trunk page +** 4 Number of leaf pointers on this page +** * zero or more pages numbers of leaves +*/ +/* #include "sqliteInt.h" */ + + +/* The following value is the maximum cell size assuming a maximum page +** size give above. +*/ +#define MX_CELL_SIZE(pBt) ((int)(pBt->pageSize-8)) + +/* The maximum number of cells on a single page of the database. This +** assumes a minimum cell size of 6 bytes (4 bytes for the cell itself +** plus 2 bytes for the index to the cell in the page header). Such +** small cells will be rare, but they are possible. +*/ +#define MX_CELL(pBt) ((pBt->pageSize-8)/6) + +/* Forward declarations */ +typedef struct MemPage MemPage; +typedef struct BtLock BtLock; +typedef struct CellInfo CellInfo; + +/* +** This is a magic string that appears at the beginning of every +** SQLite database in order to identify the file as a real database. +** +** You can change this value at compile-time by specifying a +** -DSQLITE_FILE_HEADER="..." on the compiler command-line. The +** header must be exactly 16 bytes including the zero-terminator so +** the string itself should be 15 characters long. If you change +** the header, then your custom library will not be able to read +** databases generated by the standard tools and the standard tools +** will not be able to read databases created by your custom library. +*/ +#ifndef SQLITE_FILE_HEADER /* 123456789 123456 */ +# define SQLITE_FILE_HEADER "SQLite format 3" +#endif + +/* +** Page type flags. An ORed combination of these flags appear as the +** first byte of on-disk image of every BTree page. +*/ +#define PTF_INTKEY 0x01 +#define PTF_ZERODATA 0x02 +#define PTF_LEAFDATA 0x04 +#define PTF_LEAF 0x08 + +/* +** An instance of this object stores information about each a single database +** page that has been loaded into memory. The information in this object +** is derived from the raw on-disk page content. +** +** As each database page is loaded into memory, the pager allocats an +** instance of this object and zeros the first 8 bytes. (This is the +** "extra" information associated with each page of the pager.) +** +** Access to all fields of this structure is controlled by the mutex +** stored in MemPage.pBt->mutex. +*/ +struct MemPage { + u8 isInit; /* True if previously initialized. MUST BE FIRST! */ + u8 bBusy; /* Prevent endless loops on corrupt database files */ + u8 intKey; /* True if table b-trees. False for index b-trees */ + u8 intKeyLeaf; /* True if the leaf of an intKey table */ + Pgno pgno; /* Page number for this page */ + /* Only the first 8 bytes (above) are zeroed by pager.c when a new page + ** is allocated. All fields that follow must be initialized before use */ + u8 leaf; /* True if a leaf page */ + u8 hdrOffset; /* 100 for page 1. 0 otherwise */ + u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ + u8 max1bytePayload; /* min(maxLocal,127) */ + u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ + u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ + u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ + u16 cellOffset; /* Index in aData of first cell pointer */ + int nFree; /* Number of free bytes on the page. -1 for unknown */ + u16 nCell; /* Number of cells on this page, local and ovfl */ + u16 maskPage; /* Mask for page offset */ + u16 aiOvfl[4]; /* Insert the i-th overflow cell before the aiOvfl-th + ** non-overflow cell */ + u8 *apOvfl[4]; /* Pointers to the body of overflow cells */ + BtShared *pBt; /* Pointer to BtShared that this page is part of */ + u8 *aData; /* Pointer to disk image of the page data */ + u8 *aDataEnd; /* One byte past the end of usable data */ + u8 *aCellIdx; /* The cell index area */ + u8 *aDataOfst; /* Same as aData for leaves. aData+4 for interior */ + DbPage *pDbPage; /* Pager page handle */ + u16 (*xCellSize)(MemPage*,u8*); /* cellSizePtr method */ + void (*xParseCell)(MemPage*,u8*,CellInfo*); /* btreeParseCell method */ +}; + +/* +** A linked list of the following structures is stored at BtShared.pLock. +** Locks are added (or upgraded from READ_LOCK to WRITE_LOCK) when a cursor +** is opened on the table with root page BtShared.iTable. Locks are removed +** from this list when a transaction is committed or rolled back, or when +** a btree handle is closed. +*/ +struct BtLock { + Btree *pBtree; /* Btree handle holding this lock */ + Pgno iTable; /* Root page of table */ + u8 eLock; /* READ_LOCK or WRITE_LOCK */ + BtLock *pNext; /* Next in BtShared.pLock list */ +}; + +/* Candidate values for BtLock.eLock */ +#define READ_LOCK 1 +#define WRITE_LOCK 2 + +/* A Btree handle +** +** A database connection contains a pointer to an instance of +** this object for every database file that it has open. This structure +** is opaque to the database connection. The database connection cannot +** see the internals of this structure and only deals with pointers to +** this structure. +** +** For some database files, the same underlying database cache might be +** shared between multiple connections. In that case, each connection +** has it own instance of this object. But each instance of this object +** points to the same BtShared object. The database cache and the +** schema associated with the database file are all contained within +** the BtShared object. +** +** All fields in this structure are accessed under sqlite3.mutex. +** The pBt pointer itself may not be changed while there exists cursors +** in the referenced BtShared that point back to this Btree since those +** cursors have to go through this Btree to find their BtShared and +** they often do so without holding sqlite3.mutex. +*/ +struct Btree { + sqlite3 *db; /* The database connection holding this btree */ + BtShared *pBt; /* Sharable content of this btree */ + u8 inTrans; /* TRANS_NONE, TRANS_READ or TRANS_WRITE */ + u8 sharable; /* True if we can share pBt with another db */ + u8 locked; /* True if db currently has pBt locked */ + u8 hasIncrblobCur; /* True if there are one or more Incrblob cursors */ + int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ + int nBackup; /* Number of backup operations reading this btree */ + u32 iBDataVersion; /* Combines with pBt->pPager->iDataVersion */ + Btree *pNext; /* List of other sharable Btrees from the same db */ + Btree *pPrev; /* Back pointer of the same list */ +#ifdef SQLITE_DEBUG + u64 nSeek; /* Calls to sqlite3BtreeMovetoUnpacked() */ +#endif +#ifndef SQLITE_OMIT_SHARED_CACHE + BtLock lock; /* Object used to lock page 1 */ +#endif +}; + +/* +** Btree.inTrans may take one of the following values. +** +** If the shared-data extension is enabled, there may be multiple users +** of the Btree structure. At most one of these may open a write transaction, +** but any number may have active read transactions. +** +** These values must match SQLITE_TXN_NONE, SQLITE_TXN_READ, and +** SQLITE_TXN_WRITE +*/ +#define TRANS_NONE 0 +#define TRANS_READ 1 +#define TRANS_WRITE 2 + +#if TRANS_NONE!=SQLITE_TXN_NONE +# error wrong numeric code for no-transaction +#endif +#if TRANS_READ!=SQLITE_TXN_READ +# error wrong numeric code for read-transaction +#endif +#if TRANS_WRITE!=SQLITE_TXN_WRITE +# error wrong numeric code for write-transaction +#endif + + +/* +** An instance of this object represents a single database file. +** +** A single database file can be in use at the same time by two +** or more database connections. When two or more connections are +** sharing the same database file, each connection has it own +** private Btree object for the file and each of those Btrees points +** to this one BtShared object. BtShared.nRef is the number of +** connections currently sharing this database file. +** +** Fields in this structure are accessed under the BtShared.mutex +** mutex, except for nRef and pNext which are accessed under the +** global SQLITE_MUTEX_STATIC_MAIN mutex. The pPager field +** may not be modified once it is initially set as long as nRef>0. +** The pSchema field may be set once under BtShared.mutex and +** thereafter is unchanged as long as nRef>0. +** +** isPending: +** +** If a BtShared client fails to obtain a write-lock on a database +** table (because there exists one or more read-locks on the table), +** the shared-cache enters 'pending-lock' state and isPending is +** set to true. +** +** The shared-cache leaves the 'pending lock' state when either of +** the following occur: +** +** 1) The current writer (BtShared.pWriter) concludes its transaction, OR +** 2) The number of locks held by other connections drops to zero. +** +** while in the 'pending-lock' state, no connection may start a new +** transaction. +** +** This feature is included to help prevent writer-starvation. +*/ +struct BtShared { + Pager *pPager; /* The page cache */ + sqlite3 *db; /* Database connection currently using this Btree */ + BtCursor *pCursor; /* A list of all open cursors */ + MemPage *pPage1; /* First page of the database */ + u8 openFlags; /* Flags to sqlite3BtreeOpen() */ +#ifndef SQLITE_OMIT_AUTOVACUUM + u8 autoVacuum; /* True if auto-vacuum is enabled */ + u8 incrVacuum; /* True if incr-vacuum is enabled */ + u8 bDoTruncate; /* True to truncate db on commit */ +#endif + u8 inTransaction; /* Transaction state */ + u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ + u8 nReserveWanted; /* Desired number of extra bytes per page */ + u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ + u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ + u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ + u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */ + u16 minLeaf; /* Minimum local payload in a LEAFDATA table */ + u32 pageSize; /* Total number of bytes on a page */ + u32 usableSize; /* Number of usable bytes on each page */ + int nTransaction; /* Number of open transactions (read + write) */ + u32 nPage; /* Number of pages in the database */ + void *pSchema; /* Pointer to space allocated by sqlite3BtreeSchema() */ + void (*xFreeSchema)(void*); /* Destructor for BtShared.pSchema */ + sqlite3_mutex *mutex; /* Non-recursive mutex required to access this object */ + Bitvec *pHasContent; /* Set of pages moved to free-list this transaction */ +#ifndef SQLITE_OMIT_SHARED_CACHE + int nRef; /* Number of references to this structure */ + BtShared *pNext; /* Next on a list of sharable BtShared structs */ + BtLock *pLock; /* List of locks held on this shared-btree struct */ + Btree *pWriter; /* Btree with currently open write transaction */ +#endif + u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ + int nPreformatSize; /* Size of last cell written by TransferRow() */ +}; + +/* +** Allowed values for BtShared.btsFlags +*/ +#define BTS_READ_ONLY 0x0001 /* Underlying file is readonly */ +#define BTS_PAGESIZE_FIXED 0x0002 /* Page size can no longer be changed */ +#define BTS_SECURE_DELETE 0x0004 /* PRAGMA secure_delete is enabled */ +#define BTS_OVERWRITE 0x0008 /* Overwrite deleted content with zeros */ +#define BTS_FAST_SECURE 0x000c /* Combination of the previous two */ +#define BTS_INITIALLY_EMPTY 0x0010 /* Database was empty at trans start */ +#define BTS_NO_WAL 0x0020 /* Do not open write-ahead-log files */ +#define BTS_EXCLUSIVE 0x0040 /* pWriter has an exclusive lock */ +#define BTS_PENDING 0x0080 /* Waiting for read-locks to clear */ + +/* +** An instance of the following structure is used to hold information +** about a cell. The parseCellPtr() function fills in this structure +** based on information extract from the raw disk page. +*/ +struct CellInfo { + i64 nKey; /* The key for INTKEY tables, or nPayload otherwise */ + u8 *pPayload; /* Pointer to the start of payload */ + u32 nPayload; /* Bytes of payload */ + u16 nLocal; /* Amount of payload held locally, not on overflow */ + u16 nSize; /* Size of the cell content on the main b-tree page */ +}; + +/* +** Maximum depth of an SQLite B-Tree structure. Any B-Tree deeper than +** this will be declared corrupt. This value is calculated based on a +** maximum database size of 2^31 pages a minimum fanout of 2 for a +** root-node and 3 for all other internal nodes. +** +** If a tree that appears to be taller than this is encountered, it is +** assumed that the database is corrupt. +*/ +#define BTCURSOR_MAX_DEPTH 20 + +/* +** A cursor is a pointer to a particular entry within a particular +** b-tree within a database file. +** +** The entry is identified by its MemPage and the index in +** MemPage.aCell[] of the entry. +** +** A single database file can be shared by two more database connections, +** but cursors cannot be shared. Each cursor is associated with a +** particular database connection identified BtCursor.pBtree.db. +** +** Fields in this structure are accessed under the BtShared.mutex +** found at self->pBt->mutex. +** +** skipNext meaning: +** The meaning of skipNext depends on the value of eState: +** +** eState Meaning of skipNext +** VALID skipNext is meaningless and is ignored +** INVALID skipNext is meaningless and is ignored +** SKIPNEXT sqlite3BtreeNext() is a no-op if skipNext>0 and +** sqlite3BtreePrevious() is no-op if skipNext<0. +** REQUIRESEEK restoreCursorPosition() restores the cursor to +** eState=SKIPNEXT if skipNext!=0 +** FAULT skipNext holds the cursor fault error code. +*/ +struct BtCursor { + u8 eState; /* One of the CURSOR_XXX constants (see below) */ + u8 curFlags; /* zero or more BTCF_* flags defined below */ + u8 curPagerFlags; /* Flags to send to sqlite3PagerGet() */ + u8 hints; /* As configured by CursorSetHints() */ + int skipNext; /* Prev() is noop if negative. Next() is noop if positive. + ** Error code if eState==CURSOR_FAULT */ + Btree *pBtree; /* The Btree to which this cursor belongs */ + Pgno *aOverflow; /* Cache of overflow page locations */ + void *pKey; /* Saved key that was cursor last known position */ + /* All fields above are zeroed when the cursor is allocated. See + ** sqlite3BtreeCursorZero(). Fields that follow must be manually + ** initialized. */ +#define BTCURSOR_FIRST_UNINIT pBt /* Name of first uninitialized field */ + BtShared *pBt; /* The BtShared this cursor points to */ + BtCursor *pNext; /* Forms a linked list of all cursors */ + CellInfo info; /* A parse of the cell we are pointing at */ + i64 nKey; /* Size of pKey, or last integer key */ + Pgno pgnoRoot; /* The root page of this tree */ + i8 iPage; /* Index of current page in apPage */ + u8 curIntKey; /* Value of apPage[0]->intKey */ + u16 ix; /* Current index for apPage[iPage] */ + u16 aiIdx[BTCURSOR_MAX_DEPTH-1]; /* Current index in apPage[i] */ + struct KeyInfo *pKeyInfo; /* Arg passed to comparison function */ + MemPage *pPage; /* Current page */ + MemPage *apPage[BTCURSOR_MAX_DEPTH-1]; /* Stack of parents of current page */ +}; + +/* +** Legal values for BtCursor.curFlags +*/ +#define BTCF_WriteFlag 0x01 /* True if a write cursor */ +#define BTCF_ValidNKey 0x02 /* True if info.nKey is valid */ +#define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ +#define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ +#define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ +#define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ +#define BTCF_Pinned 0x40 /* Cursor is busy and cannot be moved */ + +/* +** Potential values for BtCursor.eState. +** +** CURSOR_INVALID: +** Cursor does not point to a valid entry. This can happen (for example) +** because the table is empty or because BtreeCursorFirst() has not been +** called. +** +** CURSOR_VALID: +** Cursor points to a valid entry. getPayload() etc. may be called. +** +** CURSOR_SKIPNEXT: +** Cursor is valid except that the Cursor.skipNext field is non-zero +** indicating that the next sqlite3BtreeNext() or sqlite3BtreePrevious() +** operation should be a no-op. +** +** CURSOR_REQUIRESEEK: +** The table that this cursor was opened on still exists, but has been +** modified since the cursor was last used. The cursor position is saved +** in variables BtCursor.pKey and BtCursor.nKey. When a cursor is in +** this state, restoreCursorPosition() can be called to attempt to +** seek the cursor to the saved position. +** +** CURSOR_FAULT: +** An unrecoverable error (an I/O error or a malloc failure) has occurred +** on a different connection that shares the BtShared cache with this +** cursor. The error has left the cache in an inconsistent state. +** Do nothing else with this cursor. Any attempt to use the cursor +** should return the error code stored in BtCursor.skipNext +*/ +#define CURSOR_VALID 0 +#define CURSOR_INVALID 1 +#define CURSOR_SKIPNEXT 2 +#define CURSOR_REQUIRESEEK 3 +#define CURSOR_FAULT 4 + +/* +** The database page the PENDING_BYTE occupies. This page is never used. +*/ +# define PENDING_BYTE_PAGE(pBt) PAGER_MJ_PGNO(pBt) + +/* +** These macros define the location of the pointer-map entry for a +** database page. The first argument to each is the number of usable +** bytes on each page of the database (often 1024). The second is the +** page number to look up in the pointer map. +** +** PTRMAP_PAGENO returns the database page number of the pointer-map +** page that stores the required pointer. PTRMAP_PTROFFSET returns +** the offset of the requested map entry. +** +** If the pgno argument passed to PTRMAP_PAGENO is a pointer-map page, +** then pgno is returned. So (pgno==PTRMAP_PAGENO(pgsz, pgno)) can be +** used to test if pgno is a pointer-map page. PTRMAP_ISPAGE implements +** this test. +*/ +#define PTRMAP_PAGENO(pBt, pgno) ptrmapPageno(pBt, pgno) +#define PTRMAP_PTROFFSET(pgptrmap, pgno) (5*(pgno-pgptrmap-1)) +#define PTRMAP_ISPAGE(pBt, pgno) (PTRMAP_PAGENO((pBt),(pgno))==(pgno)) + +/* +** The pointer map is a lookup table that identifies the parent page for +** each child page in the database file. The parent page is the page that +** contains a pointer to the child. Every page in the database contains +** 0 or 1 parent pages. (In this context 'database page' refers +** to any page that is not part of the pointer map itself.) Each pointer map +** entry consists of a single byte 'type' and a 4 byte parent page number. +** The PTRMAP_XXX identifiers below are the valid types. +** +** The purpose of the pointer map is to facility moving pages from one +** position in the file to another as part of autovacuum. When a page +** is moved, the pointer in its parent must be updated to point to the +** new location. The pointer map is used to locate the parent page quickly. +** +** PTRMAP_ROOTPAGE: The database page is a root-page. The page-number is not +** used in this case. +** +** PTRMAP_FREEPAGE: The database page is an unused (free) page. The page-number +** is not used in this case. +** +** PTRMAP_OVERFLOW1: The database page is the first page in a list of +** overflow pages. The page number identifies the page that +** contains the cell with a pointer to this overflow page. +** +** PTRMAP_OVERFLOW2: The database page is the second or later page in a list of +** overflow pages. The page-number identifies the previous +** page in the overflow page list. +** +** PTRMAP_BTREE: The database page is a non-root btree page. The page number +** identifies the parent page in the btree. +*/ +#define PTRMAP_ROOTPAGE 1 +#define PTRMAP_FREEPAGE 2 +#define PTRMAP_OVERFLOW1 3 +#define PTRMAP_OVERFLOW2 4 +#define PTRMAP_BTREE 5 + +/* A bunch of assert() statements to check the transaction state variables +** of handle p (type Btree*) are internally consistent. +*/ +#define btreeIntegrity(p) \ + assert( p->pBt->inTransaction!=TRANS_NONE || p->pBt->nTransaction==0 ); \ + assert( p->pBt->inTransaction>=p->inTrans ); + + +/* +** The ISAUTOVACUUM macro is used within balance_nonroot() to determine +** if the database supports auto-vacuum or not. Because it is used +** within an expression that is an argument to another macro +** (sqliteMallocRaw), it is not possible to use conditional compilation. +** So, this macro is defined instead. +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +#define ISAUTOVACUUM (pBt->autoVacuum) +#else +#define ISAUTOVACUUM 0 +#endif + + +/* +** This structure is passed around through all the sanity checking routines +** in order to keep track of some global state information. +** +** The aRef[] array is allocated so that there is 1 bit for each page in +** the database. As the integrity-check proceeds, for each page used in +** the database the corresponding bit is set. This allows integrity-check to +** detect pages that are used twice and orphaned pages (both of which +** indicate corruption). +*/ +typedef struct IntegrityCk IntegrityCk; +struct IntegrityCk { + BtShared *pBt; /* The tree being checked out */ + Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ + u8 *aPgRef; /* 1 bit per page in the db (see above) */ + Pgno nPage; /* Number of pages in the database */ + int mxErr; /* Stop accumulating errors when this reaches zero */ + int nErr; /* Number of messages written to zErrMsg so far */ + int bOomFault; /* A memory allocation error has occurred */ + const char *zPfx; /* Error message prefix */ + Pgno v1; /* Value for first %u substitution in zPfx */ + int v2; /* Value for second %d substitution in zPfx */ + StrAccum errMsg; /* Accumulate the error message text here */ + u32 *heap; /* Min-heap used for analyzing cell coverage */ + sqlite3 *db; /* Database connection running the check */ +}; + +/* +** Routines to read or write a two- and four-byte big-endian integer values. +*/ +#define get2byte(x) ((x)[0]<<8 | (x)[1]) +#define put2byte(p,v) ((p)[0] = (u8)((v)>>8), (p)[1] = (u8)(v)) +#define get4byte sqlite3Get4byte +#define put4byte sqlite3Put4byte + +/* +** get2byteAligned(), unlike get2byte(), requires that its argument point to a +** two-byte aligned address. get2bytea() is only used for accessing the +** cell addresses in a btree header. +*/ +#if SQLITE_BYTEORDER==4321 +# define get2byteAligned(x) (*(u16*)(x)) +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000 +# define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 +# define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) +#else +# define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) +#endif + +/************** End of btreeInt.h ********************************************/ +/************** Continuing where we left off in btmutex.c ********************/ +#ifndef SQLITE_OMIT_SHARED_CACHE +#if SQLITE_THREADSAFE + +/* +** Obtain the BtShared mutex associated with B-Tree handle p. Also, +** set BtShared.db to the database handle associated with p and the +** p->locked boolean to true. +*/ +static void lockBtreeMutex(Btree *p){ + assert( p->locked==0 ); + assert( sqlite3_mutex_notheld(p->pBt->mutex) ); + assert( sqlite3_mutex_held(p->db->mutex) ); + + sqlite3_mutex_enter(p->pBt->mutex); + p->pBt->db = p->db; + p->locked = 1; +} + +/* +** Release the BtShared mutex associated with B-Tree handle p and +** clear the p->locked boolean. +*/ +static void SQLITE_NOINLINE unlockBtreeMutex(Btree *p){ + BtShared *pBt = p->pBt; + assert( p->locked==1 ); + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( sqlite3_mutex_held(p->db->mutex) ); + assert( p->db==pBt->db ); + + sqlite3_mutex_leave(pBt->mutex); + p->locked = 0; +} + +/* Forward reference */ +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p); + +/* +** Enter a mutex on the given BTree object. +** +** If the object is not sharable, then no mutex is ever required +** and this routine is a no-op. The underlying mutex is non-recursive. +** But we keep a reference count in Btree.wantToLock so the behavior +** of this interface is recursive. +** +** To avoid deadlocks, multiple Btrees are locked in the same order +** by all database connections. The p->pNext is a list of other +** Btrees belonging to the same database connection as the p Btree +** which need to be locked after p. If we cannot get a lock on +** p, then first unlock all of the others on p->pNext, then wait +** for the lock to become available on p, then relock all of the +** subsequent Btrees that desire a lock. +*/ +SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ + /* Some basic sanity checking on the Btree. The list of Btrees + ** connected by pNext and pPrev should be in sorted order by + ** Btree.pBt value. All elements of the list should belong to + ** the same connection. Only shared Btrees are on the list. */ + assert( p->pNext==0 || p->pNext->pBt>p->pBt ); + assert( p->pPrev==0 || p->pPrev->pBtpBt ); + assert( p->pNext==0 || p->pNext->db==p->db ); + assert( p->pPrev==0 || p->pPrev->db==p->db ); + assert( p->sharable || (p->pNext==0 && p->pPrev==0) ); + + /* Check for locking consistency */ + assert( !p->locked || p->wantToLock>0 ); + assert( p->sharable || p->wantToLock==0 ); + + /* We should already hold a lock on the database connection */ + assert( sqlite3_mutex_held(p->db->mutex) ); + + /* Unless the database is sharable and unlocked, then BtShared.db + ** should already be set correctly. */ + assert( (p->locked==0 && p->sharable) || p->pBt->db==p->db ); + + if( !p->sharable ) return; + p->wantToLock++; + if( p->locked ) return; + btreeLockCarefully(p); +} + +/* This is a helper function for sqlite3BtreeLock(). By moving +** complex, but seldom used logic, out of sqlite3BtreeLock() and +** into this routine, we avoid unnecessary stack pointer changes +** and thus help the sqlite3BtreeLock() routine to run much faster +** in the common case. +*/ +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p){ + Btree *pLater; + + /* In most cases, we should be able to acquire the lock we + ** want without having to go through the ascending lock + ** procedure that follows. Just be sure not to block. + */ + if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){ + p->pBt->db = p->db; + p->locked = 1; + return; + } + + /* To avoid deadlock, first release all locks with a larger + ** BtShared address. Then acquire our lock. Then reacquire + ** the other BtShared locks that we used to hold in ascending + ** order. + */ + for(pLater=p->pNext; pLater; pLater=pLater->pNext){ + assert( pLater->sharable ); + assert( pLater->pNext==0 || pLater->pNext->pBt>pLater->pBt ); + assert( !pLater->locked || pLater->wantToLock>0 ); + if( pLater->locked ){ + unlockBtreeMutex(pLater); + } + } + lockBtreeMutex(p); + for(pLater=p->pNext; pLater; pLater=pLater->pNext){ + if( pLater->wantToLock ){ + lockBtreeMutex(pLater); + } + } +} + + +/* +** Exit the recursive mutex on a Btree. +*/ +SQLITE_PRIVATE void sqlite3BtreeLeave(Btree *p){ + assert( sqlite3_mutex_held(p->db->mutex) ); + if( p->sharable ){ + assert( p->wantToLock>0 ); + p->wantToLock--; + if( p->wantToLock==0 ){ + unlockBtreeMutex(p); + } + } +} + +#ifndef NDEBUG +/* +** Return true if the BtShared mutex is held on the btree, or if the +** B-Tree is not marked as sharable. +** +** This routine is used only from within assert() statements. +*/ +SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree *p){ + assert( p->sharable==0 || p->locked==0 || p->wantToLock>0 ); + assert( p->sharable==0 || p->locked==0 || p->db==p->pBt->db ); + assert( p->sharable==0 || p->locked==0 || sqlite3_mutex_held(p->pBt->mutex) ); + assert( p->sharable==0 || p->locked==0 || sqlite3_mutex_held(p->db->mutex) ); + + return (p->sharable==0 || p->locked); +} +#endif + + +/* +** Enter the mutex on every Btree associated with a database +** connection. This is needed (for example) prior to parsing +** a statement since we will be comparing table and column names +** against all schemas and we do not want those schemas being +** reset out from under us. +** +** There is a corresponding leave-all procedures. +** +** Enter the mutexes in accending order by BtShared pointer address +** to avoid the possibility of deadlock when two threads with +** two or more btrees in common both try to lock all their btrees +** at the same instant. +*/ +static void SQLITE_NOINLINE btreeEnterAll(sqlite3 *db){ + int i; + int skipOk = 1; + Btree *p; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inDb; i++){ + p = db->aDb[i].pBt; + if( p && p->sharable ){ + sqlite3BtreeEnter(p); + skipOk = 0; + } + } + db->noSharedCache = skipOk; +} +SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ + if( db->noSharedCache==0 ) btreeEnterAll(db); +} +static void SQLITE_NOINLINE btreeLeaveAll(sqlite3 *db){ + int i; + Btree *p; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inDb; i++){ + p = db->aDb[i].pBt; + if( p ) sqlite3BtreeLeave(p); + } +} +SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3 *db){ + if( db->noSharedCache==0 ) btreeLeaveAll(db); +} + +#ifndef NDEBUG +/* +** Return true if the current thread holds the database connection +** mutex and all required BtShared mutexes. +** +** This routine is used inside assert() statements only. +*/ +SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){ + int i; + if( !sqlite3_mutex_held(db->mutex) ){ + return 0; + } + for(i=0; inDb; i++){ + Btree *p; + p = db->aDb[i].pBt; + if( p && p->sharable && + (p->wantToLock==0 || !sqlite3_mutex_held(p->pBt->mutex)) ){ + return 0; + } + } + return 1; +} +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Return true if the correct mutexes are held for accessing the +** db->aDb[iDb].pSchema structure. The mutexes required for schema +** access are: +** +** (1) The mutex on db +** (2) if iDb!=1, then the mutex on db->aDb[iDb].pBt. +** +** If pSchema is not NULL, then iDb is computed from pSchema and +** db using sqlite3SchemaToIndex(). +*/ +SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3 *db, int iDb, Schema *pSchema){ + Btree *p; + assert( db!=0 ); + if( pSchema ) iDb = sqlite3SchemaToIndex(db, pSchema); + assert( iDb>=0 && iDbnDb ); + if( !sqlite3_mutex_held(db->mutex) ) return 0; + if( iDb==1 ) return 1; + p = db->aDb[iDb].pBt; + assert( p!=0 ); + return p->sharable==0 || p->locked==1; +} +#endif /* NDEBUG */ + +#else /* SQLITE_THREADSAFE>0 above. SQLITE_THREADSAFE==0 below */ +/* +** The following are special cases for mutex enter routines for use +** in single threaded applications that use shared cache. Except for +** these two routines, all mutex operations are no-ops in that case and +** are null #defines in btree.h. +** +** If shared cache is disabled, then all btree mutex routines, including +** the ones below, are no-ops and are null #defines in btree.h. +*/ + +SQLITE_PRIVATE void sqlite3BtreeEnter(Btree *p){ + p->pBt->db = p->db; +} +SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ + int i; + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p ){ + p->pBt->db = p->db; + } + } +} +#endif /* if SQLITE_THREADSAFE */ + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Enter a mutex on a Btree given a cursor owned by that Btree. +** +** These entry points are used by incremental I/O only. Enter() is required +** any time OMIT_SHARED_CACHE is not defined, regardless of whether or not +** the build is threadsafe. Leave() is only required by threadsafe builds. +*/ +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor *pCur){ + sqlite3BtreeEnter(pCur->pBtree); +} +# if SQLITE_THREADSAFE +SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor *pCur){ + sqlite3BtreeLeave(pCur->pBtree); +} +# endif +#endif /* ifndef SQLITE_OMIT_INCRBLOB */ + +#endif /* ifndef SQLITE_OMIT_SHARED_CACHE */ + +/************** End of btmutex.c *********************************************/ +/************** Begin file btree.c *******************************************/ +/* +** 2004 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements an external (disk-based) database using BTrees. +** See the header comment on "btreeInt.h" for additional information. +** Including a description of file format and an overview of operation. +*/ +/* #include "btreeInt.h" */ + +/* +** The header string that appears at the beginning of every +** SQLite database. +*/ +static const char zMagicHeader[] = SQLITE_FILE_HEADER; + +/* +** Set this global variable to 1 to enable tracing using the TRACE +** macro. +*/ +#if 0 +int sqlite3BtreeTrace=1; /* True to enable tracing */ +# define TRACE(X) if(sqlite3BtreeTrace){printf X;fflush(stdout);} +#else +# define TRACE(X) +#endif + +/* +** Extract a 2-byte big-endian integer from an array of unsigned bytes. +** But if the value is zero, make it 65536. +** +** This routine is used to extract the "offset to cell content area" value +** from the header of a btree page. If the page size is 65536 and the page +** is empty, the offset should be 65536, but the 2-byte value stores zero. +** This routine makes the necessary adjustment to 65536. +*/ +#define get2byteNotZero(X) (((((int)get2byte(X))-1)&0xffff)+1) + +/* +** Values passed as the 5th argument to allocateBtreePage() +*/ +#define BTALLOC_ANY 0 /* Allocate any page */ +#define BTALLOC_EXACT 1 /* Allocate exact page if possible */ +#define BTALLOC_LE 2 /* Allocate any page <= the parameter */ + +/* +** Macro IfNotOmitAV(x) returns (x) if SQLITE_OMIT_AUTOVACUUM is not +** defined, or 0 if it is. For example: +** +** bIncrVacuum = IfNotOmitAV(pBtShared->incrVacuum); +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +#define IfNotOmitAV(expr) (expr) +#else +#define IfNotOmitAV(expr) 0 +#endif + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** A list of BtShared objects that are eligible for participation +** in shared cache. This variable has file scope during normal builds, +** but the test harness needs to access it so we make it global for +** test builds. +** +** Access to this variable is protected by SQLITE_MUTEX_STATIC_MAIN. +*/ +#ifdef SQLITE_TEST +SQLITE_PRIVATE BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; +#else +static BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; +#endif +#endif /* SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Enable or disable the shared pager and schema features. +** +** This routine has no effect on existing database connections. +** The shared cache setting effects only future calls to +** sqlite3_open(), sqlite3_open16(), or sqlite3_open_v2(). +*/ +SQLITE_API int sqlite3_enable_shared_cache(int enable){ + sqlite3GlobalConfig.sharedCacheEnabled = enable; + return SQLITE_OK; +} +#endif + + + +#ifdef SQLITE_OMIT_SHARED_CACHE + /* + ** The functions querySharedCacheTableLock(), setSharedCacheTableLock(), + ** and clearAllSharedCacheTableLocks() + ** manipulate entries in the BtShared.pLock linked list used to store + ** shared-cache table level locks. If the library is compiled with the + ** shared-cache feature disabled, then there is only ever one user + ** of each BtShared structure and so this locking is not necessary. + ** So define the lock related functions as no-ops. + */ + #define querySharedCacheTableLock(a,b,c) SQLITE_OK + #define setSharedCacheTableLock(a,b,c) SQLITE_OK + #define clearAllSharedCacheTableLocks(a) + #define downgradeAllSharedCacheTableLocks(a) + #define hasSharedCacheTableLock(a,b,c,d) 1 + #define hasReadConflicts(a, b) 0 +#endif + +#ifdef SQLITE_DEBUG +/* +** Return and reset the seek counter for a Btree object. +*/ +SQLITE_PRIVATE sqlite3_uint64 sqlite3BtreeSeekCount(Btree *pBt){ + u64 n = pBt->nSeek; + pBt->nSeek = 0; + return n; +} +#endif + +/* +** Implementation of the SQLITE_CORRUPT_PAGE() macro. Takes a single +** (MemPage*) as an argument. The (MemPage*) must not be NULL. +** +** If SQLITE_DEBUG is not defined, then this macro is equivalent to +** SQLITE_CORRUPT_BKPT. Or, if SQLITE_DEBUG is set, then the log message +** normally produced as a side-effect of SQLITE_CORRUPT_BKPT is augmented +** with the page number and filename associated with the (MemPage*). +*/ +#ifdef SQLITE_DEBUG +int corruptPageError(int lineno, MemPage *p){ + char *zMsg; + sqlite3BeginBenignMalloc(); + zMsg = sqlite3_mprintf("database corruption page %d of %s", + (int)p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) + ); + sqlite3EndBenignMalloc(); + if( zMsg ){ + sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); + } + sqlite3_free(zMsg); + return SQLITE_CORRUPT_BKPT; +} +# define SQLITE_CORRUPT_PAGE(pMemPage) corruptPageError(__LINE__, pMemPage) +#else +# define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno) +#endif + +#ifndef SQLITE_OMIT_SHARED_CACHE + +#ifdef SQLITE_DEBUG +/* +**** This function is only used as part of an assert() statement. *** +** +** Check to see if pBtree holds the required locks to read or write to the +** table with root page iRoot. Return 1 if it does and 0 if not. +** +** For example, when writing to a table with root-page iRoot via +** Btree connection pBtree: +** +** assert( hasSharedCacheTableLock(pBtree, iRoot, 0, WRITE_LOCK) ); +** +** When writing to an index that resides in a sharable database, the +** caller should have first obtained a lock specifying the root page of +** the corresponding table. This makes things a bit more complicated, +** as this module treats each table as a separate structure. To determine +** the table corresponding to the index being written, this +** function has to search through the database schema. +** +** Instead of a lock on the table/index rooted at page iRoot, the caller may +** hold a write-lock on the schema table (root page 1). This is also +** acceptable. +*/ +static int hasSharedCacheTableLock( + Btree *pBtree, /* Handle that must hold lock */ + Pgno iRoot, /* Root page of b-tree */ + int isIndex, /* True if iRoot is the root of an index b-tree */ + int eLockType /* Required lock type (READ_LOCK or WRITE_LOCK) */ +){ + Schema *pSchema = (Schema *)pBtree->pBt->pSchema; + Pgno iTab = 0; + BtLock *pLock; + + /* If this database is not shareable, or if the client is reading + ** and has the read-uncommitted flag set, then no lock is required. + ** Return true immediately. + */ + if( (pBtree->sharable==0) + || (eLockType==READ_LOCK && (pBtree->db->flags & SQLITE_ReadUncommit)) + ){ + return 1; + } + + /* If the client is reading or writing an index and the schema is + ** not loaded, then it is too difficult to actually check to see if + ** the correct locks are held. So do not bother - just return true. + ** This case does not come up very often anyhow. + */ + if( isIndex && (!pSchema || (pSchema->schemaFlags&DB_SchemaLoaded)==0) ){ + return 1; + } + + /* Figure out the root-page that the lock should be held on. For table + ** b-trees, this is just the root page of the b-tree being read or + ** written. For index b-trees, it is the root page of the associated + ** table. */ + if( isIndex ){ + HashElem *p; + int bSeen = 0; + for(p=sqliteHashFirst(&pSchema->idxHash); p; p=sqliteHashNext(p)){ + Index *pIdx = (Index *)sqliteHashData(p); + if( pIdx->tnum==(int)iRoot ){ + if( bSeen ){ + /* Two or more indexes share the same root page. There must + ** be imposter tables. So just return true. The assert is not + ** useful in that case. */ + return 1; + } + iTab = pIdx->pTable->tnum; + bSeen = 1; + } + } + }else{ + iTab = iRoot; + } + + /* Search for the required lock. Either a write-lock on root-page iTab, a + ** write-lock on the schema table, or (if the client is reading) a + ** read-lock on iTab will suffice. Return 1 if any of these are found. */ + for(pLock=pBtree->pBt->pLock; pLock; pLock=pLock->pNext){ + if( pLock->pBtree==pBtree + && (pLock->iTable==iTab || (pLock->eLock==WRITE_LOCK && pLock->iTable==1)) + && pLock->eLock>=eLockType + ){ + return 1; + } + } + + /* Failed to find the required lock. */ + return 0; +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* +**** This function may be used as part of assert() statements only. **** +** +** Return true if it would be illegal for pBtree to write into the +** table or index rooted at iRoot because other shared connections are +** simultaneously reading that same table or index. +** +** It is illegal for pBtree to write if some other Btree object that +** shares the same BtShared object is currently reading or writing +** the iRoot table. Except, if the other Btree object has the +** read-uncommitted flag set, then it is OK for the other object to +** have a read cursor. +** +** For example, before writing to any part of the table or index +** rooted at page iRoot, one should call: +** +** assert( !hasReadConflicts(pBtree, iRoot) ); +*/ +static int hasReadConflicts(Btree *pBtree, Pgno iRoot){ + BtCursor *p; + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + if( p->pgnoRoot==iRoot + && p->pBtree!=pBtree + && 0==(p->pBtree->db->flags & SQLITE_ReadUncommit) + ){ + return 1; + } + } + return 0; +} +#endif /* #ifdef SQLITE_DEBUG */ + +/* +** Query to see if Btree handle p may obtain a lock of type eLock +** (READ_LOCK or WRITE_LOCK) on the table with root-page iTab. Return +** SQLITE_OK if the lock may be obtained (by calling +** setSharedCacheTableLock()), or SQLITE_LOCKED if not. +*/ +static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ + BtShared *pBt = p->pBt; + BtLock *pIter; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); + assert( p->db!=0 ); + assert( !(p->db->flags&SQLITE_ReadUncommit)||eLock==WRITE_LOCK||iTab==1 ); + + /* If requesting a write-lock, then the Btree must have an open write + ** transaction on this file. And, obviously, for this to be so there + ** must be an open write transaction on the file itself. + */ + assert( eLock==READ_LOCK || (p==pBt->pWriter && p->inTrans==TRANS_WRITE) ); + assert( eLock==READ_LOCK || pBt->inTransaction==TRANS_WRITE ); + + /* This routine is a no-op if the shared-cache is not enabled */ + if( !p->sharable ){ + return SQLITE_OK; + } + + /* If some other connection is holding an exclusive lock, the + ** requested lock may not be obtained. + */ + if( pBt->pWriter!=p && (pBt->btsFlags & BTS_EXCLUSIVE)!=0 ){ + sqlite3ConnectionBlocked(p->db, pBt->pWriter->db); + return SQLITE_LOCKED_SHAREDCACHE; + } + + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + /* The condition (pIter->eLock!=eLock) in the following if(...) + ** statement is a simplification of: + ** + ** (eLock==WRITE_LOCK || pIter->eLock==WRITE_LOCK) + ** + ** since we know that if eLock==WRITE_LOCK, then no other connection + ** may hold a WRITE_LOCK on any table in this file (since there can + ** only be a single writer). + */ + assert( pIter->eLock==READ_LOCK || pIter->eLock==WRITE_LOCK ); + assert( eLock==READ_LOCK || pIter->pBtree==p || pIter->eLock==READ_LOCK); + if( pIter->pBtree!=p && pIter->iTable==iTab && pIter->eLock!=eLock ){ + sqlite3ConnectionBlocked(p->db, pIter->pBtree->db); + if( eLock==WRITE_LOCK ){ + assert( p==pBt->pWriter ); + pBt->btsFlags |= BTS_PENDING; + } + return SQLITE_LOCKED_SHAREDCACHE; + } + } + return SQLITE_OK; +} +#endif /* !SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Add a lock on the table with root-page iTable to the shared-btree used +** by Btree handle p. Parameter eLock must be either READ_LOCK or +** WRITE_LOCK. +** +** This function assumes the following: +** +** (a) The specified Btree object p is connected to a sharable +** database (one with the BtShared.sharable flag set), and +** +** (b) No other Btree objects hold a lock that conflicts +** with the requested lock (i.e. querySharedCacheTableLock() has +** already been called and returned SQLITE_OK). +** +** SQLITE_OK is returned if the lock is added successfully. SQLITE_NOMEM +** is returned if a malloc attempt fails. +*/ +static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ + BtShared *pBt = p->pBt; + BtLock *pLock = 0; + BtLock *pIter; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); + assert( p->db!=0 ); + + /* A connection with the read-uncommitted flag set will never try to + ** obtain a read-lock using this function. The only read-lock obtained + ** by a connection in read-uncommitted mode is on the sqlite_schema + ** table, and that lock is obtained in BtreeBeginTrans(). */ + assert( 0==(p->db->flags&SQLITE_ReadUncommit) || eLock==WRITE_LOCK ); + + /* This function should only be called on a sharable b-tree after it + ** has been determined that no other b-tree holds a conflicting lock. */ + assert( p->sharable ); + assert( SQLITE_OK==querySharedCacheTableLock(p, iTable, eLock) ); + + /* First search the list for an existing lock on this table. */ + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->iTable==iTable && pIter->pBtree==p ){ + pLock = pIter; + break; + } + } + + /* If the above search did not find a BtLock struct associating Btree p + ** with table iTable, allocate one and link it into the list. + */ + if( !pLock ){ + pLock = (BtLock *)sqlite3MallocZero(sizeof(BtLock)); + if( !pLock ){ + return SQLITE_NOMEM_BKPT; + } + pLock->iTable = iTable; + pLock->pBtree = p; + pLock->pNext = pBt->pLock; + pBt->pLock = pLock; + } + + /* Set the BtLock.eLock variable to the maximum of the current lock + ** and the requested lock. This means if a write-lock was already held + ** and a read-lock requested, we don't incorrectly downgrade the lock. + */ + assert( WRITE_LOCK>READ_LOCK ); + if( eLock>pLock->eLock ){ + pLock->eLock = eLock; + } + + return SQLITE_OK; +} +#endif /* !SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Release all the table locks (locks obtained via calls to +** the setSharedCacheTableLock() procedure) held by Btree object p. +** +** This function assumes that Btree p has an open read or write +** transaction. If it does not, then the BTS_PENDING flag +** may be incorrectly cleared. +*/ +static void clearAllSharedCacheTableLocks(Btree *p){ + BtShared *pBt = p->pBt; + BtLock **ppIter = &pBt->pLock; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( p->sharable || 0==*ppIter ); + assert( p->inTrans>0 ); + + while( *ppIter ){ + BtLock *pLock = *ppIter; + assert( (pBt->btsFlags & BTS_EXCLUSIVE)==0 || pBt->pWriter==pLock->pBtree ); + assert( pLock->pBtree->inTrans>=pLock->eLock ); + if( pLock->pBtree==p ){ + *ppIter = pLock->pNext; + assert( pLock->iTable!=1 || pLock==&p->lock ); + if( pLock->iTable!=1 ){ + sqlite3_free(pLock); + } + }else{ + ppIter = &pLock->pNext; + } + } + + assert( (pBt->btsFlags & BTS_PENDING)==0 || pBt->pWriter ); + if( pBt->pWriter==p ){ + pBt->pWriter = 0; + pBt->btsFlags &= ~(BTS_EXCLUSIVE|BTS_PENDING); + }else if( pBt->nTransaction==2 ){ + /* This function is called when Btree p is concluding its + ** transaction. If there currently exists a writer, and p is not + ** that writer, then the number of locks held by connections other + ** than the writer must be about to drop to zero. In this case + ** set the BTS_PENDING flag to 0. + ** + ** If there is not currently a writer, then BTS_PENDING must + ** be zero already. So this next line is harmless in that case. + */ + pBt->btsFlags &= ~BTS_PENDING; + } +} + +/* +** This function changes all write-locks held by Btree p into read-locks. +*/ +static void downgradeAllSharedCacheTableLocks(Btree *p){ + BtShared *pBt = p->pBt; + if( pBt->pWriter==p ){ + BtLock *pLock; + pBt->pWriter = 0; + pBt->btsFlags &= ~(BTS_EXCLUSIVE|BTS_PENDING); + for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ + assert( pLock->eLock==READ_LOCK || pLock->pBtree==p ); + pLock->eLock = READ_LOCK; + } + } +} + +#endif /* SQLITE_OMIT_SHARED_CACHE */ + +static void releasePage(MemPage *pPage); /* Forward reference */ +static void releasePageOne(MemPage *pPage); /* Forward reference */ +static void releasePageNotNull(MemPage *pPage); /* Forward reference */ + +/* +***** This routine is used inside of assert() only **** +** +** Verify that the cursor holds the mutex on its BtShared +*/ +#ifdef SQLITE_DEBUG +static int cursorHoldsMutex(BtCursor *p){ + return sqlite3_mutex_held(p->pBt->mutex); +} + +/* Verify that the cursor and the BtShared agree about what is the current +** database connetion. This is important in shared-cache mode. If the database +** connection pointers get out-of-sync, it is possible for routines like +** btreeInitPage() to reference an stale connection pointer that references a +** a connection that has already closed. This routine is used inside assert() +** statements only and for the purpose of double-checking that the btree code +** does keep the database connection pointers up-to-date. +*/ +static int cursorOwnsBtShared(BtCursor *p){ + assert( cursorHoldsMutex(p) ); + return (p->pBtree->db==p->pBt->db); +} +#endif + +/* +** Invalidate the overflow cache of the cursor passed as the first argument. +** on the shared btree structure pBt. +*/ +#define invalidateOverflowCache(pCur) (pCur->curFlags &= ~BTCF_ValidOvfl) + +/* +** Invalidate the overflow page-list cache for all cursors opened +** on the shared btree structure pBt. +*/ +static void invalidateAllOverflowCache(BtShared *pBt){ + BtCursor *p; + assert( sqlite3_mutex_held(pBt->mutex) ); + for(p=pBt->pCursor; p; p=p->pNext){ + invalidateOverflowCache(p); + } +} + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** This function is called before modifying the contents of a table +** to invalidate any incrblob cursors that are open on the +** row or one of the rows being modified. +** +** If argument isClearTable is true, then the entire contents of the +** table is about to be deleted. In this case invalidate all incrblob +** cursors open on any row within the table with root-page pgnoRoot. +** +** Otherwise, if argument isClearTable is false, then the row with +** rowid iRow is being replaced or deleted. In this case invalidate +** only those incrblob cursors open on that specific row. +*/ +static void invalidateIncrblobCursors( + Btree *pBtree, /* The database file to check */ + Pgno pgnoRoot, /* The table that might be changing */ + i64 iRow, /* The rowid that might be changing */ + int isClearTable /* True if all rows are being deleted */ +){ + BtCursor *p; + assert( pBtree->hasIncrblobCur ); + assert( sqlite3BtreeHoldsMutex(pBtree) ); + pBtree->hasIncrblobCur = 0; + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + if( (p->curFlags & BTCF_Incrblob)!=0 ){ + pBtree->hasIncrblobCur = 1; + if( p->pgnoRoot==pgnoRoot && (isClearTable || p->info.nKey==iRow) ){ + p->eState = CURSOR_INVALID; + } + } + } +} + +#else + /* Stub function when INCRBLOB is omitted */ + #define invalidateIncrblobCursors(w,x,y,z) +#endif /* SQLITE_OMIT_INCRBLOB */ + +/* +** Set bit pgno of the BtShared.pHasContent bitvec. This is called +** when a page that previously contained data becomes a free-list leaf +** page. +** +** The BtShared.pHasContent bitvec exists to work around an obscure +** bug caused by the interaction of two useful IO optimizations surrounding +** free-list leaf pages: +** +** 1) When all data is deleted from a page and the page becomes +** a free-list leaf page, the page is not written to the database +** (as free-list leaf pages contain no meaningful data). Sometimes +** such a page is not even journalled (as it will not be modified, +** why bother journalling it?). +** +** 2) When a free-list leaf page is reused, its content is not read +** from the database or written to the journal file (why should it +** be, if it is not at all meaningful?). +** +** By themselves, these optimizations work fine and provide a handy +** performance boost to bulk delete or insert operations. However, if +** a page is moved to the free-list and then reused within the same +** transaction, a problem comes up. If the page is not journalled when +** it is moved to the free-list and it is also not journalled when it +** is extracted from the free-list and reused, then the original data +** may be lost. In the event of a rollback, it may not be possible +** to restore the database to its original configuration. +** +** The solution is the BtShared.pHasContent bitvec. Whenever a page is +** moved to become a free-list leaf page, the corresponding bit is +** set in the bitvec. Whenever a leaf page is extracted from the free-list, +** optimization 2 above is omitted if the corresponding bit is already +** set in BtShared.pHasContent. The contents of the bitvec are cleared +** at the end of every transaction. +*/ +static int btreeSetHasContent(BtShared *pBt, Pgno pgno){ + int rc = SQLITE_OK; + if( !pBt->pHasContent ){ + assert( pgno<=pBt->nPage ); + pBt->pHasContent = sqlite3BitvecCreate(pBt->nPage); + if( !pBt->pHasContent ){ + rc = SQLITE_NOMEM_BKPT; + } + } + if( rc==SQLITE_OK && pgno<=sqlite3BitvecSize(pBt->pHasContent) ){ + rc = sqlite3BitvecSet(pBt->pHasContent, pgno); + } + return rc; +} + +/* +** Query the BtShared.pHasContent vector. +** +** This function is called when a free-list leaf page is removed from the +** free-list for reuse. It returns false if it is safe to retrieve the +** page from the pager layer with the 'no-content' flag set. True otherwise. +*/ +static int btreeGetHasContent(BtShared *pBt, Pgno pgno){ + Bitvec *p = pBt->pHasContent; + return p && (pgno>sqlite3BitvecSize(p) || sqlite3BitvecTestNotNull(p, pgno)); +} + +/* +** Clear (destroy) the BtShared.pHasContent bitvec. This should be +** invoked at the conclusion of each write-transaction. +*/ +static void btreeClearHasContent(BtShared *pBt){ + sqlite3BitvecDestroy(pBt->pHasContent); + pBt->pHasContent = 0; +} + +/* +** Release all of the apPage[] pages for a cursor. +*/ +static void btreeReleaseAllCursorPages(BtCursor *pCur){ + int i; + if( pCur->iPage>=0 ){ + for(i=0; iiPage; i++){ + releasePageNotNull(pCur->apPage[i]); + } + releasePageNotNull(pCur->pPage); + pCur->iPage = -1; + } +} + +/* +** The cursor passed as the only argument must point to a valid entry +** when this function is called (i.e. have eState==CURSOR_VALID). This +** function saves the current cursor key in variables pCur->nKey and +** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error +** code otherwise. +** +** If the cursor is open on an intkey table, then the integer key +** (the rowid) is stored in pCur->nKey and pCur->pKey is left set to +** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is +** set to point to a malloced buffer pCur->nKey bytes in size containing +** the key. +*/ +static int saveCursorKey(BtCursor *pCur){ + int rc = SQLITE_OK; + assert( CURSOR_VALID==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + + if( pCur->curIntKey ){ + /* Only the rowid is required for a table btree */ + pCur->nKey = sqlite3BtreeIntegerKey(pCur); + }else{ + /* For an index btree, save the complete key content. It is possible + ** that the current key is corrupt. In that case, it is possible that + ** the sqlite3VdbeRecordUnpack() function may overread the buffer by + ** up to the size of 1 varint plus 1 8-byte value when the cursor + ** position is restored. Hence the 17 bytes of padding allocated + ** below. */ + void *pKey; + pCur->nKey = sqlite3BtreePayloadSize(pCur); + pKey = sqlite3Malloc( pCur->nKey + 9 + 8 ); + if( pKey ){ + rc = sqlite3BtreePayload(pCur, 0, (int)pCur->nKey, pKey); + if( rc==SQLITE_OK ){ + memset(((u8*)pKey)+pCur->nKey, 0, 9+8); + pCur->pKey = pKey; + }else{ + sqlite3_free(pKey); + } + }else{ + rc = SQLITE_NOMEM_BKPT; + } + } + assert( !pCur->curIntKey || !pCur->pKey ); + return rc; +} + +/* +** Save the current cursor position in the variables BtCursor.nKey +** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. +** +** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) +** prior to calling this routine. +*/ +static int saveCursorPosition(BtCursor *pCur){ + int rc; + + assert( CURSOR_VALID==pCur->eState || CURSOR_SKIPNEXT==pCur->eState ); + assert( 0==pCur->pKey ); + assert( cursorHoldsMutex(pCur) ); + + if( pCur->curFlags & BTCF_Pinned ){ + return SQLITE_CONSTRAINT_PINNED; + } + if( pCur->eState==CURSOR_SKIPNEXT ){ + pCur->eState = CURSOR_VALID; + }else{ + pCur->skipNext = 0; + } + + rc = saveCursorKey(pCur); + if( rc==SQLITE_OK ){ + btreeReleaseAllCursorPages(pCur); + pCur->eState = CURSOR_REQUIRESEEK; + } + + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl|BTCF_AtLast); + return rc; +} + +/* Forward reference */ +static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); + +/* +** Save the positions of all cursors (except pExcept) that are open on +** the table with root-page iRoot. "Saving the cursor position" means that +** the location in the btree is remembered in such a way that it can be +** moved back to the same spot after the btree has been modified. This +** routine is called just before cursor pExcept is used to modify the +** table, for example in BtreeDelete() or BtreeInsert(). +** +** If there are two or more cursors on the same btree, then all such +** cursors should have their BTCF_Multiple flag set. The btreeCursor() +** routine enforces that rule. This routine only needs to be called in +** the uncommon case when pExpect has the BTCF_Multiple flag set. +** +** If pExpect!=NULL and if no other cursors are found on the same root-page, +** then the BTCF_Multiple flag on pExpect is cleared, to avoid another +** pointless call to this routine. +** +** Implementation note: This routine merely checks to see if any cursors +** need to be saved. It calls out to saveCursorsOnList() in the (unusual) +** event that cursors are in need to being saved. +*/ +static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ + BtCursor *p; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pExcept==0 || pExcept->pBt==pBt ); + for(p=pBt->pCursor; p; p=p->pNext){ + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ) break; + } + if( p ) return saveCursorsOnList(p, iRoot, pExcept); + if( pExcept ) pExcept->curFlags &= ~BTCF_Multiple; + return SQLITE_OK; +} + +/* This helper routine to saveAllCursors does the actual work of saving +** the cursors if and when a cursor is found that actually requires saving. +** The common case is that no cursors need to be saved, so this routine is +** broken out from its caller to avoid unnecessary stack pointer movement. +*/ +static int SQLITE_NOINLINE saveCursorsOnList( + BtCursor *p, /* The first cursor that needs saving */ + Pgno iRoot, /* Only save cursor with this iRoot. Save all if zero */ + BtCursor *pExcept /* Do not save this cursor */ +){ + do{ + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ + if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){ + int rc = saveCursorPosition(p); + if( SQLITE_OK!=rc ){ + return rc; + } + }else{ + testcase( p->iPage>=0 ); + btreeReleaseAllCursorPages(p); + } + } + p = p->pNext; + }while( p ); + return SQLITE_OK; +} + +/* +** Clear the current cursor position. +*/ +SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + sqlite3_free(pCur->pKey); + pCur->pKey = 0; + pCur->eState = CURSOR_INVALID; +} + +/* +** In this version of BtreeMoveto, pKey is a packed index record +** such as is generated by the OP_MakeRecord opcode. Unpack the +** record and then call BtreeMovetoUnpacked() to do the work. +*/ +static int btreeMoveto( + BtCursor *pCur, /* Cursor open on the btree to be searched */ + const void *pKey, /* Packed key if the btree is an index */ + i64 nKey, /* Integer key for tables. Size of pKey for indices */ + int bias, /* Bias search to the high end */ + int *pRes /* Write search results here */ +){ + int rc; /* Status code */ + UnpackedRecord *pIdxKey; /* Unpacked index key */ + + if( pKey ){ + KeyInfo *pKeyInfo = pCur->pKeyInfo; + assert( nKey==(i64)(int)nKey ); + pIdxKey = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); + if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; + sqlite3VdbeRecordUnpack(pKeyInfo, (int)nKey, pKey, pIdxKey); + if( pIdxKey->nField==0 || pIdxKey->nField>pKeyInfo->nAllField ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = sqlite3BtreeIndexMoveto(pCur, pIdxKey, pRes); + } + sqlite3DbFree(pCur->pKeyInfo->db, pIdxKey); + }else{ + pIdxKey = 0; + rc = sqlite3BtreeTableMoveto(pCur, nKey, bias, pRes); + } + return rc; +} + +/* +** Restore the cursor to the position it was in (or as close to as possible) +** when saveCursorPosition() was called. Note that this call deletes the +** saved position info stored by saveCursorPosition(), so there can be +** at most one effective restoreCursorPosition() call after each +** saveCursorPosition(). +*/ +static int btreeRestoreCursorPosition(BtCursor *pCur){ + int rc; + int skipNext = 0; + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->eState>=CURSOR_REQUIRESEEK ); + if( pCur->eState==CURSOR_FAULT ){ + return pCur->skipNext; + } + pCur->eState = CURSOR_INVALID; + if( sqlite3FaultSim(410) ){ + rc = SQLITE_IOERR; + }else{ + rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); + } + if( rc==SQLITE_OK ){ + sqlite3_free(pCur->pKey); + pCur->pKey = 0; + assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID ); + if( skipNext ) pCur->skipNext = skipNext; + if( pCur->skipNext && pCur->eState==CURSOR_VALID ){ + pCur->eState = CURSOR_SKIPNEXT; + } + } + return rc; +} + +#define restoreCursorPosition(p) \ + (p->eState>=CURSOR_REQUIRESEEK ? \ + btreeRestoreCursorPosition(p) : \ + SQLITE_OK) + +/* +** Determine whether or not a cursor has moved from the position where +** it was last placed, or has been invalidated for any other reason. +** Cursors can move when the row they are pointing at is deleted out +** from under them, for example. Cursor might also move if a btree +** is rebalanced. +** +** Calling this routine with a NULL cursor pointer returns false. +** +** Use the separate sqlite3BtreeCursorRestore() routine to restore a cursor +** back to where it ought to be if this routine returns true. +*/ +SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor *pCur){ + assert( EIGHT_BYTE_ALIGNMENT(pCur) + || pCur==sqlite3BtreeFakeValidCursor() ); + assert( offsetof(BtCursor, eState)==0 ); + assert( sizeof(pCur->eState)==1 ); + return CURSOR_VALID != *(u8*)pCur; +} + +/* +** Return a pointer to a fake BtCursor object that will always answer +** false to the sqlite3BtreeCursorHasMoved() routine above. The fake +** cursor returned must not be used with any other Btree interface. +*/ +SQLITE_PRIVATE BtCursor *sqlite3BtreeFakeValidCursor(void){ + static u8 fakeCursor = CURSOR_VALID; + assert( offsetof(BtCursor, eState)==0 ); + return (BtCursor*)&fakeCursor; +} + +/* +** This routine restores a cursor back to its original position after it +** has been moved by some outside activity (such as a btree rebalance or +** a row having been deleted out from under the cursor). +** +** On success, the *pDifferentRow parameter is false if the cursor is left +** pointing at exactly the same row. *pDifferntRow is the row the cursor +** was pointing to has been deleted, forcing the cursor to point to some +** nearby row. +** +** This routine should only be called for a cursor that just returned +** TRUE from sqlite3BtreeCursorHasMoved(). +*/ +SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow){ + int rc; + + assert( pCur!=0 ); + assert( pCur->eState!=CURSOR_VALID ); + rc = restoreCursorPosition(pCur); + if( rc ){ + *pDifferentRow = 1; + return rc; + } + if( pCur->eState!=CURSOR_VALID ){ + *pDifferentRow = 1; + }else{ + *pDifferentRow = 0; + } + return SQLITE_OK; +} + +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Provide hints to the cursor. The particular hint given (and the type +** and number of the varargs parameters) is determined by the eHintType +** parameter. See the definitions of the BTREE_HINT_* macros for details. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ + /* Used only by system that substitute their own storage engine */ +} +#endif + +/* +** Provide flag hints to the cursor. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor *pCur, unsigned x){ + assert( x==BTREE_SEEK_EQ || x==BTREE_BULKLOAD || x==0 ); + pCur->hints = x; +} + + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Given a page number of a regular database page, return the page +** number for the pointer-map page that contains the entry for the +** input page number. +** +** Return 0 (not a valid page) for pgno==1 since there is +** no pointer map associated with page 1. The integrity_check logic +** requires that ptrmapPageno(*,1)!=1. +*/ +static Pgno ptrmapPageno(BtShared *pBt, Pgno pgno){ + int nPagesPerMapPage; + Pgno iPtrMap, ret; + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno<2 ) return 0; + nPagesPerMapPage = (pBt->usableSize/5)+1; + iPtrMap = (pgno-2)/nPagesPerMapPage; + ret = (iPtrMap*nPagesPerMapPage) + 2; + if( ret==PENDING_BYTE_PAGE(pBt) ){ + ret++; + } + return ret; +} + +/* +** Write an entry into the pointer map. +** +** This routine updates the pointer map entry for page number 'key' +** so that it maps to type 'eType' and parent page number 'pgno'. +** +** If *pRC is initially non-zero (non-SQLITE_OK) then this routine is +** a no-op. If an error occurs, the appropriate error code is written +** into *pRC. +*/ +static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ + DbPage *pDbPage; /* The pointer map page */ + u8 *pPtrmap; /* The pointer map data */ + Pgno iPtrmap; /* The pointer map page number */ + int offset; /* Offset in pointer map page */ + int rc; /* Return code from subfunctions */ + + if( *pRC ) return; + + assert( sqlite3_mutex_held(pBt->mutex) ); + /* The super-journal page number must never be used as a pointer map page */ + assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); + + assert( pBt->autoVacuum ); + if( key==0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + iPtrmap = PTRMAP_PAGENO(pBt, key); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage, 0); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + if( ((char*)sqlite3PagerGetExtra(pDbPage))[0]!=0 ){ + /* The first byte of the extra data is the MemPage.isInit byte. + ** If that byte is set, it means this page is also being used + ** as a btree page. */ + *pRC = SQLITE_CORRUPT_BKPT; + goto ptrmap_exit; + } + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + *pRC = SQLITE_CORRUPT_BKPT; + goto ptrmap_exit; + } + assert( offset <= (int)pBt->usableSize-5 ); + pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); + + if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ + TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); + *pRC= rc = sqlite3PagerWrite(pDbPage); + if( rc==SQLITE_OK ){ + pPtrmap[offset] = eType; + put4byte(&pPtrmap[offset+1], parent); + } + } + +ptrmap_exit: + sqlite3PagerUnref(pDbPage); +} + +/* +** Read an entry from the pointer map. +** +** This routine retrieves the pointer map entry for page 'key', writing +** the type and parent page number to *pEType and *pPgno respectively. +** An error code is returned if something goes wrong, otherwise SQLITE_OK. +*/ +static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ + DbPage *pDbPage; /* The pointer map page */ + int iPtrmap; /* Pointer map page index */ + u8 *pPtrmap; /* Pointer map page data */ + int offset; /* Offset of entry in pointer map */ + int rc; + + assert( sqlite3_mutex_held(pBt->mutex) ); + + iPtrmap = PTRMAP_PAGENO(pBt, key); + rc = sqlite3PagerGet(pBt->pPager, iPtrmap, &pDbPage, 0); + if( rc!=0 ){ + return rc; + } + pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); + + offset = PTRMAP_PTROFFSET(iPtrmap, key); + if( offset<0 ){ + sqlite3PagerUnref(pDbPage); + return SQLITE_CORRUPT_BKPT; + } + assert( offset <= (int)pBt->usableSize-5 ); + assert( pEType!=0 ); + *pEType = pPtrmap[offset]; + if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); + + sqlite3PagerUnref(pDbPage); + if( *pEType<1 || *pEType>5 ) return SQLITE_CORRUPT_PGNO(iPtrmap); + return SQLITE_OK; +} + +#else /* if defined SQLITE_OMIT_AUTOVACUUM */ + #define ptrmapPut(w,x,y,z,rc) + #define ptrmapGet(w,x,y,z) SQLITE_OK + #define ptrmapPutOvflPtr(x, y, z, rc) +#endif + +/* +** Given a btree page and a cell index (0 means the first cell on +** the page, 1 means the second cell, and so forth) return a pointer +** to the cell content. +** +** findCellPastPtr() does the same except it skips past the initial +** 4-byte child pointer found on interior pages, if there is one. +** +** This routine works only for pages that do not contain overflow cells. +*/ +#define findCell(P,I) \ + ((P)->aData + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) +#define findCellPastPtr(P,I) \ + ((P)->aDataOfst + ((P)->maskPage & get2byteAligned(&(P)->aCellIdx[2*(I)]))) + + +/* +** This is common tail processing for btreeParseCellPtr() and +** btreeParseCellPtrIndex() for the case when the cell does not fit entirely +** on a single B-tree page. Make necessary adjustments to the CellInfo +** structure. +*/ +static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + /* If the payload will not fit completely on the local page, we have + ** to decide how much to store locally and how much to spill onto + ** overflow pages. The strategy is to minimize the amount of unused + ** space on overflow pages while keeping the amount of local storage + ** in between minLocal and maxLocal. + ** + ** Warning: changing the way overflow payload is distributed in any + ** way will result in an incompatible file format. + */ + int minLocal; /* Minimum amount of payload held locally */ + int maxLocal; /* Maximum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ + + minLocal = pPage->minLocal; + maxLocal = pPage->maxLocal; + surplus = minLocal + (pInfo->nPayload - minLocal)%(pPage->pBt->usableSize-4); + testcase( surplus==maxLocal ); + testcase( surplus==maxLocal+1 ); + if( surplus <= maxLocal ){ + pInfo->nLocal = (u16)surplus; + }else{ + pInfo->nLocal = (u16)minLocal; + } + pInfo->nSize = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell) + 4; +} + +/* +** Given a record with nPayload bytes of payload stored within btree +** page pPage, return the number of bytes of payload stored locally. +*/ +static int btreePayloadToLocal(MemPage *pPage, i64 nPayload){ + int maxLocal; /* Maximum amount of payload held locally */ + maxLocal = pPage->maxLocal; + if( nPayload<=maxLocal ){ + return nPayload; + }else{ + int minLocal; /* Minimum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ + minLocal = pPage->minLocal; + surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize-4); + return ( surplus <= maxLocal ) ? surplus : minLocal; + } +} + +/* +** The following routines are implementations of the MemPage.xParseCell() +** method. +** +** Parse a cell content block and fill in the CellInfo structure. +** +** btreeParseCellPtr() => table btree leaf nodes +** btreeParseCellNoPayload() => table btree internal nodes +** btreeParseCellPtrIndex() => index btree nodes +** +** There is also a wrapper function btreeParseCell() that works for +** all MemPage types and that references the cell by index rather than +** by pointer. +*/ +static void btreeParseCellPtrNoPayload( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 ); + assert( pPage->childPtrSize==4 ); +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER(pPage); +#endif + pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); + pInfo->nPayload = 0; + pInfo->nLocal = 0; + pInfo->pPayload = 0; + return; +} +static void btreeParseCellPtr( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + u8 *pIter; /* For scanning through pCell */ + u32 nPayload; /* Number of bytes of cell payload */ + u64 iKey; /* Extracted Key value */ + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 || pPage->leaf==1 ); + assert( pPage->intKeyLeaf ); + assert( pPage->childPtrSize==0 ); + pIter = pCell; + + /* The next block of code is equivalent to: + ** + ** pIter += getVarint32(pIter, nPayload); + ** + ** The code is inlined to avoid a function call. + */ + nPayload = *pIter; + if( nPayload>=0x80 ){ + u8 *pEnd = &pIter[8]; + nPayload &= 0x7f; + do{ + nPayload = (nPayload<<7) | (*++pIter & 0x7f); + }while( (*pIter)>=0x80 && pIternKey); + ** + ** The code is inlined to avoid a function call. + */ + iKey = *pIter; + if( iKey>=0x80 ){ + u8 *pEnd = &pIter[7]; + iKey &= 0x7f; + while(1){ + iKey = (iKey<<7) | (*++pIter & 0x7f); + if( (*pIter)<0x80 ) break; + if( pIter>=pEnd ){ + iKey = (iKey<<8) | *++pIter; + break; + } + } + } + pIter++; + + pInfo->nKey = *(i64*)&iKey; + pInfo->nPayload = nPayload; + pInfo->pPayload = pIter; + testcase( nPayload==pPage->maxLocal ); + testcase( nPayload==pPage->maxLocal+1 ); + if( nPayload<=pPage->maxLocal ){ + /* This is the (easy) common case where the entire payload fits + ** on the local page. No overflow is required. + */ + pInfo->nSize = nPayload + (u16)(pIter - pCell); + if( pInfo->nSize<4 ) pInfo->nSize = 4; + pInfo->nLocal = (u16)nPayload; + }else{ + btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); + } +} +static void btreeParseCellPtrIndex( + MemPage *pPage, /* Page containing the cell */ + u8 *pCell, /* Pointer to the cell text. */ + CellInfo *pInfo /* Fill in this structure */ +){ + u8 *pIter; /* For scanning through pCell */ + u32 nPayload; /* Number of bytes of cell payload */ + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->leaf==0 || pPage->leaf==1 ); + assert( pPage->intKeyLeaf==0 ); + pIter = pCell + pPage->childPtrSize; + nPayload = *pIter; + if( nPayload>=0x80 ){ + u8 *pEnd = &pIter[8]; + nPayload &= 0x7f; + do{ + nPayload = (nPayload<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIternKey = nPayload; + pInfo->nPayload = nPayload; + pInfo->pPayload = pIter; + testcase( nPayload==pPage->maxLocal ); + testcase( nPayload==pPage->maxLocal+1 ); + if( nPayload<=pPage->maxLocal ){ + /* This is the (easy) common case where the entire payload fits + ** on the local page. No overflow is required. + */ + pInfo->nSize = nPayload + (u16)(pIter - pCell); + if( pInfo->nSize<4 ) pInfo->nSize = 4; + pInfo->nLocal = (u16)nPayload; + }else{ + btreeParseCellAdjustSizeForOverflow(pPage, pCell, pInfo); + } +} +static void btreeParseCell( + MemPage *pPage, /* Page containing the cell */ + int iCell, /* The cell index. First cell is 0 */ + CellInfo *pInfo /* Fill in this structure */ +){ + pPage->xParseCell(pPage, findCell(pPage, iCell), pInfo); +} + +/* +** The following routines are implementations of the MemPage.xCellSize +** method. +** +** Compute the total number of bytes that a Cell needs in the cell +** data area of the btree-page. The return number includes the cell +** data header and the local payload, but not any overflow page or +** the space used by the cell pointer. +** +** cellSizePtrNoPayload() => table internal nodes +** cellSizePtr() => all index nodes & table leaf nodes +*/ +static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#endif + + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[8]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIterintKey ){ + /* pIter now points at the 64-bit integer key value, a variable length + ** integer. The following block moves pIter to point at the first byte + ** past the end of the key value. */ + pEnd = &pIter[9]; + while( (*pIter++)&0x80 && pItermaxLocal ); + testcase( nSize==pPage->maxLocal+1 ); + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + if( nSize<4 ) nSize = 4; + }else{ + int minLocal = pPage->minLocal; + nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); + testcase( nSize==pPage->maxLocal ); + testcase( nSize==pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + nSize = minLocal; + } + nSize += 4 + (u16)(pIter - pCell); + } + assert( nSize==debuginfo.nSize || CORRUPT_DB ); + return (u16)nSize; +} +static u16 cellSizePtrNoPayload(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell + 4; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#else + UNUSED_PARAMETER(pPage); +#endif + + assert( pPage->childPtrSize==4 ); + pEnd = pIter + 9; + while( (*pIter++)&0x80 && pIterxCellSize(pPage, findCell(pPage, iCell)); +} +#endif + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** The cell pCell is currently part of page pSrc but will ultimately be part +** of pPage. (pSrc and pPager are often the same.) If pCell contains a +** pointer to an overflow page, insert an entry into the pointer-map for +** the overflow page that will be valid after pCell has been moved to pPage. +*/ +static void ptrmapPutOvflPtr(MemPage *pPage, MemPage *pSrc, u8 *pCell,int *pRC){ + CellInfo info; + if( *pRC ) return; + assert( pCell!=0 ); + pPage->xParseCell(pPage, pCell, &info); + if( info.nLocalaDataEnd, pCell, pCell+info.nLocal) ){ + testcase( pSrc!=pPage ); + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + ovfl = get4byte(&pCell[info.nSize-4]); + ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); + } +} +#endif + + +/* +** Defragment the page given. This routine reorganizes cells within the +** page so that there are no free-blocks on the free-block list. +** +** Parameter nMaxFrag is the maximum amount of fragmented space that may be +** present in the page after this routine returns. +** +** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a +** b-tree page so that there are no freeblocks or fragment bytes, all +** unused bytes are contained in the unallocated space region, and all +** cells are packed tightly at the end of the page. +*/ +static int defragmentPage(MemPage *pPage, int nMaxFrag){ + int i; /* Loop counter */ + int pc; /* Address of the i-th cell */ + int hdr; /* Offset to the page header */ + int size; /* Size of a cell */ + int usableSize; /* Number of usable bytes on a page */ + int cellOffset; /* Offset to the cell pointer array */ + int cbrk; /* Offset to the cell content area */ + int nCell; /* Number of cells on the page */ + unsigned char *data; /* The page data */ + unsigned char *temp; /* Temp area for cell content */ + unsigned char *src; /* Source of content */ + int iCellFirst; /* First allowable cell index */ + int iCellLast; /* Last possible cell index */ + int iCellStart; /* First cell offset in input */ + + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( pPage->pBt!=0 ); + assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); + assert( pPage->nOverflow==0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + temp = 0; + src = data = pPage->aData; + hdr = pPage->hdrOffset; + cellOffset = pPage->cellOffset; + nCell = pPage->nCell; + assert( nCell==get2byte(&data[hdr+3]) || CORRUPT_DB ); + iCellFirst = cellOffset + 2*nCell; + usableSize = pPage->pBt->usableSize; + + /* This block handles pages with two or fewer free blocks and nMaxFrag + ** or fewer fragmented bytes. In this case it is faster to move the + ** two (or one) blocks of cells using memmove() and add the required + ** offsets to each pointer in the cell-pointer array than it is to + ** reconstruct the entire page. */ + if( (int)data[hdr+7]<=nMaxFrag ){ + int iFree = get2byte(&data[hdr+1]); + if( iFree>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); + if( iFree ){ + int iFree2 = get2byte(&data[iFree]); + if( iFree2>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); + if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ + u8 *pEnd = &data[cellOffset + nCell*2]; + u8 *pAddr; + int sz2 = 0; + int sz = get2byte(&data[iFree+2]); + int top = get2byte(&data[hdr+5]); + if( top>=iFree ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + if( iFree2 ){ + if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_PAGE(pPage); + sz2 = get2byte(&data[iFree2+2]); + if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); + memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); + sz += sz2; + }else if( iFree+sz>usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + + cbrk = top+sz; + assert( cbrk+(iFree-top) <= usableSize ); + memmove(&data[cbrk], &data[top], iFree-top); + for(pAddr=&data[cellOffset]; pAddriCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pc>=iCellStart && pc<=iCellLast ); + size = pPage->xCellSize(pPage, &src[pc]); + cbrk -= size; + if( cbrkusableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( cbrk+size<=usableSize && cbrk>=iCellStart ); + testcase( cbrk+size==usableSize ); + testcase( pc+size==usableSize ); + put2byte(pAddr, cbrk); + if( temp==0 ){ + if( cbrk==pc ) continue; + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + memcpy(&temp[iCellStart], &data[iCellStart], usableSize - iCellStart); + src = temp; + } + memcpy(&data[cbrk], &src[pc], size); + } + data[hdr+7] = 0; + + defragment_out: + assert( pPage->nFree>=0 ); + if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( cbrk>=iCellFirst ); + put2byte(&data[hdr+5], cbrk); + data[hdr+1] = 0; + data[hdr+2] = 0; + memset(&data[iCellFirst], 0, cbrk-iCellFirst); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + return SQLITE_OK; +} + +/* +** Search the free-list on page pPg for space to store a cell nByte bytes in +** size. If one can be found, return a pointer to the space and remove it +** from the free-list. +** +** If no suitable space can be found on the free-list, return NULL. +** +** This function may detect corruption within pPg. If corruption is +** detected then *pRc is set to SQLITE_CORRUPT and NULL is returned. +** +** Slots on the free list that are between 1 and 3 bytes larger than nByte +** will be ignored if adding the extra space to the fragmentation count +** causes the fragmentation count to exceed 60. +*/ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ + const int hdr = pPg->hdrOffset; /* Offset to page header */ + u8 * const aData = pPg->aData; /* Page data */ + int iAddr = hdr + 1; /* Address of ptr to pc */ + int pc = get2byte(&aData[iAddr]); /* Address of a free slot */ + int x; /* Excess size of the slot */ + int maxPC = pPg->pBt->usableSize - nByte; /* Max address for a usable slot */ + int size; /* Size of the free slot */ + + assert( pc>0 ); + while( pc<=maxPC ){ + /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each + ** freeblock form a big-endian integer which is the size of the freeblock + ** in bytes, including the 4-byte header. */ + size = get2byte(&aData[pc+2]); + if( (x = size - nByte)>=0 ){ + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total + ** number of bytes in fragments may not exceed 60. */ + if( aData[hdr+7]>57 ) return 0; + + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else if( x+pc > maxPC ){ + /* This slot extends off the end of the usable part of the page */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + iAddr = pc; + pc = get2byte(&aData[pc]); + if( pc<=iAddr+size ){ + if( pc ){ + /* The next slot in the chain is not past the end of the current slot */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + } + return 0; + } + } + if( pc>maxPC+nByte-4 ){ + /* The free slot chain extends off the end of the page */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + } + return 0; +} + +/* +** Allocate nByte bytes of space from within the B-Tree page passed +** as the first argument. Write into *pIdx the index into pPage->aData[] +** of the first byte of allocated space. Return either SQLITE_OK or +** an error code (usually SQLITE_CORRUPT). +** +** The caller guarantees that there is sufficient space to make the +** allocation. This routine might need to defragment in order to bring +** all the space together, however. This routine will avoid using +** the first two bytes past the cell pointer area since presumably this +** allocation is being made in order to insert a new cell, so we will +** also end up needing a new cell pointer. +*/ +static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ + const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ + u8 * const data = pPage->aData; /* Local cache of pPage->aData */ + int top; /* First byte of cell content area */ + int rc = SQLITE_OK; /* Integer return code */ + int gap; /* First byte of gap between cell pointers and cell content */ + + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( pPage->pBt ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( nByte>=0 ); /* Minimum cell size is 4 */ + assert( pPage->nFree>=nByte ); + assert( pPage->nOverflow==0 ); + assert( nByte < (int)(pPage->pBt->usableSize-8) ); + + assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); + gap = pPage->cellOffset + 2*pPage->nCell; + assert( gap<=65536 ); + /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size + ** and the reserved space is zero (the usual value for reserved space) + ** then the cell content offset of an empty page wants to be 65536. + ** However, that integer is too large to be stored in a 2-byte unsigned + ** integer, so a value of 0 is used in its place. */ + top = get2byte(&data[hdr+5]); + assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */ + if( gap>top ){ + if( top==0 && pPage->pBt->usableSize==65536 ){ + top = 65536; + }else{ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + + /* If there is enough space between gap and top for one more cell pointer, + ** and if the freelist is not empty, then search the + ** freelist looking for a slot big enough to satisfy the request. + */ + testcase( gap+2==top ); + testcase( gap+1==top ); + testcase( gap==top ); + if( (data[hdr+2] || data[hdr+1]) && gap+2<=top ){ + u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + if( pSpace ){ + int g2; + assert( pSpace+nByte<=data+pPage->pBt->usableSize ); + *pIdx = g2 = (int)(pSpace-data); + if( g2<=gap ){ + return SQLITE_CORRUPT_PAGE(pPage); + }else{ + return SQLITE_OK; + } + }else if( rc ){ + return rc; + } + } + + /* The request could not be fulfilled using a freelist slot. Check + ** to see if defragmentation is necessary. + */ + testcase( gap+2+nByte==top ); + if( gap+2+nByte>top ){ + assert( pPage->nCell>0 || CORRUPT_DB ); + assert( pPage->nFree>=0 ); + rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); + if( rc ) return rc; + top = get2byteNotZero(&data[hdr+5]); + assert( gap+2+nByte<=top ); + } + + + /* Allocate memory from the gap in between the cell pointer array + ** and the cell content area. The btreeComputeFreeSpace() call has already + ** validated the freelist. Given that the freelist is valid, there + ** is no way that the allocation can extend off the end of the page. + ** The assert() below verifies the previous sentence. + */ + top -= nByte; + put2byte(&data[hdr+5], top); + assert( top+nByte <= (int)pPage->pBt->usableSize ); + *pIdx = top; + return SQLITE_OK; +} + +/* +** Return a section of the pPage->aData to the freelist. +** The first byte of the new free block is pPage->aData[iStart] +** and the size of the block is iSize bytes. +** +** Adjacent freeblocks are coalesced. +** +** Even though the freeblock list was checked by btreeComputeFreeSpace(), +** that routine will not detect overlap between cells or freeblocks. Nor +** does it detect cells or freeblocks that encrouch into the reserved bytes +** at the end of the page. So do additional corruption checks inside this +** routine and return SQLITE_CORRUPT if any problems are found. +*/ +static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ + u16 iPtr; /* Address of ptr to next freeblock */ + u16 iFreeBlk; /* Address of the next freeblock */ + u8 hdr; /* Page header size. 0 or 100 */ + u8 nFrag = 0; /* Reduction in fragmentation */ + u16 iOrigSize = iSize; /* Original value of iSize */ + u16 x; /* Offset to cell content area */ + u32 iEnd = iStart + iSize; /* First byte past the iStart buffer */ + unsigned char *data = pPage->aData; /* Page content */ + + assert( pPage->pBt!=0 ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( CORRUPT_DB || iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); + assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( iSize>=4 ); /* Minimum cell size is 4 */ + assert( iStart<=pPage->pBt->usableSize-4 ); + + /* The list of freeblocks must be in ascending order. Find the + ** spot on the list where iStart should be inserted. + */ + hdr = pPage->hdrOffset; + iPtr = hdr + 1; + if( data[iPtr+1]==0 && data[iPtr]==0 ){ + iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ + }else{ + while( (iFreeBlk = get2byte(&data[iPtr]))pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( iFreeBlk>iPtr || iFreeBlk==0 ); + + /* At this point: + ** iFreeBlk: First freeblock after iStart, or zero if none + ** iPtr: The address of a pointer to iFreeBlk + ** + ** Check to see if iFreeBlk should be coalesced onto the end of iStart. + */ + if( iFreeBlk && iEnd+3>=iFreeBlk ){ + nFrag = iFreeBlk - iEnd; + if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage); + iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); + if( iEnd > pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + iSize = iEnd - iStart; + iFreeBlk = get2byte(&data[iFreeBlk]); + } + + /* If iPtr is another freeblock (that is, if iPtr is not the freelist + ** pointer in the page header) then check to see if iStart should be + ** coalesced onto the end of iPtr. + */ + if( iPtr>hdr+1 ){ + int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); + if( iPtrEnd+3>=iStart ){ + if( iPtrEnd>iStart ) return SQLITE_CORRUPT_PAGE(pPage); + nFrag += iStart - iPtrEnd; + iSize = iEnd - iPtr; + iStart = iPtr; + } + } + if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PAGE(pPage); + data[hdr+7] -= nFrag; + } + x = get2byte(&data[hdr+5]); + if( iStart<=x ){ + /* The new freeblock is at the beginning of the cell content area, + ** so just extend the cell content area rather than create another + ** freelist entry */ + if( iStartpBt->btsFlags & BTS_FAST_SECURE ){ + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ + memset(&data[iStart], 0, iSize); + } + put2byte(&data[iStart], iFreeBlk); + put2byte(&data[iStart+2], iSize); + pPage->nFree += iOrigSize; + return SQLITE_OK; +} + +/* +** Decode the flags byte (the first byte of the header) for a page +** and initialize fields of the MemPage structure accordingly. +** +** Only the following combinations are supported. Anything different +** indicates a corrupt database files: +** +** PTF_ZERODATA +** PTF_ZERODATA | PTF_LEAF +** PTF_LEAFDATA | PTF_INTKEY +** PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF +*/ +static int decodeFlags(MemPage *pPage, int flagByte){ + BtShared *pBt; /* A copy of pPage->pBt */ + + assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); + flagByte &= ~PTF_LEAF; + pPage->childPtrSize = 4-4*pPage->leaf; + pPage->xCellSize = cellSizePtr; + pBt = pPage->pBt; + if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ + /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an + ** interior table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); + /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a + ** leaf table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); + pPage->intKey = 1; + if( pPage->leaf ){ + pPage->intKeyLeaf = 1; + pPage->xParseCell = btreeParseCellPtr; + }else{ + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtrNoPayload; + pPage->xParseCell = btreeParseCellPtrNoPayload; + } + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else if( flagByte==PTF_ZERODATA ){ + /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an + ** interior index b-tree page. */ + assert( (PTF_ZERODATA)==2 ); + /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a + ** leaf index b-tree page. */ + assert( (PTF_ZERODATA|PTF_LEAF)==10 ); + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + }else{ + /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is + ** an error. */ + return SQLITE_CORRUPT_PAGE(pPage); + } + pPage->max1bytePayload = pBt->max1bytePayload; + return SQLITE_OK; +} + +/* +** Compute the amount of freespace on the page. In other words, fill +** in the pPage->nFree field. +*/ +static int btreeComputeFreeSpace(MemPage *pPage){ + int pc; /* Address of a freeblock within pPage->aData[] */ + u8 hdr; /* Offset to beginning of page header */ + u8 *data; /* Equal to pPage->aData */ + int usableSize; /* Amount of usable space on each page */ + int nFree; /* Number of unused bytes on the page */ + int top; /* First byte of the cell content area */ + int iCellFirst; /* First allowable cell or freeblock offset */ + int iCellLast; /* Last possible cell or freeblock offset */ + + assert( pPage->pBt!=0 ); + assert( pPage->pBt->db!=0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); + assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); + assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); + assert( pPage->isInit==1 ); + assert( pPage->nFree<0 ); + + usableSize = pPage->pBt->usableSize; + hdr = pPage->hdrOffset; + data = pPage->aData; + /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates + ** the start of the cell content area. A zero value for this integer is + ** interpreted as 65536. */ + top = get2byteNotZero(&data[hdr+5]); + iCellFirst = hdr + 8 + pPage->childPtrSize + 2*pPage->nCell; + iCellLast = usableSize - 4; + + /* Compute the total free space on the page + ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the + ** start of the first freeblock on the page, or is zero if there are no + ** freeblocks. */ + pc = get2byte(&data[hdr+1]); + nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ + if( pc>0 ){ + u32 next, size; + if( pciCellLast ){ + /* Freeblock off the end of the page */ + return SQLITE_CORRUPT_PAGE(pPage); + } + next = get2byte(&data[pc]); + size = get2byte(&data[pc+2]); + nFree = nFree + size; + if( next<=pc+size+3 ) break; + pc = next; + } + if( next>0 ){ + /* Freeblock not in ascending order */ + return SQLITE_CORRUPT_PAGE(pPage); + } + if( pc+size>(unsigned int)usableSize ){ + /* Last freeblock extends past page end */ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + + /* At this point, nFree contains the sum of the offset to the start + ** of the cell-content area plus the number of free bytes within + ** the cell-content area. If this is greater than the usable-size + ** of the page, then the page must be corrupted. This check also + ** serves to verify that the offset to the start of the cell-content + ** area, according to the page header, lies within the page. + */ + if( nFree>usableSize || nFreenFree = (u16)(nFree - iCellFirst); + return SQLITE_OK; +} + +/* +** Do additional sanity check after btreeInitPage() if +** PRAGMA cell_size_check=ON +*/ +static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ + int iCellFirst; /* First allowable cell or freeblock offset */ + int iCellLast; /* Last possible cell or freeblock offset */ + int i; /* Index into the cell pointer array */ + int sz; /* Size of a cell */ + int pc; /* Address of a freeblock within pPage->aData[] */ + u8 *data; /* Equal to pPage->aData */ + int usableSize; /* Maximum usable space on the page */ + int cellOffset; /* Start of cell content area */ + + iCellFirst = pPage->cellOffset + 2*pPage->nCell; + usableSize = pPage->pBt->usableSize; + iCellLast = usableSize - 4; + data = pPage->aData; + cellOffset = pPage->cellOffset; + if( !pPage->leaf ) iCellLast--; + for(i=0; inCell; i++){ + pc = get2byteAligned(&data[cellOffset+i*2]); + testcase( pc==iCellFirst ); + testcase( pc==iCellLast ); + if( pciCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + sz = pPage->xCellSize(pPage, &data[pc]); + testcase( pc+sz==usableSize ); + if( pc+sz>usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + return SQLITE_OK; +} + +/* +** Initialize the auxiliary information for a disk block. +** +** Return SQLITE_OK on success. If we see that the page does +** not contain a well-formed database page, then return +** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not +** guarantee that the page is well-formed. It only shows that +** we failed to detect any corruption. +*/ +static int btreeInitPage(MemPage *pPage){ + u8 *data; /* Equal to pPage->aData */ + BtShared *pBt; /* The main btree structure */ + + assert( pPage->pBt!=0 ); + assert( pPage->pBt->db!=0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); + assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); + assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); + assert( pPage->isInit==0 ); + + pBt = pPage->pBt; + data = pPage->aData + pPage->hdrOffset; + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ + if( decodeFlags(pPage, data[0]) ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nOverflow = 0; + pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize; + pPage->aCellIdx = data + pPage->childPtrSize + 8; + pPage->aDataEnd = pPage->aData + pBt->usableSize; + pPage->aDataOfst = pPage->aData + pPage->childPtrSize; + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + pPage->nCell = get2byte(&data[3]); + if( pPage->nCell>MX_CELL(pBt) ){ + /* To many cells for a single page. The page must be corrupt */ + return SQLITE_CORRUPT_PAGE(pPage); + } + testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only + ** possible for a root page of a table that contains no rows) then the + ** offset to the cell content area will equal the page size minus the + ** bytes of reserved space. */ + assert( pPage->nCell>0 + || get2byteNotZero(&data[5])==(int)pBt->usableSize + || CORRUPT_DB ); + pPage->nFree = -1; /* Indicate that this value is yet uncomputed */ + pPage->isInit = 1; + if( pBt->db->flags & SQLITE_CellSizeCk ){ + return btreeCellSizeCheck(pPage); + } + return SQLITE_OK; +} + +/* +** Set up a raw page so that it looks like a database page holding +** no entries. +*/ +static void zeroPage(MemPage *pPage, int flags){ + unsigned char *data = pPage->aData; + BtShared *pBt = pPage->pBt; + u8 hdr = pPage->hdrOffset; + u16 first; + + assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage) == data ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pBt->btsFlags & BTS_FAST_SECURE ){ + memset(&data[hdr], 0, pBt->usableSize - hdr); + } + data[hdr] = (char)flags; + first = hdr + ((flags&PTF_LEAF)==0 ? 12 : 8); + memset(&data[hdr+1], 0, 4); + data[hdr+7] = 0; + put2byte(&data[hdr+5], pBt->usableSize); + pPage->nFree = (u16)(pBt->usableSize - first); + decodeFlags(pPage, flags); + pPage->cellOffset = first; + pPage->aDataEnd = &data[pBt->usableSize]; + pPage->aCellIdx = &data[first]; + pPage->aDataOfst = &data[pPage->childPtrSize]; + pPage->nOverflow = 0; + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nCell = 0; + pPage->isInit = 1; +} + + +/* +** Convert a DbPage obtained from the pager into a MemPage used by +** the btree layer. +*/ +static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ + MemPage *pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + if( pgno!=pPage->pgno ){ + pPage->aData = sqlite3PagerGetData(pDbPage); + pPage->pDbPage = pDbPage; + pPage->pBt = pBt; + pPage->pgno = pgno; + pPage->hdrOffset = pgno==1 ? 100 : 0; + } + assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); + return pPage; +} + +/* +** Get a page from the pager. Initialize the MemPage.pBt and +** MemPage.aData elements if needed. See also: btreeGetUnusedPage(). +** +** If the PAGER_GET_NOCONTENT flag is set, it means that we do not care +** about the content of the page at this time. So do not go to the disk +** to fetch the content. Just fill in the content with zeros for now. +** If in the future we call sqlite3PagerWrite() on this page, that +** means we have started to be concerned about content and the disk +** read should occur at that point. +*/ +static int btreeGetPage( + BtShared *pBt, /* The btree */ + Pgno pgno, /* Number of the page to fetch */ + MemPage **ppPage, /* Return the page in this parameter */ + int flags /* PAGER_GET_NOCONTENT or PAGER_GET_READONLY */ +){ + int rc; + DbPage *pDbPage; + + assert( flags==0 || flags==PAGER_GET_NOCONTENT || flags==PAGER_GET_READONLY ); + assert( sqlite3_mutex_held(pBt->mutex) ); + rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); + if( rc ) return rc; + *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); + return SQLITE_OK; +} + +/* +** Retrieve a page from the pager cache. If the requested page is not +** already in the pager cache return NULL. Initialize the MemPage.pBt and +** MemPage.aData elements if needed. +*/ +static MemPage *btreePageLookup(BtShared *pBt, Pgno pgno){ + DbPage *pDbPage; + assert( sqlite3_mutex_held(pBt->mutex) ); + pDbPage = sqlite3PagerLookup(pBt->pPager, pgno); + if( pDbPage ){ + return btreePageFromDbPage(pDbPage, pgno, pBt); + } + return 0; +} + +/* +** Return the size of the database file in pages. If there is any kind of +** error, return ((unsigned int)-1). +*/ +static Pgno btreePagecount(BtShared *pBt){ + return pBt->nPage; +} +SQLITE_PRIVATE Pgno sqlite3BtreeLastPage(Btree *p){ + assert( sqlite3BtreeHoldsMutex(p) ); + return btreePagecount(p->pBt); +} + +/* +** Get a page from the pager and initialize it. +** +** If pCur!=0 then the page is being fetched as part of a moveToChild() +** call. Do additional sanity checking on the page in this case. +** And if the fetch fails, this routine must decrement pCur->iPage. +** +** The page is fetched as read-write unless pCur is not NULL and is +** a read-only cursor. +** +** If an error occurs, then *ppPage is undefined. It +** may remain unchanged, or it may be set to an invalid value. +*/ +static int getAndInitPage( + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage, /* Write the page pointer here */ + BtCursor *pCur, /* Cursor to receive the page, or NULL */ + int bReadOnly /* True for a read-only page */ +){ + int rc; + DbPage *pDbPage; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pCur==0 || ppPage==&pCur->pPage ); + assert( pCur==0 || bReadOnly==pCur->curPagerFlags ); + assert( pCur==0 || pCur->iPage>0 ); + + if( pgno>btreePagecount(pBt) ){ + rc = SQLITE_CORRUPT_BKPT; + goto getAndInitPage_error1; + } + rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); + if( rc ){ + goto getAndInitPage_error1; + } + *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + if( (*ppPage)->isInit==0 ){ + btreePageFromDbPage(pDbPage, pgno, pBt); + rc = btreeInitPage(*ppPage); + if( rc!=SQLITE_OK ){ + goto getAndInitPage_error2; + } + } + assert( (*ppPage)->pgno==pgno ); + assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) ); + + /* If obtaining a child page for a cursor, we must verify that the page is + ** compatible with the root page. */ + if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ + rc = SQLITE_CORRUPT_PGNO(pgno); + goto getAndInitPage_error2; + } + return SQLITE_OK; + +getAndInitPage_error2: + releasePage(*ppPage); +getAndInitPage_error1: + if( pCur ){ + pCur->iPage--; + pCur->pPage = pCur->apPage[pCur->iPage]; + } + testcase( pgno==0 ); + assert( pgno!=0 || rc==SQLITE_CORRUPT ); + return rc; +} + +/* +** Release a MemPage. This should be called once for each prior +** call to btreeGetPage. +** +** Page1 is a special case and must be released using releasePageOne(). +*/ +static void releasePageNotNull(MemPage *pPage){ + assert( pPage->aData ); + assert( pPage->pBt ); + assert( pPage->pDbPage!=0 ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnrefNotNull(pPage->pDbPage); +} +static void releasePage(MemPage *pPage){ + if( pPage ) releasePageNotNull(pPage); +} +static void releasePageOne(MemPage *pPage){ + assert( pPage!=0 ); + assert( pPage->aData ); + assert( pPage->pBt ); + assert( pPage->pDbPage!=0 ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnrefPageOne(pPage->pDbPage); +} + +/* +** Get an unused page. +** +** This works just like btreeGetPage() with the addition: +** +** * If the page is already in use for some other purpose, immediately +** release it and return an SQLITE_CURRUPT error. +** * Make sure the isInit flag is clear +*/ +static int btreeGetUnusedPage( + BtShared *pBt, /* The btree */ + Pgno pgno, /* Number of the page to fetch */ + MemPage **ppPage, /* Return the page in this parameter */ + int flags /* PAGER_GET_NOCONTENT or PAGER_GET_READONLY */ +){ + int rc = btreeGetPage(pBt, pgno, ppPage, flags); + if( rc==SQLITE_OK ){ + if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ + releasePage(*ppPage); + *ppPage = 0; + return SQLITE_CORRUPT_BKPT; + } + (*ppPage)->isInit = 0; + }else{ + *ppPage = 0; + } + return rc; +} + + +/* +** During a rollback, when the pager reloads information into the cache +** so that the cache is restored to its original state at the start of +** the transaction, for each page restored this routine is called. +** +** This routine needs to reset the extra data section at the end of the +** page to agree with the restored data. +*/ +static void pageReinit(DbPage *pData){ + MemPage *pPage; + pPage = (MemPage *)sqlite3PagerGetExtra(pData); + assert( sqlite3PagerPageRefcount(pData)>0 ); + if( pPage->isInit ){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + pPage->isInit = 0; + if( sqlite3PagerPageRefcount(pData)>1 ){ + /* pPage might not be a btree page; it might be an overflow page + ** or ptrmap page or a free page. In those cases, the following + ** call to btreeInitPage() will likely return SQLITE_CORRUPT. + ** But no harm is done by this. And it is very important that + ** btreeInitPage() be called on every btree page so we make + ** the call for every page that comes in for re-initing. */ + btreeInitPage(pPage); + } + } +} + +/* +** Invoke the busy handler for a btree. +*/ +static int btreeInvokeBusyHandler(void *pArg){ + BtShared *pBt = (BtShared*)pArg; + assert( pBt->db ); + assert( sqlite3_mutex_held(pBt->db->mutex) ); + return sqlite3InvokeBusyHandler(&pBt->db->busyHandler); +} + +/* +** Open a database file. +** +** zFilename is the name of the database file. If zFilename is NULL +** then an ephemeral database is created. The ephemeral database might +** be exclusively in memory, or it might use a disk-based memory cache. +** Either way, the ephemeral database will be automatically deleted +** when sqlite3BtreeClose() is called. +** +** If zFilename is ":memory:" then an in-memory database is created +** that is automatically destroyed when it is closed. +** +** The "flags" parameter is a bitmask that might contain bits like +** BTREE_OMIT_JOURNAL and/or BTREE_MEMORY. +** +** If the database is already opened in the same database connection +** and we are in shared cache mode, then the open will fail with an +** SQLITE_CONSTRAINT error. We cannot allow two or more BtShared +** objects in the same database connection since doing so will lead +** to problems with locking. +*/ +SQLITE_PRIVATE int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use for this b-tree */ + const char *zFilename, /* Name of the file containing the BTree database */ + sqlite3 *db, /* Associated database handle */ + Btree **ppBtree, /* Pointer to new Btree object written here */ + int flags, /* Options */ + int vfsFlags /* Flags passed through to sqlite3_vfs.xOpen() */ +){ + BtShared *pBt = 0; /* Shared part of btree structure */ + Btree *p; /* Handle to return */ + sqlite3_mutex *mutexOpen = 0; /* Prevents a race condition. Ticket #3537 */ + int rc = SQLITE_OK; /* Result code from this function */ + u8 nReserve; /* Byte of unused space on each page */ + unsigned char zDbHeader[100]; /* Database header content */ + + /* True if opening an ephemeral, temporary database */ + const int isTempDb = zFilename==0 || zFilename[0]==0; + + /* Set the variable isMemdb to true for an in-memory database, or + ** false for a file-based database. + */ +#ifdef SQLITE_OMIT_MEMORYDB + const int isMemdb = 0; +#else + const int isMemdb = (zFilename && strcmp(zFilename, ":memory:")==0) + || (isTempDb && sqlite3TempInMemory(db)) + || (vfsFlags & SQLITE_OPEN_MEMORY)!=0; +#endif + + assert( db!=0 ); + assert( pVfs!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( (flags&0xff)==flags ); /* flags fit in 8 bits */ + + /* Only a BTREE_SINGLE database can be BTREE_UNORDERED */ + assert( (flags & BTREE_UNORDERED)==0 || (flags & BTREE_SINGLE)!=0 ); + + /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ + assert( (flags & BTREE_SINGLE)==0 || isTempDb ); + + if( isMemdb ){ + flags |= BTREE_MEMORY; + } + if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ + vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; + } + p = sqlite3MallocZero(sizeof(Btree)); + if( !p ){ + return SQLITE_NOMEM_BKPT; + } + p->inTrans = TRANS_NONE; + p->db = db; +#ifndef SQLITE_OMIT_SHARED_CACHE + p->lock.pBtree = p; + p->lock.iTable = 1; +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* + ** If this Btree is a candidate for shared cache, try to find an + ** existing BtShared object that we can share with + */ + if( isTempDb==0 && (isMemdb==0 || (vfsFlags&SQLITE_OPEN_URI)!=0) ){ + if( vfsFlags & SQLITE_OPEN_SHAREDCACHE ){ + int nFilename = sqlite3Strlen30(zFilename)+1; + int nFullPathname = pVfs->mxPathname+1; + char *zFullPathname = sqlite3Malloc(MAX(nFullPathname,nFilename)); + MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) + + p->sharable = 1; + if( !zFullPathname ){ + sqlite3_free(p); + return SQLITE_NOMEM_BKPT; + } + if( isMemdb ){ + memcpy(zFullPathname, zFilename, nFilename); + }else{ + rc = sqlite3OsFullPathname(pVfs, zFilename, + nFullPathname, zFullPathname); + if( rc ){ + if( rc==SQLITE_OK_SYMLINK ){ + rc = SQLITE_OK; + }else{ + sqlite3_free(zFullPathname); + sqlite3_free(p); + return rc; + } + } + } +#if SQLITE_THREADSAFE + mutexOpen = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_OPEN); + sqlite3_mutex_enter(mutexOpen); + mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); + sqlite3_mutex_enter(mutexShared); +#endif + for(pBt=GLOBAL(BtShared*,sqlite3SharedCacheList); pBt; pBt=pBt->pNext){ + assert( pBt->nRef>0 ); + if( 0==strcmp(zFullPathname, sqlite3PagerFilename(pBt->pPager, 0)) + && sqlite3PagerVfs(pBt->pPager)==pVfs ){ + int iDb; + for(iDb=db->nDb-1; iDb>=0; iDb--){ + Btree *pExisting = db->aDb[iDb].pBt; + if( pExisting && pExisting->pBt==pBt ){ + sqlite3_mutex_leave(mutexShared); + sqlite3_mutex_leave(mutexOpen); + sqlite3_free(zFullPathname); + sqlite3_free(p); + return SQLITE_CONSTRAINT; + } + } + p->pBt = pBt; + pBt->nRef++; + break; + } + } + sqlite3_mutex_leave(mutexShared); + sqlite3_free(zFullPathname); + } +#ifdef SQLITE_DEBUG + else{ + /* In debug mode, we mark all persistent databases as sharable + ** even when they are not. This exercises the locking code and + ** gives more opportunity for asserts(sqlite3_mutex_held()) + ** statements to find locking problems. + */ + p->sharable = 1; + } +#endif + } +#endif + if( pBt==0 ){ + /* + ** The following asserts make sure that structures used by the btree are + ** the right size. This is to guard against size changes that result + ** when compiling on a different architecture. + */ + assert( sizeof(i64)==8 ); + assert( sizeof(u64)==8 ); + assert( sizeof(u32)==4 ); + assert( sizeof(u16)==2 ); + assert( sizeof(Pgno)==4 ); + + pBt = sqlite3MallocZero( sizeof(*pBt) ); + if( pBt==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto btree_open_out; + } + rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, + sizeof(MemPage), flags, vfsFlags, pageReinit); + if( rc==SQLITE_OK ){ + sqlite3PagerSetMmapLimit(pBt->pPager, db->szMmap); + rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); + } + if( rc!=SQLITE_OK ){ + goto btree_open_out; + } + pBt->openFlags = (u8)flags; + pBt->db = db; + sqlite3PagerSetBusyHandler(pBt->pPager, btreeInvokeBusyHandler, pBt); + p->pBt = pBt; + + pBt->pCursor = 0; + pBt->pPage1 = 0; + if( sqlite3PagerIsreadonly(pBt->pPager) ) pBt->btsFlags |= BTS_READ_ONLY; +#if defined(SQLITE_SECURE_DELETE) + pBt->btsFlags |= BTS_SECURE_DELETE; +#elif defined(SQLITE_FAST_SECURE_DELETE) + pBt->btsFlags |= BTS_OVERWRITE; +#endif + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ + pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); + if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE + || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ + pBt->pageSize = 0; +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the magic name ":memory:" will create an in-memory database, then + ** leave the autoVacuum mode at 0 (do not auto-vacuum), even if + ** SQLITE_DEFAULT_AUTOVACUUM is true. On the other hand, if + ** SQLITE_OMIT_MEMORYDB has been defined, then ":memory:" is just a + ** regular file-name. In this case the auto-vacuum applies as per normal. + */ + if( zFilename && !isMemdb ){ + pBt->autoVacuum = (SQLITE_DEFAULT_AUTOVACUUM ? 1 : 0); + pBt->incrVacuum = (SQLITE_DEFAULT_AUTOVACUUM==2 ? 1 : 0); + } +#endif + nReserve = 0; + }else{ + /* EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ + nReserve = zDbHeader[20]; + pBt->btsFlags |= BTS_PAGESIZE_FIXED; +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->autoVacuum = (get4byte(&zDbHeader[36 + 4*4])?1:0); + pBt->incrVacuum = (get4byte(&zDbHeader[36 + 7*4])?1:0); +#endif + } + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); + if( rc ) goto btree_open_out; + pBt->usableSize = pBt->pageSize - nReserve; + assert( (pBt->pageSize & 7)==0 ); /* 8-byte alignment of pageSize */ + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* Add the new BtShared object to the linked list sharable BtShareds. + */ + pBt->nRef = 1; + if( p->sharable ){ + MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) + MUTEX_LOGIC( mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN);) + if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ + pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); + if( pBt->mutex==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto btree_open_out; + } + } + sqlite3_mutex_enter(mutexShared); + pBt->pNext = GLOBAL(BtShared*,sqlite3SharedCacheList); + GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt; + sqlite3_mutex_leave(mutexShared); + } +#endif + } + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) + /* If the new Btree uses a sharable pBtShared, then link the new + ** Btree into the list of all sharable Btrees for the same connection. + ** The list is kept in ascending order by pBt address. + */ + if( p->sharable ){ + int i; + Btree *pSib; + for(i=0; inDb; i++){ + if( (pSib = db->aDb[i].pBt)!=0 && pSib->sharable ){ + while( pSib->pPrev ){ pSib = pSib->pPrev; } + if( (uptr)p->pBt<(uptr)pSib->pBt ){ + p->pNext = pSib; + p->pPrev = 0; + pSib->pPrev = p; + }else{ + while( pSib->pNext && (uptr)pSib->pNext->pBt<(uptr)p->pBt ){ + pSib = pSib->pNext; + } + p->pNext = pSib->pNext; + p->pPrev = pSib; + if( p->pNext ){ + p->pNext->pPrev = p; + } + pSib->pNext = p; + } + break; + } + } + } +#endif + *ppBtree = p; + +btree_open_out: + if( rc!=SQLITE_OK ){ + if( pBt && pBt->pPager ){ + sqlite3PagerClose(pBt->pPager, 0); + } + sqlite3_free(pBt); + sqlite3_free(p); + *ppBtree = 0; + }else{ + sqlite3_file *pFile; + + /* If the B-Tree was successfully opened, set the pager-cache size to the + ** default value. Except, when opening on an existing shared pager-cache, + ** do not change the pager-cache size. + */ + if( sqlite3BtreeSchema(p, 0, 0)==0 ){ + sqlite3BtreeSetCacheSize(p, SQLITE_DEFAULT_CACHE_SIZE); + } + + pFile = sqlite3PagerFile(pBt->pPager); + if( pFile->pMethods ){ + sqlite3OsFileControlHint(pFile, SQLITE_FCNTL_PDB, (void*)&pBt->db); + } + } + if( mutexOpen ){ + assert( sqlite3_mutex_held(mutexOpen) ); + sqlite3_mutex_leave(mutexOpen); + } + assert( rc!=SQLITE_OK || sqlite3BtreeConnectionCount(*ppBtree)>0 ); + return rc; +} + +/* +** Decrement the BtShared.nRef counter. When it reaches zero, +** remove the BtShared structure from the sharing list. Return +** true if the BtShared.nRef counter reaches zero and return +** false if it is still positive. +*/ +static int removeFromSharingList(BtShared *pBt){ +#ifndef SQLITE_OMIT_SHARED_CACHE + MUTEX_LOGIC( sqlite3_mutex *pMainMtx; ) + BtShared *pList; + int removed = 0; + + assert( sqlite3_mutex_notheld(pBt->mutex) ); + MUTEX_LOGIC( pMainMtx = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) + sqlite3_mutex_enter(pMainMtx); + pBt->nRef--; + if( pBt->nRef<=0 ){ + if( GLOBAL(BtShared*,sqlite3SharedCacheList)==pBt ){ + GLOBAL(BtShared*,sqlite3SharedCacheList) = pBt->pNext; + }else{ + pList = GLOBAL(BtShared*,sqlite3SharedCacheList); + while( ALWAYS(pList) && pList->pNext!=pBt ){ + pList=pList->pNext; + } + if( ALWAYS(pList) ){ + pList->pNext = pBt->pNext; + } + } + if( SQLITE_THREADSAFE ){ + sqlite3_mutex_free(pBt->mutex); + } + removed = 1; + } + sqlite3_mutex_leave(pMainMtx); + return removed; +#else + return 1; +#endif +} + +/* +** Make sure pBt->pTmpSpace points to an allocation of +** MX_CELL_SIZE(pBt) bytes with a 4-byte prefix for a left-child +** pointer. +*/ +static void allocateTempSpace(BtShared *pBt){ + if( !pBt->pTmpSpace ){ + pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize ); + + /* One of the uses of pBt->pTmpSpace is to format cells before + ** inserting them into a leaf page (function fillInCell()). If + ** a cell is less than 4 bytes in size, it is rounded up to 4 bytes + ** by the various routines that manipulate binary cells. Which + ** can mean that fillInCell() only initializes the first 2 or 3 + ** bytes of pTmpSpace, but that the first 4 bytes are copied from + ** it into a database page. This is not actually a problem, but it + ** does cause a valgrind error when the 1 or 2 bytes of unitialized + ** data is passed to system call write(). So to avoid this error, + ** zero the first 4 bytes of temp space here. + ** + ** Also: Provide four bytes of initialized space before the + ** beginning of pTmpSpace as an area available to prepend the + ** left-child pointer to the beginning of a cell. + */ + if( pBt->pTmpSpace ){ + memset(pBt->pTmpSpace, 0, 8); + pBt->pTmpSpace += 4; + } + } +} + +/* +** Free the pBt->pTmpSpace allocation +*/ +static void freeTempSpace(BtShared *pBt){ + if( pBt->pTmpSpace ){ + pBt->pTmpSpace -= 4; + sqlite3PageFree(pBt->pTmpSpace); + pBt->pTmpSpace = 0; + } +} + +/* +** Close an open database and invalidate all cursors. +*/ +SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ + BtShared *pBt = p->pBt; + + /* Close all cursors opened via this handle. */ + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + + /* Verify that no other cursors have this Btree open */ +#ifdef SQLITE_DEBUG + { + BtCursor *pCur = pBt->pCursor; + while( pCur ){ + BtCursor *pTmp = pCur; + pCur = pCur->pNext; + assert( pTmp->pBtree!=p ); + + } + } +#endif + + /* Rollback any active transaction and free the handle structure. + ** The call to sqlite3BtreeRollback() drops any table-locks held by + ** this handle. + */ + sqlite3BtreeRollback(p, SQLITE_OK, 0); + sqlite3BtreeLeave(p); + + /* If there are still other outstanding references to the shared-btree + ** structure, return now. The remainder of this procedure cleans + ** up the shared-btree. + */ + assert( p->wantToLock==0 && p->locked==0 ); + if( !p->sharable || removeFromSharingList(pBt) ){ + /* The pBt is no longer on the sharing list, so we can access + ** it without having to hold the mutex. + ** + ** Clean out and delete the BtShared object. + */ + assert( !pBt->pCursor ); + sqlite3PagerClose(pBt->pPager, p->db); + if( pBt->xFreeSchema && pBt->pSchema ){ + pBt->xFreeSchema(pBt->pSchema); + } + sqlite3DbFree(0, pBt->pSchema); + freeTempSpace(pBt); + sqlite3_free(pBt); + } + +#ifndef SQLITE_OMIT_SHARED_CACHE + assert( p->wantToLock==0 ); + assert( p->locked==0 ); + if( p->pPrev ) p->pPrev->pNext = p->pNext; + if( p->pNext ) p->pNext->pPrev = p->pPrev; +#endif + + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Change the "soft" limit on the number of pages in the cache. +** Unused and unmodified pages will be recycled when the number of +** pages in the cache exceeds this soft limit. But the size of the +** cache is allowed to grow larger than this limit if it contains +** dirty pages or pages still in active use. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetCachesize(pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + +/* +** Change the "spill" limit on the number of pages in the cache. +** If the number of pages exceeds this limit during a write transaction, +** the pager might attempt to "spill" pages to the journal early in +** order to free up memory. +** +** The value returned is the current spill size. If zero is passed +** as an argument, no changes are made to the spill size setting, so +** using mxPage of 0 is a way to query the current spill size. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetSpillSize(Btree *p, int mxPage){ + BtShared *pBt = p->pBt; + int res; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + res = sqlite3PagerSetSpillsize(pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return res; +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapLimit(pBt->pPager, szMmap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* +** Change the way data is synced to disk in order to increase or decrease +** how well the database resists damage due to OS crashes and power +** failures. Level 1 is the same as asynchronous (no syncs() occur and +** there is a high probability of damage) Level 2 is the default. There +** is a very low but non-zero probability of damage. Level 3 reduces the +** probability of damage to near zero but with a write performance reduction. +*/ +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags( + Btree *p, /* The btree to set the safety level on */ + unsigned pgFlags /* Various PAGER_* flags */ +){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetFlags(pBt->pPager, pgFlags); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} +#endif + +/* +** Change the default pages size and the number of reserved bytes per page. +** Or, if the page size has already been fixed, return SQLITE_READONLY +** without changing anything. +** +** The page size must be a power of 2 between 512 and 65536. If the page +** size supplied does not meet this constraint then the page size is not +** changed. +** +** Page sizes are constrained to be a power of two so that the region +** of the database file used for locking (beginning at PENDING_BYTE, +** the first byte past the 1GB boundary, 0x40000000) needs to occur +** at the beginning of a page. +** +** If parameter nReserve is less than zero, then the number of reserved +** bytes per page is left unchanged. +** +** If the iFix!=0 then the BTS_PAGESIZE_FIXED flag is set so that the page size +** and autovacuum mode can no longer be changed. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ + int rc = SQLITE_OK; + int x; + BtShared *pBt = p->pBt; + assert( nReserve>=0 && nReserve<=255 ); + sqlite3BtreeEnter(p); + pBt->nReserveWanted = nReserve; + x = pBt->pageSize - pBt->usableSize; + if( nReservebtsFlags & BTS_PAGESIZE_FIXED ){ + sqlite3BtreeLeave(p); + return SQLITE_READONLY; + } + assert( nReserve>=0 && nReserve<=255 ); + if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && + ((pageSize-1)&pageSize)==0 ){ + assert( (pageSize & 7)==0 ); + assert( !pBt->pCursor ); + if( nReserve>32 && pageSize==512 ) pageSize = 1024; + pBt->pageSize = (u32)pageSize; + freeTempSpace(pBt); + } + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, nReserve); + pBt->usableSize = pBt->pageSize - (u16)nReserve; + if( iFix ) pBt->btsFlags |= BTS_PAGESIZE_FIXED; + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Return the currently defined page size +*/ +SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree *p){ + return p->pBt->pageSize; +} + +/* +** This function is similar to sqlite3BtreeGetReserve(), except that it +** may only be called if it is guaranteed that the b-tree mutex is already +** held. +** +** This is useful in one special case in the backup API code where it is +** known that the shared b-tree mutex is held, but the mutex on the +** database handle that owns *p is not. In this case if sqlite3BtreeEnter() +** were to be called, it might collide with some other operation on the +** database handle that owns *p, causing undefined behavior. +*/ +SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p){ + int n; + assert( sqlite3_mutex_held(p->pBt->mutex) ); + n = p->pBt->pageSize - p->pBt->usableSize; + return n; +} + +/* +** Return the number of bytes of space at the end of every page that +** are intentually left unused. This is the "reserved" space that is +** sometimes used by extensions. +** +** The value returned is the larger of the current reserve size and +** the latest reserve size requested by SQLITE_FILECTRL_RESERVE_BYTES. +** The amount of reserve can only grow - never shrink. +*/ +SQLITE_PRIVATE int sqlite3BtreeGetRequestedReserve(Btree *p){ + int n1, n2; + sqlite3BtreeEnter(p); + n1 = (int)p->pBt->nReserveWanted; + n2 = sqlite3BtreeGetReserveNoMutex(p); + sqlite3BtreeLeave(p); + return n1>n2 ? n1 : n2; +} + + +/* +** Set the maximum page count for a database if mxPage is positive. +** No changes are made if mxPage is 0 or negative. +** Regardless of the value of mxPage, return the maximum page count. +*/ +SQLITE_PRIVATE Pgno sqlite3BtreeMaxPageCount(Btree *p, Pgno mxPage){ + Pgno n; + sqlite3BtreeEnter(p); + n = sqlite3PagerMaxPageCount(p->pBt->pPager, mxPage); + sqlite3BtreeLeave(p); + return n; +} + +/* +** Change the values for the BTS_SECURE_DELETE and BTS_OVERWRITE flags: +** +** newFlag==0 Both BTS_SECURE_DELETE and BTS_OVERWRITE are cleared +** newFlag==1 BTS_SECURE_DELETE set and BTS_OVERWRITE is cleared +** newFlag==2 BTS_SECURE_DELETE cleared and BTS_OVERWRITE is set +** newFlag==(-1) No changes +** +** This routine acts as a query if newFlag is less than zero +** +** With BTS_OVERWRITE set, deleted content is overwritten by zeros, but +** freelist leaf pages are not written back to the database. Thus in-page +** deleted content is cleared, but freelist deleted content is not. +** +** With BTS_SECURE_DELETE, operation is like BTS_OVERWRITE with the addition +** that freelist leaf pages are written back into the database, increasing +** the amount of disk I/O. +*/ +SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ + int b; + if( p==0 ) return 0; + sqlite3BtreeEnter(p); + assert( BTS_OVERWRITE==BTS_SECURE_DELETE*2 ); + assert( BTS_FAST_SECURE==(BTS_OVERWRITE|BTS_SECURE_DELETE) ); + if( newFlag>=0 ){ + p->pBt->btsFlags &= ~BTS_FAST_SECURE; + p->pBt->btsFlags |= BTS_SECURE_DELETE*newFlag; + } + b = (p->pBt->btsFlags & BTS_FAST_SECURE)/BTS_SECURE_DELETE; + sqlite3BtreeLeave(p); + return b; +} + +/* +** Change the 'auto-vacuum' property of the database. If the 'autoVacuum' +** parameter is non-zero, then auto-vacuum mode is enabled. If zero, it +** is disabled. The default value for the auto-vacuum property is +** determined by the SQLITE_DEFAULT_AUTOVACUUM macro. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ +#ifdef SQLITE_OMIT_AUTOVACUUM + return SQLITE_READONLY; +#else + BtShared *pBt = p->pBt; + int rc = SQLITE_OK; + u8 av = (u8)autoVacuum; + + sqlite3BtreeEnter(p); + if( (pBt->btsFlags & BTS_PAGESIZE_FIXED)!=0 && (av ?1:0)!=pBt->autoVacuum ){ + rc = SQLITE_READONLY; + }else{ + pBt->autoVacuum = av ?1:0; + pBt->incrVacuum = av==2 ?1:0; + } + sqlite3BtreeLeave(p); + return rc; +#endif +} + +/* +** Return the value of the 'auto-vacuum' property. If auto-vacuum is +** enabled 1 is returned. Otherwise 0. +*/ +SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *p){ +#ifdef SQLITE_OMIT_AUTOVACUUM + return BTREE_AUTOVACUUM_NONE; +#else + int rc; + sqlite3BtreeEnter(p); + rc = ( + (!p->pBt->autoVacuum)?BTREE_AUTOVACUUM_NONE: + (!p->pBt->incrVacuum)?BTREE_AUTOVACUUM_FULL: + BTREE_AUTOVACUUM_INCR + ); + sqlite3BtreeLeave(p); + return rc; +#endif +} + +/* +** If the user has not set the safety-level for this database connection +** using "PRAGMA synchronous", and if the safety-level is not already +** set to the value passed to this function as the second parameter, +** set it so. +*/ +#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS \ + && !defined(SQLITE_OMIT_WAL) +static void setDefaultSyncFlag(BtShared *pBt, u8 safety_level){ + sqlite3 *db; + Db *pDb; + if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ + while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } + if( pDb->bSyncSet==0 + && pDb->safety_level!=safety_level + && pDb!=&db->aDb[1] + ){ + pDb->safety_level = safety_level; + sqlite3PagerSetFlags(pBt->pPager, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); + } + } +} +#else +# define setDefaultSyncFlag(pBt,safety_level) +#endif + +/* Forward declaration */ +static int newDatabase(BtShared*); + + +/* +** Get a reference to pPage1 of the database file. This will +** also acquire a readlock on that file. +** +** SQLITE_OK is returned on success. If the file is not a +** well-formed database file, then SQLITE_CORRUPT is returned. +** SQLITE_BUSY is returned if the database is locked. SQLITE_NOMEM +** is returned if we run out of memory. +*/ +static int lockBtree(BtShared *pBt){ + int rc; /* Result code from subfunctions */ + MemPage *pPage1; /* Page 1 of the database file */ + u32 nPage; /* Number of pages in the database */ + u32 nPageFile = 0; /* Number of pages in the database file */ + u32 nPageHeader; /* Number of pages in the database according to hdr */ + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pBt->pPage1==0 ); + rc = sqlite3PagerSharedLock(pBt->pPager); + if( rc!=SQLITE_OK ) return rc; + rc = btreeGetPage(pBt, 1, &pPage1, 0); + if( rc!=SQLITE_OK ) return rc; + + /* Do some checking to help insure the file we opened really is + ** a valid database file. + */ + nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); + sqlite3PagerPagecount(pBt->pPager, (int*)&nPageFile); + if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ + nPage = nPageFile; + } + if( (pBt->db->flags & SQLITE_ResetDatabase)!=0 ){ + nPage = 0; + } + if( nPage>0 ){ + u32 pageSize; + u32 usableSize; + u8 *page1 = pPage1->aData; + rc = SQLITE_NOTADB; + /* EVIDENCE-OF: R-43737-39999 Every valid SQLite database file begins + ** with the following 16 bytes (in hex): 53 51 4c 69 74 65 20 66 6f 72 6d + ** 61 74 20 33 00. */ + if( memcmp(page1, zMagicHeader, 16)!=0 ){ + goto page1_init_failed; + } + +#ifdef SQLITE_OMIT_WAL + if( page1[18]>1 ){ + pBt->btsFlags |= BTS_READ_ONLY; + } + if( page1[19]>1 ){ + goto page1_init_failed; + } +#else + if( page1[18]>2 ){ + pBt->btsFlags |= BTS_READ_ONLY; + } + if( page1[19]>2 ){ + goto page1_init_failed; + } + + /* If the read version is set to 2, this database should be accessed + ** in WAL mode. If the log is not already open, open it now. Then + ** return SQLITE_OK and return without populating BtShared.pPage1. + ** The caller detects this and calls this function again. This is + ** required as the version of page 1 currently in the page1 buffer + ** may not be the latest version - there may be a newer one in the log + ** file. + */ + if( page1[19]==2 && (pBt->btsFlags & BTS_NO_WAL)==0 ){ + int isOpen = 0; + rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); + if( rc!=SQLITE_OK ){ + goto page1_init_failed; + }else{ + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_WAL_SYNCHRONOUS+1); + if( isOpen==0 ){ + releasePageOne(pPage1); + return SQLITE_OK; + } + } + rc = SQLITE_NOTADB; + }else{ + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_SYNCHRONOUS+1); + } +#endif + + /* EVIDENCE-OF: R-15465-20813 The maximum and minimum embedded payload + ** fractions and the leaf payload fraction values must be 64, 32, and 32. + ** + ** The original design allowed these amounts to vary, but as of + ** version 3.6.0, we require them to be fixed. + */ + if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ + goto page1_init_failed; + } + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ + pageSize = (page1[16]<<8) | (page1[17]<<16); + /* EVIDENCE-OF: R-25008-21688 The size of a page is a power of two + ** between 512 and 65536 inclusive. */ + if( ((pageSize-1)&pageSize)!=0 + || pageSize>SQLITE_MAX_PAGE_SIZE + || pageSize<=256 + ){ + goto page1_init_failed; + } + pBt->btsFlags |= BTS_PAGESIZE_FIXED; + assert( (pageSize & 7)==0 ); + /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte + ** integer at offset 20 is the number of bytes of space at the end of + ** each page to reserve for extensions. + ** + ** EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ + usableSize = pageSize - page1[20]; + if( (u32)pageSize!=pBt->pageSize ){ + /* After reading the first page of the database assuming a page size + ** of BtShared.pageSize, we have discovered that the page-size is + ** actually pageSize. Unlock the database, leave pBt->pPage1 at + ** zero and return SQLITE_OK. The caller will call this function + ** again with the correct page-size. + */ + releasePageOne(pPage1); + pBt->usableSize = usableSize; + pBt->pageSize = pageSize; + freeTempSpace(pBt); + rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, + pageSize-usableSize); + return rc; + } + if( sqlite3WritableSchema(pBt->db)==0 && nPage>nPageFile ){ + rc = SQLITE_CORRUPT_BKPT; + goto page1_init_failed; + } + /* EVIDENCE-OF: R-28312-64704 However, the usable size is not allowed to + ** be less than 480. In other words, if the page size is 512, then the + ** reserved space size cannot exceed 32. */ + if( usableSize<480 ){ + goto page1_init_failed; + } + pBt->pageSize = pageSize; + pBt->usableSize = usableSize; +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->autoVacuum = (get4byte(&page1[36 + 4*4])?1:0); + pBt->incrVacuum = (get4byte(&page1[36 + 7*4])?1:0); +#endif + } + + /* maxLocal is the maximum amount of payload to store locally for + ** a cell. Make sure it is small enough so that at least minFanout + ** cells can will fit on one page. We assume a 10-byte page header. + ** Besides the payload, the cell must store: + ** 2-byte pointer to the cell + ** 4-byte child pointer + ** 9-byte nKey value + ** 4-byte nData value + ** 4-byte overflow page pointer + ** So a cell consists of a 2-byte pointer, a header which is as much as + ** 17 bytes long, 0 to N bytes of payload, and an optional 4 byte overflow + ** page pointer. + */ + pBt->maxLocal = (u16)((pBt->usableSize-12)*64/255 - 23); + pBt->minLocal = (u16)((pBt->usableSize-12)*32/255 - 23); + pBt->maxLeaf = (u16)(pBt->usableSize - 35); + pBt->minLeaf = (u16)((pBt->usableSize-12)*32/255 - 23); + if( pBt->maxLocal>127 ){ + pBt->max1bytePayload = 127; + }else{ + pBt->max1bytePayload = (u8)pBt->maxLocal; + } + assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) ); + pBt->pPage1 = pPage1; + pBt->nPage = nPage; + return SQLITE_OK; + +page1_init_failed: + releasePageOne(pPage1); + pBt->pPage1 = 0; + return rc; +} + +#ifndef NDEBUG +/* +** Return the number of cursors open on pBt. This is for use +** in assert() expressions, so it is only compiled if NDEBUG is not +** defined. +** +** Only write cursors are counted if wrOnly is true. If wrOnly is +** false then all cursors are counted. +** +** For the purposes of this routine, a cursor is any cursor that +** is capable of reading or writing to the database. Cursors that +** have been tripped into the CURSOR_FAULT state are not counted. +*/ +static int countValidCursors(BtShared *pBt, int wrOnly){ + BtCursor *pCur; + int r = 0; + for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ + if( (wrOnly==0 || (pCur->curFlags & BTCF_WriteFlag)!=0) + && pCur->eState!=CURSOR_FAULT ) r++; + } + return r; +} +#endif + +/* +** If there are no outstanding cursors and we are not in the middle +** of a transaction but there is a read lock on the database, then +** this routine unrefs the first page of the database file which +** has the effect of releasing the read lock. +** +** If there is a transaction in progress, this routine is a no-op. +*/ +static void unlockBtreeIfUnused(BtShared *pBt){ + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE ); + if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ + MemPage *pPage1 = pBt->pPage1; + assert( pPage1->aData ); + assert( sqlite3PagerRefcount(pBt->pPager)==1 ); + pBt->pPage1 = 0; + releasePageOne(pPage1); + } +} + +/* +** If pBt points to an empty file then convert that empty file +** into a new empty database by initializing the first page of +** the database. +*/ +static int newDatabase(BtShared *pBt){ + MemPage *pP1; + unsigned char *data; + int rc; + + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pBt->nPage>0 ){ + return SQLITE_OK; + } + pP1 = pBt->pPage1; + assert( pP1!=0 ); + data = pP1->aData; + rc = sqlite3PagerWrite(pP1->pDbPage); + if( rc ) return rc; + memcpy(data, zMagicHeader, sizeof(zMagicHeader)); + assert( sizeof(zMagicHeader)==16 ); + data[16] = (u8)((pBt->pageSize>>8)&0xff); + data[17] = (u8)((pBt->pageSize>>16)&0xff); + data[18] = 1; + data[19] = 1; + assert( pBt->usableSize<=pBt->pageSize && pBt->usableSize+255>=pBt->pageSize); + data[20] = (u8)(pBt->pageSize - pBt->usableSize); + data[21] = 64; + data[22] = 32; + data[23] = 32; + memset(&data[24], 0, 100-24); + zeroPage(pP1, PTF_INTKEY|PTF_LEAF|PTF_LEAFDATA ); + pBt->btsFlags |= BTS_PAGESIZE_FIXED; +#ifndef SQLITE_OMIT_AUTOVACUUM + assert( pBt->autoVacuum==1 || pBt->autoVacuum==0 ); + assert( pBt->incrVacuum==1 || pBt->incrVacuum==0 ); + put4byte(&data[36 + 4*4], pBt->autoVacuum); + put4byte(&data[36 + 7*4], pBt->incrVacuum); +#endif + pBt->nPage = 1; + data[31] = 1; + return SQLITE_OK; +} + +/* +** Initialize the first page of the database file (creating a database +** consisting of a single page and no schema objects). Return SQLITE_OK +** if successful, or an SQLite error code otherwise. +*/ +SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ + int rc; + sqlite3BtreeEnter(p); + p->pBt->nPage = 0; + rc = newDatabase(p->pBt); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Attempt to start a new transaction. A write-transaction +** is started if the second argument is nonzero, otherwise a read- +** transaction. If the second argument is 2 or more and exclusive +** transaction is started, meaning that no other process is allowed +** to access the database. A preexisting transaction may not be +** upgraded to exclusive by calling this routine a second time - the +** exclusivity flag only works for a new transaction. +** +** A write-transaction must be started before attempting any +** changes to the database. None of the following routines +** will work unless a transaction is started first: +** +** sqlite3BtreeCreateTable() +** sqlite3BtreeCreateIndex() +** sqlite3BtreeClearTable() +** sqlite3BtreeDropTable() +** sqlite3BtreeInsert() +** sqlite3BtreeDelete() +** sqlite3BtreeUpdateMeta() +** +** If an initial attempt to acquire the lock fails because of lock contention +** and the database was previously unlocked, then invoke the busy handler +** if there is one. But if there was previously a read-lock, do not +** invoke the busy handler - just return SQLITE_BUSY. SQLITE_BUSY is +** returned when there is already a read-lock in order to avoid a deadlock. +** +** Suppose there are two processes A and B. A has a read lock and B has +** a reserved lock. B tries to promote to exclusive but is blocked because +** of A's read lock. A tries to promote to reserved but is blocked by B. +** One or the other of the two processes must give way or there can be +** no progress. By returning SQLITE_BUSY and not invoking the busy callback +** when A already has a read lock, we encourage A to give up and let B +** proceed. +*/ +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ + BtShared *pBt = p->pBt; + Pager *pPager = pBt->pPager; + int rc = SQLITE_OK; + + sqlite3BtreeEnter(p); + btreeIntegrity(p); + + /* If the btree is already in a write-transaction, or it + ** is already in a read-transaction and a read-transaction + ** is requested, this is a no-op. + */ + if( p->inTrans==TRANS_WRITE || (p->inTrans==TRANS_READ && !wrflag) ){ + goto trans_begun; + } + assert( pBt->inTransaction==TRANS_WRITE || IfNotOmitAV(pBt->bDoTruncate)==0 ); + + if( (p->db->flags & SQLITE_ResetDatabase) + && sqlite3PagerIsreadonly(pPager)==0 + ){ + pBt->btsFlags &= ~BTS_READ_ONLY; + } + + /* Write transactions are not possible on a read-only database */ + if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){ + rc = SQLITE_READONLY; + goto trans_begun; + } + +#ifndef SQLITE_OMIT_SHARED_CACHE + { + sqlite3 *pBlock = 0; + /* If another database handle has already opened a write transaction + ** on this shared-btree structure and a second write transaction is + ** requested, return SQLITE_LOCKED. + */ + if( (wrflag && pBt->inTransaction==TRANS_WRITE) + || (pBt->btsFlags & BTS_PENDING)!=0 + ){ + pBlock = pBt->pWriter->db; + }else if( wrflag>1 ){ + BtLock *pIter; + for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ + if( pIter->pBtree!=p ){ + pBlock = pIter->pBtree->db; + break; + } + } + } + if( pBlock ){ + sqlite3ConnectionBlocked(p->db, pBlock); + rc = SQLITE_LOCKED_SHAREDCACHE; + goto trans_begun; + } + } +#endif + + /* Any read-only or read-write transaction implies a read-lock on + ** page 1. So if some other shared-cache client already has a write-lock + ** on page 1, the transaction cannot be opened. */ + rc = querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK); + if( SQLITE_OK!=rc ) goto trans_begun; + + pBt->btsFlags &= ~BTS_INITIALLY_EMPTY; + if( pBt->nPage==0 ) pBt->btsFlags |= BTS_INITIALLY_EMPTY; + do { + sqlite3PagerWalDb(pPager, p->db); + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* If transitioning from no transaction directly to a write transaction, + ** block for the WRITER lock first if possible. */ + if( pBt->pPage1==0 && wrflag ){ + assert( pBt->inTransaction==TRANS_NONE ); + rc = sqlite3PagerWalWriteLock(pPager, 1); + if( rc!=SQLITE_BUSY && rc!=SQLITE_OK ) break; + } +#endif + + /* Call lockBtree() until either pBt->pPage1 is populated or + ** lockBtree() returns something other than SQLITE_OK. lockBtree() + ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after + ** reading page 1 it discovers that the page-size of the database + ** file is not pBt->pageSize. In this case lockBtree() will update + ** pBt->pageSize to the page-size of the file on disk. + */ + while( pBt->pPage1==0 && SQLITE_OK==(rc = lockBtree(pBt)) ); + + if( rc==SQLITE_OK && wrflag ){ + if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ + rc = SQLITE_READONLY; + }else{ + rc = sqlite3PagerBegin(pPager, wrflag>1, sqlite3TempInMemory(p->db)); + if( rc==SQLITE_OK ){ + rc = newDatabase(pBt); + }else if( rc==SQLITE_BUSY_SNAPSHOT && pBt->inTransaction==TRANS_NONE ){ + /* if there was no transaction opened when this function was + ** called and SQLITE_BUSY_SNAPSHOT is returned, change the error + ** code to SQLITE_BUSY. */ + rc = SQLITE_BUSY; + } + } + } + + if( rc!=SQLITE_OK ){ + (void)sqlite3PagerWalWriteLock(pPager, 0); + unlockBtreeIfUnused(pBt); + } + }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && + btreeInvokeBusyHandler(pBt) ); + sqlite3PagerWalDb(pPager, 0); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; +#endif + + if( rc==SQLITE_OK ){ + if( p->inTrans==TRANS_NONE ){ + pBt->nTransaction++; +#ifndef SQLITE_OMIT_SHARED_CACHE + if( p->sharable ){ + assert( p->lock.pBtree==p && p->lock.iTable==1 ); + p->lock.eLock = READ_LOCK; + p->lock.pNext = pBt->pLock; + pBt->pLock = &p->lock; + } +#endif + } + p->inTrans = (wrflag?TRANS_WRITE:TRANS_READ); + if( p->inTrans>pBt->inTransaction ){ + pBt->inTransaction = p->inTrans; + } + if( wrflag ){ + MemPage *pPage1 = pBt->pPage1; +#ifndef SQLITE_OMIT_SHARED_CACHE + assert( !pBt->pWriter ); + pBt->pWriter = p; + pBt->btsFlags &= ~BTS_EXCLUSIVE; + if( wrflag>1 ) pBt->btsFlags |= BTS_EXCLUSIVE; +#endif + + /* If the db-size header field is incorrect (as it may be if an old + ** client has been writing the database file), update it now. Doing + ** this sooner rather than later means the database size can safely + ** re-read the database size from page 1 if a savepoint or transaction + ** rollback occurs within the transaction. + */ + if( pBt->nPage!=get4byte(&pPage1->aData[28]) ){ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pPage1->aData[28], pBt->nPage); + } + } + } + } + +trans_begun: + if( rc==SQLITE_OK ){ + if( pSchemaVersion ){ + *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]); + } + if( wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + rc = sqlite3PagerOpenSavepoint(pPager, p->db->nSavepoint); + } + } + + btreeIntegrity(p); + sqlite3BtreeLeave(p); + return rc; +} + +#ifndef SQLITE_OMIT_AUTOVACUUM + +/* +** Set the pointer-map entries for all children of page pPage. Also, if +** pPage contains cells that point to overflow pages, set the pointer +** map entries for the overflow pages as well. +*/ +static int setChildPtrmaps(MemPage *pPage){ + int i; /* Counter variable */ + int nCell; /* Number of cells in page pPage */ + int rc; /* Return code */ + BtShared *pBt = pPage->pBt; + Pgno pgno = pPage->pgno; + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + rc = pPage->isInit ? SQLITE_OK : btreeInitPage(pPage); + if( rc!=SQLITE_OK ) return rc; + nCell = pPage->nCell; + + for(i=0; ileaf ){ + Pgno childPgno = get4byte(pCell); + ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno, &rc); + } + } + + if( !pPage->leaf ){ + Pgno childPgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno, &rc); + } + + return rc; +} + +/* +** Somewhere on pPage is a pointer to page iFrom. Modify this pointer so +** that it points to iTo. Parameter eType describes the type of pointer to +** be modified, as follows: +** +** PTRMAP_BTREE: pPage is a btree-page. The pointer points at a child +** page of pPage. +** +** PTRMAP_OVERFLOW1: pPage is a btree-page. The pointer points at an overflow +** page pointed to by one of the cells on pPage. +** +** PTRMAP_OVERFLOW2: pPage is an overflow-page. The pointer points at the next +** overflow page in the list. +*/ +static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + if( eType==PTRMAP_OVERFLOW2 ){ + /* The pointer is always the first 4 bytes of the page in this case. */ + if( get4byte(pPage->aData)!=iFrom ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + put4byte(pPage->aData, iTo); + }else{ + int i; + int nCell; + int rc; + + rc = pPage->isInit ? SQLITE_OK : btreeInitPage(pPage); + if( rc ) return rc; + nCell = pPage->nCell; + + for(i=0; ixParseCell(pPage, pCell, &info); + if( info.nLocal pPage->aData+pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + if( iFrom==get4byte(pCell+info.nSize-4) ){ + put4byte(pCell+info.nSize-4, iTo); + break; + } + } + }else{ + if( get4byte(pCell)==iFrom ){ + put4byte(pCell, iTo); + break; + } + } + } + + if( i==nCell ){ + if( eType!=PTRMAP_BTREE || + get4byte(&pPage->aData[pPage->hdrOffset+8])!=iFrom ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + put4byte(&pPage->aData[pPage->hdrOffset+8], iTo); + } + } + return SQLITE_OK; +} + + +/* +** Move the open database page pDbPage to location iFreePage in the +** database. The pDbPage reference remains valid. +** +** The isCommit flag indicates that there is no need to remember that +** the journal needs to be sync()ed before database page pDbPage->pgno +** can be written to. The caller has already promised not to write to that +** page. +*/ +static int relocatePage( + BtShared *pBt, /* Btree */ + MemPage *pDbPage, /* Open page to move */ + u8 eType, /* Pointer map 'type' entry for pDbPage */ + Pgno iPtrPage, /* Pointer map 'page-no' entry for pDbPage */ + Pgno iFreePage, /* The location to move pDbPage to */ + int isCommit /* isCommit flag passed to sqlite3PagerMovepage */ +){ + MemPage *pPtrPage; /* The page that contains a pointer to pDbPage */ + Pgno iDbPage = pDbPage->pgno; + Pager *pPager = pBt->pPager; + int rc; + + assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 || + eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ); + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( pDbPage->pBt==pBt ); + if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT; + + /* Move page iDbPage from its current location to page number iFreePage */ + TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", + iDbPage, iFreePage, iPtrPage, eType)); + rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); + if( rc!=SQLITE_OK ){ + return rc; + } + pDbPage->pgno = iFreePage; + + /* If pDbPage was a btree-page, then it may have child pages and/or cells + ** that point to overflow pages. The pointer map entries for all these + ** pages need to be changed. + ** + ** If pDbPage is an overflow page, then the first 4 bytes may store a + ** pointer to a subsequent overflow page. If this is the case, then + ** the pointer map needs to be updated for the subsequent overflow page. + */ + if( eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ){ + rc = setChildPtrmaps(pDbPage); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + Pgno nextOvfl = get4byte(pDbPage->aData); + if( nextOvfl!=0 ){ + ptrmapPut(pBt, nextOvfl, PTRMAP_OVERFLOW2, iFreePage, &rc); + if( rc!=SQLITE_OK ){ + return rc; + } + } + } + + /* Fix the database pointer on page iPtrPage that pointed at iDbPage so + ** that it points at iFreePage. Also fix the pointer map entry for + ** iPtrPage. + */ + if( eType!=PTRMAP_ROOTPAGE ){ + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = sqlite3PagerWrite(pPtrPage->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pPtrPage); + return rc; + } + rc = modifyPagePointer(pPtrPage, iDbPage, iFreePage, eType); + releasePage(pPtrPage); + if( rc==SQLITE_OK ){ + ptrmapPut(pBt, iFreePage, eType, iPtrPage, &rc); + } + } + return rc; +} + +/* Forward declaration required by incrVacuumStep(). */ +static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); + +/* +** Perform a single step of an incremental-vacuum. If successful, return +** SQLITE_OK. If there is no work to do (and therefore no point in +** calling this function again), return SQLITE_DONE. Or, if an error +** occurs, return some other error code. +** +** More specifically, this function attempts to re-organize the database so +** that the last page of the file currently in use is no longer in use. +** +** Parameter nFin is the number of pages that this database would contain +** were this function called until it returns SQLITE_DONE. +** +** If the bCommit parameter is non-zero, this function assumes that the +** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE +** or an error. bCommit is passed true for an auto-vacuum-on-commit +** operation, or false for an incremental vacuum. +*/ +static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ + Pgno nFreeList; /* Number of pages still on the free-list */ + int rc; + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( iLastPg>nFin ); + + if( !PTRMAP_ISPAGE(pBt, iLastPg) && iLastPg!=PENDING_BYTE_PAGE(pBt) ){ + u8 eType; + Pgno iPtrPage; + + nFreeList = get4byte(&pBt->pPage1->aData[36]); + if( nFreeList==0 ){ + return SQLITE_DONE; + } + + rc = ptrmapGet(pBt, iLastPg, &eType, &iPtrPage); + if( rc!=SQLITE_OK ){ + return rc; + } + if( eType==PTRMAP_ROOTPAGE ){ + return SQLITE_CORRUPT_BKPT; + } + + if( eType==PTRMAP_FREEPAGE ){ + if( bCommit==0 ){ + /* Remove the page from the files free-list. This is not required + ** if bCommit is non-zero. In that case, the free-list will be + ** truncated to zero after this function returns, so it doesn't + ** matter if it still contains some garbage entries. + */ + Pgno iFreePg; + MemPage *pFreePg; + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iLastPg, BTALLOC_EXACT); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( iFreePg==iLastPg ); + releasePage(pFreePg); + } + } else { + Pgno iFreePg; /* Index of free page to move pLastPg to */ + MemPage *pLastPg; + u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ + Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ + + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* If bCommit is zero, this loop runs exactly once and page pLastPg + ** is swapped with the first free page pulled off the free list. + ** + ** On the other hand, if bCommit is greater than zero, then keep + ** looping until a free-page located within the first nFin pages + ** of the file is found. + */ + if( bCommit==0 ){ + eMode = BTALLOC_LE; + iNear = nFin; + } + do { + MemPage *pFreePg; + rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iNear, eMode); + if( rc!=SQLITE_OK ){ + releasePage(pLastPg); + return rc; + } + releasePage(pFreePg); + }while( bCommit && iFreePg>nFin ); + assert( iFreePgbDoTruncate = 1; + pBt->nPage = iLastPg; + } + return SQLITE_OK; +} + +/* +** The database opened by the first argument is an auto-vacuum database +** nOrig pages in size containing nFree free pages. Return the expected +** size of the database in pages following an auto-vacuum operation. +*/ +static Pgno finalDbSize(BtShared *pBt, Pgno nOrig, Pgno nFree){ + int nEntry; /* Number of entries on one ptrmap page */ + Pgno nPtrmap; /* Number of PtrMap pages to be freed */ + Pgno nFin; /* Return value */ + + nEntry = pBt->usableSize/5; + nPtrmap = (nFree-nOrig+PTRMAP_PAGENO(pBt, nOrig)+nEntry)/nEntry; + nFin = nOrig - nFree - nPtrmap; + if( nOrig>PENDING_BYTE_PAGE(pBt) && nFinpBt; + + sqlite3BtreeEnter(p); + assert( pBt->inTransaction==TRANS_WRITE && p->inTrans==TRANS_WRITE ); + if( !pBt->autoVacuum ){ + rc = SQLITE_DONE; + }else{ + Pgno nOrig = btreePagecount(pBt); + Pgno nFree = get4byte(&pBt->pPage1->aData[36]); + Pgno nFin = finalDbSize(pBt, nOrig, nFree); + + if( nOrig=nOrig ){ + rc = SQLITE_CORRUPT_BKPT; + }else if( nFree>0 ){ + rc = saveAllCursors(pBt, 0, 0); + if( rc==SQLITE_OK ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[28], pBt->nPage); + } + }else{ + rc = SQLITE_DONE; + } + } + sqlite3BtreeLeave(p); + return rc; +} + +/* +** This routine is called prior to sqlite3PagerCommit when a transaction +** is committed for an auto-vacuum database. +** +** If SQLITE_OK is returned, then *pnTrunc is set to the number of pages +** the database file should be truncated to during the commit process. +** i.e. the database has been reorganized so that only the first *pnTrunc +** pages are in use. +*/ +static int autoVacuumCommit(BtShared *pBt){ + int rc = SQLITE_OK; + Pager *pPager = pBt->pPager; + VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager); ) + + assert( sqlite3_mutex_held(pBt->mutex) ); + invalidateAllOverflowCache(pBt); + assert(pBt->autoVacuum); + if( !pBt->incrVacuum ){ + Pgno nFin; /* Number of pages in database after autovacuuming */ + Pgno nFree; /* Number of pages on the freelist initially */ + Pgno iFree; /* The next page to be freed */ + Pgno nOrig; /* Database size before freeing */ + + nOrig = btreePagecount(pBt); + if( PTRMAP_ISPAGE(pBt, nOrig) || nOrig==PENDING_BYTE_PAGE(pBt) ){ + /* It is not possible to create a database for which the final page + ** is either a pointer-map page or the pending-byte page. If one + ** is encountered, this indicates corruption. + */ + return SQLITE_CORRUPT_BKPT; + } + + nFree = get4byte(&pBt->pPage1->aData[36]); + nFin = finalDbSize(pBt, nOrig, nFree); + if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT; + if( nFinnFin && rc==SQLITE_OK; iFree--){ + rc = incrVacuumStep(pBt, nFin, iFree, 1); + } + if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + put4byte(&pBt->pPage1->aData[32], 0); + put4byte(&pBt->pPage1->aData[36], 0); + put4byte(&pBt->pPage1->aData[28], nFin); + pBt->bDoTruncate = 1; + pBt->nPage = nFin; + } + if( rc!=SQLITE_OK ){ + sqlite3PagerRollback(pPager); + } + } + + assert( nRef>=sqlite3PagerRefcount(pPager) ); + return rc; +} + +#else /* ifndef SQLITE_OMIT_AUTOVACUUM */ +# define setChildPtrmaps(x) SQLITE_OK +#endif + +/* +** This routine does the first phase of a two-phase commit. This routine +** causes a rollback journal to be created (if it does not already exist) +** and populated with enough information so that if a power loss occurs +** the database can be restored to its original state by playing back +** the journal. Then the contents of the journal are flushed out to +** the disk. After the journal is safely on oxide, the changes to the +** database are written into the database file and flushed to oxide. +** At the end of this call, the rollback journal still exists on the +** disk and we are still holding all locks, so the transaction has not +** committed. See sqlite3BtreeCommitPhaseTwo() for the second phase of the +** commit process. +** +** This call is a no-op if no write-transaction is currently active on pBt. +** +** Otherwise, sync the database file for the btree pBt. zSuperJrnl points to +** the name of a super-journal file that should be written into the +** individual journal file, or is NULL, indicating no super-journal file +** (single database transaction). +** +** When this is called, the super-journal should already have been +** created, populated with this journal pointer and synced to disk. +** +** Once this is routine has returned, the only thing required to commit +** the write-transaction for this database file is to delete the journal. +*/ +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zSuperJrnl){ + int rc = SQLITE_OK; + if( p->inTrans==TRANS_WRITE ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + rc = autoVacuumCommit(pBt); + if( rc!=SQLITE_OK ){ + sqlite3BtreeLeave(p); + return rc; + } + } + if( pBt->bDoTruncate ){ + sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); + } +#endif + rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0); + sqlite3BtreeLeave(p); + } + return rc; +} + +/* +** This function is called from both BtreeCommitPhaseTwo() and BtreeRollback() +** at the conclusion of a transaction. +*/ +static void btreeEndTransaction(Btree *p){ + BtShared *pBt = p->pBt; + sqlite3 *db = p->db; + assert( sqlite3BtreeHoldsMutex(p) ); + +#ifndef SQLITE_OMIT_AUTOVACUUM + pBt->bDoTruncate = 0; +#endif + if( p->inTrans>TRANS_NONE && db->nVdbeRead>1 ){ + /* If there are other active statements that belong to this database + ** handle, downgrade to a read-only transaction. The other statements + ** may still be reading from the database. */ + downgradeAllSharedCacheTableLocks(p); + p->inTrans = TRANS_READ; + }else{ + /* If the handle had any kind of transaction open, decrement the + ** transaction count of the shared btree. If the transaction count + ** reaches 0, set the shared state to TRANS_NONE. The unlockBtreeIfUnused() + ** call below will unlock the pager. */ + if( p->inTrans!=TRANS_NONE ){ + clearAllSharedCacheTableLocks(p); + pBt->nTransaction--; + if( 0==pBt->nTransaction ){ + pBt->inTransaction = TRANS_NONE; + } + } + + /* Set the current transaction state to TRANS_NONE and unlock the + ** pager if this call closed the only read or write transaction. */ + p->inTrans = TRANS_NONE; + unlockBtreeIfUnused(pBt); + } + + btreeIntegrity(p); +} + +/* +** Commit the transaction currently in progress. +** +** This routine implements the second phase of a 2-phase commit. The +** sqlite3BtreeCommitPhaseOne() routine does the first phase and should +** be invoked prior to calling this routine. The sqlite3BtreeCommitPhaseOne() +** routine did all the work of writing information out to disk and flushing the +** contents so that they are written onto the disk platter. All this +** routine has to do is delete or truncate or zero the header in the +** the rollback journal (which causes the transaction to commit) and +** drop locks. +** +** Normally, if an error occurs while the pager layer is attempting to +** finalize the underlying journal file, this function returns an error and +** the upper layer will attempt a rollback. However, if the second argument +** is non-zero then this b-tree transaction is part of a multi-file +** transaction. In this case, the transaction has already been committed +** (by deleting a super-journal file) and the caller will ignore this +** functions return code. So, even if an error occurs in the pager layer, +** reset the b-tree objects internal state to indicate that the write +** transaction has been closed. This is quite safe, as the pager will have +** transitioned to the error state. +** +** This will release the write lock on the database file. If there +** are no active cursors, it also releases the read lock. +*/ +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ + + if( p->inTrans==TRANS_NONE ) return SQLITE_OK; + sqlite3BtreeEnter(p); + btreeIntegrity(p); + + /* If the handle has a write-transaction open, commit the shared-btrees + ** transaction and set the shared state to TRANS_READ. + */ + if( p->inTrans==TRANS_WRITE ){ + int rc; + BtShared *pBt = p->pBt; + assert( pBt->inTransaction==TRANS_WRITE ); + assert( pBt->nTransaction>0 ); + rc = sqlite3PagerCommitPhaseTwo(pBt->pPager); + if( rc!=SQLITE_OK && bCleanup==0 ){ + sqlite3BtreeLeave(p); + return rc; + } + p->iBDataVersion--; /* Compensate for pPager->iDataVersion++; */ + pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); + } + + btreeEndTransaction(p); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + +/* +** Do both phases of a commit. +*/ +SQLITE_PRIVATE int sqlite3BtreeCommit(Btree *p){ + int rc; + sqlite3BtreeEnter(p); + rc = sqlite3BtreeCommitPhaseOne(p, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeCommitPhaseTwo(p, 0); + } + sqlite3BtreeLeave(p); + return rc; +} + +/* +** This routine sets the state to CURSOR_FAULT and the error +** code to errCode for every cursor on any BtShared that pBtree +** references. Or if the writeOnly flag is set to 1, then only +** trip write cursors and leave read cursors unchanged. +** +** Every cursor is a candidate to be tripped, including cursors +** that belong to other database connections that happen to be +** sharing the cache with pBtree. +** +** This routine gets called when a rollback occurs. If the writeOnly +** flag is true, then only write-cursors need be tripped - read-only +** cursors save their current positions so that they may continue +** following the rollback. Or, if writeOnly is false, all cursors are +** tripped. In general, writeOnly is false if the transaction being +** rolled back modified the database schema. In this case b-tree root +** pages may be moved or deleted from the database altogether, making +** it unsafe for read cursors to continue. +** +** If the writeOnly flag is true and an error is encountered while +** saving the current position of a read-only cursor, all cursors, +** including all read-cursors are tripped. +** +** SQLITE_OK is returned if successful, or if an error occurs while +** saving a cursor position, an SQLite error code. +*/ +SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ + BtCursor *p; + int rc = SQLITE_OK; + + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); + if( pBtree ){ + sqlite3BtreeEnter(pBtree); + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ + if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){ + rc = saveCursorPosition(p); + if( rc!=SQLITE_OK ){ + (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); + break; + } + } + }else{ + sqlite3BtreeClearCursor(p); + p->eState = CURSOR_FAULT; + p->skipNext = errCode; + } + btreeReleaseAllCursorPages(p); + } + sqlite3BtreeLeave(pBtree); + } + return rc; +} + +/* +** Set the pBt->nPage field correctly, according to the current +** state of the database. Assume pBt->pPage1 is valid. +*/ +static void btreeSetNPage(BtShared *pBt, MemPage *pPage1){ + int nPage = get4byte(&pPage1->aData[28]); + testcase( nPage==0 ); + if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); + testcase( pBt->nPage!=nPage ); + pBt->nPage = nPage; +} + +/* +** Rollback the transaction in progress. +** +** If tripCode is not SQLITE_OK then cursors will be invalidated (tripped). +** Only write cursors are tripped if writeOnly is true but all cursors are +** tripped if writeOnly is false. Any attempt to use +** a tripped cursor will result in an error. +** +** This will release the write lock on the database file. If there +** are no active cursors, it also releases the read lock. +*/ +SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ + int rc; + BtShared *pBt = p->pBt; + MemPage *pPage1; + + assert( writeOnly==1 || writeOnly==0 ); + assert( tripCode==SQLITE_ABORT_ROLLBACK || tripCode==SQLITE_OK ); + sqlite3BtreeEnter(p); + if( tripCode==SQLITE_OK ){ + rc = tripCode = saveAllCursors(pBt, 0, 0); + if( rc ) writeOnly = 0; + }else{ + rc = SQLITE_OK; + } + if( tripCode ){ + int rc2 = sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + assert( rc==SQLITE_OK || (writeOnly==0 && rc2==SQLITE_OK) ); + if( rc2!=SQLITE_OK ) rc = rc2; + } + btreeIntegrity(p); + + if( p->inTrans==TRANS_WRITE ){ + int rc2; + + assert( TRANS_WRITE==pBt->inTransaction ); + rc2 = sqlite3PagerRollback(pBt->pPager); + if( rc2!=SQLITE_OK ){ + rc = rc2; + } + + /* The rollback may have destroyed the pPage1->aData value. So + ** call btreeGetPage() on page 1 again to make + ** sure pPage1->aData is set correctly. */ + if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + btreeSetNPage(pBt, pPage1); + releasePageOne(pPage1); + } + assert( countValidCursors(pBt, 1)==0 ); + pBt->inTransaction = TRANS_READ; + btreeClearHasContent(pBt); + } + + btreeEndTransaction(p); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Start a statement subtransaction. The subtransaction can be rolled +** back independently of the main transaction. You must start a transaction +** before starting a subtransaction. The subtransaction is ended automatically +** if the main transaction commits or rolls back. +** +** Statement subtransactions are used around individual SQL statements +** that are contained within a BEGIN...COMMIT block. If a constraint +** error occurs within the statement, the effect of that one statement +** can be rolled back without having to rollback the entire transaction. +** +** A statement sub-transaction is implemented as an anonymous savepoint. The +** value passed as the second parameter is the total number of savepoints, +** including the new anonymous savepoint, open on the B-Tree. i.e. if there +** are no active savepoints and no other statement-transactions open, +** iStatement is 1. This anonymous savepoint can be released or rolled back +** using the sqlite3BtreeSavepoint() function. +*/ +SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree *p, int iStatement){ + int rc; + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); + assert( iStatement>0 ); + assert( iStatement>p->db->nSavepoint ); + assert( pBt->inTransaction==TRANS_WRITE ); + /* At the pager level, a statement transaction is a savepoint with + ** an index greater than all savepoints created explicitly using + ** SQL statements. It is illegal to open, release or rollback any + ** such savepoints while the statement transaction savepoint is active. + */ + rc = sqlite3PagerOpenSavepoint(pBt->pPager, iStatement); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** The second argument to this function, op, is always SAVEPOINT_ROLLBACK +** or SAVEPOINT_RELEASE. This function either releases or rolls back the +** savepoint identified by parameter iSavepoint, depending on the value +** of op. +** +** Normally, iSavepoint is greater than or equal to zero. However, if op is +** SAVEPOINT_ROLLBACK, then iSavepoint may also be -1. In this case the +** contents of the entire transaction are rolled back. This is different +** from a normal transaction rollback, as no locks are released and the +** transaction remains open. +*/ +SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ + int rc = SQLITE_OK; + if( p && p->inTrans==TRANS_WRITE ){ + BtShared *pBt = p->pBt; + assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); + assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); + sqlite3BtreeEnter(p); + if( op==SAVEPOINT_ROLLBACK ){ + rc = saveAllCursors(pBt, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + } + if( rc==SQLITE_OK ){ + if( iSavepoint<0 && (pBt->btsFlags & BTS_INITIALLY_EMPTY)!=0 ){ + pBt->nPage = 0; + } + rc = newDatabase(pBt); + btreeSetNPage(pBt, pBt->pPage1); + + /* pBt->nPage might be zero if the database was corrupt when + ** the transaction was started. Otherwise, it must be at least 1. */ + assert( CORRUPT_DB || pBt->nPage>0 ); + } + sqlite3BtreeLeave(p); + } + return rc; +} + +/* +** Create a new cursor for the BTree whose root is on the page +** iTable. If a read-only cursor is requested, it is assumed that +** the caller already has at least a read-only transaction open +** on the database already. If a write-cursor is requested, then +** the caller is assumed to have an open write transaction. +** +** If the BTREE_WRCSR bit of wrFlag is clear, then the cursor can only +** be used for reading. If the BTREE_WRCSR bit is set, then the cursor +** can be used for reading or for writing if other conditions for writing +** are also met. These are the conditions that must be met in order +** for writing to be allowed: +** +** 1: The cursor must have been opened with wrFlag containing BTREE_WRCSR +** +** 2: Other database connections that share the same pager cache +** but which are not in the READ_UNCOMMITTED state may not have +** cursors open with wrFlag==0 on the same table. Otherwise +** the changes made by this write cursor would be visible to +** the read cursors in the other database connection. +** +** 3: The database must be writable (not on read-only media) +** +** 4: There must be an active transaction. +** +** The BTREE_FORDELETE bit of wrFlag may optionally be set if BTREE_WRCSR +** is set. If FORDELETE is set, that is a hint to the implementation that +** this cursor will only be used to seek to and delete entries of an index +** as part of a larger DELETE statement. The FORDELETE hint is not used by +** this implementation. But in a hypothetical alternative storage engine +** in which index entries are automatically deleted when corresponding table +** rows are deleted, the FORDELETE flag is a hint that all SEEK and DELETE +** operations on this cursor can be no-ops and all READ operations can +** return a null row (2-bytes: 0x01 0x00). +** +** No checking is done to make sure that page iTable really is the +** root page of a b-tree. If it is not, then the cursor acquired +** will not work correctly. +** +** It is assumed that the sqlite3BtreeCursorZero() has been called +** on pCur to initialize the memory space prior to invoking this routine. +*/ +static int btreeCursor( + Btree *p, /* The btree */ + Pgno iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to comparison function */ + BtCursor *pCur /* Space for new cursor */ +){ + BtShared *pBt = p->pBt; /* Shared b-tree handle */ + BtCursor *pX; /* Looping over other all cursors */ + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( wrFlag==0 + || wrFlag==BTREE_WRCSR + || wrFlag==(BTREE_WRCSR|BTREE_FORDELETE) + ); + + /* The following assert statements verify that if this is a sharable + ** b-tree database, the connection is holding the required table locks, + ** and that no other connection has any open cursor that conflicts with + ** this lock. The iTable<1 term disables the check for corrupt schemas. */ + assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) + || iTable<1 ); + assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); + + /* Assert that the caller has opened the required transaction. */ + assert( p->inTrans>TRANS_NONE ); + assert( wrFlag==0 || p->inTrans==TRANS_WRITE ); + assert( pBt->pPage1 && pBt->pPage1->aData ); + assert( wrFlag==0 || (pBt->btsFlags & BTS_READ_ONLY)==0 ); + + if( wrFlag ){ + allocateTempSpace(pBt); + if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM_BKPT; + } + if( iTable<=1 ){ + if( iTable<1 ){ + return SQLITE_CORRUPT_BKPT; + }else if( btreePagecount(pBt)==0 ){ + assert( wrFlag==0 ); + iTable = 0; + } + } + + /* Now that no other errors can occur, finish filling in the BtCursor + ** variables and link the cursor into the BtShared list. */ + pCur->pgnoRoot = iTable; + pCur->iPage = -1; + pCur->pKeyInfo = pKeyInfo; + pCur->pBtree = p; + pCur->pBt = pBt; + pCur->curFlags = wrFlag ? BTCF_WriteFlag : 0; + pCur->curPagerFlags = wrFlag ? 0 : PAGER_GET_READONLY; + /* If there are two or more cursors on the same btree, then all such + ** cursors *must* have the BTCF_Multiple flag set. */ + for(pX=pBt->pCursor; pX; pX=pX->pNext){ + if( pX->pgnoRoot==iTable ){ + pX->curFlags |= BTCF_Multiple; + pCur->curFlags |= BTCF_Multiple; + } + } + pCur->pNext = pBt->pCursor; + pBt->pCursor = pCur; + pCur->eState = CURSOR_INVALID; + return SQLITE_OK; +} +static int btreeCursorWithLock( + Btree *p, /* The btree */ + Pgno iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to comparison function */ + BtCursor *pCur /* Space for new cursor */ +){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + sqlite3BtreeLeave(p); + return rc; +} +SQLITE_PRIVATE int sqlite3BtreeCursor( + Btree *p, /* The btree */ + Pgno iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to xCompare() */ + BtCursor *pCur /* Write new cursor here */ +){ + if( p->sharable ){ + return btreeCursorWithLock(p, iTable, wrFlag, pKeyInfo, pCur); + }else{ + return btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + } +} + +/* +** Return the size of a BtCursor object in bytes. +** +** This interfaces is needed so that users of cursors can preallocate +** sufficient storage to hold a cursor. The BtCursor object is opaque +** to users so they cannot do the sizeof() themselves - they must call +** this routine. +*/ +SQLITE_PRIVATE int sqlite3BtreeCursorSize(void){ + return ROUND8(sizeof(BtCursor)); +} + +/* +** Initialize memory that will be converted into a BtCursor object. +** +** The simple approach here would be to memset() the entire object +** to zero. But it turns out that the apPage[] and aiIdx[] arrays +** do not need to be zeroed and they are large, so we can save a lot +** of run-time by skipping the initialization of those elements. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor *p){ + memset(p, 0, offsetof(BtCursor, BTCURSOR_FIRST_UNINIT)); +} + +/* +** Close a cursor. The read lock on the database file is released +** when the last cursor is closed. +*/ +SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ + Btree *pBtree = pCur->pBtree; + if( pBtree ){ + BtShared *pBt = pCur->pBt; + sqlite3BtreeEnter(pBtree); + assert( pBt->pCursor!=0 ); + if( pBt->pCursor==pCur ){ + pBt->pCursor = pCur->pNext; + }else{ + BtCursor *pPrev = pBt->pCursor; + do{ + if( pPrev->pNext==pCur ){ + pPrev->pNext = pCur->pNext; + break; + } + pPrev = pPrev->pNext; + }while( ALWAYS(pPrev) ); + } + btreeReleaseAllCursorPages(pCur); + unlockBtreeIfUnused(pBt); + sqlite3_free(pCur->aOverflow); + sqlite3_free(pCur->pKey); + if( (pBt->openFlags & BTREE_SINGLE) && pBt->pCursor==0 ){ + /* Since the BtShared is not sharable, there is no need to + ** worry about the missing sqlite3BtreeLeave() call here. */ + assert( pBtree->sharable==0 ); + sqlite3BtreeClose(pBtree); + }else{ + sqlite3BtreeLeave(pBtree); + } + pCur->pBtree = 0; + } + return SQLITE_OK; +} + +/* +** Make sure the BtCursor* given in the argument has a valid +** BtCursor.info structure. If it is not already valid, call +** btreeParseCell() to fill it in. +** +** BtCursor.info is a cache of the information in the current cell. +** Using this cache reduces the number of calls to btreeParseCell(). +*/ +#ifndef NDEBUG + static int cellInfoEqual(CellInfo *a, CellInfo *b){ + if( a->nKey!=b->nKey ) return 0; + if( a->pPayload!=b->pPayload ) return 0; + if( a->nPayload!=b->nPayload ) return 0; + if( a->nLocal!=b->nLocal ) return 0; + if( a->nSize!=b->nSize ) return 0; + return 1; + } + static void assertCellInfo(BtCursor *pCur){ + CellInfo info; + memset(&info, 0, sizeof(info)); + btreeParseCell(pCur->pPage, pCur->ix, &info); + assert( CORRUPT_DB || cellInfoEqual(&info, &pCur->info) ); + } +#else + #define assertCellInfo(x) +#endif +static SQLITE_NOINLINE void getCellInfo(BtCursor *pCur){ + if( pCur->info.nSize==0 ){ + pCur->curFlags |= BTCF_ValidNKey; + btreeParseCell(pCur->pPage,pCur->ix,&pCur->info); + }else{ + assertCellInfo(pCur); + } +} + +#ifndef NDEBUG /* The next routine used only within assert() statements */ +/* +** Return true if the given BtCursor is valid. A valid cursor is one +** that is currently pointing to a row in a (non-empty) table. +** This is a verification routine is used only within assert() statements. +*/ +SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor *pCur){ + return pCur && pCur->eState==CURSOR_VALID; +} +#endif /* NDEBUG */ +SQLITE_PRIVATE int sqlite3BtreeCursorIsValidNN(BtCursor *pCur){ + assert( pCur!=0 ); + return pCur->eState==CURSOR_VALID; +} + +/* +** Return the value of the integer key or "rowid" for a table btree. +** This routine is only valid for a cursor that is pointing into a +** ordinary table btree. If the cursor points to an index btree or +** is invalid, the result of this routine is undefined. +*/ +SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->curIntKey ); + getCellInfo(pCur); + return pCur->info.nKey; +} + +/* +** Pin or unpin a cursor. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorPin(BtCursor *pCur){ + assert( (pCur->curFlags & BTCF_Pinned)==0 ); + pCur->curFlags |= BTCF_Pinned; +} +SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor *pCur){ + assert( (pCur->curFlags & BTCF_Pinned)!=0 ); + pCur->curFlags &= ~BTCF_Pinned; +} + +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC +/* +** Return the offset into the database file for the start of the +** payload to which the cursor is pointing. +*/ +SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + getCellInfo(pCur); + return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) + + (i64)(pCur->info.pPayload - pCur->pPage->aData); +} +#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */ + +/* +** Return the number of bytes of payload for the entry that pCur is +** currently pointing to. For table btrees, this will be the amount +** of data. For index btrees, this will be the size of the key. +** +** The caller must guarantee that the cursor is pointing to a non-NULL +** valid entry. In other words, the calling procedure must guarantee +** that the cursor has Cursor.eState==CURSOR_VALID. +*/ +SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + getCellInfo(pCur); + return pCur->info.nPayload; +} + +/* +** Return an upper bound on the size of any record for the table +** that the cursor is pointing into. +** +** This is an optimization. Everything will still work if this +** routine always returns 2147483647 (which is the largest record +** that SQLite can handle) or more. But returning a smaller value might +** prevent large memory allocations when trying to interpret a +** corrupt datrabase. +** +** The current implementation merely returns the size of the underlying +** database file. +*/ +SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + return pCur->pBt->pageSize * (sqlite3_int64)pCur->pBt->nPage; +} + +/* +** Given the page number of an overflow page in the database (parameter +** ovfl), this function finds the page number of the next page in the +** linked list of overflow pages. If possible, it uses the auto-vacuum +** pointer-map data instead of reading the content of page ovfl to do so. +** +** If an error occurs an SQLite error code is returned. Otherwise: +** +** The page number of the next overflow page in the linked list is +** written to *pPgnoNext. If page ovfl is the last page in its linked +** list, *pPgnoNext is set to zero. +** +** If ppPage is not NULL, and a reference to the MemPage object corresponding +** to page number pOvfl was obtained, then *ppPage is set to point to that +** reference. It is the responsibility of the caller to call releasePage() +** on *ppPage to free the reference. In no reference was obtained (because +** the pointer-map was used to obtain the value for *pPgnoNext), then +** *ppPage is set to zero. +*/ +static int getOverflowPage( + BtShared *pBt, /* The database file */ + Pgno ovfl, /* Current overflow page number */ + MemPage **ppPage, /* OUT: MemPage handle (may be NULL) */ + Pgno *pPgnoNext /* OUT: Next overflow page number */ +){ + Pgno next = 0; + MemPage *pPage = 0; + int rc = SQLITE_OK; + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert(pPgnoNext); + +#ifndef SQLITE_OMIT_AUTOVACUUM + /* Try to find the next page in the overflow list using the + ** autovacuum pointer-map pages. Guess that the next page in + ** the overflow list is page number (ovfl+1). If that guess turns + ** out to be wrong, fall back to loading the data of page + ** number ovfl to determine the next page number. + */ + if( pBt->autoVacuum ){ + Pgno pgno; + Pgno iGuess = ovfl+1; + u8 eType; + + while( PTRMAP_ISPAGE(pBt, iGuess) || iGuess==PENDING_BYTE_PAGE(pBt) ){ + iGuess++; + } + + if( iGuess<=btreePagecount(pBt) ){ + rc = ptrmapGet(pBt, iGuess, &eType, &pgno); + if( rc==SQLITE_OK && eType==PTRMAP_OVERFLOW2 && pgno==ovfl ){ + next = iGuess; + rc = SQLITE_DONE; + } + } + } +#endif + + assert( next==0 || rc==SQLITE_DONE ); + if( rc==SQLITE_OK ){ + rc = btreeGetPage(pBt, ovfl, &pPage, (ppPage==0) ? PAGER_GET_READONLY : 0); + assert( rc==SQLITE_OK || pPage==0 ); + if( rc==SQLITE_OK ){ + next = get4byte(pPage->aData); + } + } + + *pPgnoNext = next; + if( ppPage ){ + *ppPage = pPage; + }else{ + releasePage(pPage); + } + return (rc==SQLITE_DONE ? SQLITE_OK : rc); +} + +/* +** Copy data from a buffer to a page, or from a page to a buffer. +** +** pPayload is a pointer to data stored on database page pDbPage. +** If argument eOp is false, then nByte bytes of data are copied +** from pPayload to the buffer pointed at by pBuf. If eOp is true, +** then sqlite3PagerWrite() is called on pDbPage and nByte bytes +** of data are copied from the buffer pBuf to pPayload. +** +** SQLITE_OK is returned on success, otherwise an error code. +*/ +static int copyPayload( + void *pPayload, /* Pointer to page data */ + void *pBuf, /* Pointer to buffer */ + int nByte, /* Number of bytes to copy */ + int eOp, /* 0 -> copy from page, 1 -> copy to page */ + DbPage *pDbPage /* Page containing pPayload */ +){ + if( eOp ){ + /* Copy data from buffer to page (a write operation) */ + int rc = sqlite3PagerWrite(pDbPage); + if( rc!=SQLITE_OK ){ + return rc; + } + memcpy(pPayload, pBuf, nByte); + }else{ + /* Copy data from page to buffer (a read operation) */ + memcpy(pBuf, pPayload, nByte); + } + return SQLITE_OK; +} + +/* +** This function is used to read or overwrite payload information +** for the entry that the pCur cursor is pointing to. The eOp +** argument is interpreted as follows: +** +** 0: The operation is a read. Populate the overflow cache. +** 1: The operation is a write. Populate the overflow cache. +** +** A total of "amt" bytes are read or written beginning at "offset". +** Data is read to or from the buffer pBuf. +** +** The content being read or written might appear on the main page +** or be scattered out on multiple overflow pages. +** +** If the current cursor entry uses one or more overflow pages +** this function may allocate space for and lazily populate +** the overflow page-list cache array (BtCursor.aOverflow). +** Subsequent calls use this cache to make seeking to the supplied offset +** more efficient. +** +** Once an overflow page-list cache has been allocated, it must be +** invalidated if some other cursor writes to the same table, or if +** the cursor is moved to a different row. Additionally, in auto-vacuum +** mode, the following events may invalidate an overflow page-list cache. +** +** * An incremental vacuum, +** * A commit in auto_vacuum="full" mode, +** * Creating a table (may require moving an overflow page). +*/ +static int accessPayload( + BtCursor *pCur, /* Cursor pointing to entry to read from */ + u32 offset, /* Begin reading this far into payload */ + u32 amt, /* Read this many bytes */ + unsigned char *pBuf, /* Write the bytes into this buffer */ + int eOp /* zero to read. non-zero to write. */ +){ + unsigned char *aPayload; + int rc = SQLITE_OK; + int iIdx = 0; + MemPage *pPage = pCur->pPage; /* Btree page of current entry */ + BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ +#ifdef SQLITE_DIRECT_OVERFLOW_READ + unsigned char * const pBufStart = pBuf; /* Start of original out buffer */ +#endif + + assert( pPage ); + assert( eOp==0 || eOp==1 ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->ixnCell ); + assert( cursorHoldsMutex(pCur) ); + + getCellInfo(pCur); + aPayload = pCur->info.pPayload; + assert( offset+amt <= pCur->info.nPayload ); + + assert( aPayload > pPage->aData ); + if( (uptr)(aPayload - pPage->aData) > (pBt->usableSize - pCur->info.nLocal) ){ + /* Trying to read or write past the end of the data is an error. The + ** conditional above is really: + ** &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] + ** but is recast into its current form to avoid integer overflow problems + */ + return SQLITE_CORRUPT_PAGE(pPage); + } + + /* Check if data must be read/written to/from the btree page itself. */ + if( offsetinfo.nLocal ){ + int a = amt; + if( a+offset>pCur->info.nLocal ){ + a = pCur->info.nLocal - offset; + } + rc = copyPayload(&aPayload[offset], pBuf, a, eOp, pPage->pDbPage); + offset = 0; + pBuf += a; + amt -= a; + }else{ + offset -= pCur->info.nLocal; + } + + + if( rc==SQLITE_OK && amt>0 ){ + const u32 ovflSize = pBt->usableSize - 4; /* Bytes content per ovfl page */ + Pgno nextPage; + + nextPage = get4byte(&aPayload[pCur->info.nLocal]); + + /* If the BtCursor.aOverflow[] has not been allocated, allocate it now. + ** + ** The aOverflow[] array is sized at one entry for each overflow page + ** in the overflow chain. The page number of the first overflow page is + ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array + ** means "not yet known" (the cache is lazily populated). + */ + if( (pCur->curFlags & BTCF_ValidOvfl)==0 ){ + int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; + if( pCur->aOverflow==0 + || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) + ){ + Pgno *aNew = (Pgno*)sqlite3Realloc( + pCur->aOverflow, nOvfl*2*sizeof(Pgno) + ); + if( aNew==0 ){ + return SQLITE_NOMEM_BKPT; + }else{ + pCur->aOverflow = aNew; + } + } + memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); + pCur->curFlags |= BTCF_ValidOvfl; + }else{ + /* If the overflow page-list cache has been allocated and the + ** entry for the first required overflow page is valid, skip + ** directly to it. + */ + if( pCur->aOverflow[offset/ovflSize] ){ + iIdx = (offset/ovflSize); + nextPage = pCur->aOverflow[iIdx]; + offset = (offset%ovflSize); + } + } + + assert( rc==SQLITE_OK && amt>0 ); + while( nextPage ){ + /* If required, populate the overflow page-list cache. */ + if( nextPage > pBt->nPage ) return SQLITE_CORRUPT_BKPT; + assert( pCur->aOverflow[iIdx]==0 + || pCur->aOverflow[iIdx]==nextPage + || CORRUPT_DB ); + pCur->aOverflow[iIdx] = nextPage; + + if( offset>=ovflSize ){ + /* The only reason to read this page is to obtain the page + ** number for the next page in the overflow chain. The page + ** data is not required. So first try to lookup the overflow + ** page-list cache, if any, then fall back to the getOverflowPage() + ** function. + */ + assert( pCur->curFlags & BTCF_ValidOvfl ); + assert( pCur->pBtree->db==pBt->db ); + if( pCur->aOverflow[iIdx+1] ){ + nextPage = pCur->aOverflow[iIdx+1]; + }else{ + rc = getOverflowPage(pBt, nextPage, 0, &nextPage); + } + offset -= ovflSize; + }else{ + /* Need to read this page properly. It contains some of the + ** range of data that is being read (eOp==0) or written (eOp!=0). + */ + int a = amt; + if( a + offset > ovflSize ){ + a = ovflSize - offset; + } + +#ifdef SQLITE_DIRECT_OVERFLOW_READ + /* If all the following are true: + ** + ** 1) this is a read operation, and + ** 2) data is required from the start of this overflow page, and + ** 3) there are no dirty pages in the page-cache + ** 4) the database is file-backed, and + ** 5) the page is not in the WAL file + ** 6) at least 4 bytes have already been read into the output buffer + ** + ** then data can be read directly from the database file into the + ** output buffer, bypassing the page-cache altogether. This speeds + ** up loading large records that span many overflow pages. + */ + if( eOp==0 /* (1) */ + && offset==0 /* (2) */ + && sqlite3PagerDirectReadOk(pBt->pPager, nextPage) /* (3,4,5) */ + && &pBuf[-4]>=pBufStart /* (6) */ + ){ + sqlite3_file *fd = sqlite3PagerFile(pBt->pPager); + u8 aSave[4]; + u8 *aWrite = &pBuf[-4]; + assert( aWrite>=pBufStart ); /* due to (6) */ + memcpy(aSave, aWrite, 4); + rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); + if( rc && nextPage>pBt->nPage ) rc = SQLITE_CORRUPT_BKPT; + nextPage = get4byte(aWrite); + memcpy(aWrite, aSave, 4); + }else +#endif + + { + DbPage *pDbPage; + rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage, + (eOp==0 ? PAGER_GET_READONLY : 0) + ); + if( rc==SQLITE_OK ){ + aPayload = sqlite3PagerGetData(pDbPage); + nextPage = get4byte(aPayload); + rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); + sqlite3PagerUnref(pDbPage); + offset = 0; + } + } + amt -= a; + if( amt==0 ) return rc; + pBuf += a; + } + if( rc ) break; + iIdx++; + } + } + + if( rc==SQLITE_OK && amt>0 ){ + /* Overflow chain ends prematurely */ + return SQLITE_CORRUPT_PAGE(pPage); + } + return rc; +} + +/* +** Read part of the payload for the row at which that cursor pCur is currently +** pointing. "amt" bytes will be transferred into pBuf[]. The transfer +** begins at "offset". +** +** pCur can be pointing to either a table or an index b-tree. +** If pointing to a table btree, then the content section is read. If +** pCur is pointing to an index b-tree then the key section is read. +** +** For sqlite3BtreePayload(), the caller must ensure that pCur is pointing +** to a valid row in the table. For sqlite3BtreePayloadChecked(), the +** cursor might be invalid or might need to be restored before being read. +** +** Return SQLITE_OK on success or an error code if anything goes +** wrong. An error is returned if "offset+amt" is larger than +** the available payload. +*/ +SQLITE_PRIVATE int sqlite3BtreePayload(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>=0 && pCur->pPage ); + assert( pCur->ixpPage->nCell ); + return accessPayload(pCur, offset, amt, (unsigned char*)pBuf, 0); +} + +/* +** This variant of sqlite3BtreePayload() works even if the cursor has not +** in the CURSOR_VALID state. It is only used by the sqlite3_blob_read() +** interface. +*/ +#ifndef SQLITE_OMIT_INCRBLOB +static SQLITE_NOINLINE int accessPayloadChecked( + BtCursor *pCur, + u32 offset, + u32 amt, + void *pBuf +){ + int rc; + if ( pCur->eState==CURSOR_INVALID ){ + return SQLITE_ABORT; + } + assert( cursorOwnsBtShared(pCur) ); + rc = btreeRestoreCursorPosition(pCur); + return rc ? rc : accessPayload(pCur, offset, amt, pBuf, 0); +} +SQLITE_PRIVATE int sqlite3BtreePayloadChecked(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ + if( pCur->eState==CURSOR_VALID ){ + assert( cursorOwnsBtShared(pCur) ); + return accessPayload(pCur, offset, amt, pBuf, 0); + }else{ + return accessPayloadChecked(pCur, offset, amt, pBuf); + } +} +#endif /* SQLITE_OMIT_INCRBLOB */ + +/* +** Return a pointer to payload information from the entry that the +** pCur cursor is pointing to. The pointer is to the beginning of +** the key if index btrees (pPage->intKey==0) and is the data for +** table btrees (pPage->intKey==1). The number of bytes of available +** key/data is written into *pAmt. If *pAmt==0, then the value +** returned will not be a valid pointer. +** +** This routine is an optimization. It is common for the entire key +** and data to fit on the local page and for there to be no overflow +** pages. When that is so, this routine can be used to access the +** key and data without making a copy. If the key and/or data spills +** onto overflow pages, then accessPayload() must be used to reassemble +** the key/data and copy it into a preallocated buffer. +** +** The pointer returned by this routine looks directly into the cached +** page of the database. The data might change or move the next time +** any btree routine is called. +*/ +static const void *fetchPayload( + BtCursor *pCur, /* Cursor pointing to entry to read from */ + u32 *pAmt /* Write the number of available bytes here */ +){ + int amt; + assert( pCur!=0 && pCur->iPage>=0 && pCur->pPage); + assert( pCur->eState==CURSOR_VALID ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->ixpPage->nCell ); + assert( pCur->info.nSize>0 ); + assert( pCur->info.pPayload>pCur->pPage->aData || CORRUPT_DB ); + assert( pCur->info.pPayloadpPage->aDataEnd ||CORRUPT_DB); + amt = pCur->info.nLocal; + if( amt>(int)(pCur->pPage->aDataEnd - pCur->info.pPayload) ){ + /* There is too little space on the page for the expected amount + ** of local content. Database must be corrupt. */ + assert( CORRUPT_DB ); + amt = MAX(0, (int)(pCur->pPage->aDataEnd - pCur->info.pPayload)); + } + *pAmt = (u32)amt; + return (void*)pCur->info.pPayload; +} + + +/* +** For the entry that cursor pCur is point to, return as +** many bytes of the key or data as are available on the local +** b-tree page. Write the number of available bytes into *pAmt. +** +** The pointer returned is ephemeral. The key/data may move +** or be destroyed on the next call to any Btree routine, +** including calls from other threads against the same cache. +** Hence, a mutex on the BtShared should be held prior to calling +** this routine. +** +** These routines is used to get quick access to key and data +** in the common case where no overflow pages are used. +*/ +SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ + return fetchPayload(pCur, pAmt); +} + + +/* +** Move the cursor down to a new child page. The newPgno argument is the +** page number of the child page to move to. +** +** This function returns SQLITE_CORRUPT if the page-header flags field of +** the new child page does not match the flags field of the parent (i.e. +** if an intkey page appears to be the parent of a non-intkey page, or +** vice-versa). +*/ +static int moveToChild(BtCursor *pCur, u32 newPgno){ + BtShared *pBt = pCur->pBt; + + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPageiPage>=0 ); + if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ + return SQLITE_CORRUPT_BKPT; + } + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + pCur->aiIdx[pCur->iPage] = pCur->ix; + pCur->apPage[pCur->iPage] = pCur->pPage; + pCur->ix = 0; + pCur->iPage++; + return getAndInitPage(pBt, newPgno, &pCur->pPage, pCur, pCur->curPagerFlags); +} + +#ifdef SQLITE_DEBUG +/* +** Page pParent is an internal (non-leaf) tree page. This function +** asserts that page number iChild is the left-child if the iIdx'th +** cell in page pParent. Or, if iIdx is equal to the total number of +** cells in pParent, that page number iChild is the right-child of +** the page. +*/ +static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ + if( CORRUPT_DB ) return; /* The conditions tested below might not be true + ** in a corrupt database */ + assert( iIdx<=pParent->nCell ); + if( iIdx==pParent->nCell ){ + assert( get4byte(&pParent->aData[pParent->hdrOffset+8])==iChild ); + }else{ + assert( get4byte(findCell(pParent, iIdx))==iChild ); + } +} +#else +# define assertParentIndex(x,y,z) +#endif + +/* +** Move the cursor up to the parent page. +** +** pCur->idx is set to the cell index that contains the pointer +** to the page we are coming from. If we are coming from the +** right-most child page then pCur->idx is set to one more than +** the largest cell index. +*/ +static void moveToParent(BtCursor *pCur){ + MemPage *pLeaf; + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->iPage>0 ); + assert( pCur->pPage ); + assertParentIndex( + pCur->apPage[pCur->iPage-1], + pCur->aiIdx[pCur->iPage-1], + pCur->pPage->pgno + ); + testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell ); + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + pCur->ix = pCur->aiIdx[pCur->iPage-1]; + pLeaf = pCur->pPage; + pCur->pPage = pCur->apPage[--pCur->iPage]; + releasePageNotNull(pLeaf); +} + +/* +** Move the cursor to point to the root page of its b-tree structure. +** +** If the table has a virtual root page, then the cursor is moved to point +** to the virtual root page instead of the actual root page. A table has a +** virtual root page when the actual root page contains no cells and a +** single child page. This can only happen with the table rooted at page 1. +** +** If the b-tree structure is empty, the cursor state is set to +** CURSOR_INVALID and this routine returns SQLITE_EMPTY. Otherwise, +** the cursor is set to point to the first cell located on the root +** (or virtual root) page and the cursor state is set to CURSOR_VALID. +** +** If this function returns successfully, it may be assumed that the +** page-header flags indicate that the [virtual] root-page is the expected +** kind of b-tree page (i.e. if when opening the cursor the caller did not +** specify a KeyInfo structure the flags byte is set to 0x05 or 0x0D, +** indicating a table b-tree, or if the caller did specify a KeyInfo +** structure the flags byte is set to 0x02 or 0x0A, indicating an index +** b-tree). +*/ +static int moveToRoot(BtCursor *pCur){ + MemPage *pRoot; + int rc = SQLITE_OK; + + assert( cursorOwnsBtShared(pCur) ); + assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); + assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); + assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); + assert( pCur->eState < CURSOR_REQUIRESEEK || pCur->iPage<0 ); + assert( pCur->pgnoRoot>0 || pCur->iPage<0 ); + + if( pCur->iPage>=0 ){ + if( pCur->iPage ){ + releasePageNotNull(pCur->pPage); + while( --pCur->iPage ){ + releasePageNotNull(pCur->apPage[pCur->iPage]); + } + pCur->pPage = pCur->apPage[0]; + goto skip_init; + } + }else if( pCur->pgnoRoot==0 ){ + pCur->eState = CURSOR_INVALID; + return SQLITE_EMPTY; + }else{ + assert( pCur->iPage==(-1) ); + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + if( pCur->eState==CURSOR_FAULT ){ + assert( pCur->skipNext!=SQLITE_OK ); + return pCur->skipNext; + } + sqlite3BtreeClearCursor(pCur); + } + rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->pPage, + 0, pCur->curPagerFlags); + if( rc!=SQLITE_OK ){ + pCur->eState = CURSOR_INVALID; + return rc; + } + pCur->iPage = 0; + pCur->curIntKey = pCur->pPage->intKey; + } + pRoot = pCur->pPage; + assert( pRoot->pgno==pCur->pgnoRoot ); + + /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor + ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is + ** NULL, the caller expects a table b-tree. If this is not the case, + ** return an SQLITE_CORRUPT error. + ** + ** Earlier versions of SQLite assumed that this test could not fail + ** if the root page was already loaded when this function was called (i.e. + ** if pCur->iPage>=0). But this is not so if the database is corrupted + ** in such a way that page pRoot is linked into a second b-tree table + ** (or the freelist). */ + assert( pRoot->intKey==1 || pRoot->intKey==0 ); + if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){ + return SQLITE_CORRUPT_PAGE(pCur->pPage); + } + +skip_init: + pCur->ix = 0; + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl); + + pRoot = pCur->pPage; + if( pRoot->nCell>0 ){ + pCur->eState = CURSOR_VALID; + }else if( !pRoot->leaf ){ + Pgno subpage; + if( pRoot->pgno!=1 ) return SQLITE_CORRUPT_BKPT; + subpage = get4byte(&pRoot->aData[pRoot->hdrOffset+8]); + pCur->eState = CURSOR_VALID; + rc = moveToChild(pCur, subpage); + }else{ + pCur->eState = CURSOR_INVALID; + rc = SQLITE_EMPTY; + } + return rc; +} + +/* +** Move the cursor down to the left-most leaf entry beneath the +** entry to which it is currently pointing. +** +** The left-most leaf is the one with the smallest key - the first +** in ascending order. +*/ +static int moveToLeftmost(BtCursor *pCur){ + Pgno pgno; + int rc = SQLITE_OK; + MemPage *pPage; + + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + while( rc==SQLITE_OK && !(pPage = pCur->pPage)->leaf ){ + assert( pCur->ixnCell ); + pgno = get4byte(findCell(pPage, pCur->ix)); + rc = moveToChild(pCur, pgno); + } + return rc; +} + +/* +** Move the cursor down to the right-most leaf entry beneath the +** page to which it is currently pointing. Notice the difference +** between moveToLeftmost() and moveToRightmost(). moveToLeftmost() +** finds the left-most entry beneath the *entry* whereas moveToRightmost() +** finds the right-most entry beneath the *page*. +** +** The right-most entry is the one with the largest key - the last +** key in ascending order. +*/ +static int moveToRightmost(BtCursor *pCur){ + Pgno pgno; + int rc = SQLITE_OK; + MemPage *pPage = 0; + + assert( cursorOwnsBtShared(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + while( !(pPage = pCur->pPage)->leaf ){ + pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); + pCur->ix = pPage->nCell; + rc = moveToChild(pCur, pgno); + if( rc ) return rc; + } + pCur->ix = pPage->nCell-1; + assert( pCur->info.nSize==0 ); + assert( (pCur->curFlags & BTCF_ValidNKey)==0 ); + return SQLITE_OK; +} + +/* Move the cursor to the first entry in the table. Return SQLITE_OK +** on success. Set *pRes to 0 if the cursor actually points to something +** or set *pRes to 1 if the table is empty. +*/ +SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + assert( pCur->pPage->nCell>0 ); + *pRes = 0; + rc = moveToLeftmost(pCur); + }else if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = 1; + rc = SQLITE_OK; + } + return rc; +} + +/* Move the cursor to the last entry in the table. Return SQLITE_OK +** on success. Set *pRes to 0 if the cursor actually points to something +** or set *pRes to 1 if the table is empty. +*/ +SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ + int rc; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* If the cursor already points to the last entry, this is a no-op. */ + if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ +#ifdef SQLITE_DEBUG + /* This block serves to assert() that the cursor really does point + ** to the last entry in the b-tree. */ + int ii; + for(ii=0; iiiPage; ii++){ + assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); + } + assert( pCur->ix==pCur->pPage->nCell-1 || CORRUPT_DB ); + testcase( pCur->ix!=pCur->pPage->nCell-1 ); + /* ^-- dbsqlfuzz b92b72e4de80b5140c30ab71372ca719b8feb618 */ + assert( pCur->pPage->leaf ); +#endif + *pRes = 0; + return SQLITE_OK; + } + + rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + assert( pCur->eState==CURSOR_VALID ); + *pRes = 0; + rc = moveToRightmost(pCur); + if( rc==SQLITE_OK ){ + pCur->curFlags |= BTCF_AtLast; + }else{ + pCur->curFlags &= ~BTCF_AtLast; + } + }else if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = 1; + rc = SQLITE_OK; + } + return rc; +} + +/* Move the cursor so that it points to an entry in a table (a.k.a INTKEY) +** table near the key intKey. Return a success code. +** +** If an exact match is not found, then the cursor is always +** left pointing at a leaf page which would hold the entry if it +** were present. The cursor might point to an entry that comes +** before or after the key. +** +** An integer is written into *pRes which is the result of +** comparing the key with the entry to which the cursor is +** pointing. The meaning of the integer written into +** *pRes is as follows: +** +** *pRes<0 The cursor is left pointing at an entry that +** is smaller than intKey or if the table is empty +** and the cursor is therefore left point to nothing. +** +** *pRes==0 The cursor is left pointing at an entry that +** exactly matches intKey. +** +** *pRes>0 The cursor is left pointing at an entry that +** is larger than intKey. +*/ +SQLITE_PRIVATE int sqlite3BtreeTableMoveto( + BtCursor *pCur, /* The cursor to be moved */ + i64 intKey, /* The table key */ + int biasRight, /* If true, bias the search to the high end */ + int *pRes /* Write search results here */ +){ + int rc; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + assert( pRes ); + assert( pCur->pKeyInfo==0 ); + assert( pCur->eState!=CURSOR_VALID || pCur->curIntKey!=0 ); + + /* If the cursor is already positioned at the point we are trying + ** to move to, then just return without doing any work */ + if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 ){ + if( pCur->info.nKey==intKey ){ + *pRes = 0; + return SQLITE_OK; + } + if( pCur->info.nKeycurFlags & BTCF_AtLast)!=0 ){ + *pRes = -1; + return SQLITE_OK; + } + /* If the requested key is one more than the previous key, then + ** try to get there using sqlite3BtreeNext() rather than a full + ** binary search. This is an optimization only. The correct answer + ** is still obtained without this case, only a little more slowely */ + if( pCur->info.nKey+1==intKey ){ + *pRes = 0; + rc = sqlite3BtreeNext(pCur, 0); + if( rc==SQLITE_OK ){ + getCellInfo(pCur); + if( pCur->info.nKey==intKey ){ + return SQLITE_OK; + } + }else if( rc!=SQLITE_DONE ){ + return rc; + } + } + } + } + +#ifdef SQLITE_DEBUG + pCur->pBtree->nSeek++; /* Performance measurement during testing */ +#endif + + rc = moveToRoot(pCur); + if( rc ){ + if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = -1; + return SQLITE_OK; + } + return rc; + } + assert( pCur->pPage ); + assert( pCur->pPage->isInit ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->pPage->nCell > 0 ); + assert( pCur->iPage==0 || pCur->apPage[0]->intKey==pCur->curIntKey ); + assert( pCur->curIntKey ); + + for(;;){ + int lwr, upr, idx, c; + Pgno chldPg; + MemPage *pPage = pCur->pPage; + u8 *pCell; /* Pointer to current cell in pPage */ + + /* pPage->nCell must be greater than zero. If this is the root-page + ** the cursor would have been INVALID above and this for(;;) loop + ** not run. If this is not the root-page, then the moveToChild() routine + ** would have already detected db corruption. Similarly, pPage must + ** be the right kind (index or table) of b-tree page. Otherwise + ** a moveToChild() or moveToRoot() call would have detected corruption. */ + assert( pPage->nCell>0 ); + assert( pPage->intKey ); + lwr = 0; + upr = pPage->nCell-1; + assert( biasRight==0 || biasRight==1 ); + idx = upr>>(1-biasRight); /* idx = biasRight ? upr : (lwr+upr)/2; */ + pCur->ix = (u16)idx; + for(;;){ + i64 nCellKey; + pCell = findCellPastPtr(pPage, idx); + if( pPage->intKeyLeaf ){ + while( 0x80 <= *(pCell++) ){ + if( pCell>=pPage->aDataEnd ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + } + getVarint(pCell, (u64*)&nCellKey); + if( nCellKeyupr ){ c = -1; break; } + }else if( nCellKey>intKey ){ + upr = idx-1; + if( lwr>upr ){ c = +1; break; } + }else{ + assert( nCellKey==intKey ); + pCur->ix = (u16)idx; + if( !pPage->leaf ){ + lwr = idx; + goto moveto_table_next_layer; + }else{ + pCur->curFlags |= BTCF_ValidNKey; + pCur->info.nKey = nCellKey; + pCur->info.nSize = 0; + *pRes = 0; + return SQLITE_OK; + } + } + assert( lwr+upr>=0 ); + idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2; */ + } + assert( lwr==upr+1 || !pPage->leaf ); + assert( pPage->isInit ); + if( pPage->leaf ){ + assert( pCur->ixpPage->nCell ); + pCur->ix = (u16)idx; + *pRes = c; + rc = SQLITE_OK; + goto moveto_table_finish; + } +moveto_table_next_layer: + if( lwr>=pPage->nCell ){ + chldPg = get4byte(&pPage->aData[pPage->hdrOffset+8]); + }else{ + chldPg = get4byte(findCell(pPage, lwr)); + } + pCur->ix = (u16)lwr; + rc = moveToChild(pCur, chldPg); + if( rc ) break; + } +moveto_table_finish: + pCur->info.nSize = 0; + assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); + return rc; +} + +/* Move the cursor so that it points to an entry in an index table +** near the key pIdxKey. Return a success code. +** +** If an exact match is not found, then the cursor is always +** left pointing at a leaf page which would hold the entry if it +** were present. The cursor might point to an entry that comes +** before or after the key. +** +** An integer is written into *pRes which is the result of +** comparing the key with the entry to which the cursor is +** pointing. The meaning of the integer written into +** *pRes is as follows: +** +** *pRes<0 The cursor is left pointing at an entry that +** is smaller than pIdxKey or if the table is empty +** and the cursor is therefore left point to nothing. +** +** *pRes==0 The cursor is left pointing at an entry that +** exactly matches pIdxKey. +** +** *pRes>0 The cursor is left pointing at an entry that +** is larger than pIdxKey. +** +** The pIdxKey->eqSeen field is set to 1 if there +** exists an entry in the table that exactly matches pIdxKey. +*/ +SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( + BtCursor *pCur, /* The cursor to be moved */ + UnpackedRecord *pIdxKey, /* Unpacked index key */ + int *pRes /* Write search results here */ +){ + int rc; + RecordCompare xRecordCompare; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + assert( pRes ); + assert( pCur->pKeyInfo!=0 ); + +#ifdef SQLITE_DEBUG + pCur->pBtree->nSeek++; /* Performance measurement during testing */ +#endif + + xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); + pIdxKey->errCode = 0; + assert( pIdxKey->default_rc==1 + || pIdxKey->default_rc==0 + || pIdxKey->default_rc==-1 + ); + + rc = moveToRoot(pCur); + if( rc ){ + if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = -1; + return SQLITE_OK; + } + return rc; + } + assert( pCur->pPage ); + assert( pCur->pPage->isInit ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->pPage->nCell > 0 ); + assert( pCur->iPage==0 || pCur->apPage[0]->intKey==pCur->curIntKey ); + assert( pCur->curIntKey || pIdxKey ); + for(;;){ + int lwr, upr, idx, c; + Pgno chldPg; + MemPage *pPage = pCur->pPage; + u8 *pCell; /* Pointer to current cell in pPage */ + + /* pPage->nCell must be greater than zero. If this is the root-page + ** the cursor would have been INVALID above and this for(;;) loop + ** not run. If this is not the root-page, then the moveToChild() routine + ** would have already detected db corruption. Similarly, pPage must + ** be the right kind (index or table) of b-tree page. Otherwise + ** a moveToChild() or moveToRoot() call would have detected corruption. */ + assert( pPage->nCell>0 ); + assert( pPage->intKey==(pIdxKey==0) ); + lwr = 0; + upr = pPage->nCell-1; + idx = upr>>1; /* idx = (lwr+upr)/2; */ + pCur->ix = (u16)idx; + for(;;){ + int nCell; /* Size of the pCell cell in bytes */ + pCell = findCellPastPtr(pPage, idx); + + /* The maximum supported page-size is 65536 bytes. This means that + ** the maximum number of record bytes stored on an index B-Tree + ** page is less than 16384 bytes and may be stored as a 2-byte + ** varint. This information is used to attempt to avoid parsing + ** the entire cell by checking for the cases where the record is + ** stored entirely within the b-tree page by inspecting the first + ** 2 bytes of the cell. + */ + nCell = pCell[0]; + if( nCell<=pPage->max1bytePayload ){ + /* This branch runs if the record-size field of the cell is a + ** single byte varint and the record fits entirely on the main + ** b-tree page. */ + testcase( pCell+nCell+1==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); + }else if( !(pCell[1] & 0x80) + && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal + ){ + /* The record-size field is a 2 byte varint and the record + ** fits entirely on the main b-tree page. */ + testcase( pCell+nCell+2==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); + }else{ + /* The record flows over onto one or more overflow pages. In + ** this case the whole cell needs to be parsed, a buffer allocated + ** and accessPayload() used to retrieve the record into the + ** buffer before VdbeRecordCompare() can be called. + ** + ** If the record is corrupt, the xRecordCompare routine may read + ** up to two varints past the end of the buffer. An extra 18 + ** bytes of padding is allocated at the end of the buffer in + ** case this happens. */ + void *pCellKey; + u8 * const pCellBody = pCell - pPage->childPtrSize; + const int nOverrun = 18; /* Size of the overrun padding */ + pPage->xParseCell(pPage, pCellBody, &pCur->info); + nCell = (int)pCur->info.nKey; + testcase( nCell<0 ); /* True if key size is 2^32 or more */ + testcase( nCell==0 ); /* Invalid key size: 0x80 0x80 0x00 */ + testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ + testcase( nCell==2 ); /* Minimum legal index key size */ + if( nCell<2 || nCell/pCur->pBt->usableSize>pCur->pBt->nPage ){ + rc = SQLITE_CORRUPT_PAGE(pPage); + goto moveto_index_finish; + } + pCellKey = sqlite3Malloc( nCell+nOverrun ); + if( pCellKey==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto moveto_index_finish; + } + pCur->ix = (u16)idx; + rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + memset(((u8*)pCellKey)+nCell,0,nOverrun); /* Fix uninit warnings */ + pCur->curFlags &= ~BTCF_ValidOvfl; + if( rc ){ + sqlite3_free(pCellKey); + goto moveto_index_finish; + } + c = sqlite3VdbeRecordCompare(nCell, pCellKey, pIdxKey); + sqlite3_free(pCellKey); + } + assert( + (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) + && (pIdxKey->errCode!=SQLITE_NOMEM || pCur->pBtree->db->mallocFailed) + ); + if( c<0 ){ + lwr = idx+1; + }else if( c>0 ){ + upr = idx-1; + }else{ + assert( c==0 ); + *pRes = 0; + rc = SQLITE_OK; + pCur->ix = (u16)idx; + if( pIdxKey->errCode ) rc = SQLITE_CORRUPT_BKPT; + goto moveto_index_finish; + } + if( lwr>upr ) break; + assert( lwr+upr>=0 ); + idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2 */ + } + assert( lwr==upr+1 || (pPage->intKey && !pPage->leaf) ); + assert( pPage->isInit ); + if( pPage->leaf ){ + assert( pCur->ixpPage->nCell ); + pCur->ix = (u16)idx; + *pRes = c; + rc = SQLITE_OK; + goto moveto_index_finish; + } + if( lwr>=pPage->nCell ){ + chldPg = get4byte(&pPage->aData[pPage->hdrOffset+8]); + }else{ + chldPg = get4byte(findCell(pPage, lwr)); + } + pCur->ix = (u16)lwr; + rc = moveToChild(pCur, chldPg); + if( rc ) break; + } +moveto_index_finish: + pCur->info.nSize = 0; + assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); + return rc; +} + + +/* +** Return TRUE if the cursor is not pointing at an entry of the table. +** +** TRUE will be returned after a call to sqlite3BtreeNext() moves +** past the last entry in the table or sqlite3BtreePrev() moves past +** the first entry. TRUE is also returned if the table is empty. +*/ +SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor *pCur){ + /* TODO: What if the cursor is in CURSOR_REQUIRESEEK but all table entries + ** have been deleted? This API will need to change to return an error code + ** as well as the boolean result value. + */ + return (CURSOR_VALID!=pCur->eState); +} + +/* +** Return an estimate for the number of rows in the table that pCur is +** pointing to. Return a negative number if no estimate is currently +** available. +*/ +SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ + i64 n; + u8 i; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* Currently this interface is only called by the OP_IfSmaller + ** opcode, and it that case the cursor will always be valid and + ** will always point to a leaf node. */ + if( NEVER(pCur->eState!=CURSOR_VALID) ) return -1; + if( NEVER(pCur->pPage->leaf==0) ) return -1; + + n = pCur->pPage->nCell; + for(i=0; iiPage; i++){ + n *= pCur->apPage[i]->nCell; + } + return n; +} + +/* +** Advance the cursor to the next entry in the database. +** Return value: +** +** SQLITE_OK success +** SQLITE_DONE cursor is already pointing at the last element +** otherwise some kind of error occurred +** +** The main entry point is sqlite3BtreeNext(). That routine is optimized +** for the common case of merely incrementing the cell counter BtCursor.aiIdx +** to the next cell on the current page. The (slower) btreeNext() helper +** routine is called when it is necessary to move to a different page or +** to restore the cursor. +** +** If bit 0x01 of the F argument in sqlite3BtreeNext(C,F) is 1, then the +** cursor corresponds to an SQL index and this routine could have been +** skipped if the SQL index had been a unique index. The F argument +** is a hint to the implement. SQLite btree implementation does not use +** this hint, but COMDB2 does. +*/ +static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ + int rc; + int idx; + MemPage *pPage; + + assert( cursorOwnsBtShared(pCur) ); + if( pCur->eState!=CURSOR_VALID ){ + assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); + rc = restoreCursorPosition(pCur); + if( rc!=SQLITE_OK ){ + return rc; + } + if( CURSOR_INVALID==pCur->eState ){ + return SQLITE_DONE; + } + if( pCur->eState==CURSOR_SKIPNEXT ){ + pCur->eState = CURSOR_VALID; + if( pCur->skipNext>0 ) return SQLITE_OK; + } + } + + pPage = pCur->pPage; + idx = ++pCur->ix; + if( !pPage->isInit || sqlite3FaultSim(412) ){ + /* The only known way for this to happen is for there to be a + ** recursive SQL function that does a DELETE operation as part of a + ** SELECT which deletes content out from under an active cursor + ** in a corrupt database file where the table being DELETE-ed from + ** has pages in common with the table being queried. See TH3 + ** module cov1/btree78.test testcase 220 (2018-06-08) for an + ** example. */ + return SQLITE_CORRUPT_BKPT; + } + + /* If the database file is corrupt, it is possible for the value of idx + ** to be invalid here. This can only occur if a second cursor modifies + ** the page while cursor pCur is holding a reference to it. Which can + ** only happen if the database is corrupt in such a way as to link the + ** page into more than one b-tree structure. + ** + ** Update 2019-12-23: appears to long longer be possible after the + ** addition of anotherValidCursor() condition on balance_deeper(). */ + harmless( idx>pPage->nCell ); + + if( idx>=pPage->nCell ){ + if( !pPage->leaf ){ + rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); + if( rc ) return rc; + return moveToLeftmost(pCur); + } + do{ + if( pCur->iPage==0 ){ + pCur->eState = CURSOR_INVALID; + return SQLITE_DONE; + } + moveToParent(pCur); + pPage = pCur->pPage; + }while( pCur->ix>=pPage->nCell ); + if( pPage->intKey ){ + return sqlite3BtreeNext(pCur, 0); + }else{ + return SQLITE_OK; + } + } + if( pPage->leaf ){ + return SQLITE_OK; + }else{ + return moveToLeftmost(pCur); + } +} +SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int flags){ + MemPage *pPage; + UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ + assert( cursorOwnsBtShared(pCur) ); + assert( flags==0 || flags==1 ); + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur); + pPage = pCur->pPage; + if( (++pCur->ix)>=pPage->nCell ){ + pCur->ix--; + return btreeNext(pCur); + } + if( pPage->leaf ){ + return SQLITE_OK; + }else{ + return moveToLeftmost(pCur); + } +} + +/* +** Step the cursor to the back to the previous entry in the database. +** Return values: +** +** SQLITE_OK success +** SQLITE_DONE the cursor is already on the first element of the table +** otherwise some kind of error occurred +** +** The main entry point is sqlite3BtreePrevious(). That routine is optimized +** for the common case of merely decrementing the cell counter BtCursor.aiIdx +** to the previous cell on the current page. The (slower) btreePrevious() +** helper routine is called when it is necessary to move to a different page +** or to restore the cursor. +** +** If bit 0x01 of the F argument to sqlite3BtreePrevious(C,F) is 1, then +** the cursor corresponds to an SQL index and this routine could have been +** skipped if the SQL index had been a unique index. The F argument is a +** hint to the implement. The native SQLite btree implementation does not +** use this hint, but COMDB2 does. +*/ +static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){ + int rc; + MemPage *pPage; + + assert( cursorOwnsBtShared(pCur) ); + assert( (pCur->curFlags & (BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey))==0 ); + assert( pCur->info.nSize==0 ); + if( pCur->eState!=CURSOR_VALID ){ + rc = restoreCursorPosition(pCur); + if( rc!=SQLITE_OK ){ + return rc; + } + if( CURSOR_INVALID==pCur->eState ){ + return SQLITE_DONE; + } + if( CURSOR_SKIPNEXT==pCur->eState ){ + pCur->eState = CURSOR_VALID; + if( pCur->skipNext<0 ) return SQLITE_OK; + } + } + + pPage = pCur->pPage; + assert( pPage->isInit ); + if( !pPage->leaf ){ + int idx = pCur->ix; + rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); + if( rc ) return rc; + rc = moveToRightmost(pCur); + }else{ + while( pCur->ix==0 ){ + if( pCur->iPage==0 ){ + pCur->eState = CURSOR_INVALID; + return SQLITE_DONE; + } + moveToParent(pCur); + } + assert( pCur->info.nSize==0 ); + assert( (pCur->curFlags & (BTCF_ValidOvfl))==0 ); + + pCur->ix--; + pPage = pCur->pPage; + if( pPage->intKey && !pPage->leaf ){ + rc = sqlite3BtreePrevious(pCur, 0); + }else{ + rc = SQLITE_OK; + } + } + return rc; +} +SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int flags){ + assert( cursorOwnsBtShared(pCur) ); + assert( flags==0 || flags==1 ); + UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey); + pCur->info.nSize = 0; + if( pCur->eState!=CURSOR_VALID + || pCur->ix==0 + || pCur->pPage->leaf==0 + ){ + return btreePrevious(pCur); + } + pCur->ix--; + return SQLITE_OK; +} + +/* +** Allocate a new page from the database file. +** +** The new page is marked as dirty. (In other words, sqlite3PagerWrite() +** has already been called on the new page.) The new page has also +** been referenced and the calling routine is responsible for calling +** sqlite3PagerUnref() on the new page when it is done. +** +** SQLITE_OK is returned on success. Any other return value indicates +** an error. *ppPage is set to NULL in the event of an error. +** +** If the "nearby" parameter is not 0, then an effort is made to +** locate a page close to the page number "nearby". This can be used in an +** attempt to keep related pages close to each other in the database file, +** which in turn can make database access faster. +** +** If the eMode parameter is BTALLOC_EXACT and the nearby page exists +** anywhere on the free-list, then it is guaranteed to be returned. If +** eMode is BTALLOC_LT then the page returned will be less than or equal +** to nearby if any such page exists. If eMode is BTALLOC_ANY then there +** are no restrictions on which page is returned. +*/ +static int allocateBtreePage( + BtShared *pBt, /* The btree */ + MemPage **ppPage, /* Store pointer to the allocated page here */ + Pgno *pPgno, /* Store the page number here */ + Pgno nearby, /* Search for a page near this one */ + u8 eMode /* BTALLOC_EXACT, BTALLOC_LT, or BTALLOC_ANY */ +){ + MemPage *pPage1; + int rc; + u32 n; /* Number of pages on the freelist */ + u32 k; /* Number of leaves on the trunk of the freelist */ + MemPage *pTrunk = 0; + MemPage *pPrevTrunk = 0; + Pgno mxPage; /* Total size of the database file */ + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); + pPage1 = pBt->pPage1; + mxPage = btreePagecount(pBt); + /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 + ** stores stores the total number of pages on the freelist. */ + n = get4byte(&pPage1->aData[36]); + testcase( n==mxPage-1 ); + if( n>=mxPage ){ + return SQLITE_CORRUPT_BKPT; + } + if( n>0 ){ + /* There are pages on the freelist. Reuse one of those pages. */ + Pgno iTrunk; + u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ + u32 nSearch = 0; /* Count of the number of search attempts */ + + /* If eMode==BTALLOC_EXACT and a query of the pointer-map + ** shows that the page 'nearby' is somewhere on the free-list, then + ** the entire-list will be searched for that page. + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + if( eMode==BTALLOC_EXACT ){ + if( nearby<=mxPage ){ + u8 eType; + assert( nearby>0 ); + assert( pBt->autoVacuum ); + rc = ptrmapGet(pBt, nearby, &eType, 0); + if( rc ) return rc; + if( eType==PTRMAP_FREEPAGE ){ + searchList = 1; + } + } + }else if( eMode==BTALLOC_LE ){ + searchList = 1; + } +#endif + + /* Decrement the free-list count by 1. Set iTrunk to the index of the + ** first free-list trunk page. iPrevTrunk is initially 1. + */ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc ) return rc; + put4byte(&pPage1->aData[36], n-1); + + /* The code within this loop is run only once if the 'searchList' variable + ** is not true. Otherwise, it runs once for each trunk-page on the + ** free-list until the page 'nearby' is located (eMode==BTALLOC_EXACT) + ** or until a page less than 'nearby' is located (eMode==BTALLOC_LT) + */ + do { + pPrevTrunk = pTrunk; + if( pPrevTrunk ){ + /* EVIDENCE-OF: R-01506-11053 The first integer on a freelist trunk page + ** is the page number of the next freelist trunk page in the list or + ** zero if this is the last freelist trunk page. */ + iTrunk = get4byte(&pPrevTrunk->aData[0]); + }else{ + /* EVIDENCE-OF: R-59841-13798 The 4-byte big-endian integer at offset 32 + ** stores the page number of the first page of the freelist, or zero if + ** the freelist is empty. */ + iTrunk = get4byte(&pPage1->aData[32]); + } + testcase( iTrunk==mxPage ); + if( iTrunk>mxPage || nSearch++ > n ){ + rc = SQLITE_CORRUPT_PGNO(pPrevTrunk ? pPrevTrunk->pgno : 1); + }else{ + rc = btreeGetUnusedPage(pBt, iTrunk, &pTrunk, 0); + } + if( rc ){ + pTrunk = 0; + goto end_allocate_page; + } + assert( pTrunk!=0 ); + assert( pTrunk->aData!=0 ); + /* EVIDENCE-OF: R-13523-04394 The second integer on a freelist trunk page + ** is the number of leaf page pointers to follow. */ + k = get4byte(&pTrunk->aData[4]); + if( k==0 && !searchList ){ + /* The trunk has no leaves and the list is not being searched. + ** So extract the trunk page itself and use it as the newly + ** allocated page */ + assert( pPrevTrunk==0 ); + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc ){ + goto end_allocate_page; + } + *pPgno = iTrunk; + memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); + *ppPage = pTrunk; + pTrunk = 0; + TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + }else if( k>(u32)(pBt->usableSize/4 - 2) ){ + /* Value of k is out of range. Database corruption */ + rc = SQLITE_CORRUPT_PGNO(iTrunk); + goto end_allocate_page; +#ifndef SQLITE_OMIT_AUTOVACUUM + }else if( searchList + && (nearby==iTrunk || (iTrunkpDbPage); + if( rc ){ + goto end_allocate_page; + } + if( k==0 ){ + if( !pPrevTrunk ){ + memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); + }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } + memcpy(&pPrevTrunk->aData[0], &pTrunk->aData[0], 4); + } + }else{ + /* The trunk page is required by the caller but it contains + ** pointers to free-list leaves. The first leaf becomes a trunk + ** page in this case. + */ + MemPage *pNewTrunk; + Pgno iNewTrunk = get4byte(&pTrunk->aData[8]); + if( iNewTrunk>mxPage ){ + rc = SQLITE_CORRUPT_PGNO(iTrunk); + goto end_allocate_page; + } + testcase( iNewTrunk==mxPage ); + rc = btreeGetUnusedPage(pBt, iNewTrunk, &pNewTrunk, 0); + if( rc!=SQLITE_OK ){ + goto end_allocate_page; + } + rc = sqlite3PagerWrite(pNewTrunk->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pNewTrunk); + goto end_allocate_page; + } + memcpy(&pNewTrunk->aData[0], &pTrunk->aData[0], 4); + put4byte(&pNewTrunk->aData[4], k-1); + memcpy(&pNewTrunk->aData[8], &pTrunk->aData[12], (k-1)*4); + releasePage(pNewTrunk); + if( !pPrevTrunk ){ + assert( sqlite3PagerIswriteable(pPage1->pDbPage) ); + put4byte(&pPage1->aData[32], iNewTrunk); + }else{ + rc = sqlite3PagerWrite(pPrevTrunk->pDbPage); + if( rc ){ + goto end_allocate_page; + } + put4byte(&pPrevTrunk->aData[0], iNewTrunk); + } + } + pTrunk = 0; + TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); +#endif + }else if( k>0 ){ + /* Extract a leaf from the trunk */ + u32 closest; + Pgno iPage; + unsigned char *aData = pTrunk->aData; + if( nearby>0 ){ + u32 i; + closest = 0; + if( eMode==BTALLOC_LE ){ + for(i=0; imxPage || iPage<2 ){ + rc = SQLITE_CORRUPT_PGNO(iTrunk); + goto end_allocate_page; + } + testcase( iPage==mxPage ); + if( !searchList + || (iPage==nearby || (iPagepgno, n-1)); + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc ) goto end_allocate_page; + if( closestpDbPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + *ppPage = 0; + } + } + searchList = 0; + } + } + releasePage(pPrevTrunk); + pPrevTrunk = 0; + }while( searchList ); + }else{ + /* There are no pages on the freelist, so append a new page to the + ** database image. + ** + ** Normally, new pages allocated by this block can be requested from the + ** pager layer with the 'no-content' flag set. This prevents the pager + ** from trying to read the pages content from disk. However, if the + ** current transaction has already run one or more incremental-vacuum + ** steps, then the page we are about to allocate may contain content + ** that is required in the event of a rollback. In this case, do + ** not set the no-content flag. This causes the pager to load and journal + ** the current page content before overwriting it. + ** + ** Note that the pager will not actually attempt to load or journal + ** content for any page that really does lie past the end of the database + ** file on disk. So the effects of disabling the no-content optimization + ** here are confined to those pages that lie between the end of the + ** database image and the end of the database file. + */ + int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate))? PAGER_GET_NOCONTENT:0; + + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc ) return rc; + pBt->nPage++; + if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ) pBt->nPage++; + +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum && PTRMAP_ISPAGE(pBt, pBt->nPage) ){ + /* If *pPgno refers to a pointer-map page, allocate two new pages + ** at the end of the file instead of one. The first allocated page + ** becomes a new pointer-map page, the second is used by the caller. + */ + MemPage *pPg = 0; + TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); + assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); + rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pPg->pDbPage); + releasePage(pPg); + } + if( rc ) return rc; + pBt->nPage++; + if( pBt->nPage==PENDING_BYTE_PAGE(pBt) ){ pBt->nPage++; } + } +#endif + put4byte(28 + (u8*)pBt->pPage1->aData, pBt->nPage); + *pPgno = pBt->nPage; + + assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); + rc = btreeGetUnusedPage(pBt, *pPgno, ppPage, bNoContent); + if( rc ) return rc; + rc = sqlite3PagerWrite((*ppPage)->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(*ppPage); + *ppPage = 0; + } + TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); + } + + assert( CORRUPT_DB || *pPgno!=PENDING_BYTE_PAGE(pBt) ); + +end_allocate_page: + releasePage(pTrunk); + releasePage(pPrevTrunk); + assert( rc!=SQLITE_OK || sqlite3PagerPageRefcount((*ppPage)->pDbPage)<=1 ); + assert( rc!=SQLITE_OK || (*ppPage)->isInit==0 ); + return rc; +} + +/* +** This function is used to add page iPage to the database file free-list. +** It is assumed that the page is not already a part of the free-list. +** +** The value passed as the second argument to this function is optional. +** If the caller happens to have a pointer to the MemPage object +** corresponding to page iPage handy, it may pass it as the second value. +** Otherwise, it may pass NULL. +** +** If a pointer to a MemPage object is passed as the second argument, +** its reference count is not altered by this function. +*/ +static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ + MemPage *pTrunk = 0; /* Free-list trunk page */ + Pgno iTrunk = 0; /* Page number of free-list trunk page */ + MemPage *pPage1 = pBt->pPage1; /* Local reference to page 1 */ + MemPage *pPage; /* Page being freed. May be NULL. */ + int rc; /* Return Code */ + u32 nFree; /* Initial number of pages on free-list */ + + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( CORRUPT_DB || iPage>1 ); + assert( !pMemPage || pMemPage->pgno==iPage ); + + if( iPage<2 || iPage>pBt->nPage ){ + return SQLITE_CORRUPT_BKPT; + } + if( pMemPage ){ + pPage = pMemPage; + sqlite3PagerRef(pPage->pDbPage); + }else{ + pPage = btreePageLookup(pBt, iPage); + } + + /* Increment the free page count on pPage1 */ + rc = sqlite3PagerWrite(pPage1->pDbPage); + if( rc ) goto freepage_out; + nFree = get4byte(&pPage1->aData[36]); + put4byte(&pPage1->aData[36], nFree+1); + + if( pBt->btsFlags & BTS_SECURE_DELETE ){ + /* If the secure_delete option is enabled, then + ** always fully overwrite deleted information with zeros. + */ + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) + ){ + goto freepage_out; + } + memset(pPage->aData, 0, pPage->pBt->pageSize); + } + + /* If the database supports auto-vacuum, write an entry in the pointer-map + ** to indicate that the page is free. + */ + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); + if( rc ) goto freepage_out; + } + + /* Now manipulate the actual database free-list structure. There are two + ** possibilities. If the free-list is currently empty, or if the first + ** trunk page in the free-list is full, then this page will become a + ** new free-list trunk page. Otherwise, it will become a leaf of the + ** first trunk page in the current free-list. This block tests if it + ** is possible to add the page as a new free-list leaf. + */ + if( nFree!=0 ){ + u32 nLeaf; /* Initial number of leaf cells on trunk page */ + + iTrunk = get4byte(&pPage1->aData[32]); + if( iTrunk>btreePagecount(pBt) ){ + rc = SQLITE_CORRUPT_BKPT; + goto freepage_out; + } + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + if( rc!=SQLITE_OK ){ + goto freepage_out; + } + + nLeaf = get4byte(&pTrunk->aData[4]); + assert( pBt->usableSize>32 ); + if( nLeaf > (u32)pBt->usableSize/4 - 2 ){ + rc = SQLITE_CORRUPT_BKPT; + goto freepage_out; + } + if( nLeaf < (u32)pBt->usableSize/4 - 8 ){ + /* In this case there is room on the trunk page to insert the page + ** being freed as a new leaf. + ** + ** Note that the trunk page is not really full until it contains + ** usableSize/4 - 2 entries, not usableSize/4 - 8 entries as we have + ** coded. But due to a coding error in versions of SQLite prior to + ** 3.6.0, databases with freelist trunk pages holding more than + ** usableSize/4 - 8 entries will be reported as corrupt. In order + ** to maintain backwards compatibility with older versions of SQLite, + ** we will continue to restrict the number of entries to usableSize/4 - 8 + ** for now. At some point in the future (once everyone has upgraded + ** to 3.6.0 or later) we should consider fixing the conditional above + ** to read "usableSize/4-2" instead of "usableSize/4-8". + ** + ** EVIDENCE-OF: R-19920-11576 However, newer versions of SQLite still + ** avoid using the last six entries in the freelist trunk page array in + ** order that database files created by newer versions of SQLite can be + ** read by older versions of SQLite. + */ + rc = sqlite3PagerWrite(pTrunk->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pTrunk->aData[4], nLeaf+1); + put4byte(&pTrunk->aData[8+nLeaf*4], iPage); + if( pPage && (pBt->btsFlags & BTS_SECURE_DELETE)==0 ){ + sqlite3PagerDontWrite(pPage->pDbPage); + } + rc = btreeSetHasContent(pBt, iPage); + } + TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); + goto freepage_out; + } + } + + /* If control flows to this point, then it was not possible to add the + ** the page being freed as a leaf page of the first trunk in the free-list. + ** Possibly because the free-list is empty, or possibly because the + ** first trunk in the free-list is full. Either way, the page being freed + ** will become the new first trunk page in the free-list. + */ + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + goto freepage_out; + } + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc!=SQLITE_OK ){ + goto freepage_out; + } + put4byte(pPage->aData, iTrunk); + put4byte(&pPage->aData[4], 0); + put4byte(&pPage1->aData[32], iPage); + TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", pPage->pgno, iTrunk)); + +freepage_out: + if( pPage ){ + pPage->isInit = 0; + } + releasePage(pPage); + releasePage(pTrunk); + return rc; +} +static void freePage(MemPage *pPage, int *pRC){ + if( (*pRC)==SQLITE_OK ){ + *pRC = freePage2(pPage->pBt, pPage, pPage->pgno); + } +} + +/* +** Free the overflow pages associated with the given Cell. +*/ +static SQLITE_NOINLINE int clearCellOverflow( + MemPage *pPage, /* The page that contains the Cell */ + unsigned char *pCell, /* First byte of the Cell */ + CellInfo *pInfo /* Size information about the cell */ +){ + BtShared *pBt; + Pgno ovflPgno; + int rc; + int nOvfl; + u32 ovflPageSize; + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pInfo->nLocal!=pInfo->nPayload ); + testcase( pCell + pInfo->nSize == pPage->aDataEnd ); + testcase( pCell + (pInfo->nSize-1) == pPage->aDataEnd ); + if( pCell + pInfo->nSize > pPage->aDataEnd ){ + /* Cell extends past end of page */ + return SQLITE_CORRUPT_PAGE(pPage); + } + ovflPgno = get4byte(pCell + pInfo->nSize - 4); + pBt = pPage->pBt; + assert( pBt->usableSize > 4 ); + ovflPageSize = pBt->usableSize - 4; + nOvfl = (pInfo->nPayload - pInfo->nLocal + ovflPageSize - 1)/ovflPageSize; + assert( nOvfl>0 || + (CORRUPT_DB && (pInfo->nPayload + ovflPageSize)btreePagecount(pBt) ){ + /* 0 is not a legal page number and page 1 cannot be an + ** overflow page. Therefore if ovflPgno<2 or past the end of the + ** file the database must be corrupt. */ + return SQLITE_CORRUPT_BKPT; + } + if( nOvfl ){ + rc = getOverflowPage(pBt, ovflPgno, &pOvfl, &iNext); + if( rc ) return rc; + } + + if( ( pOvfl || ((pOvfl = btreePageLookup(pBt, ovflPgno))!=0) ) + && sqlite3PagerPageRefcount(pOvfl->pDbPage)!=1 + ){ + /* There is no reason any cursor should have an outstanding reference + ** to an overflow page belonging to a cell that is being deleted/updated. + ** So if there exists more than one reference to this page, then it + ** must not really be an overflow page and the database must be corrupt. + ** It is helpful to detect this before calling freePage2(), as + ** freePage2() may zero the page contents if secure-delete mode is + ** enabled. If this 'overflow' page happens to be a page that the + ** caller is iterating through or using in some other way, this + ** can be problematic. + */ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = freePage2(pBt, pOvfl, ovflPgno); + } + + if( pOvfl ){ + sqlite3PagerUnref(pOvfl->pDbPage); + } + if( rc ) return rc; + ovflPgno = iNext; + } + return SQLITE_OK; +} + +/* Call xParseCell to compute the size of a cell. If the cell contains +** overflow, then invoke cellClearOverflow to clear out that overflow. +** STore the result code (SQLITE_OK or some error code) in rc. +** +** Implemented as macro to force inlining for performance. +*/ +#define BTREE_CLEAR_CELL(rc, pPage, pCell, sInfo) \ + pPage->xParseCell(pPage, pCell, &sInfo); \ + if( sInfo.nLocal!=sInfo.nPayload ){ \ + rc = clearCellOverflow(pPage, pCell, &sInfo); \ + }else{ \ + rc = SQLITE_OK; \ + } + + +/* +** Create the byte sequence used to represent a cell on page pPage +** and write that byte sequence into pCell[]. Overflow pages are +** allocated and filled in as necessary. The calling procedure +** is responsible for making sure sufficient space has been allocated +** for pCell[]. +** +** Note that pCell does not necessary need to point to the pPage->aData +** area. pCell might point to some temporary storage. The cell will +** be constructed in this temporary area then copied into pPage->aData +** later. +*/ +static int fillInCell( + MemPage *pPage, /* The page that contains the cell */ + unsigned char *pCell, /* Complete text of the cell */ + const BtreePayload *pX, /* Payload with which to construct the cell */ + int *pnSize /* Write cell size here */ +){ + int nPayload; + const u8 *pSrc; + int nSrc, n, rc, mn; + int spaceLeft; + MemPage *pToRelease; + unsigned char *pPrior; + unsigned char *pPayload; + BtShared *pBt; + Pgno pgnoOvfl; + int nHeader; + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + + /* pPage is not necessarily writeable since pCell might be auxiliary + ** buffer space that is separate from the pPage buffer area */ + assert( pCellaData || pCell>=&pPage->aData[pPage->pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + /* Fill in the header. */ + nHeader = pPage->childPtrSize; + if( pPage->intKey ){ + nPayload = pX->nData + pX->nZero; + pSrc = pX->pData; + nSrc = pX->nData; + assert( pPage->intKeyLeaf ); /* fillInCell() only called for leaves */ + nHeader += putVarint32(&pCell[nHeader], nPayload); + nHeader += putVarint(&pCell[nHeader], *(u64*)&pX->nKey); + }else{ + assert( pX->nKey<=0x7fffffff && pX->pKey!=0 ); + nSrc = nPayload = (int)pX->nKey; + pSrc = pX->pKey; + nHeader += putVarint32(&pCell[nHeader], nPayload); + } + + /* Fill in the payload */ + pPayload = &pCell[nHeader]; + if( nPayload<=pPage->maxLocal ){ + /* This is the common case where everything fits on the btree page + ** and no overflow pages are required. */ + n = nHeader + nPayload; + testcase( n==3 ); + testcase( n==4 ); + if( n<4 ) n = 4; + *pnSize = n; + assert( nSrc<=nPayload ); + testcase( nSrcminLocal; + n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4); + testcase( n==pPage->maxLocal ); + testcase( n==pPage->maxLocal+1 ); + if( n > pPage->maxLocal ) n = mn; + spaceLeft = n; + *pnSize = n + nHeader + 4; + pPrior = &pCell[nHeader+n]; + pToRelease = 0; + pgnoOvfl = 0; + pBt = pPage->pBt; + + /* At this point variables should be set as follows: + ** + ** nPayload Total payload size in bytes + ** pPayload Begin writing payload here + ** spaceLeft Space available at pPayload. If nPayload>spaceLeft, + ** that means content must spill into overflow pages. + ** *pnSize Size of the local cell (not counting overflow pages) + ** pPrior Where to write the pgno of the first overflow page + ** + ** Use a call to btreeParseCellPtr() to verify that the values above + ** were computed correctly. + */ +#ifdef SQLITE_DEBUG + { + CellInfo info; + pPage->xParseCell(pPage, pCell, &info); + assert( nHeader==(int)(info.pPayload - pCell) ); + assert( info.nKey==pX->nKey ); + assert( *pnSize == info.nSize ); + assert( spaceLeft == info.nLocal ); + } +#endif + + /* Write the payload into the local Cell and any extra into overflow pages */ + while( 1 ){ + n = nPayload; + if( n>spaceLeft ) n = spaceLeft; + + /* If pToRelease is not zero than pPayload points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); + + /* If pPayload is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPayloadaData || pPayload>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + if( nSrc>=n ){ + memcpy(pPayload, pSrc, n); + }else if( nSrc>0 ){ + n = nSrc; + memcpy(pPayload, pSrc, n); + }else{ + memset(pPayload, 0, n); + } + nPayload -= n; + if( nPayload<=0 ) break; + pPayload += n; + pSrc += n; + nSrc -= n; + spaceLeft -= n; + if( spaceLeft==0 ){ + MemPage *pOvfl = 0; +#ifndef SQLITE_OMIT_AUTOVACUUM + Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */ + if( pBt->autoVacuum ){ + do{ + pgnoOvfl++; + } while( + PTRMAP_ISPAGE(pBt, pgnoOvfl) || pgnoOvfl==PENDING_BYTE_PAGE(pBt) + ); + } +#endif + rc = allocateBtreePage(pBt, &pOvfl, &pgnoOvfl, pgnoOvfl, 0); +#ifndef SQLITE_OMIT_AUTOVACUUM + /* If the database supports auto-vacuum, and the second or subsequent + ** overflow page is being allocated, add an entry to the pointer-map + ** for that page now. + ** + ** If this is the first overflow page, then write a partial entry + ** to the pointer-map. If we write nothing to this pointer-map slot, + ** then the optimistic overflow chain processing in clearCell() + ** may misinterpret the uninitialized values and delete the + ** wrong pages from the database. + */ + if( pBt->autoVacuum && rc==SQLITE_OK ){ + u8 eType = (pgnoPtrmap?PTRMAP_OVERFLOW2:PTRMAP_OVERFLOW1); + ptrmapPut(pBt, pgnoOvfl, eType, pgnoPtrmap, &rc); + if( rc ){ + releasePage(pOvfl); + } + } +#endif + if( rc ){ + releasePage(pToRelease); + return rc; + } + + /* If pToRelease is not zero than pPrior points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); + + /* If pPrior is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPrioraData || pPrior>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + put4byte(pPrior, pgnoOvfl); + releasePage(pToRelease); + pToRelease = pOvfl; + pPrior = pOvfl->aData; + put4byte(pPrior, 0); + pPayload = &pOvfl->aData[4]; + spaceLeft = pBt->usableSize - 4; + } + } + releasePage(pToRelease); + return SQLITE_OK; +} + +/* +** Remove the i-th cell from pPage. This routine effects pPage only. +** The cell content is not freed or deallocated. It is assumed that +** the cell content has been copied someplace else. This routine just +** removes the reference to the cell from pPage. +** +** "sz" must be the number of bytes in the cell. +*/ +static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ + u32 pc; /* Offset to cell content of cell being deleted */ + u8 *data; /* pPage->aData */ + u8 *ptr; /* Used to move bytes around within data[] */ + int rc; /* The return code */ + int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ + + if( *pRC ) return; + assert( idx>=0 && idxnCell ); + assert( CORRUPT_DB || sz==cellSize(pPage, idx) ); + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->nFree>=0 ); + data = pPage->aData; + ptr = &pPage->aCellIdx[2*idx]; + pc = get2byte(ptr); + hdr = pPage->hdrOffset; + testcase( pc==get2byte(&data[hdr+5]) ); + testcase( pc+sz==pPage->pBt->usableSize ); + if( pc+sz > pPage->pBt->usableSize ){ + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + rc = freeSpace(pPage, pc, sz); + if( rc ){ + *pRC = rc; + return; + } + pPage->nCell--; + if( pPage->nCell==0 ){ + memset(&data[hdr+1], 0, 4); + data[hdr+7] = 0; + put2byte(&data[hdr+5], pPage->pBt->usableSize); + pPage->nFree = pPage->pBt->usableSize - pPage->hdrOffset + - pPage->childPtrSize - 8; + }else{ + memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); + put2byte(&data[hdr+3], pPage->nCell); + pPage->nFree += 2; + } +} + +/* +** Insert a new cell on pPage at cell index "i". pCell points to the +** content of the cell. +** +** If the cell content will fit on the page, then put it there. If it +** will not fit, then make a copy of the cell content into pTemp if +** pTemp is not null. Regardless of pTemp, allocate a new entry +** in pPage->apOvfl[] and make it point to the cell content (either +** in pTemp or the original pCell) and also record its index. +** Allocating a new entry in pPage->aCell[] implies that +** pPage->nOverflow is incremented. +** +** *pRC must be SQLITE_OK when this routine is called. +*/ +static void insertCell( + MemPage *pPage, /* Page into which we are copying */ + int i, /* New cell becomes the i-th cell of the page */ + u8 *pCell, /* Content of the new cell */ + int sz, /* Bytes of content in pCell */ + u8 *pTemp, /* Temp storage space for pCell, if needed */ + Pgno iChild, /* If non-zero, replace first 4 bytes with this value */ + int *pRC /* Read and write return code from here */ +){ + int idx = 0; /* Where to write new cell content in data[] */ + int j; /* Loop counter */ + u8 *data; /* The content of the whole page */ + u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ + + assert( *pRC==SQLITE_OK ); + assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); + assert( MX_CELL(pPage->pBt)<=10921 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); + assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); + assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); + assert( pPage->nFree>=0 ); + if( pPage->nOverflow || sz+2>pPage->nFree ){ + if( pTemp ){ + memcpy(pTemp, pCell, sz); + pCell = pTemp; + } + if( iChild ){ + put4byte(pCell, iChild); + } + j = pPage->nOverflow++; + /* Comparison against ArraySize-1 since we hold back one extra slot + ** as a contingency. In other words, never need more than 3 overflow + ** slots but 4 are allocated, just to be safe. */ + assert( j < ArraySize(pPage->apOvfl)-1 ); + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = (u16)i; + + /* When multiple overflows occur, they are always sequential and in + ** sorted order. This invariants arise because multiple overflows can + ** only occur when inserting divider cells into the parent page during + ** balancing, and the dividers are adjacent and sorted. + */ + assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ + assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ + }else{ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + data = pPage->aData; + assert( &data[pPage->cellOffset]==pPage->aCellIdx ); + rc = allocateSpace(pPage, sz, &idx); + if( rc ){ *pRC = rc; return; } + /* The allocateSpace() routine guarantees the following properties + ** if it returns successfully */ + assert( idx >= 0 ); + assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); + assert( idx+sz <= (int)pPage->pBt->usableSize ); + pPage->nFree -= (u16)(2 + sz); + if( iChild ){ + /* In a corrupt database where an entry in the cell index section of + ** a btree page has a value of 3 or less, the pCell value might point + ** as many as 4 bytes in front of the start of the aData buffer for + ** the source page. Make sure this does not cause problems by not + ** reading the first 4 bytes */ + memcpy(&data[idx+4], pCell+4, sz-4); + put4byte(&data[idx], iChild); + }else{ + memcpy(&data[idx], pCell, sz); + } + pIns = pPage->aCellIdx + i*2; + memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + put2byte(pIns, idx); + pPage->nCell++; + /* increment the cell count */ + if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pPage->pBt->autoVacuum ){ + /* The cell may contain a pointer to an overflow page. If so, write + ** the entry for the overflow page into the pointer map. + */ + ptrmapPutOvflPtr(pPage, pPage, pCell, pRC); + } +#endif + } +} + +/* +** The following parameters determine how many adjacent pages get involved +** in a balancing operation. NN is the number of neighbors on either side +** of the page that participate in the balancing operation. NB is the +** total number of pages that participate, including the target page and +** NN neighbors on either side. +** +** The minimum value of NN is 1 (of course). Increasing NN above 1 +** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance +** in exchange for a larger degradation in INSERT and UPDATE performance. +** The value of NN appears to give the best results overall. +** +** (Later:) The description above makes it seem as if these values are +** tunable - as if you could change them and recompile and it would all work. +** But that is unlikely. NB has been 3 since the inception of SQLite and +** we have never tested any other value. +*/ +#define NN 1 /* Number of neighbors on either side of pPage */ +#define NB 3 /* (NN*2+1): Total pages involved in the balance */ + +/* +** A CellArray object contains a cache of pointers and sizes for a +** consecutive sequence of cells that might be held on multiple pages. +** +** The cells in this array are the divider cell or cells from the pParent +** page plus up to three child pages. There are a total of nCell cells. +** +** pRef is a pointer to one of the pages that contributes cells. This is +** used to access information such as MemPage.intKey and MemPage.pBt->pageSize +** which should be common to all pages that contribute cells to this array. +** +** apCell[] and szCell[] hold, respectively, pointers to the start of each +** cell and the size of each cell. Some of the apCell[] pointers might refer +** to overflow cells. In other words, some apCel[] pointers might not point +** to content area of the pages. +** +** A szCell[] of zero means the size of that cell has not yet been computed. +** +** The cells come from as many as four different pages: +** +** ----------- +** | Parent | +** ----------- +** / | \ +** / | \ +** --------- --------- --------- +** |Child-1| |Child-2| |Child-3| +** --------- --------- --------- +** +** The order of cells is in the array is for an index btree is: +** +** 1. All cells from Child-1 in order +** 2. The first divider cell from Parent +** 3. All cells from Child-2 in order +** 4. The second divider cell from Parent +** 5. All cells from Child-3 in order +** +** For a table-btree (with rowids) the items 2 and 4 are empty because +** content exists only in leaves and there are no divider cells. +** +** For an index btree, the apEnd[] array holds pointer to the end of page +** for Child-1, the Parent, Child-2, the Parent (again), and Child-3, +** respectively. The ixNx[] array holds the number of cells contained in +** each of these 5 stages, and all stages to the left. Hence: +** +** ixNx[0] = Number of cells in Child-1. +** ixNx[1] = Number of cells in Child-1 plus 1 for first divider. +** ixNx[2] = Number of cells in Child-1 and Child-2 + 1 for 1st divider. +** ixNx[3] = Number of cells in Child-1 and Child-2 + both divider cells +** ixNx[4] = Total number of cells. +** +** For a table-btree, the concept is similar, except only apEnd[0]..apEnd[2] +** are used and they point to the leaf pages only, and the ixNx value are: +** +** ixNx[0] = Number of cells in Child-1. +** ixNx[1] = Number of cells in Child-1 and Child-2. +** ixNx[2] = Total number of cells. +** +** Sometimes when deleting, a child page can have zero cells. In those +** cases, ixNx[] entries with higher indexes, and the corresponding apEnd[] +** entries, shift down. The end result is that each ixNx[] entry should +** be larger than the previous +*/ +typedef struct CellArray CellArray; +struct CellArray { + int nCell; /* Number of cells in apCell[] */ + MemPage *pRef; /* Reference page */ + u8 **apCell; /* All cells begin balanced */ + u16 *szCell; /* Local size of all cells in apCell[] */ + u8 *apEnd[NB*2]; /* MemPage.aDataEnd values */ + int ixNx[NB*2]; /* Index of at which we move to the next apEnd[] */ +}; + +/* +** Make sure the cell sizes at idx, idx+1, ..., idx+N-1 have been +** computed. +*/ +static void populateCellCache(CellArray *p, int idx, int N){ + assert( idx>=0 && idx+N<=p->nCell ); + while( N>0 ){ + assert( p->apCell[idx]!=0 ); + if( p->szCell[idx]==0 ){ + p->szCell[idx] = p->pRef->xCellSize(p->pRef, p->apCell[idx]); + }else{ + assert( CORRUPT_DB || + p->szCell[idx]==p->pRef->xCellSize(p->pRef, p->apCell[idx]) ); + } + idx++; + N--; + } +} + +/* +** Return the size of the Nth element of the cell array +*/ +static SQLITE_NOINLINE u16 computeCellSize(CellArray *p, int N){ + assert( N>=0 && NnCell ); + assert( p->szCell[N]==0 ); + p->szCell[N] = p->pRef->xCellSize(p->pRef, p->apCell[N]); + return p->szCell[N]; +} +static u16 cachedCellSize(CellArray *p, int N){ + assert( N>=0 && NnCell ); + if( p->szCell[N] ) return p->szCell[N]; + return computeCellSize(p, N); +} + +/* +** Array apCell[] contains pointers to nCell b-tree page cells. The +** szCell[] array contains the size in bytes of each cell. This function +** replaces the current contents of page pPg with the contents of the cell +** array. +** +** Some of the cells in apCell[] may currently be stored in pPg. This +** function works around problems caused by this by making a copy of any +** such cells before overwriting the page data. +** +** The MemPage.nFree field is invalidated by this function. It is the +** responsibility of the caller to set it correctly. +*/ +static int rebuildPage( + CellArray *pCArray, /* Content to be added to page pPg */ + int iFirst, /* First cell in pCArray to use */ + int nCell, /* Final number of cells on page */ + MemPage *pPg /* The page to be reconstructed */ +){ + const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ + u8 * const aData = pPg->aData; /* Pointer to data for pPg */ + const int usableSize = pPg->pBt->usableSize; + u8 * const pEnd = &aData[usableSize]; + int i = iFirst; /* Which cell to copy from pCArray*/ + u32 j; /* Start of cell content area */ + int iEnd = i+nCell; /* Loop terminator */ + u8 *pCellptr = pPg->aCellIdx; + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + u8 *pData; + int k; /* Current slot in pCArray->apEnd[] */ + u8 *pSrcEnd; /* Current pCArray->apEnd[k] value */ + + assert( i(u32)usableSize) ){ j = 0; } + memcpy(&pTmp[j], &aData[j], usableSize - j); + + for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kapEnd[k]; + + pData = pEnd; + while( 1/*exit by break*/ ){ + u8 *pCell = pCArray->apCell[i]; + u16 sz = pCArray->szCell[i]; + assert( sz>0 ); + if( SQLITE_WITHIN(pCell,aData+j,pEnd) ){ + if( ((uptr)(pCell+sz))>(uptr)pEnd ) return SQLITE_CORRUPT_BKPT; + pCell = &pTmp[pCell - aData]; + }else if( (uptr)(pCell+sz)>(uptr)pSrcEnd + && (uptr)(pCell)<(uptr)pSrcEnd + ){ + return SQLITE_CORRUPT_BKPT; + } + + pData -= sz; + put2byte(pCellptr, (pData - aData)); + pCellptr += 2; + if( pData < pCellptr ) return SQLITE_CORRUPT_BKPT; + memmove(pData, pCell, sz); + assert( sz==pPg->xCellSize(pPg, pCell) || CORRUPT_DB ); + i++; + if( i>=iEnd ) break; + if( pCArray->ixNx[k]<=i ){ + k++; + pSrcEnd = pCArray->apEnd[k]; + } + } + + /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ + pPg->nCell = nCell; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+1], 0); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + aData[hdr+7] = 0x00; + return SQLITE_OK; +} + +/* +** The pCArray objects contains pointers to b-tree cells and the cell sizes. +** This function attempts to add the cells stored in the array to page pPg. +** If it cannot (because the page needs to be defragmented before the cells +** will fit), non-zero is returned. Otherwise, if the cells are added +** successfully, zero is returned. +** +** Argument pCellptr points to the first entry in the cell-pointer array +** (part of page pPg) to populate. After cell apCell[0] is written to the +** page body, a 16-bit offset is written to pCellptr. And so on, for each +** cell in the array. It is the responsibility of the caller to ensure +** that it is safe to overwrite this part of the cell-pointer array. +** +** When this function is called, *ppData points to the start of the +** content area on page pPg. If the size of the content area is extended, +** *ppData is updated to point to the new start of the content area +** before returning. +** +** Finally, argument pBegin points to the byte immediately following the +** end of the space required by this page for the cell-pointer area (for +** all cells - not just those inserted by the current call). If the content +** area must be extended to before this point in order to accomodate all +** cells in apCell[], then the cells do not fit and non-zero is returned. +*/ +static int pageInsertArray( + MemPage *pPg, /* Page to add cells to */ + u8 *pBegin, /* End of cell-pointer array */ + u8 **ppData, /* IN/OUT: Page content-area pointer */ + u8 *pCellptr, /* Pointer to cell-pointer area */ + int iFirst, /* Index of first cell to add */ + int nCell, /* Number of cells to add to pPg */ + CellArray *pCArray /* Array of cells */ +){ + int i = iFirst; /* Loop counter - cell index to insert */ + u8 *aData = pPg->aData; /* Complete page */ + u8 *pData = *ppData; /* Content area. A subset of aData[] */ + int iEnd = iFirst + nCell; /* End of loop. One past last cell to ins */ + int k; /* Current slot in pCArray->apEnd[] */ + u8 *pEnd; /* Maximum extent of cell data */ + assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ + if( iEnd<=iFirst ) return 0; + for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kapEnd[k]; + while( 1 /*Exit by break*/ ){ + int sz, rc; + u8 *pSlot; + assert( pCArray->szCell[i]!=0 ); + sz = pCArray->szCell[i]; + if( (aData[1]==0 && aData[2]==0) || (pSlot = pageFindSlot(pPg,sz,&rc))==0 ){ + if( (pData - pBegin)apCell[i] will never overlap on a well-formed + ** database. But they might for a corrupt database. Hence use memmove() + ** since memcpy() sends SIGABORT with overlapping buffers on OpenBSD */ + assert( (pSlot+sz)<=pCArray->apCell[i] + || pSlot>=(pCArray->apCell[i]+sz) + || CORRUPT_DB ); + if( (uptr)(pCArray->apCell[i]+sz)>(uptr)pEnd + && (uptr)(pCArray->apCell[i])<(uptr)pEnd + ){ + assert( CORRUPT_DB ); + (void)SQLITE_CORRUPT_BKPT; + return 1; + } + memmove(pSlot, pCArray->apCell[i], sz); + put2byte(pCellptr, (pSlot - aData)); + pCellptr += 2; + i++; + if( i>=iEnd ) break; + if( pCArray->ixNx[k]<=i ){ + k++; + pEnd = pCArray->apEnd[k]; + } + } + *ppData = pData; + return 0; +} + +/* +** The pCArray object contains pointers to b-tree cells and their sizes. +** +** This function adds the space associated with each cell in the array +** that is currently stored within the body of pPg to the pPg free-list. +** The cell-pointers and other fields of the page are not updated. +** +** This function returns the total number of cells added to the free-list. +*/ +static int pageFreeArray( + MemPage *pPg, /* Page to edit */ + int iFirst, /* First cell to delete */ + int nCell, /* Cells to delete */ + CellArray *pCArray /* Array of cells */ +){ + u8 * const aData = pPg->aData; + u8 * const pEnd = &aData[pPg->pBt->usableSize]; + u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; + int nRet = 0; + int i; + int iEnd = iFirst + nCell; + u8 *pFree = 0; + int szFree = 0; + + for(i=iFirst; iapCell[i]; + if( SQLITE_WITHIN(pCell, pStart, pEnd) ){ + int sz; + /* No need to use cachedCellSize() here. The sizes of all cells that + ** are to be freed have already been computing while deciding which + ** cells need freeing */ + sz = pCArray->szCell[i]; assert( sz>0 ); + if( pFree!=(pCell + sz) ){ + if( pFree ){ + assert( pFree>aData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } + pFree = pCell; + szFree = sz; + if( pFree+sz>pEnd ){ + return 0; + } + }else{ + pFree = pCell; + szFree += sz; + } + nRet++; + } + } + if( pFree ){ + assert( pFree>aData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } + return nRet; +} + +/* +** pCArray contains pointers to and sizes of all cells in the page being +** balanced. The current page, pPg, has pPg->nCell cells starting with +** pCArray->apCell[iOld]. After balancing, this page should hold nNew cells +** starting at apCell[iNew]. +** +** This routine makes the necessary adjustments to pPg so that it contains +** the correct cells after being balanced. +** +** The pPg->nFree field is invalid when this function returns. It is the +** responsibility of the caller to set it correctly. +*/ +static int editPage( + MemPage *pPg, /* Edit this page */ + int iOld, /* Index of first cell currently on page */ + int iNew, /* Index of new first cell on page */ + int nNew, /* Final number of cells on page */ + CellArray *pCArray /* Array of cells and sizes */ +){ + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + int iNewEnd = iNew + nNew; + +#ifdef SQLITE_DEBUG + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); +#endif + + /* Remove cells from the start and end of the page */ + assert( nCell>=0 ); + if( iOldnCell) ) return SQLITE_CORRUPT_BKPT; + memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + nCell -= nShift; + } + if( iNewEnd < iOldEnd ){ + int nTail = pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); + assert( nCell>=nTail ); + nCell -= nTail; + } + + pData = &aData[get2byteNotZero(&aData[hdr+5])]; + if( pDatapPg->aDataEnd) ) goto editpage_fail; + + /* Add cells to the start of the page */ + if( iNew=0 ); + pCellptr = pPg->aCellIdx; + memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iNew, nAdd, pCArray + ) ) goto editpage_fail; + nCell += nAdd; + } + + /* Add any overflow cells */ + for(i=0; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + if( nCell>iCell ){ + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + } + nCell++; + cachedCellSize(pCArray, iCell+iNew); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iCell+iNew, 1, pCArray + ) ) goto editpage_fail; + } + } + + /* Append cells to the end of the page */ + assert( nCell>=0 ); + pCellptr = &pPg->aCellIdx[nCell*2]; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + iNew+nCell, nNew-nCell, pCArray + ) ) goto editpage_fail; + + pPg->nCell = nNew; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + +#ifdef SQLITE_DEBUG + for(i=0; iapCell[i+iNew]; + int iOff = get2byteAligned(&pPg->aCellIdx[i*2]); + if( SQLITE_WITHIN(pCell, aData, &aData[pPg->pBt->usableSize]) ){ + pCell = &pTmp[pCell - aData]; + } + assert( 0==memcmp(pCell, &aData[iOff], + pCArray->pRef->xCellSize(pCArray->pRef, pCArray->apCell[i+iNew])) ); + } +#endif + + return SQLITE_OK; + editpage_fail: + /* Unable to edit this page. Rebuild it from scratch instead. */ + populateCellCache(pCArray, iNew, nNew); + return rebuildPage(pCArray, iNew, nNew, pPg); +} + + +#ifndef SQLITE_OMIT_QUICKBALANCE +/* +** This version of balance() handles the common special case where +** a new entry is being inserted on the extreme right-end of the +** tree, in other words, when the new entry will become the largest +** entry in the tree. +** +** Instead of trying to balance the 3 right-most leaf pages, just add +** a new page to the right-hand side and put the one new entry in +** that page. This leaves the right side of the tree somewhat +** unbalanced. But odds are that we will be inserting new entries +** at the end soon afterwards so the nearly empty page will quickly +** fill up. On average. +** +** pPage is the leaf page which is the right-most page in the tree. +** pParent is its parent. pPage must have a single overflow entry +** which is also the right-most entry on the page. +** +** The pSpace buffer is used to store a temporary copy of the divider +** cell that will be inserted into pParent. Such a cell consists of a 4 +** byte page number followed by a variable length integer. In other +** words, at most 13 bytes. Hence the pSpace buffer must be at +** least 13 bytes in size. +*/ +static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ + BtShared *const pBt = pPage->pBt; /* B-Tree Database */ + MemPage *pNew; /* Newly allocated page */ + int rc; /* Return Code */ + Pgno pgnoNew; /* Page number of pNew */ + + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + assert( pPage->nOverflow==1 ); + + if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; /* dbfuzz001.test */ + assert( pPage->nFree>=0 ); + assert( pParent->nFree>=0 ); + + /* Allocate a new page. This page will become the right-sibling of + ** pPage. Make the parent page writable, so that the new divider cell + ** may be inserted. If both these operations are successful, proceed. + */ + rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); + + if( rc==SQLITE_OK ){ + + u8 *pOut = &pSpace[4]; + u8 *pCell = pPage->apOvfl[0]; + u16 szCell = pPage->xCellSize(pPage, pCell); + u8 *pStop; + CellArray b; + + assert( sqlite3PagerIswriteable(pNew->pDbPage) ); + assert( CORRUPT_DB || pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); + zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); + b.nCell = 1; + b.pRef = pPage; + b.apCell = &pCell; + b.szCell = &szCell; + b.apEnd[0] = pPage->aDataEnd; + b.ixNx[0] = 2; + rc = rebuildPage(&b, 0, 1, pNew); + if( NEVER(rc) ){ + releasePage(pNew); + return rc; + } + pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; + + /* If this is an auto-vacuum database, update the pointer map + ** with entries for the new page, and any pointer from the + ** cell on the page to an overflow page. If either of these + ** operations fails, the return code is set, but the contents + ** of the parent page are still manipulated by thh code below. + ** That is Ok, at this point the parent page is guaranteed to + ** be marked as dirty. Returning an error code will cause a + ** rollback, undoing any changes made to the parent page. + */ + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); + if( szCell>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pNew, pCell, &rc); + } + } + + /* Create a divider cell to insert into pParent. The divider cell + ** consists of a 4-byte page number (the page number of pPage) and + ** a variable length key value (which must be the same value as the + ** largest key on pPage). + ** + ** To find the largest key value on pPage, first find the right-most + ** cell on pPage. The first two fields of this cell are the + ** record-length (a variable length integer at most 32-bits in size) + ** and the key value (a variable length integer, may have any value). + ** The first of the while(...) loops below skips over the record-length + ** field. The second while(...) loop copies the key value from the + ** cell on pPage into the pSpace buffer. + */ + pCell = findCell(pPage, pPage->nCell-1); + pStop = &pCell[9]; + while( (*(pCell++)&0x80) && pCellnCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno, &rc); + } + + /* Set the right-child pointer of pParent to point to the new page. */ + put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); + + /* Release the reference to the new page. */ + releasePage(pNew); + } + + return rc; +} +#endif /* SQLITE_OMIT_QUICKBALANCE */ + +#if 0 +/* +** This function does not contribute anything to the operation of SQLite. +** it is sometimes activated temporarily while debugging code responsible +** for setting pointer-map entries. +*/ +static int ptrmapCheckPages(MemPage **apPage, int nPage){ + int i, j; + for(i=0; ipBt; + assert( pPage->isInit ); + + for(j=0; jnCell; j++){ + CellInfo info; + u8 *z; + + z = findCell(pPage, j); + pPage->xParseCell(pPage, z, &info); + if( info.nLocalpgno && e==PTRMAP_OVERFLOW1 ); + } + if( !pPage->leaf ){ + Pgno child = get4byte(z); + ptrmapGet(pBt, child, &e, &n); + assert( n==pPage->pgno && e==PTRMAP_BTREE ); + } + } + if( !pPage->leaf ){ + Pgno child = get4byte(&pPage->aData[pPage->hdrOffset+8]); + ptrmapGet(pBt, child, &e, &n); + assert( n==pPage->pgno && e==PTRMAP_BTREE ); + } + } + return 1; +} +#endif + +/* +** This function is used to copy the contents of the b-tree node stored +** on page pFrom to page pTo. If page pFrom was not a leaf page, then +** the pointer-map entries for each child page are updated so that the +** parent page stored in the pointer map is page pTo. If pFrom contained +** any cells with overflow page pointers, then the corresponding pointer +** map entries are also updated so that the parent page is page pTo. +** +** If pFrom is currently carrying any overflow cells (entries in the +** MemPage.apOvfl[] array), they are not copied to pTo. +** +** Before returning, page pTo is reinitialized using btreeInitPage(). +** +** The performance of this function is not critical. It is only used by +** the balance_shallower() and balance_deeper() procedures, neither of +** which are called often under normal circumstances. +*/ +static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ + if( (*pRC)==SQLITE_OK ){ + BtShared * const pBt = pFrom->pBt; + u8 * const aFrom = pFrom->aData; + u8 * const aTo = pTo->aData; + int const iFromHdr = pFrom->hdrOffset; + int const iToHdr = ((pTo->pgno==1) ? 100 : 0); + int rc; + int iData; + + + assert( pFrom->isInit ); + assert( pFrom->nFree>=iToHdr ); + assert( get2byte(&aFrom[iFromHdr+5]) <= (int)pBt->usableSize ); + + /* Copy the b-tree node content from page pFrom to page pTo. */ + iData = get2byte(&aFrom[iFromHdr+5]); + memcpy(&aTo[iData], &aFrom[iData], pBt->usableSize-iData); + memcpy(&aTo[iToHdr], &aFrom[iFromHdr], pFrom->cellOffset + 2*pFrom->nCell); + + /* Reinitialize page pTo so that the contents of the MemPage structure + ** match the new data. The initialization of pTo can actually fail under + ** fairly obscure circumstances, even though it is a copy of initialized + ** page pFrom. + */ + pTo->isInit = 0; + rc = btreeInitPage(pTo); + if( rc==SQLITE_OK ) rc = btreeComputeFreeSpace(pTo); + if( rc!=SQLITE_OK ){ + *pRC = rc; + return; + } + + /* If this is an auto-vacuum database, update the pointer-map entries + ** for any b-tree or overflow pages that pTo now contains the pointers to. + */ + if( ISAUTOVACUUM ){ + *pRC = setChildPtrmaps(pTo); + } + } +} + +/* +** This routine redistributes cells on the iParentIdx'th child of pParent +** (hereafter "the page") and up to 2 siblings so that all pages have about the +** same amount of free space. Usually a single sibling on either side of the +** page are used in the balancing, though both siblings might come from one +** side if the page is the first or last child of its parent. If the page +** has fewer than 2 siblings (something which can only happen if the page +** is a root page or a child of a root page) then all available siblings +** participate in the balancing. +** +** The number of siblings of the page might be increased or decreased by +** one or two in an effort to keep pages nearly full but not over full. +** +** Note that when this routine is called, some of the cells on the page +** might not actually be stored in MemPage.aData[]. This can happen +** if the page is overfull. This routine ensures that all cells allocated +** to the page and its siblings fit into MemPage.aData[] before returning. +** +** In the course of balancing the page and its siblings, cells may be +** inserted into or removed from the parent page (pParent). Doing so +** may cause the parent page to become overfull or underfull. If this +** happens, it is the responsibility of the caller to invoke the correct +** balancing routine to fix this problem (see the balance() routine). +** +** If this routine fails for any reason, it might leave the database +** in a corrupted state. So if this routine fails, the database should +** be rolled back. +** +** The third argument to this function, aOvflSpace, is a pointer to a +** buffer big enough to hold one page. If while inserting cells into the parent +** page (pParent) the parent page becomes overfull, this buffer is +** used to store the parent's overflow cells. Because this function inserts +** a maximum of four divider cells into the parent page, and the maximum +** size of a cell stored within an internal node is always less than 1/4 +** of the page-size, the aOvflSpace[] buffer is guaranteed to be large +** enough for all overflow cells. +** +** If aOvflSpace is set to a null pointer, this function returns +** SQLITE_NOMEM. +*/ +static int balance_nonroot( + MemPage *pParent, /* Parent page of siblings being balanced */ + int iParentIdx, /* Index of "the page" in pParent */ + u8 *aOvflSpace, /* page-size bytes of space for parent ovfl */ + int isRoot, /* True if pParent is a root-page */ + int bBulk /* True if this call is part of a bulk load */ +){ + BtShared *pBt; /* The whole database */ + int nMaxCells = 0; /* Allocated size of apCell, szCell, aFrom. */ + int nNew = 0; /* Number of pages in apNew[] */ + int nOld; /* Number of pages in apOld[] */ + int i, j, k; /* Loop counters */ + int nxDiv; /* Next divider slot in pParent->aCell[] */ + int rc = SQLITE_OK; /* The return code */ + u16 leafCorrection; /* 4 if pPage is a leaf. 0 if not */ + int leafData; /* True if pPage is a leaf of a LEAFDATA tree */ + int usableSpace; /* Bytes in pPage beyond the header */ + int pageFlags; /* Value of pPage->aData[0] */ + int iSpace1 = 0; /* First unused byte of aSpace1[] */ + int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ + int szScratch; /* Size of scratch memory requested */ + MemPage *apOld[NB]; /* pPage and up to two siblings */ + MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ + u8 *pRight; /* Location in parent of right-sibling pointer */ + u8 *apDiv[NB-1]; /* Divider cells in pParent */ + int cntNew[NB+2]; /* Index in b.paCell[] of cell after i-th page */ + int cntOld[NB+2]; /* Old index in b.apCell[] */ + int szNew[NB+2]; /* Combined size of cells placed on i-th page */ + u8 *aSpace1; /* Space for copies of dividers cells */ + Pgno pgno; /* Temp var to store a page number in */ + u8 abDone[NB+2]; /* True after i'th new page is populated */ + Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ + u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ + CellArray b; /* Parsed information on cells being balanced */ + + memset(abDone, 0, sizeof(abDone)); + b.nCell = 0; + b.apCell = 0; + pBt = pParent->pBt; + assert( sqlite3_mutex_held(pBt->mutex) ); + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + + /* At this point pParent may have at most one overflow cell. And if + ** this overflow cell is present, it must be the cell with + ** index iParentIdx. This scenario comes about when this function + ** is called (indirectly) from sqlite3BtreeDelete(). + */ + assert( pParent->nOverflow==0 || pParent->nOverflow==1 ); + assert( pParent->nOverflow==0 || pParent->aiOvfl[0]==iParentIdx ); + + if( !aOvflSpace ){ + return SQLITE_NOMEM_BKPT; + } + assert( pParent->nFree>=0 ); + + /* Find the sibling pages to balance. Also locate the cells in pParent + ** that divide the siblings. An attempt is made to find NN siblings on + ** either side of pPage. More siblings are taken from one side, however, + ** if there are fewer than NN siblings on the other side. If pParent + ** has NB or fewer children then all children of pParent are taken. + ** + ** This loop also drops the divider cells from the parent page. This + ** way, the remainder of the function does not have to deal with any + ** overflow cells in the parent page, since if any existed they will + ** have already been removed. + */ + i = pParent->nOverflow + pParent->nCell; + if( i<2 ){ + nxDiv = 0; + }else{ + assert( bBulk==0 || bBulk==1 ); + if( iParentIdx==0 ){ + nxDiv = 0; + }else if( iParentIdx==i ){ + nxDiv = i-2+bBulk; + }else{ + nxDiv = iParentIdx-1; + } + i = 2-bBulk; + } + nOld = i+1; + if( (i+nxDiv-pParent->nOverflow)==pParent->nCell ){ + pRight = &pParent->aData[pParent->hdrOffset+8]; + }else{ + pRight = findCell(pParent, i+nxDiv-pParent->nOverflow); + } + pgno = get4byte(pRight); + while( 1 ){ + if( rc==SQLITE_OK ){ + rc = getAndInitPage(pBt, pgno, &apOld[i], 0, 0); + } + if( rc ){ + memset(apOld, 0, (i+1)*sizeof(MemPage*)); + goto balance_cleanup; + } + if( apOld[i]->nFree<0 ){ + rc = btreeComputeFreeSpace(apOld[i]); + if( rc ){ + memset(apOld, 0, (i)*sizeof(MemPage*)); + goto balance_cleanup; + } + } + nMaxCells += apOld[i]->nCell + ArraySize(pParent->apOvfl); + if( (i--)==0 ) break; + + if( pParent->nOverflow && i+nxDiv==pParent->aiOvfl[0] ){ + apDiv[i] = pParent->apOvfl[0]; + pgno = get4byte(apDiv[i]); + szNew[i] = pParent->xCellSize(pParent, apDiv[i]); + pParent->nOverflow = 0; + }else{ + apDiv[i] = findCell(pParent, i+nxDiv-pParent->nOverflow); + pgno = get4byte(apDiv[i]); + szNew[i] = pParent->xCellSize(pParent, apDiv[i]); + + /* Drop the cell from the parent page. apDiv[i] still points to + ** the cell within the parent, even though it has been dropped. + ** This is safe because dropping a cell only overwrites the first + ** four bytes of it, and this function does not need the first + ** four bytes of the divider cell. So the pointer is safe to use + ** later on. + ** + ** But not if we are in secure-delete mode. In secure-delete mode, + ** the dropCell() routine will overwrite the entire cell with zeroes. + ** In this case, temporarily copy the cell into the aOvflSpace[] + ** buffer. It will be copied out again as soon as the aSpace[] buffer + ** is allocated. */ + if( pBt->btsFlags & BTS_FAST_SECURE ){ + int iOff; + + /* If the following if() condition is not true, the db is corrupted. + ** The call to dropCell() below will detect this. */ + iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); + if( (iOff+szNew[i])<=(int)pBt->usableSize ){ + memcpy(&aOvflSpace[iOff], apDiv[i], szNew[i]); + apDiv[i] = &aOvflSpace[apDiv[i]-pParent->aData]; + } + } + dropCell(pParent, i+nxDiv-pParent->nOverflow, szNew[i], &rc); + } + } + + /* Make nMaxCells a multiple of 4 in order to preserve 8-byte + ** alignment */ + nMaxCells = (nMaxCells + 3)&~3; + + /* + ** Allocate space for memory structures + */ + szScratch = + nMaxCells*sizeof(u8*) /* b.apCell */ + + nMaxCells*sizeof(u16) /* b.szCell */ + + pBt->pageSize; /* aSpace1 */ + + assert( szScratch<=7*(int)pBt->pageSize ); + b.apCell = sqlite3StackAllocRaw(0, szScratch ); + if( b.apCell==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto balance_cleanup; + } + b.szCell = (u16*)&b.apCell[nMaxCells]; + aSpace1 = (u8*)&b.szCell[nMaxCells]; + assert( EIGHT_BYTE_ALIGNMENT(aSpace1) ); + + /* + ** Load pointers to all cells on sibling pages and the divider cells + ** into the local b.apCell[] array. Make copies of the divider cells + ** into space obtained from aSpace1[]. The divider cells have already + ** been removed from pParent. + ** + ** If the siblings are on leaf pages, then the child pointers of the + ** divider cells are stripped from the cells before they are copied + ** into aSpace1[]. In this way, all cells in b.apCell[] are without + ** child pointers. If siblings are not leaves, then all cell in + ** b.apCell[] include child pointers. Either way, all cells in b.apCell[] + ** are alike. + ** + ** leafCorrection: 4 if pPage is a leaf. 0 if pPage is not a leaf. + ** leafData: 1 if pPage holds key+data and pParent holds only keys. + */ + b.pRef = apOld[0]; + leafCorrection = b.pRef->leaf*4; + leafData = b.pRef->intKeyLeaf; + for(i=0; inCell; + u8 *aData = pOld->aData; + u16 maskPage = pOld->maskPage; + u8 *piCell = aData + pOld->cellOffset; + u8 *piEnd; + VVA_ONLY( int nCellAtStart = b.nCell; ) + + /* Verify that all sibling pages are of the same "type" (table-leaf, + ** table-interior, index-leaf, or index-interior). + */ + if( pOld->aData[0]!=apOld[0]->aData[0] ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + + /* Load b.apCell[] with pointers to all cells in pOld. If pOld + ** contains overflow cells, include them in the b.apCell[] array + ** in the correct spot. + ** + ** Note that when there are multiple overflow cells, it is always the + ** case that they are sequential and adjacent. This invariant arises + ** because multiple overflows can only occurs when inserting divider + ** cells into a parent on a prior balance, and divider cells are always + ** adjacent and are inserted in order. There is an assert() tagged + ** with "NOTE 1" in the overflow cell insertion loop to prove this + ** invariant. + ** + ** This must be done in advance. Once the balance starts, the cell + ** offset section of the btree page will be overwritten and we will no + ** long be able to find the cells if a pointer to each cell is not saved + ** first. + */ + memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); + if( pOld->nOverflow>0 ){ + if( NEVER(limitaiOvfl[0]) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + limit = pOld->aiOvfl[0]; + for(j=0; jnOverflow; k++){ + assert( k==0 || pOld->aiOvfl[k-1]+1==pOld->aiOvfl[k] );/* NOTE 1 */ + b.apCell[b.nCell] = pOld->apOvfl[k]; + b.nCell++; + } + } + piEnd = aData + pOld->cellOffset + 2*pOld->nCell; + while( piCellnCell+pOld->nOverflow) ); + + cntOld[i] = b.nCell; + if( imaxLocal+23 ); + assert( iSpace1 <= (int)pBt->pageSize ); + memcpy(pTemp, apDiv[i], sz); + b.apCell[b.nCell] = pTemp+leafCorrection; + assert( leafCorrection==0 || leafCorrection==4 ); + b.szCell[b.nCell] = b.szCell[b.nCell] - leafCorrection; + if( !pOld->leaf ){ + assert( leafCorrection==0 ); + assert( pOld->hdrOffset==0 || CORRUPT_DB ); + /* The right pointer of the child page pOld becomes the left + ** pointer of the divider cell */ + memcpy(b.apCell[b.nCell], &pOld->aData[8], 4); + }else{ + assert( leafCorrection==4 ); + while( b.szCell[b.nCell]<4 ){ + /* Do not allow any cells smaller than 4 bytes. If a smaller cell + ** does exist, pad it with 0x00 bytes. */ + assert( b.szCell[b.nCell]==3 || CORRUPT_DB ); + assert( b.apCell[b.nCell]==&aSpace1[iSpace1-3] || CORRUPT_DB ); + aSpace1[iSpace1++] = 0x00; + b.szCell[b.nCell]++; + } + } + b.nCell++; + } + } + + /* + ** Figure out the number of pages needed to hold all b.nCell cells. + ** Store this number in "k". Also compute szNew[] which is the total + ** size of all cells on the i-th page and cntNew[] which is the index + ** in b.apCell[] of the cell that divides page i from page i+1. + ** cntNew[k] should equal b.nCell. + ** + ** Values computed by this block: + ** + ** k: The total number of sibling pages + ** szNew[i]: Spaced used on the i-th sibling page. + ** cntNew[i]: Index in b.apCell[] and b.szCell[] for the first cell to + ** the right of the i-th sibling page. + ** usableSpace: Number of bytes of space available on each sibling. + ** + */ + usableSpace = pBt->usableSize - 12 + leafCorrection; + for(i=k=0; iaDataEnd; + b.ixNx[k] = cntOld[i]; + if( k && b.ixNx[k]==b.ixNx[k-1] ){ + k--; /* Omit b.ixNx[] entry for child pages with no cells */ + } + if( !leafData ){ + k++; + b.apEnd[k] = pParent->aDataEnd; + b.ixNx[k] = cntOld[i]+1; + } + assert( p->nFree>=0 ); + szNew[i] = usableSpace - p->nFree; + for(j=0; jnOverflow; j++){ + szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); + } + cntNew[i] = cntOld[i]; + } + k = nOld; + for(i=0; iusableSpace ){ + if( i+1>=k ){ + k = i+2; + if( k>NB+2 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } + szNew[k-1] = 0; + cntNew[k-1] = b.nCell; + } + sz = 2 + cachedCellSize(&b, cntNew[i]-1); + szNew[i] -= sz; + if( !leafData ){ + if( cntNew[i]usableSpace ) break; + szNew[i] += sz; + cntNew[i]++; + if( !leafData ){ + if( cntNew[i]=b.nCell ){ + k = i+1; + }else if( cntNew[i] <= (i>0 ? cntNew[i-1] : 0) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + } + + /* + ** The packing computed by the previous block is biased toward the siblings + ** on the left side (siblings with smaller keys). The left siblings are + ** always nearly full, while the right-most sibling might be nearly empty. + ** The next block of code attempts to adjust the packing of siblings to + ** get a better balance. + ** + ** This adjustment is more than an optimization. The packing above might + ** be so out of balance as to be illegal. For example, the right-most + ** sibling might be completely empty. This adjustment is not optional. + */ + for(i=k-1; i>0; i--){ + int szRight = szNew[i]; /* Size of sibling on the right */ + int szLeft = szNew[i-1]; /* Size of sibling on the left */ + int r; /* Index of right-most cell in left sibling */ + int d; /* Index of first cell to the left of right sibling */ + + r = cntNew[i-1] - 1; + d = r + 1 - leafData; + (void)cachedCellSize(&b, d); + do{ + assert( d szLeft-(b.szCell[r]+(i==k-1?0:2)))){ + break; + } + szRight += b.szCell[d] + 2; + szLeft -= b.szCell[r] + 2; + cntNew[i-1] = r; + r--; + d--; + }while( r>=0 ); + szNew[i] = szRight; + szNew[i-1] = szLeft; + if( cntNew[i-1] <= (i>1 ? cntNew[i-2] : 0) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + } + + /* Sanity check: For a non-corrupt database file one of the follwing + ** must be true: + ** (1) We found one or more cells (cntNew[0])>0), or + ** (2) pPage is a virtual root page. A virtual root page is when + ** the real root page is page 1 and we are the only child of + ** that page. + */ + assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); + TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + apOld[0]->pgno, apOld[0]->nCell, + nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, + nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 + )); + + /* + ** Allocate k new pages. Reuse old pages where possible. + */ + pageFlags = apOld[0]->aData[0]; + for(i=0; ipDbPage); + nNew++; + if( sqlite3PagerPageRefcount(pNew->pDbPage)!=1+(i==(iParentIdx-nxDiv)) + && rc==SQLITE_OK + ){ + rc = SQLITE_CORRUPT_BKPT; + } + if( rc ) goto balance_cleanup; + }else{ + assert( i>0 ); + rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0); + if( rc ) goto balance_cleanup; + zeroPage(pNew, pageFlags); + apNew[i] = pNew; + nNew++; + cntOld[i] = b.nCell; + + /* Set the pointer-map entry for the new sibling page. */ + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); + if( rc!=SQLITE_OK ){ + goto balance_cleanup; + } + } + } + } + + /* + ** Reassign page numbers so that the new pages are in ascending order. + ** This helps to keep entries in the disk file in order so that a scan + ** of the table is closer to a linear scan through the file. That in turn + ** helps the operating system to deliver pages from the disk more rapidly. + ** + ** An O(n^2) insertion sort algorithm is used, but since n is never more + ** than (NB+2) (a small constant), that should not be a problem. + ** + ** When NB==3, this one optimization makes the database about 25% faster + ** for large insertions and deletions. + */ + for(i=0; ipgno; + aPgFlags[i] = apNew[i]->pDbPage->flags; + for(j=0; ji ){ + sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); + } + sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); + apNew[i]->pgno = pgno; + } + } + + TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " + "%d(%d nc=%d) %d(%d nc=%d)\n", + apNew[0]->pgno, szNew[0], cntNew[0], + nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, + nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, + nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, + nNew>=3 ? cntNew[2] - cntNew[1] - !leafData : 0, + nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, + nNew>=4 ? cntNew[3] - cntNew[2] - !leafData : 0, + nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0, + nNew>=5 ? cntNew[4] - cntNew[3] - !leafData : 0 + )); + + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + assert( nNew>=1 && nNew<=ArraySize(apNew) ); + assert( apNew[nNew-1]!=0 ); + put4byte(pRight, apNew[nNew-1]->pgno); + + /* If the sibling pages are not leaves, ensure that the right-child pointer + ** of the right-most new sibling page is set to the value that was + ** originally in the same field of the right-most old sibling page. */ + if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){ + MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1]; + memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); + } + + /* Make any required updates to pointer map entries associated with + ** cells stored on sibling pages following the balance operation. Pointer + ** map entries associated with divider cells are set by the insertCell() + ** routine. The associated pointer map entries are: + ** + ** a) if the cell contains a reference to an overflow chain, the + ** entry associated with the first page in the overflow chain, and + ** + ** b) if the sibling pages are not leaves, the child page associated + ** with the cell. + ** + ** If the sibling pages are not leaves, then the pointer map entry + ** associated with the right-child of each sibling may also need to be + ** updated. This happens below, after the sibling pages have been + ** populated, not here. + */ + if( ISAUTOVACUUM ){ + MemPage *pOld; + MemPage *pNew = pOld = apNew[0]; + int cntOldNext = pNew->nCell + pNew->nOverflow; + int iNew = 0; + int iOld = 0; + + for(i=0; i=0 && iOldnCell + pOld->nOverflow + !leafData; + } + if( i==cntNew[iNew] ){ + pNew = apNew[++iNew]; + if( !leafData ) continue; + } + + /* Cell pCell is destined for new sibling page pNew. Originally, it + ** was either part of sibling page iOld (possibly an overflow cell), + ** or else the divider cell to the left of sibling page iOld. So, + ** if sibling page iOld had the same page number as pNew, and if + ** pCell really was a part of sibling page iOld (not a divider or + ** overflow cell), we can skip updating the pointer map entries. */ + if( iOld>=nNew + || pNew->pgno!=aPgno[iOld] + || !SQLITE_WITHIN(pCell,pOld->aData,pOld->aDataEnd) + ){ + if( !leafCorrection ){ + ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); + } + if( cachedCellSize(&b,i)>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pOld, pCell, &rc); + } + if( rc ) goto balance_cleanup; + } + } + } + + /* Insert new divider cells into pParent. */ + for(i=0; ileaf ){ + memcpy(&pNew->aData[8], pCell, 4); + }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in b.apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ + CellInfo info; + j--; + pNew->xParseCell(pNew, b.apCell[j], &info); + pCell = pTemp; + sz = 4 + putVarint(&pCell[4], info.nKey); + pTemp = 0; + }else{ + pCell -= 4; + /* Obscure case for non-leaf-data trees: If the cell at pCell was + ** previously stored on a leaf node, and its reported size was 4 + ** bytes, then it may actually be smaller than this + ** (see btreeParseCellPtr(), 4 bytes is the minimum size of + ** any cell). But it is important to pass the correct size to + ** insertCell(), so reparse the cell now. + ** + ** This can only happen for b-trees used to evaluate "IN (SELECT ...)" + ** and WITHOUT ROWID tables with exactly one column which is the + ** primary key. + */ + if( b.szCell[j]==4 ){ + assert(leafCorrection==4); + sz = pParent->xCellSize(pParent, pCell); + } + } + iOvflSpace += sz; + assert( sz<=pBt->maxLocal+23 ); + assert( iOvflSpace <= (int)pBt->pageSize ); + for(k=0; b.ixNx[k]<=i && ALWAYS(kpgno, &rc); + if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + } + + /* Now update the actual sibling pages. The order in which they are updated + ** is important, as this code needs to avoid disrupting any page from which + ** cells may still to be read. In practice, this means: + ** + ** (1) If cells are moving left (from apNew[iPg] to apNew[iPg-1]) + ** then it is not safe to update page apNew[iPg] until after + ** the left-hand sibling apNew[iPg-1] has been updated. + ** + ** (2) If cells are moving right (from apNew[iPg] to apNew[iPg+1]) + ** then it is not safe to update page apNew[iPg] until after + ** the right-hand sibling apNew[iPg+1] has been updated. + ** + ** If neither of the above apply, the page is safe to update. + ** + ** The iPg value in the following loop starts at nNew-1 goes down + ** to 0, then back up to nNew-1 again, thus making two passes over + ** the pages. On the initial downward pass, only condition (1) above + ** needs to be tested because (2) will always be true from the previous + ** step. On the upward pass, both conditions are always true, so the + ** upwards pass simply processes pages that were missed on the downward + ** pass. + */ + for(i=1-nNew; i=0 && iPg=0 /* On the upwards pass, or... */ + || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ + ){ + int iNew; + int iOld; + int nNewCell; + + /* Verify condition (1): If cells are moving left, update iPg + ** only after iPg-1 has already been updated. */ + assert( iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1] ); + + /* Verify condition (2): If cells are moving right, update iPg + ** only after iPg+1 has already been updated. */ + assert( cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1] ); + + if( iPg==0 ){ + iNew = iOld = 0; + nNewCell = cntNew[0]; + }else{ + iOld = iPgnFree = usableSpace-szNew[iPg]; + assert( apNew[iPg]->nOverflow==0 ); + assert( apNew[iPg]->nCell==nNewCell ); + } + } + + /* All pages have been processed exactly once */ + assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); + + assert( nOld>0 ); + assert( nNew>0 ); + + if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ + /* The root page of the b-tree now contains no cells. The only sibling + ** page is the right-child of the parent. Copy the contents of the + ** child page into the parent, decreasing the overall height of the + ** b-tree structure by one. This is described as the "balance-shallower" + ** sub-algorithm in some documentation. + ** + ** If this is an auto-vacuum database, the call to copyNodeContent() + ** sets all pointer-map entries corresponding to database image pages + ** for which the pointer is stored within the content being copied. + ** + ** It is critical that the child page be defragmented before being + ** copied into the parent, because if the parent is page 1 then it will + ** by smaller than the child due to the database header, and so all the + ** free space needs to be up front. + */ + assert( nNew==1 || CORRUPT_DB ); + rc = defragmentPage(apNew[0], -1); + testcase( rc!=SQLITE_OK ); + assert( apNew[0]->nFree == + (get2byteNotZero(&apNew[0]->aData[5]) - apNew[0]->cellOffset + - apNew[0]->nCell*2) + || rc!=SQLITE_OK + ); + copyNodeContent(apNew[0], pParent, &rc); + freePage(apNew[0], &rc); + }else if( ISAUTOVACUUM && !leafCorrection ){ + /* Fix the pointer map entries associated with the right-child of each + ** sibling page. All other pointer map entries have already been taken + ** care of. */ + for(i=0; iaData[8]); + ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); + } + } + + assert( pParent->isInit ); + TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + nOld, nNew, b.nCell)); + + /* Free any old pages that were not reused as new pages. + */ + for(i=nNew; iisInit ){ + /* The ptrmapCheckPages() contains assert() statements that verify that + ** all pointer map pages are set correctly. This is helpful while + ** debugging. This is usually disabled because a corrupt database may + ** cause an assert() statement to fail. */ + ptrmapCheckPages(apNew, nNew); + ptrmapCheckPages(&pParent, 1); + } +#endif + + /* + ** Cleanup before returning. + */ +balance_cleanup: + sqlite3StackFree(0, b.apCell); + for(i=0; ipBt; /* The BTree */ + + assert( pRoot->nOverflow>0 ); + assert( sqlite3_mutex_held(pBt->mutex) ); + + /* Make pRoot, the root page of the b-tree, writable. Allocate a new + ** page that will become the new right-child of pPage. Copy the contents + ** of the node stored on pRoot into the new child page. + */ + rc = sqlite3PagerWrite(pRoot->pDbPage); + if( rc==SQLITE_OK ){ + rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); + copyNodeContent(pRoot, pChild, &rc); + if( ISAUTOVACUUM ){ + ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); + } + } + if( rc ){ + *ppChild = 0; + releasePage(pChild); + return rc; + } + assert( sqlite3PagerIswriteable(pChild->pDbPage) ); + assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); + assert( pChild->nCell==pRoot->nCell || CORRUPT_DB ); + + TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); + + /* Copy the overflow cells from pRoot to pChild */ + memcpy(pChild->aiOvfl, pRoot->aiOvfl, + pRoot->nOverflow*sizeof(pRoot->aiOvfl[0])); + memcpy(pChild->apOvfl, pRoot->apOvfl, + pRoot->nOverflow*sizeof(pRoot->apOvfl[0])); + pChild->nOverflow = pRoot->nOverflow; + + /* Zero the contents of pRoot. Then install pChild as the right-child. */ + zeroPage(pRoot, pChild->aData[0] & ~PTF_LEAF); + put4byte(&pRoot->aData[pRoot->hdrOffset+8], pgnoChild); + + *ppChild = pChild; + return SQLITE_OK; +} + +/* +** Return SQLITE_CORRUPT if any cursor other than pCur is currently valid +** on the same B-tree as pCur. +** +** This can if a database is corrupt with two or more SQL tables +** pointing to the same b-tree. If an insert occurs on one SQL table +** and causes a BEFORE TRIGGER to do a secondary insert on the other SQL +** table linked to the same b-tree. If the secondary insert causes a +** rebalance, that can change content out from under the cursor on the +** first SQL table, violating invariants on the first insert. +*/ +static int anotherValidCursor(BtCursor *pCur){ + BtCursor *pOther; + for(pOther=pCur->pBt->pCursor; pOther; pOther=pOther->pNext){ + if( pOther!=pCur + && pOther->eState==CURSOR_VALID + && pOther->pPage==pCur->pPage + ){ + return SQLITE_CORRUPT_BKPT; + } + } + return SQLITE_OK; +} + +/* +** The page that pCur currently points to has just been modified in +** some way. This function figures out if this modification means the +** tree needs to be balanced, and if so calls the appropriate balancing +** routine. Balancing routines are: +** +** balance_quick() +** balance_deeper() +** balance_nonroot() +*/ +static int balance(BtCursor *pCur){ + int rc = SQLITE_OK; + const int nMin = pCur->pBt->usableSize * 2 / 3; + u8 aBalanceQuickSpace[13]; + u8 *pFree = 0; + + VVA_ONLY( int balance_quick_called = 0 ); + VVA_ONLY( int balance_deeper_called = 0 ); + + do { + int iPage; + MemPage *pPage = pCur->pPage; + + if( NEVER(pPage->nFree<0) && btreeComputeFreeSpace(pPage) ) break; + if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ + break; + }else if( (iPage = pCur->iPage)==0 ){ + if( pPage->nOverflow && (rc = anotherValidCursor(pCur))==SQLITE_OK ){ + /* The root page of the b-tree is overfull. In this case call the + ** balance_deeper() function to create a new child for the root-page + ** and copy the current contents of the root-page to it. The + ** next iteration of the do-loop will balance the child page. + */ + assert( balance_deeper_called==0 ); + VVA_ONLY( balance_deeper_called++ ); + rc = balance_deeper(pPage, &pCur->apPage[1]); + if( rc==SQLITE_OK ){ + pCur->iPage = 1; + pCur->ix = 0; + pCur->aiIdx[0] = 0; + pCur->apPage[0] = pPage; + pCur->pPage = pCur->apPage[1]; + assert( pCur->pPage->nOverflow ); + } + }else{ + break; + } + }else{ + MemPage * const pParent = pCur->apPage[iPage-1]; + int const iIdx = pCur->aiIdx[iPage-1]; + + rc = sqlite3PagerWrite(pParent->pDbPage); + if( rc==SQLITE_OK && pParent->nFree<0 ){ + rc = btreeComputeFreeSpace(pParent); + } + if( rc==SQLITE_OK ){ +#ifndef SQLITE_OMIT_QUICKBALANCE + if( pPage->intKeyLeaf + && pPage->nOverflow==1 + && pPage->aiOvfl[0]==pPage->nCell + && pParent->pgno!=1 + && pParent->nCell==iIdx + ){ + /* Call balance_quick() to create a new sibling of pPage on which + ** to store the overflow cell. balance_quick() inserts a new cell + ** into pParent, which may cause pParent overflow. If this + ** happens, the next iteration of the do-loop will balance pParent + ** use either balance_nonroot() or balance_deeper(). Until this + ** happens, the overflow cell is stored in the aBalanceQuickSpace[] + ** buffer. + ** + ** The purpose of the following assert() is to check that only a + ** single call to balance_quick() is made for each call to this + ** function. If this were not verified, a subtle bug involving reuse + ** of the aBalanceQuickSpace[] might sneak in. + */ + assert( balance_quick_called==0 ); + VVA_ONLY( balance_quick_called++ ); + rc = balance_quick(pParent, pPage, aBalanceQuickSpace); + }else +#endif + { + /* In this case, call balance_nonroot() to redistribute cells + ** between pPage and up to 2 of its sibling pages. This involves + ** modifying the contents of pParent, which may cause pParent to + ** become overfull or underfull. The next iteration of the do-loop + ** will balance the parent page to correct this. + ** + ** If the parent page becomes overfull, the overflow cell or cells + ** are stored in the pSpace buffer allocated immediately below. + ** A subsequent iteration of the do-loop will deal with this by + ** calling balance_nonroot() (balance_deeper() may be called first, + ** but it doesn't deal with overflow cells - just moves them to a + ** different page). Once this subsequent call to balance_nonroot() + ** has completed, it is safe to release the pSpace buffer used by + ** the previous call, as the overflow cell data will have been + ** copied either into the body of a database page or into the new + ** pSpace buffer passed to the latter call to balance_nonroot(). + */ + u8 *pSpace = sqlite3PageMalloc(pCur->pBt->pageSize); + rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, + pCur->hints&BTREE_BULKLOAD); + if( pFree ){ + /* If pFree is not NULL, it points to the pSpace buffer used + ** by a previous call to balance_nonroot(). Its contents are + ** now stored either on real database pages or within the + ** new pSpace buffer, so it may be safely freed here. */ + sqlite3PageFree(pFree); + } + + /* The pSpace buffer will be freed after the next call to + ** balance_nonroot(), or just before this function returns, whichever + ** comes first. */ + pFree = pSpace; + } + } + + pPage->nOverflow = 0; + + /* The next iteration of the do-loop balances the parent page. */ + releasePage(pPage); + pCur->iPage--; + assert( pCur->iPage>=0 ); + pCur->pPage = pCur->apPage[pCur->iPage]; + } + }while( rc==SQLITE_OK ); + + if( pFree ){ + sqlite3PageFree(pFree); + } + return rc; +} + +/* Overwrite content from pX into pDest. Only do the write if the +** content is different from what is already there. +*/ +static int btreeOverwriteContent( + MemPage *pPage, /* MemPage on which writing will occur */ + u8 *pDest, /* Pointer to the place to start writing */ + const BtreePayload *pX, /* Source of data to write */ + int iOffset, /* Offset of first byte to write */ + int iAmt /* Number of bytes to be written */ +){ + int nData = pX->nData - iOffset; + if( nData<=0 ){ + /* Overwritting with zeros */ + int i; + for(i=0; ipDbPage); + if( rc ) return rc; + memset(pDest + i, 0, iAmt - i); + } + }else{ + if( nDatapData) + iOffset, iAmt)!=0 ){ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ) return rc; + /* In a corrupt database, it is possible for the source and destination + ** buffers to overlap. This is harmless since the database is already + ** corrupt but it does cause valgrind and ASAN warnings. So use + ** memmove(). */ + memmove(pDest, ((u8*)pX->pData) + iOffset, iAmt); + } + } + return SQLITE_OK; +} + +/* +** Overwrite the cell that cursor pCur is pointing to with fresh content +** contained in pX. +*/ +static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ + int iOffset; /* Next byte of pX->pData to write */ + int nTotal = pX->nData + pX->nZero; /* Total bytes of to write */ + int rc; /* Return code */ + MemPage *pPage = pCur->pPage; /* Page being written */ + BtShared *pBt; /* Btree */ + Pgno ovflPgno; /* Next overflow page to write */ + u32 ovflPageSize; /* Size to write on overflow page */ + + if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd + || pCur->info.pPayload < pPage->aData + pPage->cellOffset + ){ + return SQLITE_CORRUPT_BKPT; + } + /* Overwrite the local portion first */ + rc = btreeOverwriteContent(pPage, pCur->info.pPayload, pX, + 0, pCur->info.nLocal); + if( rc ) return rc; + if( pCur->info.nLocal==nTotal ) return SQLITE_OK; + + /* Now overwrite the overflow pages */ + iOffset = pCur->info.nLocal; + assert( nTotal>=0 ); + assert( iOffset>=0 ); + ovflPgno = get4byte(pCur->info.pPayload + iOffset); + pBt = pPage->pBt; + ovflPageSize = pBt->usableSize - 4; + do{ + rc = btreeGetPage(pBt, ovflPgno, &pPage, 0); + if( rc ) return rc; + if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + if( iOffset+ovflPageSize<(u32)nTotal ){ + ovflPgno = get4byte(pPage->aData); + }else{ + ovflPageSize = nTotal - iOffset; + } + rc = btreeOverwriteContent(pPage, pPage->aData+4, pX, + iOffset, ovflPageSize); + } + sqlite3PagerUnref(pPage->pDbPage); + if( rc ) return rc; + iOffset += ovflPageSize; + }while( iOffset0 then pCur points to a cell +** that is larger than (pKey,nKey). +** +** If seekResult==0, that means pCur is pointing at some unknown location. +** In that case, this routine must seek the cursor to the correct insertion +** point for (pKey,nKey) before doing the insertion. For index btrees, +** if pX->nMem is non-zero, then pX->aMem contains pointers to the unpacked +** key values and pX->aMem can be used instead of pX->pKey to avoid having +** to decode the key. +*/ +SQLITE_PRIVATE int sqlite3BtreeInsert( + BtCursor *pCur, /* Insert data into the table of this cursor */ + const BtreePayload *pX, /* Content of the row to be inserted */ + int flags, /* True if this is likely an append */ + int seekResult /* Result of prior MovetoUnpacked() call */ +){ + int rc; + int loc = seekResult; /* -1: before desired location +1: after */ + int szNew = 0; + int idx; + MemPage *pPage; + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; + unsigned char *oldCell; + unsigned char *newCell = 0; + + assert( (flags & (BTREE_SAVEPOSITION|BTREE_APPEND|BTREE_PREFORMAT))==flags ); + assert( (flags & BTREE_PREFORMAT)==0 || seekResult || pCur->pKeyInfo==0 ); + + if( pCur->eState==CURSOR_FAULT ){ + assert( pCur->skipNext!=SQLITE_OK ); + return pCur->skipNext; + } + + assert( cursorOwnsBtShared(pCur) ); + assert( (pCur->curFlags & BTCF_WriteFlag)!=0 + && pBt->inTransaction==TRANS_WRITE + && (pBt->btsFlags & BTS_READ_ONLY)==0 ); + assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); + + /* Assert that the caller has been consistent. If this cursor was opened + ** expecting an index b-tree, then the caller should be inserting blob + ** keys with no associated data. If the cursor was opened expecting an + ** intkey table, the caller should be inserting integer keys with a + ** blob of associated data. */ + assert( (flags & BTREE_PREFORMAT) || (pX->pKey==0)==(pCur->pKeyInfo==0) ); + + /* Save the positions of any other cursors open on this table. + ** + ** In some cases, the call to btreeMoveto() below is a no-op. For + ** example, when inserting data into a table with auto-generated integer + ** keys, the VDBE layer invokes sqlite3BtreeLast() to figure out the + ** integer key to use. It then calls this function to actually insert the + ** data into the intkey B-Tree. In this case btreeMoveto() recognizes + ** that the cursor is already where it needs to be and returns without + ** doing any work. To avoid thwarting these optimizations, it is important + ** not to clear the cursor here. + */ + if( pCur->curFlags & BTCF_Multiple ){ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + if( loc && pCur->iPage<0 ){ + /* This can only happen if the schema is corrupt such that there is more + ** than one table or index with the same root page as used by the cursor. + ** Which can only happen if the SQLITE_NoSchemaError flag was set when + ** the schema was loaded. This cannot be asserted though, as a user might + ** set the flag, load the schema, and then unset the flag. */ + return SQLITE_CORRUPT_BKPT; + } + } + + if( pCur->pKeyInfo==0 ){ + assert( pX->pKey==0 ); + /* If this is an insert into a table b-tree, invalidate any incrblob + ** cursors open on the row being replaced */ + if( p->hasIncrblobCur ){ + invalidateIncrblobCursors(p, pCur->pgnoRoot, pX->nKey, 0); + } + + /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing + ** to a row with the same key as the new entry being inserted. + */ +#ifdef SQLITE_DEBUG + if( flags & BTREE_SAVEPOSITION ){ + assert( pCur->curFlags & BTCF_ValidNKey ); + assert( pX->nKey==pCur->info.nKey ); + assert( loc==0 ); + } +#endif + + /* On the other hand, BTREE_SAVEPOSITION==0 does not imply + ** that the cursor is not pointing to a row to be overwritten. + ** So do a complete check. + */ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey ){ + /* The cursor is pointing to the entry that is to be + ** overwritten */ + assert( pX->nData>=0 && pX->nZero>=0 ); + if( pCur->info.nSize!=0 + && pCur->info.nPayload==(u32)pX->nData+pX->nZero + ){ + /* New entry is the same size as the old. Do an overwrite */ + return btreeOverwriteCell(pCur, pX); + } + assert( loc==0 ); + }else if( loc==0 ){ + /* The cursor is *not* pointing to the cell to be overwritten, nor + ** to an adjacent cell. Move the cursor so that it is pointing either + ** to the cell to be overwritten or an adjacent cell. + */ + rc = sqlite3BtreeTableMoveto(pCur, pX->nKey, + (flags & BTREE_APPEND)!=0, &loc); + if( rc ) return rc; + } + }else{ + /* This is an index or a WITHOUT ROWID table */ + + /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing + ** to a row with the same key as the new entry being inserted. + */ + assert( (flags & BTREE_SAVEPOSITION)==0 || loc==0 ); + + /* If the cursor is not already pointing either to the cell to be + ** overwritten, or if a new cell is being inserted, if the cursor is + ** not pointing to an immediately adjacent cell, then move the cursor + ** so that it does. + */ + if( loc==0 && (flags & BTREE_SAVEPOSITION)==0 ){ + if( pX->nMem ){ + UnpackedRecord r; + r.pKeyInfo = pCur->pKeyInfo; + r.aMem = pX->aMem; + r.nField = pX->nMem; + r.default_rc = 0; + r.eqSeen = 0; + rc = sqlite3BtreeIndexMoveto(pCur, &r, &loc); + }else{ + rc = btreeMoveto(pCur, pX->pKey, pX->nKey, + (flags & BTREE_APPEND)!=0, &loc); + } + if( rc ) return rc; + } + + /* If the cursor is currently pointing to an entry to be overwritten + ** and the new content is the same as as the old, then use the + ** overwrite optimization. + */ + if( loc==0 ){ + getCellInfo(pCur); + if( pCur->info.nKey==pX->nKey ){ + BtreePayload x2; + x2.pData = pX->pKey; + x2.nData = pX->nKey; + x2.nZero = 0; + return btreeOverwriteCell(pCur, &x2); + } + } + } + assert( pCur->eState==CURSOR_VALID + || (pCur->eState==CURSOR_INVALID && loc) + || CORRUPT_DB ); + + pPage = pCur->pPage; + assert( pPage->intKey || pX->nKey>=0 || (flags & BTREE_PREFORMAT) ); + assert( pPage->leaf || !pPage->intKey ); + if( pPage->nFree<0 ){ + if( NEVER(pCur->eState>CURSOR_INVALID) ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = btreeComputeFreeSpace(pPage); + } + if( rc ) return rc; + } + + TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", + pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, + loc==0 ? "overwrite" : "new entry")); + assert( pPage->isInit ); + newCell = pBt->pTmpSpace; + assert( newCell!=0 ); + if( flags & BTREE_PREFORMAT ){ + rc = SQLITE_OK; + szNew = pBt->nPreformatSize; + if( szNew<4 ) szNew = 4; + if( ISAUTOVACUUM && szNew>pPage->maxLocal ){ + CellInfo info; + pPage->xParseCell(pPage, newCell, &info); + if( info.nPayload!=info.nLocal ){ + Pgno ovfl = get4byte(&newCell[szNew-4]); + ptrmapPut(pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); + } + } + }else{ + rc = fillInCell(pPage, newCell, pX, &szNew); + } + if( rc ) goto end_insert; + assert( szNew==pPage->xCellSize(pPage, newCell) ); + assert( szNew <= MX_CELL_SIZE(pBt) ); + idx = pCur->ix; + if( loc==0 ){ + CellInfo info; + assert( idxnCell ); + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ){ + goto end_insert; + } + oldCell = findCell(pPage, idx); + if( !pPage->leaf ){ + memcpy(newCell, oldCell, 4); + } + BTREE_CLEAR_CELL(rc, pPage, oldCell, info); + testcase( pCur->curFlags & BTCF_ValidOvfl ); + invalidateOverflowCache(pCur); + if( info.nSize==szNew && info.nLocal==info.nPayload + && (!ISAUTOVACUUM || szNewminLocal) + ){ + /* Overwrite the old cell with the new if they are the same size. + ** We could also try to do this if the old cell is smaller, then add + ** the leftover space to the free list. But experiments show that + ** doing that is no faster then skipping this optimization and just + ** calling dropCell() and insertCell(). + ** + ** This optimization cannot be used on an autovacuum database if the + ** new entry uses overflow pages, as the insertCell() call below is + ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ + assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ + if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ + return SQLITE_CORRUPT_BKPT; + } + if( oldCell+szNew > pPage->aDataEnd ){ + return SQLITE_CORRUPT_BKPT; + } + memcpy(oldCell, newCell, szNew); + return SQLITE_OK; + } + dropCell(pPage, idx, info.nSize, &rc); + if( rc ) goto end_insert; + }else if( loc<0 && pPage->nCell>0 ){ + assert( pPage->leaf ); + idx = ++pCur->ix; + pCur->curFlags &= ~BTCF_ValidNKey; + }else{ + assert( pPage->leaf ); + } + insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + assert( pPage->nOverflow==0 || rc==SQLITE_OK ); + assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); + + /* If no error has occurred and pPage has an overflow cell, call balance() + ** to redistribute the cells within the tree. Since balance() may move + ** the cursor, zero the BtCursor.info.nSize and BTCF_ValidNKey + ** variables. + ** + ** Previous versions of SQLite called moveToRoot() to move the cursor + ** back to the root page as balance() used to invalidate the contents + ** of BtCursor.apPage[] and BtCursor.aiIdx[]. Instead of doing that, + ** set the cursor state to "invalid". This makes common insert operations + ** slightly faster. + ** + ** There is a subtle but important optimization here too. When inserting + ** multiple records into an intkey b-tree using a single cursor (as can + ** happen while processing an "INSERT INTO ... SELECT" statement), it + ** is advantageous to leave the cursor pointing to the last entry in + ** the b-tree if possible. If the cursor is left pointing to the last + ** entry in the table, and the next row inserted has an integer key + ** larger than the largest existing key, it is possible to insert the + ** row without seeking the cursor. This can be a big performance boost. + */ + pCur->info.nSize = 0; + if( pPage->nOverflow ){ + assert( rc==SQLITE_OK ); + pCur->curFlags &= ~(BTCF_ValidNKey); + rc = balance(pCur); + + /* Must make sure nOverflow is reset to zero even if the balance() + ** fails. Internal data structure corruption will result otherwise. + ** Also, set the cursor state to invalid. This stops saveCursorPosition() + ** from trying to save the current position of the cursor. */ + pCur->pPage->nOverflow = 0; + pCur->eState = CURSOR_INVALID; + if( (flags & BTREE_SAVEPOSITION) && rc==SQLITE_OK ){ + btreeReleaseAllCursorPages(pCur); + if( pCur->pKeyInfo ){ + assert( pCur->pKey==0 ); + pCur->pKey = sqlite3Malloc( pX->nKey ); + if( pCur->pKey==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCur->pKey, pX->pKey, pX->nKey); + } + } + pCur->eState = CURSOR_REQUIRESEEK; + pCur->nKey = pX->nKey; + } + } + assert( pCur->iPage<0 || pCur->pPage->nOverflow==0 ); + +end_insert: + return rc; +} + +/* +** This function is used as part of copying the current row from cursor +** pSrc into cursor pDest. If the cursors are open on intkey tables, then +** parameter iKey is used as the rowid value when the record is copied +** into pDest. Otherwise, the record is copied verbatim. +** +** This function does not actually write the new value to cursor pDest. +** Instead, it creates and populates any required overflow pages and +** writes the data for the new cell into the BtShared.pTmpSpace buffer +** for the destination database. The size of the cell, in bytes, is left +** in BtShared.nPreformatSize. The caller completes the insertion by +** calling sqlite3BtreeInsert() with the BTREE_PREFORMAT flag specified. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ + int rc = SQLITE_OK; + BtShared *pBt = pDest->pBt; + u8 *aOut = pBt->pTmpSpace; /* Pointer to next output buffer */ + const u8 *aIn; /* Pointer to next input buffer */ + u32 nIn; /* Size of input buffer aIn[] */ + u32 nRem; /* Bytes of data still to copy */ + + getCellInfo(pSrc); + aOut += putVarint32(aOut, pSrc->info.nPayload); + if( pDest->pKeyInfo==0 ) aOut += putVarint(aOut, iKey); + nIn = pSrc->info.nLocal; + aIn = pSrc->info.pPayload; + if( aIn+nIn>pSrc->pPage->aDataEnd ){ + return SQLITE_CORRUPT_BKPT; + } + nRem = pSrc->info.nPayload; + if( nIn==nRem && nInpPage->maxLocal ){ + memcpy(aOut, aIn, nIn); + pBt->nPreformatSize = nIn + (aOut - pBt->pTmpSpace); + }else{ + Pager *pSrcPager = pSrc->pBt->pPager; + u8 *pPgnoOut = 0; + Pgno ovflIn = 0; + DbPage *pPageIn = 0; + MemPage *pPageOut = 0; + u32 nOut; /* Size of output buffer aOut[] */ + + nOut = btreePayloadToLocal(pDest->pPage, pSrc->info.nPayload); + pBt->nPreformatSize = nOut + (aOut - pBt->pTmpSpace); + if( nOutinfo.nPayload ){ + pPgnoOut = &aOut[nOut]; + pBt->nPreformatSize += 4; + } + + if( nRem>nIn ){ + if( aIn+nIn+4>pSrc->pPage->aDataEnd ){ + return SQLITE_CORRUPT_BKPT; + } + ovflIn = get4byte(&pSrc->info.pPayload[nIn]); + } + + do { + nRem -= nOut; + do{ + assert( nOut>0 ); + if( nIn>0 ){ + int nCopy = MIN(nOut, nIn); + memcpy(aOut, aIn, nCopy); + nOut -= nCopy; + nIn -= nCopy; + aOut += nCopy; + aIn += nCopy; + } + if( nOut>0 ){ + sqlite3PagerUnref(pPageIn); + pPageIn = 0; + rc = sqlite3PagerGet(pSrcPager, ovflIn, &pPageIn, PAGER_GET_READONLY); + if( rc==SQLITE_OK ){ + aIn = (const u8*)sqlite3PagerGetData(pPageIn); + ovflIn = get4byte(aIn); + aIn += 4; + nIn = pSrc->pBt->usableSize - 4; + } + } + }while( rc==SQLITE_OK && nOut>0 ); + + if( rc==SQLITE_OK && nRem>0 && ALWAYS(pPgnoOut) ){ + Pgno pgnoNew; + MemPage *pNew = 0; + rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); + put4byte(pPgnoOut, pgnoNew); + if( ISAUTOVACUUM && pPageOut ){ + ptrmapPut(pBt, pgnoNew, PTRMAP_OVERFLOW2, pPageOut->pgno, &rc); + } + releasePage(pPageOut); + pPageOut = pNew; + if( pPageOut ){ + pPgnoOut = pPageOut->aData; + put4byte(pPgnoOut, 0); + aOut = &pPgnoOut[4]; + nOut = MIN(pBt->usableSize - 4, nRem); + } + } + }while( nRem>0 && rc==SQLITE_OK ); + + releasePage(pPageOut); + sqlite3PagerUnref(pPageIn); + } + + return rc; +} + +/* +** Delete the entry that the cursor is pointing to. +** +** If the BTREE_SAVEPOSITION bit of the flags parameter is zero, then +** the cursor is left pointing at an arbitrary location after the delete. +** But if that bit is set, then the cursor is left in a state such that +** the next call to BtreeNext() or BtreePrev() moves it to the same row +** as it would have been on if the call to BtreeDelete() had been omitted. +** +** The BTREE_AUXDELETE bit of flags indicates that is one of several deletes +** associated with a single table entry and its indexes. Only one of those +** deletes is considered the "primary" delete. The primary delete occurs +** on a cursor that is not a BTREE_FORDELETE cursor. All but one delete +** operation on non-FORDELETE cursors is tagged with the AUXDELETE flag. +** The BTREE_AUXDELETE bit is a hint that is not used by this implementation, +** but which might be used by alternative storage engines. +*/ +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ + Btree *p = pCur->pBtree; + BtShared *pBt = p->pBt; + int rc; /* Return code */ + MemPage *pPage; /* Page to delete cell from */ + unsigned char *pCell; /* Pointer to cell to delete */ + int iCellIdx; /* Index of cell to delete */ + int iCellDepth; /* Depth of node containing pCell */ + CellInfo info; /* Size of the cell being deleted */ + int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ + u8 bPreserve = flags & BTREE_SAVEPOSITION; /* Keep cursor valid */ + + assert( cursorOwnsBtShared(pCur) ); + assert( pBt->inTransaction==TRANS_WRITE ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); + assert( pCur->curFlags & BTCF_WriteFlag ); + assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); + assert( !hasReadConflicts(p, pCur->pgnoRoot) ); + assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); + if( pCur->eState==CURSOR_REQUIRESEEK ){ + rc = btreeRestoreCursorPosition(pCur); + assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); + if( rc || pCur->eState!=CURSOR_VALID ) return rc; + } + assert( CORRUPT_DB || pCur->eState==CURSOR_VALID ); + + iCellDepth = pCur->iPage; + iCellIdx = pCur->ix; + pPage = pCur->pPage; + pCell = findCell(pPage, iCellIdx); + if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ) return SQLITE_CORRUPT; + + /* If the bPreserve flag is set to true, then the cursor position must + ** be preserved following this delete operation. If the current delete + ** will cause a b-tree rebalance, then this is done by saving the cursor + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. + ** + ** Or, if the current delete will not cause a rebalance, then the cursor + ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately + ** before or after the deleted entry. In this case set bSkipnext to true. */ + if( bPreserve ){ + if( !pPage->leaf + || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + || pPage->nCell==1 /* See dbfuzz001.test for a test case */ + ){ + /* A b-tree rebalance will be required after deleting this entry. + ** Save the cursor key. */ + rc = saveCursorKey(pCur); + if( rc ) return rc; + }else{ + bSkipnext = 1; + } + } + + /* If the page containing the entry to delete is not a leaf page, move + ** the cursor to the largest entry in the tree that is smaller than + ** the entry being deleted. This cell will replace the cell being deleted + ** from the internal node. The 'previous' entry is used for this instead + ** of the 'next' entry, as the previous entry is always a part of the + ** sub-tree headed by the child page of the cell being deleted. This makes + ** balancing the tree following the delete operation easier. */ + if( !pPage->leaf ){ + rc = sqlite3BtreePrevious(pCur, 0); + assert( rc!=SQLITE_DONE ); + if( rc ) return rc; + } + + /* Save the positions of any other cursors open on this table before + ** making any modifications. */ + if( pCur->curFlags & BTCF_Multiple ){ + rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + } + + /* If this is a delete operation to remove a row from a table b-tree, + ** invalidate any incrblob cursors open on the row being deleted. */ + if( pCur->pKeyInfo==0 && p->hasIncrblobCur ){ + invalidateIncrblobCursors(p, pCur->pgnoRoot, pCur->info.nKey, 0); + } + + /* Make the page containing the entry to be deleted writable. Then free any + ** overflow pages associated with the entry and finally remove the cell + ** itself from within the page. */ + rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ) return rc; + BTREE_CLEAR_CELL(rc, pPage, pCell, info); + dropCell(pPage, iCellIdx, info.nSize, &rc); + if( rc ) return rc; + + /* If the cell deleted was not located on a leaf page, then the cursor + ** is currently pointing to the largest entry in the sub-tree headed + ** by the child-page of the cell that was just deleted from an internal + ** node. The cell from the leaf node needs to be moved to the internal + ** node to replace the deleted cell. */ + if( !pPage->leaf ){ + MemPage *pLeaf = pCur->pPage; + int nCell; + Pgno n; + unsigned char *pTmp; + + if( pLeaf->nFree<0 ){ + rc = btreeComputeFreeSpace(pLeaf); + if( rc ) return rc; + } + if( iCellDepthiPage-1 ){ + n = pCur->apPage[iCellDepth+1]->pgno; + }else{ + n = pCur->pPage->pgno; + } + pCell = findCell(pLeaf, pLeaf->nCell-1); + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + nCell = pLeaf->xCellSize(pLeaf, pCell); + assert( MX_CELL_SIZE(pBt) >= nCell ); + pTmp = pBt->pTmpSpace; + assert( pTmp!=0 ); + rc = sqlite3PagerWrite(pLeaf->pDbPage); + if( rc==SQLITE_OK ){ + insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + } + dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); + if( rc ) return rc; + } + + /* Balance the tree. If the entry deleted was located on a leaf page, + ** then the cursor still points to that page. In this case the first + ** call to balance() repairs the tree, and the if(...) condition is + ** never true. + ** + ** Otherwise, if the entry deleted was on an internal node page, then + ** pCur is pointing to the leaf page from which a cell was removed to + ** replace the cell deleted from the internal node. This is slightly + ** tricky as the leaf node may be underfull, and the internal node may + ** be either under or overfull. In this case run the balancing algorithm + ** on the leaf node first. If the balance proceeds far enough up the + ** tree that we can be sure that any problem in the internal node has + ** been corrected, so be it. Otherwise, after balancing the leaf node, + ** walk the cursor up the tree to the internal node and balance it as + ** well. */ + rc = balance(pCur); + if( rc==SQLITE_OK && pCur->iPage>iCellDepth ){ + releasePageNotNull(pCur->pPage); + pCur->iPage--; + while( pCur->iPage>iCellDepth ){ + releasePage(pCur->apPage[pCur->iPage--]); + } + pCur->pPage = pCur->apPage[pCur->iPage]; + rc = balance(pCur); + } + + if( rc==SQLITE_OK ){ + if( bSkipnext ){ + assert( bPreserve && (pCur->iPage==iCellDepth || CORRUPT_DB) ); + assert( pPage==pCur->pPage || CORRUPT_DB ); + assert( (pPage->nCell>0 || CORRUPT_DB) && iCellIdx<=pPage->nCell ); + pCur->eState = CURSOR_SKIPNEXT; + if( iCellIdx>=pPage->nCell ){ + pCur->skipNext = -1; + pCur->ix = pPage->nCell-1; + }else{ + pCur->skipNext = 1; + } + }else{ + rc = moveToRoot(pCur); + if( bPreserve ){ + btreeReleaseAllCursorPages(pCur); + pCur->eState = CURSOR_REQUIRESEEK; + } + if( rc==SQLITE_EMPTY ) rc = SQLITE_OK; + } + } + return rc; +} + +/* +** Create a new BTree table. Write into *piTable the page +** number for the root page of the new table. +** +** The type of type is determined by the flags parameter. Only the +** following values of flags are currently in use. Other values for +** flags might not work: +** +** BTREE_INTKEY|BTREE_LEAFDATA Used for SQL tables with rowid keys +** BTREE_ZERODATA Used for SQL indices +*/ +static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ + BtShared *pBt = p->pBt; + MemPage *pRoot; + Pgno pgnoRoot; + int rc; + int ptfFlags; /* Page-type flage for the root page of new table */ + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( pBt->inTransaction==TRANS_WRITE ); + assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); + +#ifdef SQLITE_OMIT_AUTOVACUUM + rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); + if( rc ){ + return rc; + } +#else + if( pBt->autoVacuum ){ + Pgno pgnoMove; /* Move a page here to make room for the root-page */ + MemPage *pPageMove; /* The page to move to. */ + + /* Creating a new table may probably require moving an existing database + ** to make room for the new tables root page. In case this page turns + ** out to be an overflow page, delete all overflow page-map caches + ** held by open cursors. + */ + invalidateAllOverflowCache(pBt); + + /* Read the value of meta[3] from the database to determine where the + ** root page of the new table should go. meta[3] is the largest root-page + ** created so far, so the new root-page is (meta[3]+1). + */ + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); + if( pgnoRoot>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } + pgnoRoot++; + + /* The new root-page may not be allocated on a pointer-map page, or the + ** PENDING_BYTE page. + */ + while( pgnoRoot==PTRMAP_PAGENO(pBt, pgnoRoot) || + pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ + pgnoRoot++; + } + assert( pgnoRoot>=3 ); + + /* Allocate a page. The page that currently resides at pgnoRoot will + ** be moved to the allocated page (unless the allocated page happens + ** to reside at pgnoRoot). + */ + rc = allocateBtreePage(pBt, &pPageMove, &pgnoMove, pgnoRoot, BTALLOC_EXACT); + if( rc!=SQLITE_OK ){ + return rc; + } + + if( pgnoMove!=pgnoRoot ){ + /* pgnoRoot is the page that will be used for the root-page of + ** the new table (assuming an error did not occur). But we were + ** allocated pgnoMove. If required (i.e. if it was not allocated + ** by extending the file), the current page at position pgnoMove + ** is already journaled. + */ + u8 eType = 0; + Pgno iPtrPage = 0; + + /* Save the positions of any open cursors. This is required in + ** case they are holding a reference to an xFetch reference + ** corresponding to page pgnoRoot. */ + rc = saveAllCursors(pBt, 0, 0); + releasePage(pPageMove); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Move the page currently at pgnoRoot to pgnoMove. */ + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); + if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ + rc = SQLITE_CORRUPT_BKPT; + } + if( rc!=SQLITE_OK ){ + releasePage(pRoot); + return rc; + } + assert( eType!=PTRMAP_ROOTPAGE ); + assert( eType!=PTRMAP_FREEPAGE ); + rc = relocatePage(pBt, pRoot, eType, iPtrPage, pgnoMove, 0); + releasePage(pRoot); + + /* Obtain the page at pgnoRoot */ + if( rc!=SQLITE_OK ){ + return rc; + } + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = sqlite3PagerWrite(pRoot->pDbPage); + if( rc!=SQLITE_OK ){ + releasePage(pRoot); + return rc; + } + }else{ + pRoot = pPageMove; + } + + /* Update the pointer-map and meta-data with the new root-page number. */ + ptrmapPut(pBt, pgnoRoot, PTRMAP_ROOTPAGE, 0, &rc); + if( rc ){ + releasePage(pRoot); + return rc; + } + + /* When the new root page was allocated, page 1 was made writable in + ** order either to increase the database filesize, or to decrement the + ** freelist count. Hence, the sqlite3BtreeUpdateMeta() call cannot fail. + */ + assert( sqlite3PagerIswriteable(pBt->pPage1->pDbPage) ); + rc = sqlite3BtreeUpdateMeta(p, 4, pgnoRoot); + if( NEVER(rc) ){ + releasePage(pRoot); + return rc; + } + + }else{ + rc = allocateBtreePage(pBt, &pRoot, &pgnoRoot, 1, 0); + if( rc ) return rc; + } +#endif + assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); + if( createTabFlags & BTREE_INTKEY ){ + ptfFlags = PTF_INTKEY | PTF_LEAFDATA | PTF_LEAF; + }else{ + ptfFlags = PTF_ZERODATA | PTF_LEAF; + } + zeroPage(pRoot, ptfFlags); + sqlite3PagerUnref(pRoot->pDbPage); + assert( (pBt->openFlags & BTREE_SINGLE)==0 || pgnoRoot==2 ); + *piTable = pgnoRoot; + return SQLITE_OK; +} +SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree *p, Pgno *piTable, int flags){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeCreateTable(p, piTable, flags); + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Erase the given database page and all its children. Return +** the page to the freelist. +*/ +static int clearDatabasePage( + BtShared *pBt, /* The BTree that contains the table */ + Pgno pgno, /* Page number to clear */ + int freePageFlag, /* Deallocate page if true */ + i64 *pnChange /* Add number of Cells freed to this counter */ +){ + MemPage *pPage; + int rc; + unsigned char *pCell; + int i; + int hdr; + CellInfo info; + + assert( sqlite3_mutex_held(pBt->mutex) ); + if( pgno>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } + rc = getAndInitPage(pBt, pgno, &pPage, 0, 0); + if( rc ) return rc; + if( pPage->bBusy ){ + rc = SQLITE_CORRUPT_BKPT; + goto cleardatabasepage_out; + } + pPage->bBusy = 1; + hdr = pPage->hdrOffset; + for(i=0; inCell; i++){ + pCell = findCell(pPage, i); + if( !pPage->leaf ){ + rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); + if( rc ) goto cleardatabasepage_out; + } + BTREE_CLEAR_CELL(rc, pPage, pCell, info); + if( rc ) goto cleardatabasepage_out; + } + if( !pPage->leaf ){ + rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); + if( rc ) goto cleardatabasepage_out; + if( pPage->intKey ) pnChange = 0; + } + if( pnChange ){ + testcase( !pPage->intKey ); + *pnChange += pPage->nCell; + } + if( freePageFlag ){ + freePage(pPage, &rc); + }else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){ + zeroPage(pPage, pPage->aData[hdr] | PTF_LEAF); + } + +cleardatabasepage_out: + pPage->bBusy = 0; + releasePage(pPage); + return rc; +} + +/* +** Delete all information from a single table in the database. iTable is +** the page number of the root of the table. After this routine returns, +** the root page is empty, but still exists. +** +** This routine will fail with SQLITE_LOCKED if there are any open +** read cursors on the table. Open write cursors are moved to the +** root of the table. +** +** If pnChange is not NULL, then the integer value pointed to by pnChange +** is incremented by the number of entries in the table. +*/ +SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree *p, int iTable, i64 *pnChange){ + int rc; + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); + + rc = saveAllCursors(pBt, (Pgno)iTable, 0); + + if( SQLITE_OK==rc ){ + /* Invalidate all incrblob cursors open on table iTable (assuming iTable + ** is the root of a table b-tree - if it is not, the following call is + ** a no-op). */ + if( p->hasIncrblobCur ){ + invalidateIncrblobCursors(p, (Pgno)iTable, 0, 1); + } + rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); + } + sqlite3BtreeLeave(p); + return rc; +} + +/* +** Delete all information from the single table that pCur is open on. +** +** This routine only work for pCur on an ephemeral table. +*/ +SQLITE_PRIVATE int sqlite3BtreeClearTableOfCursor(BtCursor *pCur){ + return sqlite3BtreeClearTable(pCur->pBtree, pCur->pgnoRoot, 0); +} + +/* +** Erase all information in a table and add the root of the table to +** the freelist. Except, the root of the principle table (the one on +** page 1) is never added to the freelist. +** +** This routine will fail with SQLITE_LOCKED if there are any open +** cursors on the table. +** +** If AUTOVACUUM is enabled and the page at iTable is not the last +** root page in the database file, then the last root page +** in the database file is moved into the slot formerly occupied by +** iTable and that last slot formerly occupied by the last root page +** is added to the freelist instead of iTable. In this say, all +** root pages are kept at the beginning of the database file, which +** is necessary for AUTOVACUUM to work right. *piMoved is set to the +** page number that used to be the last root page in the file before +** the move. If no page gets moved, *piMoved is set to 0. +** The last root page is recorded in meta[3] and the value of +** meta[3] is updated by this procedure. +*/ +static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ + int rc; + MemPage *pPage = 0; + BtShared *pBt = p->pBt; + + assert( sqlite3BtreeHoldsMutex(p) ); + assert( p->inTrans==TRANS_WRITE ); + assert( iTable>=2 ); + if( iTable>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } + + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + if( rc ) return rc; + rc = sqlite3BtreeClearTable(p, iTable, 0); + if( rc ){ + releasePage(pPage); + return rc; + } + + *piMoved = 0; + +#ifdef SQLITE_OMIT_AUTOVACUUM + freePage(pPage, &rc); + releasePage(pPage); +#else + if( pBt->autoVacuum ){ + Pgno maxRootPgno; + sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &maxRootPgno); + + if( iTable==maxRootPgno ){ + /* If the table being dropped is the table with the largest root-page + ** number in the database, put the root page on the free list. + */ + freePage(pPage, &rc); + releasePage(pPage); + if( rc!=SQLITE_OK ){ + return rc; + } + }else{ + /* The table being dropped does not have the largest root-page + ** number in the database. So move the page that does into the + ** gap left by the deleted root-page. + */ + MemPage *pMove; + releasePage(pPage); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = relocatePage(pBt, pMove, PTRMAP_ROOTPAGE, 0, iTable, 0); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + pMove = 0; + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + freePage(pMove, &rc); + releasePage(pMove); + if( rc!=SQLITE_OK ){ + return rc; + } + *piMoved = maxRootPgno; + } + + /* Set the new 'max-root-page' value in the database header. This + ** is the old value less one, less one more if that happens to + ** be a root-page number, less one again if that is the + ** PENDING_BYTE_PAGE. + */ + maxRootPgno--; + while( maxRootPgno==PENDING_BYTE_PAGE(pBt) + || PTRMAP_ISPAGE(pBt, maxRootPgno) ){ + maxRootPgno--; + } + assert( maxRootPgno!=PENDING_BYTE_PAGE(pBt) ); + + rc = sqlite3BtreeUpdateMeta(p, 4, maxRootPgno); + }else{ + freePage(pPage, &rc); + releasePage(pPage); + } +#endif + return rc; +} +SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeDropTable(p, iTable, piMoved); + sqlite3BtreeLeave(p); + return rc; +} + + +/* +** This function may only be called if the b-tree connection already +** has a read or write transaction open on the database. +** +** Read the meta-information out of a database file. Meta[0] +** is the number of free pages currently in the database. Meta[1] +** through meta[15] are available for use by higher layers. Meta[0] +** is read-only, the others are read/write. +** +** The schema layer numbers meta values differently. At the schema +** layer (and the SetCookie and ReadCookie opcodes) the number of +** free pages is not visible. So Cookie[0] is the same as Meta[1]. +** +** This routine treats Meta[BTREE_DATA_VERSION] as a special case. Instead +** of reading the value out of the header, it instead loads the "DataVersion" +** from the pager. The BTREE_DATA_VERSION value is not actually stored in the +** database file. It is a number computed by the pager. But its access +** pattern is the same as header meta values, and so it is convenient to +** read it from this routine. +*/ +SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ + BtShared *pBt = p->pBt; + + sqlite3BtreeEnter(p); + assert( p->inTrans>TRANS_NONE ); + assert( SQLITE_OK==querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK) ); + assert( pBt->pPage1 ); + assert( idx>=0 && idx<=15 ); + + if( idx==BTREE_DATA_VERSION ){ + *pMeta = sqlite3PagerDataVersion(pBt->pPager) + p->iBDataVersion; + }else{ + *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + } + + /* If auto-vacuum is disabled in this build and this is an auto-vacuum + ** database, mark the database as read-only. */ +#ifdef SQLITE_OMIT_AUTOVACUUM + if( idx==BTREE_LARGEST_ROOT_PAGE && *pMeta>0 ){ + pBt->btsFlags |= BTS_READ_ONLY; + } +#endif + + sqlite3BtreeLeave(p); +} + +/* +** Write meta-information back into the database. Meta[0] is +** read-only and may not be written. +*/ +SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ + BtShared *pBt = p->pBt; + unsigned char *pP1; + int rc; + assert( idx>=1 && idx<=15 ); + sqlite3BtreeEnter(p); + assert( p->inTrans==TRANS_WRITE ); + assert( pBt->pPage1!=0 ); + pP1 = pBt->pPage1->aData; + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc==SQLITE_OK ){ + put4byte(&pP1[36 + idx*4], iMeta); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( idx==BTREE_INCR_VACUUM ){ + assert( pBt->autoVacuum || iMeta==0 ); + assert( iMeta==0 || iMeta==1 ); + pBt->incrVacuum = (u8)iMeta; + } +#endif + } + sqlite3BtreeLeave(p); + return rc; +} + +/* +** The first argument, pCur, is a cursor opened on some b-tree. Count the +** number of entries in the b-tree and write the result to *pnEntry. +** +** SQLITE_OK is returned if the operation is successfully executed. +** Otherwise, if an error is encountered (i.e. an IO error or database +** corruption) an SQLite error code is returned. +*/ +SQLITE_PRIVATE int sqlite3BtreeCount(sqlite3 *db, BtCursor *pCur, i64 *pnEntry){ + i64 nEntry = 0; /* Value to return in *pnEntry */ + int rc; /* Return code */ + + rc = moveToRoot(pCur); + if( rc==SQLITE_EMPTY ){ + *pnEntry = 0; + return SQLITE_OK; + } + + /* Unless an error occurs, the following loop runs one iteration for each + ** page in the B-Tree structure (not including overflow pages). + */ + while( rc==SQLITE_OK && !AtomicLoad(&db->u1.isInterrupted) ){ + int iIdx; /* Index of child node in parent */ + MemPage *pPage; /* Current page of the b-tree */ + + /* If this is a leaf page or the tree is not an int-key tree, then + ** this page contains countable entries. Increment the entry counter + ** accordingly. + */ + pPage = pCur->pPage; + if( pPage->leaf || !pPage->intKey ){ + nEntry += pPage->nCell; + } + + /* pPage is a leaf node. This loop navigates the cursor so that it + ** points to the first interior cell that it points to the parent of + ** the next page in the tree that has not yet been visited. The + ** pCur->aiIdx[pCur->iPage] value is set to the index of the parent cell + ** of the page, or to the number of cells in the page if the next page + ** to visit is the right-child of its parent. + ** + ** If all pages in the tree have been visited, return SQLITE_OK to the + ** caller. + */ + if( pPage->leaf ){ + do { + if( pCur->iPage==0 ){ + /* All pages of the b-tree have been visited. Return successfully. */ + *pnEntry = nEntry; + return moveToRoot(pCur); + } + moveToParent(pCur); + }while ( pCur->ix>=pCur->pPage->nCell ); + + pCur->ix++; + pPage = pCur->pPage; + } + + /* Descend to the child node of the cell that the cursor currently + ** points at. This is the right-child if (iIdx==pPage->nCell). + */ + iIdx = pCur->ix; + if( iIdx==pPage->nCell ){ + rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); + }else{ + rc = moveToChild(pCur, get4byte(findCell(pPage, iIdx))); + } + } + + /* An error has occurred. Return an error code. */ + return rc; +} + +/* +** Return the pager associated with a BTree. This routine is used for +** testing and debugging only. +*/ +SQLITE_PRIVATE Pager *sqlite3BtreePager(Btree *p){ + return p->pBt->pPager; +} + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* +** Append a message to the error message string. +*/ +static void checkAppendMsg( + IntegrityCk *pCheck, + const char *zFormat, + ... +){ + va_list ap; + if( !pCheck->mxErr ) return; + pCheck->mxErr--; + pCheck->nErr++; + va_start(ap, zFormat); + if( pCheck->errMsg.nChar ){ + sqlite3_str_append(&pCheck->errMsg, "\n", 1); + } + if( pCheck->zPfx ){ + sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); + } + sqlite3_str_vappendf(&pCheck->errMsg, zFormat, ap); + va_end(ap); + if( pCheck->errMsg.accError==SQLITE_NOMEM ){ + pCheck->bOomFault = 1; + } +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK + +/* +** Return non-zero if the bit in the IntegrityCk.aPgRef[] array that +** corresponds to page iPg is already set. +*/ +static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ + assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + return (pCheck->aPgRef[iPg/8] & (1 << (iPg & 0x07))); +} + +/* +** Set the bit in the IntegrityCk.aPgRef[] array that corresponds to page iPg. +*/ +static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ + assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + pCheck->aPgRef[iPg/8] |= (1 << (iPg & 0x07)); +} + + +/* +** Add 1 to the reference count for page iPage. If this is the second +** reference to the page, add an error message to pCheck->zErrMsg. +** Return 1 if there are 2 or more references to the page and 0 if +** if this is the first reference to the page. +** +** Also check that the page number is in bounds. +*/ +static int checkRef(IntegrityCk *pCheck, Pgno iPage){ + if( iPage>pCheck->nPage || iPage==0 ){ + checkAppendMsg(pCheck, "invalid page number %d", iPage); + return 1; + } + if( getPageReferenced(pCheck, iPage) ){ + checkAppendMsg(pCheck, "2nd reference to page %d", iPage); + return 1; + } + if( AtomicLoad(&pCheck->db->u1.isInterrupted) ) return 1; + setPageReferenced(pCheck, iPage); + return 0; +} + +#ifndef SQLITE_OMIT_AUTOVACUUM +/* +** Check that the entry in the pointer-map for page iChild maps to +** page iParent, pointer type ptrType. If not, append an error message +** to pCheck. +*/ +static void checkPtrmap( + IntegrityCk *pCheck, /* Integrity check context */ + Pgno iChild, /* Child page number */ + u8 eType, /* Expected pointer map type */ + Pgno iParent /* Expected pointer map parent page number */ +){ + int rc; + u8 ePtrmapType; + Pgno iPtrmapParent; + + rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->bOomFault = 1; + checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); + return; + } + + if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ + checkAppendMsg(pCheck, + "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", + iChild, eType, iParent, ePtrmapType, iPtrmapParent); + } +} +#endif + +/* +** Check the integrity of the freelist or of an overflow page list. +** Verify that the number of pages on the list is N. +*/ +static void checkList( + IntegrityCk *pCheck, /* Integrity checking context */ + int isFreeList, /* True for a freelist. False for overflow page list */ + Pgno iPage, /* Page number for first page in the list */ + u32 N /* Expected number of pages in the list */ +){ + int i; + u32 expected = N; + int nErrAtStart = pCheck->nErr; + while( iPage!=0 && pCheck->mxErr ){ + DbPage *pOvflPage; + unsigned char *pOvflData; + if( checkRef(pCheck, iPage) ) break; + N--; + if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ + checkAppendMsg(pCheck, "failed to get page %d", iPage); + break; + } + pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); + if( isFreeList ){ + u32 n = (u32)get4byte(&pOvflData[4]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pCheck->pBt->autoVacuum ){ + checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0); + } +#endif + if( n>pCheck->pBt->usableSize/4-2 ){ + checkAppendMsg(pCheck, + "freelist leaf count too big on page %d", iPage); + N--; + }else{ + for(i=0; i<(int)n; i++){ + Pgno iFreePage = get4byte(&pOvflData[8+i*4]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pCheck->pBt->autoVacuum ){ + checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0); + } +#endif + checkRef(pCheck, iFreePage); + } + N -= n; + } + } +#ifndef SQLITE_OMIT_AUTOVACUUM + else{ + /* If this database supports auto-vacuum and iPage is not the last + ** page in this overflow list, check that the pointer-map entry for + ** the following page matches iPage. + */ + if( pCheck->pBt->autoVacuum && N>0 ){ + i = get4byte(pOvflData); + checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage); + } + } +#endif + iPage = get4byte(pOvflData); + sqlite3PagerUnref(pOvflPage); + } + if( N && nErrAtStart==pCheck->nErr ){ + checkAppendMsg(pCheck, + "%s is %d but should be %d", + isFreeList ? "size" : "overflow list length", + expected-N, expected); + } +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +/* +** An implementation of a min-heap. +** +** aHeap[0] is the number of elements on the heap. aHeap[1] is the +** root element. The daughter nodes of aHeap[N] are aHeap[N*2] +** and aHeap[N*2+1]. +** +** The heap property is this: Every node is less than or equal to both +** of its daughter nodes. A consequence of the heap property is that the +** root node aHeap[1] is always the minimum value currently in the heap. +** +** The btreeHeapInsert() routine inserts an unsigned 32-bit number onto +** the heap, preserving the heap property. The btreeHeapPull() routine +** removes the root element from the heap (the minimum value in the heap) +** and then moves other nodes around as necessary to preserve the heap +** property. +** +** This heap is used for cell overlap and coverage testing. Each u32 +** entry represents the span of a cell or freeblock on a btree page. +** The upper 16 bits are the index of the first byte of a range and the +** lower 16 bits are the index of the last byte of that range. +*/ +static void btreeHeapInsert(u32 *aHeap, u32 x){ + u32 j, i = ++aHeap[0]; + aHeap[i] = x; + while( (j = i/2)>0 && aHeap[j]>aHeap[i] ){ + x = aHeap[j]; + aHeap[j] = aHeap[i]; + aHeap[i] = x; + i = j; + } +} +static int btreeHeapPull(u32 *aHeap, u32 *pOut){ + u32 j, i, x; + if( (x = aHeap[0])==0 ) return 0; + *pOut = aHeap[1]; + aHeap[1] = aHeap[x]; + aHeap[x] = 0xffffffff; + aHeap[0]--; + i = 1; + while( (j = i*2)<=aHeap[0] ){ + if( aHeap[j]>aHeap[j+1] ) j++; + if( aHeap[i]zPfx; + int saved_v1 = pCheck->v1; + int saved_v2 = pCheck->v2; + u8 savedIsInit = 0; + + /* Check that the page exists + */ + pBt = pCheck->pBt; + usableSize = pBt->usableSize; + if( iPage==0 ) return 0; + if( checkRef(pCheck, iPage) ) return 0; + pCheck->zPfx = "Page %u: "; + pCheck->v1 = iPage; + if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ + checkAppendMsg(pCheck, + "unable to get the page. error code=%d", rc); + goto end_of_check; + } + + /* Clear MemPage.isInit to make sure the corruption detection code in + ** btreeInitPage() is executed. */ + savedIsInit = pPage->isInit; + pPage->isInit = 0; + if( (rc = btreeInitPage(pPage))!=0 ){ + assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ + checkAppendMsg(pCheck, + "btreeInitPage() returns error code %d", rc); + goto end_of_check; + } + if( (rc = btreeComputeFreeSpace(pPage))!=0 ){ + assert( rc==SQLITE_CORRUPT ); + checkAppendMsg(pCheck, "free space corruption", rc); + goto end_of_check; + } + data = pPage->aData; + hdr = pPage->hdrOffset; + + /* Set up for cell analysis */ + pCheck->zPfx = "On tree page %u cell %d: "; + contentOffset = get2byteNotZero(&data[hdr+5]); + assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ + + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + nCell = get2byte(&data[hdr+3]); + assert( pPage->nCell==nCell ); + + /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page + ** immediately follows the b-tree page header. */ + cellStart = hdr + 12 - 4*pPage->leaf; + assert( pPage->aCellIdx==&data[cellStart] ); + pCellIdx = &data[cellStart + 2*(nCell-1)]; + + if( !pPage->leaf ){ + /* Analyze the right-child page of internal pages */ + pgno = get4byte(&data[hdr+8]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + pCheck->zPfx = "On page %u at right child: "; + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); + } +#endif + depth = checkTreePage(pCheck, pgno, &maxKey, maxKey); + keyCanBeEqual = 0; + }else{ + /* For leaf pages, the coverage check will occur in the same loop + ** as the other cell checks, so initialize the heap. */ + heap = pCheck->heap; + heap[0] = 0; + } + + /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte + ** integer offsets to the cell contents. */ + for(i=nCell-1; i>=0 && pCheck->mxErr; i--){ + CellInfo info; + + /* Check cell size */ + pCheck->v2 = i; + assert( pCellIdx==&data[cellStart + i*2] ); + pc = get2byteAligned(pCellIdx); + pCellIdx -= 2; + if( pcusableSize-4 ){ + checkAppendMsg(pCheck, "Offset %d out of range %d..%d", + pc, contentOffset, usableSize-4); + doCoverageCheck = 0; + continue; + } + pCell = &data[pc]; + pPage->xParseCell(pPage, pCell, &info); + if( pc+info.nSize>usableSize ){ + checkAppendMsg(pCheck, "Extends off end of page"); + doCoverageCheck = 0; + continue; + } + + /* Check for integer primary key out of range */ + if( pPage->intKey ){ + if( keyCanBeEqual ? (info.nKey > maxKey) : (info.nKey >= maxKey) ){ + checkAppendMsg(pCheck, "Rowid %lld out of order", info.nKey); + } + maxKey = info.nKey; + keyCanBeEqual = 0; /* Only the first key on the page may ==maxKey */ + } + + /* Check the content overflow list */ + if( info.nPayload>info.nLocal ){ + u32 nPage; /* Number of pages on the overflow chain */ + Pgno pgnoOvfl; /* First page of the overflow chain */ + assert( pc + info.nSize - 4 <= usableSize ); + nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4); + pgnoOvfl = get4byte(&pCell[info.nSize - 4]); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage); + } +#endif + checkList(pCheck, 0, pgnoOvfl, nPage); + } + + if( !pPage->leaf ){ + /* Check sanity of left child page for internal pages */ + pgno = get4byte(pCell); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); + } +#endif + d2 = checkTreePage(pCheck, pgno, &maxKey, maxKey); + keyCanBeEqual = 0; + if( d2!=depth ){ + checkAppendMsg(pCheck, "Child page depth differs"); + depth = d2; + } + }else{ + /* Populate the coverage-checking heap for leaf pages */ + btreeHeapInsert(heap, (pc<<16)|(pc+info.nSize-1)); + } + } + *piMinKey = maxKey; + + /* Check for complete coverage of the page + */ + pCheck->zPfx = 0; + if( doCoverageCheck && pCheck->mxErr>0 ){ + /* For leaf pages, the min-heap has already been initialized and the + ** cells have already been inserted. But for internal pages, that has + ** not yet been done, so do it now */ + if( !pPage->leaf ){ + heap = pCheck->heap; + heap[0] = 0; + for(i=nCell-1; i>=0; i--){ + u32 size; + pc = get2byteAligned(&data[cellStart+i*2]); + size = pPage->xCellSize(pPage, &data[pc]); + btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); + } + } + /* Add the freeblocks to the min-heap + ** + ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header + ** is the offset of the first freeblock, or zero if there are no + ** freeblocks on the page. + */ + i = get2byte(&data[hdr+1]); + while( i>0 ){ + int size, j; + assert( (u32)i<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ + size = get2byte(&data[i+2]); + assert( (u32)(i+size)<=usableSize ); /* due to btreeComputeFreeSpace() */ + btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1)); + /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a + ** big-endian integer which is the offset in the b-tree page of the next + ** freeblock in the chain, or zero if the freeblock is the last on the + ** chain. */ + j = get2byte(&data[i]); + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ + assert( j==0 || j>i+size ); /* Enforced by btreeComputeFreeSpace() */ + assert( (u32)j<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ + i = j; + } + /* Analyze the min-heap looking for overlap between cells and/or + ** freeblocks, and counting the number of untracked bytes in nFrag. + ** + ** Each min-heap entry is of the form: (start_address<<16)|end_address. + ** There is an implied first entry the covers the page header, the cell + ** pointer index, and the gap between the cell pointer index and the start + ** of cell content. + ** + ** The loop below pulls entries from the min-heap in order and compares + ** the start_address against the previous end_address. If there is an + ** overlap, that means bytes are used multiple times. If there is a gap, + ** that gap is added to the fragmentation count. + */ + nFrag = 0; + prev = contentOffset - 1; /* Implied first min-heap entry */ + while( btreeHeapPull(heap,&x) ){ + if( (prev&0xffff)>=(x>>16) ){ + checkAppendMsg(pCheck, + "Multiple uses for byte %u of page %u", x>>16, iPage); + break; + }else{ + nFrag += (x>>16) - (prev&0xffff) - 1; + prev = x; + } + } + nFrag += usableSize - (prev&0xffff) - 1; + /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments + ** is stored in the fifth field of the b-tree page header. + ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the + ** number of fragmented free bytes within the cell content area. + */ + if( heap[0]==0 && nFrag!=data[hdr+7] ){ + checkAppendMsg(pCheck, + "Fragmentation of %d bytes reported as %d on page %u", + nFrag, data[hdr+7], iPage); + } + } + +end_of_check: + if( !doCoverageCheck ) pPage->isInit = savedIsInit; + releasePage(pPage); + pCheck->zPfx = saved_zPfx; + pCheck->v1 = saved_v1; + pCheck->v2 = saved_v2; + return depth+1; +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* +** This routine does a complete check of the given BTree file. aRoot[] is +** an array of pages numbers were each page number is the root page of +** a table. nRoot is the number of entries in aRoot. +** +** A read-only or read-write transaction must be opened before calling +** this function. +** +** Write the number of error seen in *pnErr. Except for some memory +** allocation errors, an error message held in memory obtained from +** malloc is returned if *pnErr is non-zero. If *pnErr==0 then NULL is +** returned. If a memory allocation error occurs, NULL is returned. +** +** If the first entry in aRoot[] is 0, that indicates that the list of +** root pages is incomplete. This is a "partial integrity-check". This +** happens when performing an integrity check on a single table. The +** zero is skipped, of course. But in addition, the freelist checks +** and the checks to make sure every page is referenced are also skipped, +** since obviously it is not possible to know which pages are covered by +** the unverified btrees. Except, if aRoot[1] is 1, then the freelist +** checks are still performed. +*/ +SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( + sqlite3 *db, /* Database connection that is running the check */ + Btree *p, /* The btree to be checked */ + Pgno *aRoot, /* An array of root pages numbers for individual trees */ + int nRoot, /* Number of entries in aRoot[] */ + int mxErr, /* Stop reporting errors after this many */ + int *pnErr /* Write number of errors seen to this variable */ +){ + Pgno i; + IntegrityCk sCheck; + BtShared *pBt = p->pBt; + u64 savedDbFlags = pBt->db->flags; + char zErr[100]; + int bPartial = 0; /* True if not checking all btrees */ + int bCkFreelist = 1; /* True to scan the freelist */ + VVA_ONLY( int nRef ); + assert( nRoot>0 ); + + /* aRoot[0]==0 means this is a partial check */ + if( aRoot[0]==0 ){ + assert( nRoot>1 ); + bPartial = 1; + if( aRoot[1]!=1 ) bCkFreelist = 0; + } + + sqlite3BtreeEnter(p); + assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); + VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); + assert( nRef>=0 ); + sCheck.db = db; + sCheck.pBt = pBt; + sCheck.pPager = pBt->pPager; + sCheck.nPage = btreePagecount(sCheck.pBt); + sCheck.mxErr = mxErr; + sCheck.nErr = 0; + sCheck.bOomFault = 0; + sCheck.zPfx = 0; + sCheck.v1 = 0; + sCheck.v2 = 0; + sCheck.aPgRef = 0; + sCheck.heap = 0; + sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); + sCheck.errMsg.printfFlags = SQLITE_PRINTF_INTERNAL; + if( sCheck.nPage==0 ){ + goto integrity_ck_cleanup; + } + + sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); + if( !sCheck.aPgRef ){ + sCheck.bOomFault = 1; + goto integrity_ck_cleanup; + } + sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); + if( sCheck.heap==0 ){ + sCheck.bOomFault = 1; + goto integrity_ck_cleanup; + } + + i = PENDING_BYTE_PAGE(pBt); + if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); + + /* Check the integrity of the freelist + */ + if( bCkFreelist ){ + sCheck.zPfx = "Main freelist: "; + checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), + get4byte(&pBt->pPage1->aData[36])); + sCheck.zPfx = 0; + } + + /* Check all the tables. + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + if( !bPartial ){ + if( pBt->autoVacuum ){ + Pgno mx = 0; + Pgno mxInHdr; + for(i=0; (int)ipPage1->aData[52]); + if( mx!=mxInHdr ){ + checkAppendMsg(&sCheck, + "max rootpage (%d) disagrees with header (%d)", + mx, mxInHdr + ); + } + }else if( get4byte(&pBt->pPage1->aData[64])!=0 ){ + checkAppendMsg(&sCheck, + "incremental_vacuum enabled with a max rootpage of zero" + ); + } + } +#endif + testcase( pBt->db->flags & SQLITE_CellSizeCk ); + pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; + for(i=0; (int)iautoVacuum && aRoot[i]>1 && !bPartial ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); + } +#endif + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + } + pBt->db->flags = savedDbFlags; + + /* Make sure every page in the file is referenced + */ + if( !bPartial ){ + for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ +#ifdef SQLITE_OMIT_AUTOVACUUM + if( getPageReferenced(&sCheck, i)==0 ){ + checkAppendMsg(&sCheck, "Page %d is never used", i); + } +#else + /* If the database supports auto-vacuum, make sure no tables contain + ** references to pointer-map pages. + */ + if( getPageReferenced(&sCheck, i)==0 && + (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ + checkAppendMsg(&sCheck, "Page %d is never used", i); + } + if( getPageReferenced(&sCheck, i)!=0 && + (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ + checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i); + } +#endif + } + } + + /* Clean up and report errors. + */ +integrity_ck_cleanup: + sqlite3PageFree(sCheck.heap); + sqlite3_free(sCheck.aPgRef); + if( sCheck.bOomFault ){ + sqlite3_str_reset(&sCheck.errMsg); + sCheck.nErr++; + } + *pnErr = sCheck.nErr; + if( sCheck.nErr==0 ) sqlite3_str_reset(&sCheck.errMsg); + /* Make sure this analysis did not leave any unref() pages. */ + assert( nRef==sqlite3PagerRefcount(pBt->pPager) ); + sqlite3BtreeLeave(p); + return sqlite3StrAccumFinish(&sCheck.errMsg); +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +/* +** Return the full pathname of the underlying database file. Return +** an empty string if the database is in-memory or a TEMP database. +** +** The pager filename is invariant as long as the pager is +** open so it is safe to access without the BtShared mutex. +*/ +SQLITE_PRIVATE const char *sqlite3BtreeGetFilename(Btree *p){ + assert( p->pBt->pPager!=0 ); + return sqlite3PagerFilename(p->pBt->pPager, 1); +} + +/* +** Return the pathname of the journal file for this database. The return +** value of this routine is the same regardless of whether the journal file +** has been created or not. +** +** The pager journal filename is invariant as long as the pager is +** open so it is safe to access without the BtShared mutex. +*/ +SQLITE_PRIVATE const char *sqlite3BtreeGetJournalname(Btree *p){ + assert( p->pBt->pPager!=0 ); + return sqlite3PagerJournalname(p->pBt->pPager); +} + +/* +** Return one of SQLITE_TXN_NONE, SQLITE_TXN_READ, or SQLITE_TXN_WRITE +** to describe the current transaction state of Btree p. +*/ +SQLITE_PRIVATE int sqlite3BtreeTxnState(Btree *p){ + assert( p==0 || sqlite3_mutex_held(p->db->mutex) ); + return p ? p->inTrans : 0; +} + +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on the Btree passed as the first argument. +** +** Return SQLITE_LOCKED if this or any other connection has an open +** transaction on the shared-cache the argument Btree is connected to. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +*/ +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; + if( p ){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + if( pBt->inTransaction!=TRANS_NONE ){ + rc = SQLITE_LOCKED; + }else{ + rc = sqlite3PagerCheckpoint(pBt->pPager, p->db, eMode, pnLog, pnCkpt); + } + sqlite3BtreeLeave(p); + } + return rc; +} +#endif + +/* +** Return true if there is currently a backup running on Btree p. +*/ +SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree *p){ + assert( p ); + assert( sqlite3_mutex_held(p->db->mutex) ); + return p->nBackup!=0; +} + +/* +** This function returns a pointer to a blob of memory associated with +** a single shared-btree. The memory is used by client code for its own +** purposes (for example, to store a high-level schema associated with +** the shared-btree). The btree layer manages reference counting issues. +** +** The first time this is called on a shared-btree, nBytes bytes of memory +** are allocated, zeroed, and returned to the caller. For each subsequent +** call the nBytes parameter is ignored and a pointer to the same blob +** of memory returned. +** +** If the nBytes parameter is 0 and the blob of memory has not yet been +** allocated, a null pointer is returned. If the blob has already been +** allocated, it is returned as normal. +** +** Just before the shared-btree is closed, the function passed as the +** xFree argument when the memory allocation was made is invoked on the +** blob of allocated memory. The xFree function should not call sqlite3_free() +** on the memory, the btree layer does that. +*/ +SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void *)){ + BtShared *pBt = p->pBt; + sqlite3BtreeEnter(p); + if( !pBt->pSchema && nBytes ){ + pBt->pSchema = sqlite3DbMallocZero(0, nBytes); + pBt->xFreeSchema = xFree; + } + sqlite3BtreeLeave(p); + return pBt->pSchema; +} + +/* +** Return SQLITE_LOCKED_SHAREDCACHE if another user of the same shared +** btree as the argument handle holds an exclusive lock on the +** sqlite_schema table. Otherwise SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *p){ + int rc; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + rc = querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK); + assert( rc==SQLITE_OK || rc==SQLITE_LOCKED_SHAREDCACHE ); + sqlite3BtreeLeave(p); + return rc; +} + + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** Obtain a lock on the table whose root page is iTab. The +** lock is a write lock if isWritelock is true or a read lock +** if it is false. +*/ +SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ + int rc = SQLITE_OK; + assert( p->inTrans!=TRANS_NONE ); + if( p->sharable ){ + u8 lockType = READ_LOCK + isWriteLock; + assert( READ_LOCK+1==WRITE_LOCK ); + assert( isWriteLock==0 || isWriteLock==1 ); + + sqlite3BtreeEnter(p); + rc = querySharedCacheTableLock(p, iTab, lockType); + if( rc==SQLITE_OK ){ + rc = setSharedCacheTableLock(p, iTab, lockType); + } + sqlite3BtreeLeave(p); + } + return rc; +} +#endif + +#ifndef SQLITE_OMIT_INCRBLOB +/* +** Argument pCsr must be a cursor opened for writing on an +** INTKEY table currently pointing at a valid table entry. +** This function modifies the data stored as part of that entry. +** +** Only the data content may only be modified, it is not possible to +** change the length of the data stored. If this function is called with +** parameters that attempt to write past the end of the existing data, +** no modifications are made and SQLITE_CORRUPT is returned. +*/ +SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ + int rc; + assert( cursorOwnsBtShared(pCsr) ); + assert( sqlite3_mutex_held(pCsr->pBtree->db->mutex) ); + assert( pCsr->curFlags & BTCF_Incrblob ); + + rc = restoreCursorPosition(pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } + assert( pCsr->eState!=CURSOR_REQUIRESEEK ); + if( pCsr->eState!=CURSOR_VALID ){ + return SQLITE_ABORT; + } + + /* Save the positions of all other cursors open on this table. This is + ** required in case any of them are holding references to an xFetch + ** version of the b-tree page modified by the accessPayload call below. + ** + ** Note that pCsr must be open on a INTKEY table and saveCursorPosition() + ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence + ** saveAllCursors can only return SQLITE_OK. + */ + VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + assert( rc==SQLITE_OK ); + + /* Check some assumptions: + ** (a) the cursor is open for writing, + ** (b) there is a read/write transaction open, + ** (c) the connection holds a write-lock on the table (if required), + ** (d) there are no conflicting read-locks, and + ** (e) the cursor points at a valid row of an intKey table. + */ + if( (pCsr->curFlags & BTCF_WriteFlag)==0 ){ + return SQLITE_READONLY; + } + assert( (pCsr->pBt->btsFlags & BTS_READ_ONLY)==0 + && pCsr->pBt->inTransaction==TRANS_WRITE ); + assert( hasSharedCacheTableLock(pCsr->pBtree, pCsr->pgnoRoot, 0, 2) ); + assert( !hasReadConflicts(pCsr->pBtree, pCsr->pgnoRoot) ); + assert( pCsr->pPage->intKey ); + + return accessPayload(pCsr, offset, amt, (unsigned char *)z, 1); +} + +/* +** Mark this cursor as an incremental blob cursor. +*/ +SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ + pCur->curFlags |= BTCF_Incrblob; + pCur->pBtree->hasIncrblobCur = 1; +} +#endif + +/* +** Set both the "read version" (single byte at byte offset 18) and +** "write version" (single byte at byte offset 19) fields in the database +** header to iVersion. +*/ +SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ + BtShared *pBt = pBtree->pBt; + int rc; /* Return code */ + + assert( iVersion==1 || iVersion==2 ); + + /* If setting the version fields to 1, do not automatically open the + ** WAL connection, even if the version fields are currently set to 2. + */ + pBt->btsFlags &= ~BTS_NO_WAL; + if( iVersion==1 ) pBt->btsFlags |= BTS_NO_WAL; + + rc = sqlite3BtreeBeginTrans(pBtree, 0, 0); + if( rc==SQLITE_OK ){ + u8 *aData = pBt->pPage1->aData; + if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ + rc = sqlite3BtreeBeginTrans(pBtree, 2, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc==SQLITE_OK ){ + aData[18] = (u8)iVersion; + aData[19] = (u8)iVersion; + } + } + } + } + + pBt->btsFlags &= ~BTS_NO_WAL; + return rc; +} + +/* +** Return true if the cursor has a hint specified. This routine is +** only used from within assert() statements +*/ +SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor *pCsr, unsigned int mask){ + return (pCsr->hints & mask)!=0; +} + +/* +** Return true if the given Btree is read-only. +*/ +SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ + return (p->pBt->btsFlags & BTS_READ_ONLY)!=0; +} + +/* +** Return the size of the header added to each page by this module. +*/ +SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } + +#if !defined(SQLITE_OMIT_SHARED_CACHE) +/* +** Return true if the Btree passed as the only argument is sharable. +*/ +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree *p){ + return p->sharable; +} + +/* +** Return the number of connections to the BtShared object accessed by +** the Btree handle passed as the only argument. For private caches +** this is always 1. For shared caches it may be 1 or greater. +*/ +SQLITE_PRIVATE int sqlite3BtreeConnectionCount(Btree *p){ + testcase( p->sharable ); + return p->pBt->nRef; +} +#endif + +/************** End of btree.c ***********************************************/ +/************** Begin file backup.c ******************************************/ +/* +** 2009 January 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the implementation of the sqlite3_backup_XXX() +** API functions and the related features. +*/ +/* #include "sqliteInt.h" */ +/* #include "btreeInt.h" */ + +/* +** Structure allocated for each backup operation. +*/ +struct sqlite3_backup { + sqlite3* pDestDb; /* Destination database handle */ + Btree *pDest; /* Destination b-tree file */ + u32 iDestSchema; /* Original schema cookie in destination */ + int bDestLocked; /* True once a write-transaction is open on pDest */ + + Pgno iNext; /* Page number of the next source page to copy */ + sqlite3* pSrcDb; /* Source database handle */ + Btree *pSrc; /* Source b-tree file */ + + int rc; /* Backup process error code */ + + /* These two variables are set by every call to backup_step(). They are + ** read by calls to backup_remaining() and backup_pagecount(). + */ + Pgno nRemaining; /* Number of pages left to copy */ + Pgno nPagecount; /* Total number of pages to copy */ + + int isAttached; /* True once backup has been registered with pager */ + sqlite3_backup *pNext; /* Next backup associated with source pager */ +}; + +/* +** THREAD SAFETY NOTES: +** +** Once it has been created using backup_init(), a single sqlite3_backup +** structure may be accessed via two groups of thread-safe entry points: +** +** * Via the sqlite3_backup_XXX() API function backup_step() and +** backup_finish(). Both these functions obtain the source database +** handle mutex and the mutex associated with the source BtShared +** structure, in that order. +** +** * Via the BackupUpdate() and BackupRestart() functions, which are +** invoked by the pager layer to report various state changes in +** the page cache associated with the source database. The mutex +** associated with the source database BtShared structure will always +** be held when either of these functions are invoked. +** +** The other sqlite3_backup_XXX() API functions, backup_remaining() and +** backup_pagecount() are not thread-safe functions. If they are called +** while some other thread is calling backup_step() or backup_finish(), +** the values returned may be invalid. There is no way for a call to +** BackupUpdate() or BackupRestart() to interfere with backup_remaining() +** or backup_pagecount(). +** +** Depending on the SQLite configuration, the database handles and/or +** the Btree objects may have their own mutexes that require locking. +** Non-sharable Btrees (in-memory databases for example), do not have +** associated mutexes. +*/ + +/* +** Return a pointer corresponding to database zDb (i.e. "main", "temp") +** in connection handle pDb. If such a database cannot be found, return +** a NULL pointer and write an error message to pErrorDb. +** +** If the "temp" database is requested, it may need to be opened by this +** function. If an error occurs while doing so, return 0 and write an +** error message to pErrorDb. +*/ +static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ + int i = sqlite3FindDbName(pDb, zDb); + + if( i==1 ){ + Parse sParse; + int rc = 0; + memset(&sParse, 0, sizeof(sParse)); + sParse.db = pDb; + if( sqlite3OpenTempDatabase(&sParse) ){ + sqlite3ErrorWithMsg(pErrorDb, sParse.rc, "%s", sParse.zErrMsg); + rc = SQLITE_ERROR; + } + sqlite3DbFree(pErrorDb, sParse.zErrMsg); + sqlite3ParserReset(&sParse); + if( rc ){ + return 0; + } + } + + if( i<0 ){ + sqlite3ErrorWithMsg(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb); + return 0; + } + + return pDb->aDb[i].pBt; +} + +/* +** Attempt to set the page size of the destination to match the page size +** of the source. +*/ +static int setDestPgsz(sqlite3_backup *p){ + int rc; + rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),0,0); + return rc; +} + +/* +** Check that there is no open read-transaction on the b-tree passed as the +** second argument. If there is not, return SQLITE_OK. Otherwise, if there +** is an open read-transaction, return SQLITE_ERROR and leave an error +** message in database handle db. +*/ +static int checkReadTransaction(sqlite3 *db, Btree *p){ + if( sqlite3BtreeTxnState(p)!=SQLITE_TXN_NONE ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "destination database is in use"); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + +/* +** Create an sqlite3_backup process to copy the contents of zSrcDb from +** connection handle pSrcDb to zDestDb in pDestDb. If successful, return +** a pointer to the new sqlite3_backup object. +** +** If an error occurs, NULL is returned and an error code and error message +** stored in database handle pDestDb. +*/ +SQLITE_API sqlite3_backup *sqlite3_backup_init( + sqlite3* pDestDb, /* Database to write to */ + const char *zDestDb, /* Name of database within pDestDb */ + sqlite3* pSrcDb, /* Database connection to read from */ + const char *zSrcDb /* Name of database within pSrcDb */ +){ + sqlite3_backup *p; /* Value to return */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pSrcDb)||!sqlite3SafetyCheckOk(pDestDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + /* Lock the source database handle. The destination database + ** handle is not locked in this routine, but it is locked in + ** sqlite3_backup_step(). The user is required to ensure that no + ** other thread accesses the destination handle for the duration + ** of the backup operation. Any attempt to use the destination + ** database connection while a backup is in progress may cause + ** a malfunction or a deadlock. + */ + sqlite3_mutex_enter(pSrcDb->mutex); + sqlite3_mutex_enter(pDestDb->mutex); + + if( pSrcDb==pDestDb ){ + sqlite3ErrorWithMsg( + pDestDb, SQLITE_ERROR, "source and destination must be distinct" + ); + p = 0; + }else { + /* Allocate space for a new sqlite3_backup object... + ** EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ + p = (sqlite3_backup *)sqlite3MallocZero(sizeof(sqlite3_backup)); + if( !p ){ + sqlite3Error(pDestDb, SQLITE_NOMEM_BKPT); + } + } + + /* If the allocation succeeded, populate the new object. */ + if( p ){ + p->pSrc = findBtree(pDestDb, pSrcDb, zSrcDb); + p->pDest = findBtree(pDestDb, pDestDb, zDestDb); + p->pDestDb = pDestDb; + p->pSrcDb = pSrcDb; + p->iNext = 1; + p->isAttached = 0; + + if( 0==p->pSrc || 0==p->pDest + || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK + ){ + /* One (or both) of the named databases did not exist or an OOM + ** error was hit. Or there is a transaction open on the destination + ** database. The error has already been written into the pDestDb + ** handle. All that is left to do here is free the sqlite3_backup + ** structure. */ + sqlite3_free(p); + p = 0; + } + } + if( p ){ + p->pSrc->nBackup++; + } + + sqlite3_mutex_leave(pDestDb->mutex); + sqlite3_mutex_leave(pSrcDb->mutex); + return p; +} + +/* +** Argument rc is an SQLite error code. Return true if this error is +** considered fatal if encountered during a backup operation. All errors +** are considered fatal except for SQLITE_BUSY and SQLITE_LOCKED. +*/ +static int isFatalError(int rc){ + return (rc!=SQLITE_OK && rc!=SQLITE_BUSY && ALWAYS(rc!=SQLITE_LOCKED)); +} + +/* +** Parameter zSrcData points to a buffer containing the data for +** page iSrcPg from the source database. Copy this data into the +** destination database. +*/ +static int backupOnePage( + sqlite3_backup *p, /* Backup handle */ + Pgno iSrcPg, /* Source database page to backup */ + const u8 *zSrcData, /* Source database page data */ + int bUpdate /* True for an update, false otherwise */ +){ + Pager * const pDestPager = sqlite3BtreePager(p->pDest); + const int nSrcPgsz = sqlite3BtreeGetPageSize(p->pSrc); + int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); + const int nCopy = MIN(nSrcPgsz, nDestPgsz); + const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; + int rc = SQLITE_OK; + i64 iOff; + + assert( sqlite3BtreeGetReserveNoMutex(p->pSrc)>=0 ); + assert( p->bDestLocked ); + assert( !isFatalError(p->rc) ); + assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); + assert( zSrcData ); + + /* Catch the case where the destination is an in-memory database and the + ** page sizes of the source and destination differ. + */ + if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){ + rc = SQLITE_READONLY; + } + + /* This loop runs once for each destination page spanned by the source + ** page. For each iteration, variable iOff is set to the byte offset + ** of the destination page. + */ + for(iOff=iEnd-(i64)nSrcPgsz; rc==SQLITE_OK && iOffpDest->pBt) ) continue; + if( SQLITE_OK==(rc = sqlite3PagerGet(pDestPager, iDest, &pDestPg, 0)) + && SQLITE_OK==(rc = sqlite3PagerWrite(pDestPg)) + ){ + const u8 *zIn = &zSrcData[iOff%nSrcPgsz]; + u8 *zDestData = sqlite3PagerGetData(pDestPg); + u8 *zOut = &zDestData[iOff%nDestPgsz]; + + /* Copy the data from the source page into the destination page. + ** Then clear the Btree layer MemPage.isInit flag. Both this module + ** and the pager code use this trick (clearing the first byte + ** of the page 'extra' space to invalidate the Btree layers + ** cached parse of the page). MemPage.isInit is marked + ** "MUST BE FIRST" for this purpose. + */ + memcpy(zOut, zIn, nCopy); + ((u8 *)sqlite3PagerGetExtra(pDestPg))[0] = 0; + if( iOff==0 && bUpdate==0 ){ + sqlite3Put4byte(&zOut[28], sqlite3BtreeLastPage(p->pSrc)); + } + } + sqlite3PagerUnref(pDestPg); + } + + return rc; +} + +/* +** If pFile is currently larger than iSize bytes, then truncate it to +** exactly iSize bytes. If pFile is not larger than iSize bytes, then +** this function is a no-op. +** +** Return SQLITE_OK if everything is successful, or an SQLite error +** code if an error occurs. +*/ +static int backupTruncateFile(sqlite3_file *pFile, i64 iSize){ + i64 iCurrent; + int rc = sqlite3OsFileSize(pFile, &iCurrent); + if( rc==SQLITE_OK && iCurrent>iSize ){ + rc = sqlite3OsTruncate(pFile, iSize); + } + return rc; +} + +/* +** Register this backup object with the associated source pager for +** callbacks when pages are changed or the cache invalidated. +*/ +static void attachBackupObject(sqlite3_backup *p){ + sqlite3_backup **pp; + assert( sqlite3BtreeHoldsMutex(p->pSrc) ); + pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + p->pNext = *pp; + *pp = p; + p->isAttached = 1; +} + +/* +** Copy nPage pages from the source b-tree to the destination. +*/ +SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ + int rc; + int destMode; /* Destination journal mode */ + int pgszSrc = 0; /* Source page size */ + int pgszDest = 0; /* Destination page size */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(p->pSrcDb->mutex); + sqlite3BtreeEnter(p->pSrc); + if( p->pDestDb ){ + sqlite3_mutex_enter(p->pDestDb->mutex); + } + + rc = p->rc; + if( !isFatalError(rc) ){ + Pager * const pSrcPager = sqlite3BtreePager(p->pSrc); /* Source pager */ + Pager * const pDestPager = sqlite3BtreePager(p->pDest); /* Dest pager */ + int ii; /* Iterator variable */ + int nSrcPage = -1; /* Size of source db in pages */ + int bCloseTrans = 0; /* True if src db requires unlocking */ + + /* If the source pager is currently in a write-transaction, return + ** SQLITE_BUSY immediately. + */ + if( p->pDestDb && p->pSrc->pBt->inTransaction==TRANS_WRITE ){ + rc = SQLITE_BUSY; + }else{ + rc = SQLITE_OK; + } + + /* If there is no open read-transaction on the source database, open + ** one now. If a transaction is opened here, then it will be closed + ** before this function exits. + */ + if( rc==SQLITE_OK && SQLITE_TXN_NONE==sqlite3BtreeTxnState(p->pSrc) ){ + rc = sqlite3BtreeBeginTrans(p->pSrc, 0, 0); + bCloseTrans = 1; + } + + /* If the destination database has not yet been locked (i.e. if this + ** is the first call to backup_step() for the current backup operation), + ** try to set its page size to the same as the source database. This + ** is especially important on ZipVFS systems, as in that case it is + ** not possible to create a database file that uses one page size by + ** writing to it with another. */ + if( p->bDestLocked==0 && rc==SQLITE_OK && setDestPgsz(p)==SQLITE_NOMEM ){ + rc = SQLITE_NOMEM; + } + + /* Lock the destination database, if it is not locked already. */ + if( SQLITE_OK==rc && p->bDestLocked==0 + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2, + (int*)&p->iDestSchema)) + ){ + p->bDestLocked = 1; + } + + /* Do not allow backup if the destination database is in WAL mode + ** and the page sizes are different between source and destination */ + pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); + pgszDest = sqlite3BtreeGetPageSize(p->pDest); + destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest)); + if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){ + rc = SQLITE_READONLY; + } + + /* Now that there is a read-lock on the source database, query the + ** source pager for the number of pages in the database. + */ + nSrcPage = (int)sqlite3BtreeLastPage(p->pSrc); + assert( nSrcPage>=0 ); + for(ii=0; (nPage<0 || iiiNext<=(Pgno)nSrcPage && !rc; ii++){ + const Pgno iSrcPg = p->iNext; /* Source page number */ + if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ + DbPage *pSrcPg; /* Source page object */ + rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg,PAGER_GET_READONLY); + if( rc==SQLITE_OK ){ + rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); + sqlite3PagerUnref(pSrcPg); + } + } + p->iNext++; + } + if( rc==SQLITE_OK ){ + p->nPagecount = nSrcPage; + p->nRemaining = nSrcPage+1-p->iNext; + if( p->iNext>(Pgno)nSrcPage ){ + rc = SQLITE_DONE; + }else if( !p->isAttached ){ + attachBackupObject(p); + } + } + + /* Update the schema version field in the destination database. This + ** is to make sure that the schema-version really does change in + ** the case where the source and destination databases have the + ** same schema version. + */ + if( rc==SQLITE_DONE ){ + if( nSrcPage==0 ){ + rc = sqlite3BtreeNewDb(p->pDest); + nSrcPage = 1; + } + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + rc = sqlite3BtreeUpdateMeta(p->pDest,1,p->iDestSchema+1); + } + if( rc==SQLITE_OK ){ + if( p->pDestDb ){ + sqlite3ResetAllSchemasOfConnection(p->pDestDb); + } + if( destMode==PAGER_JOURNALMODE_WAL ){ + rc = sqlite3BtreeSetVersion(p->pDest, 2); + } + } + if( rc==SQLITE_OK ){ + int nDestTruncate; + /* Set nDestTruncate to the final number of pages in the destination + ** database. The complication here is that the destination page + ** size may be different to the source page size. + ** + ** If the source page size is smaller than the destination page size, + ** round up. In this case the call to sqlite3OsTruncate() below will + ** fix the size of the file. However it is important to call + ** sqlite3PagerTruncateImage() here so that any pages in the + ** destination file that lie beyond the nDestTruncate page mark are + ** journalled by PagerCommitPhaseOne() before they are destroyed + ** by the file truncation. + */ + assert( pgszSrc==sqlite3BtreeGetPageSize(p->pSrc) ); + assert( pgszDest==sqlite3BtreeGetPageSize(p->pDest) ); + if( pgszSrcpDest->pBt) ){ + nDestTruncate--; + } + }else{ + nDestTruncate = nSrcPage * (pgszSrc/pgszDest); + } + assert( nDestTruncate>0 ); + + if( pgszSrc= iSize || ( + nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1) + && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest + )); + + /* This block ensures that all data required to recreate the original + ** database has been stored in the journal for pDestPager and the + ** journal synced to disk. So at this point we may safely modify + ** the database file in any way, knowing that if a power failure + ** occurs, the original database will be reconstructed from the + ** journal file. */ + sqlite3PagerPagecount(pDestPager, &nDstPage); + for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){ + if( iPg!=PENDING_BYTE_PAGE(p->pDest->pBt) ){ + DbPage *pPg; + rc = sqlite3PagerGet(pDestPager, iPg, &pPg, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pPg); + sqlite3PagerUnref(pPg); + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 1); + } + + /* Write the extra pages and truncate the database file as required */ + iEnd = MIN(PENDING_BYTE + pgszDest, iSize); + for( + iOff=PENDING_BYTE+pgszSrc; + rc==SQLITE_OK && iOffpDest, 0)) + ){ + rc = SQLITE_DONE; + } + } + } + + /* If bCloseTrans is true, then this function opened a read transaction + ** on the source database. Close the read transaction here. There is + ** no need to check the return values of the btree methods here, as + ** "committing" a read-only transaction cannot fail. + */ + if( bCloseTrans ){ + TESTONLY( int rc2 ); + TESTONLY( rc2 = ) sqlite3BtreeCommitPhaseOne(p->pSrc, 0); + TESTONLY( rc2 |= ) sqlite3BtreeCommitPhaseTwo(p->pSrc, 0); + assert( rc2==SQLITE_OK ); + } + + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM_BKPT; + } + p->rc = rc; + } + if( p->pDestDb ){ + sqlite3_mutex_leave(p->pDestDb->mutex); + } + sqlite3BtreeLeave(p->pSrc); + sqlite3_mutex_leave(p->pSrcDb->mutex); + return rc; +} + +/* +** Release all resources associated with an sqlite3_backup* handle. +*/ +SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ + sqlite3_backup **pp; /* Ptr to head of pagers backup list */ + sqlite3 *pSrcDb; /* Source database connection */ + int rc; /* Value to return */ + + /* Enter the mutexes */ + if( p==0 ) return SQLITE_OK; + pSrcDb = p->pSrcDb; + sqlite3_mutex_enter(pSrcDb->mutex); + sqlite3BtreeEnter(p->pSrc); + if( p->pDestDb ){ + sqlite3_mutex_enter(p->pDestDb->mutex); + } + + /* Detach this backup from the source pager. */ + if( p->pDestDb ){ + p->pSrc->nBackup--; + } + if( p->isAttached ){ + pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + assert( pp!=0 ); + while( *pp!=p ){ + pp = &(*pp)->pNext; + assert( pp!=0 ); + } + *pp = p->pNext; + } + + /* If a transaction is still open on the Btree, roll it back. */ + sqlite3BtreeRollback(p->pDest, SQLITE_OK, 0); + + /* Set the error code of the destination database handle. */ + rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; + if( p->pDestDb ){ + sqlite3Error(p->pDestDb, rc); + + /* Exit the mutexes and free the backup context structure. */ + sqlite3LeaveMutexAndCloseZombie(p->pDestDb); + } + sqlite3BtreeLeave(p->pSrc); + if( p->pDestDb ){ + /* EVIDENCE-OF: R-64852-21591 The sqlite3_backup object is created by a + ** call to sqlite3_backup_init() and is destroyed by a call to + ** sqlite3_backup_finish(). */ + sqlite3_free(p); + } + sqlite3LeaveMutexAndCloseZombie(pSrcDb); + return rc; +} + +/* +** Return the number of pages still to be backed up as of the most recent +** call to sqlite3_backup_step(). +*/ +SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return p->nRemaining; +} + +/* +** Return the total number of pages in the source database as of the most +** recent call to sqlite3_backup_step(). +*/ +SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return p->nPagecount; +} + +/* +** This function is called after the contents of page iPage of the +** source database have been modified. If page iPage has already been +** copied into the destination database, then the data written to the +** destination is now invalidated. The destination copy of iPage needs +** to be updated with the new data before the backup operation is +** complete. +** +** It is assumed that the mutex associated with the BtShared object +** corresponding to the source database is held when this function is +** called. +*/ +static SQLITE_NOINLINE void backupUpdate( + sqlite3_backup *p, + Pgno iPage, + const u8 *aData +){ + assert( p!=0 ); + do{ + assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); + if( !isFatalError(p->rc) && iPageiNext ){ + /* The backup process p has already copied page iPage. But now it + ** has been modified by a transaction on the source pager. Copy + ** the new data into the backup. + */ + int rc; + assert( p->pDestDb ); + sqlite3_mutex_enter(p->pDestDb->mutex); + rc = backupOnePage(p, iPage, aData, 1); + sqlite3_mutex_leave(p->pDestDb->mutex); + assert( rc!=SQLITE_BUSY && rc!=SQLITE_LOCKED ); + if( rc!=SQLITE_OK ){ + p->rc = rc; + } + } + }while( (p = p->pNext)!=0 ); +} +SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, const u8 *aData){ + if( pBackup ) backupUpdate(pBackup, iPage, aData); +} + +/* +** Restart the backup process. This is called when the pager layer +** detects that the database has been modified by an external database +** connection. In this case there is no way of knowing which of the +** pages that have been copied into the destination database are still +** valid and which are not, so the entire process needs to be restarted. +** +** It is assumed that the mutex associated with the BtShared object +** corresponding to the source database is held when this function is +** called. +*/ +SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *pBackup){ + sqlite3_backup *p; /* Iterator variable */ + for(p=pBackup; p; p=p->pNext){ + assert( sqlite3_mutex_held(p->pSrc->pBt->mutex) ); + p->iNext = 1; + } +} + +#ifndef SQLITE_OMIT_VACUUM +/* +** Copy the complete content of pBtFrom into pBtTo. A transaction +** must be active for both files. +** +** The size of file pTo may be reduced by this operation. If anything +** goes wrong, the transaction on pTo is rolled back. If successful, the +** transaction is committed before returning. +*/ +SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ + int rc; + sqlite3_file *pFd; /* File descriptor for database pTo */ + sqlite3_backup b; + sqlite3BtreeEnter(pTo); + sqlite3BtreeEnter(pFrom); + + assert( sqlite3BtreeTxnState(pTo)==SQLITE_TXN_WRITE ); + pFd = sqlite3PagerFile(sqlite3BtreePager(pTo)); + if( pFd->pMethods ){ + i64 nByte = sqlite3BtreeGetPageSize(pFrom)*(i64)sqlite3BtreeLastPage(pFrom); + rc = sqlite3OsFileControl(pFd, SQLITE_FCNTL_OVERWRITE, &nByte); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc ) goto copy_finished; + } + + /* Set up an sqlite3_backup object. sqlite3_backup.pDestDb must be set + ** to 0. This is used by the implementations of sqlite3_backup_step() + ** and sqlite3_backup_finish() to detect that they are being called + ** from this function, not directly by the user. + */ + memset(&b, 0, sizeof(b)); + b.pSrcDb = pFrom->db; + b.pSrc = pFrom; + b.pDest = pTo; + b.iNext = 1; + + /* 0x7FFFFFFF is the hard limit for the number of pages in a database + ** file. By passing this as the number of pages to copy to + ** sqlite3_backup_step(), we can guarantee that the copy finishes + ** within a single call (unless an error occurs). The assert() statement + ** checks this assumption - (p->rc) should be set to either SQLITE_DONE + ** or an error code. */ + sqlite3_backup_step(&b, 0x7FFFFFFF); + assert( b.rc!=SQLITE_OK ); + + rc = sqlite3_backup_finish(&b); + if( rc==SQLITE_OK ){ + pTo->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; + }else{ + sqlite3PagerClearCache(sqlite3BtreePager(b.pDest)); + } + + assert( sqlite3BtreeTxnState(pTo)!=SQLITE_TXN_WRITE ); +copy_finished: + sqlite3BtreeLeave(pFrom); + sqlite3BtreeLeave(pTo); + return rc; +} +#endif /* SQLITE_OMIT_VACUUM */ + +/************** End of backup.c **********************************************/ +/************** Begin file vdbemem.c *****************************************/ +/* +** 2004 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code use to manipulate "Mem" structure. A "Mem" +** stores a single value in the VDBE. Mem is an opaque structure visible +** only within the VDBE. Interface routines refer to a Mem using the +** name sqlite_value +*/ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ + +/* True if X is a power of two. 0 is considered a power of two here. +** In other words, return true if X has at most one bit set. +*/ +#define ISPOWEROF2(X) (((X)&((X)-1))==0) + +#ifdef SQLITE_DEBUG +/* +** Check invariants on a Mem object. +** +** This routine is intended for use inside of assert() statements, like +** this: assert( sqlite3VdbeCheckMemInvariants(pMem) ); +*/ +SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ + /* If MEM_Dyn is set then Mem.xDel!=0. + ** Mem.xDel might not be initialized if MEM_Dyn is clear. + */ + assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); + + /* MEM_Dyn may only be set if Mem.szMalloc==0. In this way we + ** ensure that if Mem.szMalloc>0 then it is safe to do + ** Mem.z = Mem.zMalloc without having to check Mem.flags&MEM_Dyn. + ** That saves a few cycles in inner loops. */ + assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); + + /* Cannot have more than one of MEM_Int, MEM_Real, or MEM_IntReal */ + assert( ISPOWEROF2(p->flags & (MEM_Int|MEM_Real|MEM_IntReal)) ); + + if( p->flags & MEM_Null ){ + /* Cannot be both MEM_Null and some other type */ + assert( (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob|MEM_Agg))==0 ); + + /* If MEM_Null is set, then either the value is a pure NULL (the usual + ** case) or it is a pointer set using sqlite3_bind_pointer() or + ** sqlite3_result_pointer(). If a pointer, then MEM_Term must also be + ** set. + */ + if( (p->flags & (MEM_Term|MEM_Subtype))==(MEM_Term|MEM_Subtype) ){ + /* This is a pointer type. There may be a flag to indicate what to + ** do with the pointer. */ + assert( ((p->flags&MEM_Dyn)!=0 ? 1 : 0) + + ((p->flags&MEM_Ephem)!=0 ? 1 : 0) + + ((p->flags&MEM_Static)!=0 ? 1 : 0) <= 1 ); + + /* No other bits set */ + assert( (p->flags & ~(MEM_Null|MEM_Term|MEM_Subtype|MEM_FromBind + |MEM_Dyn|MEM_Ephem|MEM_Static))==0 ); + }else{ + /* A pure NULL might have other flags, such as MEM_Static, MEM_Dyn, + ** MEM_Ephem, MEM_Cleared, or MEM_Subtype */ + } + }else{ + /* The MEM_Cleared bit is only allowed on NULLs */ + assert( (p->flags & MEM_Cleared)==0 ); + } + + /* The szMalloc field holds the correct memory allocation size */ + assert( p->szMalloc==0 + || (p->flags==MEM_Undefined + && p->szMalloc<=sqlite3DbMallocSize(p->db,p->zMalloc)) + || p->szMalloc==sqlite3DbMallocSize(p->db,p->zMalloc)); + + /* If p holds a string or blob, the Mem.z must point to exactly + ** one of the following: + ** + ** (1) Memory in Mem.zMalloc and managed by the Mem object + ** (2) Memory to be freed using Mem.xDel + ** (3) An ephemeral string or blob + ** (4) A static string or blob + */ + if( (p->flags & (MEM_Str|MEM_Blob)) && p->n>0 ){ + assert( + ((p->szMalloc>0 && p->z==p->zMalloc)? 1 : 0) + + ((p->flags&MEM_Dyn)!=0 ? 1 : 0) + + ((p->flags&MEM_Ephem)!=0 ? 1 : 0) + + ((p->flags&MEM_Static)!=0 ? 1 : 0) == 1 + ); + } + return 1; +} +#endif + +/* +** Render a Mem object which is one of MEM_Int, MEM_Real, or MEM_IntReal +** into a buffer. +*/ +static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ + StrAccum acc; + assert( p->flags & (MEM_Int|MEM_Real|MEM_IntReal) ); + assert( sz>22 ); + if( p->flags & MEM_Int ){ +#if GCC_VERSION>=7000000 + /* Work-around for GCC bug + ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96270 */ + i64 x; + assert( (p->flags&MEM_Int)*2==sizeof(x) ); + memcpy(&x, (char*)&p->u, (p->flags&MEM_Int)*2); + sqlite3Int64ToText(x, zBuf); +#else + sqlite3Int64ToText(p->u.i, zBuf); +#endif + }else{ + sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); + sqlite3_str_appendf(&acc, "%!.15g", + (p->flags & MEM_IntReal)!=0 ? (double)p->u.i : p->u.r); + assert( acc.zText==zBuf && acc.mxAlloc<=0 ); + zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ + } +} + +#ifdef SQLITE_DEBUG +/* +** Validity checks on pMem. pMem holds a string. +** +** (1) Check that string value of pMem agrees with its integer or real value. +** (2) Check that the string is correctly zero terminated +** +** A single int or real value always converts to the same strings. But +** many different strings can be converted into the same int or real. +** If a table contains a numeric value and an index is based on the +** corresponding string value, then it is important that the string be +** derived from the numeric value, not the other way around, to ensure +** that the index and table are consistent. See ticket +** https://www.sqlite.org/src/info/343634942dd54ab (2018-01-31) for +** an example. +** +** This routine looks at pMem to verify that if it has both a numeric +** representation and a string representation then the string rep has +** been derived from the numeric and not the other way around. It returns +** true if everything is ok and false if there is a problem. +** +** This routine is for use inside of assert() statements only. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ + char zBuf[100]; + char *z; + int i, j, incr; + if( (p->flags & MEM_Str)==0 ) return 1; + if( p->flags & MEM_Term ){ + /* Insure that the string is properly zero-terminated. Pay particular + ** attention to the case where p->n is odd */ + if( p->szMalloc>0 && p->z==p->zMalloc ){ + assert( p->enc==SQLITE_UTF8 || p->szMalloc >= ((p->n+1)&~1)+2 ); + assert( p->enc!=SQLITE_UTF8 || p->szMalloc >= p->n+1 ); + } + assert( p->z[p->n]==0 ); + assert( p->enc==SQLITE_UTF8 || p->z[(p->n+1)&~1]==0 ); + assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); + } + if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; + vdbeMemRenderNum(sizeof(zBuf), zBuf, p); + z = p->z; + i = j = 0; + incr = 1; + if( p->enc!=SQLITE_UTF8 ){ + incr = 2; + if( p->enc==SQLITE_UTF16BE ) z++; + } + while( zBuf[j] ){ + if( zBuf[j++]!=z[i] ) return 0; + i += incr; + } + return 1; +} +#endif /* SQLITE_DEBUG */ + +/* +** If pMem is an object with a valid string representation, this routine +** ensures the internal encoding for the string representation is +** 'desiredEnc', one of SQLITE_UTF8, SQLITE_UTF16LE or SQLITE_UTF16BE. +** +** If pMem is not a string object, or the encoding of the string +** representation is already stored using the requested encoding, then this +** routine is a no-op. +** +** SQLITE_OK is returned if the conversion is successful (or not required). +** SQLITE_NOMEM may be returned if a malloc() fails during conversion +** between formats. +*/ +SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ +#ifndef SQLITE_OMIT_UTF16 + int rc; +#endif + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE + || desiredEnc==SQLITE_UTF16BE ); + if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){ + return SQLITE_OK; + } + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); +#ifdef SQLITE_OMIT_UTF16 + return SQLITE_ERROR; +#else + + /* MemTranslate() may return SQLITE_OK or SQLITE_NOMEM. If NOMEM is returned, + ** then the encoding of the value may not have changed. + */ + rc = sqlite3VdbeMemTranslate(pMem, (u8)desiredEnc); + assert(rc==SQLITE_OK || rc==SQLITE_NOMEM); + assert(rc==SQLITE_OK || pMem->enc!=desiredEnc); + assert(rc==SQLITE_NOMEM || pMem->enc==desiredEnc); + return rc; +#endif +} + +/* +** Make sure pMem->z points to a writable allocation of at least n bytes. +** +** If the bPreserve argument is true, then copy of the content of +** pMem->z into the new allocation. pMem must be either a string or +** blob if bPreserve is true. If bPreserve is false, any prior content +** in pMem->z is discarded. +*/ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ + assert( sqlite3VdbeCheckMemInvariants(pMem) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + testcase( pMem->db==0 ); + + /* If the bPreserve flag is set to true, then the memory cell must already + ** contain a valid string or blob value. */ + assert( bPreserve==0 || pMem->flags&(MEM_Blob|MEM_Str) ); + testcase( bPreserve && pMem->z==0 ); + + assert( pMem->szMalloc==0 + || (pMem->flags==MEM_Undefined + && pMem->szMalloc<=sqlite3DbMallocSize(pMem->db,pMem->zMalloc)) + || pMem->szMalloc==sqlite3DbMallocSize(pMem->db,pMem->zMalloc)); + if( pMem->szMalloc>0 && bPreserve && pMem->z==pMem->zMalloc ){ + if( pMem->db ){ + pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); + }else{ + pMem->zMalloc = sqlite3Realloc(pMem->z, n); + if( pMem->zMalloc==0 ) sqlite3_free(pMem->z); + pMem->z = pMem->zMalloc; + } + bPreserve = 0; + }else{ + if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); + pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); + } + if( pMem->zMalloc==0 ){ + sqlite3VdbeMemSetNull(pMem); + pMem->z = 0; + pMem->szMalloc = 0; + return SQLITE_NOMEM_BKPT; + }else{ + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); + } + + if( bPreserve && pMem->z ){ + assert( pMem->z!=pMem->zMalloc ); + memcpy(pMem->zMalloc, pMem->z, pMem->n); + } + if( (pMem->flags&MEM_Dyn)!=0 ){ + assert( pMem->xDel!=0 && pMem->xDel!=SQLITE_DYNAMIC ); + pMem->xDel((void *)(pMem->z)); + } + + pMem->z = pMem->zMalloc; + pMem->flags &= ~(MEM_Dyn|MEM_Ephem|MEM_Static); + return SQLITE_OK; +} + +/* +** Change the pMem->zMalloc allocation to be at least szNew bytes. +** If pMem->zMalloc already meets or exceeds the requested size, this +** routine is a no-op. +** +** Any prior string or blob content in the pMem object may be discarded. +** The pMem->xDel destructor is called, if it exists. Though MEM_Str +** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, MEM_IntReal, +** and MEM_Null values are preserved. +** +** Return SQLITE_OK on success or an error code (probably SQLITE_NOMEM) +** if unable to complete the resizing. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ + assert( CORRUPT_DB || szNew>0 ); + assert( (pMem->flags & MEM_Dyn)==0 || pMem->szMalloc==0 ); + if( pMem->szMallocflags & MEM_Dyn)==0 ); + pMem->z = pMem->zMalloc; + pMem->flags &= (MEM_Null|MEM_Int|MEM_Real|MEM_IntReal); + return SQLITE_OK; +} + +/* +** It is already known that pMem contains an unterminated string. +** Add the zero terminator. +** +** Three bytes of zero are added. In this way, there is guaranteed +** to be a double-zero byte at an even byte boundary in order to +** terminate a UTF16 string, even if the initial size of the buffer +** is an odd number of bytes. +*/ +static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ + if( sqlite3VdbeMemGrow(pMem, pMem->n+3, 1) ){ + return SQLITE_NOMEM_BKPT; + } + pMem->z[pMem->n] = 0; + pMem->z[pMem->n+1] = 0; + pMem->z[pMem->n+2] = 0; + pMem->flags |= MEM_Term; + return SQLITE_OK; +} + +/* +** Change pMem so that its MEM_Str or MEM_Blob value is stored in +** MEM.zMalloc, where it can be safely written. +** +** Return SQLITE_OK on success or SQLITE_NOMEM if malloc fails. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + if( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ){ + if( ExpandBlob(pMem) ) return SQLITE_NOMEM; + if( pMem->szMalloc==0 || pMem->z!=pMem->zMalloc ){ + int rc = vdbeMemAddTerminator(pMem); + if( rc ) return rc; + } + } + pMem->flags &= ~MEM_Ephem; +#ifdef SQLITE_DEBUG + pMem->pScopyFrom = 0; +#endif + + return SQLITE_OK; +} + +/* +** If the given Mem* has a zero-filled tail, turn it into an ordinary +** blob stored in dynamically allocated space. +*/ +#ifndef SQLITE_OMIT_INCRBLOB +SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ + int nByte; + assert( pMem->flags & MEM_Zero ); + assert( (pMem->flags&MEM_Blob)!=0 || MemNullNochng(pMem) ); + testcase( sqlite3_value_nochange(pMem) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + + /* Set nByte to the number of bytes required to store the expanded blob. */ + nByte = pMem->n + pMem->u.nZero; + if( nByte<=0 ){ + if( (pMem->flags & MEM_Blob)==0 ) return SQLITE_OK; + nByte = 1; + } + if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ + return SQLITE_NOMEM_BKPT; + } + + memset(&pMem->z[pMem->n], 0, pMem->u.nZero); + pMem->n += pMem->u.nZero; + pMem->flags &= ~(MEM_Zero|MEM_Term); + return SQLITE_OK; +} +#endif + +/* +** Make sure the given Mem is \u0000 terminated. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + testcase( (pMem->flags & (MEM_Term|MEM_Str))==(MEM_Term|MEM_Str) ); + testcase( (pMem->flags & (MEM_Term|MEM_Str))==0 ); + if( (pMem->flags & (MEM_Term|MEM_Str))!=MEM_Str ){ + return SQLITE_OK; /* Nothing to do */ + }else{ + return vdbeMemAddTerminator(pMem); + } +} + +/* +** Add MEM_Str to the set of representations for the given Mem. This +** routine is only called if pMem is a number of some kind, not a NULL +** or a BLOB. +** +** Existing representations MEM_Int, MEM_Real, or MEM_IntReal are invalidated +** if bForce is true but are retained if bForce is false. +** +** A MEM_Null value will never be passed to this function. This function is +** used for converting values to text for returning to the user (i.e. via +** sqlite3_value_text()), or for ensuring that values to be used as btree +** keys are strings. In the former case a NULL pointer is returned the +** user and the latter is an internal programming error. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ + const int nByte = 32; + + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( !(pMem->flags&MEM_Zero) ); + assert( !(pMem->flags&(MEM_Str|MEM_Blob)) ); + assert( pMem->flags&(MEM_Int|MEM_Real|MEM_IntReal) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + + + if( sqlite3VdbeMemClearAndResize(pMem, nByte) ){ + pMem->enc = 0; + return SQLITE_NOMEM_BKPT; + } + + vdbeMemRenderNum(nByte, pMem->z, pMem); + assert( pMem->z!=0 ); + pMem->n = sqlite3Strlen30NN(pMem->z); + pMem->enc = SQLITE_UTF8; + pMem->flags |= MEM_Str|MEM_Term; + if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + sqlite3VdbeChangeEncoding(pMem, enc); + return SQLITE_OK; +} + +/* +** Memory cell pMem contains the context of an aggregate function. +** This routine calls the finalize method for that function. The +** result of the aggregate is stored back into pMem. +** +** Return SQLITE_ERROR if the finalizer reports an error. SQLITE_OK +** otherwise. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ + sqlite3_context ctx; + Mem t; + assert( pFunc!=0 ); + assert( pFunc->xFinalize!=0 ); + assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + memset(&ctx, 0, sizeof(ctx)); + memset(&t, 0, sizeof(t)); + t.flags = MEM_Null; + t.db = pMem->db; + ctx.pOut = &t; + ctx.pMem = pMem; + ctx.pFunc = pFunc; + pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ + assert( (pMem->flags & MEM_Dyn)==0 ); + if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); + memcpy(pMem, &t, sizeof(t)); + return ctx.isError; +} + +/* +** Memory cell pAccum contains the context of an aggregate function. +** This routine calls the xValue method for that function and stores +** the results in memory cell pMem. +** +** SQLITE_ERROR is returned if xValue() reports an error. SQLITE_OK +** otherwise. +*/ +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem *pAccum, Mem *pOut, FuncDef *pFunc){ + sqlite3_context ctx; + assert( pFunc!=0 ); + assert( pFunc->xValue!=0 ); + assert( (pAccum->flags & MEM_Null)!=0 || pFunc==pAccum->u.pDef ); + assert( pAccum->db==0 || sqlite3_mutex_held(pAccum->db->mutex) ); + memset(&ctx, 0, sizeof(ctx)); + sqlite3VdbeMemSetNull(pOut); + ctx.pOut = pOut; + ctx.pMem = pAccum; + ctx.pFunc = pFunc; + pFunc->xValue(&ctx); + return ctx.isError; +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** If the memory cell contains a value that must be freed by +** invoking the external callback in Mem.xDel, then this routine +** will free that value. It also sets Mem.flags to MEM_Null. +** +** This is a helper routine for sqlite3VdbeMemSetNull() and +** for sqlite3VdbeMemRelease(). Use those other routines as the +** entry point for releasing Mem resources. +*/ +static SQLITE_NOINLINE void vdbeMemClearExternAndSetNull(Mem *p){ + assert( p->db==0 || sqlite3_mutex_held(p->db->mutex) ); + assert( VdbeMemDynamic(p) ); + if( p->flags&MEM_Agg ){ + sqlite3VdbeMemFinalize(p, p->u.pDef); + assert( (p->flags & MEM_Agg)==0 ); + testcase( p->flags & MEM_Dyn ); + } + if( p->flags&MEM_Dyn ){ + assert( p->xDel!=SQLITE_DYNAMIC && p->xDel!=0 ); + p->xDel((void *)p->z); + } + p->flags = MEM_Null; +} + +/* +** Release memory held by the Mem p, both external memory cleared +** by p->xDel and memory in p->zMalloc. +** +** This is a helper routine invoked by sqlite3VdbeMemRelease() in +** the unusual case where there really is memory in p that needs +** to be freed. +*/ +static SQLITE_NOINLINE void vdbeMemClear(Mem *p){ + if( VdbeMemDynamic(p) ){ + vdbeMemClearExternAndSetNull(p); + } + if( p->szMalloc ){ + sqlite3DbFreeNN(p->db, p->zMalloc); + p->szMalloc = 0; + } + p->z = 0; +} + +/* +** Release any memory resources held by the Mem. Both the memory that is +** free by Mem.xDel and the Mem.zMalloc allocation are freed. +** +** Use this routine prior to clean up prior to abandoning a Mem, or to +** reset a Mem back to its minimum memory utilization. +** +** Use sqlite3VdbeMemSetNull() to release just the Mem.xDel space +** prior to inserting new content into the Mem. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p){ + assert( sqlite3VdbeCheckMemInvariants(p) ); + if( VdbeMemDynamic(p) || p->szMalloc ){ + vdbeMemClear(p); + } +} + +/* +** Convert a 64-bit IEEE double into a 64-bit signed integer. +** If the double is out of range of a 64-bit signed integer then +** return the closest available 64-bit signed integer. +*/ +static SQLITE_NOINLINE i64 doubleToInt64(double r){ +#ifdef SQLITE_OMIT_FLOATING_POINT + /* When floating-point is omitted, double and int64 are the same thing */ + return r; +#else + /* + ** Many compilers we encounter do not define constants for the + ** minimum and maximum 64-bit integers, or they define them + ** inconsistently. And many do not understand the "LL" notation. + ** So we define our own static constants here using nothing + ** larger than a 32-bit integer constant. + */ + static const i64 maxInt = LARGEST_INT64; + static const i64 minInt = SMALLEST_INT64; + + if( r<=(double)minInt ){ + return minInt; + }else if( r>=(double)maxInt ){ + return maxInt; + }else{ + return (i64)r; + } +#endif +} + +/* +** Return some kind of integer value which is the best we can do +** at representing the value that *pMem describes as an integer. +** If pMem is an integer, then the value is exact. If pMem is +** a floating-point then the value returned is the integer part. +** If pMem is a string or blob, then we make an attempt to convert +** it into an integer and return that. If pMem represents an +** an SQL-NULL value, return 0. +** +** If pMem represents a string value, its encoding might be changed. +*/ +static SQLITE_NOINLINE i64 memIntValue(Mem *pMem){ + i64 value = 0; + sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); + return value; +} +SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ + int flags; + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + flags = pMem->flags; + if( flags & (MEM_Int|MEM_IntReal) ){ + testcase( flags & MEM_IntReal ); + return pMem->u.i; + }else if( flags & MEM_Real ){ + return doubleToInt64(pMem->u.r); + }else if( (flags & (MEM_Str|MEM_Blob))!=0 && pMem->z!=0 ){ + return memIntValue(pMem); + }else{ + return 0; + } +} + +/* +** Return the best representation of pMem that we can get into a +** double. If pMem is already a double or an integer, return its +** value. If it is a string or blob, try to convert it to a double. +** If it is a NULL, return 0.0. +*/ +static SQLITE_NOINLINE double memRealValue(Mem *pMem){ + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + double val = (double)0; + sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); + return val; +} +SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + if( pMem->flags & MEM_Real ){ + return pMem->u.r; + }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_IntReal ); + return (double)pMem->u.i; + }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ + return memRealValue(pMem); + }else{ + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + return (double)0; + } +} + +/* +** Return 1 if pMem represents true, and return 0 if pMem represents false. +** Return the value ifNull if pMem is NULL. +*/ +SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ + testcase( pMem->flags & MEM_IntReal ); + if( pMem->flags & (MEM_Int|MEM_IntReal) ) return pMem->u.i!=0; + if( pMem->flags & MEM_Null ) return ifNull; + return sqlite3VdbeRealValue(pMem)!=0.0; +} + +/* +** The MEM structure is already a MEM_Real. Try to also make it a +** MEM_Int if we can. +*/ +SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ + i64 ix; + assert( pMem->flags & MEM_Real ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + + ix = doubleToInt64(pMem->u.r); + + /* Only mark the value as an integer if + ** + ** (1) the round-trip conversion real->int->real is a no-op, and + ** (2) The integer is neither the largest nor the smallest + ** possible integer (ticket #3922) + ** + ** The second and third terms in the following conditional enforces + ** the second condition under the assumption that addition overflow causes + ** values to wrap around. + */ + if( pMem->u.r==ix && ix>SMALLEST_INT64 && ixu.i = ix; + MemSetTypeFlag(pMem, MEM_Int); + } +} + +/* +** Convert pMem to type integer. Invalidate any prior representations. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + + pMem->u.i = sqlite3VdbeIntValue(pMem); + MemSetTypeFlag(pMem, MEM_Int); + return SQLITE_OK; +} + +/* +** Convert pMem so that it is of type MEM_Real. +** Invalidate any prior representations. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + + pMem->u.r = sqlite3VdbeRealValue(pMem); + MemSetTypeFlag(pMem, MEM_Real); + return SQLITE_OK; +} + +/* Compare a floating point value to an integer. Return true if the two +** values are the same within the precision of the floating point value. +** +** This function assumes that i was obtained by assignment from r1. +** +** For some versions of GCC on 32-bit machines, if you do the more obvious +** comparison of "r1==(double)i" you sometimes get an answer of false even +** though the r1 and (double)i values are bit-for-bit the same. +*/ +SQLITE_PRIVATE int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ + double r2 = (double)i; + return r1==0.0 + || (memcmp(&r1, &r2, sizeof(r1))==0 + && i >= -2251799813685248LL && i < 2251799813685248LL); +} + +/* +** Convert pMem so that it has type MEM_Real or MEM_Int. +** Invalidate any prior representations. +** +** Every effort is made to force the conversion, even if the input +** is a string that does not look completely like a number. Convert +** as much of the string as we can and ignore the rest. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_Real ); + testcase( pMem->flags & MEM_IntReal ); + testcase( pMem->flags & MEM_Null ); + if( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))==0 ){ + int rc; + sqlite3_int64 ix; + assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) + || sqlite3RealSameAsInt(pMem->u.r, (ix = (i64)pMem->u.r)) + ){ + pMem->u.i = ix; + MemSetTypeFlag(pMem, MEM_Int); + }else{ + MemSetTypeFlag(pMem, MEM_Real); + } + } + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))!=0 ); + pMem->flags &= ~(MEM_Str|MEM_Blob|MEM_Zero); + return SQLITE_OK; +} + +/* +** Cast the datatype of the value in pMem according to the affinity +** "aff". Casting is different from applying affinity in that a cast +** is forced. In other words, the value is converted into the desired +** affinity even if that results in loss of data. This routine is +** used (for example) to implement the SQL "cast()" operator. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ + if( pMem->flags & MEM_Null ) return SQLITE_OK; + switch( aff ){ + case SQLITE_AFF_BLOB: { /* Really a cast to BLOB */ + if( (pMem->flags & MEM_Blob)==0 ){ + sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); + assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); + if( pMem->flags & MEM_Str ) MemSetTypeFlag(pMem, MEM_Blob); + }else{ + pMem->flags &= ~(MEM_TypeMask&~MEM_Blob); + } + break; + } + case SQLITE_AFF_NUMERIC: { + sqlite3VdbeMemNumerify(pMem); + break; + } + case SQLITE_AFF_INTEGER: { + sqlite3VdbeMemIntegerify(pMem); + break; + } + case SQLITE_AFF_REAL: { + sqlite3VdbeMemRealify(pMem); + break; + } + default: { + assert( aff==SQLITE_AFF_TEXT ); + assert( MEM_Str==(MEM_Blob>>3) ); + pMem->flags |= (pMem->flags&MEM_Blob)>>3; + sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); + assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); + pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); + return sqlite3VdbeChangeEncoding(pMem, encoding); + } + } + return SQLITE_OK; +} + +/* +** Initialize bulk memory to be a consistent Mem object. +** +** The minimum amount of initialization feasible is performed. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemInit(Mem *pMem, sqlite3 *db, u16 flags){ + assert( (flags & ~MEM_TypeMask)==0 ); + pMem->flags = flags; + pMem->db = db; + pMem->szMalloc = 0; +} + + +/* +** Delete any previous value and set the value stored in *pMem to NULL. +** +** This routine calls the Mem.xDel destructor to dispose of values that +** require the destructor. But it preserves the Mem.zMalloc memory allocation. +** To free all resources, use sqlite3VdbeMemRelease(), which both calls this +** routine to invoke the destructor and deallocates Mem.zMalloc. +** +** Use this routine to reset the Mem prior to insert a new value. +** +** Use sqlite3VdbeMemRelease() to complete erase the Mem prior to abandoning it. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem *pMem){ + if( VdbeMemDynamic(pMem) ){ + vdbeMemClearExternAndSetNull(pMem); + }else{ + pMem->flags = MEM_Null; + } +} +SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value *p){ + sqlite3VdbeMemSetNull((Mem*)p); +} + +/* +** Delete any previous value and set the value to be a BLOB of length +** n containing all zeros. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ + sqlite3VdbeMemRelease(pMem); + pMem->flags = MEM_Blob|MEM_Zero; + pMem->n = 0; + if( n<0 ) n = 0; + pMem->u.nZero = n; + pMem->enc = SQLITE_UTF8; + pMem->z = 0; +} + +/* +** The pMem is known to contain content that needs to be destroyed prior +** to a value change. So invoke the destructor, then set the value to +** a 64-bit integer. +*/ +static SQLITE_NOINLINE void vdbeReleaseAndSetInt64(Mem *pMem, i64 val){ + sqlite3VdbeMemSetNull(pMem); + pMem->u.i = val; + pMem->flags = MEM_Int; +} + +/* +** Delete any previous value and set the value stored in *pMem to val, +** manifest type INTEGER. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ + if( VdbeMemDynamic(pMem) ){ + vdbeReleaseAndSetInt64(pMem, val); + }else{ + pMem->u.i = val; + pMem->flags = MEM_Int; + } +} + +/* A no-op destructor */ +SQLITE_PRIVATE void sqlite3NoopDestructor(void *p){ UNUSED_PARAMETER(p); } + +/* +** Set the value stored in *pMem should already be a NULL. +** Also store a pointer to go with it. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetPointer( + Mem *pMem, + void *pPtr, + const char *zPType, + void (*xDestructor)(void*) +){ + assert( pMem->flags==MEM_Null ); + pMem->u.zPType = zPType ? zPType : ""; + pMem->z = pPtr; + pMem->flags = MEM_Null|MEM_Dyn|MEM_Subtype|MEM_Term; + pMem->eSubtype = 'p'; + pMem->xDel = xDestructor ? xDestructor : sqlite3NoopDestructor; +} + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Delete any previous value and set the value stored in *pMem to val, +** manifest type REAL. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ + sqlite3VdbeMemSetNull(pMem); + if( !sqlite3IsNaN(val) ){ + pMem->u.r = val; + pMem->flags = MEM_Real; + } +} +#endif + +#ifdef SQLITE_DEBUG +/* +** Return true if the Mem holds a RowSet object. This routine is intended +** for use inside of assert() statements. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem *pMem){ + return (pMem->flags&(MEM_Blob|MEM_Dyn))==(MEM_Blob|MEM_Dyn) + && pMem->xDel==sqlite3RowSetDelete; +} +#endif + +/* +** Delete any previous value and set the value of pMem to be an +** empty boolean index. +** +** Return SQLITE_OK on success and SQLITE_NOMEM if a memory allocation +** error occurs. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem *pMem){ + sqlite3 *db = pMem->db; + RowSet *p; + assert( db!=0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + sqlite3VdbeMemRelease(pMem); + p = sqlite3RowSetInit(db); + if( p==0 ) return SQLITE_NOMEM; + pMem->z = (char*)p; + pMem->flags = MEM_Blob|MEM_Dyn; + pMem->xDel = sqlite3RowSetDelete; + return SQLITE_OK; +} + +/* +** Return true if the Mem object contains a TEXT or BLOB that is +** too large - whose size exceeds SQLITE_MAX_LENGTH. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ + assert( p->db!=0 ); + if( p->flags & (MEM_Str|MEM_Blob) ){ + int n = p->n; + if( p->flags & MEM_Zero ){ + n += p->u.nZero; + } + return n>p->db->aLimit[SQLITE_LIMIT_LENGTH]; + } + return 0; +} + +#ifdef SQLITE_DEBUG +/* +** This routine prepares a memory cell for modification by breaking +** its link to a shallow copy and by marking any current shallow +** copies of this cell as invalid. +** +** This is used for testing and debugging only - to help ensure that shallow +** copies (created by OP_SCopy) are not misused. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ + int i; + Mem *pX; + for(i=1, pX=pVdbe->aMem+1; inMem; i++, pX++){ + if( pX->pScopyFrom==pMem ){ + u16 mFlags; + if( pVdbe->db->flags & SQLITE_VdbeTrace ){ + sqlite3DebugPrintf("Invalidate R[%d] due to change in R[%d]\n", + (int)(pX - pVdbe->aMem), (int)(pMem - pVdbe->aMem)); + } + /* If pX is marked as a shallow copy of pMem, then try to verify that + ** no significant changes have been made to pX since the OP_SCopy. + ** A significant change would indicated a missed call to this + ** function for pX. Minor changes, such as adding or removing a + ** dual type, are allowed, as long as the underlying value is the + ** same. */ + mFlags = pMem->flags & pX->flags & pX->mScopyFlags; + assert( (mFlags&(MEM_Int|MEM_IntReal))==0 || pMem->u.i==pX->u.i ); + + /* pMem is the register that is changing. But also mark pX as + ** undefined so that we can quickly detect the shallow-copy error */ + pX->flags = MEM_Undefined; + pX->pScopyFrom = 0; + } + } + pMem->pScopyFrom = 0; +} +#endif /* SQLITE_DEBUG */ + +/* +** Make an shallow copy of pFrom into pTo. Prior contents of +** pTo are freed. The pFrom->z field is not duplicated. If +** pFrom->z is used, then pTo->z points to the same thing as pFrom->z +** and flags gets srcType (either MEM_Ephem or MEM_Static). +*/ +static SQLITE_NOINLINE void vdbeClrCopy(Mem *pTo, const Mem *pFrom, int eType){ + vdbeMemClearExternAndSetNull(pTo); + assert( !VdbeMemDynamic(pTo) ); + sqlite3VdbeMemShallowCopy(pTo, pFrom, eType); +} +SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ + assert( !sqlite3VdbeMemIsRowSet(pFrom) ); + assert( pTo->db==pFrom->db ); + if( VdbeMemDynamic(pTo) ){ vdbeClrCopy(pTo,pFrom,srcType); return; } + memcpy(pTo, pFrom, MEMCELLSIZE); + if( (pFrom->flags&MEM_Static)==0 ){ + pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); + assert( srcType==MEM_Ephem || srcType==MEM_Static ); + pTo->flags |= srcType; + } +} + +/* +** Make a full copy of pFrom into pTo. Prior contents of pTo are +** freed before the copy is made. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ + int rc = SQLITE_OK; + + assert( !sqlite3VdbeMemIsRowSet(pFrom) ); + if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); + memcpy(pTo, pFrom, MEMCELLSIZE); + pTo->flags &= ~MEM_Dyn; + if( pTo->flags&(MEM_Str|MEM_Blob) ){ + if( 0==(pFrom->flags&MEM_Static) ){ + pTo->flags |= MEM_Ephem; + rc = sqlite3VdbeMemMakeWriteable(pTo); + } + } + + return rc; +} + +/* +** Transfer the contents of pFrom to pTo. Any existing value in pTo is +** freed. If pFrom contains ephemeral data, a copy is made. +** +** pFrom contains an SQL NULL when this routine returns. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ + assert( pFrom->db==0 || sqlite3_mutex_held(pFrom->db->mutex) ); + assert( pTo->db==0 || sqlite3_mutex_held(pTo->db->mutex) ); + assert( pFrom->db==0 || pTo->db==0 || pFrom->db==pTo->db ); + + sqlite3VdbeMemRelease(pTo); + memcpy(pTo, pFrom, sizeof(Mem)); + pFrom->flags = MEM_Null; + pFrom->szMalloc = 0; +} + +/* +** Change the value of a Mem to be a string or a BLOB. +** +** The memory management strategy depends on the value of the xDel +** parameter. If the value passed is SQLITE_TRANSIENT, then the +** string is copied into a (possibly existing) buffer managed by the +** Mem structure. Otherwise, any existing buffer is freed and the +** pointer copied. +** +** If the string is too large (if it exceeds the SQLITE_LIMIT_LENGTH +** size limit) then no memory allocation occurs. If the string can be +** stored without allocating memory, then it is. If a memory allocation +** is required to store the string, then value of pMem is unchanged. In +** either case, SQLITE_TOOBIG is returned. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemSetStr( + Mem *pMem, /* Memory cell to set to string value */ + const char *z, /* String pointer */ + i64 n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*) /* Destructor function */ +){ + i64 nByte = n; /* New value for pMem->n */ + int iLimit; /* Maximum allowed string or blob size */ + u16 flags = 0; /* New value for pMem->flags */ + + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + + /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ + if( !z ){ + sqlite3VdbeMemSetNull(pMem); + return SQLITE_OK; + } + + if( pMem->db ){ + iLimit = pMem->db->aLimit[SQLITE_LIMIT_LENGTH]; + }else{ + iLimit = SQLITE_MAX_LENGTH; + } + flags = (enc==0?MEM_Blob:MEM_Str); + if( nByte<0 ){ + assert( enc!=0 ); + if( enc==SQLITE_UTF8 ){ + nByte = strlen(z); + }else{ + for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} + } + flags |= MEM_Term; + } + + /* The following block sets the new values of Mem.z and Mem.xDel. It + ** also sets a flag in local variable "flags" to indicate the memory + ** management (one of MEM_Dyn or MEM_Static). + */ + if( xDel==SQLITE_TRANSIENT ){ + i64 nAlloc = nByte; + if( flags&MEM_Term ){ + nAlloc += (enc==SQLITE_UTF8?1:2); + } + if( nByte>iLimit ){ + return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); + } + testcase( nAlloc==0 ); + testcase( nAlloc==31 ); + testcase( nAlloc==32 ); + if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ + return SQLITE_NOMEM_BKPT; + } + memcpy(pMem->z, z, nAlloc); + }else{ + sqlite3VdbeMemRelease(pMem); + pMem->z = (char *)z; + if( xDel==SQLITE_DYNAMIC ){ + pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); + }else{ + pMem->xDel = xDel; + flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); + } + } + + pMem->n = (int)(nByte & 0x7fffffff); + pMem->flags = flags; + if( enc ){ + pMem->enc = enc; +#ifdef SQLITE_ENABLE_SESSION + }else if( pMem->db==0 ){ + pMem->enc = SQLITE_UTF8; +#endif + }else{ + assert( pMem->db!=0 ); + pMem->enc = ENC(pMem->db); + } + +#ifndef SQLITE_OMIT_UTF16 + if( enc>SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ + return SQLITE_NOMEM_BKPT; + } +#endif + + if( nByte>iLimit ){ + return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); + } + + return SQLITE_OK; +} + +/* +** Move data out of a btree key or data field and into a Mem structure. +** The data is payload from the entry that pCur is currently pointing +** to. offset and amt determine what portion of the data or key to retrieve. +** The result is written into the pMem element. +** +** The pMem object must have been initialized. This routine will use +** pMem->zMalloc to hold the content from the btree, if possible. New +** pMem->zMalloc space will be allocated if necessary. The calling routine +** is responsible for making sure that the pMem object is eventually +** destroyed. +** +** If this routine fails for any reason (malloc returns NULL or unable +** to read from the disk) then the pMem is left in an inconsistent state. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( + BtCursor *pCur, /* Cursor pointing at record to retrieve. */ + u32 offset, /* Offset from the start of data to return bytes from. */ + u32 amt, /* Number of bytes to return. */ + Mem *pMem /* OUT: Return data in this Mem structure. */ +){ + int rc; + pMem->flags = MEM_Null; + if( sqlite3BtreeMaxRecordSize(pCur)z); + if( rc==SQLITE_OK ){ + pMem->z[amt] = 0; /* Overrun area used when reading malformed records */ + pMem->flags = MEM_Blob; + pMem->n = (int)amt; + }else{ + sqlite3VdbeMemRelease(pMem); + } + } + return rc; +} +SQLITE_PRIVATE int sqlite3VdbeMemFromBtreeZeroOffset( + BtCursor *pCur, /* Cursor pointing at record to retrieve. */ + u32 amt, /* Number of bytes to return. */ + Mem *pMem /* OUT: Return data in this Mem structure. */ +){ + u32 available = 0; /* Number of bytes available on the local btree page */ + int rc = SQLITE_OK; /* Return code */ + + assert( sqlite3BtreeCursorIsValid(pCur) ); + assert( !VdbeMemDynamic(pMem) ); + + /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() + ** that both the BtShared and database handle mutexes are held. */ + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + pMem->z = (char *)sqlite3BtreePayloadFetch(pCur, &available); + assert( pMem->z!=0 ); + + if( amt<=available ){ + pMem->flags = MEM_Blob|MEM_Ephem; + pMem->n = (int)amt; + }else{ + rc = sqlite3VdbeMemFromBtree(pCur, 0, amt, pMem); + } + + return rc; +} + +/* +** The pVal argument is known to be a value other than NULL. +** Convert it into a string with encoding enc and return a pointer +** to a zero-terminated version of that string. +*/ +static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ + assert( pVal!=0 ); + assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); + assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); + assert( !sqlite3VdbeMemIsRowSet(pVal) ); + assert( (pVal->flags & (MEM_Null))==0 ); + if( pVal->flags & (MEM_Blob|MEM_Str) ){ + if( ExpandBlob(pVal) ) return 0; + pVal->flags |= MEM_Str; + if( pVal->enc != (enc & ~SQLITE_UTF16_ALIGNED) ){ + sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); + } + if( (enc & SQLITE_UTF16_ALIGNED)!=0 && 1==(1&SQLITE_PTR_TO_INT(pVal->z)) ){ + assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); + if( sqlite3VdbeMemMakeWriteable(pVal)!=SQLITE_OK ){ + return 0; + } + } + sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-31275-44060 */ + }else{ + sqlite3VdbeMemStringify(pVal, enc, 0); + assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); + } + assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 + || pVal->db->mallocFailed ); + if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ + assert( sqlite3VdbeMemValidStrRep(pVal) ); + return pVal->z; + }else{ + return 0; + } +} + +/* This function is only available internally, it is not part of the +** external API. It works in a similar way to sqlite3_value_text(), +** except the data returned is in the encoding specified by the second +** parameter, which must be one of SQLITE_UTF16BE, SQLITE_UTF16LE or +** SQLITE_UTF8. +** +** (2006-02-16:) The enc value can be or-ed with SQLITE_UTF16_ALIGNED. +** If that is the case, then the result must be aligned on an even byte +** boundary. +*/ +SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ + if( !pVal ) return 0; + assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); + assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); + assert( !sqlite3VdbeMemIsRowSet(pVal) ); + if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ + assert( sqlite3VdbeMemValidStrRep(pVal) ); + return pVal->z; + } + if( pVal->flags&MEM_Null ){ + return 0; + } + return valueToText(pVal, enc); +} + +/* +** Create a new sqlite3_value object. +*/ +SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *db){ + Mem *p = sqlite3DbMallocZero(db, sizeof(*p)); + if( p ){ + p->flags = MEM_Null; + p->db = db; + } + return p; +} + +/* +** Context object passed by sqlite3Stat4ProbeSetValue() through to +** valueNew(). See comments above valueNew() for details. +*/ +struct ValueNewStat4Ctx { + Parse *pParse; + Index *pIdx; + UnpackedRecord **ppRec; + int iVal; +}; + +/* +** Allocate and return a pointer to a new sqlite3_value object. If +** the second argument to this function is NULL, the object is allocated +** by calling sqlite3ValueNew(). +** +** Otherwise, if the second argument is non-zero, then this function is +** being called indirectly by sqlite3Stat4ProbeSetValue(). If it has not +** already been allocated, allocate the UnpackedRecord structure that +** that function will return to its caller here. Then return a pointer to +** an sqlite3_value within the UnpackedRecord.a[] array. +*/ +static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ +#ifdef SQLITE_ENABLE_STAT4 + if( p ){ + UnpackedRecord *pRec = p->ppRec[0]; + + if( pRec==0 ){ + Index *pIdx = p->pIdx; /* Index being probed */ + int nByte; /* Bytes of space to allocate */ + int i; /* Counter variable */ + int nCol = pIdx->nColumn; /* Number of index columns including rowid */ + + nByte = sizeof(Mem) * nCol + ROUND8(sizeof(UnpackedRecord)); + pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); + if( pRec ){ + pRec->pKeyInfo = sqlite3KeyInfoOfIndex(p->pParse, pIdx); + if( pRec->pKeyInfo ){ + assert( pRec->pKeyInfo->nAllField==nCol ); + assert( pRec->pKeyInfo->enc==ENC(db) ); + pRec->aMem = (Mem *)((u8*)pRec + ROUND8(sizeof(UnpackedRecord))); + for(i=0; iaMem[i].flags = MEM_Null; + pRec->aMem[i].db = db; + } + }else{ + sqlite3DbFreeNN(db, pRec); + pRec = 0; + } + } + if( pRec==0 ) return 0; + p->ppRec[0] = pRec; + } + + pRec->nField = p->iVal+1; + return &pRec->aMem[p->iVal]; + } +#else + UNUSED_PARAMETER(p); +#endif /* defined(SQLITE_ENABLE_STAT4) */ + return sqlite3ValueNew(db); +} + +/* +** The expression object indicated by the second argument is guaranteed +** to be a scalar SQL function. If +** +** * all function arguments are SQL literals, +** * one of the SQLITE_FUNC_CONSTANT or _SLOCHNG function flags is set, and +** * the SQLITE_FUNC_NEEDCOLL function flag is not set, +** +** then this routine attempts to invoke the SQL function. Assuming no +** error occurs, output parameter (*ppVal) is set to point to a value +** object containing the result before returning SQLITE_OK. +** +** Affinity aff is applied to the result of the function before returning. +** If the result is a text value, the sqlite3_value object uses encoding +** enc. +** +** If the conditions above are not met, this function returns SQLITE_OK +** and sets (*ppVal) to NULL. Or, if an error occurs, (*ppVal) is set to +** NULL and an SQLite error code returned. +*/ +#ifdef SQLITE_ENABLE_STAT4 +static int valueFromFunction( + sqlite3 *db, /* The database connection */ + const Expr *p, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 aff, /* Affinity to use */ + sqlite3_value **ppVal, /* Write the new value here */ + struct ValueNewStat4Ctx *pCtx /* Second argument for valueNew() */ +){ + sqlite3_context ctx; /* Context object for function invocation */ + sqlite3_value **apVal = 0; /* Function arguments */ + int nVal = 0; /* Size of apVal[] array */ + FuncDef *pFunc = 0; /* Function definition */ + sqlite3_value *pVal = 0; /* New value */ + int rc = SQLITE_OK; /* Return code */ + ExprList *pList = 0; /* Function arguments */ + int i; /* Iterator variable */ + + assert( pCtx!=0 ); + assert( (p->flags & EP_TokenOnly)==0 ); + pList = p->x.pList; + if( pList ) nVal = pList->nExpr; + pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0); + assert( pFunc ); + if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + ){ + return SQLITE_OK; + } + + if( pList ){ + apVal = (sqlite3_value**)sqlite3DbMallocZero(db, sizeof(apVal[0]) * nVal); + if( apVal==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto value_from_function_out; + } + for(i=0; ia[i].pExpr, enc, aff, &apVal[i]); + if( apVal[i]==0 || rc!=SQLITE_OK ) goto value_from_function_out; + } + } + + pVal = valueNew(db, pCtx); + if( pVal==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto value_from_function_out; + } + + assert( pCtx->pParse->rc==SQLITE_OK ); + memset(&ctx, 0, sizeof(ctx)); + ctx.pOut = pVal; + ctx.pFunc = pFunc; + pFunc->xSFunc(&ctx, nVal, apVal); + if( ctx.isError ){ + rc = ctx.isError; + sqlite3ErrorMsg(pCtx->pParse, "%s", sqlite3_value_text(pVal)); + }else{ + sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8); + assert( rc==SQLITE_OK ); + rc = sqlite3VdbeChangeEncoding(pVal, enc); + if( rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal) ){ + rc = SQLITE_TOOBIG; + pCtx->pParse->nErr++; + } + } + pCtx->pParse->rc = rc; + + value_from_function_out: + if( rc!=SQLITE_OK ){ + pVal = 0; + } + if( apVal ){ + for(i=0; iop)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft; +#if defined(SQLITE_ENABLE_STAT4) + if( op==TK_REGISTER ) op = pExpr->op2; +#else + if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; +#endif + + /* Compressed expressions only appear when parsing the DEFAULT clause + ** on a table column definition, and hence only when pCtx==0. This + ** check ensures that an EP_TokenOnly expression is never passed down + ** into valueFromFunction(). */ + assert( (pExpr->flags & EP_TokenOnly)==0 || pCtx==0 ); + + if( op==TK_CAST ){ + u8 aff = sqlite3AffinityType(pExpr->u.zToken,0); + rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); + testcase( rc!=SQLITE_OK ); + if( *ppVal ){ + sqlite3VdbeMemCast(*ppVal, aff, SQLITE_UTF8); + sqlite3ValueApplyAffinity(*ppVal, affinity, SQLITE_UTF8); + } + return rc; + } + + /* Handle negative integers in a single step. This is needed in the + ** case when the value is -9223372036854775808. + */ + if( op==TK_UMINUS + && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ + pExpr = pExpr->pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; + } + + if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ + pVal = valueNew(db, pCtx); + if( pVal==0 ) goto no_mem; + if( ExprHasProperty(pExpr, EP_IntValue) ){ + sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); + }else{ + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); + if( zVal==0 ) goto no_mem; + sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + } + if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){ + sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + }else{ + sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); + } + assert( (pVal->flags & MEM_IntReal)==0 ); + if( pVal->flags & (MEM_Int|MEM_IntReal|MEM_Real) ){ + testcase( pVal->flags & MEM_Int ); + testcase( pVal->flags & MEM_Real ); + pVal->flags &= ~MEM_Str; + } + if( enc!=SQLITE_UTF8 ){ + rc = sqlite3VdbeChangeEncoding(pVal, enc); + } + }else if( op==TK_UMINUS ) { + /* This branch happens for multiple negative signs. Ex: -(-5) */ + if( SQLITE_OK==valueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal,pCtx) + && pVal!=0 + ){ + sqlite3VdbeMemNumerify(pVal); + if( pVal->flags & MEM_Real ){ + pVal->u.r = -pVal->u.r; + }else if( pVal->u.i==SMALLEST_INT64 ){ +#ifndef SQLITE_OMIT_FLOATING_POINT + pVal->u.r = -(double)SMALLEST_INT64; +#else + pVal->u.r = LARGEST_INT64; +#endif + MemSetTypeFlag(pVal, MEM_Real); + }else{ + pVal->u.i = -pVal->u.i; + } + sqlite3ValueApplyAffinity(pVal, affinity, enc); + } + }else if( op==TK_NULL ){ + pVal = valueNew(db, pCtx); + if( pVal==0 ) goto no_mem; + sqlite3VdbeMemSetNull(pVal); + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + else if( op==TK_BLOB ){ + int nVal; + assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); + assert( pExpr->u.zToken[1]=='\'' ); + pVal = valueNew(db, pCtx); + if( !pVal ) goto no_mem; + zVal = &pExpr->u.zToken[2]; + nVal = sqlite3Strlen30(zVal)-1; + assert( zVal[nVal]=='\'' ); + sqlite3VdbeMemSetStr(pVal, sqlite3HexToBlob(db, zVal, nVal), nVal/2, + 0, SQLITE_DYNAMIC); + } +#endif +#ifdef SQLITE_ENABLE_STAT4 + else if( op==TK_FUNCTION && pCtx!=0 ){ + rc = valueFromFunction(db, pExpr, enc, affinity, &pVal, pCtx); + } +#endif + else if( op==TK_TRUEFALSE ){ + pVal = valueNew(db, pCtx); + if( pVal ){ + pVal->flags = MEM_Int; + pVal->u.i = pExpr->u.zToken[4]==0; + } + } + + *ppVal = pVal; + return rc; + +no_mem: +#ifdef SQLITE_ENABLE_STAT4 + if( pCtx==0 || pCtx->pParse->nErr==0 ) +#endif + sqlite3OomFault(db); + sqlite3DbFree(db, zVal); + assert( *ppVal==0 ); +#ifdef SQLITE_ENABLE_STAT4 + if( pCtx==0 ) sqlite3ValueFree(pVal); +#else + assert( pCtx==0 ); sqlite3ValueFree(pVal); +#endif + return SQLITE_NOMEM_BKPT; +} + +/* +** Create a new sqlite3_value object, containing the value of pExpr. +** +** This only works for very simple expressions that consist of one constant +** token (i.e. "5", "5.1", "'a string'"). If the expression can +** be converted directly into a value, then the value is allocated and +** a pointer written to *ppVal. The caller is responsible for deallocating +** the value by passing it to sqlite3ValueFree() later on. If the expression +** cannot be converted to a value, then *ppVal is set to NULL. +*/ +SQLITE_PRIVATE int sqlite3ValueFromExpr( + sqlite3 *db, /* The database connection */ + const Expr *pExpr, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal /* Write the new value here */ +){ + return pExpr ? valueFromExpr(db, pExpr, enc, affinity, ppVal, 0) : 0; +} + +#ifdef SQLITE_ENABLE_STAT4 +/* +** Attempt to extract a value from pExpr and use it to construct *ppVal. +** +** If pAlloc is not NULL, then an UnpackedRecord object is created for +** pAlloc if one does not exist and the new value is added to the +** UnpackedRecord object. +** +** A value is extracted in the following cases: +** +** * (pExpr==0). In this case the value is assumed to be an SQL NULL, +** +** * The expression is a bound variable, and this is a reprepare, or +** +** * The expression is a literal value. +** +** On success, *ppVal is made to point to the extracted value. The caller +** is responsible for ensuring that the value is eventually freed. +*/ +static int stat4ValueFromExpr( + Parse *pParse, /* Parse context */ + Expr *pExpr, /* The expression to extract a value from */ + u8 affinity, /* Affinity to use */ + struct ValueNewStat4Ctx *pAlloc,/* How to allocate space. Or NULL */ + sqlite3_value **ppVal /* OUT: New value object (or NULL) */ +){ + int rc = SQLITE_OK; + sqlite3_value *pVal = 0; + sqlite3 *db = pParse->db; + + /* Skip over any TK_COLLATE nodes */ + pExpr = sqlite3ExprSkipCollate(pExpr); + + assert( pExpr==0 || pExpr->op!=TK_REGISTER || pExpr->op2!=TK_VARIABLE ); + if( !pExpr ){ + pVal = valueNew(db, pAlloc); + if( pVal ){ + sqlite3VdbeMemSetNull((Mem*)pVal); + } + }else if( pExpr->op==TK_VARIABLE && (db->flags & SQLITE_EnableQPSG)==0 ){ + Vdbe *v; + int iBindVar = pExpr->iColumn; + sqlite3VdbeSetVarmask(pParse->pVdbe, iBindVar); + if( (v = pParse->pReprepare)!=0 ){ + pVal = valueNew(db, pAlloc); + if( pVal ){ + rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iBindVar-1]); + sqlite3ValueApplyAffinity(pVal, affinity, ENC(db)); + pVal->db = pParse->db; + } + } + }else{ + rc = valueFromExpr(db, pExpr, ENC(db), affinity, &pVal, pAlloc); + } + + assert( pVal==0 || pVal->db==db ); + *ppVal = pVal; + return rc; +} + +/* +** This function is used to allocate and populate UnpackedRecord +** structures intended to be compared against sample index keys stored +** in the sqlite_stat4 table. +** +** A single call to this function populates zero or more fields of the +** record starting with field iVal (fields are numbered from left to +** right starting with 0). A single field is populated if: +** +** * (pExpr==0). In this case the value is assumed to be an SQL NULL, +** +** * The expression is a bound variable, and this is a reprepare, or +** +** * The sqlite3ValueFromExpr() function is able to extract a value +** from the expression (i.e. the expression is a literal value). +** +** Or, if pExpr is a TK_VECTOR, one field is populated for each of the +** vector components that match either of the two latter criteria listed +** above. +** +** Before any value is appended to the record, the affinity of the +** corresponding column within index pIdx is applied to it. Before +** this function returns, output parameter *pnExtract is set to the +** number of values appended to the record. +** +** When this function is called, *ppRec must either point to an object +** allocated by an earlier call to this function, or must be NULL. If it +** is NULL and a value can be successfully extracted, a new UnpackedRecord +** is allocated (and *ppRec set to point to it) before returning. +** +** Unless an error is encountered, SQLITE_OK is returned. It is not an +** error if a value cannot be extracted from pExpr. If an error does +** occur, an SQLite error code is returned. +*/ +SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue( + Parse *pParse, /* Parse context */ + Index *pIdx, /* Index being probed */ + UnpackedRecord **ppRec, /* IN/OUT: Probe record */ + Expr *pExpr, /* The expression to extract a value from */ + int nElem, /* Maximum number of values to append */ + int iVal, /* Array element to populate */ + int *pnExtract /* OUT: Values appended to the record */ +){ + int rc = SQLITE_OK; + int nExtract = 0; + + if( pExpr==0 || pExpr->op!=TK_SELECT ){ + int i; + struct ValueNewStat4Ctx alloc; + + alloc.pParse = pParse; + alloc.pIdx = pIdx; + alloc.ppRec = ppRec; + + for(i=0; idb, pIdx, iVal+i); + alloc.iVal = iVal+i; + rc = stat4ValueFromExpr(pParse, pElem, aff, &alloc, &pVal); + if( !pVal ) break; + nExtract++; + } + } + + *pnExtract = nExtract; + return rc; +} + +/* +** Attempt to extract a value from expression pExpr using the methods +** as described for sqlite3Stat4ProbeSetValue() above. +** +** If successful, set *ppVal to point to a new value object and return +** SQLITE_OK. If no value can be extracted, but no other error occurs +** (e.g. OOM), return SQLITE_OK and set *ppVal to NULL. Or, if an error +** does occur, return an SQLite error code. The final value of *ppVal +** is undefined in this case. +*/ +SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr( + Parse *pParse, /* Parse context */ + Expr *pExpr, /* The expression to extract a value from */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal /* OUT: New value object (or NULL) */ +){ + return stat4ValueFromExpr(pParse, pExpr, affinity, 0, ppVal); +} + +/* +** Extract the iCol-th column from the nRec-byte record in pRec. Write +** the column value into *ppVal. If *ppVal is initially NULL then a new +** sqlite3_value object is allocated. +** +** If *ppVal is initially NULL then the caller is responsible for +** ensuring that the value written into *ppVal is eventually freed. +*/ +SQLITE_PRIVATE int sqlite3Stat4Column( + sqlite3 *db, /* Database handle */ + const void *pRec, /* Pointer to buffer containing record */ + int nRec, /* Size of buffer pRec in bytes */ + int iCol, /* Column to extract */ + sqlite3_value **ppVal /* OUT: Extracted value */ +){ + u32 t = 0; /* a column type code */ + int nHdr; /* Size of the header in the record */ + int iHdr; /* Next unread header byte */ + int iField; /* Next unread data byte */ + int szField = 0; /* Size of the current data field */ + int i; /* Column index */ + u8 *a = (u8*)pRec; /* Typecast byte array */ + Mem *pMem = *ppVal; /* Write result into this Mem object */ + + assert( iCol>0 ); + iHdr = getVarint32(a, nHdr); + if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; + iField = nHdr; + for(i=0; i<=iCol; i++){ + iHdr += getVarint32(&a[iHdr], t); + testcase( iHdr==nHdr ); + testcase( iHdr==nHdr+1 ); + if( iHdr>nHdr ) return SQLITE_CORRUPT_BKPT; + szField = sqlite3VdbeSerialTypeLen(t); + iField += szField; + } + testcase( iField==nRec ); + testcase( iField==nRec+1 ); + if( iField>nRec ) return SQLITE_CORRUPT_BKPT; + if( pMem==0 ){ + pMem = *ppVal = sqlite3ValueNew(db); + if( pMem==0 ) return SQLITE_NOMEM_BKPT; + } + sqlite3VdbeSerialGet(&a[iField-szField], t, pMem); + pMem->enc = ENC(db); + return SQLITE_OK; +} + +/* +** Unless it is NULL, the argument must be an UnpackedRecord object returned +** by an earlier call to sqlite3Stat4ProbeSetValue(). This call deletes +** the object. +*/ +SQLITE_PRIVATE void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ + if( pRec ){ + int i; + int nCol = pRec->pKeyInfo->nAllField; + Mem *aMem = pRec->aMem; + sqlite3 *db = aMem[0].db; + for(i=0; ipKeyInfo); + sqlite3DbFreeNN(db, pRec); + } +} +#endif /* ifdef SQLITE_ENABLE_STAT4 */ + +/* +** Change the string value of an sqlite3_value object +*/ +SQLITE_PRIVATE void sqlite3ValueSetStr( + sqlite3_value *v, /* Value to be set */ + int n, /* Length of string z */ + const void *z, /* Text of the new string */ + u8 enc, /* Encoding to use */ + void (*xDel)(void*) /* Destructor for the string */ +){ + if( v ) sqlite3VdbeMemSetStr((Mem *)v, z, n, enc, xDel); +} + +/* +** Free an sqlite3_value object +*/ +SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value *v){ + if( !v ) return; + sqlite3VdbeMemRelease((Mem *)v); + sqlite3DbFreeNN(((Mem*)v)->db, v); +} + +/* +** The sqlite3ValueBytes() routine returns the number of bytes in the +** sqlite3_value object assuming that it uses the encoding "enc". +** The valueBytes() routine is a helper function. +*/ +static SQLITE_NOINLINE int valueBytes(sqlite3_value *pVal, u8 enc){ + return valueToText(pVal, enc)!=0 ? pVal->n : 0; +} +SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ + Mem *p = (Mem*)pVal; + assert( (p->flags & MEM_Null)==0 || (p->flags & (MEM_Str|MEM_Blob))==0 ); + if( (p->flags & MEM_Str)!=0 && pVal->enc==enc ){ + return p->n; + } + if( (p->flags & MEM_Blob)!=0 ){ + if( p->flags & MEM_Zero ){ + return p->n + p->u.nZero; + }else{ + return p->n; + } + } + if( p->flags & MEM_Null ) return 0; + return valueBytes(pVal, enc); +} + +/************** End of vdbemem.c *********************************************/ +/************** Begin file vdbeaux.c *****************************************/ +/* +** 2003 September 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used for creating, destroying, and populating +** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) +*/ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ + +/* Forward references */ +static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef); +static void vdbeFreeOpArray(sqlite3 *, Op *, int); + +/* +** Create a new virtual database engine. +*/ +SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ + sqlite3 *db = pParse->db; + Vdbe *p; + p = sqlite3DbMallocRawNN(db, sizeof(Vdbe) ); + if( p==0 ) return 0; + memset(&p->aOp, 0, sizeof(Vdbe)-offsetof(Vdbe,aOp)); + p->db = db; + if( db->pVdbe ){ + db->pVdbe->pPrev = p; + } + p->pNext = db->pVdbe; + p->pPrev = 0; + db->pVdbe = p; + p->iVdbeMagic = VDBE_MAGIC_INIT; + p->pParse = pParse; + pParse->pVdbe = p; + assert( pParse->aLabel==0 ); + assert( pParse->nLabel==0 ); + assert( p->nOpAlloc==0 ); + assert( pParse->szOpAlloc==0 ); + sqlite3VdbeAddOp2(p, OP_Init, 0, 1); + return p; +} + +/* +** Return the Parse object that owns a Vdbe object. +*/ +SQLITE_PRIVATE Parse *sqlite3VdbeParser(Vdbe *p){ + return p->pParse; +} + +/* +** Change the error string stored in Vdbe.zErrMsg +*/ +SQLITE_PRIVATE void sqlite3VdbeError(Vdbe *p, const char *zFormat, ...){ + va_list ap; + sqlite3DbFree(p->db, p->zErrMsg); + va_start(ap, zFormat); + p->zErrMsg = sqlite3VMPrintf(p->db, zFormat, ap); + va_end(ap); +} + +/* +** Remember the SQL string for a prepared statement. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){ + if( p==0 ) return; + p->prepFlags = prepFlags; + if( (prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){ + p->expmask = 0; + } + assert( p->zSql==0 ); + p->zSql = sqlite3DbStrNDup(p->db, z, n); +} + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Add a new element to the Vdbe->pDblStr list. +*/ +SQLITE_PRIVATE void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){ + if( p ){ + int n = sqlite3Strlen30(z); + DblquoteStr *pStr = sqlite3DbMallocRawNN(db, + sizeof(*pStr)+n+1-sizeof(pStr->z)); + if( pStr ){ + pStr->pNextStr = p->pDblStr; + p->pDblStr = pStr; + memcpy(pStr->z, z, n+1); + } + } +} +#endif + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** zId of length nId is a double-quoted identifier. Check to see if +** that identifier is really used as a string literal. +*/ +SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString( + Vdbe *pVdbe, /* The prepared statement */ + const char *zId /* The double-quoted identifier, already dequoted */ +){ + DblquoteStr *pStr; + assert( zId!=0 ); + if( pVdbe->pDblStr==0 ) return 0; + for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){ + if( strcmp(zId, pStr->z)==0 ) return 1; + } + return 0; +} +#endif + +/* +** Swap all content between two VDBE structures. +*/ +SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ + Vdbe tmp, *pTmp; + char *zTmp; + assert( pA->db==pB->db ); + tmp = *pA; + *pA = *pB; + *pB = tmp; + pTmp = pA->pNext; + pA->pNext = pB->pNext; + pB->pNext = pTmp; + pTmp = pA->pPrev; + pA->pPrev = pB->pPrev; + pB->pPrev = pTmp; + zTmp = pA->zSql; + pA->zSql = pB->zSql; + pB->zSql = zTmp; +#ifdef SQLITE_ENABLE_NORMALIZE + zTmp = pA->zNormSql; + pA->zNormSql = pB->zNormSql; + pB->zNormSql = zTmp; +#endif + pB->expmask = pA->expmask; + pB->prepFlags = pA->prepFlags; + memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter)); + pB->aCounter[SQLITE_STMTSTATUS_REPREPARE]++; +} + +/* +** Resize the Vdbe.aOp array so that it is at least nOp elements larger +** than its current size. nOp is guaranteed to be less than or equal +** to 1024/sizeof(Op). +** +** If an out-of-memory error occurs while resizing the array, return +** SQLITE_NOMEM. In this case Vdbe.aOp and Vdbe.nOpAlloc remain +** unchanged (this is so that any opcodes already allocated can be +** correctly deallocated along with the rest of the Vdbe). +*/ +static int growOpArray(Vdbe *v, int nOp){ + VdbeOp *pNew; + Parse *p = v->pParse; + + /* The SQLITE_TEST_REALLOC_STRESS compile-time option is designed to force + ** more frequent reallocs and hence provide more opportunities for + ** simulated OOM faults. SQLITE_TEST_REALLOC_STRESS is generally used + ** during testing only. With SQLITE_TEST_REALLOC_STRESS grow the op array + ** by the minimum* amount required until the size reaches 512. Normal + ** operation (without SQLITE_TEST_REALLOC_STRESS) is to double the current + ** size of the op array or add 1KB of space, whichever is smaller. */ +#ifdef SQLITE_TEST_REALLOC_STRESS + sqlite3_int64 nNew = (v->nOpAlloc>=512 ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)v->nOpAlloc+nOp); +#else + sqlite3_int64 nNew = (v->nOpAlloc ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)(1024/sizeof(Op))); + UNUSED_PARAMETER(nOp); +#endif + + /* Ensure that the size of a VDBE does not grow too large */ + if( nNew > p->db->aLimit[SQLITE_LIMIT_VDBE_OP] ){ + sqlite3OomFault(p->db); + return SQLITE_NOMEM; + } + + assert( nOp<=(1024/sizeof(Op)) ); + assert( nNew>=(v->nOpAlloc+nOp) ); + pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); + if( pNew ){ + p->szOpAlloc = sqlite3DbMallocSize(p->db, pNew); + v->nOpAlloc = p->szOpAlloc/sizeof(Op); + v->aOp = pNew; + } + return (pNew ? SQLITE_OK : SQLITE_NOMEM_BKPT); +} + +#ifdef SQLITE_DEBUG +/* This routine is just a convenient place to set a breakpoint that will +** fire after each opcode is inserted and displayed using +** "PRAGMA vdbe_addoptrace=on". Parameters "pc" (program counter) and +** pOp are available to make the breakpoint conditional. +** +** Other useful labels for breakpoints include: +** test_trace_breakpoint(pc,pOp) +** sqlite3CorruptError(lineno) +** sqlite3MisuseError(lineno) +** sqlite3CantopenError(lineno) +*/ +static void test_addop_breakpoint(int pc, Op *pOp){ + static int n = 0; + n++; +} +#endif + +/* +** Add a new instruction to the list of instructions current in the +** VDBE. Return the address of the new instruction. +** +** Parameters: +** +** p Pointer to the VDBE +** +** op The opcode for this instruction +** +** p1, p2, p3 Operands +** +** Use the sqlite3VdbeResolveLabel() function to fix an address and +** the sqlite3VdbeChangeP4() function to change the value of the P4 +** operand. +*/ +static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ + assert( p->nOpAlloc<=p->nOp ); + if( growOpArray(p, 1) ) return 1; + assert( p->nOpAlloc>p->nOp ); + return sqlite3VdbeAddOp3(p, op, p1, p2, p3); +} +SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ + int i; + VdbeOp *pOp; + + i = p->nOp; + assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); + assert( op>=0 && op<0xff ); + if( p->nOpAlloc<=i ){ + return growOp3(p, op, p1, p2, p3); + } + p->nOp++; + pOp = &p->aOp[i]; + pOp->opcode = (u8)op; + pOp->p5 = 0; + pOp->p1 = p1; + pOp->p2 = p2; + pOp->p3 = p3; + pOp->p4.p = 0; + pOp->p4type = P4_NOTUSED; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + pOp->zComment = 0; +#endif +#ifdef SQLITE_DEBUG + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i, &p->aOp[i]); + test_addop_breakpoint(i, &p->aOp[i]); + } +#endif +#ifdef VDBE_PROFILE + pOp->cycles = 0; + pOp->cnt = 0; +#endif +#ifdef SQLITE_VDBE_COVERAGE + pOp->iSrcLine = 0; +#endif + return i; +} +SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe *p, int op){ + return sqlite3VdbeAddOp3(p, op, 0, 0, 0); +} +SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe *p, int op, int p1){ + return sqlite3VdbeAddOp3(p, op, p1, 0, 0); +} +SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ + return sqlite3VdbeAddOp3(p, op, p1, p2, 0); +} + +/* Generate code for an unconditional jump to instruction iDest +*/ +SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe *p, int iDest){ + return sqlite3VdbeAddOp3(p, OP_Goto, 0, iDest, 0); +} + +/* Generate code to cause the string zStr to be loaded into +** register iDest +*/ +SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe *p, int iDest, const char *zStr){ + return sqlite3VdbeAddOp4(p, OP_String8, 0, iDest, 0, zStr, 0); +} + +/* +** Generate code that initializes multiple registers to string or integer +** constants. The registers begin with iDest and increase consecutively. +** One register is initialized for each characgter in zTypes[]. For each +** "s" character in zTypes[], the register is a string if the argument is +** not NULL, or OP_Null if the value is a null pointer. For each "i" character +** in zTypes[], the register is initialized to an integer. +** +** If the input string does not end with "X" then an OP_ResultRow instruction +** is generated for the values inserted. +*/ +SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe *p, int iDest, const char *zTypes, ...){ + va_list ap; + int i; + char c; + va_start(ap, zTypes); + for(i=0; (c = zTypes[i])!=0; i++){ + if( c=='s' ){ + const char *z = va_arg(ap, const char*); + sqlite3VdbeAddOp4(p, z==0 ? OP_Null : OP_String8, 0, iDest+i, 0, z, 0); + }else if( c=='i' ){ + sqlite3VdbeAddOp2(p, OP_Integer, va_arg(ap, int), iDest+i); + }else{ + goto skip_op_resultrow; + } + } + sqlite3VdbeAddOp2(p, OP_ResultRow, iDest, i); +skip_op_resultrow: + va_end(ap); +} + +/* +** Add an opcode that includes the p4 value as a pointer. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOp4( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + const char *zP4, /* The P4 operand */ + int p4type /* P4 operand type */ +){ + int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); + sqlite3VdbeChangeP4(p, addr, zP4, p4type); + return addr; +} + +/* +** Add an OP_Function or OP_PureFunc opcode. +** +** The eCallCtx argument is information (typically taken from Expr.op2) +** that describes the calling context of the function. 0 means a general +** function call. NC_IsCheck means called by a check constraint, +** NC_IdxExpr means called as part of an index expression. NC_PartIdx +** means in the WHERE clause of a partial index. NC_GenCol means called +** while computing a generated column value. 0 is the usual case. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall( + Parse *pParse, /* Parsing context */ + int p1, /* Constant argument mask */ + int p2, /* First argument register */ + int p3, /* Register into which results are written */ + int nArg, /* Number of argument */ + const FuncDef *pFunc, /* The function to be invoked */ + int eCallCtx /* Calling context */ +){ + Vdbe *v = pParse->pVdbe; + int nByte; + int addr; + sqlite3_context *pCtx; + assert( v ); + nByte = sizeof(*pCtx) + (nArg-1)*sizeof(sqlite3_value*); + pCtx = sqlite3DbMallocRawNN(pParse->db, nByte); + if( pCtx==0 ){ + assert( pParse->db->mallocFailed ); + freeEphemeralFunction(pParse->db, (FuncDef*)pFunc); + return 0; + } + pCtx->pOut = 0; + pCtx->pFunc = (FuncDef*)pFunc; + pCtx->pVdbe = 0; + pCtx->isError = 0; + pCtx->argc = nArg; + pCtx->iOp = sqlite3VdbeCurrentAddr(v); + addr = sqlite3VdbeAddOp4(v, eCallCtx ? OP_PureFunc : OP_Function, + p1, p2, p3, (char*)pCtx, P4_FUNCCTX); + sqlite3VdbeChangeP5(v, eCallCtx & NC_SelfRef); + return addr; +} + +/* +** Add an opcode that includes the p4 value with a P4_INT64 or +** P4_REAL type. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + const u8 *zP4, /* The P4 operand */ + int p4type /* P4 operand type */ +){ + char *p4copy = sqlite3DbMallocRawNN(sqlite3VdbeDb(p), 8); + if( p4copy ) memcpy(p4copy, zP4, 8); + return sqlite3VdbeAddOp4(p, op, p1, p2, p3, p4copy, p4type); +} + +#ifndef SQLITE_OMIT_EXPLAIN +/* +** Return the address of the current EXPLAIN QUERY PLAN baseline. +** 0 means "none". +*/ +SQLITE_PRIVATE int sqlite3VdbeExplainParent(Parse *pParse){ + VdbeOp *pOp; + if( pParse->addrExplain==0 ) return 0; + pOp = sqlite3VdbeGetOp(pParse->pVdbe, pParse->addrExplain); + return pOp->p2; +} + +/* +** Set a debugger breakpoint on the following routine in order to +** monitor the EXPLAIN QUERY PLAN code generation. +*/ +#if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char *z1, const char *z2){ + (void)z1; + (void)z2; +} +#endif + +/* +** Add a new OP_Explain opcode. +** +** If the bPush flag is true, then make this opcode the parent for +** subsequent Explains until sqlite3VdbeExplainPop() is called. +*/ +SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ +#ifndef SQLITE_DEBUG + /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined. + ** But omit them (for performance) during production builds */ + if( pParse->explain==2 ) +#endif + { + char *zMsg; + Vdbe *v; + va_list ap; + int iThis; + va_start(ap, zFmt); + zMsg = sqlite3VMPrintf(pParse->db, zFmt, ap); + va_end(ap); + v = pParse->pVdbe; + iThis = v->nOp; + sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, + zMsg, P4_DYNAMIC); + sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetOp(v,-1)->p4.z); + if( bPush){ + pParse->addrExplain = iThis; + } + } +} + +/* +** Pop the EXPLAIN QUERY PLAN stack one level. +*/ +SQLITE_PRIVATE void sqlite3VdbeExplainPop(Parse *pParse){ + sqlite3ExplainBreakpoint("POP", 0); + pParse->addrExplain = sqlite3VdbeExplainParent(pParse); +} +#endif /* SQLITE_OMIT_EXPLAIN */ + +/* +** Add an OP_ParseSchema opcode. This routine is broken out from +** sqlite3VdbeAddOp4() since it needs to also needs to mark all btrees +** as having been used. +** +** The zWhere string must have been obtained from sqlite3_malloc(). +** This routine will take ownership of the allocated memory. +*/ +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere, u16 p5){ + int j; + sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeChangeP5(p, p5); + for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); + sqlite3MayAbort(p->pParse); +} + +/* +** Add an opcode that includes the p4 value as an integer. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + int p4 /* The P4 operand as an integer */ +){ + int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); + if( p->db->mallocFailed==0 ){ + VdbeOp *pOp = &p->aOp[addr]; + pOp->p4type = P4_INT32; + pOp->p4.i = p4; + } + return addr; +} + +/* Insert the end of a co-routine +*/ +SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe *v, int regYield){ + sqlite3VdbeAddOp1(v, OP_EndCoroutine, regYield); + + /* Clear the temporary register cache, thereby ensuring that each + ** co-routine has its own independent set of registers, because co-routines + ** might expect their registers to be preserved across an OP_Yield, and + ** that could cause problems if two or more co-routines are using the same + ** temporary register. + */ + v->pParse->nTempReg = 0; + v->pParse->nRangeReg = 0; +} + +/* +** Create a new symbolic label for an instruction that has yet to be +** coded. The symbolic label is really just a negative number. The +** label can be used as the P2 value of an operation. Later, when +** the label is resolved to a specific address, the VDBE will scan +** through its operation list and change all values of P2 which match +** the label into the resolved address. +** +** The VDBE knows that a P2 value is a label because labels are +** always negative and P2 values are suppose to be non-negative. +** Hence, a negative P2 value is a label that has yet to be resolved. +** (Later:) This is only true for opcodes that have the OPFLG_JUMP +** property. +** +** Variable usage notes: +** +** Parse.aLabel[x] Stores the address that the x-th label resolves +** into. For testing (SQLITE_DEBUG), unresolved +** labels stores -1, but that is not required. +** Parse.nLabelAlloc Number of slots allocated to Parse.aLabel[] +** Parse.nLabel The *negative* of the number of labels that have +** been issued. The negative is stored because +** that gives a performance improvement over storing +** the equivalent positive value. +*/ +SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse *pParse){ + return --pParse->nLabel; +} + +/* +** Resolve label "x" to be the address of the next instruction to +** be inserted. The parameter "x" must have been obtained from +** a prior call to sqlite3VdbeMakeLabel(). +*/ +static SQLITE_NOINLINE void resizeResolveLabel(Parse *p, Vdbe *v, int j){ + int nNewSize = 10 - p->nLabel; + p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, + nNewSize*sizeof(p->aLabel[0])); + if( p->aLabel==0 ){ + p->nLabelAlloc = 0; + }else{ +#ifdef SQLITE_DEBUG + int i; + for(i=p->nLabelAlloc; iaLabel[i] = -1; +#endif + p->nLabelAlloc = nNewSize; + p->aLabel[j] = v->nOp; + } +} +SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ + Parse *p = v->pParse; + int j = ADDR(x); + assert( v->iVdbeMagic==VDBE_MAGIC_INIT ); + assert( j<-p->nLabel ); + assert( j>=0 ); +#ifdef SQLITE_DEBUG + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + printf("RESOLVE LABEL %d to %d\n", x, v->nOp); + } +#endif + if( p->nLabelAlloc + p->nLabel < 0 ){ + resizeResolveLabel(p,v,j); + }else{ + assert( p->aLabel[j]==(-1) ); /* Labels may only be resolved once */ + p->aLabel[j] = v->nOp; + } +} + +/* +** Mark the VDBE as one that can only be run one time. +*/ +SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe *p){ + p->runOnlyOnce = 1; +} + +/* +** Mark the VDBE as one that can only be run multiple times. +*/ +SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe *p){ + p->runOnlyOnce = 0; +} + +#ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */ + +/* +** The following type and function are used to iterate through all opcodes +** in a Vdbe main program and each of the sub-programs (triggers) it may +** invoke directly or indirectly. It should be used as follows: +** +** Op *pOp; +** VdbeOpIter sIter; +** +** memset(&sIter, 0, sizeof(sIter)); +** sIter.v = v; // v is of type Vdbe* +** while( (pOp = opIterNext(&sIter)) ){ +** // Do something with pOp +** } +** sqlite3DbFree(v->db, sIter.apSub); +** +*/ +typedef struct VdbeOpIter VdbeOpIter; +struct VdbeOpIter { + Vdbe *v; /* Vdbe to iterate through the opcodes of */ + SubProgram **apSub; /* Array of subprograms */ + int nSub; /* Number of entries in apSub */ + int iAddr; /* Address of next instruction to return */ + int iSub; /* 0 = main program, 1 = first sub-program etc. */ +}; +static Op *opIterNext(VdbeOpIter *p){ + Vdbe *v = p->v; + Op *pRet = 0; + Op *aOp; + int nOp; + + if( p->iSub<=p->nSub ){ + + if( p->iSub==0 ){ + aOp = v->aOp; + nOp = v->nOp; + }else{ + aOp = p->apSub[p->iSub-1]->aOp; + nOp = p->apSub[p->iSub-1]->nOp; + } + assert( p->iAddriAddr]; + p->iAddr++; + if( p->iAddr==nOp ){ + p->iSub++; + p->iAddr = 0; + } + + if( pRet->p4type==P4_SUBPROGRAM ){ + int nByte = (p->nSub+1)*sizeof(SubProgram*); + int j; + for(j=0; jnSub; j++){ + if( p->apSub[j]==pRet->p4.pProgram ) break; + } + if( j==p->nSub ){ + p->apSub = sqlite3DbReallocOrFree(v->db, p->apSub, nByte); + if( !p->apSub ){ + pRet = 0; + }else{ + p->apSub[p->nSub++] = pRet->p4.pProgram; + } + } + } + } + + return pRet; +} + +/* +** Check if the program stored in the VM associated with pParse may +** throw an ABORT exception (causing the statement, but not entire transaction +** to be rolled back). This condition is true if the main program or any +** sub-programs contains any of the following: +** +** * OP_Halt with P1=SQLITE_CONSTRAINT and P2=OE_Abort. +** * OP_HaltIfNull with P1=SQLITE_CONSTRAINT and P2=OE_Abort. +** * OP_Destroy +** * OP_VUpdate +** * OP_VCreate +** * OP_VRename +** * OP_FkCounter with P2==0 (immediate foreign key constraint) +** * OP_CreateBtree/BTREE_INTKEY and OP_InitCoroutine +** (for CREATE TABLE AS SELECT ...) +** +** Then check that the value of Parse.mayAbort is true if an +** ABORT may be thrown, or false otherwise. Return true if it does +** match, or false otherwise. This function is intended to be used as +** part of an assert statement in the compiler. Similar to: +** +** assert( sqlite3VdbeAssertMayAbort(pParse->pVdbe, pParse->mayAbort) ); +*/ +SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ + int hasAbort = 0; + int hasFkCounter = 0; + int hasCreateTable = 0; + int hasCreateIndex = 0; + int hasInitCoroutine = 0; + Op *pOp; + VdbeOpIter sIter; + memset(&sIter, 0, sizeof(sIter)); + sIter.v = v; + + while( (pOp = opIterNext(&sIter))!=0 ){ + int opcode = pOp->opcode; + if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename + || opcode==OP_VDestroy + || opcode==OP_VCreate + || opcode==OP_ParseSchema + || ((opcode==OP_Halt || opcode==OP_HaltIfNull) + && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort)) + ){ + hasAbort = 1; + break; + } + if( opcode==OP_CreateBtree && pOp->p3==BTREE_INTKEY ) hasCreateTable = 1; + if( mayAbort ){ + /* hasCreateIndex may also be set for some DELETE statements that use + ** OP_Clear. So this routine may end up returning true in the case + ** where a "DELETE FROM tbl" has a statement-journal but does not + ** require one. This is not so bad - it is an inefficiency, not a bug. */ + if( opcode==OP_CreateBtree && pOp->p3==BTREE_BLOBKEY ) hasCreateIndex = 1; + if( opcode==OP_Clear ) hasCreateIndex = 1; + } + if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1; +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ + hasFkCounter = 1; + } +#endif + } + sqlite3DbFree(v->db, sIter.apSub); + + /* Return true if hasAbort==mayAbort. Or if a malloc failure occurred. + ** If malloc failed, then the while() loop above may not have iterated + ** through all opcodes and hasAbort may be set incorrectly. Return + ** true for this case to prevent the assert() in the callers frame + ** from failing. */ + return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter + || (hasCreateTable && hasInitCoroutine) || hasCreateIndex + ); +} +#endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ + +#ifdef SQLITE_DEBUG +/* +** Increment the nWrite counter in the VDBE if the cursor is not an +** ephemeral cursor, or if the cursor argument is NULL. +*/ +SQLITE_PRIVATE void sqlite3VdbeIncrWriteCounter(Vdbe *p, VdbeCursor *pC){ + if( pC==0 + || (pC->eCurType!=CURTYPE_SORTER + && pC->eCurType!=CURTYPE_PSEUDO + && !pC->isEphemeral) + ){ + p->nWrite++; + } +} +#endif + +#ifdef SQLITE_DEBUG +/* +** Assert if an Abort at this point in time might result in a corrupt +** database. +*/ +SQLITE_PRIVATE void sqlite3VdbeAssertAbortable(Vdbe *p){ + assert( p->nWrite==0 || p->usesStmtJournal ); +} +#endif + +/* +** This routine is called after all opcodes have been inserted. It loops +** through all the opcodes and fixes up some details. +** +** (1) For each jump instruction with a negative P2 value (a label) +** resolve the P2 value to an actual address. +** +** (2) Compute the maximum number of arguments used by any SQL function +** and store that value in *pMaxFuncArgs. +** +** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately +** indicate what the prepared statement actually does. +** +** (4) Initialize the p4.xAdvance pointer on opcodes that use it. +** +** (5) Reclaim the memory allocated for storing labels. +** +** This routine will only function correctly if the mkopcodeh.tcl generator +** script numbers the opcodes correctly. Changes to this routine must be +** coordinated with changes to mkopcodeh.tcl. +*/ +static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ + int nMaxArgs = *pMaxFuncArgs; + Op *pOp; + Parse *pParse = p->pParse; + int *aLabel = pParse->aLabel; + p->readOnly = 1; + p->bIsReader = 0; + pOp = &p->aOp[p->nOp-1]; + while(1){ + + /* Only JUMP opcodes and the short list of special opcodes in the switch + ** below need to be considered. The mkopcodeh.tcl generator script groups + ** all these opcodes together near the front of the opcode list. Skip + ** any opcode that does not need processing by virtual of the fact that + ** it is larger than SQLITE_MX_JUMP_OPCODE, as a performance optimization. + */ + if( pOp->opcode<=SQLITE_MX_JUMP_OPCODE ){ + /* NOTE: Be sure to update mkopcodeh.tcl when adding or removing + ** cases from this switch! */ + switch( pOp->opcode ){ + case OP_Transaction: { + if( pOp->p2!=0 ) p->readOnly = 0; + /* no break */ deliberate_fall_through + } + case OP_AutoCommit: + case OP_Savepoint: { + p->bIsReader = 1; + break; + } +#ifndef SQLITE_OMIT_WAL + case OP_Checkpoint: +#endif + case OP_Vacuum: + case OP_JournalMode: { + p->readOnly = 0; + p->bIsReader = 1; + break; + } + case OP_Next: + case OP_SorterNext: { + pOp->p4.xAdvance = sqlite3BtreeNext; + pOp->p4type = P4_ADVANCE; + /* The code generator never codes any of these opcodes as a jump + ** to a label. They are always coded as a jump backwards to a + ** known address */ + assert( pOp->p2>=0 ); + break; + } + case OP_Prev: { + pOp->p4.xAdvance = sqlite3BtreePrevious; + pOp->p4type = P4_ADVANCE; + /* The code generator never codes any of these opcodes as a jump + ** to a label. They are always coded as a jump backwards to a + ** known address */ + assert( pOp->p2>=0 ); + break; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + case OP_VUpdate: { + if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; + break; + } + case OP_VFilter: { + int n; + assert( (pOp - p->aOp) >= 3 ); + assert( pOp[-1].opcode==OP_Integer ); + n = pOp[-1].p1; + if( n>nMaxArgs ) nMaxArgs = n; + /* Fall through into the default case */ + /* no break */ deliberate_fall_through + } +#endif + default: { + if( pOp->p2<0 ){ + /* The mkopcodeh.tcl script has so arranged things that the only + ** non-jump opcodes less than SQLITE_MX_JUMP_CODE are guaranteed to + ** have non-negative values for P2. */ + assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ); + assert( ADDR(pOp->p2)<-pParse->nLabel ); + pOp->p2 = aLabel[ADDR(pOp->p2)]; + } + break; + } + } + /* The mkopcodeh.tcl script has so arranged things that the only + ** non-jump opcodes less than SQLITE_MX_JUMP_CODE are guaranteed to + ** have non-negative values for P2. */ + assert( (sqlite3OpcodeProperty[pOp->opcode]&OPFLG_JUMP)==0 || pOp->p2>=0); + } + if( pOp==p->aOp ) break; + pOp--; + } + sqlite3DbFree(p->db, pParse->aLabel); + pParse->aLabel = 0; + pParse->nLabel = 0; + *pMaxFuncArgs = nMaxArgs; + assert( p->bIsReader!=0 || DbMaskAllZero(p->btreeMask) ); +} + +/* +** Return the address of the next instruction to be inserted. +*/ +SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe *p){ + assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); + return p->nOp; +} + +/* +** Verify that at least N opcode slots are available in p without +** having to malloc for more space (except when compiled using +** SQLITE_TEST_REALLOC_STRESS). This interface is used during testing +** to verify that certain calls to sqlite3VdbeAddOpList() can never +** fail due to a OOM fault and hence that the return value from +** sqlite3VdbeAddOpList() will always be non-NULL. +*/ +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) +SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N){ + assert( p->nOp + N <= p->nOpAlloc ); +} +#endif + +/* +** Verify that the VM passed as the only argument does not contain +** an OP_ResultRow opcode. Fail an assert() if it does. This is used +** by code in pragma.c to ensure that the implementation of certain +** pragmas comports with the flags specified in the mkpragmatab.tcl +** script. +*/ +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) +SQLITE_PRIVATE void sqlite3VdbeVerifyNoResultRow(Vdbe *p){ + int i; + for(i=0; inOp; i++){ + assert( p->aOp[i].opcode!=OP_ResultRow ); + } +} +#endif + +/* +** Generate code (a single OP_Abortable opcode) that will +** verify that the VDBE program can safely call Abort in the current +** context. +*/ +#if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3VdbeVerifyAbortable(Vdbe *p, int onError){ + if( onError==OE_Abort ) sqlite3VdbeAddOp0(p, OP_Abortable); +} +#endif + +/* +** This function returns a pointer to the array of opcodes associated with +** the Vdbe passed as the first argument. It is the callers responsibility +** to arrange for the returned array to be eventually freed using the +** vdbeFreeOpArray() function. +** +** Before returning, *pnOp is set to the number of entries in the returned +** array. Also, *pnMaxArg is set to the larger of its current value and +** the number of entries in the Vdbe.apArg[] array required to execute the +** returned program. +*/ +SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){ + VdbeOp *aOp = p->aOp; + assert( aOp && !p->db->mallocFailed ); + + /* Check that sqlite3VdbeUsesBtree() was not called on this VM */ + assert( DbMaskAllZero(p->btreeMask) ); + + resolveP2Values(p, pnMaxArg); + *pnOp = p->nOp; + p->aOp = 0; + return aOp; +} + +/* +** Add a whole list of operations to the operation stack. Return a +** pointer to the first operation inserted. +** +** Non-zero P2 arguments to jump instructions are automatically adjusted +** so that the jump target is relative to the first operation inserted. +*/ +SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList( + Vdbe *p, /* Add opcodes to the prepared statement */ + int nOp, /* Number of opcodes to add */ + VdbeOpList const *aOp, /* The opcodes to be added */ + int iLineno /* Source-file line number of first opcode */ +){ + int i; + VdbeOp *pOut, *pFirst; + assert( nOp>0 ); + assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); + if( p->nOp + nOp > p->nOpAlloc && growOpArray(p, nOp) ){ + return 0; + } + pFirst = pOut = &p->aOp[p->nOp]; + for(i=0; iopcode = aOp->opcode; + pOut->p1 = aOp->p1; + pOut->p2 = aOp->p2; + assert( aOp->p2>=0 ); + if( (sqlite3OpcodeProperty[aOp->opcode] & OPFLG_JUMP)!=0 && aOp->p2>0 ){ + pOut->p2 += p->nOp; + } + pOut->p3 = aOp->p3; + pOut->p4type = P4_NOTUSED; + pOut->p4.p = 0; + pOut->p5 = 0; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + pOut->zComment = 0; +#endif +#ifdef SQLITE_VDBE_COVERAGE + pOut->iSrcLine = iLineno+i; +#else + (void)iLineno; +#endif +#ifdef SQLITE_DEBUG + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i+p->nOp, &p->aOp[i+p->nOp]); + } +#endif + } + p->nOp += nOp; + return pFirst; +} + +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) +/* +** Add an entry to the array of counters managed by sqlite3_stmt_scanstatus(). +*/ +SQLITE_PRIVATE void sqlite3VdbeScanStatus( + Vdbe *p, /* VM to add scanstatus() to */ + int addrExplain, /* Address of OP_Explain (or 0) */ + int addrLoop, /* Address of loop counter */ + int addrVisit, /* Address of rows visited counter */ + LogEst nEst, /* Estimated number of output rows */ + const char *zName /* Name of table or index being scanned */ +){ + sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanStatus *pNew = &aNew[p->nScan++]; + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; + } +} +#endif + + +/* +** Change the value of the opcode, or P1, P2, P3, or P5 operands +** for a specific instruction. +*/ +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, int addr, u8 iNewOpcode){ + sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; +} +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ + sqlite3VdbeGetOp(p,addr)->p1 = val; +} +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ + sqlite3VdbeGetOp(p,addr)->p2 = val; +} +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ + sqlite3VdbeGetOp(p,addr)->p3 = val; +} +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ + assert( p->nOp>0 || p->db->mallocFailed ); + if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5; +} + +/* +** Change the P2 operand of instruction addr so that it points to +** the address of the next instruction to be coded. +*/ +SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ + sqlite3VdbeChangeP2(p, addr, p->nOp); +} + +/* +** Change the P2 operand of the jump instruction at addr so that +** the jump lands on the next opcode. Or if the jump instruction was +** the previous opcode (and is thus a no-op) then simply back up +** the next instruction counter by one slot so that the jump is +** overwritten by the next inserted opcode. +** +** This routine is an optimization of sqlite3VdbeJumpHere() that +** strives to omit useless byte-code like this: +** +** 7 Once 0 8 0 +** 8 ... +*/ +SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ + if( addr==p->nOp-1 ){ + assert( p->aOp[addr].opcode==OP_Once + || p->aOp[addr].opcode==OP_If + || p->aOp[addr].opcode==OP_FkIfZero ); + assert( p->aOp[addr].p4type==0 ); +#ifdef SQLITE_VDBE_COVERAGE + sqlite3VdbeGetOp(p,-1)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ +#endif + p->nOp--; + }else{ + sqlite3VdbeChangeP2(p, addr, p->nOp); + } +} + + +/* +** If the input FuncDef structure is ephemeral, then free it. If +** the FuncDef is not ephermal, then do nothing. +*/ +static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ + if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ + sqlite3DbFreeNN(db, pDef); + } +} + +/* +** Delete a P4 value if necessary. +*/ +static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){ + if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); + sqlite3DbFreeNN(db, p); +} +static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){ + freeEphemeralFunction(db, p->pFunc); + sqlite3DbFreeNN(db, p); +} +static void freeP4(sqlite3 *db, int p4type, void *p4){ + assert( db ); + switch( p4type ){ + case P4_FUNCCTX: { + freeP4FuncCtx(db, (sqlite3_context*)p4); + break; + } + case P4_REAL: + case P4_INT64: + case P4_DYNAMIC: + case P4_DYNBLOB: + case P4_INTARRAY: { + sqlite3DbFree(db, p4); + break; + } + case P4_KEYINFO: { + if( db->pnBytesFreed==0 ) sqlite3KeyInfoUnref((KeyInfo*)p4); + break; + } +#ifdef SQLITE_ENABLE_CURSOR_HINTS + case P4_EXPR: { + sqlite3ExprDelete(db, (Expr*)p4); + break; + } +#endif + case P4_FUNCDEF: { + freeEphemeralFunction(db, (FuncDef*)p4); + break; + } + case P4_MEM: { + if( db->pnBytesFreed==0 ){ + sqlite3ValueFree((sqlite3_value*)p4); + }else{ + freeP4Mem(db, (Mem*)p4); + } + break; + } + case P4_VTAB : { + if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); + break; + } + } +} + +/* +** Free the space allocated for aOp and any p4 values allocated for the +** opcodes contained within. If aOp is not NULL it is assumed to contain +** nOp entries. +*/ +static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ + if( aOp ){ + Op *pOp; + for(pOp=&aOp[nOp-1]; pOp>=aOp; pOp--){ + if( pOp->p4type <= P4_FREE_IF_LE ) freeP4(db, pOp->p4type, pOp->p4.p); +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + sqlite3DbFree(db, pOp->zComment); +#endif + } + sqlite3DbFreeNN(db, aOp); + } +} + +/* +** Link the SubProgram object passed as the second argument into the linked +** list at Vdbe.pSubProgram. This list is used to delete all sub-program +** objects when the VM is no longer required. +*/ +SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ + p->pNext = pVdbe->pProgram; + pVdbe->pProgram = p; +} + +/* +** Return true if the given Vdbe has any SubPrograms. +*/ +SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe *pVdbe){ + return pVdbe->pProgram!=0; +} + +/* +** Change the opcode at addr into OP_Noop +*/ +SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ + VdbeOp *pOp; + if( p->db->mallocFailed ) return 0; + assert( addr>=0 && addrnOp ); + pOp = &p->aOp[addr]; + freeP4(p->db, pOp->p4type, pOp->p4.p); + pOp->p4type = P4_NOTUSED; + pOp->p4.z = 0; + pOp->opcode = OP_Noop; + return 1; +} + +/* +** If the last opcode is "op" and it is not a jump destination, +** then remove it. Return true if and only if an opcode was removed. +*/ +SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ + if( p->nOp>0 && p->aOp[p->nOp-1].opcode==op ){ + return sqlite3VdbeChangeToNoop(p, p->nOp-1); + }else{ + return 0; + } +} + +#ifdef SQLITE_DEBUG +/* +** Generate an OP_ReleaseReg opcode to indicate that a range of +** registers, except any identified by mask, are no longer in use. +*/ +SQLITE_PRIVATE void sqlite3VdbeReleaseRegisters( + Parse *pParse, /* Parsing context */ + int iFirst, /* Index of first register to be released */ + int N, /* Number of registers to release */ + u32 mask, /* Mask of registers to NOT release */ + int bUndefine /* If true, mark registers as undefined */ +){ + if( N==0 ) return; + assert( pParse->pVdbe ); + assert( iFirst>=1 ); + assert( iFirst+N-1<=pParse->nMem ); + if( N<=31 && mask!=0 ){ + while( N>0 && (mask&1)!=0 ){ + mask >>= 1; + iFirst++; + N--; + } + while( N>0 && N<=32 && (mask & MASKBIT32(N-1))!=0 ){ + mask &= ~MASKBIT32(N-1); + N--; + } + } + if( N>0 ){ + sqlite3VdbeAddOp3(pParse->pVdbe, OP_ReleaseReg, iFirst, N, *(int*)&mask); + if( bUndefine ) sqlite3VdbeChangeP5(pParse->pVdbe, 1); + } +} +#endif /* SQLITE_DEBUG */ + + +/* +** Change the value of the P4 operand for a specific instruction. +** This routine is useful when a large program is loaded from a +** static array using sqlite3VdbeAddOpList but we want to make a +** few minor changes to the program. +** +** If n>=0 then the P4 operand is dynamic, meaning that a copy of +** the string is made into memory obtained from sqlite3_malloc(). +** A value of n==0 means copy bytes of zP4 up to and including the +** first null byte. If n>0 then copy n+1 bytes of zP4. +** +** Other values of n (P4_STATIC, P4_COLLSEQ etc.) indicate that zP4 points +** to a string or structure that is guaranteed to exist for the lifetime of +** the Vdbe. In these cases we can just copy the pointer. +** +** If addr<0 then change P4 on the most recently inserted instruction. +*/ +static void SQLITE_NOINLINE vdbeChangeP4Full( + Vdbe *p, + Op *pOp, + const char *zP4, + int n +){ + if( pOp->p4type ){ + freeP4(p->db, pOp->p4type, pOp->p4.p); + pOp->p4type = 0; + pOp->p4.p = 0; + } + if( n<0 ){ + sqlite3VdbeChangeP4(p, (int)(pOp - p->aOp), zP4, n); + }else{ + if( n==0 ) n = sqlite3Strlen30(zP4); + pOp->p4.z = sqlite3DbStrNDup(p->db, zP4, n); + pOp->p4type = P4_DYNAMIC; + } +} +SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ + Op *pOp; + sqlite3 *db; + assert( p!=0 ); + db = p->db; + assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); + assert( p->aOp!=0 || db->mallocFailed ); + if( db->mallocFailed ){ + if( n!=P4_VTAB ) freeP4(db, n, (void*)*(char**)&zP4); + return; + } + assert( p->nOp>0 ); + assert( addrnOp ); + if( addr<0 ){ + addr = p->nOp - 1; + } + pOp = &p->aOp[addr]; + if( n>=0 || pOp->p4type ){ + vdbeChangeP4Full(p, pOp, zP4, n); + return; + } + if( n==P4_INT32 ){ + /* Note: this cast is safe, because the origin data point was an int + ** that was cast to a (const char *). */ + pOp->p4.i = SQLITE_PTR_TO_INT(zP4); + pOp->p4type = P4_INT32; + }else if( zP4!=0 ){ + assert( n<0 ); + pOp->p4.p = (void*)zP4; + pOp->p4type = (signed char)n; + if( n==P4_VTAB ) sqlite3VtabLock((VTable*)zP4); + } +} + +/* +** Change the P4 operand of the most recently coded instruction +** to the value defined by the arguments. This is a high-speed +** version of sqlite3VdbeChangeP4(). +** +** The P4 operand must not have been previously defined. And the new +** P4 must not be P4_INT32. Use sqlite3VdbeChangeP4() in either of +** those cases. +*/ +SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe *p, void *pP4, int n){ + VdbeOp *pOp; + assert( n!=P4_INT32 && n!=P4_VTAB ); + assert( n<=0 ); + if( p->db->mallocFailed ){ + freeP4(p->db, n, pP4); + }else{ + assert( pP4!=0 ); + assert( p->nOp>0 ); + pOp = &p->aOp[p->nOp-1]; + assert( pOp->p4type==P4_NOTUSED ); + pOp->p4type = n; + pOp->p4.p = pP4; + } +} + +/* +** Set the P4 on the most recently added opcode to the KeyInfo for the +** index given. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse *pParse, Index *pIdx){ + Vdbe *v = pParse->pVdbe; + KeyInfo *pKeyInfo; + assert( v!=0 ); + assert( pIdx!=0 ); + pKeyInfo = sqlite3KeyInfoOfIndex(pParse, pIdx); + if( pKeyInfo ) sqlite3VdbeAppendP4(v, pKeyInfo, P4_KEYINFO); +} + +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +/* +** Change the comment on the most recently coded instruction. Or +** insert a No-op and add the comment to that new instruction. This +** makes the code easier to read during debugging. None of this happens +** in a production build. +*/ +static void vdbeVComment(Vdbe *p, const char *zFormat, va_list ap){ + assert( p->nOp>0 || p->aOp==0 ); + assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed + || p->pParse->nErr>0 ); + if( p->nOp ){ + assert( p->aOp ); + sqlite3DbFree(p->db, p->aOp[p->nOp-1].zComment); + p->aOp[p->nOp-1].zComment = sqlite3VMPrintf(p->db, zFormat, ap); + } +} +SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe *p, const char *zFormat, ...){ + va_list ap; + if( p ){ + va_start(ap, zFormat); + vdbeVComment(p, zFormat, ap); + va_end(ap); + } +} +SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ + va_list ap; + if( p ){ + sqlite3VdbeAddOp0(p, OP_Noop); + va_start(ap, zFormat); + vdbeVComment(p, zFormat, ap); + va_end(ap); + } +} +#endif /* NDEBUG */ + +#ifdef SQLITE_VDBE_COVERAGE +/* +** Set the value if the iSrcLine field for the previously coded instruction. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){ + sqlite3VdbeGetOp(v,-1)->iSrcLine = iLine; +} +#endif /* SQLITE_VDBE_COVERAGE */ + +/* +** Return the opcode for a given address. If the address is -1, then +** return the most recently inserted opcode. +** +** If a memory allocation error has occurred prior to the calling of this +** routine, then a pointer to a dummy VdbeOp will be returned. That opcode +** is readable but not writable, though it is cast to a writable value. +** The return of a dummy opcode allows the call to continue functioning +** after an OOM fault without having to check to see if the return from +** this routine is a valid pointer. But because the dummy.opcode is 0, +** dummy will never be written to. This is verified by code inspection and +** by running with Valgrind. +*/ +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ + /* C89 specifies that the constant "dummy" will be initialized to all + ** zeros, which is correct. MSVC generates a warning, nevertheless. */ + static VdbeOp dummy; /* Ignore the MSVC warning about no initializer */ + assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); + if( addr<0 ){ + addr = p->nOp - 1; + } + assert( (addr>=0 && addrnOp) || p->db->mallocFailed ); + if( p->db->mallocFailed ){ + return (VdbeOp*)&dummy; + }else{ + return &p->aOp[addr]; + } +} + +#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) +/* +** Return an integer value for one of the parameters to the opcode pOp +** determined by character c. +*/ +static int translateP(char c, const Op *pOp){ + if( c=='1' ) return pOp->p1; + if( c=='2' ) return pOp->p2; + if( c=='3' ) return pOp->p3; + if( c=='4' ) return pOp->p4.i; + return pOp->p5; +} + +/* +** Compute a string for the "comment" field of a VDBE opcode listing. +** +** The Synopsis: field in comments in the vdbe.c source file gets converted +** to an extra string that is appended to the sqlite3OpcodeName(). In the +** absence of other comments, this synopsis becomes the comment on the opcode. +** Some translation occurs: +** +** "PX" -> "r[X]" +** "PX@PY" -> "r[X..X+Y-1]" or "r[x]" if y is 0 or 1 +** "PX@PY+1" -> "r[X..X+Y]" or "r[x]" if y is 0 +** "PY..PY" -> "r[X..Y]" or "r[x]" if y<=x +*/ +SQLITE_PRIVATE char *sqlite3VdbeDisplayComment( + sqlite3 *db, /* Optional - Oom error reporting only */ + const Op *pOp, /* The opcode to be commented */ + const char *zP4 /* Previously obtained value for P4 */ +){ + const char *zOpName; + const char *zSynopsis; + int nOpName; + int ii; + char zAlt[50]; + StrAccum x; + + sqlite3StrAccumInit(&x, 0, 0, 0, SQLITE_MAX_LENGTH); + zOpName = sqlite3OpcodeName(pOp->opcode); + nOpName = sqlite3Strlen30(zOpName); + if( zOpName[nOpName+1] ){ + int seenCom = 0; + char c; + zSynopsis = zOpName += nOpName + 1; + if( strncmp(zSynopsis,"IF ",3)==0 ){ + sqlite3_snprintf(sizeof(zAlt), zAlt, "if %s goto P2", zSynopsis+3); + zSynopsis = zAlt; + } + for(ii=0; (c = zSynopsis[ii])!=0; ii++){ + if( c=='P' ){ + c = zSynopsis[++ii]; + if( c=='4' ){ + sqlite3_str_appendall(&x, zP4); + }else if( c=='X' ){ + sqlite3_str_appendall(&x, pOp->zComment); + seenCom = 1; + }else{ + int v1 = translateP(c, pOp); + int v2; + if( strncmp(zSynopsis+ii+1, "@P", 2)==0 ){ + ii += 3; + v2 = translateP(zSynopsis[ii], pOp); + if( strncmp(zSynopsis+ii+1,"+1",2)==0 ){ + ii += 2; + v2++; + } + if( v2<2 ){ + sqlite3_str_appendf(&x, "%d", v1); + }else{ + sqlite3_str_appendf(&x, "%d..%d", v1, v1+v2-1); + } + }else if( strncmp(zSynopsis+ii+1, "@NP", 3)==0 ){ + sqlite3_context *pCtx = pOp->p4.pCtx; + if( pOp->p4type!=P4_FUNCCTX || pCtx->argc==1 ){ + sqlite3_str_appendf(&x, "%d", v1); + }else if( pCtx->argc>1 ){ + sqlite3_str_appendf(&x, "%d..%d", v1, v1+pCtx->argc-1); + }else if( x.accError==0 ){ + assert( x.nChar>2 ); + x.nChar -= 2; + ii++; + } + ii += 3; + }else{ + sqlite3_str_appendf(&x, "%d", v1); + if( strncmp(zSynopsis+ii+1, "..P3", 4)==0 && pOp->p3==0 ){ + ii += 4; + } + } + } + }else{ + sqlite3_str_appendchar(&x, 1, c); + } + } + if( !seenCom && pOp->zComment ){ + sqlite3_str_appendf(&x, "; %s", pOp->zComment); + } + }else if( pOp->zComment ){ + sqlite3_str_appendall(&x, pOp->zComment); + } + if( (x.accError & SQLITE_NOMEM)!=0 && db!=0 ){ + sqlite3OomFault(db); + } + return sqlite3StrAccumFinish(&x); +} +#endif /* SQLITE_ENABLE_EXPLAIN_COMMENTS */ + +#if VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) +/* +** Translate the P4.pExpr value for an OP_CursorHint opcode into text +** that can be displayed in the P4 column of EXPLAIN output. +*/ +static void displayP4Expr(StrAccum *p, Expr *pExpr){ + const char *zOp = 0; + switch( pExpr->op ){ + case TK_STRING: + sqlite3_str_appendf(p, "%Q", pExpr->u.zToken); + break; + case TK_INTEGER: + sqlite3_str_appendf(p, "%d", pExpr->u.iValue); + break; + case TK_NULL: + sqlite3_str_appendf(p, "NULL"); + break; + case TK_REGISTER: { + sqlite3_str_appendf(p, "r[%d]", pExpr->iTable); + break; + } + case TK_COLUMN: { + if( pExpr->iColumn<0 ){ + sqlite3_str_appendf(p, "rowid"); + }else{ + sqlite3_str_appendf(p, "c%d", (int)pExpr->iColumn); + } + break; + } + case TK_LT: zOp = "LT"; break; + case TK_LE: zOp = "LE"; break; + case TK_GT: zOp = "GT"; break; + case TK_GE: zOp = "GE"; break; + case TK_NE: zOp = "NE"; break; + case TK_EQ: zOp = "EQ"; break; + case TK_IS: zOp = "IS"; break; + case TK_ISNOT: zOp = "ISNOT"; break; + case TK_AND: zOp = "AND"; break; + case TK_OR: zOp = "OR"; break; + case TK_PLUS: zOp = "ADD"; break; + case TK_STAR: zOp = "MUL"; break; + case TK_MINUS: zOp = "SUB"; break; + case TK_REM: zOp = "REM"; break; + case TK_BITAND: zOp = "BITAND"; break; + case TK_BITOR: zOp = "BITOR"; break; + case TK_SLASH: zOp = "DIV"; break; + case TK_LSHIFT: zOp = "LSHIFT"; break; + case TK_RSHIFT: zOp = "RSHIFT"; break; + case TK_CONCAT: zOp = "CONCAT"; break; + case TK_UMINUS: zOp = "MINUS"; break; + case TK_UPLUS: zOp = "PLUS"; break; + case TK_BITNOT: zOp = "BITNOT"; break; + case TK_NOT: zOp = "NOT"; break; + case TK_ISNULL: zOp = "ISNULL"; break; + case TK_NOTNULL: zOp = "NOTNULL"; break; + + default: + sqlite3_str_appendf(p, "%s", "expr"); + break; + } + + if( zOp ){ + sqlite3_str_appendf(p, "%s(", zOp); + displayP4Expr(p, pExpr->pLeft); + if( pExpr->pRight ){ + sqlite3_str_append(p, ",", 1); + displayP4Expr(p, pExpr->pRight); + } + sqlite3_str_append(p, ")", 1); + } +} +#endif /* VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) */ + + +#if VDBE_DISPLAY_P4 +/* +** Compute a string that describes the P4 parameter for an opcode. +** Use zTemp for any required temporary buffer space. +*/ +SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ + char *zP4 = 0; + StrAccum x; + + sqlite3StrAccumInit(&x, 0, 0, 0, SQLITE_MAX_LENGTH); + switch( pOp->p4type ){ + case P4_KEYINFO: { + int j; + KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; + assert( pKeyInfo->aSortFlags!=0 ); + sqlite3_str_appendf(&x, "k(%d", pKeyInfo->nKeyField); + for(j=0; jnKeyField; j++){ + CollSeq *pColl = pKeyInfo->aColl[j]; + const char *zColl = pColl ? pColl->zName : ""; + if( strcmp(zColl, "BINARY")==0 ) zColl = "B"; + sqlite3_str_appendf(&x, ",%s%s%s", + (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_DESC) ? "-" : "", + (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_BIGNULL)? "N." : "", + zColl); + } + sqlite3_str_append(&x, ")", 1); + break; + } +#ifdef SQLITE_ENABLE_CURSOR_HINTS + case P4_EXPR: { + displayP4Expr(&x, pOp->p4.pExpr); + break; + } +#endif + case P4_COLLSEQ: { + static const char *const encnames[] = {"?", "8", "16LE", "16BE"}; + CollSeq *pColl = pOp->p4.pColl; + assert( pColl->enc<4 ); + sqlite3_str_appendf(&x, "%.18s-%s", pColl->zName, + encnames[pColl->enc]); + break; + } + case P4_FUNCDEF: { + FuncDef *pDef = pOp->p4.pFunc; + sqlite3_str_appendf(&x, "%s(%d)", pDef->zName, pDef->nArg); + break; + } + case P4_FUNCCTX: { + FuncDef *pDef = pOp->p4.pCtx->pFunc; + sqlite3_str_appendf(&x, "%s(%d)", pDef->zName, pDef->nArg); + break; + } + case P4_INT64: { + sqlite3_str_appendf(&x, "%lld", *pOp->p4.pI64); + break; + } + case P4_INT32: { + sqlite3_str_appendf(&x, "%d", pOp->p4.i); + break; + } + case P4_REAL: { + sqlite3_str_appendf(&x, "%.16g", *pOp->p4.pReal); + break; + } + case P4_MEM: { + Mem *pMem = pOp->p4.pMem; + if( pMem->flags & MEM_Str ){ + zP4 = pMem->z; + }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + sqlite3_str_appendf(&x, "%lld", pMem->u.i); + }else if( pMem->flags & MEM_Real ){ + sqlite3_str_appendf(&x, "%.16g", pMem->u.r); + }else if( pMem->flags & MEM_Null ){ + zP4 = "NULL"; + }else{ + assert( pMem->flags & MEM_Blob ); + zP4 = "(blob)"; + } + break; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + case P4_VTAB: { + sqlite3_vtab *pVtab = pOp->p4.pVtab->pVtab; + sqlite3_str_appendf(&x, "vtab:%p", pVtab); + break; + } +#endif + case P4_INTARRAY: { + u32 i; + u32 *ai = pOp->p4.ai; + u32 n = ai[0]; /* The first element of an INTARRAY is always the + ** count of the number of elements to follow */ + for(i=1; i<=n; i++){ + sqlite3_str_appendf(&x, "%c%u", (i==1 ? '[' : ','), ai[i]); + } + sqlite3_str_append(&x, "]", 1); + break; + } + case P4_SUBPROGRAM: { + zP4 = "program"; + break; + } + case P4_DYNBLOB: + case P4_ADVANCE: { + break; + } + case P4_TABLE: { + zP4 = pOp->p4.pTab->zName; + break; + } + default: { + zP4 = pOp->p4.z; + } + } + if( zP4 ) sqlite3_str_appendall(&x, zP4); + if( (x.accError & SQLITE_NOMEM)!=0 ){ + sqlite3OomFault(db); + } + return sqlite3StrAccumFinish(&x); +} +#endif /* VDBE_DISPLAY_P4 */ + +/* +** Declare to the Vdbe that the BTree object at db->aDb[i] is used. +** +** The prepared statements need to know in advance the complete set of +** attached databases that will be use. A mask of these databases +** is maintained in p->btreeMask. The p->lockMask value is the subset of +** p->btreeMask of databases that will require a lock. +*/ +SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe *p, int i){ + assert( i>=0 && idb->nDb && i<(int)sizeof(yDbMask)*8 ); + assert( i<(int)sizeof(p->btreeMask)*8 ); + DbMaskSet(p->btreeMask, i); + if( i!=1 && sqlite3BtreeSharable(p->db->aDb[i].pBt) ){ + DbMaskSet(p->lockMask, i); + } +} + +#if !defined(SQLITE_OMIT_SHARED_CACHE) +/* +** If SQLite is compiled to support shared-cache mode and to be threadsafe, +** this routine obtains the mutex associated with each BtShared structure +** that may be accessed by the VM passed as an argument. In doing so it also +** sets the BtShared.db member of each of the BtShared structures, ensuring +** that the correct busy-handler callback is invoked if required. +** +** If SQLite is not threadsafe but does support shared-cache mode, then +** sqlite3BtreeEnter() is invoked to set the BtShared.db variables +** of all of BtShared structures accessible via the database handle +** associated with the VM. +** +** If SQLite is not threadsafe and does not support shared-cache mode, this +** function is a no-op. +** +** The p->btreeMask field is a bitmask of all btrees that the prepared +** statement p will ever use. Let N be the number of bits in p->btreeMask +** corresponding to btrees that use shared cache. Then the runtime of +** this routine is N*N. But as N is rarely more than 1, this should not +** be a problem. +*/ +SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe *p){ + int i; + sqlite3 *db; + Db *aDb; + int nDb; + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ + db = p->db; + aDb = db->aDb; + nDb = db->nDb; + for(i=0; ilockMask,i) && ALWAYS(aDb[i].pBt!=0) ){ + sqlite3BtreeEnter(aDb[i].pBt); + } + } +} +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 +/* +** Unlock all of the btrees previously locked by a call to sqlite3VdbeEnter(). +*/ +static SQLITE_NOINLINE void vdbeLeave(Vdbe *p){ + int i; + sqlite3 *db; + Db *aDb; + int nDb; + db = p->db; + aDb = db->aDb; + nDb = db->nDb; + for(i=0; ilockMask,i) && ALWAYS(aDb[i].pBt!=0) ){ + sqlite3BtreeLeave(aDb[i].pBt); + } + } +} +SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ + vdbeLeave(p); +} +#endif + +#if defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) +/* +** Print a single opcode. This routine is used for debugging only. +*/ +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, VdbeOp *pOp){ + char *zP4; + char *zCom; + sqlite3 dummyDb; + static const char *zFormat1 = "%4d %-13s %4d %4d %4d %-13s %.2X %s\n"; + if( pOut==0 ) pOut = stdout; + sqlite3BeginBenignMalloc(); + dummyDb.mallocFailed = 1; + zP4 = sqlite3VdbeDisplayP4(&dummyDb, pOp); +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + zCom = sqlite3VdbeDisplayComment(0, pOp, zP4); +#else + zCom = 0; +#endif + /* NB: The sqlite3OpcodeName() function is implemented by code created + ** by the mkopcodeh.awk and mkopcodec.awk scripts which extract the + ** information from the vdbe.c source text */ + fprintf(pOut, zFormat1, pc, + sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3, + zP4 ? zP4 : "", pOp->p5, + zCom ? zCom : "" + ); + fflush(pOut); + sqlite3_free(zP4); + sqlite3_free(zCom); + sqlite3EndBenignMalloc(); +} +#endif + +/* +** Initialize an array of N Mem element. +*/ +static void initMemArray(Mem *p, int N, sqlite3 *db, u16 flags){ + while( (N--)>0 ){ + p->db = db; + p->flags = flags; + p->szMalloc = 0; +#ifdef SQLITE_DEBUG + p->pScopyFrom = 0; +#endif + p++; + } +} + +/* +** Release an array of N Mem elements +*/ +static void releaseMemArray(Mem *p, int N){ + if( p && N ){ + Mem *pEnd = &p[N]; + sqlite3 *db = p->db; + if( db->pnBytesFreed ){ + do{ + if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); + }while( (++p)flags & MEM_Agg ); + testcase( p->flags & MEM_Dyn ); + testcase( p->xDel==sqlite3VdbeFrameMemDel ); + if( p->flags&(MEM_Agg|MEM_Dyn) ){ + sqlite3VdbeMemRelease(p); + }else if( p->szMalloc ){ + sqlite3DbFreeNN(db, p->zMalloc); + p->szMalloc = 0; + } + + p->flags = MEM_Undefined; + }while( (++p)iFrameMagic!=SQLITE_FRAME_MAGIC ) return 0; + return 1; +} +#endif + + +/* +** This is a destructor on a Mem object (which is really an sqlite3_value) +** that deletes the Frame object that is attached to it as a blob. +** +** This routine does not delete the Frame right away. It merely adds the +** frame to a list of frames to be deleted when the Vdbe halts. +*/ +SQLITE_PRIVATE void sqlite3VdbeFrameMemDel(void *pArg){ + VdbeFrame *pFrame = (VdbeFrame*)pArg; + assert( sqlite3VdbeFrameIsValid(pFrame) ); + pFrame->pParent = pFrame->v->pDelFrame; + pFrame->v->pDelFrame = pFrame; +} + +#if defined(SQLITE_ENABLE_BYTECODE_VTAB) || !defined(SQLITE_OMIT_EXPLAIN) +/* +** Locate the next opcode to be displayed in EXPLAIN or EXPLAIN +** QUERY PLAN output. +** +** Return SQLITE_ROW on success. Return SQLITE_DONE if there are no +** more opcodes to be displayed. +*/ +SQLITE_PRIVATE int sqlite3VdbeNextOpcode( + Vdbe *p, /* The statement being explained */ + Mem *pSub, /* Storage for keeping track of subprogram nesting */ + int eMode, /* 0: normal. 1: EQP. 2: TablesUsed */ + int *piPc, /* IN/OUT: Current rowid. Overwritten with next rowid */ + int *piAddr, /* OUT: Write index into (*paOp)[] here */ + Op **paOp /* OUT: Write the opcode array here */ +){ + int nRow; /* Stop when row count reaches this */ + int nSub = 0; /* Number of sub-vdbes seen so far */ + SubProgram **apSub = 0; /* Array of sub-vdbes */ + int i; /* Next instruction address */ + int rc = SQLITE_OK; /* Result code */ + Op *aOp = 0; /* Opcode array */ + int iPc; /* Rowid. Copy of value in *piPc */ + + /* When the number of output rows reaches nRow, that means the + ** listing has finished and sqlite3_step() should return SQLITE_DONE. + ** nRow is the sum of the number of rows in the main program, plus + ** the sum of the number of rows in all trigger subprograms encountered + ** so far. The nRow value will increase as new trigger subprograms are + ** encountered, but p->pc will eventually catch up to nRow. + */ + nRow = p->nOp; + if( pSub!=0 ){ + if( pSub->flags&MEM_Blob ){ + /* pSub is initiallly NULL. It is initialized to a BLOB by + ** the P4_SUBPROGRAM processing logic below */ + nSub = pSub->n/sizeof(Vdbe*); + apSub = (SubProgram **)pSub->z; + } + for(i=0; inOp; + } + } + iPc = *piPc; + while(1){ /* Loop exits via break */ + i = iPc++; + if( i>=nRow ){ + p->rc = SQLITE_OK; + rc = SQLITE_DONE; + break; + } + if( inOp ){ + /* The rowid is small enough that we are still in the + ** main program. */ + aOp = p->aOp; + }else{ + /* We are currently listing subprograms. Figure out which one and + ** pick up the appropriate opcode. */ + int j; + i -= p->nOp; + assert( apSub!=0 ); + assert( nSub>0 ); + for(j=0; i>=apSub[j]->nOp; j++){ + i -= apSub[j]->nOp; + assert( inOp || j+1aOp; + } + + /* When an OP_Program opcode is encounter (the only opcode that has + ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms + ** kept in p->aMem[9].z to hold the new program - assuming this subprogram + ** has not already been seen. + */ + if( pSub!=0 && aOp[i].p4type==P4_SUBPROGRAM ){ + int nByte = (nSub+1)*sizeof(SubProgram*); + int j; + for(j=0; jrc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; + break; + } + apSub = (SubProgram **)pSub->z; + apSub[nSub++] = aOp[i].p4.pProgram; + MemSetTypeFlag(pSub, MEM_Blob); + pSub->n = nSub*sizeof(SubProgram*); + nRow += aOp[i].p4.pProgram->nOp; + } + } + if( eMode==0 ) break; +#ifdef SQLITE_ENABLE_BYTECODE_VTAB + if( eMode==2 ){ + Op *pOp = aOp + i; + if( pOp->opcode==OP_OpenRead ) break; + if( pOp->opcode==OP_OpenWrite && (pOp->p5 & OPFLAG_P2ISREG)==0 ) break; + if( pOp->opcode==OP_ReopenIdx ) break; + }else +#endif + { + assert( eMode==1 ); + if( aOp[i].opcode==OP_Explain ) break; + if( aOp[i].opcode==OP_Init && iPc>1 ) break; + } + } + *piPc = iPc; + *piAddr = i; + *paOp = aOp; + return rc; +} +#endif /* SQLITE_ENABLE_BYTECODE_VTAB || !SQLITE_OMIT_EXPLAIN */ + + +/* +** Delete a VdbeFrame object and its contents. VdbeFrame objects are +** allocated by the OP_Program opcode in sqlite3VdbeExec(). +*/ +SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ + int i; + Mem *aMem = VdbeFrameMem(p); + VdbeCursor **apCsr = (VdbeCursor **)&aMem[p->nChildMem]; + assert( sqlite3VdbeFrameIsValid(p) ); + for(i=0; inChildCsr; i++){ + sqlite3VdbeFreeCursor(p->v, apCsr[i]); + } + releaseMemArray(aMem, p->nChildMem); + sqlite3VdbeDeleteAuxData(p->v->db, &p->pAuxData, -1, 0); + sqlite3DbFree(p->v->db, p); +} + +#ifndef SQLITE_OMIT_EXPLAIN +/* +** Give a listing of the program in the virtual machine. +** +** The interface is the same as sqlite3VdbeExec(). But instead of +** running the code, it invokes the callback once for each instruction. +** This feature is used to implement "EXPLAIN". +** +** When p->explain==1, each instruction is listed. When +** p->explain==2, only OP_Explain instructions are listed and these +** are shown in a different format. p->explain==2 is used to implement +** EXPLAIN QUERY PLAN. +** 2018-04-24: In p->explain==2 mode, the OP_Init opcodes of triggers +** are also shown, so that the boundaries between the main program and +** each trigger are clear. +** +** When p->explain==1, first the main program is listed, then each of +** the trigger subprograms are listed one by one. +*/ +SQLITE_PRIVATE int sqlite3VdbeList( + Vdbe *p /* The VDBE */ +){ + Mem *pSub = 0; /* Memory cell hold array of subprogs */ + sqlite3 *db = p->db; /* The database connection */ + int i; /* Loop counter */ + int rc = SQLITE_OK; /* Return code */ + Mem *pMem = &p->aMem[1]; /* First Mem of result set */ + int bListSubprogs = (p->explain==1 || (db->flags & SQLITE_TriggerEQP)!=0); + Op *aOp; /* Array of opcodes */ + Op *pOp; /* Current opcode */ + + assert( p->explain ); + assert( p->iVdbeMagic==VDBE_MAGIC_RUN ); + assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY || p->rc==SQLITE_NOMEM ); + + /* Even though this opcode does not use dynamic strings for + ** the result, result columns may become dynamic if the user calls + ** sqlite3_column_text16(), causing a translation to UTF-16 encoding. + */ + releaseMemArray(pMem, 8); + p->pResultSet = 0; + + if( p->rc==SQLITE_NOMEM ){ + /* This happens if a malloc() inside a call to sqlite3_column_text() or + ** sqlite3_column_text16() failed. */ + sqlite3OomFault(db); + return SQLITE_ERROR; + } + + if( bListSubprogs ){ + /* The first 8 memory cells are used for the result set. So we will + ** commandeer the 9th cell to use as storage for an array of pointers + ** to trigger subprograms. The VDBE is guaranteed to have at least 9 + ** cells. */ + assert( p->nMem>9 ); + pSub = &p->aMem[9]; + }else{ + pSub = 0; + } + + /* Figure out which opcode is next to display */ + rc = sqlite3VdbeNextOpcode(p, pSub, p->explain==2, &p->pc, &i, &aOp); + + if( rc==SQLITE_OK ){ + pOp = aOp + i; + if( AtomicLoad(&db->u1.isInterrupted) ){ + p->rc = SQLITE_INTERRUPT; + rc = SQLITE_ERROR; + sqlite3VdbeError(p, sqlite3ErrStr(p->rc)); + }else{ + char *zP4 = sqlite3VdbeDisplayP4(db, pOp); + if( p->explain==2 ){ + sqlite3VdbeMemSetInt64(pMem, pOp->p1); + sqlite3VdbeMemSetInt64(pMem+1, pOp->p2); + sqlite3VdbeMemSetInt64(pMem+2, pOp->p3); + sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free); + p->nResColumn = 4; + }else{ + sqlite3VdbeMemSetInt64(pMem+0, i); + sqlite3VdbeMemSetStr(pMem+1, (char*)sqlite3OpcodeName(pOp->opcode), + -1, SQLITE_UTF8, SQLITE_STATIC); + sqlite3VdbeMemSetInt64(pMem+2, pOp->p1); + sqlite3VdbeMemSetInt64(pMem+3, pOp->p2); + sqlite3VdbeMemSetInt64(pMem+4, pOp->p3); + /* pMem+5 for p4 is done last */ + sqlite3VdbeMemSetInt64(pMem+6, pOp->p5); +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + { + char *zCom = sqlite3VdbeDisplayComment(db, pOp, zP4); + sqlite3VdbeMemSetStr(pMem+7, zCom, -1, SQLITE_UTF8, sqlite3_free); + } +#else + sqlite3VdbeMemSetNull(pMem+7); +#endif + sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free); + p->nResColumn = 8; + } + p->pResultSet = pMem; + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM; + rc = SQLITE_ERROR; + }else{ + p->rc = SQLITE_OK; + rc = SQLITE_ROW; + } + } + } + return rc; +} +#endif /* SQLITE_OMIT_EXPLAIN */ + +#ifdef SQLITE_DEBUG +/* +** Print the SQL that was used to generate a VDBE program. +*/ +SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe *p){ + const char *z = 0; + if( p->zSql ){ + z = p->zSql; + }else if( p->nOp>=1 ){ + const VdbeOp *pOp = &p->aOp[0]; + if( pOp->opcode==OP_Init && pOp->p4.z!=0 ){ + z = pOp->p4.z; + while( sqlite3Isspace(*z) ) z++; + } + } + if( z ) printf("SQL: [%s]\n", z); +} +#endif + +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) +/* +** Print an IOTRACE message showing SQL content. +*/ +SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe *p){ + int nOp = p->nOp; + VdbeOp *pOp; + if( sqlite3IoTrace==0 ) return; + if( nOp<1 ) return; + pOp = &p->aOp[0]; + if( pOp->opcode==OP_Init && pOp->p4.z!=0 ){ + int i, j; + char z[1000]; + sqlite3_snprintf(sizeof(z), z, "%s", pOp->p4.z); + for(i=0; sqlite3Isspace(z[i]); i++){} + for(j=0; z[i]; i++){ + if( sqlite3Isspace(z[i]) ){ + if( z[i-1]!=' ' ){ + z[j++] = ' '; + } + }else{ + z[j++] = z[i]; + } + } + z[j] = 0; + sqlite3IoTrace("SQL %s\n", z); + } +} +#endif /* !SQLITE_OMIT_TRACE && SQLITE_ENABLE_IOTRACE */ + +/* An instance of this object describes bulk memory available for use +** by subcomponents of a prepared statement. Space is allocated out +** of a ReusableSpace object by the allocSpace() routine below. +*/ +struct ReusableSpace { + u8 *pSpace; /* Available memory */ + sqlite3_int64 nFree; /* Bytes of available memory */ + sqlite3_int64 nNeeded; /* Total bytes that could not be allocated */ +}; + +/* Try to allocate nByte bytes of 8-byte aligned bulk memory for pBuf +** from the ReusableSpace object. Return a pointer to the allocated +** memory on success. If insufficient memory is available in the +** ReusableSpace object, increase the ReusableSpace.nNeeded +** value by the amount needed and return NULL. +** +** If pBuf is not initially NULL, that means that the memory has already +** been allocated by a prior call to this routine, so just return a copy +** of pBuf and leave ReusableSpace unchanged. +** +** This allocator is employed to repurpose unused slots at the end of the +** opcode array of prepared state for other memory needs of the prepared +** statement. +*/ +static void *allocSpace( + struct ReusableSpace *p, /* Bulk memory available for allocation */ + void *pBuf, /* Pointer to a prior allocation */ + sqlite3_int64 nByte /* Bytes of memory needed */ +){ + assert( EIGHT_BYTE_ALIGNMENT(p->pSpace) ); + if( pBuf==0 ){ + nByte = ROUND8(nByte); + if( nByte <= p->nFree ){ + p->nFree -= nByte; + pBuf = &p->pSpace[p->nFree]; + }else{ + p->nNeeded += nByte; + } + } + assert( EIGHT_BYTE_ALIGNMENT(pBuf) ); + return pBuf; +} + +/* +** Rewind the VDBE back to the beginning in preparation for +** running it. +*/ +SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + int i; +#endif + assert( p!=0 ); + assert( p->iVdbeMagic==VDBE_MAGIC_INIT || p->iVdbeMagic==VDBE_MAGIC_RESET ); + + /* There should be at least one opcode. + */ + assert( p->nOp>0 ); + + /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. */ + p->iVdbeMagic = VDBE_MAGIC_RUN; + +#ifdef SQLITE_DEBUG + for(i=0; inMem; i++){ + assert( p->aMem[i].db==p->db ); + } +#endif + p->pc = -1; + p->rc = SQLITE_OK; + p->errorAction = OE_Abort; + p->nChange = 0; + p->cacheCtr = 1; + p->minWriteFileFormat = 255; + p->iStatement = 0; + p->nFkConstraint = 0; +#ifdef VDBE_PROFILE + for(i=0; inOp; i++){ + p->aOp[i].cnt = 0; + p->aOp[i].cycles = 0; + } +#endif +} + +/* +** Prepare a virtual machine for execution for the first time after +** creating the virtual machine. This involves things such +** as allocating registers and initializing the program counter. +** After the VDBE has be prepped, it can be executed by one or more +** calls to sqlite3VdbeExec(). +** +** This function may be called exactly once on each virtual machine. +** After this routine is called the VM has been "packaged" and is ready +** to run. After this routine is called, further calls to +** sqlite3VdbeAddOp() functions are prohibited. This routine disconnects +** the Vdbe from the Parse object that helped generate it so that the +** the Vdbe becomes an independent entity and the Parse object can be +** destroyed. +** +** Use the sqlite3VdbeRewind() procedure to restore a virtual machine back +** to its initial state after it has been run. +*/ +SQLITE_PRIVATE void sqlite3VdbeMakeReady( + Vdbe *p, /* The VDBE */ + Parse *pParse /* Parsing context */ +){ + sqlite3 *db; /* The database connection */ + int nVar; /* Number of parameters */ + int nMem; /* Number of VM memory registers */ + int nCursor; /* Number of cursors required */ + int nArg; /* Number of arguments in subprograms */ + int n; /* Loop counter */ + struct ReusableSpace x; /* Reusable bulk memory */ + + assert( p!=0 ); + assert( p->nOp>0 ); + assert( pParse!=0 ); + assert( p->iVdbeMagic==VDBE_MAGIC_INIT ); + assert( pParse==p->pParse ); + p->pVList = pParse->pVList; + pParse->pVList = 0; + db = p->db; + assert( db->mallocFailed==0 ); + nVar = pParse->nVar; + nMem = pParse->nMem; + nCursor = pParse->nTab; + nArg = pParse->nMaxArg; + + /* Each cursor uses a memory cell. The first cursor (cursor 0) can + ** use aMem[0] which is not otherwise used by the VDBE program. Allocate + ** space at the end of aMem[] for cursors 1 and greater. + ** See also: allocateCursor(). + */ + nMem += nCursor; + if( nCursor==0 && nMem>0 ) nMem++; /* Space for aMem[0] even if not used */ + + /* Figure out how much reusable memory is available at the end of the + ** opcode array. This extra memory will be reallocated for other elements + ** of the prepared statement. + */ + n = ROUND8(sizeof(Op)*p->nOp); /* Bytes of opcode memory used */ + x.pSpace = &((u8*)p->aOp)[n]; /* Unused opcode memory */ + assert( EIGHT_BYTE_ALIGNMENT(x.pSpace) ); + x.nFree = ROUNDDOWN8(pParse->szOpAlloc - n); /* Bytes of unused memory */ + assert( x.nFree>=0 ); + assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) ); + + resolveP2Values(p, &nArg); + p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); + if( pParse->explain ){ + static const char * const azColName[] = { + "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", + "id", "parent", "notused", "detail" + }; + int iFirst, mx, i; + if( nMem<10 ) nMem = 10; + p->explain = pParse->explain; + if( pParse->explain==2 ){ + sqlite3VdbeSetNumCols(p, 4); + iFirst = 8; + mx = 12; + }else{ + sqlite3VdbeSetNumCols(p, 8); + iFirst = 0; + mx = 8; + } + for(i=iFirst; iexpired = 0; + + /* Memory for registers, parameters, cursor, etc, is allocated in one or two + ** passes. On the first pass, we try to reuse unused memory at the + ** end of the opcode array. If we are unable to satisfy all memory + ** requirements by reusing the opcode array tail, then the second + ** pass will fill in the remainder using a fresh memory allocation. + ** + ** This two-pass approach that reuses as much memory as possible from + ** the leftover memory at the end of the opcode array. This can significantly + ** reduce the amount of memory held by a prepared statement. + */ + x.nNeeded = 0; + p->aMem = allocSpace(&x, 0, nMem*sizeof(Mem)); + p->aVar = allocSpace(&x, 0, nVar*sizeof(Mem)); + p->apArg = allocSpace(&x, 0, nArg*sizeof(Mem*)); + p->apCsr = allocSpace(&x, 0, nCursor*sizeof(VdbeCursor*)); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = allocSpace(&x, 0, p->nOp*sizeof(i64)); +#endif + if( x.nNeeded ){ + x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); + x.nFree = x.nNeeded; + if( !db->mallocFailed ){ + p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); + p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); + p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); + p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); +#endif + } + } + + if( db->mallocFailed ){ + p->nVar = 0; + p->nCursor = 0; + p->nMem = 0; + }else{ + p->nCursor = nCursor; + p->nVar = (ynVar)nVar; + initMemArray(p->aVar, nVar, db, MEM_Null); + p->nMem = nMem; + initMemArray(p->aMem, nMem, db, MEM_Undefined); + memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*)); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + memset(p->anExec, 0, p->nOp*sizeof(i64)); +#endif + } + sqlite3VdbeRewind(p); +} + +/* +** Close a VDBE cursor and release all the resources that cursor +** happens to hold. +*/ +SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ + if( pCx==0 ){ + return; + } + assert( pCx->pBtx==0 || pCx->eCurType==CURTYPE_BTREE ); + assert( pCx->pBtx==0 || pCx->isEphemeral ); + switch( pCx->eCurType ){ + case CURTYPE_SORTER: { + sqlite3VdbeSorterClose(p->db, pCx); + break; + } + case CURTYPE_BTREE: { + assert( pCx->uc.pCursor!=0 ); + sqlite3BtreeCloseCursor(pCx->uc.pCursor); + break; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + case CURTYPE_VTAB: { + sqlite3_vtab_cursor *pVCur = pCx->uc.pVCur; + const sqlite3_module *pModule = pVCur->pVtab->pModule; + assert( pVCur->pVtab->nRef>0 ); + pVCur->pVtab->nRef--; + pModule->xClose(pVCur); + break; + } +#endif + } +} + +/* +** Close all cursors in the current frame. +*/ +static void closeCursorsInFrame(Vdbe *p){ + if( p->apCsr ){ + int i; + for(i=0; inCursor; i++){ + VdbeCursor *pC = p->apCsr[i]; + if( pC ){ + sqlite3VdbeFreeCursor(p, pC); + p->apCsr[i] = 0; + } + } + } +} + +/* +** Copy the values stored in the VdbeFrame structure to its Vdbe. This +** is used, for example, when a trigger sub-program is halted to restore +** control to the main program. +*/ +SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ + Vdbe *v = pFrame->v; + closeCursorsInFrame(v); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + v->anExec = pFrame->anExec; +#endif + v->aOp = pFrame->aOp; + v->nOp = pFrame->nOp; + v->aMem = pFrame->aMem; + v->nMem = pFrame->nMem; + v->apCsr = pFrame->apCsr; + v->nCursor = pFrame->nCursor; + v->db->lastRowid = pFrame->lastRowid; + v->nChange = pFrame->nChange; + v->db->nChange = pFrame->nDbChange; + sqlite3VdbeDeleteAuxData(v->db, &v->pAuxData, -1, 0); + v->pAuxData = pFrame->pAuxData; + pFrame->pAuxData = 0; + return pFrame->pc; +} + +/* +** Close all cursors. +** +** Also release any dynamic memory held by the VM in the Vdbe.aMem memory +** cell array. This is necessary as the memory cell array may contain +** pointers to VdbeFrame objects, which may in turn contain pointers to +** open cursors. +*/ +static void closeAllCursors(Vdbe *p){ + if( p->pFrame ){ + VdbeFrame *pFrame; + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + sqlite3VdbeFrameRestore(pFrame); + p->pFrame = 0; + p->nFrame = 0; + } + assert( p->nFrame==0 ); + closeCursorsInFrame(p); + if( p->aMem ){ + releaseMemArray(p->aMem, p->nMem); + } + while( p->pDelFrame ){ + VdbeFrame *pDel = p->pDelFrame; + p->pDelFrame = pDel->pParent; + sqlite3VdbeFrameDelete(pDel); + } + + /* Delete any auxdata allocations made by the VM */ + if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p->db, &p->pAuxData, -1, 0); + assert( p->pAuxData==0 ); +} + +/* +** Set the number of result columns that will be returned by this SQL +** statement. This is now set at compile time, rather than during +** execution of the vdbe program so that sqlite3_column_count() can +** be called on an SQL statement before sqlite3_step(). +*/ +SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){ + int n; + sqlite3 *db = p->db; + + if( p->nResColumn ){ + releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + sqlite3DbFree(db, p->aColName); + } + n = nResColumn*COLNAME_N; + p->nResColumn = (u16)nResColumn; + p->aColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n ); + if( p->aColName==0 ) return; + initMemArray(p->aColName, n, db, MEM_Null); +} + +/* +** Set the name of the idx'th column to be returned by the SQL statement. +** zName must be a pointer to a nul terminated string. +** +** This call must be made after a call to sqlite3VdbeSetNumCols(). +** +** The final parameter, xDel, must be one of SQLITE_DYNAMIC, SQLITE_STATIC +** or SQLITE_TRANSIENT. If it is SQLITE_DYNAMIC, then the buffer pointed +** to by zName will be freed by sqlite3DbFree() when the vdbe is destroyed. +*/ +SQLITE_PRIVATE int sqlite3VdbeSetColName( + Vdbe *p, /* Vdbe being configured */ + int idx, /* Index of column zName applies to */ + int var, /* One of the COLNAME_* constants */ + const char *zName, /* Pointer to buffer containing name */ + void (*xDel)(void*) /* Memory management strategy for zName */ +){ + int rc; + Mem *pColName; + assert( idxnResColumn ); + assert( vardb->mallocFailed ){ + assert( !zName || xDel!=SQLITE_DYNAMIC ); + return SQLITE_NOMEM_BKPT; + } + assert( p->aColName!=0 ); + pColName = &(p->aColName[idx+var*p->nResColumn]); + rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); + assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); + return rc; +} + +/* +** A read or write transaction may or may not be active on database handle +** db. If a transaction is active, commit it. If there is a +** write-transaction spanning more than one database file, this routine +** takes care of the super-journal trickery. +*/ +static int vdbeCommit(sqlite3 *db, Vdbe *p){ + int i; + int nTrans = 0; /* Number of databases with an active write-transaction + ** that are candidates for a two-phase commit using a + ** super-journal */ + int rc = SQLITE_OK; + int needXcommit = 0; + +#ifdef SQLITE_OMIT_VIRTUALTABLE + /* With this option, sqlite3VtabSync() is defined to be simply + ** SQLITE_OK so p is not used. + */ + UNUSED_PARAMETER(p); +#endif + + /* Before doing anything else, call the xSync() callback for any + ** virtual module tables written in this transaction. This has to + ** be done before determining whether a super-journal file is + ** required, as an xSync() callback may add an attached database + ** to the transaction. + */ + rc = sqlite3VtabSync(db, p); + + /* This loop determines (a) if the commit hook should be invoked and + ** (b) how many database files have open write transactions, not + ** including the temp database. (b) is important because if more than + ** one database file has an open write transaction, a super-journal + ** file is required for an atomic commit. + */ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){ + /* Whether or not a database might need a super-journal depends upon + ** its journal mode (among other things). This matrix determines which + ** journal modes use a super-journal and which do not */ + static const u8 aMJNeeded[] = { + /* DELETE */ 1, + /* PERSIST */ 1, + /* OFF */ 0, + /* TRUNCATE */ 1, + /* MEMORY */ 0, + /* WAL */ 0 + }; + Pager *pPager; /* Pager associated with pBt */ + needXcommit = 1; + sqlite3BtreeEnter(pBt); + pPager = sqlite3BtreePager(pBt); + if( db->aDb[i].safety_level!=PAGER_SYNCHRONOUS_OFF + && aMJNeeded[sqlite3PagerGetJournalMode(pPager)] + && sqlite3PagerIsMemdb(pPager)==0 + ){ + assert( i!=1 ); + nTrans++; + } + rc = sqlite3PagerExclusiveLock(pPager); + sqlite3BtreeLeave(pBt); + } + } + if( rc!=SQLITE_OK ){ + return rc; + } + + /* If there are any write-transactions at all, invoke the commit hook */ + if( needXcommit && db->xCommitCallback ){ + rc = db->xCommitCallback(db->pCommitArg); + if( rc ){ + return SQLITE_CONSTRAINT_COMMITHOOK; + } + } + + /* The simple case - no more than one database file (not counting the + ** TEMP database) has a transaction active. There is no need for the + ** super-journal. + ** + ** If the return value of sqlite3BtreeGetFilename() is a zero length + ** string, it means the main database is :memory: or a temp file. In + ** that case we do not support atomic multi-file commits, so use the + ** simple case then too. + */ + if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(db->aDb[0].pBt)) + || nTrans<=1 + ){ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + rc = sqlite3BtreeCommitPhaseOne(pBt, 0); + } + } + + /* Do the commit only if all databases successfully complete phase 1. + ** If one of the BtreeCommitPhaseOne() calls fails, this indicates an + ** IO error while deleting or truncating a journal file. It is unlikely, + ** but could happen. In this case abandon processing and return the error. + */ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + rc = sqlite3BtreeCommitPhaseTwo(pBt, 0); + } + } + if( rc==SQLITE_OK ){ + sqlite3VtabCommit(db); + } + } + + /* The complex case - There is a multi-file write-transaction active. + ** This requires a super-journal file to ensure the transaction is + ** committed atomically. + */ +#ifndef SQLITE_OMIT_DISKIO + else{ + sqlite3_vfs *pVfs = db->pVfs; + char *zSuper = 0; /* File-name for the super-journal */ + char const *zMainFile = sqlite3BtreeGetFilename(db->aDb[0].pBt); + sqlite3_file *pSuperJrnl = 0; + i64 offset = 0; + int res; + int retryCount = 0; + int nMainFile; + + /* Select a super-journal file name */ + nMainFile = sqlite3Strlen30(zMainFile); + zSuper = sqlite3MPrintf(db, "%.4c%s%.16c", 0,zMainFile,0); + if( zSuper==0 ) return SQLITE_NOMEM_BKPT; + zSuper += 4; + do { + u32 iRandom; + if( retryCount ){ + if( retryCount>100 ){ + sqlite3_log(SQLITE_FULL, "MJ delete: %s", zSuper); + sqlite3OsDelete(pVfs, zSuper, 0); + break; + }else if( retryCount==1 ){ + sqlite3_log(SQLITE_FULL, "MJ collide: %s", zSuper); + } + } + retryCount++; + sqlite3_randomness(sizeof(iRandom), &iRandom); + sqlite3_snprintf(13, &zSuper[nMainFile], "-mj%06X9%02X", + (iRandom>>8)&0xffffff, iRandom&0xff); + /* The antipenultimate character of the super-journal name must + ** be "9" to avoid name collisions when using 8+3 filenames. */ + assert( zSuper[sqlite3Strlen30(zSuper)-3]=='9' ); + sqlite3FileSuffix3(zMainFile, zSuper); + rc = sqlite3OsAccess(pVfs, zSuper, SQLITE_ACCESS_EXISTS, &res); + }while( rc==SQLITE_OK && res ); + if( rc==SQLITE_OK ){ + /* Open the super-journal. */ + rc = sqlite3OsOpenMalloc(pVfs, zSuper, &pSuperJrnl, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| + SQLITE_OPEN_EXCLUSIVE|SQLITE_OPEN_SUPER_JOURNAL, 0 + ); + } + if( rc!=SQLITE_OK ){ + sqlite3DbFree(db, zSuper-4); + return rc; + } + + /* Write the name of each database file in the transaction into the new + ** super-journal file. If an error occurs at this point close + ** and delete the super-journal file. All the individual journal files + ** still have 'null' as the super-journal pointer, so they will roll + ** back independently if a failure occurs. + */ + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){ + char const *zFile = sqlite3BtreeGetJournalname(pBt); + if( zFile==0 ){ + continue; /* Ignore TEMP and :memory: databases */ + } + assert( zFile[0]!=0 ); + rc = sqlite3OsWrite(pSuperJrnl, zFile, sqlite3Strlen30(zFile)+1,offset); + offset += sqlite3Strlen30(zFile)+1; + if( rc!=SQLITE_OK ){ + sqlite3OsCloseFree(pSuperJrnl); + sqlite3OsDelete(pVfs, zSuper, 0); + sqlite3DbFree(db, zSuper-4); + return rc; + } + } + } + + /* Sync the super-journal file. If the IOCAP_SEQUENTIAL device + ** flag is set this is not required. + */ + if( 0==(sqlite3OsDeviceCharacteristics(pSuperJrnl)&SQLITE_IOCAP_SEQUENTIAL) + && SQLITE_OK!=(rc = sqlite3OsSync(pSuperJrnl, SQLITE_SYNC_NORMAL)) + ){ + sqlite3OsCloseFree(pSuperJrnl); + sqlite3OsDelete(pVfs, zSuper, 0); + sqlite3DbFree(db, zSuper-4); + return rc; + } + + /* Sync all the db files involved in the transaction. The same call + ** sets the super-journal pointer in each individual journal. If + ** an error occurs here, do not delete the super-journal file. + ** + ** If the error occurs during the first call to + ** sqlite3BtreeCommitPhaseOne(), then there is a chance that the + ** super-journal file will be orphaned. But we cannot delete it, + ** in case the super-journal file name was written into the journal + ** file before the failure occurred. + */ + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + rc = sqlite3BtreeCommitPhaseOne(pBt, zSuper); + } + } + sqlite3OsCloseFree(pSuperJrnl); + assert( rc!=SQLITE_BUSY ); + if( rc!=SQLITE_OK ){ + sqlite3DbFree(db, zSuper-4); + return rc; + } + + /* Delete the super-journal file. This commits the transaction. After + ** doing this the directory is synced again before any individual + ** transaction files are deleted. + */ + rc = sqlite3OsDelete(pVfs, zSuper, 1); + sqlite3DbFree(db, zSuper-4); + zSuper = 0; + if( rc ){ + return rc; + } + + /* All files and directories have already been synced, so the following + ** calls to sqlite3BtreeCommitPhaseTwo() are only closing files and + ** deleting or truncating journals. If something goes wrong while + ** this is happening we don't really care. The integrity of the + ** transaction is already guaranteed, but some stray 'cold' journals + ** may be lying around. Returning an error code won't help matters. + */ + disable_simulated_io_errors(); + sqlite3BeginBenignMalloc(); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + sqlite3BtreeCommitPhaseTwo(pBt, 1); + } + } + sqlite3EndBenignMalloc(); + enable_simulated_io_errors(); + + sqlite3VtabCommit(db); + } +#endif + + return rc; +} + +/* +** This routine checks that the sqlite3.nVdbeActive count variable +** matches the number of vdbe's in the list sqlite3.pVdbe that are +** currently active. An assertion fails if the two counts do not match. +** This is an internal self-check only - it is not an essential processing +** step. +** +** This is a no-op if NDEBUG is defined. +*/ +#ifndef NDEBUG +static void checkActiveVdbeCnt(sqlite3 *db){ + Vdbe *p; + int cnt = 0; + int nWrite = 0; + int nRead = 0; + p = db->pVdbe; + while( p ){ + if( sqlite3_stmt_busy((sqlite3_stmt*)p) ){ + cnt++; + if( p->readOnly==0 ) nWrite++; + if( p->bIsReader ) nRead++; + } + p = p->pNext; + } + assert( cnt==db->nVdbeActive ); + assert( nWrite==db->nVdbeWrite ); + assert( nRead==db->nVdbeRead ); +} +#else +#define checkActiveVdbeCnt(x) +#endif + +/* +** If the Vdbe passed as the first argument opened a statement-transaction, +** close it now. Argument eOp must be either SAVEPOINT_ROLLBACK or +** SAVEPOINT_RELEASE. If it is SAVEPOINT_ROLLBACK, then the statement +** transaction is rolled back. If eOp is SAVEPOINT_RELEASE, then the +** statement transaction is committed. +** +** If an IO error occurs, an SQLITE_IOERR_XXX error code is returned. +** Otherwise SQLITE_OK. +*/ +static SQLITE_NOINLINE int vdbeCloseStatement(Vdbe *p, int eOp){ + sqlite3 *const db = p->db; + int rc = SQLITE_OK; + int i; + const int iSavepoint = p->iStatement-1; + + assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE); + assert( db->nStatement>0 ); + assert( p->iStatement==(db->nStatement+db->nSavepoint) ); + + for(i=0; inDb; i++){ + int rc2 = SQLITE_OK; + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + if( eOp==SAVEPOINT_ROLLBACK ){ + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc2==SQLITE_OK ){ + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint); + } + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + db->nStatement--; + p->iStatement = 0; + + if( rc==SQLITE_OK ){ + if( eOp==SAVEPOINT_ROLLBACK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc==SQLITE_OK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + } + } + + /* If the statement transaction is being rolled back, also restore the + ** database handles deferred constraint counter to the value it had when + ** the statement transaction was opened. */ + if( eOp==SAVEPOINT_ROLLBACK ){ + db->nDeferredCons = p->nStmtDefCons; + db->nDeferredImmCons = p->nStmtDefImmCons; + } + return rc; +} +SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ + if( p->db->nStatement && p->iStatement ){ + return vdbeCloseStatement(p, eOp); + } + return SQLITE_OK; +} + + +/* +** This function is called when a transaction opened by the database +** handle associated with the VM passed as an argument is about to be +** committed. If there are outstanding deferred foreign key constraint +** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK. +** +** If there are outstanding FK violations and this function returns +** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY +** and write an error message to it. Then return SQLITE_ERROR. +*/ +#ifndef SQLITE_OMIT_FOREIGN_KEY +SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ + sqlite3 *db = p->db; + if( (deferred && (db->nDeferredCons+db->nDeferredImmCons)>0) + || (!deferred && p->nFkConstraint>0) + ){ + p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; + p->errorAction = OE_Abort; + sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); + return SQLITE_ERROR; + } + return SQLITE_OK; +} +#endif + +/* +** This routine is called the when a VDBE tries to halt. If the VDBE +** has made changes and is in autocommit mode, then commit those +** changes. If a rollback is needed, then do the rollback. +** +** This routine is the only way to move the sqlite3eOpenState of a VM from +** SQLITE_STATE_RUN to SQLITE_STATE_HALT. It is harmless to +** call this on a VM that is in the SQLITE_STATE_HALT state. +** +** Return an error code. If the commit could not complete because of +** lock contention, return SQLITE_BUSY. If SQLITE_BUSY is returned, it +** means the close did not happen and needs to be repeated. +*/ +SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ + int rc; /* Used to store transient return codes */ + sqlite3 *db = p->db; + + /* This function contains the logic that determines if a statement or + ** transaction will be committed or rolled back as a result of the + ** execution of this virtual machine. + ** + ** If any of the following errors occur: + ** + ** SQLITE_NOMEM + ** SQLITE_IOERR + ** SQLITE_FULL + ** SQLITE_INTERRUPT + ** + ** Then the internal cache might have been left in an inconsistent + ** state. We need to rollback the statement transaction, if there is + ** one, or the complete transaction if there is no statement transaction. + */ + + if( p->iVdbeMagic!=VDBE_MAGIC_RUN ){ + return SQLITE_OK; + } + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM_BKPT; + } + closeAllCursors(p); + checkActiveVdbeCnt(db); + + /* No commit or rollback needed if the program never started or if the + ** SQL statement does not read or write a database file. */ + if( p->pc>=0 && p->bIsReader ){ + int mrc; /* Primary error code from p->rc */ + int eStatementOp = 0; + int isSpecialError; /* Set to true if a 'special' error */ + + /* Lock all btrees used by the statement */ + sqlite3VdbeEnter(p); + + /* Check for one of the special errors */ + mrc = p->rc & 0xff; + isSpecialError = mrc==SQLITE_NOMEM || mrc==SQLITE_IOERR + || mrc==SQLITE_INTERRUPT || mrc==SQLITE_FULL; + if( isSpecialError ){ + /* If the query was read-only and the error code is SQLITE_INTERRUPT, + ** no rollback is necessary. Otherwise, at least a savepoint + ** transaction must be rolled back to restore the database to a + ** consistent state. + ** + ** Even if the statement is read-only, it is important to perform + ** a statement or transaction rollback operation. If the error + ** occurred while writing to the journal, sub-journal or database + ** file as part of an effort to free up cache space (see function + ** pagerStress() in pager.c), the rollback is required to restore + ** the pager to a consistent state. + */ + if( !p->readOnly || mrc!=SQLITE_INTERRUPT ){ + if( (mrc==SQLITE_NOMEM || mrc==SQLITE_FULL) && p->usesStmtJournal ){ + eStatementOp = SAVEPOINT_ROLLBACK; + }else{ + /* We are forced to roll back the active transaction. Before doing + ** so, abort any other statements this handle currently has active. + */ + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); + sqlite3CloseSavepoints(db); + db->autoCommit = 1; + p->nChange = 0; + } + } + } + + /* Check for immediate foreign key violations. */ + if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ + sqlite3VdbeCheckFk(p, 0); + } + + /* If the auto-commit flag is set and this is the only active writer + ** VM, then we do either a commit or rollback of the current transaction. + ** + ** Note: This block also runs if one of the special errors handled + ** above has occurred. + */ + if( !sqlite3VtabInSync(db) + && db->autoCommit + && db->nVdbeWrite==(p->readOnly==0) + ){ + if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ + rc = sqlite3VdbeCheckFk(p, 1); + if( rc!=SQLITE_OK ){ + if( NEVER(p->readOnly) ){ + sqlite3VdbeLeave(p); + return SQLITE_ERROR; + } + rc = SQLITE_CONSTRAINT_FOREIGNKEY; + }else{ + /* The auto-commit flag is true, the vdbe program was successful + ** or hit an 'OR FAIL' constraint and there are no deferred foreign + ** key constraints to hold up the transaction. This means a commit + ** is required. */ + rc = vdbeCommit(db, p); + } + if( rc==SQLITE_BUSY && p->readOnly ){ + sqlite3VdbeLeave(p); + return SQLITE_BUSY; + }else if( rc!=SQLITE_OK ){ + p->rc = rc; + sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; + }else{ + db->nDeferredCons = 0; + db->nDeferredImmCons = 0; + db->flags &= ~(u64)SQLITE_DeferFKs; + sqlite3CommitInternalChanges(db); + } + }else{ + sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; + } + db->nStatement = 0; + }else if( eStatementOp==0 ){ + if( p->rc==SQLITE_OK || p->errorAction==OE_Fail ){ + eStatementOp = SAVEPOINT_RELEASE; + }else if( p->errorAction==OE_Abort ){ + eStatementOp = SAVEPOINT_ROLLBACK; + }else{ + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); + sqlite3CloseSavepoints(db); + db->autoCommit = 1; + p->nChange = 0; + } + } + + /* If eStatementOp is non-zero, then a statement transaction needs to + ** be committed or rolled back. Call sqlite3VdbeCloseStatement() to + ** do so. If this operation returns an error, and the current statement + ** error code is SQLITE_OK or SQLITE_CONSTRAINT, then promote the + ** current statement error code. + */ + if( eStatementOp ){ + rc = sqlite3VdbeCloseStatement(p, eStatementOp); + if( rc ){ + if( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT ){ + p->rc = rc; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = 0; + } + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); + sqlite3CloseSavepoints(db); + db->autoCommit = 1; + p->nChange = 0; + } + } + + /* If this was an INSERT, UPDATE or DELETE and no statement transaction + ** has been rolled back, update the database connection change-counter. + */ + if( p->changeCntOn ){ + if( eStatementOp!=SAVEPOINT_ROLLBACK ){ + sqlite3VdbeSetChanges(db, p->nChange); + }else{ + sqlite3VdbeSetChanges(db, 0); + } + p->nChange = 0; + } + + /* Release the locks */ + sqlite3VdbeLeave(p); + } + + /* We have successfully halted and closed the VM. Record this fact. */ + if( p->pc>=0 ){ + db->nVdbeActive--; + if( !p->readOnly ) db->nVdbeWrite--; + if( p->bIsReader ) db->nVdbeRead--; + assert( db->nVdbeActive>=db->nVdbeRead ); + assert( db->nVdbeRead>=db->nVdbeWrite ); + assert( db->nVdbeWrite>=0 ); + } + p->iVdbeMagic = VDBE_MAGIC_HALT; + checkActiveVdbeCnt(db); + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM_BKPT; + } + + /* If the auto-commit flag is set to true, then any locks that were held + ** by connection db have now been released. Call sqlite3ConnectionUnlocked() + ** to invoke any required unlock-notify callbacks. + */ + if( db->autoCommit ){ + sqlite3ConnectionUnlocked(db); + } + + assert( db->nVdbeActive>0 || db->autoCommit==0 || db->nStatement==0 ); + return (p->rc==SQLITE_BUSY ? SQLITE_BUSY : SQLITE_OK); +} + + +/* +** Each VDBE holds the result of the most recent sqlite3_step() call +** in p->rc. This routine sets that result back to SQLITE_OK. +*/ +SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe *p){ + p->rc = SQLITE_OK; +} + +/* +** Copy the error code and error message belonging to the VDBE passed +** as the first argument to its database handle (so that they will be +** returned by calls to sqlite3_errcode() and sqlite3_errmsg()). +** +** This function does not clear the VDBE error code or message, just +** copies them to the database handle. +*/ +SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p){ + sqlite3 *db = p->db; + int rc = p->rc; + if( p->zErrMsg ){ + db->bBenignMalloc++; + sqlite3BeginBenignMalloc(); + if( db->pErr==0 ) db->pErr = sqlite3ValueNew(db); + sqlite3ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE_UTF8, SQLITE_TRANSIENT); + sqlite3EndBenignMalloc(); + db->bBenignMalloc--; + }else if( db->pErr ){ + sqlite3ValueSetNull(db->pErr); + } + db->errCode = rc; + return rc; +} + +#ifdef SQLITE_ENABLE_SQLLOG +/* +** If an SQLITE_CONFIG_SQLLOG hook is registered and the VM has been run, +** invoke it. +*/ +static void vdbeInvokeSqllog(Vdbe *v){ + if( sqlite3GlobalConfig.xSqllog && v->rc==SQLITE_OK && v->zSql && v->pc>=0 ){ + char *zExpanded = sqlite3VdbeExpandSql(v, v->zSql); + assert( v->db->init.busy==0 ); + if( zExpanded ){ + sqlite3GlobalConfig.xSqllog( + sqlite3GlobalConfig.pSqllogArg, v->db, zExpanded, 1 + ); + sqlite3DbFree(v->db, zExpanded); + } + } +} +#else +# define vdbeInvokeSqllog(x) +#endif + +/* +** Clean up a VDBE after execution but do not delete the VDBE just yet. +** Write any error messages into *pzErrMsg. Return the result code. +** +** After this routine is run, the VDBE should be ready to be executed +** again. +** +** To look at it another way, this routine resets the state of the +** virtual machine from VDBE_MAGIC_RUN or VDBE_MAGIC_HALT back to +** VDBE_MAGIC_INIT. +*/ +SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + int i; +#endif + + sqlite3 *db; + db = p->db; + + /* If the VM did not run to completion or if it encountered an + ** error, then it might not have been halted properly. So halt + ** it now. + */ + sqlite3VdbeHalt(p); + + /* If the VDBE has been run even partially, then transfer the error code + ** and error message from the VDBE into the main database structure. But + ** if the VDBE has just been set to run but has not actually executed any + ** instructions yet, leave the main database error information unchanged. + */ + if( p->pc>=0 ){ + vdbeInvokeSqllog(p); + if( db->pErr || p->zErrMsg ){ + sqlite3VdbeTransferError(p); + }else{ + db->errCode = p->rc; + } + if( p->runOnlyOnce ) p->expired = 1; + }else if( p->rc && p->expired ){ + /* The expired flag was set on the VDBE before the first call + ** to sqlite3_step(). For consistency (since sqlite3_step() was + ** called), set the database error in this case as well. + */ + sqlite3ErrorWithMsg(db, p->rc, p->zErrMsg ? "%s" : 0, p->zErrMsg); + } + + /* Reset register contents and reclaim error message memory. + */ +#ifdef SQLITE_DEBUG + /* Execute assert() statements to ensure that the Vdbe.apCsr[] and + ** Vdbe.aMem[] arrays have already been cleaned up. */ + if( p->apCsr ) for(i=0; inCursor; i++) assert( p->apCsr[i]==0 ); + if( p->aMem ){ + for(i=0; inMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); + } +#endif + if( p->zErrMsg ){ + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = 0; + } + p->pResultSet = 0; +#ifdef SQLITE_DEBUG + p->nWrite = 0; +#endif + + /* Save profiling information from this VDBE run. + */ +#ifdef VDBE_PROFILE + { + FILE *out = fopen("vdbe_profile.out", "a"); + if( out ){ + fprintf(out, "---- "); + for(i=0; inOp; i++){ + fprintf(out, "%02x", p->aOp[i].opcode); + } + fprintf(out, "\n"); + if( p->zSql ){ + char c, pc = 0; + fprintf(out, "-- "); + for(i=0; (c = p->zSql[i])!=0; i++){ + if( pc=='\n' ) fprintf(out, "-- "); + putc(c, out); + pc = c; + } + if( pc!='\n' ) fprintf(out, "\n"); + } + for(i=0; inOp; i++){ + char zHdr[100]; + sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ", + p->aOp[i].cnt, + p->aOp[i].cycles, + p->aOp[i].cnt>0 ? p->aOp[i].cycles/p->aOp[i].cnt : 0 + ); + fprintf(out, "%s", zHdr); + sqlite3VdbePrintOp(out, i, &p->aOp[i]); + } + fclose(out); + } + } +#endif + p->iVdbeMagic = VDBE_MAGIC_RESET; + return p->rc & db->errMask; +} + +/* +** Clean up and delete a VDBE after execution. Return an integer which is +** the result code. Write any error message text into *pzErrMsg. +*/ +SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe *p){ + int rc = SQLITE_OK; + if( p->iVdbeMagic==VDBE_MAGIC_RUN || p->iVdbeMagic==VDBE_MAGIC_HALT ){ + rc = sqlite3VdbeReset(p); + assert( (rc & p->db->errMask)==rc ); + } + sqlite3VdbeDelete(p); + return rc; +} + +/* +** If parameter iOp is less than zero, then invoke the destructor for +** all auxiliary data pointers currently cached by the VM passed as +** the first argument. +** +** Or, if iOp is greater than or equal to zero, then the destructor is +** only invoked for those auxiliary data pointers created by the user +** function invoked by the OP_Function opcode at instruction iOp of +** VM pVdbe, and only then if: +** +** * the associated function parameter is the 32nd or later (counting +** from left to right), or +** +** * the corresponding bit in argument mask is clear (where the first +** function parameter corresponds to bit 0 etc.). +*/ +SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, int mask){ + while( *pp ){ + AuxData *pAux = *pp; + if( (iOp<0) + || (pAux->iAuxOp==iOp + && pAux->iAuxArg>=0 + && (pAux->iAuxArg>31 || !(mask & MASKBIT32(pAux->iAuxArg)))) + ){ + testcase( pAux->iAuxArg==31 ); + if( pAux->xDeleteAux ){ + pAux->xDeleteAux(pAux->pAux); + } + *pp = pAux->pNextAux; + sqlite3DbFree(db, pAux); + }else{ + pp= &pAux->pNextAux; + } + } +} + +/* +** Free all memory associated with the Vdbe passed as the second argument, +** except for object itself, which is preserved. +** +** The difference between this function and sqlite3VdbeDelete() is that +** VdbeDelete() also unlinks the Vdbe from the list of VMs associated with +** the database connection and frees the object itself. +*/ +SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ + SubProgram *pSub, *pNext; + assert( p->db==0 || p->db==db ); + releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + for(pSub=p->pProgram; pSub; pSub=pNext){ + pNext = pSub->pNext; + vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); + sqlite3DbFree(db, pSub); + } + if( p->iVdbeMagic!=VDBE_MAGIC_INIT ){ + releaseMemArray(p->aVar, p->nVar); + sqlite3DbFree(db, p->pVList); + sqlite3DbFree(db, p->pFree); + } + vdbeFreeOpArray(db, p->aOp, p->nOp); + sqlite3DbFree(db, p->aColName); + sqlite3DbFree(db, p->zSql); +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3DbFree(db, p->zNormSql); + { + DblquoteStr *pThis, *pNext; + for(pThis=p->pDblStr; pThis; pThis=pNext){ + pNext = pThis->pNextStr; + sqlite3DbFree(db, pThis); + } + } +#endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + { + int i; + for(i=0; inScan; i++){ + sqlite3DbFree(db, p->aScan[i].zName); + } + sqlite3DbFree(db, p->aScan); + } +#endif +} + +/* +** Delete an entire VDBE. +*/ +SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ + sqlite3 *db; + + assert( p!=0 ); + db = p->db; + assert( sqlite3_mutex_held(db->mutex) ); + sqlite3VdbeClearObject(db, p); + if( p->pPrev ){ + p->pPrev->pNext = p->pNext; + }else{ + assert( db->pVdbe==p ); + db->pVdbe = p->pNext; + } + if( p->pNext ){ + p->pNext->pPrev = p->pPrev; + } + p->iVdbeMagic = VDBE_MAGIC_DEAD; + p->db = 0; + sqlite3DbFreeNN(db, p); +} + +/* +** The cursor "p" has a pending seek operation that has not yet been +** carried out. Seek the cursor now. If an error occurs, return +** the appropriate error code. +*/ +SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeFinishMoveto(VdbeCursor *p){ + int res, rc; +#ifdef SQLITE_TEST + extern int sqlite3_search_count; +#endif + assert( p->deferredMoveto ); + assert( p->isTable ); + assert( p->eCurType==CURTYPE_BTREE ); + rc = sqlite3BtreeTableMoveto(p->uc.pCursor, p->movetoTarget, 0, &res); + if( rc ) return rc; + if( res!=0 ) return SQLITE_CORRUPT_BKPT; +#ifdef SQLITE_TEST + sqlite3_search_count++; +#endif + p->deferredMoveto = 0; + p->cacheStatus = CACHE_STALE; + return SQLITE_OK; +} + +/* +** Something has moved cursor "p" out of place. Maybe the row it was +** pointed to was deleted out from under it. Or maybe the btree was +** rebalanced. Whatever the cause, try to restore "p" to the place it +** is supposed to be pointing. If the row was deleted out from under the +** cursor, set the cursor to point to a NULL row. +*/ +static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ + int isDifferentRow, rc; + assert( p->eCurType==CURTYPE_BTREE ); + assert( p->uc.pCursor!=0 ); + assert( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ); + rc = sqlite3BtreeCursorRestore(p->uc.pCursor, &isDifferentRow); + p->cacheStatus = CACHE_STALE; + if( isDifferentRow ) p->nullRow = 1; + return rc; +} + +/* +** Check to ensure that the cursor is valid. Restore the cursor +** if need be. Return any I/O error from the restore operation. +*/ +SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor *p){ + assert( p->eCurType==CURTYPE_BTREE ); + if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ + return handleMovedCursor(p); + } + return SQLITE_OK; +} + +/* +** Make sure the cursor p is ready to read or write the row to which it +** was last positioned. Return an error code if an OOM fault or I/O error +** prevents us from positioning the cursor to its correct position. +** +** If a MoveTo operation is pending on the given cursor, then do that +** MoveTo now. If no move is pending, check to see if the row has been +** deleted out from under the cursor and if it has, mark the row as +** a NULL row. +** +** If the cursor is already pointing to the correct row and that row has +** not been deleted out from under the cursor, then this routine is a no-op. +*/ +SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, u32 *piCol){ + VdbeCursor *p = *pp; + assert( p->eCurType==CURTYPE_BTREE || p->eCurType==CURTYPE_PSEUDO ); + if( p->deferredMoveto ){ + u32 iMap; + assert( !p->isEphemeral ); + if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 && !p->nullRow ){ + *pp = p->pAltCursor; + *piCol = iMap - 1; + return SQLITE_OK; + } + return sqlite3VdbeFinishMoveto(p); + } + if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ + return handleMovedCursor(p); + } + return SQLITE_OK; +} + +/* +** The following functions: +** +** sqlite3VdbeSerialType() +** sqlite3VdbeSerialTypeLen() +** sqlite3VdbeSerialLen() +** sqlite3VdbeSerialPut() +** sqlite3VdbeSerialGet() +** +** encapsulate the code that serializes values for storage in SQLite +** data and index records. Each serialized value consists of a +** 'serial-type' and a blob of data. The serial type is an 8-byte unsigned +** integer, stored as a varint. +** +** In an SQLite index record, the serial type is stored directly before +** the blob of data that it corresponds to. In a table record, all serial +** types are stored at the start of the record, and the blobs of data at +** the end. Hence these functions allow the caller to handle the +** serial-type and data blob separately. +** +** The following table describes the various storage classes for data: +** +** serial type bytes of data type +** -------------- --------------- --------------- +** 0 0 NULL +** 1 1 signed integer +** 2 2 signed integer +** 3 3 signed integer +** 4 4 signed integer +** 5 6 signed integer +** 6 8 signed integer +** 7 8 IEEE float +** 8 0 Integer constant 0 +** 9 0 Integer constant 1 +** 10,11 reserved for expansion +** N>=12 and even (N-12)/2 BLOB +** N>=13 and odd (N-13)/2 text +** +** The 8 and 9 types were added in 3.3.0, file format 4. Prior versions +** of SQLite will not understand those serial types. +*/ + +#if 0 /* Inlined into the OP_MakeRecord opcode */ +/* +** Return the serial-type for the value stored in pMem. +** +** This routine might convert a large MEM_IntReal value into MEM_Real. +** +** 2019-07-11: The primary user of this subroutine was the OP_MakeRecord +** opcode in the byte-code engine. But by moving this routine in-line, we +** can omit some redundant tests and make that opcode a lot faster. So +** this routine is now only used by the STAT3 logic and STAT3 support has +** ended. The code is kept here for historical reference only. +*/ +SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ + int flags = pMem->flags; + u32 n; + + assert( pLen!=0 ); + if( flags&MEM_Null ){ + *pLen = 0; + return 0; + } + if( flags&(MEM_Int|MEM_IntReal) ){ + /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ +# define MAX_6BYTE ((((i64)0x00008000)<<32)-1) + i64 i = pMem->u.i; + u64 u; + testcase( flags & MEM_Int ); + testcase( flags & MEM_IntReal ); + if( i<0 ){ + u = ~i; + }else{ + u = i; + } + if( u<=127 ){ + if( (i&1)==i && file_format>=4 ){ + *pLen = 0; + return 8+(u32)u; + }else{ + *pLen = 1; + return 1; + } + } + if( u<=32767 ){ *pLen = 2; return 2; } + if( u<=8388607 ){ *pLen = 3; return 3; } + if( u<=2147483647 ){ *pLen = 4; return 4; } + if( u<=MAX_6BYTE ){ *pLen = 6; return 5; } + *pLen = 8; + if( flags&MEM_IntReal ){ + /* If the value is IntReal and is going to take up 8 bytes to store + ** as an integer, then we might as well make it an 8-byte floating + ** point value */ + pMem->u.r = (double)pMem->u.i; + pMem->flags &= ~MEM_IntReal; + pMem->flags |= MEM_Real; + return 7; + } + return 6; + } + if( flags&MEM_Real ){ + *pLen = 8; + return 7; + } + assert( pMem->db->mallocFailed || flags&(MEM_Str|MEM_Blob) ); + assert( pMem->n>=0 ); + n = (u32)pMem->n; + if( flags & MEM_Zero ){ + n += pMem->u.nZero; + } + *pLen = n; + return ((n*2) + 12 + ((flags&MEM_Str)!=0)); +} +#endif /* inlined into OP_MakeRecord */ + +/* +** The sizes for serial types less than 128 +*/ +static const u8 sqlite3SmallTypeSizes[] = { + /* 0 1 2 3 4 5 6 7 8 9 */ +/* 0 */ 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, +/* 10 */ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, +/* 20 */ 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, +/* 30 */ 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, +/* 40 */ 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, +/* 50 */ 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, +/* 60 */ 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, +/* 70 */ 29, 29, 30, 30, 31, 31, 32, 32, 33, 33, +/* 80 */ 34, 34, 35, 35, 36, 36, 37, 37, 38, 38, +/* 90 */ 39, 39, 40, 40, 41, 41, 42, 42, 43, 43, +/* 100 */ 44, 44, 45, 45, 46, 46, 47, 47, 48, 48, +/* 110 */ 49, 49, 50, 50, 51, 51, 52, 52, 53, 53, +/* 120 */ 54, 54, 55, 55, 56, 56, 57, 57 +}; + +/* +** Return the length of the data corresponding to the supplied serial-type. +*/ +SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32 serial_type){ + if( serial_type>=128 ){ + return (serial_type-12)/2; + }else{ + assert( serial_type<12 + || sqlite3SmallTypeSizes[serial_type]==(serial_type - 12)/2 ); + return sqlite3SmallTypeSizes[serial_type]; + } +} +SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8 serial_type){ + assert( serial_type<128 ); + return sqlite3SmallTypeSizes[serial_type]; +} + +/* +** If we are on an architecture with mixed-endian floating +** points (ex: ARM7) then swap the lower 4 bytes with the +** upper 4 bytes. Return the result. +** +** For most architectures, this is a no-op. +** +** (later): It is reported to me that the mixed-endian problem +** on ARM7 is an issue with GCC, not with the ARM7 chip. It seems +** that early versions of GCC stored the two words of a 64-bit +** float in the wrong order. And that error has been propagated +** ever since. The blame is not necessarily with GCC, though. +** GCC might have just copying the problem from a prior compiler. +** I am also told that newer versions of GCC that follow a different +** ABI get the byte order right. +** +** Developers using SQLite on an ARM7 should compile and run their +** application using -DSQLITE_DEBUG=1 at least once. With DEBUG +** enabled, some asserts below will ensure that the byte order of +** floating point values is correct. +** +** (2007-08-30) Frank van Vugt has studied this problem closely +** and has send his findings to the SQLite developers. Frank +** writes that some Linux kernels offer floating point hardware +** emulation that uses only 32-bit mantissas instead of a full +** 48-bits as required by the IEEE standard. (This is the +** CONFIG_FPE_FASTFPE option.) On such systems, floating point +** byte swapping becomes very complicated. To avoid problems, +** the necessary byte swapping is carried out using a 64-bit integer +** rather than a 64-bit float. Frank assures us that the code here +** works for him. We, the developers, have no way to independently +** verify this, but Frank seems to know what he is talking about +** so we trust him. +*/ +#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT +static u64 floatSwap(u64 in){ + union { + u64 r; + u32 i[2]; + } u; + u32 t; + + u.r = in; + t = u.i[0]; + u.i[0] = u.i[1]; + u.i[1] = t; + return u.r; +} +# define swapMixedEndianFloat(X) X = floatSwap(X) +#else +# define swapMixedEndianFloat(X) +#endif + +/* +** Write the serialized data blob for the value stored in pMem into +** buf. It is assumed that the caller has allocated sufficient space. +** Return the number of bytes written. +** +** nBuf is the amount of space left in buf[]. The caller is responsible +** for allocating enough space to buf[] to hold the entire field, exclusive +** of the pMem->u.nZero bytes for a MEM_Zero value. +** +** Return the number of bytes actually written into buf[]. The number +** of bytes in the zero-filled tail is included in the return value only +** if those bytes were zeroed in buf[]. +*/ +SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ + u32 len; + + /* Integer and Real */ + if( serial_type<=7 && serial_type>0 ){ + u64 v; + u32 i; + if( serial_type==7 ){ + assert( sizeof(v)==sizeof(pMem->u.r) ); + memcpy(&v, &pMem->u.r, sizeof(v)); + swapMixedEndianFloat(v); + }else{ + v = pMem->u.i; + } + len = i = sqlite3SmallTypeSizes[serial_type]; + assert( i>0 ); + do{ + buf[--i] = (u8)(v&0xFF); + v >>= 8; + }while( i ); + return len; + } + + /* String or blob */ + if( serial_type>=12 ){ + assert( pMem->n + ((pMem->flags & MEM_Zero)?pMem->u.nZero:0) + == (int)sqlite3VdbeSerialTypeLen(serial_type) ); + len = pMem->n; + if( len>0 ) memcpy(buf, pMem->z, len); + return len; + } + + /* NULL or constants 0 or 1 */ + return 0; +} + +/* Input "x" is a sequence of unsigned characters that represent a +** big-endian integer. Return the equivalent native integer +*/ +#define ONE_BYTE_INT(x) ((i8)(x)[0]) +#define TWO_BYTE_INT(x) (256*(i8)((x)[0])|(x)[1]) +#define THREE_BYTE_INT(x) (65536*(i8)((x)[0])|((x)[1]<<8)|(x)[2]) +#define FOUR_BYTE_UINT(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) +#define FOUR_BYTE_INT(x) (16777216*(i8)((x)[0])|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) + +/* +** Deserialize the data blob pointed to by buf as serial type serial_type +** and store the result in pMem. Return the number of bytes read. +** +** This function is implemented as two separate routines for performance. +** The few cases that require local variables are broken out into a separate +** routine so that in most cases the overhead of moving the stack pointer +** is avoided. +*/ +static u32 serialGet( + const unsigned char *buf, /* Buffer to deserialize from */ + u32 serial_type, /* Serial type to deserialize */ + Mem *pMem /* Memory cell to write value into */ +){ + u64 x = FOUR_BYTE_UINT(buf); + u32 y = FOUR_BYTE_UINT(buf+4); + x = (x<<32) + y; + if( serial_type==6 ){ + /* EVIDENCE-OF: R-29851-52272 Value is a big-endian 64-bit + ** twos-complement integer. */ + pMem->u.i = *(i64*)&x; + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + }else{ + /* EVIDENCE-OF: R-57343-49114 Value is a big-endian IEEE 754-2008 64-bit + ** floating point number. */ +#if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) + /* Verify that integers and floating point values use the same + ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is + ** defined that 64-bit floating point values really are mixed + ** endian. + */ + static const u64 t1 = ((u64)0x3ff00000)<<32; + static const double r1 = 1.0; + u64 t2 = t1; + swapMixedEndianFloat(t2); + assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); +#endif + assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->u.r, &x, sizeof(x)); + pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real; + } + return 8; +} +SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( + const unsigned char *buf, /* Buffer to deserialize from */ + u32 serial_type, /* Serial type to deserialize */ + Mem *pMem /* Memory cell to write value into */ +){ + switch( serial_type ){ + case 10: { /* Internal use only: NULL with virtual table + ** UPDATE no-change flag set */ + pMem->flags = MEM_Null|MEM_Zero; + pMem->n = 0; + pMem->u.nZero = 0; + break; + } + case 11: /* Reserved for future use */ + case 0: { /* Null */ + /* EVIDENCE-OF: R-24078-09375 Value is a NULL. */ + pMem->flags = MEM_Null; + break; + } + case 1: { + /* EVIDENCE-OF: R-44885-25196 Value is an 8-bit twos-complement + ** integer. */ + pMem->u.i = ONE_BYTE_INT(buf); + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 1; + } + case 2: { /* 2-byte signed integer */ + /* EVIDENCE-OF: R-49794-35026 Value is a big-endian 16-bit + ** twos-complement integer. */ + pMem->u.i = TWO_BYTE_INT(buf); + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 2; + } + case 3: { /* 3-byte signed integer */ + /* EVIDENCE-OF: R-37839-54301 Value is a big-endian 24-bit + ** twos-complement integer. */ + pMem->u.i = THREE_BYTE_INT(buf); + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 3; + } + case 4: { /* 4-byte signed integer */ + /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit + ** twos-complement integer. */ + pMem->u.i = FOUR_BYTE_INT(buf); +#ifdef __HP_cc + /* Work around a sign-extension bug in the HP compiler for HP/UX */ + if( buf[0]&0x80 ) pMem->u.i |= 0xffffffff80000000LL; +#endif + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 4; + } + case 5: { /* 6-byte signed integer */ + /* EVIDENCE-OF: R-50385-09674 Value is a big-endian 48-bit + ** twos-complement integer. */ + pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 6; + } + case 6: /* 8-byte signed integer */ + case 7: { /* IEEE floating point */ + /* These use local variables, so do them in a separate routine + ** to avoid having to move the frame pointer in the common case */ + return serialGet(buf,serial_type,pMem); + } + case 8: /* Integer 0 */ + case 9: { /* Integer 1 */ + /* EVIDENCE-OF: R-12976-22893 Value is the integer 0. */ + /* EVIDENCE-OF: R-18143-12121 Value is the integer 1. */ + pMem->u.i = serial_type-8; + pMem->flags = MEM_Int; + return 0; + } + default: { + /* EVIDENCE-OF: R-14606-31564 Value is a BLOB that is (N-12)/2 bytes in + ** length. + ** EVIDENCE-OF: R-28401-00140 Value is a string in the text encoding and + ** (N-13)/2 bytes in length. */ + static const u16 aFlag[] = { MEM_Blob|MEM_Ephem, MEM_Str|MEM_Ephem }; + pMem->z = (char *)buf; + pMem->n = (serial_type-12)/2; + pMem->flags = aFlag[serial_type&1]; + return pMem->n; + } + } + return 0; +} +/* +** This routine is used to allocate sufficient space for an UnpackedRecord +** structure large enough to be used with sqlite3VdbeRecordUnpack() if +** the first argument is a pointer to KeyInfo structure pKeyInfo. +** +** The space is either allocated using sqlite3DbMallocRaw() or from within +** the unaligned buffer passed via the second and third arguments (presumably +** stack space). If the former, then *ppFree is set to a pointer that should +** be eventually freed by the caller using sqlite3DbFree(). Or, if the +** allocation comes from the pSpace/szSpace buffer, *ppFree is set to NULL +** before returning. +** +** If an OOM error occurs, NULL is returned. +*/ +SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( + KeyInfo *pKeyInfo /* Description of the record */ +){ + UnpackedRecord *p; /* Unpacked record to return */ + int nByte; /* Number of bytes required for *p */ + nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nKeyField+1); + p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); + if( !p ) return 0; + p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; + assert( pKeyInfo->aSortFlags!=0 ); + p->pKeyInfo = pKeyInfo; + p->nField = pKeyInfo->nKeyField + 1; + return p; +} + +/* +** Given the nKey-byte encoding of a record in pKey[], populate the +** UnpackedRecord structure indicated by the fourth argument with the +** contents of the decoded record. +*/ +SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( + KeyInfo *pKeyInfo, /* Information about the record format */ + int nKey, /* Size of the binary record */ + const void *pKey, /* The binary record */ + UnpackedRecord *p /* Populate this structure before returning. */ +){ + const unsigned char *aKey = (const unsigned char *)pKey; + u32 d; + u32 idx; /* Offset in aKey[] to read from */ + u16 u; /* Unsigned loop counter */ + u32 szHdr; + Mem *pMem = p->aMem; + + p->default_rc = 0; + assert( EIGHT_BYTE_ALIGNMENT(pMem) ); + idx = getVarint32(aKey, szHdr); + d = szHdr; + u = 0; + while( idxenc = pKeyInfo->enc; + pMem->db = pKeyInfo->db; + /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ + pMem->szMalloc = 0; + pMem->z = 0; + d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); + pMem++; + if( (++u)>=p->nField ) break; + } + if( d>(u32)nKey && u ){ + assert( CORRUPT_DB ); + /* In a corrupt record entry, the last pMem might have been set up using + ** uninitialized memory. Overwrite its value with NULL, to prevent + ** warnings from MSAN. */ + sqlite3VdbeMemSetNull(pMem-1); + } + assert( u<=pKeyInfo->nKeyField + 1 ); + p->nField = u; +} + +#ifdef SQLITE_DEBUG +/* +** This function compares two index or table record keys in the same way +** as the sqlite3VdbeRecordCompare() routine. Unlike VdbeRecordCompare(), +** this function deserializes and compares values using the +** sqlite3VdbeSerialGet() and sqlite3MemCompare() functions. It is used +** in assert() statements to ensure that the optimized code in +** sqlite3VdbeRecordCompare() returns results with these two primitives. +** +** Return true if the result of comparison is equivalent to desiredResult. +** Return false if there is a disagreement. +*/ +static int vdbeRecordCompareDebug( + int nKey1, const void *pKey1, /* Left key */ + const UnpackedRecord *pPKey2, /* Right key */ + int desiredResult /* Correct answer */ +){ + u32 d1; /* Offset into aKey[] of next data element */ + u32 idx1; /* Offset into aKey[] of next header element */ + u32 szHdr1; /* Number of bytes in header */ + int i = 0; + int rc = 0; + const unsigned char *aKey1 = (const unsigned char *)pKey1; + KeyInfo *pKeyInfo; + Mem mem1; + + pKeyInfo = pPKey2->pKeyInfo; + if( pKeyInfo->db==0 ) return 1; + mem1.enc = pKeyInfo->enc; + mem1.db = pKeyInfo->db; + /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ + VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ + + /* Compilers may complain that mem1.u.i is potentially uninitialized. + ** We could initialize it, as shown here, to silence those complaints. + ** But in fact, mem1.u.i will never actually be used uninitialized, and doing + ** the unnecessary initialization has a measurable negative performance + ** impact, since this routine is a very high runner. And so, we choose + ** to ignore the compiler warnings and leave this variable uninitialized. + */ + /* mem1.u.i = 0; // not needed, here to silence compiler warning */ + + idx1 = getVarint32(aKey1, szHdr1); + if( szHdr1>98307 ) return SQLITE_CORRUPT; + d1 = szHdr1; + assert( pKeyInfo->nAllField>=pPKey2->nField || CORRUPT_DB ); + assert( pKeyInfo->aSortFlags!=0 ); + assert( pKeyInfo->nKeyField>0 ); + assert( idx1<=szHdr1 || CORRUPT_DB ); + do{ + u32 serial_type1; + + /* Read the serial types for the next element in each key. */ + idx1 += getVarint32( aKey1+idx1, serial_type1 ); + + /* Verify that there is enough key space remaining to avoid + ** a buffer overread. The "d1+serial_type1+2" subexpression will + ** always be greater than or equal to the amount of required key space. + ** Use that approximation to avoid the more expensive call to + ** sqlite3VdbeSerialTypeLen() in the common case. + */ + if( d1+(u64)serial_type1+2>(u64)nKey1 + && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)>(u64)nKey1 + ){ + break; + } + + /* Extract the values to be compared. + */ + d1 += sqlite3VdbeSerialGet(&aKey1[d1], serial_type1, &mem1); + + /* Do the comparison + */ + rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], + pKeyInfo->nAllField>i ? pKeyInfo->aColl[i] : 0); + if( rc!=0 ){ + assert( mem1.szMalloc==0 ); /* See comment below */ + if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) + && ((mem1.flags & MEM_Null) || (pPKey2->aMem[i].flags & MEM_Null)) + ){ + rc = -rc; + } + if( pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC ){ + rc = -rc; /* Invert the result for DESC sort order. */ + } + goto debugCompareEnd; + } + i++; + }while( idx1nField ); + + /* No memory allocation is ever used on mem1. Prove this using + ** the following assert(). If the assert() fails, it indicates a + ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). + */ + assert( mem1.szMalloc==0 ); + + /* rc==0 here means that one of the keys ran out of fields and + ** all the fields up to that point were equal. Return the default_rc + ** value. */ + rc = pPKey2->default_rc; + +debugCompareEnd: + if( desiredResult==0 && rc==0 ) return 1; + if( desiredResult<0 && rc<0 ) return 1; + if( desiredResult>0 && rc>0 ) return 1; + if( CORRUPT_DB ) return 1; + if( pKeyInfo->db->mallocFailed ) return 1; + return 0; +} +#endif + +#ifdef SQLITE_DEBUG +/* +** Count the number of fields (a.k.a. columns) in the record given by +** pKey,nKey. The verify that this count is less than or equal to the +** limit given by pKeyInfo->nAllField. +** +** If this constraint is not satisfied, it means that the high-speed +** vdbeRecordCompareInt() and vdbeRecordCompareString() routines will +** not work correctly. If this assert() ever fires, it probably means +** that the KeyInfo.nKeyField or KeyInfo.nAllField values were computed +** incorrectly. +*/ +static void vdbeAssertFieldCountWithinLimits( + int nKey, const void *pKey, /* The record to verify */ + const KeyInfo *pKeyInfo /* Compare size with this KeyInfo */ +){ + int nField = 0; + u32 szHdr; + u32 idx; + u32 notUsed; + const unsigned char *aKey = (const unsigned char*)pKey; + + if( CORRUPT_DB ) return; + idx = getVarint32(aKey, szHdr); + assert( nKey>=0 ); + assert( szHdr<=(u32)nKey ); + while( idxnAllField ); +} +#else +# define vdbeAssertFieldCountWithinLimits(A,B,C) +#endif + +/* +** Both *pMem1 and *pMem2 contain string values. Compare the two values +** using the collation sequence pColl. As usual, return a negative , zero +** or positive value if *pMem1 is less than, equal to or greater than +** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);". +*/ +static int vdbeCompareMemString( + const Mem *pMem1, + const Mem *pMem2, + const CollSeq *pColl, + u8 *prcErr /* If an OOM occurs, set to SQLITE_NOMEM */ +){ + if( pMem1->enc==pColl->enc ){ + /* The strings are already in the correct encoding. Call the + ** comparison function directly */ + return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); + }else{ + int rc; + const void *v1, *v2; + Mem c1; + Mem c2; + sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); + sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); + sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); + sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); + v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); + v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); + if( (v1==0 || v2==0) ){ + if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT; + rc = 0; + }else{ + rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2); + } + sqlite3VdbeMemRelease(&c1); + sqlite3VdbeMemRelease(&c2); + return rc; + } +} + +/* +** The input pBlob is guaranteed to be a Blob that is not marked +** with MEM_Zero. Return true if it could be a zero-blob. +*/ +static int isAllZero(const char *z, int n){ + int i; + for(i=0; in; + int n2 = pB2->n; + + /* It is possible to have a Blob value that has some non-zero content + ** followed by zero content. But that only comes up for Blobs formed + ** by the OP_MakeRecord opcode, and such Blobs never get passed into + ** sqlite3MemCompare(). */ + assert( (pB1->flags & MEM_Zero)==0 || n1==0 ); + assert( (pB2->flags & MEM_Zero)==0 || n2==0 ); + + if( (pB1->flags|pB2->flags) & MEM_Zero ){ + if( pB1->flags & pB2->flags & MEM_Zero ){ + return pB1->u.nZero - pB2->u.nZero; + }else if( pB1->flags & MEM_Zero ){ + if( !isAllZero(pB2->z, pB2->n) ) return -1; + return pB1->u.nZero - n2; + }else{ + if( !isAllZero(pB1->z, pB1->n) ) return +1; + return n1 - pB2->u.nZero; + } + } + c = memcmp(pB1->z, pB2->z, n1>n2 ? n2 : n1); + if( c ) return c; + return n1 - n2; +} + +/* +** Do a comparison between a 64-bit signed integer and a 64-bit floating-point +** number. Return negative, zero, or positive if the first (i64) is less than, +** equal to, or greater than the second (double). +*/ +SQLITE_PRIVATE int sqlite3IntFloatCompare(i64 i, double r){ + if( sizeof(LONGDOUBLE_TYPE)>8 ){ + LONGDOUBLE_TYPE x = (LONGDOUBLE_TYPE)i; + testcase( xr ); + testcase( x==r ); + if( xr ) return +1; /*NO_TEST*/ /* work around bugs in gcov */ + return 0; /*NO_TEST*/ /* work around bugs in gcov */ + }else{ + i64 y; + double s; + if( r<-9223372036854775808.0 ) return +1; + if( r>=9223372036854775808.0 ) return -1; + y = (i64)r; + if( iy ) return +1; + s = (double)i; + if( sr ) return +1; + return 0; + } +} + +/* +** Compare the values contained by the two memory cells, returning +** negative, zero or positive if pMem1 is less than, equal to, or greater +** than pMem2. Sorting order is NULL's first, followed by numbers (integers +** and reals) sorted numerically, followed by text ordered by the collating +** sequence pColl and finally blob's ordered by memcmp(). +** +** Two NULL values are considered equal by this function. +*/ +SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ + int f1, f2; + int combined_flags; + + f1 = pMem1->flags; + f2 = pMem2->flags; + combined_flags = f1|f2; + assert( !sqlite3VdbeMemIsRowSet(pMem1) && !sqlite3VdbeMemIsRowSet(pMem2) ); + + /* If one value is NULL, it is less than the other. If both values + ** are NULL, return 0. + */ + if( combined_flags&MEM_Null ){ + return (f2&MEM_Null) - (f1&MEM_Null); + } + + /* At least one of the two values is a number + */ + if( combined_flags&(MEM_Int|MEM_Real|MEM_IntReal) ){ + testcase( combined_flags & MEM_Int ); + testcase( combined_flags & MEM_Real ); + testcase( combined_flags & MEM_IntReal ); + if( (f1 & f2 & (MEM_Int|MEM_IntReal))!=0 ){ + testcase( f1 & f2 & MEM_Int ); + testcase( f1 & f2 & MEM_IntReal ); + if( pMem1->u.i < pMem2->u.i ) return -1; + if( pMem1->u.i > pMem2->u.i ) return +1; + return 0; + } + if( (f1 & f2 & MEM_Real)!=0 ){ + if( pMem1->u.r < pMem2->u.r ) return -1; + if( pMem1->u.r > pMem2->u.r ) return +1; + return 0; + } + if( (f1&(MEM_Int|MEM_IntReal))!=0 ){ + testcase( f1 & MEM_Int ); + testcase( f1 & MEM_IntReal ); + if( (f2&MEM_Real)!=0 ){ + return sqlite3IntFloatCompare(pMem1->u.i, pMem2->u.r); + }else if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ + if( pMem1->u.i < pMem2->u.i ) return -1; + if( pMem1->u.i > pMem2->u.i ) return +1; + return 0; + }else{ + return -1; + } + } + if( (f1&MEM_Real)!=0 ){ + if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ + testcase( f2 & MEM_Int ); + testcase( f2 & MEM_IntReal ); + return -sqlite3IntFloatCompare(pMem2->u.i, pMem1->u.r); + }else{ + return -1; + } + } + return +1; + } + + /* If one value is a string and the other is a blob, the string is less. + ** If both are strings, compare using the collating functions. + */ + if( combined_flags&MEM_Str ){ + if( (f1 & MEM_Str)==0 ){ + return 1; + } + if( (f2 & MEM_Str)==0 ){ + return -1; + } + + assert( pMem1->enc==pMem2->enc || pMem1->db->mallocFailed ); + assert( pMem1->enc==SQLITE_UTF8 || + pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); + + /* The collation sequence must be defined at this point, even if + ** the user deletes the collation sequence after the vdbe program is + ** compiled (this was not always the case). + */ + assert( !pColl || pColl->xCmp ); + + if( pColl ){ + return vdbeCompareMemString(pMem1, pMem2, pColl, 0); + } + /* If a NULL pointer was passed as the collate function, fall through + ** to the blob case and use memcmp(). */ + } + + /* Both values must be blobs. Compare using memcmp(). */ + return sqlite3BlobCompare(pMem1, pMem2); +} + + +/* +** The first argument passed to this function is a serial-type that +** corresponds to an integer - all values between 1 and 9 inclusive +** except 7. The second points to a buffer containing an integer value +** serialized according to serial_type. This function deserializes +** and returns the value. +*/ +static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ + u32 y; + assert( CORRUPT_DB || (serial_type>=1 && serial_type<=9 && serial_type!=7) ); + switch( serial_type ){ + case 0: + case 1: + testcase( aKey[0]&0x80 ); + return ONE_BYTE_INT(aKey); + case 2: + testcase( aKey[0]&0x80 ); + return TWO_BYTE_INT(aKey); + case 3: + testcase( aKey[0]&0x80 ); + return THREE_BYTE_INT(aKey); + case 4: { + testcase( aKey[0]&0x80 ); + y = FOUR_BYTE_UINT(aKey); + return (i64)*(int*)&y; + } + case 5: { + testcase( aKey[0]&0x80 ); + return FOUR_BYTE_UINT(aKey+2) + (((i64)1)<<32)*TWO_BYTE_INT(aKey); + } + case 6: { + u64 x = FOUR_BYTE_UINT(aKey); + testcase( aKey[0]&0x80 ); + x = (x<<32) | FOUR_BYTE_UINT(aKey+4); + return (i64)*(i64*)&x; + } + } + + return (serial_type - 8); +} + +/* +** This function compares the two table rows or index records +** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero +** or positive integer if key1 is less than, equal to or +** greater than key2. The {nKey1, pKey1} key must be a blob +** created by the OP_MakeRecord opcode of the VDBE. The pPKey2 +** key must be a parsed key such as obtained from +** sqlite3VdbeParseRecord. +** +** If argument bSkip is non-zero, it is assumed that the caller has already +** determined that the first fields of the keys are equal. +** +** Key1 and Key2 do not have to contain the same number of fields. If all +** fields that appear in both keys are equal, then pPKey2->default_rc is +** returned. +** +** If database corruption is discovered, set pPKey2->errCode to +** SQLITE_CORRUPT and return 0. If an OOM error is encountered, +** pPKey2->errCode is set to SQLITE_NOMEM and, if it is not NULL, the +** malloc-failed flag set on database handle (pPKey2->pKeyInfo->db). +*/ +SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2, /* Right key */ + int bSkip /* If true, skip the first field */ +){ + u32 d1; /* Offset into aKey[] of next data element */ + int i; /* Index of next field to compare */ + u32 szHdr1; /* Size of record header in bytes */ + u32 idx1; /* Offset of first type in header */ + int rc = 0; /* Return value */ + Mem *pRhs = pPKey2->aMem; /* Next field of pPKey2 to compare */ + KeyInfo *pKeyInfo; + const unsigned char *aKey1 = (const unsigned char *)pKey1; + Mem mem1; + + /* If bSkip is true, then the caller has already determined that the first + ** two elements in the keys are equal. Fix the various stack variables so + ** that this routine begins comparing at the second field. */ + if( bSkip ){ + u32 s1; + idx1 = 1 + getVarint32(&aKey1[1], s1); + szHdr1 = aKey1[0]; + d1 = szHdr1 + sqlite3VdbeSerialTypeLen(s1); + i = 1; + pRhs++; + }else{ + idx1 = getVarint32(aKey1, szHdr1); + d1 = szHdr1; + i = 0; + } + if( d1>(unsigned)nKey1 ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } + + VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ + assert( pPKey2->pKeyInfo->nAllField>=pPKey2->nField + || CORRUPT_DB ); + assert( pPKey2->pKeyInfo->aSortFlags!=0 ); + assert( pPKey2->pKeyInfo->nKeyField>0 ); + assert( idx1<=szHdr1 || CORRUPT_DB ); + do{ + u32 serial_type; + + /* RHS is an integer */ + if( pRhs->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pRhs->flags & MEM_Int ); + testcase( pRhs->flags & MEM_IntReal ); + serial_type = aKey1[idx1]; + testcase( serial_type==12 ); + if( serial_type>=10 ){ + rc = +1; + }else if( serial_type==0 ){ + rc = -1; + }else if( serial_type==7 ){ + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r); + }else{ + i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); + i64 rhs = pRhs->u.i; + if( lhsrhs ){ + rc = +1; + } + } + } + + /* RHS is real */ + else if( pRhs->flags & MEM_Real ){ + serial_type = aKey1[idx1]; + if( serial_type>=10 ){ + /* Serial types 12 or greater are strings and blobs (greater than + ** numbers). Types 10 and 11 are currently "reserved for future + ** use", so it doesn't really matter what the results of comparing + ** them to numberic values are. */ + rc = +1; + }else if( serial_type==0 ){ + rc = -1; + }else{ + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + if( serial_type==7 ){ + if( mem1.u.ru.r ){ + rc = -1; + }else if( mem1.u.r>pRhs->u.r ){ + rc = +1; + } + }else{ + rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r); + } + } + } + + /* RHS is a string */ + else if( pRhs->flags & MEM_Str ){ + getVarint32NR(&aKey1[idx1], serial_type); + testcase( serial_type==12 ); + if( serial_type<12 ){ + rc = -1; + }else if( !(serial_type & 0x01) ){ + rc = +1; + }else{ + mem1.n = (serial_type - 12) / 2; + testcase( (d1+mem1.n)==(unsigned)nKey1 ); + testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); + if( (d1+mem1.n) > (unsigned)nKey1 + || (pKeyInfo = pPKey2->pKeyInfo)->nAllField<=i + ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + }else if( pKeyInfo->aColl[i] ){ + mem1.enc = pKeyInfo->enc; + mem1.db = pKeyInfo->db; + mem1.flags = MEM_Str; + mem1.z = (char*)&aKey1[d1]; + rc = vdbeCompareMemString( + &mem1, pRhs, pKeyInfo->aColl[i], &pPKey2->errCode + ); + }else{ + int nCmp = MIN(mem1.n, pRhs->n); + rc = memcmp(&aKey1[d1], pRhs->z, nCmp); + if( rc==0 ) rc = mem1.n - pRhs->n; + } + } + } + + /* RHS is a blob */ + else if( pRhs->flags & MEM_Blob ){ + assert( (pRhs->flags & MEM_Zero)==0 || pRhs->n==0 ); + getVarint32NR(&aKey1[idx1], serial_type); + testcase( serial_type==12 ); + if( serial_type<12 || (serial_type & 0x01) ){ + rc = -1; + }else{ + int nStr = (serial_type - 12) / 2; + testcase( (d1+nStr)==(unsigned)nKey1 ); + testcase( (d1+nStr+1)==(unsigned)nKey1 ); + if( (d1+nStr) > (unsigned)nKey1 ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + }else if( pRhs->flags & MEM_Zero ){ + if( !isAllZero((const char*)&aKey1[d1],nStr) ){ + rc = 1; + }else{ + rc = nStr - pRhs->u.nZero; + } + }else{ + int nCmp = MIN(nStr, pRhs->n); + rc = memcmp(&aKey1[d1], pRhs->z, nCmp); + if( rc==0 ) rc = nStr - pRhs->n; + } + } + } + + /* RHS is null */ + else{ + serial_type = aKey1[idx1]; + rc = (serial_type!=0); + } + + if( rc!=0 ){ + int sortFlags = pPKey2->pKeyInfo->aSortFlags[i]; + if( sortFlags ){ + if( (sortFlags & KEYINFO_ORDER_BIGNULL)==0 + || ((sortFlags & KEYINFO_ORDER_DESC) + !=(serial_type==0 || (pRhs->flags&MEM_Null))) + ){ + rc = -rc; + } + } + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); + assert( mem1.szMalloc==0 ); /* See comment below */ + return rc; + } + + i++; + if( i==pPKey2->nField ) break; + pRhs++; + d1 += sqlite3VdbeSerialTypeLen(serial_type); + idx1 += sqlite3VarintLen(serial_type); + }while( idx1<(unsigned)szHdr1 && d1<=(unsigned)nKey1 ); + + /* No memory allocation is ever used on mem1. Prove this using + ** the following assert(). If the assert() fails, it indicates a + ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ + assert( mem1.szMalloc==0 ); + + /* rc==0 here means that one or both of the keys ran out of fields and + ** all the fields up to that point were equal. Return the default_rc + ** value. */ + assert( CORRUPT_DB + || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) + || pPKey2->pKeyInfo->db->mallocFailed + ); + pPKey2->eqSeen = 1; + return pPKey2->default_rc; +} +SQLITE_PRIVATE int sqlite3VdbeRecordCompare( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2 /* Right key */ +){ + return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 0); +} + + +/* +** This function is an optimized version of sqlite3VdbeRecordCompare() +** that (a) the first field of pPKey2 is an integer, and (b) the +** size-of-header varint at the start of (pKey1/nKey1) fits in a single +** byte (i.e. is less than 128). +** +** To avoid concerns about buffer overreads, this routine is only used +** on schemas where the maximum valid header size is 63 bytes or less. +*/ +static int vdbeRecordCompareInt( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2 /* Right key */ +){ + const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; + int serial_type = ((const u8*)pKey1)[1]; + int res; + u32 y; + u64 x; + i64 v; + i64 lhs; + + vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); + assert( (*(u8*)pKey1)<=0x3F || CORRUPT_DB ); + switch( serial_type ){ + case 1: { /* 1-byte signed integer */ + lhs = ONE_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 2: { /* 2-byte signed integer */ + lhs = TWO_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 3: { /* 3-byte signed integer */ + lhs = THREE_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 4: { /* 4-byte signed integer */ + y = FOUR_BYTE_UINT(aKey); + lhs = (i64)*(int*)&y; + testcase( lhs<0 ); + break; + } + case 5: { /* 6-byte signed integer */ + lhs = FOUR_BYTE_UINT(aKey+2) + (((i64)1)<<32)*TWO_BYTE_INT(aKey); + testcase( lhs<0 ); + break; + } + case 6: { /* 8-byte signed integer */ + x = FOUR_BYTE_UINT(aKey); + x = (x<<32) | FOUR_BYTE_UINT(aKey+4); + lhs = *(i64*)&x; + testcase( lhs<0 ); + break; + } + case 8: + lhs = 0; + break; + case 9: + lhs = 1; + break; + + /* This case could be removed without changing the results of running + ** this code. Including it causes gcc to generate a faster switch + ** statement (since the range of switch targets now starts at zero and + ** is contiguous) but does not cause any duplicate code to be generated + ** (as gcc is clever enough to combine the two like cases). Other + ** compilers might be similar. */ + case 0: case 7: + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); + + default: + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); + } + + v = pPKey2->aMem[0].u.i; + if( v>lhs ){ + res = pPKey2->r1; + }else if( vr2; + }else if( pPKey2->nField>1 ){ + /* The first fields of the two keys are equal. Compare the trailing + ** fields. */ + res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); + }else{ + /* The first fields of the two keys are equal and there are no trailing + ** fields. Return pPKey2->default_rc in this case. */ + res = pPKey2->default_rc; + pPKey2->eqSeen = 1; + } + + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) ); + return res; +} + +/* +** This function is an optimized version of sqlite3VdbeRecordCompare() +** that (a) the first field of pPKey2 is a string, that (b) the first field +** uses the collation sequence BINARY and (c) that the size-of-header varint +** at the start of (pKey1/nKey1) fits in a single byte. +*/ +static int vdbeRecordCompareString( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2 /* Right key */ +){ + const u8 *aKey1 = (const u8*)pKey1; + int serial_type; + int res; + + assert( pPKey2->aMem[0].flags & MEM_Str ); + vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); + serial_type = (u8)(aKey1[1]); + if( serial_type >= 0x80 ){ + sqlite3GetVarint32(&aKey1[1], (u32*)&serial_type); + } + if( serial_type<12 ){ + res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ + }else if( !(serial_type & 0x01) ){ + res = pPKey2->r2; /* (pKey1/nKey1) is a blob */ + }else{ + int nCmp; + int nStr; + int szHdr = aKey1[0]; + + nStr = (serial_type-12) / 2; + if( (szHdr + nStr) > nKey1 ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } + nCmp = MIN( pPKey2->aMem[0].n, nStr ); + res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); + + if( res>0 ){ + res = pPKey2->r2; + }else if( res<0 ){ + res = pPKey2->r1; + }else{ + res = nStr - pPKey2->aMem[0].n; + if( res==0 ){ + if( pPKey2->nField>1 ){ + res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); + }else{ + res = pPKey2->default_rc; + pPKey2->eqSeen = 1; + } + }else if( res>0 ){ + res = pPKey2->r2; + }else{ + res = pPKey2->r1; + } + } + } + + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) + || CORRUPT_DB + || pPKey2->pKeyInfo->db->mallocFailed + ); + return res; +} + +/* +** Return a pointer to an sqlite3VdbeRecordCompare() compatible function +** suitable for comparing serialized records to the unpacked record passed +** as the only argument. +*/ +SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ + /* varintRecordCompareInt() and varintRecordCompareString() both assume + ** that the size-of-header varint that occurs at the start of each record + ** fits in a single byte (i.e. is 127 or less). varintRecordCompareInt() + ** also assumes that it is safe to overread a buffer by at least the + ** maximum possible legal header size plus 8 bytes. Because there is + ** guaranteed to be at least 74 (but not 136) bytes of padding following each + ** buffer passed to varintRecordCompareInt() this makes it convenient to + ** limit the size of the header to 64 bytes in cases where the first field + ** is an integer. + ** + ** The easiest way to enforce this limit is to consider only records with + ** 13 fields or less. If the first field is an integer, the maximum legal + ** header size is (12*5 + 1 + 1) bytes. */ + if( p->pKeyInfo->nAllField<=13 ){ + int flags = p->aMem[0].flags; + if( p->pKeyInfo->aSortFlags[0] ){ + if( p->pKeyInfo->aSortFlags[0] & KEYINFO_ORDER_BIGNULL ){ + return sqlite3VdbeRecordCompare; + } + p->r1 = 1; + p->r2 = -1; + }else{ + p->r1 = -1; + p->r2 = 1; + } + if( (flags & MEM_Int) ){ + return vdbeRecordCompareInt; + } + testcase( flags & MEM_Real ); + testcase( flags & MEM_Null ); + testcase( flags & MEM_Blob ); + if( (flags & (MEM_Real|MEM_IntReal|MEM_Null|MEM_Blob))==0 + && p->pKeyInfo->aColl[0]==0 + ){ + assert( flags & MEM_Str ); + return vdbeRecordCompareString; + } + } + + return sqlite3VdbeRecordCompare; +} + +/* +** pCur points at an index entry created using the OP_MakeRecord opcode. +** Read the rowid (the last field in the record) and store it in *rowid. +** Return SQLITE_OK if everything works, or an error code otherwise. +** +** pCur might be pointing to text obtained from a corrupt database file. +** So the content cannot be trusted. Do appropriate checks on the content. +*/ +SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ + i64 nCellKey = 0; + int rc; + u32 szHdr; /* Size of the header */ + u32 typeRowid; /* Serial type of the rowid */ + u32 lenRowid; /* Size of the rowid */ + Mem m, v; + + /* Get the size of the index entry. Only indices entries of less + ** than 2GiB are support - anything large must be database corruption. + ** Any corruption is detected in sqlite3BtreeParseCellPtr(), though, so + ** this code can safely assume that nCellKey is 32-bits + */ + assert( sqlite3BtreeCursorIsValid(pCur) ); + nCellKey = sqlite3BtreePayloadSize(pCur); + assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey ); + + /* Read in the complete content of the index entry */ + sqlite3VdbeMemInit(&m, db, 0); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m); + if( rc ){ + return rc; + } + + /* The index entry must begin with a header size */ + getVarint32NR((u8*)m.z, szHdr); + testcase( szHdr==3 ); + testcase( szHdr==m.n ); + testcase( szHdr>0x7fffffff ); + assert( m.n>=0 ); + if( unlikely(szHdr<3 || szHdr>(unsigned)m.n) ){ + goto idx_rowid_corruption; + } + + /* The last field of the index should be an integer - the ROWID. + ** Verify that the last entry really is an integer. */ + getVarint32NR((u8*)&m.z[szHdr-1], typeRowid); + testcase( typeRowid==1 ); + testcase( typeRowid==2 ); + testcase( typeRowid==3 ); + testcase( typeRowid==4 ); + testcase( typeRowid==5 ); + testcase( typeRowid==6 ); + testcase( typeRowid==8 ); + testcase( typeRowid==9 ); + if( unlikely(typeRowid<1 || typeRowid>9 || typeRowid==7) ){ + goto idx_rowid_corruption; + } + lenRowid = sqlite3SmallTypeSizes[typeRowid]; + testcase( (u32)m.n==szHdr+lenRowid ); + if( unlikely((u32)m.neCurType==CURTYPE_BTREE ); + pCur = pC->uc.pCursor; + assert( sqlite3BtreeCursorIsValid(pCur) ); + nCellKey = sqlite3BtreePayloadSize(pCur); + /* nCellKey will always be between 0 and 0xffffffff because of the way + ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ + if( nCellKey<=0 || nCellKey>0x7fffffff ){ + *res = 0; + return SQLITE_CORRUPT_BKPT; + } + sqlite3VdbeMemInit(&m, db, 0); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m); + if( rc ){ + return rc; + } + *res = sqlite3VdbeRecordCompareWithSkip(m.n, m.z, pUnpacked, 0); + sqlite3VdbeMemRelease(&m); + return SQLITE_OK; +} + +/* +** This routine sets the value to be returned by subsequent calls to +** sqlite3_changes() on the database handle 'db'. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *db, i64 nChange){ + assert( sqlite3_mutex_held(db->mutex) ); + db->nChange = nChange; + db->nTotalChange += nChange; +} + +/* +** Set a flag in the vdbe to update the change counter when it is finalised +** or reset. +*/ +SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe *v){ + v->changeCntOn = 1; +} + +/* +** Mark every prepared statement associated with a database connection +** as expired. +** +** An expired statement means that recompilation of the statement is +** recommend. Statements expire when things happen that make their +** programs obsolete. Removing user-defined functions or collating +** sequences, or changing an authorization function are the types of +** things that make prepared statements obsolete. +** +** If iCode is 1, then expiration is advisory. The statement should +** be reprepared before being restarted, but if it is already running +** it is allowed to run to completion. +** +** Internally, this function just sets the Vdbe.expired flag on all +** prepared statements. The flag is set to 1 for an immediate expiration +** and set to 2 for an advisory expiration. +*/ +SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db, int iCode){ + Vdbe *p; + for(p = db->pVdbe; p; p=p->pNext){ + p->expired = iCode+1; + } +} + +/* +** Return the database associated with the Vdbe. +*/ +SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe *v){ + return v->db; +} + +/* +** Return the SQLITE_PREPARE flags for a Vdbe. +*/ +SQLITE_PRIVATE u8 sqlite3VdbePrepareFlags(Vdbe *v){ + return v->prepFlags; +} + +/* +** Return a pointer to an sqlite3_value structure containing the value bound +** parameter iVar of VM v. Except, if the value is an SQL NULL, return +** 0 instead. Unless it is NULL, apply affinity aff (one of the SQLITE_AFF_* +** constants) to the value before returning it. +** +** The returned value must be freed by the caller using sqlite3ValueFree(). +*/ +SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff){ + assert( iVar>0 ); + if( v ){ + Mem *pMem = &v->aVar[iVar-1]; + assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + if( 0==(pMem->flags & MEM_Null) ){ + sqlite3_value *pRet = sqlite3ValueNew(v->db); + if( pRet ){ + sqlite3VdbeMemCopy((Mem *)pRet, pMem); + sqlite3ValueApplyAffinity(pRet, aff, SQLITE_UTF8); + } + return pRet; + } + } + return 0; +} + +/* +** Configure SQL variable iVar so that binding a new value to it signals +** to sqlite3_reoptimize() that re-preparing the statement may result +** in a better query plan. +*/ +SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ + assert( iVar>0 ); + assert( (v->db->flags & SQLITE_EnableQPSG)==0 ); + if( iVar>=32 ){ + v->expmask |= 0x80000000; + }else{ + v->expmask |= ((u32)1 << (iVar-1)); + } +} + +/* +** Cause a function to throw an error if it was call from OP_PureFunc +** rather than OP_Function. +** +** OP_PureFunc means that the function must be deterministic, and should +** throw an error if it is given inputs that would make it non-deterministic. +** This routine is invoked by date/time functions that use non-deterministic +** features such as 'now'. +*/ +SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context *pCtx){ + const VdbeOp *pOp; +#ifdef SQLITE_ENABLE_STAT4 + if( pCtx->pVdbe==0 ) return 1; +#endif + pOp = pCtx->pVdbe->aOp + pCtx->iOp; + if( pOp->opcode==OP_PureFunc ){ + const char *zContext; + char *zMsg; + if( pOp->p5 & NC_IsCheck ){ + zContext = "a CHECK constraint"; + }else if( pOp->p5 & NC_GenCol ){ + zContext = "a generated column"; + }else{ + zContext = "an index"; + } + zMsg = sqlite3_mprintf("non-deterministic use of %s() in %s", + pCtx->pFunc->zName, zContext); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); + return 0; + } + return 1; +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored +** in memory obtained from sqlite3_malloc) into a Vdbe.zErrMsg (text stored +** in memory obtained from sqlite3DbMalloc). +*/ +SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ + if( pVtab->zErrMsg ){ + sqlite3 *db = p->db; + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, pVtab->zErrMsg); + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; + } +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + +/* +** If the second argument is not NULL, release any allocations associated +** with the memory cells in the p->aMem[] array. Also free the UnpackedRecord +** structure itself, using sqlite3DbFree(). +** +** This function is used to free UnpackedRecord structures allocated by +** the vdbeUnpackRecord() function found in vdbeapi.c. +*/ +static void vdbeFreeUnpacked(sqlite3 *db, int nField, UnpackedRecord *p){ + if( p ){ + int i; + for(i=0; iaMem[i]; + if( pMem->zMalloc ) sqlite3VdbeMemRelease(pMem); + } + sqlite3DbFreeNN(db, p); + } +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** Invoke the pre-update hook. If this is an UPDATE or DELETE pre-update call, +** then cursor passed as the second argument should point to the row about +** to be update or deleted. If the application calls sqlite3_preupdate_old(), +** the required value will be read from the row the cursor points to. +*/ +SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( + Vdbe *v, /* Vdbe pre-update hook is invoked by */ + VdbeCursor *pCsr, /* Cursor to grab old.* values from */ + int op, /* SQLITE_INSERT, UPDATE or DELETE */ + const char *zDb, /* Database name */ + Table *pTab, /* Modified table */ + i64 iKey1, /* Initial key value */ + int iReg, /* Register for new.* record */ + int iBlobWrite +){ + sqlite3 *db = v->db; + i64 iKey2; + PreUpdate preupdate; + const char *zTbl = pTab->zName; + static const u8 fakeSortOrder = 0; + + assert( db->pPreUpdate==0 ); + memset(&preupdate, 0, sizeof(PreUpdate)); + if( HasRowid(pTab)==0 ){ + iKey1 = iKey2 = 0; + preupdate.pPk = sqlite3PrimaryKeyIndex(pTab); + }else{ + if( op==SQLITE_UPDATE ){ + iKey2 = v->aMem[iReg].u.i; + }else{ + iKey2 = iKey1; + } + } + + assert( pCsr->nField==pTab->nCol + || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1) + ); + + preupdate.v = v; + preupdate.pCsr = pCsr; + preupdate.op = op; + preupdate.iNewReg = iReg; + preupdate.keyinfo.db = db; + preupdate.keyinfo.enc = ENC(db); + preupdate.keyinfo.nKeyField = pTab->nCol; + preupdate.keyinfo.aSortFlags = (u8*)&fakeSortOrder; + preupdate.iKey1 = iKey1; + preupdate.iKey2 = iKey2; + preupdate.pTab = pTab; + preupdate.iBlobWrite = iBlobWrite; + + db->pPreUpdate = &preupdate; + db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2); + db->pPreUpdate = 0; + sqlite3DbFree(db, preupdate.aRecord); + vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pUnpacked); + vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pNewUnpacked); + if( preupdate.aNew ){ + int i; + for(i=0; inField; i++){ + sqlite3VdbeMemRelease(&preupdate.aNew[i]); + } + sqlite3DbFreeNN(db, preupdate.aNew); + } +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +/************** End of vdbeaux.c *********************************************/ +/************** Begin file vdbeapi.c *****************************************/ +/* +** 2004 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code use to implement APIs that are part of the +** VDBE. +*/ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Return TRUE (non-zero) of the statement supplied as an argument needs +** to be recompiled. A statement needs to be recompiled whenever the +** execution environment changes in a way that would alter the program +** that sqlite3_prepare() generates. For example, if new functions or +** collating sequences are registered or if an authorizer function is +** added or changed. +*/ +SQLITE_API int sqlite3_expired(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + return p==0 || p->expired; +} +#endif + +/* +** Check on a Vdbe to make sure it has not been finalized. Log +** an error and return true if it has been finalized (or is otherwise +** invalid). Return false if it is ok. +*/ +static int vdbeSafety(Vdbe *p){ + if( p->db==0 ){ + sqlite3_log(SQLITE_MISUSE, "API called with finalized prepared statement"); + return 1; + }else{ + return 0; + } +} +static int vdbeSafetyNotNull(Vdbe *p){ + if( p==0 ){ + sqlite3_log(SQLITE_MISUSE, "API called with NULL prepared statement"); + return 1; + }else{ + return vdbeSafety(p); + } +} + +#ifndef SQLITE_OMIT_TRACE +/* +** Invoke the profile callback. This routine is only called if we already +** know that the profile callback is defined and needs to be invoked. +*/ +static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ + sqlite3_int64 iNow; + sqlite3_int64 iElapse; + assert( p->startTime>0 ); + assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 ); + assert( db->init.busy==0 ); + assert( p->zSql!=0 ); + sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); + iElapse = (iNow - p->startTime)*1000000; +#ifndef SQLITE_OMIT_DEPRECATED + if( db->xProfile ){ + db->xProfile(db->pProfileArg, p->zSql, iElapse); + } +#endif + if( db->mTrace & SQLITE_TRACE_PROFILE ){ + db->trace.xV2(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse); + } + p->startTime = 0; +} +/* +** The checkProfileCallback(DB,P) macro checks to see if a profile callback +** is needed, and it invokes the callback if it is needed. +*/ +# define checkProfileCallback(DB,P) \ + if( ((P)->startTime)>0 ){ invokeProfileCallback(DB,P); } +#else +# define checkProfileCallback(DB,P) /*no-op*/ +#endif + +/* +** The following routine destroys a virtual machine that is created by +** the sqlite3_compile() routine. The integer returned is an SQLITE_ +** success/failure code that describes the result of executing the virtual +** machine. +** +** This routine sets the error code and string returned by +** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). +*/ +SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ + int rc; + if( pStmt==0 ){ + /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL + ** pointer is a harmless no-op. */ + rc = SQLITE_OK; + }else{ + Vdbe *v = (Vdbe*)pStmt; + sqlite3 *db = v->db; + if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; + sqlite3_mutex_enter(db->mutex); + checkProfileCallback(db, v); + rc = sqlite3VdbeFinalize(v); + rc = sqlite3ApiExit(db, rc); + sqlite3LeaveMutexAndCloseZombie(db); + } + return rc; +} + +/* +** Terminate the current execution of an SQL statement and reset it +** back to its starting state so that it can be reused. A success code from +** the prior execution is returned. +** +** This routine sets the error code and string returned by +** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). +*/ +SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt){ + int rc; + if( pStmt==0 ){ + rc = SQLITE_OK; + }else{ + Vdbe *v = (Vdbe*)pStmt; + sqlite3 *db = v->db; + sqlite3_mutex_enter(db->mutex); + checkProfileCallback(db, v); + rc = sqlite3VdbeReset(v); + sqlite3VdbeRewind(v); + assert( (rc & (db->errMask))==rc ); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + } + return rc; +} + +/* +** Set all the parameters in the compiled SQL statement to NULL. +*/ +SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ + int i; + int rc = SQLITE_OK; + Vdbe *p = (Vdbe*)pStmt; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = ((Vdbe*)pStmt)->db->mutex; +#endif + sqlite3_mutex_enter(mutex); + for(i=0; inVar; i++){ + sqlite3VdbeMemRelease(&p->aVar[i]); + p->aVar[i].flags = MEM_Null; + } + assert( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 || p->expmask==0 ); + if( p->expmask ){ + p->expired = 1; + } + sqlite3_mutex_leave(mutex); + return rc; +} + + +/**************************** sqlite3_value_ ******************************* +** The following routines extract information from a Mem or sqlite3_value +** structure. +*/ +SQLITE_API const void *sqlite3_value_blob(sqlite3_value *pVal){ + Mem *p = (Mem*)pVal; + if( p->flags & (MEM_Blob|MEM_Str) ){ + if( ExpandBlob(p)!=SQLITE_OK ){ + assert( p->flags==MEM_Null && p->z==0 ); + return 0; + } + p->flags |= MEM_Blob; + return p->n ? p->z : 0; + }else{ + return sqlite3_value_text(pVal); + } +} +SQLITE_API int sqlite3_value_bytes(sqlite3_value *pVal){ + return sqlite3ValueBytes(pVal, SQLITE_UTF8); +} +SQLITE_API int sqlite3_value_bytes16(sqlite3_value *pVal){ + return sqlite3ValueBytes(pVal, SQLITE_UTF16NATIVE); +} +SQLITE_API double sqlite3_value_double(sqlite3_value *pVal){ + return sqlite3VdbeRealValue((Mem*)pVal); +} +SQLITE_API int sqlite3_value_int(sqlite3_value *pVal){ + return (int)sqlite3VdbeIntValue((Mem*)pVal); +} +SQLITE_API sqlite_int64 sqlite3_value_int64(sqlite3_value *pVal){ + return sqlite3VdbeIntValue((Mem*)pVal); +} +SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value *pVal){ + Mem *pMem = (Mem*)pVal; + return ((pMem->flags & MEM_Subtype) ? pMem->eSubtype : 0); +} +SQLITE_API void *sqlite3_value_pointer(sqlite3_value *pVal, const char *zPType){ + Mem *p = (Mem*)pVal; + if( (p->flags&(MEM_TypeMask|MEM_Term|MEM_Subtype)) == + (MEM_Null|MEM_Term|MEM_Subtype) + && zPType!=0 + && p->eSubtype=='p' + && strcmp(p->u.zPType, zPType)==0 + ){ + return (void*)p->z; + }else{ + return 0; + } +} +SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value *pVal){ + return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_value_text16(sqlite3_value* pVal){ + return sqlite3ValueText(pVal, SQLITE_UTF16NATIVE); +} +SQLITE_API const void *sqlite3_value_text16be(sqlite3_value *pVal){ + return sqlite3ValueText(pVal, SQLITE_UTF16BE); +} +SQLITE_API const void *sqlite3_value_text16le(sqlite3_value *pVal){ + return sqlite3ValueText(pVal, SQLITE_UTF16LE); +} +#endif /* SQLITE_OMIT_UTF16 */ +/* EVIDENCE-OF: R-12793-43283 Every value in SQLite has one of five +** fundamental datatypes: 64-bit signed integer 64-bit IEEE floating +** point number string BLOB NULL +*/ +SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ + static const u8 aType[] = { + SQLITE_BLOB, /* 0x00 (not possible) */ + SQLITE_NULL, /* 0x01 NULL */ + SQLITE_TEXT, /* 0x02 TEXT */ + SQLITE_NULL, /* 0x03 (not possible) */ + SQLITE_INTEGER, /* 0x04 INTEGER */ + SQLITE_NULL, /* 0x05 (not possible) */ + SQLITE_INTEGER, /* 0x06 INTEGER + TEXT */ + SQLITE_NULL, /* 0x07 (not possible) */ + SQLITE_FLOAT, /* 0x08 FLOAT */ + SQLITE_NULL, /* 0x09 (not possible) */ + SQLITE_FLOAT, /* 0x0a FLOAT + TEXT */ + SQLITE_NULL, /* 0x0b (not possible) */ + SQLITE_INTEGER, /* 0x0c (not possible) */ + SQLITE_NULL, /* 0x0d (not possible) */ + SQLITE_INTEGER, /* 0x0e (not possible) */ + SQLITE_NULL, /* 0x0f (not possible) */ + SQLITE_BLOB, /* 0x10 BLOB */ + SQLITE_NULL, /* 0x11 (not possible) */ + SQLITE_TEXT, /* 0x12 (not possible) */ + SQLITE_NULL, /* 0x13 (not possible) */ + SQLITE_INTEGER, /* 0x14 INTEGER + BLOB */ + SQLITE_NULL, /* 0x15 (not possible) */ + SQLITE_INTEGER, /* 0x16 (not possible) */ + SQLITE_NULL, /* 0x17 (not possible) */ + SQLITE_FLOAT, /* 0x18 FLOAT + BLOB */ + SQLITE_NULL, /* 0x19 (not possible) */ + SQLITE_FLOAT, /* 0x1a (not possible) */ + SQLITE_NULL, /* 0x1b (not possible) */ + SQLITE_INTEGER, /* 0x1c (not possible) */ + SQLITE_NULL, /* 0x1d (not possible) */ + SQLITE_INTEGER, /* 0x1e (not possible) */ + SQLITE_NULL, /* 0x1f (not possible) */ + SQLITE_FLOAT, /* 0x20 INTREAL */ + SQLITE_NULL, /* 0x21 (not possible) */ + SQLITE_TEXT, /* 0x22 INTREAL + TEXT */ + SQLITE_NULL, /* 0x23 (not possible) */ + SQLITE_FLOAT, /* 0x24 (not possible) */ + SQLITE_NULL, /* 0x25 (not possible) */ + SQLITE_FLOAT, /* 0x26 (not possible) */ + SQLITE_NULL, /* 0x27 (not possible) */ + SQLITE_FLOAT, /* 0x28 (not possible) */ + SQLITE_NULL, /* 0x29 (not possible) */ + SQLITE_FLOAT, /* 0x2a (not possible) */ + SQLITE_NULL, /* 0x2b (not possible) */ + SQLITE_FLOAT, /* 0x2c (not possible) */ + SQLITE_NULL, /* 0x2d (not possible) */ + SQLITE_FLOAT, /* 0x2e (not possible) */ + SQLITE_NULL, /* 0x2f (not possible) */ + SQLITE_BLOB, /* 0x30 (not possible) */ + SQLITE_NULL, /* 0x31 (not possible) */ + SQLITE_TEXT, /* 0x32 (not possible) */ + SQLITE_NULL, /* 0x33 (not possible) */ + SQLITE_FLOAT, /* 0x34 (not possible) */ + SQLITE_NULL, /* 0x35 (not possible) */ + SQLITE_FLOAT, /* 0x36 (not possible) */ + SQLITE_NULL, /* 0x37 (not possible) */ + SQLITE_FLOAT, /* 0x38 (not possible) */ + SQLITE_NULL, /* 0x39 (not possible) */ + SQLITE_FLOAT, /* 0x3a (not possible) */ + SQLITE_NULL, /* 0x3b (not possible) */ + SQLITE_FLOAT, /* 0x3c (not possible) */ + SQLITE_NULL, /* 0x3d (not possible) */ + SQLITE_FLOAT, /* 0x3e (not possible) */ + SQLITE_NULL, /* 0x3f (not possible) */ + }; +#ifdef SQLITE_DEBUG + { + int eType = SQLITE_BLOB; + if( pVal->flags & MEM_Null ){ + eType = SQLITE_NULL; + }else if( pVal->flags & (MEM_Real|MEM_IntReal) ){ + eType = SQLITE_FLOAT; + }else if( pVal->flags & MEM_Int ){ + eType = SQLITE_INTEGER; + }else if( pVal->flags & MEM_Str ){ + eType = SQLITE_TEXT; + } + assert( eType == aType[pVal->flags&MEM_AffMask] ); + } +#endif + return aType[pVal->flags&MEM_AffMask]; +} + +/* Return true if a parameter to xUpdate represents an unchanged column */ +SQLITE_API int sqlite3_value_nochange(sqlite3_value *pVal){ + return (pVal->flags&(MEM_Null|MEM_Zero))==(MEM_Null|MEM_Zero); +} + +/* Return true if a parameter value originated from an sqlite3_bind() */ +SQLITE_API int sqlite3_value_frombind(sqlite3_value *pVal){ + return (pVal->flags&MEM_FromBind)!=0; +} + +/* Make a copy of an sqlite3_value object +*/ +SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value *pOrig){ + sqlite3_value *pNew; + if( pOrig==0 ) return 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return 0; + memset(pNew, 0, sizeof(*pNew)); + memcpy(pNew, pOrig, MEMCELLSIZE); + pNew->flags &= ~MEM_Dyn; + pNew->db = 0; + if( pNew->flags&(MEM_Str|MEM_Blob) ){ + pNew->flags &= ~(MEM_Static|MEM_Dyn); + pNew->flags |= MEM_Ephem; + if( sqlite3VdbeMemMakeWriteable(pNew)!=SQLITE_OK ){ + sqlite3ValueFree(pNew); + pNew = 0; + } + } + return pNew; +} + +/* Destroy an sqlite3_value object previously obtained from +** sqlite3_value_dup(). +*/ +SQLITE_API void sqlite3_value_free(sqlite3_value *pOld){ + sqlite3ValueFree(pOld); +} + + +/**************************** sqlite3_result_ ******************************* +** The following routines are used by user-defined functions to specify +** the function result. +** +** The setStrOrError() function calls sqlite3VdbeMemSetStr() to store the +** result as a string or blob but if the string or blob is too large, it +** then sets the error code to SQLITE_TOOBIG +** +** The invokeValueDestructor(P,X) routine invokes destructor function X() +** on value P is not going to be used and need to be destroyed. +*/ +static void setResultStrOrError( + sqlite3_context *pCtx, /* Function context */ + const char *z, /* String pointer */ + int n, /* Bytes in string, or negative */ + u8 enc, /* Encoding of z. 0 for BLOBs */ + void (*xDel)(void*) /* Destructor function */ +){ + if( sqlite3VdbeMemSetStr(pCtx->pOut, z, n, enc, xDel)==SQLITE_TOOBIG ){ + sqlite3_result_error_toobig(pCtx); + } +} +static int invokeValueDestructor( + const void *p, /* Value to destroy */ + void (*xDel)(void*), /* The destructor */ + sqlite3_context *pCtx /* Set a SQLITE_TOOBIG error if no NULL */ +){ + assert( xDel!=SQLITE_DYNAMIC ); + if( xDel==0 ){ + /* noop */ + }else if( xDel==SQLITE_TRANSIENT ){ + /* noop */ + }else{ + xDel((void*)p); + } + sqlite3_result_error_toobig(pCtx); + return SQLITE_TOOBIG; +} +SQLITE_API void sqlite3_result_blob( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( n>=0 ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + setResultStrOrError(pCtx, z, n, 0, xDel); +} +SQLITE_API void sqlite3_result_blob64( + sqlite3_context *pCtx, + const void *z, + sqlite3_uint64 n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + assert( xDel!=SQLITE_DYNAMIC ); + if( n>0x7fffffff ){ + (void)invokeValueDestructor(z, xDel, pCtx); + }else{ + setResultStrOrError(pCtx, z, (int)n, 0, xDel); + } +} +SQLITE_API void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetDouble(pCtx->pOut, rVal); +} +SQLITE_API void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + pCtx->isError = SQLITE_ERROR; + sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + pCtx->isError = SQLITE_ERROR; + sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); +} +#endif +SQLITE_API void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetInt64(pCtx->pOut, (i64)iVal); +} +SQLITE_API void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetInt64(pCtx->pOut, iVal); +} +SQLITE_API void sqlite3_result_null(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetNull(pCtx->pOut); +} +SQLITE_API void sqlite3_result_pointer( + sqlite3_context *pCtx, + void *pPtr, + const char *zPType, + void (*xDestructor)(void*) +){ + Mem *pOut = pCtx->pOut; + assert( sqlite3_mutex_held(pOut->db->mutex) ); + sqlite3VdbeMemRelease(pOut); + pOut->flags = MEM_Null; + sqlite3VdbeMemSetPointer(pOut, pPtr, zPType, xDestructor); +} +SQLITE_API void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ + Mem *pOut = pCtx->pOut; + assert( sqlite3_mutex_held(pOut->db->mutex) ); + pOut->eSubtype = eSubtype & 0xff; + pOut->flags |= MEM_Subtype; +} +SQLITE_API void sqlite3_result_text( + sqlite3_context *pCtx, + const char *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); +} +SQLITE_API void sqlite3_result_text64( + sqlite3_context *pCtx, + const char *z, + sqlite3_uint64 n, + void (*xDel)(void *), + unsigned char enc +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + assert( xDel!=SQLITE_DYNAMIC ); + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( n>0x7fffffff ){ + (void)invokeValueDestructor(z, xDel, pCtx); + }else{ + setResultStrOrError(pCtx, z, (int)n, enc, xDel); + } +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API void sqlite3_result_text16( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); +} +SQLITE_API void sqlite3_result_text16be( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); +} +SQLITE_API void sqlite3_result_text16le( + sqlite3_context *pCtx, + const void *z, + int n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); +} +#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_API void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemCopy(pCtx->pOut, pValue); +} +SQLITE_API void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetZeroBlob(pCtx->pOut, n); +} +SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ + Mem *pOut = pCtx->pOut; + assert( sqlite3_mutex_held(pOut->db->mutex) ); + if( n>(u64)pOut->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + return SQLITE_TOOBIG; + } + sqlite3VdbeMemSetZeroBlob(pCtx->pOut, (int)n); + return SQLITE_OK; +} +SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ + pCtx->isError = errCode ? errCode : -1; +#ifdef SQLITE_DEBUG + if( pCtx->pVdbe ) pCtx->pVdbe->rcApp = errCode; +#endif + if( pCtx->pOut->flags & MEM_Null ){ + sqlite3VdbeMemSetStr(pCtx->pOut, sqlite3ErrStr(errCode), -1, + SQLITE_UTF8, SQLITE_STATIC); + } +} + +/* Force an SQLITE_TOOBIG error. */ +SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + pCtx->isError = SQLITE_TOOBIG; + sqlite3VdbeMemSetStr(pCtx->pOut, "string or blob too big", -1, + SQLITE_UTF8, SQLITE_STATIC); +} + +/* An SQLITE_NOMEM error. */ +SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetNull(pCtx->pOut); + pCtx->isError = SQLITE_NOMEM_BKPT; + sqlite3OomFault(pCtx->pOut->db); +} + +#ifndef SQLITE_UNTESTABLE +/* Force the INT64 value currently stored as the result to be +** a MEM_IntReal value. See the SQLITE_TESTCTRL_RESULT_INTREAL +** test-control. +*/ +SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + if( pCtx->pOut->flags & MEM_Int ){ + pCtx->pOut->flags &= ~MEM_Int; + pCtx->pOut->flags |= MEM_IntReal; + } +} +#endif + + +/* +** This function is called after a transaction has been committed. It +** invokes callbacks registered with sqlite3_wal_hook() as required. +*/ +static int doWalCallbacks(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_WAL + int i; + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + int nEntry; + sqlite3BtreeEnter(pBt); + nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); + if( nEntry>0 && db->xWalCallback && rc==SQLITE_OK ){ + rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zDbSName, nEntry); + } + } + } +#endif + return rc; +} + + +/* +** Execute the statement pStmt, either until a row of data is ready, the +** statement is completely executed or an error occurs. +** +** This routine implements the bulk of the logic behind the sqlite_step() +** API. The only thing omitted is the automatic recompile if a +** schema change has occurred. That detail is handled by the +** outer sqlite3_step() wrapper procedure. +*/ +static int sqlite3Step(Vdbe *p){ + sqlite3 *db; + int rc; + + assert(p); + if( p->iVdbeMagic!=VDBE_MAGIC_RUN ){ + /* We used to require that sqlite3_reset() be called before retrying + ** sqlite3_step() after any error or after SQLITE_DONE. But beginning + ** with version 3.7.0, we changed this so that sqlite3_reset() would + ** be called automatically instead of throwing the SQLITE_MISUSE error. + ** This "automatic-reset" change is not technically an incompatibility, + ** since any application that receives an SQLITE_MISUSE is broken by + ** definition. + ** + ** Nevertheless, some published applications that were originally written + ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE + ** returns, and those were broken by the automatic-reset change. As a + ** a work-around, the SQLITE_OMIT_AUTORESET compile-time restores the + ** legacy behavior of returning SQLITE_MISUSE for cases where the + ** previous sqlite3_step() returned something other than a SQLITE_LOCKED + ** or SQLITE_BUSY error. + */ +#ifdef SQLITE_OMIT_AUTORESET + if( (rc = p->rc&0xff)==SQLITE_BUSY || rc==SQLITE_LOCKED ){ + sqlite3_reset((sqlite3_stmt*)p); + }else{ + return SQLITE_MISUSE_BKPT; + } +#else + sqlite3_reset((sqlite3_stmt*)p); +#endif + } + + /* Check that malloc() has not failed. If it has, return early. */ + db = p->db; + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; + } + + if( p->pc<0 && p->expired ){ + p->rc = SQLITE_SCHEMA; + rc = SQLITE_ERROR; + if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){ + /* If this statement was prepared using saved SQL and an + ** error has occurred, then return the error code in p->rc to the + ** caller. Set the error code in the database handle to the same value. + */ + rc = sqlite3VdbeTransferError(p); + } + goto end_of_step; + } + if( p->pc<0 ){ + /* If there are no other statements currently running, then + ** reset the interrupt flag. This prevents a call to sqlite3_interrupt + ** from interrupting a statement that has not yet started. + */ + if( db->nVdbeActive==0 ){ + AtomicStore(&db->u1.isInterrupted, 0); + } + + assert( db->nVdbeWrite>0 || db->autoCommit==0 + || (db->nDeferredCons==0 && db->nDeferredImmCons==0) + ); + +#ifndef SQLITE_OMIT_TRACE + if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 + && !db->init.busy && p->zSql ){ + sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); + }else{ + assert( p->startTime==0 ); + } +#endif + + db->nVdbeActive++; + if( p->readOnly==0 ) db->nVdbeWrite++; + if( p->bIsReader ) db->nVdbeRead++; + p->pc = 0; + } +#ifdef SQLITE_DEBUG + p->rcApp = SQLITE_OK; +#endif +#ifndef SQLITE_OMIT_EXPLAIN + if( p->explain ){ + rc = sqlite3VdbeList(p); + }else +#endif /* SQLITE_OMIT_EXPLAIN */ + { + db->nVdbeExec++; + rc = sqlite3VdbeExec(p); + db->nVdbeExec--; + } + + if( rc!=SQLITE_ROW ){ +#ifndef SQLITE_OMIT_TRACE + /* If the statement completed successfully, invoke the profile callback */ + checkProfileCallback(db, p); +#endif + + if( rc==SQLITE_DONE && db->autoCommit ){ + assert( p->rc==SQLITE_OK ); + p->rc = doWalCallbacks(db); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; + } + }else if( rc!=SQLITE_DONE && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){ + /* If this statement was prepared using saved SQL and an + ** error has occurred, then return the error code in p->rc to the + ** caller. Set the error code in the database handle to the same value. + */ + rc = sqlite3VdbeTransferError(p); + } + } + + db->errCode = rc; + if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){ + p->rc = SQLITE_NOMEM_BKPT; + if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ) rc = p->rc; + } +end_of_step: + /* There are only a limited number of result codes allowed from the + ** statements prepared using the legacy sqlite3_prepare() interface */ + assert( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 + || rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR + || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE + ); + return (rc&db->errMask); +} + +/* +** This is the top-level implementation of sqlite3_step(). Call +** sqlite3Step() to do most of the work. If a schema error occurs, +** call sqlite3Reprepare() and try again. +*/ +SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ + int rc = SQLITE_OK; /* Result from sqlite3Step() */ + Vdbe *v = (Vdbe*)pStmt; /* the prepared statement */ + int cnt = 0; /* Counter to prevent infinite loop of reprepares */ + sqlite3 *db; /* The database connection */ + + if( vdbeSafetyNotNull(v) ){ + return SQLITE_MISUSE_BKPT; + } + db = v->db; + sqlite3_mutex_enter(db->mutex); + v->doingRerun = 0; + while( (rc = sqlite3Step(v))==SQLITE_SCHEMA + && cnt++ < SQLITE_MAX_SCHEMA_RETRY ){ + int savedPc = v->pc; + rc = sqlite3Reprepare(v); + if( rc!=SQLITE_OK ){ + /* This case occurs after failing to recompile an sql statement. + ** The error message from the SQL compiler has already been loaded + ** into the database handle. This block copies the error message + ** from the database handle into the statement and sets the statement + ** program counter to 0 to ensure that when the statement is + ** finalized or reset the parser error message is available via + ** sqlite3_errmsg() and sqlite3_errcode(). + */ + const char *zErr = (const char *)sqlite3_value_text(db->pErr); + sqlite3DbFree(db, v->zErrMsg); + if( !db->mallocFailed ){ + v->zErrMsg = sqlite3DbStrDup(db, zErr); + v->rc = rc = sqlite3ApiExit(db, rc); + } else { + v->zErrMsg = 0; + v->rc = rc = SQLITE_NOMEM_BKPT; + } + break; + } + sqlite3_reset(pStmt); + if( savedPc>=0 ) v->doingRerun = 1; + assert( v->expired==0 ); + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + + +/* +** Extract the user data from a sqlite3_context structure and return a +** pointer to it. +*/ +SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ + assert( p && p->pFunc ); + return p->pFunc->pUserData; +} + +/* +** Extract the user data from a sqlite3_context structure and return a +** pointer to it. +** +** IMPLEMENTATION-OF: R-46798-50301 The sqlite3_context_db_handle() interface +** returns a copy of the pointer to the database connection (the 1st +** parameter) of the sqlite3_create_function() and +** sqlite3_create_function16() routines that originally registered the +** application defined function. +*/ +SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ + assert( p && p->pOut ); + return p->pOut->db; +} + +/* +** If this routine is invoked from within an xColumn method of a virtual +** table, then it returns true if and only if the the call is during an +** UPDATE operation and the value of the column will not be modified +** by the UPDATE. +** +** If this routine is called from any context other than within the +** xColumn method of a virtual table, then the return value is meaningless +** and arbitrary. +** +** Virtual table implements might use this routine to optimize their +** performance by substituting a NULL result, or some other light-weight +** value, as a signal to the xUpdate routine that the column is unchanged. +*/ +SQLITE_API int sqlite3_vtab_nochange(sqlite3_context *p){ + assert( p ); + return sqlite3_value_nochange(p->pOut); +} + +/* +** Return the current time for a statement. If the current time +** is requested more than once within the same run of a single prepared +** statement, the exact same time is returned for each invocation regardless +** of the amount of time that elapses between invocations. In other words, +** the time returned is always the time of the first call. +*/ +SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context *p){ + int rc; +#ifndef SQLITE_ENABLE_STAT4 + sqlite3_int64 *piTime = &p->pVdbe->iCurrentTime; + assert( p->pVdbe!=0 ); +#else + sqlite3_int64 iTime = 0; + sqlite3_int64 *piTime = p->pVdbe!=0 ? &p->pVdbe->iCurrentTime : &iTime; +#endif + if( *piTime==0 ){ + rc = sqlite3OsCurrentTimeInt64(p->pOut->db->pVfs, piTime); + if( rc ) *piTime = 0; + } + return *piTime; +} + +/* +** Create a new aggregate context for p and return a pointer to +** its pMem->z element. +*/ +static SQLITE_NOINLINE void *createAggContext(sqlite3_context *p, int nByte){ + Mem *pMem = p->pMem; + assert( (pMem->flags & MEM_Agg)==0 ); + if( nByte<=0 ){ + sqlite3VdbeMemSetNull(pMem); + pMem->z = 0; + }else{ + sqlite3VdbeMemClearAndResize(pMem, nByte); + pMem->flags = MEM_Agg; + pMem->u.pDef = p->pFunc; + if( pMem->z ){ + memset(pMem->z, 0, nByte); + } + } + return (void*)pMem->z; +} + +/* +** Allocate or return the aggregate context for a user function. A new +** context is allocated on the first call. Subsequent calls return the +** same context that was returned on prior calls. +*/ +SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ + assert( p && p->pFunc && p->pFunc->xFinalize ); + assert( sqlite3_mutex_held(p->pOut->db->mutex) ); + testcase( nByte<0 ); + if( (p->pMem->flags & MEM_Agg)==0 ){ + return createAggContext(p, nByte); + }else{ + return (void*)p->pMem->z; + } +} + +/* +** Return the auxiliary data pointer, if any, for the iArg'th argument to +** the user-function defined by pCtx. +** +** The left-most argument is 0. +** +** Undocumented behavior: If iArg is negative then access a cache of +** auxiliary data pointers that is available to all functions within a +** single prepared statement. The iArg values must match. +*/ +SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ + AuxData *pAuxData; + + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); +#if SQLITE_ENABLE_STAT4 + if( pCtx->pVdbe==0 ) return 0; +#else + assert( pCtx->pVdbe!=0 ); +#endif + for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNextAux){ + if( pAuxData->iAuxArg==iArg && (pAuxData->iAuxOp==pCtx->iOp || iArg<0) ){ + return pAuxData->pAux; + } + } + return 0; +} + +/* +** Set the auxiliary data pointer and delete function, for the iArg'th +** argument to the user-function defined by pCtx. Any previous value is +** deleted by calling the delete function specified when it was set. +** +** The left-most argument is 0. +** +** Undocumented behavior: If iArg is negative then make the data available +** to all functions within the current prepared statement using iArg as an +** access code. +*/ +SQLITE_API void sqlite3_set_auxdata( + sqlite3_context *pCtx, + int iArg, + void *pAux, + void (*xDelete)(void*) +){ + AuxData *pAuxData; + Vdbe *pVdbe = pCtx->pVdbe; + + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); +#ifdef SQLITE_ENABLE_STAT4 + if( pVdbe==0 ) goto failed; +#else + assert( pVdbe!=0 ); +#endif + + for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNextAux){ + if( pAuxData->iAuxArg==iArg && (pAuxData->iAuxOp==pCtx->iOp || iArg<0) ){ + break; + } + } + if( pAuxData==0 ){ + pAuxData = sqlite3DbMallocZero(pVdbe->db, sizeof(AuxData)); + if( !pAuxData ) goto failed; + pAuxData->iAuxOp = pCtx->iOp; + pAuxData->iAuxArg = iArg; + pAuxData->pNextAux = pVdbe->pAuxData; + pVdbe->pAuxData = pAuxData; + if( pCtx->isError==0 ) pCtx->isError = -1; + }else if( pAuxData->xDeleteAux ){ + pAuxData->xDeleteAux(pAuxData->pAux); + } + + pAuxData->pAux = pAux; + pAuxData->xDeleteAux = xDelete; + return; + +failed: + if( xDelete ){ + xDelete(pAux); + } +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Return the number of times the Step function of an aggregate has been +** called. +** +** This function is deprecated. Do not use it for new code. It is +** provide only to avoid breaking legacy code. New aggregate function +** implementations should keep their own counts within their aggregate +** context. +*/ +SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ + assert( p && p->pMem && p->pFunc && p->pFunc->xFinalize ); + return p->pMem->n; +} +#endif + +/* +** Return the number of columns in the result set for the statement pStmt. +*/ +SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ + Vdbe *pVm = (Vdbe *)pStmt; + return pVm ? pVm->nResColumn : 0; +} + +/* +** Return the number of values available from the current row of the +** currently executing statement pStmt. +*/ +SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt){ + Vdbe *pVm = (Vdbe *)pStmt; + if( pVm==0 || pVm->pResultSet==0 ) return 0; + return pVm->nResColumn; +} + +/* +** Return a pointer to static memory containing an SQL NULL value. +*/ +static const Mem *columnNullValue(void){ + /* Even though the Mem structure contains an element + ** of type i64, on certain architectures (x86) with certain compiler + ** switches (-Os), gcc may align this Mem object on a 4-byte boundary + ** instead of an 8-byte one. This all works fine, except that when + ** running with SQLITE_DEBUG defined the SQLite code sometimes assert()s + ** that a Mem structure is located on an 8-byte boundary. To prevent + ** these assert()s from failing, when building with SQLITE_DEBUG defined + ** using gcc, we force nullMem to be 8-byte aligned using the magical + ** __attribute__((aligned(8))) macro. */ + static const Mem nullMem +#if defined(SQLITE_DEBUG) && defined(__GNUC__) + __attribute__((aligned(8))) +#endif + = { + /* .u = */ {0}, + /* .flags = */ (u16)MEM_Null, + /* .enc = */ (u8)0, + /* .eSubtype = */ (u8)0, + /* .n = */ (int)0, + /* .z = */ (char*)0, + /* .zMalloc = */ (char*)0, + /* .szMalloc = */ (int)0, + /* .uTemp = */ (u32)0, + /* .db = */ (sqlite3*)0, + /* .xDel = */ (void(*)(void*))0, +#ifdef SQLITE_DEBUG + /* .pScopyFrom = */ (Mem*)0, + /* .mScopyFlags= */ 0, +#endif + }; + return &nullMem; +} + +/* +** Check to see if column iCol of the given statement is valid. If +** it is, return a pointer to the Mem for the value of that column. +** If iCol is not valid, return a pointer to a Mem which has a value +** of NULL. +*/ +static Mem *columnMem(sqlite3_stmt *pStmt, int i){ + Vdbe *pVm; + Mem *pOut; + + pVm = (Vdbe *)pStmt; + if( pVm==0 ) return (Mem*)columnNullValue(); + assert( pVm->db ); + sqlite3_mutex_enter(pVm->db->mutex); + if( pVm->pResultSet!=0 && inResColumn && i>=0 ){ + pOut = &pVm->pResultSet[i]; + }else{ + sqlite3Error(pVm->db, SQLITE_RANGE); + pOut = (Mem*)columnNullValue(); + } + return pOut; +} + +/* +** This function is called after invoking an sqlite3_value_XXX function on a +** column value (i.e. a value returned by evaluating an SQL expression in the +** select list of a SELECT statement) that may cause a malloc() failure. If +** malloc() has failed, the threads mallocFailed flag is cleared and the result +** code of statement pStmt set to SQLITE_NOMEM. +** +** Specifically, this is called from within: +** +** sqlite3_column_int() +** sqlite3_column_int64() +** sqlite3_column_text() +** sqlite3_column_text16() +** sqlite3_column_real() +** sqlite3_column_bytes() +** sqlite3_column_bytes16() +** sqiite3_column_blob() +*/ +static void columnMallocFailure(sqlite3_stmt *pStmt) +{ + /* If malloc() failed during an encoding conversion within an + ** sqlite3_column_XXX API, then set the return code of the statement to + ** SQLITE_NOMEM. The next call to _step() (if any) will return SQLITE_ERROR + ** and _finalize() will return NOMEM. + */ + Vdbe *p = (Vdbe *)pStmt; + if( p ){ + assert( p->db!=0 ); + assert( sqlite3_mutex_held(p->db->mutex) ); + p->rc = sqlite3ApiExit(p->db, p->rc); + sqlite3_mutex_leave(p->db->mutex); + } +} + +/**************************** sqlite3_column_ ******************************* +** The following routines are used to access elements of the current row +** in the result set. +*/ +SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ + const void *val; + val = sqlite3_value_blob( columnMem(pStmt,i) ); + /* Even though there is no encoding conversion, value_blob() might + ** need to call malloc() to expand the result of a zeroblob() + ** expression. + */ + columnMallocFailure(pStmt); + return val; +} +SQLITE_API int sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ + int val = sqlite3_value_bytes( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt *pStmt, int i){ + int val = sqlite3_value_bytes16( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API double sqlite3_column_double(sqlite3_stmt *pStmt, int i){ + double val = sqlite3_value_double( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API int sqlite3_column_int(sqlite3_stmt *pStmt, int i){ + int val = sqlite3_value_int( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API sqlite_int64 sqlite3_column_int64(sqlite3_stmt *pStmt, int i){ + sqlite_int64 val = sqlite3_value_int64( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt *pStmt, int i){ + const unsigned char *val = sqlite3_value_text( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt *pStmt, int i){ + Mem *pOut = columnMem(pStmt, i); + if( pOut->flags&MEM_Static ){ + pOut->flags &= ~MEM_Static; + pOut->flags |= MEM_Ephem; + } + columnMallocFailure(pStmt); + return (sqlite3_value *)pOut; +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt *pStmt, int i){ + const void *val = sqlite3_value_text16( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return val; +} +#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_API int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ + int iType = sqlite3_value_type( columnMem(pStmt,i) ); + columnMallocFailure(pStmt); + return iType; +} + +/* +** Convert the N-th element of pStmt->pColName[] into a string using +** xFunc() then return that string. If N is out of range, return 0. +** +** There are up to 5 names for each column. useType determines which +** name is returned. Here are the names: +** +** 0 The column name as it should be displayed for output +** 1 The datatype name for the column +** 2 The name of the database that the column derives from +** 3 The name of the table that the column derives from +** 4 The name of the table column that the result column derives from +** +** If the result is not a simple column reference (if it is an expression +** or a constant) then useTypes 2, 3, and 4 return NULL. +*/ +static const void *columnName( + sqlite3_stmt *pStmt, /* The statement */ + int N, /* Which column to get the name for */ + int useUtf16, /* True to return the name as UTF16 */ + int useType /* What type of name */ +){ + const void *ret; + Vdbe *p; + int n; + sqlite3 *db; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + ret = 0; + p = (Vdbe *)pStmt; + db = p->db; + assert( db!=0 ); + n = sqlite3_column_count(pStmt); + if( N=0 ){ + N += useType*n; + sqlite3_mutex_enter(db->mutex); + assert( db->mallocFailed==0 ); +#ifndef SQLITE_OMIT_UTF16 + if( useUtf16 ){ + ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); + }else +#endif + { + ret = sqlite3_value_text((sqlite3_value*)&p->aColName[N]); + } + /* A malloc may have failed inside of the _text() call. If this + ** is the case, clear the mallocFailed flag and return NULL. + */ + if( db->mallocFailed ){ + sqlite3OomClear(db); + ret = 0; + } + sqlite3_mutex_leave(db->mutex); + } + return ret; +} + +/* +** Return the name of the Nth column of the result set returned by SQL +** statement pStmt. +*/ +SQLITE_API const char *sqlite3_column_name(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 0, COLNAME_NAME); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 1, COLNAME_NAME); +} +#endif + +/* +** Constraint: If you have ENABLE_COLUMN_METADATA then you must +** not define OMIT_DECLTYPE. +*/ +#if defined(SQLITE_OMIT_DECLTYPE) && defined(SQLITE_ENABLE_COLUMN_METADATA) +# error "Must not define both SQLITE_OMIT_DECLTYPE \ + and SQLITE_ENABLE_COLUMN_METADATA" +#endif + +#ifndef SQLITE_OMIT_DECLTYPE +/* +** Return the column declaration type (if applicable) of the 'i'th column +** of the result set of SQL statement pStmt. +*/ +SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 0, COLNAME_DECLTYPE); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 1, COLNAME_DECLTYPE); +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_OMIT_DECLTYPE */ + +#ifdef SQLITE_ENABLE_COLUMN_METADATA +/* +** Return the name of the database from which a result column derives. +** NULL is returned if the result column is an expression or constant or +** anything else which is not an unambiguous reference to a database column. +*/ +SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 0, COLNAME_DATABASE); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 1, COLNAME_DATABASE); +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Return the name of the table from which a result column derives. +** NULL is returned if the result column is an expression or constant or +** anything else which is not an unambiguous reference to a database column. +*/ +SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 0, COLNAME_TABLE); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 1, COLNAME_TABLE); +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Return the name of the table column from which a result column derives. +** NULL is returned if the result column is an expression or constant or +** anything else which is not an unambiguous reference to a database column. +*/ +SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 0, COLNAME_COLUMN); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 1, COLNAME_COLUMN); +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_ENABLE_COLUMN_METADATA */ + + +/******************************* sqlite3_bind_ *************************** +** +** Routines used to attach values to wildcards in a compiled SQL statement. +*/ +/* +** Unbind the value bound to variable i in virtual machine p. This is the +** the same as binding a NULL value to the column. If the "i" parameter is +** out of range, then SQLITE_RANGE is returned. Othewise SQLITE_OK. +** +** A successful evaluation of this routine acquires the mutex on p. +** the mutex is released if any kind of error occurs. +** +** The error code stored in database p->db is overwritten with the return +** value in any case. +*/ +static int vdbeUnbind(Vdbe *p, int i){ + Mem *pVar; + if( vdbeSafetyNotNull(p) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(p->db->mutex); + if( p->iVdbeMagic!=VDBE_MAGIC_RUN || p->pc>=0 ){ + sqlite3Error(p->db, SQLITE_MISUSE); + sqlite3_mutex_leave(p->db->mutex); + sqlite3_log(SQLITE_MISUSE, + "bind on a busy prepared statement: [%s]", p->zSql); + return SQLITE_MISUSE_BKPT; + } + if( i<1 || i>p->nVar ){ + sqlite3Error(p->db, SQLITE_RANGE); + sqlite3_mutex_leave(p->db->mutex); + return SQLITE_RANGE; + } + i--; + pVar = &p->aVar[i]; + sqlite3VdbeMemRelease(pVar); + pVar->flags = MEM_Null; + p->db->errCode = SQLITE_OK; + + /* If the bit corresponding to this variable in Vdbe.expmask is set, then + ** binding a new value to this variable invalidates the current query plan. + ** + ** IMPLEMENTATION-OF: R-57496-20354 If the specific value bound to a host + ** parameter in the WHERE clause might influence the choice of query plan + ** for a statement, then the statement will be automatically recompiled, + ** as if there had been a schema change, on the first sqlite3_step() call + ** following any change to the bindings of that parameter. + */ + assert( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 || p->expmask==0 ); + if( p->expmask!=0 && (p->expmask & (i>=31 ? 0x80000000 : (u32)1<expired = 1; + } + return SQLITE_OK; +} + +/* +** Bind a text or BLOB value. +*/ +static int bindText( + sqlite3_stmt *pStmt, /* The statement to bind against */ + int i, /* Index of the parameter to bind */ + const void *zData, /* Pointer to the data to be bound */ + i64 nData, /* Number of bytes of data to be bound */ + void (*xDel)(void*), /* Destructor for the data */ + u8 encoding /* Encoding for the data */ +){ + Vdbe *p = (Vdbe *)pStmt; + Mem *pVar; + int rc; + + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + if( zData!=0 ){ + pVar = &p->aVar[i-1]; + rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); + if( rc==SQLITE_OK && encoding!=0 ){ + rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); + } + if( rc ){ + sqlite3Error(p->db, rc); + rc = sqlite3ApiExit(p->db, rc); + } + } + sqlite3_mutex_leave(p->db->mutex); + }else if( xDel!=SQLITE_STATIC && xDel!=SQLITE_TRANSIENT ){ + xDel((void*)zData); + } + return rc; +} + + +/* +** Bind a blob value to an SQL statement variable. +*/ +SQLITE_API int sqlite3_bind_blob( + sqlite3_stmt *pStmt, + int i, + const void *zData, + int nData, + void (*xDel)(void*) +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( nData<0 ) return SQLITE_MISUSE_BKPT; +#endif + return bindText(pStmt, i, zData, nData, xDel, 0); +} +SQLITE_API int sqlite3_bind_blob64( + sqlite3_stmt *pStmt, + int i, + const void *zData, + sqlite3_uint64 nData, + void (*xDel)(void*) +){ + assert( xDel!=SQLITE_DYNAMIC ); + return bindText(pStmt, i, zData, nData, xDel, 0); +} +SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue); + sqlite3_mutex_leave(p->db->mutex); + } + return rc; +} +SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ + return sqlite3_bind_int64(p, i, (i64)iValue); +} +SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); + sqlite3_mutex_leave(p->db->mutex); + } + return rc; +} +SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ + int rc; + Vdbe *p = (Vdbe*)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3_mutex_leave(p->db->mutex); + } + return rc; +} +SQLITE_API int sqlite3_bind_pointer( + sqlite3_stmt *pStmt, + int i, + void *pPtr, + const char *zPTtype, + void (*xDestructor)(void*) +){ + int rc; + Vdbe *p = (Vdbe*)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zPTtype, xDestructor); + sqlite3_mutex_leave(p->db->mutex); + }else if( xDestructor ){ + xDestructor(pPtr); + } + return rc; +} +SQLITE_API int sqlite3_bind_text( + sqlite3_stmt *pStmt, + int i, + const char *zData, + int nData, + void (*xDel)(void*) +){ + return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); +} +SQLITE_API int sqlite3_bind_text64( + sqlite3_stmt *pStmt, + int i, + const char *zData, + sqlite3_uint64 nData, + void (*xDel)(void*), + unsigned char enc +){ + assert( xDel!=SQLITE_DYNAMIC ); + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + return bindText(pStmt, i, zData, nData, xDel, enc); +} +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API int sqlite3_bind_text16( + sqlite3_stmt *pStmt, + int i, + const void *zData, + int nData, + void (*xDel)(void*) +){ + return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); +} +#endif /* SQLITE_OMIT_UTF16 */ +SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ + int rc; + switch( sqlite3_value_type((sqlite3_value*)pValue) ){ + case SQLITE_INTEGER: { + rc = sqlite3_bind_int64(pStmt, i, pValue->u.i); + break; + } + case SQLITE_FLOAT: { + rc = sqlite3_bind_double(pStmt, i, pValue->u.r); + break; + } + case SQLITE_BLOB: { + if( pValue->flags & MEM_Zero ){ + rc = sqlite3_bind_zeroblob(pStmt, i, pValue->u.nZero); + }else{ + rc = sqlite3_bind_blob(pStmt, i, pValue->z, pValue->n,SQLITE_TRANSIENT); + } + break; + } + case SQLITE_TEXT: { + rc = bindText(pStmt,i, pValue->z, pValue->n, SQLITE_TRANSIENT, + pValue->enc); + break; + } + default: { + rc = sqlite3_bind_null(pStmt, i); + break; + } + } + return rc; +} +SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + rc = vdbeUnbind(p, i); + if( rc==SQLITE_OK ){ + sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); + sqlite3_mutex_leave(p->db->mutex); + } + return rc; +} +SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i, sqlite3_uint64 n){ + int rc; + Vdbe *p = (Vdbe *)pStmt; + sqlite3_mutex_enter(p->db->mutex); + if( n>(u64)p->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + rc = SQLITE_TOOBIG; + }else{ + assert( (n & 0x7FFFFFFF)==n ); + rc = sqlite3_bind_zeroblob(pStmt, i, n); + } + rc = sqlite3ApiExit(p->db, rc); + sqlite3_mutex_leave(p->db->mutex); + return rc; +} + +/* +** Return the number of wildcards that can be potentially bound to. +** This routine is added to support DBD::SQLite. +*/ +SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + return p ? p->nVar : 0; +} + +/* +** Return the name of a wildcard parameter. Return NULL if the index +** is out of range or if the wildcard is unnamed. +** +** The result is always UTF-8. +*/ +SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ + Vdbe *p = (Vdbe*)pStmt; + if( p==0 ) return 0; + return sqlite3VListNumToName(p->pVList, i); +} + +/* +** Given a wildcard parameter name, return the index of the variable +** with that name. If there is no variable with the given name, +** return 0. +*/ +SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){ + if( p==0 || zName==0 ) return 0; + return sqlite3VListNameToNum(p->pVList, zName, nName); +} +SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ + return sqlite3VdbeParameterIndex((Vdbe*)pStmt, zName, sqlite3Strlen30(zName)); +} + +/* +** Transfer all bindings from the first statement over to the second. +*/ +SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ + Vdbe *pFrom = (Vdbe*)pFromStmt; + Vdbe *pTo = (Vdbe*)pToStmt; + int i; + assert( pTo->db==pFrom->db ); + assert( pTo->nVar==pFrom->nVar ); + sqlite3_mutex_enter(pTo->db->mutex); + for(i=0; inVar; i++){ + sqlite3VdbeMemMove(&pTo->aVar[i], &pFrom->aVar[i]); + } + sqlite3_mutex_leave(pTo->db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Deprecated external interface. Internal/core SQLite code +** should call sqlite3TransferBindings. +** +** It is misuse to call this routine with statements from different +** database connections. But as this is a deprecated interface, we +** will not bother to check for that condition. +** +** If the two statements contain a different number of bindings, then +** an SQLITE_ERROR is returned. Nothing else can go wrong, so otherwise +** SQLITE_OK is returned. +*/ +SQLITE_API int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ + Vdbe *pFrom = (Vdbe*)pFromStmt; + Vdbe *pTo = (Vdbe*)pToStmt; + if( pFrom->nVar!=pTo->nVar ){ + return SQLITE_ERROR; + } + assert( (pTo->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 || pTo->expmask==0 ); + if( pTo->expmask ){ + pTo->expired = 1; + } + assert( (pFrom->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 || pFrom->expmask==0 ); + if( pFrom->expmask ){ + pFrom->expired = 1; + } + return sqlite3TransferBindings(pFromStmt, pToStmt); +} +#endif + +/* +** Return the sqlite3* database handle to which the prepared statement given +** in the argument belongs. This is the same database handle that was +** the first argument to the sqlite3_prepare() that was used to create +** the statement in the first place. +*/ +SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->db : 0; +} + +/* +** Return true if the prepared statement is guaranteed to not modify the +** database. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->readOnly : 1; +} + +/* +** Return 1 if the statement is an EXPLAIN and return 2 if the +** statement is an EXPLAIN QUERY PLAN +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->explain : 0; +} + +/* +** Return true if the prepared statement is in need of being reset. +*/ +SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ + Vdbe *v = (Vdbe*)pStmt; + return v!=0 && v->iVdbeMagic==VDBE_MAGIC_RUN && v->pc>=0; +} + +/* +** Return a pointer to the next prepared statement after pStmt associated +** with database connection pDb. If pStmt is NULL, return the first +** prepared statement for the database connection. Return NULL if there +** are no more. +*/ +SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ + sqlite3_stmt *pNext; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(pDb->mutex); + if( pStmt==0 ){ + pNext = (sqlite3_stmt*)pDb->pVdbe; + }else{ + pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pNext; + } + sqlite3_mutex_leave(pDb->mutex); + return pNext; +} + +/* +** Return the value of a status counter for a prepared statement +*/ +SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ + Vdbe *pVdbe = (Vdbe*)pStmt; + u32 v; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !pStmt + || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||op>=ArraySize(pVdbe->aCounter))) + ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + if( op==SQLITE_STMTSTATUS_MEMUSED ){ + sqlite3 *db = pVdbe->db; + sqlite3_mutex_enter(db->mutex); + v = 0; + db->pnBytesFreed = (int*)&v; + sqlite3VdbeClearObject(db, pVdbe); + sqlite3DbFree(db, pVdbe); + db->pnBytesFreed = 0; + sqlite3_mutex_leave(db->mutex); + }else{ + v = pVdbe->aCounter[op]; + if( resetFlag ) pVdbe->aCounter[op] = 0; + } + return (int)v; +} + +/* +** Return the SQL associated with a prepared statement +*/ +SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe *)pStmt; + return p ? p->zSql : 0; +} + +/* +** Return the SQL associated with a prepared statement with +** bound parameters expanded. Space to hold the returned string is +** obtained from sqlite3_malloc(). The caller is responsible for +** freeing the returned string by passing it to sqlite3_free(). +** +** The SQLITE_TRACE_SIZE_LIMIT puts an upper bound on the size of +** expanded bound parameters. +*/ +SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){ +#ifdef SQLITE_OMIT_TRACE + return 0; +#else + char *z = 0; + const char *zSql = sqlite3_sql(pStmt); + if( zSql ){ + Vdbe *p = (Vdbe *)pStmt; + sqlite3_mutex_enter(p->db->mutex); + z = sqlite3VdbeExpandSql(p, zSql); + sqlite3_mutex_leave(p->db->mutex); + } + return z; +#endif +} + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Return the normalized SQL associated with a prepared statement. +*/ +SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe *)pStmt; + if( p==0 ) return 0; + if( p->zNormSql==0 && ALWAYS(p->zSql!=0) ){ + sqlite3_mutex_enter(p->db->mutex); + p->zNormSql = sqlite3Normalize(p, p->zSql); + sqlite3_mutex_leave(p->db->mutex); + } + return p->zNormSql; +} +#endif /* SQLITE_ENABLE_NORMALIZE */ + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** Allocate and populate an UnpackedRecord structure based on the serialized +** record in nKey/pKey. Return a pointer to the new UnpackedRecord structure +** if successful, or a NULL pointer if an OOM error is encountered. +*/ +static UnpackedRecord *vdbeUnpackRecord( + KeyInfo *pKeyInfo, + int nKey, + const void *pKey +){ + UnpackedRecord *pRet; /* Return value */ + + pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); + if( pRet ){ + memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nKeyField+1)); + sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet); + } + return pRet; +} + +/* +** This function is called from within a pre-update callback to retrieve +** a field of the row currently being updated or deleted. +*/ +SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ + PreUpdate *p = db->pPreUpdate; + Mem *pMem; + int rc = SQLITE_OK; + + /* Test that this call is being made from within an SQLITE_DELETE or + ** SQLITE_UPDATE pre-update callback, and that iIdx is within range. */ + if( !p || p->op==SQLITE_INSERT ){ + rc = SQLITE_MISUSE_BKPT; + goto preupdate_old_out; + } + if( p->pPk ){ + iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); + } + if( iIdx>=p->pCsr->nField || iIdx<0 ){ + rc = SQLITE_RANGE; + goto preupdate_old_out; + } + + /* If the old.* record has not yet been loaded into memory, do so now. */ + if( p->pUnpacked==0 ){ + u32 nRec; + u8 *aRec; + + nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor); + aRec = sqlite3DbMallocRaw(db, nRec); + if( !aRec ) goto preupdate_old_out; + rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec); + if( rc==SQLITE_OK ){ + p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec); + if( !p->pUnpacked ) rc = SQLITE_NOMEM; + } + if( rc!=SQLITE_OK ){ + sqlite3DbFree(db, aRec); + goto preupdate_old_out; + } + p->aRecord = aRec; + } + + pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey1); + }else if( iIdx>=p->pUnpacked->nField ){ + *ppValue = (sqlite3_value *)columnNullValue(); + }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ + if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_IntReal ); + sqlite3VdbeMemRealify(pMem); + } + } + + preupdate_old_out: + sqlite3Error(db, rc); + return sqlite3ApiExit(db, rc); +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** This function is called from within a pre-update callback to retrieve +** the number of columns in the row being updated, deleted or inserted. +*/ +SQLITE_API int sqlite3_preupdate_count(sqlite3 *db){ + PreUpdate *p = db->pPreUpdate; + return (p ? p->keyinfo.nKeyField : 0); +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** This function is designed to be called from within a pre-update callback +** only. It returns zero if the change that caused the callback was made +** immediately by a user SQL statement. Or, if the change was made by a +** trigger program, it returns the number of trigger programs currently +** on the stack (1 for a top-level trigger, 2 for a trigger fired by a +** top-level trigger etc.). +** +** For the purposes of the previous paragraph, a foreign key CASCADE, SET NULL +** or SET DEFAULT action is considered a trigger. +*/ +SQLITE_API int sqlite3_preupdate_depth(sqlite3 *db){ + PreUpdate *p = db->pPreUpdate; + return (p ? p->v->nFrame : 0); +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** This function is designed to be called from within a pre-update callback +** only. +*/ +SQLITE_API int sqlite3_preupdate_blobwrite(sqlite3 *db){ + PreUpdate *p = db->pPreUpdate; + return (p ? p->iBlobWrite : -1); +} +#endif + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** This function is called from within a pre-update callback to retrieve +** a field of the row currently being updated or inserted. +*/ +SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ + PreUpdate *p = db->pPreUpdate; + int rc = SQLITE_OK; + Mem *pMem; + + if( !p || p->op==SQLITE_DELETE ){ + rc = SQLITE_MISUSE_BKPT; + goto preupdate_new_out; + } + if( p->pPk && p->op!=SQLITE_UPDATE ){ + iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); + } + if( iIdx>=p->pCsr->nField || iIdx<0 ){ + rc = SQLITE_RANGE; + goto preupdate_new_out; + } + + if( p->op==SQLITE_INSERT ){ + /* For an INSERT, memory cell p->iNewReg contains the serialized record + ** that is being inserted. Deserialize it. */ + UnpackedRecord *pUnpack = p->pNewUnpacked; + if( !pUnpack ){ + Mem *pData = &p->v->aMem[p->iNewReg]; + rc = ExpandBlob(pData); + if( rc!=SQLITE_OK ) goto preupdate_new_out; + pUnpack = vdbeUnpackRecord(&p->keyinfo, pData->n, pData->z); + if( !pUnpack ){ + rc = SQLITE_NOMEM; + goto preupdate_new_out; + } + p->pNewUnpacked = pUnpack; + } + pMem = &pUnpack->aMem[iIdx]; + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey2); + }else if( iIdx>=pUnpack->nField ){ + pMem = (sqlite3_value *)columnNullValue(); + } + }else{ + /* For an UPDATE, memory cell (p->iNewReg+1+iIdx) contains the required + ** value. Make a copy of the cell contents and return a pointer to it. + ** It is not safe to return a pointer to the memory cell itself as the + ** caller may modify the value text encoding. + */ + assert( p->op==SQLITE_UPDATE ); + if( !p->aNew ){ + p->aNew = (Mem *)sqlite3DbMallocZero(db, sizeof(Mem) * p->pCsr->nField); + if( !p->aNew ){ + rc = SQLITE_NOMEM; + goto preupdate_new_out; + } + } + assert( iIdx>=0 && iIdxpCsr->nField ); + pMem = &p->aNew[iIdx]; + if( pMem->flags==0 ){ + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey2); + }else{ + rc = sqlite3VdbeMemCopy(pMem, &p->v->aMem[p->iNewReg+1+iIdx]); + if( rc!=SQLITE_OK ) goto preupdate_new_out; + } + } + } + *ppValue = pMem; + + preupdate_new_out: + sqlite3Error(db, rc); + return sqlite3ApiExit(db, rc); +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Return status data for a single loop within query pStmt. +*/ +SQLITE_API int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement being queried */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Which metric to return */ + void *pOut /* OUT: Write the answer here */ +){ + Vdbe *p = (Vdbe*)pStmt; + ScanStatus *pScan; + if( idx<0 || idx>=p->nScan ) return 1; + pScan = &p->aScan[idx]; + switch( iScanStatusOp ){ + case SQLITE_SCANSTAT_NLOOP: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + break; + } + case SQLITE_SCANSTAT_NVISIT: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + break; + } + case SQLITE_SCANSTAT_EST: { + double r = 1.0; + LogEst x = pScan->nEst; + while( x<100 ){ + x += 10; + r *= 0.5; + } + *(double*)pOut = r*sqlite3LogEstToInt(x); + break; + } + case SQLITE_SCANSTAT_NAME: { + *(const char**)pOut = pScan->zName; + break; + } + case SQLITE_SCANSTAT_EXPLAIN: { + if( pScan->addrExplain ){ + *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + }else{ + *(const char**)pOut = 0; + } + break; + } + case SQLITE_SCANSTAT_SELECTID: { + if( pScan->addrExplain ){ + *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + }else{ + *(int*)pOut = -1; + } + break; + } + default: { + return 1; + } + } + return 0; +} + +/* +** Zero all counters associated with the sqlite3_stmt_scanstatus() data. +*/ +SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + memset(p->anExec, 0, p->nOp * sizeof(i64)); +} +#endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ + +/************** End of vdbeapi.c *********************************************/ +/************** Begin file vdbetrace.c ***************************************/ +/* +** 2009 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code used to insert the values of host parameters +** (aka "wildcards") into the SQL text output by sqlite3_trace(). +** +** The Vdbe parse-tree explainer is also found here. +*/ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ + +#ifndef SQLITE_OMIT_TRACE + +/* +** zSql is a zero-terminated string of UTF-8 SQL text. Return the number of +** bytes in this text up to but excluding the first character in +** a host parameter. If the text contains no host parameters, return +** the total number of bytes in the text. +*/ +static int findNextHostParameter(const char *zSql, int *pnToken){ + int tokenType; + int nTotal = 0; + int n; + + *pnToken = 0; + while( zSql[0] ){ + n = sqlite3GetToken((u8*)zSql, &tokenType); + assert( n>0 && tokenType!=TK_ILLEGAL ); + if( tokenType==TK_VARIABLE ){ + *pnToken = n; + break; + } + nTotal += n; + zSql += n; + } + return nTotal; +} + +/* +** This function returns a pointer to a nul-terminated string in memory +** obtained from sqlite3DbMalloc(). If sqlite3.nVdbeExec is 1, then the +** string contains a copy of zRawSql but with host parameters expanded to +** their current bindings. Or, if sqlite3.nVdbeExec is greater than 1, +** then the returned string holds a copy of zRawSql with "-- " prepended +** to each line of text. +** +** If the SQLITE_TRACE_SIZE_LIMIT macro is defined to an integer, then +** then long strings and blobs are truncated to that many bytes. This +** can be used to prevent unreasonably large trace strings when dealing +** with large (multi-megabyte) strings and blobs. +** +** The calling function is responsible for making sure the memory returned +** is eventually freed. +** +** ALGORITHM: Scan the input string looking for host parameters in any of +** these forms: ?, ?N, $A, @A, :A. Take care to avoid text within +** string literals, quoted identifier names, and comments. For text forms, +** the host parameter index is found by scanning the prepared +** statement for the corresponding OP_Variable opcode. Once the host +** parameter index is known, locate the value in p->aVar[]. Then render +** the value as a literal in place of the host parameter name. +*/ +SQLITE_PRIVATE char *sqlite3VdbeExpandSql( + Vdbe *p, /* The prepared statement being evaluated */ + const char *zRawSql /* Raw text of the SQL statement */ +){ + sqlite3 *db; /* The database connection */ + int idx = 0; /* Index of a host parameter */ + int nextIndex = 1; /* Index of next ? host parameter */ + int n; /* Length of a token prefix */ + int nToken; /* Length of the parameter token */ + int i; /* Loop counter */ + Mem *pVar; /* Value of a host parameter */ + StrAccum out; /* Accumulate the output here */ +#ifndef SQLITE_OMIT_UTF16 + Mem utf8; /* Used to convert UTF16 into UTF8 for display */ +#endif + + db = p->db; + sqlite3StrAccumInit(&out, 0, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); + if( db->nVdbeExec>1 ){ + while( *zRawSql ){ + const char *zStart = zRawSql; + while( *(zRawSql++)!='\n' && *zRawSql ); + sqlite3_str_append(&out, "-- ", 3); + assert( (zRawSql - zStart) > 0 ); + sqlite3_str_append(&out, zStart, (int)(zRawSql-zStart)); + } + }else if( p->nVar==0 ){ + sqlite3_str_append(&out, zRawSql, sqlite3Strlen30(zRawSql)); + }else{ + while( zRawSql[0] ){ + n = findNextHostParameter(zRawSql, &nToken); + assert( n>0 ); + sqlite3_str_append(&out, zRawSql, n); + zRawSql += n; + assert( zRawSql[0] || nToken==0 ); + if( nToken==0 ) break; + if( zRawSql[0]=='?' ){ + if( nToken>1 ){ + assert( sqlite3Isdigit(zRawSql[1]) ); + sqlite3GetInt32(&zRawSql[1], &idx); + }else{ + idx = nextIndex; + } + }else{ + assert( zRawSql[0]==':' || zRawSql[0]=='$' || + zRawSql[0]=='@' || zRawSql[0]=='#' ); + testcase( zRawSql[0]==':' ); + testcase( zRawSql[0]=='$' ); + testcase( zRawSql[0]=='@' ); + testcase( zRawSql[0]=='#' ); + idx = sqlite3VdbeParameterIndex(p, zRawSql, nToken); + assert( idx>0 ); + } + zRawSql += nToken; + nextIndex = MAX(idx + 1, nextIndex); + assert( idx>0 && idx<=p->nVar ); + pVar = &p->aVar[idx-1]; + if( pVar->flags & MEM_Null ){ + sqlite3_str_append(&out, "NULL", 4); + }else if( pVar->flags & (MEM_Int|MEM_IntReal) ){ + sqlite3_str_appendf(&out, "%lld", pVar->u.i); + }else if( pVar->flags & MEM_Real ){ + sqlite3_str_appendf(&out, "%!.15g", pVar->u.r); + }else if( pVar->flags & MEM_Str ){ + int nOut; /* Number of bytes of the string text to include in output */ +#ifndef SQLITE_OMIT_UTF16 + u8 enc = ENC(db); + if( enc!=SQLITE_UTF8 ){ + memset(&utf8, 0, sizeof(utf8)); + utf8.db = db; + sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); + if( SQLITE_NOMEM==sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8) ){ + out.accError = SQLITE_NOMEM; + out.nAlloc = 0; + } + pVar = &utf8; + } +#endif + nOut = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut>SQLITE_TRACE_SIZE_LIMIT ){ + nOut = SQLITE_TRACE_SIZE_LIMIT; + while( nOutn && (pVar->z[nOut]&0xc0)==0x80 ){ nOut++; } + } +#endif + sqlite3_str_appendf(&out, "'%.*q'", nOut, pVar->z); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOutn ){ + sqlite3_str_appendf(&out, "/*+%d bytes*/", pVar->n-nOut); + } +#endif +#ifndef SQLITE_OMIT_UTF16 + if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); +#endif + }else if( pVar->flags & MEM_Zero ){ + sqlite3_str_appendf(&out, "zeroblob(%d)", pVar->u.nZero); + }else{ + int nOut; /* Number of bytes of the blob to include in output */ + assert( pVar->flags & MEM_Blob ); + sqlite3_str_append(&out, "x'", 2); + nOut = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOut>SQLITE_TRACE_SIZE_LIMIT ) nOut = SQLITE_TRACE_SIZE_LIMIT; +#endif + for(i=0; iz[i]&0xff); + } + sqlite3_str_append(&out, "'", 1); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nOutn ){ + sqlite3_str_appendf(&out, "/*+%d bytes*/", pVar->n-nOut); + } +#endif + } + } + } + if( out.accError ) sqlite3_str_reset(&out); + return sqlite3StrAccumFinish(&out); +} + +#endif /* #ifndef SQLITE_OMIT_TRACE */ + +/************** End of vdbetrace.c *******************************************/ +/************** Begin file vdbe.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** The code in this file implements the function that runs the +** bytecode of a prepared statement. +** +** Various scripts scan this source file in order to generate HTML +** documentation, headers files, or other derived files. The formatting +** of the code in this file is, therefore, important. See other comments +** in this file for details. If in doubt, do not deviate from existing +** commenting and indentation practices when changing or adding code. +*/ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ + +/* +** Invoke this macro on memory cells just prior to changing the +** value of the cell. This macro verifies that shallow copies are +** not misused. A shallow copy of a string or blob just copies a +** pointer to the string or blob, not the content. If the original +** is changed while the copy is still in use, the string or blob might +** be changed out from under the copy. This macro verifies that nothing +** like that ever happens. +*/ +#ifdef SQLITE_DEBUG +# define memAboutToChange(P,M) sqlite3VdbeMemAboutToChange(P,M) +#else +# define memAboutToChange(P,M) +#endif + +/* +** The following global variable is incremented every time a cursor +** moves, either by the OP_SeekXX, OP_Next, or OP_Prev opcodes. The test +** procedures use this information to make sure that indices are +** working correctly. This variable has no function other than to +** help verify the correct operation of the library. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_search_count = 0; +#endif + +/* +** When this global variable is positive, it gets decremented once before +** each instruction in the VDBE. When it reaches zero, the u1.isInterrupted +** field of the sqlite3 structure is set in order to simulate an interrupt. +** +** This facility is used for testing purposes only. It does not function +** in an ordinary build. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_interrupt_count = 0; +#endif + +/* +** The next global variable is incremented each type the OP_Sort opcode +** is executed. The test procedures use this information to make sure that +** sorting is occurring or not occurring at appropriate times. This variable +** has no function other than to help verify the correct operation of the +** library. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_sort_count = 0; +#endif + +/* +** The next global variable records the size of the largest MEM_Blob +** or MEM_Str that has been used by a VDBE opcode. The test procedures +** use this information to make sure that the zero-blob functionality +** is working correctly. This variable has no function other than to +** help verify the correct operation of the library. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_max_blobsize = 0; +static void updateMaxBlobsize(Mem *p){ + if( (p->flags & (MEM_Str|MEM_Blob))!=0 && p->n>sqlite3_max_blobsize ){ + sqlite3_max_blobsize = p->n; + } +} +#endif + +/* +** This macro evaluates to true if either the update hook or the preupdate +** hook are enabled for database connect DB. +*/ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +# define HAS_UPDATE_HOOK(DB) ((DB)->xPreUpdateCallback||(DB)->xUpdateCallback) +#else +# define HAS_UPDATE_HOOK(DB) ((DB)->xUpdateCallback) +#endif + +/* +** The next global variable is incremented each time the OP_Found opcode +** is executed. This is used to test whether or not the foreign key +** operation implemented using OP_FkIsZero is working. This variable +** has no function other than to help verify the correct operation of the +** library. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_found_count = 0; +#endif + +/* +** Test a register to see if it exceeds the current maximum blob size. +** If it does, record the new maximum blob size. +*/ +#if defined(SQLITE_TEST) && !defined(SQLITE_UNTESTABLE) +# define UPDATE_MAX_BLOBSIZE(P) updateMaxBlobsize(P) +#else +# define UPDATE_MAX_BLOBSIZE(P) +#endif + +#ifdef SQLITE_DEBUG +/* This routine provides a convenient place to set a breakpoint during +** tracing with PRAGMA vdbe_trace=on. The breakpoint fires right after +** each opcode is printed. Variables "pc" (program counter) and pOp are +** available to add conditionals to the breakpoint. GDB example: +** +** break test_trace_breakpoint if pc=22 +** +** Other useful labels for breakpoints include: +** test_addop_breakpoint(pc,pOp) +** sqlite3CorruptError(lineno) +** sqlite3MisuseError(lineno) +** sqlite3CantopenError(lineno) +*/ +static void test_trace_breakpoint(int pc, Op *pOp, Vdbe *v){ + static int n = 0; + n++; +} +#endif + +/* +** Invoke the VDBE coverage callback, if that callback is defined. This +** feature is used for test suite validation only and does not appear an +** production builds. +** +** M is the type of branch. I is the direction taken for this instance of +** the branch. +** +** M: 2 - two-way branch (I=0: fall-thru 1: jump ) +** 3 - two-way + NULL (I=0: fall-thru 1: jump 2: NULL ) +** 4 - OP_Jump (I=0: jump p1 1: jump p2 2: jump p3) +** +** In other words, if M is 2, then I is either 0 (for fall-through) or +** 1 (for when the branch is taken). If M is 3, the I is 0 for an +** ordinary fall-through, I is 1 if the branch was taken, and I is 2 +** if the result of comparison is NULL. For M=3, I=2 the jump may or +** may not be taken, depending on the SQLITE_JUMPIFNULL flags in p5. +** When M is 4, that means that an OP_Jump is being run. I is 0, 1, or 2 +** depending on if the operands are less than, equal, or greater than. +** +** iSrcLine is the source code line (from the __LINE__ macro) that +** generated the VDBE instruction combined with flag bits. The source +** code line number is in the lower 24 bits of iSrcLine and the upper +** 8 bytes are flags. The lower three bits of the flags indicate +** values for I that should never occur. For example, if the branch is +** always taken, the flags should be 0x05 since the fall-through and +** alternate branch are never taken. If a branch is never taken then +** flags should be 0x06 since only the fall-through approach is allowed. +** +** Bit 0x08 of the flags indicates an OP_Jump opcode that is only +** interested in equal or not-equal. In other words, I==0 and I==2 +** should be treated as equivalent +** +** Since only a line number is retained, not the filename, this macro +** only works for amalgamation builds. But that is ok, since these macros +** should be no-ops except for special builds used to measure test coverage. +*/ +#if !defined(SQLITE_VDBE_COVERAGE) +# define VdbeBranchTaken(I,M) +#else +# define VdbeBranchTaken(I,M) vdbeTakeBranch(pOp->iSrcLine,I,M) + static void vdbeTakeBranch(u32 iSrcLine, u8 I, u8 M){ + u8 mNever; + assert( I<=2 ); /* 0: fall through, 1: taken, 2: alternate taken */ + assert( M<=4 ); /* 2: two-way branch, 3: three-way branch, 4: OP_Jump */ + assert( I> 24; + assert( (I & mNever)==0 ); + if( sqlite3GlobalConfig.xVdbeBranch==0 ) return; /*NO_TEST*/ + /* Invoke the branch coverage callback with three arguments: + ** iSrcLine - the line number of the VdbeCoverage() macro, with + ** flags removed. + ** I - Mask of bits 0x07 indicating which cases are are + ** fulfilled by this instance of the jump. 0x01 means + ** fall-thru, 0x02 means taken, 0x04 means NULL. Any + ** impossible cases (ex: if the comparison is never NULL) + ** are filled in automatically so that the coverage + ** measurement logic does not flag those impossible cases + ** as missed coverage. + ** M - Type of jump. Same as M argument above + */ + I |= mNever; + if( M==2 ) I |= 0x04; + if( M==4 ){ + I |= 0x08; + if( (mNever&0x08)!=0 && (I&0x05)!=0) I |= 0x05; /*NO_TEST*/ + } + sqlite3GlobalConfig.xVdbeBranch(sqlite3GlobalConfig.pVdbeBranchArg, + iSrcLine&0xffffff, I, M); + } +#endif + +/* +** An ephemeral string value (signified by the MEM_Ephem flag) contains +** a pointer to a dynamically allocated string where some other entity +** is responsible for deallocating that string. Because the register +** does not control the string, it might be deleted without the register +** knowing it. +** +** This routine converts an ephemeral string into a dynamically allocated +** string that the register itself controls. In other words, it +** converts an MEM_Ephem string into a string with P.z==P.zMalloc. +*/ +#define Deephemeralize(P) \ + if( ((P)->flags&MEM_Ephem)!=0 \ + && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} + +/* Return true if the cursor was opened using the OP_OpenSorter opcode. */ +#define isSorter(x) ((x)->eCurType==CURTYPE_SORTER) + +/* +** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL +** if we run out of memory. +*/ +static VdbeCursor *allocateCursor( + Vdbe *p, /* The virtual machine */ + int iCur, /* Index of the new VdbeCursor */ + int nField, /* Number of fields in the table or index */ + int iDb, /* Database the cursor belongs to, or -1 */ + u8 eCurType /* Type of the new cursor */ +){ + /* Find the memory cell that will be used to store the blob of memory + ** required for this VdbeCursor structure. It is convenient to use a + ** vdbe memory cell to manage the memory allocation required for a + ** VdbeCursor structure for the following reasons: + ** + ** * Sometimes cursor numbers are used for a couple of different + ** purposes in a vdbe program. The different uses might require + ** different sized allocations. Memory cells provide growable + ** allocations. + ** + ** * When using ENABLE_MEMORY_MANAGEMENT, memory cell buffers can + ** be freed lazily via the sqlite3_release_memory() API. This + ** minimizes the number of malloc calls made by the system. + ** + ** The memory cell for cursor 0 is aMem[0]. The rest are allocated from + ** the top of the register space. Cursor 1 is at Mem[p->nMem-1]. + ** Cursor 2 is at Mem[p->nMem-2]. And so forth. + */ + Mem *pMem = iCur>0 ? &p->aMem[p->nMem-iCur] : p->aMem; + + int nByte; + VdbeCursor *pCx = 0; + nByte = + ROUND8(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + + (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); + + assert( iCur>=0 && iCurnCursor ); + if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ + sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); + p->apCsr[iCur] = 0; + } + + /* There used to be a call to sqlite3VdbeMemClearAndResize() to make sure + ** the pMem used to hold space for the cursor has enough storage available + ** in pMem->zMalloc. But for the special case of the aMem[] entries used + ** to hold cursors, it is faster to in-line the logic. */ + assert( pMem->flags==MEM_Undefined ); + assert( (pMem->flags & MEM_Dyn)==0 ); + assert( pMem->szMalloc==0 || pMem->z==pMem->zMalloc ); + if( pMem->szMallocszMalloc>0 ){ + sqlite3DbFreeNN(pMem->db, pMem->zMalloc); + } + pMem->z = pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, nByte); + if( pMem->zMalloc==0 ){ + pMem->szMalloc = 0; + return 0; + } + pMem->szMalloc = nByte; + } + + p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->zMalloc; + memset(pCx, 0, offsetof(VdbeCursor,pAltCursor)); + pCx->eCurType = eCurType; + pCx->iDb = iDb; + pCx->nField = nField; + pCx->aOffset = &pCx->aType[nField]; + if( eCurType==CURTYPE_BTREE ){ + pCx->uc.pCursor = (BtCursor*) + &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; + sqlite3BtreeCursorZero(pCx->uc.pCursor); + } + return pCx; +} + +/* +** The string in pRec is known to look like an integer and to have a +** floating point value of rValue. Return true and set *piValue to the +** integer value if the string is in range to be an integer. Otherwise, +** return false. +*/ +static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ + i64 iValue = (double)rValue; + if( sqlite3RealSameAsInt(rValue,iValue) ){ + *piValue = iValue; + return 1; + } + return 0==sqlite3Atoi64(pRec->z, piValue, pRec->n, pRec->enc); +} + +/* +** Try to convert a value into a numeric representation if we can +** do so without loss of information. In other words, if the string +** looks like a number, convert it into a number. If it does not +** look like a number, leave it alone. +** +** If the bTryForInt flag is true, then extra effort is made to give +** an integer representation. Strings that look like floating point +** values but which have no fractional component (example: '48.00') +** will have a MEM_Int representation when bTryForInt is true. +** +** If bTryForInt is false, then if the input string contains a decimal +** point or exponential notation, the result is only MEM_Real, even +** if there is an exact integer representation of the quantity. +*/ +static void applyNumericAffinity(Mem *pRec, int bTryForInt){ + double rValue; + u8 enc = pRec->enc; + int rc; + assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real|MEM_IntReal))==MEM_Str ); + rc = sqlite3AtoF(pRec->z, &rValue, pRec->n, enc); + if( rc<=0 ) return; + if( rc==1 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ + pRec->flags |= MEM_Int; + }else{ + pRec->u.r = rValue; + pRec->flags |= MEM_Real; + if( bTryForInt ) sqlite3VdbeIntegerAffinity(pRec); + } + /* TEXT->NUMERIC is many->one. Hence, it is important to invalidate the + ** string representation after computing a numeric equivalent, because the + ** string representation might not be the canonical representation for the + ** numeric value. Ticket [343634942dd54ab57b7024] 2018-01-31. */ + pRec->flags &= ~MEM_Str; +} + +/* +** Processing is determine by the affinity parameter: +** +** SQLITE_AFF_INTEGER: +** SQLITE_AFF_REAL: +** SQLITE_AFF_NUMERIC: +** Try to convert pRec to an integer representation or a +** floating-point representation if an integer representation +** is not possible. Note that the integer representation is +** always preferred, even if the affinity is REAL, because +** an integer representation is more space efficient on disk. +** +** SQLITE_AFF_TEXT: +** Convert pRec to a text representation. +** +** SQLITE_AFF_BLOB: +** SQLITE_AFF_NONE: +** No-op. pRec is unchanged. +*/ +static void applyAffinity( + Mem *pRec, /* The value to apply affinity to */ + char affinity, /* The affinity to be applied */ + u8 enc /* Use this text encoding */ +){ + if( affinity>=SQLITE_AFF_NUMERIC ){ + assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL + || affinity==SQLITE_AFF_NUMERIC ); + if( (pRec->flags & MEM_Int)==0 ){ /*OPTIMIZATION-IF-FALSE*/ + if( (pRec->flags & MEM_Real)==0 ){ + if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); + }else{ + sqlite3VdbeIntegerAffinity(pRec); + } + } + }else if( affinity==SQLITE_AFF_TEXT ){ + /* Only attempt the conversion to TEXT if there is an integer or real + ** representation (blob and NULL do not get converted) but no string + ** representation. It would be harmless to repeat the conversion if + ** there is already a string rep, but it is pointless to waste those + ** CPU cycles. */ + if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/ + if( (pRec->flags&(MEM_Real|MEM_Int|MEM_IntReal)) ){ + testcase( pRec->flags & MEM_Int ); + testcase( pRec->flags & MEM_Real ); + testcase( pRec->flags & MEM_IntReal ); + sqlite3VdbeMemStringify(pRec, enc, 1); + } + } + pRec->flags &= ~(MEM_Real|MEM_Int|MEM_IntReal); + } +} + +/* +** Try to convert the type of a function argument or a result column +** into a numeric representation. Use either INTEGER or REAL whichever +** is appropriate. But only do the conversion if it is possible without +** loss of information and return the revised type of the argument. +*/ +SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ + int eType = sqlite3_value_type(pVal); + if( eType==SQLITE_TEXT ){ + Mem *pMem = (Mem*)pVal; + applyNumericAffinity(pMem, 0); + eType = sqlite3_value_type(pVal); + } + return eType; +} + +/* +** Exported version of applyAffinity(). This one works on sqlite3_value*, +** not the internal Mem* type. +*/ +SQLITE_PRIVATE void sqlite3ValueApplyAffinity( + sqlite3_value *pVal, + u8 affinity, + u8 enc +){ + applyAffinity((Mem *)pVal, affinity, enc); +} + +/* +** pMem currently only holds a string type (or maybe a BLOB that we can +** interpret as a string if we want to). Compute its corresponding +** numeric type, if has one. Set the pMem->u.r and pMem->u.i fields +** accordingly. +*/ +static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ + int rc; + sqlite3_int64 ix; + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ); + assert( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ); + if( ExpandBlob(pMem) ){ + pMem->u.i = 0; + return MEM_Int; + } + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + if( rc<=0 ){ + if( rc==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ + pMem->u.i = ix; + return MEM_Int; + }else{ + return MEM_Real; + } + }else if( rc==1 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ + pMem->u.i = ix; + return MEM_Int; + } + return MEM_Real; +} + +/* +** Return the numeric type for pMem, either MEM_Int or MEM_Real or both or +** none. +** +** Unlike applyNumericAffinity(), this routine does not modify pMem->flags. +** But it does set pMem->u.r and pMem->u.i appropriately. +*/ +static u16 numericType(Mem *pMem){ + if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_Real ); + testcase( pMem->flags & MEM_IntReal ); + return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal); + } + if( pMem->flags & (MEM_Str|MEM_Blob) ){ + testcase( pMem->flags & MEM_Str ); + testcase( pMem->flags & MEM_Blob ); + return computeNumericType(pMem); + } + return 0; +} + +#ifdef SQLITE_DEBUG +/* +** Write a nice string representation of the contents of cell pMem +** into buffer zBuf, length nBuf. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr){ + int f = pMem->flags; + static const char *const encnames[] = {"(X)", "(8)", "(16LE)", "(16BE)"}; + if( f&MEM_Blob ){ + int i; + char c; + if( f & MEM_Dyn ){ + c = 'z'; + assert( (f & (MEM_Static|MEM_Ephem))==0 ); + }else if( f & MEM_Static ){ + c = 't'; + assert( (f & (MEM_Dyn|MEM_Ephem))==0 ); + }else if( f & MEM_Ephem ){ + c = 'e'; + assert( (f & (MEM_Static|MEM_Dyn))==0 ); + }else{ + c = 's'; + } + sqlite3_str_appendf(pStr, "%cx[", c); + for(i=0; i<25 && in; i++){ + sqlite3_str_appendf(pStr, "%02X", ((int)pMem->z[i] & 0xFF)); + } + sqlite3_str_appendf(pStr, "|"); + for(i=0; i<25 && in; i++){ + char z = pMem->z[i]; + sqlite3_str_appendchar(pStr, 1, (z<32||z>126)?'.':z); + } + sqlite3_str_appendf(pStr,"]"); + if( f & MEM_Zero ){ + sqlite3_str_appendf(pStr, "+%dz",pMem->u.nZero); + } + }else if( f & MEM_Str ){ + int j; + u8 c; + if( f & MEM_Dyn ){ + c = 'z'; + assert( (f & (MEM_Static|MEM_Ephem))==0 ); + }else if( f & MEM_Static ){ + c = 't'; + assert( (f & (MEM_Dyn|MEM_Ephem))==0 ); + }else if( f & MEM_Ephem ){ + c = 'e'; + assert( (f & (MEM_Static|MEM_Dyn))==0 ); + }else{ + c = 's'; + } + sqlite3_str_appendf(pStr, " %c%d[", c, pMem->n); + for(j=0; j<25 && jn; j++){ + c = pMem->z[j]; + sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.'); + } + sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]); + } +} +#endif + +#ifdef SQLITE_DEBUG +/* +** Print the value of a register for tracing purposes: +*/ +static void memTracePrint(Mem *p){ + if( p->flags & MEM_Undefined ){ + printf(" undefined"); + }else if( p->flags & MEM_Null ){ + printf(p->flags & MEM_Zero ? " NULL-nochng" : " NULL"); + }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ + printf(" si:%lld", p->u.i); + }else if( (p->flags & (MEM_IntReal))!=0 ){ + printf(" ir:%lld", p->u.i); + }else if( p->flags & MEM_Int ){ + printf(" i:%lld", p->u.i); +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( p->flags & MEM_Real ){ + printf(" r:%.17g", p->u.r); +#endif + }else if( sqlite3VdbeMemIsRowSet(p) ){ + printf(" (rowset)"); + }else{ + StrAccum acc; + char zBuf[1000]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3VdbeMemPrettyPrint(p, &acc); + printf(" %s", sqlite3StrAccumFinish(&acc)); + } + if( p->flags & MEM_Subtype ) printf(" subtype=0x%02x", p->eSubtype); +} +static void registerTrace(int iReg, Mem *p){ + printf("R[%d] = ", iReg); + memTracePrint(p); + if( p->pScopyFrom ){ + printf(" <== R[%d]", (int)(p->pScopyFrom - &p[-iReg])); + } + printf("\n"); + sqlite3VdbeCheckMemInvariants(p); +} +/**/ void sqlite3PrintMem(Mem *pMem){ + memTracePrint(pMem); + printf("\n"); + fflush(stdout); +} +#endif + +#ifdef SQLITE_DEBUG +/* +** Show the values of all registers in the virtual machine. Used for +** interactive debugging. +*/ +SQLITE_PRIVATE void sqlite3VdbeRegisterDump(Vdbe *v){ + int i; + for(i=1; inMem; i++) registerTrace(i, v->aMem+i); +} +#endif /* SQLITE_DEBUG */ + + +#ifdef SQLITE_DEBUG +# define REGISTER_TRACE(R,M) if(db->flags&SQLITE_VdbeTrace)registerTrace(R,M) +#else +# define REGISTER_TRACE(R,M) +#endif + + +#ifdef VDBE_PROFILE + +/* +** hwtime.h contains inline assembler code for implementing +** high-performance timing routines. +*/ +/* #include "hwtime.h" */ + +#endif + +#ifndef NDEBUG +/* +** This function is only called from within an assert() expression. It +** checks that the sqlite3.nTransaction variable is correctly set to +** the number of non-transaction savepoints currently in the +** linked list starting at sqlite3.pSavepoint. +** +** Usage: +** +** assert( checkSavepointCount(db) ); +*/ +static int checkSavepointCount(sqlite3 *db){ + int n = 0; + Savepoint *p; + for(p=db->pSavepoint; p; p=p->pNext) n++; + assert( n==(db->nSavepoint + db->isTransactionSavepoint) ); + return 1; +} +#endif + +/* +** Return the register of pOp->p2 after first preparing it to be +** overwritten with an integer value. +*/ +static SQLITE_NOINLINE Mem *out2PrereleaseWithClear(Mem *pOut){ + sqlite3VdbeMemSetNull(pOut); + pOut->flags = MEM_Int; + return pOut; +} +static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){ + Mem *pOut; + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); + pOut = &p->aMem[pOp->p2]; + memAboutToChange(p, pOut); + if( VdbeMemDynamic(pOut) ){ /*OPTIMIZATION-IF-FALSE*/ + return out2PrereleaseWithClear(pOut); + }else{ + pOut->flags = MEM_Int; + return pOut; + } +} + +/* +** Return the symbolic name for the data type of a pMem +*/ +static const char *vdbeMemTypeName(Mem *pMem){ + static const char *azTypes[] = { + /* SQLITE_INTEGER */ "INT", + /* SQLITE_FLOAT */ "REAL", + /* SQLITE_TEXT */ "TEXT", + /* SQLITE_BLOB */ "BLOB", + /* SQLITE_NULL */ "NULL" + }; + return azTypes[sqlite3_value_type(pMem)-1]; +} + +/* +** Execute as much of a VDBE program as we can. +** This is the core of sqlite3_step(). +*/ +SQLITE_PRIVATE int sqlite3VdbeExec( + Vdbe *p /* The VDBE */ +){ + Op *aOp = p->aOp; /* Copy of p->aOp */ + Op *pOp = aOp; /* Current operation */ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + Op *pOrigOp; /* Value of pOp at the top of the loop */ +#endif +#ifdef SQLITE_DEBUG + int nExtraDelete = 0; /* Verifies FORDELETE and AUXDELETE flags */ +#endif + int rc = SQLITE_OK; /* Value to return */ + sqlite3 *db = p->db; /* The database */ + u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */ + u8 encoding = ENC(db); /* The database encoding */ + int iCompare = 0; /* Result of last comparison */ + u64 nVmStep = 0; /* Number of virtual machine steps */ +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + u64 nProgressLimit; /* Invoke xProgress() when nVmStep reaches this */ +#endif + Mem *aMem = p->aMem; /* Copy of p->aMem */ + Mem *pIn1 = 0; /* 1st input operand */ + Mem *pIn2 = 0; /* 2nd input operand */ + Mem *pIn3 = 0; /* 3rd input operand */ + Mem *pOut = 0; /* Output operand */ +#ifdef VDBE_PROFILE + u64 start; /* CPU clock count at start of opcode */ +#endif + /*** INSERT STACK UNION HERE ***/ + + assert( p->iVdbeMagic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */ + sqlite3VdbeEnter(p); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress ){ + u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; + assert( 0 < db->nProgressOps ); + nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); + }else{ + nProgressLimit = LARGEST_UINT64; + } +#endif + if( p->rc==SQLITE_NOMEM ){ + /* This happens if a malloc() inside a call to sqlite3_column_text() or + ** sqlite3_column_text16() failed. */ + goto no_mem; + } + assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); + testcase( p->rc!=SQLITE_OK ); + p->rc = SQLITE_OK; + assert( p->bIsReader || p->readOnly!=0 ); + p->iCurrentTime = 0; + assert( p->explain==0 ); + p->pResultSet = 0; + db->busyHandler.nBusy = 0; + if( AtomicLoad(&db->u1.isInterrupted) ) goto abort_due_to_interrupt; + sqlite3VdbeIOTraceSql(p); +#ifdef SQLITE_DEBUG + sqlite3BeginBenignMalloc(); + if( p->pc==0 + && (p->db->flags & (SQLITE_VdbeListing|SQLITE_VdbeEQP|SQLITE_VdbeTrace))!=0 + ){ + int i; + int once = 1; + sqlite3VdbePrintSql(p); + if( p->db->flags & SQLITE_VdbeListing ){ + printf("VDBE Program Listing:\n"); + for(i=0; inOp; i++){ + sqlite3VdbePrintOp(stdout, i, &aOp[i]); + } + } + if( p->db->flags & SQLITE_VdbeEQP ){ + for(i=0; inOp; i++){ + if( aOp[i].opcode==OP_Explain ){ + if( once ) printf("VDBE Query Plan:\n"); + printf("%s\n", aOp[i].p4.z); + once = 0; + } + } + } + if( p->db->flags & SQLITE_VdbeTrace ) printf("VDBE Trace:\n"); + } + sqlite3EndBenignMalloc(); +#endif + for(pOp=&aOp[p->pc]; 1; pOp++){ + /* Errors are detected by individual opcodes, with an immediate + ** jumps to abort_due_to_error. */ + assert( rc==SQLITE_OK ); + + assert( pOp>=aOp && pOp<&aOp[p->nOp]); +#ifdef VDBE_PROFILE + start = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); +#endif + nVmStep++; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + if( p->anExec ) p->anExec[(int)(pOp-aOp)]++; +#endif + + /* Only allow tracing if SQLITE_DEBUG is defined. + */ +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeTrace ){ + sqlite3VdbePrintOp(stdout, (int)(pOp - aOp), pOp); + test_trace_breakpoint((int)(pOp - aOp),pOp,p); + } +#endif + + + /* Check to see if we need to simulate an interrupt. This only happens + ** if we have a special test build. + */ +#ifdef SQLITE_TEST + if( sqlite3_interrupt_count>0 ){ + sqlite3_interrupt_count--; + if( sqlite3_interrupt_count==0 ){ + sqlite3_interrupt(db); + } + } +#endif + + /* Sanity checking on other operands */ +#ifdef SQLITE_DEBUG + { + u8 opProperty = sqlite3OpcodeProperty[pOp->opcode]; + if( (opProperty & OPFLG_IN1)!=0 ){ + assert( pOp->p1>0 ); + assert( pOp->p1<=(p->nMem+1 - p->nCursor) ); + assert( memIsValid(&aMem[pOp->p1]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) ); + REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); + } + if( (opProperty & OPFLG_IN2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); + assert( memIsValid(&aMem[pOp->p2]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) ); + REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); + } + if( (opProperty & OPFLG_IN3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( memIsValid(&aMem[pOp->p3]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) ); + REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); + } + if( (opProperty & OPFLG_OUT2)!=0 ){ + assert( pOp->p2>0 ); + assert( pOp->p2<=(p->nMem+1 - p->nCursor) ); + memAboutToChange(p, &aMem[pOp->p2]); + } + if( (opProperty & OPFLG_OUT3)!=0 ){ + assert( pOp->p3>0 ); + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); + memAboutToChange(p, &aMem[pOp->p3]); + } + } +#endif +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + pOrigOp = pOp; +#endif + + switch( pOp->opcode ){ + +/***************************************************************************** +** What follows is a massive switch statement where each case implements a +** separate instruction in the virtual machine. If we follow the usual +** indentation conventions, each case should be indented by 6 spaces. But +** that is a lot of wasted space on the left margin. So the code within +** the switch statement will break with convention and be flush-left. Another +** big comment (similar to this one) will mark the point in the code where +** we transition back to normal indentation. +** +** The formatting of each case is important. The makefile for SQLite +** generates two C files "opcodes.h" and "opcodes.c" by scanning this +** file looking for lines that begin with "case OP_". The opcodes.h files +** will be filled with #defines that give unique integer values to each +** opcode and the opcodes.c file is filled with an array of strings where +** each string is the symbolic name for the corresponding opcode. If the +** case statement is followed by a comment of the form "/# same as ... #/" +** that comment is used to determine the particular value of the opcode. +** +** Other keywords in the comment that follows each case are used to +** construct the OPFLG_INITIALIZER value that initializes opcodeProperty[]. +** Keywords include: in1, in2, in3, out2, out3. See +** the mkopcodeh.awk script for additional information. +** +** Documentation about VDBE opcodes is generated by scanning this file +** for lines of that contain "Opcode:". That line and all subsequent +** comment lines are used in the generation of the opcode.html documentation +** file. +** +** SUMMARY: +** +** Formatting is important to scripts that scan this file. +** Do not deviate from the formatting style currently in use. +** +*****************************************************************************/ + +/* Opcode: Goto * P2 * * * +** +** An unconditional jump to address P2. +** The next instruction executed will be +** the one at index P2 from the beginning of +** the program. +** +** The P1 parameter is not actually used by this opcode. However, it +** is sometimes set to 1 instead of 0 as a hint to the command-line shell +** that this Goto is the bottom of a loop and that the lines from P2 down +** to the current line should be indented for EXPLAIN output. +*/ +case OP_Goto: { /* jump */ + +#ifdef SQLITE_DEBUG + /* In debuggging mode, when the p5 flags is set on an OP_Goto, that + ** means we should really jump back to the preceeding OP_ReleaseReg + ** instruction. */ + if( pOp->p5 ){ + assert( pOp->p2 < (int)(pOp - aOp) ); + assert( pOp->p2 > 1 ); + pOp = &aOp[pOp->p2 - 2]; + assert( pOp[1].opcode==OP_ReleaseReg ); + goto check_for_interrupt; + } +#endif + +jump_to_p2_and_check_for_interrupt: + pOp = &aOp[pOp->p2 - 1]; + + /* Opcodes that are used as the bottom of a loop (OP_Next, OP_Prev, + ** OP_VNext, or OP_SorterNext) all jump here upon + ** completion. Check to see if sqlite3_interrupt() has been called + ** or if the progress callback needs to be invoked. + ** + ** This code uses unstructured "goto" statements and does not look clean. + ** But that is not due to sloppy coding habits. The code is written this + ** way for performance, to avoid having to run the interrupt and progress + ** checks on every opcode. This helps sqlite3_step() to run about 1.5% + ** faster according to "valgrind --tool=cachegrind" */ +check_for_interrupt: + if( AtomicLoad(&db->u1.isInterrupted) ) goto abort_due_to_interrupt; +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + /* Call the progress callback if it is configured and the required number + ** of VDBE ops have been executed (either since this invocation of + ** sqlite3VdbeExec() or since last time the progress callback was called). + ** If the progress callback returns non-zero, exit the virtual machine with + ** a return code SQLITE_ABORT. + */ + while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ + assert( db->nProgressOps!=0 ); + nProgressLimit += db->nProgressOps; + if( db->xProgress(db->pProgressArg) ){ + nProgressLimit = LARGEST_UINT64; + rc = SQLITE_INTERRUPT; + goto abort_due_to_error; + } + } +#endif + + break; +} + +/* Opcode: Gosub P1 P2 * * * +** +** Write the current address onto register P1 +** and then jump to address P2. +*/ +case OP_Gosub: { /* jump */ + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + pIn1 = &aMem[pOp->p1]; + assert( VdbeMemDynamic(pIn1)==0 ); + memAboutToChange(p, pIn1); + pIn1->flags = MEM_Int; + pIn1->u.i = (int)(pOp-aOp); + REGISTER_TRACE(pOp->p1, pIn1); + + /* Most jump operations do a goto to this spot in order to update + ** the pOp pointer. */ +jump_to_p2: + pOp = &aOp[pOp->p2 - 1]; + break; +} + +/* Opcode: Return P1 * * * * +** +** Jump to the next instruction after the address in register P1. After +** the jump, register P1 becomes undefined. +*/ +case OP_Return: { /* in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags==MEM_Int ); + pOp = &aOp[pIn1->u.i]; + pIn1->flags = MEM_Undefined; + break; +} + +/* Opcode: InitCoroutine P1 P2 P3 * * +** +** Set up register P1 so that it will Yield to the coroutine +** located at address P3. +** +** If P2!=0 then the coroutine implementation immediately follows +** this opcode. So jump over the coroutine implementation to +** address P2. +** +** See also: EndCoroutine +*/ +case OP_InitCoroutine: { /* jump */ + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p2>=0 && pOp->p2nOp ); + assert( pOp->p3>=0 && pOp->p3nOp ); + pOut = &aMem[pOp->p1]; + assert( !VdbeMemDynamic(pOut) ); + pOut->u.i = pOp->p3 - 1; + pOut->flags = MEM_Int; + if( pOp->p2 ) goto jump_to_p2; + break; +} + +/* Opcode: EndCoroutine P1 * * * * +** +** The instruction at the address in register P1 is a Yield. +** Jump to the P2 parameter of that Yield. +** After the jump, register P1 becomes undefined. +** +** See also: InitCoroutine +*/ +case OP_EndCoroutine: { /* in1 */ + VdbeOp *pCaller; + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags==MEM_Int ); + assert( pIn1->u.i>=0 && pIn1->u.inOp ); + pCaller = &aOp[pIn1->u.i]; + assert( pCaller->opcode==OP_Yield ); + assert( pCaller->p2>=0 && pCaller->p2nOp ); + pOp = &aOp[pCaller->p2 - 1]; + pIn1->flags = MEM_Undefined; + break; +} + +/* Opcode: Yield P1 P2 * * * +** +** Swap the program counter with the value in register P1. This +** has the effect of yielding to a coroutine. +** +** If the coroutine that is launched by this instruction ends with +** Yield or Return then continue to the next instruction. But if +** the coroutine launched by this instruction ends with +** EndCoroutine, then jump to P2 rather than continuing with the +** next instruction. +** +** See also: InitCoroutine +*/ +case OP_Yield: { /* in1, jump */ + int pcDest; + pIn1 = &aMem[pOp->p1]; + assert( VdbeMemDynamic(pIn1)==0 ); + pIn1->flags = MEM_Int; + pcDest = (int)pIn1->u.i; + pIn1->u.i = (int)(pOp - aOp); + REGISTER_TRACE(pOp->p1, pIn1); + pOp = &aOp[pcDest]; + break; +} + +/* Opcode: HaltIfNull P1 P2 P3 P4 P5 +** Synopsis: if r[P3]=null halt +** +** Check the value in register P3. If it is NULL then Halt using +** parameter P1, P2, and P4 as if this were a Halt instruction. If the +** value in register P3 is not NULL, then this routine is a no-op. +** The P5 parameter should be 1. +*/ +case OP_HaltIfNull: { /* in3 */ + pIn3 = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + if( pOp->p2==OE_Abort ){ sqlite3VdbeAssertAbortable(p); } +#endif + if( (pIn3->flags & MEM_Null)==0 ) break; + /* Fall through into OP_Halt */ + /* no break */ deliberate_fall_through +} + +/* Opcode: Halt P1 P2 * P4 P5 +** +** Exit immediately. All open cursors, etc are closed +** automatically. +** +** P1 is the result code returned by sqlite3_exec(), sqlite3_reset(), +** or sqlite3_finalize(). For a normal halt, this should be SQLITE_OK (0). +** For errors, it can be some other value. If P1!=0 then P2 will determine +** whether or not to rollback the current transaction. Do not rollback +** if P2==OE_Fail. Do the rollback if P2==OE_Rollback. If P2==OE_Abort, +** then back out all changes that have occurred during this execution of the +** VDBE, but do not rollback the transaction. +** +** If P4 is not null then it is an error message string. +** +** P5 is a value between 0 and 4, inclusive, that modifies the P4 string. +** +** 0: (no change) +** 1: NOT NULL contraint failed: P4 +** 2: UNIQUE constraint failed: P4 +** 3: CHECK constraint failed: P4 +** 4: FOREIGN KEY constraint failed: P4 +** +** If P5 is not zero and P4 is NULL, then everything after the ":" is +** omitted. +** +** There is an implied "Halt 0 0 0" instruction inserted at the very end of +** every program. So a jump past the last instruction of the program +** is the same as executing Halt. +*/ +case OP_Halt: { + VdbeFrame *pFrame; + int pcx; + + pcx = (int)(pOp - aOp); +#ifdef SQLITE_DEBUG + if( pOp->p2==OE_Abort ){ sqlite3VdbeAssertAbortable(p); } +#endif + if( pOp->p1==SQLITE_OK && p->pFrame ){ + /* Halt the sub-program. Return control to the parent frame. */ + pFrame = p->pFrame; + p->pFrame = pFrame->pParent; + p->nFrame--; + sqlite3VdbeSetChanges(db, p->nChange); + pcx = sqlite3VdbeFrameRestore(pFrame); + if( pOp->p2==OE_Ignore ){ + /* Instruction pcx is the OP_Program that invoked the sub-program + ** currently being halted. If the p2 instruction of this OP_Halt + ** instruction is set to OE_Ignore, then the sub-program is throwing + ** an IGNORE exception. In this case jump to the address specified + ** as the p2 of the calling OP_Program. */ + pcx = p->aOp[pcx].p2-1; + } + aOp = p->aOp; + aMem = p->aMem; + pOp = &aOp[pcx]; + break; + } + p->rc = pOp->p1; + p->errorAction = (u8)pOp->p2; + p->pc = pcx; + assert( pOp->p5<=4 ); + if( p->rc ){ + if( pOp->p5 ){ + static const char * const azType[] = { "NOT NULL", "UNIQUE", "CHECK", + "FOREIGN KEY" }; + testcase( pOp->p5==1 ); + testcase( pOp->p5==2 ); + testcase( pOp->p5==3 ); + testcase( pOp->p5==4 ); + sqlite3VdbeError(p, "%s constraint failed", azType[pOp->p5-1]); + if( pOp->p4.z ){ + p->zErrMsg = sqlite3MPrintf(db, "%z: %s", p->zErrMsg, pOp->p4.z); + } + }else{ + sqlite3VdbeError(p, "%s", pOp->p4.z); + } + sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pcx, p->zSql, p->zErrMsg); + } + rc = sqlite3VdbeHalt(p); + assert( rc==SQLITE_BUSY || rc==SQLITE_OK || rc==SQLITE_ERROR ); + if( rc==SQLITE_BUSY ){ + p->rc = SQLITE_BUSY; + }else{ + assert( rc==SQLITE_OK || (p->rc&0xff)==SQLITE_CONSTRAINT ); + assert( rc==SQLITE_OK || db->nDeferredCons>0 || db->nDeferredImmCons>0 ); + rc = p->rc ? SQLITE_ERROR : SQLITE_DONE; + } + goto vdbe_return; +} + +/* Opcode: Integer P1 P2 * * * +** Synopsis: r[P2]=P1 +** +** The 32-bit integer value P1 is written into register P2. +*/ +case OP_Integer: { /* out2 */ + pOut = out2Prerelease(p, pOp); + pOut->u.i = pOp->p1; + break; +} + +/* Opcode: Int64 * P2 * P4 * +** Synopsis: r[P2]=P4 +** +** P4 is a pointer to a 64-bit integer value. +** Write that value into register P2. +*/ +case OP_Int64: { /* out2 */ + pOut = out2Prerelease(p, pOp); + assert( pOp->p4.pI64!=0 ); + pOut->u.i = *pOp->p4.pI64; + break; +} + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* Opcode: Real * P2 * P4 * +** Synopsis: r[P2]=P4 +** +** P4 is a pointer to a 64-bit floating point value. +** Write that value into register P2. +*/ +case OP_Real: { /* same as TK_FLOAT, out2 */ + pOut = out2Prerelease(p, pOp); + pOut->flags = MEM_Real; + assert( !sqlite3IsNaN(*pOp->p4.pReal) ); + pOut->u.r = *pOp->p4.pReal; + break; +} +#endif + +/* Opcode: String8 * P2 * P4 * +** Synopsis: r[P2]='P4' +** +** P4 points to a nul terminated UTF-8 string. This opcode is transformed +** into a String opcode before it is executed for the first time. During +** this transformation, the length of string P4 is computed and stored +** as the P1 parameter. +*/ +case OP_String8: { /* same as TK_STRING, out2 */ + assert( pOp->p4.z!=0 ); + pOut = out2Prerelease(p, pOp); + pOp->p1 = sqlite3Strlen30(pOp->p4.z); + +#ifndef SQLITE_OMIT_UTF16 + if( encoding!=SQLITE_UTF8 ){ + rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); + assert( rc==SQLITE_OK || rc==SQLITE_TOOBIG ); + if( rc ) goto too_big; + if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; + assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z ); + assert( VdbeMemDynamic(pOut)==0 ); + pOut->szMalloc = 0; + pOut->flags |= MEM_Static; + if( pOp->p4type==P4_DYNAMIC ){ + sqlite3DbFree(db, pOp->p4.z); + } + pOp->p4type = P4_DYNAMIC; + pOp->p4.z = pOut->z; + pOp->p1 = pOut->n; + } +#endif + if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + pOp->opcode = OP_String; + assert( rc==SQLITE_OK ); + /* Fall through to the next case, OP_String */ + /* no break */ deliberate_fall_through +} + +/* Opcode: String P1 P2 P3 P4 P5 +** Synopsis: r[P2]='P4' (len=P1) +** +** The string value P4 of length P1 (bytes) is stored in register P2. +** +** If P3 is not zero and the content of register P3 is equal to P5, then +** the datatype of the register P2 is converted to BLOB. The content is +** the same sequence of bytes, it is merely interpreted as a BLOB instead +** of a string, as if it had been CAST. In other words: +** +** if( P3!=0 and reg[P3]==P5 ) reg[P2] := CAST(reg[P2] as BLOB) +*/ +case OP_String: { /* out2 */ + assert( pOp->p4.z!=0 ); + pOut = out2Prerelease(p, pOp); + pOut->flags = MEM_Str|MEM_Static|MEM_Term; + pOut->z = pOp->p4.z; + pOut->n = pOp->p1; + pOut->enc = encoding; + UPDATE_MAX_BLOBSIZE(pOut); +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( pOp->p3>0 ){ + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); + pIn3 = &aMem[pOp->p3]; + assert( pIn3->flags & MEM_Int ); + if( pIn3->u.i==pOp->p5 ) pOut->flags = MEM_Blob|MEM_Static|MEM_Term; + } +#endif + break; +} + +/* Opcode: Null P1 P2 P3 * * +** Synopsis: r[P2..P3]=NULL +** +** Write a NULL into registers P2. If P3 greater than P2, then also write +** NULL into register P3 and every register in between P2 and P3. If P3 +** is less than P2 (typically P3 is zero) then only register P2 is +** set to NULL. +** +** If the P1 value is non-zero, then also set the MEM_Cleared flag so that +** NULL values will not compare equal even if SQLITE_NULLEQ is set on +** OP_Ne or OP_Eq. +*/ +case OP_Null: { /* out2 */ + int cnt; + u16 nullFlag; + pOut = out2Prerelease(p, pOp); + cnt = pOp->p3-pOp->p2; + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); + pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; + pOut->n = 0; +#ifdef SQLITE_DEBUG + pOut->uTemp = 0; +#endif + while( cnt>0 ){ + pOut++; + memAboutToChange(p, pOut); + sqlite3VdbeMemSetNull(pOut); + pOut->flags = nullFlag; + pOut->n = 0; + cnt--; + } + break; +} + +/* Opcode: SoftNull P1 * * * * +** Synopsis: r[P1]=NULL +** +** Set register P1 to have the value NULL as seen by the OP_MakeRecord +** instruction, but do not free any string or blob memory associated with +** the register, so that if the value was a string or blob that was +** previously copied using OP_SCopy, the copies will continue to be valid. +*/ +case OP_SoftNull: { + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + pOut = &aMem[pOp->p1]; + pOut->flags = (pOut->flags&~(MEM_Undefined|MEM_AffMask))|MEM_Null; + break; +} + +/* Opcode: Blob P1 P2 * P4 * +** Synopsis: r[P2]=P4 (len=P1) +** +** P4 points to a blob of data P1 bytes long. Store this +** blob in register P2. +*/ +case OP_Blob: { /* out2 */ + assert( pOp->p1 <= SQLITE_MAX_LENGTH ); + pOut = out2Prerelease(p, pOp); + sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); + pOut->enc = encoding; + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: Variable P1 P2 * P4 * +** Synopsis: r[P2]=parameter(P1,P4) +** +** Transfer the values of bound parameter P1 into register P2 +** +** If the parameter is named, then its name appears in P4. +** The P4 value is used by sqlite3_bind_parameter_name(). +*/ +case OP_Variable: { /* out2 */ + Mem *pVar; /* Value being transferred */ + + assert( pOp->p1>0 && pOp->p1<=p->nVar ); + assert( pOp->p4.z==0 || pOp->p4.z==sqlite3VListNumToName(p->pVList,pOp->p1) ); + pVar = &p->aVar[pOp->p1 - 1]; + if( sqlite3VdbeMemTooBig(pVar) ){ + goto too_big; + } + pOut = &aMem[pOp->p2]; + if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); + memcpy(pOut, pVar, MEMCELLSIZE); + pOut->flags &= ~(MEM_Dyn|MEM_Ephem); + pOut->flags |= MEM_Static|MEM_FromBind; + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: Move P1 P2 P3 * * +** Synopsis: r[P2@P3]=r[P1@P3] +** +** Move the P3 values in register P1..P1+P3-1 over into +** registers P2..P2+P3-1. Registers P1..P1+P3-1 are +** left holding a NULL. It is an error for register ranges +** P1..P1+P3-1 and P2..P2+P3-1 to overlap. It is an error +** for P3 to be less than 1. +*/ +case OP_Move: { + int n; /* Number of registers left to copy */ + int p1; /* Register to copy from */ + int p2; /* Register to copy to */ + + n = pOp->p3; + p1 = pOp->p1; + p2 = pOp->p2; + assert( n>0 && p1>0 && p2>0 ); + assert( p1+n<=p2 || p2+n<=p1 ); + + pIn1 = &aMem[p1]; + pOut = &aMem[p2]; + do{ + assert( pOut<=&aMem[(p->nMem+1 - p->nCursor)] ); + assert( pIn1<=&aMem[(p->nMem+1 - p->nCursor)] ); + assert( memIsValid(pIn1) ); + memAboutToChange(p, pOut); + sqlite3VdbeMemMove(pOut, pIn1); +#ifdef SQLITE_DEBUG + pIn1->pScopyFrom = 0; + { int i; + for(i=1; inMem; i++){ + if( aMem[i].pScopyFrom==pIn1 ){ + aMem[i].pScopyFrom = pOut; + } + } + } +#endif + Deephemeralize(pOut); + REGISTER_TRACE(p2++, pOut); + pIn1++; + pOut++; + }while( --n ); + break; +} + +/* Opcode: Copy P1 P2 P3 * * +** Synopsis: r[P2@P3+1]=r[P1@P3+1] +** +** Make a copy of registers P1..P1+P3 into registers P2..P2+P3. +** +** This instruction makes a deep copy of the value. A duplicate +** is made of any string or blob constant. See also OP_SCopy. +*/ +case OP_Copy: { + int n; + + n = pOp->p3; + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + assert( pOut!=pIn1 ); + while( 1 ){ + memAboutToChange(p, pOut); + sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); + Deephemeralize(pOut); +#ifdef SQLITE_DEBUG + pOut->pScopyFrom = 0; +#endif + REGISTER_TRACE(pOp->p2+pOp->p3-n, pOut); + if( (n--)==0 ) break; + pOut++; + pIn1++; + } + break; +} + +/* Opcode: SCopy P1 P2 * * * +** Synopsis: r[P2]=r[P1] +** +** Make a shallow copy of register P1 into register P2. +** +** This instruction makes a shallow copy of the value. If the value +** is a string or blob, then the copy is only a pointer to the +** original and hence if the original changes so will the copy. +** Worse, if the original is deallocated, the copy becomes invalid. +** Thus the program must guarantee that the original will not change +** during the lifetime of the copy. Use OP_Copy to make a complete +** copy. +*/ +case OP_SCopy: { /* out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + assert( pOut!=pIn1 ); + sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); +#ifdef SQLITE_DEBUG + pOut->pScopyFrom = pIn1; + pOut->mScopyFlags = pIn1->flags; +#endif + break; +} + +/* Opcode: IntCopy P1 P2 * * * +** Synopsis: r[P2]=r[P1] +** +** Transfer the integer value held in register P1 into register P2. +** +** This is an optimized version of SCopy that works only for integer +** values. +*/ +case OP_IntCopy: { /* out2 */ + pIn1 = &aMem[pOp->p1]; + assert( (pIn1->flags & MEM_Int)!=0 ); + pOut = &aMem[pOp->p2]; + sqlite3VdbeMemSetInt64(pOut, pIn1->u.i); + break; +} + +/* Opcode: ChngCntRow P1 P2 * * * +** Synopsis: output=r[P1] +** +** Output value in register P1 as the chance count for a DML statement, +** due to the "PRAGMA count_changes=ON" setting. Or, if there was a +** foreign key error in the statement, trigger the error now. +** +** This opcode is a variant of OP_ResultRow that checks the foreign key +** immediate constraint count and throws an error if the count is +** non-zero. The P2 opcode must be 1. +*/ +case OP_ChngCntRow: { + assert( pOp->p2==1 ); + if( (rc = sqlite3VdbeCheckFk(p,0))!=SQLITE_OK ){ + goto abort_due_to_error; + } + /* Fall through to the next case, OP_ResultRow */ + /* no break */ deliberate_fall_through +} + +/* Opcode: ResultRow P1 P2 * * * +** Synopsis: output=r[P1@P2] +** +** The registers P1 through P1+P2-1 contain a single row of +** results. This opcode causes the sqlite3_step() call to terminate +** with an SQLITE_ROW return code and it sets up the sqlite3_stmt +** structure to provide access to the r(P1)..r(P1+P2-1) values as +** the result row. +*/ +case OP_ResultRow: { + Mem *pMem; + int i; + assert( p->nResColumn==pOp->p2 ); + assert( pOp->p1>0 || CORRUPT_DB ); + assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); + + /* Invalidate all ephemeral cursor row caches */ + p->cacheCtr = (p->cacheCtr + 2)|1; + + /* Make sure the results of the current row are \000 terminated + ** and have an assigned type. The results are de-ephemeralized as + ** a side effect. + */ + pMem = p->pResultSet = &aMem[pOp->p1]; + for(i=0; ip2; i++){ + assert( memIsValid(&pMem[i]) ); + Deephemeralize(&pMem[i]); + assert( (pMem[i].flags & MEM_Ephem)==0 + || (pMem[i].flags & (MEM_Str|MEM_Blob))==0 ); + sqlite3VdbeMemNulTerminate(&pMem[i]); + REGISTER_TRACE(pOp->p1+i, &pMem[i]); +#ifdef SQLITE_DEBUG + /* The registers in the result will not be used again when the + ** prepared statement restarts. This is because sqlite3_column() + ** APIs might have caused type conversions of made other changes to + ** the register values. Therefore, we can go ahead and break any + ** OP_SCopy dependencies. */ + pMem[i].pScopyFrom = 0; +#endif + } + if( db->mallocFailed ) goto no_mem; + + if( db->mTrace & SQLITE_TRACE_ROW ){ + db->trace.xV2(SQLITE_TRACE_ROW, db->pTraceArg, p, 0); + } + + + /* Return SQLITE_ROW + */ + p->pc = (int)(pOp - aOp) + 1; + rc = SQLITE_ROW; + goto vdbe_return; +} + +/* Opcode: Concat P1 P2 P3 * * +** Synopsis: r[P3]=r[P2]+r[P1] +** +** Add the text in register P1 onto the end of the text in +** register P2 and store the result in register P3. +** If either the P1 or P2 text are NULL then store NULL in P3. +** +** P3 = P2 || P1 +** +** It is illegal for P1 and P3 to be the same register. Sometimes, +** if P3 is the same register as P2, the implementation is able +** to avoid a memcpy(). +*/ +case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ + i64 nByte; /* Total size of the output string or blob */ + u16 flags1; /* Initial flags for P1 */ + u16 flags2; /* Initial flags for P2 */ + + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + pOut = &aMem[pOp->p3]; + testcase( pOut==pIn2 ); + assert( pIn1!=pOut ); + flags1 = pIn1->flags; + testcase( flags1 & MEM_Null ); + testcase( pIn2->flags & MEM_Null ); + if( (flags1 | pIn2->flags) & MEM_Null ){ + sqlite3VdbeMemSetNull(pOut); + break; + } + if( (flags1 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn1,encoding,0) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + }else if( (flags1 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + } + flags2 = pIn2->flags; + if( (flags2 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn2,encoding,0) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + }else if( (flags2 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + } + nByte = pIn1->n + pIn2->n; + if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + if( sqlite3VdbeMemGrow(pOut, (int)nByte+3, pOut==pIn2) ){ + goto no_mem; + } + MemSetTypeFlag(pOut, MEM_Str); + if( pOut!=pIn2 ){ + memcpy(pOut->z, pIn2->z, pIn2->n); + assert( (pIn2->flags & MEM_Dyn) == (flags2 & MEM_Dyn) ); + pIn2->flags = flags2; + } + memcpy(&pOut->z[pIn2->n], pIn1->z, pIn1->n); + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; + pOut->z[nByte]=0; + pOut->z[nByte+1] = 0; + pOut->z[nByte+2] = 0; + pOut->flags |= MEM_Term; + pOut->n = (int)nByte; + pOut->enc = encoding; + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: Add P1 P2 P3 * * +** Synopsis: r[P3]=r[P1]+r[P2] +** +** Add the value in register P1 to the value in register P2 +** and store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: Multiply P1 P2 P3 * * +** Synopsis: r[P3]=r[P1]*r[P2] +** +** +** Multiply the value in register P1 by the value in register P2 +** and store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: Subtract P1 P2 P3 * * +** Synopsis: r[P3]=r[P2]-r[P1] +** +** Subtract the value in register P1 from the value in register P2 +** and store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: Divide P1 P2 P3 * * +** Synopsis: r[P3]=r[P2]/r[P1] +** +** Divide the value in register P1 by the value in register P2 +** and store the result in register P3 (P3=P2/P1). If the value in +** register P1 is zero, then the result is NULL. If either input is +** NULL, the result is NULL. +*/ +/* Opcode: Remainder P1 P2 P3 * * +** Synopsis: r[P3]=r[P2]%r[P1] +** +** Compute the remainder after integer register P2 is divided by +** register P1 and store the result in register P3. +** If the value in register P1 is zero the result is NULL. +** If either operand is NULL, the result is NULL. +*/ +case OP_Add: /* same as TK_PLUS, in1, in2, out3 */ +case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ +case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ +case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ +case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ + u16 flags; /* Combined MEM_* flags from both inputs */ + u16 type1; /* Numeric type of left operand */ + u16 type2; /* Numeric type of right operand */ + i64 iA; /* Integer value of left operand */ + i64 iB; /* Integer value of right operand */ + double rA; /* Real value of left operand */ + double rB; /* Real value of right operand */ + + pIn1 = &aMem[pOp->p1]; + type1 = numericType(pIn1); + pIn2 = &aMem[pOp->p2]; + type2 = numericType(pIn2); + pOut = &aMem[pOp->p3]; + flags = pIn1->flags | pIn2->flags; + if( (type1 & type2 & MEM_Int)!=0 ){ + iA = pIn1->u.i; + iB = pIn2->u.i; + switch( pOp->opcode ){ + case OP_Add: if( sqlite3AddInt64(&iB,iA) ) goto fp_math; break; + case OP_Subtract: if( sqlite3SubInt64(&iB,iA) ) goto fp_math; break; + case OP_Multiply: if( sqlite3MulInt64(&iB,iA) ) goto fp_math; break; + case OP_Divide: { + if( iA==0 ) goto arithmetic_result_is_null; + if( iA==-1 && iB==SMALLEST_INT64 ) goto fp_math; + iB /= iA; + break; + } + default: { + if( iA==0 ) goto arithmetic_result_is_null; + if( iA==-1 ) iA = 1; + iB %= iA; + break; + } + } + pOut->u.i = iB; + MemSetTypeFlag(pOut, MEM_Int); + }else if( (flags & MEM_Null)!=0 ){ + goto arithmetic_result_is_null; + }else{ +fp_math: + rA = sqlite3VdbeRealValue(pIn1); + rB = sqlite3VdbeRealValue(pIn2); + switch( pOp->opcode ){ + case OP_Add: rB += rA; break; + case OP_Subtract: rB -= rA; break; + case OP_Multiply: rB *= rA; break; + case OP_Divide: { + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + if( rA==(double)0 ) goto arithmetic_result_is_null; + rB /= rA; + break; + } + default: { + iA = sqlite3VdbeIntValue(pIn1); + iB = sqlite3VdbeIntValue(pIn2); + if( iA==0 ) goto arithmetic_result_is_null; + if( iA==-1 ) iA = 1; + rB = (double)(iB % iA); + break; + } + } +#ifdef SQLITE_OMIT_FLOATING_POINT + pOut->u.i = rB; + MemSetTypeFlag(pOut, MEM_Int); +#else + if( sqlite3IsNaN(rB) ){ + goto arithmetic_result_is_null; + } + pOut->u.r = rB; + MemSetTypeFlag(pOut, MEM_Real); +#endif + } + break; + +arithmetic_result_is_null: + sqlite3VdbeMemSetNull(pOut); + break; +} + +/* Opcode: CollSeq P1 * * P4 +** +** P4 is a pointer to a CollSeq object. If the next call to a user function +** or aggregate calls sqlite3GetFuncCollSeq(), this collation sequence will +** be returned. This is used by the built-in min(), max() and nullif() +** functions. +** +** If P1 is not zero, then it is a register that a subsequent min() or +** max() aggregate will set to 1 if the current row is not the minimum or +** maximum. The P1 register is initialized to 0 by this instruction. +** +** The interface used by the implementation of the aforementioned functions +** to retrieve the collation sequence set by this opcode is not available +** publicly. Only built-in functions have access to this feature. +*/ +case OP_CollSeq: { + assert( pOp->p4type==P4_COLLSEQ ); + if( pOp->p1 ){ + sqlite3VdbeMemSetInt64(&aMem[pOp->p1], 0); + } + break; +} + +/* Opcode: BitAnd P1 P2 P3 * * +** Synopsis: r[P3]=r[P1]&r[P2] +** +** Take the bit-wise AND of the values in register P1 and P2 and +** store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: BitOr P1 P2 P3 * * +** Synopsis: r[P3]=r[P1]|r[P2] +** +** Take the bit-wise OR of the values in register P1 and P2 and +** store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +/* Opcode: ShiftLeft P1 P2 P3 * * +** Synopsis: r[P3]=r[P2]<>r[P1] +** +** Shift the integer value in register P2 to the right by the +** number of bits specified by the integer in register P1. +** Store the result in register P3. +** If either input is NULL, the result is NULL. +*/ +case OP_BitAnd: /* same as TK_BITAND, in1, in2, out3 */ +case OP_BitOr: /* same as TK_BITOR, in1, in2, out3 */ +case OP_ShiftLeft: /* same as TK_LSHIFT, in1, in2, out3 */ +case OP_ShiftRight: { /* same as TK_RSHIFT, in1, in2, out3 */ + i64 iA; + u64 uA; + i64 iB; + u8 op; + + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + pOut = &aMem[pOp->p3]; + if( (pIn1->flags | pIn2->flags) & MEM_Null ){ + sqlite3VdbeMemSetNull(pOut); + break; + } + iA = sqlite3VdbeIntValue(pIn2); + iB = sqlite3VdbeIntValue(pIn1); + op = pOp->opcode; + if( op==OP_BitAnd ){ + iA &= iB; + }else if( op==OP_BitOr ){ + iA |= iB; + }else if( iB!=0 ){ + assert( op==OP_ShiftRight || op==OP_ShiftLeft ); + + /* If shifting by a negative amount, shift in the other direction */ + if( iB<0 ){ + assert( OP_ShiftRight==OP_ShiftLeft+1 ); + op = 2*OP_ShiftLeft + 1 - op; + iB = iB>(-64) ? -iB : 64; + } + + if( iB>=64 ){ + iA = (iA>=0 || op==OP_ShiftLeft) ? 0 : -1; + }else{ + memcpy(&uA, &iA, sizeof(uA)); + if( op==OP_ShiftLeft ){ + uA <<= iB; + }else{ + uA >>= iB; + /* Sign-extend on a right shift of a negative number */ + if( iA<0 ) uA |= ((((u64)0xffffffff)<<32)|0xffffffff) << (64-iB); + } + memcpy(&iA, &uA, sizeof(iA)); + } + } + pOut->u.i = iA; + MemSetTypeFlag(pOut, MEM_Int); + break; +} + +/* Opcode: AddImm P1 P2 * * * +** Synopsis: r[P1]=r[P1]+P2 +** +** Add the constant P2 to the value in register P1. +** The result is always an integer. +** +** To force any register to be an integer, just add 0. +*/ +case OP_AddImm: { /* in1 */ + pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); + sqlite3VdbeMemIntegerify(pIn1); + pIn1->u.i += pOp->p2; + break; +} + +/* Opcode: MustBeInt P1 P2 * * * +** +** Force the value in register P1 to be an integer. If the value +** in P1 is not an integer and cannot be converted into an integer +** without data loss, then jump immediately to P2, or if P2==0 +** raise an SQLITE_MISMATCH exception. +*/ +case OP_MustBeInt: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + if( (pIn1->flags & MEM_Int)==0 ){ + applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); + if( (pIn1->flags & MEM_Int)==0 ){ + VdbeBranchTaken(1, 2); + if( pOp->p2==0 ){ + rc = SQLITE_MISMATCH; + goto abort_due_to_error; + }else{ + goto jump_to_p2; + } + } + } + VdbeBranchTaken(0, 2); + MemSetTypeFlag(pIn1, MEM_Int); + break; +} + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* Opcode: RealAffinity P1 * * * * +** +** If register P1 holds an integer convert it to a real value. +** +** This opcode is used when extracting information from a column that +** has REAL affinity. Such column values may still be stored as +** integers, for space efficiency, but after extraction we want them +** to have only a real value. +*/ +case OP_RealAffinity: { /* in1 */ + pIn1 = &aMem[pOp->p1]; + if( pIn1->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pIn1->flags & MEM_Int ); + testcase( pIn1->flags & MEM_IntReal ); + sqlite3VdbeMemRealify(pIn1); + REGISTER_TRACE(pOp->p1, pIn1); + } + break; +} +#endif + +#ifndef SQLITE_OMIT_CAST +/* Opcode: Cast P1 P2 * * * +** Synopsis: affinity(r[P1]) +** +** Force the value in register P1 to be the type defined by P2. +** +**
      +**
    • P2=='A' → BLOB +**
    • P2=='B' → TEXT +**
    • P2=='C' → NUMERIC +**
    • P2=='D' → INTEGER +**
    • P2=='E' → REAL +**
    +** +** A NULL value is not changed by this routine. It remains NULL. +*/ +case OP_Cast: { /* in1 */ + assert( pOp->p2>=SQLITE_AFF_BLOB && pOp->p2<=SQLITE_AFF_REAL ); + testcase( pOp->p2==SQLITE_AFF_TEXT ); + testcase( pOp->p2==SQLITE_AFF_BLOB ); + testcase( pOp->p2==SQLITE_AFF_NUMERIC ); + testcase( pOp->p2==SQLITE_AFF_INTEGER ); + testcase( pOp->p2==SQLITE_AFF_REAL ); + pIn1 = &aMem[pOp->p1]; + memAboutToChange(p, pIn1); + rc = ExpandBlob(pIn1); + if( rc ) goto abort_due_to_error; + rc = sqlite3VdbeMemCast(pIn1, pOp->p2, encoding); + if( rc ) goto abort_due_to_error; + UPDATE_MAX_BLOBSIZE(pIn1); + REGISTER_TRACE(pOp->p1, pIn1); + break; +} +#endif /* SQLITE_OMIT_CAST */ + +/* Opcode: Eq P1 P2 P3 P4 P5 +** Synopsis: IF r[P3]==r[P1] +** +** Compare the values in register P1 and P3. If reg(P3)==reg(P1) then +** jump to address P2. +** +** The SQLITE_AFF_MASK portion of P5 must be an affinity character - +** SQLITE_AFF_TEXT, SQLITE_AFF_INTEGER, and so forth. An attempt is made +** to coerce both inputs according to this affinity before the +** comparison is made. If the SQLITE_AFF_MASK is 0x00, then numeric +** affinity is used. Note that the affinity conversions are stored +** back into the input registers P1 and P3. So this opcode can cause +** persistent changes to registers P1 and P3. +** +** Once any conversions have taken place, and neither value is NULL, +** the values are compared. If both values are blobs then memcmp() is +** used to determine the results of the comparison. If both values +** are text, then the appropriate collating function specified in +** P4 is used to do the comparison. If P4 is not specified then +** memcmp() is used to compare text string. If both values are +** numeric, then a numeric comparison is used. If the two values +** are of different types, then numbers are considered less than +** strings and strings are considered less than blobs. +** +** If SQLITE_NULLEQ is set in P5 then the result of comparison is always either +** true or false and is never NULL. If both operands are NULL then the result +** of comparison is true. If either operand is NULL then the result is false. +** If neither operand is NULL the result is the same as it would be if +** the SQLITE_NULLEQ flag were omitted from P5. +** +** This opcode saves the result of comparison for use by the new +** OP_Jump opcode. +*/ +/* Opcode: Ne P1 P2 P3 P4 P5 +** Synopsis: IF r[P3]!=r[P1] +** +** This works just like the Eq opcode except that the jump is taken if +** the operands in registers P1 and P3 are not equal. See the Eq opcode for +** additional information. +*/ +/* Opcode: Lt P1 P2 P3 P4 P5 +** Synopsis: IF r[P3]r[P1] +** +** This works just like the Lt opcode except that the jump is taken if +** the content of register P3 is greater than the content of +** register P1. See the Lt opcode for additional information. +*/ +/* Opcode: Ge P1 P2 P3 P4 P5 +** Synopsis: IF r[P3]>=r[P1] +** +** This works just like the Lt opcode except that the jump is taken if +** the content of register P3 is greater than or equal to the content of +** register P1. See the Lt opcode for additional information. +*/ +case OP_Eq: /* same as TK_EQ, jump, in1, in3 */ +case OP_Ne: /* same as TK_NE, jump, in1, in3 */ +case OP_Lt: /* same as TK_LT, jump, in1, in3 */ +case OP_Le: /* same as TK_LE, jump, in1, in3 */ +case OP_Gt: /* same as TK_GT, jump, in1, in3 */ +case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ + int res, res2; /* Result of the comparison of pIn1 against pIn3 */ + char affinity; /* Affinity to use for comparison */ + u16 flags1; /* Copy of initial value of pIn1->flags */ + u16 flags3; /* Copy of initial value of pIn3->flags */ + + pIn1 = &aMem[pOp->p1]; + pIn3 = &aMem[pOp->p3]; + flags1 = pIn1->flags; + flags3 = pIn3->flags; + if( (flags1 & flags3 & MEM_Int)!=0 ){ + assert( (pOp->p5 & SQLITE_AFF_MASK)!=SQLITE_AFF_TEXT || CORRUPT_DB ); + /* Common case of comparison of two integers */ + if( pIn3->u.i > pIn1->u.i ){ + iCompare = +1; + if( sqlite3aGTb[pOp->opcode] ){ + VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); + goto jump_to_p2; + } + }else if( pIn3->u.i < pIn1->u.i ){ + iCompare = -1; + if( sqlite3aLTb[pOp->opcode] ){ + VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); + goto jump_to_p2; + } + }else{ + iCompare = 0; + if( sqlite3aEQb[pOp->opcode] ){ + VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); + goto jump_to_p2; + } + } + VdbeBranchTaken(0, (pOp->p5 & SQLITE_NULLEQ)?2:3); + break; + } + if( (flags1 | flags3)&MEM_Null ){ + /* One or both operands are NULL */ + if( pOp->p5 & SQLITE_NULLEQ ){ + /* If SQLITE_NULLEQ is set (which will only happen if the operator is + ** OP_Eq or OP_Ne) then take the jump or not depending on whether + ** or not both operands are null. + */ + assert( (flags1 & MEM_Cleared)==0 ); + assert( (pOp->p5 & SQLITE_JUMPIFNULL)==0 || CORRUPT_DB ); + testcase( (pOp->p5 & SQLITE_JUMPIFNULL)!=0 ); + if( (flags1&flags3&MEM_Null)!=0 + && (flags3&MEM_Cleared)==0 + ){ + res = 0; /* Operands are equal */ + }else{ + res = ((flags3 & MEM_Null) ? -1 : +1); /* Operands are not equal */ + } + }else{ + /* SQLITE_NULLEQ is clear and at least one operand is NULL, + ** then the result is always NULL. + ** The jump is taken if the SQLITE_JUMPIFNULL bit is set. + */ + iCompare = 1; /* Operands are not equal */ + VdbeBranchTaken(2,3); + if( pOp->p5 & SQLITE_JUMPIFNULL ){ + goto jump_to_p2; + } + break; + } + }else{ + /* Neither operand is NULL and we couldn't do the special high-speed + ** integer comparison case. So do a general-case comparison. */ + affinity = pOp->p5 & SQLITE_AFF_MASK; + if( affinity>=SQLITE_AFF_NUMERIC ){ + if( (flags1 | flags3)&MEM_Str ){ + if( (flags1 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ + applyNumericAffinity(pIn1,0); + testcase( flags3==pIn3->flags ); + flags3 = pIn3->flags; + } + if( (flags3 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ + applyNumericAffinity(pIn3,0); + } + } + }else if( affinity==SQLITE_AFF_TEXT ){ + if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + testcase( pIn1->flags & MEM_Int ); + testcase( pIn1->flags & MEM_Real ); + testcase( pIn1->flags & MEM_IntReal ); + sqlite3VdbeMemStringify(pIn1, encoding, 1); + testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); + flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); + if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str; + } + if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ + testcase( pIn3->flags & MEM_Int ); + testcase( pIn3->flags & MEM_Real ); + testcase( pIn3->flags & MEM_IntReal ); + sqlite3VdbeMemStringify(pIn3, encoding, 1); + testcase( (flags3&MEM_Dyn) != (pIn3->flags&MEM_Dyn) ); + flags3 = (pIn3->flags & ~MEM_TypeMask) | (flags3 & MEM_TypeMask); + } + } + assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); + res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); + } + + /* At this point, res is negative, zero, or positive if reg[P1] is + ** less than, equal to, or greater than reg[P3], respectively. Compute + ** the answer to this operator in res2, depending on what the comparison + ** operator actually is. The next block of code depends on the fact + ** that the 6 comparison operators are consecutive integers in this + ** order: NE, EQ, GT, LE, LT, GE */ + assert( OP_Eq==OP_Ne+1 ); assert( OP_Gt==OP_Ne+2 ); assert( OP_Le==OP_Ne+3 ); + assert( OP_Lt==OP_Ne+4 ); assert( OP_Ge==OP_Ne+5 ); + if( res<0 ){ + res2 = sqlite3aLTb[pOp->opcode]; + }else if( res==0 ){ + res2 = sqlite3aEQb[pOp->opcode]; + }else{ + res2 = sqlite3aGTb[pOp->opcode]; + } + iCompare = res; + + /* Undo any changes made by applyAffinity() to the input registers. */ + assert( (pIn3->flags & MEM_Dyn) == (flags3 & MEM_Dyn) ); + pIn3->flags = flags3; + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; + + VdbeBranchTaken(res2!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); + if( res2 ){ + goto jump_to_p2; + } + break; +} + +/* Opcode: ElseEq * P2 * * * +** +** This opcode must follow an OP_Lt or OP_Gt comparison operator. There +** can be zero or more OP_ReleaseReg opcodes intervening, but no other +** opcodes are allowed to occur between this instruction and the previous +** OP_Lt or OP_Gt. +** +** If result of an OP_Eq comparison on the same two operands as the +** prior OP_Lt or OP_Gt would have been true, then jump to P2. +** If the result of an OP_Eq comparison on the two previous +** operands would have been false or NULL, then fall through. +*/ +case OP_ElseEq: { /* same as TK_ESCAPE, jump */ + +#ifdef SQLITE_DEBUG + /* Verify the preconditions of this opcode - that it follows an OP_Lt or + ** OP_Gt with zero or more intervening OP_ReleaseReg opcodes */ + int iAddr; + for(iAddr = (int)(pOp - aOp) - 1; ALWAYS(iAddr>=0); iAddr--){ + if( aOp[iAddr].opcode==OP_ReleaseReg ) continue; + assert( aOp[iAddr].opcode==OP_Lt || aOp[iAddr].opcode==OP_Gt ); + break; + } +#endif /* SQLITE_DEBUG */ + VdbeBranchTaken(iCompare==0, 2); + if( iCompare==0 ) goto jump_to_p2; + break; +} + + +/* Opcode: Permutation * * * P4 * +** +** Set the permutation used by the OP_Compare operator in the next +** instruction. The permutation is stored in the P4 operand. +** +** The permutation is only valid until the next OP_Compare that has +** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should +** occur immediately prior to the OP_Compare. +** +** The first integer in the P4 integer array is the length of the array +** and does not become part of the permutation. +*/ +case OP_Permutation: { + assert( pOp->p4type==P4_INTARRAY ); + assert( pOp->p4.ai ); + assert( pOp[1].opcode==OP_Compare ); + assert( pOp[1].p5 & OPFLAG_PERMUTE ); + break; +} + +/* Opcode: Compare P1 P2 P3 P4 P5 +** Synopsis: r[P1@P3] <-> r[P2@P3] +** +** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this +** vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of +** the comparison for use by the next OP_Jump instruct. +** +** If P5 has the OPFLAG_PERMUTE bit set, then the order of comparison is +** determined by the most recent OP_Permutation operator. If the +** OPFLAG_PERMUTE bit is clear, then register are compared in sequential +** order. +** +** P4 is a KeyInfo structure that defines collating sequences and sort +** orders for the comparison. The permutation applies to registers +** only. The KeyInfo elements are used sequentially. +** +** The comparison is a sort comparison, so NULLs compare equal, +** NULLs are less than numbers, numbers are less than strings, +** and strings are less than blobs. +*/ +case OP_Compare: { + int n; + int i; + int p1; + int p2; + const KeyInfo *pKeyInfo; + u32 idx; + CollSeq *pColl; /* Collating sequence to use on this term */ + int bRev; /* True for DESCENDING sort order */ + u32 *aPermute; /* The permutation */ + + if( (pOp->p5 & OPFLAG_PERMUTE)==0 ){ + aPermute = 0; + }else{ + assert( pOp>aOp ); + assert( pOp[-1].opcode==OP_Permutation ); + assert( pOp[-1].p4type==P4_INTARRAY ); + aPermute = pOp[-1].p4.ai + 1; + assert( aPermute!=0 ); + } + n = pOp->p3; + pKeyInfo = pOp->p4.pKeyInfo; + assert( n>0 ); + assert( pKeyInfo!=0 ); + p1 = pOp->p1; + p2 = pOp->p2; +#ifdef SQLITE_DEBUG + if( aPermute ){ + int k, mx = 0; + for(k=0; k(u32)mx ) mx = aPermute[k]; + assert( p1>0 && p1+mx<=(p->nMem+1 - p->nCursor)+1 ); + assert( p2>0 && p2+mx<=(p->nMem+1 - p->nCursor)+1 ); + }else{ + assert( p1>0 && p1+n<=(p->nMem+1 - p->nCursor)+1 ); + assert( p2>0 && p2+n<=(p->nMem+1 - p->nCursor)+1 ); + } +#endif /* SQLITE_DEBUG */ + for(i=0; inKeyField ); + pColl = pKeyInfo->aColl[i]; + bRev = (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC); + iCompare = sqlite3MemCompare(&aMem[p1+idx], &aMem[p2+idx], pColl); + if( iCompare ){ + if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) + && ((aMem[p1+idx].flags & MEM_Null) || (aMem[p2+idx].flags & MEM_Null)) + ){ + iCompare = -iCompare; + } + if( bRev ) iCompare = -iCompare; + break; + } + } + break; +} + +/* Opcode: Jump P1 P2 P3 * * +** +** Jump to the instruction at address P1, P2, or P3 depending on whether +** in the most recent OP_Compare instruction the P1 vector was less than +** equal to, or greater than the P2 vector, respectively. +*/ +case OP_Jump: { /* jump */ + if( iCompare<0 ){ + VdbeBranchTaken(0,4); pOp = &aOp[pOp->p1 - 1]; + }else if( iCompare==0 ){ + VdbeBranchTaken(1,4); pOp = &aOp[pOp->p2 - 1]; + }else{ + VdbeBranchTaken(2,4); pOp = &aOp[pOp->p3 - 1]; + } + break; +} + +/* Opcode: And P1 P2 P3 * * +** Synopsis: r[P3]=(r[P1] && r[P2]) +** +** Take the logical AND of the values in registers P1 and P2 and +** write the result into register P3. +** +** If either P1 or P2 is 0 (false) then the result is 0 even if +** the other input is NULL. A NULL and true or two NULLs give +** a NULL output. +*/ +/* Opcode: Or P1 P2 P3 * * +** Synopsis: r[P3]=(r[P1] || r[P2]) +** +** Take the logical OR of the values in register P1 and P2 and +** store the answer in register P3. +** +** If either P1 or P2 is nonzero (true) then the result is 1 (true) +** even if the other input is NULL. A NULL and false or two NULLs +** give a NULL output. +*/ +case OP_And: /* same as TK_AND, in1, in2, out3 */ +case OP_Or: { /* same as TK_OR, in1, in2, out3 */ + int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ + int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ + + v1 = sqlite3VdbeBooleanValue(&aMem[pOp->p1], 2); + v2 = sqlite3VdbeBooleanValue(&aMem[pOp->p2], 2); + if( pOp->opcode==OP_And ){ + static const unsigned char and_logic[] = { 0, 0, 0, 0, 1, 2, 0, 2, 2 }; + v1 = and_logic[v1*3+v2]; + }else{ + static const unsigned char or_logic[] = { 0, 1, 2, 1, 1, 1, 2, 1, 2 }; + v1 = or_logic[v1*3+v2]; + } + pOut = &aMem[pOp->p3]; + if( v1==2 ){ + MemSetTypeFlag(pOut, MEM_Null); + }else{ + pOut->u.i = v1; + MemSetTypeFlag(pOut, MEM_Int); + } + break; +} + +/* Opcode: IsTrue P1 P2 P3 P4 * +** Synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 +** +** This opcode implements the IS TRUE, IS FALSE, IS NOT TRUE, and +** IS NOT FALSE operators. +** +** Interpret the value in register P1 as a boolean value. Store that +** boolean (a 0 or 1) in register P2. Or if the value in register P1 is +** NULL, then the P3 is stored in register P2. Invert the answer if P4 +** is 1. +** +** The logic is summarized like this: +** +**
      +**
    • If P3==0 and P4==0 then r[P2] := r[P1] IS TRUE +**
    • If P3==1 and P4==1 then r[P2] := r[P1] IS FALSE +**
    • If P3==0 and P4==1 then r[P2] := r[P1] IS NOT TRUE +**
    • If P3==1 and P4==0 then r[P2] := r[P1] IS NOT FALSE +**
    +*/ +case OP_IsTrue: { /* in1, out2 */ + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p4.i==0 || pOp->p4.i==1 ); + assert( pOp->p3==0 || pOp->p3==1 ); + sqlite3VdbeMemSetInt64(&aMem[pOp->p2], + sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3) ^ pOp->p4.i); + break; +} + +/* Opcode: Not P1 P2 * * * +** Synopsis: r[P2]= !r[P1] +** +** Interpret the value in register P1 as a boolean value. Store the +** boolean complement in register P2. If the value in register P1 is +** NULL, then a NULL is stored in P2. +*/ +case OP_Not: { /* same as TK_NOT, in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( (pIn1->flags & MEM_Null)==0 ){ + sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeBooleanValue(pIn1,0)); + }else{ + sqlite3VdbeMemSetNull(pOut); + } + break; +} + +/* Opcode: BitNot P1 P2 * * * +** Synopsis: r[P2]= ~r[P1] +** +** Interpret the content of register P1 as an integer. Store the +** ones-complement of the P1 value into register P2. If P1 holds +** a NULL then store a NULL in P2. +*/ +case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + sqlite3VdbeMemSetNull(pOut); + if( (pIn1->flags & MEM_Null)==0 ){ + pOut->flags = MEM_Int; + pOut->u.i = ~sqlite3VdbeIntValue(pIn1); + } + break; +} + +/* Opcode: Once P1 P2 * * * +** +** Fall through to the next instruction the first time this opcode is +** encountered on each invocation of the byte-code program. Jump to P2 +** on the second and all subsequent encounters during the same invocation. +** +** Top-level programs determine first invocation by comparing the P1 +** operand against the P1 operand on the OP_Init opcode at the beginning +** of the program. If the P1 values differ, then fall through and make +** the P1 of this opcode equal to the P1 of OP_Init. If P1 values are +** the same then take the jump. +** +** For subprograms, there is a bitmask in the VdbeFrame that determines +** whether or not the jump should be taken. The bitmask is necessary +** because the self-altering code trick does not work for recursive +** triggers. +*/ +case OP_Once: { /* jump */ + u32 iAddr; /* Address of this instruction */ + assert( p->aOp[0].opcode==OP_Init ); + if( p->pFrame ){ + iAddr = (int)(pOp - p->aOp); + if( (p->pFrame->aOnce[iAddr/8] & (1<<(iAddr & 7)))!=0 ){ + VdbeBranchTaken(1, 2); + goto jump_to_p2; + } + p->pFrame->aOnce[iAddr/8] |= 1<<(iAddr & 7); + }else{ + if( p->aOp[0].p1==pOp->p1 ){ + VdbeBranchTaken(1, 2); + goto jump_to_p2; + } + } + VdbeBranchTaken(0, 2); + pOp->p1 = p->aOp[0].p1; + break; +} + +/* Opcode: If P1 P2 P3 * * +** +** Jump to P2 if the value in register P1 is true. The value +** is considered true if it is numeric and non-zero. If the value +** in P1 is NULL then take the jump if and only if P3 is non-zero. +*/ +case OP_If: { /* jump, in1 */ + int c; + c = sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3); + VdbeBranchTaken(c!=0, 2); + if( c ) goto jump_to_p2; + break; +} + +/* Opcode: IfNot P1 P2 P3 * * +** +** Jump to P2 if the value in register P1 is False. The value +** is considered false if it has a numeric value of zero. If the value +** in P1 is NULL then take the jump if and only if P3 is non-zero. +*/ +case OP_IfNot: { /* jump, in1 */ + int c; + c = !sqlite3VdbeBooleanValue(&aMem[pOp->p1], !pOp->p3); + VdbeBranchTaken(c!=0, 2); + if( c ) goto jump_to_p2; + break; +} + +/* Opcode: IsNull P1 P2 * * * +** Synopsis: if r[P1]==NULL goto P2 +** +** Jump to P2 if the value in register P1 is NULL. +*/ +case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ + pIn1 = &aMem[pOp->p1]; + VdbeBranchTaken( (pIn1->flags & MEM_Null)!=0, 2); + if( (pIn1->flags & MEM_Null)!=0 ){ + goto jump_to_p2; + } + break; +} + +/* Opcode: IsNullOrType P1 P2 P3 * * +** Synopsis: if typeof(r[P1]) IN (P3,5) goto P2 +** +** Jump to P2 if the value in register P1 is NULL or has a datatype P3. +** P3 is an integer which should be one of SQLITE_INTEGER, SQLITE_FLOAT, +** SQLITE_BLOB, SQLITE_NULL, or SQLITE_TEXT. +*/ +case OP_IsNullOrType: { /* jump, in1 */ + int doTheJump; + pIn1 = &aMem[pOp->p1]; + doTheJump = (pIn1->flags & MEM_Null)!=0 || sqlite3_value_type(pIn1)==pOp->p3; + VdbeBranchTaken( doTheJump, 2); + if( doTheJump ) goto jump_to_p2; + break; +} + +/* Opcode: ZeroOrNull P1 P2 P3 * * +** Synopsis: r[P2] = 0 OR NULL +** +** If all both registers P1 and P3 are NOT NULL, then store a zero in +** register P2. If either registers P1 or P3 are NULL then put +** a NULL in register P2. +*/ +case OP_ZeroOrNull: { /* in1, in2, out2, in3 */ + if( (aMem[pOp->p1].flags & MEM_Null)!=0 + || (aMem[pOp->p3].flags & MEM_Null)!=0 + ){ + sqlite3VdbeMemSetNull(aMem + pOp->p2); + }else{ + sqlite3VdbeMemSetInt64(aMem + pOp->p2, 0); + } + break; +} + +/* Opcode: NotNull P1 P2 * * * +** Synopsis: if r[P1]!=NULL goto P2 +** +** Jump to P2 if the value in register P1 is not NULL. +*/ +case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ + pIn1 = &aMem[pOp->p1]; + VdbeBranchTaken( (pIn1->flags & MEM_Null)==0, 2); + if( (pIn1->flags & MEM_Null)==0 ){ + goto jump_to_p2; + } + break; +} + +/* Opcode: IfNullRow P1 P2 P3 * * +** Synopsis: if P1.nullRow then r[P3]=NULL, goto P2 +** +** Check the cursor P1 to see if it is currently pointing at a NULL row. +** If it is, then set register P3 to NULL and jump immediately to P2. +** If P1 is not on a NULL row, then fall through without making any +** changes. +*/ +case OP_IfNullRow: { /* jump */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( p->apCsr[pOp->p1]!=0 ); + if( p->apCsr[pOp->p1]->nullRow ){ + sqlite3VdbeMemSetNull(aMem + pOp->p3); + goto jump_to_p2; + } + break; +} + +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC +/* Opcode: Offset P1 P2 P3 * * +** Synopsis: r[P3] = sqlite_offset(P1) +** +** Store in register r[P3] the byte offset into the database file that is the +** start of the payload for the record at which that cursor P1 is currently +** pointing. +** +** P2 is the column number for the argument to the sqlite_offset() function. +** This opcode does not use P2 itself, but the P2 value is used by the +** code generator. The P1, P2, and P3 operands to this opcode are the +** same as for OP_Column. +** +** This opcode is only available if SQLite is compiled with the +** -DSQLITE_ENABLE_OFFSET_SQL_FUNC option. +*/ +case OP_Offset: { /* out3 */ + VdbeCursor *pC; /* The VDBE cursor */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + pOut = &p->aMem[pOp->p3]; + if( NEVER(pC==0) || pC->eCurType!=CURTYPE_BTREE ){ + sqlite3VdbeMemSetNull(pOut); + }else{ + sqlite3VdbeMemSetInt64(pOut, sqlite3BtreeOffset(pC->uc.pCursor)); + } + break; +} +#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */ + +/* Opcode: Column P1 P2 P3 P4 P5 +** Synopsis: r[P3]=PX +** +** Interpret the data that cursor P1 points to as a structure built using +** the MakeRecord instruction. (See the MakeRecord opcode for additional +** information about the format of the data.) Extract the P2-th column +** from this record. If there are less that (P2+1) +** values in the record, extract a NULL. +** +** The value extracted is stored in register P3. +** +** If the record contains fewer than P2 fields, then extract a NULL. Or, +** if the P4 argument is a P4_MEM use the value of the P4 argument as +** the result. +** +** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 then +** the result is guaranteed to only be used as the argument of a length() +** or typeof() function, respectively. The loading of large blobs can be +** skipped for length() and all content loading can be skipped for typeof(). +*/ +case OP_Column: { + u32 p2; /* column number to retrieve */ + VdbeCursor *pC; /* The VDBE cursor */ + BtCursor *pCrsr; /* The BTree cursor */ + u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ + int len; /* The length of the serialized data for the column */ + int i; /* Loop counter */ + Mem *pDest; /* Where to write the extracted value */ + Mem sMem; /* For storing the record being decoded */ + const u8 *zData; /* Part of the record being decoded */ + const u8 *zHdr; /* Next unparsed byte of the header */ + const u8 *zEndHdr; /* Pointer to first byte after the header */ + u64 offset64; /* 64-bit offset */ + u32 t; /* A type code from the record header */ + Mem *pReg; /* PseudoTable input register */ + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + p2 = (u32)pOp->p2; + + /* If the cursor cache is stale (meaning it is not currently point at + ** the correct row) then bring it up-to-date by doing the necessary + ** B-Tree seek. */ + rc = sqlite3VdbeCursorMoveto(&pC, &p2); + if( rc ) goto abort_due_to_error; + + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + pDest = &aMem[pOp->p3]; + memAboutToChange(p, pDest); + assert( pC!=0 ); + assert( p2<(u32)pC->nField ); + aOffset = pC->aOffset; + assert( pC->eCurType!=CURTYPE_VTAB ); + assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow ); + assert( pC->eCurType!=CURTYPE_SORTER ); + + if( pC->cacheStatus!=p->cacheCtr ){ /*OPTIMIZATION-IF-FALSE*/ + if( pC->nullRow ){ + if( pC->eCurType==CURTYPE_PSEUDO ){ + /* For the special case of as pseudo-cursor, the seekResult field + ** identifies the register that holds the record */ + assert( pC->seekResult>0 ); + pReg = &aMem[pC->seekResult]; + assert( pReg->flags & MEM_Blob ); + assert( memIsValid(pReg) ); + pC->payloadSize = pC->szRow = pReg->n; + pC->aRow = (u8*)pReg->z; + }else{ + sqlite3VdbeMemSetNull(pDest); + goto op_column_out; + } + }else{ + pCrsr = pC->uc.pCursor; + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pCrsr ); + assert( sqlite3BtreeCursorIsValid(pCrsr) ); + pC->payloadSize = sqlite3BtreePayloadSize(pCrsr); + pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &pC->szRow); + assert( pC->szRow<=pC->payloadSize ); + assert( pC->szRow<=65536 ); /* Maximum page size is 64KiB */ + if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + } + pC->cacheStatus = p->cacheCtr; + pC->iHdrOffset = getVarint32(pC->aRow, aOffset[0]); + pC->nHdrParsed = 0; + + + if( pC->szRowaRow does not have to hold the entire row, but it does at least + ** need to cover the header of the record. If pC->aRow does not contain + ** the complete header, then set it to zero, forcing the header to be + ** dynamically allocated. */ + pC->aRow = 0; + pC->szRow = 0; + + /* Make sure a corrupt database has not given us an oversize header. + ** Do this now to avoid an oversize memory allocation. + ** + ** Type entries can be between 1 and 5 bytes each. But 4 and 5 byte + ** types use so much data space that there can only be 4096 and 32 of + ** them, respectively. So the maximum header length results from a + ** 3-byte type for each of the maximum of 32768 columns plus three + ** extra bytes for the header length itself. 32768*3 + 3 = 98307. + */ + if( aOffset[0] > 98307 || aOffset[0] > pC->payloadSize ){ + goto op_column_corrupt; + } + }else{ + /* This is an optimization. By skipping over the first few tests + ** (ex: pC->nHdrParsed<=p2) in the next section, we achieve a + ** measurable performance gain. + ** + ** This branch is taken even if aOffset[0]==0. Such a record is never + ** generated by SQLite, and could be considered corruption, but we + ** accept it for historical reasons. When aOffset[0]==0, the code this + ** branch jumps to reads past the end of the record, but never more + ** than a few bytes. Even if the record occurs at the end of the page + ** content area, the "page header" comes after the page content and so + ** this overread is harmless. Similar overreads can occur for a corrupt + ** database file. + */ + zData = pC->aRow; + assert( pC->nHdrParsed<=p2 ); /* Conditional skipped */ + testcase( aOffset[0]==0 ); + goto op_column_read_header; + } + } + + /* Make sure at least the first p2+1 entries of the header have been + ** parsed and valid information is in aOffset[] and pC->aType[]. + */ + if( pC->nHdrParsed<=p2 ){ + /* If there is more header available for parsing in the record, try + ** to extract additional fields up through the p2+1-th field + */ + if( pC->iHdrOffsetaRow==0 ){ + memset(&sMem, 0, sizeof(sMem)); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pC->uc.pCursor,aOffset[0],&sMem); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + zData = (u8*)sMem.z; + }else{ + zData = pC->aRow; + } + + /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */ + op_column_read_header: + i = pC->nHdrParsed; + offset64 = aOffset[i]; + zHdr = zData + pC->iHdrOffset; + zEndHdr = zData + aOffset[0]; + testcase( zHdr>=zEndHdr ); + do{ + if( (pC->aType[i] = t = zHdr[0])<0x80 ){ + zHdr++; + offset64 += sqlite3VdbeOneByteSerialTypeLen(t); + }else{ + zHdr += sqlite3GetVarint32(zHdr, &t); + pC->aType[i] = t; + offset64 += sqlite3VdbeSerialTypeLen(t); + } + aOffset[++i] = (u32)(offset64 & 0xffffffff); + }while( (u32)i<=p2 && zHdr=zEndHdr && (zHdr>zEndHdr || offset64!=pC->payloadSize)) + || (offset64 > pC->payloadSize) + ){ + if( aOffset[0]==0 ){ + i = 0; + zHdr = zEndHdr; + }else{ + if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); + goto op_column_corrupt; + } + } + + pC->nHdrParsed = i; + pC->iHdrOffset = (u32)(zHdr - zData); + if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); + }else{ + t = 0; + } + + /* If after trying to extract new entries from the header, nHdrParsed is + ** still not up to p2, that means that the record has fewer than p2 + ** columns. So the result will be either the default value or a NULL. + */ + if( pC->nHdrParsed<=p2 ){ + if( pOp->p4type==P4_MEM ){ + sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); + }else{ + sqlite3VdbeMemSetNull(pDest); + } + goto op_column_out; + } + }else{ + t = pC->aType[p2]; + } + + /* Extract the content for the p2+1-th column. Control can only + ** reach this point if aOffset[p2], aOffset[p2+1], and pC->aType[p2] are + ** all valid. + */ + assert( p2nHdrParsed ); + assert( rc==SQLITE_OK ); + assert( sqlite3VdbeCheckMemInvariants(pDest) ); + if( VdbeMemDynamic(pDest) ){ + sqlite3VdbeMemSetNull(pDest); + } + assert( t==pC->aType[p2] ); + if( pC->szRow>=aOffset[p2+1] ){ + /* This is the common case where the desired content fits on the original + ** page - where the content is not on an overflow page */ + zData = pC->aRow + aOffset[p2]; + if( t<12 ){ + sqlite3VdbeSerialGet(zData, t, pDest); + }else{ + /* If the column value is a string, we need a persistent value, not + ** a MEM_Ephem value. This branch is a fast short-cut that is equivalent + ** to calling sqlite3VdbeSerialGet() and sqlite3VdbeDeephemeralize(). + */ + static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term }; + pDest->n = len = (t-12)/2; + pDest->enc = encoding; + if( pDest->szMalloc < len+2 ){ + pDest->flags = MEM_Null; + if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem; + }else{ + pDest->z = pDest->zMalloc; + } + memcpy(pDest->z, zData, len); + pDest->z[len] = 0; + pDest->z[len+1] = 0; + pDest->flags = aFlag[t&1]; + } + }else{ + pDest->enc = encoding; + /* This branch happens only when content is on overflow pages */ + if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 + && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) + || (len = sqlite3VdbeSerialTypeLen(t))==0 + ){ + /* Content is irrelevant for + ** 1. the typeof() function, + ** 2. the length(X) function if X is a blob, and + ** 3. if the content length is zero. + ** So we might as well use bogus content rather than reading + ** content from disk. + ** + ** Although sqlite3VdbeSerialGet() may read at most 8 bytes from the + ** buffer passed to it, debugging function VdbeMemPrettyPrint() may + ** read more. Use the global constant sqlite3CtypeMap[] as the array, + ** as that array is 256 bytes long (plenty for VdbeMemPrettyPrint()) + ** and it begins with a bunch of zeros. + */ + sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest); + }else{ + rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + pDest->flags &= ~MEM_Ephem; + } + } + +op_column_out: + UPDATE_MAX_BLOBSIZE(pDest); + REGISTER_TRACE(pOp->p3, pDest); + break; + +op_column_corrupt: + if( aOp[0].p3>0 ){ + pOp = &aOp[aOp[0].p3-1]; + break; + }else{ + rc = SQLITE_CORRUPT_BKPT; + goto abort_due_to_error; + } +} + +/* Opcode: TypeCheck P1 P2 * P4 * +** Synopsis: typecheck(r[P1@P2]) +** +** Apply affinities to the range of P2 registers beginning with P1. +** Take the affinities from the Table object in P4. If any value +** cannot be coerced into the correct type, then raise an error. +** +** This opcode is similar to OP_Affinity except that this opcode +** forces the register type to the Table column type. This is used +** to implement "strict affinity". +** +** Preconditions: +** +**
      +**
    • P2 should be the number of non-virtual columns in the +** table of P4. +**
    • Table P4 should be a STRICT table. +**
    +** +** If any precondition is false, an assertion fault occurs. +*/ +case OP_TypeCheck: { + Table *pTab; + Column *aCol; + int i; + + assert( pOp->p4type==P4_TABLE ); + pTab = pOp->p4.pTab; + assert( pTab->tabFlags & TF_Strict ); + assert( pTab->nNVCol==pOp->p2 ); + aCol = pTab->aCol; + pIn1 = &aMem[pOp->p1]; + for(i=0; inCol; i++){ + if( aCol[i].colFlags & COLFLAG_VIRTUAL ) continue; + assert( pIn1 < &aMem[pOp->p1+pOp->p2] ); + applyAffinity(pIn1, aCol[i].affinity, encoding); + if( (pIn1->flags & MEM_Null)==0 ){ + switch( aCol[i].eCType ){ + case COLTYPE_BLOB: { + if( (pIn1->flags & MEM_Blob)==0 ) goto vdbe_type_error; + break; + } + case COLTYPE_INTEGER: + case COLTYPE_INT: { + if( (pIn1->flags & MEM_Int)==0 ) goto vdbe_type_error; + break; + } + case COLTYPE_TEXT: { + if( (pIn1->flags & MEM_Str)==0 ) goto vdbe_type_error; + break; + } + case COLTYPE_REAL: { + if( pIn1->flags & MEM_Int ){ + /* When applying REAL affinity, if the result is still an MEM_Int + ** that will fit in 6 bytes, then change the type to MEM_IntReal + ** so that we keep the high-resolution integer value but know that + ** the type really wants to be REAL. */ + testcase( pIn1->u.i==140737488355328LL ); + testcase( pIn1->u.i==140737488355327LL ); + testcase( pIn1->u.i==-140737488355328LL ); + testcase( pIn1->u.i==-140737488355329LL ); + if( pIn1->u.i<=140737488355327LL && pIn1->u.i>=-140737488355328LL){ + pIn1->flags |= MEM_IntReal; + pIn1->flags &= ~MEM_Int; + }else{ + pIn1->u.r = (double)pIn1->u.i; + pIn1->flags |= MEM_Real; + pIn1->flags &= ~MEM_Int; + } + }else if( (pIn1->flags & MEM_Real)==0 ){ + goto vdbe_type_error; + } + break; + } + default: { + /* COLTYPE_ANY. Accept anything. */ + break; + } + } + } + REGISTER_TRACE((int)(pIn1-aMem), pIn1); + pIn1++; + } + assert( pIn1 == &aMem[pOp->p1+pOp->p2] ); + break; + +vdbe_type_error: + sqlite3VdbeError(p, "cannot store %s value in %s column %s.%s", + vdbeMemTypeName(pIn1), sqlite3StdType[aCol[i].eCType-1], + pTab->zName, aCol[i].zCnName); + rc = SQLITE_CONSTRAINT_DATATYPE; + goto abort_due_to_error; +} + +/* Opcode: Affinity P1 P2 * P4 * +** Synopsis: affinity(r[P1@P2]) +** +** Apply affinities to a range of P2 registers starting with P1. +** +** P4 is a string that is P2 characters long. The N-th character of the +** string indicates the column affinity that should be used for the N-th +** memory cell in the range. +*/ +case OP_Affinity: { + const char *zAffinity; /* The affinity to be applied */ + + zAffinity = pOp->p4.z; + assert( zAffinity!=0 ); + assert( pOp->p2>0 ); + assert( zAffinity[pOp->p2]==0 ); + pIn1 = &aMem[pOp->p1]; + while( 1 /*exit-by-break*/ ){ + assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); + assert( zAffinity[0]==SQLITE_AFF_NONE || memIsValid(pIn1) ); + applyAffinity(pIn1, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pIn1->flags & MEM_Int)!=0 ){ + /* When applying REAL affinity, if the result is still an MEM_Int + ** that will fit in 6 bytes, then change the type to MEM_IntReal + ** so that we keep the high-resolution integer value but know that + ** the type really wants to be REAL. */ + testcase( pIn1->u.i==140737488355328LL ); + testcase( pIn1->u.i==140737488355327LL ); + testcase( pIn1->u.i==-140737488355328LL ); + testcase( pIn1->u.i==-140737488355329LL ); + if( pIn1->u.i<=140737488355327LL && pIn1->u.i>=-140737488355328LL ){ + pIn1->flags |= MEM_IntReal; + pIn1->flags &= ~MEM_Int; + }else{ + pIn1->u.r = (double)pIn1->u.i; + pIn1->flags |= MEM_Real; + pIn1->flags &= ~MEM_Int; + } + } + REGISTER_TRACE((int)(pIn1-aMem), pIn1); + zAffinity++; + if( zAffinity[0]==0 ) break; + pIn1++; + } + break; +} + +/* Opcode: MakeRecord P1 P2 P3 P4 * +** Synopsis: r[P3]=mkrec(r[P1@P2]) +** +** Convert P2 registers beginning with P1 into the [record format] +** use as a data record in a database table or as a key +** in an index. The OP_Column opcode can decode the record later. +** +** P4 may be a string that is P2 characters long. The N-th character of the +** string indicates the column affinity that should be used for the N-th +** field of the index key. +** +** The mapping from character to affinity is given by the SQLITE_AFF_ +** macros defined in sqliteInt.h. +** +** If P4 is NULL then all index fields have the affinity BLOB. +** +** The meaning of P5 depends on whether or not the SQLITE_ENABLE_NULL_TRIM +** compile-time option is enabled: +** +** * If SQLITE_ENABLE_NULL_TRIM is enabled, then the P5 is the index +** of the right-most table that can be null-trimmed. +** +** * If SQLITE_ENABLE_NULL_TRIM is omitted, then P5 has the value +** OPFLAG_NOCHNG_MAGIC if the OP_MakeRecord opcode is allowed to +** accept no-change records with serial_type 10. This value is +** only used inside an assert() and does not affect the end result. +*/ +case OP_MakeRecord: { + Mem *pRec; /* The new record */ + u64 nData; /* Number of bytes of data space */ + int nHdr; /* Number of bytes of header space */ + i64 nByte; /* Data space required for this record */ + i64 nZero; /* Number of zero bytes at the end of the record */ + int nVarint; /* Number of bytes in a varint */ + u32 serial_type; /* Type field */ + Mem *pData0; /* First field to be combined into the record */ + Mem *pLast; /* Last field of the record */ + int nField; /* Number of fields in the record */ + char *zAffinity; /* The affinity string for the record */ + int file_format; /* File format to use for encoding */ + u32 len; /* Length of a field */ + u8 *zHdr; /* Where to write next byte of the header */ + u8 *zPayload; /* Where to write next byte of the payload */ + + /* Assuming the record contains N fields, the record format looks + ** like this: + ** + ** ------------------------------------------------------------------------ + ** | hdr-size | type 0 | type 1 | ... | type N-1 | data0 | ... | data N-1 | + ** ------------------------------------------------------------------------ + ** + ** Data(0) is taken from register P1. Data(1) comes from register P1+1 + ** and so forth. + ** + ** Each type field is a varint representing the serial type of the + ** corresponding data element (see sqlite3VdbeSerialType()). The + ** hdr-size field is also a varint which is the offset from the beginning + ** of the record to data0. + */ + nData = 0; /* Number of bytes of data space */ + nHdr = 0; /* Number of bytes of header space */ + nZero = 0; /* Number of zero bytes at the end of the record */ + nField = pOp->p1; + zAffinity = pOp->p4.z; + assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem+1 - p->nCursor)+1 ); + pData0 = &aMem[nField]; + nField = pOp->p2; + pLast = &pData0[nField-1]; + file_format = p->minWriteFileFormat; + + /* Identify the output register */ + assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); + pOut = &aMem[pOp->p3]; + memAboutToChange(p, pOut); + + /* Apply the requested affinity to all inputs + */ + assert( pData0<=pLast ); + if( zAffinity ){ + pRec = pData0; + do{ + applyAffinity(pRec, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pRec->flags & MEM_Int) ){ + pRec->flags |= MEM_IntReal; + pRec->flags &= ~(MEM_Int); + } + REGISTER_TRACE((int)(pRec-aMem), pRec); + zAffinity++; + pRec++; + assert( zAffinity[0]==0 || pRec<=pLast ); + }while( zAffinity[0] ); + } + +#ifdef SQLITE_ENABLE_NULL_TRIM + /* NULLs can be safely trimmed from the end of the record, as long as + ** as the schema format is 2 or more and none of the omitted columns + ** have a non-NULL default value. Also, the record must be left with + ** at least one field. If P5>0 then it will be one more than the + ** index of the right-most column with a non-NULL default value */ + if( pOp->p5 ){ + while( (pLast->flags & MEM_Null)!=0 && nField>pOp->p5 ){ + pLast--; + nField--; + } + } +#endif + + /* Loop through the elements that will make up the record to figure + ** out how much space is required for the new record. After this loop, + ** the Mem.uTemp field of each term should hold the serial-type that will + ** be used for that term in the generated record: + ** + ** Mem.uTemp value type + ** --------------- --------------- + ** 0 NULL + ** 1 1-byte signed integer + ** 2 2-byte signed integer + ** 3 3-byte signed integer + ** 4 4-byte signed integer + ** 5 6-byte signed integer + ** 6 8-byte signed integer + ** 7 IEEE float + ** 8 Integer constant 0 + ** 9 Integer constant 1 + ** 10,11 reserved for expansion + ** N>=12 and even BLOB + ** N>=13 and odd text + ** + ** The following additional values are computed: + ** nHdr Number of bytes needed for the record header + ** nData Number of bytes of data space needed for the record + ** nZero Zero bytes at the end of the record + */ + pRec = pLast; + do{ + assert( memIsValid(pRec) ); + if( pRec->flags & MEM_Null ){ + if( pRec->flags & MEM_Zero ){ + /* Values with MEM_Null and MEM_Zero are created by xColumn virtual + ** table methods that never invoke sqlite3_result_xxxxx() while + ** computing an unchanging column value in an UPDATE statement. + ** Give such values a special internal-use-only serial-type of 10 + ** so that they can be passed through to xUpdate and have + ** a true sqlite3_value_nochange(). */ +#ifndef SQLITE_ENABLE_NULL_TRIM + assert( pOp->p5==OPFLAG_NOCHNG_MAGIC || CORRUPT_DB ); +#endif + pRec->uTemp = 10; + }else{ + pRec->uTemp = 0; + } + nHdr++; + }else if( pRec->flags & (MEM_Int|MEM_IntReal) ){ + /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ + i64 i = pRec->u.i; + u64 uu; + testcase( pRec->flags & MEM_Int ); + testcase( pRec->flags & MEM_IntReal ); + if( i<0 ){ + uu = ~i; + }else{ + uu = i; + } + nHdr++; + testcase( uu==127 ); testcase( uu==128 ); + testcase( uu==32767 ); testcase( uu==32768 ); + testcase( uu==8388607 ); testcase( uu==8388608 ); + testcase( uu==2147483647 ); testcase( uu==2147483648 ); + testcase( uu==140737488355327LL ); testcase( uu==140737488355328LL ); + if( uu<=127 ){ + if( (i&1)==i && file_format>=4 ){ + pRec->uTemp = 8+(u32)uu; + }else{ + nData++; + pRec->uTemp = 1; + } + }else if( uu<=32767 ){ + nData += 2; + pRec->uTemp = 2; + }else if( uu<=8388607 ){ + nData += 3; + pRec->uTemp = 3; + }else if( uu<=2147483647 ){ + nData += 4; + pRec->uTemp = 4; + }else if( uu<=140737488355327LL ){ + nData += 6; + pRec->uTemp = 5; + }else{ + nData += 8; + if( pRec->flags & MEM_IntReal ){ + /* If the value is IntReal and is going to take up 8 bytes to store + ** as an integer, then we might as well make it an 8-byte floating + ** point value */ + pRec->u.r = (double)pRec->u.i; + pRec->flags &= ~MEM_IntReal; + pRec->flags |= MEM_Real; + pRec->uTemp = 7; + }else{ + pRec->uTemp = 6; + } + } + }else if( pRec->flags & MEM_Real ){ + nHdr++; + nData += 8; + pRec->uTemp = 7; + }else{ + assert( db->mallocFailed || pRec->flags&(MEM_Str|MEM_Blob) ); + assert( pRec->n>=0 ); + len = (u32)pRec->n; + serial_type = (len*2) + 12 + ((pRec->flags & MEM_Str)!=0); + if( pRec->flags & MEM_Zero ){ + serial_type += pRec->u.nZero*2; + if( nData ){ + if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; + len += pRec->u.nZero; + }else{ + nZero += pRec->u.nZero; + } + } + nData += len; + nHdr += sqlite3VarintLen(serial_type); + pRec->uTemp = serial_type; + } + if( pRec==pData0 ) break; + pRec--; + }while(1); + + /* EVIDENCE-OF: R-22564-11647 The header begins with a single varint + ** which determines the total number of bytes in the header. The varint + ** value is the size of the header in bytes including the size varint + ** itself. */ + testcase( nHdr==126 ); + testcase( nHdr==127 ); + if( nHdr<=126 ){ + /* The common case */ + nHdr += 1; + }else{ + /* Rare case of a really large header */ + nVarint = sqlite3VarintLen(nHdr); + nHdr += nVarint; + if( nVarintp3) is not allowed to + ** be one of the input registers (because the following call to + ** sqlite3VdbeMemClearAndResize() could clobber the value before it is used). + */ + if( nByte+nZero<=pOut->szMalloc ){ + /* The output register is already large enough to hold the record. + ** No error checks or buffer enlargement is required */ + pOut->z = pOut->zMalloc; + }else{ + /* Need to make sure that the output is not too big and then enlarge + ** the output register to hold the full result */ + if( nByte+nZero>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + if( sqlite3VdbeMemClearAndResize(pOut, (int)nByte) ){ + goto no_mem; + } + } + pOut->n = (int)nByte; + pOut->flags = MEM_Blob; + if( nZero ){ + pOut->u.nZero = nZero; + pOut->flags |= MEM_Zero; + } + UPDATE_MAX_BLOBSIZE(pOut); + zHdr = (u8 *)pOut->z; + zPayload = zHdr + nHdr; + + /* Write the record */ + zHdr += putVarint32(zHdr, nHdr); + assert( pData0<=pLast ); + pRec = pData0; + do{ + serial_type = pRec->uTemp; + /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more + ** additional varints, one per column. */ + zHdr += putVarint32(zHdr, serial_type); /* serial type */ + /* EVIDENCE-OF: R-64536-51728 The values for each column in the record + ** immediately follow the header. */ + zPayload += sqlite3VdbeSerialPut(zPayload, pRec, serial_type); /* content */ + }while( (++pRec)<=pLast ); + assert( nHdr==(int)(zHdr - (u8*)pOut->z) ); + assert( nByte==(int)(zPayload - (u8*)pOut->z) ); + + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + REGISTER_TRACE(pOp->p3, pOut); + break; +} + +/* Opcode: Count P1 P2 p3 * * +** Synopsis: r[P2]=count() +** +** Store the number of entries (an integer value) in the table or index +** opened by cursor P1 in register P2. +** +** If P3==0, then an exact count is obtained, which involves visiting +** every btree page of the table. But if P3 is non-zero, an estimate +** is returned based on the current cursor position. +*/ +case OP_Count: { /* out2 */ + i64 nEntry; + BtCursor *pCrsr; + + assert( p->apCsr[pOp->p1]->eCurType==CURTYPE_BTREE ); + pCrsr = p->apCsr[pOp->p1]->uc.pCursor; + assert( pCrsr ); + if( pOp->p3 ){ + nEntry = sqlite3BtreeRowCountEst(pCrsr); + }else{ + nEntry = 0; /* Not needed. Only used to silence a warning. */ + rc = sqlite3BtreeCount(db, pCrsr, &nEntry); + if( rc ) goto abort_due_to_error; + } + pOut = out2Prerelease(p, pOp); + pOut->u.i = nEntry; + goto check_for_interrupt; +} + +/* Opcode: Savepoint P1 * * P4 * +** +** Open, release or rollback the savepoint named by parameter P4, depending +** on the value of P1. To open a new savepoint set P1==0 (SAVEPOINT_BEGIN). +** To release (commit) an existing savepoint set P1==1 (SAVEPOINT_RELEASE). +** To rollback an existing savepoint set P1==2 (SAVEPOINT_ROLLBACK). +*/ +case OP_Savepoint: { + int p1; /* Value of P1 operand */ + char *zName; /* Name of savepoint */ + int nName; + Savepoint *pNew; + Savepoint *pSavepoint; + Savepoint *pTmp; + int iSavepoint; + int ii; + + p1 = pOp->p1; + zName = pOp->p4.z; + + /* Assert that the p1 parameter is valid. Also that if there is no open + ** transaction, then there cannot be any savepoints. + */ + assert( db->pSavepoint==0 || db->autoCommit==0 ); + assert( p1==SAVEPOINT_BEGIN||p1==SAVEPOINT_RELEASE||p1==SAVEPOINT_ROLLBACK ); + assert( db->pSavepoint || db->isTransactionSavepoint==0 ); + assert( checkSavepointCount(db) ); + assert( p->bIsReader ); + + if( p1==SAVEPOINT_BEGIN ){ + if( db->nVdbeWrite>0 ){ + /* A new savepoint cannot be created if there are active write + ** statements (i.e. open read/write incremental blob handles). + */ + sqlite3VdbeError(p, "cannot open savepoint - SQL statements in progress"); + rc = SQLITE_BUSY; + }else{ + nName = sqlite3Strlen30(zName); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* This call is Ok even if this savepoint is actually a transaction + ** savepoint (and therefore should not prompt xSavepoint()) callbacks. + ** If this is a transaction savepoint being opened, it is guaranteed + ** that the db->aVTrans[] array is empty. */ + assert( db->autoCommit==0 || db->nVTrans==0 ); + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, + db->nStatement+db->nSavepoint); + if( rc!=SQLITE_OK ) goto abort_due_to_error; +#endif + + /* Create a new savepoint structure. */ + pNew = sqlite3DbMallocRawNN(db, sizeof(Savepoint)+nName+1); + if( pNew ){ + pNew->zName = (char *)&pNew[1]; + memcpy(pNew->zName, zName, nName+1); + + /* If there is no open transaction, then mark this as a special + ** "transaction savepoint". */ + if( db->autoCommit ){ + db->autoCommit = 0; + db->isTransactionSavepoint = 1; + }else{ + db->nSavepoint++; + } + + /* Link the new savepoint into the database handle's list. */ + pNew->pNext = db->pSavepoint; + db->pSavepoint = pNew; + pNew->nDeferredCons = db->nDeferredCons; + pNew->nDeferredImmCons = db->nDeferredImmCons; + } + } + }else{ + assert( p1==SAVEPOINT_RELEASE || p1==SAVEPOINT_ROLLBACK ); + iSavepoint = 0; + + /* Find the named savepoint. If there is no such savepoint, then an + ** an error is returned to the user. */ + for( + pSavepoint = db->pSavepoint; + pSavepoint && sqlite3StrICmp(pSavepoint->zName, zName); + pSavepoint = pSavepoint->pNext + ){ + iSavepoint++; + } + if( !pSavepoint ){ + sqlite3VdbeError(p, "no such savepoint: %s", zName); + rc = SQLITE_ERROR; + }else if( db->nVdbeWrite>0 && p1==SAVEPOINT_RELEASE ){ + /* It is not possible to release (commit) a savepoint if there are + ** active write statements. + */ + sqlite3VdbeError(p, "cannot release savepoint - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + }else{ + + /* Determine whether or not this is a transaction savepoint. If so, + ** and this is a RELEASE command, then the current transaction + ** is committed. + */ + int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint; + if( isTransaction && p1==SAVEPOINT_RELEASE ){ + if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + goto vdbe_return; + } + db->autoCommit = 1; + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + db->autoCommit = 0; + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; + } + rc = p->rc; + if( rc ){ + db->autoCommit = 0; + }else{ + db->isTransactionSavepoint = 0; + } + }else{ + int isSchemaChange; + iSavepoint = db->nSavepoint - iSavepoint - 1; + if( p1==SAVEPOINT_ROLLBACK ){ + isSchemaChange = (db->mDbFlags & DBFLAG_SchemaChange)!=0; + for(ii=0; iinDb; ii++){ + rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, + SQLITE_ABORT_ROLLBACK, + isSchemaChange==0); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + } + }else{ + assert( p1==SAVEPOINT_RELEASE ); + isSchemaChange = 0; + } + for(ii=0; iinDb; ii++){ + rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + } + if( isSchemaChange ){ + sqlite3ExpirePreparedStatements(db, 0); + sqlite3ResetAllSchemasOfConnection(db); + db->mDbFlags |= DBFLAG_SchemaChange; + } + } + if( rc ) goto abort_due_to_error; + + /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all + ** savepoints nested inside of the savepoint being operated on. */ + while( db->pSavepoint!=pSavepoint ){ + pTmp = db->pSavepoint; + db->pSavepoint = pTmp->pNext; + sqlite3DbFree(db, pTmp); + db->nSavepoint--; + } + + /* If it is a RELEASE, then destroy the savepoint being operated on + ** too. If it is a ROLLBACK TO, then set the number of deferred + ** constraint violations present in the database to the value stored + ** when the savepoint was created. */ + if( p1==SAVEPOINT_RELEASE ){ + assert( pSavepoint==db->pSavepoint ); + db->pSavepoint = pSavepoint->pNext; + sqlite3DbFree(db, pSavepoint); + if( !isTransaction ){ + db->nSavepoint--; + } + }else{ + assert( p1==SAVEPOINT_ROLLBACK ); + db->nDeferredCons = pSavepoint->nDeferredCons; + db->nDeferredImmCons = pSavepoint->nDeferredImmCons; + } + + if( !isTransaction || p1==SAVEPOINT_ROLLBACK ){ + rc = sqlite3VtabSavepoint(db, p1, iSavepoint); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + } + } + } + if( rc ) goto abort_due_to_error; + + break; +} + +/* Opcode: AutoCommit P1 P2 * * * +** +** Set the database auto-commit flag to P1 (1 or 0). If P2 is true, roll +** back any currently active btree transactions. If there are any active +** VMs (apart from this one), then a ROLLBACK fails. A COMMIT fails if +** there are active writing VMs or active VMs that use shared cache. +** +** This instruction causes the VM to halt. +*/ +case OP_AutoCommit: { + int desiredAutoCommit; + int iRollback; + + desiredAutoCommit = pOp->p1; + iRollback = pOp->p2; + assert( desiredAutoCommit==1 || desiredAutoCommit==0 ); + assert( desiredAutoCommit==1 || iRollback==0 ); + assert( db->nVdbeActive>0 ); /* At least this one VM is active */ + assert( p->bIsReader ); + + if( desiredAutoCommit!=db->autoCommit ){ + if( iRollback ){ + assert( desiredAutoCommit==1 ); + sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); + db->autoCommit = 1; + }else if( desiredAutoCommit && db->nVdbeWrite>0 ){ + /* If this instruction implements a COMMIT and other VMs are writing + ** return an error indicating that the other VMs must complete first. + */ + sqlite3VdbeError(p, "cannot commit transaction - " + "SQL statements in progress"); + rc = SQLITE_BUSY; + goto abort_due_to_error; + }else if( (rc = sqlite3VdbeCheckFk(p, 1))!=SQLITE_OK ){ + goto vdbe_return; + }else{ + db->autoCommit = (u8)desiredAutoCommit; + } + if( sqlite3VdbeHalt(p)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + db->autoCommit = (u8)(1-desiredAutoCommit); + p->rc = rc = SQLITE_BUSY; + goto vdbe_return; + } + sqlite3CloseSavepoints(db); + if( p->rc==SQLITE_OK ){ + rc = SQLITE_DONE; + }else{ + rc = SQLITE_ERROR; + } + goto vdbe_return; + }else{ + sqlite3VdbeError(p, + (!desiredAutoCommit)?"cannot start a transaction within a transaction":( + (iRollback)?"cannot rollback - no transaction is active": + "cannot commit - no transaction is active")); + + rc = SQLITE_ERROR; + goto abort_due_to_error; + } + /*NOTREACHED*/ assert(0); +} + +/* Opcode: Transaction P1 P2 P3 P4 P5 +** +** Begin a transaction on database P1 if a transaction is not already +** active. +** If P2 is non-zero, then a write-transaction is started, or if a +** read-transaction is already active, it is upgraded to a write-transaction. +** If P2 is zero, then a read-transaction is started. If P2 is 2 or more +** then an exclusive transaction is started. +** +** P1 is the index of the database file on which the transaction is +** started. Index 0 is the main database file and index 1 is the +** file used for temporary tables. Indices of 2 or more are used for +** attached databases. +** +** If a write-transaction is started and the Vdbe.usesStmtJournal flag is +** true (this flag is set if the Vdbe may modify more than one row and may +** throw an ABORT exception), a statement transaction may also be opened. +** More specifically, a statement transaction is opened iff the database +** connection is currently not in autocommit mode, or if there are other +** active statements. A statement transaction allows the changes made by this +** VDBE to be rolled back after an error without having to roll back the +** entire transaction. If no error is encountered, the statement transaction +** will automatically commit when the VDBE halts. +** +** If P5!=0 then this opcode also checks the schema cookie against P3 +** and the schema generation counter against P4. +** The cookie changes its value whenever the database schema changes. +** This operation is used to detect when that the cookie has changed +** and that the current process needs to reread the schema. If the schema +** cookie in P3 differs from the schema cookie in the database header or +** if the schema generation counter in P4 differs from the current +** generation counter, then an SQLITE_SCHEMA error is raised and execution +** halts. The sqlite3_step() wrapper function might then reprepare the +** statement and rerun it from the beginning. +*/ +case OP_Transaction: { + Btree *pBt; + int iMeta = 0; + + assert( p->bIsReader ); + assert( p->readOnly==0 || pOp->p2==0 ); + assert( pOp->p2>=0 && pOp->p2<=2 ); + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); + if( pOp->p2 && (db->flags & SQLITE_QueryOnly)!=0 ){ + rc = SQLITE_READONLY; + goto abort_due_to_error; + } + pBt = db->aDb[pOp->p1].pBt; + + if( pBt ){ + rc = sqlite3BtreeBeginTrans(pBt, pOp->p2, &iMeta); + testcase( rc==SQLITE_BUSY_SNAPSHOT ); + testcase( rc==SQLITE_BUSY_RECOVERY ); + if( rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + p->rc = rc; + goto vdbe_return; + } + goto abort_due_to_error; + } + + if( p->usesStmtJournal + && pOp->p2 + && (db->autoCommit==0 || db->nVdbeRead>1) + ){ + assert( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ); + if( p->iStatement==0 ){ + assert( db->nStatement>=0 && db->nSavepoint>=0 ); + db->nStatement++; + p->iStatement = db->nSavepoint + db->nStatement; + } + + rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, p->iStatement-1); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginStmt(pBt, p->iStatement); + } + + /* Store the current value of the database handles deferred constraint + ** counter. If the statement transaction needs to be rolled back, + ** the value of this counter needs to be restored too. */ + p->nStmtDefCons = db->nDeferredCons; + p->nStmtDefImmCons = db->nDeferredImmCons; + } + } + assert( pOp->p5==0 || pOp->p4type==P4_INT32 ); + if( pOp->p5 + && (iMeta!=pOp->p3 + || db->aDb[pOp->p1].pSchema->iGeneration!=pOp->p4.i) + ){ + /* + ** IMPLEMENTATION-OF: R-03189-51135 As each SQL statement runs, the schema + ** version is checked to ensure that the schema has not changed since the + ** SQL statement was prepared. + */ + sqlite3DbFree(db, p->zErrMsg); + p->zErrMsg = sqlite3DbStrDup(db, "database schema has changed"); + /* If the schema-cookie from the database file matches the cookie + ** stored with the in-memory representation of the schema, do + ** not reload the schema from the database file. + ** + ** If virtual-tables are in use, this is not just an optimization. + ** Often, v-tables store their data in other SQLite tables, which + ** are queried from within xNext() and other v-table methods using + ** prepared queries. If such a query is out-of-date, we do not want to + ** discard the database schema, as the user code implementing the + ** v-table would have to be ready for the sqlite3_vtab structure itself + ** to be invalidated whenever sqlite3_step() is called from within + ** a v-table method. + */ + if( db->aDb[pOp->p1].pSchema->schema_cookie!=iMeta ){ + sqlite3ResetOneSchema(db, pOp->p1); + } + p->expired = 1; + rc = SQLITE_SCHEMA; + } + if( rc ) goto abort_due_to_error; + break; +} + +/* Opcode: ReadCookie P1 P2 P3 * * +** +** Read cookie number P3 from database P1 and write it into register P2. +** P3==1 is the schema version. P3==2 is the database format. +** P3==3 is the recommended pager cache size, and so forth. P1==0 is +** the main database file and P1==1 is the database file used to store +** temporary tables. +** +** There must be a read-lock on the database (either a transaction +** must be started or there must be an open cursor) before +** executing this instruction. +*/ +case OP_ReadCookie: { /* out2 */ + int iMeta; + int iDb; + int iCookie; + + assert( p->bIsReader ); + iDb = pOp->p1; + iCookie = pOp->p3; + assert( pOp->p3=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 ); + assert( DbMaskTest(p->btreeMask, iDb) ); + + sqlite3BtreeGetMeta(db->aDb[iDb].pBt, iCookie, (u32 *)&iMeta); + pOut = out2Prerelease(p, pOp); + pOut->u.i = iMeta; + break; +} + +/* Opcode: SetCookie P1 P2 P3 * P5 +** +** Write the integer value P3 into cookie number P2 of database P1. +** P2==1 is the schema version. P2==2 is the database format. +** P2==3 is the recommended pager cache +** size, and so forth. P1==0 is the main database file and P1==1 is the +** database file used to store temporary tables. +** +** A transaction must be started before executing this opcode. +** +** If P2 is the SCHEMA_VERSION cookie (cookie number 1) then the internal +** schema version is set to P3-P5. The "PRAGMA schema_version=N" statement +** has P5 set to 1, so that the internal schema version will be different +** from the database schema version, resulting in a schema reset. +*/ +case OP_SetCookie: { + Db *pDb; + + sqlite3VdbeIncrWriteCounter(p, 0); + assert( pOp->p2p1>=0 && pOp->p1nDb ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); + assert( p->readOnly==0 ); + pDb = &db->aDb[pOp->p1]; + assert( pDb->pBt!=0 ); + assert( sqlite3SchemaMutexHeld(db, pOp->p1, 0) ); + /* See note about index shifting on OP_ReadCookie */ + rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3); + if( pOp->p2==BTREE_SCHEMA_VERSION ){ + /* When the schema cookie changes, record the new cookie internally */ + pDb->pSchema->schema_cookie = pOp->p3 - pOp->p5; + db->mDbFlags |= DBFLAG_SchemaChange; + }else if( pOp->p2==BTREE_FILE_FORMAT ){ + /* Record changes in the file format */ + pDb->pSchema->file_format = pOp->p3; + } + if( pOp->p1==1 ){ + /* Invalidate all prepared statements whenever the TEMP database + ** schema is changed. Ticket #1644 */ + sqlite3ExpirePreparedStatements(db, 0); + p->expired = 0; + } + if( rc ) goto abort_due_to_error; + break; +} + +/* Opcode: OpenRead P1 P2 P3 P4 P5 +** Synopsis: root=P2 iDb=P3 +** +** Open a read-only cursor for the database table whose root page is +** P2 in a database file. The database file is determined by P3. +** P3==0 means the main database, P3==1 means the database used for +** temporary tables, and P3>1 means used the corresponding attached +** database. Give the new cursor an identifier of P1. The P1 +** values need not be contiguous but all P1 values should be small integers. +** It is an error for P1 to be negative. +** +** Allowed P5 bits: +**
      +**
    • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxLT) +**
    +** +** The P4 value may be either an integer (P4_INT32) or a pointer to +** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo +** object, then table being opened must be an [index b-tree] where the +** KeyInfo object defines the content and collating +** sequence of that index b-tree. Otherwise, if P4 is an integer +** value, then the table being opened must be a [table b-tree] with a +** number of columns no less than the value of P4. +** +** See also: OpenWrite, ReopenIdx +*/ +/* Opcode: ReopenIdx P1 P2 P3 P4 P5 +** Synopsis: root=P2 iDb=P3 +** +** The ReopenIdx opcode works like OP_OpenRead except that it first +** checks to see if the cursor on P1 is already open on the same +** b-tree and if it is this opcode becomes a no-op. In other words, +** if the cursor is already open, do not reopen it. +** +** The ReopenIdx opcode may only be used with P5==0 or P5==OPFLAG_SEEKEQ +** and with P4 being a P4_KEYINFO object. Furthermore, the P3 value must +** be the same as every other ReopenIdx or OpenRead for the same cursor +** number. +** +** Allowed P5 bits: +**
      +**
    • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxLT) +**
    +** +** See also: OP_OpenRead, OP_OpenWrite +*/ +/* Opcode: OpenWrite P1 P2 P3 P4 P5 +** Synopsis: root=P2 iDb=P3 +** +** Open a read/write cursor named P1 on the table or index whose root +** page is P2 (or whose root page is held in register P2 if the +** OPFLAG_P2ISREG bit is set in P5 - see below). +** +** The P4 value may be either an integer (P4_INT32) or a pointer to +** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo +** object, then table being opened must be an [index b-tree] where the +** KeyInfo object defines the content and collating +** sequence of that index b-tree. Otherwise, if P4 is an integer +** value, then the table being opened must be a [table b-tree] with a +** number of columns no less than the value of P4. +** +** Allowed P5 bits: +**
      +**
    • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxLT) +**
    • 0x08 OPFLAG_FORDELETE: This cursor is used only to seek +** and subsequently delete entries in an index btree. This is a +** hint to the storage engine that the storage engine is allowed to +** ignore. The hint is not used by the official SQLite b*tree storage +** engine, but is used by COMDB2. +**
    • 0x10 OPFLAG_P2ISREG: Use the content of register P2 +** as the root page, not the value of P2 itself. +**
    +** +** This instruction works like OpenRead except that it opens the cursor +** in read/write mode. +** +** See also: OP_OpenRead, OP_ReopenIdx +*/ +case OP_ReopenIdx: { + int nField; + KeyInfo *pKeyInfo; + u32 p2; + int iDb; + int wrFlag; + Btree *pX; + VdbeCursor *pCur; + Db *pDb; + + assert( pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); + assert( pOp->p4type==P4_KEYINFO ); + pCur = p->apCsr[pOp->p1]; + if( pCur && pCur->pgnoRoot==(u32)pOp->p2 ){ + assert( pCur->iDb==pOp->p3 ); /* Guaranteed by the code generator */ + assert( pCur->eCurType==CURTYPE_BTREE ); + sqlite3BtreeClearCursor(pCur->uc.pCursor); + goto open_cursor_set_hints; + } + /* If the cursor is not currently open or is open on a different + ** index, then fall through into OP_OpenRead to force a reopen */ +case OP_OpenRead: +case OP_OpenWrite: + + assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); + assert( p->bIsReader ); + assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx + || p->readOnly==0 ); + + if( p->expired==1 ){ + rc = SQLITE_ABORT_ROLLBACK; + goto abort_due_to_error; + } + + nField = 0; + pKeyInfo = 0; + p2 = (u32)pOp->p2; + iDb = pOp->p3; + assert( iDb>=0 && iDbnDb ); + assert( DbMaskTest(p->btreeMask, iDb) ); + pDb = &db->aDb[iDb]; + pX = pDb->pBt; + assert( pX!=0 ); + if( pOp->opcode==OP_OpenWrite ){ + assert( OPFLAG_FORDELETE==BTREE_FORDELETE ); + wrFlag = BTREE_WRCSR | (pOp->p5 & OPFLAG_FORDELETE); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( pDb->pSchema->file_format < p->minWriteFileFormat ){ + p->minWriteFileFormat = pDb->pSchema->file_format; + } + }else{ + wrFlag = 0; + } + if( pOp->p5 & OPFLAG_P2ISREG ){ + assert( p2>0 ); + assert( p2<=(u32)(p->nMem+1 - p->nCursor) ); + assert( pOp->opcode==OP_OpenWrite ); + pIn2 = &aMem[p2]; + assert( memIsValid(pIn2) ); + assert( (pIn2->flags & MEM_Int)!=0 ); + sqlite3VdbeMemIntegerify(pIn2); + p2 = (int)pIn2->u.i; + /* The p2 value always comes from a prior OP_CreateBtree opcode and + ** that opcode will always set the p2 value to 2 or more or else fail. + ** If there were a failure, the prepared statement would have halted + ** before reaching this instruction. */ + assert( p2>=2 ); + } + if( pOp->p4type==P4_KEYINFO ){ + pKeyInfo = pOp->p4.pKeyInfo; + assert( pKeyInfo->enc==ENC(db) ); + assert( pKeyInfo->db==db ); + nField = pKeyInfo->nAllField; + }else if( pOp->p4type==P4_INT32 ){ + nField = pOp->p4.i; + } + assert( pOp->p1>=0 ); + assert( nField>=0 ); + testcase( nField==0 ); /* Table with INTEGER PRIMARY KEY and nothing else */ + pCur = allocateCursor(p, pOp->p1, nField, iDb, CURTYPE_BTREE); + if( pCur==0 ) goto no_mem; + pCur->nullRow = 1; + pCur->isOrdered = 1; + pCur->pgnoRoot = p2; +#ifdef SQLITE_DEBUG + pCur->wrFlag = wrFlag; +#endif + rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->uc.pCursor); + pCur->pKeyInfo = pKeyInfo; + /* Set the VdbeCursor.isTable variable. Previous versions of + ** SQLite used to check if the root-page flags were sane at this point + ** and report database corruption if they were not, but this check has + ** since moved into the btree layer. */ + pCur->isTable = pOp->p4type!=P4_KEYINFO; + +open_cursor_set_hints: + assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); + assert( OPFLAG_SEEKEQ==BTREE_SEEK_EQ ); + testcase( pOp->p5 & OPFLAG_BULKCSR ); + testcase( pOp->p2 & OPFLAG_SEEKEQ ); + sqlite3BtreeCursorHintFlags(pCur->uc.pCursor, + (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ))); + if( rc ) goto abort_due_to_error; + break; +} + +/* Opcode: OpenDup P1 P2 * * * +** +** Open a new cursor P1 that points to the same ephemeral table as +** cursor P2. The P2 cursor must have been opened by a prior OP_OpenEphemeral +** opcode. Only ephemeral cursors may be duplicated. +** +** Duplicate ephemeral cursors are used for self-joins of materialized views. +*/ +case OP_OpenDup: { + VdbeCursor *pOrig; /* The original cursor to be duplicated */ + VdbeCursor *pCx; /* The new cursor */ + + pOrig = p->apCsr[pOp->p2]; + assert( pOrig ); + assert( pOrig->isEphemeral ); /* Only ephemeral cursors can be duplicated */ + + pCx = allocateCursor(p, pOp->p1, pOrig->nField, -1, CURTYPE_BTREE); + if( pCx==0 ) goto no_mem; + pCx->nullRow = 1; + pCx->isEphemeral = 1; + pCx->pKeyInfo = pOrig->pKeyInfo; + pCx->isTable = pOrig->isTable; + pCx->pgnoRoot = pOrig->pgnoRoot; + pCx->isOrdered = pOrig->isOrdered; + pCx->pBtx = pOrig->pBtx; + pCx->hasBeenDuped = 1; + pOrig->hasBeenDuped = 1; + rc = sqlite3BtreeCursor(pCx->pBtx, pCx->pgnoRoot, BTREE_WRCSR, + pCx->pKeyInfo, pCx->uc.pCursor); + /* The sqlite3BtreeCursor() routine can only fail for the first cursor + ** opened for a database. Since there is already an open cursor when this + ** opcode is run, the sqlite3BtreeCursor() cannot fail */ + assert( rc==SQLITE_OK ); + break; +} + + +/* Opcode: OpenEphemeral P1 P2 P3 P4 P5 +** Synopsis: nColumn=P2 +** +** Open a new cursor P1 to a transient table. +** The cursor is always opened read/write even if +** the main database is read-only. The ephemeral +** table is deleted automatically when the cursor is closed. +** +** If the cursor P1 is already opened on an ephemeral table, the table +** is cleared (all content is erased). +** +** P2 is the number of columns in the ephemeral table. +** The cursor points to a BTree table if P4==0 and to a BTree index +** if P4 is not 0. If P4 is not NULL, it points to a KeyInfo structure +** that defines the format of keys in the index. +** +** The P5 parameter can be a mask of the BTREE_* flags defined +** in btree.h. These flags control aspects of the operation of +** the btree. The BTREE_OMIT_JOURNAL and BTREE_SINGLE flags are +** added automatically. +** +** If P3 is positive, then reg[P3] is modified slightly so that it +** can be used as zero-length data for OP_Insert. This is an optimization +** that avoids an extra OP_Blob opcode to initialize that register. +*/ +/* Opcode: OpenAutoindex P1 P2 * P4 * +** Synopsis: nColumn=P2 +** +** This opcode works the same as OP_OpenEphemeral. It has a +** different name to distinguish its use. Tables created using +** by this opcode will be used for automatically created transient +** indices in joins. +*/ +case OP_OpenAutoindex: +case OP_OpenEphemeral: { + VdbeCursor *pCx; + KeyInfo *pKeyInfo; + + static const int vfsFlags = + SQLITE_OPEN_READWRITE | + SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_TRANSIENT_DB; + assert( pOp->p1>=0 ); + assert( pOp->p2>=0 ); + if( pOp->p3>0 ){ + /* Make register reg[P3] into a value that can be used as the data + ** form sqlite3BtreeInsert() where the length of the data is zero. */ + assert( pOp->p2==0 ); /* Only used when number of columns is zero */ + assert( pOp->opcode==OP_OpenEphemeral ); + assert( aMem[pOp->p3].flags & MEM_Null ); + aMem[pOp->p3].n = 0; + aMem[pOp->p3].z = ""; + } + pCx = p->apCsr[pOp->p1]; + if( pCx && !pCx->hasBeenDuped ){ + /* If the ephermeral table is already open and has no duplicates from + ** OP_OpenDup, then erase all existing content so that the table is + ** empty again, rather than creating a new table. */ + assert( pCx->isEphemeral ); + pCx->seqCount = 0; + pCx->cacheStatus = CACHE_STALE; + rc = sqlite3BtreeClearTable(pCx->pBtx, pCx->pgnoRoot, 0); + }else{ + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_BTREE); + if( pCx==0 ) goto no_mem; + pCx->isEphemeral = 1; + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBtx, + BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, + vfsFlags); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(pCx->pBtx, 1, 0); + if( rc==SQLITE_OK ){ + /* If a transient index is required, create it by calling + ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before + ** opening it. If a transient table is required, just use the + ** automatically created table with root-page 1 (an BLOB_INTKEY table). + */ + if( (pCx->pKeyInfo = pKeyInfo = pOp->p4.pKeyInfo)!=0 ){ + assert( pOp->p4type==P4_KEYINFO ); + rc = sqlite3BtreeCreateTable(pCx->pBtx, &pCx->pgnoRoot, + BTREE_BLOBKEY | pOp->p5); + if( rc==SQLITE_OK ){ + assert( pCx->pgnoRoot==SCHEMA_ROOT+1 ); + assert( pKeyInfo->db==db ); + assert( pKeyInfo->enc==ENC(db) ); + rc = sqlite3BtreeCursor(pCx->pBtx, pCx->pgnoRoot, BTREE_WRCSR, + pKeyInfo, pCx->uc.pCursor); + } + pCx->isTable = 0; + }else{ + pCx->pgnoRoot = SCHEMA_ROOT; + rc = sqlite3BtreeCursor(pCx->pBtx, SCHEMA_ROOT, BTREE_WRCSR, + 0, pCx->uc.pCursor); + pCx->isTable = 1; + } + } + pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); + if( rc ){ + sqlite3BtreeClose(pCx->pBtx); + } + } + } + if( rc ) goto abort_due_to_error; + pCx->nullRow = 1; + break; +} + +/* Opcode: SorterOpen P1 P2 P3 P4 * +** +** This opcode works like OP_OpenEphemeral except that it opens +** a transient index that is specifically designed to sort large +** tables using an external merge-sort algorithm. +** +** If argument P3 is non-zero, then it indicates that the sorter may +** assume that a stable sort considering the first P3 fields of each +** key is sufficient to produce the required results. +*/ +case OP_SorterOpen: { + VdbeCursor *pCx; + + assert( pOp->p1>=0 ); + assert( pOp->p2>=0 ); + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_SORTER); + if( pCx==0 ) goto no_mem; + pCx->pKeyInfo = pOp->p4.pKeyInfo; + assert( pCx->pKeyInfo->db==db ); + assert( pCx->pKeyInfo->enc==ENC(db) ); + rc = sqlite3VdbeSorterInit(db, pOp->p3, pCx); + if( rc ) goto abort_due_to_error; + break; +} + +/* Opcode: SequenceTest P1 P2 * * * +** Synopsis: if( cursor[P1].ctr++ ) pc = P2 +** +** P1 is a sorter cursor. If the sequence counter is currently zero, jump +** to P2. Regardless of whether or not the jump is taken, increment the +** the sequence value. +*/ +case OP_SequenceTest: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC) ); + if( (pC->seqCount++)==0 ){ + goto jump_to_p2; + } + break; +} + +/* Opcode: OpenPseudo P1 P2 P3 * * +** Synopsis: P3 columns in r[P2] +** +** Open a new cursor that points to a fake table that contains a single +** row of data. The content of that one row is the content of memory +** register P2. In other words, cursor P1 becomes an alias for the +** MEM_Blob content contained in register P2. +** +** A pseudo-table created by this opcode is used to hold a single +** row output from the sorter so that the row can be decomposed into +** individual columns using the OP_Column opcode. The OP_Column opcode +** is the only cursor opcode that works with a pseudo-table. +** +** P3 is the number of fields in the records that will be stored by +** the pseudo-table. +*/ +case OP_OpenPseudo: { + VdbeCursor *pCx; + + assert( pOp->p1>=0 ); + assert( pOp->p3>=0 ); + pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, CURTYPE_PSEUDO); + if( pCx==0 ) goto no_mem; + pCx->nullRow = 1; + pCx->seekResult = pOp->p2; + pCx->isTable = 1; + /* Give this pseudo-cursor a fake BtCursor pointer so that pCx + ** can be safely passed to sqlite3VdbeCursorMoveto(). This avoids a test + ** for pCx->eCurType==CURTYPE_BTREE inside of sqlite3VdbeCursorMoveto() + ** which is a performance optimization */ + pCx->uc.pCursor = sqlite3BtreeFakeValidCursor(); + assert( pOp->p5==0 ); + break; +} + +/* Opcode: Close P1 * * * * +** +** Close a cursor previously opened as P1. If P1 is not +** currently open, this instruction is a no-op. +*/ +case OP_Close: { + assert( pOp->p1>=0 && pOp->p1nCursor ); + sqlite3VdbeFreeCursor(p, p->apCsr[pOp->p1]); + p->apCsr[pOp->p1] = 0; + break; +} + +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK +/* Opcode: ColumnsUsed P1 * * P4 * +** +** This opcode (which only exists if SQLite was compiled with +** SQLITE_ENABLE_COLUMN_USED_MASK) identifies which columns of the +** table or index for cursor P1 are used. P4 is a 64-bit integer +** (P4_INT64) in which the first 63 bits are one for each of the +** first 63 columns of the table or index that are actually used +** by the cursor. The high-order bit is set if any column after +** the 64th is used. +*/ +case OP_ColumnsUsed: { + VdbeCursor *pC; + pC = p->apCsr[pOp->p1]; + assert( pC->eCurType==CURTYPE_BTREE ); + pC->maskUsed = *(u64*)pOp->p4.pI64; + break; +} +#endif + +/* Opcode: SeekGE P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as the key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. +** +** Reposition cursor P1 so that it points to the smallest entry that +** is greater than or equal to the key value. If there are no records +** greater than or equal to the key and P2 is not zero, then jump to P2. +** +** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this +** opcode will either land on a record that exactly matches the key, or +** else it will cause a jump to P2. When the cursor is OPFLAG_SEEKEQ, +** this opcode must be followed by an IdxLE opcode with the same arguments. +** The IdxGT opcode will be skipped if this opcode succeeds, but the +** IdxGT opcode will be used on subsequent loop iterations. The +** OPFLAG_SEEKEQ flags is a hint to the btree layer to say that this +** is an equality search. +** +** This opcode leaves the cursor configured to move in forward order, +** from the beginning toward the end. In other words, the cursor is +** configured to use Next, not Prev. +** +** See also: Found, NotFound, SeekLt, SeekGt, SeekLe +*/ +/* Opcode: SeekGT P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. +** +** Reposition cursor P1 so that it points to the smallest entry that +** is greater than the key value. If there are no records greater than +** the key and P2 is not zero, then jump to P2. +** +** This opcode leaves the cursor configured to move in forward order, +** from the beginning toward the end. In other words, the cursor is +** configured to use Next, not Prev. +** +** See also: Found, NotFound, SeekLt, SeekGe, SeekLe +*/ +/* Opcode: SeekLT P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. +** +** Reposition cursor P1 so that it points to the largest entry that +** is less than the key value. If there are no records less than +** the key and P2 is not zero, then jump to P2. +** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use Prev, not Next. +** +** See also: Found, NotFound, SeekGt, SeekGe, SeekLe +*/ +/* Opcode: SeekLE P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. +** +** Reposition cursor P1 so that it points to the largest entry that +** is less than or equal to the key value. If there are no records +** less than or equal to the key and P2 is not zero, then jump to P2. +** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use Prev, not Next. +** +** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this +** opcode will either land on a record that exactly matches the key, or +** else it will cause a jump to P2. When the cursor is OPFLAG_SEEKEQ, +** this opcode must be followed by an IdxLE opcode with the same arguments. +** The IdxGE opcode will be skipped if this opcode succeeds, but the +** IdxGE opcode will be used on subsequent loop iterations. The +** OPFLAG_SEEKEQ flags is a hint to the btree layer to say that this +** is an equality search. +** +** See also: Found, NotFound, SeekGt, SeekGe, SeekLt +*/ +case OP_SeekLT: /* jump, in3, group */ +case OP_SeekLE: /* jump, in3, group */ +case OP_SeekGE: /* jump, in3, group */ +case OP_SeekGT: { /* jump, in3, group */ + int res; /* Comparison result */ + int oc; /* Opcode */ + VdbeCursor *pC; /* The cursor to seek */ + UnpackedRecord r; /* The key to seek for */ + int nField; /* Number of columns or fields in the key */ + i64 iKey; /* The rowid we are to seek to */ + int eqOnly; /* Only interested in == results */ + + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p2!=0 ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( OP_SeekLE == OP_SeekLT+1 ); + assert( OP_SeekGE == OP_SeekLT+2 ); + assert( OP_SeekGT == OP_SeekLT+3 ); + assert( pC->isOrdered ); + assert( pC->uc.pCursor!=0 ); + oc = pOp->opcode; + eqOnly = 0; + pC->nullRow = 0; +#ifdef SQLITE_DEBUG + pC->seekOp = pOp->opcode; +#endif + + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + if( pC->isTable ){ + u16 flags3, newType; + /* The OPFLAG_SEEKEQ/BTREE_SEEK_EQ flag is only set on index cursors */ + assert( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ)==0 + || CORRUPT_DB ); + + /* The input value in P3 might be of any type: integer, real, string, + ** blob, or NULL. But it needs to be an integer before we can do + ** the seek, so convert it. */ + pIn3 = &aMem[pOp->p3]; + flags3 = pIn3->flags; + if( (flags3 & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Str))==MEM_Str ){ + applyNumericAffinity(pIn3, 0); + } + iKey = sqlite3VdbeIntValue(pIn3); /* Get the integer key value */ + newType = pIn3->flags; /* Record the type after applying numeric affinity */ + pIn3->flags = flags3; /* But convert the type back to its original */ + + /* If the P3 value could not be converted into an integer without + ** loss of information, then special processing is required... */ + if( (newType & (MEM_Int|MEM_IntReal))==0 ){ + int c; + if( (newType & MEM_Real)==0 ){ + if( (newType & MEM_Null) || oc>=OP_SeekGE ){ + VdbeBranchTaken(1,2); + goto jump_to_p2; + }else{ + rc = sqlite3BtreeLast(pC->uc.pCursor, &res); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + goto seek_not_found; + } + } + c = sqlite3IntFloatCompare(iKey, pIn3->u.r); + + /* If the approximation iKey is larger than the actual real search + ** term, substitute >= for > and < for <=. e.g. if the search term + ** is 4.9 and the integer approximation 5: + ** + ** (x > 4.9) -> (x >= 5) + ** (x <= 4.9) -> (x < 5) + */ + if( c>0 ){ + assert( OP_SeekGE==(OP_SeekGT-1) ); + assert( OP_SeekLT==(OP_SeekLE-1) ); + assert( (OP_SeekLE & 0x0001)==(OP_SeekGT & 0x0001) ); + if( (oc & 0x0001)==(OP_SeekGT & 0x0001) ) oc--; + } + + /* If the approximation iKey is smaller than the actual real search + ** term, substitute <= for < and > for >=. */ + else if( c<0 ){ + assert( OP_SeekLE==(OP_SeekLT+1) ); + assert( OP_SeekGT==(OP_SeekGE+1) ); + assert( (OP_SeekLT & 0x0001)==(OP_SeekGE & 0x0001) ); + if( (oc & 0x0001)==(OP_SeekLT & 0x0001) ) oc++; + } + } + rc = sqlite3BtreeTableMoveto(pC->uc.pCursor, (u64)iKey, 0, &res); + pC->movetoTarget = iKey; /* Used by OP_Delete */ + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + }else{ + /* For a cursor with the OPFLAG_SEEKEQ/BTREE_SEEK_EQ hint, only the + ** OP_SeekGE and OP_SeekLE opcodes are allowed, and these must be + ** immediately followed by an OP_IdxGT or OP_IdxLT opcode, respectively, + ** with the same key. + */ + if( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ) ){ + eqOnly = 1; + assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE ); + assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); + assert( pOp->opcode==OP_SeekGE || pOp[1].opcode==OP_IdxLT ); + assert( pOp->opcode==OP_SeekLE || pOp[1].opcode==OP_IdxGT ); + assert( pOp[1].p1==pOp[0].p1 ); + assert( pOp[1].p2==pOp[0].p2 ); + assert( pOp[1].p3==pOp[0].p3 ); + assert( pOp[1].p4.i==pOp[0].p4.i ); + } + + nField = pOp->p4.i; + assert( pOp->p4type==P4_INT32 ); + assert( nField>0 ); + r.pKeyInfo = pC->pKeyInfo; + r.nField = (u16)nField; + + /* The next line of code computes as follows, only faster: + ** if( oc==OP_SeekGT || oc==OP_SeekLE ){ + ** r.default_rc = -1; + ** }else{ + ** r.default_rc = +1; + ** } + */ + r.default_rc = ((1 & (oc - OP_SeekLT)) ? -1 : +1); + assert( oc!=OP_SeekGT || r.default_rc==-1 ); + assert( oc!=OP_SeekLE || r.default_rc==-1 ); + assert( oc!=OP_SeekGE || r.default_rc==+1 ); + assert( oc!=OP_SeekLT || r.default_rc==+1 ); + + r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { int i; for(i=0; iuc.pCursor, &r, &res); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + if( eqOnly && r.eqSeen==0 ){ + assert( res!=0 ); + goto seek_not_found; + } + } +#ifdef SQLITE_TEST + sqlite3_search_count++; +#endif + if( oc>=OP_SeekGE ){ assert( oc==OP_SeekGE || oc==OP_SeekGT ); + if( res<0 || (res==0 && oc==OP_SeekGT) ){ + res = 0; + rc = sqlite3BtreeNext(pC->uc.pCursor, 0); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + res = 1; + }else{ + goto abort_due_to_error; + } + } + }else{ + res = 0; + } + }else{ + assert( oc==OP_SeekLT || oc==OP_SeekLE ); + if( res>0 || (res==0 && oc==OP_SeekLT) ){ + res = 0; + rc = sqlite3BtreePrevious(pC->uc.pCursor, 0); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + res = 1; + }else{ + goto abort_due_to_error; + } + } + }else{ + /* res might be negative because the table is empty. Check to + ** see if this is the case. + */ + res = sqlite3BtreeEof(pC->uc.pCursor); + } + } +seek_not_found: + assert( pOp->p2>0 ); + VdbeBranchTaken(res!=0,2); + if( res ){ + goto jump_to_p2; + }else if( eqOnly ){ + assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); + pOp++; /* Skip the OP_IdxLt or OP_IdxGT that follows */ + } + break; +} + + +/* Opcode: SeekScan P1 P2 * * * +** Synopsis: Scan-ahead up to P1 rows +** +** This opcode is a prefix opcode to OP_SeekGE. In other words, this +** opcode must be immediately followed by OP_SeekGE. This constraint is +** checked by assert() statements. +** +** This opcode uses the P1 through P4 operands of the subsequent +** OP_SeekGE. In the text that follows, the operands of the subsequent +** OP_SeekGE opcode are denoted as SeekOP.P1 through SeekOP.P4. Only +** the P1 and P2 operands of this opcode are also used, and are called +** This.P1 and This.P2. +** +** This opcode helps to optimize IN operators on a multi-column index +** where the IN operator is on the later terms of the index by avoiding +** unnecessary seeks on the btree, substituting steps to the next row +** of the b-tree instead. A correct answer is obtained if this opcode +** is omitted or is a no-op. +** +** The SeekGE.P3 and SeekGE.P4 operands identify an unpacked key which +** is the desired entry that we want the cursor SeekGE.P1 to be pointing +** to. Call this SeekGE.P4/P5 row the "target". +** +** If the SeekGE.P1 cursor is not currently pointing to a valid row, +** then this opcode is a no-op and control passes through into the OP_SeekGE. +** +** If the SeekGE.P1 cursor is pointing to a valid row, then that row +** might be the target row, or it might be near and slightly before the +** target row. This opcode attempts to position the cursor on the target +** row by, perhaps by invoking sqlite3BtreeStep() on the cursor +** between 0 and This.P1 times. +** +** There are three possible outcomes from this opcode:
      +** +**
    1. If after This.P1 steps, the cursor is still pointing to a place that +** is earlier in the btree than the target row, then fall through +** into the subsquence OP_SeekGE opcode. +** +**
    2. If the cursor is successfully moved to the target row by 0 or more +** sqlite3BtreeNext() calls, then jump to This.P2, which will land just +** past the OP_IdxGT or OP_IdxGE opcode that follows the OP_SeekGE. +** +**
    3. If the cursor ends up past the target row (indicating the the target +** row does not exist in the btree) then jump to SeekOP.P2. +**
    +*/ +case OP_SeekScan: { + VdbeCursor *pC; + int res; + int nStep; + UnpackedRecord r; + + assert( pOp[1].opcode==OP_SeekGE ); + + /* pOp->p2 points to the first instruction past the OP_IdxGT that + ** follows the OP_SeekGE. */ + assert( pOp->p2>=(int)(pOp-aOp)+2 ); + assert( aOp[pOp->p2-1].opcode==OP_IdxGT || aOp[pOp->p2-1].opcode==OP_IdxGE ); + testcase( aOp[pOp->p2-1].opcode==OP_IdxGE ); + assert( pOp[1].p1==aOp[pOp->p2-1].p1 ); + assert( pOp[1].p2==aOp[pOp->p2-1].p2 ); + assert( pOp[1].p3==aOp[pOp->p2-1].p3 ); + + assert( pOp->p1>0 ); + pC = p->apCsr[pOp[1].p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( !pC->isTable ); + if( !sqlite3BtreeCursorIsValidNN(pC->uc.pCursor) ){ +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("... cursor not valid - fall through\n"); + } +#endif + break; + } + nStep = pOp->p1; + assert( nStep>=1 ); + r.pKeyInfo = pC->pKeyInfo; + r.nField = (u16)pOp[1].p4.i; + r.default_rc = 0; + r.aMem = &aMem[pOp[1].p3]; +#ifdef SQLITE_DEBUG + { + int i; + for(i=0; i0 ){ + seekscan_search_fail: +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("... %d steps and then skip\n", pOp->p1 - nStep); + } +#endif + VdbeBranchTaken(1,3); + pOp++; + goto jump_to_p2; + } + if( res==0 ){ +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("... %d steps and then success\n", pOp->p1 - nStep); + } +#endif + VdbeBranchTaken(2,3); + goto jump_to_p2; + break; + } + if( nStep<=0 ){ +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("... fall through after %d steps\n", pOp->p1); + } +#endif + VdbeBranchTaken(0,3); + break; + } + nStep--; + rc = sqlite3BtreeNext(pC->uc.pCursor, 0); + if( rc ){ + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + goto seekscan_search_fail; + }else{ + goto abort_due_to_error; + } + } + } + + break; +} + + +/* Opcode: SeekHit P1 P2 P3 * * +** Synopsis: set P2<=seekHit<=P3 +** +** Increase or decrease the seekHit value for cursor P1, if necessary, +** so that it is no less than P2 and no greater than P3. +** +** The seekHit integer represents the maximum of terms in an index for which +** there is known to be at least one match. If the seekHit value is smaller +** than the total number of equality terms in an index lookup, then the +** OP_IfNoHope opcode might run to see if the IN loop can be abandoned +** early, thus saving work. This is part of the IN-early-out optimization. +** +** P1 must be a valid b-tree cursor. +*/ +case OP_SeekHit: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pOp->p3>=pOp->p2 ); + if( pC->seekHitp2 ){ +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("seekHit changes from %d to %d\n", pC->seekHit, pOp->p2); + } +#endif + pC->seekHit = pOp->p2; + }else if( pC->seekHit>pOp->p3 ){ +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("seekHit changes from %d to %d\n", pC->seekHit, pOp->p3); + } +#endif + pC->seekHit = pOp->p3; + } + break; +} + +/* Opcode: IfNotOpen P1 P2 * * * +** Synopsis: if( !csr[P1] ) goto P2 +** +** If cursor P1 is not open, jump to instruction P2. Otherwise, fall through. +*/ +case OP_IfNotOpen: { /* jump */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + VdbeBranchTaken(p->apCsr[pOp->p1]==0, 2); + if( !p->apCsr[pOp->p1] ){ + goto jump_to_p2_and_check_for_interrupt; + } + break; +} + +/* Opcode: Found P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If P4==0 then register P3 holds a blob constructed by MakeRecord. If +** P4>0 then register P3 is the first of P4 registers that form an unpacked +** record. +** +** Cursor P1 is on an index btree. If the record identified by P3 and P4 +** is a prefix of any entry in P1 then a jump is made to P2 and +** P1 is left pointing at the matching entry. +** +** This operation leaves the cursor in a state where it can be +** advanced in the forward direction. The Next instruction will work, +** but not the Prev instruction. +** +** See also: NotFound, NoConflict, NotExists. SeekGe +*/ +/* Opcode: NotFound P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If P4==0 then register P3 holds a blob constructed by MakeRecord. If +** P4>0 then register P3 is the first of P4 registers that form an unpacked +** record. +** +** Cursor P1 is on an index btree. If the record identified by P3 and P4 +** is not the prefix of any entry in P1 then a jump is made to P2. If P1 +** does contain an entry whose prefix matches the P3/P4 record then control +** falls through to the next instruction and P1 is left pointing at the +** matching entry. +** +** This operation leaves the cursor in a state where it cannot be +** advanced in either direction. In other words, the Next and Prev +** opcodes do not work after this operation. +** +** See also: Found, NotExists, NoConflict, IfNoHope +*/ +/* Opcode: IfNoHope P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** Register P3 is the first of P4 registers that form an unpacked +** record. Cursor P1 is an index btree. P2 is a jump destination. +** In other words, the operands to this opcode are the same as the +** operands to OP_NotFound and OP_IdxGT. +** +** This opcode is an optimization attempt only. If this opcode always +** falls through, the correct answer is still obtained, but extra works +** is performed. +** +** A value of N in the seekHit flag of cursor P1 means that there exists +** a key P3:N that will match some record in the index. We want to know +** if it is possible for a record P3:P4 to match some record in the +** index. If it is not possible, we can skips some work. So if seekHit +** is less than P4, attempt to find out if a match is possible by running +** OP_NotFound. +** +** This opcode is used in IN clause processing for a multi-column key. +** If an IN clause is attached to an element of the key other than the +** left-most element, and if there are no matches on the most recent +** seek over the whole key, then it might be that one of the key element +** to the left is prohibiting a match, and hence there is "no hope" of +** any match regardless of how many IN clause elements are checked. +** In such a case, we abandon the IN clause search early, using this +** opcode. The opcode name comes from the fact that the +** jump is taken if there is "no hope" of achieving a match. +** +** See also: NotFound, SeekHit +*/ +/* Opcode: NoConflict P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** If P4==0 then register P3 holds a blob constructed by MakeRecord. If +** P4>0 then register P3 is the first of P4 registers that form an unpacked +** record. +** +** Cursor P1 is on an index btree. If the record identified by P3 and P4 +** contains any NULL value, jump immediately to P2. If all terms of the +** record are not-NULL then a check is done to determine if any row in the +** P1 index btree has a matching key prefix. If there are no matches, jump +** immediately to P2. If there is a match, fall through and leave the P1 +** cursor pointing to the matching row. +** +** This opcode is similar to OP_NotFound with the exceptions that the +** branch is always taken if any part of the search key input is NULL. +** +** This operation leaves the cursor in a state where it cannot be +** advanced in either direction. In other words, the Next and Prev +** opcodes do not work after this operation. +** +** See also: NotFound, Found, NotExists +*/ +case OP_IfNoHope: { /* jump, in3 */ + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("seekHit is %d\n", pC->seekHit); + } +#endif + if( pC->seekHit>=pOp->p4.i ) break; + /* Fall through into OP_NotFound */ + /* no break */ deliberate_fall_through +} +case OP_NoConflict: /* jump, in3 */ +case OP_NotFound: /* jump, in3 */ +case OP_Found: { /* jump, in3 */ + int alreadyExists; + int takeJump; + int ii; + VdbeCursor *pC; + int res; + UnpackedRecord *pFree; + UnpackedRecord *pIdxKey; + UnpackedRecord r; + +#ifdef SQLITE_TEST + if( pOp->opcode!=OP_NoConflict ) sqlite3_found_count++; +#endif + + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_INT32 ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); +#ifdef SQLITE_DEBUG + pC->seekOp = pOp->opcode; +#endif + pIn3 = &aMem[pOp->p3]; + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); + assert( pC->isTable==0 ); + if( pOp->p4.i>0 ){ + r.pKeyInfo = pC->pKeyInfo; + r.nField = (u16)pOp->p4.i; + r.aMem = pIn3; +#ifdef SQLITE_DEBUG + for(ii=0; iip3+ii, &r.aMem[ii]); + } +#endif + pIdxKey = &r; + pFree = 0; + }else{ + assert( pIn3->flags & MEM_Blob ); + rc = ExpandBlob(pIn3); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + if( rc ) goto no_mem; + pFree = pIdxKey = sqlite3VdbeAllocUnpackedRecord(pC->pKeyInfo); + if( pIdxKey==0 ) goto no_mem; + sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); + } + pIdxKey->default_rc = 0; + takeJump = 0; + if( pOp->opcode==OP_NoConflict ){ + /* For the OP_NoConflict opcode, take the jump if any of the + ** input fields are NULL, since any key with a NULL will not + ** conflict */ + for(ii=0; iinField; ii++){ + if( pIdxKey->aMem[ii].flags & MEM_Null ){ + takeJump = 1; + break; + } + } + } + rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, pIdxKey, &res); + if( pFree ) sqlite3DbFreeNN(db, pFree); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + pC->seekResult = res; + alreadyExists = (res==0); + pC->nullRow = 1-alreadyExists; + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + if( pOp->opcode==OP_Found ){ + VdbeBranchTaken(alreadyExists!=0,2); + if( alreadyExists ) goto jump_to_p2; + }else{ + VdbeBranchTaken(takeJump||alreadyExists==0,2); + if( takeJump || !alreadyExists ) goto jump_to_p2; + if( pOp->opcode==OP_IfNoHope ) pC->seekHit = pOp->p4.i; + } + break; +} + +/* Opcode: SeekRowid P1 P2 P3 * * +** Synopsis: intkey=r[P3] +** +** P1 is the index of a cursor open on an SQL table btree (with integer +** keys). If register P3 does not contain an integer or if P1 does not +** contain a record with rowid P3 then jump immediately to P2. +** Or, if P2 is 0, raise an SQLITE_CORRUPT error. If P1 does contain +** a record with rowid P3 then +** leave the cursor pointing at that record and fall through to the next +** instruction. +** +** The OP_NotExists opcode performs the same operation, but with OP_NotExists +** the P3 register must be guaranteed to contain an integer value. With this +** opcode, register P3 might not contain an integer. +** +** The OP_NotFound opcode performs the same operation on index btrees +** (with arbitrary multi-value keys). +** +** This opcode leaves the cursor in a state where it cannot be advanced +** in either direction. In other words, the Next and Prev opcodes will +** not work following this opcode. +** +** See also: Found, NotFound, NoConflict, SeekRowid +*/ +/* Opcode: NotExists P1 P2 P3 * * +** Synopsis: intkey=r[P3] +** +** P1 is the index of a cursor open on an SQL table btree (with integer +** keys). P3 is an integer rowid. If P1 does not contain a record with +** rowid P3 then jump immediately to P2. Or, if P2 is 0, raise an +** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then +** leave the cursor pointing at that record and fall through to the next +** instruction. +** +** The OP_SeekRowid opcode performs the same operation but also allows the +** P3 register to contain a non-integer value, in which case the jump is +** always taken. This opcode requires that P3 always contain an integer. +** +** The OP_NotFound opcode performs the same operation on index btrees +** (with arbitrary multi-value keys). +** +** This opcode leaves the cursor in a state where it cannot be advanced +** in either direction. In other words, the Next and Prev opcodes will +** not work following this opcode. +** +** See also: Found, NotFound, NoConflict, SeekRowid +*/ +case OP_SeekRowid: { /* jump, in3 */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + u64 iKey; + + pIn3 = &aMem[pOp->p3]; + testcase( pIn3->flags & MEM_Int ); + testcase( pIn3->flags & MEM_IntReal ); + testcase( pIn3->flags & MEM_Real ); + testcase( (pIn3->flags & (MEM_Str|MEM_Int))==MEM_Str ); + if( (pIn3->flags & (MEM_Int|MEM_IntReal))==0 ){ + /* If pIn3->u.i does not contain an integer, compute iKey as the + ** integer value of pIn3. Jump to P2 if pIn3 cannot be converted + ** into an integer without loss of information. Take care to avoid + ** changing the datatype of pIn3, however, as it is used by other + ** parts of the prepared statement. */ + Mem x = pIn3[0]; + applyAffinity(&x, SQLITE_AFF_NUMERIC, encoding); + if( (x.flags & MEM_Int)==0 ) goto jump_to_p2; + iKey = x.u.i; + goto notExistsWithKey; + } + /* Fall through into OP_NotExists */ + /* no break */ deliberate_fall_through +case OP_NotExists: /* jump, in3 */ + pIn3 = &aMem[pOp->p3]; + assert( (pIn3->flags & MEM_Int)!=0 || pOp->opcode==OP_SeekRowid ); + assert( pOp->p1>=0 && pOp->p1nCursor ); + iKey = pIn3->u.i; +notExistsWithKey: + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); +#ifdef SQLITE_DEBUG + if( pOp->opcode==OP_SeekRowid ) pC->seekOp = OP_SeekRowid; +#endif + assert( pC->isTable ); + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; + assert( pCrsr!=0 ); + res = 0; + rc = sqlite3BtreeTableMoveto(pCrsr, iKey, 0, &res); + assert( rc==SQLITE_OK || res==0 ); + pC->movetoTarget = iKey; /* Used by OP_Delete */ + pC->nullRow = 0; + pC->cacheStatus = CACHE_STALE; + pC->deferredMoveto = 0; + VdbeBranchTaken(res!=0,2); + pC->seekResult = res; + if( res!=0 ){ + assert( rc==SQLITE_OK ); + if( pOp->p2==0 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + goto jump_to_p2; + } + } + if( rc ) goto abort_due_to_error; + break; +} + +/* Opcode: Sequence P1 P2 * * * +** Synopsis: r[P2]=cursor[P1].ctr++ +** +** Find the next available sequence number for cursor P1. +** Write the sequence number into register P2. +** The sequence number on the cursor is incremented after this +** instruction. +*/ +case OP_Sequence: { /* out2 */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( p->apCsr[pOp->p1]!=0 ); + assert( p->apCsr[pOp->p1]->eCurType!=CURTYPE_VTAB ); + pOut = out2Prerelease(p, pOp); + pOut->u.i = p->apCsr[pOp->p1]->seqCount++; + break; +} + + +/* Opcode: NewRowid P1 P2 P3 * * +** Synopsis: r[P2]=rowid +** +** Get a new integer record number (a.k.a "rowid") used as the key to a table. +** The record number is not previously used as a key in the database +** table that cursor P1 points to. The new record number is written +** written to register P2. +** +** If P3>0 then P3 is a register in the root frame of this VDBE that holds +** the largest previously generated record number. No new record numbers are +** allowed to be less than this value. When this value reaches its maximum, +** an SQLITE_FULL error is generated. The P3 register is updated with the ' +** generated record number. This P3 mechanism is used to help implement the +** AUTOINCREMENT feature. +*/ +case OP_NewRowid: { /* out2 */ + i64 v; /* The new rowid */ + VdbeCursor *pC; /* Cursor of table to get the new rowid */ + int res; /* Result of an sqlite3BtreeLast() */ + int cnt; /* Counter to limit the number of searches */ +#ifndef SQLITE_OMIT_AUTOINCREMENT + Mem *pMem; /* Register holding largest rowid for AUTOINCREMENT */ + VdbeFrame *pFrame; /* Root frame of VDBE */ +#endif + + v = 0; + res = 0; + pOut = out2Prerelease(p, pOp); + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->isTable ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); + { + /* The next rowid or record number (different terms for the same + ** thing) is obtained in a two-step algorithm. + ** + ** First we attempt to find the largest existing rowid and add one + ** to that. But if the largest existing rowid is already the maximum + ** positive integer, we have to fall through to the second + ** probabilistic algorithm + ** + ** The second algorithm is to select a rowid at random and see if + ** it already exists in the table. If it does not exist, we have + ** succeeded. If the random rowid does exist, we select a new one + ** and try again, up to 100 times. + */ + assert( pC->isTable ); + +#ifdef SQLITE_32BIT_ROWID +# define MAX_ROWID 0x7fffffff +#else + /* Some compilers complain about constants of the form 0x7fffffffffffffff. + ** Others complain about 0x7ffffffffffffffffLL. The following macro seems + ** to provide the constant while making all compilers happy. + */ +# define MAX_ROWID (i64)( (((u64)0x7fffffff)<<32) | (u64)0xffffffff ) +#endif + + if( !pC->useRandomRowid ){ + rc = sqlite3BtreeLast(pC->uc.pCursor, &res); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + if( res ){ + v = 1; /* IMP: R-61914-48074 */ + }else{ + assert( sqlite3BtreeCursorIsValid(pC->uc.pCursor) ); + v = sqlite3BtreeIntegerKey(pC->uc.pCursor); + if( v>=MAX_ROWID ){ + pC->useRandomRowid = 1; + }else{ + v++; /* IMP: R-29538-34987 */ + } + } + } + +#ifndef SQLITE_OMIT_AUTOINCREMENT + if( pOp->p3 ){ + /* Assert that P3 is a valid memory cell. */ + assert( pOp->p3>0 ); + if( p->pFrame ){ + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + /* Assert that P3 is a valid memory cell. */ + assert( pOp->p3<=pFrame->nMem ); + pMem = &pFrame->aMem[pOp->p3]; + }else{ + /* Assert that P3 is a valid memory cell. */ + assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); + pMem = &aMem[pOp->p3]; + memAboutToChange(p, pMem); + } + assert( memIsValid(pMem) ); + + REGISTER_TRACE(pOp->p3, pMem); + sqlite3VdbeMemIntegerify(pMem); + assert( (pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */ + if( pMem->u.i==MAX_ROWID || pC->useRandomRowid ){ + rc = SQLITE_FULL; /* IMP: R-17817-00630 */ + goto abort_due_to_error; + } + if( vu.i+1 ){ + v = pMem->u.i + 1; + } + pMem->u.i = v; + } +#endif + if( pC->useRandomRowid ){ + /* IMPLEMENTATION-OF: R-07677-41881 If the largest ROWID is equal to the + ** largest possible integer (9223372036854775807) then the database + ** engine starts picking positive candidate ROWIDs at random until + ** it finds one that is not previously used. */ + assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is + ** an AUTOINCREMENT table. */ + cnt = 0; + do{ + sqlite3_randomness(sizeof(v), &v); + v &= (MAX_ROWID>>1); v++; /* Ensure that v is greater than zero */ + }while( ((rc = sqlite3BtreeTableMoveto(pC->uc.pCursor, (u64)v, + 0, &res))==SQLITE_OK) + && (res==0) + && (++cnt<100)); + if( rc ) goto abort_due_to_error; + if( res==0 ){ + rc = SQLITE_FULL; /* IMP: R-38219-53002 */ + goto abort_due_to_error; + } + assert( v>0 ); /* EV: R-40812-03570 */ + } + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + } + pOut->u.i = v; + break; +} + +/* Opcode: Insert P1 P2 P3 P4 P5 +** Synopsis: intkey=r[P3] data=r[P2] +** +** Write an entry into the table of cursor P1. A new entry is +** created if it doesn't already exist or the data for an existing +** entry is overwritten. The data is the value MEM_Blob stored in register +** number P2. The key is stored in register P3. The key must +** be a MEM_Int. +** +** If the OPFLAG_NCHANGE flag of P5 is set, then the row change count is +** incremented (otherwise not). If the OPFLAG_LASTROWID flag of P5 is set, +** then rowid is stored for subsequent return by the +** sqlite3_last_insert_rowid() function (otherwise it is unmodified). +** +** If the OPFLAG_USESEEKRESULT flag of P5 is set, the implementation might +** run faster by avoiding an unnecessary seek on cursor P1. However, +** the OPFLAG_USESEEKRESULT flag must only be set if there have been no prior +** seeks on the cursor or if the most recent seek used a key equal to P3. +** +** If the OPFLAG_ISUPDATE flag is set, then this opcode is part of an +** UPDATE operation. Otherwise (if the flag is clear) then this opcode +** is part of an INSERT operation. The difference is only important to +** the update hook. +** +** Parameter P4 may point to a Table structure, or may be NULL. If it is +** not NULL, then the update-hook (sqlite3.xUpdateCallback) is invoked +** following a successful insert. +** +** (WARNING/TODO: If P1 is a pseudo-cursor and P2 is dynamically +** allocated, then ownership of P2 is transferred to the pseudo-cursor +** and register P2 becomes ephemeral. If the cursor is changed, the +** value of register P2 will then change. Make sure this does not +** cause any problems.) +** +** This instruction only works on tables. The equivalent instruction +** for indices is OP_IdxInsert. +*/ +case OP_Insert: { + Mem *pData; /* MEM cell holding data for the record to be inserted */ + Mem *pKey; /* MEM cell holding key for the record */ + VdbeCursor *pC; /* Cursor to table into which insert is written */ + int seekResult; /* Result of prior seek or 0 if no USESEEKRESULT flag */ + const char *zDb; /* database name - used by the update hook */ + Table *pTab; /* Table structure - used by update and pre-update hooks */ + BtreePayload x; /* Payload to be inserted */ + + pData = &aMem[pOp->p2]; + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( memIsValid(pData) ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->deferredMoveto==0 ); + assert( pC->uc.pCursor!=0 ); + assert( (pOp->p5 & OPFLAG_ISNOOP) || pC->isTable ); + assert( pOp->p4type==P4_TABLE || pOp->p4type>=P4_STATIC ); + REGISTER_TRACE(pOp->p2, pData); + sqlite3VdbeIncrWriteCounter(p, pC); + + pKey = &aMem[pOp->p3]; + assert( pKey->flags & MEM_Int ); + assert( memIsValid(pKey) ); + REGISTER_TRACE(pOp->p3, pKey); + x.nKey = pKey->u.i; + + if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ + assert( pC->iDb>=0 ); + zDb = db->aDb[pC->iDb].zDbSName; + pTab = pOp->p4.pTab; + assert( (pOp->p5 & OPFLAG_ISNOOP) || HasRowid(pTab) ); + }else{ + pTab = 0; + zDb = 0; /* Not needed. Silence a compiler warning. */ + } + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + /* Invoke the pre-update hook, if any */ + if( pTab ){ + if( db->xPreUpdateCallback && !(pOp->p5 & OPFLAG_ISUPDATE) ){ + sqlite3VdbePreUpdateHook(p,pC,SQLITE_INSERT,zDb,pTab,x.nKey,pOp->p2,-1); + } + if( db->xUpdateCallback==0 || pTab->aCol==0 ){ + /* Prevent post-update hook from running in cases when it should not */ + pTab = 0; + } + } + if( pOp->p5 & OPFLAG_ISNOOP ) break; +#endif + + if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; + assert( (pData->flags & (MEM_Blob|MEM_Str))!=0 || pData->n==0 ); + x.pData = pData->z; + x.nData = pData->n; + seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0); + if( pData->flags & MEM_Zero ){ + x.nZero = pData->u.nZero; + }else{ + x.nZero = 0; + } + x.pKey = 0; + rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, + (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)), + seekResult + ); + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + + /* Invoke the update-hook if required. */ + if( rc ) goto abort_due_to_error; + if( pTab ){ + assert( db->xUpdateCallback!=0 ); + assert( pTab->aCol!=0 ); + db->xUpdateCallback(db->pUpdateArg, + (pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT, + zDb, pTab->zName, x.nKey); + } + break; +} + +/* Opcode: RowCell P1 P2 P3 * * +** +** P1 and P2 are both open cursors. Both must be opened on the same type +** of table - intkey or index. This opcode is used as part of copying +** the current row from P2 into P1. If the cursors are opened on intkey +** tables, register P3 contains the rowid to use with the new record in +** P1. If they are opened on index tables, P3 is not used. +** +** This opcode must be followed by either an Insert or InsertIdx opcode +** with the OPFLAG_PREFORMAT flag set to complete the insert operation. +*/ +case OP_RowCell: { + VdbeCursor *pDest; /* Cursor to write to */ + VdbeCursor *pSrc; /* Cursor to read from */ + i64 iKey; /* Rowid value to insert with */ + assert( pOp[1].opcode==OP_Insert || pOp[1].opcode==OP_IdxInsert ); + assert( pOp[1].opcode==OP_Insert || pOp->p3==0 ); + assert( pOp[1].opcode==OP_IdxInsert || pOp->p3>0 ); + assert( pOp[1].p5 & OPFLAG_PREFORMAT ); + pDest = p->apCsr[pOp->p1]; + pSrc = p->apCsr[pOp->p2]; + iKey = pOp->p3 ? aMem[pOp->p3].u.i : 0; + rc = sqlite3BtreeTransferRow(pDest->uc.pCursor, pSrc->uc.pCursor, iKey); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + break; +}; + +/* Opcode: Delete P1 P2 P3 P4 P5 +** +** Delete the record at which the P1 cursor is currently pointing. +** +** If the OPFLAG_SAVEPOSITION bit of the P5 parameter is set, then +** the cursor will be left pointing at either the next or the previous +** record in the table. If it is left pointing at the next record, then +** the next Next instruction will be a no-op. As a result, in this case +** it is ok to delete a record from within a Next loop. If +** OPFLAG_SAVEPOSITION bit of P5 is clear, then the cursor will be +** left in an undefined state. +** +** If the OPFLAG_AUXDELETE bit is set on P5, that indicates that this +** delete one of several associated with deleting a table row and all its +** associated index entries. Exactly one of those deletes is the "primary" +** delete. The others are all on OPFLAG_FORDELETE cursors or else are +** marked with the AUXDELETE flag. +** +** If the OPFLAG_NCHANGE flag of P2 (NB: P2 not P5) is set, then the row +** change count is incremented (otherwise not). +** +** P1 must not be pseudo-table. It has to be a real table with +** multiple rows. +** +** If P4 is not NULL then it points to a Table object. In this case either +** the update or pre-update hook, or both, may be invoked. The P1 cursor must +** have been positioned using OP_NotFound prior to invoking this opcode in +** this case. Specifically, if one is configured, the pre-update hook is +** invoked if P4 is not NULL. The update-hook is invoked if one is configured, +** P4 is not NULL, and the OPFLAG_NCHANGE flag is set in P2. +** +** If the OPFLAG_ISUPDATE flag is set in P2, then P3 contains the address +** of the memory cell that contains the value that the rowid of the row will +** be set to by the update. +*/ +case OP_Delete: { + VdbeCursor *pC; + const char *zDb; + Table *pTab; + int opflags; + + opflags = pOp->p2; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); + assert( pC->deferredMoveto==0 ); + sqlite3VdbeIncrWriteCounter(p, pC); + +#ifdef SQLITE_DEBUG + if( pOp->p4type==P4_TABLE + && HasRowid(pOp->p4.pTab) + && pOp->p5==0 + && sqlite3BtreeCursorIsValidNN(pC->uc.pCursor) + ){ + /* If p5 is zero, the seek operation that positioned the cursor prior to + ** OP_Delete will have also set the pC->movetoTarget field to the rowid of + ** the row that is being deleted */ + i64 iKey = sqlite3BtreeIntegerKey(pC->uc.pCursor); + assert( CORRUPT_DB || pC->movetoTarget==iKey ); + } +#endif + + /* If the update-hook or pre-update-hook will be invoked, set zDb to + ** the name of the db to pass as to it. Also set local pTab to a copy + ** of p4.pTab. Finally, if p5 is true, indicating that this cursor was + ** last moved with OP_Next or OP_Prev, not Seek or NotFound, set + ** VdbeCursor.movetoTarget to the current rowid. */ + if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ + assert( pC->iDb>=0 ); + assert( pOp->p4.pTab!=0 ); + zDb = db->aDb[pC->iDb].zDbSName; + pTab = pOp->p4.pTab; + if( (pOp->p5 & OPFLAG_SAVEPOSITION)!=0 && pC->isTable ){ + pC->movetoTarget = sqlite3BtreeIntegerKey(pC->uc.pCursor); + } + }else{ + zDb = 0; /* Not needed. Silence a compiler warning. */ + pTab = 0; /* Not needed. Silence a compiler warning. */ + } + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + /* Invoke the pre-update-hook if required. */ + if( db->xPreUpdateCallback && pOp->p4.pTab ){ + assert( !(opflags & OPFLAG_ISUPDATE) + || HasRowid(pTab)==0 + || (aMem[pOp->p3].flags & MEM_Int) + ); + sqlite3VdbePreUpdateHook(p, pC, + (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, + zDb, pTab, pC->movetoTarget, + pOp->p3, -1 + ); + } + if( opflags & OPFLAG_ISNOOP ) break; +#endif + + /* Only flags that can be set are SAVEPOISTION and AUXDELETE */ + assert( (pOp->p5 & ~(OPFLAG_SAVEPOSITION|OPFLAG_AUXDELETE))==0 ); + assert( OPFLAG_SAVEPOSITION==BTREE_SAVEPOSITION ); + assert( OPFLAG_AUXDELETE==BTREE_AUXDELETE ); + +#ifdef SQLITE_DEBUG + if( p->pFrame==0 ){ + if( pC->isEphemeral==0 + && (pOp->p5 & OPFLAG_AUXDELETE)==0 + && (pC->wrFlag & OPFLAG_FORDELETE)==0 + ){ + nExtraDelete++; + } + if( pOp->p2 & OPFLAG_NCHANGE ){ + nExtraDelete--; + } + } +#endif + + rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5); + pC->cacheStatus = CACHE_STALE; + pC->seekResult = 0; + if( rc ) goto abort_due_to_error; + + /* Invoke the update-hook if required. */ + if( opflags & OPFLAG_NCHANGE ){ + p->nChange++; + if( db->xUpdateCallback && HasRowid(pTab) ){ + db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, zDb, pTab->zName, + pC->movetoTarget); + assert( pC->iDb>=0 ); + } + } + + break; +} +/* Opcode: ResetCount * * * * * +** +** The value of the change counter is copied to the database handle +** change counter (returned by subsequent calls to sqlite3_changes()). +** Then the VMs internal change counter resets to 0. +** This is used by trigger programs. +*/ +case OP_ResetCount: { + sqlite3VdbeSetChanges(db, p->nChange); + p->nChange = 0; + break; +} + +/* Opcode: SorterCompare P1 P2 P3 P4 +** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2 +** +** P1 is a sorter cursor. This instruction compares a prefix of the +** record blob in register P3 against a prefix of the entry that +** the sorter cursor currently points to. Only the first P4 fields +** of r[P3] and the sorter record are compared. +** +** If either P3 or the sorter contains a NULL in one of their significant +** fields (not counting the P4 fields at the end which are ignored) then +** the comparison is assumed to be equal. +** +** Fall through to next instruction if the two records compare equal to +** each other. Jump to P2 if they are different. +*/ +case OP_SorterCompare: { + VdbeCursor *pC; + int res; + int nKeyCol; + + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC) ); + assert( pOp->p4type==P4_INT32 ); + pIn3 = &aMem[pOp->p3]; + nKeyCol = pOp->p4.i; + res = 0; + rc = sqlite3VdbeSorterCompare(pC, pIn3, nKeyCol, &res); + VdbeBranchTaken(res!=0,2); + if( rc ) goto abort_due_to_error; + if( res ) goto jump_to_p2; + break; +}; + +/* Opcode: SorterData P1 P2 P3 * * +** Synopsis: r[P2]=data +** +** Write into register P2 the current sorter data for sorter cursor P1. +** Then clear the column header cache on cursor P3. +** +** This opcode is normally use to move a record out of the sorter and into +** a register that is the source for a pseudo-table cursor created using +** OpenPseudo. That pseudo-table cursor is the one that is identified by +** parameter P3. Clearing the P3 column cache as part of this opcode saves +** us from having to issue a separate NullRow instruction to clear that cache. +*/ +case OP_SorterData: { + VdbeCursor *pC; + + pOut = &aMem[pOp->p2]; + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC) ); + rc = sqlite3VdbeSorterRowkey(pC, pOut); + assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); + assert( pOp->p1>=0 && pOp->p1nCursor ); + if( rc ) goto abort_due_to_error; + p->apCsr[pOp->p3]->cacheStatus = CACHE_STALE; + break; +} + +/* Opcode: RowData P1 P2 P3 * * +** Synopsis: r[P2]=data +** +** Write into register P2 the complete row content for the row at +** which cursor P1 is currently pointing. +** There is no interpretation of the data. +** It is just copied onto the P2 register exactly as +** it is found in the database file. +** +** If cursor P1 is an index, then the content is the key of the row. +** If cursor P2 is a table, then the content extracted is the data. +** +** If the P1 cursor must be pointing to a valid row (not a NULL row) +** of a real table, not a pseudo-table. +** +** If P3!=0 then this opcode is allowed to make an ephemeral pointer +** into the database page. That means that the content of the output +** register will be invalidated as soon as the cursor moves - including +** moves caused by other cursors that "save" the current cursors +** position in order that they can write to the same table. If P3==0 +** then a copy of the data is made into memory. P3!=0 is faster, but +** P3==0 is safer. +** +** If P3!=0 then the content of the P2 register is unsuitable for use +** in OP_Result and any OP_Result will invalidate the P2 register content. +** The P2 register content is invalidated by opcodes like OP_Function or +** by any use of another cursor pointing to the same table. +*/ +case OP_RowData: { + VdbeCursor *pC; + BtCursor *pCrsr; + u32 n; + + pOut = out2Prerelease(p, pOp); + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( isSorter(pC)==0 ); + assert( pC->nullRow==0 ); + assert( pC->uc.pCursor!=0 ); + pCrsr = pC->uc.pCursor; + + /* The OP_RowData opcodes always follow OP_NotExists or + ** OP_SeekRowid or OP_Rewind/Op_Next with no intervening instructions + ** that might invalidate the cursor. + ** If this where not the case, on of the following assert()s + ** would fail. Should this ever change (because of changes in the code + ** generator) then the fix would be to insert a call to + ** sqlite3VdbeCursorMoveto(). + */ + assert( pC->deferredMoveto==0 ); + assert( sqlite3BtreeCursorIsValid(pCrsr) ); + + n = sqlite3BtreePayloadSize(pCrsr); + if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + testcase( n==0 ); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pCrsr, n, pOut); + if( rc ) goto abort_due_to_error; + if( !pOp->p3 ) Deephemeralize(pOut); + UPDATE_MAX_BLOBSIZE(pOut); + REGISTER_TRACE(pOp->p2, pOut); + break; +} + +/* Opcode: Rowid P1 P2 * * * +** Synopsis: r[P2]=rowid +** +** Store in register P2 an integer which is the key of the table entry that +** P1 is currently point to. +** +** P1 can be either an ordinary table or a virtual table. There used to +** be a separate OP_VRowid opcode for use with virtual tables, but this +** one opcode now works for both table types. +*/ +case OP_Rowid: { /* out2 */ + VdbeCursor *pC; + i64 v; + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + + pOut = out2Prerelease(p, pOp); + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow ); + if( pC->nullRow ){ + pOut->flags = MEM_Null; + break; + }else if( pC->deferredMoveto ){ + v = pC->movetoTarget; +#ifndef SQLITE_OMIT_VIRTUALTABLE + }else if( pC->eCurType==CURTYPE_VTAB ){ + assert( pC->uc.pVCur!=0 ); + pVtab = pC->uc.pVCur->pVtab; + pModule = pVtab->pModule; + assert( pModule->xRowid ); + rc = pModule->xRowid(pC->uc.pVCur, &v); + sqlite3VtabImportErrmsg(p, pVtab); + if( rc ) goto abort_due_to_error; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + }else{ + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); + rc = sqlite3VdbeCursorRestore(pC); + if( rc ) goto abort_due_to_error; + if( pC->nullRow ){ + pOut->flags = MEM_Null; + break; + } + v = sqlite3BtreeIntegerKey(pC->uc.pCursor); + } + pOut->u.i = v; + break; +} + +/* Opcode: NullRow P1 * * * * +** +** Move the cursor P1 to a null row. Any OP_Column operations +** that occur while the cursor is on the null row will always +** write a NULL. +*/ +case OP_NullRow: { + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + pC->nullRow = 1; + pC->cacheStatus = CACHE_STALE; + if( pC->eCurType==CURTYPE_BTREE ){ + assert( pC->uc.pCursor!=0 ); + sqlite3BtreeClearCursor(pC->uc.pCursor); + } +#ifdef SQLITE_DEBUG + if( pC->seekOp==0 ) pC->seekOp = OP_NullRow; +#endif + break; +} + +/* Opcode: SeekEnd P1 * * * * +** +** Position cursor P1 at the end of the btree for the purpose of +** appending a new entry onto the btree. +** +** It is assumed that the cursor is used only for appending and so +** if the cursor is valid, then the cursor must already be pointing +** at the end of the btree and so no changes are made to +** the cursor. +*/ +/* Opcode: Last P1 P2 * * * +** +** The next use of the Rowid or Column or Prev instruction for P1 +** will refer to the last entry in the database table or index. +** If the table or index is empty and P2>0, then jump immediately to P2. +** If P2 is 0 or if the table or index is not empty, fall through +** to the following instruction. +** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use Prev, not Next. +*/ +case OP_SeekEnd: +case OP_Last: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; + res = 0; + assert( pCrsr!=0 ); +#ifdef SQLITE_DEBUG + pC->seekOp = pOp->opcode; +#endif + if( pOp->opcode==OP_SeekEnd ){ + assert( pOp->p2==0 ); + pC->seekResult = -1; + if( sqlite3BtreeCursorIsValidNN(pCrsr) ){ + break; + } + } + rc = sqlite3BtreeLast(pCrsr, &res); + pC->nullRow = (u8)res; + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + if( rc ) goto abort_due_to_error; + if( pOp->p2>0 ){ + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + } + break; +} + +/* Opcode: IfSmaller P1 P2 P3 * * +** +** Estimate the number of rows in the table P1. Jump to P2 if that +** estimate is less than approximately 2**(0.1*P3). +*/ +case OP_IfSmaller: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + i64 sz; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + pCrsr = pC->uc.pCursor; + assert( pCrsr ); + rc = sqlite3BtreeFirst(pCrsr, &res); + if( rc ) goto abort_due_to_error; + if( res==0 ){ + sz = sqlite3BtreeRowCountEst(pCrsr); + if( ALWAYS(sz>=0) && sqlite3LogEst((u64)sz)p3 ) res = 1; + } + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} + + +/* Opcode: SorterSort P1 P2 * * * +** +** After all records have been inserted into the Sorter object +** identified by P1, invoke this opcode to actually do the sorting. +** Jump to P2 if there are no records to be sorted. +** +** This opcode is an alias for OP_Sort and OP_Rewind that is used +** for Sorter objects. +*/ +/* Opcode: Sort P1 P2 * * * +** +** This opcode does exactly the same thing as OP_Rewind except that +** it increments an undocumented global variable used for testing. +** +** Sorting is accomplished by writing records into a sorting index, +** then rewinding that index and playing it back from beginning to +** end. We use the OP_Sort opcode instead of OP_Rewind to do the +** rewinding so that the global variable will be incremented and +** regression tests can determine whether or not the optimizer is +** correctly optimizing out sorts. +*/ +case OP_SorterSort: /* jump */ +case OP_Sort: { /* jump */ +#ifdef SQLITE_TEST + sqlite3_sort_count++; + sqlite3_search_count--; +#endif + p->aCounter[SQLITE_STMTSTATUS_SORT]++; + /* Fall through into OP_Rewind */ + /* no break */ deliberate_fall_through +} +/* Opcode: Rewind P1 P2 * * * +** +** The next use of the Rowid or Column or Next instruction for P1 +** will refer to the first entry in the database table or index. +** If the table or index is empty, jump immediately to P2. +** If the table or index is not empty, fall through to the following +** instruction. +** +** This opcode leaves the cursor configured to move in forward order, +** from the beginning toward the end. In other words, the cursor is +** configured to use Next, not Prev. +*/ +case OP_Rewind: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p5==0 ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( isSorter(pC)==(pOp->opcode==OP_SorterSort) ); + res = 1; +#ifdef SQLITE_DEBUG + pC->seekOp = OP_Rewind; +#endif + if( isSorter(pC) ){ + rc = sqlite3VdbeSorterRewind(pC, &res); + }else{ + assert( pC->eCurType==CURTYPE_BTREE ); + pCrsr = pC->uc.pCursor; + assert( pCrsr ); + rc = sqlite3BtreeFirst(pCrsr, &res); + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + } + if( rc ) goto abort_due_to_error; + pC->nullRow = (u8)res; + assert( pOp->p2>0 && pOp->p2nOp ); + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} + +/* Opcode: Next P1 P2 P3 P4 P5 +** +** Advance cursor P1 so that it points to the next key/data pair in its +** table or index. If there are no more key/value pairs then fall through +** to the following instruction. But if the cursor advance was successful, +** jump immediately to P2. +** +** The Next opcode is only valid following an SeekGT, SeekGE, or +** OP_Rewind opcode used to position the cursor. Next is not allowed +** to follow SeekLT, SeekLE, or OP_Last. +** +** The P1 cursor must be for a real table, not a pseudo-table. P1 must have +** been opened prior to this opcode or the program will segfault. +** +** The P3 value is a hint to the btree implementation. If P3==1, that +** means P1 is an SQL index and that this instruction could have been +** omitted if that index had been unique. P3 is usually 0. P3 is +** always either 0 or 1. +** +** P4 is always of type P4_ADVANCE. The function pointer points to +** sqlite3BtreeNext(). +** +** If P5 is positive and the jump is taken, then event counter +** number P5-1 in the prepared statement is incremented. +** +** See also: Prev +*/ +/* Opcode: Prev P1 P2 P3 P4 P5 +** +** Back up cursor P1 so that it points to the previous key/data pair in its +** table or index. If there is no previous key/value pairs then fall through +** to the following instruction. But if the cursor backup was successful, +** jump immediately to P2. +** +** +** The Prev opcode is only valid following an SeekLT, SeekLE, or +** OP_Last opcode used to position the cursor. Prev is not allowed +** to follow SeekGT, SeekGE, or OP_Rewind. +** +** The P1 cursor must be for a real table, not a pseudo-table. If P1 is +** not open then the behavior is undefined. +** +** The P3 value is a hint to the btree implementation. If P3==1, that +** means P1 is an SQL index and that this instruction could have been +** omitted if that index had been unique. P3 is usually 0. P3 is +** always either 0 or 1. +** +** P4 is always of type P4_ADVANCE. The function pointer points to +** sqlite3BtreePrevious(). +** +** If P5 is positive and the jump is taken, then event counter +** number P5-1 in the prepared statement is incremented. +*/ +/* Opcode: SorterNext P1 P2 * * P5 +** +** This opcode works just like OP_Next except that P1 must be a +** sorter object for which the OP_SorterSort opcode has been +** invoked. This opcode advances the cursor to the next sorted +** record, or jumps to P2 if there are no more sorted records. +*/ +case OP_SorterNext: { /* jump */ + VdbeCursor *pC; + + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC) ); + rc = sqlite3VdbeSorterNext(db, pC); + goto next_tail; +case OP_Prev: /* jump */ +case OP_Next: /* jump */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p5aCounter) ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->deferredMoveto==0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); + assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); + + /* The Next opcode is only used after SeekGT, SeekGE, Rewind, and Found. + ** The Prev opcode is only used after SeekLT, SeekLE, and Last. */ + assert( pOp->opcode!=OP_Next + || pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE + || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found + || pC->seekOp==OP_NullRow|| pC->seekOp==OP_SeekRowid + || pC->seekOp==OP_IfNoHope); + assert( pOp->opcode!=OP_Prev + || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE + || pC->seekOp==OP_Last || pC->seekOp==OP_IfNoHope + || pC->seekOp==OP_NullRow); + + rc = pOp->p4.xAdvance(pC->uc.pCursor, pOp->p3); +next_tail: + pC->cacheStatus = CACHE_STALE; + VdbeBranchTaken(rc==SQLITE_OK,2); + if( rc==SQLITE_OK ){ + pC->nullRow = 0; + p->aCounter[pOp->p5]++; +#ifdef SQLITE_TEST + sqlite3_search_count++; +#endif + goto jump_to_p2_and_check_for_interrupt; + } + if( rc!=SQLITE_DONE ) goto abort_due_to_error; + rc = SQLITE_OK; + pC->nullRow = 1; + goto check_for_interrupt; +} + +/* Opcode: IdxInsert P1 P2 P3 P4 P5 +** Synopsis: key=r[P2] +** +** Register P2 holds an SQL index key made using the +** MakeRecord instructions. This opcode writes that key +** into the index P1. Data for the entry is nil. +** +** If P4 is not zero, then it is the number of values in the unpacked +** key of reg(P2). In that case, P3 is the index of the first register +** for the unpacked key. The availability of the unpacked key can sometimes +** be an optimization. +** +** If P5 has the OPFLAG_APPEND bit set, that is a hint to the b-tree layer +** that this insert is likely to be an append. +** +** If P5 has the OPFLAG_NCHANGE bit set, then the change counter is +** incremented by this instruction. If the OPFLAG_NCHANGE bit is clear, +** then the change counter is unchanged. +** +** If the OPFLAG_USESEEKRESULT flag of P5 is set, the implementation might +** run faster by avoiding an unnecessary seek on cursor P1. However, +** the OPFLAG_USESEEKRESULT flag must only be set if there have been no prior +** seeks on the cursor or if the most recent seek used a key equivalent +** to P2. +** +** This instruction only works for indices. The equivalent instruction +** for tables is OP_Insert. +*/ +case OP_IdxInsert: { /* in2 */ + VdbeCursor *pC; + BtreePayload x; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + sqlite3VdbeIncrWriteCounter(p, pC); + assert( pC!=0 ); + assert( !isSorter(pC) ); + pIn2 = &aMem[pOp->p2]; + assert( (pIn2->flags & MEM_Blob) || (pOp->p5 & OPFLAG_PREFORMAT) ); + if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->isTable==0 ); + rc = ExpandBlob(pIn2); + if( rc ) goto abort_due_to_error; + x.nKey = pIn2->n; + x.pKey = pIn2->z; + x.aMem = aMem + pOp->p3; + x.nMem = (u16)pOp->p4.i; + rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, + (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)), + ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) + ); + assert( pC->deferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; + if( rc) goto abort_due_to_error; + break; +} + +/* Opcode: SorterInsert P1 P2 * * * +** Synopsis: key=r[P2] +** +** Register P2 holds an SQL index key made using the +** MakeRecord instructions. This opcode writes that key +** into the sorter P1. Data for the entry is nil. +*/ +case OP_SorterInsert: { /* in2 */ + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + sqlite3VdbeIncrWriteCounter(p, pC); + assert( pC!=0 ); + assert( isSorter(pC) ); + pIn2 = &aMem[pOp->p2]; + assert( pIn2->flags & MEM_Blob ); + assert( pC->isTable==0 ); + rc = ExpandBlob(pIn2); + if( rc ) goto abort_due_to_error; + rc = sqlite3VdbeSorterWrite(pC, pIn2); + if( rc) goto abort_due_to_error; + break; +} + +/* Opcode: IdxDelete P1 P2 P3 * P5 +** Synopsis: key=r[P2@P3] +** +** The content of P3 registers starting at register P2 form +** an unpacked index key. This opcode removes that entry from the +** index opened by cursor P1. +** +** If P5 is not zero, then raise an SQLITE_CORRUPT_INDEX error +** if no matching index entry is found. This happens when running +** an UPDATE or DELETE statement and the index entry to be updated +** or deleted is not found. For some uses of IdxDelete +** (example: the EXCEPT operator) it does not matter that no matching +** entry is found. For those cases, P5 is zero. Also, do not raise +** this (self-correcting and non-critical) error if in writable_schema mode. +*/ +case OP_IdxDelete: { + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + UnpackedRecord r; + + assert( pOp->p3>0 ); + assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem+1 - p->nCursor)+1 ); + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + sqlite3VdbeIncrWriteCounter(p, pC); + pCrsr = pC->uc.pCursor; + assert( pCrsr!=0 ); + r.pKeyInfo = pC->pKeyInfo; + r.nField = (u16)pOp->p3; + r.default_rc = 0; + r.aMem = &aMem[pOp->p2]; + rc = sqlite3BtreeIndexMoveto(pCrsr, &r, &res); + if( rc ) goto abort_due_to_error; + if( res==0 ){ + rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); + if( rc ) goto abort_due_to_error; + }else if( pOp->p5 && !sqlite3WritableSchema(db) ){ + rc = sqlite3ReportError(SQLITE_CORRUPT_INDEX, __LINE__, "index corruption"); + goto abort_due_to_error; + } + assert( pC->deferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; + pC->seekResult = 0; + break; +} + +/* Opcode: DeferredSeek P1 * P3 P4 * +** Synopsis: Move P3 to P1.rowid if needed +** +** P1 is an open index cursor and P3 is a cursor on the corresponding +** table. This opcode does a deferred seek of the P3 table cursor +** to the row that corresponds to the current row of P1. +** +** This is a deferred seek. Nothing actually happens until +** the cursor is used to read a record. That way, if no reads +** occur, no unnecessary I/O happens. +** +** P4 may be an array of integers (type P4_INTARRAY) containing +** one entry for each column in the P3 table. If array entry a(i) +** is non-zero, then reading column a(i)-1 from cursor P3 is +** equivalent to performing the deferred seek and then reading column i +** from P1. This information is stored in P3 and used to redirect +** reads against P3 over to P1, thus possibly avoiding the need to +** seek and read cursor P3. +*/ +/* Opcode: IdxRowid P1 P2 * * * +** Synopsis: r[P2]=rowid +** +** Write into register P2 an integer which is the last entry in the record at +** the end of the index key pointed to by cursor P1. This integer should be +** the rowid of the table entry to which this index entry points. +** +** See also: Rowid, MakeRecord. +*/ +case OP_DeferredSeek: +case OP_IdxRowid: { /* out2 */ + VdbeCursor *pC; /* The P1 index cursor */ + VdbeCursor *pTabCur; /* The P2 table cursor (OP_DeferredSeek only) */ + i64 rowid; /* Rowid that P1 current points to */ + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0 ); + assert( pC->isTable==0 ); + assert( pC->deferredMoveto==0 ); + assert( !pC->nullRow || pOp->opcode==OP_IdxRowid ); + + /* The IdxRowid and Seek opcodes are combined because of the commonality + ** of sqlite3VdbeCursorRestore() and sqlite3VdbeIdxRowid(). */ + rc = sqlite3VdbeCursorRestore(pC); + + /* sqlite3VbeCursorRestore() can only fail if the record has been deleted + ** out from under the cursor. That will never happens for an IdxRowid + ** or Seek opcode */ + if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + + if( !pC->nullRow ){ + rowid = 0; /* Not needed. Only used to silence a warning. */ + rc = sqlite3VdbeIdxRowid(db, pC->uc.pCursor, &rowid); + if( rc!=SQLITE_OK ){ + goto abort_due_to_error; + } + if( pOp->opcode==OP_DeferredSeek ){ + assert( pOp->p3>=0 && pOp->p3nCursor ); + pTabCur = p->apCsr[pOp->p3]; + assert( pTabCur!=0 ); + assert( pTabCur->eCurType==CURTYPE_BTREE ); + assert( pTabCur->uc.pCursor!=0 ); + assert( pTabCur->isTable ); + pTabCur->nullRow = 0; + pTabCur->movetoTarget = rowid; + pTabCur->deferredMoveto = 1; + assert( pOp->p4type==P4_INTARRAY || pOp->p4.ai==0 ); + pTabCur->aAltMap = pOp->p4.ai; + assert( !pC->isEphemeral ); + assert( !pTabCur->isEphemeral ); + pTabCur->pAltCursor = pC; + }else{ + pOut = out2Prerelease(p, pOp); + pOut->u.i = rowid; + } + }else{ + assert( pOp->opcode==OP_IdxRowid ); + sqlite3VdbeMemSetNull(&aMem[pOp->p2]); + } + break; +} + +/* Opcode: FinishSeek P1 * * * * +** +** If cursor P1 was previously moved via OP_DeferredSeek, complete that +** seek operation now, without further delay. If the cursor seek has +** already occurred, this instruction is a no-op. +*/ +case OP_FinishSeek: { + VdbeCursor *pC; /* The P1 index cursor */ + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + if( pC->deferredMoveto ){ + rc = sqlite3VdbeFinishMoveto(pC); + if( rc ) goto abort_due_to_error; + } + break; +} + +/* Opcode: IdxGE P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** The P4 register values beginning with P3 form an unpacked index +** key that omits the PRIMARY KEY. Compare this key value against the index +** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID +** fields at the end. +** +** If the P1 index entry is greater than or equal to the key value +** then jump to P2. Otherwise fall through to the next instruction. +*/ +/* Opcode: IdxGT P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** The P4 register values beginning with P3 form an unpacked index +** key that omits the PRIMARY KEY. Compare this key value against the index +** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID +** fields at the end. +** +** If the P1 index entry is greater than the key value +** then jump to P2. Otherwise fall through to the next instruction. +*/ +/* Opcode: IdxLT P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** The P4 register values beginning with P3 form an unpacked index +** key that omits the PRIMARY KEY or ROWID. Compare this key value against +** the index that P1 is currently pointing to, ignoring the PRIMARY KEY or +** ROWID on the P1 index. +** +** If the P1 index entry is less than the key value then jump to P2. +** Otherwise fall through to the next instruction. +*/ +/* Opcode: IdxLE P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** The P4 register values beginning with P3 form an unpacked index +** key that omits the PRIMARY KEY or ROWID. Compare this key value against +** the index that P1 is currently pointing to, ignoring the PRIMARY KEY or +** ROWID on the P1 index. +** +** If the P1 index entry is less than or equal to the key value then jump +** to P2. Otherwise fall through to the next instruction. +*/ +case OP_IdxLE: /* jump */ +case OP_IdxGT: /* jump */ +case OP_IdxLT: /* jump */ +case OP_IdxGE: { /* jump */ + VdbeCursor *pC; + int res; + UnpackedRecord r; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->isOrdered ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->uc.pCursor!=0); + assert( pC->deferredMoveto==0 ); + assert( pOp->p4type==P4_INT32 ); + r.pKeyInfo = pC->pKeyInfo; + r.nField = (u16)pOp->p4.i; + if( pOp->opcodeopcode==OP_IdxLE || pOp->opcode==OP_IdxGT ); + r.default_rc = -1; + }else{ + assert( pOp->opcode==OP_IdxGE || pOp->opcode==OP_IdxLT ); + r.default_rc = 0; + } + r.aMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + { + int i; + for(i=0; ip3+i, &aMem[pOp->p3+i]); + } + } +#endif + + /* Inlined version of sqlite3VdbeIdxKeyCompare() */ + { + i64 nCellKey = 0; + BtCursor *pCur; + Mem m; + + assert( pC->eCurType==CURTYPE_BTREE ); + pCur = pC->uc.pCursor; + assert( sqlite3BtreeCursorIsValid(pCur) ); + nCellKey = sqlite3BtreePayloadSize(pCur); + /* nCellKey will always be between 0 and 0xffffffff because of the way + ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ + if( nCellKey<=0 || nCellKey>0x7fffffff ){ + rc = SQLITE_CORRUPT_BKPT; + goto abort_due_to_error; + } + sqlite3VdbeMemInit(&m, db, 0); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m); + if( rc ) goto abort_due_to_error; + res = sqlite3VdbeRecordCompareWithSkip(m.n, m.z, &r, 0); + sqlite3VdbeMemRelease(&m); + } + /* End of inlined sqlite3VdbeIdxKeyCompare() */ + + assert( (OP_IdxLE&1)==(OP_IdxLT&1) && (OP_IdxGE&1)==(OP_IdxGT&1) ); + if( (pOp->opcode&1)==(OP_IdxLT&1) ){ + assert( pOp->opcode==OP_IdxLE || pOp->opcode==OP_IdxLT ); + res = -res; + }else{ + assert( pOp->opcode==OP_IdxGE || pOp->opcode==OP_IdxGT ); + res++; + } + VdbeBranchTaken(res>0,2); + assert( rc==SQLITE_OK ); + if( res>0 ) goto jump_to_p2; + break; +} + +/* Opcode: Destroy P1 P2 P3 * * +** +** Delete an entire database table or index whose root page in the database +** file is given by P1. +** +** The table being destroyed is in the main database file if P3==0. If +** P3==1 then the table to be clear is in the auxiliary database file +** that is used to store tables create using CREATE TEMPORARY TABLE. +** +** If AUTOVACUUM is enabled then it is possible that another root page +** might be moved into the newly deleted root page in order to keep all +** root pages contiguous at the beginning of the database. The former +** value of the root page that moved - its value before the move occurred - +** is stored in register P2. If no page movement was required (because the +** table being dropped was already the last one in the database) then a +** zero is stored in register P2. If AUTOVACUUM is disabled then a zero +** is stored in register P2. +** +** This opcode throws an error if there are any active reader VMs when +** it is invoked. This is done to avoid the difficulty associated with +** updating existing cursors when a root page is moved in an AUTOVACUUM +** database. This error is thrown even if the database is not an AUTOVACUUM +** db in order to avoid introducing an incompatibility between autovacuum +** and non-autovacuum modes. +** +** See also: Clear +*/ +case OP_Destroy: { /* out2 */ + int iMoved; + int iDb; + + sqlite3VdbeIncrWriteCounter(p, 0); + assert( p->readOnly==0 ); + assert( pOp->p1>1 ); + pOut = out2Prerelease(p, pOp); + pOut->flags = MEM_Null; + if( db->nVdbeRead > db->nVDestroy+1 ){ + rc = SQLITE_LOCKED; + p->errorAction = OE_Abort; + goto abort_due_to_error; + }else{ + iDb = pOp->p3; + assert( DbMaskTest(p->btreeMask, iDb) ); + iMoved = 0; /* Not needed. Only to silence a warning. */ + rc = sqlite3BtreeDropTable(db->aDb[iDb].pBt, pOp->p1, &iMoved); + pOut->flags = MEM_Int; + pOut->u.i = iMoved; + if( rc ) goto abort_due_to_error; +#ifndef SQLITE_OMIT_AUTOVACUUM + if( iMoved!=0 ){ + sqlite3RootPageMoved(db, iDb, iMoved, pOp->p1); + /* All OP_Destroy operations occur on the same btree */ + assert( resetSchemaOnFault==0 || resetSchemaOnFault==iDb+1 ); + resetSchemaOnFault = iDb+1; + } +#endif + } + break; +} + +/* Opcode: Clear P1 P2 P3 +** +** Delete all contents of the database table or index whose root page +** in the database file is given by P1. But, unlike Destroy, do not +** remove the table or index from the database file. +** +** The table being clear is in the main database file if P2==0. If +** P2==1 then the table to be clear is in the auxiliary database file +** that is used to store tables create using CREATE TEMPORARY TABLE. +** +** If the P3 value is non-zero, then the row change count is incremented +** by the number of rows in the table being cleared. If P3 is greater +** than zero, then the value stored in register P3 is also incremented +** by the number of rows in the table being cleared. +** +** See also: Destroy +*/ +case OP_Clear: { + i64 nChange; + + sqlite3VdbeIncrWriteCounter(p, 0); + nChange = 0; + assert( p->readOnly==0 ); + assert( DbMaskTest(p->btreeMask, pOp->p2) ); + rc = sqlite3BtreeClearTable(db->aDb[pOp->p2].pBt, (u32)pOp->p1, &nChange); + if( pOp->p3 ){ + p->nChange += nChange; + if( pOp->p3>0 ){ + assert( memIsValid(&aMem[pOp->p3]) ); + memAboutToChange(p, &aMem[pOp->p3]); + aMem[pOp->p3].u.i += nChange; + } + } + if( rc ) goto abort_due_to_error; + break; +} + +/* Opcode: ResetSorter P1 * * * * +** +** Delete all contents from the ephemeral table or sorter +** that is open on cursor P1. +** +** This opcode only works for cursors used for sorting and +** opened with OP_OpenEphemeral or OP_SorterOpen. +*/ +case OP_ResetSorter: { + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + if( isSorter(pC) ){ + sqlite3VdbeSorterReset(db, pC->uc.pSorter); + }else{ + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->isEphemeral ); + rc = sqlite3BtreeClearTableOfCursor(pC->uc.pCursor); + if( rc ) goto abort_due_to_error; + } + break; +} + +/* Opcode: CreateBtree P1 P2 P3 * * +** Synopsis: r[P2]=root iDb=P1 flags=P3 +** +** Allocate a new b-tree in the main database file if P1==0 or in the +** TEMP database file if P1==1 or in an attached database if +** P1>1. The P3 argument must be 1 (BTREE_INTKEY) for a rowid table +** it must be 2 (BTREE_BLOBKEY) for an index or WITHOUT ROWID table. +** The root page number of the new b-tree is stored in register P2. +*/ +case OP_CreateBtree: { /* out2 */ + Pgno pgno; + Db *pDb; + + sqlite3VdbeIncrWriteCounter(p, 0); + pOut = out2Prerelease(p, pOp); + pgno = 0; + assert( pOp->p3==BTREE_INTKEY || pOp->p3==BTREE_BLOBKEY ); + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); + assert( p->readOnly==0 ); + pDb = &db->aDb[pOp->p1]; + assert( pDb->pBt!=0 ); + rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, pOp->p3); + if( rc ) goto abort_due_to_error; + pOut->u.i = pgno; + break; +} + +/* Opcode: SqlExec * * * P4 * +** +** Run the SQL statement or statements specified in the P4 string. +*/ +case OP_SqlExec: { + sqlite3VdbeIncrWriteCounter(p, 0); + db->nSqlExec++; + rc = sqlite3_exec(db, pOp->p4.z, 0, 0, 0); + db->nSqlExec--; + if( rc ) goto abort_due_to_error; + break; +} + +/* Opcode: ParseSchema P1 * * P4 * +** +** Read and parse all entries from the schema table of database P1 +** that match the WHERE clause P4. If P4 is a NULL pointer, then the +** entire schema for P1 is reparsed. +** +** This opcode invokes the parser to create a new virtual machine, +** then runs the new virtual machine. It is thus a re-entrant opcode. +*/ +case OP_ParseSchema: { + int iDb; + const char *zSchema; + char *zSql; + InitData initData; + + /* Any prepared statement that invokes this opcode will hold mutexes + ** on every btree. This is a prerequisite for invoking + ** sqlite3InitCallback(). + */ +#ifdef SQLITE_DEBUG + for(iDb=0; iDbnDb; iDb++){ + assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); + } +#endif + + iDb = pOp->p1; + assert( iDb>=0 && iDbnDb ); + assert( DbHasProperty(db, iDb, DB_SchemaLoaded) + || db->mallocFailed + || (CORRUPT_DB && (db->flags & SQLITE_NoSchemaError)!=0) ); + +#ifndef SQLITE_OMIT_ALTERTABLE + if( pOp->p4.z==0 ){ + sqlite3SchemaClear(db->aDb[iDb].pSchema); + db->mDbFlags &= ~DBFLAG_SchemaKnownOk; + rc = sqlite3InitOne(db, iDb, &p->zErrMsg, pOp->p5); + db->mDbFlags |= DBFLAG_SchemaChange; + p->expired = 0; + }else +#endif + { + zSchema = DFLT_SCHEMA_TABLE; + initData.db = db; + initData.iDb = iDb; + initData.pzErrMsg = &p->zErrMsg; + initData.mInitFlags = 0; + initData.mxPage = sqlite3BtreeLastPage(db->aDb[iDb].pBt); + zSql = sqlite3MPrintf(db, + "SELECT*FROM\"%w\".%s WHERE %s ORDER BY rowid", + db->aDb[iDb].zDbSName, zSchema, pOp->p4.z); + if( zSql==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + assert( db->init.busy==0 ); + db->init.busy = 1; + initData.rc = SQLITE_OK; + initData.nInitRow = 0; + assert( !db->mallocFailed ); + rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0); + if( rc==SQLITE_OK ) rc = initData.rc; + if( rc==SQLITE_OK && initData.nInitRow==0 ){ + /* The OP_ParseSchema opcode with a non-NULL P4 argument should parse + ** at least one SQL statement. Any less than that indicates that + ** the sqlite_schema table is corrupt. */ + rc = SQLITE_CORRUPT_BKPT; + } + sqlite3DbFreeNN(db, zSql); + db->init.busy = 0; + } + } + if( rc ){ + sqlite3ResetAllSchemasOfConnection(db); + if( rc==SQLITE_NOMEM ){ + goto no_mem; + } + goto abort_due_to_error; + } + break; +} + +#if !defined(SQLITE_OMIT_ANALYZE) +/* Opcode: LoadAnalysis P1 * * * * +** +** Read the sqlite_stat1 table for database P1 and load the content +** of that table into the internal index hash table. This will cause +** the analysis to be used when preparing all subsequent queries. +*/ +case OP_LoadAnalysis: { + assert( pOp->p1>=0 && pOp->p1nDb ); + rc = sqlite3AnalysisLoad(db, pOp->p1); + if( rc ) goto abort_due_to_error; + break; +} +#endif /* !defined(SQLITE_OMIT_ANALYZE) */ + +/* Opcode: DropTable P1 * * P4 * +** +** Remove the internal (in-memory) data structures that describe +** the table named P4 in database P1. This is called after a table +** is dropped from disk (using the Destroy opcode) in order to keep +** the internal representation of the +** schema consistent with what is on disk. +*/ +case OP_DropTable: { + sqlite3VdbeIncrWriteCounter(p, 0); + sqlite3UnlinkAndDeleteTable(db, pOp->p1, pOp->p4.z); + break; +} + +/* Opcode: DropIndex P1 * * P4 * +** +** Remove the internal (in-memory) data structures that describe +** the index named P4 in database P1. This is called after an index +** is dropped from disk (using the Destroy opcode) +** in order to keep the internal representation of the +** schema consistent with what is on disk. +*/ +case OP_DropIndex: { + sqlite3VdbeIncrWriteCounter(p, 0); + sqlite3UnlinkAndDeleteIndex(db, pOp->p1, pOp->p4.z); + break; +} + +/* Opcode: DropTrigger P1 * * P4 * +** +** Remove the internal (in-memory) data structures that describe +** the trigger named P4 in database P1. This is called after a trigger +** is dropped from disk (using the Destroy opcode) in order to keep +** the internal representation of the +** schema consistent with what is on disk. +*/ +case OP_DropTrigger: { + sqlite3VdbeIncrWriteCounter(p, 0); + sqlite3UnlinkAndDeleteTrigger(db, pOp->p1, pOp->p4.z); + break; +} + + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* Opcode: IntegrityCk P1 P2 P3 P4 P5 +** +** Do an analysis of the currently open database. Store in +** register P1 the text of an error message describing any problems. +** If no problems are found, store a NULL in register P1. +** +** The register P3 contains one less than the maximum number of allowed errors. +** At most reg(P3) errors will be reported. +** In other words, the analysis stops as soon as reg(P1) errors are +** seen. Reg(P1) is updated with the number of errors remaining. +** +** The root page numbers of all tables in the database are integers +** stored in P4_INTARRAY argument. +** +** If P5 is not zero, the check is done on the auxiliary database +** file, not the main database file. +** +** This opcode is used to implement the integrity_check pragma. +*/ +case OP_IntegrityCk: { + int nRoot; /* Number of tables to check. (Number of root pages.) */ + Pgno *aRoot; /* Array of rootpage numbers for tables to be checked */ + int nErr; /* Number of errors reported */ + char *z; /* Text of the error report */ + Mem *pnErr; /* Register keeping track of errors remaining */ + + assert( p->bIsReader ); + nRoot = pOp->p2; + aRoot = pOp->p4.ai; + assert( nRoot>0 ); + assert( aRoot[0]==(Pgno)nRoot ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + pnErr = &aMem[pOp->p3]; + assert( (pnErr->flags & MEM_Int)!=0 ); + assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); + pIn1 = &aMem[pOp->p1]; + assert( pOp->p5nDb ); + assert( DbMaskTest(p->btreeMask, pOp->p5) ); + z = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], nRoot, + (int)pnErr->u.i+1, &nErr); + sqlite3VdbeMemSetNull(pIn1); + if( nErr==0 ){ + assert( z==0 ); + }else if( z==0 ){ + goto no_mem; + }else{ + pnErr->u.i -= nErr-1; + sqlite3VdbeMemSetStr(pIn1, z, -1, SQLITE_UTF8, sqlite3_free); + } + UPDATE_MAX_BLOBSIZE(pIn1); + sqlite3VdbeChangeEncoding(pIn1, encoding); + goto check_for_interrupt; +} +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +/* Opcode: RowSetAdd P1 P2 * * * +** Synopsis: rowset(P1)=r[P2] +** +** Insert the integer value held by register P2 into a RowSet object +** held in register P1. +** +** An assertion fails if P2 is not an integer. +*/ +case OP_RowSetAdd: { /* in1, in2 */ + pIn1 = &aMem[pOp->p1]; + pIn2 = &aMem[pOp->p2]; + assert( (pIn2->flags & MEM_Int)!=0 ); + if( (pIn1->flags & MEM_Blob)==0 ){ + if( sqlite3VdbeMemSetRowSet(pIn1) ) goto no_mem; + } + assert( sqlite3VdbeMemIsRowSet(pIn1) ); + sqlite3RowSetInsert((RowSet*)pIn1->z, pIn2->u.i); + break; +} + +/* Opcode: RowSetRead P1 P2 P3 * * +** Synopsis: r[P3]=rowset(P1) +** +** Extract the smallest value from the RowSet object in P1 +** and put that value into register P3. +** Or, if RowSet object P1 is initially empty, leave P3 +** unchanged and jump to instruction P2. +*/ +case OP_RowSetRead: { /* jump, in1, out3 */ + i64 val; + + pIn1 = &aMem[pOp->p1]; + assert( (pIn1->flags & MEM_Blob)==0 || sqlite3VdbeMemIsRowSet(pIn1) ); + if( (pIn1->flags & MEM_Blob)==0 + || sqlite3RowSetNext((RowSet*)pIn1->z, &val)==0 + ){ + /* The boolean index is empty */ + sqlite3VdbeMemSetNull(pIn1); + VdbeBranchTaken(1,2); + goto jump_to_p2_and_check_for_interrupt; + }else{ + /* A value was pulled from the index */ + VdbeBranchTaken(0,2); + sqlite3VdbeMemSetInt64(&aMem[pOp->p3], val); + } + goto check_for_interrupt; +} + +/* Opcode: RowSetTest P1 P2 P3 P4 +** Synopsis: if r[P3] in rowset(P1) goto P2 +** +** Register P3 is assumed to hold a 64-bit integer value. If register P1 +** contains a RowSet object and that RowSet object contains +** the value held in P3, jump to register P2. Otherwise, insert the +** integer in P3 into the RowSet and continue on to the +** next opcode. +** +** The RowSet object is optimized for the case where sets of integers +** are inserted in distinct phases, which each set contains no duplicates. +** Each set is identified by a unique P4 value. The first set +** must have P4==0, the final set must have P4==-1, and for all other sets +** must have P4>0. +** +** This allows optimizations: (a) when P4==0 there is no need to test +** the RowSet object for P3, as it is guaranteed not to contain it, +** (b) when P4==-1 there is no need to insert the value, as it will +** never be tested for, and (c) when a value that is part of set X is +** inserted, there is no need to search to see if the same value was +** previously inserted as part of set X (only if it was previously +** inserted as part of some other set). +*/ +case OP_RowSetTest: { /* jump, in1, in3 */ + int iSet; + int exists; + + pIn1 = &aMem[pOp->p1]; + pIn3 = &aMem[pOp->p3]; + iSet = pOp->p4.i; + assert( pIn3->flags&MEM_Int ); + + /* If there is anything other than a rowset object in memory cell P1, + ** delete it now and initialize P1 with an empty rowset + */ + if( (pIn1->flags & MEM_Blob)==0 ){ + if( sqlite3VdbeMemSetRowSet(pIn1) ) goto no_mem; + } + assert( sqlite3VdbeMemIsRowSet(pIn1) ); + assert( pOp->p4type==P4_INT32 ); + assert( iSet==-1 || iSet>=0 ); + if( iSet ){ + exists = sqlite3RowSetTest((RowSet*)pIn1->z, iSet, pIn3->u.i); + VdbeBranchTaken(exists!=0,2); + if( exists ) goto jump_to_p2; + } + if( iSet>=0 ){ + sqlite3RowSetInsert((RowSet*)pIn1->z, pIn3->u.i); + } + break; +} + + +#ifndef SQLITE_OMIT_TRIGGER + +/* Opcode: Program P1 P2 P3 P4 P5 +** +** Execute the trigger program passed as P4 (type P4_SUBPROGRAM). +** +** P1 contains the address of the memory cell that contains the first memory +** cell in an array of values used as arguments to the sub-program. P2 +** contains the address to jump to if the sub-program throws an IGNORE +** exception using the RAISE() function. Register P3 contains the address +** of a memory cell in this (the parent) VM that is used to allocate the +** memory required by the sub-vdbe at runtime. +** +** P4 is a pointer to the VM containing the trigger program. +** +** If P5 is non-zero, then recursive program invocation is enabled. +*/ +case OP_Program: { /* jump */ + int nMem; /* Number of memory registers for sub-program */ + int nByte; /* Bytes of runtime space required for sub-program */ + Mem *pRt; /* Register to allocate runtime space */ + Mem *pMem; /* Used to iterate through memory cells */ + Mem *pEnd; /* Last memory cell in new array */ + VdbeFrame *pFrame; /* New vdbe frame to execute in */ + SubProgram *pProgram; /* Sub-program to execute */ + void *t; /* Token identifying trigger */ + + pProgram = pOp->p4.pProgram; + pRt = &aMem[pOp->p3]; + assert( pProgram->nOp>0 ); + + /* If the p5 flag is clear, then recursive invocation of triggers is + ** disabled for backwards compatibility (p5 is set if this sub-program + ** is really a trigger, not a foreign key action, and the flag set + ** and cleared by the "PRAGMA recursive_triggers" command is clear). + ** + ** It is recursive invocation of triggers, at the SQL level, that is + ** disabled. In some cases a single trigger may generate more than one + ** SubProgram (if the trigger may be executed with more than one different + ** ON CONFLICT algorithm). SubProgram structures associated with a + ** single trigger all have the same value for the SubProgram.token + ** variable. */ + if( pOp->p5 ){ + t = pProgram->token; + for(pFrame=p->pFrame; pFrame && pFrame->token!=t; pFrame=pFrame->pParent); + if( pFrame ) break; + } + + if( p->nFrame>=db->aLimit[SQLITE_LIMIT_TRIGGER_DEPTH] ){ + rc = SQLITE_ERROR; + sqlite3VdbeError(p, "too many levels of trigger recursion"); + goto abort_due_to_error; + } + + /* Register pRt is used to store the memory required to save the state + ** of the current program, and the memory required at runtime to execute + ** the trigger program. If this trigger has been fired before, then pRt + ** is already allocated. Otherwise, it must be initialized. */ + if( (pRt->flags&MEM_Blob)==0 ){ + /* SubProgram.nMem is set to the number of memory cells used by the + ** program stored in SubProgram.aOp. As well as these, one memory + ** cell is required for each cursor used by the program. Set local + ** variable nMem (and later, VdbeFrame.nChildMem) to this value. + */ + nMem = pProgram->nMem + pProgram->nCsr; + assert( nMem>0 ); + if( pProgram->nCsr==0 ) nMem++; + nByte = ROUND8(sizeof(VdbeFrame)) + + nMem * sizeof(Mem) + + pProgram->nCsr * sizeof(VdbeCursor*) + + (pProgram->nOp + 7)/8; + pFrame = sqlite3DbMallocZero(db, nByte); + if( !pFrame ){ + goto no_mem; + } + sqlite3VdbeMemRelease(pRt); + pRt->flags = MEM_Blob|MEM_Dyn; + pRt->z = (char*)pFrame; + pRt->n = nByte; + pRt->xDel = sqlite3VdbeFrameMemDel; + + pFrame->v = p; + pFrame->nChildMem = nMem; + pFrame->nChildCsr = pProgram->nCsr; + pFrame->pc = (int)(pOp - aOp); + pFrame->aMem = p->aMem; + pFrame->nMem = p->nMem; + pFrame->apCsr = p->apCsr; + pFrame->nCursor = p->nCursor; + pFrame->aOp = p->aOp; + pFrame->nOp = p->nOp; + pFrame->token = pProgram->token; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pFrame->anExec = p->anExec; +#endif +#ifdef SQLITE_DEBUG + pFrame->iFrameMagic = SQLITE_FRAME_MAGIC; +#endif + + pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; + for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ + pMem->flags = MEM_Undefined; + pMem->db = db; + } + }else{ + pFrame = (VdbeFrame*)pRt->z; + assert( pRt->xDel==sqlite3VdbeFrameMemDel ); + assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem + || (pProgram->nCsr==0 && pProgram->nMem+1==pFrame->nChildMem) ); + assert( pProgram->nCsr==pFrame->nChildCsr ); + assert( (int)(pOp - aOp)==pFrame->pc ); + } + + p->nFrame++; + pFrame->pParent = p->pFrame; + pFrame->lastRowid = db->lastRowid; + pFrame->nChange = p->nChange; + pFrame->nDbChange = p->db->nChange; + assert( pFrame->pAuxData==0 ); + pFrame->pAuxData = p->pAuxData; + p->pAuxData = 0; + p->nChange = 0; + p->pFrame = pFrame; + p->aMem = aMem = VdbeFrameMem(pFrame); + p->nMem = pFrame->nChildMem; + p->nCursor = (u16)pFrame->nChildCsr; + p->apCsr = (VdbeCursor **)&aMem[p->nMem]; + pFrame->aOnce = (u8*)&p->apCsr[pProgram->nCsr]; + memset(pFrame->aOnce, 0, (pProgram->nOp + 7)/8); + p->aOp = aOp = pProgram->aOp; + p->nOp = pProgram->nOp; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = 0; +#endif +#ifdef SQLITE_DEBUG + /* Verify that second and subsequent executions of the same trigger do not + ** try to reuse register values from the first use. */ + { + int i; + for(i=0; inMem; i++){ + aMem[i].pScopyFrom = 0; /* Prevent false-positive AboutToChange() errs */ + MemSetTypeFlag(&aMem[i], MEM_Undefined); /* Fault if this reg is reused */ + } + } +#endif + pOp = &aOp[-1]; + goto check_for_interrupt; +} + +/* Opcode: Param P1 P2 * * * +** +** This opcode is only ever present in sub-programs called via the +** OP_Program instruction. Copy a value currently stored in a memory +** cell of the calling (parent) frame to cell P2 in the current frames +** address space. This is used by trigger programs to access the new.* +** and old.* values. +** +** The address of the cell in the parent frame is determined by adding +** the value of the P1 argument to the value of the P1 argument to the +** calling OP_Program instruction. +*/ +case OP_Param: { /* out2 */ + VdbeFrame *pFrame; + Mem *pIn; + pOut = out2Prerelease(p, pOp); + pFrame = p->pFrame; + pIn = &pFrame->aMem[pOp->p1 + pFrame->aOp[pFrame->pc].p1]; + sqlite3VdbeMemShallowCopy(pOut, pIn, MEM_Ephem); + break; +} + +#endif /* #ifndef SQLITE_OMIT_TRIGGER */ + +#ifndef SQLITE_OMIT_FOREIGN_KEY +/* Opcode: FkCounter P1 P2 * * * +** Synopsis: fkctr[P1]+=P2 +** +** Increment a "constraint counter" by P2 (P2 may be negative or positive). +** If P1 is non-zero, the database constraint counter is incremented +** (deferred foreign key constraints). Otherwise, if P1 is zero, the +** statement counter is incremented (immediate foreign key constraints). +*/ +case OP_FkCounter: { + if( db->flags & SQLITE_DeferFKs ){ + db->nDeferredImmCons += pOp->p2; + }else if( pOp->p1 ){ + db->nDeferredCons += pOp->p2; + }else{ + p->nFkConstraint += pOp->p2; + } + break; +} + +/* Opcode: FkIfZero P1 P2 * * * +** Synopsis: if fkctr[P1]==0 goto P2 +** +** This opcode tests if a foreign key constraint-counter is currently zero. +** If so, jump to instruction P2. Otherwise, fall through to the next +** instruction. +** +** If P1 is non-zero, then the jump is taken if the database constraint-counter +** is zero (the one that counts deferred constraint violations). If P1 is +** zero, the jump is taken if the statement constraint-counter is zero +** (immediate foreign key constraint violations). +*/ +case OP_FkIfZero: { /* jump */ + if( pOp->p1 ){ + VdbeBranchTaken(db->nDeferredCons==0 && db->nDeferredImmCons==0, 2); + if( db->nDeferredCons==0 && db->nDeferredImmCons==0 ) goto jump_to_p2; + }else{ + VdbeBranchTaken(p->nFkConstraint==0 && db->nDeferredImmCons==0, 2); + if( p->nFkConstraint==0 && db->nDeferredImmCons==0 ) goto jump_to_p2; + } + break; +} +#endif /* #ifndef SQLITE_OMIT_FOREIGN_KEY */ + +#ifndef SQLITE_OMIT_AUTOINCREMENT +/* Opcode: MemMax P1 P2 * * * +** Synopsis: r[P1]=max(r[P1],r[P2]) +** +** P1 is a register in the root frame of this VM (the root frame is +** different from the current frame if this instruction is being executed +** within a sub-program). Set the value of register P1 to the maximum of +** its current value and the value in register P2. +** +** This instruction throws an error if the memory cell is not initially +** an integer. +*/ +case OP_MemMax: { /* in2 */ + VdbeFrame *pFrame; + if( p->pFrame ){ + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + pIn1 = &pFrame->aMem[pOp->p1]; + }else{ + pIn1 = &aMem[pOp->p1]; + } + assert( memIsValid(pIn1) ); + sqlite3VdbeMemIntegerify(pIn1); + pIn2 = &aMem[pOp->p2]; + sqlite3VdbeMemIntegerify(pIn2); + if( pIn1->u.iu.i){ + pIn1->u.i = pIn2->u.i; + } + break; +} +#endif /* SQLITE_OMIT_AUTOINCREMENT */ + +/* Opcode: IfPos P1 P2 P3 * * +** Synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 +** +** Register P1 must contain an integer. +** If the value of register P1 is 1 or greater, subtract P3 from the +** value in P1 and jump to P2. +** +** If the initial value of register P1 is less than 1, then the +** value is unchanged and control passes through to the next instruction. +*/ +case OP_IfPos: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + VdbeBranchTaken( pIn1->u.i>0, 2); + if( pIn1->u.i>0 ){ + pIn1->u.i -= pOp->p3; + goto jump_to_p2; + } + break; +} + +/* Opcode: OffsetLimit P1 P2 P3 * * +** Synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) +** +** This opcode performs a commonly used computation associated with +** LIMIT and OFFSET process. r[P1] holds the limit counter. r[P3] +** holds the offset counter. The opcode computes the combined value +** of the LIMIT and OFFSET and stores that value in r[P2]. The r[P2] +** value computed is the total number of rows that will need to be +** visited in order to complete the query. +** +** If r[P3] is zero or negative, that means there is no OFFSET +** and r[P2] is set to be the value of the LIMIT, r[P1]. +** +** if r[P1] is zero or negative, that means there is no LIMIT +** and r[P2] is set to -1. +** +** Otherwise, r[P2] is set to the sum of r[P1] and r[P3]. +*/ +case OP_OffsetLimit: { /* in1, out2, in3 */ + i64 x; + pIn1 = &aMem[pOp->p1]; + pIn3 = &aMem[pOp->p3]; + pOut = out2Prerelease(p, pOp); + assert( pIn1->flags & MEM_Int ); + assert( pIn3->flags & MEM_Int ); + x = pIn1->u.i; + if( x<=0 || sqlite3AddInt64(&x, pIn3->u.i>0?pIn3->u.i:0) ){ + /* If the LIMIT is less than or equal to zero, loop forever. This + ** is documented. But also, if the LIMIT+OFFSET exceeds 2^63 then + ** also loop forever. This is undocumented. In fact, one could argue + ** that the loop should terminate. But assuming 1 billion iterations + ** per second (far exceeding the capabilities of any current hardware) + ** it would take nearly 300 years to actually reach the limit. So + ** looping forever is a reasonable approximation. */ + pOut->u.i = -1; + }else{ + pOut->u.i = x; + } + break; +} + +/* Opcode: IfNotZero P1 P2 * * * +** Synopsis: if r[P1]!=0 then r[P1]--, goto P2 +** +** Register P1 must contain an integer. If the content of register P1 is +** initially greater than zero, then decrement the value in register P1. +** If it is non-zero (negative or positive) and then also jump to P2. +** If register P1 is initially zero, leave it unchanged and fall through. +*/ +case OP_IfNotZero: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + VdbeBranchTaken(pIn1->u.i<0, 2); + if( pIn1->u.i ){ + if( pIn1->u.i>0 ) pIn1->u.i--; + goto jump_to_p2; + } + break; +} + +/* Opcode: DecrJumpZero P1 P2 * * * +** Synopsis: if (--r[P1])==0 goto P2 +** +** Register P1 must hold an integer. Decrement the value in P1 +** and jump to P2 if the new value is exactly zero. +*/ +case OP_DecrJumpZero: { /* jump, in1 */ + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags&MEM_Int ); + if( pIn1->u.i>SMALLEST_INT64 ) pIn1->u.i--; + VdbeBranchTaken(pIn1->u.i==0, 2); + if( pIn1->u.i==0 ) goto jump_to_p2; + break; +} + + +/* Opcode: AggStep * P2 P3 P4 P5 +** Synopsis: accum=r[P3] step(r[P2@P5]) +** +** Execute the xStep function for an aggregate. +** The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the +** accumulator. +** +** The P5 arguments are taken from register P2 and its +** successors. +*/ +/* Opcode: AggInverse * P2 P3 P4 P5 +** Synopsis: accum=r[P3] inverse(r[P2@P5]) +** +** Execute the xInverse function for an aggregate. +** The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the +** accumulator. +** +** The P5 arguments are taken from register P2 and its +** successors. +*/ +/* Opcode: AggStep1 P1 P2 P3 P4 P5 +** Synopsis: accum=r[P3] step(r[P2@P5]) +** +** Execute the xStep (if P1==0) or xInverse (if P1!=0) function for an +** aggregate. The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the +** accumulator. +** +** The P5 arguments are taken from register P2 and its +** successors. +** +** This opcode is initially coded as OP_AggStep0. On first evaluation, +** the FuncDef stored in P4 is converted into an sqlite3_context and +** the opcode is changed. In this way, the initialization of the +** sqlite3_context only happens once, instead of on each call to the +** step function. +*/ +case OP_AggInverse: +case OP_AggStep: { + int n; + sqlite3_context *pCtx; + + assert( pOp->p4type==P4_FUNCDEF ); + n = pOp->p5; + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); + assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); + pCtx = sqlite3DbMallocRawNN(db, n*sizeof(sqlite3_value*) + + (sizeof(pCtx[0]) + sizeof(Mem) - sizeof(sqlite3_value*))); + if( pCtx==0 ) goto no_mem; + pCtx->pMem = 0; + pCtx->pOut = (Mem*)&(pCtx->argv[n]); + sqlite3VdbeMemInit(pCtx->pOut, db, MEM_Null); + pCtx->pFunc = pOp->p4.pFunc; + pCtx->iOp = (int)(pOp - aOp); + pCtx->pVdbe = p; + pCtx->skipFlag = 0; + pCtx->isError = 0; + pCtx->argc = n; + pOp->p4type = P4_FUNCCTX; + pOp->p4.pCtx = pCtx; + + /* OP_AggInverse must have P1==1 and OP_AggStep must have P1==0 */ + assert( pOp->p1==(pOp->opcode==OP_AggInverse) ); + + pOp->opcode = OP_AggStep1; + /* Fall through into OP_AggStep */ + /* no break */ deliberate_fall_through +} +case OP_AggStep1: { + int i; + sqlite3_context *pCtx; + Mem *pMem; + + assert( pOp->p4type==P4_FUNCCTX ); + pCtx = pOp->p4.pCtx; + pMem = &aMem[pOp->p3]; + +#ifdef SQLITE_DEBUG + if( pOp->p1 ){ + /* This is an OP_AggInverse call. Verify that xStep has always + ** been called at least once prior to any xInverse call. */ + assert( pMem->uTemp==0x1122e0e3 ); + }else{ + /* This is an OP_AggStep call. Mark it as such. */ + pMem->uTemp = 0x1122e0e3; + } +#endif + + /* If this function is inside of a trigger, the register array in aMem[] + ** might change from one evaluation to the next. The next block of code + ** checks to see if the register array has changed, and if so it + ** reinitializes the relavant parts of the sqlite3_context object */ + if( pCtx->pMem != pMem ){ + pCtx->pMem = pMem; + for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; + } + +#ifdef SQLITE_DEBUG + for(i=0; iargc; i++){ + assert( memIsValid(pCtx->argv[i]) ); + REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); + } +#endif + + pMem->n++; + assert( pCtx->pOut->flags==MEM_Null ); + assert( pCtx->isError==0 ); + assert( pCtx->skipFlag==0 ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pOp->p1 ){ + (pCtx->pFunc->xInverse)(pCtx,pCtx->argc,pCtx->argv); + }else +#endif + (pCtx->pFunc->xSFunc)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ + + if( pCtx->isError ){ + if( pCtx->isError>0 ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); + rc = pCtx->isError; + } + if( pCtx->skipFlag ){ + assert( pOp[-1].opcode==OP_CollSeq ); + i = pOp[-1].p1; + if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1); + pCtx->skipFlag = 0; + } + sqlite3VdbeMemRelease(pCtx->pOut); + pCtx->pOut->flags = MEM_Null; + pCtx->isError = 0; + if( rc ) goto abort_due_to_error; + } + assert( pCtx->pOut->flags==MEM_Null ); + assert( pCtx->skipFlag==0 ); + break; +} + +/* Opcode: AggFinal P1 P2 * P4 * +** Synopsis: accum=r[P1] N=P2 +** +** P1 is the memory location that is the accumulator for an aggregate +** or window function. Execute the finalizer function +** for an aggregate and store the result in P1. +** +** P2 is the number of arguments that the step function takes and +** P4 is a pointer to the FuncDef for this function. The P2 +** argument is not used by this opcode. It is only there to disambiguate +** functions that can take varying numbers of arguments. The +** P4 argument is only needed for the case where +** the step function was not previously called. +*/ +/* Opcode: AggValue * P2 P3 P4 * +** Synopsis: r[P3]=value N=P2 +** +** Invoke the xValue() function and store the result in register P3. +** +** P2 is the number of arguments that the step function takes and +** P4 is a pointer to the FuncDef for this function. The P2 +** argument is not used by this opcode. It is only there to disambiguate +** functions that can take varying numbers of arguments. The +** P4 argument is only needed for the case where +** the step function was not previously called. +*/ +case OP_AggValue: +case OP_AggFinal: { + Mem *pMem; + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p3==0 || pOp->opcode==OP_AggValue ); + pMem = &aMem[pOp->p1]; + assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pOp->p3 ){ + memAboutToChange(p, &aMem[pOp->p3]); + rc = sqlite3VdbeMemAggValue(pMem, &aMem[pOp->p3], pOp->p4.pFunc); + pMem = &aMem[pOp->p3]; + }else +#endif + { + rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); + } + + if( rc ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pMem)); + goto abort_due_to_error; + } + sqlite3VdbeChangeEncoding(pMem, encoding); + UPDATE_MAX_BLOBSIZE(pMem); + if( sqlite3VdbeMemTooBig(pMem) ){ + goto too_big; + } + break; +} + +#ifndef SQLITE_OMIT_WAL +/* Opcode: Checkpoint P1 P2 P3 * * +** +** Checkpoint database P1. This is a no-op if P1 is not currently in +** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL, +** RESTART, or TRUNCATE. Write 1 or 0 into mem[P3] if the checkpoint returns +** SQLITE_BUSY or not, respectively. Write the number of pages in the +** WAL after the checkpoint into mem[P3+1] and the number of pages +** in the WAL that have been checkpointed after the checkpoint +** completes into mem[P3+2]. However on an error, mem[P3+1] and +** mem[P3+2] are initialized to -1. +*/ +case OP_Checkpoint: { + int i; /* Loop counter */ + int aRes[3]; /* Results */ + Mem *pMem; /* Write results here */ + + assert( p->readOnly==0 ); + aRes[0] = 0; + aRes[1] = aRes[2] = -1; + assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE + || pOp->p2==SQLITE_CHECKPOINT_FULL + || pOp->p2==SQLITE_CHECKPOINT_RESTART + || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE + ); + rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); + if( rc ){ + if( rc!=SQLITE_BUSY ) goto abort_due_to_error; + rc = SQLITE_OK; + aRes[0] = 1; + } + for(i=0, pMem = &aMem[pOp->p3]; i<3; i++, pMem++){ + sqlite3VdbeMemSetInt64(pMem, (i64)aRes[i]); + } + break; +}; +#endif + +#ifndef SQLITE_OMIT_PRAGMA +/* Opcode: JournalMode P1 P2 P3 * * +** +** Change the journal mode of database P1 to P3. P3 must be one of the +** PAGER_JOURNALMODE_XXX values. If changing between the various rollback +** modes (delete, truncate, persist, off and memory), this is a simple +** operation. No IO is required. +** +** If changing into or out of WAL mode the procedure is more complicated. +** +** Write a string containing the final journal-mode to register P2. +*/ +case OP_JournalMode: { /* out2 */ + Btree *pBt; /* Btree to change journal mode of */ + Pager *pPager; /* Pager associated with pBt */ + int eNew; /* New journal mode */ + int eOld; /* The old journal mode */ +#ifndef SQLITE_OMIT_WAL + const char *zFilename; /* Name of database file for pPager */ +#endif + + pOut = out2Prerelease(p, pOp); + eNew = pOp->p3; + assert( eNew==PAGER_JOURNALMODE_DELETE + || eNew==PAGER_JOURNALMODE_TRUNCATE + || eNew==PAGER_JOURNALMODE_PERSIST + || eNew==PAGER_JOURNALMODE_OFF + || eNew==PAGER_JOURNALMODE_MEMORY + || eNew==PAGER_JOURNALMODE_WAL + || eNew==PAGER_JOURNALMODE_QUERY + ); + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( p->readOnly==0 ); + + pBt = db->aDb[pOp->p1].pBt; + pPager = sqlite3BtreePager(pBt); + eOld = sqlite3PagerGetJournalMode(pPager); + if( eNew==PAGER_JOURNALMODE_QUERY ) eNew = eOld; + assert( sqlite3BtreeHoldsMutex(pBt) ); + if( !sqlite3PagerOkToChangeJournalMode(pPager) ) eNew = eOld; + +#ifndef SQLITE_OMIT_WAL + zFilename = sqlite3PagerFilename(pPager, 1); + + /* Do not allow a transition to journal_mode=WAL for a database + ** in temporary storage or if the VFS does not support shared memory + */ + if( eNew==PAGER_JOURNALMODE_WAL + && (sqlite3Strlen30(zFilename)==0 /* Temp file */ + || !sqlite3PagerWalSupported(pPager)) /* No shared-memory support */ + ){ + eNew = eOld; + } + + if( (eNew!=eOld) + && (eOld==PAGER_JOURNALMODE_WAL || eNew==PAGER_JOURNALMODE_WAL) + ){ + if( !db->autoCommit || db->nVdbeRead>1 ){ + rc = SQLITE_ERROR; + sqlite3VdbeError(p, + "cannot change %s wal mode from within a transaction", + (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of") + ); + goto abort_due_to_error; + }else{ + + if( eOld==PAGER_JOURNALMODE_WAL ){ + /* If leaving WAL mode, close the log file. If successful, the call + ** to PagerCloseWal() checkpoints and deletes the write-ahead-log + ** file. An EXCLUSIVE lock may still be held on the database file + ** after a successful return. + */ + rc = sqlite3PagerCloseWal(pPager, db); + if( rc==SQLITE_OK ){ + sqlite3PagerSetJournalMode(pPager, eNew); + } + }else if( eOld==PAGER_JOURNALMODE_MEMORY ){ + /* Cannot transition directly from MEMORY to WAL. Use mode OFF + ** as an intermediate */ + sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF); + } + + /* Open a transaction on the database file. Regardless of the journal + ** mode, this transaction always uses a rollback journal. + */ + assert( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE ); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeSetVersion(pBt, (eNew==PAGER_JOURNALMODE_WAL ? 2 : 1)); + } + } + } +#endif /* ifndef SQLITE_OMIT_WAL */ + + if( rc ) eNew = eOld; + eNew = sqlite3PagerSetJournalMode(pPager, eNew); + + pOut->flags = MEM_Str|MEM_Static|MEM_Term; + pOut->z = (char *)sqlite3JournalModename(eNew); + pOut->n = sqlite3Strlen30(pOut->z); + pOut->enc = SQLITE_UTF8; + sqlite3VdbeChangeEncoding(pOut, encoding); + if( rc ) goto abort_due_to_error; + break; +}; +#endif /* SQLITE_OMIT_PRAGMA */ + +#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) +/* Opcode: Vacuum P1 P2 * * * +** +** Vacuum the entire database P1. P1 is 0 for "main", and 2 or more +** for an attached database. The "temp" database may not be vacuumed. +** +** If P2 is not zero, then it is a register holding a string which is +** the file into which the result of vacuum should be written. When +** P2 is zero, the vacuum overwrites the original database. +*/ +case OP_Vacuum: { + assert( p->readOnly==0 ); + rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1, + pOp->p2 ? &aMem[pOp->p2] : 0); + if( rc ) goto abort_due_to_error; + break; +} +#endif + +#if !defined(SQLITE_OMIT_AUTOVACUUM) +/* Opcode: IncrVacuum P1 P2 * * * +** +** Perform a single step of the incremental vacuum procedure on +** the P1 database. If the vacuum has finished, jump to instruction +** P2. Otherwise, fall through to the next instruction. +*/ +case OP_IncrVacuum: { /* jump */ + Btree *pBt; + + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); + assert( p->readOnly==0 ); + pBt = db->aDb[pOp->p1].pBt; + rc = sqlite3BtreeIncrVacuum(pBt); + VdbeBranchTaken(rc==SQLITE_DONE,2); + if( rc ){ + if( rc!=SQLITE_DONE ) goto abort_due_to_error; + rc = SQLITE_OK; + goto jump_to_p2; + } + break; +} +#endif + +/* Opcode: Expire P1 P2 * * * +** +** Cause precompiled statements to expire. When an expired statement +** is executed using sqlite3_step() it will either automatically +** reprepare itself (if it was originally created using sqlite3_prepare_v2()) +** or it will fail with SQLITE_SCHEMA. +** +** If P1 is 0, then all SQL statements become expired. If P1 is non-zero, +** then only the currently executing statement is expired. +** +** If P2 is 0, then SQL statements are expired immediately. If P2 is 1, +** then running SQL statements are allowed to continue to run to completion. +** The P2==1 case occurs when a CREATE INDEX or similar schema change happens +** that might help the statement run faster but which does not affect the +** correctness of operation. +*/ +case OP_Expire: { + assert( pOp->p2==0 || pOp->p2==1 ); + if( !pOp->p1 ){ + sqlite3ExpirePreparedStatements(db, pOp->p2); + }else{ + p->expired = pOp->p2+1; + } + break; +} + +/* Opcode: CursorLock P1 * * * * +** +** Lock the btree to which cursor P1 is pointing so that the btree cannot be +** written by an other cursor. +*/ +case OP_CursorLock: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + sqlite3BtreeCursorPin(pC->uc.pCursor); + break; +} + +/* Opcode: CursorUnlock P1 * * * * +** +** Unlock the btree to which cursor P1 is pointing so that it can be +** written by other cursors. +*/ +case OP_CursorUnlock: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + sqlite3BtreeCursorUnpin(pC->uc.pCursor); + break; +} + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* Opcode: TableLock P1 P2 P3 P4 * +** Synopsis: iDb=P1 root=P2 write=P3 +** +** Obtain a lock on a particular table. This instruction is only used when +** the shared-cache feature is enabled. +** +** P1 is the index of the database in sqlite3.aDb[] of the database +** on which the lock is acquired. A readlock is obtained if P3==0 or +** a write lock if P3==1. +** +** P2 contains the root-page of the table to lock. +** +** P4 contains a pointer to the name of the table being locked. This is only +** used to generate an error message if the lock cannot be obtained. +*/ +case OP_TableLock: { + u8 isWriteLock = (u8)pOp->p3; + if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommit) ){ + int p1 = pOp->p1; + assert( p1>=0 && p1nDb ); + assert( DbMaskTest(p->btreeMask, p1) ); + assert( isWriteLock==0 || isWriteLock==1 ); + rc = sqlite3BtreeLockTable(db->aDb[p1].pBt, pOp->p2, isWriteLock); + if( rc ){ + if( (rc&0xFF)==SQLITE_LOCKED ){ + const char *z = pOp->p4.z; + sqlite3VdbeError(p, "database table is locked: %s", z); + } + goto abort_due_to_error; + } + } + break; +} +#endif /* SQLITE_OMIT_SHARED_CACHE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VBegin * * * P4 * +** +** P4 may be a pointer to an sqlite3_vtab structure. If so, call the +** xBegin method for that table. +** +** Also, whether or not P4 is set, check that this is not being called from +** within a callback to a virtual table xSync() method. If it is, the error +** code will be set to SQLITE_LOCKED. +*/ +case OP_VBegin: { + VTable *pVTab; + pVTab = pOp->p4.pVtab; + rc = sqlite3VtabBegin(db, pVTab); + if( pVTab ) sqlite3VtabImportErrmsg(p, pVTab->pVtab); + if( rc ) goto abort_due_to_error; + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VCreate P1 P2 * * * +** +** P2 is a register that holds the name of a virtual table in database +** P1. Call the xCreate method for that table. +*/ +case OP_VCreate: { + Mem sMem; /* For storing the record being decoded */ + const char *zTab; /* Name of the virtual table */ + + memset(&sMem, 0, sizeof(sMem)); + sMem.db = db; + /* Because P2 is always a static string, it is impossible for the + ** sqlite3VdbeMemCopy() to fail */ + assert( (aMem[pOp->p2].flags & MEM_Str)!=0 ); + assert( (aMem[pOp->p2].flags & MEM_Static)!=0 ); + rc = sqlite3VdbeMemCopy(&sMem, &aMem[pOp->p2]); + assert( rc==SQLITE_OK ); + zTab = (const char*)sqlite3_value_text(&sMem); + assert( zTab || db->mallocFailed ); + if( zTab ){ + rc = sqlite3VtabCallCreate(db, pOp->p1, zTab, &p->zErrMsg); + } + sqlite3VdbeMemRelease(&sMem); + if( rc ) goto abort_due_to_error; + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VDestroy P1 * * P4 * +** +** P4 is the name of a virtual table in database P1. Call the xDestroy method +** of that table. +*/ +case OP_VDestroy: { + db->nVDestroy++; + rc = sqlite3VtabCallDestroy(db, pOp->p1, pOp->p4.z); + db->nVDestroy--; + assert( p->errorAction==OE_Abort && p->usesStmtJournal ); + if( rc ) goto abort_due_to_error; + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VOpen P1 * * P4 * +** +** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. +** P1 is a cursor number. This opcode opens a cursor to the virtual +** table and stores that cursor in P1. +*/ +case OP_VOpen: { + VdbeCursor *pCur; + sqlite3_vtab_cursor *pVCur; + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + + assert( p->bIsReader ); + pCur = 0; + pVCur = 0; + pVtab = pOp->p4.pVtab->pVtab; + if( pVtab==0 || NEVER(pVtab->pModule==0) ){ + rc = SQLITE_LOCKED; + goto abort_due_to_error; + } + pModule = pVtab->pModule; + rc = pModule->xOpen(pVtab, &pVCur); + sqlite3VtabImportErrmsg(p, pVtab); + if( rc ) goto abort_due_to_error; + + /* Initialize sqlite3_vtab_cursor base class */ + pVCur->pVtab = pVtab; + + /* Initialize vdbe cursor object */ + pCur = allocateCursor(p, pOp->p1, 0, -1, CURTYPE_VTAB); + if( pCur ){ + pCur->uc.pVCur = pVCur; + pVtab->nRef++; + }else{ + assert( db->mallocFailed ); + pModule->xClose(pVCur); + goto no_mem; + } + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VFilter P1 P2 P3 P4 * +** Synopsis: iplan=r[P3] zplan='P4' +** +** P1 is a cursor opened using VOpen. P2 is an address to jump to if +** the filtered result set is empty. +** +** P4 is either NULL or a string that was generated by the xBestIndex +** method of the module. The interpretation of the P4 string is left +** to the module implementation. +** +** This opcode invokes the xFilter method on the virtual table specified +** by P1. The integer query plan parameter to xFilter is stored in register +** P3. Register P3+1 stores the argc parameter to be passed to the +** xFilter method. Registers P3+2..P3+1+argc are the argc +** additional parameters which are passed to +** xFilter as argv. Register P3+2 becomes argv[0] when passed to xFilter. +** +** A jump is made to P2 if the result set after filtering would be empty. +*/ +case OP_VFilter: { /* jump */ + int nArg; + int iQuery; + const sqlite3_module *pModule; + Mem *pQuery; + Mem *pArgc; + sqlite3_vtab_cursor *pVCur; + sqlite3_vtab *pVtab; + VdbeCursor *pCur; + int res; + int i; + Mem **apArg; + + pQuery = &aMem[pOp->p3]; + pArgc = &pQuery[1]; + pCur = p->apCsr[pOp->p1]; + assert( memIsValid(pQuery) ); + REGISTER_TRACE(pOp->p3, pQuery); + assert( pCur->eCurType==CURTYPE_VTAB ); + pVCur = pCur->uc.pVCur; + pVtab = pVCur->pVtab; + pModule = pVtab->pModule; + + /* Grab the index number and argc parameters */ + assert( (pQuery->flags&MEM_Int)!=0 && pArgc->flags==MEM_Int ); + nArg = (int)pArgc->u.i; + iQuery = (int)pQuery->u.i; + + /* Invoke the xFilter method */ + apArg = p->apArg; + for(i = 0; ixFilter(pVCur, iQuery, pOp->p4.z, nArg, apArg); + sqlite3VtabImportErrmsg(p, pVtab); + if( rc ) goto abort_due_to_error; + res = pModule->xEof(pVCur); + pCur->nullRow = 0; + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VColumn P1 P2 P3 * P5 +** Synopsis: r[P3]=vcolumn(P2) +** +** Store in register P3 the value of the P2-th column of +** the current row of the virtual-table of cursor P1. +** +** If the VColumn opcode is being used to fetch the value of +** an unchanging column during an UPDATE operation, then the P5 +** value is OPFLAG_NOCHNG. This will cause the sqlite3_vtab_nochange() +** function to return true inside the xColumn method of the virtual +** table implementation. The P5 column might also contain other +** bits (OPFLAG_LENGTHARG or OPFLAG_TYPEOFARG) but those bits are +** unused by OP_VColumn. +*/ +case OP_VColumn: { + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + Mem *pDest; + sqlite3_context sContext; + + VdbeCursor *pCur = p->apCsr[pOp->p1]; + assert( pCur->eCurType==CURTYPE_VTAB ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + pDest = &aMem[pOp->p3]; + memAboutToChange(p, pDest); + if( pCur->nullRow ){ + sqlite3VdbeMemSetNull(pDest); + break; + } + pVtab = pCur->uc.pVCur->pVtab; + pModule = pVtab->pModule; + assert( pModule->xColumn ); + memset(&sContext, 0, sizeof(sContext)); + sContext.pOut = pDest; + assert( pOp->p5==OPFLAG_NOCHNG || pOp->p5==0 ); + if( pOp->p5 & OPFLAG_NOCHNG ){ + sqlite3VdbeMemSetNull(pDest); + pDest->flags = MEM_Null|MEM_Zero; + pDest->u.nZero = 0; + }else{ + MemSetTypeFlag(pDest, MEM_Null); + } + rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2); + sqlite3VtabImportErrmsg(p, pVtab); + if( sContext.isError>0 ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pDest)); + rc = sContext.isError; + } + sqlite3VdbeChangeEncoding(pDest, encoding); + REGISTER_TRACE(pOp->p3, pDest); + UPDATE_MAX_BLOBSIZE(pDest); + + if( sqlite3VdbeMemTooBig(pDest) ){ + goto too_big; + } + if( rc ) goto abort_due_to_error; + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VNext P1 P2 * * * +** +** Advance virtual table P1 to the next row in its result set and +** jump to instruction P2. Or, if the virtual table has reached +** the end of its result set, then fall through to the next instruction. +*/ +case OP_VNext: { /* jump */ + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + int res; + VdbeCursor *pCur; + + pCur = p->apCsr[pOp->p1]; + assert( pCur->eCurType==CURTYPE_VTAB ); + if( pCur->nullRow ){ + break; + } + pVtab = pCur->uc.pVCur->pVtab; + pModule = pVtab->pModule; + assert( pModule->xNext ); + + /* Invoke the xNext() method of the module. There is no way for the + ** underlying implementation to return an error if one occurs during + ** xNext(). Instead, if an error occurs, true is returned (indicating that + ** data is available) and the error code returned when xColumn or + ** some other method is next invoked on the save virtual table cursor. + */ + rc = pModule->xNext(pCur->uc.pVCur); + sqlite3VtabImportErrmsg(p, pVtab); + if( rc ) goto abort_due_to_error; + res = pModule->xEof(pCur->uc.pVCur); + VdbeBranchTaken(!res,2); + if( !res ){ + /* If there is data, jump to P2 */ + goto jump_to_p2_and_check_for_interrupt; + } + goto check_for_interrupt; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VRename P1 * * P4 * +** +** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. +** This opcode invokes the corresponding xRename method. The value +** in register P1 is passed as the zName argument to the xRename method. +*/ +case OP_VRename: { + sqlite3_vtab *pVtab; + Mem *pName; + int isLegacy; + + isLegacy = (db->flags & SQLITE_LegacyAlter); + db->flags |= SQLITE_LegacyAlter; + pVtab = pOp->p4.pVtab->pVtab; + pName = &aMem[pOp->p1]; + assert( pVtab->pModule->xRename ); + assert( memIsValid(pName) ); + assert( p->readOnly==0 ); + REGISTER_TRACE(pOp->p1, pName); + assert( pName->flags & MEM_Str ); + testcase( pName->enc==SQLITE_UTF8 ); + testcase( pName->enc==SQLITE_UTF16BE ); + testcase( pName->enc==SQLITE_UTF16LE ); + rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8); + if( rc ) goto abort_due_to_error; + rc = pVtab->pModule->xRename(pVtab, pName->z); + if( isLegacy==0 ) db->flags &= ~(u64)SQLITE_LegacyAlter; + sqlite3VtabImportErrmsg(p, pVtab); + p->expired = 0; + if( rc ) goto abort_due_to_error; + break; +} +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VUpdate P1 P2 P3 P4 P5 +** Synopsis: data=r[P3@P2] +** +** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. +** This opcode invokes the corresponding xUpdate method. P2 values +** are contiguous memory cells starting at P3 to pass to the xUpdate +** invocation. The value in register (P3+P2-1) corresponds to the +** p2th element of the argv array passed to xUpdate. +** +** The xUpdate method will do a DELETE or an INSERT or both. +** The argv[0] element (which corresponds to memory cell P3) +** is the rowid of a row to delete. If argv[0] is NULL then no +** deletion occurs. The argv[1] element is the rowid of the new +** row. This can be NULL to have the virtual table select the new +** rowid for itself. The subsequent elements in the array are +** the values of columns in the new row. +** +** If P2==1 then no insert is performed. argv[0] is the rowid of +** a row to delete. +** +** P1 is a boolean flag. If it is set to true and the xUpdate call +** is successful, then the value returned by sqlite3_last_insert_rowid() +** is set to the value of the rowid for the row just inserted. +** +** P5 is the error actions (OE_Replace, OE_Fail, OE_Ignore, etc) to +** apply in the case of a constraint failure on an insert or update. +*/ +case OP_VUpdate: { + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + int nArg; + int i; + sqlite_int64 rowid; + Mem **apArg; + Mem *pX; + + assert( pOp->p2==1 || pOp->p5==OE_Fail || pOp->p5==OE_Rollback + || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace + ); + assert( p->readOnly==0 ); + if( db->mallocFailed ) goto no_mem; + sqlite3VdbeIncrWriteCounter(p, 0); + pVtab = pOp->p4.pVtab->pVtab; + if( pVtab==0 || NEVER(pVtab->pModule==0) ){ + rc = SQLITE_LOCKED; + goto abort_due_to_error; + } + pModule = pVtab->pModule; + nArg = pOp->p2; + assert( pOp->p4type==P4_VTAB ); + if( ALWAYS(pModule->xUpdate) ){ + u8 vtabOnConflict = db->vtabOnConflict; + apArg = p->apArg; + pX = &aMem[pOp->p3]; + for(i=0; ivtabOnConflict = pOp->p5; + rc = pModule->xUpdate(pVtab, nArg, apArg, &rowid); + db->vtabOnConflict = vtabOnConflict; + sqlite3VtabImportErrmsg(p, pVtab); + if( rc==SQLITE_OK && pOp->p1 ){ + assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); + db->lastRowid = rowid; + } + if( (rc&0xff)==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ + if( pOp->p5==OE_Ignore ){ + rc = SQLITE_OK; + }else{ + p->errorAction = ((pOp->p5==OE_Replace) ? OE_Abort : pOp->p5); + } + }else{ + p->nChange++; + } + if( rc ) goto abort_due_to_error; + } + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* Opcode: Pagecount P1 P2 * * * +** +** Write the current number of pages in database P1 to memory cell P2. +*/ +case OP_Pagecount: { /* out2 */ + pOut = out2Prerelease(p, pOp); + pOut->u.i = sqlite3BtreeLastPage(db->aDb[pOp->p1].pBt); + break; +} +#endif + + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* Opcode: MaxPgcnt P1 P2 P3 * * +** +** Try to set the maximum page count for database P1 to the value in P3. +** Do not let the maximum page count fall below the current page count and +** do not change the maximum page count value if P3==0. +** +** Store the maximum page count after the change in register P2. +*/ +case OP_MaxPgcnt: { /* out2 */ + unsigned int newMax; + Btree *pBt; + + pOut = out2Prerelease(p, pOp); + pBt = db->aDb[pOp->p1].pBt; + newMax = 0; + if( pOp->p3 ){ + newMax = sqlite3BtreeLastPage(pBt); + if( newMax < (unsigned)pOp->p3 ) newMax = (unsigned)pOp->p3; + } + pOut->u.i = sqlite3BtreeMaxPageCount(pBt, newMax); + break; +} +#endif + +/* Opcode: Function P1 P2 P3 P4 * +** Synopsis: r[P3]=func(r[P2@NP]) +** +** Invoke a user function (P4 is a pointer to an sqlite3_context object that +** contains a pointer to the function to be run) with arguments taken +** from register P2 and successors. The number of arguments is in +** the sqlite3_context object that P4 points to. +** The result of the function is stored +** in register P3. Register P3 must not be one of the function inputs. +** +** P1 is a 32-bit bitmask indicating whether or not each argument to the +** function was determined to be constant at compile time. If the first +** argument was constant then bit 0 of P1 is set. This is used to determine +** whether meta data associated with a user function argument using the +** sqlite3_set_auxdata() API may be safely retained until the next +** invocation of this opcode. +** +** See also: AggStep, AggFinal, PureFunc +*/ +/* Opcode: PureFunc P1 P2 P3 P4 * +** Synopsis: r[P3]=func(r[P2@NP]) +** +** Invoke a user function (P4 is a pointer to an sqlite3_context object that +** contains a pointer to the function to be run) with arguments taken +** from register P2 and successors. The number of arguments is in +** the sqlite3_context object that P4 points to. +** The result of the function is stored +** in register P3. Register P3 must not be one of the function inputs. +** +** P1 is a 32-bit bitmask indicating whether or not each argument to the +** function was determined to be constant at compile time. If the first +** argument was constant then bit 0 of P1 is set. This is used to determine +** whether meta data associated with a user function argument using the +** sqlite3_set_auxdata() API may be safely retained until the next +** invocation of this opcode. +** +** This opcode works exactly like OP_Function. The only difference is in +** its name. This opcode is used in places where the function must be +** purely non-deterministic. Some built-in date/time functions can be +** either determinitic of non-deterministic, depending on their arguments. +** When those function are used in a non-deterministic way, they will check +** to see if they were called using OP_PureFunc instead of OP_Function, and +** if they were, they throw an error. +** +** See also: AggStep, AggFinal, Function +*/ +case OP_PureFunc: /* group */ +case OP_Function: { /* group */ + int i; + sqlite3_context *pCtx; + + assert( pOp->p4type==P4_FUNCCTX ); + pCtx = pOp->p4.pCtx; + + /* If this function is inside of a trigger, the register array in aMem[] + ** might change from one evaluation to the next. The next block of code + ** checks to see if the register array has changed, and if so it + ** reinitializes the relavant parts of the sqlite3_context object */ + pOut = &aMem[pOp->p3]; + if( pCtx->pOut != pOut ){ + pCtx->pVdbe = p; + pCtx->pOut = pOut; + for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; + } + assert( pCtx->pVdbe==p ); + + memAboutToChange(p, pOut); +#ifdef SQLITE_DEBUG + for(i=0; iargc; i++){ + assert( memIsValid(pCtx->argv[i]) ); + REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); + } +#endif + MemSetTypeFlag(pOut, MEM_Null); + assert( pCtx->isError==0 ); + (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ + + /* If the function returned an error, throw an exception */ + if( pCtx->isError ){ + if( pCtx->isError>0 ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pOut)); + rc = pCtx->isError; + } + sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1); + pCtx->isError = 0; + if( rc ) goto abort_due_to_error; + } + + /* Copy the result of the function into register P3 */ + if( pOut->flags & (MEM_Str|MEM_Blob) ){ + sqlite3VdbeChangeEncoding(pOut, encoding); + if( sqlite3VdbeMemTooBig(pOut) ) goto too_big; + } + + REGISTER_TRACE(pOp->p3, pOut); + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: Trace P1 P2 * P4 * +** +** Write P4 on the statement trace output if statement tracing is +** enabled. +** +** Operand P1 must be 0x7fffffff and P2 must positive. +*/ +/* Opcode: Init P1 P2 P3 P4 * +** Synopsis: Start at P2 +** +** Programs contain a single instance of this opcode as the very first +** opcode. +** +** If tracing is enabled (by the sqlite3_trace()) interface, then +** the UTF-8 string contained in P4 is emitted on the trace callback. +** Or if P4 is blank, use the string returned by sqlite3_sql(). +** +** If P2 is not zero, jump to instruction P2. +** +** Increment the value of P1 so that OP_Once opcodes will jump the +** first time they are evaluated for this run. +** +** If P3 is not zero, then it is an address to jump to if an SQLITE_CORRUPT +** error is encountered. +*/ +case OP_Trace: +case OP_Init: { /* jump */ + int i; +#ifndef SQLITE_OMIT_TRACE + char *zTrace; +#endif + + /* If the P4 argument is not NULL, then it must be an SQL comment string. + ** The "--" string is broken up to prevent false-positives with srcck1.c. + ** + ** This assert() provides evidence for: + ** EVIDENCE-OF: R-50676-09860 The callback can compute the same text that + ** would have been returned by the legacy sqlite3_trace() interface by + ** using the X argument when X begins with "--" and invoking + ** sqlite3_expanded_sql(P) otherwise. + */ + assert( pOp->p4.z==0 || strncmp(pOp->p4.z, "-" "- ", 3)==0 ); + + /* OP_Init is always instruction 0 */ + assert( pOp==p->aOp || pOp->opcode==OP_Trace ); + +#ifndef SQLITE_OMIT_TRACE + if( (db->mTrace & (SQLITE_TRACE_STMT|SQLITE_TRACE_LEGACY))!=0 + && !p->doingRerun + && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 + ){ +#ifndef SQLITE_OMIT_DEPRECATED + if( db->mTrace & SQLITE_TRACE_LEGACY ){ + char *z = sqlite3VdbeExpandSql(p, zTrace); + db->trace.xLegacy(db->pTraceArg, z); + sqlite3_free(z); + }else +#endif + if( db->nVdbeExec>1 ){ + char *z = sqlite3MPrintf(db, "-- %s", zTrace); + (void)db->trace.xV2(SQLITE_TRACE_STMT, db->pTraceArg, p, z); + sqlite3DbFree(db, z); + }else{ + (void)db->trace.xV2(SQLITE_TRACE_STMT, db->pTraceArg, p, zTrace); + } + } +#ifdef SQLITE_USE_FCNTL_TRACE + zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql); + if( zTrace ){ + int j; + for(j=0; jnDb; j++){ + if( DbMaskTest(p->btreeMask, j)==0 ) continue; + sqlite3_file_control(db, db->aDb[j].zDbSName, SQLITE_FCNTL_TRACE, zTrace); + } + } +#endif /* SQLITE_USE_FCNTL_TRACE */ +#ifdef SQLITE_DEBUG + if( (db->flags & SQLITE_SqlTrace)!=0 + && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 + ){ + sqlite3DebugPrintf("SQL-trace: %s\n", zTrace); + } +#endif /* SQLITE_DEBUG */ +#endif /* SQLITE_OMIT_TRACE */ + assert( pOp->p2>0 ); + if( pOp->p1>=sqlite3GlobalConfig.iOnceResetThreshold ){ + if( pOp->opcode==OP_Trace ) break; + for(i=1; inOp; i++){ + if( p->aOp[i].opcode==OP_Once ) p->aOp[i].p1 = 0; + } + pOp->p1 = 0; + } + pOp->p1++; + p->aCounter[SQLITE_STMTSTATUS_RUN]++; + goto jump_to_p2; +} + +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* Opcode: CursorHint P1 * * P4 * +** +** Provide a hint to cursor P1 that it only needs to return rows that +** satisfy the Expr in P4. TK_REGISTER terms in the P4 expression refer +** to values currently held in registers. TK_COLUMN terms in the P4 +** expression refer to columns in the b-tree to which cursor P1 is pointing. +*/ +case OP_CursorHint: { + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_EXPR ); + pC = p->apCsr[pOp->p1]; + if( pC ){ + assert( pC->eCurType==CURTYPE_BTREE ); + sqlite3BtreeCursorHint(pC->uc.pCursor, BTREE_HINT_RANGE, + pOp->p4.pExpr, aMem); + } + break; +} +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ + +#ifdef SQLITE_DEBUG +/* Opcode: Abortable * * * * * +** +** Verify that an Abort can happen. Assert if an Abort at this point +** might cause database corruption. This opcode only appears in debugging +** builds. +** +** An Abort is safe if either there have been no writes, or if there is +** an active statement journal. +*/ +case OP_Abortable: { + sqlite3VdbeAssertAbortable(p); + break; +} +#endif + +#ifdef SQLITE_DEBUG +/* Opcode: ReleaseReg P1 P2 P3 * P5 +** Synopsis: release r[P1@P2] mask P3 +** +** Release registers from service. Any content that was in the +** the registers is unreliable after this opcode completes. +** +** The registers released will be the P2 registers starting at P1, +** except if bit ii of P3 set, then do not release register P1+ii. +** In other words, P3 is a mask of registers to preserve. +** +** Releasing a register clears the Mem.pScopyFrom pointer. That means +** that if the content of the released register was set using OP_SCopy, +** a change to the value of the source register for the OP_SCopy will no longer +** generate an assertion fault in sqlite3VdbeMemAboutToChange(). +** +** If P5 is set, then all released registers have their type set +** to MEM_Undefined so that any subsequent attempt to read the released +** register (before it is reinitialized) will generate an assertion fault. +** +** P5 ought to be set on every call to this opcode. +** However, there are places in the code generator will release registers +** before their are used, under the (valid) assumption that the registers +** will not be reallocated for some other purpose before they are used and +** hence are safe to release. +** +** This opcode is only available in testing and debugging builds. It is +** not generated for release builds. The purpose of this opcode is to help +** validate the generated bytecode. This opcode does not actually contribute +** to computing an answer. +*/ +case OP_ReleaseReg: { + Mem *pMem; + int i; + u32 constMask; + assert( pOp->p1>0 ); + assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); + pMem = &aMem[pOp->p1]; + constMask = pOp->p3; + for(i=0; ip2; i++, pMem++){ + if( i>=32 || (constMask & MASKBIT32(i))==0 ){ + pMem->pScopyFrom = 0; + if( i<32 && pOp->p5 ) MemSetTypeFlag(pMem, MEM_Undefined); + } + } + break; +} +#endif + +/* Opcode: Noop * * * * * +** +** Do nothing. This instruction is often useful as a jump +** destination. +*/ +/* +** The magic Explain opcode are only inserted when explain==2 (which +** is to say when the EXPLAIN QUERY PLAN syntax is used.) +** This opcode records information from the optimizer. It is the +** the same as a no-op. This opcodesnever appears in a real VM program. +*/ +default: { /* This is really OP_Noop, OP_Explain */ + assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain ); + + break; +} + +/***************************************************************************** +** The cases of the switch statement above this line should all be indented +** by 6 spaces. But the left-most 6 spaces have been removed to improve the +** readability. From this point on down, the normal indentation rules are +** restored. +*****************************************************************************/ + } + +#ifdef VDBE_PROFILE + { + u64 endTime = sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + if( endTime>start ) pOrigOp->cycles += endTime - start; + pOrigOp->cnt++; + } +#endif + + /* The following code adds nothing to the actual functionality + ** of the program. It is only here for testing and debugging. + ** On the other hand, it does burn CPU cycles every time through + ** the evaluator loop. So we can leave it out when NDEBUG is defined. + */ +#ifndef NDEBUG + assert( pOp>=&aOp[-1] && pOp<&aOp[p->nOp-1] ); + +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeTrace ){ + u8 opProperty = sqlite3OpcodeProperty[pOrigOp->opcode]; + if( rc!=0 ) printf("rc=%d\n",rc); + if( opProperty & (OPFLG_OUT2) ){ + registerTrace(pOrigOp->p2, &aMem[pOrigOp->p2]); + } + if( opProperty & OPFLG_OUT3 ){ + registerTrace(pOrigOp->p3, &aMem[pOrigOp->p3]); + } + if( opProperty==0xff ){ + /* Never happens. This code exists to avoid a harmless linkage + ** warning aboud sqlite3VdbeRegisterDump() being defined but not + ** used. */ + sqlite3VdbeRegisterDump(p); + } + } +#endif /* SQLITE_DEBUG */ +#endif /* NDEBUG */ + } /* The end of the for(;;) loop the loops through opcodes */ + + /* If we reach this point, it means that execution is finished with + ** an error of some kind. + */ +abort_due_to_error: + if( db->mallocFailed ){ + rc = SQLITE_NOMEM_BKPT; + }else if( rc==SQLITE_IOERR_CORRUPTFS ){ + rc = SQLITE_CORRUPT_BKPT; + } + assert( rc ); +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeTrace ){ + printf("ABORT-due-to-error. rc=%d\n", rc); + } +#endif + if( p->zErrMsg==0 && rc!=SQLITE_IOERR_NOMEM ){ + sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); + } + p->rc = rc; + sqlite3SystemError(db, rc); + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(rc, "statement aborts at %d: [%s] %s", + (int)(pOp - aOp), p->zSql, p->zErrMsg); + sqlite3VdbeHalt(p); + if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); + rc = SQLITE_ERROR; + if( resetSchemaOnFault>0 ){ + sqlite3ResetOneSchema(db, resetSchemaOnFault-1); + } + + /* This is the only way out of this procedure. We have to + ** release the mutexes on btrees that were acquired at the + ** top. */ +vdbe_return: +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ + nProgressLimit += db->nProgressOps; + if( db->xProgress(db->pProgressArg) ){ + nProgressLimit = LARGEST_UINT64; + rc = SQLITE_INTERRUPT; + goto abort_due_to_error; + } + } +#endif + p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; + sqlite3VdbeLeave(p); + assert( rc!=SQLITE_OK || nExtraDelete==0 + || sqlite3_strlike("DELETE%",p->zSql,0)!=0 + ); + return rc; + + /* Jump to here if a string or blob larger than SQLITE_MAX_LENGTH + ** is encountered. + */ +too_big: + sqlite3VdbeError(p, "string or blob too big"); + rc = SQLITE_TOOBIG; + goto abort_due_to_error; + + /* Jump to here if a malloc() fails. + */ +no_mem: + sqlite3OomFault(db); + sqlite3VdbeError(p, "out of memory"); + rc = SQLITE_NOMEM_BKPT; + goto abort_due_to_error; + + /* Jump to here if the sqlite3_interrupt() API sets the interrupt + ** flag. + */ +abort_due_to_interrupt: + assert( AtomicLoad(&db->u1.isInterrupted) ); + rc = SQLITE_INTERRUPT; + goto abort_due_to_error; +} + + +/************** End of vdbe.c ************************************************/ +/************** Begin file vdbeblob.c ****************************************/ +/* +** 2007 May 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code used to implement incremental BLOB I/O. +*/ + +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ + +#ifndef SQLITE_OMIT_INCRBLOB + +/* +** Valid sqlite3_blob* handles point to Incrblob structures. +*/ +typedef struct Incrblob Incrblob; +struct Incrblob { + int nByte; /* Size of open blob, in bytes */ + int iOffset; /* Byte offset of blob in cursor data */ + u16 iCol; /* Table column this handle is open on */ + BtCursor *pCsr; /* Cursor pointing at blob row */ + sqlite3_stmt *pStmt; /* Statement holding cursor open */ + sqlite3 *db; /* The associated database */ + char *zDb; /* Database name */ + Table *pTab; /* Table object */ +}; + + +/* +** This function is used by both blob_open() and blob_reopen(). It seeks +** the b-tree cursor associated with blob handle p to point to row iRow. +** If successful, SQLITE_OK is returned and subsequent calls to +** sqlite3_blob_read() or sqlite3_blob_write() access the specified row. +** +** If an error occurs, or if the specified row does not exist or does not +** contain a value of type TEXT or BLOB in the column nominated when the +** blob handle was opened, then an error code is returned and *pzErr may +** be set to point to a buffer containing an error message. It is the +** responsibility of the caller to free the error message buffer using +** sqlite3DbFree(). +** +** If an error does occur, then the b-tree cursor is closed. All subsequent +** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will +** immediately return SQLITE_ABORT. +*/ +static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ + int rc; /* Error code */ + char *zErr = 0; /* Error message */ + Vdbe *v = (Vdbe *)p->pStmt; + + /* Set the value of register r[1] in the SQL statement to integer iRow. + ** This is done directly as a performance optimization + */ + v->aMem[1].flags = MEM_Int; + v->aMem[1].u.i = iRow; + + /* If the statement has been run before (and is paused at the OP_ResultRow) + ** then back it up to the point where it does the OP_NotExists. This could + ** have been down with an extra OP_Goto, but simply setting the program + ** counter is faster. */ + if( v->pc>4 ){ + v->pc = 4; + assert( v->aOp[v->pc].opcode==OP_NotExists ); + rc = sqlite3VdbeExec(v); + }else{ + rc = sqlite3_step(p->pStmt); + } + if( rc==SQLITE_ROW ){ + VdbeCursor *pC = v->apCsr[0]; + u32 type = pC->nHdrParsed>p->iCol ? pC->aType[p->iCol] : 0; + testcase( pC->nHdrParsed==p->iCol ); + testcase( pC->nHdrParsed==p->iCol+1 ); + if( type<12 ){ + zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", + type==0?"null": type==7?"real": "integer" + ); + rc = SQLITE_ERROR; + sqlite3_finalize(p->pStmt); + p->pStmt = 0; + }else{ + p->iOffset = pC->aType[p->iCol + pC->nField]; + p->nByte = sqlite3VdbeSerialTypeLen(type); + p->pCsr = pC->uc.pCursor; + sqlite3BtreeIncrblobCursor(p->pCsr); + } + } + + if( rc==SQLITE_ROW ){ + rc = SQLITE_OK; + }else if( p->pStmt ){ + rc = sqlite3_finalize(p->pStmt); + p->pStmt = 0; + if( rc==SQLITE_OK ){ + zErr = sqlite3MPrintf(p->db, "no such rowid: %lld", iRow); + rc = SQLITE_ERROR; + }else{ + zErr = sqlite3MPrintf(p->db, "%s", sqlite3_errmsg(p->db)); + } + } + + assert( rc!=SQLITE_OK || zErr==0 ); + assert( rc!=SQLITE_ROW && rc!=SQLITE_DONE ); + + *pzErr = zErr; + return rc; +} + +/* +** Open a blob handle. +*/ +SQLITE_API int sqlite3_blob_open( + sqlite3* db, /* The database connection */ + const char *zDb, /* The attached database containing the blob */ + const char *zTable, /* The table containing the blob */ + const char *zColumn, /* The column containing the blob */ + sqlite_int64 iRow, /* The row containing the glob */ + int wrFlag, /* True -> read/write access, false -> read-only */ + sqlite3_blob **ppBlob /* Handle for accessing the blob returned here */ +){ + int nAttempt = 0; + int iCol; /* Index of zColumn in row-record */ + int rc = SQLITE_OK; + char *zErr = 0; + Table *pTab; + Incrblob *pBlob = 0; + Parse sParse; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppBlob==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + *ppBlob = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zTable==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + wrFlag = !!wrFlag; /* wrFlag = (wrFlag ? 1 : 0); */ + + sqlite3_mutex_enter(db->mutex); + + pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); + do { + memset(&sParse, 0, sizeof(Parse)); + if( !pBlob ) goto blob_open_out; + sParse.db = db; + sqlite3DbFree(db, zErr); + zErr = 0; + + sqlite3BtreeEnterAll(db); + pTab = sqlite3LocateTable(&sParse, 0, zTable, zDb); + if( pTab && IsVirtual(pTab) ){ + pTab = 0; + sqlite3ErrorMsg(&sParse, "cannot open virtual table: %s", zTable); + } + if( pTab && !HasRowid(pTab) ){ + pTab = 0; + sqlite3ErrorMsg(&sParse, "cannot open table without rowid: %s", zTable); + } +#ifndef SQLITE_OMIT_VIEW + if( pTab && IsView(pTab) ){ + pTab = 0; + sqlite3ErrorMsg(&sParse, "cannot open view: %s", zTable); + } +#endif + if( !pTab ){ + if( sParse.zErrMsg ){ + sqlite3DbFree(db, zErr); + zErr = sParse.zErrMsg; + sParse.zErrMsg = 0; + } + rc = SQLITE_ERROR; + sqlite3BtreeLeaveAll(db); + goto blob_open_out; + } + pBlob->pTab = pTab; + pBlob->zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName; + + /* Now search pTab for the exact column. */ + for(iCol=0; iColnCol; iCol++) { + if( sqlite3StrICmp(pTab->aCol[iCol].zCnName, zColumn)==0 ){ + break; + } + } + if( iCol==pTab->nCol ){ + sqlite3DbFree(db, zErr); + zErr = sqlite3MPrintf(db, "no such column: \"%s\"", zColumn); + rc = SQLITE_ERROR; + sqlite3BtreeLeaveAll(db); + goto blob_open_out; + } + + /* If the value is being opened for writing, check that the + ** column is not indexed, and that it is not part of a foreign key. + */ + if( wrFlag ){ + const char *zFault = 0; + Index *pIdx; +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( db->flags&SQLITE_ForeignKeys ){ + /* Check that the column is not part of an FK child key definition. It + ** is not necessary to check if it is part of a parent key, as parent + ** key columns must be indexed. The check below will pick up this + ** case. */ + FKey *pFKey; + assert( !IsVirtual(pTab) ); + for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pFKey->pNextFrom){ + int j; + for(j=0; jnCol; j++){ + if( pFKey->aCol[j].iFrom==iCol ){ + zFault = "foreign key"; + } + } + } + } +#endif + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int j; + for(j=0; jnKeyCol; j++){ + /* FIXME: Be smarter about indexes that use expressions */ + if( pIdx->aiColumn[j]==iCol || pIdx->aiColumn[j]==XN_EXPR ){ + zFault = "indexed"; + } + } + } + if( zFault ){ + sqlite3DbFree(db, zErr); + zErr = sqlite3MPrintf(db, "cannot open %s column for writing", zFault); + rc = SQLITE_ERROR; + sqlite3BtreeLeaveAll(db); + goto blob_open_out; + } + } + + pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(&sParse); + assert( pBlob->pStmt || db->mallocFailed ); + if( pBlob->pStmt ){ + + /* This VDBE program seeks a btree cursor to the identified + ** db/table/row entry. The reason for using a vdbe program instead + ** of writing code to use the b-tree layer directly is that the + ** vdbe program will take advantage of the various transaction, + ** locking and error handling infrastructure built into the vdbe. + ** + ** After seeking the cursor, the vdbe executes an OP_ResultRow. + ** Code external to the Vdbe then "borrows" the b-tree cursor and + ** uses it to implement the blob_read(), blob_write() and + ** blob_bytes() functions. + ** + ** The sqlite3_blob_close() function finalizes the vdbe program, + ** which closes the b-tree cursor and (possibly) commits the + ** transaction. + */ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList openBlob[] = { + {OP_TableLock, 0, 0, 0}, /* 0: Acquire a read or write lock */ + {OP_OpenRead, 0, 0, 0}, /* 1: Open a cursor */ + /* blobSeekToRow() will initialize r[1] to the desired rowid */ + {OP_NotExists, 0, 5, 1}, /* 2: Seek the cursor to rowid=r[1] */ + {OP_Column, 0, 0, 1}, /* 3 */ + {OP_ResultRow, 1, 0, 0}, /* 4 */ + {OP_Halt, 0, 0, 0}, /* 5 */ + }; + Vdbe *v = (Vdbe *)pBlob->pStmt; + int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + VdbeOp *aOp; + + sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, wrFlag, + pTab->pSchema->schema_cookie, + pTab->pSchema->iGeneration); + sqlite3VdbeChangeP5(v, 1); + assert( sqlite3VdbeCurrentAddr(v)==2 || db->mallocFailed ); + aOp = sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); + + /* Make sure a mutex is held on the table to be accessed */ + sqlite3VdbeUsesBtree(v, iDb); + + if( db->mallocFailed==0 ){ + assert( aOp!=0 ); + /* Configure the OP_TableLock instruction */ +#ifdef SQLITE_OMIT_SHARED_CACHE + aOp[0].opcode = OP_Noop; +#else + aOp[0].p1 = iDb; + aOp[0].p2 = pTab->tnum; + aOp[0].p3 = wrFlag; + sqlite3VdbeChangeP4(v, 2, pTab->zName, P4_TRANSIENT); + } + if( db->mallocFailed==0 ){ +#endif + + /* Remove either the OP_OpenWrite or OpenRead. Set the P2 + ** parameter of the other to pTab->tnum. */ + if( wrFlag ) aOp[1].opcode = OP_OpenWrite; + aOp[1].p2 = pTab->tnum; + aOp[1].p3 = iDb; + + /* Configure the number of columns. Configure the cursor to + ** think that the table has one more column than it really + ** does. An OP_Column to retrieve this imaginary column will + ** always return an SQL NULL. This is useful because it means + ** we can invoke OP_Column to fill in the vdbe cursors type + ** and offset cache without causing any IO. + */ + aOp[1].p4type = P4_INT32; + aOp[1].p4.i = pTab->nCol+1; + aOp[3].p2 = pTab->nCol; + + sParse.nVar = 0; + sParse.nMem = 1; + sParse.nTab = 1; + sqlite3VdbeMakeReady(v, &sParse); + } + } + + pBlob->iCol = iCol; + pBlob->db = db; + sqlite3BtreeLeaveAll(db); + if( db->mallocFailed ){ + goto blob_open_out; + } + rc = blobSeekToRow(pBlob, iRow, &zErr); + } while( (++nAttempt)mallocFailed==0 ){ + *ppBlob = (sqlite3_blob *)pBlob; + }else{ + if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); + sqlite3DbFree(db, pBlob); + } + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + sqlite3ParserReset(&sParse); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Close a blob handle that was previously created using +** sqlite3_blob_open(). +*/ +SQLITE_API int sqlite3_blob_close(sqlite3_blob *pBlob){ + Incrblob *p = (Incrblob *)pBlob; + int rc; + sqlite3 *db; + + if( p ){ + sqlite3_stmt *pStmt = p->pStmt; + db = p->db; + sqlite3_mutex_enter(db->mutex); + sqlite3DbFree(db, p); + sqlite3_mutex_leave(db->mutex); + rc = sqlite3_finalize(pStmt); + }else{ + rc = SQLITE_OK; + } + return rc; +} + +/* +** Perform a read or write operation on a blob +*/ +static int blobReadWrite( + sqlite3_blob *pBlob, + void *z, + int n, + int iOffset, + int (*xCall)(BtCursor*, u32, u32, void*) +){ + int rc; + Incrblob *p = (Incrblob *)pBlob; + Vdbe *v; + sqlite3 *db; + + if( p==0 ) return SQLITE_MISUSE_BKPT; + db = p->db; + sqlite3_mutex_enter(db->mutex); + v = (Vdbe*)p->pStmt; + + if( n<0 || iOffset<0 || ((sqlite3_int64)iOffset+n)>p->nByte ){ + /* Request is out of range. Return a transient error. */ + rc = SQLITE_ERROR; + }else if( v==0 ){ + /* If there is no statement handle, then the blob-handle has + ** already been invalidated. Return SQLITE_ABORT in this case. + */ + rc = SQLITE_ABORT; + }else{ + /* Call either BtreeData() or BtreePutData(). If SQLITE_ABORT is + ** returned, clean-up the statement handle. + */ + assert( db == v->db ); + sqlite3BtreeEnterCursor(p->pCsr); + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( xCall==sqlite3BtreePutData && db->xPreUpdateCallback ){ + /* If a pre-update hook is registered and this is a write cursor, + ** invoke it here. + ** + ** TODO: The preupdate-hook is passed SQLITE_DELETE, even though this + ** operation should really be an SQLITE_UPDATE. This is probably + ** incorrect, but is convenient because at this point the new.* values + ** are not easily obtainable. And for the sessions module, an + ** SQLITE_UPDATE where the PK columns do not change is handled in the + ** same way as an SQLITE_DELETE (the SQLITE_DELETE code is actually + ** slightly more efficient). Since you cannot write to a PK column + ** using the incremental-blob API, this works. For the sessions module + ** anyhow. + */ + sqlite3_int64 iKey; + iKey = sqlite3BtreeIntegerKey(p->pCsr); + sqlite3VdbePreUpdateHook( + v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol + ); + } +#endif + + rc = xCall(p->pCsr, iOffset+p->iOffset, n, z); + sqlite3BtreeLeaveCursor(p->pCsr); + if( rc==SQLITE_ABORT ){ + sqlite3VdbeFinalize(v); + p->pStmt = 0; + }else{ + v->rc = rc; + } + } + sqlite3Error(db, rc); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Read data from a blob handle. +*/ +SQLITE_API int sqlite3_blob_read(sqlite3_blob *pBlob, void *z, int n, int iOffset){ + return blobReadWrite(pBlob, z, n, iOffset, sqlite3BtreePayloadChecked); +} + +/* +** Write data to a blob handle. +*/ +SQLITE_API int sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int iOffset){ + return blobReadWrite(pBlob, (void *)z, n, iOffset, sqlite3BtreePutData); +} + +/* +** Query a blob handle for the size of the data. +** +** The Incrblob.nByte field is fixed for the lifetime of the Incrblob +** so no mutex is required for access. +*/ +SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){ + Incrblob *p = (Incrblob *)pBlob; + return (p && p->pStmt) ? p->nByte : 0; +} + +/* +** Move an existing blob handle to point to a different row of the same +** database table. +** +** If an error occurs, or if the specified row does not exist or does not +** contain a blob or text value, then an error code is returned and the +** database handle error code and message set. If this happens, then all +** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) +** immediately return SQLITE_ABORT. +*/ +SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ + int rc; + Incrblob *p = (Incrblob *)pBlob; + sqlite3 *db; + + if( p==0 ) return SQLITE_MISUSE_BKPT; + db = p->db; + sqlite3_mutex_enter(db->mutex); + + if( p->pStmt==0 ){ + /* If there is no statement handle, then the blob-handle has + ** already been invalidated. Return SQLITE_ABORT in this case. + */ + rc = SQLITE_ABORT; + }else{ + char *zErr; + ((Vdbe*)p->pStmt)->rc = SQLITE_OK; + rc = blobSeekToRow(p, iRow, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + } + assert( rc!=SQLITE_SCHEMA ); + } + + rc = sqlite3ApiExit(db, rc); + assert( rc==SQLITE_OK || p->pStmt==0 ); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#endif /* #ifndef SQLITE_OMIT_INCRBLOB */ + +/************** End of vdbeblob.c ********************************************/ +/************** Begin file vdbesort.c ****************************************/ +/* +** 2011-07-09 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code for the VdbeSorter object, used in concert with +** a VdbeCursor to sort large numbers of keys for CREATE INDEX statements +** or by SELECT statements with ORDER BY clauses that cannot be satisfied +** using indexes and without LIMIT clauses. +** +** The VdbeSorter object implements a multi-threaded external merge sort +** algorithm that is efficient even if the number of elements being sorted +** exceeds the available memory. +** +** Here is the (internal, non-API) interface between this module and the +** rest of the SQLite system: +** +** sqlite3VdbeSorterInit() Create a new VdbeSorter object. +** +** sqlite3VdbeSorterWrite() Add a single new row to the VdbeSorter +** object. The row is a binary blob in the +** OP_MakeRecord format that contains both +** the ORDER BY key columns and result columns +** in the case of a SELECT w/ ORDER BY, or +** the complete record for an index entry +** in the case of a CREATE INDEX. +** +** sqlite3VdbeSorterRewind() Sort all content previously added. +** Position the read cursor on the +** first sorted element. +** +** sqlite3VdbeSorterNext() Advance the read cursor to the next sorted +** element. +** +** sqlite3VdbeSorterRowkey() Return the complete binary blob for the +** row currently under the read cursor. +** +** sqlite3VdbeSorterCompare() Compare the binary blob for the row +** currently under the read cursor against +** another binary blob X and report if +** X is strictly less than the read cursor. +** Used to enforce uniqueness in a +** CREATE UNIQUE INDEX statement. +** +** sqlite3VdbeSorterClose() Close the VdbeSorter object and reclaim +** all resources. +** +** sqlite3VdbeSorterReset() Refurbish the VdbeSorter for reuse. This +** is like Close() followed by Init() only +** much faster. +** +** The interfaces above must be called in a particular order. Write() can +** only occur in between Init()/Reset() and Rewind(). Next(), Rowkey(), and +** Compare() can only occur in between Rewind() and Close()/Reset(). i.e. +** +** Init() +** for each record: Write() +** Rewind() +** Rowkey()/Compare() +** Next() +** Close() +** +** Algorithm: +** +** Records passed to the sorter via calls to Write() are initially held +** unsorted in main memory. Assuming the amount of memory used never exceeds +** a threshold, when Rewind() is called the set of records is sorted using +** an in-memory merge sort. In this case, no temporary files are required +** and subsequent calls to Rowkey(), Next() and Compare() read records +** directly from main memory. +** +** If the amount of space used to store records in main memory exceeds the +** threshold, then the set of records currently in memory are sorted and +** written to a temporary file in "Packed Memory Array" (PMA) format. +** A PMA created at this point is known as a "level-0 PMA". Higher levels +** of PMAs may be created by merging existing PMAs together - for example +** merging two or more level-0 PMAs together creates a level-1 PMA. +** +** The threshold for the amount of main memory to use before flushing +** records to a PMA is roughly the same as the limit configured for the +** page-cache of the main database. Specifically, the threshold is set to +** the value returned by "PRAGMA main.page_size" multipled by +** that returned by "PRAGMA main.cache_size", in bytes. +** +** If the sorter is running in single-threaded mode, then all PMAs generated +** are appended to a single temporary file. Or, if the sorter is running in +** multi-threaded mode then up to (N+1) temporary files may be opened, where +** N is the configured number of worker threads. In this case, instead of +** sorting the records and writing the PMA to a temporary file itself, the +** calling thread usually launches a worker thread to do so. Except, if +** there are already N worker threads running, the main thread does the work +** itself. +** +** The sorter is running in multi-threaded mode if (a) the library was built +** with pre-processor symbol SQLITE_MAX_WORKER_THREADS set to a value greater +** than zero, and (b) worker threads have been enabled at runtime by calling +** "PRAGMA threads=N" with some value of N greater than 0. +** +** When Rewind() is called, any data remaining in memory is flushed to a +** final PMA. So at this point the data is stored in some number of sorted +** PMAs within temporary files on disk. +** +** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the +** sorter is running in single-threaded mode, then these PMAs are merged +** incrementally as keys are retreived from the sorter by the VDBE. The +** MergeEngine object, described in further detail below, performs this +** merge. +** +** Or, if running in multi-threaded mode, then a background thread is +** launched to merge the existing PMAs. Once the background thread has +** merged T bytes of data into a single sorted PMA, the main thread +** begins reading keys from that PMA while the background thread proceeds +** with merging the next T bytes of data. And so on. +** +** Parameter T is set to half the value of the memory threshold used +** by Write() above to determine when to create a new PMA. +** +** If there are more than SORTER_MAX_MERGE_COUNT PMAs in total when +** Rewind() is called, then a hierarchy of incremental-merges is used. +** First, T bytes of data from the first SORTER_MAX_MERGE_COUNT PMAs on +** disk are merged together. Then T bytes of data from the second set, and +** so on, such that no operation ever merges more than SORTER_MAX_MERGE_COUNT +** PMAs at a time. This done is to improve locality. +** +** If running in multi-threaded mode and there are more than +** SORTER_MAX_MERGE_COUNT PMAs on disk when Rewind() is called, then more +** than one background thread may be created. Specifically, there may be +** one background thread for each temporary file on disk, and one background +** thread to merge the output of each of the others to a single PMA for +** the main thread to read from. +*/ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ + +/* +** If SQLITE_DEBUG_SORTER_THREADS is defined, this module outputs various +** messages to stderr that may be helpful in understanding the performance +** characteristics of the sorter in multi-threaded mode. +*/ +#if 0 +# define SQLITE_DEBUG_SORTER_THREADS 1 +#endif + +/* +** Hard-coded maximum amount of data to accumulate in memory before flushing +** to a level 0 PMA. The purpose of this limit is to prevent various integer +** overflows. 512MiB. +*/ +#define SQLITE_MAX_PMASZ (1<<29) + +/* +** Private objects used by the sorter +*/ +typedef struct MergeEngine MergeEngine; /* Merge PMAs together */ +typedef struct PmaReader PmaReader; /* Incrementally read one PMA */ +typedef struct PmaWriter PmaWriter; /* Incrementally write one PMA */ +typedef struct SorterRecord SorterRecord; /* A record being sorted */ +typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ +typedef struct SorterFile SorterFile; /* Temporary file object wrapper */ +typedef struct SorterList SorterList; /* In-memory list of records */ +typedef struct IncrMerger IncrMerger; /* Read & merge multiple PMAs */ + +/* +** A container for a temp file handle and the current amount of data +** stored in the file. +*/ +struct SorterFile { + sqlite3_file *pFd; /* File handle */ + i64 iEof; /* Bytes of data stored in pFd */ +}; + +/* +** An in-memory list of objects to be sorted. +** +** If aMemory==0 then each object is allocated separately and the objects +** are connected using SorterRecord.u.pNext. If aMemory!=0 then all objects +** are stored in the aMemory[] bulk memory, one right after the other, and +** are connected using SorterRecord.u.iNext. +*/ +struct SorterList { + SorterRecord *pList; /* Linked list of records */ + u8 *aMemory; /* If non-NULL, bulk memory to hold pList */ + int szPMA; /* Size of pList as PMA in bytes */ +}; + +/* +** The MergeEngine object is used to combine two or more smaller PMAs into +** one big PMA using a merge operation. Separate PMAs all need to be +** combined into one big PMA in order to be able to step through the sorted +** records in order. +** +** The aReadr[] array contains a PmaReader object for each of the PMAs being +** merged. An aReadr[] object either points to a valid key or else is at EOF. +** ("EOF" means "End Of File". When aReadr[] is at EOF there is no more data.) +** For the purposes of the paragraphs below, we assume that the array is +** actually N elements in size, where N is the smallest power of 2 greater +** to or equal to the number of PMAs being merged. The extra aReadr[] elements +** are treated as if they are empty (always at EOF). +** +** The aTree[] array is also N elements in size. The value of N is stored in +** the MergeEngine.nTree variable. +** +** The final (N/2) elements of aTree[] contain the results of comparing +** pairs of PMA keys together. Element i contains the result of +** comparing aReadr[2*i-N] and aReadr[2*i-N+1]. Whichever key is smaller, the +** aTree element is set to the index of it. +** +** For the purposes of this comparison, EOF is considered greater than any +** other key value. If the keys are equal (only possible with two EOF +** values), it doesn't matter which index is stored. +** +** The (N/4) elements of aTree[] that precede the final (N/2) described +** above contains the index of the smallest of each block of 4 PmaReaders +** And so on. So that aTree[1] contains the index of the PmaReader that +** currently points to the smallest key value. aTree[0] is unused. +** +** Example: +** +** aReadr[0] -> Banana +** aReadr[1] -> Feijoa +** aReadr[2] -> Elderberry +** aReadr[3] -> Currant +** aReadr[4] -> Grapefruit +** aReadr[5] -> Apple +** aReadr[6] -> Durian +** aReadr[7] -> EOF +** +** aTree[] = { X, 5 0, 5 0, 3, 5, 6 } +** +** The current element is "Apple" (the value of the key indicated by +** PmaReader 5). When the Next() operation is invoked, PmaReader 5 will +** be advanced to the next key in its segment. Say the next key is +** "Eggplant": +** +** aReadr[5] -> Eggplant +** +** The contents of aTree[] are updated first by comparing the new PmaReader +** 5 key to the current key of PmaReader 4 (still "Grapefruit"). The PmaReader +** 5 value is still smaller, so aTree[6] is set to 5. And so on up the tree. +** The value of PmaReader 6 - "Durian" - is now smaller than that of PmaReader +** 5, so aTree[3] is set to 6. Key 0 is smaller than key 6 (Bananafile2. And instead of using a +** background thread to prepare data for the PmaReader, with a single +** threaded IncrMerger the allocate part of pTask->file2 is "refilled" with +** keys from pMerger by the calling thread whenever the PmaReader runs out +** of data. +*/ +struct IncrMerger { + SortSubtask *pTask; /* Task that owns this merger */ + MergeEngine *pMerger; /* Merge engine thread reads data from */ + i64 iStartOff; /* Offset to start writing file at */ + int mxSz; /* Maximum bytes of data to store */ + int bEof; /* Set to true when merge is finished */ + int bUseThread; /* True to use a bg thread for this object */ + SorterFile aFile[2]; /* aFile[0] for reading, [1] for writing */ +}; + +/* +** An instance of this object is used for writing a PMA. +** +** The PMA is written one record at a time. Each record is of an arbitrary +** size. But I/O is more efficient if it occurs in page-sized blocks where +** each block is aligned on a page boundary. This object caches writes to +** the PMA so that aligned, page-size blocks are written. +*/ +struct PmaWriter { + int eFWErr; /* Non-zero if in an error state */ + u8 *aBuffer; /* Pointer to write buffer */ + int nBuffer; /* Size of write buffer in bytes */ + int iBufStart; /* First byte of buffer to write */ + int iBufEnd; /* Last byte of buffer to write */ + i64 iWriteOff; /* Offset of start of buffer in file */ + sqlite3_file *pFd; /* File handle to write to */ +}; + +/* +** This object is the header on a single record while that record is being +** held in memory and prior to being written out as part of a PMA. +** +** How the linked list is connected depends on how memory is being managed +** by this module. If using a separate allocation for each in-memory record +** (VdbeSorter.list.aMemory==0), then the list is always connected using the +** SorterRecord.u.pNext pointers. +** +** Or, if using the single large allocation method (VdbeSorter.list.aMemory!=0), +** then while records are being accumulated the list is linked using the +** SorterRecord.u.iNext offset. This is because the aMemory[] array may +** be sqlite3Realloc()ed while records are being accumulated. Once the VM +** has finished passing records to the sorter, or when the in-memory buffer +** is full, the list is sorted. As part of the sorting process, it is +** converted to use the SorterRecord.u.pNext pointers. See function +** vdbeSorterSort() for details. +*/ +struct SorterRecord { + int nVal; /* Size of the record in bytes */ + union { + SorterRecord *pNext; /* Pointer to next record in list */ + int iNext; /* Offset within aMemory of next record */ + } u; + /* The data for the record immediately follows this header */ +}; + +/* Return a pointer to the buffer containing the record data for SorterRecord +** object p. Should be used as if: +** +** void *SRVAL(SorterRecord *p) { return (void*)&p[1]; } +*/ +#define SRVAL(p) ((void*)((SorterRecord*)(p) + 1)) + + +/* Maximum number of PMAs that a single MergeEngine can merge */ +#define SORTER_MAX_MERGE_COUNT 16 + +static int vdbeIncrSwap(IncrMerger*); +static void vdbeIncrFree(IncrMerger *); + +/* +** Free all memory belonging to the PmaReader object passed as the +** argument. All structure fields are set to zero before returning. +*/ +static void vdbePmaReaderClear(PmaReader *pReadr){ + sqlite3_free(pReadr->aAlloc); + sqlite3_free(pReadr->aBuffer); + if( pReadr->aMap ) sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); + vdbeIncrFree(pReadr->pIncr); + memset(pReadr, 0, sizeof(PmaReader)); +} + +/* +** Read the next nByte bytes of data from the PMA p. +** If successful, set *ppOut to point to a buffer containing the data +** and return SQLITE_OK. Otherwise, if an error occurs, return an SQLite +** error code. +** +** The buffer returned in *ppOut is only valid until the +** next call to this function. +*/ +static int vdbePmaReadBlob( + PmaReader *p, /* PmaReader from which to take the blob */ + int nByte, /* Bytes of data to read */ + u8 **ppOut /* OUT: Pointer to buffer containing data */ +){ + int iBuf; /* Offset within buffer to read from */ + int nAvail; /* Bytes of data available in buffer */ + + if( p->aMap ){ + *ppOut = &p->aMap[p->iReadOff]; + p->iReadOff += nByte; + return SQLITE_OK; + } + + assert( p->aBuffer ); + + /* If there is no more data to be read from the buffer, read the next + ** p->nBuffer bytes of data from the file into it. Or, if there are less + ** than p->nBuffer bytes remaining in the PMA, read all remaining data. */ + iBuf = p->iReadOff % p->nBuffer; + if( iBuf==0 ){ + int nRead; /* Bytes to read from disk */ + int rc; /* sqlite3OsRead() return code */ + + /* Determine how many bytes of data to read. */ + if( (p->iEof - p->iReadOff) > (i64)p->nBuffer ){ + nRead = p->nBuffer; + }else{ + nRead = (int)(p->iEof - p->iReadOff); + } + assert( nRead>0 ); + + /* Readr data from the file. Return early if an error occurs. */ + rc = sqlite3OsRead(p->pFd, p->aBuffer, nRead, p->iReadOff); + assert( rc!=SQLITE_IOERR_SHORT_READ ); + if( rc!=SQLITE_OK ) return rc; + } + nAvail = p->nBuffer - iBuf; + + if( nByte<=nAvail ){ + /* The requested data is available in the in-memory buffer. In this + ** case there is no need to make a copy of the data, just return a + ** pointer into the buffer to the caller. */ + *ppOut = &p->aBuffer[iBuf]; + p->iReadOff += nByte; + }else{ + /* The requested data is not all available in the in-memory buffer. + ** In this case, allocate space at p->aAlloc[] to copy the requested + ** range into. Then return a copy of pointer p->aAlloc to the caller. */ + int nRem; /* Bytes remaining to copy */ + + /* Extend the p->aAlloc[] allocation if required. */ + if( p->nAllocnAlloc); + while( nByte>nNew ) nNew = nNew*2; + aNew = sqlite3Realloc(p->aAlloc, nNew); + if( !aNew ) return SQLITE_NOMEM_BKPT; + p->nAlloc = nNew; + p->aAlloc = aNew; + } + + /* Copy as much data as is available in the buffer into the start of + ** p->aAlloc[]. */ + memcpy(p->aAlloc, &p->aBuffer[iBuf], nAvail); + p->iReadOff += nAvail; + nRem = nByte - nAvail; + + /* The following loop copies up to p->nBuffer bytes per iteration into + ** the p->aAlloc[] buffer. */ + while( nRem>0 ){ + int rc; /* vdbePmaReadBlob() return code */ + int nCopy; /* Number of bytes to copy */ + u8 *aNext; /* Pointer to buffer to copy data from */ + + nCopy = nRem; + if( nRem>p->nBuffer ) nCopy = p->nBuffer; + rc = vdbePmaReadBlob(p, nCopy, &aNext); + if( rc!=SQLITE_OK ) return rc; + assert( aNext!=p->aAlloc ); + memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); + nRem -= nCopy; + } + + *ppOut = p->aAlloc; + } + + return SQLITE_OK; +} + +/* +** Read a varint from the stream of data accessed by p. Set *pnOut to +** the value read. +*/ +static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ + int iBuf; + + if( p->aMap ){ + p->iReadOff += sqlite3GetVarint(&p->aMap[p->iReadOff], pnOut); + }else{ + iBuf = p->iReadOff % p->nBuffer; + if( iBuf && (p->nBuffer-iBuf)>=9 ){ + p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + }else{ + u8 aVarint[16], *a; + int i = 0, rc; + do{ + rc = vdbePmaReadBlob(p, 1, &a); + if( rc ) return rc; + aVarint[(i++)&0xf] = a[0]; + }while( (a[0]&0x80)!=0 ); + sqlite3GetVarint(aVarint, pnOut); + } + } + + return SQLITE_OK; +} + +/* +** Attempt to memory map file pFile. If successful, set *pp to point to the +** new mapping and return SQLITE_OK. If the mapping is not attempted +** (because the file is too large or the VFS layer is configured not to use +** mmap), return SQLITE_OK and set *pp to NULL. +** +** Or, if an error occurs, return an SQLite error code. The final value of +** *pp is undefined in this case. +*/ +static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ + int rc = SQLITE_OK; + if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) ){ + sqlite3_file *pFd = pFile->pFd; + if( pFd->pMethods->iVersion>=3 ){ + rc = sqlite3OsFetch(pFd, 0, (int)pFile->iEof, (void**)pp); + testcase( rc!=SQLITE_OK ); + } + } + return rc; +} + +/* +** Attach PmaReader pReadr to file pFile (if it is not already attached to +** that file) and seek it to offset iOff within the file. Return SQLITE_OK +** if successful, or an SQLite error code if an error occurs. +*/ +static int vdbePmaReaderSeek( + SortSubtask *pTask, /* Task context */ + PmaReader *pReadr, /* Reader whose cursor is to be moved */ + SorterFile *pFile, /* Sorter file to read from */ + i64 iOff /* Offset in pFile */ +){ + int rc = SQLITE_OK; + + assert( pReadr->pIncr==0 || pReadr->pIncr->bEof==0 ); + + if( sqlite3FaultSim(201) ) return SQLITE_IOERR_READ; + if( pReadr->aMap ){ + sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); + pReadr->aMap = 0; + } + pReadr->iReadOff = iOff; + pReadr->iEof = pFile->iEof; + pReadr->pFd = pFile->pFd; + + rc = vdbeSorterMapFile(pTask, pFile, &pReadr->aMap); + if( rc==SQLITE_OK && pReadr->aMap==0 ){ + int pgsz = pTask->pSorter->pgsz; + int iBuf = pReadr->iReadOff % pgsz; + if( pReadr->aBuffer==0 ){ + pReadr->aBuffer = (u8*)sqlite3Malloc(pgsz); + if( pReadr->aBuffer==0 ) rc = SQLITE_NOMEM_BKPT; + pReadr->nBuffer = pgsz; + } + if( rc==SQLITE_OK && iBuf ){ + int nRead = pgsz - iBuf; + if( (pReadr->iReadOff + nRead) > pReadr->iEof ){ + nRead = (int)(pReadr->iEof - pReadr->iReadOff); + } + rc = sqlite3OsRead( + pReadr->pFd, &pReadr->aBuffer[iBuf], nRead, pReadr->iReadOff + ); + testcase( rc!=SQLITE_OK ); + } + } + + return rc; +} + +/* +** Advance PmaReader pReadr to the next key in its PMA. Return SQLITE_OK if +** no error occurs, or an SQLite error code if one does. +*/ +static int vdbePmaReaderNext(PmaReader *pReadr){ + int rc = SQLITE_OK; /* Return Code */ + u64 nRec = 0; /* Size of record in bytes */ + + + if( pReadr->iReadOff>=pReadr->iEof ){ + IncrMerger *pIncr = pReadr->pIncr; + int bEof = 1; + if( pIncr ){ + rc = vdbeIncrSwap(pIncr); + if( rc==SQLITE_OK && pIncr->bEof==0 ){ + rc = vdbePmaReaderSeek( + pIncr->pTask, pReadr, &pIncr->aFile[0], pIncr->iStartOff + ); + bEof = 0; + } + } + + if( bEof ){ + /* This is an EOF condition */ + vdbePmaReaderClear(pReadr); + testcase( rc!=SQLITE_OK ); + return rc; + } + } + + if( rc==SQLITE_OK ){ + rc = vdbePmaReadVarint(pReadr, &nRec); + } + if( rc==SQLITE_OK ){ + pReadr->nKey = (int)nRec; + rc = vdbePmaReadBlob(pReadr, (int)nRec, &pReadr->aKey); + testcase( rc!=SQLITE_OK ); + } + + return rc; +} + +/* +** Initialize PmaReader pReadr to scan through the PMA stored in file pFile +** starting at offset iStart and ending at offset iEof-1. This function +** leaves the PmaReader pointing to the first key in the PMA (or EOF if the +** PMA is empty). +** +** If the pnByte parameter is NULL, then it is assumed that the file +** contains a single PMA, and that that PMA omits the initial length varint. +*/ +static int vdbePmaReaderInit( + SortSubtask *pTask, /* Task context */ + SorterFile *pFile, /* Sorter file to read from */ + i64 iStart, /* Start offset in pFile */ + PmaReader *pReadr, /* PmaReader to populate */ + i64 *pnByte /* IN/OUT: Increment this value by PMA size */ +){ + int rc; + + assert( pFile->iEof>iStart ); + assert( pReadr->aAlloc==0 && pReadr->nAlloc==0 ); + assert( pReadr->aBuffer==0 ); + assert( pReadr->aMap==0 ); + + rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); + if( rc==SQLITE_OK ){ + u64 nByte = 0; /* Size of PMA in bytes */ + rc = vdbePmaReadVarint(pReadr, &nByte); + pReadr->iEof = pReadr->iReadOff + nByte; + *pnByte += nByte; + } + + if( rc==SQLITE_OK ){ + rc = vdbePmaReaderNext(pReadr); + } + return rc; +} + +/* +** A version of vdbeSorterCompare() that assumes that it has already been +** determined that the first field of key1 is equal to the first field of +** key2. +*/ +static int vdbeSorterCompareTail( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + UnpackedRecord *r2 = pTask->pUnpacked; + if( *pbKey2Cached==0 ){ + sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + *pbKey2Cached = 1; + } + return sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, r2, 1); +} + +/* +** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, +** size nKey2 bytes). Use (pTask->pKeyInfo) for the collation sequences +** used by the comparison. Return the result of the comparison. +** +** If IN/OUT parameter *pbKey2Cached is true when this function is called, +** it is assumed that (pTask->pUnpacked) contains the unpacked version +** of key2. If it is false, (pTask->pUnpacked) is populated with the unpacked +** version of key2 and *pbKey2Cached set to true before returning. +** +** If an OOM error is encountered, (pTask->pUnpacked->error_rc) is set +** to SQLITE_NOMEM. +*/ +static int vdbeSorterCompare( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + UnpackedRecord *r2 = pTask->pUnpacked; + if( !*pbKey2Cached ){ + sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); + *pbKey2Cached = 1; + } + return sqlite3VdbeRecordCompare(nKey1, pKey1, r2); +} + +/* +** A specially optimized version of vdbeSorterCompare() that assumes that +** the first field of each key is a TEXT value and that the collation +** sequence to compare them with is BINARY. +*/ +static int vdbeSorterCompareText( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + const u8 * const p1 = (const u8 * const)pKey1; + const u8 * const p2 = (const u8 * const)pKey2; + const u8 * const v1 = &p1[ p1[0] ]; /* Pointer to value 1 */ + const u8 * const v2 = &p2[ p2[0] ]; /* Pointer to value 2 */ + + int n1; + int n2; + int res; + + getVarint32NR(&p1[1], n1); + getVarint32NR(&p2[1], n2); + res = memcmp(v1, v2, (MIN(n1, n2) - 13)/2); + if( res==0 ){ + res = n1 - n2; + } + + if( res==0 ){ + if( pTask->pSorter->pKeyInfo->nKeyField>1 ){ + res = vdbeSorterCompareTail( + pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 + ); + } + }else{ + assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); + if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ + res = res * -1; + } + } + + return res; +} + +/* +** A specially optimized version of vdbeSorterCompare() that assumes that +** the first field of each key is an INTEGER value. +*/ +static int vdbeSorterCompareInt( + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ + int *pbKey2Cached, /* True if pTask->pUnpacked is pKey2 */ + const void *pKey1, int nKey1, /* Left side of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ +){ + const u8 * const p1 = (const u8 * const)pKey1; + const u8 * const p2 = (const u8 * const)pKey2; + const int s1 = p1[1]; /* Left hand serial type */ + const int s2 = p2[1]; /* Right hand serial type */ + const u8 * const v1 = &p1[ p1[0] ]; /* Pointer to value 1 */ + const u8 * const v2 = &p2[ p2[0] ]; /* Pointer to value 2 */ + int res; /* Return value */ + + assert( (s1>0 && s1<7) || s1==8 || s1==9 ); + assert( (s2>0 && s2<7) || s2==8 || s2==9 ); + + if( s1==s2 ){ + /* The two values have the same sign. Compare using memcmp(). */ + static const u8 aLen[] = {0, 1, 2, 3, 4, 6, 8, 0, 0, 0 }; + const u8 n = aLen[s1]; + int i; + res = 0; + for(i=0; i7 && s2>7 ){ + res = s1 - s2; + }else{ + if( s2>7 ){ + res = +1; + }else if( s1>7 ){ + res = -1; + }else{ + res = s1 - s2; + } + assert( res!=0 ); + + if( res>0 ){ + if( *v1 & 0x80 ) res = -1; + }else{ + if( *v2 & 0x80 ) res = +1; + } + } + + if( res==0 ){ + if( pTask->pSorter->pKeyInfo->nKeyField>1 ){ + res = vdbeSorterCompareTail( + pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 + ); + } + }else if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ + assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); + res = res * -1; + } + + return res; +} + +/* +** Initialize the temporary index cursor just opened as a sorter cursor. +** +** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nKeyField) +** to determine the number of fields that should be compared from the +** records being sorted. However, if the value passed as argument nField +** is non-zero and the sorter is able to guarantee a stable sort, nField +** is used instead. This is used when sorting records for a CREATE INDEX +** statement. In this case, keys are always delivered to the sorter in +** order of the primary key, which happens to be make up the final part +** of the records being sorted. So if the sort is stable, there is never +** any reason to compare PK fields and they can be ignored for a small +** performance boost. +** +** The sorter can guarantee a stable sort when running in single-threaded +** mode, but not in multi-threaded mode. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterInit( + sqlite3 *db, /* Database connection (for malloc()) */ + int nField, /* Number of key fields in each record */ + VdbeCursor *pCsr /* Cursor that holds the new sorter */ +){ + int pgsz; /* Page size of main database */ + int i; /* Used to iterate through aTask[] */ + VdbeSorter *pSorter; /* The new sorter */ + KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ + int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ + int sz; /* Size of pSorter in bytes */ + int rc = SQLITE_OK; +#if SQLITE_MAX_WORKER_THREADS==0 +# define nWorker 0 +#else + int nWorker; +#endif + + /* Initialize the upper limit on the number of worker threads */ +#if SQLITE_MAX_WORKER_THREADS>0 + if( sqlite3TempInMemory(db) || sqlite3GlobalConfig.bCoreMutex==0 ){ + nWorker = 0; + }else{ + nWorker = db->aLimit[SQLITE_LIMIT_WORKER_THREADS]; + } +#endif + + /* Do not allow the total number of threads (main thread + all workers) + ** to exceed the maximum merge count */ +#if SQLITE_MAX_WORKER_THREADS>=SORTER_MAX_MERGE_COUNT + if( nWorker>=SORTER_MAX_MERGE_COUNT ){ + nWorker = SORTER_MAX_MERGE_COUNT-1; + } +#endif + + assert( pCsr->pKeyInfo && pCsr->pBtx==0 ); + assert( pCsr->eCurType==CURTYPE_SORTER ); + szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nKeyField-1)*sizeof(CollSeq*); + sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); + + pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); + pCsr->uc.pSorter = pSorter; + if( pSorter==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + Btree *pBt = db->aDb[0].pBt; + pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); + memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); + pKeyInfo->db = 0; + if( nField && nWorker==0 ){ + pKeyInfo->nKeyField = nField; + } + sqlite3BtreeEnter(pBt); + pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(pBt); + sqlite3BtreeLeave(pBt); + pSorter->nTask = nWorker + 1; + pSorter->iPrev = (u8)(nWorker - 1); + pSorter->bUseThreads = (pSorter->nTask>1); + pSorter->db = db; + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + pTask->pSorter = pSorter; + } + + if( !sqlite3TempInMemory(db) ){ + i64 mxCache; /* Cache size in bytes*/ + u32 szPma = sqlite3GlobalConfig.szPma; + pSorter->mnPmaSize = szPma * pgsz; + + mxCache = db->aDb[0].pSchema->cache_size; + if( mxCache<0 ){ + /* A negative cache-size value C indicates that the cache is abs(C) + ** KiB in size. */ + mxCache = mxCache * -1024; + }else{ + mxCache = mxCache * pgsz; + } + mxCache = MIN(mxCache, SQLITE_MAX_PMASZ); + pSorter->mxPmaSize = MAX(pSorter->mnPmaSize, (int)mxCache); + + /* Avoid large memory allocations if the application has requested + ** SQLITE_CONFIG_SMALL_MALLOC. */ + if( sqlite3GlobalConfig.bSmallMalloc==0 ){ + assert( pSorter->iMemory==0 ); + pSorter->nMemory = pgsz; + pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); + if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM_BKPT; + } + } + + if( pKeyInfo->nAllField<13 + && (pKeyInfo->aColl[0]==0 || pKeyInfo->aColl[0]==db->pDfltColl) + && (pKeyInfo->aSortFlags[0] & KEYINFO_ORDER_BIGNULL)==0 + ){ + pSorter->typeMask = SORTER_TYPE_INTEGER | SORTER_TYPE_TEXT; + } + } + + return rc; +} +#undef nWorker /* Defined at the top of this function */ + +/* +** Free the list of sorted records starting at pRecord. +*/ +static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ + SorterRecord *p; + SorterRecord *pNext; + for(p=pRecord; p; p=pNext){ + pNext = p->u.pNext; + sqlite3DbFree(db, p); + } +} + +/* +** Free all resources owned by the object indicated by argument pTask. All +** fields of *pTask are zeroed before returning. +*/ +static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ + sqlite3DbFree(db, pTask->pUnpacked); +#if SQLITE_MAX_WORKER_THREADS>0 + /* pTask->list.aMemory can only be non-zero if it was handed memory + ** from the main thread. That only occurs SQLITE_MAX_WORKER_THREADS>0 */ + if( pTask->list.aMemory ){ + sqlite3_free(pTask->list.aMemory); + }else +#endif + { + assert( pTask->list.aMemory==0 ); + vdbeSorterRecordFree(0, pTask->list.pList); + } + if( pTask->file.pFd ){ + sqlite3OsCloseFree(pTask->file.pFd); + } + if( pTask->file2.pFd ){ + sqlite3OsCloseFree(pTask->file2.pFd); + } + memset(pTask, 0, sizeof(SortSubtask)); +} + +#ifdef SQLITE_DEBUG_SORTER_THREADS +static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); + fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterRewindDebug(const char *zEvent){ + i64 t = 0; + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + if( ALWAYS(pVfs) ) sqlite3OsCurrentTimeInt64(pVfs, &t); + fprintf(stderr, "%lld:X %s\n", t, zEvent); +} +static void vdbeSorterPopulateDebug( + SortSubtask *pTask, + const char *zEvent +){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); + fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterBlockDebug( + SortSubtask *pTask, + int bBlocked, + const char *zEvent +){ + if( bBlocked ){ + i64 t; + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); + fprintf(stderr, "%lld:main %s\n", t, zEvent); + } +} +#else +# define vdbeSorterWorkDebug(x,y) +# define vdbeSorterRewindDebug(y) +# define vdbeSorterPopulateDebug(x,y) +# define vdbeSorterBlockDebug(x,y,z) +#endif + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** Join thread pTask->thread. +*/ +static int vdbeSorterJoinThread(SortSubtask *pTask){ + int rc = SQLITE_OK; + if( pTask->pThread ){ +#ifdef SQLITE_DEBUG_SORTER_THREADS + int bDone = pTask->bDone; +#endif + void *pRet = SQLITE_INT_TO_PTR(SQLITE_ERROR); + vdbeSorterBlockDebug(pTask, !bDone, "enter"); + (void)sqlite3ThreadJoin(pTask->pThread, &pRet); + vdbeSorterBlockDebug(pTask, !bDone, "exit"); + rc = SQLITE_PTR_TO_INT(pRet); + assert( pTask->bDone==1 ); + pTask->bDone = 0; + pTask->pThread = 0; + } + return rc; +} + +/* +** Launch a background thread to run xTask(pIn). +*/ +static int vdbeSorterCreateThread( + SortSubtask *pTask, /* Thread will use this task object */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + assert( pTask->pThread==0 && pTask->bDone==0 ); + return sqlite3ThreadCreate(&pTask->pThread, xTask, pIn); +} + +/* +** Join all outstanding threads launched by SorterWrite() to create +** level-0 PMAs. +*/ +static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ + int rc = rcin; + int i; + + /* This function is always called by the main user thread. + ** + ** If this function is being called after SorterRewind() has been called, + ** it is possible that thread pSorter->aTask[pSorter->nTask-1].pThread + ** is currently attempt to join one of the other threads. To avoid a race + ** condition where this thread also attempts to join the same object, join + ** thread pSorter->aTask[pSorter->nTask-1].pThread first. */ + for(i=pSorter->nTask-1; i>=0; i--){ + SortSubtask *pTask = &pSorter->aTask[i]; + int rc2 = vdbeSorterJoinThread(pTask); + if( rc==SQLITE_OK ) rc = rc2; + } + return rc; +} +#else +# define vdbeSorterJoinAll(x,rcin) (rcin) +# define vdbeSorterJoinThread(pTask) SQLITE_OK +#endif + +/* +** Allocate a new MergeEngine object capable of handling up to +** nReader PmaReader inputs. +** +** nReader is automatically rounded up to the next power of two. +** nReader may not exceed SORTER_MAX_MERGE_COUNT even after rounding up. +*/ +static MergeEngine *vdbeMergeEngineNew(int nReader){ + int N = 2; /* Smallest power of two >= nReader */ + int nByte; /* Total bytes of space to allocate */ + MergeEngine *pNew; /* Pointer to allocated object to return */ + + assert( nReader<=SORTER_MAX_MERGE_COUNT ); + + while( NnTree = N; + pNew->pTask = 0; + pNew->aReadr = (PmaReader*)&pNew[1]; + pNew->aTree = (int*)&pNew->aReadr[N]; + } + return pNew; +} + +/* +** Free the MergeEngine object passed as the only argument. +*/ +static void vdbeMergeEngineFree(MergeEngine *pMerger){ + int i; + if( pMerger ){ + for(i=0; inTree; i++){ + vdbePmaReaderClear(&pMerger->aReadr[i]); + } + } + sqlite3_free(pMerger); +} + +/* +** Free all resources associated with the IncrMerger object indicated by +** the first argument. +*/ +static void vdbeIncrFree(IncrMerger *pIncr){ + if( pIncr ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + vdbeSorterJoinThread(pIncr->pTask); + if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); + if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); + } +#endif + vdbeMergeEngineFree(pIncr->pMerger); + sqlite3_free(pIncr); + } +} + +/* +** Reset a sorting cursor back to its original empty state. +*/ +SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ + int i; + (void)vdbeSorterJoinAll(pSorter, SQLITE_OK); + assert( pSorter->bUseThreads || pSorter->pReader==0 ); +#if SQLITE_MAX_WORKER_THREADS>0 + if( pSorter->pReader ){ + vdbePmaReaderClear(pSorter->pReader); + sqlite3DbFree(db, pSorter->pReader); + pSorter->pReader = 0; + } +#endif + vdbeMergeEngineFree(pSorter->pMerger); + pSorter->pMerger = 0; + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + vdbeSortSubtaskCleanup(db, pTask); + pTask->pSorter = pSorter; + } + if( pSorter->list.aMemory==0 ){ + vdbeSorterRecordFree(0, pSorter->list.pList); + } + pSorter->list.pList = 0; + pSorter->list.szPMA = 0; + pSorter->bUsePMA = 0; + pSorter->iMemory = 0; + pSorter->mxKeysize = 0; + sqlite3DbFree(db, pSorter->pUnpacked); + pSorter->pUnpacked = 0; +} + +/* +** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. +*/ +SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ + VdbeSorter *pSorter; + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; + if( pSorter ){ + sqlite3VdbeSorterReset(db, pSorter); + sqlite3_free(pSorter->list.aMemory); + sqlite3DbFree(db, pSorter); + pCsr->uc.pSorter = 0; + } +} + +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** The first argument is a file-handle open on a temporary file. The file +** is guaranteed to be nByte bytes or smaller in size. This function +** attempts to extend the file to nByte bytes in size and to ensure that +** the VFS has memory mapped it. +** +** Whether or not the file does end up memory mapped of course depends on +** the specific VFS implementation. +*/ +static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ + if( nByte<=(i64)(db->nMaxSorterMmap) && pFd->pMethods->iVersion>=3 ){ + void *p = 0; + int chunksize = 4*1024; + sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_CHUNK_SIZE, &chunksize); + sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_SIZE_HINT, &nByte); + sqlite3OsFetch(pFd, 0, (int)nByte, &p); + sqlite3OsUnfetch(pFd, 0, p); + } +} +#else +# define vdbeSorterExtendFile(x,y,z) +#endif + +/* +** Allocate space for a file-handle and open a temporary file. If successful, +** set *ppFd to point to the malloc'd file-handle and return SQLITE_OK. +** Otherwise, set *ppFd to 0 and return an SQLite error code. +*/ +static int vdbeSorterOpenTempFile( + sqlite3 *db, /* Database handle doing sort */ + i64 nExtend, /* Attempt to extend file to this size */ + sqlite3_file **ppFd +){ + int rc; + if( sqlite3FaultSim(202) ) return SQLITE_IOERR_ACCESS; + rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFd, + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &rc + ); + if( rc==SQLITE_OK ){ + i64 max = SQLITE_MAX_MMAP_SIZE; + sqlite3OsFileControlHint(*ppFd, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + if( nExtend>0 ){ + vdbeSorterExtendFile(db, *ppFd, nExtend); + } + } + return rc; +} + +/* +** If it has not already been allocated, allocate the UnpackedRecord +** structure at pTask->pUnpacked. Return SQLITE_OK if successful (or +** if no allocation was required), or SQLITE_NOMEM otherwise. +*/ +static int vdbeSortAllocUnpacked(SortSubtask *pTask){ + if( pTask->pUnpacked==0 ){ + pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pTask->pSorter->pKeyInfo); + if( pTask->pUnpacked==0 ) return SQLITE_NOMEM_BKPT; + pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nKeyField; + pTask->pUnpacked->errCode = 0; + } + return SQLITE_OK; +} + + +/* +** Merge the two sorted lists p1 and p2 into a single list. +*/ +static SorterRecord *vdbeSorterMerge( + SortSubtask *pTask, /* Calling thread context */ + SorterRecord *p1, /* First list to merge */ + SorterRecord *p2 /* Second list to merge */ +){ + SorterRecord *pFinal = 0; + SorterRecord **pp = &pFinal; + int bCached = 0; + + assert( p1!=0 && p2!=0 ); + for(;;){ + int res; + res = pTask->xCompare( + pTask, &bCached, SRVAL(p1), p1->nVal, SRVAL(p2), p2->nVal + ); + + if( res<=0 ){ + *pp = p1; + pp = &p1->u.pNext; + p1 = p1->u.pNext; + if( p1==0 ){ + *pp = p2; + break; + } + }else{ + *pp = p2; + pp = &p2->u.pNext; + p2 = p2->u.pNext; + bCached = 0; + if( p2==0 ){ + *pp = p1; + break; + } + } + } + return pFinal; +} + +/* +** Return the SorterCompare function to compare values collected by the +** sorter object passed as the only argument. +*/ +static SorterCompare vdbeSorterGetCompare(VdbeSorter *p){ + if( p->typeMask==SORTER_TYPE_INTEGER ){ + return vdbeSorterCompareInt; + }else if( p->typeMask==SORTER_TYPE_TEXT ){ + return vdbeSorterCompareText; + } + return vdbeSorterCompare; +} + +/* +** Sort the linked list of records headed at pTask->pList. Return +** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if +** an error occurs. +*/ +static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ + int i; + SorterRecord *p; + int rc; + SorterRecord *aSlot[64]; + + rc = vdbeSortAllocUnpacked(pTask); + if( rc!=SQLITE_OK ) return rc; + + p = pList->pList; + pTask->xCompare = vdbeSorterGetCompare(pTask->pSorter); + memset(aSlot, 0, sizeof(aSlot)); + + while( p ){ + SorterRecord *pNext; + if( pList->aMemory ){ + if( (u8*)p==pList->aMemory ){ + pNext = 0; + }else{ + assert( p->u.iNextaMemory) ); + pNext = (SorterRecord*)&pList->aMemory[p->u.iNext]; + } + }else{ + pNext = p->u.pNext; + } + + p->u.pNext = 0; + for(i=0; aSlot[i]; i++){ + p = vdbeSorterMerge(pTask, p, aSlot[i]); + aSlot[i] = 0; + } + aSlot[i] = p; + p = pNext; + } + + p = 0; + for(i=0; ipList = p; + + assert( pTask->pUnpacked->errCode==SQLITE_OK + || pTask->pUnpacked->errCode==SQLITE_NOMEM + ); + return pTask->pUnpacked->errCode; +} + +/* +** Initialize a PMA-writer object. +*/ +static void vdbePmaWriterInit( + sqlite3_file *pFd, /* File handle to write to */ + PmaWriter *p, /* Object to populate */ + int nBuf, /* Buffer size */ + i64 iStart /* Offset of pFd to begin writing at */ +){ + memset(p, 0, sizeof(PmaWriter)); + p->aBuffer = (u8*)sqlite3Malloc(nBuf); + if( !p->aBuffer ){ + p->eFWErr = SQLITE_NOMEM_BKPT; + }else{ + p->iBufEnd = p->iBufStart = (iStart % nBuf); + p->iWriteOff = iStart - p->iBufStart; + p->nBuffer = nBuf; + p->pFd = pFd; + } +} + +/* +** Write nData bytes of data to the PMA. Return SQLITE_OK +** if successful, or an SQLite error code if an error occurs. +*/ +static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ + int nRem = nData; + while( nRem>0 && p->eFWErr==0 ){ + int nCopy = nRem; + if( nCopy>(p->nBuffer - p->iBufEnd) ){ + nCopy = p->nBuffer - p->iBufEnd; + } + + memcpy(&p->aBuffer[p->iBufEnd], &pData[nData-nRem], nCopy); + p->iBufEnd += nCopy; + if( p->iBufEnd==p->nBuffer ){ + p->eFWErr = sqlite3OsWrite(p->pFd, + &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, + p->iWriteOff + p->iBufStart + ); + p->iBufStart = p->iBufEnd = 0; + p->iWriteOff += p->nBuffer; + } + assert( p->iBufEndnBuffer ); + + nRem -= nCopy; + } +} + +/* +** Flush any buffered data to disk and clean up the PMA-writer object. +** The results of using the PMA-writer after this call are undefined. +** Return SQLITE_OK if flushing the buffered data succeeds or is not +** required. Otherwise, return an SQLite error code. +** +** Before returning, set *piEof to the offset immediately following the +** last byte written to the file. +*/ +static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ + int rc; + if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ + p->eFWErr = sqlite3OsWrite(p->pFd, + &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, + p->iWriteOff + p->iBufStart + ); + } + *piEof = (p->iWriteOff + p->iBufEnd); + sqlite3_free(p->aBuffer); + rc = p->eFWErr; + memset(p, 0, sizeof(PmaWriter)); + return rc; +} + +/* +** Write value iVal encoded as a varint to the PMA. Return +** SQLITE_OK if successful, or an SQLite error code if an error occurs. +*/ +static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ + int nByte; + u8 aByte[10]; + nByte = sqlite3PutVarint(aByte, iVal); + vdbePmaWriteBlob(p, aByte, nByte); +} + +/* +** Write the current contents of in-memory linked-list pList to a level-0 +** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if +** successful, or an SQLite error code otherwise. +** +** The format of a PMA is: +** +** * A varint. This varint contains the total number of bytes of content +** in the PMA (not including the varint itself). +** +** * One or more records packed end-to-end in order of ascending keys. +** Each record consists of a varint followed by a blob of data (the +** key). The varint is the number of bytes in the blob of data. +*/ +static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ + sqlite3 *db = pTask->pSorter->db; + int rc = SQLITE_OK; /* Return code */ + PmaWriter writer; /* Object used to write to the file */ + +#ifdef SQLITE_DEBUG + /* Set iSz to the expected size of file pTask->file after writing the PMA. + ** This is used by an assert() statement at the end of this function. */ + i64 iSz = pList->szPMA + sqlite3VarintLen(pList->szPMA) + pTask->file.iEof; +#endif + + vdbeSorterWorkDebug(pTask, "enter"); + memset(&writer, 0, sizeof(PmaWriter)); + assert( pList->szPMA>0 ); + + /* If the first temporary PMA file has not been opened, open it now. */ + if( pTask->file.pFd==0 ){ + rc = vdbeSorterOpenTempFile(db, 0, &pTask->file.pFd); + assert( rc!=SQLITE_OK || pTask->file.pFd ); + assert( pTask->file.iEof==0 ); + assert( pTask->nPMA==0 ); + } + + /* Try to get the file to memory map */ + if( rc==SQLITE_OK ){ + vdbeSorterExtendFile(db, pTask->file.pFd, pTask->file.iEof+pList->szPMA+9); + } + + /* Sort the list */ + if( rc==SQLITE_OK ){ + rc = vdbeSorterSort(pTask, pList); + } + + if( rc==SQLITE_OK ){ + SorterRecord *p; + SorterRecord *pNext = 0; + + vdbePmaWriterInit(pTask->file.pFd, &writer, pTask->pSorter->pgsz, + pTask->file.iEof); + pTask->nPMA++; + vdbePmaWriteVarint(&writer, pList->szPMA); + for(p=pList->pList; p; p=pNext){ + pNext = p->u.pNext; + vdbePmaWriteVarint(&writer, p->nVal); + vdbePmaWriteBlob(&writer, SRVAL(p), p->nVal); + if( pList->aMemory==0 ) sqlite3_free(p); + } + pList->pList = p; + rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof); + } + + vdbeSorterWorkDebug(pTask, "exit"); + assert( rc!=SQLITE_OK || pList->pList==0 ); + assert( rc!=SQLITE_OK || pTask->file.iEof==iSz ); + return rc; +} + +/* +** Advance the MergeEngine to its next entry. +** Set *pbEof to true there is no next entry because +** the MergeEngine has reached the end of all its inputs. +** +** Return SQLITE_OK if successful or an error code if an error occurs. +*/ +static int vdbeMergeEngineStep( + MergeEngine *pMerger, /* The merge engine to advance to the next row */ + int *pbEof /* Set TRUE at EOF. Set false for more content */ +){ + int rc; + int iPrev = pMerger->aTree[1];/* Index of PmaReader to advance */ + SortSubtask *pTask = pMerger->pTask; + + /* Advance the current PmaReader */ + rc = vdbePmaReaderNext(&pMerger->aReadr[iPrev]); + + /* Update contents of aTree[] */ + if( rc==SQLITE_OK ){ + int i; /* Index of aTree[] to recalculate */ + PmaReader *pReadr1; /* First PmaReader to compare */ + PmaReader *pReadr2; /* Second PmaReader to compare */ + int bCached = 0; + + /* Find the first two PmaReaders to compare. The one that was just + ** advanced (iPrev) and the one next to it in the array. */ + pReadr1 = &pMerger->aReadr[(iPrev & 0xFFFE)]; + pReadr2 = &pMerger->aReadr[(iPrev | 0x0001)]; + + for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ + /* Compare pReadr1 and pReadr2. Store the result in variable iRes. */ + int iRes; + if( pReadr1->pFd==0 ){ + iRes = +1; + }else if( pReadr2->pFd==0 ){ + iRes = -1; + }else{ + iRes = pTask->xCompare(pTask, &bCached, + pReadr1->aKey, pReadr1->nKey, pReadr2->aKey, pReadr2->nKey + ); + } + + /* If pReadr1 contained the smaller value, set aTree[i] to its index. + ** Then set pReadr2 to the next PmaReader to compare to pReadr1. In this + ** case there is no cache of pReadr2 in pTask->pUnpacked, so set + ** pKey2 to point to the record belonging to pReadr2. + ** + ** Alternatively, if pReadr2 contains the smaller of the two values, + ** set aTree[i] to its index and update pReadr1. If vdbeSorterCompare() + ** was actually called above, then pTask->pUnpacked now contains + ** a value equivalent to pReadr2. So set pKey2 to NULL to prevent + ** vdbeSorterCompare() from decoding pReadr2 again. + ** + ** If the two values were equal, then the value from the oldest + ** PMA should be considered smaller. The VdbeSorter.aReadr[] array + ** is sorted from oldest to newest, so pReadr1 contains older values + ** than pReadr2 iff (pReadr1aTree[i] = (int)(pReadr1 - pMerger->aReadr); + pReadr2 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; + bCached = 0; + }else{ + if( pReadr1->pFd ) bCached = 0; + pMerger->aTree[i] = (int)(pReadr2 - pMerger->aReadr); + pReadr1 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; + } + } + *pbEof = (pMerger->aReadr[pMerger->aTree[1]].pFd==0); + } + + return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for background threads that write level-0 PMAs. +*/ +static void *vdbeSorterFlushThread(void *pCtx){ + SortSubtask *pTask = (SortSubtask*)pCtx; + int rc; /* Return code */ + assert( pTask->bDone==0 ); + rc = vdbeSorterListToPMA(pTask, &pTask->list); + pTask->bDone = 1; + return SQLITE_INT_TO_PTR(rc); +} +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ + +/* +** Flush the current contents of VdbeSorter.list to a new PMA, possibly +** using a background thread. +*/ +static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ +#if SQLITE_MAX_WORKER_THREADS==0 + pSorter->bUsePMA = 1; + return vdbeSorterListToPMA(&pSorter->aTask[0], &pSorter->list); +#else + int rc = SQLITE_OK; + int i; + SortSubtask *pTask = 0; /* Thread context used to create new PMA */ + int nWorker = (pSorter->nTask-1); + + /* Set the flag to indicate that at least one PMA has been written. + ** Or will be, anyhow. */ + pSorter->bUsePMA = 1; + + /* Select a sub-task to sort and flush the current list of in-memory + ** records to disk. If the sorter is running in multi-threaded mode, + ** round-robin between the first (pSorter->nTask-1) tasks. Except, if + ** the background thread from a sub-tasks previous turn is still running, + ** skip it. If the first (pSorter->nTask-1) sub-tasks are all still busy, + ** fall back to using the final sub-task. The first (pSorter->nTask-1) + ** sub-tasks are prefered as they use background threads - the final + ** sub-task uses the main thread. */ + for(i=0; iiPrev + i + 1) % nWorker; + pTask = &pSorter->aTask[iTest]; + if( pTask->bDone ){ + rc = vdbeSorterJoinThread(pTask); + } + if( rc!=SQLITE_OK || pTask->pThread==0 ) break; + } + + if( rc==SQLITE_OK ){ + if( i==nWorker ){ + /* Use the foreground thread for this operation */ + rc = vdbeSorterListToPMA(&pSorter->aTask[nWorker], &pSorter->list); + }else{ + /* Launch a background thread for this operation */ + u8 *aMem; + void *pCtx; + + assert( pTask!=0 ); + assert( pTask->pThread==0 && pTask->bDone==0 ); + assert( pTask->list.pList==0 ); + assert( pTask->list.aMemory==0 || pSorter->list.aMemory!=0 ); + + aMem = pTask->list.aMemory; + pCtx = (void*)pTask; + pSorter->iPrev = (u8)(pTask - pSorter->aTask); + pTask->list = pSorter->list; + pSorter->list.pList = 0; + pSorter->list.szPMA = 0; + if( aMem ){ + pSorter->list.aMemory = aMem; + pSorter->nMemory = sqlite3MallocSize(aMem); + }else if( pSorter->list.aMemory ){ + pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory); + if( !pSorter->list.aMemory ) return SQLITE_NOMEM_BKPT; + } + + rc = vdbeSorterCreateThread(pTask, vdbeSorterFlushThread, pCtx); + } + } + + return rc; +#endif /* SQLITE_MAX_WORKER_THREADS!=0 */ +} + +/* +** Add a record to the sorter. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterWrite( + const VdbeCursor *pCsr, /* Sorter cursor */ + Mem *pVal /* Memory cell containing record */ +){ + VdbeSorter *pSorter; + int rc = SQLITE_OK; /* Return Code */ + SorterRecord *pNew; /* New list element */ + int bFlush; /* True to flush contents of memory to PMA */ + int nReq; /* Bytes of memory required */ + int nPMA; /* Bytes of PMA space required */ + int t; /* serial type of first record field */ + + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; + getVarint32NR((const u8*)&pVal->z[1], t); + if( t>0 && t<10 && t!=7 ){ + pSorter->typeMask &= SORTER_TYPE_INTEGER; + }else if( t>10 && (t & 0x01) ){ + pSorter->typeMask &= SORTER_TYPE_TEXT; + }else{ + pSorter->typeMask = 0; + } + + assert( pSorter ); + + /* Figure out whether or not the current contents of memory should be + ** flushed to a PMA before continuing. If so, do so. + ** + ** If using the single large allocation mode (pSorter->aMemory!=0), then + ** flush the contents of memory to a new PMA if (a) at least one value is + ** already in memory and (b) the new value will not fit in memory. + ** + ** Or, if using separate allocations for each record, flush the contents + ** of memory to a PMA if either of the following are true: + ** + ** * The total memory allocated for the in-memory list is greater + ** than (page-size * cache-size), or + ** + ** * The total memory allocated for the in-memory list is greater + ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. + */ + nReq = pVal->n + sizeof(SorterRecord); + nPMA = pVal->n + sqlite3VarintLen(pVal->n); + if( pSorter->mxPmaSize ){ + if( pSorter->list.aMemory ){ + bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; + }else{ + bFlush = ( + (pSorter->list.szPMA > pSorter->mxPmaSize) + || (pSorter->list.szPMA > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + ); + } + if( bFlush ){ + rc = vdbeSorterFlushPMA(pSorter); + pSorter->list.szPMA = 0; + pSorter->iMemory = 0; + assert( rc!=SQLITE_OK || pSorter->list.pList==0 ); + } + } + + pSorter->list.szPMA += nPMA; + if( nPMA>pSorter->mxKeysize ){ + pSorter->mxKeysize = nPMA; + } + + if( pSorter->list.aMemory ){ + int nMin = pSorter->iMemory + nReq; + + if( nMin>pSorter->nMemory ){ + u8 *aNew; + sqlite3_int64 nNew = 2 * (sqlite3_int64)pSorter->nMemory; + int iListOff = -1; + if( pSorter->list.pList ){ + iListOff = (u8*)pSorter->list.pList - pSorter->list.aMemory; + } + while( nNew < nMin ) nNew = nNew*2; + if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; + if( nNew < nMin ) nNew = nMin; + aNew = sqlite3Realloc(pSorter->list.aMemory, nNew); + if( !aNew ) return SQLITE_NOMEM_BKPT; + if( iListOff>=0 ){ + pSorter->list.pList = (SorterRecord*)&aNew[iListOff]; + } + pSorter->list.aMemory = aNew; + pSorter->nMemory = nNew; + } + + pNew = (SorterRecord*)&pSorter->list.aMemory[pSorter->iMemory]; + pSorter->iMemory += ROUND8(nReq); + if( pSorter->list.pList ){ + pNew->u.iNext = (int)((u8*)(pSorter->list.pList) - pSorter->list.aMemory); + } + }else{ + pNew = (SorterRecord *)sqlite3Malloc(nReq); + if( pNew==0 ){ + return SQLITE_NOMEM_BKPT; + } + pNew->u.pNext = pSorter->list.pList; + } + + memcpy(SRVAL(pNew), pVal->z, pVal->n); + pNew->nVal = pVal->n; + pSorter->list.pList = pNew; + + return rc; +} + +/* +** Read keys from pIncr->pMerger and populate pIncr->aFile[1]. The format +** of the data stored in aFile[1] is the same as that used by regular PMAs, +** except that the number-of-bytes varint is omitted from the start. +*/ +static int vdbeIncrPopulate(IncrMerger *pIncr){ + int rc = SQLITE_OK; + int rc2; + i64 iStart = pIncr->iStartOff; + SorterFile *pOut = &pIncr->aFile[1]; + SortSubtask *pTask = pIncr->pTask; + MergeEngine *pMerger = pIncr->pMerger; + PmaWriter writer; + assert( pIncr->bEof==0 ); + + vdbeSorterPopulateDebug(pTask, "enter"); + + vdbePmaWriterInit(pOut->pFd, &writer, pTask->pSorter->pgsz, iStart); + while( rc==SQLITE_OK ){ + int dummy; + PmaReader *pReader = &pMerger->aReadr[ pMerger->aTree[1] ]; + int nKey = pReader->nKey; + i64 iEof = writer.iWriteOff + writer.iBufEnd; + + /* Check if the output file is full or if the input has been exhausted. + ** In either case exit the loop. */ + if( pReader->pFd==0 ) break; + if( (iEof + nKey + sqlite3VarintLen(nKey))>(iStart + pIncr->mxSz) ) break; + + /* Write the next key to the output. */ + vdbePmaWriteVarint(&writer, nKey); + vdbePmaWriteBlob(&writer, pReader->aKey, nKey); + assert( pIncr->pMerger->pTask==pTask ); + rc = vdbeMergeEngineStep(pIncr->pMerger, &dummy); + } + + rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); + if( rc==SQLITE_OK ) rc = rc2; + vdbeSorterPopulateDebug(pTask, "exit"); + return rc; +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for background threads that populate aFile[1] of +** multi-threaded IncrMerger objects. +*/ +static void *vdbeIncrPopulateThread(void *pCtx){ + IncrMerger *pIncr = (IncrMerger*)pCtx; + void *pRet = SQLITE_INT_TO_PTR( vdbeIncrPopulate(pIncr) ); + pIncr->pTask->bDone = 1; + return pRet; +} + +/* +** Launch a background thread to populate aFile[1] of pIncr. +*/ +static int vdbeIncrBgPopulate(IncrMerger *pIncr){ + void *p = (void*)pIncr; + assert( pIncr->bUseThread ); + return vdbeSorterCreateThread(pIncr->pTask, vdbeIncrPopulateThread, p); +} +#endif + +/* +** This function is called when the PmaReader corresponding to pIncr has +** finished reading the contents of aFile[0]. Its purpose is to "refill" +** aFile[0] such that the PmaReader should start rereading it from the +** beginning. +** +** For single-threaded objects, this is accomplished by literally reading +** keys from pIncr->pMerger and repopulating aFile[0]. +** +** For multi-threaded objects, all that is required is to wait until the +** background thread is finished (if it is not already) and then swap +** aFile[0] and aFile[1] in place. If the contents of pMerger have not +** been exhausted, this function also launches a new background thread +** to populate the new aFile[1]. +** +** SQLITE_OK is returned on success, or an SQLite error code otherwise. +*/ +static int vdbeIncrSwap(IncrMerger *pIncr){ + int rc = SQLITE_OK; + +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + rc = vdbeSorterJoinThread(pIncr->pTask); + + if( rc==SQLITE_OK ){ + SorterFile f0 = pIncr->aFile[0]; + pIncr->aFile[0] = pIncr->aFile[1]; + pIncr->aFile[1] = f0; + } + + if( rc==SQLITE_OK ){ + if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ + pIncr->bEof = 1; + }else{ + rc = vdbeIncrBgPopulate(pIncr); + } + } + }else +#endif + { + rc = vdbeIncrPopulate(pIncr); + pIncr->aFile[0] = pIncr->aFile[1]; + if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ + pIncr->bEof = 1; + } + } + + return rc; +} + +/* +** Allocate and return a new IncrMerger object to read data from pMerger. +** +** If an OOM condition is encountered, return NULL. In this case free the +** pMerger argument before returning. +*/ +static int vdbeIncrMergerNew( + SortSubtask *pTask, /* The thread that will be using the new IncrMerger */ + MergeEngine *pMerger, /* The MergeEngine that the IncrMerger will control */ + IncrMerger **ppOut /* Write the new IncrMerger here */ +){ + int rc = SQLITE_OK; + IncrMerger *pIncr = *ppOut = (IncrMerger*) + (sqlite3FaultSim(100) ? 0 : sqlite3MallocZero(sizeof(*pIncr))); + if( pIncr ){ + pIncr->pMerger = pMerger; + pIncr->pTask = pTask; + pIncr->mxSz = MAX(pTask->pSorter->mxKeysize+9,pTask->pSorter->mxPmaSize/2); + pTask->file2.iEof += pIncr->mxSz; + }else{ + vdbeMergeEngineFree(pMerger); + rc = SQLITE_NOMEM_BKPT; + } + return rc; +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** Set the "use-threads" flag on object pIncr. +*/ +static void vdbeIncrMergerSetThreads(IncrMerger *pIncr){ + pIncr->bUseThread = 1; + pIncr->pTask->file2.iEof -= pIncr->mxSz; +} +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ + + + +/* +** Recompute pMerger->aTree[iOut] by comparing the next keys on the +** two PmaReaders that feed that entry. Neither of the PmaReaders +** are advanced. This routine merely does the comparison. +*/ +static void vdbeMergeEngineCompare( + MergeEngine *pMerger, /* Merge engine containing PmaReaders to compare */ + int iOut /* Store the result in pMerger->aTree[iOut] */ +){ + int i1; + int i2; + int iRes; + PmaReader *p1; + PmaReader *p2; + + assert( iOutnTree && iOut>0 ); + + if( iOut>=(pMerger->nTree/2) ){ + i1 = (iOut - pMerger->nTree/2) * 2; + i2 = i1 + 1; + }else{ + i1 = pMerger->aTree[iOut*2]; + i2 = pMerger->aTree[iOut*2+1]; + } + + p1 = &pMerger->aReadr[i1]; + p2 = &pMerger->aReadr[i2]; + + if( p1->pFd==0 ){ + iRes = i2; + }else if( p2->pFd==0 ){ + iRes = i1; + }else{ + SortSubtask *pTask = pMerger->pTask; + int bCached = 0; + int res; + assert( pTask->pUnpacked!=0 ); /* from vdbeSortSubtaskMain() */ + res = pTask->xCompare( + pTask, &bCached, p1->aKey, p1->nKey, p2->aKey, p2->nKey + ); + if( res<=0 ){ + iRes = i1; + }else{ + iRes = i2; + } + } + + pMerger->aTree[iOut] = iRes; +} + +/* +** Allowed values for the eMode parameter to vdbeMergeEngineInit() +** and vdbePmaReaderIncrMergeInit(). +** +** Only INCRINIT_NORMAL is valid in single-threaded builds (when +** SQLITE_MAX_WORKER_THREADS==0). The other values are only used +** when there exists one or more separate worker threads. +*/ +#define INCRINIT_NORMAL 0 +#define INCRINIT_TASK 1 +#define INCRINIT_ROOT 2 + +/* +** Forward reference required as the vdbeIncrMergeInit() and +** vdbePmaReaderIncrInit() routines are called mutually recursively when +** building a merge tree. +*/ +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); + +/* +** Initialize the MergeEngine object passed as the second argument. Once this +** function returns, the first key of merged data may be read from the +** MergeEngine object in the usual fashion. +** +** If argument eMode is INCRINIT_ROOT, then it is assumed that any IncrMerge +** objects attached to the PmaReader objects that the merger reads from have +** already been populated, but that they have not yet populated aFile[0] and +** set the PmaReader objects up to read from it. In this case all that is +** required is to call vdbePmaReaderNext() on each PmaReader to point it at +** its first key. +** +** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use +** vdbePmaReaderIncrMergeInit() to initialize each PmaReader that feeds data +** to pMerger. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int vdbeMergeEngineInit( + SortSubtask *pTask, /* Thread that will run pMerger */ + MergeEngine *pMerger, /* MergeEngine to initialize */ + int eMode /* One of the INCRINIT_XXX constants */ +){ + int rc = SQLITE_OK; /* Return code */ + int i; /* For looping over PmaReader objects */ + int nTree; /* Number of subtrees to merge */ + + /* Failure to allocate the merge would have been detected prior to + ** invoking this routine */ + assert( pMerger!=0 ); + + /* eMode is always INCRINIT_NORMAL in single-threaded mode */ + assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); + + /* Verify that the MergeEngine is assigned to a single thread */ + assert( pMerger->pTask==0 ); + pMerger->pTask = pTask; + + nTree = pMerger->nTree; + for(i=0; i0 && eMode==INCRINIT_ROOT ){ + /* PmaReaders should be normally initialized in order, as if they are + ** reading from the same temp file this makes for more linear file IO. + ** However, in the INCRINIT_ROOT case, if PmaReader aReadr[nTask-1] is + ** in use it will block the vdbePmaReaderNext() call while it uses + ** the main thread to fill its buffer. So calling PmaReaderNext() + ** on this PmaReader before any of the multi-threaded PmaReaders takes + ** better advantage of multi-processor hardware. */ + rc = vdbePmaReaderNext(&pMerger->aReadr[nTree-i-1]); + }else{ + rc = vdbePmaReaderIncrInit(&pMerger->aReadr[i], INCRINIT_NORMAL); + } + if( rc!=SQLITE_OK ) return rc; + } + + for(i=pMerger->nTree-1; i>0; i--){ + vdbeMergeEngineCompare(pMerger, i); + } + return pTask->pUnpacked->errCode; +} + +/* +** The PmaReader passed as the first argument is guaranteed to be an +** incremental-reader (pReadr->pIncr!=0). This function serves to open +** and/or initialize the temp file related fields of the IncrMerge +** object at (pReadr->pIncr). +** +** If argument eMode is set to INCRINIT_NORMAL, then all PmaReaders +** in the sub-tree headed by pReadr are also initialized. Data is then +** loaded into the buffers belonging to pReadr and it is set to point to +** the first key in its range. +** +** If argument eMode is set to INCRINIT_TASK, then pReadr is guaranteed +** to be a multi-threaded PmaReader and this function is being called in a +** background thread. In this case all PmaReaders in the sub-tree are +** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to +** pReadr is populated. However, pReadr itself is not set up to point +** to its first key. A call to vdbePmaReaderNext() is still required to do +** that. +** +** The reason this function does not call vdbePmaReaderNext() immediately +** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that it has +** to block on thread (pTask->thread) before accessing aFile[1]. But, since +** this entire function is being run by thread (pTask->thread), that will +** lead to the current background thread attempting to join itself. +** +** Finally, if argument eMode is set to INCRINIT_ROOT, it may be assumed +** that pReadr->pIncr is a multi-threaded IncrMerge objects, and that all +** child-trees have already been initialized using IncrInit(INCRINIT_TASK). +** In this case vdbePmaReaderNext() is called on all child PmaReaders and +** the current PmaReader set to point to the first key in its range. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ + int rc = SQLITE_OK; + IncrMerger *pIncr = pReadr->pIncr; + SortSubtask *pTask = pIncr->pTask; + sqlite3 *db = pTask->pSorter->db; + + /* eMode is always INCRINIT_NORMAL in single-threaded mode */ + assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); + + rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); + + /* Set up the required files for pIncr. A multi-theaded IncrMerge object + ** requires two temp files to itself, whereas a single-threaded object + ** only requires a region of pTask->file2. */ + if( rc==SQLITE_OK ){ + int mxSz = pIncr->mxSz; +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); + } + }else +#endif + /*if( !pIncr->bUseThread )*/{ + if( pTask->file2.pFd==0 ){ + assert( pTask->file2.iEof>0 ); + rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); + pTask->file2.iEof = 0; + } + if( rc==SQLITE_OK ){ + pIncr->aFile[1].pFd = pTask->file2.pFd; + pIncr->iStartOff = pTask->file2.iEof; + pTask->file2.iEof += mxSz; + } + } + } + +#if SQLITE_MAX_WORKER_THREADS>0 + if( rc==SQLITE_OK && pIncr->bUseThread ){ + /* Use the current thread to populate aFile[1], even though this + ** PmaReader is multi-threaded. If this is an INCRINIT_TASK object, + ** then this function is already running in background thread + ** pIncr->pTask->thread. + ** + ** If this is the INCRINIT_ROOT object, then it is running in the + ** main VDBE thread. But that is Ok, as that thread cannot return + ** control to the VDBE or proceed with anything useful until the + ** first results are ready from this merger object anyway. + */ + assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); + rc = vdbeIncrPopulate(pIncr); + } +#endif + + if( rc==SQLITE_OK && (SQLITE_MAX_WORKER_THREADS==0 || eMode!=INCRINIT_TASK) ){ + rc = vdbePmaReaderNext(pReadr); + } + + return rc; +} + +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for vdbePmaReaderIncrMergeInit() operations run in +** background threads. +*/ +static void *vdbePmaReaderBgIncrInit(void *pCtx){ + PmaReader *pReader = (PmaReader*)pCtx; + void *pRet = SQLITE_INT_TO_PTR( + vdbePmaReaderIncrMergeInit(pReader,INCRINIT_TASK) + ); + pReader->pIncr->pTask->bDone = 1; + return pRet; +} +#endif + +/* +** If the PmaReader passed as the first argument is not an incremental-reader +** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it invokes +** the vdbePmaReaderIncrMergeInit() function with the parameters passed to +** this routine to initialize the incremental merge. +** +** If the IncrMerger object is multi-threaded (IncrMerger.bUseThread==1), +** then a background thread is launched to call vdbePmaReaderIncrMergeInit(). +** Or, if the IncrMerger is single threaded, the same function is called +** using the current thread. +*/ +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){ + IncrMerger *pIncr = pReadr->pIncr; /* Incremental merger */ + int rc = SQLITE_OK; /* Return code */ + if( pIncr ){ +#if SQLITE_MAX_WORKER_THREADS>0 + assert( pIncr->bUseThread==0 || eMode==INCRINIT_TASK ); + if( pIncr->bUseThread ){ + void *pCtx = (void*)pReadr; + rc = vdbeSorterCreateThread(pIncr->pTask, vdbePmaReaderBgIncrInit, pCtx); + }else +#endif + { + rc = vdbePmaReaderIncrMergeInit(pReadr, eMode); + } + } + return rc; +} + +/* +** Allocate a new MergeEngine object to merge the contents of nPMA level-0 +** PMAs from pTask->file. If no error occurs, set *ppOut to point to +** the new object and return SQLITE_OK. Or, if an error does occur, set *ppOut +** to NULL and return an SQLite error code. +** +** When this function is called, *piOffset is set to the offset of the +** first PMA to read from pTask->file. Assuming no error occurs, it is +** set to the offset immediately following the last byte of the last +** PMA before returning. If an error does occur, then the final value of +** *piOffset is undefined. +*/ +static int vdbeMergeEngineLevel0( + SortSubtask *pTask, /* Sorter task to read from */ + int nPMA, /* Number of PMAs to read */ + i64 *piOffset, /* IN/OUT: Readr offset in pTask->file */ + MergeEngine **ppOut /* OUT: New merge-engine */ +){ + MergeEngine *pNew; /* Merge engine to return */ + i64 iOff = *piOffset; + int i; + int rc = SQLITE_OK; + + *ppOut = pNew = vdbeMergeEngineNew(nPMA); + if( pNew==0 ) rc = SQLITE_NOMEM_BKPT; + + for(i=0; iaReadr[i]; + rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pReadr, &nDummy); + iOff = pReadr->iEof; + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pNew); + *ppOut = 0; + } + *piOffset = iOff; + return rc; +} + +/* +** Return the depth of a tree comprising nPMA PMAs, assuming a fanout of +** SORTER_MAX_MERGE_COUNT. The returned value does not include leaf nodes. +** +** i.e. +** +** nPMA<=16 -> TreeDepth() == 0 +** nPMA<=256 -> TreeDepth() == 1 +** nPMA<=65536 -> TreeDepth() == 2 +*/ +static int vdbeSorterTreeDepth(int nPMA){ + int nDepth = 0; + i64 nDiv = SORTER_MAX_MERGE_COUNT; + while( nDiv < (i64)nPMA ){ + nDiv = nDiv * SORTER_MAX_MERGE_COUNT; + nDepth++; + } + return nDepth; +} + +/* +** pRoot is the root of an incremental merge-tree with depth nDepth (according +** to vdbeSorterTreeDepth()). pLeaf is the iSeq'th leaf to be added to the +** tree, counting from zero. This function adds pLeaf to the tree. +** +** If successful, SQLITE_OK is returned. If an error occurs, an SQLite error +** code is returned and pLeaf is freed. +*/ +static int vdbeSorterAddToTree( + SortSubtask *pTask, /* Task context */ + int nDepth, /* Depth of tree according to TreeDepth() */ + int iSeq, /* Sequence number of leaf within tree */ + MergeEngine *pRoot, /* Root of tree */ + MergeEngine *pLeaf /* Leaf to add to tree */ +){ + int rc = SQLITE_OK; + int nDiv = 1; + int i; + MergeEngine *p = pRoot; + IncrMerger *pIncr; + + rc = vdbeIncrMergerNew(pTask, pLeaf, &pIncr); + + for(i=1; iaReadr[iIter]; + + if( pReadr->pIncr==0 ){ + MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pNew==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + rc = vdbeIncrMergerNew(pTask, pNew, &pReadr->pIncr); + } + } + if( rc==SQLITE_OK ){ + p = pReadr->pIncr->pMerger; + nDiv = nDiv / SORTER_MAX_MERGE_COUNT; + } + } + + if( rc==SQLITE_OK ){ + p->aReadr[iSeq % SORTER_MAX_MERGE_COUNT].pIncr = pIncr; + }else{ + vdbeIncrFree(pIncr); + } + return rc; +} + +/* +** This function is called as part of a SorterRewind() operation on a sorter +** that has already written two or more level-0 PMAs to one or more temp +** files. It builds a tree of MergeEngine/IncrMerger/PmaReader objects that +** can be used to incrementally merge all PMAs on disk. +** +** If successful, SQLITE_OK is returned and *ppOut set to point to the +** MergeEngine object at the root of the tree before returning. Or, if an +** error occurs, an SQLite error code is returned and the final value +** of *ppOut is undefined. +*/ +static int vdbeSorterMergeTreeBuild( + VdbeSorter *pSorter, /* The VDBE cursor that implements the sort */ + MergeEngine **ppOut /* Write the MergeEngine here */ +){ + MergeEngine *pMain = 0; + int rc = SQLITE_OK; + int iTask; + +#if SQLITE_MAX_WORKER_THREADS>0 + /* If the sorter uses more than one task, then create the top-level + ** MergeEngine here. This MergeEngine will read data from exactly + ** one PmaReader per sub-task. */ + assert( pSorter->bUseThreads || pSorter->nTask==1 ); + if( pSorter->nTask>1 ){ + pMain = vdbeMergeEngineNew(pSorter->nTask); + if( pMain==0 ) rc = SQLITE_NOMEM_BKPT; + } +#endif + + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + SortSubtask *pTask = &pSorter->aTask[iTask]; + assert( pTask->nPMA>0 || SQLITE_MAX_WORKER_THREADS>0 ); + if( SQLITE_MAX_WORKER_THREADS==0 || pTask->nPMA ){ + MergeEngine *pRoot = 0; /* Root node of tree for this task */ + int nDepth = vdbeSorterTreeDepth(pTask->nPMA); + i64 iReadOff = 0; + + if( pTask->nPMA<=SORTER_MAX_MERGE_COUNT ){ + rc = vdbeMergeEngineLevel0(pTask, pTask->nPMA, &iReadOff, &pRoot); + }else{ + int i; + int iSeq = 0; + pRoot = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pRoot==0 ) rc = SQLITE_NOMEM_BKPT; + for(i=0; inPMA && rc==SQLITE_OK; i += SORTER_MAX_MERGE_COUNT){ + MergeEngine *pMerger = 0; /* New level-0 PMA merger */ + int nReader; /* Number of level-0 PMAs to merge */ + + nReader = MIN(pTask->nPMA - i, SORTER_MAX_MERGE_COUNT); + rc = vdbeMergeEngineLevel0(pTask, nReader, &iReadOff, &pMerger); + if( rc==SQLITE_OK ){ + rc = vdbeSorterAddToTree(pTask, nDepth, iSeq++, pRoot, pMerger); + } + } + } + + if( rc==SQLITE_OK ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pMain!=0 ){ + rc = vdbeIncrMergerNew(pTask, pRoot, &pMain->aReadr[iTask].pIncr); + }else +#endif + { + assert( pMain==0 ); + pMain = pRoot; + } + }else{ + vdbeMergeEngineFree(pRoot); + } + } + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pMain); + pMain = 0; + } + *ppOut = pMain; + return rc; +} + +/* +** This function is called as part of an sqlite3VdbeSorterRewind() operation +** on a sorter that has written two or more PMAs to temporary files. It sets +** up either VdbeSorter.pMerger (for single threaded sorters) or pReader +** (for multi-threaded sorters) so that it can be used to iterate through +** all records stored in the sorter. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ + int rc; /* Return code */ + SortSubtask *pTask0 = &pSorter->aTask[0]; + MergeEngine *pMain = 0; +#if SQLITE_MAX_WORKER_THREADS + sqlite3 *db = pTask0->pSorter->db; + int i; + SorterCompare xCompare = vdbeSorterGetCompare(pSorter); + for(i=0; inTask; i++){ + pSorter->aTask[i].xCompare = xCompare; + } +#endif + + rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); + if( rc==SQLITE_OK ){ +#if SQLITE_MAX_WORKER_THREADS + assert( pSorter->bUseThreads==0 || pSorter->nTask>1 ); + if( pSorter->bUseThreads ){ + int iTask; + PmaReader *pReadr = 0; + SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; + rc = vdbeSortAllocUnpacked(pLast); + if( rc==SQLITE_OK ){ + pReadr = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); + pSorter->pReader = pReadr; + if( pReadr==0 ) rc = SQLITE_NOMEM_BKPT; + } + if( rc==SQLITE_OK ){ + rc = vdbeIncrMergerNew(pLast, pMain, &pReadr->pIncr); + if( rc==SQLITE_OK ){ + vdbeIncrMergerSetThreads(pReadr->pIncr); + for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ + IncrMerger *pIncr; + if( (pIncr = pMain->aReadr[iTask].pIncr) ){ + vdbeIncrMergerSetThreads(pIncr); + assert( pIncr->pTask!=pLast ); + } + } + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + /* Check that: + ** + ** a) The incremental merge object is configured to use the + ** right task, and + ** b) If it is using task (nTask-1), it is configured to run + ** in single-threaded mode. This is important, as the + ** root merge (INCRINIT_ROOT) will be using the same task + ** object. + */ + PmaReader *p = &pMain->aReadr[iTask]; + assert( p->pIncr==0 || ( + (p->pIncr->pTask==&pSorter->aTask[iTask]) /* a */ + && (iTask!=pSorter->nTask-1 || p->pIncr->bUseThread==0) /* b */ + )); + rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); + } + } + pMain = 0; + } + if( rc==SQLITE_OK ){ + rc = vdbePmaReaderIncrMergeInit(pReadr, INCRINIT_ROOT); + } + }else +#endif + { + rc = vdbeMergeEngineInit(pTask0, pMain, INCRINIT_NORMAL); + pSorter->pMerger = pMain; + pMain = 0; + } + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pMain); + } + return rc; +} + + +/* +** Once the sorter has been populated by calls to sqlite3VdbeSorterWrite, +** this function is called to prepare for iterating through the records +** in sorted order. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ + VdbeSorter *pSorter; + int rc = SQLITE_OK; /* Return code */ + + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; + assert( pSorter ); + + /* If no data has been written to disk, then do not do so now. Instead, + ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly + ** from the in-memory list. */ + if( pSorter->bUsePMA==0 ){ + if( pSorter->list.pList ){ + *pbEof = 0; + rc = vdbeSorterSort(&pSorter->aTask[0], &pSorter->list); + }else{ + *pbEof = 1; + } + return rc; + } + + /* Write the current in-memory list to a PMA. When the VdbeSorterWrite() + ** function flushes the contents of memory to disk, it immediately always + ** creates a new list consisting of a single key immediately afterwards. + ** So the list is never empty at this point. */ + assert( pSorter->list.pList ); + rc = vdbeSorterFlushPMA(pSorter); + + /* Join all threads */ + rc = vdbeSorterJoinAll(pSorter, rc); + + vdbeSorterRewindDebug("rewind"); + + /* Assuming no errors have occurred, set up a merger structure to + ** incrementally read and merge all remaining PMAs. */ + assert( pSorter->pReader==0 ); + if( rc==SQLITE_OK ){ + rc = vdbeSorterSetupMerge(pSorter); + *pbEof = 0; + } + + vdbeSorterRewindDebug("rewinddone"); + return rc; +} + +/* +** Advance to the next element in the sorter. Return value: +** +** SQLITE_OK success +** SQLITE_DONE end of data +** otherwise some kind of error. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr){ + VdbeSorter *pSorter; + int rc; /* Return code */ + + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; + assert( pSorter->bUsePMA || (pSorter->pReader==0 && pSorter->pMerger==0) ); + if( pSorter->bUsePMA ){ + assert( pSorter->pReader==0 || pSorter->pMerger==0 ); + assert( pSorter->bUseThreads==0 || pSorter->pReader ); + assert( pSorter->bUseThreads==1 || pSorter->pMerger ); +#if SQLITE_MAX_WORKER_THREADS>0 + if( pSorter->bUseThreads ){ + rc = vdbePmaReaderNext(pSorter->pReader); + if( rc==SQLITE_OK && pSorter->pReader->pFd==0 ) rc = SQLITE_DONE; + }else +#endif + /*if( !pSorter->bUseThreads )*/ { + int res = 0; + assert( pSorter->pMerger!=0 ); + assert( pSorter->pMerger->pTask==(&pSorter->aTask[0]) ); + rc = vdbeMergeEngineStep(pSorter->pMerger, &res); + if( rc==SQLITE_OK && res ) rc = SQLITE_DONE; + } + }else{ + SorterRecord *pFree = pSorter->list.pList; + pSorter->list.pList = pFree->u.pNext; + pFree->u.pNext = 0; + if( pSorter->list.aMemory==0 ) vdbeSorterRecordFree(db, pFree); + rc = pSorter->list.pList ? SQLITE_OK : SQLITE_DONE; + } + return rc; +} + +/* +** Return a pointer to a buffer owned by the sorter that contains the +** current key. +*/ +static void *vdbeSorterRowkey( + const VdbeSorter *pSorter, /* Sorter object */ + int *pnKey /* OUT: Size of current key in bytes */ +){ + void *pKey; + if( pSorter->bUsePMA ){ + PmaReader *pReader; +#if SQLITE_MAX_WORKER_THREADS>0 + if( pSorter->bUseThreads ){ + pReader = pSorter->pReader; + }else +#endif + /*if( !pSorter->bUseThreads )*/{ + pReader = &pSorter->pMerger->aReadr[pSorter->pMerger->aTree[1]]; + } + *pnKey = pReader->nKey; + pKey = pReader->aKey; + }else{ + *pnKey = pSorter->list.pList->nVal; + pKey = SRVAL(pSorter->list.pList); + } + return pKey; +} + +/* +** Copy the current sorter key into the memory cell pOut. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ + VdbeSorter *pSorter; + void *pKey; int nKey; /* Sorter key to copy into pOut */ + + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; + pKey = vdbeSorterRowkey(pSorter, &nKey); + if( sqlite3VdbeMemClearAndResize(pOut, nKey) ){ + return SQLITE_NOMEM_BKPT; + } + pOut->n = nKey; + MemSetTypeFlag(pOut, MEM_Blob); + memcpy(pOut->z, pKey, nKey); + + return SQLITE_OK; +} + +/* +** Compare the key in memory cell pVal with the key that the sorter cursor +** passed as the first argument currently points to. For the purposes of +** the comparison, ignore the rowid field at the end of each record. +** +** If the sorter cursor key contains any NULL values, consider it to be +** less than pVal. Even if pVal also contains NULL values. +** +** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). +** Otherwise, set *pRes to a negative, zero or positive value if the +** key in pVal is smaller than, equal to or larger than the current sorter +** key. +** +** This routine forms the core of the OP_SorterCompare opcode, which in +** turn is used to verify uniqueness when constructing a UNIQUE INDEX. +*/ +SQLITE_PRIVATE int sqlite3VdbeSorterCompare( + const VdbeCursor *pCsr, /* Sorter cursor */ + Mem *pVal, /* Value to compare to current sorter key */ + int nKeyCol, /* Compare this many columns */ + int *pRes /* OUT: Result of comparison */ +){ + VdbeSorter *pSorter; + UnpackedRecord *r2; + KeyInfo *pKeyInfo; + int i; + void *pKey; int nKey; /* Sorter key to compare pVal with */ + + assert( pCsr->eCurType==CURTYPE_SORTER ); + pSorter = pCsr->uc.pSorter; + r2 = pSorter->pUnpacked; + pKeyInfo = pCsr->pKeyInfo; + if( r2==0 ){ + r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); + if( r2==0 ) return SQLITE_NOMEM_BKPT; + r2->nField = nKeyCol; + } + assert( r2->nField==nKeyCol ); + + pKey = vdbeSorterRowkey(pSorter, &nKey); + sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); + for(i=0; iaMem[i].flags & MEM_Null ){ + *pRes = -1; + return SQLITE_OK; + } + } + + *pRes = sqlite3VdbeRecordCompare(pVal->n, pVal->z, r2); + return SQLITE_OK; +} + +/************** End of vdbesort.c ********************************************/ +/************** Begin file vdbevtab.c ****************************************/ +/* +** 2020-03-23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements virtual-tables for examining the bytecode content +** of a prepared statement. +*/ +/* #include "sqliteInt.h" */ +#if defined(SQLITE_ENABLE_BYTECODE_VTAB) && !defined(SQLITE_OMIT_VIRTUALTABLE) +/* #include "vdbeInt.h" */ + +/* An instance of the bytecode() table-valued function. +*/ +typedef struct bytecodevtab bytecodevtab; +struct bytecodevtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection */ + int bTablesUsed; /* 2 for tables_used(). 0 for bytecode(). */ +}; + +/* A cursor for scanning through the bytecode +*/ +typedef struct bytecodevtab_cursor bytecodevtab_cursor; +struct bytecodevtab_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_stmt *pStmt; /* The statement whose bytecode is displayed */ + int iRowid; /* The rowid of the output table */ + int iAddr; /* Address */ + int needFinalize; /* Cursors owns pStmt and must finalize it */ + int showSubprograms; /* Provide a listing of subprograms */ + Op *aOp; /* Operand array */ + char *zP4; /* Rendered P4 value */ + const char *zType; /* tables_used.type */ + const char *zSchema; /* tables_used.schema */ + const char *zName; /* tables_used.name */ + Mem sub; /* Subprograms */ +}; + +/* +** Create a new bytecode() table-valued function. +*/ +static int bytecodevtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + bytecodevtab *pNew; + int rc; + int isTabUsed = pAux!=0; + const char *azSchema[2] = { + /* bytecode() schema */ + "CREATE TABLE x(" + "addr INT," + "opcode TEXT," + "p1 INT," + "p2 INT," + "p3 INT," + "p4 TEXT," + "p5 INT," + "comment TEXT," + "subprog TEXT," + "stmt HIDDEN" + ");", + + /* Tables_used() schema */ + "CREATE TABLE x(" + "type TEXT," + "schema TEXT," + "name TEXT," + "wr INT," + "subprog TEXT," + "stmt HIDDEN" + ");" + }; + + rc = sqlite3_declare_vtab(db, azSchema[isTabUsed]); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->bTablesUsed = isTabUsed*2; + } + return rc; +} + +/* +** This method is the destructor for bytecodevtab objects. +*/ +static int bytecodevtabDisconnect(sqlite3_vtab *pVtab){ + bytecodevtab *p = (bytecodevtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new bytecodevtab_cursor object. +*/ +static int bytecodevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + bytecodevtab *pVTab = (bytecodevtab*)p; + bytecodevtab_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + sqlite3VdbeMemInit(&pCur->sub, pVTab->db, 1); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Clear all internal content from a bytecodevtab cursor. +*/ +static void bytecodevtabCursorClear(bytecodevtab_cursor *pCur){ + sqlite3_free(pCur->zP4); + pCur->zP4 = 0; + sqlite3VdbeMemRelease(&pCur->sub); + sqlite3VdbeMemSetNull(&pCur->sub); + if( pCur->needFinalize ){ + sqlite3_finalize(pCur->pStmt); + } + pCur->pStmt = 0; + pCur->needFinalize = 0; + pCur->zType = 0; + pCur->zSchema = 0; + pCur->zName = 0; +} + +/* +** Destructor for a bytecodevtab_cursor. +*/ +static int bytecodevtabClose(sqlite3_vtab_cursor *cur){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + bytecodevtabCursorClear(pCur); + sqlite3_free(pCur); + return SQLITE_OK; +} + + +/* +** Advance a bytecodevtab_cursor to its next row of output. +*/ +static int bytecodevtabNext(sqlite3_vtab_cursor *cur){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + bytecodevtab *pTab = (bytecodevtab*)cur->pVtab; + int rc; + if( pCur->zP4 ){ + sqlite3_free(pCur->zP4); + pCur->zP4 = 0; + } + if( pCur->zName ){ + pCur->zName = 0; + pCur->zType = 0; + pCur->zSchema = 0; + } + rc = sqlite3VdbeNextOpcode( + (Vdbe*)pCur->pStmt, + pCur->showSubprograms ? &pCur->sub : 0, + pTab->bTablesUsed, + &pCur->iRowid, + &pCur->iAddr, + &pCur->aOp); + if( rc!=SQLITE_OK ){ + sqlite3VdbeMemSetNull(&pCur->sub); + pCur->aOp = 0; + } + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int bytecodevtabEof(sqlite3_vtab_cursor *cur){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + return pCur->aOp==0; +} + +/* +** Return values of columns for the row at which the bytecodevtab_cursor +** is currently pointing. +*/ +static int bytecodevtabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + bytecodevtab *pVTab = (bytecodevtab*)cur->pVtab; + Op *pOp = pCur->aOp + pCur->iAddr; + if( pVTab->bTablesUsed ){ + if( i==4 ){ + i = 8; + }else{ + if( i<=2 && pCur->zType==0 ){ + Schema *pSchema; + HashElem *k; + int iDb = pOp->p3; + Pgno iRoot = (Pgno)pOp->p2; + sqlite3 *db = pVTab->db; + pSchema = db->aDb[iDb].pSchema; + pCur->zSchema = db->aDb[iDb].zDbSName; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pTab = (Table*)sqliteHashData(k); + if( !IsVirtual(pTab) && pTab->tnum==iRoot ){ + pCur->zName = pTab->zName; + pCur->zType = "table"; + break; + } + } + if( pCur->zName==0 ){ + for(k=sqliteHashFirst(&pSchema->idxHash); k; k=sqliteHashNext(k)){ + Index *pIdx = (Index*)sqliteHashData(k); + if( pIdx->tnum==iRoot ){ + pCur->zName = pIdx->zName; + pCur->zType = "index"; + } + } + } + } + i += 10; + } + } + switch( i ){ + case 0: /* addr */ + sqlite3_result_int(ctx, pCur->iAddr); + break; + case 1: /* opcode */ + sqlite3_result_text(ctx, (char*)sqlite3OpcodeName(pOp->opcode), + -1, SQLITE_STATIC); + break; + case 2: /* p1 */ + sqlite3_result_int(ctx, pOp->p1); + break; + case 3: /* p2 */ + sqlite3_result_int(ctx, pOp->p2); + break; + case 4: /* p3 */ + sqlite3_result_int(ctx, pOp->p3); + break; + case 5: /* p4 */ + case 7: /* comment */ + if( pCur->zP4==0 ){ + pCur->zP4 = sqlite3VdbeDisplayP4(pVTab->db, pOp); + } + if( i==5 ){ + sqlite3_result_text(ctx, pCur->zP4, -1, SQLITE_STATIC); + }else{ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + char *zCom = sqlite3VdbeDisplayComment(pVTab->db, pOp, pCur->zP4); + sqlite3_result_text(ctx, zCom, -1, sqlite3_free); +#endif + } + break; + case 6: /* p5 */ + sqlite3_result_int(ctx, pOp->p5); + break; + case 8: { /* subprog */ + Op *aOp = pCur->aOp; + assert( aOp[0].opcode==OP_Init ); + assert( aOp[0].p4.z==0 || strncmp(aOp[0].p4.z,"-" "- ",3)==0 ); + if( pCur->iRowid==pCur->iAddr+1 ){ + break; /* Result is NULL for the main program */ + }else if( aOp[0].p4.z!=0 ){ + sqlite3_result_text(ctx, aOp[0].p4.z+3, -1, SQLITE_STATIC); + }else{ + sqlite3_result_text(ctx, "(FK)", 4, SQLITE_STATIC); + } + break; + } + case 10: /* tables_used.type */ + sqlite3_result_text(ctx, pCur->zType, -1, SQLITE_STATIC); + break; + case 11: /* tables_used.schema */ + sqlite3_result_text(ctx, pCur->zSchema, -1, SQLITE_STATIC); + break; + case 12: /* tables_used.name */ + sqlite3_result_text(ctx, pCur->zName, -1, SQLITE_STATIC); + break; + case 13: /* tables_used.wr */ + sqlite3_result_int(ctx, pOp->opcode==OP_OpenWrite); + break; + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int bytecodevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Initialize a cursor. +** +** idxNum==0 means show all subprograms +** idxNum==1 means show only the main bytecode and omit subprograms. +*/ +static int bytecodevtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor *)pVtabCursor; + bytecodevtab *pVTab = (bytecodevtab *)pVtabCursor->pVtab; + int rc = SQLITE_OK; + + bytecodevtabCursorClear(pCur); + pCur->iRowid = 0; + pCur->iAddr = 0; + pCur->showSubprograms = idxNum==0; + assert( argc==1 ); + if( sqlite3_value_type(argv[0])==SQLITE_TEXT ){ + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pVTab->db, zSql, -1, &pCur->pStmt, 0); + pCur->needFinalize = 1; + } + }else{ + pCur->pStmt = (sqlite3_stmt*)sqlite3_value_pointer(argv[0],"stmt-pointer"); + } + if( pCur->pStmt==0 ){ + pVTab->base.zErrMsg = sqlite3_mprintf( + "argument to %s() is not a valid SQL statement", + pVTab->bTablesUsed ? "tables_used" : "bytecode" + ); + rc = SQLITE_ERROR; + }else{ + bytecodevtabNext(pVtabCursor); + } + return rc; +} + +/* +** We must have a single stmt=? constraint that will be passed through +** into the xFilter method. If there is no valid stmt=? constraint, +** then return an SQLITE_CONSTRAINT error. +*/ +static int bytecodevtabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + int rc = SQLITE_CONSTRAINT; + struct sqlite3_index_constraint *p; + bytecodevtab *pVTab = (bytecodevtab*)tab; + int iBaseCol = pVTab->bTablesUsed ? 4 : 8; + pIdxInfo->estimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 0; + for(i=0, p=pIdxInfo->aConstraint; inConstraint; i++, p++){ + if( p->usable==0 ) continue; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==iBaseCol+1 ){ + rc = SQLITE_OK; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + } + if( p->op==SQLITE_INDEX_CONSTRAINT_ISNULL && p->iColumn==iBaseCol ){ + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->idxNum = 1; + } + } + return rc; +} + +/* +** This following structure defines all the methods for the +** virtual table. +*/ +static sqlite3_module bytecodevtabModule = { + /* iVersion */ 0, + /* xCreate */ 0, + /* xConnect */ bytecodevtabConnect, + /* xBestIndex */ bytecodevtabBestIndex, + /* xDisconnect */ bytecodevtabDisconnect, + /* xDestroy */ 0, + /* xOpen */ bytecodevtabOpen, + /* xClose */ bytecodevtabClose, + /* xFilter */ bytecodevtabFilter, + /* xNext */ bytecodevtabNext, + /* xEof */ bytecodevtabEof, + /* xColumn */ bytecodevtabColumn, + /* xRowid */ bytecodevtabRowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0 +}; + + +SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3 *db){ + int rc; + rc = sqlite3_create_module(db, "bytecode", &bytecodevtabModule, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "tables_used", &bytecodevtabModule, &db); + } + return rc; +} +#elif defined(SQLITE_ENABLE_BYTECODE_VTAB) +SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3 *db){ return SQLITE_OK; } +#endif /* SQLITE_ENABLE_BYTECODE_VTAB */ + +/************** End of vdbevtab.c ********************************************/ +/************** Begin file memjournal.c **************************************/ +/* +** 2008 October 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code use to implement an in-memory rollback journal. +** The in-memory rollback journal is used to journal transactions for +** ":memory:" databases and when the journal_mode=MEMORY pragma is used. +** +** Update: The in-memory journal is also used to temporarily cache +** smaller journals that are not critical for power-loss recovery. +** For example, statement journals that are not too big will be held +** entirely in memory, thus reducing the number of file I/O calls, and +** more importantly, reducing temporary file creation events. If these +** journals become too large for memory, they are spilled to disk. But +** in the common case, they are usually small and no file I/O needs to +** occur. +*/ +/* #include "sqliteInt.h" */ + +/* Forward references to internal structures */ +typedef struct MemJournal MemJournal; +typedef struct FilePoint FilePoint; +typedef struct FileChunk FileChunk; + +/* +** The rollback journal is composed of a linked list of these structures. +** +** The zChunk array is always at least 8 bytes in size - usually much more. +** Its actual size is stored in the MemJournal.nChunkSize variable. +*/ +struct FileChunk { + FileChunk *pNext; /* Next chunk in the journal */ + u8 zChunk[8]; /* Content of this chunk */ +}; + +/* +** By default, allocate this many bytes of memory for each FileChunk object. +*/ +#define MEMJOURNAL_DFLT_FILECHUNKSIZE 1024 + +/* +** For chunk size nChunkSize, return the number of bytes that should +** be allocated for each FileChunk structure. +*/ +#define fileChunkSize(nChunkSize) (sizeof(FileChunk) + ((nChunkSize)-8)) + +/* +** An instance of this object serves as a cursor into the rollback journal. +** The cursor can be either for reading or writing. +*/ +struct FilePoint { + sqlite3_int64 iOffset; /* Offset from the beginning of the file */ + FileChunk *pChunk; /* Specific chunk into which cursor points */ +}; + +/* +** This structure is a subclass of sqlite3_file. Each open memory-journal +** is an instance of this class. +*/ +struct MemJournal { + const sqlite3_io_methods *pMethod; /* Parent class. MUST BE FIRST */ + int nChunkSize; /* In-memory chunk-size */ + + int nSpill; /* Bytes of data before flushing */ + FileChunk *pFirst; /* Head of in-memory chunk-list */ + FilePoint endpoint; /* Pointer to the end of the file */ + FilePoint readpoint; /* Pointer to the end of the last xRead() */ + + int flags; /* xOpen flags */ + sqlite3_vfs *pVfs; /* The "real" underlying VFS */ + const char *zJournal; /* Name of the journal file */ +}; + +/* +** Read data from the in-memory journal file. This is the implementation +** of the sqlite3_vfs.xRead method. +*/ +static int memjrnlRead( + sqlite3_file *pJfd, /* The journal file from which to read */ + void *zBuf, /* Put the results here */ + int iAmt, /* Number of bytes to read */ + sqlite_int64 iOfst /* Begin reading at this offset */ +){ + MemJournal *p = (MemJournal *)pJfd; + u8 *zOut = zBuf; + int nRead = iAmt; + int iChunkOffset; + FileChunk *pChunk; + + if( (iAmt+iOfst)>p->endpoint.iOffset ){ + return SQLITE_IOERR_SHORT_READ; + } + assert( p->readpoint.iOffset==0 || p->readpoint.pChunk!=0 ); + if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ + sqlite3_int64 iOff = 0; + for(pChunk=p->pFirst; + ALWAYS(pChunk) && (iOff+p->nChunkSize)<=iOfst; + pChunk=pChunk->pNext + ){ + iOff += p->nChunkSize; + } + }else{ + pChunk = p->readpoint.pChunk; + assert( pChunk!=0 ); + } + + iChunkOffset = (int)(iOfst%p->nChunkSize); + do { + int iSpace = p->nChunkSize - iChunkOffset; + int nCopy = MIN(nRead, (p->nChunkSize - iChunkOffset)); + memcpy(zOut, (u8*)pChunk->zChunk + iChunkOffset, nCopy); + zOut += nCopy; + nRead -= iSpace; + iChunkOffset = 0; + } while( nRead>=0 && (pChunk=pChunk->pNext)!=0 && nRead>0 ); + p->readpoint.iOffset = pChunk ? iOfst+iAmt : 0; + p->readpoint.pChunk = pChunk; + + return SQLITE_OK; +} + +/* +** Free the list of FileChunk structures headed at MemJournal.pFirst. +*/ +static void memjrnlFreeChunks(FileChunk *pFirst){ + FileChunk *pIter; + FileChunk *pNext; + for(pIter=pFirst; pIter; pIter=pNext){ + pNext = pIter->pNext; + sqlite3_free(pIter); + } +} + +/* +** Flush the contents of memory to a real file on disk. +*/ +static int memjrnlCreateFile(MemJournal *p){ + int rc; + sqlite3_file *pReal = (sqlite3_file*)p; + MemJournal copy = *p; + + memset(p, 0, sizeof(MemJournal)); + rc = sqlite3OsOpen(copy.pVfs, copy.zJournal, pReal, copy.flags, 0); + if( rc==SQLITE_OK ){ + int nChunk = copy.nChunkSize; + i64 iOff = 0; + FileChunk *pIter; + for(pIter=copy.pFirst; pIter; pIter=pIter->pNext){ + if( iOff + nChunk > copy.endpoint.iOffset ){ + nChunk = copy.endpoint.iOffset - iOff; + } + rc = sqlite3OsWrite(pReal, (u8*)pIter->zChunk, nChunk, iOff); + if( rc ) break; + iOff += nChunk; + } + if( rc==SQLITE_OK ){ + /* No error has occurred. Free the in-memory buffers. */ + memjrnlFreeChunks(copy.pFirst); + } + } + if( rc!=SQLITE_OK ){ + /* If an error occurred while creating or writing to the file, restore + ** the original before returning. This way, SQLite uses the in-memory + ** journal data to roll back changes made to the internal page-cache + ** before this function was called. */ + sqlite3OsClose(pReal); + *p = copy; + } + return rc; +} + + +/* +** Write data to the file. +*/ +static int memjrnlWrite( + sqlite3_file *pJfd, /* The journal file into which to write */ + const void *zBuf, /* Take data to be written from here */ + int iAmt, /* Number of bytes to write */ + sqlite_int64 iOfst /* Begin writing at this offset into the file */ +){ + MemJournal *p = (MemJournal *)pJfd; + int nWrite = iAmt; + u8 *zWrite = (u8 *)zBuf; + + /* If the file should be created now, create it and write the new data + ** into the file on disk. */ + if( p->nSpill>0 && (iAmt+iOfst)>p->nSpill ){ + int rc = memjrnlCreateFile(p); + if( rc==SQLITE_OK ){ + rc = sqlite3OsWrite(pJfd, zBuf, iAmt, iOfst); + } + return rc; + } + + /* If the contents of this write should be stored in memory */ + else{ + /* An in-memory journal file should only ever be appended to. Random + ** access writes are not required. The only exception to this is when + ** the in-memory journal is being used by a connection using the + ** atomic-write optimization. In this case the first 28 bytes of the + ** journal file may be written as part of committing the transaction. */ + assert( iOfst==p->endpoint.iOffset || iOfst==0 ); +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + if( iOfst==0 && p->pFirst ){ + assert( p->nChunkSize>iAmt ); + memcpy((u8*)p->pFirst->zChunk, zBuf, iAmt); + }else +#else + assert( iOfst>0 || p->pFirst==0 ); +#endif + { + while( nWrite>0 ){ + FileChunk *pChunk = p->endpoint.pChunk; + int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize); + int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset); + + if( iChunkOffset==0 ){ + /* New chunk is required to extend the file. */ + FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize)); + if( !pNew ){ + return SQLITE_IOERR_NOMEM_BKPT; + } + pNew->pNext = 0; + if( pChunk ){ + assert( p->pFirst ); + pChunk->pNext = pNew; + }else{ + assert( !p->pFirst ); + p->pFirst = pNew; + } + p->endpoint.pChunk = pNew; + } + + memcpy((u8*)p->endpoint.pChunk->zChunk + iChunkOffset, zWrite, iSpace); + zWrite += iSpace; + nWrite -= iSpace; + p->endpoint.iOffset += iSpace; + } + } + } + + return SQLITE_OK; +} + +/* +** Truncate the in-memory file. +*/ +static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ + MemJournal *p = (MemJournal *)pJfd; + assert( p->endpoint.pChunk==0 || p->endpoint.pChunk->pNext==0 ); + if( sizeendpoint.iOffset ){ + FileChunk *pIter = 0; + if( size==0 ){ + memjrnlFreeChunks(p->pFirst); + p->pFirst = 0; + }else{ + i64 iOff = p->nChunkSize; + for(pIter=p->pFirst; ALWAYS(pIter) && iOff<=size; pIter=pIter->pNext){ + iOff += p->nChunkSize; + } + if( ALWAYS(pIter) ){ + memjrnlFreeChunks(pIter->pNext); + pIter->pNext = 0; + } + } + + p->endpoint.pChunk = pIter; + p->endpoint.iOffset = size; + p->readpoint.pChunk = 0; + p->readpoint.iOffset = 0; + } + return SQLITE_OK; +} + +/* +** Close the file. +*/ +static int memjrnlClose(sqlite3_file *pJfd){ + MemJournal *p = (MemJournal *)pJfd; + memjrnlFreeChunks(p->pFirst); + return SQLITE_OK; +} + +/* +** Sync the file. +** +** If the real file has been created, call its xSync method. Otherwise, +** syncing an in-memory journal is a no-op. +*/ +static int memjrnlSync(sqlite3_file *pJfd, int flags){ + UNUSED_PARAMETER2(pJfd, flags); + return SQLITE_OK; +} + +/* +** Query the size of the file in bytes. +*/ +static int memjrnlFileSize(sqlite3_file *pJfd, sqlite_int64 *pSize){ + MemJournal *p = (MemJournal *)pJfd; + *pSize = (sqlite_int64) p->endpoint.iOffset; + return SQLITE_OK; +} + +/* +** Table of methods for MemJournal sqlite3_file object. +*/ +static const struct sqlite3_io_methods MemJournalMethods = { + 1, /* iVersion */ + memjrnlClose, /* xClose */ + memjrnlRead, /* xRead */ + memjrnlWrite, /* xWrite */ + memjrnlTruncate, /* xTruncate */ + memjrnlSync, /* xSync */ + memjrnlFileSize, /* xFileSize */ + 0, /* xLock */ + 0, /* xUnlock */ + 0, /* xCheckReservedLock */ + 0, /* xFileControl */ + 0, /* xSectorSize */ + 0, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/* +** Open a journal file. +** +** The behaviour of the journal file depends on the value of parameter +** nSpill. If nSpill is 0, then the journal file is always create and +** accessed using the underlying VFS. If nSpill is less than zero, then +** all content is always stored in main-memory. Finally, if nSpill is a +** positive value, then the journal file is initially created in-memory +** but may be flushed to disk later on. In this case the journal file is +** flushed to disk either when it grows larger than nSpill bytes in size, +** or when sqlite3JournalCreate() is called. +*/ +SQLITE_PRIVATE int sqlite3JournalOpen( + sqlite3_vfs *pVfs, /* The VFS to use for actual file I/O */ + const char *zName, /* Name of the journal file */ + sqlite3_file *pJfd, /* Preallocated, blank file handle */ + int flags, /* Opening flags */ + int nSpill /* Bytes buffered before opening the file */ +){ + MemJournal *p = (MemJournal*)pJfd; + + /* Zero the file-handle object. If nSpill was passed zero, initialize + ** it using the sqlite3OsOpen() function of the underlying VFS. In this + ** case none of the code in this module is executed as a result of calls + ** made on the journal file-handle. */ + memset(p, 0, sizeof(MemJournal)); + if( nSpill==0 ){ + return sqlite3OsOpen(pVfs, zName, pJfd, flags, 0); + } + + if( nSpill>0 ){ + p->nChunkSize = nSpill; + }else{ + p->nChunkSize = 8 + MEMJOURNAL_DFLT_FILECHUNKSIZE - sizeof(FileChunk); + assert( MEMJOURNAL_DFLT_FILECHUNKSIZE==fileChunkSize(p->nChunkSize) ); + } + + pJfd->pMethods = (const sqlite3_io_methods*)&MemJournalMethods; + p->nSpill = nSpill; + p->flags = flags; + p->zJournal = zName; + p->pVfs = pVfs; + return SQLITE_OK; +} + +/* +** Open an in-memory journal file. +*/ +SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *pJfd){ + sqlite3JournalOpen(0, 0, pJfd, 0, -1); +} + +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +/* +** If the argument p points to a MemJournal structure that is not an +** in-memory-only journal file (i.e. is one that was opened with a +ve +** nSpill parameter or as SQLITE_OPEN_MAIN_JOURNAL), and the underlying +** file has not yet been created, create it now. +*/ +SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *pJfd){ + int rc = SQLITE_OK; + MemJournal *p = (MemJournal*)pJfd; + if( pJfd->pMethods==&MemJournalMethods && ( +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + p->nSpill>0 +#else + /* While this appears to not be possible without ATOMIC_WRITE, the + ** paths are complex, so it seems prudent to leave the test in as + ** a NEVER(), in case our analysis is subtly flawed. */ + NEVER(p->nSpill>0) +#endif +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + || (p->flags & SQLITE_OPEN_MAIN_JOURNAL) +#endif + )){ + rc = memjrnlCreateFile(p); + } + return rc; +} +#endif + +/* +** The file-handle passed as the only argument is open on a journal file. +** Return true if this "journal file" is currently stored in heap memory, +** or false otherwise. +*/ +SQLITE_PRIVATE int sqlite3JournalIsInMemory(sqlite3_file *p){ + return p->pMethods==&MemJournalMethods; +} + +/* +** Return the number of bytes required to store a JournalFile that uses vfs +** pVfs to create the underlying on-disk files. +*/ +SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ + return MAX(pVfs->szOsFile, (int)sizeof(MemJournal)); +} + +/************** End of memjournal.c ******************************************/ +/************** Begin file walker.c ******************************************/ +/* +** 2008 August 16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains routines used for walking the parser tree for +** an SQL statement. +*/ +/* #include "sqliteInt.h" */ +/* #include */ +/* #include */ + + +#if !defined(SQLITE_OMIT_WINDOWFUNC) +/* +** Walk all expressions linked into the list of Window objects passed +** as the second argument. +*/ +static int walkWindowList(Walker *pWalker, Window *pList, int bOneOnly){ + Window *pWin; + for(pWin=pList; pWin; pWin=pWin->pNextWin){ + int rc; + rc = sqlite3WalkExprList(pWalker, pWin->pOrderBy); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExprList(pWalker, pWin->pPartition); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExpr(pWalker, pWin->pFilter); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExpr(pWalker, pWin->pStart); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExpr(pWalker, pWin->pEnd); + if( rc ) return WRC_Abort; + if( bOneOnly ) break; + } + return WRC_Continue; +} +#endif + +/* +** Walk an expression tree. Invoke the callback once for each node +** of the expression, while descending. (In other words, the callback +** is invoked before visiting children.) +** +** The return value from the callback should be one of the WRC_* +** constants to specify how to proceed with the walk. +** +** WRC_Continue Continue descending down the tree. +** +** WRC_Prune Do not descend into child nodes, but allow +** the walk to continue with sibling nodes. +** +** WRC_Abort Do no more callbacks. Unwind the stack and +** return from the top-level walk call. +** +** The return value from this routine is WRC_Abort to abandon the tree walk +** and WRC_Continue to continue. +*/ +static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ + int rc; + testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); + testcase( ExprHasProperty(pExpr, EP_Reduced) ); + while(1){ + rc = pWalker->xExprCallback(pWalker, pExpr); + if( rc ) return rc & WRC_Abort; + if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ + assert( pExpr->x.pList==0 || pExpr->pRight==0 ); + if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; + if( pExpr->pRight ){ + assert( !ExprHasProperty(pExpr, EP_WinFunc) ); + pExpr = pExpr->pRight; + continue; + }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + assert( !ExprHasProperty(pExpr, EP_WinFunc) ); + if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; + }else{ + if( pExpr->x.pList ){ + if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + if( walkWindowList(pWalker, pExpr->y.pWin, 1) ) return WRC_Abort; + } +#endif + } + } + break; + } + return WRC_Continue; +} +SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ + return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; +} + +/* +** Call sqlite3WalkExpr() for every expression in list p or until +** an abort request is seen. +*/ +SQLITE_PRIVATE int sqlite3WalkExprList(Walker *pWalker, ExprList *p){ + int i; + struct ExprList_item *pItem; + if( p ){ + for(i=p->nExpr, pItem=p->a; i>0; i--, pItem++){ + if( sqlite3WalkExpr(pWalker, pItem->pExpr) ) return WRC_Abort; + } + } + return WRC_Continue; +} + +/* +** This is a no-op callback for Walker->xSelectCallback2. If this +** callback is set, then the Select->pWinDefn list is traversed. +*/ +SQLITE_PRIVATE void sqlite3WalkWinDefnDummyCallback(Walker *pWalker, Select *p){ + UNUSED_PARAMETER(pWalker); + UNUSED_PARAMETER(p); + /* No-op */ +} + +/* +** Walk all expressions associated with SELECT statement p. Do +** not invoke the SELECT callback on p, but do (of course) invoke +** any expr callbacks and SELECT callbacks that come from subqueries. +** Return WRC_Abort or WRC_Continue. +*/ +SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker *pWalker, Select *p){ + if( sqlite3WalkExprList(pWalker, p->pEList) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, p->pWhere) ) return WRC_Abort; + if( sqlite3WalkExprList(pWalker, p->pGroupBy) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, p->pHaving) ) return WRC_Abort; + if( sqlite3WalkExprList(pWalker, p->pOrderBy) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, p->pLimit) ) return WRC_Abort; +#if !defined(SQLITE_OMIT_WINDOWFUNC) + if( p->pWinDefn ){ + Parse *pParse; + if( pWalker->xSelectCallback2==sqlite3WalkWinDefnDummyCallback + || ((pParse = pWalker->pParse)!=0 && IN_RENAME_OBJECT) +#ifndef SQLITE_OMIT_CTE + || pWalker->xSelectCallback2==sqlite3SelectPopWith +#endif + ){ + /* The following may return WRC_Abort if there are unresolvable + ** symbols (e.g. a table that does not exist) in a window definition. */ + int rc = walkWindowList(pWalker, p->pWinDefn, 0); + return rc; + } + } +#endif + return WRC_Continue; +} + +/* +** Walk the parse trees associated with all subqueries in the +** FROM clause of SELECT statement p. Do not invoke the select +** callback on p, but do invoke it on each FROM clause subquery +** and on any subqueries further down in the tree. Return +** WRC_Abort or WRC_Continue; +*/ +SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ + SrcList *pSrc; + int i; + SrcItem *pItem; + + pSrc = p->pSrc; + if( ALWAYS(pSrc) ){ + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + if( pItem->pSelect && sqlite3WalkSelect(pWalker, pItem->pSelect) ){ + return WRC_Abort; + } + if( pItem->fg.isTabFunc + && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } + } + } + return WRC_Continue; +} + +/* +** Call sqlite3WalkExpr() for every expression in Select statement p. +** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and +** on the compound select chain, p->pPrior. +** +** If it is not NULL, the xSelectCallback() callback is invoked before +** the walk of the expressions and FROM clause. The xSelectCallback2() +** method is invoked following the walk of the expressions and FROM clause, +** but only if both xSelectCallback and xSelectCallback2 are both non-NULL +** and if the expressions and FROM clause both return WRC_Continue; +** +** Return WRC_Continue under normal conditions. Return WRC_Abort if +** there is an abort request. +** +** If the Walker does not have an xSelectCallback() then this routine +** is a no-op returning WRC_Continue. +*/ +SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ + int rc; + if( p==0 ) return WRC_Continue; + if( pWalker->xSelectCallback==0 ) return WRC_Continue; + do{ + rc = pWalker->xSelectCallback(pWalker, p); + if( rc ) return rc & WRC_Abort; + if( sqlite3WalkSelectExpr(pWalker, p) + || sqlite3WalkSelectFrom(pWalker, p) + ){ + return WRC_Abort; + } + if( pWalker->xSelectCallback2 ){ + pWalker->xSelectCallback2(pWalker, p); + } + p = p->pPrior; + }while( p!=0 ); + return WRC_Continue; +} + +/* Increase the walkerDepth when entering a subquery, and +** descrease when leaving the subquery. +*/ +SQLITE_PRIVATE int sqlite3WalkerDepthIncrease(Walker *pWalker, Select *pSelect){ + UNUSED_PARAMETER(pSelect); + pWalker->walkerDepth++; + return WRC_Continue; +} +SQLITE_PRIVATE void sqlite3WalkerDepthDecrease(Walker *pWalker, Select *pSelect){ + UNUSED_PARAMETER(pSelect); + pWalker->walkerDepth--; +} + + +/* +** No-op routine for the parse-tree walker. +** +** When this routine is the Walker.xExprCallback then expression trees +** are walked without any actions being taken at each node. Presumably, +** when this routine is used for Walker.xExprCallback then +** Walker.xSelectCallback is set to do something useful for every +** subquery in the parser tree. +*/ +SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return WRC_Continue; +} + +/* +** No-op routine for the parse-tree walker for SELECT statements. +** subquery in the parser tree. +*/ +SQLITE_PRIVATE int sqlite3SelectWalkNoop(Walker *NotUsed, Select *NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return WRC_Continue; +} + +/************** End of walker.c **********************************************/ +/************** Begin file resolve.c *****************************************/ +/* +** 2008 August 18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains routines used for walking the parser tree and +** resolve all identifiers by associating them with a particular +** table and column. +*/ +/* #include "sqliteInt.h" */ + +/* +** Magic table number to mean the EXCLUDED table in an UPSERT statement. +*/ +#define EXCLUDED_TABLE_NUMBER 2 + +/* +** Walk the expression tree pExpr and increase the aggregate function +** depth (the Expr.op2 field) by N on every TK_AGG_FUNCTION node. +** This needs to occur when copying a TK_AGG_FUNCTION node from an +** outer query into an inner subquery. +** +** incrAggFunctionDepth(pExpr,n) is the main routine. incrAggDepth(..) +** is a helper function - a callback for the tree walker. +** +** See also the sqlite3WindowExtraAggFuncDepth() routine in window.c +*/ +static int incrAggDepth(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n; + return WRC_Continue; +} +static void incrAggFunctionDepth(Expr *pExpr, int N){ + if( N>0 ){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = incrAggDepth; + w.u.n = N; + sqlite3WalkExpr(&w, pExpr); + } +} + +/* +** Turn the pExpr expression into an alias for the iCol-th column of the +** result set in pEList. +** +** If the reference is followed by a COLLATE operator, then make sure +** the COLLATE operator is preserved. For example: +** +** SELECT a+b, c+d FROM t1 ORDER BY 1 COLLATE nocase; +** +** Should be transformed into: +** +** SELECT a+b, c+d FROM t1 ORDER BY (a+b) COLLATE nocase; +** +** The nSubquery parameter specifies how many levels of subquery the +** alias is removed from the original expression. The usual value is +** zero but it might be more if the alias is contained within a subquery +** of the original expression. The Expr.op2 field of TK_AGG_FUNCTION +** structures must be increased by the nSubquery amount. +*/ +static void resolveAlias( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* A result set */ + int iCol, /* A column in the result set. 0..pEList->nExpr-1 */ + Expr *pExpr, /* Transform this into an alias to the result set */ + int nSubquery /* Number of subqueries that the label is moving */ +){ + Expr *pOrig; /* The iCol-th column of the result set */ + Expr *pDup; /* Copy of pOrig */ + sqlite3 *db; /* The database connection */ + + assert( iCol>=0 && iColnExpr ); + pOrig = pEList->a[iCol].pExpr; + assert( pOrig!=0 ); + db = pParse->db; + pDup = sqlite3ExprDup(db, pOrig, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDup); + pDup = 0; + }else{ + incrAggFunctionDepth(pDup, nSubquery); + if( pExpr->op==TK_COLLATE ){ + pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); + } + + /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This + ** prevents ExprDelete() from deleting the Expr structure itself, + ** allowing it to be repopulated by the memcpy() on the following line. + ** The pExpr->u.zToken might point into memory that will be freed by the + ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to + ** make a copy of the token before doing the sqlite3DbFree(). + */ + ExprSetProperty(pExpr, EP_Static); + sqlite3ExprDelete(db, pExpr); + memcpy(pExpr, pDup, sizeof(*pExpr)); + if( !ExprHasProperty(pExpr, EP_IntValue) && pExpr->u.zToken!=0 ){ + assert( (pExpr->flags & (EP_Reduced|EP_TokenOnly))==0 ); + pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken); + pExpr->flags |= EP_MemToken; + } + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + if( ALWAYS(pExpr->y.pWin!=0) ){ + pExpr->y.pWin->pOwner = pExpr; + } + } + sqlite3DbFree(db, pDup); + } +} + + +/* +** Return TRUE if the name zCol occurs anywhere in the USING clause. +** +** Return FALSE if the USING clause is NULL or if it does not contain +** zCol. +*/ +static int nameInUsingClause(IdList *pUsing, const char *zCol){ + if( pUsing ){ + int k; + for(k=0; knId; k++){ + if( sqlite3StrICmp(pUsing->a[k].zName, zCol)==0 ) return 1; + } + } + return 0; +} + +/* +** Subqueries stores the original database, table and column names for their +** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN". +** Check to see if the zSpan given to this routine matches the zDb, zTab, +** and zCol. If any of zDb, zTab, and zCol are NULL then those fields will +** match anything. +*/ +SQLITE_PRIVATE int sqlite3MatchEName( + const struct ExprList_item *pItem, + const char *zCol, + const char *zTab, + const char *zDb +){ + int n; + const char *zSpan; + if( pItem->eEName!=ENAME_TAB ) return 0; + zSpan = pItem->zEName; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zDb && (sqlite3StrNICmp(zSpan, zDb, n)!=0 || zDb[n]!=0) ){ + return 0; + } + zSpan += n+1; + for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} + if( zTab && (sqlite3StrNICmp(zSpan, zTab, n)!=0 || zTab[n]!=0) ){ + return 0; + } + zSpan += n+1; + if( zCol && sqlite3StrICmp(zSpan, zCol)!=0 ){ + return 0; + } + return 1; +} + +/* +** Return TRUE if the double-quoted string mis-feature should be supported. +*/ +static int areDoubleQuotedStringsEnabled(sqlite3 *db, NameContext *pTopNC){ + if( db->init.busy ) return 1; /* Always support for legacy schemas */ + if( pTopNC->ncFlags & NC_IsDDL ){ + /* Currently parsing a DDL statement */ + if( sqlite3WritableSchema(db) && (db->flags & SQLITE_DqsDML)!=0 ){ + return 1; + } + return (db->flags & SQLITE_DqsDDL)!=0; + }else{ + /* Currently parsing a DML statement */ + return (db->flags & SQLITE_DqsDML)!=0; + } +} + +/* +** The argument is guaranteed to be a non-NULL Expr node of type TK_COLUMN. +** return the appropriate colUsed mask. +*/ +SQLITE_PRIVATE Bitmask sqlite3ExprColUsed(Expr *pExpr){ + int n; + Table *pExTab; + + n = pExpr->iColumn; + pExTab = pExpr->y.pTab; + assert( pExTab!=0 ); + if( (pExTab->tabFlags & TF_HasGenerated)!=0 + && (pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0 + ){ + testcase( pExTab->nCol==BMS-1 ); + testcase( pExTab->nCol==BMS ); + return pExTab->nCol>=BMS ? ALLBITS : MASKBIT(pExTab->nCol)-1; + }else{ + testcase( n==BMS-1 ); + testcase( n==BMS ); + if( n>=BMS ) n = BMS-1; + return ((Bitmask)1)<iDb Set the index in db->aDb[] of the database X +** (even if X is implied). +** pExpr->iTable Set to the cursor number for the table obtained +** from pSrcList. +** pExpr->y.pTab Points to the Table structure of X.Y (even if +** X and/or Y are implied.) +** pExpr->iColumn Set to the column number within the table. +** pExpr->op Set to TK_COLUMN. +** pExpr->pLeft Any expression this points to is deleted +** pExpr->pRight Any expression this points to is deleted. +** +** The zDb variable is the name of the database (the "X"). This value may be +** NULL meaning that name is of the form Y.Z or Z. Any available database +** can be used. The zTable variable is the name of the table (the "Y"). This +** value can be NULL if zDb is also NULL. If zTable is NULL it +** means that the form of the name is Z and that columns from any table +** can be used. +** +** If the name cannot be resolved unambiguously, leave an error message +** in pParse and return WRC_Abort. Return WRC_Prune on success. +*/ +static int lookupName( + Parse *pParse, /* The parsing context */ + const char *zDb, /* Name of the database containing table, or NULL */ + const char *zTab, /* Name of table containing column, or NULL */ + const char *zCol, /* Name of the column. */ + NameContext *pNC, /* The name context used to resolve the name */ + Expr *pExpr /* Make this EXPR node point to the selected column */ +){ + int i, j; /* Loop counters */ + int cnt = 0; /* Number of matching column names */ + int cntTab = 0; /* Number of matching table names */ + int nSubquery = 0; /* How many levels of subquery */ + sqlite3 *db = pParse->db; /* The database connection */ + SrcItem *pItem; /* Use for looping over pSrcList items */ + SrcItem *pMatch = 0; /* The matching pSrcList item */ + NameContext *pTopNC = pNC; /* First namecontext in the list */ + Schema *pSchema = 0; /* Schema of the expression */ + int eNewExprOp = TK_COLUMN; /* New value for pExpr->op on success */ + Table *pTab = 0; /* Table hold the row */ + Column *pCol; /* A column of pTab */ + + assert( pNC ); /* the name context cannot be NULL. */ + assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + + /* Initialize the node to no-match */ + pExpr->iTable = -1; + ExprSetVVAProperty(pExpr, EP_NoReduce); + + /* Translate the schema name in zDb into a pointer to the corresponding + ** schema. If not found, pSchema will remain NULL and nothing will match + ** resulting in an appropriate error message toward the end of this routine + */ + if( zDb ){ + testcase( pNC->ncFlags & NC_PartIdx ); + testcase( pNC->ncFlags & NC_IsCheck ); + if( (pNC->ncFlags & (NC_PartIdx|NC_IsCheck))!=0 ){ + /* Silently ignore database qualifiers inside CHECK constraints and + ** partial indices. Do not raise errors because that might break + ** legacy and because it does not hurt anything to just ignore the + ** database name. */ + zDb = 0; + }else{ + for(i=0; inDb; i++){ + assert( db->aDb[i].zDbSName ); + if( sqlite3StrICmp(db->aDb[i].zDbSName,zDb)==0 ){ + pSchema = db->aDb[i].pSchema; + break; + } + } + if( i==db->nDb && sqlite3StrICmp("main", zDb)==0 ){ + /* This branch is taken when the main database has been renamed + ** using SQLITE_DBCONFIG_MAINDBNAME. */ + pSchema = db->aDb[0].pSchema; + zDb = db->aDb[0].zDbSName; + } + } + } + + /* Start at the inner-most context and move outward until a match is found */ + assert( pNC && cnt==0 ); + do{ + ExprList *pEList; + SrcList *pSrcList = pNC->pSrcList; + + if( pSrcList ){ + for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ + u8 hCol; + pTab = pItem->pTab; + assert( pTab!=0 && pTab->zName!=0 ); + assert( pTab->nCol>0 || pParse->nErr ); + if( pItem->pSelect && (pItem->pSelect->selFlags & SF_NestedFrom)!=0 ){ + int hit = 0; + pEList = pItem->pSelect->pEList; + for(j=0; jnExpr; j++){ + if( sqlite3MatchEName(&pEList->a[j], zCol, zTab, zDb) ){ + cnt++; + cntTab = 2; + pMatch = pItem; + pExpr->iColumn = j; + hit = 1; + } + } + if( hit || zTab==0 ) continue; + } + if( zDb && pTab->pSchema!=pSchema ){ + continue; + } + if( zTab ){ + const char *zTabName = pItem->zAlias ? pItem->zAlias : pTab->zName; + assert( zTabName!=0 ); + if( sqlite3StrICmp(zTabName, zTab)!=0 ){ + continue; + } + if( IN_RENAME_OBJECT && pItem->zAlias ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)&pExpr->y.pTab); + } + } + hCol = sqlite3StrIHash(zCol); + for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ + if( pCol->hName==hCol + && sqlite3StrICmp(pCol->zCnName, zCol)==0 + ){ + /* If there has been exactly one prior match and this match + ** is for the right-hand table of a NATURAL JOIN or is in a + ** USING clause, then skip this match. + */ + if( cnt==1 ){ + if( pItem->fg.jointype & JT_NATURAL ) continue; + if( nameInUsingClause(pItem->pUsing, zCol) ) continue; + } + cnt++; + pMatch = pItem; + /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ + pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; + break; + } + } + if( 0==cnt && VisibleRowid(pTab) ){ + cntTab++; + pMatch = pItem; + } + } + if( pMatch ){ + pExpr->iTable = pMatch->iCursor; + pExpr->y.pTab = pMatch->pTab; + /* RIGHT JOIN not (yet) supported */ + assert( (pMatch->fg.jointype & JT_RIGHT)==0 ); + if( (pMatch->fg.jointype & JT_LEFT)!=0 ){ + ExprSetProperty(pExpr, EP_CanBeNull); + } + pSchema = pExpr->y.pTab->pSchema; + } + } /* if( pSrcList ) */ + +#if !defined(SQLITE_OMIT_TRIGGER) || !defined(SQLITE_OMIT_UPSERT) + /* If we have not already resolved the name, then maybe + ** it is a new.* or old.* trigger argument reference. Or + ** maybe it is an excluded.* from an upsert. Or maybe it is + ** a reference in the RETURNING clause to a table being modified. + */ + if( cnt==0 && zDb==0 ){ + pTab = 0; +#ifndef SQLITE_OMIT_TRIGGER + if( pParse->pTriggerTab!=0 ){ + int op = pParse->eTriggerOp; + assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); + if( pParse->bReturning ){ + if( (pNC->ncFlags & NC_UBaseReg)!=0 + && (zTab==0 || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0) + ){ + pExpr->iTable = op!=TK_DELETE; + pTab = pParse->pTriggerTab; + } + }else if( op!=TK_DELETE && zTab && sqlite3StrICmp("new",zTab) == 0 ){ + pExpr->iTable = 1; + pTab = pParse->pTriggerTab; + }else if( op!=TK_INSERT && zTab && sqlite3StrICmp("old",zTab)==0 ){ + pExpr->iTable = 0; + pTab = pParse->pTriggerTab; + } + } +#endif /* SQLITE_OMIT_TRIGGER */ +#ifndef SQLITE_OMIT_UPSERT + if( (pNC->ncFlags & NC_UUpsert)!=0 && zTab!=0 ){ + Upsert *pUpsert = pNC->uNC.pUpsert; + if( pUpsert && sqlite3StrICmp("excluded",zTab)==0 ){ + pTab = pUpsert->pUpsertSrc->a[0].pTab; + pExpr->iTable = EXCLUDED_TABLE_NUMBER; + } + } +#endif /* SQLITE_OMIT_UPSERT */ + + if( pTab ){ + int iCol; + u8 hCol = sqlite3StrIHash(zCol); + pSchema = pTab->pSchema; + cntTab++; + for(iCol=0, pCol=pTab->aCol; iColnCol; iCol++, pCol++){ + if( pCol->hName==hCol + && sqlite3StrICmp(pCol->zCnName, zCol)==0 + ){ + if( iCol==pTab->iPKey ){ + iCol = -1; + } + break; + } + } + if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && VisibleRowid(pTab) ){ + /* IMP: R-51414-32910 */ + iCol = -1; + } + if( iColnCol ){ + cnt++; + pMatch = 0; +#ifndef SQLITE_OMIT_UPSERT + if( pExpr->iTable==EXCLUDED_TABLE_NUMBER ){ + testcase( iCol==(-1) ); + if( IN_RENAME_OBJECT ){ + pExpr->iColumn = iCol; + pExpr->y.pTab = pTab; + eNewExprOp = TK_COLUMN; + }else{ + pExpr->iTable = pNC->uNC.pUpsert->regData + + sqlite3TableColumnToStorage(pTab, iCol); + eNewExprOp = TK_REGISTER; + } + }else +#endif /* SQLITE_OMIT_UPSERT */ + { + pExpr->y.pTab = pTab; + if( pParse->bReturning ){ + eNewExprOp = TK_REGISTER; + pExpr->iTable = pNC->uNC.iBaseReg + (pTab->nCol+1)*pExpr->iTable + + sqlite3TableColumnToStorage(pTab, iCol) + 1; + }else{ + pExpr->iColumn = (i16)iCol; + eNewExprOp = TK_TRIGGER; +#ifndef SQLITE_OMIT_TRIGGER + if( iCol<0 ){ + pExpr->affExpr = SQLITE_AFF_INTEGER; + }else if( pExpr->iTable==0 ){ + testcase( iCol==31 ); + testcase( iCol==32 ); + pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<ncFlags & (NC_IdxExpr|NC_GenCol))==0 + && sqlite3IsRowid(zCol) + && ALWAYS(VisibleRowid(pMatch->pTab)) + ){ + cnt = 1; + pExpr->iColumn = -1; + pExpr->affExpr = SQLITE_AFF_INTEGER; + } + + /* + ** If the input is of the form Z (not Y.Z or X.Y.Z) then the name Z + ** might refer to an result-set alias. This happens, for example, when + ** we are resolving names in the WHERE clause of the following command: + ** + ** SELECT a+b AS x FROM table WHERE x<10; + ** + ** In cases like this, replace pExpr with a copy of the expression that + ** forms the result set entry ("a+b" in the example) and return immediately. + ** Note that the expression in the result set should have already been + ** resolved by the time the WHERE clause is resolved. + ** + ** The ability to use an output result-set column in the WHERE, GROUP BY, + ** or HAVING clauses, or as part of a larger expression in the ORDER BY + ** clause is not standard SQL. This is a (goofy) SQLite extension, that + ** is supported for backwards compatibility only. Hence, we issue a warning + ** on sqlite3_log() whenever the capability is used. + */ + if( cnt==0 + && (pNC->ncFlags & NC_UEList)!=0 + && zTab==0 + ){ + pEList = pNC->uNC.pEList; + assert( pEList!=0 ); + for(j=0; jnExpr; j++){ + char *zAs = pEList->a[j].zEName; + if( pEList->a[j].eEName==ENAME_NAME + && sqlite3_stricmp(zAs, zCol)==0 + ){ + Expr *pOrig; + assert( pExpr->pLeft==0 && pExpr->pRight==0 ); + assert( pExpr->x.pList==0 ); + assert( pExpr->x.pSelect==0 ); + pOrig = pEList->a[j].pExpr; + if( (pNC->ncFlags&NC_AllowAgg)==0 && ExprHasProperty(pOrig, EP_Agg) ){ + sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); + return WRC_Abort; + } + if( ExprHasProperty(pOrig, EP_Win) + && ((pNC->ncFlags&NC_AllowWin)==0 || pNC!=pTopNC ) + ){ + sqlite3ErrorMsg(pParse, "misuse of aliased window function %s",zAs); + return WRC_Abort; + } + if( sqlite3ExprVectorSize(pOrig)!=1 ){ + sqlite3ErrorMsg(pParse, "row value misused"); + return WRC_Abort; + } + resolveAlias(pParse, pEList, j, pExpr, nSubquery); + cnt = 1; + pMatch = 0; + assert( zTab==0 && zDb==0 ); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pExpr); + } + goto lookupname_end; + } + } + } + + /* Advance to the next name context. The loop will exit when either + ** we have a match (cnt>0) or when we run out of name contexts. + */ + if( cnt ) break; + pNC = pNC->pNext; + nSubquery++; + }while( pNC ); + + + /* + ** If X and Y are NULL (in other words if only the column name Z is + ** supplied) and the value of Z is enclosed in double-quotes, then + ** Z is a string literal if it doesn't match any column names. In that + ** case, we need to return right away and not make any changes to + ** pExpr. + ** + ** Because no reference was made to outer contexts, the pNC->nRef + ** fields are not changed in any context. + */ + if( cnt==0 && zTab==0 ){ + assert( pExpr->op==TK_ID ); + if( ExprHasProperty(pExpr,EP_DblQuoted) + && areDoubleQuotedStringsEnabled(db, pTopNC) + ){ + /* If a double-quoted identifier does not match any known column name, + ** then treat it as a string. + ** + ** This hack was added in the early days of SQLite in a misguided attempt + ** to be compatible with MySQL 3.x, which used double-quotes for strings. + ** I now sorely regret putting in this hack. The effect of this hack is + ** that misspelled identifier names are silently converted into strings + ** rather than causing an error, to the frustration of countless + ** programmers. To all those frustrated programmers, my apologies. + ** + ** Someday, I hope to get rid of this hack. Unfortunately there is + ** a huge amount of legacy SQL that uses it. So for now, we just + ** issue a warning. + */ + sqlite3_log(SQLITE_WARNING, + "double-quoted string literal: \"%w\"", zCol); +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3VdbeAddDblquoteStr(db, pParse->pVdbe, zCol); +#endif + pExpr->op = TK_STRING; + pExpr->y.pTab = 0; + return WRC_Prune; + } + if( sqlite3ExprIdToTrueFalse(pExpr) ){ + return WRC_Prune; + } + } + + /* + ** cnt==0 means there was not match. cnt>1 means there were two or + ** more matches. Either way, we have an error. + */ + if( cnt!=1 ){ + const char *zErr; + zErr = cnt==0 ? "no such column" : "ambiguous column name"; + if( zDb ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); + }else if( zTab ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); + } + pParse->checkSchema = 1; + pTopNC->nNcErr++; + } + + /* If a column from a table in pSrcList is referenced, then record + ** this fact in the pSrcList.a[].colUsed bitmask. Column 0 causes + ** bit 0 to be set. Column 1 sets bit 1. And so forth. Bit 63 is + ** set if the 63rd or any subsequent column is used. + ** + ** The colUsed mask is an optimization used to help determine if an + ** index is a covering index. The correct answer is still obtained + ** if the mask contains extra set bits. However, it is important to + ** avoid setting bits beyond the maximum column number of the table. + ** (See ticket [b92e5e8ec2cdbaa1]). + ** + ** If a generated column is referenced, set bits for every column + ** of the table. + */ + if( pExpr->iColumn>=0 && pMatch!=0 ){ + pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + } + + /* Clean up and return + */ + if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ + sqlite3ExprDelete(db, pExpr->pLeft); + pExpr->pLeft = 0; + sqlite3ExprDelete(db, pExpr->pRight); + pExpr->pRight = 0; + } + pExpr->op = eNewExprOp; + ExprSetProperty(pExpr, EP_Leaf); +lookupname_end: + if( cnt==1 ){ + assert( pNC!=0 ); +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pParse->db->xAuth + && (pExpr->op==TK_COLUMN || pExpr->op==TK_TRIGGER) + ){ + sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + } +#endif + /* Increment the nRef value on all name contexts from TopNC up to + ** the point where the name matched. */ + for(;;){ + assert( pTopNC!=0 ); + pTopNC->nRef++; + if( pTopNC==pNC ) break; + pTopNC = pTopNC->pNext; + } + return WRC_Prune; + } else { + return WRC_Abort; + } +} + +/* +** Allocate and return a pointer to an expression to load the column iCol +** from datasource iSrc in SrcList pSrc. +*/ +SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ + Expr *p = sqlite3ExprAlloc(db, TK_COLUMN, 0, 0); + if( p ){ + SrcItem *pItem = &pSrc->a[iSrc]; + Table *pTab = p->y.pTab = pItem->pTab; + p->iTable = pItem->iCursor; + if( p->y.pTab->iPKey==iCol ){ + p->iColumn = -1; + }else{ + p->iColumn = (ynVar)iCol; + if( (pTab->tabFlags & TF_HasGenerated)!=0 + && (pTab->aCol[iCol].colFlags & COLFLAG_GENERATED)!=0 + ){ + testcase( pTab->nCol==63 ); + testcase( pTab->nCol==64 ); + pItem->colUsed = pTab->nCol>=64 ? ALLBITS : MASKBIT(pTab->nCol)-1; + }else{ + testcase( iCol==BMS ); + testcase( iCol==BMS-1 ); + pItem->colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } + } + } + return p; +} + +/* +** Report an error that an expression is not valid for some set of +** pNC->ncFlags values determined by validMask. +** +** static void notValid( +** Parse *pParse, // Leave error message here +** NameContext *pNC, // The name context +** const char *zMsg, // Type of error +** int validMask, // Set of contexts for which prohibited +** Expr *pExpr // Invalidate this expression on error +** ){...} +** +** As an optimization, since the conditional is almost always false +** (because errors are rare), the conditional is moved outside of the +** function call using a macro. +*/ +static void notValidImpl( + Parse *pParse, /* Leave error message here */ + NameContext *pNC, /* The name context */ + const char *zMsg, /* Type of error */ + Expr *pExpr /* Invalidate this expression on error */ +){ + const char *zIn = "partial index WHERE clauses"; + if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; +#ifndef SQLITE_OMIT_CHECK + else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; +#endif +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + else if( pNC->ncFlags & NC_GenCol ) zIn = "generated columns"; +#endif + sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); + if( pExpr ) pExpr->op = TK_NULL; +} +#define sqlite3ResolveNotValid(P,N,M,X,E) \ + assert( ((X)&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol))==0 ); \ + if( ((N)->ncFlags & (X))!=0 ) notValidImpl(P,N,M,E); + +/* +** Expression p should encode a floating point value between 1.0 and 0.0. +** Return 1024 times this value. Or return -1 if p is not a floating point +** value between 1.0 and 0.0. +*/ +static int exprProbability(Expr *p){ + double r = -1.0; + if( p->op!=TK_FLOAT ) return -1; + sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); + assert( r>=0.0 ); + if( r>1.0 ) return -1; + return (int)(r*134217728.0); +} + +/* +** This routine is callback for sqlite3WalkExpr(). +** +** Resolve symbolic names into TK_COLUMN operators for the current +** node in the expression tree. Return 0 to continue the search down +** the tree or 2 to abort the tree walk. +** +** This routine also does error checking and name resolution for +** function names. The operator for aggregate functions is changed +** to TK_AGG_FUNCTION. +*/ +static int resolveExprStep(Walker *pWalker, Expr *pExpr){ + NameContext *pNC; + Parse *pParse; + + pNC = pWalker->u.pNC; + assert( pNC!=0 ); + pParse = pNC->pParse; + assert( pParse==pWalker->pParse ); + +#ifndef NDEBUG + if( pNC->pSrcList && pNC->pSrcList->nAlloc>0 ){ + SrcList *pSrcList = pNC->pSrcList; + int i; + for(i=0; ipSrcList->nSrc; i++){ + assert( pSrcList->a[i].iCursor>=0 && pSrcList->a[i].iCursornTab); + } + } +#endif + switch( pExpr->op ){ + + /* The special operator TK_ROW means use the rowid for the first + ** column in the FROM clause. This is used by the LIMIT and ORDER BY + ** clause processing on UPDATE and DELETE statements, and by + ** UPDATE ... FROM statement processing. + */ + case TK_ROW: { + SrcList *pSrcList = pNC->pSrcList; + SrcItem *pItem; + assert( pSrcList && pSrcList->nSrc>=1 ); + pItem = pSrcList->a; + pExpr->op = TK_COLUMN; + pExpr->y.pTab = pItem->pTab; + pExpr->iTable = pItem->iCursor; + pExpr->iColumn--; + pExpr->affExpr = SQLITE_AFF_INTEGER; + break; + } + + /* An optimization: Attempt to convert + ** + ** "expr IS NOT NULL" --> "TRUE" + ** "expr IS NULL" --> "FALSE" + ** + ** if we can prove that "expr" is never NULL. Call this the + ** "NOT NULL strength reduction optimization". + ** + ** If this optimization occurs, also restore the NameContext ref-counts + ** to the state they where in before the "column" LHS expression was + ** resolved. This prevents "column" from being counted as having been + ** referenced, which might prevent a SELECT from being erroneously + ** marked as correlated. + */ + case TK_NOTNULL: + case TK_ISNULL: { + int anRef[8]; + NameContext *p; + int i; + for(i=0, p=pNC; p && ipNext, i++){ + anRef[i] = p->nRef; + } + sqlite3WalkExpr(pWalker, pExpr->pLeft); + if( 0==sqlite3ExprCanBeNull(pExpr->pLeft) && !IN_RENAME_OBJECT ){ + testcase( ExprHasProperty(pExpr, EP_FromJoin) ); + if( pExpr->op==TK_NOTNULL ){ + pExpr->u.zToken = "true"; + ExprSetProperty(pExpr, EP_IsTrue); + }else{ + pExpr->u.zToken = "false"; + ExprSetProperty(pExpr, EP_IsFalse); + } + pExpr->op = TK_TRUEFALSE; + for(i=0, p=pNC; p && ipNext, i++){ + p->nRef = anRef[i]; + } + sqlite3ExprDelete(pParse->db, pExpr->pLeft); + pExpr->pLeft = 0; + } + return WRC_Prune; + } + + /* A column name: ID + ** Or table name and column name: ID.ID + ** Or a database, table and column: ID.ID.ID + ** + ** The TK_ID and TK_OUT cases are combined so that there will only + ** be one call to lookupName(). Then the compiler will in-line + ** lookupName() for a size reduction and performance increase. + */ + case TK_ID: + case TK_DOT: { + const char *zColumn; + const char *zTable; + const char *zDb; + Expr *pRight; + + if( pExpr->op==TK_ID ){ + zDb = 0; + zTable = 0; + zColumn = pExpr->u.zToken; + }else{ + Expr *pLeft = pExpr->pLeft; + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + sqlite3ResolveNotValid(pParse, pNC, "the \".\" operator", + NC_IdxExpr|NC_GenCol, 0); + pRight = pExpr->pRight; + if( pRight->op==TK_ID ){ + zDb = 0; + }else{ + assert( pRight->op==TK_DOT ); + zDb = pLeft->u.zToken; + pLeft = pRight->pLeft; + pRight = pRight->pRight; + } + zTable = pLeft->u.zToken; + zColumn = pRight->u.zToken; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, (void*)pExpr, (void*)pRight); + sqlite3RenameTokenRemap(pParse, (void*)&pExpr->y.pTab, (void*)pLeft); + } + } + return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); + } + + /* Resolve function names + */ + case TK_FUNCTION: { + ExprList *pList = pExpr->x.pList; /* The argument list */ + int n = pList ? pList->nExpr : 0; /* Number of arguments */ + int no_such_func = 0; /* True if no such function exists */ + int wrong_num_args = 0; /* True if wrong number of arguments */ + int is_agg = 0; /* True if is an aggregate function */ + int nId; /* Number of characters in function name */ + const char *zId; /* The function name. */ + FuncDef *pDef; /* Information about the function */ + u8 enc = ENC(pParse->db); /* The database encoding */ + int savedAllowFlags = (pNC->ncFlags & (NC_AllowAgg | NC_AllowWin)); +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin = (IsWindowFunc(pExpr) ? pExpr->y.pWin : 0); +#endif + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + zId = pExpr->u.zToken; + nId = sqlite3Strlen30(zId); + pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); + if( pDef==0 ){ + pDef = sqlite3FindFunction(pParse->db, zId, -2, enc, 0); + if( pDef==0 ){ + no_such_func = 1; + }else{ + wrong_num_args = 1; + } + }else{ + is_agg = pDef->xFinalize!=0; + if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ + ExprSetProperty(pExpr, EP_Unlikely); + if( n==2 ){ + pExpr->iTable = exprProbability(pList->a[1].pExpr); + if( pExpr->iTable<0 ){ + sqlite3ErrorMsg(pParse, + "second argument to likelihood() must be a " + "constant between 0.0 and 1.0"); + pNC->nNcErr++; + } + }else{ + /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is + ** equivalent to likelihood(X, 0.0625). + ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is + ** short-hand for likelihood(X,0.0625). + ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand + ** for likelihood(X,0.9375). + ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent + ** to likelihood(X,0.9375). */ + /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ + pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; + } + } +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0,pDef->zName,0); + if( auth!=SQLITE_OK ){ + if( auth==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized to use function: %s", + pDef->zName); + pNC->nNcErr++; + } + pExpr->op = TK_NULL; + return WRC_Prune; + } + } +#endif + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ + /* For the purposes of the EP_ConstFunc flag, date and time + ** functions and other functions that change slowly are considered + ** constant because they are constant for the duration of one query. + ** This allows them to be factored out of inner loops. */ + ExprSetProperty(pExpr,EP_ConstFunc); + } + if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ + /* Clearly non-deterministic functions like random(), but also + ** date/time functions that use 'now', and other functions like + ** sqlite_version() that might change over time cannot be used + ** in an index or generated column. Curiously, they can be used + ** in a CHECK constraint. SQLServer, MySQL, and PostgreSQL all + ** all this. */ + sqlite3ResolveNotValid(pParse, pNC, "non-deterministic functions", + NC_IdxExpr|NC_PartIdx|NC_GenCol, 0); + }else{ + assert( (NC_SelfRef & 0xff)==NC_SelfRef ); /* Must fit in 8 bits */ + pExpr->op2 = pNC->ncFlags & NC_SelfRef; + if( pNC->ncFlags & NC_FromDDL ) ExprSetProperty(pExpr, EP_FromDDL); + } + if( (pDef->funcFlags & SQLITE_FUNC_INTERNAL)!=0 + && pParse->nested==0 + && (pParse->db->mDbFlags & DBFLAG_InternalFunc)==0 + ){ + /* Internal-use-only functions are disallowed unless the + ** SQL is being compiled using sqlite3NestedParse() or + ** the SQLITE_TESTCTRL_INTERNAL_FUNCTIONS test-control has be + ** used to activate internal functionsn for testing purposes */ + no_such_func = 1; + pDef = 0; + }else + if( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 + && !IN_RENAME_OBJECT + ){ + sqlite3ExprFunctionUsable(pParse, pExpr, pDef); + } + } + + if( 0==IN_RENAME_OBJECT ){ +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( is_agg==0 || (pDef->funcFlags & SQLITE_FUNC_MINMAX) + || (pDef->xValue==0 && pDef->xInverse==0) + || (pDef->xValue && pDef->xInverse && pDef->xSFunc && pDef->xFinalize) + ); + if( pDef && pDef->xValue==0 && pWin ){ + sqlite3ErrorMsg(pParse, + "%.*s() may not be used as a window function", nId, zId + ); + pNC->nNcErr++; + }else if( + (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) + || (is_agg && (pDef->funcFlags&SQLITE_FUNC_WINDOW) && !pWin) + || (is_agg && pWin && (pNC->ncFlags & NC_AllowWin)==0) + ){ + const char *zType; + if( (pDef->funcFlags & SQLITE_FUNC_WINDOW) || pWin ){ + zType = "window"; + }else{ + zType = "aggregate"; + } + sqlite3ErrorMsg(pParse, "misuse of %s function %.*s()",zType,nId,zId); + pNC->nNcErr++; + is_agg = 0; + } +#else + if( (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) ){ + sqlite3ErrorMsg(pParse,"misuse of aggregate function %.*s()",nId,zId); + pNC->nNcErr++; + is_agg = 0; + } +#endif + else if( no_such_func && pParse->db->init.busy==0 +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + && pParse->explain==0 +#endif + ){ + sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); + pNC->nNcErr++; + }else if( wrong_num_args ){ + sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", + nId, zId); + pNC->nNcErr++; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + else if( is_agg==0 && ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ErrorMsg(pParse, + "FILTER may not be used with non-aggregate %.*s()", + nId, zId + ); + pNC->nNcErr++; + } +#endif + if( is_agg ){ + /* Window functions may not be arguments of aggregate functions. + ** Or arguments of other window functions. But aggregate functions + ** may be arguments for window functions. */ +#ifndef SQLITE_OMIT_WINDOWFUNC + pNC->ncFlags &= ~(NC_AllowWin | (!pWin ? NC_AllowAgg : 0)); +#else + pNC->ncFlags &= ~NC_AllowAgg; +#endif + } + } +#ifndef SQLITE_OMIT_WINDOWFUNC + else if( ExprHasProperty(pExpr, EP_WinFunc) ){ + is_agg = 1; + } +#endif + sqlite3WalkExprList(pWalker, pList); + if( is_agg ){ +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + Select *pSel = pNC->pWinSelect; + assert( pWin==pExpr->y.pWin ); + if( IN_RENAME_OBJECT==0 ){ + sqlite3WindowUpdate(pParse, pSel ? pSel->pWinDefn : 0, pWin, pDef); + if( pParse->db->mallocFailed ) break; + } + sqlite3WalkExprList(pWalker, pWin->pPartition); + sqlite3WalkExprList(pWalker, pWin->pOrderBy); + sqlite3WalkExpr(pWalker, pWin->pFilter); + sqlite3WindowLink(pSel, pWin); + pNC->ncFlags |= NC_HasWin; + }else +#endif /* SQLITE_OMIT_WINDOWFUNC */ + { + NameContext *pNC2 = pNC; + pExpr->op = TK_AGG_FUNCTION; + pExpr->op2 = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3WalkExpr(pWalker, pExpr->y.pWin->pFilter); + } +#endif + while( pNC2 && !sqlite3FunctionUsesThisSrc(pExpr, pNC2->pSrcList) ){ + pExpr->op2++; + pNC2 = pNC2->pNext; + } + assert( pDef!=0 || IN_RENAME_OBJECT ); + if( pNC2 && pDef ){ + assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); + assert( SQLITE_FUNC_ANYORDER==NC_OrderAgg ); + testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); + testcase( (pDef->funcFlags & SQLITE_FUNC_ANYORDER)!=0 ); + pNC2->ncFlags |= NC_HasAgg + | ((pDef->funcFlags^SQLITE_FUNC_ANYORDER) + & (SQLITE_FUNC_MINMAX|SQLITE_FUNC_ANYORDER)); + } + } + pNC->ncFlags |= savedAllowFlags; + } + /* FIX ME: Compute pExpr->affinity based on the expected return + ** type of the function + */ + return WRC_Prune; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_SELECT: + case TK_EXISTS: testcase( pExpr->op==TK_EXISTS ); +#endif + case TK_IN: { + testcase( pExpr->op==TK_IN ); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + int nRef = pNC->nRef; + testcase( pNC->ncFlags & NC_IsCheck ); + testcase( pNC->ncFlags & NC_PartIdx ); + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + if( pNC->ncFlags & NC_SelfRef ){ + notValidImpl(pParse, pNC, "subqueries", pExpr); + }else{ + sqlite3WalkSelect(pWalker, pExpr->x.pSelect); + } + assert( pNC->nRef>=nRef ); + if( nRef!=pNC->nRef ){ + ExprSetProperty(pExpr, EP_VarSelect); + pNC->ncFlags |= NC_VarSelect; + } + } + break; + } + case TK_VARIABLE: { + testcase( pNC->ncFlags & NC_IsCheck ); + testcase( pNC->ncFlags & NC_PartIdx ); + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + sqlite3ResolveNotValid(pParse, pNC, "parameters", + NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol, pExpr); + break; + } + case TK_IS: + case TK_ISNOT: { + Expr *pRight = sqlite3ExprSkipCollateAndLikely(pExpr->pRight); + assert( !ExprHasProperty(pExpr, EP_Reduced) ); + /* Handle special cases of "x IS TRUE", "x IS FALSE", "x IS NOT TRUE", + ** and "x IS NOT FALSE". */ + if( ALWAYS(pRight) && (pRight->op==TK_ID || pRight->op==TK_TRUEFALSE) ){ + int rc = resolveExprStep(pWalker, pRight); + if( rc==WRC_Abort ) return WRC_Abort; + if( pRight->op==TK_TRUEFALSE ){ + pExpr->op2 = pExpr->op; + pExpr->op = TK_TRUTH; + return WRC_Continue; + } + } + /* no break */ deliberate_fall_through + } + case TK_BETWEEN: + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: { + int nLeft, nRight; + if( pParse->db->mallocFailed ) break; + assert( pExpr->pLeft!=0 ); + nLeft = sqlite3ExprVectorSize(pExpr->pLeft); + if( pExpr->op==TK_BETWEEN ){ + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[0].pExpr); + if( nRight==nLeft ){ + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[1].pExpr); + } + }else{ + assert( pExpr->pRight!=0 ); + nRight = sqlite3ExprVectorSize(pExpr->pRight); + } + if( nLeft!=nRight ){ + testcase( pExpr->op==TK_EQ ); + testcase( pExpr->op==TK_NE ); + testcase( pExpr->op==TK_LT ); + testcase( pExpr->op==TK_LE ); + testcase( pExpr->op==TK_GT ); + testcase( pExpr->op==TK_GE ); + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + testcase( pExpr->op==TK_BETWEEN ); + sqlite3ErrorMsg(pParse, "row value misused"); + } + break; + } + } + return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; +} + +/* +** pEList is a list of expressions which are really the result set of the +** a SELECT statement. pE is a term in an ORDER BY or GROUP BY clause. +** This routine checks to see if pE is a simple identifier which corresponds +** to the AS-name of one of the terms of the expression list. If it is, +** this routine return an integer between 1 and N where N is the number of +** elements in pEList, corresponding to the matching entry. If there is +** no match, or if pE is not a simple identifier, then this routine +** return 0. +** +** pEList has been resolved. pE has not. +*/ +static int resolveAsName( + Parse *pParse, /* Parsing context for error messages */ + ExprList *pEList, /* List of expressions to scan */ + Expr *pE /* Expression we are trying to match */ +){ + int i; /* Loop counter */ + + UNUSED_PARAMETER(pParse); + + if( pE->op==TK_ID ){ + char *zCol = pE->u.zToken; + for(i=0; inExpr; i++){ + if( pEList->a[i].eEName==ENAME_NAME + && sqlite3_stricmp(pEList->a[i].zEName, zCol)==0 + ){ + return i+1; + } + } + } + return 0; +} + +/* +** pE is a pointer to an expression which is a single term in the +** ORDER BY of a compound SELECT. The expression has not been +** name resolved. +** +** At the point this routine is called, we already know that the +** ORDER BY term is not an integer index into the result set. That +** case is handled by the calling routine. +** +** Attempt to match pE against result set columns in the left-most +** SELECT statement. Return the index i of the matching column, +** as an indication to the caller that it should sort by the i-th column. +** The left-most column is 1. In other words, the value returned is the +** same integer value that would be used in the SQL statement to indicate +** the column. +** +** If there is no match, return 0. Return -1 if an error occurs. +*/ +static int resolveOrderByTermToExprList( + Parse *pParse, /* Parsing context for error messages */ + Select *pSelect, /* The SELECT statement with the ORDER BY clause */ + Expr *pE /* The specific ORDER BY term */ +){ + int i; /* Loop counter */ + ExprList *pEList; /* The columns of the result set */ + NameContext nc; /* Name context for resolving pE */ + sqlite3 *db; /* Database connection */ + int rc; /* Return code from subprocedures */ + u8 savedSuppErr; /* Saved value of db->suppressErr */ + + assert( sqlite3ExprIsInteger(pE, &i)==0 ); + pEList = pSelect->pEList; + + /* Resolve all names in the ORDER BY term expression + */ + memset(&nc, 0, sizeof(nc)); + nc.pParse = pParse; + nc.pSrcList = pSelect->pSrc; + nc.uNC.pEList = pEList; + nc.ncFlags = NC_AllowAgg|NC_UEList|NC_NoSelect; + nc.nNcErr = 0; + db = pParse->db; + savedSuppErr = db->suppressErr; + db->suppressErr = 1; + rc = sqlite3ResolveExprNames(&nc, pE); + db->suppressErr = savedSuppErr; + if( rc ) return 0; + + /* Try to match the ORDER BY expression against an expression + ** in the result set. Return an 1-based index of the matching + ** result-set entry. + */ + for(i=0; inExpr; i++){ + if( sqlite3ExprCompare(0, pEList->a[i].pExpr, pE, -1)<2 ){ + return i+1; + } + } + + /* If no match, return 0. */ + return 0; +} + +/* +** Generate an ORDER BY or GROUP BY term out-of-range error. +*/ +static void resolveOutOfRangeError( + Parse *pParse, /* The error context into which to write the error */ + const char *zType, /* "ORDER" or "GROUP" */ + int i, /* The index (1-based) of the term out of range */ + int mx /* Largest permissible value of i */ +){ + sqlite3ErrorMsg(pParse, + "%r %s BY term out of range - should be " + "between 1 and %d", i, zType, mx); +} + +/* +** Analyze the ORDER BY clause in a compound SELECT statement. Modify +** each term of the ORDER BY clause is a constant integer between 1 +** and N where N is the number of columns in the compound SELECT. +** +** ORDER BY terms that are already an integer between 1 and N are +** unmodified. ORDER BY terms that are integers outside the range of +** 1 through N generate an error. ORDER BY terms that are expressions +** are matched against result set expressions of compound SELECT +** beginning with the left-most SELECT and working toward the right. +** At the first match, the ORDER BY expression is transformed into +** the integer column number. +** +** Return the number of errors seen. +*/ +static int resolveCompoundOrderBy( + Parse *pParse, /* Parsing context. Leave error messages here */ + Select *pSelect /* The SELECT statement containing the ORDER BY */ +){ + int i; + ExprList *pOrderBy; + ExprList *pEList; + sqlite3 *db; + int moreToDo = 1; + + pOrderBy = pSelect->pOrderBy; + if( pOrderBy==0 ) return 0; + db = pParse->db; + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); + return 1; + } + for(i=0; inExpr; i++){ + pOrderBy->a[i].done = 0; + } + pSelect->pNext = 0; + while( pSelect->pPrior ){ + pSelect->pPrior->pNext = pSelect; + pSelect = pSelect->pPrior; + } + while( pSelect && moreToDo ){ + struct ExprList_item *pItem; + moreToDo = 0; + pEList = pSelect->pEList; + assert( pEList!=0 ); + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + int iCol = -1; + Expr *pE, *pDup; + if( pItem->done ) continue; + pE = sqlite3ExprSkipCollateAndLikely(pItem->pExpr); + if( NEVER(pE==0) ) continue; + if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( iCol<=0 || iCol>pEList->nExpr ){ + resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); + return 1; + } + }else{ + iCol = resolveAsName(pParse, pEList, pE); + if( iCol==0 ){ + /* Now test if expression pE matches one of the values returned + ** by pSelect. In the usual case this is done by duplicating the + ** expression, resolving any symbols in it, and then comparing + ** it against each expression returned by the SELECT statement. + ** Once the comparisons are finished, the duplicate expression + ** is deleted. + ** + ** If this is running as part of an ALTER TABLE operation and + ** the symbols resolve successfully, also resolve the symbols in the + ** actual expression. This allows the code in alter.c to modify + ** column references within the ORDER BY expression as required. */ + pDup = sqlite3ExprDup(db, pE, 0); + if( !db->mallocFailed ){ + assert(pDup); + iCol = resolveOrderByTermToExprList(pParse, pSelect, pDup); + if( IN_RENAME_OBJECT && iCol>0 ){ + resolveOrderByTermToExprList(pParse, pSelect, pE); + } + } + sqlite3ExprDelete(db, pDup); + } + } + if( iCol>0 ){ + /* Convert the ORDER BY term into an integer column number iCol, + ** taking care to preserve the COLLATE clause if it exists. */ + if( !IN_RENAME_OBJECT ){ + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + if( pNew==0 ) return 1; + pNew->flags |= EP_IntValue; + pNew->u.iValue = iCol; + if( pItem->pExpr==pE ){ + pItem->pExpr = pNew; + }else{ + Expr *pParent = pItem->pExpr; + assert( pParent->op==TK_COLLATE ); + while( pParent->pLeft->op==TK_COLLATE ) pParent = pParent->pLeft; + assert( pParent->pLeft==pE ); + pParent->pLeft = pNew; + } + sqlite3ExprDelete(db, pE); + pItem->u.x.iOrderByCol = (u16)iCol; + } + pItem->done = 1; + }else{ + moreToDo = 1; + } + } + pSelect = pSelect->pNext; + } + for(i=0; inExpr; i++){ + if( pOrderBy->a[i].done==0 ){ + sqlite3ErrorMsg(pParse, "%r ORDER BY term does not match any " + "column in the result set", i+1); + return 1; + } + } + return 0; +} + +/* +** Check every term in the ORDER BY or GROUP BY clause pOrderBy of +** the SELECT statement pSelect. If any term is reference to a +** result set expression (as determined by the ExprList.a.u.x.iOrderByCol +** field) then convert that term into a copy of the corresponding result set +** column. +** +** If any errors are detected, add an error message to pParse and +** return non-zero. Return zero if no errors are seen. +*/ +SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( + Parse *pParse, /* Parsing context. Leave error messages here */ + Select *pSelect, /* The SELECT statement containing the clause */ + ExprList *pOrderBy, /* The ORDER BY or GROUP BY clause to be processed */ + const char *zType /* "ORDER" or "GROUP" */ +){ + int i; + sqlite3 *db = pParse->db; + ExprList *pEList; + struct ExprList_item *pItem; + + if( pOrderBy==0 || pParse->db->mallocFailed || IN_RENAME_OBJECT ) return 0; + if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); + return 1; + } + pEList = pSelect->pEList; + assert( pEList!=0 ); /* sqlite3SelectNew() guarantees this */ + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + if( pItem->u.x.iOrderByCol ){ + if( pItem->u.x.iOrderByCol>pEList->nExpr ){ + resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); + return 1; + } + resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr,0); + } + } + return 0; +} + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Walker callback for windowRemoveExprFromSelect(). +*/ +static int resolveRemoveWindowsCb(Walker *pWalker, Expr *pExpr){ + UNUSED_PARAMETER(pWalker); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + Window *pWin = pExpr->y.pWin; + sqlite3WindowUnlinkFromSelect(pWin); + } + return WRC_Continue; +} + +/* +** Remove any Window objects owned by the expression pExpr from the +** Select.pWin list of Select object pSelect. +*/ +static void windowRemoveExprFromSelect(Select *pSelect, Expr *pExpr){ + if( pSelect->pWin ){ + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.xExprCallback = resolveRemoveWindowsCb; + sWalker.u.pSelect = pSelect; + sqlite3WalkExpr(&sWalker, pExpr); + } +} +#else +# define windowRemoveExprFromSelect(a, b) +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** pOrderBy is an ORDER BY or GROUP BY clause in SELECT statement pSelect. +** The Name context of the SELECT statement is pNC. zType is either +** "ORDER" or "GROUP" depending on which type of clause pOrderBy is. +** +** This routine resolves each term of the clause into an expression. +** If the order-by term is an integer I between 1 and N (where N is the +** number of columns in the result set of the SELECT) then the expression +** in the resolution is a copy of the I-th result-set expression. If +** the order-by term is an identifier that corresponds to the AS-name of +** a result-set expression, then the term resolves to a copy of the +** result-set expression. Otherwise, the expression is resolved in +** the usual way - using sqlite3ResolveExprNames(). +** +** This routine returns the number of errors. If errors occur, then +** an appropriate error message might be left in pParse. (OOM errors +** excepted.) +*/ +static int resolveOrderGroupBy( + NameContext *pNC, /* The name context of the SELECT statement */ + Select *pSelect, /* The SELECT statement holding pOrderBy */ + ExprList *pOrderBy, /* An ORDER BY or GROUP BY clause to resolve */ + const char *zType /* Either "ORDER" or "GROUP", as appropriate */ +){ + int i, j; /* Loop counters */ + int iCol; /* Column number */ + struct ExprList_item *pItem; /* A term of the ORDER BY clause */ + Parse *pParse; /* Parsing context */ + int nResult; /* Number of terms in the result set */ + + assert( pOrderBy!=0 ); + nResult = pSelect->pEList->nExpr; + pParse = pNC->pParse; + for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ + Expr *pE = pItem->pExpr; + Expr *pE2 = sqlite3ExprSkipCollateAndLikely(pE); + if( NEVER(pE2==0) ) continue; + if( zType[0]!='G' ){ + iCol = resolveAsName(pParse, pSelect->pEList, pE2); + if( iCol>0 ){ + /* If an AS-name match is found, mark this ORDER BY column as being + ** a copy of the iCol-th result-set column. The subsequent call to + ** sqlite3ResolveOrderGroupBy() will convert the expression to a + ** copy of the iCol-th result-set expression. */ + pItem->u.x.iOrderByCol = (u16)iCol; + continue; + } + } + if( sqlite3ExprIsInteger(pE2, &iCol) ){ + /* The ORDER BY term is an integer constant. Again, set the column + ** number so that sqlite3ResolveOrderGroupBy() will convert the + ** order-by term to a copy of the result-set expression */ + if( iCol<1 || iCol>0xffff ){ + resolveOutOfRangeError(pParse, zType, i+1, nResult); + return 1; + } + pItem->u.x.iOrderByCol = (u16)iCol; + continue; + } + + /* Otherwise, treat the ORDER BY term as an ordinary expression */ + pItem->u.x.iOrderByCol = 0; + if( sqlite3ResolveExprNames(pNC, pE) ){ + return 1; + } + for(j=0; jpEList->nExpr; j++){ + if( sqlite3ExprCompare(0, pE, pSelect->pEList->a[j].pExpr, -1)==0 ){ + /* Since this expresion is being changed into a reference + ** to an identical expression in the result set, remove all Window + ** objects belonging to the expression from the Select.pWin list. */ + windowRemoveExprFromSelect(pSelect, pE); + pItem->u.x.iOrderByCol = j+1; + } + } + } + return sqlite3ResolveOrderGroupBy(pParse, pSelect, pOrderBy, zType); +} + +/* +** Resolve names in the SELECT statement p and all of its descendants. +*/ +static int resolveSelectStep(Walker *pWalker, Select *p){ + NameContext *pOuterNC; /* Context that contains this SELECT */ + NameContext sNC; /* Name context of this SELECT */ + int isCompound; /* True if p is a compound select */ + int nCompound; /* Number of compound terms processed so far */ + Parse *pParse; /* Parsing context */ + int i; /* Loop counter */ + ExprList *pGroupBy; /* The GROUP BY clause */ + Select *pLeftmost; /* Left-most of SELECT of a compound */ + sqlite3 *db; /* Database connection */ + + + assert( p!=0 ); + if( p->selFlags & SF_Resolved ){ + return WRC_Prune; + } + pOuterNC = pWalker->u.pNC; + pParse = pWalker->pParse; + db = pParse->db; + + /* Normally sqlite3SelectExpand() will be called first and will have + ** already expanded this SELECT. However, if this is a subquery within + ** an expression, sqlite3ResolveExprNames() will be called without a + ** prior call to sqlite3SelectExpand(). When that happens, let + ** sqlite3SelectPrep() do all of the processing for this SELECT. + ** sqlite3SelectPrep() will invoke both sqlite3SelectExpand() and + ** this routine in the correct order. + */ + if( (p->selFlags & SF_Expanded)==0 ){ + sqlite3SelectPrep(pParse, p, pOuterNC); + return (pParse->nErr || db->mallocFailed) ? WRC_Abort : WRC_Prune; + } + + isCompound = p->pPrior!=0; + nCompound = 0; + pLeftmost = p; + while( p ){ + assert( (p->selFlags & SF_Expanded)!=0 ); + assert( (p->selFlags & SF_Resolved)==0 ); + assert( db->suppressErr==0 ); /* SF_Resolved not set if errors suppressed */ + p->selFlags |= SF_Resolved; + + + /* Resolve the expressions in the LIMIT and OFFSET clauses. These + ** are not allowed to refer to any names, so pass an empty NameContext. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pWinSelect = p; + if( sqlite3ResolveExprNames(&sNC, p->pLimit) ){ + return WRC_Abort; + } + + /* If the SF_Converted flags is set, then this Select object was + ** was created by the convertCompoundSelectToSubquery() function. + ** In this case the ORDER BY clause (p->pOrderBy) should be resolved + ** as if it were part of the sub-query, not the parent. This block + ** moves the pOrderBy down to the sub-query. It will be moved back + ** after the names have been resolved. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + assert( p->pSrc->nSrc==1 && p->pOrderBy ); + assert( pSub->pPrior && pSub->pOrderBy==0 ); + pSub->pOrderBy = p->pOrderBy; + p->pOrderBy = 0; + } + + /* Recursively resolve names in all subqueries in the FROM clause + */ + for(i=0; ipSrc->nSrc; i++){ + SrcItem *pItem = &p->pSrc->a[i]; + if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){ + int nRef = pOuterNC ? pOuterNC->nRef : 0; + const char *zSavedContext = pParse->zAuthContext; + + if( pItem->zName ) pParse->zAuthContext = pItem->zName; + sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); + pParse->zAuthContext = zSavedContext; + if( pParse->nErr || db->mallocFailed ) return WRC_Abort; + + /* If the number of references to the outer context changed when + ** expressions in the sub-select were resolved, the sub-select + ** is correlated. It is not required to check the refcount on any + ** but the innermost outer context object, as lookupName() increments + ** the refcount on all contexts between the current one and the + ** context containing the column when it resolves a name. */ + if( pOuterNC ){ + assert( pItem->fg.isCorrelated==0 && pOuterNC->nRef>=nRef ); + pItem->fg.isCorrelated = (pOuterNC->nRef>nRef); + } + } + } + + /* Set up the local name-context to pass to sqlite3ResolveExprNames() to + ** resolve the result-set expression list. + */ + sNC.ncFlags = NC_AllowAgg|NC_AllowWin; + sNC.pSrcList = p->pSrc; + sNC.pNext = pOuterNC; + + /* Resolve names in the result set. */ + if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; + sNC.ncFlags &= ~NC_AllowWin; + + /* If there are no aggregate functions in the result-set, and no GROUP BY + ** expression, do not allow aggregates in any of the other expressions. + */ + assert( (p->selFlags & SF_Aggregate)==0 ); + pGroupBy = p->pGroupBy; + if( pGroupBy || (sNC.ncFlags & NC_HasAgg)!=0 ){ + assert( NC_MinMaxAgg==SF_MinMaxAgg ); + assert( NC_OrderAgg==SF_OrderByReqd ); + p->selFlags |= SF_Aggregate | (sNC.ncFlags&(NC_MinMaxAgg|NC_OrderAgg)); + }else{ + sNC.ncFlags &= ~NC_AllowAgg; + } + + /* Add the output column list to the name-context before parsing the + ** other expressions in the SELECT statement. This is so that + ** expressions in the WHERE clause (etc.) can refer to expressions by + ** aliases in the result set. + ** + ** Minor point: If this is the case, then the expression will be + ** re-evaluated for each reference to it. + */ + assert( (sNC.ncFlags & (NC_UAggInfo|NC_UUpsert|NC_UBaseReg))==0 ); + sNC.uNC.pEList = p->pEList; + sNC.ncFlags |= NC_UEList; + if( p->pHaving ){ + if( !pGroupBy ){ + sqlite3ErrorMsg(pParse, "a GROUP BY clause is required before HAVING"); + return WRC_Abort; + } + if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; + } + if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + + /* Resolve names in table-valued-function arguments */ + for(i=0; ipSrc->nSrc; i++){ + SrcItem *pItem = &p->pSrc->a[i]; + if( pItem->fg.isTabFunc + && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } + } + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( IN_RENAME_OBJECT ){ + Window *pWin; + for(pWin=p->pWinDefn; pWin; pWin=pWin->pNextWin){ + if( sqlite3ResolveExprListNames(&sNC, pWin->pOrderBy) + || sqlite3ResolveExprListNames(&sNC, pWin->pPartition) + ){ + return WRC_Abort; + } + } + } +#endif + + /* The ORDER BY and GROUP BY clauses may not refer to terms in + ** outer queries + */ + sNC.pNext = 0; + sNC.ncFlags |= NC_AllowAgg|NC_AllowWin; + + /* If this is a converted compound query, move the ORDER BY clause from + ** the sub-query back to the parent query. At this point each term + ** within the ORDER BY clause has been transformed to an integer value. + ** These integers will be replaced by copies of the corresponding result + ** set expressions by the call to resolveOrderGroupBy() below. */ + if( p->selFlags & SF_Converted ){ + Select *pSub = p->pSrc->a[0].pSelect; + p->pOrderBy = pSub->pOrderBy; + pSub->pOrderBy = 0; + } + + /* Process the ORDER BY clause for singleton SELECT statements. + ** The ORDER BY clause for compounds SELECT statements is handled + ** below, after all of the result-sets for all of the elements of + ** the compound have been resolved. + ** + ** If there is an ORDER BY clause on a term of a compound-select other + ** than the right-most term, then that is a syntax error. But the error + ** is not detected until much later, and so we need to go ahead and + ** resolve those symbols on the incorrect ORDER BY for consistency. + */ + if( p->pOrderBy!=0 + && isCompound<=nCompound /* Defer right-most ORDER BY of a compound */ + && resolveOrderGroupBy(&sNC, p, p->pOrderBy, "ORDER") + ){ + return WRC_Abort; + } + if( db->mallocFailed ){ + return WRC_Abort; + } + sNC.ncFlags &= ~NC_AllowWin; + + /* Resolve the GROUP BY clause. At the same time, make sure + ** the GROUP BY clause does not contain aggregate functions. + */ + if( pGroupBy ){ + struct ExprList_item *pItem; + + if( resolveOrderGroupBy(&sNC, p, pGroupBy, "GROUP") || db->mallocFailed ){ + return WRC_Abort; + } + for(i=0, pItem=pGroupBy->a; inExpr; i++, pItem++){ + if( ExprHasProperty(pItem->pExpr, EP_Agg) ){ + sqlite3ErrorMsg(pParse, "aggregate functions are not allowed in " + "the GROUP BY clause"); + return WRC_Abort; + } + } + } + + /* If this is part of a compound SELECT, check that it has the right + ** number of expressions in the select list. */ + if( p->pNext && p->pEList->nExpr!=p->pNext->pEList->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, p->pNext); + return WRC_Abort; + } + + /* Advance to the next term of the compound + */ + p = p->pPrior; + nCompound++; + } + + /* Resolve the ORDER BY on a compound SELECT after all terms of + ** the compound have been resolved. + */ + if( isCompound && resolveCompoundOrderBy(pParse, pLeftmost) ){ + return WRC_Abort; + } + + return WRC_Prune; +} + +/* +** This routine walks an expression tree and resolves references to +** table columns and result-set columns. At the same time, do error +** checking on function usage and set a flag if any aggregate functions +** are seen. +** +** To resolve table columns references we look for nodes (or subtrees) of the +** form X.Y.Z or Y.Z or just Z where +** +** X: The name of a database. Ex: "main" or "temp" or +** the symbolic name assigned to an ATTACH-ed database. +** +** Y: The name of a table in a FROM clause. Or in a trigger +** one of the special names "old" or "new". +** +** Z: The name of a column in table Y. +** +** The node at the root of the subtree is modified as follows: +** +** Expr.op Changed to TK_COLUMN +** Expr.pTab Points to the Table object for X.Y +** Expr.iColumn The column index in X.Y. -1 for the rowid. +** Expr.iTable The VDBE cursor number for X.Y +** +** +** To resolve result-set references, look for expression nodes of the +** form Z (with no X and Y prefix) where the Z matches the right-hand +** size of an AS clause in the result-set of a SELECT. The Z expression +** is replaced by a copy of the left-hand side of the result-set expression. +** Table-name and function resolution occurs on the substituted expression +** tree. For example, in: +** +** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY x; +** +** The "x" term of the order by is replaced by "a+b" to render: +** +** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY a+b; +** +** Function calls are checked to make sure that the function is +** defined and that the correct number of arguments are specified. +** If the function is an aggregate function, then the NC_HasAgg flag is +** set and the opcode is changed from TK_FUNCTION to TK_AGG_FUNCTION. +** If an expression contains aggregate functions then the EP_Agg +** property on the expression is set. +** +** An error message is left in pParse if anything is amiss. The number +** if errors is returned. +*/ +SQLITE_PRIVATE int sqlite3ResolveExprNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + Expr *pExpr /* The expression to be analyzed. */ +){ + int savedHasAgg; + Walker w; + + if( pExpr==0 ) return SQLITE_OK; + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); + w.pParse = pNC->pParse; + w.xExprCallback = resolveExprStep; + w.xSelectCallback = (pNC->ncFlags & NC_NoSelect) ? 0 : resolveSelectStep; + w.xSelectCallback2 = 0; + w.u.pNC = pNC; +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight += pExpr->nHeight; + if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ + return SQLITE_ERROR; + } +#endif + sqlite3WalkExpr(&w, pExpr); +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight -= pExpr->nHeight; +#endif + assert( EP_Agg==NC_HasAgg ); + assert( EP_Win==NC_HasWin ); + testcase( pNC->ncFlags & NC_HasAgg ); + testcase( pNC->ncFlags & NC_HasWin ); + ExprSetProperty(pExpr, pNC->ncFlags & (NC_HasAgg|NC_HasWin) ); + pNC->ncFlags |= savedHasAgg; + return pNC->nNcErr>0 || w.pParse->nErr>0; +} + +/* +** Resolve all names for all expression in an expression list. This is +** just like sqlite3ResolveExprNames() except that it works for an expression +** list rather than a single expression. +*/ +SQLITE_PRIVATE int sqlite3ResolveExprListNames( + NameContext *pNC, /* Namespace to resolve expressions in. */ + ExprList *pList /* The expression list to be analyzed. */ +){ + int i; + int savedHasAgg = 0; + Walker w; + if( pList==0 ) return WRC_Continue; + w.pParse = pNC->pParse; + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.xSelectCallback2 = 0; + w.u.pNC = pNC; + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + if( pExpr==0 ) continue; +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight += pExpr->nHeight; + if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ + return WRC_Abort; + } +#endif + sqlite3WalkExpr(&w, pExpr); +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight -= pExpr->nHeight; +#endif + assert( EP_Agg==NC_HasAgg ); + assert( EP_Win==NC_HasWin ); + testcase( pNC->ncFlags & NC_HasAgg ); + testcase( pNC->ncFlags & NC_HasWin ); + if( pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg) ){ + ExprSetProperty(pExpr, pNC->ncFlags & (NC_HasAgg|NC_HasWin) ); + savedHasAgg |= pNC->ncFlags & + (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); + } + if( w.pParse->nErr>0 ) return WRC_Abort; + } + pNC->ncFlags |= savedHasAgg; + return WRC_Continue; +} + +/* +** Resolve all names in all expressions of a SELECT and in all +** decendents of the SELECT, including compounds off of p->pPrior, +** subqueries in expressions, and subqueries used as FROM clause +** terms. +** +** See sqlite3ResolveExprNames() for a description of the kinds of +** transformations that occur. +** +** All SELECT statements should have been expanded using +** sqlite3SelectExpand() prior to invoking this routine. +*/ +SQLITE_PRIVATE void sqlite3ResolveSelectNames( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + NameContext *pOuterNC /* Name context for parent SELECT statement */ +){ + Walker w; + + assert( p!=0 ); + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.xSelectCallback2 = 0; + w.pParse = pParse; + w.u.pNC = pOuterNC; + sqlite3WalkSelect(&w, p); +} + +/* +** Resolve names in expressions that can only reference a single table +** or which cannot reference any tables at all. Examples: +** +** "type" flag +** ------------ +** (1) CHECK constraints NC_IsCheck +** (2) WHERE clauses on partial indices NC_PartIdx +** (3) Expressions in indexes on expressions NC_IdxExpr +** (4) Expression arguments to VACUUM INTO. 0 +** (5) GENERATED ALWAYS as expressions NC_GenCol +** +** In all cases except (4), the Expr.iTable value for Expr.op==TK_COLUMN +** nodes of the expression is set to -1 and the Expr.iColumn value is +** set to the column number. In case (4), TK_COLUMN nodes cause an error. +** +** Any errors cause an error message to be set in pParse. +*/ +SQLITE_PRIVATE int sqlite3ResolveSelfReference( + Parse *pParse, /* Parsing context */ + Table *pTab, /* The table being referenced, or NULL */ + int type, /* NC_IsCheck, NC_PartIdx, NC_IdxExpr, NC_GenCol, or 0 */ + Expr *pExpr, /* Expression to resolve. May be NULL. */ + ExprList *pList /* Expression list to resolve. May be NULL. */ +){ + SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ + NameContext sNC; /* Name context for pParse->pNewTable */ + int rc; + + assert( type==0 || pTab!=0 ); + assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr + || type==NC_GenCol || pTab==0 ); + memset(&sNC, 0, sizeof(sNC)); + memset(&sSrc, 0, sizeof(sSrc)); + if( pTab ){ + sSrc.nSrc = 1; + sSrc.a[0].zName = pTab->zName; + sSrc.a[0].pTab = pTab; + sSrc.a[0].iCursor = -1; + if( pTab->pSchema!=pParse->db->aDb[1].pSchema ){ + /* Cause EP_FromDDL to be set on TK_FUNCTION nodes of non-TEMP + ** schema elements */ + type |= NC_FromDDL; + } + } + sNC.pParse = pParse; + sNC.pSrcList = &sSrc; + sNC.ncFlags = type | NC_IsDDL; + if( (rc = sqlite3ResolveExprNames(&sNC, pExpr))!=SQLITE_OK ) return rc; + if( pList ) rc = sqlite3ResolveExprListNames(&sNC, pList); + return rc; +} + +/************** End of resolve.c *********************************************/ +/************** Begin file expr.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains routines used for analyzing expressions and +** for generating VDBE code that evaluates expressions in SQLite. +*/ +/* #include "sqliteInt.h" */ + +/* Forward declarations */ +static void exprCodeBetween(Parse*,Expr*,int,void(*)(Parse*,Expr*,int,int),int); +static int exprCodeVector(Parse *pParse, Expr *p, int *piToFree); + +/* +** Return the affinity character for a single column of a table. +*/ +SQLITE_PRIVATE char sqlite3TableColumnAffinity(const Table *pTab, int iCol){ + assert( iColnCol ); + return iCol>=0 ? pTab->aCol[iCol].affinity : SQLITE_AFF_INTEGER; +} + +/* +** Return the 'affinity' of the expression pExpr if any. +** +** If pExpr is a column, a reference to a column via an 'AS' alias, +** or a sub-select with a column as the return value, then the +** affinity of that column is returned. Otherwise, 0x00 is returned, +** indicating no affinity for the expression. +** +** i.e. the WHERE clause expressions in the following statements all +** have an affinity: +** +** CREATE TABLE t1(a); +** SELECT * FROM t1 WHERE a; +** SELECT a AS b FROM t1 WHERE b; +** SELECT * FROM t1 WHERE (select a from t1); +*/ +SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr){ + int op; + while( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){ + assert( pExpr->op==TK_COLLATE + || pExpr->op==TK_IF_NULL_ROW + || (pExpr->op==TK_REGISTER && pExpr->op2==TK_IF_NULL_ROW) ); + pExpr = pExpr->pLeft; + assert( pExpr!=0 ); + } + op = pExpr->op; + if( op==TK_REGISTER ) op = pExpr->op2; + if( (op==TK_COLUMN || op==TK_AGG_COLUMN) && pExpr->y.pTab ){ + return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); + } + if( op==TK_SELECT ){ + assert( pExpr->flags&EP_xIsSelect ); + assert( pExpr->x.pSelect!=0 ); + assert( pExpr->x.pSelect->pEList!=0 ); + assert( pExpr->x.pSelect->pEList->a[0].pExpr!=0 ); + return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); + } +#ifndef SQLITE_OMIT_CAST + if( op==TK_CAST ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + return sqlite3AffinityType(pExpr->u.zToken, 0); + } +#endif + if( op==TK_SELECT_COLUMN ){ + assert( pExpr->pLeft->flags&EP_xIsSelect ); + assert( pExpr->iColumn < pExpr->iTable ); + assert( pExpr->iTable==pExpr->pLeft->x.pSelect->pEList->nExpr ); + return sqlite3ExprAffinity( + pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr + ); + } + if( op==TK_VECTOR ){ + return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); + } + return pExpr->affExpr; +} + +/* +** Set the collating sequence for expression pExpr to be the collating +** sequence named by pToken. Return a pointer to a new Expr node that +** implements the COLLATE operator. +** +** If a memory allocation error occurs, that fact is recorded in pParse->db +** and the pExpr parameter is returned unchanged. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken( + const Parse *pParse, /* Parsing context */ + Expr *pExpr, /* Add the "COLLATE" clause to this expression */ + const Token *pCollName, /* Name of collating sequence */ + int dequote /* True to dequote pCollName */ +){ + if( pCollName->n>0 ){ + Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, dequote); + if( pNew ){ + pNew->pLeft = pExpr; + pNew->flags |= EP_Collate|EP_Skip; + pExpr = pNew; + } + } + return pExpr; +} +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString( + const Parse *pParse, /* Parsing context */ + Expr *pExpr, /* Add the "COLLATE" clause to this expression */ + const char *zC /* The collating sequence name */ +){ + Token s; + assert( zC!=0 ); + sqlite3TokenInit(&s, (char*)zC); + return sqlite3ExprAddCollateToken(pParse, pExpr, &s, 0); +} + +/* +** Skip over any TK_COLLATE operators. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ + while( pExpr && ExprHasProperty(pExpr, EP_Skip) ){ + assert( pExpr->op==TK_COLLATE ); + pExpr = pExpr->pLeft; + } + return pExpr; +} + +/* +** Skip over any TK_COLLATE operators and/or any unlikely() +** or likelihood() or likely() functions at the root of an +** expression. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ + while( pExpr && ExprHasProperty(pExpr, EP_Skip|EP_Unlikely) ){ + if( ExprHasProperty(pExpr, EP_Unlikely) ){ + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + assert( pExpr->x.pList->nExpr>0 ); + assert( pExpr->op==TK_FUNCTION ); + pExpr = pExpr->x.pList->a[0].pExpr; + }else{ + assert( pExpr->op==TK_COLLATE ); + pExpr = pExpr->pLeft; + } + } + return pExpr; +} + +/* +** Return the collation sequence for the expression pExpr. If +** there is no defined collating sequence, return NULL. +** +** See also: sqlite3ExprNNCollSeq() +** +** The sqlite3ExprNNCollSeq() works the same exact that it returns the +** default collation if pExpr has no defined collation. +** +** The collating sequence might be determined by a COLLATE operator +** or by the presence of a column with a defined collating sequence. +** COLLATE operators take first precedence. Left operands take +** precedence over right operands. +*/ +SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ + sqlite3 *db = pParse->db; + CollSeq *pColl = 0; + const Expr *p = pExpr; + while( p ){ + int op = p->op; + if( op==TK_REGISTER ) op = p->op2; + if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER) + && p->y.pTab!=0 + ){ + /* op==TK_REGISTER && p->y.pTab!=0 happens when pExpr was originally + ** a TK_COLUMN but was previously evaluated and cached in a register */ + int j = p->iColumn; + if( j>=0 ){ + const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]); + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); + } + break; + } + if( op==TK_CAST || op==TK_UPLUS ){ + p = p->pLeft; + continue; + } + if( op==TK_VECTOR ){ + p = p->x.pList->a[0].pExpr; + continue; + } + if( op==TK_COLLATE ){ + pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); + break; + } + if( p->flags & EP_Collate ){ + if( p->pLeft && (p->pLeft->flags & EP_Collate)!=0 ){ + p = p->pLeft; + }else{ + Expr *pNext = p->pRight; + /* The Expr.x union is never used at the same time as Expr.pRight */ + assert( p->x.pList==0 || p->pRight==0 ); + if( p->x.pList!=0 + && !db->mallocFailed + && ALWAYS(!ExprHasProperty(p, EP_xIsSelect)) + ){ + int i; + for(i=0; ALWAYS(ix.pList->nExpr); i++){ + if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ + pNext = p->x.pList->a[i].pExpr; + break; + } + } + } + p = pNext; + } + }else{ + break; + } + } + if( sqlite3CheckCollSeq(pParse, pColl) ){ + pColl = 0; + } + return pColl; +} + +/* +** Return the collation sequence for the expression pExpr. If +** there is no defined collating sequence, return a pointer to the +** defautl collation sequence. +** +** See also: sqlite3ExprCollSeq() +** +** The sqlite3ExprCollSeq() routine works the same except that it +** returns NULL if there is no defined collation. +*/ +SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, const Expr *pExpr){ + CollSeq *p = sqlite3ExprCollSeq(pParse, pExpr); + if( p==0 ) p = pParse->db->pDfltColl; + assert( p!=0 ); + return p; +} + +/* +** Return TRUE if the two expressions have equivalent collating sequences. +*/ +SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse *pParse, const Expr *pE1, const Expr *pE2){ + CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pE1); + CollSeq *pColl2 = sqlite3ExprNNCollSeq(pParse, pE2); + return sqlite3StrICmp(pColl1->zName, pColl2->zName)==0; +} + +/* +** pExpr is an operand of a comparison operator. aff2 is the +** type affinity of the other operand. This routine returns the +** type affinity that should be used for the comparison operator. +*/ +SQLITE_PRIVATE char sqlite3CompareAffinity(const Expr *pExpr, char aff2){ + char aff1 = sqlite3ExprAffinity(pExpr); + if( aff1>SQLITE_AFF_NONE && aff2>SQLITE_AFF_NONE ){ + /* Both sides of the comparison are columns. If one has numeric + ** affinity, use that. Otherwise use no affinity. + */ + if( sqlite3IsNumericAffinity(aff1) || sqlite3IsNumericAffinity(aff2) ){ + return SQLITE_AFF_NUMERIC; + }else{ + return SQLITE_AFF_BLOB; + } + }else{ + /* One side is a column, the other is not. Use the columns affinity. */ + assert( aff1<=SQLITE_AFF_NONE || aff2<=SQLITE_AFF_NONE ); + return (aff1<=SQLITE_AFF_NONE ? aff2 : aff1) | SQLITE_AFF_NONE; + } +} + +/* +** pExpr is a comparison operator. Return the type affinity that should +** be applied to both operands prior to doing the comparison. +*/ +static char comparisonAffinity(const Expr *pExpr){ + char aff; + assert( pExpr->op==TK_EQ || pExpr->op==TK_IN || pExpr->op==TK_LT || + pExpr->op==TK_GT || pExpr->op==TK_GE || pExpr->op==TK_LE || + pExpr->op==TK_NE || pExpr->op==TK_IS || pExpr->op==TK_ISNOT ); + assert( pExpr->pLeft ); + aff = sqlite3ExprAffinity(pExpr->pLeft); + if( pExpr->pRight ){ + aff = sqlite3CompareAffinity(pExpr->pRight, aff); + }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); + }else if( aff==0 ){ + aff = SQLITE_AFF_BLOB; + } + return aff; +} + +/* +** pExpr is a comparison expression, eg. '=', '<', IN(...) etc. +** idx_affinity is the affinity of an indexed column. Return true +** if the index with affinity idx_affinity may be used to implement +** the comparison in pExpr. +*/ +SQLITE_PRIVATE int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity){ + char aff = comparisonAffinity(pExpr); + if( affflags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pLeft); + }else if( pRight && (pRight->flags & EP_Collate)!=0 ){ + pColl = sqlite3ExprCollSeq(pParse, pRight); + }else{ + pColl = sqlite3ExprCollSeq(pParse, pLeft); + if( !pColl ){ + pColl = sqlite3ExprCollSeq(pParse, pRight); + } + } + return pColl; +} + +/* Expresssion p is a comparison operator. Return a collation sequence +** appropriate for the comparison operator. +** +** This is normally just a wrapper around sqlite3BinaryCompareCollSeq(). +** However, if the OP_Commuted flag is set, then the order of the operands +** is reversed in the sqlite3BinaryCompareCollSeq() call so that the +** correct collating sequence is found. +*/ +SQLITE_PRIVATE CollSeq *sqlite3ExprCompareCollSeq(Parse *pParse, const Expr *p){ + if( ExprHasProperty(p, EP_Commuted) ){ + return sqlite3BinaryCompareCollSeq(pParse, p->pRight, p->pLeft); + }else{ + return sqlite3BinaryCompareCollSeq(pParse, p->pLeft, p->pRight); + } +} + +/* +** Generate code for a comparison operator. +*/ +static int codeCompare( + Parse *pParse, /* The parsing (and code generating) context */ + Expr *pLeft, /* The left operand */ + Expr *pRight, /* The right operand */ + int opcode, /* The comparison opcode */ + int in1, int in2, /* Register holding operands */ + int dest, /* Jump here if true. */ + int jumpIfNull, /* If true, jump if either operand is NULL */ + int isCommuted /* The comparison has been commuted */ +){ + int p5; + int addr; + CollSeq *p4; + + if( pParse->nErr ) return 0; + if( isCommuted ){ + p4 = sqlite3BinaryCompareCollSeq(pParse, pRight, pLeft); + }else{ + p4 = sqlite3BinaryCompareCollSeq(pParse, pLeft, pRight); + } + p5 = binaryCompareP5(pLeft, pRight, jumpIfNull); + addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1, + (void*)p4, P4_COLLSEQ); + sqlite3VdbeChangeP5(pParse->pVdbe, (u8)p5); + return addr; +} + +/* +** Return true if expression pExpr is a vector, or false otherwise. +** +** A vector is defined as any expression that results in two or more +** columns of result. Every TK_VECTOR node is an vector because the +** parser will not generate a TK_VECTOR with fewer than two entries. +** But a TK_SELECT might be either a vector or a scalar. It is only +** considered a vector if it has two or more result columns. +*/ +SQLITE_PRIVATE int sqlite3ExprIsVector(const Expr *pExpr){ + return sqlite3ExprVectorSize(pExpr)>1; +} + +/* +** If the expression passed as the only argument is of type TK_VECTOR +** return the number of expressions in the vector. Or, if the expression +** is a sub-select, return the number of columns in the sub-select. For +** any other type of expression, return 1. +*/ +SQLITE_PRIVATE int sqlite3ExprVectorSize(const Expr *pExpr){ + u8 op = pExpr->op; + if( op==TK_REGISTER ) op = pExpr->op2; + if( op==TK_VECTOR ){ + return pExpr->x.pList->nExpr; + }else if( op==TK_SELECT ){ + return pExpr->x.pSelect->pEList->nExpr; + }else{ + return 1; + } +} + +/* +** Return a pointer to a subexpression of pVector that is the i-th +** column of the vector (numbered starting with 0). The caller must +** ensure that i is within range. +** +** If pVector is really a scalar (and "scalar" here includes subqueries +** that return a single column!) then return pVector unmodified. +** +** pVector retains ownership of the returned subexpression. +** +** If the vector is a (SELECT ...) then the expression returned is +** just the expression for the i-th term of the result set, and may +** not be ready for evaluation because the table cursor has not yet +** been positioned. +*/ +SQLITE_PRIVATE Expr *sqlite3VectorFieldSubexpr(Expr *pVector, int i){ + assert( iop==TK_ERROR ); + if( sqlite3ExprIsVector(pVector) ){ + assert( pVector->op2==0 || pVector->op==TK_REGISTER ); + if( pVector->op==TK_SELECT || pVector->op2==TK_SELECT ){ + return pVector->x.pSelect->pEList->a[i].pExpr; + }else{ + return pVector->x.pList->a[i].pExpr; + } + } + return pVector; +} + +/* +** Compute and return a new Expr object which when passed to +** sqlite3ExprCode() will generate all necessary code to compute +** the iField-th column of the vector expression pVector. +** +** It is ok for pVector to be a scalar (as long as iField==0). +** In that case, this routine works like sqlite3ExprDup(). +** +** The caller owns the returned Expr object and is responsible for +** ensuring that the returned value eventually gets freed. +** +** The caller retains ownership of pVector. If pVector is a TK_SELECT, +** then the returned object will reference pVector and so pVector must remain +** valid for the life of the returned object. If pVector is a TK_VECTOR +** or a scalar expression, then it can be deleted as soon as this routine +** returns. +** +** A trick to cause a TK_SELECT pVector to be deleted together with +** the returned Expr object is to attach the pVector to the pRight field +** of the returned TK_SELECT_COLUMN Expr object. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprForVectorField( + Parse *pParse, /* Parsing context */ + Expr *pVector, /* The vector. List of expressions or a sub-SELECT */ + int iField, /* Which column of the vector to return */ + int nField /* Total number of columns in the vector */ +){ + Expr *pRet; + if( pVector->op==TK_SELECT ){ + assert( pVector->flags & EP_xIsSelect ); + /* The TK_SELECT_COLUMN Expr node: + ** + ** pLeft: pVector containing TK_SELECT. Not deleted. + ** pRight: not used. But recursively deleted. + ** iColumn: Index of a column in pVector + ** iTable: 0 or the number of columns on the LHS of an assignment + ** pLeft->iTable: First in an array of register holding result, or 0 + ** if the result is not yet computed. + ** + ** sqlite3ExprDelete() specifically skips the recursive delete of + ** pLeft on TK_SELECT_COLUMN nodes. But pRight is followed, so pVector + ** can be attached to pRight to cause this node to take ownership of + ** pVector. Typically there will be multiple TK_SELECT_COLUMN nodes + ** with the same pLeft pointer to the pVector, but only one of them + ** will own the pVector. + */ + pRet = sqlite3PExpr(pParse, TK_SELECT_COLUMN, 0, 0); + if( pRet ){ + pRet->iTable = nField; + pRet->iColumn = iField; + pRet->pLeft = pVector; + } + }else{ + if( pVector->op==TK_VECTOR ){ + Expr **ppVector = &pVector->x.pList->a[iField].pExpr; + pVector = *ppVector; + if( IN_RENAME_OBJECT ){ + /* This must be a vector UPDATE inside a trigger */ + *ppVector = 0; + return pVector; + } + } + pRet = sqlite3ExprDup(pParse->db, pVector, 0); + } + return pRet; +} + +/* +** If expression pExpr is of type TK_SELECT, generate code to evaluate +** it. Return the register in which the result is stored (or, if the +** sub-select returns more than one column, the first in an array +** of registers in which the result is stored). +** +** If pExpr is not a TK_SELECT expression, return 0. +*/ +static int exprCodeSubselect(Parse *pParse, Expr *pExpr){ + int reg = 0; +#ifndef SQLITE_OMIT_SUBQUERY + if( pExpr->op==TK_SELECT ){ + reg = sqlite3CodeSubselect(pParse, pExpr); + } +#endif + return reg; +} + +/* +** Argument pVector points to a vector expression - either a TK_VECTOR +** or TK_SELECT that returns more than one column. This function returns +** the register number of a register that contains the value of +** element iField of the vector. +** +** If pVector is a TK_SELECT expression, then code for it must have +** already been generated using the exprCodeSubselect() routine. In this +** case parameter regSelect should be the first in an array of registers +** containing the results of the sub-select. +** +** If pVector is of type TK_VECTOR, then code for the requested field +** is generated. In this case (*pRegFree) may be set to the number of +** a temporary register to be freed by the caller before returning. +** +** Before returning, output parameter (*ppExpr) is set to point to the +** Expr object corresponding to element iElem of the vector. +*/ +static int exprVectorRegister( + Parse *pParse, /* Parse context */ + Expr *pVector, /* Vector to extract element from */ + int iField, /* Field to extract from pVector */ + int regSelect, /* First in array of registers */ + Expr **ppExpr, /* OUT: Expression element */ + int *pRegFree /* OUT: Temp register to free */ +){ + u8 op = pVector->op; + assert( op==TK_VECTOR || op==TK_REGISTER || op==TK_SELECT || op==TK_ERROR ); + if( op==TK_REGISTER ){ + *ppExpr = sqlite3VectorFieldSubexpr(pVector, iField); + return pVector->iTable+iField; + } + if( op==TK_SELECT ){ + *ppExpr = pVector->x.pSelect->pEList->a[iField].pExpr; + return regSelect+iField; + } + if( op==TK_VECTOR ){ + *ppExpr = pVector->x.pList->a[iField].pExpr; + return sqlite3ExprCodeTemp(pParse, *ppExpr, pRegFree); + } + return 0; +} + +/* +** Expression pExpr is a comparison between two vector values. Compute +** the result of the comparison (1, 0, or NULL) and write that +** result into register dest. +** +** The caller must satisfy the following preconditions: +** +** if pExpr->op==TK_IS: op==TK_EQ and p5==SQLITE_NULLEQ +** if pExpr->op==TK_ISNOT: op==TK_NE and p5==SQLITE_NULLEQ +** otherwise: op==pExpr->op and p5==0 +*/ +static void codeVectorCompare( + Parse *pParse, /* Code generator context */ + Expr *pExpr, /* The comparison operation */ + int dest, /* Write results into this register */ + u8 op, /* Comparison operator */ + u8 p5 /* SQLITE_NULLEQ or zero */ +){ + Vdbe *v = pParse->pVdbe; + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + int nLeft = sqlite3ExprVectorSize(pLeft); + int i; + int regLeft = 0; + int regRight = 0; + u8 opx = op; + int addrCmp = 0; + int addrDone = sqlite3VdbeMakeLabel(pParse); + int isCommuted = ExprHasProperty(pExpr,EP_Commuted); + + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); + if( pParse->nErr ) return; + if( nLeft!=sqlite3ExprVectorSize(pRight) ){ + sqlite3ErrorMsg(pParse, "row value misused"); + return; + } + assert( pExpr->op==TK_EQ || pExpr->op==TK_NE + || pExpr->op==TK_IS || pExpr->op==TK_ISNOT + || pExpr->op==TK_LT || pExpr->op==TK_GT + || pExpr->op==TK_LE || pExpr->op==TK_GE + ); + assert( pExpr->op==op || (pExpr->op==TK_IS && op==TK_EQ) + || (pExpr->op==TK_ISNOT && op==TK_NE) ); + assert( p5==0 || pExpr->op!=op ); + assert( p5==SQLITE_NULLEQ || pExpr->op==op ); + + if( op==TK_LE ) opx = TK_LT; + if( op==TK_GE ) opx = TK_GT; + if( op==TK_NE ) opx = TK_EQ; + + regLeft = exprCodeSubselect(pParse, pLeft); + regRight = exprCodeSubselect(pParse, pRight); + + sqlite3VdbeAddOp2(v, OP_Integer, 1, dest); + for(i=0; 1 /*Loop exits by "break"*/; i++){ + int regFree1 = 0, regFree2 = 0; + Expr *pL = 0, *pR = 0; + int r1, r2; + assert( i>=0 && i0 +/* +** Check that argument nHeight is less than or equal to the maximum +** expression depth allowed. If it is not, leave an error message in +** pParse. +*/ +SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse *pParse, int nHeight){ + int rc = SQLITE_OK; + int mxHeight = pParse->db->aLimit[SQLITE_LIMIT_EXPR_DEPTH]; + if( nHeight>mxHeight ){ + sqlite3ErrorMsg(pParse, + "Expression tree is too large (maximum depth %d)", mxHeight + ); + rc = SQLITE_ERROR; + } + return rc; +} + +/* The following three functions, heightOfExpr(), heightOfExprList() +** and heightOfSelect(), are used to determine the maximum height +** of any expression tree referenced by the structure passed as the +** first argument. +** +** If this maximum height is greater than the current value pointed +** to by pnHeight, the second parameter, then set *pnHeight to that +** value. +*/ +static void heightOfExpr(const Expr *p, int *pnHeight){ + if( p ){ + if( p->nHeight>*pnHeight ){ + *pnHeight = p->nHeight; + } + } +} +static void heightOfExprList(const ExprList *p, int *pnHeight){ + if( p ){ + int i; + for(i=0; inExpr; i++){ + heightOfExpr(p->a[i].pExpr, pnHeight); + } + } +} +static void heightOfSelect(const Select *pSelect, int *pnHeight){ + const Select *p; + for(p=pSelect; p; p=p->pPrior){ + heightOfExpr(p->pWhere, pnHeight); + heightOfExpr(p->pHaving, pnHeight); + heightOfExpr(p->pLimit, pnHeight); + heightOfExprList(p->pEList, pnHeight); + heightOfExprList(p->pGroupBy, pnHeight); + heightOfExprList(p->pOrderBy, pnHeight); + } +} + +/* +** Set the Expr.nHeight variable in the structure passed as an +** argument. An expression with no children, Expr.pList or +** Expr.pSelect member has a height of 1. Any other expression +** has a height equal to the maximum height of any other +** referenced Expr plus one. +** +** Also propagate EP_Propagate flags up from Expr.x.pList to Expr.flags, +** if appropriate. +*/ +static void exprSetHeight(Expr *p){ + int nHeight = 0; + heightOfExpr(p->pLeft, &nHeight); + heightOfExpr(p->pRight, &nHeight); + if( ExprHasProperty(p, EP_xIsSelect) ){ + heightOfSelect(p->x.pSelect, &nHeight); + }else if( p->x.pList ){ + heightOfExprList(p->x.pList, &nHeight); + p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList); + } + p->nHeight = nHeight + 1; +} + +/* +** Set the Expr.nHeight variable using the exprSetHeight() function. If +** the height is greater than the maximum allowed expression depth, +** leave an error in pParse. +** +** Also propagate all EP_Propagate flags from the Expr.x.pList into +** Expr.flags. +*/ +SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ + if( pParse->nErr ) return; + exprSetHeight(p); + sqlite3ExprCheckHeight(pParse, p->nHeight); +} + +/* +** Return the maximum height of any expression tree referenced +** by the select statement passed as an argument. +*/ +SQLITE_PRIVATE int sqlite3SelectExprHeight(const Select *p){ + int nHeight = 0; + heightOfSelect(p, &nHeight); + return nHeight; +} +#else /* ABOVE: Height enforcement enabled. BELOW: Height enforcement off */ +/* +** Propagate all EP_Propagate flags from the Expr.x.pList into +** Expr.flags. +*/ +SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ + if( pParse->nErr ) return; + if( p && p->x.pList && !ExprHasProperty(p, EP_xIsSelect) ){ + p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList); + } +} +#define exprSetHeight(y) +#endif /* SQLITE_MAX_EXPR_DEPTH>0 */ + +/* +** This routine is the core allocator for Expr nodes. +** +** Construct a new expression node and return a pointer to it. Memory +** for this node and for the pToken argument is a single allocation +** obtained from sqlite3DbMalloc(). The calling function +** is responsible for making sure the node eventually gets freed. +** +** If dequote is true, then the token (if it exists) is dequoted. +** If dequote is false, no dequoting is performed. The deQuote +** parameter is ignored if pToken is NULL or if the token does not +** appear to be quoted. If the quotes were of the form "..." (double-quotes) +** then the EP_DblQuoted flag is set on the expression node. +** +** Special case: If op==TK_INTEGER and pToken points to a string that +** can be translated into a 32-bit integer, then the token is not +** stored in u.zToken. Instead, the integer values is written +** into u.iValue and the EP_IntValue flag is set. No extra storage +** is allocated to hold the integer text and the dequote flag is ignored. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprAlloc( + sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ + int op, /* Expression opcode */ + const Token *pToken, /* Token argument. Might be NULL */ + int dequote /* True to dequote */ +){ + Expr *pNew; + int nExtra = 0; + int iValue = 0; + + assert( db!=0 ); + if( pToken ){ + if( op!=TK_INTEGER || pToken->z==0 + || sqlite3GetInt32(pToken->z, &iValue)==0 ){ + nExtra = pToken->n+1; + assert( iValue>=0 ); + } + } + pNew = sqlite3DbMallocRawNN(db, sizeof(Expr)+nExtra); + if( pNew ){ + memset(pNew, 0, sizeof(Expr)); + pNew->op = (u8)op; + pNew->iAgg = -1; + if( pToken ){ + if( nExtra==0 ){ + pNew->flags |= EP_IntValue|EP_Leaf|(iValue?EP_IsTrue:EP_IsFalse); + pNew->u.iValue = iValue; + }else{ + pNew->u.zToken = (char*)&pNew[1]; + assert( pToken->z!=0 || pToken->n==0 ); + if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); + pNew->u.zToken[pToken->n] = 0; + if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ + sqlite3DequoteExpr(pNew); + } + } + } +#if SQLITE_MAX_EXPR_DEPTH>0 + pNew->nHeight = 1; +#endif + } + return pNew; +} + +/* +** Allocate a new expression node from a zero-terminated token that has +** already been dequoted. +*/ +SQLITE_PRIVATE Expr *sqlite3Expr( + sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ + int op, /* Expression opcode */ + const char *zToken /* Token argument. Might be NULL */ +){ + Token x; + x.z = zToken; + x.n = sqlite3Strlen30(zToken); + return sqlite3ExprAlloc(db, op, &x, 0); +} + +/* +** Attach subtrees pLeft and pRight to the Expr node pRoot. +** +** If pRoot==NULL that means that a memory allocation error has occurred. +** In that case, delete the subtrees pLeft and pRight. +*/ +SQLITE_PRIVATE void sqlite3ExprAttachSubtrees( + sqlite3 *db, + Expr *pRoot, + Expr *pLeft, + Expr *pRight +){ + if( pRoot==0 ){ + assert( db->mallocFailed ); + sqlite3ExprDelete(db, pLeft); + sqlite3ExprDelete(db, pRight); + }else{ + if( pRight ){ + pRoot->pRight = pRight; + pRoot->flags |= EP_Propagate & pRight->flags; + } + if( pLeft ){ + pRoot->pLeft = pLeft; + pRoot->flags |= EP_Propagate & pLeft->flags; + } + exprSetHeight(pRoot); + } +} + +/* +** Allocate an Expr node which joins as many as two subtrees. +** +** One or both of the subtrees can be NULL. Return a pointer to the new +** Expr node. Or, if an OOM error occurs, set pParse->db->mallocFailed, +** free the subtrees and return NULL. +*/ +SQLITE_PRIVATE Expr *sqlite3PExpr( + Parse *pParse, /* Parsing context */ + int op, /* Expression opcode */ + Expr *pLeft, /* Left operand */ + Expr *pRight /* Right operand */ +){ + Expr *p; + p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)); + if( p ){ + memset(p, 0, sizeof(Expr)); + p->op = op & 0xff; + p->iAgg = -1; + sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); + sqlite3ExprCheckHeight(pParse, p->nHeight); + }else{ + sqlite3ExprDelete(pParse->db, pLeft); + sqlite3ExprDelete(pParse->db, pRight); + } + return p; +} + +/* +** Add pSelect to the Expr.x.pSelect field. Or, if pExpr is NULL (due +** do a memory allocation failure) then delete the pSelect object. +*/ +SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pSelect){ + if( pExpr ){ + pExpr->x.pSelect = pSelect; + ExprSetProperty(pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, pExpr); + }else{ + assert( pParse->db->mallocFailed ); + sqlite3SelectDelete(pParse->db, pSelect); + } +} + +/* +** Expression list pEList is a list of vector values. This function +** converts the contents of pEList to a VALUES(...) Select statement +** returning 1 row for each element of the list. For example, the +** expression list: +** +** ( (1,2), (3,4) (5,6) ) +** +** is translated to the equivalent of: +** +** VALUES(1,2), (3,4), (5,6) +** +** Each of the vector values in pEList must contain exactly nElem terms. +** If a list element that is not a vector or does not contain nElem terms, +** an error message is left in pParse. +** +** This is used as part of processing IN(...) expressions with a list +** of vectors on the RHS. e.g. "... IN ((1,2), (3,4), (5,6))". +*/ +SQLITE_PRIVATE Select *sqlite3ExprListToValues(Parse *pParse, int nElem, ExprList *pEList){ + int ii; + Select *pRet = 0; + assert( nElem>1 ); + for(ii=0; iinExpr; ii++){ + Select *pSel; + Expr *pExpr = pEList->a[ii].pExpr; + int nExprElem = (pExpr->op==TK_VECTOR ? pExpr->x.pList->nExpr : 1); + if( nExprElem!=nElem ){ + sqlite3ErrorMsg(pParse, "IN(...) element has %d term%s - expected %d", + nExprElem, nExprElem>1?"s":"", nElem + ); + break; + } + pSel = sqlite3SelectNew(pParse, pExpr->x.pList, 0, 0, 0, 0, 0, SF_Values,0); + pExpr->x.pList = 0; + if( pSel ){ + if( pRet ){ + pSel->op = TK_ALL; + pSel->pPrior = pRet; + } + pRet = pSel; + } + } + + if( pRet && pRet->pPrior ){ + pRet->selFlags |= SF_MultiValue; + } + sqlite3ExprListDelete(pParse->db, pEList); + return pRet; +} + +/* +** Join two expressions using an AND operator. If either expression is +** NULL, then just return the other expression. +** +** If one side or the other of the AND is known to be false, then instead +** of returning an AND expression, just return a constant expression with +** a value of false. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ + sqlite3 *db = pParse->db; + if( pLeft==0 ){ + return pRight; + }else if( pRight==0 ){ + return pLeft; + }else if( (ExprAlwaysFalse(pLeft) || ExprAlwaysFalse(pRight)) + && !IN_RENAME_OBJECT + ){ + sqlite3ExprDeferredDelete(pParse, pLeft); + sqlite3ExprDeferredDelete(pParse, pRight); + return sqlite3Expr(db, TK_INTEGER, "0"); + }else{ + return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); + } +} + +/* +** Construct a new expression node for a function with multiple +** arguments. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprFunction( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Argument list */ + const Token *pToken, /* Name of the function */ + int eDistinct /* SF_Distinct or SF_ALL or 0 */ +){ + Expr *pNew; + sqlite3 *db = pParse->db; + assert( pToken ); + pNew = sqlite3ExprAlloc(db, TK_FUNCTION, pToken, 1); + if( pNew==0 ){ + sqlite3ExprListDelete(db, pList); /* Avoid memory leak when malloc fails */ + return 0; + } + if( pList + && pList->nExpr > pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] + && !pParse->nested + ){ + sqlite3ErrorMsg(pParse, "too many arguments on function %T", pToken); + } + pNew->x.pList = pList; + ExprSetProperty(pNew, EP_HasFunc); + assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + sqlite3ExprSetHeightAndFlags(pParse, pNew); + if( eDistinct==SF_Distinct ) ExprSetProperty(pNew, EP_Distinct); + return pNew; +} + +/* +** Check to see if a function is usable according to current access +** rules: +** +** SQLITE_FUNC_DIRECT - Only usable from top-level SQL +** +** SQLITE_FUNC_UNSAFE - Usable if TRUSTED_SCHEMA or from +** top-level SQL +** +** If the function is not usable, create an error. +*/ +SQLITE_PRIVATE void sqlite3ExprFunctionUsable( + Parse *pParse, /* Parsing and code generating context */ + const Expr *pExpr, /* The function invocation */ + const FuncDef *pDef /* The function being invoked */ +){ + assert( !IN_RENAME_OBJECT ); + assert( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 ); + if( ExprHasProperty(pExpr, EP_FromDDL) ){ + if( (pDef->funcFlags & SQLITE_FUNC_DIRECT)!=0 + || (pParse->db->flags & SQLITE_TrustedSchema)==0 + ){ + /* Functions prohibited in triggers and views if: + ** (1) tagged with SQLITE_DIRECTONLY + ** (2) not tagged with SQLITE_INNOCUOUS (which means it + ** is tagged with SQLITE_FUNC_UNSAFE) and + ** SQLITE_DBCONFIG_TRUSTED_SCHEMA is off (meaning + ** that the schema is possibly tainted). + */ + sqlite3ErrorMsg(pParse, "unsafe use of %s()", pDef->zName); + } + } +} + +/* +** Assign a variable number to an expression that encodes a wildcard +** in the original SQL statement. +** +** Wildcards consisting of a single "?" are assigned the next sequential +** variable number. +** +** Wildcards of the form "?nnn" are assigned the number "nnn". We make +** sure "nnn" is not too big to avoid a denial of service attack when +** the SQL statement comes from an external source. +** +** Wildcards of the form ":aaa", "@aaa", or "$aaa" are assigned the same number +** as the previous instance of the same wildcard. Or if this is the first +** instance of the wildcard, the next sequential variable number is +** assigned. +*/ +SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ + sqlite3 *db = pParse->db; + const char *z; + ynVar x; + + if( pExpr==0 ) return; + assert( !ExprHasProperty(pExpr, EP_IntValue|EP_Reduced|EP_TokenOnly) ); + z = pExpr->u.zToken; + assert( z!=0 ); + assert( z[0]!=0 ); + assert( n==(u32)sqlite3Strlen30(z) ); + if( z[1]==0 ){ + /* Wildcard of the form "?". Assign the next variable number */ + assert( z[0]=='?' ); + x = (ynVar)(++pParse->nVar); + }else{ + int doAdd = 0; + if( z[0]=='?' ){ + /* Wildcard of the form "?nnn". Convert "nnn" to an integer and + ** use it as the variable number */ + i64 i; + int bOk; + if( n==2 ){ /*OPTIMIZATION-IF-TRUE*/ + i = z[1]-'0'; /* The common case of ?N for a single digit N */ + bOk = 1; + }else{ + bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + } + testcase( i==0 ); + testcase( i==1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); + testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ); + if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", + db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); + return; + } + x = (ynVar)i; + if( x>pParse->nVar ){ + pParse->nVar = (int)x; + doAdd = 1; + }else if( sqlite3VListNumToName(pParse->pVList, x)==0 ){ + doAdd = 1; + } + }else{ + /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable + ** number as the prior appearance of the same name, or if the name + ** has never appeared before, reuse the same variable number + */ + x = (ynVar)sqlite3VListNameToNum(pParse->pVList, z, n); + if( x==0 ){ + x = (ynVar)(++pParse->nVar); + doAdd = 1; + } + } + if( doAdd ){ + pParse->pVList = sqlite3VListAdd(db, pParse->pVList, z, n, x); + } + } + pExpr->iColumn = x; + if( x>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + sqlite3ErrorMsg(pParse, "too many SQL variables"); + } +} + +/* +** Recursively delete an expression tree. +*/ +static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ + assert( p!=0 ); + /* Sanity check: Assert that the IntValue is non-negative if it exists */ + assert( !ExprHasProperty(p, EP_IntValue) || p->u.iValue>=0 ); + + assert( !ExprHasProperty(p, EP_WinFunc) || p->y.pWin!=0 || db->mallocFailed ); + assert( p->op!=TK_FUNCTION || ExprHasProperty(p, EP_TokenOnly|EP_Reduced) + || p->y.pWin==0 || ExprHasProperty(p, EP_WinFunc) ); +#ifdef SQLITE_DEBUG + if( ExprHasProperty(p, EP_Leaf) && !ExprHasProperty(p, EP_TokenOnly) ){ + assert( p->pLeft==0 ); + assert( p->pRight==0 ); + assert( p->x.pSelect==0 ); + } +#endif + if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ + /* The Expr.x union is never used at the same time as Expr.pRight */ + assert( p->x.pList==0 || p->pRight==0 ); + if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); + if( p->pRight ){ + assert( !ExprHasProperty(p, EP_WinFunc) ); + sqlite3ExprDeleteNN(db, p->pRight); + }else if( ExprHasProperty(p, EP_xIsSelect) ){ + assert( !ExprHasProperty(p, EP_WinFunc) ); + sqlite3SelectDelete(db, p->x.pSelect); + }else{ + sqlite3ExprListDelete(db, p->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(p, EP_WinFunc) ){ + sqlite3WindowDelete(db, p->y.pWin); + } +#endif + } + } + if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); + if( !ExprHasProperty(p, EP_Static) ){ + sqlite3DbFreeNN(db, p); + } +} +SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ + if( p ) sqlite3ExprDeleteNN(db, p); +} + + +/* +** Arrange to cause pExpr to be deleted when the pParse is deleted. +** This is similar to sqlite3ExprDelete() except that the delete is +** deferred untilthe pParse is deleted. +** +** The pExpr might be deleted immediately on an OOM error. +** +** The deferred delete is (currently) implemented by adding the +** pExpr to the pParse->pConstExpr list with a register number of 0. +*/ +SQLITE_PRIVATE void sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ + pParse->pConstExpr = + sqlite3ExprListAppend(pParse, pParse->pConstExpr, pExpr); +} + +/* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the +** expression. +*/ +SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse *pParse, Expr *p){ + if( p ){ + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprUnmap(pParse, p); + } + sqlite3ExprDeleteNN(pParse->db, p); + } +} + +/* +** Return the number of bytes allocated for the expression structure +** passed as the first argument. This is always one of EXPR_FULLSIZE, +** EXPR_REDUCEDSIZE or EXPR_TOKENONLYSIZE. +*/ +static int exprStructSize(const Expr *p){ + if( ExprHasProperty(p, EP_TokenOnly) ) return EXPR_TOKENONLYSIZE; + if( ExprHasProperty(p, EP_Reduced) ) return EXPR_REDUCEDSIZE; + return EXPR_FULLSIZE; +} + +/* +** The dupedExpr*Size() routines each return the number of bytes required +** to store a copy of an expression or expression tree. They differ in +** how much of the tree is measured. +** +** dupedExprStructSize() Size of only the Expr structure +** dupedExprNodeSize() Size of Expr + space for token +** dupedExprSize() Expr + token + subtree components +** +*************************************************************************** +** +** The dupedExprStructSize() function returns two values OR-ed together: +** (1) the space required for a copy of the Expr structure only and +** (2) the EP_xxx flags that indicate what the structure size should be. +** The return values is always one of: +** +** EXPR_FULLSIZE +** EXPR_REDUCEDSIZE | EP_Reduced +** EXPR_TOKENONLYSIZE | EP_TokenOnly +** +** The size of the structure can be found by masking the return value +** of this routine with 0xfff. The flags can be found by masking the +** return value with EP_Reduced|EP_TokenOnly. +** +** Note that with flags==EXPRDUP_REDUCE, this routines works on full-size +** (unreduced) Expr objects as they or originally constructed by the parser. +** During expression analysis, extra information is computed and moved into +** later parts of the Expr object and that extra information might get chopped +** off if the expression is reduced. Note also that it does not work to +** make an EXPRDUP_REDUCE copy of a reduced expression. It is only legal +** to reduce a pristine expression tree from the parser. The implementation +** of dupedExprStructSize() contain multiple assert() statements that attempt +** to enforce this constraint. +*/ +static int dupedExprStructSize(const Expr *p, int flags){ + int nSize; + assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ + assert( EXPR_FULLSIZE<=0xfff ); + assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); + if( 0==flags || p->op==TK_SELECT_COLUMN +#ifndef SQLITE_OMIT_WINDOWFUNC + || ExprHasProperty(p, EP_WinFunc) +#endif + ){ + nSize = EXPR_FULLSIZE; + }else{ + assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); + assert( !ExprHasProperty(p, EP_FromJoin) ); + assert( !ExprHasProperty(p, EP_MemToken) ); + assert( !ExprHasVVAProperty(p, EP_NoReduce) ); + if( p->pLeft || p->x.pList ){ + nSize = EXPR_REDUCEDSIZE | EP_Reduced; + }else{ + assert( p->pRight==0 ); + nSize = EXPR_TOKENONLYSIZE | EP_TokenOnly; + } + } + return nSize; +} + +/* +** This function returns the space in bytes required to store the copy +** of the Expr structure and a copy of the Expr.u.zToken string (if that +** string is defined.) +*/ +static int dupedExprNodeSize(const Expr *p, int flags){ + int nByte = dupedExprStructSize(p, flags) & 0xfff; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nByte += sqlite3Strlen30NN(p->u.zToken)+1; + } + return ROUND8(nByte); +} + +/* +** Return the number of bytes required to create a duplicate of the +** expression passed as the first argument. The second argument is a +** mask containing EXPRDUP_XXX flags. +** +** The value returned includes space to create a copy of the Expr struct +** itself and the buffer referred to by Expr.u.zToken, if any. +** +** If the EXPRDUP_REDUCE flag is set, then the return value includes +** space to duplicate all Expr nodes in the tree formed by Expr.pLeft +** and Expr.pRight variables (but not for any structures pointed to or +** descended from the Expr.x.pList or Expr.x.pSelect variables). +*/ +static int dupedExprSize(const Expr *p, int flags){ + int nByte = 0; + if( p ){ + nByte = dupedExprNodeSize(p, flags); + if( flags&EXPRDUP_REDUCE ){ + nByte += dupedExprSize(p->pLeft, flags) + dupedExprSize(p->pRight, flags); + } + } + return nByte; +} + +/* +** This function is similar to sqlite3ExprDup(), except that if pzBuffer +** is not NULL then *pzBuffer is assumed to point to a buffer large enough +** to store the copy of expression p, the copies of p->u.zToken +** (if applicable), and the copies of the p->pLeft and p->pRight expressions, +** if any. Before returning, *pzBuffer is set to the first byte past the +** portion of the buffer copied into by this function. +*/ +static Expr *exprDup(sqlite3 *db, const Expr *p, int dupFlags, u8 **pzBuffer){ + Expr *pNew; /* Value to return */ + u8 *zAlloc; /* Memory space from which to build Expr object */ + u32 staticFlag; /* EP_Static if space not obtained from malloc */ + + assert( db!=0 ); + assert( p ); + assert( dupFlags==0 || dupFlags==EXPRDUP_REDUCE ); + assert( pzBuffer==0 || dupFlags==EXPRDUP_REDUCE ); + + /* Figure out where to write the new Expr structure. */ + if( pzBuffer ){ + zAlloc = *pzBuffer; + staticFlag = EP_Static; + assert( zAlloc!=0 ); + }else{ + zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, dupFlags)); + staticFlag = 0; + } + pNew = (Expr *)zAlloc; + + if( pNew ){ + /* Set nNewSize to the size allocated for the structure pointed to + ** by pNew. This is either EXPR_FULLSIZE, EXPR_REDUCEDSIZE or + ** EXPR_TOKENONLYSIZE. nToken is set to the number of bytes consumed + ** by the copy of the p->u.zToken string (if any). + */ + const unsigned nStructSize = dupedExprStructSize(p, dupFlags); + const int nNewSize = nStructSize & 0xfff; + int nToken; + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; + }else{ + nToken = 0; + } + if( dupFlags ){ + assert( ExprHasProperty(p, EP_Reduced)==0 ); + memcpy(zAlloc, p, nNewSize); + }else{ + u32 nSize = (u32)exprStructSize(p); + memcpy(zAlloc, p, nSize); + if( nSizeflags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); + pNew->flags |= staticFlag; + ExprClearVVAProperties(pNew); + if( dupFlags ){ + ExprSetVVAProperty(pNew, EP_Immutable); + } + + /* Copy the p->u.zToken string, if any. */ + if( nToken ){ + char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + memcpy(zToken, p->u.zToken, nToken); + } + + if( 0==((p->flags|pNew->flags) & (EP_TokenOnly|EP_Leaf)) ){ + /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ + if( ExprHasProperty(p, EP_xIsSelect) ){ + pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, dupFlags); + }else{ + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, dupFlags); + } + } + + /* Fill in pNew->pLeft and pNew->pRight. */ + if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly|EP_WinFunc) ){ + zAlloc += dupedExprNodeSize(p, dupFlags); + if( !ExprHasProperty(pNew, EP_TokenOnly|EP_Leaf) ){ + pNew->pLeft = p->pLeft ? + exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc) : 0; + pNew->pRight = p->pRight ? + exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(p, EP_WinFunc) ){ + pNew->y.pWin = sqlite3WindowDup(db, pNew, p->y.pWin); + assert( ExprHasProperty(pNew, EP_WinFunc) ); + } +#endif /* SQLITE_OMIT_WINDOWFUNC */ + if( pzBuffer ){ + *pzBuffer = zAlloc; + } + }else{ + if( !ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ + if( pNew->op==TK_SELECT_COLUMN ){ + pNew->pLeft = p->pLeft; + assert( p->pRight==0 || p->pRight==p->pLeft + || ExprHasProperty(p->pLeft, EP_Subquery) ); + }else{ + pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); + } + pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); + } + } + } + return pNew; +} + +/* +** Create and return a deep copy of the object passed as the second +** argument. If an OOM condition is encountered, NULL is returned +** and the db->mallocFailed flag set. +*/ +#ifndef SQLITE_OMIT_CTE +SQLITE_PRIVATE With *sqlite3WithDup(sqlite3 *db, With *p){ + With *pRet = 0; + if( p ){ + sqlite3_int64 nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + pRet = sqlite3DbMallocZero(db, nByte); + if( pRet ){ + int i; + pRet->nCte = p->nCte; + for(i=0; inCte; i++){ + pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0); + pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0); + pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName); + } + } + } + return pRet; +} +#else +# define sqlite3WithDup(x,y) 0 +#endif + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** The gatherSelectWindows() procedure and its helper routine +** gatherSelectWindowsCallback() are used to scan all the expressions +** an a newly duplicated SELECT statement and gather all of the Window +** objects found there, assembling them onto the linked list at Select->pWin. +*/ +static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_WinFunc) ){ + Select *pSelect = pWalker->u.pSelect; + Window *pWin = pExpr->y.pWin; + assert( pWin ); + assert( IsWindowFunc(pExpr) ); + assert( pWin->ppThis==0 ); + sqlite3WindowLink(pSelect, pWin); + } + return WRC_Continue; +} +static int gatherSelectWindowsSelectCallback(Walker *pWalker, Select *p){ + return p==pWalker->u.pSelect ? WRC_Continue : WRC_Prune; +} +static void gatherSelectWindows(Select *p){ + Walker w; + w.xExprCallback = gatherSelectWindowsCallback; + w.xSelectCallback = gatherSelectWindowsSelectCallback; + w.xSelectCallback2 = 0; + w.pParse = 0; + w.u.pSelect = p; + sqlite3WalkSelect(&w, p); +} +#endif + + +/* +** The following group of routines make deep copies of expressions, +** expression lists, ID lists, and select statements. The copies can +** be deleted (by being passed to their respective ...Delete() routines) +** without effecting the originals. +** +** The expression list, ID, and source lists return by sqlite3ExprListDup(), +** sqlite3IdListDup(), and sqlite3SrcListDup() can not be further expanded +** by subsequent calls to sqlite*ListAppend() routines. +** +** Any tables that the SrcList might point to are not duplicated. +** +** The flags parameter contains a combination of the EXPRDUP_XXX flags. +** If the EXPRDUP_REDUCE flag is set, then the structure returned is a +** truncated version of the usual Expr structure that will be stored as +** part of the in-memory representation of the database schema. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3 *db, const Expr *p, int flags){ + assert( flags==0 || flags==EXPRDUP_REDUCE ); + return p ? exprDup(db, p, flags, 0) : 0; +} +SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int flags){ + ExprList *pNew; + struct ExprList_item *pItem; + const struct ExprList_item *pOldItem; + int i; + Expr *pPriorSelectColOld = 0; + Expr *pPriorSelectColNew = 0; + assert( db!=0 ); + if( p==0 ) return 0; + pNew = sqlite3DbMallocRawNN(db, sqlite3DbMallocSize(db, p)); + if( pNew==0 ) return 0; + pNew->nExpr = p->nExpr; + pNew->nAlloc = p->nAlloc; + pItem = pNew->a; + pOldItem = p->a; + for(i=0; inExpr; i++, pItem++, pOldItem++){ + Expr *pOldExpr = pOldItem->pExpr; + Expr *pNewExpr; + pItem->pExpr = sqlite3ExprDup(db, pOldExpr, flags); + if( pOldExpr + && pOldExpr->op==TK_SELECT_COLUMN + && (pNewExpr = pItem->pExpr)!=0 + ){ + if( pNewExpr->pRight ){ + pPriorSelectColOld = pOldExpr->pRight; + pPriorSelectColNew = pNewExpr->pRight; + pNewExpr->pLeft = pNewExpr->pRight; + }else{ + if( pOldExpr->pLeft!=pPriorSelectColOld ){ + pPriorSelectColOld = pOldExpr->pLeft; + pPriorSelectColNew = sqlite3ExprDup(db, pPriorSelectColOld, flags); + pNewExpr->pRight = pPriorSelectColNew; + } + pNewExpr->pLeft = pPriorSelectColNew; + } + } + pItem->zEName = sqlite3DbStrDup(db, pOldItem->zEName); + pItem->sortFlags = pOldItem->sortFlags; + pItem->eEName = pOldItem->eEName; + pItem->done = 0; + pItem->bNulls = pOldItem->bNulls; + pItem->bSorterRef = pOldItem->bSorterRef; + pItem->u = pOldItem->u; + } + return pNew; +} + +/* +** If cursors, triggers, views and subqueries are all omitted from +** the build, then none of the following routines, except for +** sqlite3SelectDup(), can be called. sqlite3SelectDup() is sometimes +** called with a NULL argument. +*/ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) \ + || !defined(SQLITE_OMIT_SUBQUERY) +SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ + SrcList *pNew; + int i; + int nByte; + assert( db!=0 ); + if( p==0 ) return 0; + nByte = sizeof(*p) + (p->nSrc>0 ? sizeof(p->a[0]) * (p->nSrc-1) : 0); + pNew = sqlite3DbMallocRawNN(db, nByte ); + if( pNew==0 ) return 0; + pNew->nSrc = pNew->nAlloc = p->nSrc; + for(i=0; inSrc; i++){ + SrcItem *pNewItem = &pNew->a[i]; + const SrcItem *pOldItem = &p->a[i]; + Table *pTab; + pNewItem->pSchema = pOldItem->pSchema; + pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); + pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); + pNewItem->fg = pOldItem->fg; + pNewItem->iCursor = pOldItem->iCursor; + pNewItem->addrFillSub = pOldItem->addrFillSub; + pNewItem->regReturn = pOldItem->regReturn; + if( pNewItem->fg.isIndexedBy ){ + pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); + } + pNewItem->u2 = pOldItem->u2; + if( pNewItem->fg.isCte ){ + pNewItem->u2.pCteUse->nUse++; + } + if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = + sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + } + pTab = pNewItem->pTab = pOldItem->pTab; + if( pTab ){ + pTab->nTabRef++; + } + pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); + pNewItem->pOn = sqlite3ExprDup(db, pOldItem->pOn, flags); + pNewItem->pUsing = sqlite3IdListDup(db, pOldItem->pUsing); + pNewItem->colUsed = pOldItem->colUsed; + } + return pNew; +} +SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, const IdList *p){ + IdList *pNew; + int i; + assert( db!=0 ); + if( p==0 ) return 0; + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); + if( pNew==0 ) return 0; + pNew->nId = p->nId; + pNew->a = sqlite3DbMallocRawNN(db, p->nId*sizeof(p->a[0]) ); + if( pNew->a==0 ){ + sqlite3DbFreeNN(db, pNew); + return 0; + } + /* Note that because the size of the allocation for p->a[] is not + ** necessarily a power of two, sqlite3IdListAppend() may not be called + ** on the duplicate created by this function. */ + for(i=0; inId; i++){ + struct IdList_item *pNewItem = &pNew->a[i]; + struct IdList_item *pOldItem = &p->a[i]; + pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); + pNewItem->idx = pOldItem->idx; + } + return pNew; +} +SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int flags){ + Select *pRet = 0; + Select *pNext = 0; + Select **pp = &pRet; + const Select *p; + + assert( db!=0 ); + for(p=pDup; p; p=p->pPrior){ + Select *pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); + if( pNew==0 ) break; + pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); + pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); + pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); + pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); + pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); + pNew->op = p->op; + pNew->pNext = pNext; + pNew->pPrior = 0; + pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); + pNew->iLimit = 0; + pNew->iOffset = 0; + pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->nSelectRow = p->nSelectRow; + pNew->pWith = sqlite3WithDup(db, p->pWith); +#ifndef SQLITE_OMIT_WINDOWFUNC + pNew->pWin = 0; + pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn); + if( p->pWin && db->mallocFailed==0 ) gatherSelectWindows(pNew); +#endif + pNew->selId = p->selId; + if( db->mallocFailed ){ + /* Any prior OOM might have left the Select object incomplete. + ** Delete the whole thing rather than allow an incomplete Select + ** to be used by the code generator. */ + pNew->pNext = 0; + sqlite3SelectDelete(db, pNew); + break; + } + *pp = pNew; + pp = &pNew->pPrior; + pNext = pNew; + } + + return pRet; +} +#else +SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ + assert( p==0 ); + return 0; +} +#endif + + +/* +** Add a new element to the end of an expression list. If pList is +** initially NULL, then create a new expression list. +** +** The pList argument must be either NULL or a pointer to an ExprList +** obtained from a prior call to sqlite3ExprListAppend(). This routine +** may not be used with an ExprList obtained from sqlite3ExprListDup(). +** Reason: This routine assumes that the number of slots in pList->a[] +** is a power of two. That is true for sqlite3ExprListAppend() returns +** but is not necessarily true from the return value of sqlite3ExprListDup(). +** +** If a memory allocation error occurs, the entire list is freed and +** NULL is returned. If non-NULL is returned, then it is guaranteed +** that the new entry was successfully appended. +*/ +static const struct ExprList_item zeroItem = {0}; +SQLITE_PRIVATE SQLITE_NOINLINE ExprList *sqlite3ExprListAppendNew( + sqlite3 *db, /* Database handle. Used for memory allocation */ + Expr *pExpr /* Expression to be appended. Might be NULL */ +){ + struct ExprList_item *pItem; + ExprList *pList; + + pList = sqlite3DbMallocRawNN(db, sizeof(ExprList)+sizeof(pList->a[0])*4 ); + if( pList==0 ){ + sqlite3ExprDelete(db, pExpr); + return 0; + } + pList->nAlloc = 4; + pList->nExpr = 1; + pItem = &pList->a[0]; + *pItem = zeroItem; + pItem->pExpr = pExpr; + return pList; +} +SQLITE_PRIVATE SQLITE_NOINLINE ExprList *sqlite3ExprListAppendGrow( + sqlite3 *db, /* Database handle. Used for memory allocation */ + ExprList *pList, /* List to which to append. Might be NULL */ + Expr *pExpr /* Expression to be appended. Might be NULL */ +){ + struct ExprList_item *pItem; + ExprList *pNew; + pList->nAlloc *= 2; + pNew = sqlite3DbRealloc(db, pList, + sizeof(*pList)+(pList->nAlloc-1)*sizeof(pList->a[0])); + if( pNew==0 ){ + sqlite3ExprListDelete(db, pList); + sqlite3ExprDelete(db, pExpr); + return 0; + }else{ + pList = pNew; + } + pItem = &pList->a[pList->nExpr++]; + *pItem = zeroItem; + pItem->pExpr = pExpr; + return pList; +} +SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + Expr *pExpr /* Expression to be appended. Might be NULL */ +){ + struct ExprList_item *pItem; + if( pList==0 ){ + return sqlite3ExprListAppendNew(pParse->db,pExpr); + } + if( pList->nAllocnExpr+1 ){ + return sqlite3ExprListAppendGrow(pParse->db,pList,pExpr); + } + pItem = &pList->a[pList->nExpr++]; + *pItem = zeroItem; + pItem->pExpr = pExpr; + return pList; +} + +/* +** pColumns and pExpr form a vector assignment which is part of the SET +** clause of an UPDATE statement. Like this: +** +** (a,b,c) = (expr1,expr2,expr3) +** Or: (a,b,c) = (SELECT x,y,z FROM ....) +** +** For each term of the vector assignment, append new entries to the +** expression list pList. In the case of a subquery on the RHS, append +** TK_SELECT_COLUMN expressions. +*/ +SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + IdList *pColumns, /* List of names of LHS of the assignment */ + Expr *pExpr /* Vector expression to be appended. Might be NULL */ +){ + sqlite3 *db = pParse->db; + int n; + int i; + int iFirst = pList ? pList->nExpr : 0; + /* pColumns can only be NULL due to an OOM but an OOM will cause an + ** exit prior to this routine being invoked */ + if( NEVER(pColumns==0) ) goto vector_append_error; + if( pExpr==0 ) goto vector_append_error; + + /* If the RHS is a vector, then we can immediately check to see that + ** the size of the RHS and LHS match. But if the RHS is a SELECT, + ** wildcards ("*") in the result set of the SELECT must be expanded before + ** we can do the size check, so defer the size check until code generation. + */ + if( pExpr->op!=TK_SELECT && pColumns->nId!=(n=sqlite3ExprVectorSize(pExpr)) ){ + sqlite3ErrorMsg(pParse, "%d columns assigned %d values", + pColumns->nId, n); + goto vector_append_error; + } + + for(i=0; inId; i++){ + Expr *pSubExpr = sqlite3ExprForVectorField(pParse, pExpr, i, pColumns->nId); + assert( pSubExpr!=0 || db->mallocFailed ); + if( pSubExpr==0 ) continue; + pList = sqlite3ExprListAppend(pParse, pList, pSubExpr); + if( pList ){ + assert( pList->nExpr==iFirst+i+1 ); + pList->a[pList->nExpr-1].zEName = pColumns->a[i].zName; + pColumns->a[i].zName = 0; + } + } + + if( !db->mallocFailed && pExpr->op==TK_SELECT && ALWAYS(pList!=0) ){ + Expr *pFirst = pList->a[iFirst].pExpr; + assert( pFirst!=0 ); + assert( pFirst->op==TK_SELECT_COLUMN ); + + /* Store the SELECT statement in pRight so it will be deleted when + ** sqlite3ExprListDelete() is called */ + pFirst->pRight = pExpr; + pExpr = 0; + + /* Remember the size of the LHS in iTable so that we can check that + ** the RHS and LHS sizes match during code generation. */ + pFirst->iTable = pColumns->nId; + } + +vector_append_error: + sqlite3ExprUnmapAndDelete(pParse, pExpr); + sqlite3IdListDelete(db, pColumns); + return pList; +} + +/* +** Set the sort order for the last element on the given ExprList. +*/ +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder, int eNulls){ + struct ExprList_item *pItem; + if( p==0 ) return; + assert( p->nExpr>0 ); + + assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC==0 && SQLITE_SO_DESC>0 ); + assert( iSortOrder==SQLITE_SO_UNDEFINED + || iSortOrder==SQLITE_SO_ASC + || iSortOrder==SQLITE_SO_DESC + ); + assert( eNulls==SQLITE_SO_UNDEFINED + || eNulls==SQLITE_SO_ASC + || eNulls==SQLITE_SO_DESC + ); + + pItem = &p->a[p->nExpr-1]; + assert( pItem->bNulls==0 ); + if( iSortOrder==SQLITE_SO_UNDEFINED ){ + iSortOrder = SQLITE_SO_ASC; + } + pItem->sortFlags = (u8)iSortOrder; + + if( eNulls!=SQLITE_SO_UNDEFINED ){ + pItem->bNulls = 1; + if( iSortOrder!=eNulls ){ + pItem->sortFlags |= KEYINFO_ORDER_BIGNULL; + } + } +} + +/* +** Set the ExprList.a[].zEName element of the most recently added item +** on the expression list. +** +** pList might be NULL following an OOM error. But pName should never be +** NULL. If a memory allocation fails, the pParse->db->mallocFailed flag +** is set. +*/ +SQLITE_PRIVATE void sqlite3ExprListSetName( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to add the span. */ + const Token *pName, /* Name to be added */ + int dequote /* True to cause the name to be dequoted */ +){ + assert( pList!=0 || pParse->db->mallocFailed!=0 ); + assert( pParse->eParseMode!=PARSE_MODE_UNMAP || dequote==0 ); + if( pList ){ + struct ExprList_item *pItem; + assert( pList->nExpr>0 ); + pItem = &pList->a[pList->nExpr-1]; + assert( pItem->zEName==0 ); + assert( pItem->eEName==ENAME_NAME ); + pItem->zEName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); + if( dequote ){ + /* If dequote==0, then pName->z does not point to part of a DDL + ** statement handled by the parser. And so no token need be added + ** to the token-map. */ + sqlite3Dequote(pItem->zEName); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (const void*)pItem->zEName, pName); + } + } + } +} + +/* +** Set the ExprList.a[].zSpan element of the most recently added item +** on the expression list. +** +** pList might be NULL following an OOM error. But pSpan should never be +** NULL. If a memory allocation fails, the pParse->db->mallocFailed flag +** is set. +*/ +SQLITE_PRIVATE void sqlite3ExprListSetSpan( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to add the span. */ + const char *zStart, /* Start of the span */ + const char *zEnd /* End of the span */ +){ + sqlite3 *db = pParse->db; + assert( pList!=0 || db->mallocFailed!=0 ); + if( pList ){ + struct ExprList_item *pItem = &pList->a[pList->nExpr-1]; + assert( pList->nExpr>0 ); + if( pItem->zEName==0 ){ + pItem->zEName = sqlite3DbSpanDup(db, zStart, zEnd); + pItem->eEName = ENAME_SPAN; + } + } +} + +/* +** If the expression list pEList contains more than iLimit elements, +** leave an error message in pParse. +*/ +SQLITE_PRIVATE void sqlite3ExprListCheckLength( + Parse *pParse, + ExprList *pEList, + const char *zObject +){ + int mx = pParse->db->aLimit[SQLITE_LIMIT_COLUMN]; + testcase( pEList && pEList->nExpr==mx ); + testcase( pEList && pEList->nExpr==mx+1 ); + if( pEList && pEList->nExpr>mx ){ + sqlite3ErrorMsg(pParse, "too many columns in %s", zObject); + } +} + +/* +** Delete an entire expression list. +*/ +static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ + int i = pList->nExpr; + struct ExprList_item *pItem = pList->a; + assert( pList->nExpr>0 ); + do{ + sqlite3ExprDelete(db, pItem->pExpr); + sqlite3DbFree(db, pItem->zEName); + pItem++; + }while( --i>0 ); + sqlite3DbFreeNN(db, pList); +} +SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ + if( pList ) exprListDeleteNN(db, pList); +} + +/* +** Return the bitwise-OR of all Expr.flags fields in the given +** ExprList. +*/ +SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList *pList){ + int i; + u32 m = 0; + assert( pList!=0 ); + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + assert( pExpr!=0 ); + m |= pExpr->flags; + } + return m; +} + +/* +** This is a SELECT-node callback for the expression walker that +** always "fails". By "fail" in this case, we mean set +** pWalker->eCode to zero and abort. +** +** This callback is used by multiple expression walkers. +*/ +SQLITE_PRIVATE int sqlite3SelectWalkFail(Walker *pWalker, Select *NotUsed){ + UNUSED_PARAMETER(NotUsed); + pWalker->eCode = 0; + return WRC_Abort; +} + +/* +** Check the input string to see if it is "true" or "false" (in any case). +** +** If the string is.... Return +** "true" EP_IsTrue +** "false" EP_IsFalse +** anything else 0 +*/ +SQLITE_PRIVATE u32 sqlite3IsTrueOrFalse(const char *zIn){ + if( sqlite3StrICmp(zIn, "true")==0 ) return EP_IsTrue; + if( sqlite3StrICmp(zIn, "false")==0 ) return EP_IsFalse; + return 0; +} + + +/* +** If the input expression is an ID with the name "true" or "false" +** then convert it into an TK_TRUEFALSE term. Return non-zero if +** the conversion happened, and zero if the expression is unaltered. +*/ +SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr *pExpr){ + u32 v; + assert( pExpr->op==TK_ID || pExpr->op==TK_STRING ); + if( !ExprHasProperty(pExpr, EP_Quoted) + && (v = sqlite3IsTrueOrFalse(pExpr->u.zToken))!=0 + ){ + pExpr->op = TK_TRUEFALSE; + ExprSetProperty(pExpr, v); + return 1; + } + return 0; +} + +/* +** The argument must be a TK_TRUEFALSE Expr node. Return 1 if it is TRUE +** and 0 if it is FALSE. +*/ +SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr *pExpr){ + pExpr = sqlite3ExprSkipCollate((Expr*)pExpr); + assert( pExpr->op==TK_TRUEFALSE ); + assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0 + || sqlite3StrICmp(pExpr->u.zToken,"false")==0 ); + return pExpr->u.zToken[4]==0; +} + +/* +** If pExpr is an AND or OR expression, try to simplify it by eliminating +** terms that are always true or false. Return the simplified expression. +** Or return the original expression if no simplification is possible. +** +** Examples: +** +** (x<10) AND true => (x<10) +** (x<10) AND false => false +** (x<10) AND (y=22 OR false) => (x<10) AND (y=22) +** (x<10) AND (y=22 OR true) => (x<10) +** (y=22) OR true => true +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ + assert( pExpr!=0 ); + if( pExpr->op==TK_AND || pExpr->op==TK_OR ){ + Expr *pRight = sqlite3ExprSimplifiedAndOr(pExpr->pRight); + Expr *pLeft = sqlite3ExprSimplifiedAndOr(pExpr->pLeft); + if( ExprAlwaysTrue(pLeft) || ExprAlwaysFalse(pRight) ){ + pExpr = pExpr->op==TK_AND ? pRight : pLeft; + }else if( ExprAlwaysTrue(pRight) || ExprAlwaysFalse(pLeft) ){ + pExpr = pExpr->op==TK_AND ? pLeft : pRight; + } + } + return pExpr; +} + + +/* +** These routines are Walker callbacks used to check expressions to +** see if they are "constant" for some definition of constant. The +** Walker.eCode value determines the type of "constant" we are looking +** for. +** +** These callback routines are used to implement the following: +** +** sqlite3ExprIsConstant() pWalker->eCode==1 +** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 +** sqlite3ExprIsTableConstant() pWalker->eCode==3 +** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 +** +** In all cases, the callbacks set Walker.eCode=0 and abort if the expression +** is found to not be a constant. +** +** The sqlite3ExprIsConstantOrFunction() is used for evaluating DEFAULT +** expressions in a CREATE TABLE statement. The Walker.eCode value is 5 +** when parsing an existing schema out of the sqlite_schema table and 4 +** when processing a new CREATE TABLE statement. A bound parameter raises +** an error for new statements, but is silently converted +** to NULL for existing schemas. This allows sqlite_schema tables that +** contain a bound parameter because they were generated by older versions +** of SQLite to be parsed by newer versions of SQLite without raising a +** malformed schema error. +*/ +static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ + + /* If pWalker->eCode is 2 then any term of the expression that comes from + ** the ON or USING clauses of a left join disqualifies the expression + ** from being considered constant. */ + if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ + pWalker->eCode = 0; + return WRC_Abort; + } + + switch( pExpr->op ){ + /* Consider functions to be constant if all their arguments are constant + ** and either pWalker->eCode==4 or 5 or the function has the + ** SQLITE_FUNC_CONST flag. */ + case TK_FUNCTION: + if( (pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_ConstFunc)) + && !ExprHasProperty(pExpr, EP_WinFunc) + ){ + if( pWalker->eCode==5 ) ExprSetProperty(pExpr, EP_FromDDL); + return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; + } + case TK_ID: + /* Convert "true" or "false" in a DEFAULT clause into the + ** appropriate TK_TRUEFALSE operator */ + if( sqlite3ExprIdToTrueFalse(pExpr) ){ + return WRC_Prune; + } + /* no break */ deliberate_fall_through + case TK_COLUMN: + case TK_AGG_FUNCTION: + case TK_AGG_COLUMN: + testcase( pExpr->op==TK_ID ); + testcase( pExpr->op==TK_COLUMN ); + testcase( pExpr->op==TK_AGG_FUNCTION ); + testcase( pExpr->op==TK_AGG_COLUMN ); + if( ExprHasProperty(pExpr, EP_FixedCol) && pWalker->eCode!=2 ){ + return WRC_Continue; + } + if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ + return WRC_Continue; + } + /* no break */ deliberate_fall_through + case TK_IF_NULL_ROW: + case TK_REGISTER: + case TK_DOT: + testcase( pExpr->op==TK_REGISTER ); + testcase( pExpr->op==TK_IF_NULL_ROW ); + testcase( pExpr->op==TK_DOT ); + pWalker->eCode = 0; + return WRC_Abort; + case TK_VARIABLE: + if( pWalker->eCode==5 ){ + /* Silently convert bound parameters that appear inside of CREATE + ** statements into a NULL when parsing the CREATE statement text out + ** of the sqlite_schema table */ + pExpr->op = TK_NULL; + }else if( pWalker->eCode==4 ){ + /* A bound parameter in a CREATE statement that originates from + ** sqlite3_prepare() causes an error */ + pWalker->eCode = 0; + return WRC_Abort; + } + /* no break */ deliberate_fall_through + default: + testcase( pExpr->op==TK_SELECT ); /* sqlite3SelectWalkFail() disallows */ + testcase( pExpr->op==TK_EXISTS ); /* sqlite3SelectWalkFail() disallows */ + return WRC_Continue; + } +} +static int exprIsConst(Expr *p, int initFlag, int iCur){ + Walker w; + w.eCode = initFlag; + w.xExprCallback = exprNodeIsConstant; + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + w.u.iCur = iCur; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + +/* +** Walk an expression tree. Return non-zero if the expression is constant +** and 0 if it involves variables or function calls. +** +** For the purposes of this function, a double-quoted string (ex: "abc") +** is considered a variable but a single-quoted string (ex: 'abc') is +** a constant. +*/ +SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ + return exprIsConst(p, 1, 0); +} + +/* +** Walk an expression tree. Return non-zero if +** +** (1) the expression is constant, and +** (2) the expression does originate in the ON or USING clause +** of a LEFT JOIN, and +** (3) the expression does not contain any EP_FixedCol TK_COLUMN +** operands created by the constant propagation optimization. +** +** When this routine returns true, it indicates that the expression +** can be added to the pParse->pConstExpr list and evaluated once when +** the prepared statement starts up. See sqlite3ExprCodeRunJustOnce(). +*/ +SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ + return exprIsConst(p, 2, 0); +} + +/* +** Walk an expression tree. Return non-zero if the expression is constant +** for any single row of the table with cursor iCur. In other words, the +** expression must not refer to any non-deterministic function nor any +** table other than iCur. +*/ +SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ + return exprIsConst(p, 3, iCur); +} + + +/* +** sqlite3WalkExpr() callback used by sqlite3ExprIsConstantOrGroupBy(). +*/ +static int exprNodeIsConstantOrGroupBy(Walker *pWalker, Expr *pExpr){ + ExprList *pGroupBy = pWalker->u.pGroupBy; + int i; + + /* Check if pExpr is identical to any GROUP BY term. If so, consider + ** it constant. */ + for(i=0; inExpr; i++){ + Expr *p = pGroupBy->a[i].pExpr; + if( sqlite3ExprCompare(0, pExpr, p, -1)<2 ){ + CollSeq *pColl = sqlite3ExprNNCollSeq(pWalker->pParse, p); + if( sqlite3IsBinary(pColl) ){ + return WRC_Prune; + } + } + } + + /* Check if pExpr is a sub-select. If so, consider it variable. */ + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + pWalker->eCode = 0; + return WRC_Abort; + } + + return exprNodeIsConstant(pWalker, pExpr); +} + +/* +** Walk the expression tree passed as the first argument. Return non-zero +** if the expression consists entirely of constants or copies of terms +** in pGroupBy that sort with the BINARY collation sequence. +** +** This routine is used to determine if a term of the HAVING clause can +** be promoted into the WHERE clause. In order for such a promotion to work, +** the value of the HAVING clause term must be the same for all members of +** a "group". The requirement that the GROUP BY term must be BINARY +** assumes that no other collating sequence will have a finer-grained +** grouping than binary. In other words (A=B COLLATE binary) implies +** A=B in every other collating sequence. The requirement that the +** GROUP BY be BINARY is stricter than necessary. It would also work +** to promote HAVING clauses that use the same alternative collating +** sequence as the GROUP BY term, but that is much harder to check, +** alternative collating sequences are uncommon, and this is only an +** optimization, so we take the easy way out and simply require the +** GROUP BY to use the BINARY collating sequence. +*/ +SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprList *pGroupBy){ + Walker w; + w.eCode = 1; + w.xExprCallback = exprNodeIsConstantOrGroupBy; + w.xSelectCallback = 0; + w.u.pGroupBy = pGroupBy; + w.pParse = pParse; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + +/* +** Walk an expression tree for the DEFAULT field of a column definition +** in a CREATE TABLE statement. Return non-zero if the expression is +** acceptable for use as a DEFAULT. That is to say, return non-zero if +** the expression is constant or a function call with constant arguments. +** Return and 0 if there are any variables. +** +** isInit is true when parsing from sqlite_schema. isInit is false when +** processing a new CREATE TABLE statement. When isInit is true, parameters +** (such as ? or $abc) in the expression are converted into NULL. When +** isInit is false, parameters raise an error. Parameters should not be +** allowed in a CREATE TABLE statement, but some legacy versions of SQLite +** allowed it, so we need to support it when reading sqlite_schema for +** backwards compatibility. +** +** If isInit is true, set EP_FromDDL on every TK_FUNCTION node. +** +** For the purposes of this function, a double-quoted string (ex: "abc") +** is considered a variable but a single-quoted string (ex: 'abc') is +** a constant. +*/ +SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ + assert( isInit==0 || isInit==1 ); + return exprIsConst(p, 4+isInit, 0); +} + +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Walk an expression tree. Return 1 if the expression contains a +** subquery of some kind. Return 0 if there are no subqueries. +*/ +SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr *p){ + Walker w; + w.eCode = 1; + w.xExprCallback = sqlite3ExprWalkNoop; + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + sqlite3WalkExpr(&w, p); + return w.eCode==0; +} +#endif + +/* +** If the expression p codes a constant integer that is small enough +** to fit in a 32-bit integer, return 1 and put the value of the integer +** in *pValue. If the expression is not an integer or if it is too big +** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. +*/ +SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr *p, int *pValue){ + int rc = 0; + if( NEVER(p==0) ) return 0; /* Used to only happen following on OOM */ + + /* If an expression is an integer literal that fits in a signed 32-bit + ** integer, then the EP_IntValue flag will have already been set */ + assert( p->op!=TK_INTEGER || (p->flags & EP_IntValue)!=0 + || sqlite3GetInt32(p->u.zToken, &rc)==0 ); + + if( p->flags & EP_IntValue ){ + *pValue = p->u.iValue; + return 1; + } + switch( p->op ){ + case TK_UPLUS: { + rc = sqlite3ExprIsInteger(p->pLeft, pValue); + break; + } + case TK_UMINUS: { + int v; + if( sqlite3ExprIsInteger(p->pLeft, &v) ){ + assert( v!=(-2147483647-1) ); + *pValue = -v; + rc = 1; + } + break; + } + default: break; + } + return rc; +} + +/* +** Return FALSE if there is no chance that the expression can be NULL. +** +** If the expression might be NULL or if the expression is too complex +** to tell return TRUE. +** +** This routine is used as an optimization, to skip OP_IsNull opcodes +** when we know that a value cannot be NULL. Hence, a false positive +** (returning TRUE when in fact the expression can never be NULL) might +** be a small performance hit but is otherwise harmless. On the other +** hand, a false negative (returning FALSE when the result could be NULL) +** will likely result in an incorrect answer. So when in doubt, return +** TRUE. +*/ +SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ + u8 op; + assert( p!=0 ); + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + p = p->pLeft; + assert( p!=0 ); + } + op = p->op; + if( op==TK_REGISTER ) op = p->op2; + switch( op ){ + case TK_INTEGER: + case TK_STRING: + case TK_FLOAT: + case TK_BLOB: + return 0; + case TK_COLUMN: + return ExprHasProperty(p, EP_CanBeNull) || + p->y.pTab==0 || /* Reference to column of index on expression */ + (p->iColumn>=0 + && ALWAYS(p->y.pTab->aCol!=0) /* Defense against OOM problems */ + && p->y.pTab->aCol[p->iColumn].notNull==0); + default: + return 1; + } +} + +/* +** Return TRUE if the given expression is a constant which would be +** unchanged by OP_Affinity with the affinity given in the second +** argument. +** +** This routine is used to determine if the OP_Affinity operation +** can be omitted. When in doubt return FALSE. A false negative +** is harmless. A false positive, however, can result in the wrong +** answer. +*/ +SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ + u8 op; + int unaryMinus = 0; + if( aff==SQLITE_AFF_BLOB ) return 1; + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + if( p->op==TK_UMINUS ) unaryMinus = 1; + p = p->pLeft; + } + op = p->op; + if( op==TK_REGISTER ) op = p->op2; + switch( op ){ + case TK_INTEGER: { + return aff>=SQLITE_AFF_NUMERIC; + } + case TK_FLOAT: { + return aff>=SQLITE_AFF_NUMERIC; + } + case TK_STRING: { + return !unaryMinus && aff==SQLITE_AFF_TEXT; + } + case TK_BLOB: { + return !unaryMinus; + } + case TK_COLUMN: { + assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */ + return aff>=SQLITE_AFF_NUMERIC && p->iColumn<0; + } + default: { + return 0; + } + } +} + +/* +** Return TRUE if the given string is a row-id column name. +*/ +SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ + if( sqlite3StrICmp(z, "_ROWID_")==0 ) return 1; + if( sqlite3StrICmp(z, "ROWID")==0 ) return 1; + if( sqlite3StrICmp(z, "OID")==0 ) return 1; + return 0; +} + +/* +** pX is the RHS of an IN operator. If pX is a SELECT statement +** that can be simplified to a direct table access, then return +** a pointer to the SELECT statement. If pX is not a SELECT statement, +** or if the SELECT statement needs to be manifested into a transient +** table, then return NULL. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +static Select *isCandidateForInOpt(const Expr *pX){ + Select *p; + SrcList *pSrc; + ExprList *pEList; + Table *pTab; + int i; + if( !ExprHasProperty(pX, EP_xIsSelect) ) return 0; /* Not a subquery */ + if( ExprHasProperty(pX, EP_VarSelect) ) return 0; /* Correlated subq */ + p = pX->x.pSelect; + if( p->pPrior ) return 0; /* Not a compound SELECT */ + if( p->selFlags & (SF_Distinct|SF_Aggregate) ){ + testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); + testcase( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); + return 0; /* No DISTINCT keyword and no aggregate functions */ + } + assert( p->pGroupBy==0 ); /* Has no GROUP BY clause */ + if( p->pLimit ) return 0; /* Has no LIMIT clause */ + if( p->pWhere ) return 0; /* Has no WHERE clause */ + pSrc = p->pSrc; + assert( pSrc!=0 ); + if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ + if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ + pTab = pSrc->a[0].pTab; + assert( pTab!=0 ); + assert( !IsView(pTab) ); /* FROM clause is not a view */ + if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ + pEList = p->pEList; + assert( pEList!=0 ); + /* All SELECT results must be columns. */ + for(i=0; inExpr; i++){ + Expr *pRes = pEList->a[i].pExpr; + if( pRes->op!=TK_COLUMN ) return 0; + assert( pRes->iTable==pSrc->a[0].iCursor ); /* Not a correlated subquery */ + } + return p; +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code that checks the left-most column of index table iCur to see if +** it contains any NULL entries. Cause the register at regHasNull to be set +** to a non-NULL value if iCur contains no NULLs. Cause register regHasNull +** to be set to NULL if iCur contains one or more NULL values. +*/ +static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ + int addr1; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); + addr1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + VdbeComment((v, "first_entry_in(%d)", iCur)); + sqlite3VdbeJumpHere(v, addr1); +} +#endif + + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** The argument is an IN operator with a list (not a subquery) on the +** right-hand side. Return TRUE if that list is constant. +*/ +static int sqlite3InRhsIsConstant(Expr *pIn){ + Expr *pLHS; + int res; + assert( !ExprHasProperty(pIn, EP_xIsSelect) ); + pLHS = pIn->pLeft; + pIn->pLeft = 0; + res = sqlite3ExprIsConstant(pIn); + pIn->pLeft = pLHS; + return res; +} +#endif + +/* +** This function is used by the implementation of the IN (...) operator. +** The pX parameter is the expression on the RHS of the IN operator, which +** might be either a list of expressions or a subquery. +** +** The job of this routine is to find or create a b-tree object that can +** be used either to test for membership in the RHS set or to iterate through +** all members of the RHS set, skipping duplicates. +** +** A cursor is opened on the b-tree object that is the RHS of the IN operator +** and pX->iTable is set to the index of that cursor. +** +** The returned value of this function indicates the b-tree type, as follows: +** +** IN_INDEX_ROWID - The cursor was opened on a database table. +** IN_INDEX_INDEX_ASC - The cursor was opened on an ascending index. +** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. +** IN_INDEX_EPH - The cursor was opened on a specially created and +** populated epheremal table. +** IN_INDEX_NOOP - No cursor was allocated. The IN operator must be +** implemented as a sequence of comparisons. +** +** An existing b-tree might be used if the RHS expression pX is a simple +** subquery such as: +** +** SELECT , ... FROM +** +** If the RHS of the IN operator is a list or a more complex subquery, then +** an ephemeral table might need to be generated from the RHS and then +** pX->iTable made to point to the ephemeral table instead of an +** existing table. +** +** The inFlags parameter must contain, at a minimum, one of the bits +** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP but not both. If inFlags contains +** IN_INDEX_MEMBERSHIP, then the generated table will be used for a fast +** membership test. When the IN_INDEX_LOOP bit is set, the IN index will +** be used to loop over all values of the RHS of the IN operator. +** +** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate +** through the set members) then the b-tree must not contain duplicates. +** An epheremal table will be created unless the selected columns are guaranteed +** to be unique - either because it is an INTEGER PRIMARY KEY or due to +** a UNIQUE constraint or index. +** +** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used +** for fast set membership tests) then an epheremal table must +** be used unless is a single INTEGER PRIMARY KEY column or an +** index can be found with the specified as its left-most. +** +** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and +** if the RHS of the IN operator is a list (not a subquery) then this +** routine might decide that creating an ephemeral b-tree for membership +** testing is too expensive and return IN_INDEX_NOOP. In that case, the +** calling routine should implement the IN operator using a sequence +** of Eq or Ne comparison operations. +** +** When the b-tree is being used for membership tests, the calling function +** might need to know whether or not the RHS side of the IN operator +** contains a NULL. If prRhsHasNull is not a NULL pointer and +** if there is any chance that the (...) might contain a NULL value at +** runtime, then a register is allocated and the register number written +** to *prRhsHasNull. If there is no chance that the (...) contains a +** NULL value, then *prRhsHasNull is left unchanged. +** +** If a register is allocated and its location stored in *prRhsHasNull, then +** the value in that register will be NULL if the b-tree contains one or more +** NULL values, and it will be some non-NULL value if the b-tree contains no +** NULL values. +** +** If the aiMap parameter is not NULL, it must point to an array containing +** one element for each column returned by the SELECT statement on the RHS +** of the IN(...) operator. The i'th entry of the array is populated with the +** offset of the index column that matches the i'th column returned by the +** SELECT. For example, if the expression and selected index are: +** +** (?,?,?) IN (SELECT a, b, c FROM t1) +** CREATE INDEX i1 ON t1(b, c, a); +** +** then aiMap[] is populated with {2, 0, 1}. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +SQLITE_PRIVATE int sqlite3FindInIndex( + Parse *pParse, /* Parsing context */ + Expr *pX, /* The IN expression */ + u32 inFlags, /* IN_INDEX_LOOP, _MEMBERSHIP, and/or _NOOP_OK */ + int *prRhsHasNull, /* Register holding NULL status. See notes */ + int *aiMap, /* Mapping from Index fields to RHS fields */ + int *piTab /* OUT: index to use */ +){ + Select *p; /* SELECT to the right of IN operator */ + int eType = 0; /* Type of RHS table. IN_INDEX_* */ + int iTab = pParse->nTab++; /* Cursor of the RHS table */ + int mustBeUnique; /* True if RHS must be unique */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ + + assert( pX->op==TK_IN ); + mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; + + /* If the RHS of this IN(...) operator is a SELECT, and if it matters + ** whether or not the SELECT result contains NULL values, check whether + ** or not NULL is actually possible (it may not be, for example, due + ** to NOT NULL constraints in the schema). If no NULL values are possible, + ** set prRhsHasNull to 0 before continuing. */ + if( prRhsHasNull && (pX->flags & EP_xIsSelect) ){ + int i; + ExprList *pEList = pX->x.pSelect->pEList; + for(i=0; inExpr; i++){ + if( sqlite3ExprCanBeNull(pEList->a[i].pExpr) ) break; + } + if( i==pEList->nExpr ){ + prRhsHasNull = 0; + } + } + + /* Check to see if an existing table or index can be used to + ** satisfy the query. This is preferable to generating a new + ** ephemeral table. */ + if( pParse->nErr==0 && (p = isCandidateForInOpt(pX))!=0 ){ + sqlite3 *db = pParse->db; /* Database connection */ + Table *pTab; /* Table
    . */ + int iDb; /* Database idx for pTab */ + ExprList *pEList = p->pEList; + int nExpr = pEList->nExpr; + + assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ + assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ + pTab = p->pSrc->a[0].pTab; + + /* Code an OP_Transaction and OP_TableLock for
    . */ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 && iDbtnum, 0, pTab->zName); + + assert(v); /* sqlite3GetVdbe() has always been previously called */ + if( nExpr==1 && pEList->a[0].pExpr->iColumn<0 ){ + /* The "x IN (SELECT rowid FROM table)" case */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); + VdbeCoverage(v); + + sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + eType = IN_INDEX_ROWID; + ExplainQueryPlan((pParse, 0, + "USING ROWID SEARCH ON TABLE %s FOR IN-OPERATOR",pTab->zName)); + sqlite3VdbeJumpHere(v, iAddr); + }else{ + Index *pIdx; /* Iterator variable */ + int affinity_ok = 1; + int i; + + /* Check that the affinity that will be used to perform each + ** comparison is the same as the affinity of each column in table + ** on the RHS of the IN operator. If it not, it is not possible to + ** use any index of the RHS table. */ + for(i=0; ipLeft, i); + int iCol = pEList->a[i].pExpr->iColumn; + char idxaff = sqlite3TableColumnAffinity(pTab,iCol); /* RHS table */ + char cmpaff = sqlite3CompareAffinity(pLhs, idxaff); + testcase( cmpaff==SQLITE_AFF_BLOB ); + testcase( cmpaff==SQLITE_AFF_TEXT ); + switch( cmpaff ){ + case SQLITE_AFF_BLOB: + break; + case SQLITE_AFF_TEXT: + /* sqlite3CompareAffinity() only returns TEXT if one side or the + ** other has no affinity and the other side is TEXT. Hence, + ** the only way for cmpaff to be TEXT is for idxaff to be TEXT + ** and for the term on the LHS of the IN to have no affinity. */ + assert( idxaff==SQLITE_AFF_TEXT ); + break; + default: + affinity_ok = sqlite3IsNumericAffinity(idxaff); + } + } + + if( affinity_ok ){ + /* Search for an existing index that will work for this IN operator */ + for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){ + Bitmask colUsed; /* Columns of the index used */ + Bitmask mCol; /* Mask for the current column */ + if( pIdx->nColumnpPartIdxWhere!=0 ) continue; + /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute + ** BITMASK(nExpr) without overflowing */ + testcase( pIdx->nColumn==BMS-2 ); + testcase( pIdx->nColumn==BMS-1 ); + if( pIdx->nColumn>=BMS-1 ) continue; + if( mustBeUnique ){ + if( pIdx->nKeyCol>nExpr + ||(pIdx->nColumn>nExpr && !IsUniqueIndex(pIdx)) + ){ + continue; /* This index is not unique over the IN RHS columns */ + } + } + + colUsed = 0; /* Columns of index used so far */ + for(i=0; ipLeft, i); + Expr *pRhs = pEList->a[i].pExpr; + CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + int j; + + assert( pReq!=0 || pRhs->iColumn==XN_ROWID || pParse->nErr ); + for(j=0; jaiColumn[j]!=pRhs->iColumn ) continue; + assert( pIdx->azColl[j] ); + if( pReq!=0 && sqlite3StrICmp(pReq->zName, pIdx->azColl[j])!=0 ){ + continue; + } + break; + } + if( j==nExpr ) break; + mCol = MASKBIT(j); + if( mCol & colUsed ) break; /* Each column used only once */ + colUsed |= mCol; + if( aiMap ) aiMap[i] = j; + } + + assert( i==nExpr || colUsed!=(MASKBIT(nExpr)-1) ); + if( colUsed==(MASKBIT(nExpr)-1) ){ + /* If we reach this point, that means the index pIdx is usable */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + ExplainQueryPlan((pParse, 0, + "USING INDEX %s FOR IN-OPERATOR",pIdx->zName)); + sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; + + if( prRhsHasNull ){ +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + i64 mask = (1<nMem; + if( nExpr==1 ){ + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); + } + } + sqlite3VdbeJumpHere(v, iAddr); + } + } /* End loop over indexes */ + } /* End if( affinity_ok ) */ + } /* End if not an rowid index */ + } /* End attempt to optimize using an index */ + + /* If no preexisting index is available for the IN clause + ** and IN_INDEX_NOOP is an allowed reply + ** and the RHS of the IN operator is a list, not a subquery + ** and the RHS is not constant or has two or fewer terms, + ** then it is not worth creating an ephemeral table to evaluate + ** the IN operator so return IN_INDEX_NOOP. + */ + if( eType==0 + && (inFlags & IN_INDEX_NOOP_OK) + && !ExprHasProperty(pX, EP_xIsSelect) + && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + ){ + eType = IN_INDEX_NOOP; + } + + if( eType==0 ){ + /* Could not find an existing table or index to use as the RHS b-tree. + ** We will have to generate an ephemeral table to do the job. + */ + u32 savedNQueryLoop = pParse->nQueryLoop; + int rMayHaveNull = 0; + eType = IN_INDEX_EPH; + if( inFlags & IN_INDEX_LOOP ){ + pParse->nQueryLoop = 0; + }else if( prRhsHasNull ){ + *prRhsHasNull = rMayHaveNull = ++pParse->nMem; + } + assert( pX->op==TK_IN ); + sqlite3CodeRhsOfIN(pParse, pX, iTab); + if( rMayHaveNull ){ + sqlite3SetHasNullFlag(v, iTab, rMayHaveNull); + } + pParse->nQueryLoop = savedNQueryLoop; + } + + if( aiMap && eType!=IN_INDEX_INDEX_ASC && eType!=IN_INDEX_INDEX_DESC ){ + int i, n; + n = sqlite3ExprVectorSize(pX->pLeft); + for(i=0; ipLeft; + int nVal = sqlite3ExprVectorSize(pLeft); + Select *pSelect = (pExpr->flags & EP_xIsSelect) ? pExpr->x.pSelect : 0; + char *zRet; + + assert( pExpr->op==TK_IN ); + zRet = sqlite3DbMallocRaw(pParse->db, nVal+1); + if( zRet ){ + int i; + for(i=0; ipEList->a[i].pExpr, a); + }else{ + zRet[i] = a; + } + } + zRet[nVal] = '\0'; + } + return zRet; +} +#endif + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Load the Parse object passed as the first argument with an error +** message of the form: +** +** "sub-select returns N columns - expected M" +*/ +SQLITE_PRIVATE void sqlite3SubselectError(Parse *pParse, int nActual, int nExpect){ + if( pParse->nErr==0 ){ + const char *zFmt = "sub-select returns %d columns - expected %d"; + sqlite3ErrorMsg(pParse, zFmt, nActual, nExpect); + } +} +#endif + +/* +** Expression pExpr is a vector that has been used in a context where +** it is not permitted. If pExpr is a sub-select vector, this routine +** loads the Parse object with a message of the form: +** +** "sub-select returns N columns - expected 1" +** +** Or, if it is a regular scalar vector: +** +** "row value misused" +*/ +SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ +#ifndef SQLITE_OMIT_SUBQUERY + if( pExpr->flags & EP_xIsSelect ){ + sqlite3SubselectError(pParse, pExpr->x.pSelect->pEList->nExpr, 1); + }else +#endif + { + sqlite3ErrorMsg(pParse, "row value misused"); + } +} + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code that will construct an ephemeral table containing all terms +** in the RHS of an IN operator. The IN operator can be in either of two +** forms: +** +** x IN (4,5,11) -- IN operator with list on right-hand side +** x IN (SELECT a FROM b) -- IN operator with subquery on the right +** +** The pExpr parameter is the IN operator. The cursor number for the +** constructed ephermeral table is returned. The first time the ephemeral +** table is computed, the cursor number is also stored in pExpr->iTable, +** however the cursor number returned might not be the same, as it might +** have been duplicated using OP_OpenDup. +** +** If the LHS expression ("x" in the examples) is a column value, or +** the SELECT statement returns a column value, then the affinity of that +** column is used to build the index keys. If both 'x' and the +** SELECT... statement are columns, then numeric affinity is used +** if either column has NUMERIC or INTEGER affinity. If neither +** 'x' nor the SELECT... statement are columns, then numeric affinity +** is used. +*/ +SQLITE_PRIVATE void sqlite3CodeRhsOfIN( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The IN operator */ + int iTab /* Use this cursor number */ +){ + int addrOnce = 0; /* Address of the OP_Once instruction at top */ + int addr; /* Address of OP_OpenEphemeral instruction */ + Expr *pLeft; /* the LHS of the IN operator */ + KeyInfo *pKeyInfo = 0; /* Key information */ + int nVal; /* Size of vector pLeft */ + Vdbe *v; /* The prepared statement under construction */ + + v = pParse->pVdbe; + assert( v!=0 ); + + /* The evaluation of the IN must be repeated every time it + ** is encountered if any of the following is true: + ** + ** * The right-hand side is a correlated subquery + ** * The right-hand side is an expression list containing variables + ** * We are inside a trigger + ** + ** If all of the above are false, then we can compute the RHS just once + ** and reuse it many names. + */ + if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){ + /* Reuse of the RHS is allowed */ + /* If this routine has already been coded, but the previous code + ** might not have been invoked yet, so invoke it now as a subroutine. + */ + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d", + pExpr->x.pSelect->selId)); + } + sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr); + sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable); + sqlite3VdbeJumpHere(v, addrOnce); + return; + } + + /* Begin coding the subroutine */ + ExprSetProperty(pExpr, EP_Subrtn); + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + pExpr->y.sub.regReturn = ++pParse->nMem; + pExpr->y.sub.iAddr = + sqlite3VdbeAddOp2(v, OP_Integer, 0, pExpr->y.sub.regReturn) + 1; + VdbeComment((v, "return address")); + + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + + /* Check to see if this is a vector IN operator */ + pLeft = pExpr->pLeft; + nVal = sqlite3ExprVectorSize(pLeft); + + /* Construct the ephemeral table that will contain the content of + ** RHS of the IN operator. + */ + pExpr->iTable = iTab; + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, nVal); +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + VdbeComment((v, "Result of SELECT %u", pExpr->x.pSelect->selId)); + }else{ + VdbeComment((v, "RHS of IN operator")); + } +#endif + pKeyInfo = sqlite3KeyInfoAlloc(pParse->db, nVal, 1); + + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* Case 1: expr IN (SELECT ...) + ** + ** Generate code to write the results of the select into the temporary + ** table allocated and opened above. + */ + Select *pSelect = pExpr->x.pSelect; + ExprList *pEList = pSelect->pEList; + + ExplainQueryPlan((pParse, 1, "%sLIST SUBQUERY %d", + addrOnce?"":"CORRELATED ", pSelect->selId + )); + /* If the LHS and RHS of the IN operator do not match, that + ** error will have been caught long before we reach this point. */ + if( ALWAYS(pEList->nExpr==nVal) ){ + Select *pCopy; + SelectDest dest; + int i; + int rc; + sqlite3SelectDestInit(&dest, SRT_Set, iTab); + dest.zAffSdst = exprINAffinity(pParse, pExpr); + pSelect->iLimit = 0; + testcase( pSelect->selFlags & SF_Distinct ); + testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ + pCopy = sqlite3SelectDup(pParse->db, pSelect, 0); + rc = pParse->db->mallocFailed ? 1 :sqlite3Select(pParse, pCopy, &dest); + sqlite3SelectDelete(pParse->db, pCopy); + sqlite3DbFree(pParse->db, dest.zAffSdst); + if( rc ){ + sqlite3KeyInfoUnref(pKeyInfo); + return; + } + assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ + assert( pEList!=0 ); + assert( pEList->nExpr>0 ); + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + for(i=0; iaColl[i] = sqlite3BinaryCompareCollSeq( + pParse, p, pEList->a[i].pExpr + ); + } + } + }else if( ALWAYS(pExpr->x.pList!=0) ){ + /* Case 2: expr IN (exprlist) + ** + ** For each expression, build an index key from the evaluation and + ** store it in the temporary table. If is a column, then use + ** that columns affinity when building index keys. If is not + ** a column, use numeric affinity. + */ + char affinity; /* Affinity of the LHS of the IN */ + int i; + ExprList *pList = pExpr->x.pList; + struct ExprList_item *pItem; + int r1, r2; + affinity = sqlite3ExprAffinity(pLeft); + if( affinity<=SQLITE_AFF_NONE ){ + affinity = SQLITE_AFF_BLOB; + }else if( affinity==SQLITE_AFF_REAL ){ + affinity = SQLITE_AFF_NUMERIC; + } + if( pKeyInfo ){ + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + pKeyInfo->aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + } + + /* Loop through each expression in . */ + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempReg(pParse); + for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ + Expr *pE2 = pItem->pExpr; + + /* If the expression is not constant then we will need to + ** disable the test that was generated above that makes sure + ** this code only executes once. Because for a non-constant + ** expression we need to rerun this code each time. + */ + if( addrOnce && !sqlite3ExprIsConstant(pE2) ){ + sqlite3VdbeChangeToNoop(v, addrOnce); + ExprClearProperty(pExpr, EP_Subrtn); + addrOnce = 0; + } + + /* Evaluate the expression and insert it into the temp table */ + sqlite3ExprCode(pParse, pE2, r1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, r1, 1, r2, &affinity, 1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r2, r1, 1); + } + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempReg(pParse, r2); + } + if( pKeyInfo ){ + sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); + } + if( addrOnce ){ + sqlite3VdbeJumpHere(v, addrOnce); + /* Subroutine return */ + sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn); + sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1); + sqlite3ClearTempRegCache(pParse); + } +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +/* +** Generate code for scalar subqueries used as a subquery expression +** or EXISTS operator: +** +** (SELECT a FROM b) -- subquery +** EXISTS (SELECT a FROM b) -- EXISTS subquery +** +** The pExpr parameter is the SELECT or EXISTS operator to be coded. +** +** Return the register that holds the result. For a multi-column SELECT, +** the result is stored in a contiguous array of registers and the +** return value is the register of the left-most result column. +** Return 0 if an error occurs. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ + int addrOnce = 0; /* Address of OP_Once at top of subroutine */ + int rReg = 0; /* Register storing resulting */ + Select *pSel; /* SELECT statement to encode */ + SelectDest dest; /* How to deal with SELECT result */ + int nReg; /* Registers to allocate */ + Expr *pLimit; /* New limit expression */ + + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + if( pParse->nErr ) return 0; + testcase( pExpr->op==TK_EXISTS ); + testcase( pExpr->op==TK_SELECT ); + assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); + assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + pSel = pExpr->x.pSelect; + + /* If this routine has already been coded, then invoke it as a + ** subroutine. */ + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + ExplainQueryPlan((pParse, 0, "REUSE SUBQUERY %d", pSel->selId)); + sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr); + return pExpr->iTable; + } + + /* Begin coding the subroutine */ + ExprSetProperty(pExpr, EP_Subrtn); + pExpr->y.sub.regReturn = ++pParse->nMem; + pExpr->y.sub.iAddr = + sqlite3VdbeAddOp2(v, OP_Integer, 0, pExpr->y.sub.regReturn) + 1; + VdbeComment((v, "return address")); + + + /* The evaluation of the EXISTS/SELECT must be repeated every time it + ** is encountered if any of the following is true: + ** + ** * The right-hand side is a correlated subquery + ** * The right-hand side is an expression list containing variables + ** * We are inside a trigger + ** + ** If all of the above are false, then we can run this code just once + ** save the results, and reuse the same result on subsequent invocations. + */ + if( !ExprHasProperty(pExpr, EP_VarSelect) ){ + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + + /* For a SELECT, generate code to put the values for all columns of + ** the first row into an array of registers and return the index of + ** the first register. + ** + ** If this is an EXISTS, write an integer 0 (not exists) or 1 (exists) + ** into a register and return that register number. + ** + ** In both cases, the query is augmented with "LIMIT 1". Any + ** preexisting limit is discarded in place of the new LIMIT 1. + */ + ExplainQueryPlan((pParse, 1, "%sSCALAR SUBQUERY %d", + addrOnce?"":"CORRELATED ", pSel->selId)); + nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; + sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); + pParse->nMem += nReg; + if( pExpr->op==TK_SELECT ){ + dest.eDest = SRT_Mem; + dest.iSdst = dest.iSDParm; + dest.nSdst = nReg; + sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); + VdbeComment((v, "Init subquery result")); + }else{ + dest.eDest = SRT_Exists; + sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm); + VdbeComment((v, "Init EXISTS result")); + } + if( pSel->pLimit ){ + /* The subquery already has a limit. If the pre-existing limit is X + ** then make the new limit X<>0 so that the new limit is either 1 or 0 */ + sqlite3 *db = pParse->db; + pLimit = sqlite3Expr(db, TK_INTEGER, "0"); + if( pLimit ){ + pLimit->affExpr = SQLITE_AFF_NUMERIC; + pLimit = sqlite3PExpr(pParse, TK_NE, + sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); + } + sqlite3ExprDelete(db, pSel->pLimit->pLeft); + pSel->pLimit->pLeft = pLimit; + }else{ + /* If there is no pre-existing limit add a limit of 1 */ + pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); + pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0); + } + pSel->iLimit = 0; + if( sqlite3Select(pParse, pSel, &dest) ){ + if( pParse->nErr ){ + pExpr->op2 = pExpr->op; + pExpr->op = TK_ERROR; + } + return 0; + } + pExpr->iTable = rReg = dest.iSDParm; + ExprSetVVAProperty(pExpr, EP_NoReduce); + if( addrOnce ){ + sqlite3VdbeJumpHere(v, addrOnce); + } + + /* Subroutine return */ + sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn); + sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1); + sqlite3ClearTempRegCache(pParse); + return rReg; +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Expr pIn is an IN(...) expression. This function checks that the +** sub-select on the RHS of the IN() operator has the same number of +** columns as the vector on the LHS. Or, if the RHS of the IN() is not +** a sub-query, that the LHS is a vector of size 1. +*/ +SQLITE_PRIVATE int sqlite3ExprCheckIN(Parse *pParse, Expr *pIn){ + int nVector = sqlite3ExprVectorSize(pIn->pLeft); + if( (pIn->flags & EP_xIsSelect)!=0 && !pParse->db->mallocFailed ){ + if( nVector!=pIn->x.pSelect->pEList->nExpr ){ + sqlite3SubselectError(pParse, pIn->x.pSelect->pEList->nExpr, nVector); + return 1; + } + }else if( nVector!=1 ){ + sqlite3VectorErrorMsg(pParse, pIn->pLeft); + return 1; + } + return 0; +} +#endif + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code for an IN expression. +** +** x IN (SELECT ...) +** x IN (value, value, ...) +** +** The left-hand side (LHS) is a scalar or vector expression. The +** right-hand side (RHS) is an array of zero or more scalar values, or a +** subquery. If the RHS is a subquery, the number of result columns must +** match the number of columns in the vector on the LHS. If the RHS is +** a list of values, the LHS must be a scalar. +** +** The IN operator is true if the LHS value is contained within the RHS. +** The result is false if the LHS is definitely not in the RHS. The +** result is NULL if the presence of the LHS in the RHS cannot be +** determined due to NULLs. +** +** This routine generates code that jumps to destIfFalse if the LHS is not +** contained within the RHS. If due to NULLs we cannot determine if the LHS +** is contained in the RHS then jump to destIfNull. If the LHS is contained +** within the RHS then fall through. +** +** See the separate in-operator.md documentation file in the canonical +** SQLite source tree for additional information. +*/ +static void sqlite3ExprCodeIN( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The IN expression */ + int destIfFalse, /* Jump here if LHS is not contained in the RHS */ + int destIfNull /* Jump here if the results are unknown due to NULLs */ +){ + int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ + int eType; /* Type of the RHS */ + int rLhs; /* Register(s) holding the LHS values */ + int rLhsOrig; /* LHS values prior to reordering by aiMap[] */ + Vdbe *v; /* Statement under construction */ + int *aiMap = 0; /* Map from vector field to index column */ + char *zAff = 0; /* Affinity string for comparisons */ + int nVector; /* Size of vectors for this IN operator */ + int iDummy; /* Dummy parameter to exprCodeVector() */ + Expr *pLeft; /* The LHS of the IN operator */ + int i; /* loop counter */ + int destStep2; /* Where to jump when NULLs seen in step 2 */ + int destStep6 = 0; /* Start of code for Step 6 */ + int addrTruthOp; /* Address of opcode that determines the IN is true */ + int destNotNull; /* Jump here if a comparison is not true in step 6 */ + int addrTop; /* Top of the step-6 loop */ + int iTab = 0; /* Index to use */ + u8 okConstFactor = pParse->okConstFactor; + + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); + pLeft = pExpr->pLeft; + if( sqlite3ExprCheckIN(pParse, pExpr) ) return; + zAff = exprINAffinity(pParse, pExpr); + nVector = sqlite3ExprVectorSize(pExpr->pLeft); + aiMap = (int*)sqlite3DbMallocZero( + pParse->db, nVector*(sizeof(int) + sizeof(char)) + 1 + ); + if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; + + /* Attempt to compute the RHS. After this step, if anything other than + ** IN_INDEX_NOOP is returned, the table opened with cursor iTab + ** contains the values that make up the RHS. If IN_INDEX_NOOP is returned, + ** the RHS has not yet been coded. */ + v = pParse->pVdbe; + assert( v!=0 ); /* OOM detected prior to this routine */ + VdbeNoopComment((v, "begin IN expr")); + eType = sqlite3FindInIndex(pParse, pExpr, + IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, + destIfFalse==destIfNull ? 0 : &rRhsHasNull, + aiMap, &iTab); + + assert( pParse->nErr || nVector==1 || eType==IN_INDEX_EPH + || eType==IN_INDEX_INDEX_ASC || eType==IN_INDEX_INDEX_DESC + ); +#ifdef SQLITE_DEBUG + /* Confirm that aiMap[] contains nVector integer values between 0 and + ** nVector-1. */ + for(i=0; i from " IN (...)". If the LHS is a + ** vector, then it is stored in an array of nVector registers starting + ** at r1. + ** + ** sqlite3FindInIndex() might have reordered the fields of the LHS vector + ** so that the fields are in the same order as an existing index. The + ** aiMap[] array contains a mapping from the original LHS field order to + ** the field order that matches the RHS index. + ** + ** Avoid factoring the LHS of the IN(...) expression out of the loop, + ** even if it is constant, as OP_Affinity may be used on the register + ** by code generated below. */ + assert( pParse->okConstFactor==okConstFactor ); + pParse->okConstFactor = 0; + rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); + pParse->okConstFactor = okConstFactor; + for(i=0; ix.pList; + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + int labelOk = sqlite3VdbeMakeLabel(pParse); + int r2, regToFree; + int regCkNull = 0; + int ii; + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + if( destIfNull!=destIfFalse ){ + regCkNull = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_BitAnd, rLhs, rLhs, regCkNull); + } + for(ii=0; iinExpr; ii++){ + r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); + if( regCkNull && sqlite3ExprCanBeNull(pList->a[ii].pExpr) ){ + sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); + } + sqlite3ReleaseTempReg(pParse, regToFree); + if( iinExpr-1 || destIfNull!=destIfFalse ){ + int op = rLhs!=r2 ? OP_Eq : OP_NotNull; + sqlite3VdbeAddOp4(v, op, rLhs, labelOk, r2, + (void*)pColl, P4_COLLSEQ); + VdbeCoverageIf(v, iinExpr-1 && op==OP_Eq); + VdbeCoverageIf(v, ii==pList->nExpr-1 && op==OP_Eq); + VdbeCoverageIf(v, iinExpr-1 && op==OP_NotNull); + VdbeCoverageIf(v, ii==pList->nExpr-1 && op==OP_NotNull); + sqlite3VdbeChangeP5(v, zAff[0]); + }else{ + int op = rLhs!=r2 ? OP_Ne : OP_IsNull; + assert( destIfNull==destIfFalse ); + sqlite3VdbeAddOp4(v, op, rLhs, destIfFalse, r2, + (void*)pColl, P4_COLLSEQ); + VdbeCoverageIf(v, op==OP_Ne); + VdbeCoverageIf(v, op==OP_IsNull); + sqlite3VdbeChangeP5(v, zAff[0] | SQLITE_JUMPIFNULL); + } + } + if( regCkNull ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); + sqlite3VdbeGoto(v, destIfFalse); + } + sqlite3VdbeResolveLabel(v, labelOk); + sqlite3ReleaseTempReg(pParse, regCkNull); + goto sqlite3ExprCodeIN_finished; + } + + /* Step 2: Check to see if the LHS contains any NULL columns. If the + ** LHS does contain NULLs then the result must be either FALSE or NULL. + ** We will then skip the binary search of the RHS. + */ + if( destIfNull==destIfFalse ){ + destStep2 = destIfFalse; + }else{ + destStep2 = destStep6 = sqlite3VdbeMakeLabel(pParse); + } + if( pParse->nErr ) goto sqlite3ExprCodeIN_finished; + for(i=0; ipLeft, i); + if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; + if( sqlite3ExprCanBeNull(p) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, rLhs+i, destStep2); + VdbeCoverage(v); + } + } + + /* Step 3. The LHS is now known to be non-NULL. Do the binary search + ** of the RHS using the LHS as a probe. If found, the result is + ** true. + */ + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree and so we also + ** know that the RHS is non-NULL. Hence, we combine steps 3 and 4 + ** into a single opcode. */ + sqlite3VdbeAddOp3(v, OP_SeekRowid, iTab, destIfFalse, rLhs); + VdbeCoverage(v); + addrTruthOp = sqlite3VdbeAddOp0(v, OP_Goto); /* Return True */ + }else{ + sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); + if( destIfFalse==destIfNull ){ + /* Combine Step 3 and Step 5 into a single opcode */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, iTab, destIfFalse, + rLhs, nVector); VdbeCoverage(v); + goto sqlite3ExprCodeIN_finished; + } + /* Ordinary Step 3, for the case where FALSE and NULL are distinct */ + addrTruthOp = sqlite3VdbeAddOp4Int(v, OP_Found, iTab, 0, + rLhs, nVector); VdbeCoverage(v); + } + + /* Step 4. If the RHS is known to be non-NULL and we did not find + ** an match on the search above, then the result must be FALSE. + */ + if( rRhsHasNull && nVector==1 ){ + sqlite3VdbeAddOp2(v, OP_NotNull, rRhsHasNull, destIfFalse); + VdbeCoverage(v); + } + + /* Step 5. If we do not care about the difference between NULL and + ** FALSE, then just return false. + */ + if( destIfFalse==destIfNull ) sqlite3VdbeGoto(v, destIfFalse); + + /* Step 6: Loop through rows of the RHS. Compare each row to the LHS. + ** If any comparison is NULL, then the result is NULL. If all + ** comparisons are FALSE then the final result is FALSE. + ** + ** For a scalar LHS, it is sufficient to check just the first row + ** of the RHS. + */ + if( destStep6 ) sqlite3VdbeResolveLabel(v, destStep6); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, destIfFalse); + VdbeCoverage(v); + if( nVector>1 ){ + destNotNull = sqlite3VdbeMakeLabel(pParse); + }else{ + /* For nVector==1, combine steps 6 and 7 by immediately returning + ** FALSE if the first comparison is not NULL */ + destNotNull = destIfFalse; + } + for(i=0; i1 ){ + sqlite3VdbeResolveLabel(v, destNotNull); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addrTop+1); + VdbeCoverage(v); + + /* Step 7: If we reach this point, we know that the result must + ** be false. */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + } + + /* Jumps here in order to return true. */ + sqlite3VdbeJumpHere(v, addrTruthOp); + +sqlite3ExprCodeIN_finished: + if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); + VdbeComment((v, "end IN expr")); +sqlite3ExprCodeIN_oom_error: + sqlite3DbFree(pParse->db, aiMap); + sqlite3DbFree(pParse->db, zAff); +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Generate an instruction that will put the floating point +** value described by z[0..n-1] into register iMem. +** +** The z[] string will probably not be zero-terminated. But the +** z[n] character is guaranteed to be something that does not look +** like the continuation of the number. +*/ +static void codeReal(Vdbe *v, const char *z, int negateFlag, int iMem){ + if( ALWAYS(z!=0) ){ + double value; + sqlite3AtoF(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + assert( !sqlite3IsNaN(value) ); /* The new AtoF never returns NaN */ + if( negateFlag ) value = -value; + sqlite3VdbeAddOp4Dup8(v, OP_Real, 0, iMem, 0, (u8*)&value, P4_REAL); + } +} +#endif + + +/* +** Generate an instruction that will put the integer describe by +** text z[0..n-1] into register iMem. +** +** Expr.u.zToken is always UTF8 and zero-terminated. +*/ +static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ + Vdbe *v = pParse->pVdbe; + if( pExpr->flags & EP_IntValue ){ + int i = pExpr->u.iValue; + assert( i>=0 ); + if( negFlag ) i = -i; + sqlite3VdbeAddOp2(v, OP_Integer, i, iMem); + }else{ + int c; + i64 value; + const char *z = pExpr->u.zToken; + assert( z!=0 ); + c = sqlite3DecOrHexToI64(z, &value); + if( (c==3 && !negFlag) || (c==2) || (negFlag && value==SMALLEST_INT64)){ +#ifdef SQLITE_OMIT_FLOATING_POINT + sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); +#else +#ifndef SQLITE_OMIT_HEX_INTEGER + if( sqlite3_strnicmp(z,"0x",2)==0 ){ + sqlite3ErrorMsg(pParse, "hex literal too big: %s%s", negFlag?"-":"",z); + }else +#endif + { + codeReal(v, z, negFlag, iMem); + } +#endif + }else{ + if( negFlag ){ value = c==3 ? SMALLEST_INT64 : -value; } + sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64); + } + } +} + + +/* Generate code that will load into register regOut a value that is +** appropriate for the iIdxCol-th column of index pIdx. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn( + Parse *pParse, /* The parsing context */ + Index *pIdx, /* The index whose column is to be loaded */ + int iTabCur, /* Cursor pointing to a table row */ + int iIdxCol, /* The column of the index to be loaded */ + int regOut /* Store the index column value in this register */ +){ + i16 iTabCol = pIdx->aiColumn[iIdxCol]; + if( iTabCol==XN_EXPR ){ + assert( pIdx->aColExpr ); + assert( pIdx->aColExpr->nExpr>iIdxCol ); + pParse->iSelfTab = iTabCur + 1; + sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); + pParse->iSelfTab = 0; + }else{ + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, + iTabCol, regOut); + } +} + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* +** Generate code that will compute the value of generated column pCol +** and store the result in register regOut +*/ +SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn( + Parse *pParse, /* Parsing context */ + Table *pTab, /* Table containing the generated column */ + Column *pCol, /* The generated column */ + int regOut /* Put the result in this register */ +){ + int iAddr; + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + assert( pParse->iSelfTab!=0 ); + if( pParse->iSelfTab>0 ){ + iAddr = sqlite3VdbeAddOp3(v, OP_IfNullRow, pParse->iSelfTab-1, 0, regOut); + }else{ + iAddr = 0; + } + sqlite3ExprCodeCopy(pParse, sqlite3ColumnExpr(pTab,pCol), regOut); + if( pCol->affinity>=SQLITE_AFF_TEXT ){ + sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); + } + if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + +/* +** Generate code to extract the value of the iCol-th column of a table. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( + Vdbe *v, /* Parsing context */ + Table *pTab, /* The table containing the value */ + int iTabCur, /* The table cursor. Or the PK cursor for WITHOUT ROWID */ + int iCol, /* Index of the column to extract */ + int regOut /* Extract the value into this register */ +){ + Column *pCol; + assert( v!=0 ); + if( pTab==0 ){ + sqlite3VdbeAddOp3(v, OP_Column, iTabCur, iCol, regOut); + return; + } + if( iCol<0 || iCol==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); + }else{ + int op; + int x; + if( IsVirtual(pTab) ){ + op = OP_VColumn; + x = iCol; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( (pCol = &pTab->aCol[iCol])->colFlags & COLFLAG_VIRTUAL ){ + Parse *pParse = sqlite3VdbeParser(v); + if( pCol->colFlags & COLFLAG_BUSY ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", + pCol->zCnName); + }else{ + int savedSelfTab = pParse->iSelfTab; + pCol->colFlags |= COLFLAG_BUSY; + pParse->iSelfTab = iTabCur+1; + sqlite3ExprCodeGeneratedColumn(pParse, pTab, pCol, regOut); + pParse->iSelfTab = savedSelfTab; + pCol->colFlags &= ~COLFLAG_BUSY; + } + return; +#endif + }else if( !HasRowid(pTab) ){ + testcase( iCol!=sqlite3TableColumnToStorage(pTab, iCol) ); + x = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), iCol); + op = OP_Column; + }else{ + x = sqlite3TableColumnToStorage(pTab,iCol); + testcase( x!=iCol ); + op = OP_Column; + } + sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); + sqlite3ColumnDefault(v, pTab, iCol, regOut); + } +} + +/* +** Generate code that will extract the iColumn-th column from +** table pTab and store the column value in register iReg. +** +** There must be an open cursor to pTab in iTable when this routine +** is called. If iColumn<0 then code is generated that extracts the rowid. +*/ +SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Description of the table we are reading from */ + int iColumn, /* Index of the table column */ + int iTable, /* The cursor pointing to the table */ + int iReg, /* Store results here */ + u8 p5 /* P5 value for OP_Column + FLAGS */ +){ + assert( pParse->pVdbe!=0 ); + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg); + if( p5 ){ + VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1); + if( pOp->opcode==OP_Column ) pOp->p5 = p5; + } + return iReg; +} + +/* +** Generate code to move content from registers iFrom...iFrom+nReg-1 +** over to iTo..iTo+nReg-1. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); +} + +/* +** Convert a scalar expression node to a TK_REGISTER referencing +** register iReg. The caller must ensure that iReg already contains +** the correct value for the expression. +*/ +static void exprToRegister(Expr *pExpr, int iReg){ + Expr *p = sqlite3ExprSkipCollateAndLikely(pExpr); + if( NEVER(p==0) ) return; + p->op2 = p->op; + p->op = TK_REGISTER; + p->iTable = iReg; + ExprClearProperty(p, EP_Skip); +} + +/* +** Evaluate an expression (either a vector or a scalar expression) and store +** the result in continguous temporary registers. Return the index of +** the first register used to store the result. +** +** If the returned result register is a temporary scalar, then also write +** that register number into *piFreeable. If the returned result register +** is not a temporary or if the expression is a vector set *piFreeable +** to 0. +*/ +static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ + int iResult; + int nResult = sqlite3ExprVectorSize(p); + if( nResult==1 ){ + iResult = sqlite3ExprCodeTemp(pParse, p, piFreeable); + }else{ + *piFreeable = 0; + if( p->op==TK_SELECT ){ +#if SQLITE_OMIT_SUBQUERY + iResult = 0; +#else + iResult = sqlite3CodeSubselect(pParse, p); +#endif + }else{ + int i; + iResult = pParse->nMem+1; + pParse->nMem += nResult; + for(i=0; ix.pList->a[i].pExpr, i+iResult); + } + } + } + return iResult; +} + +/* +** If the last opcode is a OP_Copy, then set the do-not-merge flag (p5) +** so that a subsequent copy will not be merged into this one. +*/ +static void setDoNotMergeFlagOnCopy(Vdbe *v){ + if( sqlite3VdbeGetOp(v, -1)->opcode==OP_Copy ){ + sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergable */ + } +} + +/* +** Generate code to implement special SQL functions that are implemented +** in-line rather than by using the usual callbacks. +*/ +static int exprCodeInlineFunction( + Parse *pParse, /* Parsing context */ + ExprList *pFarg, /* List of function arguments */ + int iFuncId, /* Function ID. One of the INTFUNC_... values */ + int target /* Store function result in this register */ +){ + int nFarg; + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + assert( pFarg!=0 ); + nFarg = pFarg->nExpr; + assert( nFarg>0 ); /* All in-line functions have at least one argument */ + switch( iFuncId ){ + case INLINEFUNC_coalesce: { + /* Attempt a direct implementation of the built-in COALESCE() and + ** IFNULL() functions. This avoids unnecessary evaluation of + ** arguments past the first non-NULL argument. + */ + int endCoalesce = sqlite3VdbeMakeLabel(pParse); + int i; + assert( nFarg>=2 ); + sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + for(i=1; ia[i].pExpr, target); + } + setDoNotMergeFlagOnCopy(v); + sqlite3VdbeResolveLabel(v, endCoalesce); + break; + } + case INLINEFUNC_iif: { + Expr caseExpr; + memset(&caseExpr, 0, sizeof(caseExpr)); + caseExpr.op = TK_CASE; + caseExpr.x.pList = pFarg; + return sqlite3ExprCodeTarget(pParse, &caseExpr, target); + } + + default: { + /* The UNLIKELY() function is a no-op. The result is the value + ** of the first argument. + */ + assert( nFarg==1 || nFarg==2 ); + target = sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); + break; + } + + /*********************************************************************** + ** Test-only SQL functions that are only usable if enabled + ** via SQLITE_TESTCTRL_INTERNAL_FUNCTIONS + */ +#if !defined(SQLITE_UNTESTABLE) + case INLINEFUNC_expr_compare: { + /* Compare two expressions using sqlite3ExprCompare() */ + assert( nFarg==2 ); + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprCompare(0,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1), + target); + break; + } + + case INLINEFUNC_expr_implies_expr: { + /* Compare two expressions using sqlite3ExprImpliesExpr() */ + assert( nFarg==2 ); + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprImpliesExpr(pParse,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1), + target); + break; + } + + case INLINEFUNC_implies_nonnull_row: { + /* REsult of sqlite3ExprImpliesNonNullRow() */ + Expr *pA1; + assert( nFarg==2 ); + pA1 = pFarg->a[1].pExpr; + if( pA1->op==TK_COLUMN ){ + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprImpliesNonNullRow(pFarg->a[0].pExpr,pA1->iTable), + target); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + break; + } + + case INLINEFUNC_affinity: { + /* The AFFINITY() function evaluates to a string that describes + ** the type affinity of the argument. This is used for testing of + ** the SQLite type logic. + */ + const char *azAff[] = { "blob", "text", "numeric", "integer", "real" }; + char aff; + assert( nFarg==1 ); + aff = sqlite3ExprAffinity(pFarg->a[0].pExpr); + sqlite3VdbeLoadString(v, target, + (aff<=SQLITE_AFF_NONE) ? "none" : azAff[aff-SQLITE_AFF_BLOB]); + break; + } +#endif /* !defined(SQLITE_UNTESTABLE) */ + } + return target; +} + + +/* +** Generate code into the current Vdbe to evaluate the given +** expression. Attempt to store the results in register "target". +** Return the register where results are stored. +** +** With this routine, there is no guarantee that results will +** be stored in target. The result might be stored in some other +** register if it is convenient to do so. The calling function +** must check the return code and move the results to the desired +** register. +*/ +SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ + Vdbe *v = pParse->pVdbe; /* The VM under construction */ + int op; /* The opcode being coded */ + int inReg = target; /* Results stored in register inReg */ + int regFree1 = 0; /* If non-zero free this temporary register */ + int regFree2 = 0; /* If non-zero free this temporary register */ + int r1, r2; /* Various register numbers */ + Expr tempX; /* Temporary expression node */ + int p5 = 0; + + assert( target>0 && target<=pParse->nMem ); + assert( v!=0 ); + +expr_code_doover: + if( pExpr==0 ){ + op = TK_NULL; + }else{ + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); + op = pExpr->op; + } + switch( op ){ + case TK_AGG_COLUMN: { + AggInfo *pAggInfo = pExpr->pAggInfo; + struct AggInfo_col *pCol; + assert( pAggInfo!=0 ); + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + pCol = &pAggInfo->aCol[pExpr->iAgg]; + if( !pAggInfo->directMode ){ + assert( pCol->iMem>0 ); + return pCol->iMem; + }else if( pAggInfo->useSortingIdx ){ + Table *pTab = pCol->pTab; + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, + pCol->iSorterColumn, target); + if( pCol->iColumn<0 ){ + VdbeComment((v,"%s.rowid",pTab->zName)); + }else{ + VdbeComment((v,"%s.%s", + pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); + if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + } + } + return target; + } + /* Otherwise, fall thru into the TK_COLUMN case */ + /* no break */ deliberate_fall_through + } + case TK_COLUMN: { + int iTab = pExpr->iTable; + int iReg; + if( ExprHasProperty(pExpr, EP_FixedCol) ){ + /* This COLUMN expression is really a constant due to WHERE clause + ** constraints, and that constant is coded by the pExpr->pLeft + ** expresssion. However, make sure the constant has the correct + ** datatype by applying the Affinity of the table column to the + ** constant. + */ + int aff; + iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target); + if( pExpr->y.pTab ){ + aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); + }else{ + aff = pExpr->affExpr; + } + if( aff>SQLITE_AFF_BLOB ){ + static const char zAff[] = "B\000C\000D\000E"; + assert( SQLITE_AFF_BLOB=='A' ); + assert( SQLITE_AFF_TEXT=='B' ); + sqlite3VdbeAddOp4(v, OP_Affinity, iReg, 1, 0, + &zAff[(aff-'B')*2], P4_STATIC); + } + return iReg; + } + if( iTab<0 ){ + if( pParse->iSelfTab<0 ){ + /* Other columns in the same row for CHECK constraints or + ** generated columns or for inserting into partial index. + ** The row is unpacked into registers beginning at + ** 0-(pParse->iSelfTab). The rowid (if any) is in a register + ** immediately prior to the first column. + */ + Column *pCol; + Table *pTab = pExpr->y.pTab; + int iSrc; + int iCol = pExpr->iColumn; + assert( pTab!=0 ); + assert( iCol>=XN_ROWID ); + assert( iColnCol ); + if( iCol<0 ){ + return -1-pParse->iSelfTab; + } + pCol = pTab->aCol + iCol; + testcase( iCol!=sqlite3TableColumnToStorage(pTab,iCol) ); + iSrc = sqlite3TableColumnToStorage(pTab, iCol) - pParse->iSelfTab; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pCol->colFlags & COLFLAG_GENERATED ){ + if( pCol->colFlags & COLFLAG_BUSY ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", + pCol->zCnName); + return 0; + } + pCol->colFlags |= COLFLAG_BUSY; + if( pCol->colFlags & COLFLAG_NOTAVAIL ){ + sqlite3ExprCodeGeneratedColumn(pParse, pTab, pCol, iSrc); + } + pCol->colFlags &= ~(COLFLAG_BUSY|COLFLAG_NOTAVAIL); + return iSrc; + }else +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + if( pCol->affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp2(v, OP_SCopy, iSrc, target); + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + return target; + }else{ + return iSrc; + } + }else{ + /* Coding an expression that is part of an index where column names + ** in the index refer to the table to which the index belongs */ + iTab = pParse->iSelfTab - 1; + } + } + iReg = sqlite3ExprCodeGetColumn(pParse, pExpr->y.pTab, + pExpr->iColumn, iTab, target, + pExpr->op2); + if( pExpr->y.pTab==0 && pExpr->affExpr==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); + } + return iReg; + } + case TK_INTEGER: { + codeInteger(pParse, pExpr, 0, target); + return target; + } + case TK_TRUEFALSE: { + sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprTruthValue(pExpr), target); + return target; + } +#ifndef SQLITE_OMIT_FLOATING_POINT + case TK_FLOAT: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + codeReal(v, pExpr->u.zToken, 0, target); + return target; + } +#endif + case TK_STRING: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3VdbeLoadString(v, target, pExpr->u.zToken); + return target; + } + default: { + /* Make NULL the default case so that if a bug causes an illegal + ** Expr node to be passed into this function, it will be handled + ** sanely and not crash. But keep the assert() to bring the problem + ** to the attention of the developers. */ + assert( op==TK_NULL || op==TK_ERROR || pParse->db->mallocFailed ); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + return target; + } +#ifndef SQLITE_OMIT_BLOB_LITERAL + case TK_BLOB: { + int n; + const char *z; + char *zBlob; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); + assert( pExpr->u.zToken[1]=='\'' ); + z = &pExpr->u.zToken[2]; + n = sqlite3Strlen30(z) - 1; + assert( z[n]=='\'' ); + zBlob = sqlite3HexToBlob(sqlite3VdbeDb(v), z, n); + sqlite3VdbeAddOp4(v, OP_Blob, n/2, target, 0, zBlob, P4_DYNAMIC); + return target; + } +#endif + case TK_VARIABLE: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + assert( pExpr->u.zToken!=0 ); + assert( pExpr->u.zToken[0]!=0 ); + sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); + if( pExpr->u.zToken[1]!=0 ){ + const char *z = sqlite3VListNumToName(pParse->pVList, pExpr->iColumn); + assert( pExpr->u.zToken[0]=='?' || (z && !strcmp(pExpr->u.zToken, z)) ); + pParse->pVList[0] = 0; /* Indicate VList may no longer be enlarged */ + sqlite3VdbeAppendP4(v, (char*)z, P4_STATIC); + } + return target; + } + case TK_REGISTER: { + return pExpr->iTable; + } +#ifndef SQLITE_OMIT_CAST + case TK_CAST: { + /* Expressions of the form: CAST(pLeft AS token) */ + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + if( inReg!=target ){ + sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); + inReg = target; + } + sqlite3VdbeAddOp2(v, OP_Cast, target, + sqlite3AffinityType(pExpr->u.zToken, 0)); + return inReg; + } +#endif /* SQLITE_OMIT_CAST */ + case TK_IS: + case TK_ISNOT: + op = (op==TK_IS) ? TK_EQ : TK_NE; + p5 = SQLITE_NULLEQ; + /* fall-through */ + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + Expr *pLeft = pExpr->pLeft; + if( sqlite3ExprIsVector(pLeft) ){ + codeVectorCompare(pParse, pExpr, target, op, p5); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + sqlite3VdbeAddOp2(v, OP_Integer, 1, inReg); + codeCompare(pParse, pLeft, pExpr->pRight, op, r1, r2, + sqlite3VdbeCurrentAddr(v)+2, p5, + ExprHasProperty(pExpr,EP_Commuted)); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + if( p5==SQLITE_NULLEQ ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, inReg); + }else{ + sqlite3VdbeAddOp3(v, OP_ZeroOrNull, r1, inReg, r2); + } + testcase( regFree1==0 ); + testcase( regFree2==0 ); + } + break; + } + case TK_AND: + case TK_OR: + case TK_PLUS: + case TK_STAR: + case TK_MINUS: + case TK_REM: + case TK_BITAND: + case TK_BITOR: + case TK_SLASH: + case TK_LSHIFT: + case TK_RSHIFT: + case TK_CONCAT: { + assert( TK_AND==OP_And ); testcase( op==TK_AND ); + assert( TK_OR==OP_Or ); testcase( op==TK_OR ); + assert( TK_PLUS==OP_Add ); testcase( op==TK_PLUS ); + assert( TK_MINUS==OP_Subtract ); testcase( op==TK_MINUS ); + assert( TK_REM==OP_Remainder ); testcase( op==TK_REM ); + assert( TK_BITAND==OP_BitAnd ); testcase( op==TK_BITAND ); + assert( TK_BITOR==OP_BitOr ); testcase( op==TK_BITOR ); + assert( TK_SLASH==OP_Divide ); testcase( op==TK_SLASH ); + assert( TK_LSHIFT==OP_ShiftLeft ); testcase( op==TK_LSHIFT ); + assert( TK_RSHIFT==OP_ShiftRight ); testcase( op==TK_RSHIFT ); + assert( TK_CONCAT==OP_Concat ); testcase( op==TK_CONCAT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + sqlite3VdbeAddOp3(v, op, r2, r1, target); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_UMINUS: { + Expr *pLeft = pExpr->pLeft; + assert( pLeft ); + if( pLeft->op==TK_INTEGER ){ + codeInteger(pParse, pLeft, 1, target); + return target; +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( pLeft->op==TK_FLOAT ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + codeReal(v, pLeft->u.zToken, 1, target); + return target; +#endif + }else{ + tempX.op = TK_INTEGER; + tempX.flags = EP_IntValue|EP_TokenOnly; + tempX.u.iValue = 0; + ExprClearVVAProperties(&tempX); + r1 = sqlite3ExprCodeTemp(pParse, &tempX, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free2); + sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); + testcase( regFree2==0 ); + } + break; + } + case TK_BITNOT: + case TK_NOT: { + assert( TK_BITNOT==OP_BitNot ); testcase( op==TK_BITNOT ); + assert( TK_NOT==OP_Not ); testcase( op==TK_NOT ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + sqlite3VdbeAddOp2(v, op, r1, inReg); + break; + } + case TK_TRUTH: { + int isTrue; /* IS TRUE or IS NOT TRUE */ + int bNormal; /* IS TRUE or IS FALSE */ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + bNormal = pExpr->op2==TK_IS; + testcase( isTrue && bNormal); + testcase( !isTrue && bNormal); + sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !isTrue, isTrue ^ bNormal); + break; + } + case TK_ISNULL: + case TK_NOTNULL: { + int addr; + assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); + assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + addr = sqlite3VdbeAddOp1(v, op, r1); + VdbeCoverageIf(v, op==TK_ISNULL); + VdbeCoverageIf(v, op==TK_NOTNULL); + sqlite3VdbeAddOp2(v, OP_Integer, 0, target); + sqlite3VdbeJumpHere(v, addr); + break; + } + case TK_AGG_FUNCTION: { + AggInfo *pInfo = pExpr->pAggInfo; + if( pInfo==0 + || NEVER(pExpr->iAgg<0) + || NEVER(pExpr->iAgg>=pInfo->nFunc) + ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); + }else{ + return pInfo->aFunc[pExpr->iAgg].iMem; + } + break; + } + case TK_FUNCTION: { + ExprList *pFarg; /* List of function arguments */ + int nFarg; /* Number of function arguments */ + FuncDef *pDef; /* The function definition object */ + const char *zId; /* The function name */ + u32 constMask = 0; /* Mask of function arguments that are constant */ + int i; /* Loop counter */ + sqlite3 *db = pParse->db; /* The database connection */ + u8 enc = ENC(db); /* The text encoding used by this database */ + CollSeq *pColl = 0; /* A collating sequence */ + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + return pExpr->y.pWin->regResult; + } +#endif + + if( ConstFactorOk(pParse) && sqlite3ExprIsConstantNotJoin(pExpr) ){ + /* SQL functions can be expensive. So try to avoid running them + ** multiple times if we know they always give the same result */ + return sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); + } + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + assert( !ExprHasProperty(pExpr, EP_TokenOnly) ); + pFarg = pExpr->x.pList; + nFarg = pFarg ? pFarg->nExpr : 0; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + zId = pExpr->u.zToken; + pDef = sqlite3FindFunction(db, zId, nFarg, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 && pParse->explain ){ + pDef = sqlite3FindFunction(db, "unknown", nFarg, enc, 0); + } +#endif + if( pDef==0 || pDef->xFinalize!=0 ){ + sqlite3ErrorMsg(pParse, "unknown function: %s()", zId); + break; + } + if( pDef->funcFlags & SQLITE_FUNC_INLINE ){ + assert( (pDef->funcFlags & SQLITE_FUNC_UNSAFE)==0 ); + assert( (pDef->funcFlags & SQLITE_FUNC_DIRECT)==0 ); + return exprCodeInlineFunction(pParse, pFarg, + SQLITE_PTR_TO_INT(pDef->pUserData), target); + }else if( pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE) ){ + sqlite3ExprFunctionUsable(pParse, pExpr, pDef); + } + + for(i=0; ia[i].pExpr) ){ + testcase( i==31 ); + constMask |= MASKBIT32(i); + } + if( (pDef->funcFlags & SQLITE_FUNC_NEEDCOLL)!=0 && !pColl ){ + pColl = sqlite3ExprCollSeq(pParse, pFarg->a[i].pExpr); + } + } + if( pFarg ){ + if( constMask ){ + r1 = pParse->nMem+1; + pParse->nMem += nFarg; + }else{ + r1 = sqlite3GetTempRange(pParse, nFarg); + } + + /* For length() and typeof() functions with a column argument, + ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG + ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data + ** loading. + */ + if( (pDef->funcFlags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ + u8 exprOp; + assert( nFarg==1 ); + assert( pFarg->a[0].pExpr!=0 ); + exprOp = pFarg->a[0].pExpr->op; + if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ + assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); + assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); + testcase( pDef->funcFlags & OPFLAG_LENGTHARG ); + pFarg->a[0].pExpr->op2 = + pDef->funcFlags & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG); + } + } + + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, + SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); + }else{ + r1 = 0; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Possibly overload the function if the first argument is + ** a virtual table column. + ** + ** For infix functions (LIKE, GLOB, REGEXP, and MATCH) use the + ** second argument, not the first, as the argument to test to + ** see if it is a column in a virtual table. This is done because + ** the left operand of infix functions (the operand we want to + ** control overloading) ends up as the second argument to the + ** function. The expression "A glob B" is equivalent to + ** "glob(B,A). We want to use the A in "A glob B" to test + ** for function overloading. But we use the B term in "glob(B,A)". + */ + if( nFarg>=2 && ExprHasProperty(pExpr, EP_InfixFunc) ){ + pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[1].pExpr); + }else if( nFarg>0 ){ + pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[0].pExpr); + } +#endif + if( pDef->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + if( !pColl ) pColl = db->pDfltColl; + sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); + } +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + if( pDef->funcFlags & SQLITE_FUNC_OFFSET ){ + Expr *pArg = pFarg->a[0].pExpr; + if( pArg->op==TK_COLUMN ){ + sqlite3VdbeAddOp3(v, OP_Offset, pArg->iTable, pArg->iColumn, target); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + }else +#endif + { + sqlite3VdbeAddFunctionCall(pParse, constMask, r1, target, nFarg, + pDef, pExpr->op2); + } + if( nFarg ){ + if( constMask==0 ){ + sqlite3ReleaseTempRange(pParse, r1, nFarg); + }else{ + sqlite3VdbeReleaseRegisters(pParse, r1, nFarg, constMask, 1); + } + } + return target; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_EXISTS: + case TK_SELECT: { + int nCol; + testcase( op==TK_EXISTS ); + testcase( op==TK_SELECT ); + if( pParse->db->mallocFailed ){ + return 0; + }else if( op==TK_SELECT && (nCol = pExpr->x.pSelect->pEList->nExpr)!=1 ){ + sqlite3SubselectError(pParse, nCol, 1); + }else{ + return sqlite3CodeSubselect(pParse, pExpr); + } + break; + } + case TK_SELECT_COLUMN: { + int n; + if( pExpr->pLeft->iTable==0 ){ + pExpr->pLeft->iTable = sqlite3CodeSubselect(pParse, pExpr->pLeft); + } + assert( pExpr->pLeft->op==TK_SELECT || pExpr->pLeft->op==TK_ERROR ); + n = sqlite3ExprVectorSize(pExpr->pLeft); + if( pExpr->iTable!=n ){ + sqlite3ErrorMsg(pParse, "%d columns assigned %d values", + pExpr->iTable, n); + } + return pExpr->pLeft->iTable + pExpr->iColumn; + } + case TK_IN: { + int destIfFalse = sqlite3VdbeMakeLabel(pParse); + int destIfNull = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); + sqlite3VdbeAddOp2(v, OP_Integer, 1, target); + sqlite3VdbeResolveLabel(v, destIfFalse); + sqlite3VdbeAddOp2(v, OP_AddImm, target, 0); + sqlite3VdbeResolveLabel(v, destIfNull); + return target; + } +#endif /* SQLITE_OMIT_SUBQUERY */ + + + /* + ** x BETWEEN y AND z + ** + ** This is equivalent to + ** + ** x>=y AND x<=z + ** + ** X is stored in pExpr->pLeft. + ** Y is stored in pExpr->pList->a[0].pExpr. + ** Z is stored in pExpr->pList->a[1].pExpr. + */ + case TK_BETWEEN: { + exprCodeBetween(pParse, pExpr, target, 0, 0); + return target; + } + case TK_SPAN: + case TK_COLLATE: + case TK_UPLUS: { + pExpr = pExpr->pLeft; + goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. OSSFuzz. */ + } + + case TK_TRIGGER: { + /* If the opcode is TK_TRIGGER, then the expression is a reference + ** to a column in the new.* or old.* pseudo-tables available to + ** trigger programs. In this case Expr.iTable is set to 1 for the + ** new.* pseudo-table, or 0 for the old.* pseudo-table. Expr.iColumn + ** is set to the column of the pseudo-table to read, or to -1 to + ** read the rowid field. + ** + ** The expression is implemented using an OP_Param opcode. The p1 + ** parameter is set to 0 for an old.rowid reference, or to (i+1) + ** to reference another column of the old.* pseudo-table, where + ** i is the index of the column. For a new.rowid reference, p1 is + ** set to (n+1), where n is the number of columns in each pseudo-table. + ** For a reference to any other column in the new.* pseudo-table, p1 + ** is set to (n+2+i), where n and i are as defined previously. For + ** example, if the table on which triggers are being fired is + ** declared as: + ** + ** CREATE TABLE t1(a, b); + ** + ** Then p1 is interpreted as follows: + ** + ** p1==0 -> old.rowid p1==3 -> new.rowid + ** p1==1 -> old.a p1==4 -> new.a + ** p1==2 -> old.b p1==5 -> new.b + */ + Table *pTab = pExpr->y.pTab; + int iCol = pExpr->iColumn; + int p1 = pExpr->iTable * (pTab->nCol+1) + 1 + + sqlite3TableColumnToStorage(pTab, iCol); + + assert( pExpr->iTable==0 || pExpr->iTable==1 ); + assert( iCol>=-1 && iColnCol ); + assert( pTab->iPKey<0 || iCol!=pTab->iPKey ); + assert( p1>=0 && p1<(pTab->nCol*2+2) ); + + sqlite3VdbeAddOp2(v, OP_Param, p1, target); + VdbeComment((v, "r[%d]=%s.%s", target, + (pExpr->iTable ? "new" : "old"), + (pExpr->iColumn<0 ? "rowid" : pExpr->y.pTab->aCol[iCol].zCnName) + )); + +#ifndef SQLITE_OMIT_FLOATING_POINT + /* If the column has REAL affinity, it may currently be stored as an + ** integer. Use OP_RealAffinity to make sure it is really real. + ** + ** EVIDENCE-OF: R-60985-57662 SQLite will convert the value back to + ** floating point when extracting it from the record. */ + if( iCol>=0 && pTab->aCol[iCol].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + } +#endif + break; + } + + case TK_VECTOR: { + sqlite3ErrorMsg(pParse, "row value misused"); + break; + } + + /* TK_IF_NULL_ROW Expr nodes are inserted ahead of expressions + ** that derive from the right-hand table of a LEFT JOIN. The + ** Expr.iTable value is the table number for the right-hand table. + ** The expression is only evaluated if that table is not currently + ** on a LEFT JOIN NULL row. + */ + case TK_IF_NULL_ROW: { + int addrINR; + u8 okConstFactor = pParse->okConstFactor; + addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); + /* Temporarily disable factoring of constant expressions, since + ** even though expressions may appear to be constant, they are not + ** really constant because they originate from the right-hand side + ** of a LEFT JOIN. */ + pParse->okConstFactor = 0; + inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); + pParse->okConstFactor = okConstFactor; + sqlite3VdbeJumpHere(v, addrINR); + sqlite3VdbeChangeP3(v, addrINR, inReg); + break; + } + + /* + ** Form A: + ** CASE x WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END + ** + ** Form B: + ** CASE WHEN e1 THEN r1 WHEN e2 THEN r2 ... WHEN eN THEN rN ELSE y END + ** + ** Form A is can be transformed into the equivalent form B as follows: + ** CASE WHEN x=e1 THEN r1 WHEN x=e2 THEN r2 ... + ** WHEN x=eN THEN rN ELSE y END + ** + ** X (if it exists) is in pExpr->pLeft. + ** Y is in the last element of pExpr->x.pList if pExpr->x.pList->nExpr is + ** odd. The Y is also optional. If the number of elements in x.pList + ** is even, then Y is omitted and the "otherwise" result is NULL. + ** Ei is in pExpr->pList->a[i*2] and Ri is pExpr->pList->a[i*2+1]. + ** + ** The result of the expression is the Ri for the first matching Ei, + ** or if there is no matching Ei, the ELSE term Y, or if there is + ** no ELSE term, NULL. + */ + case TK_CASE: { + int endLabel; /* GOTO label for end of CASE stmt */ + int nextCase; /* GOTO label for next WHEN clause */ + int nExpr; /* 2x number of WHEN terms */ + int i; /* Loop counter */ + ExprList *pEList; /* List of WHEN terms */ + struct ExprList_item *aListelem; /* Array of WHEN terms */ + Expr opCompare; /* The X==Ei expression */ + Expr *pX; /* The X expression */ + Expr *pTest = 0; /* X==Ei (form A) or just Ei (form B) */ + Expr *pDel = 0; + sqlite3 *db = pParse->db; + + assert( !ExprHasProperty(pExpr, EP_xIsSelect) && pExpr->x.pList ); + assert(pExpr->x.pList->nExpr > 0); + pEList = pExpr->x.pList; + aListelem = pEList->a; + nExpr = pEList->nExpr; + endLabel = sqlite3VdbeMakeLabel(pParse); + if( (pX = pExpr->pLeft)!=0 ){ + pDel = sqlite3ExprDup(db, pX, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDel); + break; + } + testcase( pX->op==TK_COLUMN ); + exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + testcase( regFree1==0 ); + memset(&opCompare, 0, sizeof(opCompare)); + opCompare.op = TK_EQ; + opCompare.pLeft = pDel; + pTest = &opCompare; + /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: + ** The value in regFree1 might get SCopy-ed into the file result. + ** So make sure that the regFree1 register is not reused for other + ** purposes and possibly overwritten. */ + regFree1 = 0; + } + for(i=0; iop==TK_COLUMN ); + sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); + testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); + sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); + sqlite3VdbeGoto(v, endLabel); + sqlite3VdbeResolveLabel(v, nextCase); + } + if( (nExpr&1)!=0 ){ + sqlite3ExprCode(pParse, pEList->a[nExpr-1].pExpr, target); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + sqlite3ExprDelete(db, pDel); + setDoNotMergeFlagOnCopy(v); + sqlite3VdbeResolveLabel(v, endLabel); + break; + } +#ifndef SQLITE_OMIT_TRIGGER + case TK_RAISE: { + assert( pExpr->affExpr==OE_Rollback + || pExpr->affExpr==OE_Abort + || pExpr->affExpr==OE_Fail + || pExpr->affExpr==OE_Ignore + ); + if( !pParse->pTriggerTab && !pParse->nested ){ + sqlite3ErrorMsg(pParse, + "RAISE() may only be used within a trigger-program"); + return 0; + } + if( pExpr->affExpr==OE_Abort ){ + sqlite3MayAbort(pParse); + } + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + if( pExpr->affExpr==OE_Ignore ){ + sqlite3VdbeAddOp4( + v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); + VdbeCoverage(v); + }else{ + sqlite3HaltConstraint(pParse, + pParse->pTriggerTab ? SQLITE_CONSTRAINT_TRIGGER : SQLITE_ERROR, + pExpr->affExpr, pExpr->u.zToken, 0, 0); + } + + break; + } +#endif + } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); + return inReg; +} + +/* +** Generate code that will evaluate expression pExpr just one time +** per prepared statement execution. +** +** If the expression uses functions (that might throw an exception) then +** guard them with an OP_Once opcode to ensure that the code is only executed +** once. If no functions are involved, then factor the code out and put it at +** the end of the prepared statement in the initialization section. +** +** If regDest>=0 then the result is always stored in that register and the +** result is not reusable. If regDest<0 then this routine is free to +** store the value whereever it wants. The register where the expression +** is stored is returned. When regDest<0, two identical expressions might +** code to the same register, if they do not contain function calls and hence +** are factored out into the initialization section at the end of the +** prepared statement. +*/ +SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The expression to code when the VDBE initializes */ + int regDest /* Store the value in this register */ +){ + ExprList *p; + assert( ConstFactorOk(pParse) ); + p = pParse->pConstExpr; + if( regDest<0 && p ){ + struct ExprList_item *pItem; + int i; + for(pItem=p->a, i=p->nExpr; i>0; pItem++, i--){ + if( pItem->reusable && sqlite3ExprCompare(0,pItem->pExpr,pExpr,-1)==0 ){ + return pItem->u.iConstExprReg; + } + } + } + pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + if( pExpr!=0 && ExprHasProperty(pExpr, EP_HasFunc) ){ + Vdbe *v = pParse->pVdbe; + int addr; + assert( v ); + addr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + pParse->okConstFactor = 0; + if( !pParse->db->mallocFailed ){ + if( regDest<0 ) regDest = ++pParse->nMem; + sqlite3ExprCode(pParse, pExpr, regDest); + } + pParse->okConstFactor = 1; + sqlite3ExprDelete(pParse->db, pExpr); + sqlite3VdbeJumpHere(v, addr); + }else{ + p = sqlite3ExprListAppend(pParse, p, pExpr); + if( p ){ + struct ExprList_item *pItem = &p->a[p->nExpr-1]; + pItem->reusable = regDest<0; + if( regDest<0 ) regDest = ++pParse->nMem; + pItem->u.iConstExprReg = regDest; + } + pParse->pConstExpr = p; + } + return regDest; +} + +/* +** Generate code to evaluate an expression and store the results +** into a register. Return the register number where the results +** are stored. +** +** If the register is a temporary register that can be deallocated, +** then write its number into *pReg. If the result register is not +** a temporary, then set *pReg to zero. +** +** If pExpr is a constant, then this routine might generate this +** code to fill the register in the initialization section of the +** VDBE program, in order to factor it out of the evaluation loop. +*/ +SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ + int r2; + pExpr = sqlite3ExprSkipCollateAndLikely(pExpr); + if( ConstFactorOk(pParse) + && ALWAYS(pExpr!=0) + && pExpr->op!=TK_REGISTER + && sqlite3ExprIsConstantNotJoin(pExpr) + ){ + *pReg = 0; + r2 = sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); + }else{ + int r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); + if( r2==r1 ){ + *pReg = r1; + }else{ + sqlite3ReleaseTempReg(pParse, r1); + *pReg = 0; + } + } + return r2; +} + +/* +** Generate code that will evaluate expression pExpr and store the +** results in register target. The results are guaranteed to appear +** in register target. +*/ +SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ + int inReg; + + assert( pExpr==0 || !ExprHasVVAProperty(pExpr,EP_Immutable) ); + assert( target>0 && target<=pParse->nMem ); + assert( pParse->pVdbe!=0 || pParse->db->mallocFailed ); + if( pParse->pVdbe==0 ) return; + inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + if( inReg!=target ){ + u8 op; + if( ALWAYS(pExpr) && ExprHasProperty(pExpr,EP_Subquery) ){ + op = OP_Copy; + }else{ + op = OP_SCopy; + } + sqlite3VdbeAddOp2(pParse->pVdbe, op, inReg, target); + } +} + +/* +** Make a transient copy of expression pExpr and then code it using +** sqlite3ExprCode(). This routine works just like sqlite3ExprCode() +** except that the input expression is guaranteed to be unchanged. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ + sqlite3 *db = pParse->db; + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( !db->mallocFailed ) sqlite3ExprCode(pParse, pExpr, target); + sqlite3ExprDelete(db, pExpr); +} + +/* +** Generate code that will evaluate expression pExpr and store the +** results in register target. The results are guaranteed to appear +** in register target. If the expression is constant, then this routine +** might choose to code the expression at initialization time. +*/ +SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ + if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pExpr) ){ + sqlite3ExprCodeRunJustOnce(pParse, pExpr, target); + }else{ + sqlite3ExprCodeCopy(pParse, pExpr, target); + } +} + +/* +** Generate code that pushes the value of every element of the given +** expression list into a sequence of registers beginning at target. +** +** Return the number of elements evaluated. The number returned will +** usually be pList->nExpr but might be reduced if SQLITE_ECEL_OMITREF +** is defined. +** +** The SQLITE_ECEL_DUP flag prevents the arguments from being +** filled using OP_SCopy. OP_Copy must be used instead. +** +** The SQLITE_ECEL_FACTOR argument allows constant arguments to be +** factored out into initialization code. +** +** The SQLITE_ECEL_REF flag means that expressions in the list with +** ExprList.a[].u.x.iOrderByCol>0 have already been evaluated and stored +** in registers at srcReg, and so the value can be copied from there. +** If SQLITE_ECEL_OMITREF is also set, then the values with u.x.iOrderByCol>0 +** are simply omitted rather than being copied from srcReg. +*/ +SQLITE_PRIVATE int sqlite3ExprCodeExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* The expression list to be coded */ + int target, /* Where to write results */ + int srcReg, /* Source registers if SQLITE_ECEL_REF */ + u8 flags /* SQLITE_ECEL_* flags */ +){ + struct ExprList_item *pItem; + int i, j, n; + u8 copyOp = (flags & SQLITE_ECEL_DUP) ? OP_Copy : OP_SCopy; + Vdbe *v = pParse->pVdbe; + assert( pList!=0 ); + assert( target>0 ); + assert( pParse->pVdbe!=0 ); /* Never gets this far otherwise */ + n = pList->nExpr; + if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; + for(pItem=pList->a, i=0; ipExpr; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pItem->bSorterRef ){ + i--; + n--; + }else +#endif + if( (flags & SQLITE_ECEL_REF)!=0 && (j = pItem->u.x.iOrderByCol)>0 ){ + if( flags & SQLITE_ECEL_OMITREF ){ + i--; + n--; + }else{ + sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + } + }else if( (flags & SQLITE_ECEL_FACTOR)!=0 + && sqlite3ExprIsConstantNotJoin(pExpr) + ){ + sqlite3ExprCodeRunJustOnce(pParse, pExpr, target+i); + }else{ + int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); + if( inReg!=target+i ){ + VdbeOp *pOp; + if( copyOp==OP_Copy + && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy + && pOp->p1+pOp->p3+1==inReg + && pOp->p2+pOp->p3+1==target+i + && pOp->p5==0 /* The do-not-merge flag must be clear */ + ){ + pOp->p3++; + }else{ + sqlite3VdbeAddOp2(v, copyOp, inReg, target+i); + } + } + } + } + return n; +} + +/* +** Generate code for a BETWEEN operator. +** +** x BETWEEN y AND z +** +** The above is equivalent to +** +** x>=y AND x<=z +** +** Code it as such, taking care to do the common subexpression +** elimination of x. +** +** The xJumpIf parameter determines details: +** +** NULL: Store the boolean result in reg[dest] +** sqlite3ExprIfTrue: Jump to dest if true +** sqlite3ExprIfFalse: Jump to dest if false +** +** The jumpIfNull parameter is ignored if xJumpIf is NULL. +*/ +static void exprCodeBetween( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The BETWEEN expression */ + int dest, /* Jump destination or storage location */ + void (*xJump)(Parse*,Expr*,int,int), /* Action to take */ + int jumpIfNull /* Take the jump if the BETWEEN is NULL */ +){ + Expr exprAnd; /* The AND operator in x>=y AND x<=z */ + Expr compLeft; /* The x>=y term */ + Expr compRight; /* The x<=z term */ + int regFree1 = 0; /* Temporary use register */ + Expr *pDel = 0; + sqlite3 *db = pParse->db; + + memset(&compLeft, 0, sizeof(Expr)); + memset(&compRight, 0, sizeof(Expr)); + memset(&exprAnd, 0, sizeof(Expr)); + + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + pDel = sqlite3ExprDup(db, pExpr->pLeft, 0); + if( db->mallocFailed==0 ){ + exprAnd.op = TK_AND; + exprAnd.pLeft = &compLeft; + exprAnd.pRight = &compRight; + compLeft.op = TK_GE; + compLeft.pLeft = pDel; + compLeft.pRight = pExpr->x.pList->a[0].pExpr; + compRight.op = TK_LE; + compRight.pLeft = pDel; + compRight.pRight = pExpr->x.pList->a[1].pExpr; + exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + if( xJump ){ + xJump(pParse, &exprAnd, dest, jumpIfNull); + }else{ + /* Mark the expression is being from the ON or USING clause of a join + ** so that the sqlite3ExprCodeTarget() routine will not attempt to move + ** it into the Parse.pConstExpr list. We should use a new bit for this, + ** for clarity, but we are out of bits in the Expr.flags field so we + ** have to reuse the EP_FromJoin bit. Bummer. */ + pDel->flags |= EP_FromJoin; + sqlite3ExprCodeTarget(pParse, &exprAnd, dest); + } + sqlite3ReleaseTempReg(pParse, regFree1); + } + sqlite3ExprDelete(db, pDel); + + /* Ensure adequate test coverage */ + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==0 ); +} + +/* +** Generate code for a boolean expression such that a jump is made +** to the label "dest" if the expression is true but execution +** continues straight thru if the expression is false. +** +** If the expression evaluates to NULL (neither true nor false), then +** take the jump if the jumpIfNull flag is SQLITE_JUMPIFNULL. +** +** This code depends on the fact that certain token values (ex: TK_EQ) +** are the same as opcode values (ex: OP_Eq) that implement the corresponding +** operation. Special comments in vdbe.c and the mkopcodeh.awk script in +** the make process cause these values to align. Assert()s in the code +** below verify that the numbers are aligned correctly. +*/ +SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ + Vdbe *v = pParse->pVdbe; + int op = 0; + int regFree1 = 0; + int regFree2 = 0; + int r1, r2; + + assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ + if( NEVER(pExpr==0) ) return; /* No way this can happen */ + assert( !ExprHasVVAProperty(pExpr, EP_Immutable) ); + op = pExpr->op; + switch( op ){ + case TK_AND: + case TK_OR: { + Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + sqlite3ExprIfTrue(pParse, pAlt, dest, jumpIfNull); + }else if( op==TK_AND ){ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + }else{ + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + } + break; + } + case TK_NOT: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + break; + } + case TK_TRUTH: { + int isNot; /* IS NOT TRUE or IS NOT FALSE */ + int isTrue; /* IS TRUE or IS NOT TRUE */ + testcase( jumpIfNull==0 ); + isNot = pExpr->op2==TK_ISNOT; + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + testcase( isTrue && isNot ); + testcase( !isTrue && isNot ); + if( isTrue ^ isNot ){ + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, + isNot ? SQLITE_JUMPIFNULL : 0); + }else{ + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, + isNot ? SQLITE_JUMPIFNULL : 0); + } + break; + } + case TK_IS: + case TK_ISNOT: + testcase( op==TK_IS ); + testcase( op==TK_ISNOT ); + op = (op==TK_IS) ? TK_EQ : TK_NE; + jumpIfNull = SQLITE_NULLEQ; + /* no break */ deliberate_fall_through + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; + testcase( jumpIfNull==0 ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, jumpIfNull, ExprHasProperty(pExpr,EP_Commuted)); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_ISNULL: + case TK_NOTNULL: { + assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); + assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp2(v, op, r1, dest); + VdbeCoverageIf(v, op==TK_ISNULL); + VdbeCoverageIf(v, op==TK_NOTNULL); + testcase( regFree1==0 ); + break; + } + case TK_BETWEEN: { + testcase( jumpIfNull==0 ); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfTrue, jumpIfNull); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_IN: { + int destIfFalse = sqlite3VdbeMakeLabel(pParse); + int destIfNull = jumpIfNull ? dest : destIfFalse; + sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); + sqlite3VdbeGoto(v, dest); + sqlite3VdbeResolveLabel(v, destIfFalse); + break; + } +#endif + default: { + default_expr: + if( ExprAlwaysTrue(pExpr) ){ + sqlite3VdbeGoto(v, dest); + }else if( ExprAlwaysFalse(pExpr) ){ + /* No-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); + VdbeCoverage(v); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } + break; + } + } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); +} + +/* +** Generate code for a boolean expression such that a jump is made +** to the label "dest" if the expression is false but execution +** continues straight thru if the expression is true. +** +** If the expression evaluates to NULL (neither true nor false) then +** jump if jumpIfNull is SQLITE_JUMPIFNULL or fall through if jumpIfNull +** is 0. +*/ +SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ + Vdbe *v = pParse->pVdbe; + int op = 0; + int regFree1 = 0; + int regFree2 = 0; + int r1, r2; + + assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ + if( pExpr==0 ) return; + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); + + /* The value of pExpr->op and op are related as follows: + ** + ** pExpr->op op + ** --------- ---------- + ** TK_ISNULL OP_NotNull + ** TK_NOTNULL OP_IsNull + ** TK_NE OP_Eq + ** TK_EQ OP_Ne + ** TK_GT OP_Le + ** TK_LE OP_Gt + ** TK_GE OP_Lt + ** TK_LT OP_Ge + ** + ** For other values of pExpr->op, op is undefined and unused. + ** The value of TK_ and OP_ constants are arranged such that we + ** can compute the mapping above using the following expression. + ** Assert()s verify that the computation is correct. + */ + op = ((pExpr->op+(TK_ISNULL&1))^1)-(TK_ISNULL&1); + + /* Verify correct alignment of TK_ and OP_ constants + */ + assert( pExpr->op!=TK_ISNULL || op==OP_NotNull ); + assert( pExpr->op!=TK_NOTNULL || op==OP_IsNull ); + assert( pExpr->op!=TK_NE || op==OP_Eq ); + assert( pExpr->op!=TK_EQ || op==OP_Ne ); + assert( pExpr->op!=TK_LT || op==OP_Ge ); + assert( pExpr->op!=TK_LE || op==OP_Gt ); + assert( pExpr->op!=TK_GT || op==OP_Le ); + assert( pExpr->op!=TK_GE || op==OP_Lt ); + + switch( pExpr->op ){ + case TK_AND: + case TK_OR: { + Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + sqlite3ExprIfFalse(pParse, pAlt, dest, jumpIfNull); + }else if( pExpr->op==TK_AND ){ + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + }else{ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + } + break; + } + case TK_NOT: { + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + break; + } + case TK_TRUTH: { + int isNot; /* IS NOT TRUE or IS NOT FALSE */ + int isTrue; /* IS TRUE or IS NOT TRUE */ + testcase( jumpIfNull==0 ); + isNot = pExpr->op2==TK_ISNOT; + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + testcase( isTrue && isNot ); + testcase( !isTrue && isNot ); + if( isTrue ^ isNot ){ + /* IS TRUE and IS NOT FALSE */ + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, + isNot ? 0 : SQLITE_JUMPIFNULL); + + }else{ + /* IS FALSE and IS NOT TRUE */ + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, + isNot ? 0 : SQLITE_JUMPIFNULL); + } + break; + } + case TK_IS: + case TK_ISNOT: + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; + jumpIfNull = SQLITE_NULLEQ; + /* no break */ deliberate_fall_through + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_NE: + case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; + testcase( jumpIfNull==0 ); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, + r1, r2, dest, jumpIfNull,ExprHasProperty(pExpr,EP_Commuted)); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Eq && jumpIfNull==SQLITE_NULLEQ); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull!=SQLITE_NULLEQ); + VdbeCoverageIf(v, op==OP_Ne && jumpIfNull==SQLITE_NULLEQ); + testcase( regFree1==0 ); + testcase( regFree2==0 ); + break; + } + case TK_ISNULL: + case TK_NOTNULL: { + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp2(v, op, r1, dest); + testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); + testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); + testcase( regFree1==0 ); + break; + } + case TK_BETWEEN: { + testcase( jumpIfNull==0 ); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfFalse, jumpIfNull); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_IN: { + if( jumpIfNull ){ + sqlite3ExprCodeIN(pParse, pExpr, dest, dest); + }else{ + int destIfNull = sqlite3VdbeMakeLabel(pParse); + sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull); + sqlite3VdbeResolveLabel(v, destIfNull); + } + break; + } +#endif + default: { + default_expr: + if( ExprAlwaysFalse(pExpr) ){ + sqlite3VdbeGoto(v, dest); + }else if( ExprAlwaysTrue(pExpr) ){ + /* no-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); + VdbeCoverage(v); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } + break; + } + } + sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ReleaseTempReg(pParse, regFree2); +} + +/* +** Like sqlite3ExprIfFalse() except that a copy is made of pExpr before +** code generation, and that copy is deleted after code generation. This +** ensures that the original pExpr is unchanged. +*/ +SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse *pParse, Expr *pExpr, int dest,int jumpIfNull){ + sqlite3 *db = pParse->db; + Expr *pCopy = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed==0 ){ + sqlite3ExprIfFalse(pParse, pCopy, dest, jumpIfNull); + } + sqlite3ExprDelete(db, pCopy); +} + +/* +** Expression pVar is guaranteed to be an SQL variable. pExpr may be any +** type of expression. +** +** If pExpr is a simple SQL value - an integer, real, string, blob +** or NULL value - then the VDBE currently being prepared is configured +** to re-prepare each time a new value is bound to variable pVar. +** +** Additionally, if pExpr is a simple SQL value and the value is the +** same as that currently bound to variable pVar, non-zero is returned. +** Otherwise, if the values are not the same or if pExpr is not a simple +** SQL value, zero is returned. +*/ +static int exprCompareVariable( + const Parse *pParse, + const Expr *pVar, + const Expr *pExpr +){ + int res = 0; + int iVar; + sqlite3_value *pL, *pR = 0; + + sqlite3ValueFromExpr(pParse->db, pExpr, SQLITE_UTF8, SQLITE_AFF_BLOB, &pR); + if( pR ){ + iVar = pVar->iColumn; + sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); + pL = sqlite3VdbeGetBoundValue(pParse->pReprepare, iVar, SQLITE_AFF_BLOB); + if( pL ){ + if( sqlite3_value_type(pL)==SQLITE_TEXT ){ + sqlite3_value_text(pL); /* Make sure the encoding is UTF-8 */ + } + res = 0==sqlite3MemCompare(pL, pR, 0); + } + sqlite3ValueFree(pR); + sqlite3ValueFree(pL); + } + + return res; +} + +/* +** Do a deep comparison of two expression trees. Return 0 if the two +** expressions are completely identical. Return 1 if they differ only +** by a COLLATE operator at the top level. Return 2 if there are differences +** other than the top-level COLLATE operator. +** +** If any subelement of pB has Expr.iTable==(-1) then it is allowed +** to compare equal to an equivalent element in pA with Expr.iTable==iTab. +** +** The pA side might be using TK_REGISTER. If that is the case and pB is +** not using TK_REGISTER but is otherwise equivalent, then still return 0. +** +** Sometimes this routine will return 2 even if the two expressions +** really are equivalent. If we cannot prove that the expressions are +** identical, we return 2 just to be safe. So if this routine +** returns 2, then you do not really know for certain if the two +** expressions are the same. But if you get a 0 or 1 return, then you +** can be sure the expressions are the same. In the places where +** this routine is used, it does not hurt to get an extra 2 - that +** just might result in some slightly slower code. But returning +** an incorrect 0 or 1 could lead to a malfunction. +** +** If pParse is not NULL then TK_VARIABLE terms in pA with bindings in +** pParse->pReprepare can be matched against literals in pB. The +** pParse->pVdbe->expmask bitmask is updated for each variable referenced. +** If pParse is NULL (the normal case) then any TK_VARIABLE term in +** Argument pParse should normally be NULL. If it is not NULL and pA or +** pB causes a return value of 2. +*/ +SQLITE_PRIVATE int sqlite3ExprCompare( + const Parse *pParse, + const Expr *pA, + const Expr *pB, + int iTab +){ + u32 combinedFlags; + if( pA==0 || pB==0 ){ + return pB==pA ? 0 : 2; + } + if( pParse && pA->op==TK_VARIABLE && exprCompareVariable(pParse, pA, pB) ){ + return 0; + } + combinedFlags = pA->flags | pB->flags; + if( combinedFlags & EP_IntValue ){ + if( (pA->flags&pB->flags&EP_IntValue)!=0 && pA->u.iValue==pB->u.iValue ){ + return 0; + } + return 2; + } + if( pA->op!=pB->op || pA->op==TK_RAISE ){ + if( pA->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA->pLeft,pB,iTab)<2 ){ + return 1; + } + if( pB->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA,pB->pLeft,iTab)<2 ){ + return 1; + } + return 2; + } + if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){ + if( pA->op==TK_FUNCTION || pA->op==TK_AGG_FUNCTION ){ + if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( pA->op==pB->op ); + if( ExprHasProperty(pA,EP_WinFunc)!=ExprHasProperty(pB,EP_WinFunc) ){ + return 2; + } + if( ExprHasProperty(pA,EP_WinFunc) ){ + if( sqlite3WindowCompare(pParse, pA->y.pWin, pB->y.pWin, 1)!=0 ){ + return 2; + } + } +#endif + }else if( pA->op==TK_NULL ){ + return 0; + }else if( pA->op==TK_COLLATE ){ + if( sqlite3_stricmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; + }else if( ALWAYS(pB->u.zToken!=0) && strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + return 2; + } + } + if( (pA->flags & (EP_Distinct|EP_Commuted)) + != (pB->flags & (EP_Distinct|EP_Commuted)) ) return 2; + if( ALWAYS((combinedFlags & EP_TokenOnly)==0) ){ + if( combinedFlags & EP_xIsSelect ) return 2; + if( (combinedFlags & EP_FixedCol)==0 + && sqlite3ExprCompare(pParse, pA->pLeft, pB->pLeft, iTab) ) return 2; + if( sqlite3ExprCompare(pParse, pA->pRight, pB->pRight, iTab) ) return 2; + if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2; + if( pA->op!=TK_STRING + && pA->op!=TK_TRUEFALSE + && ALWAYS((combinedFlags & EP_Reduced)==0) + ){ + if( pA->iColumn!=pB->iColumn ) return 2; + if( pA->op2!=pB->op2 && pA->op==TK_TRUTH ) return 2; + if( pA->op!=TK_IN && pA->iTable!=pB->iTable && pA->iTable!=iTab ){ + return 2; + } + } + } + return 0; +} + +/* +** Compare two ExprList objects. Return 0 if they are identical, 1 +** if they are certainly different, or 2 if it is not possible to +** determine if they are identical or not. +** +** If any subelement of pB has Expr.iTable==(-1) then it is allowed +** to compare equal to an equivalent element in pA with Expr.iTable==iTab. +** +** This routine might return non-zero for equivalent ExprLists. The +** only consequence will be disabled optimizations. But this routine +** must never return 0 if the two ExprList objects are different, or +** a malfunction will result. +** +** Two NULL pointers are considered to be the same. But a NULL pointer +** always differs from a non-NULL pointer. +*/ +SQLITE_PRIVATE int sqlite3ExprListCompare(const ExprList *pA, const ExprList *pB, int iTab){ + int i; + if( pA==0 && pB==0 ) return 0; + if( pA==0 || pB==0 ) return 1; + if( pA->nExpr!=pB->nExpr ) return 1; + for(i=0; inExpr; i++){ + int res; + Expr *pExprA = pA->a[i].pExpr; + Expr *pExprB = pB->a[i].pExpr; + if( pA->a[i].sortFlags!=pB->a[i].sortFlags ) return 1; + if( (res = sqlite3ExprCompare(0, pExprA, pExprB, iTab)) ) return res; + } + return 0; +} + +/* +** Like sqlite3ExprCompare() except COLLATE operators at the top-level +** are ignored. +*/ +SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr *pA,Expr *pB, int iTab){ + return sqlite3ExprCompare(0, + sqlite3ExprSkipCollateAndLikely(pA), + sqlite3ExprSkipCollateAndLikely(pB), + iTab); +} + +/* +** Return non-zero if Expr p can only be true if pNN is not NULL. +** +** Or if seenNot is true, return non-zero if Expr p can only be +** non-NULL if pNN is not NULL +*/ +static int exprImpliesNotNull( + const Parse *pParse,/* Parsing context */ + const Expr *p, /* The expression to be checked */ + const Expr *pNN, /* The expression that is NOT NULL */ + int iTab, /* Table being evaluated */ + int seenNot /* Return true only if p can be any non-NULL value */ +){ + assert( p ); + assert( pNN ); + if( sqlite3ExprCompare(pParse, p, pNN, iTab)==0 ){ + return pNN->op!=TK_NULL; + } + switch( p->op ){ + case TK_IN: { + if( seenNot && ExprHasProperty(p, EP_xIsSelect) ) return 0; + assert( ExprHasProperty(p,EP_xIsSelect) + || (p->x.pList!=0 && p->x.pList->nExpr>0) ); + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_BETWEEN: { + ExprList *pList = p->x.pList; + assert( pList!=0 ); + assert( pList->nExpr==2 ); + if( seenNot ) return 0; + if( exprImpliesNotNull(pParse, pList->a[0].pExpr, pNN, iTab, 1) + || exprImpliesNotNull(pParse, pList->a[1].pExpr, pNN, iTab, 1) + ){ + return 1; + } + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_PLUS: + case TK_MINUS: + case TK_BITOR: + case TK_LSHIFT: + case TK_RSHIFT: + case TK_CONCAT: + seenNot = 1; + /* no break */ deliberate_fall_through + case TK_STAR: + case TK_REM: + case TK_BITAND: + case TK_SLASH: { + if( exprImpliesNotNull(pParse, p->pRight, pNN, iTab, seenNot) ) return 1; + /* no break */ deliberate_fall_through + } + case TK_SPAN: + case TK_COLLATE: + case TK_UPLUS: + case TK_UMINUS: { + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, seenNot); + } + case TK_TRUTH: { + if( seenNot ) return 0; + if( p->op2!=TK_IS ) return 0; + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_BITNOT: + case TK_NOT: { + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + } + return 0; +} + +/* +** Return true if we can prove the pE2 will always be true if pE1 is +** true. Return false if we cannot complete the proof or if pE2 might +** be false. Examples: +** +** pE1: x==5 pE2: x==5 Result: true +** pE1: x>0 pE2: x==5 Result: false +** pE1: x=21 pE2: x=21 OR y=43 Result: true +** pE1: x!=123 pE2: x IS NOT NULL Result: true +** pE1: x!=?1 pE2: x IS NOT NULL Result: true +** pE1: x IS NULL pE2: x IS NOT NULL Result: false +** pE1: x IS ?2 pE2: x IS NOT NULL Reuslt: false +** +** When comparing TK_COLUMN nodes between pE1 and pE2, if pE2 has +** Expr.iTable<0 then assume a table number given by iTab. +** +** If pParse is not NULL, then the values of bound variables in pE1 are +** compared against literal values in pE2 and pParse->pVdbe->expmask is +** modified to record which bound variables are referenced. If pParse +** is NULL, then false will be returned if pE1 contains any bound variables. +** +** When in doubt, return false. Returning true might give a performance +** improvement. Returning false might cause a performance reduction, but +** it will always give the correct answer and is hence always safe. +*/ +SQLITE_PRIVATE int sqlite3ExprImpliesExpr( + const Parse *pParse, + const Expr *pE1, + const Expr *pE2, + int iTab +){ + if( sqlite3ExprCompare(pParse, pE1, pE2, iTab)==0 ){ + return 1; + } + if( pE2->op==TK_OR + && (sqlite3ExprImpliesExpr(pParse, pE1, pE2->pLeft, iTab) + || sqlite3ExprImpliesExpr(pParse, pE1, pE2->pRight, iTab) ) + ){ + return 1; + } + if( pE2->op==TK_NOTNULL + && exprImpliesNotNull(pParse, pE1, pE2->pLeft, iTab, 0) + ){ + return 1; + } + return 0; +} + +/* +** This is the Expr node callback for sqlite3ExprImpliesNonNullRow(). +** If the expression node requires that the table at pWalker->iCur +** have one or more non-NULL column, then set pWalker->eCode to 1 and abort. +** +** This routine controls an optimization. False positives (setting +** pWalker->eCode to 1 when it should not be) are deadly, but false-negatives +** (never setting pWalker->eCode) is a harmless missed optimization. +*/ +static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ + testcase( pExpr->op==TK_AGG_COLUMN ); + testcase( pExpr->op==TK_AGG_FUNCTION ); + if( ExprHasProperty(pExpr, EP_FromJoin) ) return WRC_Prune; + switch( pExpr->op ){ + case TK_ISNOT: + case TK_ISNULL: + case TK_NOTNULL: + case TK_IS: + case TK_OR: + case TK_VECTOR: + case TK_CASE: + case TK_IN: + case TK_FUNCTION: + case TK_TRUTH: + testcase( pExpr->op==TK_ISNOT ); + testcase( pExpr->op==TK_ISNULL ); + testcase( pExpr->op==TK_NOTNULL ); + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_OR ); + testcase( pExpr->op==TK_VECTOR ); + testcase( pExpr->op==TK_CASE ); + testcase( pExpr->op==TK_IN ); + testcase( pExpr->op==TK_FUNCTION ); + testcase( pExpr->op==TK_TRUTH ); + return WRC_Prune; + case TK_COLUMN: + if( pWalker->u.iCur==pExpr->iTable ){ + pWalker->eCode = 1; + return WRC_Abort; + } + return WRC_Prune; + + case TK_AND: + if( pWalker->eCode==0 ){ + sqlite3WalkExpr(pWalker, pExpr->pLeft); + if( pWalker->eCode ){ + pWalker->eCode = 0; + sqlite3WalkExpr(pWalker, pExpr->pRight); + } + } + return WRC_Prune; + + case TK_BETWEEN: + if( sqlite3WalkExpr(pWalker, pExpr->pLeft)==WRC_Abort ){ + assert( pWalker->eCode ); + return WRC_Abort; + } + return WRC_Prune; + + /* Virtual tables are allowed to use constraints like x=NULL. So + ** a term of the form x=y does not prove that y is not null if x + ** is the column of a virtual table */ + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: { + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + testcase( pExpr->op==TK_EQ ); + testcase( pExpr->op==TK_NE ); + testcase( pExpr->op==TK_LT ); + testcase( pExpr->op==TK_LE ); + testcase( pExpr->op==TK_GT ); + testcase( pExpr->op==TK_GE ); + /* The y.pTab=0 assignment in wherecode.c always happens after the + ** impliesNotNullRow() test */ + if( (pLeft->op==TK_COLUMN && pLeft->y.pTab!=0 + && IsVirtual(pLeft->y.pTab)) + || (pRight->op==TK_COLUMN && pRight->y.pTab!=0 + && IsVirtual(pRight->y.pTab)) + ){ + return WRC_Prune; + } + /* no break */ deliberate_fall_through + } + default: + return WRC_Continue; + } +} + +/* +** Return true (non-zero) if expression p can only be true if at least +** one column of table iTab is non-null. In other words, return true +** if expression p will always be NULL or false if every column of iTab +** is NULL. +** +** False negatives are acceptable. In other words, it is ok to return +** zero even if expression p will never be true of every column of iTab +** is NULL. A false negative is merely a missed optimization opportunity. +** +** False positives are not allowed, however. A false positive may result +** in an incorrect answer. +** +** Terms of p that are marked with EP_FromJoin (and hence that come from +** the ON or USING clauses of LEFT JOINS) are excluded from the analysis. +** +** This routine is used to check if a LEFT JOIN can be converted into +** an ordinary JOIN. The p argument is the WHERE clause. If the WHERE +** clause requires that some column of the right table of the LEFT JOIN +** be non-NULL, then the LEFT JOIN can be safely converted into an +** ordinary join. +*/ +SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ + Walker w; + p = sqlite3ExprSkipCollateAndLikely(p); + if( p==0 ) return 0; + if( p->op==TK_NOTNULL ){ + p = p->pLeft; + }else{ + while( p->op==TK_AND ){ + if( sqlite3ExprImpliesNonNullRow(p->pLeft, iTab) ) return 1; + p = p->pRight; + } + } + w.xExprCallback = impliesNotNullRow; + w.xSelectCallback = 0; + w.xSelectCallback2 = 0; + w.eCode = 0; + w.u.iCur = iTab; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + +/* +** An instance of the following structure is used by the tree walker +** to determine if an expression can be evaluated by reference to the +** index only, without having to do a search for the corresponding +** table entry. The IdxCover.pIdx field is the index. IdxCover.iCur +** is the cursor for the table. +*/ +struct IdxCover { + Index *pIdx; /* The index to be tested for coverage */ + int iCur; /* Cursor number for the table corresponding to the index */ +}; + +/* +** Check to see if there are references to columns in table +** pWalker->u.pIdxCover->iCur can be satisfied using the index +** pWalker->u.pIdxCover->pIdx. +*/ +static int exprIdxCover(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pWalker->u.pIdxCover->iCur + && sqlite3TableColumnToIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 + ){ + pWalker->eCode = 1; + return WRC_Abort; + } + return WRC_Continue; +} + +/* +** Determine if an index pIdx on table with cursor iCur contains will +** the expression pExpr. Return true if the index does cover the +** expression and false if the pExpr expression references table columns +** that are not found in the index pIdx. +** +** An index covering an expression means that the expression can be +** evaluated using only the index and without having to lookup the +** corresponding table entry. +*/ +SQLITE_PRIVATE int sqlite3ExprCoveredByIndex( + Expr *pExpr, /* The index to be tested */ + int iCur, /* The cursor number for the corresponding table */ + Index *pIdx /* The index that might be used for coverage */ +){ + Walker w; + struct IdxCover xcov; + memset(&w, 0, sizeof(w)); + xcov.iCur = iCur; + xcov.pIdx = pIdx; + w.xExprCallback = exprIdxCover; + w.u.pIdxCover = &xcov; + sqlite3WalkExpr(&w, pExpr); + return !w.eCode; +} + + +/* +** An instance of the following structure is used by the tree walker +** to count references to table columns in the arguments of an +** aggregate function, in order to implement the +** sqlite3FunctionThisSrc() routine. +*/ +struct SrcCount { + SrcList *pSrc; /* One particular FROM clause in a nested query */ + int iSrcInner; /* Smallest cursor number in this context */ + int nThis; /* Number of references to columns in pSrcList */ + int nOther; /* Number of references to columns in other FROM clauses */ +}; + +/* +** xSelect callback for sqlite3FunctionUsesThisSrc(). If this is the first +** SELECT with a FROM clause encountered during this iteration, set +** SrcCount.iSrcInner to the cursor number of the leftmost object in +** the FROM cause. +*/ +static int selectSrcCount(Walker *pWalker, Select *pSel){ + struct SrcCount *p = pWalker->u.pSrcCount; + if( p->iSrcInner==0x7FFFFFFF && ALWAYS(pSel->pSrc) && pSel->pSrc->nSrc ){ + pWalker->u.pSrcCount->iSrcInner = pSel->pSrc->a[0].iCursor; + } + return WRC_Continue; +} + +/* +** Count the number of references to columns. +*/ +static int exprSrcCount(Walker *pWalker, Expr *pExpr){ + /* There was once a NEVER() on the second term on the grounds that + ** sqlite3FunctionUsesThisSrc() was always called before + ** sqlite3ExprAnalyzeAggregates() and so the TK_COLUMNs have not yet + ** been converted into TK_AGG_COLUMN. But this is no longer true due + ** to window functions - sqlite3WindowRewrite() may now indirectly call + ** FunctionUsesThisSrc() when creating a new sub-select. */ + if( pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN ){ + int i; + struct SrcCount *p = pWalker->u.pSrcCount; + SrcList *pSrc = p->pSrc; + int nSrc = pSrc ? pSrc->nSrc : 0; + for(i=0; iiTable==pSrc->a[i].iCursor ) break; + } + if( inThis++; + }else if( pExpr->iTableiSrcInner ){ + /* In a well-formed parse tree (no name resolution errors), + ** TK_COLUMN nodes with smaller Expr.iTable values are in an + ** outer context. Those are the only ones to count as "other" */ + p->nOther++; + } + } + return WRC_Continue; +} + +/* +** Determine if any of the arguments to the pExpr Function reference +** pSrcList. Return true if they do. Also return true if the function +** has no arguments or has only constant arguments. Return false if pExpr +** references columns but not columns of tables found in pSrcList. +*/ +SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr *pExpr, SrcList *pSrcList){ + Walker w; + struct SrcCount cnt; + assert( pExpr->op==TK_AGG_FUNCTION ); + memset(&w, 0, sizeof(w)); + w.xExprCallback = exprSrcCount; + w.xSelectCallback = selectSrcCount; + w.u.pSrcCount = &cnt; + cnt.pSrc = pSrcList; + cnt.iSrcInner = (pSrcList&&pSrcList->nSrc)?pSrcList->a[0].iCursor:0x7FFFFFFF; + cnt.nThis = 0; + cnt.nOther = 0; + sqlite3WalkExprList(&w, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3WalkExpr(&w, pExpr->y.pWin->pFilter); + } +#endif + return cnt.nThis>0 || cnt.nOther==0; +} + +/* +** This is a Walker expression node callback. +** +** For Expr nodes that contain pAggInfo pointers, make sure the AggInfo +** object that is referenced does not refer directly to the Expr. If +** it does, make a copy. This is done because the pExpr argument is +** subject to change. +** +** The copy is stored on pParse->pConstExpr with a register number of 0. +** This will cause the expression to be deleted automatically when the +** Parse object is destroyed, but the zero register number means that it +** will not generate any code in the preamble. +*/ +static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ + if( ALWAYS(!ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced)) + && pExpr->pAggInfo!=0 + ){ + AggInfo *pAggInfo = pExpr->pAggInfo; + int iAgg = pExpr->iAgg; + Parse *pParse = pWalker->pParse; + sqlite3 *db = pParse->db; + assert( pExpr->op==TK_AGG_COLUMN || pExpr->op==TK_AGG_FUNCTION ); + if( pExpr->op==TK_AGG_COLUMN ){ + assert( iAgg>=0 && iAggnColumn ); + if( pAggInfo->aCol[iAgg].pCExpr==pExpr ){ + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( pExpr ){ + pAggInfo->aCol[iAgg].pCExpr = pExpr; + sqlite3ExprDeferredDelete(pParse, pExpr); + } + } + }else{ + assert( iAgg>=0 && iAggnFunc ); + if( pAggInfo->aFunc[iAgg].pFExpr==pExpr ){ + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( pExpr ){ + pAggInfo->aFunc[iAgg].pFExpr = pExpr; + sqlite3ExprDeferredDelete(pParse, pExpr); + } + } + } + } + return WRC_Continue; +} + +/* +** Initialize a Walker object so that will persist AggInfo entries referenced +** by the tree that is walked. +*/ +SQLITE_PRIVATE void sqlite3AggInfoPersistWalkerInit(Walker *pWalker, Parse *pParse){ + memset(pWalker, 0, sizeof(*pWalker)); + pWalker->pParse = pParse; + pWalker->xExprCallback = agginfoPersistExprCb; + pWalker->xSelectCallback = sqlite3SelectWalkNoop; +} + +/* +** Add a new element to the pAggInfo->aCol[] array. Return the index of +** the new element. Return a negative number if malloc fails. +*/ +static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ + int i; + pInfo->aCol = sqlite3ArrayAllocate( + db, + pInfo->aCol, + sizeof(pInfo->aCol[0]), + &pInfo->nColumn, + &i + ); + return i; +} + +/* +** Add a new element to the pAggInfo->aFunc[] array. Return the index of +** the new element. Return a negative number if malloc fails. +*/ +static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ + int i; + pInfo->aFunc = sqlite3ArrayAllocate( + db, + pInfo->aFunc, + sizeof(pInfo->aFunc[0]), + &pInfo->nFunc, + &i + ); + return i; +} + +/* +** This is the xExprCallback for a tree walker. It is used to +** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates +** for additional information. +*/ +static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ + int i; + NameContext *pNC = pWalker->u.pNC; + Parse *pParse = pNC->pParse; + SrcList *pSrcList = pNC->pSrcList; + AggInfo *pAggInfo = pNC->uNC.pAggInfo; + + assert( pNC->ncFlags & NC_UAggInfo ); + switch( pExpr->op ){ + case TK_AGG_COLUMN: + case TK_COLUMN: { + testcase( pExpr->op==TK_AGG_COLUMN ); + testcase( pExpr->op==TK_COLUMN ); + /* Check to see if the column is in one of the tables in the FROM + ** clause of the aggregate query */ + if( ALWAYS(pSrcList!=0) ){ + SrcItem *pItem = pSrcList->a; + for(i=0; inSrc; i++, pItem++){ + struct AggInfo_col *pCol; + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + if( pExpr->iTable==pItem->iCursor ){ + /* If we reach this point, it means that pExpr refers to a table + ** that is in the FROM clause of the aggregate query. + ** + ** Make an entry for the column in pAggInfo->aCol[] if there + ** is not an entry there already. + */ + int k; + pCol = pAggInfo->aCol; + for(k=0; knColumn; k++, pCol++){ + if( pCol->iTable==pExpr->iTable && + pCol->iColumn==pExpr->iColumn ){ + break; + } + } + if( (k>=pAggInfo->nColumn) + && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 + ){ + pCol = &pAggInfo->aCol[k]; + pCol->pTab = pExpr->y.pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iMem = ++pParse->nMem; + pCol->iSorterColumn = -1; + pCol->pCExpr = pExpr; + if( pAggInfo->pGroupBy ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; jpExpr; + if( pE->op==TK_COLUMN && pE->iTable==pExpr->iTable && + pE->iColumn==pExpr->iColumn ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } + } + /* There is now an entry for pExpr in pAggInfo->aCol[] (either + ** because it was there before or because we just created it). + ** Convert the pExpr to be a TK_AGG_COLUMN referring to that + ** pAggInfo->aCol[] entry. + */ + ExprSetVVAProperty(pExpr, EP_NoReduce); + pExpr->pAggInfo = pAggInfo; + pExpr->op = TK_AGG_COLUMN; + pExpr->iAgg = (i16)k; + break; + } /* endif pExpr->iTable==pItem->iCursor */ + } /* end loop over pSrcList */ + } + return WRC_Prune; + } + case TK_AGG_FUNCTION: { + if( (pNC->ncFlags & NC_InAggFunc)==0 + && pWalker->walkerDepth==pExpr->op2 + ){ + /* Check to see if pExpr is a duplicate of another aggregate + ** function that is already in the pAggInfo structure + */ + struct AggInfo_func *pItem = pAggInfo->aFunc; + for(i=0; inFunc; i++, pItem++){ + if( pItem->pFExpr==pExpr ) break; + if( sqlite3ExprCompare(0, pItem->pFExpr, pExpr, -1)==0 ){ + break; + } + } + if( i>=pAggInfo->nFunc ){ + /* pExpr is original. Make a new entry in pAggInfo->aFunc[] + */ + u8 enc = ENC(pParse->db); + i = addAggInfoFunc(pParse->db, pAggInfo); + if( i>=0 ){ + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + pItem = &pAggInfo->aFunc[i]; + pItem->pFExpr = pExpr; + pItem->iMem = ++pParse->nMem; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + pItem->pFunc = sqlite3FindFunction(pParse->db, + pExpr->u.zToken, + pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); + if( pExpr->flags & EP_Distinct ){ + pItem->iDistinct = pParse->nTab++; + }else{ + pItem->iDistinct = -1; + } + } + } + /* Make pExpr point to the appropriate pAggInfo->aFunc[] entry + */ + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(pExpr, EP_NoReduce); + pExpr->iAgg = (i16)i; + pExpr->pAggInfo = pAggInfo; + return WRC_Prune; + }else{ + return WRC_Continue; + } + } + } + return WRC_Continue; +} + +/* +** Analyze the pExpr expression looking for aggregate functions and +** for variables that need to be added to AggInfo object that pNC->pAggInfo +** points to. Additional entries are made on the AggInfo object as +** necessary. +** +** This routine should only be called after the expression has been +** analyzed by sqlite3ResolveExprNames(). +*/ +SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ + Walker w; + w.xExprCallback = analyzeAggregate; + w.xSelectCallback = sqlite3WalkerDepthIncrease; + w.xSelectCallback2 = sqlite3WalkerDepthDecrease; + w.walkerDepth = 0; + w.u.pNC = pNC; + w.pParse = 0; + assert( pNC->pSrcList!=0 ); + sqlite3WalkExpr(&w, pExpr); +} + +/* +** Call sqlite3ExprAnalyzeAggregates() for every expression in an +** expression list. Return the number of errors. +** +** If an error is found, the analysis is cut short. +*/ +SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext *pNC, ExprList *pList){ + struct ExprList_item *pItem; + int i; + if( pList ){ + for(pItem=pList->a, i=0; inExpr; i++, pItem++){ + sqlite3ExprAnalyzeAggregates(pNC, pItem->pExpr); + } + } +} + +/* +** Allocate a single new register for use to hold some intermediate result. +*/ +SQLITE_PRIVATE int sqlite3GetTempReg(Parse *pParse){ + if( pParse->nTempReg==0 ){ + return ++pParse->nMem; + } + return pParse->aTempReg[--pParse->nTempReg]; +} + +/* +** Deallocate a register, making available for reuse for some other +** purpose. +*/ +SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ + if( iReg ){ + sqlite3VdbeReleaseRegisters(pParse, iReg, 1, 0, 0); + if( pParse->nTempRegaTempReg) ){ + pParse->aTempReg[pParse->nTempReg++] = iReg; + } + } +} + +/* +** Allocate or deallocate a block of nReg consecutive registers. +*/ +SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ + int i, n; + if( nReg==1 ) return sqlite3GetTempReg(pParse); + i = pParse->iRangeReg; + n = pParse->nRangeReg; + if( nReg<=n ){ + pParse->iRangeReg += nReg; + pParse->nRangeReg -= nReg; + }else{ + i = pParse->nMem+1; + pParse->nMem += nReg; + } + return i; +} +SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, iReg); + return; + } + sqlite3VdbeReleaseRegisters(pParse, iReg, nReg, 0, 0); + if( nReg>pParse->nRangeReg ){ + pParse->nRangeReg = nReg; + pParse->iRangeReg = iReg; + } +} + +/* +** Mark all temporary registers as being unavailable for reuse. +** +** Always invoke this procedure after coding a subroutine or co-routine +** that might be invoked from other parts of the code, to ensure that +** the sub/co-routine does not use registers in common with the code that +** invokes the sub/co-routine. +*/ +SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ + pParse->nTempReg = 0; + pParse->nRangeReg = 0; +} + +/* +** Validate that no temporary register falls within the range of +** iFirst..iLast, inclusive. This routine is only call from within assert() +** statements. +*/ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ + int i; + if( pParse->nRangeReg>0 + && pParse->iRangeReg+pParse->nRangeReg > iFirst + && pParse->iRangeReg <= iLast + ){ + return 0; + } + for(i=0; inTempReg; i++){ + if( pParse->aTempReg[i]>=iFirst && pParse->aTempReg[i]<=iLast ){ + return 0; + } + } + return 1; +} +#endif /* SQLITE_DEBUG */ + +/************** End of expr.c ************************************************/ +/************** Begin file alter.c *******************************************/ +/* +** 2005 February 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that used to generate VDBE code +** that implements the ALTER TABLE command. +*/ +/* #include "sqliteInt.h" */ + +/* +** The code in this file only exists if we are not omitting the +** ALTER TABLE logic from the build. +*/ +#ifndef SQLITE_OMIT_ALTERTABLE + +/* +** Parameter zName is the name of a table that is about to be altered +** (either with ALTER TABLE ... RENAME TO or ALTER TABLE ... ADD COLUMN). +** If the table is a system table, this function leaves an error message +** in pParse->zErr (system tables may not be altered) and returns non-zero. +** +** Or, if zName is not a system table, zero is returned. +*/ +static int isAlterableTable(Parse *pParse, Table *pTab){ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) +#ifndef SQLITE_OMIT_VIRTUALTABLE + || (pTab->tabFlags & TF_Eponymous)!=0 + || ( (pTab->tabFlags & TF_Shadow)!=0 + && sqlite3ReadOnlyShadowTables(pParse->db) + ) +#endif + ){ + sqlite3ErrorMsg(pParse, "table %s may not be altered", pTab->zName); + return 1; + } + return 0; +} + +/* +** Generate code to verify that the schemas of database zDb and, if +** bTemp is not true, database "temp", can still be parsed. This is +** called at the end of the generation of an ALTER TABLE ... RENAME ... +** statement to ensure that the operation has not rendered any schema +** objects unusable. +*/ +static void renameTestSchema( + Parse *pParse, /* Parse context */ + const char *zDb, /* Name of db to verify schema of */ + int bTemp, /* True if this is the temp db */ + const char *zWhen, /* "when" part of error message */ + int bNoDQS /* Do not allow DQS in the schema */ +){ + pParse->colNamesSet = 1; + sqlite3NestedParse(pParse, + "SELECT 1 " + "FROM \"%w\"." DFLT_SCHEMA_TABLE " " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" + " AND sqlite_rename_test(%Q, sql, type, name, %d, %Q, %d)=NULL ", + zDb, + zDb, bTemp, zWhen, bNoDQS + ); + + if( bTemp==0 ){ + sqlite3NestedParse(pParse, + "SELECT 1 " + "FROM temp." DFLT_SCHEMA_TABLE " " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" + " AND sqlite_rename_test(%Q, sql, type, name, 1, %Q, %d)=NULL ", + zDb, zWhen, bNoDQS + ); + } +} + +/* +** Generate VM code to replace any double-quoted strings (but not double-quoted +** identifiers) within the "sql" column of the sqlite_schema table in +** database zDb with their single-quoted equivalents. If argument bTemp is +** not true, similarly update all SQL statements in the sqlite_schema table +** of the temp db. +*/ +static void renameFixQuotes(Parse *pParse, const char *zDb, int bTemp){ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." DFLT_SCHEMA_TABLE + " SET sql = sqlite_rename_quotefix(%Q, sql)" + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" , zDb, zDb + ); + if( bTemp==0 ){ + sqlite3NestedParse(pParse, + "UPDATE temp." DFLT_SCHEMA_TABLE + " SET sql = sqlite_rename_quotefix('temp', sql)" + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" + ); + } +} + +/* +** Generate code to reload the schema for database iDb. And, if iDb!=1, for +** the temp database as well. +*/ +static void renameReloadSchema(Parse *pParse, int iDb, u16 p5){ + Vdbe *v = pParse->pVdbe; + if( v ){ + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, iDb, 0, p5); + if( iDb!=1 ) sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, 1, 0, p5); + } +} + +/* +** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" +** command. +*/ +SQLITE_PRIVATE void sqlite3AlterRenameTable( + Parse *pParse, /* Parser context. */ + SrcList *pSrc, /* The table to rename. */ + Token *pName /* The new table name. */ +){ + int iDb; /* Database that contains the table */ + char *zDb; /* Name of database iDb */ + Table *pTab; /* Table being renamed */ + char *zName = 0; /* NULL-terminated version of pName */ + sqlite3 *db = pParse->db; /* Database connection */ + int nTabName; /* Number of UTF-8 characters in zTabName */ + const char *zTabName; /* Original name of the table */ + Vdbe *v; + VTable *pVTab = 0; /* Non-zero if this is a v-tab with an xRename() */ + + if( NEVER(db->mallocFailed) ) goto exit_rename_table; + assert( pSrc->nSrc==1 ); + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_rename_table; + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + zDb = db->aDb[iDb].zDbSName; + + /* Get a NULL terminated version of the new table name. */ + zName = sqlite3NameFromToken(db, pName); + if( !zName ) goto exit_rename_table; + + /* Check that a table or index named 'zName' does not already exist + ** in database iDb. If so, this is an error. + */ + if( sqlite3FindTable(db, zName, zDb) + || sqlite3FindIndex(db, zName, zDb) + || sqlite3IsShadowTableOf(db, pTab, zName) + ){ + sqlite3ErrorMsg(pParse, + "there is already another table or index with this name: %s", zName); + goto exit_rename_table; + } + + /* Make sure it is not a system table being altered, or a reserved name + ** that the table is being renamed to. + */ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ){ + goto exit_rename_table; + } + if( SQLITE_OK!=sqlite3CheckObjectName(pParse,zName,"table",zName) ){ + goto exit_rename_table; + } + +#ifndef SQLITE_OMIT_VIEW + if( IsView(pTab) ){ + sqlite3ErrorMsg(pParse, "view %s may not be altered", pTab->zName); + goto exit_rename_table; + } +#endif + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + goto exit_rename_table; + } +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto exit_rename_table; + } + if( IsVirtual(pTab) ){ + pVTab = sqlite3GetVTable(db, pTab); + if( pVTab->pVtab->pModule->xRename==0 ){ + pVTab = 0; + } + } +#endif + + /* Begin a transaction for database iDb. Then modify the schema cookie + ** (since the ALTER TABLE modifies the schema). Call sqlite3MayAbort(), + ** as the scalar functions (e.g. sqlite_rename_table()) invoked by the + ** nested SQL may raise an exception. */ + v = sqlite3GetVdbe(pParse); + if( v==0 ){ + goto exit_rename_table; + } + sqlite3MayAbort(pParse); + + /* figure out how many UTF-8 characters are in zName */ + zTabName = pTab->zName; + nTabName = sqlite3Utf8CharLen(zTabName, -1); + + /* Rewrite all CREATE TABLE, INDEX, TRIGGER or VIEW statements in + ** the schema to use the new table name. */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." DFLT_SCHEMA_TABLE " SET " + "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, %d) " + "WHERE (type!='index' OR tbl_name=%Q COLLATE nocase)" + "AND name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + , zDb, zDb, zTabName, zName, (iDb==1), zTabName + ); + + /* Update the tbl_name and name columns of the sqlite_schema table + ** as required. */ + sqlite3NestedParse(pParse, + "UPDATE %Q." DFLT_SCHEMA_TABLE " SET " + "tbl_name = %Q, " + "name = CASE " + "WHEN type='table' THEN %Q " + "WHEN name LIKE 'sqliteX_autoindex%%' ESCAPE 'X' " + " AND type='index' THEN " + "'sqlite_autoindex_' || %Q || substr(name,%d+18) " + "ELSE name END " + "WHERE tbl_name=%Q COLLATE nocase AND " + "(type='table' OR type='index' OR type='trigger');", + zDb, + zName, zName, zName, + nTabName, zTabName + ); + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* If the sqlite_sequence table exists in this database, then update + ** it with the new table name. + */ + if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){ + sqlite3NestedParse(pParse, + "UPDATE \"%w\".sqlite_sequence set name = %Q WHERE name = %Q", + zDb, zName, pTab->zName); + } +#endif + + /* If the table being renamed is not itself part of the temp database, + ** edit view and trigger definitions within the temp database + ** as required. */ + if( iDb!=1 ){ + sqlite3NestedParse(pParse, + "UPDATE sqlite_temp_schema SET " + "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, 1), " + "tbl_name = " + "CASE WHEN tbl_name=%Q COLLATE nocase AND " + " sqlite_rename_test(%Q, sql, type, name, 1, 'after rename', 0) " + "THEN %Q ELSE tbl_name END " + "WHERE type IN ('view', 'trigger')" + , zDb, zTabName, zName, zTabName, zDb, zName); + } + + /* If this is a virtual table, invoke the xRename() function if + ** one is defined. The xRename() callback will modify the names + ** of any resources used by the v-table implementation (including other + ** SQLite tables) that are identified by the name of the virtual table. + */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pVTab ){ + int i = ++pParse->nMem; + sqlite3VdbeLoadString(v, i, zName); + sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); + } +#endif + + renameReloadSchema(pParse, iDb, INITFLAG_AlterRename); + renameTestSchema(pParse, zDb, iDb==1, "after rename", 0); + +exit_rename_table: + sqlite3SrcListDelete(db, pSrc); + sqlite3DbFree(db, zName); +} + +/* +** Write code that will raise an error if the table described by +** zDb and zTab is not empty. +*/ +static void sqlite3ErrorIfNotEmpty( + Parse *pParse, /* Parsing context */ + const char *zDb, /* Schema holding the table */ + const char *zTab, /* Table to check for empty */ + const char *zErr /* Error message text */ +){ + sqlite3NestedParse(pParse, + "SELECT raise(ABORT,%Q) FROM \"%w\".\"%w\"", + zErr, zDb, zTab + ); +} + +/* +** This function is called after an "ALTER TABLE ... ADD" statement +** has been parsed. Argument pColDef contains the text of the new +** column definition. +** +** The Table structure pParse->pNewTable was extended to include +** the new column during parsing. +*/ +SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ + Table *pNew; /* Copy of pParse->pNewTable */ + Table *pTab; /* Table being altered */ + int iDb; /* Database number */ + const char *zDb; /* Database name */ + const char *zTab; /* Table name */ + char *zCol; /* Null-terminated column definition */ + Column *pCol; /* The new column */ + Expr *pDflt; /* Default value for the new column */ + sqlite3 *db; /* The database connection; */ + Vdbe *v; /* The prepared statement under construction */ + int r1; /* Temporary registers */ + + db = pParse->db; + if( pParse->nErr || db->mallocFailed ) return; + pNew = pParse->pNewTable; + assert( pNew ); + + assert( sqlite3BtreeHoldsAllMutexes(db) ); + iDb = sqlite3SchemaToIndex(db, pNew->pSchema); + zDb = db->aDb[iDb].zDbSName; + zTab = &pNew->zName[16]; /* Skip the "sqlite_altertab_" prefix on the name */ + pCol = &pNew->aCol[pNew->nCol-1]; + pDflt = sqlite3ColumnExpr(pNew, pCol); + pTab = sqlite3FindTable(db, zTab, zDb); + assert( pTab ); + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + return; + } +#endif + + + /* Check that the new column is not specified as PRIMARY KEY or UNIQUE. + ** If there is a NOT NULL constraint, then the default value for the + ** column must not be NULL. + */ + if( pCol->colFlags & COLFLAG_PRIMKEY ){ + sqlite3ErrorMsg(pParse, "Cannot add a PRIMARY KEY column"); + return; + } + if( pNew->pIndex ){ + sqlite3ErrorMsg(pParse, + "Cannot add a UNIQUE column"); + return; + } + if( (pCol->colFlags & COLFLAG_GENERATED)==0 ){ + /* If the default value for the new column was specified with a + ** literal NULL, then set pDflt to 0. This simplifies checking + ** for an SQL NULL default below. + */ + assert( pDflt==0 || pDflt->op==TK_SPAN ); + if( pDflt && pDflt->pLeft->op==TK_NULL ){ + pDflt = 0; + } + if( (db->flags&SQLITE_ForeignKeys) && pNew->u.tab.pFKey && pDflt ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a REFERENCES column with non-NULL default value"); + } + if( pCol->notNull && !pDflt ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a NOT NULL column with default value NULL"); + } + + + /* Ensure the default expression is something that sqlite3ValueFromExpr() + ** can handle (i.e. not CURRENT_TIME etc.) + */ + if( pDflt ){ + sqlite3_value *pVal = 0; + int rc; + rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + if( rc!=SQLITE_OK ){ + assert( db->mallocFailed == 1 ); + return; + } + if( !pVal ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a column with non-constant default"); + } + sqlite3ValueFree(pVal); + } + }else if( pCol->colFlags & COLFLAG_STORED ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, "cannot add a STORED column"); + } + + + /* Modify the CREATE TABLE statement. */ + zCol = sqlite3DbStrNDup(db, (char*)pColDef->z, pColDef->n); + if( zCol ){ + char *zEnd = &zCol[pColDef->n-1]; + while( zEnd>zCol && (*zEnd==';' || sqlite3Isspace(*zEnd)) ){ + *zEnd-- = '\0'; + } + /* substr() operations on characters, but addColOffset is in bytes. So we + ** have to use printf() to translate between these units: */ + assert( !IsVirtual(pTab) ); + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." DFLT_SCHEMA_TABLE " SET " + "sql = printf('%%.%ds, ',sql) || %Q" + " || substr(sql,1+length(printf('%%.%ds',sql))) " + "WHERE type = 'table' AND name = %Q", + zDb, pNew->u.tab.addColOffset, zCol, pNew->u.tab.addColOffset, + zTab + ); + sqlite3DbFree(db, zCol); + } + + v = sqlite3GetVdbe(pParse); + if( v ){ + /* Make sure the schema version is at least 3. But do not upgrade + ** from less than 3 to 4, as that will corrupt any preexisting DESC + ** index. + */ + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_AddImm, r1, -2); + sqlite3VdbeAddOp2(v, OP_IfPos, r1, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); + sqlite3ReleaseTempReg(pParse, r1); + + /* Reload the table definition */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterRename); + + /* Verify that constraints are still satisfied */ + if( pNew->pCheck!=0 + || (pCol->notNull && (pCol->colFlags & COLFLAG_GENERATED)!=0) + ){ + sqlite3NestedParse(pParse, + "SELECT CASE WHEN quick_check GLOB 'CHECK*'" + " THEN raise(ABORT,'CHECK constraint failed')" + " ELSE raise(ABORT,'NOT NULL constraint failed')" + " END" + " FROM pragma_quick_check(\"%w\",\"%w\")" + " WHERE quick_check GLOB 'CHECK*' OR quick_check GLOB 'NULL*'", + zTab, zDb + ); + } + } +} + +/* +** This function is called by the parser after the table-name in +** an "ALTER TABLE ADD" statement is parsed. Argument +** pSrc is the full-name of the table being altered. +** +** This routine makes a (partial) copy of the Table structure +** for the table being altered and sets Parse.pNewTable to point +** to it. Routines called by the parser as the column definition +** is parsed (i.e. sqlite3AddColumn()) add the new Column data to +** the copy. The copy of the Table structure is deleted by tokenize.c +** after parsing is finished. +** +** Routine sqlite3AlterFinishAddColumn() will be called to complete +** coding the "ALTER TABLE ... ADD" statement. +*/ +SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ + Table *pNew; + Table *pTab; + int iDb; + int i; + int nAlloc; + sqlite3 *db = pParse->db; + + /* Look up the table being altered. */ + assert( pParse->pNewTable==0 ); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + if( db->mallocFailed ) goto exit_begin_add_column; + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_begin_add_column; + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "virtual tables may not be altered"); + goto exit_begin_add_column; + } +#endif + + /* Make sure this is not an attempt to ALTER a view. */ + if( IsView(pTab) ){ + sqlite3ErrorMsg(pParse, "Cannot add a column to a view"); + goto exit_begin_add_column; + } + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ){ + goto exit_begin_add_column; + } + + sqlite3MayAbort(pParse); + assert( pTab->u.tab.addColOffset>0 ); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + + /* Put a copy of the Table struct in Parse.pNewTable for the + ** sqlite3AddColumn() function and friends to modify. But modify + ** the name by adding an "sqlite_altertab_" prefix. By adding this + ** prefix, we insure that the name will not collide with an existing + ** table because user table are not allowed to have the "sqlite_" + ** prefix on their name. + */ + pNew = (Table*)sqlite3DbMallocZero(db, sizeof(Table)); + if( !pNew ) goto exit_begin_add_column; + pParse->pNewTable = pNew; + pNew->nTabRef = 1; + pNew->nCol = pTab->nCol; + assert( pNew->nCol>0 ); + nAlloc = (((pNew->nCol-1)/8)*8)+8; + assert( nAlloc>=pNew->nCol && nAlloc%8==0 && nAlloc-pNew->nCol<8 ); + pNew->aCol = (Column*)sqlite3DbMallocZero(db, sizeof(Column)*nAlloc); + pNew->zName = sqlite3MPrintf(db, "sqlite_altertab_%s", pTab->zName); + if( !pNew->aCol || !pNew->zName ){ + assert( db->mallocFailed ); + goto exit_begin_add_column; + } + memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); + for(i=0; inCol; i++){ + Column *pCol = &pNew->aCol[i]; + pCol->zCnName = sqlite3DbStrDup(db, pCol->zCnName); + pCol->hName = sqlite3StrIHash(pCol->zCnName); + } + assert( !IsVirtual(pNew) ); + pNew->u.tab.pDfltList = sqlite3ExprListDup(db, pTab->u.tab.pDfltList, 0); + pNew->pSchema = db->aDb[iDb].pSchema; + pNew->u.tab.addColOffset = pTab->u.tab.addColOffset; + pNew->nTabRef = 1; + +exit_begin_add_column: + sqlite3SrcListDelete(db, pSrc); + return; +} + +/* +** Parameter pTab is the subject of an ALTER TABLE ... RENAME COLUMN +** command. This function checks if the table is a view or virtual +** table (columns of views or virtual tables may not be renamed). If so, +** it loads an error message into pParse and returns non-zero. +** +** Or, if pTab is not a view or virtual table, zero is returned. +*/ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) +static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ + const char *zType = 0; +#ifndef SQLITE_OMIT_VIEW + if( IsView(pTab) ){ + zType = "view"; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + zType = "virtual table"; + } +#endif + if( zType ){ + sqlite3ErrorMsg(pParse, "cannot %s %s \"%s\"", + (bDrop ? "drop column from" : "rename columns of"), + zType, pTab->zName + ); + return 1; + } + return 0; +} +#else /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ +# define isRealTable(x,y,z) (0) +#endif + +/* +** Handles the following parser reduction: +** +** cmd ::= ALTER TABLE pSrc RENAME COLUMN pOld TO pNew +*/ +SQLITE_PRIVATE void sqlite3AlterRenameColumn( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* Table being altered. pSrc->nSrc==1 */ + Token *pOld, /* Name of column being changed */ + Token *pNew /* New column name */ +){ + sqlite3 *db = pParse->db; /* Database connection */ + Table *pTab; /* Table being updated */ + int iCol; /* Index of column being renamed */ + char *zOld = 0; /* Old column name */ + char *zNew = 0; /* New column name */ + const char *zDb; /* Name of schema containing the table */ + int iSchema; /* Index of the schema */ + int bQuote; /* True to quote the new name */ + + /* Locate the table to be altered */ + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_rename_column; + + /* Cannot alter a system table */ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ) goto exit_rename_column; + if( SQLITE_OK!=isRealTable(pParse, pTab, 0) ) goto exit_rename_column; + + /* Which schema holds the table to be altered */ + iSchema = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iSchema>=0 ); + zDb = db->aDb[iSchema].zDbSName; + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + goto exit_rename_column; + } +#endif + + /* Make sure the old name really is a column name in the table to be + ** altered. Set iCol to be the index of the column being renamed */ + zOld = sqlite3NameFromToken(db, pOld); + if( !zOld ) goto exit_rename_column; + for(iCol=0; iColnCol; iCol++){ + if( 0==sqlite3StrICmp(pTab->aCol[iCol].zCnName, zOld) ) break; + } + if( iCol==pTab->nCol ){ + sqlite3ErrorMsg(pParse, "no such column: \"%s\"", zOld); + goto exit_rename_column; + } + + /* Ensure the schema contains no double-quoted strings */ + renameTestSchema(pParse, zDb, iSchema==1, "", 0); + renameFixQuotes(pParse, zDb, iSchema==1); + + /* Do the rename operation using a recursive UPDATE statement that + ** uses the sqlite_rename_column() SQL function to compute the new + ** CREATE statement text for the sqlite_schema table. + */ + sqlite3MayAbort(pParse); + zNew = sqlite3NameFromToken(db, pNew); + if( !zNew ) goto exit_rename_column; + assert( pNew->n>0 ); + bQuote = sqlite3Isquote(pNew->z[0]); + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." DFLT_SCHEMA_TABLE " SET " + "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, %d) " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X' " + " AND (type != 'index' OR tbl_name = %Q)", + zDb, + zDb, pTab->zName, iCol, zNew, bQuote, iSchema==1, + pTab->zName + ); + + sqlite3NestedParse(pParse, + "UPDATE temp." DFLT_SCHEMA_TABLE " SET " + "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, 1) " + "WHERE type IN ('trigger', 'view')", + zDb, pTab->zName, iCol, zNew, bQuote + ); + + /* Drop and reload the database schema. */ + renameReloadSchema(pParse, iSchema, INITFLAG_AlterRename); + renameTestSchema(pParse, zDb, iSchema==1, "after rename", 1); + + exit_rename_column: + sqlite3SrcListDelete(db, pSrc); + sqlite3DbFree(db, zOld); + sqlite3DbFree(db, zNew); + return; +} + +/* +** Each RenameToken object maps an element of the parse tree into +** the token that generated that element. The parse tree element +** might be one of: +** +** * A pointer to an Expr that represents an ID +** * The name of a table column in Column.zName +** +** A list of RenameToken objects can be constructed during parsing. +** Each new object is created by sqlite3RenameTokenMap(). +** As the parse tree is transformed, the sqlite3RenameTokenRemap() +** routine is used to keep the mapping current. +** +** After the parse finishes, renameTokenFind() routine can be used +** to look up the actual token value that created some element in +** the parse tree. +*/ +struct RenameToken { + const void *p; /* Parse tree element created by token t */ + Token t; /* The token that created parse tree element p */ + RenameToken *pNext; /* Next is a list of all RenameToken objects */ +}; + +/* +** The context of an ALTER TABLE RENAME COLUMN operation that gets passed +** down into the Walker. +*/ +typedef struct RenameCtx RenameCtx; +struct RenameCtx { + RenameToken *pList; /* List of tokens to overwrite */ + int nList; /* Number of tokens in pList */ + int iCol; /* Index of column being renamed */ + Table *pTab; /* Table being ALTERed */ + const char *zOld; /* Old column name */ +}; + +#ifdef SQLITE_DEBUG +/* +** This function is only for debugging. It performs two tasks: +** +** 1. Checks that pointer pPtr does not already appear in the +** rename-token list. +** +** 2. Dereferences each pointer in the rename-token list. +** +** The second is most effective when debugging under valgrind or +** address-sanitizer or similar. If any of these pointers no longer +** point to valid objects, an exception is raised by the memory-checking +** tool. +** +** The point of this is to prevent comparisons of invalid pointer values. +** Even though this always seems to work, it is undefined according to the +** C standard. Example of undefined comparison: +** +** sqlite3_free(x); +** if( x==y ) ... +** +** Technically, as x no longer points into a valid object or to the byte +** following a valid object, it may not be used in comparison operations. +*/ +static void renameTokenCheckAll(Parse *pParse, const void *pPtr){ + if( pParse->nErr==0 && pParse->db->mallocFailed==0 ){ + const RenameToken *p; + u8 i = 0; + for(p=pParse->pRename; p; p=p->pNext){ + if( p->p ){ + assert( p->p!=pPtr ); + i += *(u8*)(p->p); + } + } + } +} +#else +# define renameTokenCheckAll(x,y) +#endif + +/* +** Remember that the parser tree element pPtr was created using +** the token pToken. +** +** In other words, construct a new RenameToken object and add it +** to the list of RenameToken objects currently being built up +** in pParse->pRename. +** +** The pPtr argument is returned so that this routine can be used +** with tail recursion in tokenExpr() routine, for a small performance +** improvement. +*/ +SQLITE_PRIVATE const void *sqlite3RenameTokenMap( + Parse *pParse, + const void *pPtr, + const Token *pToken +){ + RenameToken *pNew; + assert( pPtr || pParse->db->mallocFailed ); + renameTokenCheckAll(pParse, pPtr); + if( ALWAYS(pParse->eParseMode!=PARSE_MODE_UNMAP) ){ + pNew = sqlite3DbMallocZero(pParse->db, sizeof(RenameToken)); + if( pNew ){ + pNew->p = pPtr; + pNew->t = *pToken; + pNew->pNext = pParse->pRename; + pParse->pRename = pNew; + } + } + + return pPtr; +} + +/* +** It is assumed that there is already a RenameToken object associated +** with parse tree element pFrom. This function remaps the associated token +** to parse tree element pTo. +*/ +SQLITE_PRIVATE void sqlite3RenameTokenRemap(Parse *pParse, const void *pTo, const void *pFrom){ + RenameToken *p; + renameTokenCheckAll(pParse, pTo); + for(p=pParse->pRename; p; p=p->pNext){ + if( p->p==pFrom ){ + p->p = pTo; + break; + } + } +} + +/* +** Walker callback used by sqlite3RenameExprUnmap(). +*/ +static int renameUnmapExprCb(Walker *pWalker, Expr *pExpr){ + Parse *pParse = pWalker->pParse; + sqlite3RenameTokenRemap(pParse, 0, (const void*)pExpr); + sqlite3RenameTokenRemap(pParse, 0, (const void*)&pExpr->y.pTab); + return WRC_Continue; +} + +/* +** Iterate through the Select objects that are part of WITH clauses attached +** to select statement pSelect. +*/ +static void renameWalkWith(Walker *pWalker, Select *pSelect){ + With *pWith = pSelect->pWith; + if( pWith ){ + Parse *pParse = pWalker->pParse; + int i; + With *pCopy = 0; + assert( pWith->nCte>0 ); + if( (pWith->a[0].pSelect->selFlags & SF_Expanded)==0 ){ + /* Push a copy of the With object onto the with-stack. We use a copy + ** here as the original will be expanded and resolved (flags SF_Expanded + ** and SF_Resolved) below. And the parser code that uses the with-stack + ** fails if the Select objects on it have already been expanded and + ** resolved. */ + pCopy = sqlite3WithDup(pParse->db, pWith); + pCopy = sqlite3WithPush(pParse, pCopy, 1); + } + for(i=0; inCte; i++){ + Select *p = pWith->a[i].pSelect; + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + if( pCopy ) sqlite3SelectPrep(sNC.pParse, p, &sNC); + if( sNC.pParse->db->mallocFailed ) return; + sqlite3WalkSelect(pWalker, p); + sqlite3RenameExprlistUnmap(pParse, pWith->a[i].pCols); + } + if( pCopy && pParse->pWith==pCopy ){ + pParse->pWith = pCopy->pOuter; + } + } +} + +/* +** Unmap all tokens in the IdList object passed as the second argument. +*/ +static void unmapColumnIdlistNames( + Parse *pParse, + const IdList *pIdList +){ + if( pIdList ){ + int ii; + for(ii=0; iinId; ii++){ + sqlite3RenameTokenRemap(pParse, 0, (const void*)pIdList->a[ii].zName); + } + } +} + +/* +** Walker callback used by sqlite3RenameExprUnmap(). +*/ +static int renameUnmapSelectCb(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + int i; + if( pParse->nErr ) return WRC_Abort; + if( NEVER(p->selFlags & (SF_View|SF_CopyCte)) ){ + return WRC_Prune; + } + if( ALWAYS(p->pEList) ){ + ExprList *pList = p->pEList; + for(i=0; inExpr; i++){ + if( pList->a[i].zEName && pList->a[i].eEName==ENAME_NAME ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pList->a[i].zEName); + } + } + } + if( ALWAYS(p->pSrc) ){ /* Every Select as a SrcList, even if it is empty */ + SrcList *pSrc = p->pSrc; + for(i=0; inSrc; i++){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pSrc->a[i].zName); + sqlite3WalkExpr(pWalker, pSrc->a[i].pOn); + unmapColumnIdlistNames(pParse, pSrc->a[i].pUsing); + } + } + + renameWalkWith(pWalker, p); + return WRC_Continue; +} + +/* +** Remove all nodes that are part of expression pExpr from the rename list. +*/ +SQLITE_PRIVATE void sqlite3RenameExprUnmap(Parse *pParse, Expr *pExpr){ + u8 eMode = pParse->eParseMode; + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = renameUnmapExprCb; + sWalker.xSelectCallback = renameUnmapSelectCb; + pParse->eParseMode = PARSE_MODE_UNMAP; + sqlite3WalkExpr(&sWalker, pExpr); + pParse->eParseMode = eMode; +} + +/* +** Remove all nodes that are part of expression-list pEList from the +** rename list. +*/ +SQLITE_PRIVATE void sqlite3RenameExprlistUnmap(Parse *pParse, ExprList *pEList){ + if( pEList ){ + int i; + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = renameUnmapExprCb; + sqlite3WalkExprList(&sWalker, pEList); + for(i=0; inExpr; i++){ + if( ALWAYS(pEList->a[i].eEName==ENAME_NAME) ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pEList->a[i].zEName); + } + } + } +} + +/* +** Free the list of RenameToken objects given in the second argument +*/ +static void renameTokenFree(sqlite3 *db, RenameToken *pToken){ + RenameToken *pNext; + RenameToken *p; + for(p=pToken; p; p=pNext){ + pNext = p->pNext; + sqlite3DbFree(db, p); + } +} + +/* +** Search the Parse object passed as the first argument for a RenameToken +** object associated with parse tree element pPtr. If found, return a pointer +** to it. Otherwise, return NULL. +** +** If the second argument passed to this function is not NULL and a matching +** RenameToken object is found, remove it from the Parse object and add it to +** the list maintained by the RenameCtx object. +*/ +static RenameToken *renameTokenFind( + Parse *pParse, + struct RenameCtx *pCtx, + const void *pPtr +){ + RenameToken **pp; + if( NEVER(pPtr==0) ){ + return 0; + } + for(pp=&pParse->pRename; (*pp); pp=&(*pp)->pNext){ + if( (*pp)->p==pPtr ){ + RenameToken *pToken = *pp; + if( pCtx ){ + *pp = pToken->pNext; + pToken->pNext = pCtx->pList; + pCtx->pList = pToken; + pCtx->nList++; + } + return pToken; + } + } + return 0; +} + +/* +** This is a Walker select callback. It does nothing. It is only required +** because without a dummy callback, sqlite3WalkExpr() and similar do not +** descend into sub-select statements. +*/ +static int renameColumnSelectCb(Walker *pWalker, Select *p){ + if( p->selFlags & (SF_View|SF_CopyCte) ){ + testcase( p->selFlags & SF_View ); + testcase( p->selFlags & SF_CopyCte ); + return WRC_Prune; + } + renameWalkWith(pWalker, p); + return WRC_Continue; +} + +/* +** This is a Walker expression callback. +** +** For every TK_COLUMN node in the expression tree, search to see +** if the column being references is the column being renamed by an +** ALTER TABLE statement. If it is, then attach its associated +** RenameToken object to the list of RenameToken objects being +** constructed in RenameCtx object at pWalker->u.pRename. +*/ +static int renameColumnExprCb(Walker *pWalker, Expr *pExpr){ + RenameCtx *p = pWalker->u.pRename; + if( pExpr->op==TK_TRIGGER + && pExpr->iColumn==p->iCol + && pWalker->pParse->pTriggerTab==p->pTab + ){ + renameTokenFind(pWalker->pParse, p, (void*)pExpr); + }else if( pExpr->op==TK_COLUMN + && pExpr->iColumn==p->iCol + && p->pTab==pExpr->y.pTab + ){ + renameTokenFind(pWalker->pParse, p, (void*)pExpr); + } + return WRC_Continue; +} + +/* +** The RenameCtx contains a list of tokens that reference a column that +** is being renamed by an ALTER TABLE statement. Return the "last" +** RenameToken in the RenameCtx and remove that RenameToken from the +** RenameContext. "Last" means the last RenameToken encountered when +** the input SQL is parsed from left to right. Repeated calls to this routine +** return all column name tokens in the order that they are encountered +** in the SQL statement. +*/ +static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ + RenameToken *pBest = pCtx->pList; + RenameToken *pToken; + RenameToken **pp; + + for(pToken=pBest->pNext; pToken; pToken=pToken->pNext){ + if( pToken->t.z>pBest->t.z ) pBest = pToken; + } + for(pp=&pCtx->pList; *pp!=pBest; pp=&(*pp)->pNext); + *pp = pBest->pNext; + + return pBest; +} + +/* +** An error occured while parsing or otherwise processing a database +** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an +** ALTER TABLE RENAME COLUMN program. The error message emitted by the +** sub-routine is currently stored in pParse->zErrMsg. This function +** adds context to the error message and then stores it in pCtx. +*/ +static void renameColumnParseError( + sqlite3_context *pCtx, + const char *zWhen, + sqlite3_value *pType, + sqlite3_value *pObject, + Parse *pParse +){ + const char *zT = (const char*)sqlite3_value_text(pType); + const char *zN = (const char*)sqlite3_value_text(pObject); + char *zErr; + + zErr = sqlite3_mprintf("error in %s %s%s%s: %s", + zT, zN, (zWhen[0] ? " " : ""), zWhen, + pParse->zErrMsg + ); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); +} + +/* +** For each name in the the expression-list pEList (i.e. each +** pEList->a[i].zName) that matches the string in zOld, extract the +** corresponding rename-token from Parse object pParse and add it +** to the RenameCtx pCtx. +*/ +static void renameColumnElistNames( + Parse *pParse, + RenameCtx *pCtx, + const ExprList *pEList, + const char *zOld +){ + if( pEList ){ + int i; + for(i=0; inExpr; i++){ + const char *zName = pEList->a[i].zEName; + if( ALWAYS(pEList->a[i].eEName==ENAME_NAME) + && ALWAYS(zName!=0) + && 0==sqlite3_stricmp(zName, zOld) + ){ + renameTokenFind(pParse, pCtx, (const void*)zName); + } + } + } +} + +/* +** For each name in the the id-list pIdList (i.e. each pIdList->a[i].zName) +** that matches the string in zOld, extract the corresponding rename-token +** from Parse object pParse and add it to the RenameCtx pCtx. +*/ +static void renameColumnIdlistNames( + Parse *pParse, + RenameCtx *pCtx, + const IdList *pIdList, + const char *zOld +){ + if( pIdList ){ + int i; + for(i=0; inId; i++){ + const char *zName = pIdList->a[i].zName; + if( 0==sqlite3_stricmp(zName, zOld) ){ + renameTokenFind(pParse, pCtx, (const void*)zName); + } + } + } +} + + +/* +** Parse the SQL statement zSql using Parse object (*p). The Parse object +** is initialized by this function before it is used. +*/ +static int renameParseSql( + Parse *p, /* Memory to use for Parse object */ + const char *zDb, /* Name of schema SQL belongs to */ + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL to parse */ + int bTemp /* True if SQL is from temp schema */ +){ + int rc; + char *zErr = 0; + + db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb); + + /* Parse the SQL statement passed as the first argument. If no error + ** occurs and the parse does not result in a new table, index or + ** trigger object, the database must be corrupt. */ + memset(p, 0, sizeof(Parse)); + p->eParseMode = PARSE_MODE_RENAME; + p->db = db; + p->nQueryLoop = 1; + rc = zSql ? sqlite3RunParser(p, zSql, &zErr) : SQLITE_NOMEM; + assert( p->zErrMsg==0 ); + assert( rc!=SQLITE_OK || zErr==0 ); + p->zErrMsg = zErr; + if( db->mallocFailed ) rc = SQLITE_NOMEM; + if( rc==SQLITE_OK + && p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0 + ){ + rc = SQLITE_CORRUPT_BKPT; + } + +#ifdef SQLITE_DEBUG + /* Ensure that all mappings in the Parse.pRename list really do map to + ** a part of the input string. */ + if( rc==SQLITE_OK ){ + int nSql = sqlite3Strlen30(zSql); + RenameToken *pToken; + for(pToken=p->pRename; pToken; pToken=pToken->pNext){ + assert( pToken->t.z>=zSql && &pToken->t.z[pToken->t.n]<=&zSql[nSql] ); + } + } +#endif + + db->init.iDb = 0; + return rc; +} + +/* +** This function edits SQL statement zSql, replacing each token identified +** by the linked list pRename with the text of zNew. If argument bQuote is +** true, then zNew is always quoted first. If no error occurs, the result +** is loaded into context object pCtx as the result. +** +** Or, if an error occurs (i.e. an OOM condition), an error is left in +** pCtx and an SQLite error code returned. +*/ +static int renameEditSql( + sqlite3_context *pCtx, /* Return result here */ + RenameCtx *pRename, /* Rename context */ + const char *zSql, /* SQL statement to edit */ + const char *zNew, /* New token text */ + int bQuote /* True to always quote token */ +){ + i64 nNew = sqlite3Strlen30(zNew); + i64 nSql = sqlite3Strlen30(zSql); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + int rc = SQLITE_OK; + char *zQuot = 0; + char *zOut; + i64 nQuot = 0; + char *zBuf1 = 0; + char *zBuf2 = 0; + + if( zNew ){ + /* Set zQuot to point to a buffer containing a quoted copy of the + ** identifier zNew. If the corresponding identifier in the original + ** ALTER TABLE statement was quoted (bQuote==1), then set zNew to + ** point to zQuot so that all substitutions are made using the + ** quoted version of the new column name. */ + zQuot = sqlite3MPrintf(db, "\"%w\" ", zNew); + if( zQuot==0 ){ + return SQLITE_NOMEM; + }else{ + nQuot = sqlite3Strlen30(zQuot)-1; + } + + assert( nQuot>=nNew ); + zOut = sqlite3DbMallocZero(db, nSql + pRename->nList*nQuot + 1); + }else{ + zOut = (char*)sqlite3DbMallocZero(db, (nSql*2+1) * 3); + if( zOut ){ + zBuf1 = &zOut[nSql*2+1]; + zBuf2 = &zOut[nSql*4+2]; + } + } + + /* At this point pRename->pList contains a list of RenameToken objects + ** corresponding to all tokens in the input SQL that must be replaced + ** with the new column name, or with single-quoted versions of themselves. + ** All that remains is to construct and return the edited SQL string. */ + if( zOut ){ + int nOut = nSql; + memcpy(zOut, zSql, nSql); + while( pRename->pList ){ + int iOff; /* Offset of token to replace in zOut */ + u32 nReplace; + const char *zReplace; + RenameToken *pBest = renameColumnTokenNext(pRename); + + if( zNew ){ + if( bQuote==0 && sqlite3IsIdChar(*pBest->t.z) ){ + nReplace = nNew; + zReplace = zNew; + }else{ + nReplace = nQuot; + zReplace = zQuot; + if( pBest->t.z[pBest->t.n]=='"' ) nReplace++; + } + }else{ + /* Dequote the double-quoted token. Then requote it again, this time + ** using single quotes. If the character immediately following the + ** original token within the input SQL was a single quote ('), then + ** add another space after the new, single-quoted version of the + ** token. This is so that (SELECT "string"'alias') maps to + ** (SELECT 'string' 'alias'), and not (SELECT 'string''alias'). */ + memcpy(zBuf1, pBest->t.z, pBest->t.n); + zBuf1[pBest->t.n] = 0; + sqlite3Dequote(zBuf1); + sqlite3_snprintf(nSql*2, zBuf2, "%Q%s", zBuf1, + pBest->t.z[pBest->t.n]=='\'' ? " " : "" + ); + zReplace = zBuf2; + nReplace = sqlite3Strlen30(zReplace); + } + + iOff = pBest->t.z - zSql; + if( pBest->t.n!=nReplace ){ + memmove(&zOut[iOff + nReplace], &zOut[iOff + pBest->t.n], + nOut - (iOff + pBest->t.n) + ); + nOut += nReplace - pBest->t.n; + zOut[nOut] = '\0'; + } + memcpy(&zOut[iOff], zReplace, nReplace); + sqlite3DbFree(db, pBest); + } + + sqlite3_result_text(pCtx, zOut, -1, SQLITE_TRANSIENT); + sqlite3DbFree(db, zOut); + }else{ + rc = SQLITE_NOMEM; + } + + sqlite3_free(zQuot); + return rc; +} + +/* +** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming +** it was read from the schema of database zDb. Return SQLITE_OK if +** successful. Otherwise, return an SQLite error code and leave an error +** message in the Parse object. +*/ +static int renameResolveTrigger(Parse *pParse){ + sqlite3 *db = pParse->db; + Trigger *pNew = pParse->pNewTrigger; + TriggerStep *pStep; + NameContext sNC; + int rc = SQLITE_OK; + + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + assert( pNew->pTabSchema ); + pParse->pTriggerTab = sqlite3FindTable(db, pNew->table, + db->aDb[sqlite3SchemaToIndex(db, pNew->pTabSchema)].zDbSName + ); + pParse->eTriggerOp = pNew->op; + /* ALWAYS() because if the table of the trigger does not exist, the + ** error would have been hit before this point */ + if( ALWAYS(pParse->pTriggerTab) ){ + rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab); + } + + /* Resolve symbols in WHEN clause */ + if( rc==SQLITE_OK && pNew->pWhen ){ + rc = sqlite3ResolveExprNames(&sNC, pNew->pWhen); + } + + for(pStep=pNew->step_list; rc==SQLITE_OK && pStep; pStep=pStep->pNext){ + if( pStep->pSelect ){ + sqlite3SelectPrep(pParse, pStep->pSelect, &sNC); + if( pParse->nErr ) rc = pParse->rc; + } + if( rc==SQLITE_OK && pStep->zTarget ){ + SrcList *pSrc = sqlite3TriggerStepSrc(pParse, pStep); + if( pSrc ){ + int i; + for(i=0; inSrc && rc==SQLITE_OK; i++){ + SrcItem *p = &pSrc->a[i]; + p->iCursor = pParse->nTab++; + if( p->pSelect ){ + sqlite3SelectPrep(pParse, p->pSelect, 0); + sqlite3ExpandSubquery(pParse, p); + assert( i>0 ); + assert( pStep->pFrom->a[i-1].pSelect ); + sqlite3SelectPrep(pParse, pStep->pFrom->a[i-1].pSelect, 0); + }else{ + p->pTab = sqlite3LocateTableItem(pParse, 0, p); + if( p->pTab==0 ){ + rc = SQLITE_ERROR; + }else{ + p->pTab->nTabRef++; + rc = sqlite3ViewGetColumnNames(pParse, p->pTab); + } + } + } + sNC.pSrcList = pSrc; + if( rc==SQLITE_OK && pStep->pWhere ){ + rc = sqlite3ResolveExprNames(&sNC, pStep->pWhere); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprListNames(&sNC, pStep->pExprList); + } + assert( !pStep->pUpsert || (!pStep->pWhere && !pStep->pExprList) ); + if( pStep->pUpsert && rc==SQLITE_OK ){ + Upsert *pUpsert = pStep->pUpsert; + pUpsert->pUpsertSrc = pSrc; + sNC.uNC.pUpsert = pUpsert; + sNC.ncFlags = NC_UUpsert; + rc = sqlite3ResolveExprListNames(&sNC, pUpsert->pUpsertTarget); + if( rc==SQLITE_OK ){ + ExprList *pUpsertSet = pUpsert->pUpsertSet; + rc = sqlite3ResolveExprListNames(&sNC, pUpsertSet); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertWhere); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertTargetWhere); + } + sNC.ncFlags = 0; + } + sNC.pSrcList = 0; + sqlite3SrcListDelete(db, pSrc); + }else{ + rc = SQLITE_NOMEM; + } + } + } + return rc; +} + +/* +** Invoke sqlite3WalkExpr() or sqlite3WalkSelect() on all Select or Expr +** objects that are part of the trigger passed as the second argument. +*/ +static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ + TriggerStep *pStep; + + /* Find tokens to edit in WHEN clause */ + sqlite3WalkExpr(pWalker, pTrigger->pWhen); + + /* Find tokens to edit in trigger steps */ + for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ + sqlite3WalkSelect(pWalker, pStep->pSelect); + sqlite3WalkExpr(pWalker, pStep->pWhere); + sqlite3WalkExprList(pWalker, pStep->pExprList); + if( pStep->pUpsert ){ + Upsert *pUpsert = pStep->pUpsert; + sqlite3WalkExprList(pWalker, pUpsert->pUpsertTarget); + sqlite3WalkExprList(pWalker, pUpsert->pUpsertSet); + sqlite3WalkExpr(pWalker, pUpsert->pUpsertWhere); + sqlite3WalkExpr(pWalker, pUpsert->pUpsertTargetWhere); + } + if( pStep->pFrom ){ + int i; + for(i=0; ipFrom->nSrc; i++){ + sqlite3WalkSelect(pWalker, pStep->pFrom->a[i].pSelect); + } + } + } +} + +/* +** Free the contents of Parse object (*pParse). Do not free the memory +** occupied by the Parse object itself. +*/ +static void renameParseCleanup(Parse *pParse){ + sqlite3 *db = pParse->db; + Index *pIdx; + if( pParse->pVdbe ){ + sqlite3VdbeFinalize(pParse->pVdbe); + } + sqlite3DeleteTable(db, pParse->pNewTable); + while( (pIdx = pParse->pNewIndex)!=0 ){ + pParse->pNewIndex = pIdx->pNext; + sqlite3FreeIndex(db, pIdx); + } + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + sqlite3DbFree(db, pParse->zErrMsg); + renameTokenFree(db, pParse->pRename); + sqlite3ParserReset(pParse); +} + +/* +** SQL function: +** +** sqlite_rename_column(zSql, iCol, bQuote, zNew, zTable, zOld) +** +** 0. zSql: SQL statement to rewrite +** 1. type: Type of object ("table", "view" etc.) +** 2. object: Name of object +** 3. Database: Database name (e.g. "main") +** 4. Table: Table name +** 5. iCol: Index of column to rename +** 6. zNew: New column name +** 7. bQuote: Non-zero if the new column name should be quoted. +** 8. bTemp: True if zSql comes from temp schema +** +** Do a column rename operation on the CREATE statement given in zSql. +** The iCol-th column (left-most is 0) of table zTable is renamed from zCol +** into zNew. The name should be quoted if bQuote is true. +** +** This function is used internally by the ALTER TABLE RENAME COLUMN command. +** It is only accessible to SQL created using sqlite3NestedParse(). It is +** not reachable from ordinary SQL passed into sqlite3_prepare(). +*/ +static void renameColumnFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + RenameCtx sCtx; + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + const char *zDb = (const char*)sqlite3_value_text(argv[3]); + const char *zTable = (const char*)sqlite3_value_text(argv[4]); + int iCol = sqlite3_value_int(argv[5]); + const char *zNew = (const char*)sqlite3_value_text(argv[6]); + int bQuote = sqlite3_value_int(argv[7]); + int bTemp = sqlite3_value_int(argv[8]); + const char *zOld; + int rc; + Parse sParse; + Walker sWalker; + Index *pIdx; + int i; + Table *pTab; +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; +#endif + + UNUSED_PARAMETER(NotUsed); + if( zSql==0 ) return; + if( zTable==0 ) return; + if( zNew==0 ) return; + if( iCol<0 ) return; + sqlite3BtreeEnterAll(db); + pTab = sqlite3FindTable(db, zTable, zDb); + if( pTab==0 || iCol>=pTab->nCol ){ + sqlite3BtreeLeaveAll(db); + return; + } + zOld = pTab->aCol[iCol].zCnName; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.iCol = ((iCol==pTab->iPKey) ? -1 : iCol); + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = 0; +#endif + rc = renameParseSql(&sParse, zDb, db, zSql, bTemp); + + /* Find tokens that need to be replaced. */ + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameColumnExprCb; + sWalker.xSelectCallback = renameColumnSelectCb; + sWalker.u.pRename = &sCtx; + + sCtx.pTab = pTab; + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + if( sParse.pNewTable ){ + if( IsView(sParse.pNewTable) ){ + Select *pSelect = sParse.pNewTable->u.view.pSelect; + pSelect->selFlags &= ~SF_View; + sParse.rc = SQLITE_OK; + sqlite3SelectPrep(&sParse, pSelect, 0); + rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); + if( rc==SQLITE_OK ){ + sqlite3WalkSelect(&sWalker, pSelect); + } + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + }else if( IsOrdinaryTable(sParse.pNewTable) ){ + /* A regular table */ + int bFKOnly = sqlite3_stricmp(zTable, sParse.pNewTable->zName); + FKey *pFKey; + sCtx.pTab = sParse.pNewTable; + if( bFKOnly==0 ){ + if( iColnCol ){ + renameTokenFind( + &sParse, &sCtx, (void*)sParse.pNewTable->aCol[iCol].zCnName + ); + } + if( sCtx.iCol<0 ){ + renameTokenFind(&sParse, &sCtx, (void*)&sParse.pNewTable->iPKey); + } + sqlite3WalkExprList(&sWalker, sParse.pNewTable->pCheck); + for(pIdx=sParse.pNewTable->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3WalkExprList(&sWalker, pIdx->aColExpr); + } + for(pIdx=sParse.pNewIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3WalkExprList(&sWalker, pIdx->aColExpr); + } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + for(i=0; inCol; i++){ + Expr *pExpr = sqlite3ColumnExpr(sParse.pNewTable, + &sParse.pNewTable->aCol[i]); + sqlite3WalkExpr(&sWalker, pExpr); + } +#endif + } + + assert( !IsVirtual(sParse.pNewTable) ); + for(pFKey=sParse.pNewTable->u.tab.pFKey; pFKey; pFKey=pFKey->pNextFrom){ + for(i=0; inCol; i++){ + if( bFKOnly==0 && pFKey->aCol[i].iFrom==iCol ){ + renameTokenFind(&sParse, &sCtx, (void*)&pFKey->aCol[i]); + } + if( 0==sqlite3_stricmp(pFKey->zTo, zTable) + && 0==sqlite3_stricmp(pFKey->aCol[i].zCol, zOld) + ){ + renameTokenFind(&sParse, &sCtx, (void*)pFKey->aCol[i].zCol); + } + } + } + } + }else if( sParse.pNewIndex ){ + sqlite3WalkExprList(&sWalker, sParse.pNewIndex->aColExpr); + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + }else{ + /* A trigger */ + TriggerStep *pStep; + rc = renameResolveTrigger(&sParse); + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + + for(pStep=sParse.pNewTrigger->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget ){ + Table *pTarget = sqlite3LocateTable(&sParse, 0, pStep->zTarget, zDb); + if( pTarget==pTab ){ + if( pStep->pUpsert ){ + ExprList *pUpsertSet = pStep->pUpsert->pUpsertSet; + renameColumnElistNames(&sParse, &sCtx, pUpsertSet, zOld); + } + renameColumnIdlistNames(&sParse, &sCtx, pStep->pIdList, zOld); + renameColumnElistNames(&sParse, &sCtx, pStep->pExprList, zOld); + } + } + } + + + /* Find tokens to edit in UPDATE OF clause */ + if( sParse.pTriggerTab==pTab ){ + renameColumnIdlistNames(&sParse, &sCtx,sParse.pNewTrigger->pColumns,zOld); + } + + /* Find tokens to edit in various expressions and selects */ + renameWalkTrigger(&sWalker, sParse.pNewTrigger); + } + + assert( rc==SQLITE_OK ); + rc = renameEditSql(context, &sCtx, zSql, zNew, bQuote); + +renameColumnFunc_done: + if( rc!=SQLITE_OK ){ + if( sParse.zErrMsg ){ + renameColumnParseError(context, "", argv[1], argv[2], &sParse); + }else{ + sqlite3_result_error_code(context, rc); + } + } + + renameParseCleanup(&sParse); + renameTokenFree(db, sCtx.pList); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + sqlite3BtreeLeaveAll(db); +} + +/* +** Walker expression callback used by "RENAME TABLE". +*/ +static int renameTableExprCb(Walker *pWalker, Expr *pExpr){ + RenameCtx *p = pWalker->u.pRename; + if( pExpr->op==TK_COLUMN && p->pTab==pExpr->y.pTab ){ + renameTokenFind(pWalker->pParse, p, (void*)&pExpr->y.pTab); + } + return WRC_Continue; +} + +/* +** Walker select callback used by "RENAME TABLE". +*/ +static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ + int i; + RenameCtx *p = pWalker->u.pRename; + SrcList *pSrc = pSelect->pSrc; + if( pSelect->selFlags & (SF_View|SF_CopyCte) ){ + testcase( pSelect->selFlags & SF_View ); + testcase( pSelect->selFlags & SF_CopyCte ); + return WRC_Prune; + } + if( NEVER(pSrc==0) ){ + assert( pWalker->pParse->db->mallocFailed ); + return WRC_Abort; + } + for(i=0; inSrc; i++){ + SrcItem *pItem = &pSrc->a[i]; + if( pItem->pTab==p->pTab ){ + renameTokenFind(pWalker->pParse, p, pItem->zName); + } + } + renameWalkWith(pWalker, pSelect); + + return WRC_Continue; +} + + +/* +** This C function implements an SQL user function that is used by SQL code +** generated by the ALTER TABLE ... RENAME command to modify the definition +** of any foreign key constraints that use the table being renamed as the +** parent table. It is passed three arguments: +** +** 0: The database containing the table being renamed. +** 1. type: Type of object ("table", "view" etc.) +** 2. object: Name of object +** 3: The complete text of the schema statement being modified, +** 4: The old name of the table being renamed, and +** 5: The new name of the table being renamed. +** 6: True if the schema statement comes from the temp db. +** +** It returns the new schema statement. For example: +** +** sqlite_rename_table('main', 'CREATE TABLE t1(a REFERENCES t2)','t2','t3',0) +** -> 'CREATE TABLE t1(a REFERENCES t3)' +*/ +static void renameTableFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zDb = (const char*)sqlite3_value_text(argv[0]); + const char *zInput = (const char*)sqlite3_value_text(argv[3]); + const char *zOld = (const char*)sqlite3_value_text(argv[4]); + const char *zNew = (const char*)sqlite3_value_text(argv[5]); + int bTemp = sqlite3_value_int(argv[6]); + UNUSED_PARAMETER(NotUsed); + + if( zInput && zOld && zNew ){ + Parse sParse; + int rc; + int bQuote = 1; + RenameCtx sCtx; + Walker sWalker; + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + sqlite3BtreeEnterAll(db); + + memset(&sCtx, 0, sizeof(RenameCtx)); + sCtx.pTab = sqlite3FindTable(db, zOld, zDb); + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameTableExprCb; + sWalker.xSelectCallback = renameTableSelectCb; + sWalker.u.pRename = &sCtx; + + rc = renameParseSql(&sParse, zDb, db, zInput, bTemp); + + if( rc==SQLITE_OK ){ + int isLegacy = (db->flags & SQLITE_LegacyAlter); + if( sParse.pNewTable ){ + Table *pTab = sParse.pNewTable; + + if( IsView(pTab) ){ + if( isLegacy==0 ){ + Select *pSelect = pTab->u.view.pSelect; + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = &sParse; + + assert( pSelect->selFlags & SF_View ); + pSelect->selFlags &= ~SF_View; + sqlite3SelectPrep(&sParse, pTab->u.view.pSelect, &sNC); + if( sParse.nErr ){ + rc = sParse.rc; + }else{ + sqlite3WalkSelect(&sWalker, pTab->u.view.pSelect); + } + } + }else{ + /* Modify any FK definitions to point to the new table. */ +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( (isLegacy==0 || (db->flags & SQLITE_ForeignKeys)) + && !IsVirtual(pTab) + ){ + FKey *pFKey; + assert( !IsVirtual(pTab) ); + for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pFKey->pNextFrom){ + if( sqlite3_stricmp(pFKey->zTo, zOld)==0 ){ + renameTokenFind(&sParse, &sCtx, (void*)pFKey->zTo); + } + } + } +#endif + + /* If this is the table being altered, fix any table refs in CHECK + ** expressions. Also update the name that appears right after the + ** "CREATE [VIRTUAL] TABLE" bit. */ + if( sqlite3_stricmp(zOld, pTab->zName)==0 ){ + sCtx.pTab = pTab; + if( isLegacy==0 ){ + sqlite3WalkExprList(&sWalker, pTab->pCheck); + } + renameTokenFind(&sParse, &sCtx, pTab->zName); + } + } + } + + else if( sParse.pNewIndex ){ + renameTokenFind(&sParse, &sCtx, sParse.pNewIndex->zName); + if( isLegacy==0 ){ + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + } + } + +#ifndef SQLITE_OMIT_TRIGGER + else{ + Trigger *pTrigger = sParse.pNewTrigger; + TriggerStep *pStep; + if( 0==sqlite3_stricmp(sParse.pNewTrigger->table, zOld) + && sCtx.pTab->pSchema==pTrigger->pTabSchema + ){ + renameTokenFind(&sParse, &sCtx, sParse.pNewTrigger->table); + } + + if( isLegacy==0 ){ + rc = renameResolveTrigger(&sParse); + if( rc==SQLITE_OK ){ + renameWalkTrigger(&sWalker, pTrigger); + for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget && 0==sqlite3_stricmp(pStep->zTarget, zOld) ){ + renameTokenFind(&sParse, &sCtx, pStep->zTarget); + } + } + } + } + } +#endif + } + + if( rc==SQLITE_OK ){ + rc = renameEditSql(context, &sCtx, zInput, zNew, bQuote); + } + if( rc!=SQLITE_OK ){ + if( sParse.zErrMsg ){ + renameColumnParseError(context, "", argv[1], argv[2], &sParse); + }else{ + sqlite3_result_error_code(context, rc); + } + } + + renameParseCleanup(&sParse); + renameTokenFree(db, sCtx.pList); + sqlite3BtreeLeaveAll(db); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + } + + return; +} + +static int renameQuotefixExprCb(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_STRING && (pExpr->flags & EP_DblQuoted) ){ + renameTokenFind(pWalker->pParse, pWalker->u.pRename, (const void*)pExpr); + } + return WRC_Continue; +} + +/* +** The implementation of an SQL scalar function that rewrites DDL statements +** so that any string literals that use double-quotes are modified so that +** they use single quotes. +** +** Two arguments must be passed: +** +** 0: Database name ("main", "temp" etc.). +** 1: SQL statement to edit. +** +** The returned value is the modified SQL statement. For example, given +** the database schema: +** +** CREATE TABLE t1(a, b, c); +** +** SELECT sqlite_rename_quotefix('main', +** 'CREATE VIEW v1 AS SELECT "a", "string" FROM t1' +** ); +** +** returns the string: +** +** CREATE VIEW v1 AS SELECT "a", 'string' FROM t1 +*/ +static void renameQuotefixFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + char const *zDb = (const char*)sqlite3_value_text(argv[0]); + char const *zInput = (const char*)sqlite3_value_text(argv[1]); + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + sqlite3BtreeEnterAll(db); + + UNUSED_PARAMETER(NotUsed); + if( zDb && zInput ){ + int rc; + Parse sParse; + rc = renameParseSql(&sParse, zDb, db, zInput, 0); + + if( rc==SQLITE_OK ){ + RenameCtx sCtx; + Walker sWalker; + + /* Walker to find tokens that need to be replaced. */ + memset(&sCtx, 0, sizeof(RenameCtx)); + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameQuotefixExprCb; + sWalker.xSelectCallback = renameColumnSelectCb; + sWalker.u.pRename = &sCtx; + + if( sParse.pNewTable ){ + if( IsView(sParse.pNewTable) ){ + Select *pSelect = sParse.pNewTable->u.view.pSelect; + pSelect->selFlags &= ~SF_View; + sParse.rc = SQLITE_OK; + sqlite3SelectPrep(&sParse, pSelect, 0); + rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); + if( rc==SQLITE_OK ){ + sqlite3WalkSelect(&sWalker, pSelect); + } + }else{ + int i; + sqlite3WalkExprList(&sWalker, sParse.pNewTable->pCheck); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + for(i=0; inCol; i++){ + sqlite3WalkExpr(&sWalker, + sqlite3ColumnExpr(sParse.pNewTable, + &sParse.pNewTable->aCol[i])); + } +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + } + }else if( sParse.pNewIndex ){ + sqlite3WalkExprList(&sWalker, sParse.pNewIndex->aColExpr); + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + }else{ +#ifndef SQLITE_OMIT_TRIGGER + rc = renameResolveTrigger(&sParse); + if( rc==SQLITE_OK ){ + renameWalkTrigger(&sWalker, sParse.pNewTrigger); + } +#endif /* SQLITE_OMIT_TRIGGER */ + } + + if( rc==SQLITE_OK ){ + rc = renameEditSql(context, &sCtx, zInput, 0, 0); + } + renameTokenFree(db, sCtx.pList); + } + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(context, rc); + } + renameParseCleanup(&sParse); + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + + sqlite3BtreeLeaveAll(db); +} + +/* +** An SQL user function that checks that there are no parse or symbol +** resolution problems in a CREATE TRIGGER|TABLE|VIEW|INDEX statement. +** After an ALTER TABLE .. RENAME operation is performed and the schema +** reloaded, this function is called on each SQL statement in the schema +** to ensure that it is still usable. +** +** 0: Database name ("main", "temp" etc.). +** 1: SQL statement. +** 2: Object type ("view", "table", "trigger" or "index"). +** 3: Object name. +** 4: True if object is from temp schema. +** 5: "when" part of error message. +** 6: True to disable the DQS quirk when parsing SQL. +** +** Unless it finds an error, this function normally returns NULL. However, it +** returns integer value 1 if: +** +** * the SQL argument creates a trigger, and +** * the table that the trigger is attached to is in database zDb. +*/ +static void renameTableTest( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + char const *zDb = (const char*)sqlite3_value_text(argv[0]); + char const *zInput = (const char*)sqlite3_value_text(argv[1]); + int bTemp = sqlite3_value_int(argv[4]); + int isLegacy = (db->flags & SQLITE_LegacyAlter); + char const *zWhen = (const char*)sqlite3_value_text(argv[5]); + int bNoDQS = sqlite3_value_int(argv[6]); + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + UNUSED_PARAMETER(NotUsed); + + if( zDb && zInput ){ + int rc; + Parse sParse; + int flags = db->flags; + if( bNoDQS ) db->flags &= ~(SQLITE_DqsDML|SQLITE_DqsDDL); + rc = renameParseSql(&sParse, zDb, db, zInput, bTemp); + db->flags |= (flags & (SQLITE_DqsDML|SQLITE_DqsDDL)); + if( rc==SQLITE_OK ){ + if( isLegacy==0 && sParse.pNewTable && IsView(sParse.pNewTable) ){ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = &sParse; + sqlite3SelectPrep(&sParse, sParse.pNewTable->u.view.pSelect, &sNC); + if( sParse.nErr ) rc = sParse.rc; + } + + else if( sParse.pNewTrigger ){ + if( isLegacy==0 ){ + rc = renameResolveTrigger(&sParse); + } + if( rc==SQLITE_OK ){ + int i1 = sqlite3SchemaToIndex(db, sParse.pNewTrigger->pTabSchema); + int i2 = sqlite3FindDbName(db, zDb); + if( i1==i2 ) sqlite3_result_int(context, 1); + } + } + } + + if( rc!=SQLITE_OK && zWhen ){ + renameColumnParseError(context, zWhen, argv[2], argv[3],&sParse); + } + renameParseCleanup(&sParse); + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif +} + +/* +** The implementation of internal UDF sqlite_drop_column(). +** +** Arguments: +** +** argv[0]: An integer - the index of the schema containing the table +** argv[1]: CREATE TABLE statement to modify. +** argv[2]: An integer - the index of the column to remove. +** +** The value returned is a string containing the CREATE TABLE statement +** with column argv[2] removed. +*/ +static void dropColumnFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + int iSchema = sqlite3_value_int(argv[0]); + const char *zSql = (const char*)sqlite3_value_text(argv[1]); + int iCol = sqlite3_value_int(argv[2]); + const char *zDb = db->aDb[iSchema].zDbSName; + int rc; + Parse sParse; + RenameToken *pCol; + Table *pTab; + const char *zEnd; + char *zNew = 0; + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + UNUSED_PARAMETER(NotUsed); + rc = renameParseSql(&sParse, zDb, db, zSql, iSchema==1); + if( rc!=SQLITE_OK ) goto drop_column_done; + pTab = sParse.pNewTable; + if( pTab==0 || pTab->nCol==1 || iCol>=pTab->nCol ){ + /* This can happen if the sqlite_schema table is corrupt */ + rc = SQLITE_CORRUPT_BKPT; + goto drop_column_done; + } + + pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zCnName); + if( iColnCol-1 ){ + RenameToken *pEnd; + pEnd = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol+1].zCnName); + zEnd = (const char*)pEnd->t.z; + }else{ + assert( !IsVirtual(pTab) ); + zEnd = (const char*)&zSql[pTab->u.tab.addColOffset]; + while( ALWAYS(pCol->t.z[0]!=0) && pCol->t.z[0]!=',' ) pCol->t.z--; + } + + zNew = sqlite3MPrintf(db, "%.*s%s", pCol->t.z-zSql, zSql, zEnd); + sqlite3_result_text(context, zNew, -1, SQLITE_TRANSIENT); + sqlite3_free(zNew); + +drop_column_done: + renameParseCleanup(&sParse); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(context, rc); + } +} + +/* +** This function is called by the parser upon parsing an +** +** ALTER TABLE pSrc DROP COLUMN pName +** +** statement. Argument pSrc contains the possibly qualified name of the +** table being edited, and token pName the name of the column to drop. +*/ +SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ + sqlite3 *db = pParse->db; /* Database handle */ + Table *pTab; /* Table to modify */ + int iDb; /* Index of db containing pTab in aDb[] */ + const char *zDb; /* Database containing pTab ("main" etc.) */ + char *zCol = 0; /* Name of column to drop */ + int iCol; /* Index of column zCol in pTab->aCol[] */ + + /* Look up the table being altered. */ + assert( pParse->pNewTable==0 ); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + if( NEVER(db->mallocFailed) ) goto exit_drop_column; + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_drop_column; + + /* Make sure this is not an attempt to ALTER a view, virtual table or + ** system table. */ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ) goto exit_drop_column; + if( SQLITE_OK!=isRealTable(pParse, pTab, 1) ) goto exit_drop_column; + + /* Find the index of the column being dropped. */ + zCol = sqlite3NameFromToken(db, pName); + if( zCol==0 ){ + assert( db->mallocFailed ); + goto exit_drop_column; + } + iCol = sqlite3ColumnIndex(pTab, zCol); + if( iCol<0 ){ + sqlite3ErrorMsg(pParse, "no such column: \"%s\"", zCol); + goto exit_drop_column; + } + + /* Do not allow the user to drop a PRIMARY KEY column or a column + ** constrained by a UNIQUE constraint. */ + if( pTab->aCol[iCol].colFlags & (COLFLAG_PRIMKEY|COLFLAG_UNIQUE) ){ + sqlite3ErrorMsg(pParse, "cannot drop %s column: \"%s\"", + (pTab->aCol[iCol].colFlags&COLFLAG_PRIMKEY) ? "PRIMARY KEY" : "UNIQUE", + zCol + ); + goto exit_drop_column; + } + + /* Do not allow the number of columns to go to zero */ + if( pTab->nCol<=1 ){ + sqlite3ErrorMsg(pParse, "cannot drop column \"%s\": no other columns exist",zCol); + goto exit_drop_column; + } + + /* Edit the sqlite_schema table */ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 ); + zDb = db->aDb[iDb].zDbSName; + renameTestSchema(pParse, zDb, iDb==1, "", 0); + renameFixQuotes(pParse, zDb, iDb==1); + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." DFLT_SCHEMA_TABLE " SET " + "sql = sqlite_drop_column(%d, sql, %d) " + "WHERE (type=='table' AND tbl_name=%Q COLLATE nocase)" + , zDb, iDb, iCol, pTab->zName + ); + + /* Drop and reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDrop); + renameTestSchema(pParse, zDb, iDb==1, "after drop column", 1); + + /* Edit rows of table on disk */ + if( pParse->nErr==0 && (pTab->aCol[iCol].colFlags & COLFLAG_VIRTUAL)==0 ){ + int i; + int addr; + int reg; + int regRec; + Index *pPk = 0; + int nField = 0; /* Number of non-virtual columns after drop */ + int iCur; + Vdbe *v = sqlite3GetVdbe(pParse); + iCur = pParse->nTab++; + sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenWrite); + addr = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + reg = ++pParse->nMem; + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, reg); + pParse->nMem += pTab->nCol; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + pParse->nMem += pPk->nColumn; + for(i=0; inKeyCol; i++){ + sqlite3VdbeAddOp3(v, OP_Column, iCur, i, reg+i+1); + } + nField = pPk->nKeyCol; + } + regRec = ++pParse->nMem; + for(i=0; inCol; i++){ + if( i!=iCol && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + int regOut; + if( pPk ){ + int iPos = sqlite3TableColumnToIndex(pPk, i); + int iColPos = sqlite3TableColumnToIndex(pPk, iCol); + if( iPosnKeyCol ) continue; + regOut = reg+1+iPos-(iPos>iColPos); + }else{ + regOut = reg+1+nField; + } + if( i==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); + }else{ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + } + nField++; + } + } + if( nField==0 ){ + /* dbsqlfuzz 5f09e7bcc78b4954d06bf9f2400d7715f48d1fef */ + pParse->nMem++; + sqlite3VdbeAddOp2(v, OP_Null, 0, reg+1); + nField = 1; + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, reg+1, nField, regRec); + if( pPk ){ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iCur, regRec, reg+1, pPk->nKeyCol); + }else{ + sqlite3VdbeAddOp3(v, OP_Insert, iCur, regRec, reg); + } + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); + + sqlite3VdbeAddOp2(v, OP_Next, iCur, addr+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + } + +exit_drop_column: + sqlite3DbFree(db, zCol); + sqlite3SrcListDelete(db, pSrc); +} + +/* +** Register built-in functions used to help implement ALTER TABLE +*/ +SQLITE_PRIVATE void sqlite3AlterFunctions(void){ + static FuncDef aAlterTableFuncs[] = { + INTERNAL_FUNCTION(sqlite_rename_column, 9, renameColumnFunc), + INTERNAL_FUNCTION(sqlite_rename_table, 7, renameTableFunc), + INTERNAL_FUNCTION(sqlite_rename_test, 7, renameTableTest), + INTERNAL_FUNCTION(sqlite_drop_column, 3, dropColumnFunc), + INTERNAL_FUNCTION(sqlite_rename_quotefix,2, renameQuotefixFunc), + }; + sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); +} +#endif /* SQLITE_ALTER_TABLE */ + +/************** End of alter.c ***********************************************/ +/************** Begin file analyze.c *****************************************/ +/* +** 2005-07-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code associated with the ANALYZE command. +** +** The ANALYZE command gather statistics about the content of tables +** and indices. These statistics are made available to the query planner +** to help it make better decisions about how to perform queries. +** +** The following system tables are or have been supported: +** +** CREATE TABLE sqlite_stat1(tbl, idx, stat); +** CREATE TABLE sqlite_stat2(tbl, idx, sampleno, sample); +** CREATE TABLE sqlite_stat3(tbl, idx, nEq, nLt, nDLt, sample); +** CREATE TABLE sqlite_stat4(tbl, idx, nEq, nLt, nDLt, sample); +** +** Additional tables might be added in future releases of SQLite. +** The sqlite_stat2 table is not created or used unless the SQLite version +** is between 3.6.18 and 3.7.8, inclusive, and unless SQLite is compiled +** with SQLITE_ENABLE_STAT2. The sqlite_stat2 table is deprecated. +** The sqlite_stat2 table is superseded by sqlite_stat3, which is only +** created and used by SQLite versions 3.7.9 through 3.29.0 when +** SQLITE_ENABLE_STAT3 defined. The functionality of sqlite_stat3 +** is a superset of sqlite_stat2 and is also now deprecated. The +** sqlite_stat4 is an enhanced version of sqlite_stat3 and is only +** available when compiled with SQLITE_ENABLE_STAT4 and in SQLite +** versions 3.8.1 and later. STAT4 is the only variant that is still +** supported. +** +** For most applications, sqlite_stat1 provides all the statistics required +** for the query planner to make good choices. +** +** Format of sqlite_stat1: +** +** There is normally one row per index, with the index identified by the +** name in the idx column. The tbl column is the name of the table to +** which the index belongs. In each such row, the stat column will be +** a string consisting of a list of integers. The first integer in this +** list is the number of rows in the index. (This is the same as the +** number of rows in the table, except for partial indices.) The second +** integer is the average number of rows in the index that have the same +** value in the first column of the index. The third integer is the average +** number of rows in the index that have the same value for the first two +** columns. The N-th integer (for N>1) is the average number of rows in +** the index which have the same value for the first N-1 columns. For +** a K-column index, there will be K+1 integers in the stat column. If +** the index is unique, then the last integer will be 1. +** +** The list of integers in the stat column can optionally be followed +** by the keyword "unordered". The "unordered" keyword, if it is present, +** must be separated from the last integer by a single space. If the +** "unordered" keyword is present, then the query planner assumes that +** the index is unordered and will not use the index for a range query. +** +** If the sqlite_stat1.idx column is NULL, then the sqlite_stat1.stat +** column contains a single integer which is the (estimated) number of +** rows in the table identified by sqlite_stat1.tbl. +** +** Format of sqlite_stat2: +** +** The sqlite_stat2 is only created and is only used if SQLite is compiled +** with SQLITE_ENABLE_STAT2 and if the SQLite version number is between +** 3.6.18 and 3.7.8. The "stat2" table contains additional information +** about the distribution of keys within an index. The index is identified by +** the "idx" column and the "tbl" column is the name of the table to which +** the index belongs. There are usually 10 rows in the sqlite_stat2 +** table for each index. +** +** The sqlite_stat2 entries for an index that have sampleno between 0 and 9 +** inclusive are samples of the left-most key value in the index taken at +** evenly spaced points along the index. Let the number of samples be S +** (10 in the standard build) and let C be the number of rows in the index. +** Then the sampled rows are given by: +** +** rownumber = (i*C*2 + C)/(S*2) +** +** For i between 0 and S-1. Conceptually, the index space is divided into +** S uniform buckets and the samples are the middle row from each bucket. +** +** The format for sqlite_stat2 is recorded here for legacy reference. This +** version of SQLite does not support sqlite_stat2. It neither reads nor +** writes the sqlite_stat2 table. This version of SQLite only supports +** sqlite_stat3. +** +** Format for sqlite_stat3: +** +** The sqlite_stat3 format is a subset of sqlite_stat4. Hence, the +** sqlite_stat4 format will be described first. Further information +** about sqlite_stat3 follows the sqlite_stat4 description. +** +** Format for sqlite_stat4: +** +** As with sqlite_stat2, the sqlite_stat4 table contains histogram data +** to aid the query planner in choosing good indices based on the values +** that indexed columns are compared against in the WHERE clauses of +** queries. +** +** The sqlite_stat4 table contains multiple entries for each index. +** The idx column names the index and the tbl column is the table of the +** index. If the idx and tbl columns are the same, then the sample is +** of the INTEGER PRIMARY KEY. The sample column is a blob which is the +** binary encoding of a key from the index. The nEq column is a +** list of integers. The first integer is the approximate number +** of entries in the index whose left-most column exactly matches +** the left-most column of the sample. The second integer in nEq +** is the approximate number of entries in the index where the +** first two columns match the first two columns of the sample. +** And so forth. nLt is another list of integers that show the approximate +** number of entries that are strictly less than the sample. The first +** integer in nLt contains the number of entries in the index where the +** left-most column is less than the left-most column of the sample. +** The K-th integer in the nLt entry is the number of index entries +** where the first K columns are less than the first K columns of the +** sample. The nDLt column is like nLt except that it contains the +** number of distinct entries in the index that are less than the +** sample. +** +** There can be an arbitrary number of sqlite_stat4 entries per index. +** The ANALYZE command will typically generate sqlite_stat4 tables +** that contain between 10 and 40 samples which are distributed across +** the key space, though not uniformly, and which include samples with +** large nEq values. +** +** Format for sqlite_stat3 redux: +** +** The sqlite_stat3 table is like sqlite_stat4 except that it only +** looks at the left-most column of the index. The sqlite_stat3.sample +** column contains the actual value of the left-most column instead +** of a blob encoding of the complete index key as is found in +** sqlite_stat4.sample. The nEq, nLt, and nDLt entries of sqlite_stat3 +** all contain just a single integer which is the same as the first +** integer in the equivalent columns in sqlite_stat4. +*/ +#ifndef SQLITE_OMIT_ANALYZE +/* #include "sqliteInt.h" */ + +#if defined(SQLITE_ENABLE_STAT4) +# define IsStat4 1 +#else +# define IsStat4 0 +# undef SQLITE_STAT4_SAMPLES +# define SQLITE_STAT4_SAMPLES 1 +#endif + +/* +** This routine generates code that opens the sqlite_statN tables. +** The sqlite_stat1 table is always relevant. sqlite_stat2 is now +** obsolete. sqlite_stat3 and sqlite_stat4 are only opened when +** appropriate compile-time options are provided. +** +** If the sqlite_statN tables do not previously exist, it is created. +** +** Argument zWhere may be a pointer to a buffer containing a table name, +** or it may be a NULL pointer. If it is not NULL, then all entries in +** the sqlite_statN tables associated with the named table are deleted. +** If zWhere==0, then code is generated to delete all stat table entries. +*/ +static void openStatTable( + Parse *pParse, /* Parsing context */ + int iDb, /* The database we are looking in */ + int iStatCur, /* Open the sqlite_stat1 table on this cursor */ + const char *zWhere, /* Delete entries for this table or index */ + const char *zWhereType /* Either "tbl" or "idx" */ +){ + static const struct { + const char *zName; + const char *zCols; + } aTable[] = { + { "sqlite_stat1", "tbl,idx,stat" }, +#if defined(SQLITE_ENABLE_STAT4) + { "sqlite_stat4", "tbl,idx,neq,nlt,ndlt,sample" }, +#else + { "sqlite_stat4", 0 }, +#endif + { "sqlite_stat3", 0 }, + }; + int i; + sqlite3 *db = pParse->db; + Db *pDb; + Vdbe *v = sqlite3GetVdbe(pParse); + u32 aRoot[ArraySize(aTable)]; + u8 aCreateTbl[ArraySize(aTable)]; +#ifdef SQLITE_ENABLE_STAT4 + const int nToOpen = OptimizationEnabled(db,SQLITE_Stat4) ? 2 : 1; +#else + const int nToOpen = 1; +#endif + + if( v==0 ) return; + assert( sqlite3BtreeHoldsAllMutexes(db) ); + assert( sqlite3VdbeDb(v)==db ); + pDb = &db->aDb[iDb]; + + /* Create new statistic tables if they do not exist, or clear them + ** if they do already exist. + */ + for(i=0; izDbSName))==0 ){ + if( iregRoot. This is important + ** because the OpenWrite opcode below will be needing it. */ + sqlite3NestedParse(pParse, + "CREATE TABLE %Q.%s(%s)", pDb->zDbSName, zTab, aTable[i].zCols + ); + aRoot[i] = (u32)pParse->regRoot; + aCreateTbl[i] = OPFLAG_P2ISREG; + } + }else{ + /* The table already exists. If zWhere is not NULL, delete all entries + ** associated with the table zWhere. If zWhere is NULL, delete the + ** entire contents of the table. */ + aRoot[i] = pStat->tnum; + sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab); + if( zWhere ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE %s=%Q", + pDb->zDbSName, zTab, zWhereType, zWhere + ); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + }else if( db->xPreUpdateCallback ){ + sqlite3NestedParse(pParse, "DELETE FROM %Q.%s", pDb->zDbSName, zTab); +#endif + }else{ + /* The sqlite_stat[134] table already exists. Delete all rows. */ + sqlite3VdbeAddOp2(v, OP_Clear, (int)aRoot[i], iDb); + } + } + } + + /* Open the sqlite_stat[134] tables for writing. */ + for(i=0; inRowid ){ + sqlite3DbFree(db, p->u.aRowid); + p->nRowid = 0; + } +} +#endif + +/* Initialize the BLOB value of a ROWID +*/ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleSetRowid(sqlite3 *db, StatSample *p, int n, const u8 *pData){ + assert( db!=0 ); + if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); + p->u.aRowid = sqlite3DbMallocRawNN(db, n); + if( p->u.aRowid ){ + p->nRowid = n; + memcpy(p->u.aRowid, pData, n); + }else{ + p->nRowid = 0; + } +} +#endif + +/* Initialize the INTEGER value of a ROWID. +*/ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleSetRowidInt64(sqlite3 *db, StatSample *p, i64 iRowid){ + assert( db!=0 ); + if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); + p->nRowid = 0; + p->u.iRowid = iRowid; +} +#endif + + +/* +** Copy the contents of object (*pFrom) into (*pTo). +*/ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleCopy(StatAccum *p, StatSample *pTo, StatSample *pFrom){ + pTo->isPSample = pFrom->isPSample; + pTo->iCol = pFrom->iCol; + pTo->iHash = pFrom->iHash; + memcpy(pTo->anEq, pFrom->anEq, sizeof(tRowcnt)*p->nCol); + memcpy(pTo->anLt, pFrom->anLt, sizeof(tRowcnt)*p->nCol); + memcpy(pTo->anDLt, pFrom->anDLt, sizeof(tRowcnt)*p->nCol); + if( pFrom->nRowid ){ + sampleSetRowid(p->db, pTo, pFrom->nRowid, pFrom->u.aRowid); + }else{ + sampleSetRowidInt64(p->db, pTo, pFrom->u.iRowid); + } +} +#endif + +/* +** Reclaim all memory of a StatAccum structure. +*/ +static void statAccumDestructor(void *pOld){ + StatAccum *p = (StatAccum*)pOld; +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ){ + int i; + for(i=0; inCol; i++) sampleClear(p->db, p->aBest+i); + for(i=0; imxSample; i++) sampleClear(p->db, p->a+i); + sampleClear(p->db, &p->current); + } +#endif + sqlite3DbFree(p->db, p); +} + +/* +** Implementation of the stat_init(N,K,C,L) SQL function. The four parameters +** are: +** N: The number of columns in the index including the rowid/pk (note 1) +** K: The number of columns in the index excluding the rowid/pk. +** C: Estimated number of rows in the index +** L: A limit on the number of rows to scan, or 0 for no-limit +** +** Note 1: In the special case of the covering index that implements a +** WITHOUT ROWID table, N is the number of PRIMARY KEY columns, not the +** total number of columns in the table. +** +** For indexes on ordinary rowid tables, N==K+1. But for indexes on +** WITHOUT ROWID tables, N=K+P where P is the number of columns in the +** PRIMARY KEY of the table. The covering index that implements the +** original WITHOUT ROWID table as N==K as a special case. +** +** This routine allocates the StatAccum object in heap memory. The return +** value is a pointer to the StatAccum object. The datatype of the +** return value is BLOB, but it is really just a pointer to the StatAccum +** object. +*/ +static void statInit( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + StatAccum *p; + int nCol; /* Number of columns in index being sampled */ + int nKeyCol; /* Number of key columns */ + int nColUp; /* nCol rounded up for alignment */ + int n; /* Bytes of space to allocate */ + sqlite3 *db = sqlite3_context_db_handle(context); /* Database connection */ +#ifdef SQLITE_ENABLE_STAT4 + /* Maximum number of samples. 0 if STAT4 data is not collected */ + int mxSample = OptimizationEnabled(db,SQLITE_Stat4) ?SQLITE_STAT4_SAMPLES :0; +#endif + + /* Decode the three function arguments */ + UNUSED_PARAMETER(argc); + nCol = sqlite3_value_int(argv[0]); + assert( nCol>0 ); + nColUp = sizeof(tRowcnt)<8 ? (nCol+1)&~1 : nCol; + nKeyCol = sqlite3_value_int(argv[1]); + assert( nKeyCol<=nCol ); + assert( nKeyCol>0 ); + + /* Allocate the space required for the StatAccum object */ + n = sizeof(*p) + + sizeof(tRowcnt)*nColUp /* StatAccum.anEq */ + + sizeof(tRowcnt)*nColUp; /* StatAccum.anDLt */ +#ifdef SQLITE_ENABLE_STAT4 + if( mxSample ){ + n += sizeof(tRowcnt)*nColUp /* StatAccum.anLt */ + + sizeof(StatSample)*(nCol+mxSample) /* StatAccum.aBest[], a[] */ + + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample); + } +#endif + p = sqlite3DbMallocZero(db, n); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + p->db = db; + p->nEst = sqlite3_value_int64(argv[2]); + p->nRow = 0; + p->nLimit = sqlite3_value_int64(argv[3]); + p->nCol = nCol; + p->nKeyCol = nKeyCol; + p->nSkipAhead = 0; + p->current.anDLt = (tRowcnt*)&p[1]; + p->current.anEq = &p->current.anDLt[nColUp]; + +#ifdef SQLITE_ENABLE_STAT4 + p->mxSample = p->nLimit==0 ? mxSample : 0; + if( mxSample ){ + u8 *pSpace; /* Allocated space not yet assigned */ + int i; /* Used to iterate through p->aSample[] */ + + p->iGet = -1; + p->nPSample = (tRowcnt)(p->nEst/(mxSample/3+1) + 1); + p->current.anLt = &p->current.anEq[nColUp]; + p->iPrn = 0x689e962d*(u32)nCol ^ 0xd0944565*(u32)sqlite3_value_int(argv[2]); + + /* Set up the StatAccum.a[] and aBest[] arrays */ + p->a = (struct StatSample*)&p->current.anLt[nColUp]; + p->aBest = &p->a[mxSample]; + pSpace = (u8*)(&p->a[mxSample+nCol]); + for(i=0; i<(mxSample+nCol); i++){ + p->a[i].anEq = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + p->a[i].anLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + p->a[i].anDLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + } + assert( (pSpace - (u8*)p)==n ); + + for(i=0; iaBest[i].iCol = i; + } + } +#endif + + /* Return a pointer to the allocated object to the caller. Note that + ** only the pointer (the 2nd parameter) matters. The size of the object + ** (given by the 3rd parameter) is never used and can be any positive + ** value. */ + sqlite3_result_blob(context, p, sizeof(*p), statAccumDestructor); +} +static const FuncDef statInitFuncdef = { + 4, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statInit, /* xSFunc */ + 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ + "stat_init", /* zName */ + {0} +}; + +#ifdef SQLITE_ENABLE_STAT4 +/* +** pNew and pOld are both candidate non-periodic samples selected for +** the same column (pNew->iCol==pOld->iCol). Ignoring this column and +** considering only any trailing columns and the sample hash value, this +** function returns true if sample pNew is to be preferred over pOld. +** In other words, if we assume that the cardinalities of the selected +** column for pNew and pOld are equal, is pNew to be preferred over pOld. +** +** This function assumes that for each argument sample, the contents of +** the anEq[] array from pSample->anEq[pSample->iCol+1] onwards are valid. +*/ +static int sampleIsBetterPost( + StatAccum *pAccum, + StatSample *pNew, + StatSample *pOld +){ + int nCol = pAccum->nCol; + int i; + assert( pNew->iCol==pOld->iCol ); + for(i=pNew->iCol+1; ianEq[i]>pOld->anEq[i] ) return 1; + if( pNew->anEq[i]anEq[i] ) return 0; + } + if( pNew->iHash>pOld->iHash ) return 1; + return 0; +} +#endif + +#ifdef SQLITE_ENABLE_STAT4 +/* +** Return true if pNew is to be preferred over pOld. +** +** This function assumes that for each argument sample, the contents of +** the anEq[] array from pSample->anEq[pSample->iCol] onwards are valid. +*/ +static int sampleIsBetter( + StatAccum *pAccum, + StatSample *pNew, + StatSample *pOld +){ + tRowcnt nEqNew = pNew->anEq[pNew->iCol]; + tRowcnt nEqOld = pOld->anEq[pOld->iCol]; + + assert( pOld->isPSample==0 && pNew->isPSample==0 ); + assert( IsStat4 || (pNew->iCol==0 && pOld->iCol==0) ); + + if( (nEqNew>nEqOld) ) return 1; + if( nEqNew==nEqOld ){ + if( pNew->iColiCol ) return 1; + return (pNew->iCol==pOld->iCol && sampleIsBetterPost(pAccum, pNew, pOld)); + } + return 0; +} + +/* +** Copy the contents of sample *pNew into the p->a[] array. If necessary, +** remove the least desirable sample from p->a[] to make room. +*/ +static void sampleInsert(StatAccum *p, StatSample *pNew, int nEqZero){ + StatSample *pSample = 0; + int i; + + assert( IsStat4 || nEqZero==0 ); + + /* StatAccum.nMaxEqZero is set to the maximum number of leading 0 + ** values in the anEq[] array of any sample in StatAccum.a[]. In + ** other words, if nMaxEqZero is n, then it is guaranteed that there + ** are no samples with StatSample.anEq[m]==0 for (m>=n). */ + if( nEqZero>p->nMaxEqZero ){ + p->nMaxEqZero = nEqZero; + } + if( pNew->isPSample==0 ){ + StatSample *pUpgrade = 0; + assert( pNew->anEq[pNew->iCol]>0 ); + + /* This sample is being added because the prefix that ends in column + ** iCol occurs many times in the table. However, if we have already + ** added a sample that shares this prefix, there is no need to add + ** this one. Instead, upgrade the priority of the highest priority + ** existing sample that shares this prefix. */ + for(i=p->nSample-1; i>=0; i--){ + StatSample *pOld = &p->a[i]; + if( pOld->anEq[pNew->iCol]==0 ){ + if( pOld->isPSample ) return; + assert( pOld->iCol>pNew->iCol ); + assert( sampleIsBetter(p, pNew, pOld) ); + if( pUpgrade==0 || sampleIsBetter(p, pOld, pUpgrade) ){ + pUpgrade = pOld; + } + } + } + if( pUpgrade ){ + pUpgrade->iCol = pNew->iCol; + pUpgrade->anEq[pUpgrade->iCol] = pNew->anEq[pUpgrade->iCol]; + goto find_new_min; + } + } + + /* If necessary, remove sample iMin to make room for the new sample. */ + if( p->nSample>=p->mxSample ){ + StatSample *pMin = &p->a[p->iMin]; + tRowcnt *anEq = pMin->anEq; + tRowcnt *anLt = pMin->anLt; + tRowcnt *anDLt = pMin->anDLt; + sampleClear(p->db, pMin); + memmove(pMin, &pMin[1], sizeof(p->a[0])*(p->nSample-p->iMin-1)); + pSample = &p->a[p->nSample-1]; + pSample->nRowid = 0; + pSample->anEq = anEq; + pSample->anDLt = anDLt; + pSample->anLt = anLt; + p->nSample = p->mxSample-1; + } + + /* The "rows less-than" for the rowid column must be greater than that + ** for the last sample in the p->a[] array. Otherwise, the samples would + ** be out of order. */ + assert( p->nSample==0 + || pNew->anLt[p->nCol-1] > p->a[p->nSample-1].anLt[p->nCol-1] ); + + /* Insert the new sample */ + pSample = &p->a[p->nSample]; + sampleCopy(p, pSample, pNew); + p->nSample++; + + /* Zero the first nEqZero entries in the anEq[] array. */ + memset(pSample->anEq, 0, sizeof(tRowcnt)*nEqZero); + +find_new_min: + if( p->nSample>=p->mxSample ){ + int iMin = -1; + for(i=0; imxSample; i++){ + if( p->a[i].isPSample ) continue; + if( iMin<0 || sampleIsBetter(p, &p->a[iMin], &p->a[i]) ){ + iMin = i; + } + } + assert( iMin>=0 ); + p->iMin = iMin; + } +} +#endif /* SQLITE_ENABLE_STAT4 */ + +#ifdef SQLITE_ENABLE_STAT4 +/* +** Field iChng of the index being scanned has changed. So at this point +** p->current contains a sample that reflects the previous row of the +** index. The value of anEq[iChng] and subsequent anEq[] elements are +** correct at this point. +*/ +static void samplePushPrevious(StatAccum *p, int iChng){ + int i; + + /* Check if any samples from the aBest[] array should be pushed + ** into IndexSample.a[] at this point. */ + for(i=(p->nCol-2); i>=iChng; i--){ + StatSample *pBest = &p->aBest[i]; + pBest->anEq[i] = p->current.anEq[i]; + if( p->nSamplemxSample || sampleIsBetter(p, pBest, &p->a[p->iMin]) ){ + sampleInsert(p, pBest, i); + } + } + + /* Check that no sample contains an anEq[] entry with an index of + ** p->nMaxEqZero or greater set to zero. */ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=p->nMaxEqZero; jnCol; j++) assert( p->a[i].anEq[j]>0 ); + } + + /* Update the anEq[] fields of any samples already collected. */ + if( iChngnMaxEqZero ){ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=iChng; jnCol; j++){ + if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + } + } + p->nMaxEqZero = iChng; + } +} +#endif /* SQLITE_ENABLE_STAT4 */ + +/* +** Implementation of the stat_push SQL function: stat_push(P,C,R) +** Arguments: +** +** P Pointer to the StatAccum object created by stat_init() +** C Index of left-most column to differ from previous row +** R Rowid for the current row. Might be a key record for +** WITHOUT ROWID tables. +** +** The purpose of this routine is to collect statistical data and/or +** samples from the index being analyzed into the StatAccum object. +** The stat_get() SQL function will be used afterwards to +** retrieve the information gathered. +** +** This SQL function usually returns NULL, but might return an integer +** if it wants the byte-code to do special processing. +** +** The R parameter is only used for STAT4 +*/ +static void statPush( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + + /* The three function arguments */ + StatAccum *p = (StatAccum*)sqlite3_value_blob(argv[0]); + int iChng = sqlite3_value_int(argv[1]); + + UNUSED_PARAMETER( argc ); + UNUSED_PARAMETER( context ); + assert( p->nCol>0 ); + assert( iChngnCol ); + + if( p->nRow==0 ){ + /* This is the first call to this function. Do initialization. */ + for(i=0; inCol; i++) p->current.anEq[i] = 1; + }else{ + /* Second and subsequent calls get processed here */ +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ) samplePushPrevious(p, iChng); +#endif + + /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply + ** to the current row of the index. */ + for(i=0; icurrent.anEq[i]++; + } + for(i=iChng; inCol; i++){ + p->current.anDLt[i]++; +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ) p->current.anLt[i] += p->current.anEq[i]; +#endif + p->current.anEq[i] = 1; + } + } + + p->nRow++; +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ){ + tRowcnt nLt; + if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){ + sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2])); + }else{ + sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]), + sqlite3_value_blob(argv[2])); + } + p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; + + nLt = p->current.anLt[p->nCol-1]; + /* Check if this is to be a periodic sample. If so, add it. */ + if( (nLt/p->nPSample)!=(nLt+1)/p->nPSample ){ + p->current.isPSample = 1; + p->current.iCol = 0; + sampleInsert(p, &p->current, p->nCol-1); + p->current.isPSample = 0; + } + + /* Update the aBest[] array. */ + for(i=0; i<(p->nCol-1); i++){ + p->current.iCol = i; + if( i>=iChng || sampleIsBetterPost(p, &p->current, &p->aBest[i]) ){ + sampleCopy(p, &p->aBest[i], &p->current); + } + } + }else +#endif + if( p->nLimit && p->nRow>(tRowcnt)p->nLimit*(p->nSkipAhead+1) ){ + p->nSkipAhead++; + sqlite3_result_int(context, p->current.anDLt[0]>0); + } +} + +static const FuncDef statPushFuncdef = { + 2+IsStat4, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statPush, /* xSFunc */ + 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ + "stat_push", /* zName */ + {0} +}; + +#define STAT_GET_STAT1 0 /* "stat" column of stat1 table */ +#define STAT_GET_ROWID 1 /* "rowid" column of stat[34] entry */ +#define STAT_GET_NEQ 2 /* "neq" column of stat[34] entry */ +#define STAT_GET_NLT 3 /* "nlt" column of stat[34] entry */ +#define STAT_GET_NDLT 4 /* "ndlt" column of stat[34] entry */ + +/* +** Implementation of the stat_get(P,J) SQL function. This routine is +** used to query statistical information that has been gathered into +** the StatAccum object by prior calls to stat_push(). The P parameter +** has type BLOB but it is really just a pointer to the StatAccum object. +** The content to returned is determined by the parameter J +** which is one of the STAT_GET_xxxx values defined above. +** +** The stat_get(P,J) function is not available to generic SQL. It is +** inserted as part of a manually constructed bytecode program. (See +** the callStatGet() routine below.) It is guaranteed that the P +** parameter will always be a pointer to a StatAccum object, never a +** NULL. +** +** If STAT4 is not enabled, then J is always +** STAT_GET_STAT1 and is hence omitted and this routine becomes +** a one-parameter function, stat_get(P), that always returns the +** stat1 table entry information. +*/ +static void statGet( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + StatAccum *p = (StatAccum*)sqlite3_value_blob(argv[0]); +#ifdef SQLITE_ENABLE_STAT4 + /* STAT4 has a parameter on this routine. */ + int eCall = sqlite3_value_int(argv[1]); + assert( argc==2 ); + assert( eCall==STAT_GET_STAT1 || eCall==STAT_GET_NEQ + || eCall==STAT_GET_ROWID || eCall==STAT_GET_NLT + || eCall==STAT_GET_NDLT + ); + assert( eCall==STAT_GET_STAT1 || p->mxSample ); + if( eCall==STAT_GET_STAT1 ) +#else + assert( argc==1 ); +#endif + { + /* Return the value to store in the "stat" column of the sqlite_stat1 + ** table for this index. + ** + ** The value is a string composed of a list of integers describing + ** the index. The first integer in the list is the total number of + ** entries in the index. There is one additional integer in the list + ** for each indexed column. This additional integer is an estimate of + ** the number of rows matched by a equality query on the index using + ** a key with the corresponding number of fields. In other words, + ** if the index is on columns (a,b) and the sqlite_stat1 value is + ** "100 10 2", then SQLite estimates that: + ** + ** * the index contains 100 rows, + ** * "WHERE a=?" matches 10 rows, and + ** * "WHERE a=? AND b=?" matches 2 rows. + ** + ** If D is the count of distinct values and K is the total number of + ** rows, then each estimate is computed as: + ** + ** I = (K+D-1)/D + */ + sqlite3_str sStat; /* Text of the constructed "stat" line */ + int i; /* Loop counter */ + + sqlite3StrAccumInit(&sStat, 0, 0, 0, (p->nKeyCol+1)*100); + sqlite3_str_appendf(&sStat, "%llu", + p->nSkipAhead ? (u64)p->nEst : (u64)p->nRow); + for(i=0; inKeyCol; i++){ + u64 nDistinct = p->current.anDLt[i] + 1; + u64 iVal = (p->nRow + nDistinct - 1) / nDistinct; + sqlite3_str_appendf(&sStat, " %llu", iVal); + assert( p->current.anEq[i] ); + } + sqlite3ResultStrAccum(context, &sStat); + } +#ifdef SQLITE_ENABLE_STAT4 + else if( eCall==STAT_GET_ROWID ){ + if( p->iGet<0 ){ + samplePushPrevious(p, 0); + p->iGet = 0; + } + if( p->iGetnSample ){ + StatSample *pS = p->a + p->iGet; + if( pS->nRowid==0 ){ + sqlite3_result_int64(context, pS->u.iRowid); + }else{ + sqlite3_result_blob(context, pS->u.aRowid, pS->nRowid, + SQLITE_TRANSIENT); + } + } + }else{ + tRowcnt *aCnt = 0; + sqlite3_str sStat; + int i; + + assert( p->iGetnSample ); + switch( eCall ){ + case STAT_GET_NEQ: aCnt = p->a[p->iGet].anEq; break; + case STAT_GET_NLT: aCnt = p->a[p->iGet].anLt; break; + default: { + aCnt = p->a[p->iGet].anDLt; + p->iGet++; + break; + } + } + sqlite3StrAccumInit(&sStat, 0, 0, 0, p->nCol*100); + for(i=0; inCol; i++){ + sqlite3_str_appendf(&sStat, "%llu ", (u64)aCnt[i]); + } + if( sStat.nChar ) sStat.nChar--; + sqlite3ResultStrAccum(context, &sStat); + } +#endif /* SQLITE_ENABLE_STAT4 */ +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER( argc ); +#endif +} +static const FuncDef statGetFuncdef = { + 1+IsStat4, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + statGet, /* xSFunc */ + 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ + "stat_get", /* zName */ + {0} +}; + +static void callStatGet(Parse *pParse, int regStat, int iParam, int regOut){ +#ifdef SQLITE_ENABLE_STAT4 + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Integer, iParam, regStat+1); +#elif SQLITE_DEBUG + assert( iParam==STAT_GET_STAT1 ); +#else + UNUSED_PARAMETER( iParam ); +#endif + assert( regOut!=regStat && regOut!=regStat+1 ); + sqlite3VdbeAddFunctionCall(pParse, 0, regStat, regOut, 1+IsStat4, + &statGetFuncdef, 0); +} + +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +/* Add a comment to the most recent VDBE opcode that is the name +** of the k-th column of the pIdx index. +*/ +static void analyzeVdbeCommentIndexWithColumnName( + Vdbe *v, /* Prepared statement under construction */ + Index *pIdx, /* Index whose column is being loaded */ + int k /* Which column index */ +){ + int i; /* Index of column in the table */ + assert( k>=0 && knColumn ); + i = pIdx->aiColumn[k]; + if( NEVER(i==XN_ROWID) ){ + VdbeComment((v,"%s.rowid",pIdx->zName)); + }else if( i==XN_EXPR ){ + VdbeComment((v,"%s.expr(%d)",pIdx->zName, k)); + }else{ + VdbeComment((v,"%s.%s", pIdx->zName, pIdx->pTable->aCol[i].zCnName)); + } +} +#else +# define analyzeVdbeCommentIndexWithColumnName(a,b,c) +#endif /* SQLITE_DEBUG */ + +/* +** Generate code to do an analysis of all indices associated with +** a single table. +*/ +static void analyzeOneTable( + Parse *pParse, /* Parser context */ + Table *pTab, /* Table whose indices are to be analyzed */ + Index *pOnlyIdx, /* If not NULL, only analyze this one index */ + int iStatCur, /* Index of VdbeCursor that writes the sqlite_stat1 table */ + int iMem, /* Available memory locations begin here */ + int iTab /* Next available cursor */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + Index *pIdx; /* An index to being analyzed */ + int iIdxCur; /* Cursor open on index being analyzed */ + int iTabCur; /* Table cursor */ + Vdbe *v; /* The virtual machine being built up */ + int i; /* Loop counter */ + int jZeroRows = -1; /* Jump from here if number of rows is zero */ + int iDb; /* Index of database containing pTab */ + u8 needTableCnt = 1; /* True to count the table */ + int regNewRowid = iMem++; /* Rowid for the inserted record */ + int regStat = iMem++; /* Register to hold StatAccum object */ + int regChng = iMem++; /* Index of changed index field */ + int regRowid = iMem++; /* Rowid argument passed to stat_push() */ + int regTemp = iMem++; /* Temporary use register */ + int regTemp2 = iMem++; /* Second temporary use register */ + int regTabname = iMem++; /* Register containing table name */ + int regIdxname = iMem++; /* Register containing index name */ + int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ + int regPrev = iMem; /* MUST BE LAST (see below) */ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + Table *pStat1 = 0; +#endif + + pParse->nMem = MAX(pParse->nMem, iMem); + v = sqlite3GetVdbe(pParse); + if( v==0 || NEVER(pTab==0) ){ + return; + } + if( pTab->tnum==0 ){ + /* Do not gather statistics on views or virtual tables */ + return; + } + if( sqlite3_strlike("sqlite\\_%", pTab->zName, '\\')==0 ){ + /* Do not gather statistics on system tables */ + return; + } + assert( sqlite3BtreeHoldsAllMutexes(db) ); + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 ); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); +#ifndef SQLITE_OMIT_AUTHORIZATION + if( sqlite3AuthCheck(pParse, SQLITE_ANALYZE, pTab->zName, 0, + db->aDb[iDb].zDbSName ) ){ + return; + } +#endif + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( db->xPreUpdateCallback ){ + pStat1 = (Table*)sqlite3DbMallocZero(db, sizeof(Table) + 13); + if( pStat1==0 ) return; + pStat1->zName = (char*)&pStat1[1]; + memcpy(pStat1->zName, "sqlite_stat1", 13); + pStat1->nCol = 3; + pStat1->iPKey = -1; + sqlite3VdbeAddOp4(pParse->pVdbe, OP_Noop, 0, 0, 0,(char*)pStat1,P4_DYNBLOB); + } +#endif + + /* Establish a read-lock on the table at the shared-cache level. + ** Open a read-only cursor on the table. Also allocate a cursor number + ** to use for scanning indexes (iIdxCur). No index cursor is opened at + ** this time though. */ + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + iTabCur = iTab++; + iIdxCur = iTab++; + pParse->nTab = MAX(pParse->nTab, iTab); + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeLoadString(v, regTabname, pTab->zName); + + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int nCol; /* Number of columns in pIdx. "N" */ + int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrNextRow; /* Address of "next_row:" */ + const char *zIdxName; /* Name of the index */ + int nColTest; /* Number of columns to test for changes */ + + if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; + if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIdx) ){ + nCol = pIdx->nKeyCol; + zIdxName = pTab->zName; + nColTest = nCol - 1; + }else{ + nCol = pIdx->nColumn; + zIdxName = pIdx->zName; + nColTest = pIdx->uniqNotNull ? pIdx->nKeyCol-1 : nCol-1; + } + + /* Populate the register containing the index name. */ + sqlite3VdbeLoadString(v, regIdxname, zIdxName); + VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); + + /* + ** Pseudo-code for loop that calls stat_push(): + ** + ** Rewind csr + ** if eof(csr) goto end_of_scan; + ** regChng = 0 + ** goto chng_addr_0; + ** + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto chng_addr_N + ** + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... + ** + ** endDistinctTest: + ** regRowid = idx(rowid) + ** stat_push(P, regChng, regRowid) + ** Next csr + ** if !eof(csr) goto next_row; + ** + ** end_of_scan: + */ + + /* Make sure there are enough memory cells allocated to accommodate + ** the regPrev array and a trailing rowid (the rowid slot is required + ** when building a record to insert into the sample column of + ** the sqlite_stat4 table. */ + pParse->nMem = MAX(pParse->nMem, regPrev+nColTest); + + /* Open a read-only cursor on the index being analyzed. */ + assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); + sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + + /* Invoke the stat_init() function. The arguments are: + ** + ** (1) the number of columns in the index including the rowid + ** (or for a WITHOUT ROWID table, the number of PK columns), + ** (2) the number of columns in the key without the rowid/pk + ** (3) estimated number of rows in the index, + */ + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); + assert( regRowid==regStat+2 ); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); +#ifdef SQLITE_ENABLE_STAT4 + if( OptimizationEnabled(db, SQLITE_Stat4) ){ + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regTemp); + addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); + }else +#endif + { + addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); + } + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, + &statInitFuncdef, 0); + + /* Implementation of the following: + ** + ** Rewind csr + ** if eof(csr) goto end_of_scan; + ** regChng = 0 + ** goto next_push_0; + ** + */ + sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); + addrNextRow = sqlite3VdbeCurrentAddr(v); + + if( nColTest>0 ){ + int endDistinctTest = sqlite3VdbeMakeLabel(pParse); + int *aGotoChng; /* Array of jump instruction addresses */ + aGotoChng = sqlite3DbMallocRawNN(db, sizeof(int)*nColTest); + if( aGotoChng==0 ) continue; + + /* + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto endDistinctTest + */ + sqlite3VdbeAddOp0(v, OP_Goto); + addrNextRow = sqlite3VdbeCurrentAddr(v); + if( nColTest==1 && pIdx->nKeyCol==1 && IsUniqueIndex(pIdx) ){ + /* For a single-column UNIQUE index, once we have found a non-NULL + ** row, we know that all the rest will be distinct, so skip + ** subsequent distinctness tests. */ + sqlite3VdbeAddOp2(v, OP_NotNull, regPrev, endDistinctTest); + VdbeCoverage(v); + } + for(i=0; iazColl[i]); + sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); + analyzeVdbeCommentIndexWithColumnName(v,pIdx,i); + aGotoChng[i] = + sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); + sqlite3VdbeGoto(v, endDistinctTest); + + + /* + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... + */ + sqlite3VdbeJumpHere(v, addrNextRow-1); + for(i=0; ipTable); + int j, k, regKey; + regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); + assert( k>=0 && knColumn ); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); + analyzeVdbeCommentIndexWithColumnName(v,pIdx,k); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); + sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); + } + } +#endif + assert( regChng==(regStat+1) ); + { + sqlite3VdbeAddFunctionCall(pParse, 1, regStat, regTemp, 2+IsStat4, + &statPushFuncdef, 0); + if( db->nAnalysisLimit ){ + int j1, j2, j3; + j1 = sqlite3VdbeAddOp1(v, OP_IsNull, regTemp); VdbeCoverage(v); + j2 = sqlite3VdbeAddOp1(v, OP_If, regTemp); VdbeCoverage(v); + j3 = sqlite3VdbeAddOp4Int(v, OP_SeekGT, iIdxCur, 0, regPrev, 1); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j2); + sqlite3VdbeJumpHere(v, j3); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + } + } + + /* Add the entry to the stat1 table. */ + callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); + assert( "BBB"[0]==SQLITE_AFF_TEXT ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + sqlite3VdbeChangeP4(v, -1, (char*)pStat1, P4_TABLE); +#endif + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + + /* Add the entries to the stat4 table. */ +#ifdef SQLITE_ENABLE_STAT4 + if( OptimizationEnabled(db, SQLITE_Stat4) && db->nAnalysisLimit==0 ){ + int regEq = regStat1; + int regLt = regStat1+1; + int regDLt = regStat1+2; + int regSample = regStat1+3; + int regCol = regStat1+4; + int regSampleRowid = regCol + nCol; + int addrNext; + int addrIsNull; + u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + + pParse->nMem = MAX(pParse->nMem, regCol+nCol); + + addrNext = sqlite3VdbeCurrentAddr(v); + callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid); + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regSampleRowid); + VdbeCoverage(v); + callStatGet(pParse, regStat, STAT_GET_NEQ, regEq); + callStatGet(pParse, regStat, STAT_GET_NLT, regLt); + callStatGet(pParse, regStat, STAT_GET_NDLT, regDLt); + sqlite3VdbeAddOp4Int(v, seekOp, iTabCur, addrNext, regSampleRowid, 0); + VdbeCoverage(v); + for(i=0; izName)); + sqlite3VdbeAddOp2(v, OP_Count, iTabCur, regStat1); + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); + assert( "BBB"[0]==SQLITE_AFF_TEXT ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + sqlite3VdbeChangeP4(v, -1, (char*)pStat1, P4_TABLE); +#endif + sqlite3VdbeJumpHere(v, jZeroRows); + } +} + + +/* +** Generate code that will cause the most recent index analysis to +** be loaded into internal hash tables where is can be used. +*/ +static void loadAnalysis(Parse *pParse, int iDb){ + Vdbe *v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp1(v, OP_LoadAnalysis, iDb); + } +} + +/* +** Generate code that will do an analysis of an entire database +*/ +static void analyzeDatabase(Parse *pParse, int iDb){ + sqlite3 *db = pParse->db; + Schema *pSchema = db->aDb[iDb].pSchema; /* Schema of database iDb */ + HashElem *k; + int iStatCur; + int iMem; + int iTab; + + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 3; + openStatTable(pParse, iDb, iStatCur, 0, 0); + iMem = pParse->nMem+1; + iTab = pParse->nTab; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pTab = (Table*)sqliteHashData(k); + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); + } + loadAnalysis(pParse, iDb); +} + +/* +** Generate code that will do an analysis of a single table in +** a database. If pOnlyIdx is not NULL then it is a single index +** in pTab that should be analyzed. +*/ +static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ + int iDb; + int iStatCur; + + assert( pTab!=0 ); + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3BeginWriteOperation(pParse, 0, iDb); + iStatCur = pParse->nTab; + pParse->nTab += 3; + if( pOnlyIdx ){ + openStatTable(pParse, iDb, iStatCur, pOnlyIdx->zName, "idx"); + }else{ + openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); + } + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1,pParse->nTab); + loadAnalysis(pParse, iDb); +} + +/* +** Generate code for the ANALYZE command. The parser calls this routine +** when it recognizes an ANALYZE command. +** +** ANALYZE -- 1 +** ANALYZE -- 2 +** ANALYZE ?.? -- 3 +** +** Form 1 causes all indices in all attached databases to be analyzed. +** Form 2 analyzes all indices the single database named. +** Form 3 analyzes all indices associated with the named table. +*/ +SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ + sqlite3 *db = pParse->db; + int iDb; + int i; + char *z, *zDb; + Table *pTab; + Index *pIdx; + Token *pTableName; + Vdbe *v; + + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + return; + } + + assert( pName2!=0 || pName1==0 ); + if( pName1==0 ){ + /* Form 1: Analyze everything */ + for(i=0; inDb; i++){ + if( i==1 ) continue; /* Do not analyze the TEMP database */ + analyzeDatabase(pParse, i); + } + }else if( pName2->n==0 && (iDb = sqlite3FindDb(db, pName1))>=0 ){ + /* Analyze the schema named as the argument */ + analyzeDatabase(pParse, iDb); + }else{ + /* Form 3: Analyze the table or index named as an argument */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); + if( iDb>=0 ){ + zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; + z = sqlite3NameFromToken(db, pTableName); + if( z ){ + if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ + analyzeTable(pParse, pIdx->pTable, pIdx); + }else if( (pTab = sqlite3LocateTable(pParse, 0, z, zDb))!=0 ){ + analyzeTable(pParse, pTab, 0); + } + sqlite3DbFree(db, z); + } + } + } + if( db->nSqlExec==0 && (v = sqlite3GetVdbe(pParse))!=0 ){ + sqlite3VdbeAddOp0(v, OP_Expire); + } +} + +/* +** Used to pass information from the analyzer reader through to the +** callback routine. +*/ +typedef struct analysisInfo analysisInfo; +struct analysisInfo { + sqlite3 *db; + const char *zDatabase; +}; + +/* +** The first argument points to a nul-terminated string containing a +** list of space separated integers. Read the first nOut of these into +** the array aOut[]. +*/ +static void decodeIntArray( + char *zIntArray, /* String containing int array to decode */ + int nOut, /* Number of slots in aOut[] */ + tRowcnt *aOut, /* Store integers here */ + LogEst *aLog, /* Or, if aOut==0, here */ + Index *pIndex /* Handle extra flags for this index, if not NULL */ +){ + char *z = zIntArray; + int c; + int i; + tRowcnt v; + +#ifdef SQLITE_ENABLE_STAT4 + if( z==0 ) z = ""; +#else + assert( z!=0 ); +#endif + for(i=0; *z && i='0' && c<='9' ){ + v = v*10 + c - '0'; + z++; + } +#ifdef SQLITE_ENABLE_STAT4 + if( aOut ) aOut[i] = v; + if( aLog ) aLog[i] = sqlite3LogEst(v); +#else + assert( aOut==0 ); + UNUSED_PARAMETER(aOut); + assert( aLog!=0 ); + aLog[i] = sqlite3LogEst(v); +#endif + if( *z==' ' ) z++; + } +#ifndef SQLITE_ENABLE_STAT4 + assert( pIndex!=0 ); { +#else + if( pIndex ){ +#endif + pIndex->bUnordered = 0; + pIndex->noSkipScan = 0; + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ + pIndex->bUnordered = 1; + }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ + int sz = sqlite3Atoi(z+3); + if( sz<2 ) sz = 2; + pIndex->szIdxRow = sqlite3LogEst(sz); + }else if( sqlite3_strglob("noskipscan*", z)==0 ){ + pIndex->noSkipScan = 1; + } +#ifdef SQLITE_ENABLE_COSTMULT + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } +#endif + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; + } + } +} + +/* +** This callback is invoked once for each index when reading the +** sqlite_stat1 table. +** +** argv[0] = name of the table +** argv[1] = name of the index (might be NULL) +** argv[2] = results of analysis - on integer for each column +** +** Entries for which argv[1]==NULL simply record the number of rows in +** the table. +*/ +static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ + analysisInfo *pInfo = (analysisInfo*)pData; + Index *pIndex; + Table *pTable; + const char *z; + + assert( argc==3 ); + UNUSED_PARAMETER2(NotUsed, argc); + + if( argv==0 || argv[0]==0 || argv[2]==0 ){ + return 0; + } + pTable = sqlite3FindTable(pInfo->db, argv[0], pInfo->zDatabase); + if( pTable==0 ){ + return 0; + } + if( argv[1]==0 ){ + pIndex = 0; + }else if( sqlite3_stricmp(argv[0],argv[1])==0 ){ + pIndex = sqlite3PrimaryKeyIndex(pTable); + }else{ + pIndex = sqlite3FindIndex(pInfo->db, argv[1], pInfo->zDatabase); + } + z = argv[2]; + + if( pIndex ){ + tRowcnt *aiRowEst = 0; + int nCol = pIndex->nKeyCol+1; +#ifdef SQLITE_ENABLE_STAT4 + /* Index.aiRowEst may already be set here if there are duplicate + ** sqlite_stat1 entries for this index. In that case just clobber + ** the old data with the new instead of allocating a new array. */ + if( pIndex->aiRowEst==0 ){ + pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero(sizeof(tRowcnt) * nCol); + if( pIndex->aiRowEst==0 ) sqlite3OomFault(pInfo->db); + } + aiRowEst = pIndex->aiRowEst; +#endif + pIndex->bUnordered = 0; + decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); + pIndex->hasStat1 = 1; + if( pIndex->pPartIdxWhere==0 ){ + pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pTable->tabFlags |= TF_HasStat1; + } + }else{ + Index fakeIdx; + fakeIdx.szIdxRow = pTable->szTabRow; +#ifdef SQLITE_ENABLE_COSTMULT + fakeIdx.pTable = pTable; +#endif + decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); + pTable->szTabRow = fakeIdx.szIdxRow; + pTable->tabFlags |= TF_HasStat1; + } + + return 0; +} + +/* +** If the Index.aSample variable is not NULL, delete the aSample[] array +** and its contents. +*/ +SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ +#ifdef SQLITE_ENABLE_STAT4 + if( pIdx->aSample ){ + int j; + for(j=0; jnSample; j++){ + IndexSample *p = &pIdx->aSample[j]; + sqlite3DbFree(db, p->p); + } + sqlite3DbFree(db, pIdx->aSample); + } + if( db && db->pnBytesFreed==0 ){ + pIdx->nSample = 0; + pIdx->aSample = 0; + } +#else + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pIdx); +#endif /* SQLITE_ENABLE_STAT4 */ +} + +#ifdef SQLITE_ENABLE_STAT4 +/* +** Populate the pIdx->aAvgEq[] array based on the samples currently +** stored in pIdx->aSample[]. +*/ +static void initAvgEq(Index *pIdx){ + if( pIdx ){ + IndexSample *aSample = pIdx->aSample; + IndexSample *pFinal = &aSample[pIdx->nSample-1]; + int iCol; + int nCol = 1; + if( pIdx->nSampleCol>1 ){ + /* If this is stat4 data, then calculate aAvgEq[] values for all + ** sample columns except the last. The last is always set to 1, as + ** once the trailing PK fields are considered all index keys are + ** unique. */ + nCol = pIdx->nSampleCol-1; + pIdx->aAvgEq[nCol] = 1; + } + for(iCol=0; iColnSample; + int i; /* Used to iterate through samples */ + tRowcnt sumEq = 0; /* Sum of the nEq values */ + tRowcnt avgEq = 0; + tRowcnt nRow; /* Number of rows in index */ + i64 nSum100 = 0; /* Number of terms contributing to sumEq */ + i64 nDist100; /* Number of distinct values in index */ + + if( !pIdx->aiRowEst || iCol>=pIdx->nKeyCol || pIdx->aiRowEst[iCol+1]==0 ){ + nRow = pFinal->anLt[iCol]; + nDist100 = (i64)100 * pFinal->anDLt[iCol]; + nSample--; + }else{ + nRow = pIdx->aiRowEst[0]; + nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1]; + } + pIdx->nRowEst0 = nRow; + + /* Set nSum to the number of distinct (iCol+1) field prefixes that + ** occur in the stat4 table for this index. Set sumEq to the sum of + ** the nEq values for column iCol for the same set (adding the value + ** only once where there exist duplicate prefixes). */ + for(i=0; inSample-1) + || aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] + ){ + sumEq += aSample[i].anEq[iCol]; + nSum100 += 100; + } + } + + if( nDist100>nSum100 && sumEqaAvgEq[iCol] = avgEq; + } + } +} + +/* +** Look up an index by name. Or, if the name of a WITHOUT ROWID table +** is supplied instead, find the PRIMARY KEY index for that table. +*/ +static Index *findIndexOrPrimaryKey( + sqlite3 *db, + const char *zName, + const char *zDb +){ + Index *pIdx = sqlite3FindIndex(db, zName, zDb); + if( pIdx==0 ){ + Table *pTab = sqlite3FindTable(db, zName, zDb); + if( pTab && !HasRowid(pTab) ) pIdx = sqlite3PrimaryKeyIndex(pTab); + } + return pIdx; +} + +/* +** Load the content from either the sqlite_stat4 +** into the relevant Index.aSample[] arrays. +** +** Arguments zSql1 and zSql2 must point to SQL statements that return +** data equivalent to the following: +** +** zSql1: SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx +** zSql2: SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4 +** +** where %Q is replaced with the database name before the SQL is executed. +*/ +static int loadStatTbl( + sqlite3 *db, /* Database handle */ + const char *zSql1, /* SQL statement 1 (see above) */ + const char *zSql2, /* SQL statement 2 (see above) */ + const char *zDb /* Database name (e.g. "main") */ +){ + int rc; /* Result codes from subroutines */ + sqlite3_stmt *pStmt = 0; /* An SQL statement being run */ + char *zSql; /* Text of the SQL statement */ + Index *pPrevIdx = 0; /* Previous index in the loop */ + IndexSample *pSample; /* A slot in pIdx->aSample[] */ + + assert( db->lookaside.bDisable ); + zSql = sqlite3MPrintf(db, zSql1, zDb); + if( !zSql ){ + return SQLITE_NOMEM_BKPT; + } + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + sqlite3DbFree(db, zSql); + if( rc ) return rc; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int nIdxCol = 1; /* Number of columns in stat4 records */ + + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nSample; /* Number of samples */ + int nByte; /* Bytes of space required */ + int i; /* Bytes of space required */ + tRowcnt *pSpace; + + zIndex = (char *)sqlite3_column_text(pStmt, 0); + if( zIndex==0 ) continue; + nSample = sqlite3_column_int(pStmt, 1); + pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); + assert( pIdx==0 || pIdx->nSample==0 ); + if( pIdx==0 ) continue; + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; + } + pIdx->nSampleCol = nIdxCol; + nByte = sizeof(IndexSample) * nSample; + nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + + pIdx->aSample = sqlite3DbMallocZero(db, nByte); + if( pIdx->aSample==0 ){ + sqlite3_finalize(pStmt); + return SQLITE_NOMEM_BKPT; + } + pSpace = (tRowcnt*)&pIdx->aSample[nSample]; + pIdx->aAvgEq = pSpace; pSpace += nIdxCol; + pIdx->pTable->tabFlags |= TF_HasStat4; + for(i=0; iaSample[i].anEq = pSpace; pSpace += nIdxCol; + pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; + pIdx->aSample[i].anDLt = pSpace; pSpace += nIdxCol; + } + assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) ); + } + rc = sqlite3_finalize(pStmt); + if( rc ) return rc; + + zSql = sqlite3MPrintf(db, zSql2, zDb); + if( !zSql ){ + return SQLITE_NOMEM_BKPT; + } + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + sqlite3DbFree(db, zSql); + if( rc ) return rc; + + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int nCol = 1; /* Number of columns in index */ + + zIndex = (char *)sqlite3_column_text(pStmt, 0); + if( zIndex==0 ) continue; + pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); + if( pIdx==0 ) continue; + /* This next condition is true if data has already been loaded from + ** the sqlite_stat4 table. */ + nCol = pIdx->nSampleCol; + if( pIdx!=pPrevIdx ){ + initAvgEq(pPrevIdx); + pPrevIdx = pIdx; + } + pSample = &pIdx->aSample[pIdx->nSample]; + decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); + + /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. + ** This is in case the sample record is corrupted. In that case, the + ** sqlite3VdbeRecordCompare() may read up to two varints past the + ** end of the allocated buffer before it realizes it is dealing with + ** a corrupt record. Adding the two 0x00 bytes prevents this from causing + ** a buffer overread. */ + pSample->n = sqlite3_column_bytes(pStmt, 4); + pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); + if( pSample->p==0 ){ + sqlite3_finalize(pStmt); + return SQLITE_NOMEM_BKPT; + } + if( pSample->n ){ + memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); + } + pIdx->nSample++; + } + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) initAvgEq(pPrevIdx); + return rc; +} + +/* +** Load content from the sqlite_stat4 table into +** the Index.aSample[] arrays of all indices. +*/ +static int loadStat4(sqlite3 *db, const char *zDb){ + int rc = SQLITE_OK; /* Result codes from subroutines */ + const Table *pStat4; + + assert( db->lookaside.bDisable ); + if( (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0 + && IsOrdinaryTable(pStat4) + ){ + rc = loadStatTbl(db, + "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", + "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", + zDb + ); + } + return rc; +} +#endif /* SQLITE_ENABLE_STAT4 */ + +/* +** Load the content of the sqlite_stat1 and sqlite_stat4 tables. The +** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] +** arrays. The contents of sqlite_stat4 are used to populate the +** Index.aSample[] arrays. +** +** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR +** is returned. In this case, even if SQLITE_ENABLE_STAT4 was defined +** during compilation and the sqlite_stat4 table is present, no data is +** read from it. +** +** If SQLITE_ENABLE_STAT4 was defined during compilation and the +** sqlite_stat4 table is not present in the database, SQLITE_ERROR is +** returned. However, in this case, data is read from the sqlite_stat1 +** table (if it is present) before returning. +** +** If an OOM error occurs, this function always sets db->mallocFailed. +** This means if the caller does not care about other errors, the return +** code may be ignored. +*/ +SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ + analysisInfo sInfo; + HashElem *i; + char *zSql; + int rc = SQLITE_OK; + Schema *pSchema = db->aDb[iDb].pSchema; + const Table *pStat1; + + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pBt!=0 ); + + /* Clear any prior statistics */ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(i=sqliteHashFirst(&pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + pTab->tabFlags &= ~TF_HasStat1; + } + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + pIdx->hasStat1 = 0; +#ifdef SQLITE_ENABLE_STAT4 + sqlite3DeleteIndexSamples(db, pIdx); + pIdx->aSample = 0; +#endif + } + + /* Load new statistics out of the sqlite_stat1 table */ + sInfo.db = db; + sInfo.zDatabase = db->aDb[iDb].zDbSName; + if( (pStat1 = sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)) + && IsOrdinaryTable(pStat1) + ){ + zSql = sqlite3MPrintf(db, + "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); + if( zSql==0 ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0); + sqlite3DbFree(db, zSql); + } + } + + /* Set appropriate defaults on all indexes not in the sqlite_stat1 table */ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + if( !pIdx->hasStat1 ) sqlite3DefaultRowEst(pIdx); + } + + /* Load the statistics from the sqlite_stat4 table. */ +#ifdef SQLITE_ENABLE_STAT4 + if( rc==SQLITE_OK ){ + DisableLookaside; + rc = loadStat4(db, sInfo.zDatabase); + EnableLookaside; + } + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3_free(pIdx->aiRowEst); + pIdx->aiRowEst = 0; + } +#endif + + if( rc==SQLITE_NOMEM ){ + sqlite3OomFault(db); + } + return rc; +} + + +#endif /* SQLITE_OMIT_ANALYZE */ + +/************** End of analyze.c *********************************************/ +/************** Begin file attach.c ******************************************/ +/* +** 2003 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the ATTACH and DETACH commands. +*/ +/* #include "sqliteInt.h" */ + +#ifndef SQLITE_OMIT_ATTACH +/* +** Resolve an expression that was part of an ATTACH or DETACH statement. This +** is slightly different from resolving a normal SQL expression, because simple +** identifiers are treated as strings, not possible column names or aliases. +** +** i.e. if the parser sees: +** +** ATTACH DATABASE abc AS def +** +** it treats the two expressions as literal strings 'abc' and 'def' instead of +** looking for columns of the same name. +** +** This only applies to the root node of pExpr, so the statement: +** +** ATTACH DATABASE abc||def AS 'db2' +** +** will fail because neither abc or def can be resolved. +*/ +static int resolveAttachExpr(NameContext *pName, Expr *pExpr) +{ + int rc = SQLITE_OK; + if( pExpr ){ + if( pExpr->op!=TK_ID ){ + rc = sqlite3ResolveExprNames(pName, pExpr); + }else{ + pExpr->op = TK_STRING; + } + } + return rc; +} + +/* +** Return true if zName points to a name that may be used to refer to +** database iDb attached to handle db. +*/ +SQLITE_PRIVATE int sqlite3DbIsNamed(sqlite3 *db, int iDb, const char *zName){ + return ( + sqlite3StrICmp(db->aDb[iDb].zDbSName, zName)==0 + || (iDb==0 && sqlite3StrICmp("main", zName)==0) + ); +} + +/* +** An SQL user-function registered to do the work of an ATTACH statement. The +** three arguments to the function come directly from an attach statement: +** +** ATTACH DATABASE x AS y KEY z +** +** SELECT sqlite_attach(x, y, z) +** +** If the optional "KEY z" syntax is omitted, an SQL NULL is passed as the +** third argument. +** +** If the db->init.reopenMemdb flags is set, then instead of attaching a +** new database, close the database on db->init.iDb and reopen it as an +** empty MemDB. +*/ +static void attachFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + int i; + int rc = 0; + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zName; + const char *zFile; + char *zPath = 0; + char *zErr = 0; + unsigned int flags; + Db *aNew; /* New array of Db pointers */ + Db *pNew; /* Db object for the newly attached database */ + char *zErrDyn = 0; + sqlite3_vfs *pVfs; + + UNUSED_PARAMETER(NotUsed); + zFile = (const char *)sqlite3_value_text(argv[0]); + zName = (const char *)sqlite3_value_text(argv[1]); + if( zFile==0 ) zFile = ""; + if( zName==0 ) zName = ""; + +#ifndef SQLITE_OMIT_DESERIALIZE +# define REOPEN_AS_MEMDB(db) (db->init.reopenMemdb) +#else +# define REOPEN_AS_MEMDB(db) (0) +#endif + + if( REOPEN_AS_MEMDB(db) ){ + /* This is not a real ATTACH. Instead, this routine is being called + ** from sqlite3_deserialize() to close database db->init.iDb and + ** reopen it as a MemDB */ + pVfs = sqlite3_vfs_find("memdb"); + if( pVfs==0 ) return; + pNew = &db->aDb[db->init.iDb]; + if( pNew->pBt ) sqlite3BtreeClose(pNew->pBt); + pNew->pBt = 0; + pNew->pSchema = 0; + rc = sqlite3BtreeOpen(pVfs, "x\0", db, &pNew->pBt, 0, SQLITE_OPEN_MAIN_DB); + }else{ + /* This is a real ATTACH + ** + ** Check for the following errors: + ** + ** * Too many attached databases, + ** * Transaction currently open + ** * Specified database name already being used. + */ + if( db->nDb>=db->aLimit[SQLITE_LIMIT_ATTACHED]+2 ){ + zErrDyn = sqlite3MPrintf(db, "too many attached databases - max %d", + db->aLimit[SQLITE_LIMIT_ATTACHED] + ); + goto attach_error; + } + for(i=0; inDb; i++){ + assert( zName ); + if( sqlite3DbIsNamed(db, i, zName) ){ + zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); + goto attach_error; + } + } + + /* Allocate the new entry in the db->aDb[] array and initialize the schema + ** hash tables. + */ + if( db->aDb==db->aDbStatic ){ + aNew = sqlite3DbMallocRawNN(db, sizeof(db->aDb[0])*3 ); + if( aNew==0 ) return; + memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); + }else{ + aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); + if( aNew==0 ) return; + } + db->aDb = aNew; + pNew = &db->aDb[db->nDb]; + memset(pNew, 0, sizeof(*pNew)); + + /* Open the database file. If the btree is successfully opened, use + ** it to obtain the database schema. At this point the schema may + ** or may not be initialized. + */ + flags = db->openFlags; + rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + assert( pVfs ); + flags |= SQLITE_OPEN_MAIN_DB; + rc = sqlite3BtreeOpen(pVfs, zPath, db, &pNew->pBt, 0, flags); + db->nDb++; + pNew->zDbSName = sqlite3DbStrDup(db, zName); + } + db->noSharedCache = 0; + if( rc==SQLITE_CONSTRAINT ){ + rc = SQLITE_ERROR; + zErrDyn = sqlite3MPrintf(db, "database is already attached"); + }else if( rc==SQLITE_OK ){ + Pager *pPager; + pNew->pSchema = sqlite3SchemaGet(db, pNew->pBt); + if( !pNew->pSchema ){ + rc = SQLITE_NOMEM_BKPT; + }else if( pNew->pSchema->file_format && pNew->pSchema->enc!=ENC(db) ){ + zErrDyn = sqlite3MPrintf(db, + "attached databases must use the same text encoding as main database"); + rc = SQLITE_ERROR; + } + sqlite3BtreeEnter(pNew->pBt); + pPager = sqlite3BtreePager(pNew->pBt); + sqlite3PagerLockingMode(pPager, db->dfltLockMode); + sqlite3BtreeSecureDelete(pNew->pBt, + sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); +#ifndef SQLITE_OMIT_PAGER_PRAGMAS + sqlite3BtreeSetPagerFlags(pNew->pBt, + PAGER_SYNCHRONOUS_FULL | (db->flags & PAGER_FLAGS_MASK)); +#endif + sqlite3BtreeLeave(pNew->pBt); + } + pNew->safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; + if( rc==SQLITE_OK && pNew->zDbSName==0 ){ + rc = SQLITE_NOMEM_BKPT; + } + sqlite3_free_filename( zPath ); + + /* If the file was opened successfully, read the schema for the new database. + ** If this fails, or if opening the file failed, then close the file and + ** remove the entry from the db->aDb[] array. i.e. put everything back the + ** way we found it. + */ + if( rc==SQLITE_OK ){ + sqlite3BtreeEnterAll(db); + db->init.iDb = 0; + db->mDbFlags &= ~(DBFLAG_SchemaKnownOk); + if( !REOPEN_AS_MEMDB(db) ){ + rc = sqlite3Init(db, &zErrDyn); + } + sqlite3BtreeLeaveAll(db); + assert( zErrDyn==0 || rc!=SQLITE_OK ); + } +#ifdef SQLITE_USER_AUTHENTICATION + if( rc==SQLITE_OK && !REOPEN_AS_MEMDB(db) ){ + u8 newAuth = 0; + rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); + if( newAuthauth.authLevel ){ + rc = SQLITE_AUTH_USER; + } + } +#endif + if( rc ){ + if( !REOPEN_AS_MEMDB(db) ){ + int iDb = db->nDb - 1; + assert( iDb>=2 ); + if( db->aDb[iDb].pBt ){ + sqlite3BtreeClose(db->aDb[iDb].pBt); + db->aDb[iDb].pBt = 0; + db->aDb[iDb].pSchema = 0; + } + sqlite3ResetAllSchemasOfConnection(db); + db->nDb = iDb; + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomFault(db); + sqlite3DbFree(db, zErrDyn); + zErrDyn = sqlite3MPrintf(db, "out of memory"); + }else if( zErrDyn==0 ){ + zErrDyn = sqlite3MPrintf(db, "unable to open database: %s", zFile); + } + } + goto attach_error; + } + + return; + +attach_error: + /* Return an error if we get here */ + if( zErrDyn ){ + sqlite3_result_error(context, zErrDyn, -1); + sqlite3DbFree(db, zErrDyn); + } + if( rc ) sqlite3_result_error_code(context, rc); +} + +/* +** An SQL user-function registered to do the work of an DETACH statement. The +** three arguments to the function come directly from a detach statement: +** +** DETACH DATABASE x +** +** SELECT sqlite_detach(x) +*/ +static void detachFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + const char *zName = (const char *)sqlite3_value_text(argv[0]); + sqlite3 *db = sqlite3_context_db_handle(context); + int i; + Db *pDb = 0; + HashElem *pEntry; + char zErr[128]; + + UNUSED_PARAMETER(NotUsed); + + if( zName==0 ) zName = ""; + for(i=0; inDb; i++){ + pDb = &db->aDb[i]; + if( pDb->pBt==0 ) continue; + if( sqlite3DbIsNamed(db, i, zName) ) break; + } + + if( i>=db->nDb ){ + sqlite3_snprintf(sizeof(zErr),zErr, "no such database: %s", zName); + goto detach_error; + } + if( i<2 ){ + sqlite3_snprintf(sizeof(zErr),zErr, "cannot detach database %s", zName); + goto detach_error; + } + if( sqlite3BtreeTxnState(pDb->pBt)!=SQLITE_TXN_NONE + || sqlite3BtreeIsInBackup(pDb->pBt) + ){ + sqlite3_snprintf(sizeof(zErr),zErr, "database %s is locked", zName); + goto detach_error; + } + + /* If any TEMP triggers reference the schema being detached, move those + ** triggers to reference the TEMP schema itself. */ + assert( db->aDb[1].pSchema ); + pEntry = sqliteHashFirst(&db->aDb[1].pSchema->trigHash); + while( pEntry ){ + Trigger *pTrig = (Trigger*)sqliteHashData(pEntry); + if( pTrig->pTabSchema==pDb->pSchema ){ + pTrig->pTabSchema = pTrig->pSchema; + } + pEntry = sqliteHashNext(pEntry); + } + + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + pDb->pSchema = 0; + sqlite3CollapseDatabaseArray(db); + return; + +detach_error: + sqlite3_result_error(context, zErr, -1); +} + +/* +** This procedure generates VDBE code for a single invocation of either the +** sqlite_detach() or sqlite_attach() SQL user functions. +*/ +static void codeAttach( + Parse *pParse, /* The parser context */ + int type, /* Either SQLITE_ATTACH or SQLITE_DETACH */ + FuncDef const *pFunc,/* FuncDef wrapper for detachFunc() or attachFunc() */ + Expr *pAuthArg, /* Expression to pass to authorization callback */ + Expr *pFilename, /* Name of database file */ + Expr *pDbname, /* Name of the database to use internally */ + Expr *pKey /* Database key for encryption extension */ +){ + int rc; + NameContext sName; + Vdbe *v; + sqlite3* db = pParse->db; + int regArgs; + + if( pParse->nErr ) goto attach_end; + memset(&sName, 0, sizeof(NameContext)); + sName.pParse = pParse; + + if( + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pFilename)) || + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pDbname)) || + SQLITE_OK!=(rc = resolveAttachExpr(&sName, pKey)) + ){ + goto attach_end; + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pAuthArg ){ + char *zAuthArg; + if( pAuthArg->op==TK_STRING ){ + zAuthArg = pAuthArg->u.zToken; + }else{ + zAuthArg = 0; + } + rc = sqlite3AuthCheck(pParse, type, zAuthArg, 0, 0); + if(rc!=SQLITE_OK ){ + goto attach_end; + } + } +#endif /* SQLITE_OMIT_AUTHORIZATION */ + + + v = sqlite3GetVdbe(pParse); + regArgs = sqlite3GetTempRange(pParse, 4); + sqlite3ExprCode(pParse, pFilename, regArgs); + sqlite3ExprCode(pParse, pDbname, regArgs+1); + sqlite3ExprCode(pParse, pKey, regArgs+2); + + assert( v || db->mallocFailed ); + if( v ){ + sqlite3VdbeAddFunctionCall(pParse, 0, regArgs+3-pFunc->nArg, regArgs+3, + pFunc->nArg, pFunc, 0); + /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this + ** statement only). For DETACH, set it to false (expire all existing + ** statements). + */ + sqlite3VdbeAddOp1(v, OP_Expire, (type==SQLITE_ATTACH)); + } + +attach_end: + sqlite3ExprDelete(db, pFilename); + sqlite3ExprDelete(db, pDbname); + sqlite3ExprDelete(db, pKey); +} + +/* +** Called by the parser to compile a DETACH statement. +** +** DETACH pDbname +*/ +SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ + static const FuncDef detach_func = { + 1, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + detachFunc, /* xSFunc */ + 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ + "sqlite_detach", /* zName */ + {0} + }; + codeAttach(pParse, SQLITE_DETACH, &detach_func, pDbname, 0, 0, pDbname); +} + +/* +** Called by the parser to compile an ATTACH statement. +** +** ATTACH p AS pDbname KEY pKey +*/ +SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *pKey){ + static const FuncDef attach_func = { + 3, /* nArg */ + SQLITE_UTF8, /* funcFlags */ + 0, /* pUserData */ + 0, /* pNext */ + attachFunc, /* xSFunc */ + 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ + "sqlite_attach", /* zName */ + {0} + }; + codeAttach(pParse, SQLITE_ATTACH, &attach_func, p, p, pDbname, pKey); +} +#endif /* SQLITE_OMIT_ATTACH */ + +/* +** Expression callback used by sqlite3FixAAAA() routines. +*/ +static int fixExprCb(Walker *p, Expr *pExpr){ + DbFixer *pFix = p->u.pFix; + if( !pFix->bTemp ) ExprSetProperty(pExpr, EP_FromDDL); + if( pExpr->op==TK_VARIABLE ){ + if( pFix->pParse->db->init.busy ){ + pExpr->op = TK_NULL; + }else{ + sqlite3ErrorMsg(pFix->pParse, "%s cannot use variables", pFix->zType); + return WRC_Abort; + } + } + return WRC_Continue; +} + +/* +** Select callback used by sqlite3FixAAAA() routines. +*/ +static int fixSelectCb(Walker *p, Select *pSelect){ + DbFixer *pFix = p->u.pFix; + int i; + SrcItem *pItem; + sqlite3 *db = pFix->pParse->db; + int iDb = sqlite3FindDbName(db, pFix->zDb); + SrcList *pList = pSelect->pSrc; + + if( NEVER(pList==0) ) return WRC_Continue; + for(i=0, pItem=pList->a; inSrc; i++, pItem++){ + if( pFix->bTemp==0 ){ + if( pItem->zDatabase ){ + if( iDb!=sqlite3FindDbName(db, pItem->zDatabase) ){ + sqlite3ErrorMsg(pFix->pParse, + "%s %T cannot reference objects in database %s", + pFix->zType, pFix->pName, pItem->zDatabase); + return WRC_Abort; + } + sqlite3DbFree(db, pItem->zDatabase); + pItem->zDatabase = 0; + pItem->fg.notCte = 1; + } + pItem->pSchema = pFix->pSchema; + pItem->fg.fromDDL = 1; + } +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) + if( sqlite3WalkExpr(&pFix->w, pList->a[i].pOn) ) return WRC_Abort; +#endif + } + if( pSelect->pWith ){ + for(i=0; ipWith->nCte; i++){ + if( sqlite3WalkSelect(p, pSelect->pWith->a[i].pSelect) ){ + return WRC_Abort; + } + } + } + return WRC_Continue; +} + +/* +** Initialize a DbFixer structure. This routine must be called prior +** to passing the structure to one of the sqliteFixAAAA() routines below. +*/ +SQLITE_PRIVATE void sqlite3FixInit( + DbFixer *pFix, /* The fixer to be initialized */ + Parse *pParse, /* Error messages will be written here */ + int iDb, /* This is the database that must be used */ + const char *zType, /* "view", "trigger", or "index" */ + const Token *pName /* Name of the view, trigger, or index */ +){ + sqlite3 *db = pParse->db; + assert( db->nDb>iDb ); + pFix->pParse = pParse; + pFix->zDb = db->aDb[iDb].zDbSName; + pFix->pSchema = db->aDb[iDb].pSchema; + pFix->zType = zType; + pFix->pName = pName; + pFix->bTemp = (iDb==1); + pFix->w.pParse = pParse; + pFix->w.xExprCallback = fixExprCb; + pFix->w.xSelectCallback = fixSelectCb; + pFix->w.xSelectCallback2 = sqlite3WalkWinDefnDummyCallback; + pFix->w.walkerDepth = 0; + pFix->w.eCode = 0; + pFix->w.u.pFix = pFix; +} + +/* +** The following set of routines walk through the parse tree and assign +** a specific database to all table references where the database name +** was left unspecified in the original SQL statement. The pFix structure +** must have been initialized by a prior call to sqlite3FixInit(). +** +** These routines are used to make sure that an index, trigger, or +** view in one database does not refer to objects in a different database. +** (Exception: indices, triggers, and views in the TEMP database are +** allowed to refer to anything.) If a reference is explicitly made +** to an object in a different database, an error message is added to +** pParse->zErrMsg and these routines return non-zero. If everything +** checks out, these routines return 0. +*/ +SQLITE_PRIVATE int sqlite3FixSrcList( + DbFixer *pFix, /* Context of the fixation */ + SrcList *pList /* The Source list to check and modify */ +){ + int res = 0; + if( pList ){ + Select s; + memset(&s, 0, sizeof(s)); + s.pSrc = pList; + res = sqlite3WalkSelect(&pFix->w, &s); + } + return res; +} +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) +SQLITE_PRIVATE int sqlite3FixSelect( + DbFixer *pFix, /* Context of the fixation */ + Select *pSelect /* The SELECT statement to be fixed to one database */ +){ + return sqlite3WalkSelect(&pFix->w, pSelect); +} +SQLITE_PRIVATE int sqlite3FixExpr( + DbFixer *pFix, /* Context of the fixation */ + Expr *pExpr /* The expression to be fixed to one database */ +){ + return sqlite3WalkExpr(&pFix->w, pExpr); +} +#endif + +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE int sqlite3FixTriggerStep( + DbFixer *pFix, /* Context of the fixation */ + TriggerStep *pStep /* The trigger step be fixed to one database */ +){ + while( pStep ){ + if( sqlite3WalkSelect(&pFix->w, pStep->pSelect) + || sqlite3WalkExpr(&pFix->w, pStep->pWhere) + || sqlite3WalkExprList(&pFix->w, pStep->pExprList) + || sqlite3FixSrcList(pFix, pStep->pFrom) + ){ + return 1; + } +#ifndef SQLITE_OMIT_UPSERT + { + Upsert *pUp; + for(pUp=pStep->pUpsert; pUp; pUp=pUp->pNextUpsert){ + if( sqlite3WalkExprList(&pFix->w, pUp->pUpsertTarget) + || sqlite3WalkExpr(&pFix->w, pUp->pUpsertTargetWhere) + || sqlite3WalkExprList(&pFix->w, pUp->pUpsertSet) + || sqlite3WalkExpr(&pFix->w, pUp->pUpsertWhere) + ){ + return 1; + } + } + } +#endif + pStep = pStep->pNext; + } + + return 0; +} +#endif + +/************** End of attach.c **********************************************/ +/************** Begin file auth.c ********************************************/ +/* +** 2003 January 11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the sqlite3_set_authorizer() +** API. This facility is an optional feature of the library. Embedded +** systems that do not need this facility may omit it by recompiling +** the library with -DSQLITE_OMIT_AUTHORIZATION=1 +*/ +/* #include "sqliteInt.h" */ + +/* +** All of the code in this file may be omitted by defining a single +** macro. +*/ +#ifndef SQLITE_OMIT_AUTHORIZATION + +/* +** Set or clear the access authorization function. +** +** The access authorization function is be called during the compilation +** phase to verify that the user has read and/or write access permission on +** various fields of the database. The first argument to the auth function +** is a copy of the 3rd argument to this routine. The second argument +** to the auth function is one of these constants: +** +** SQLITE_CREATE_INDEX +** SQLITE_CREATE_TABLE +** SQLITE_CREATE_TEMP_INDEX +** SQLITE_CREATE_TEMP_TABLE +** SQLITE_CREATE_TEMP_TRIGGER +** SQLITE_CREATE_TEMP_VIEW +** SQLITE_CREATE_TRIGGER +** SQLITE_CREATE_VIEW +** SQLITE_DELETE +** SQLITE_DROP_INDEX +** SQLITE_DROP_TABLE +** SQLITE_DROP_TEMP_INDEX +** SQLITE_DROP_TEMP_TABLE +** SQLITE_DROP_TEMP_TRIGGER +** SQLITE_DROP_TEMP_VIEW +** SQLITE_DROP_TRIGGER +** SQLITE_DROP_VIEW +** SQLITE_INSERT +** SQLITE_PRAGMA +** SQLITE_READ +** SQLITE_SELECT +** SQLITE_TRANSACTION +** SQLITE_UPDATE +** +** The third and fourth arguments to the auth function are the name of +** the table and the column that are being accessed. The auth function +** should return either SQLITE_OK, SQLITE_DENY, or SQLITE_IGNORE. If +** SQLITE_OK is returned, it means that access is allowed. SQLITE_DENY +** means that the SQL statement will never-run - the sqlite3_exec() call +** will return with an error. SQLITE_IGNORE means that the SQL statement +** should run but attempts to read the specified column will return NULL +** and attempts to write the column will be ignored. +** +** Setting the auth function to NULL disables this hook. The default +** setting of the auth function is NULL. +*/ +SQLITE_API int sqlite3_set_authorizer( + sqlite3 *db, + int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xAuth = (sqlite3_xauth)xAuth; + db->pAuthArg = pArg; + if( db->xAuth ) sqlite3ExpirePreparedStatements(db, 1); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Write an error message into pParse->zErrMsg that explains that the +** user-supplied authorization function returned an illegal value. +*/ +static void sqliteAuthBadReturnCode(Parse *pParse){ + sqlite3ErrorMsg(pParse, "authorizer malfunction"); + pParse->rc = SQLITE_ERROR; +} + +/* +** Invoke the authorization callback for permission to read column zCol from +** table zTab in database zDb. This function assumes that an authorization +** callback has been registered (i.e. that sqlite3.xAuth is not NULL). +** +** If SQLITE_IGNORE is returned and pExpr is not NULL, then pExpr is changed +** to an SQL NULL expression. Otherwise, if pExpr is NULL, then SQLITE_IGNORE +** is treated as SQLITE_DENY. In this case an error is left in pParse. +*/ +SQLITE_PRIVATE int sqlite3AuthReadCol( + Parse *pParse, /* The parser context */ + const char *zTab, /* Table name */ + const char *zCol, /* Column name */ + int iDb /* Index of containing database. */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + char *zDb = db->aDb[iDb].zDbSName; /* Schema name of attached database */ + int rc; /* Auth callback return code */ + + if( db->init.busy ) return SQLITE_OK; + rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); + if( rc==SQLITE_DENY ){ + char *z = sqlite3_mprintf("%s.%s", zTab, zCol); + if( db->nDb>2 || iDb!=0 ) z = sqlite3_mprintf("%s.%z", zDb, z); + sqlite3ErrorMsg(pParse, "access to %z is prohibited", z); + pParse->rc = SQLITE_AUTH; + }else if( rc!=SQLITE_IGNORE && rc!=SQLITE_OK ){ + sqliteAuthBadReturnCode(pParse); + } + return rc; +} + +/* +** The pExpr should be a TK_COLUMN expression. The table referred to +** is in pTabList or else it is the NEW or OLD table of a trigger. +** Check to see if it is OK to read this particular column. +** +** If the auth function returns SQLITE_IGNORE, change the TK_COLUMN +** instruction into a TK_NULL. If the auth function returns SQLITE_DENY, +** then generate an error. +*/ +SQLITE_PRIVATE void sqlite3AuthRead( + Parse *pParse, /* The parser context */ + Expr *pExpr, /* The expression to check authorization on */ + Schema *pSchema, /* The schema of the expression */ + SrcList *pTabList /* All table that pExpr might refer to */ +){ + Table *pTab = 0; /* The table being read */ + const char *zCol; /* Name of the column of the table */ + int iSrc; /* Index in pTabList->a[] of table being read */ + int iDb; /* The index of the database the expression refers to */ + int iCol; /* Index of column in table */ + + assert( pExpr->op==TK_COLUMN || pExpr->op==TK_TRIGGER ); + assert( !IN_RENAME_OBJECT ); + assert( pParse->db->xAuth!=0 ); + iDb = sqlite3SchemaToIndex(pParse->db, pSchema); + if( iDb<0 ){ + /* An attempt to read a column out of a subquery or other + ** temporary table. */ + return; + } + + if( pExpr->op==TK_TRIGGER ){ + pTab = pParse->pTriggerTab; + }else{ + assert( pTabList ); + for(iSrc=0; iSrcnSrc; iSrc++){ + if( pExpr->iTable==pTabList->a[iSrc].iCursor ){ + pTab = pTabList->a[iSrc].pTab; + break; + } + } + } + iCol = pExpr->iColumn; + if( pTab==0 ) return; + + if( iCol>=0 ){ + assert( iColnCol ); + zCol = pTab->aCol[iCol].zCnName; + }else if( pTab->iPKey>=0 ){ + assert( pTab->iPKeynCol ); + zCol = pTab->aCol[pTab->iPKey].zCnName; + }else{ + zCol = "ROWID"; + } + assert( iDb>=0 && iDbdb->nDb ); + if( SQLITE_IGNORE==sqlite3AuthReadCol(pParse, pTab->zName, zCol, iDb) ){ + pExpr->op = TK_NULL; + } +} + +/* +** Do an authorization check using the code and arguments given. Return +** either SQLITE_OK (zero) or SQLITE_IGNORE or SQLITE_DENY. If SQLITE_DENY +** is returned, then the error count and error message in pParse are +** modified appropriately. +*/ +SQLITE_PRIVATE int sqlite3AuthCheck( + Parse *pParse, + int code, + const char *zArg1, + const char *zArg2, + const char *zArg3 +){ + sqlite3 *db = pParse->db; + int rc; + + /* Don't do any authorization checks if the database is initialising + ** or if the parser is being invoked from within sqlite3_declare_vtab. + */ + assert( !IN_RENAME_OBJECT || db->xAuth==0 ); + if( db->xAuth==0 || db->init.busy || IN_SPECIAL_PARSE ){ + return SQLITE_OK; + } + + /* EVIDENCE-OF: R-43249-19882 The third through sixth parameters to the + ** callback are either NULL pointers or zero-terminated strings that + ** contain additional details about the action to be authorized. + ** + ** The following testcase() macros show that any of the 3rd through 6th + ** parameters can be either NULL or a string. */ + testcase( zArg1==0 ); + testcase( zArg2==0 ); + testcase( zArg3==0 ); + testcase( pParse->zAuthContext==0 ); + + rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); + if( rc==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized"); + pParse->rc = SQLITE_AUTH; + }else if( rc!=SQLITE_OK && rc!=SQLITE_IGNORE ){ + rc = SQLITE_DENY; + sqliteAuthBadReturnCode(pParse); + } + return rc; +} + +/* +** Push an authorization context. After this routine is called, the +** zArg3 argument to authorization callbacks will be zContext until +** popped. Or if pParse==0, this routine is a no-op. +*/ +SQLITE_PRIVATE void sqlite3AuthContextPush( + Parse *pParse, + AuthContext *pContext, + const char *zContext +){ + assert( pParse ); + pContext->pParse = pParse; + pContext->zAuthContext = pParse->zAuthContext; + pParse->zAuthContext = zContext; +} + +/* +** Pop an authorization context that was previously pushed +** by sqlite3AuthContextPush +*/ +SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext *pContext){ + if( pContext->pParse ){ + pContext->pParse->zAuthContext = pContext->zAuthContext; + pContext->pParse = 0; + } +} + +#endif /* SQLITE_OMIT_AUTHORIZATION */ + +/************** End of auth.c ************************************************/ +/************** Begin file build.c *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the SQLite parser +** when syntax rules are reduced. The routines in this file handle the +** following kinds of SQL syntax: +** +** CREATE TABLE +** DROP TABLE +** CREATE INDEX +** DROP INDEX +** creating ID lists +** BEGIN TRANSACTION +** COMMIT +** ROLLBACK +*/ +/* #include "sqliteInt.h" */ + +#ifndef SQLITE_OMIT_SHARED_CACHE +/* +** The TableLock structure is only used by the sqlite3TableLock() and +** codeTableLocks() functions. +*/ +struct TableLock { + int iDb; /* The database containing the table to be locked */ + Pgno iTab; /* The root page of the table to be locked */ + u8 isWriteLock; /* True for write lock. False for a read lock */ + const char *zLockName; /* Name of the table */ +}; + +/* +** Record the fact that we want to lock a table at run-time. +** +** The table to be locked has root page iTab and is found in database iDb. +** A read or a write lock can be taken depending on isWritelock. +** +** This routine just records the fact that the lock is desired. The +** code to make the lock occur is generated by a later call to +** codeTableLocks() which occurs during sqlite3FinishCoding(). +*/ +static SQLITE_NOINLINE void lockTable( + Parse *pParse, /* Parsing context */ + int iDb, /* Index of the database containing the table to lock */ + Pgno iTab, /* Root page number of the table to be locked */ + u8 isWriteLock, /* True for a write lock */ + const char *zName /* Name of the table to be locked */ +){ + Parse *pToplevel; + int i; + int nBytes; + TableLock *p; + assert( iDb>=0 ); + + pToplevel = sqlite3ParseToplevel(pParse); + for(i=0; inTableLock; i++){ + p = &pToplevel->aTableLock[i]; + if( p->iDb==iDb && p->iTab==iTab ){ + p->isWriteLock = (p->isWriteLock || isWriteLock); + return; + } + } + + nBytes = sizeof(TableLock) * (pToplevel->nTableLock+1); + pToplevel->aTableLock = + sqlite3DbReallocOrFree(pToplevel->db, pToplevel->aTableLock, nBytes); + if( pToplevel->aTableLock ){ + p = &pToplevel->aTableLock[pToplevel->nTableLock++]; + p->iDb = iDb; + p->iTab = iTab; + p->isWriteLock = isWriteLock; + p->zLockName = zName; + }else{ + pToplevel->nTableLock = 0; + sqlite3OomFault(pToplevel->db); + } +} +SQLITE_PRIVATE void sqlite3TableLock( + Parse *pParse, /* Parsing context */ + int iDb, /* Index of the database containing the table to lock */ + Pgno iTab, /* Root page number of the table to be locked */ + u8 isWriteLock, /* True for a write lock */ + const char *zName /* Name of the table to be locked */ +){ + if( iDb==1 ) return; + if( !sqlite3BtreeSharable(pParse->db->aDb[iDb].pBt) ) return; + lockTable(pParse, iDb, iTab, isWriteLock, zName); +} + +/* +** Code an OP_TableLock instruction for each table locked by the +** statement (configured by calls to sqlite3TableLock()). +*/ +static void codeTableLocks(Parse *pParse){ + int i; + Vdbe *pVdbe = pParse->pVdbe; + assert( pVdbe!=0 ); + + for(i=0; inTableLock; i++){ + TableLock *p = &pParse->aTableLock[i]; + int p1 = p->iDb; + sqlite3VdbeAddOp4(pVdbe, OP_TableLock, p1, p->iTab, p->isWriteLock, + p->zLockName, P4_STATIC); + } +} +#else + #define codeTableLocks(x) +#endif + +/* +** Return TRUE if the given yDbMask object is empty - if it contains no +** 1 bits. This routine is used by the DbMaskAllZero() and DbMaskNotZero() +** macros when SQLITE_MAX_ATTACHED is greater than 30. +*/ +#if SQLITE_MAX_ATTACHED>30 +SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask m){ + int i; + for(i=0; ipToplevel==0 ); + db = pParse->db; + if( pParse->nested ) return; + if( db->mallocFailed || pParse->nErr ){ + if( pParse->rc==SQLITE_OK ) pParse->rc = SQLITE_ERROR; + return; + } + + /* Begin by generating some termination code at the end of the + ** vdbe program + */ + v = pParse->pVdbe; + if( v==0 ){ + if( db->init.busy ){ + pParse->rc = SQLITE_DONE; + return; + } + v = sqlite3GetVdbe(pParse); + if( v==0 ) pParse->rc = SQLITE_ERROR; + } + assert( !pParse->isMultiWrite + || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); + if( v ){ + if( pParse->bReturning ){ + Returning *pReturning = pParse->u1.pReturning; + int addrRewind; + int i; + int reg; + + addrRewind = + sqlite3VdbeAddOp1(v, OP_Rewind, pReturning->iRetCur); + VdbeCoverage(v); + reg = pReturning->iRetReg; + for(i=0; inRetCol; i++){ + sqlite3VdbeAddOp3(v, OP_Column, pReturning->iRetCur, i, reg+i); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, reg, i); + sqlite3VdbeAddOp2(v, OP_Next, pReturning->iRetCur, addrRewind+1); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrRewind); + } + sqlite3VdbeAddOp0(v, OP_Halt); + +#if SQLITE_USER_AUTHENTICATION + if( pParse->nTableLock>0 && db->init.busy==0 ){ + sqlite3UserAuthInit(db); + if( db->auth.authLevelrc = SQLITE_AUTH_USER; + return; + } + } +#endif + + /* The cookie mask contains one bit for each database file open. + ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are + ** set for each database that is used. Generate code to start a + ** transaction on each used database and to verify the schema cookie + ** on each used database. + */ + if( db->mallocFailed==0 + && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) + ){ + int iDb, i; + assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); + sqlite3VdbeJumpHere(v, 0); + for(iDb=0; iDbnDb; iDb++){ + Schema *pSchema; + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; + sqlite3VdbeUsesBtree(v, iDb); + pSchema = db->aDb[iDb].pSchema; + sqlite3VdbeAddOp4Int(v, + OP_Transaction, /* Opcode */ + iDb, /* P1 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ + pSchema->schema_cookie, /* P3 */ + pSchema->iGeneration /* P4 */ + ); + if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + for(i=0; inVtabLock; i++){ + char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); + sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); + } + pParse->nVtabLock = 0; +#endif + + /* Once all the cookies have been verified and transactions opened, + ** obtain the required table-locks. This is a no-op unless the + ** shared-cache feature is enabled. + */ + codeTableLocks(pParse); + + /* Initialize any AUTOINCREMENT data structures required. + */ + sqlite3AutoincrementBegin(pParse); + + /* Code constant expressions that where factored out of inner loops. + ** + ** The pConstExpr list might also contain expressions that we simply + ** want to keep around until the Parse object is deleted. Such + ** expressions have iConstExprReg==0. Do not generate code for + ** those expressions, of course. + */ + if( pParse->pConstExpr ){ + ExprList *pEL = pParse->pConstExpr; + pParse->okConstFactor = 0; + for(i=0; inExpr; i++){ + int iReg = pEL->a[i].u.iConstExprReg; + if( iReg>0 ){ + sqlite3ExprCode(pParse, pEL->a[i].pExpr, iReg); + } + } + } + + if( pParse->bReturning ){ + Returning *pRet = pParse->u1.pReturning; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); + } + + /* Finally, jump back to the beginning of the executable code. */ + sqlite3VdbeGoto(v, 1); + } + } + + /* Get the VDBE program ready for execution + */ + if( v && pParse->nErr==0 && !db->mallocFailed ){ + /* A minimum of one cursor is required if autoincrement is used + * See ticket [a696379c1f08866] */ + assert( pParse->pAinc==0 || pParse->nTab>0 ); + sqlite3VdbeMakeReady(v, pParse); + pParse->rc = SQLITE_DONE; + }else{ + pParse->rc = SQLITE_ERROR; + } +} + +/* +** Run the parser and code generator recursively in order to generate +** code for the SQL statement given onto the end of the pParse context +** currently under construction. Notes: +** +** * The final OP_Halt is not appended and other initialization +** and finalization steps are omitted because those are handling by the +** outermost parser. +** +** * Built-in SQL functions always take precedence over application-defined +** SQL functions. In other words, it is not possible to override a +** built-in function. +*/ +SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ + va_list ap; + char *zSql; + char *zErrMsg = 0; + sqlite3 *db = pParse->db; + u32 savedDbFlags = db->mDbFlags; + char saveBuf[PARSE_TAIL_SZ]; + + if( pParse->nErr ) return; + assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ + va_start(ap, zFormat); + zSql = sqlite3VMPrintf(db, zFormat, ap); + va_end(ap); + if( zSql==0 ){ + /* This can result either from an OOM or because the formatted string + ** exceeds SQLITE_LIMIT_LENGTH. In the latter case, we need to set + ** an error */ + if( !db->mallocFailed ) pParse->rc = SQLITE_TOOBIG; + pParse->nErr++; + return; + } + pParse->nested++; + memcpy(saveBuf, PARSE_TAIL(pParse), PARSE_TAIL_SZ); + memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); + db->mDbFlags |= DBFLAG_PreferBuiltin; + sqlite3RunParser(pParse, zSql, &zErrMsg); + db->mDbFlags = savedDbFlags; + sqlite3DbFree(db, zErrMsg); + sqlite3DbFree(db, zSql); + memcpy(PARSE_TAIL(pParse), saveBuf, PARSE_TAIL_SZ); + pParse->nested--; +} + +#if SQLITE_USER_AUTHENTICATION +/* +** Return TRUE if zTable is the name of the system table that stores the +** list of users and their access credentials. +*/ +SQLITE_PRIVATE int sqlite3UserAuthTable(const char *zTable){ + return sqlite3_stricmp(zTable, "sqlite_user")==0; +} +#endif + +/* +** Locate the in-memory structure that describes a particular database +** table given the name of that table and (optionally) the name of the +** database containing the table. Return NULL if not found. +** +** If zDatabase is 0, all databases are searched for the table and the +** first matching table is returned. (No checking for duplicate table +** names is done.) The search order is TEMP first, then MAIN, then any +** auxiliary databases added using the ATTACH command. +** +** See also sqlite3LocateTable(). +*/ +SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ + Table *p = 0; + int i; + + /* All mutexes are required for schema access. Make sure we hold them. */ + assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); +#if SQLITE_USER_AUTHENTICATION + /* Only the admin user is allowed to know that the sqlite_user table + ** exists */ + if( db->auth.authLevelnDb; i++){ + if( sqlite3StrICmp(zDatabase, db->aDb[i].zDbSName)==0 ) break; + } + if( i>=db->nDb ){ + /* No match against the official names. But always match "main" + ** to schema 0 as a legacy fallback. */ + if( sqlite3StrICmp(zDatabase,"main")==0 ){ + i = 0; + }else{ + return 0; + } + } + p = sqlite3HashFind(&db->aDb[i].pSchema->tblHash, zName); + if( p==0 && sqlite3StrNICmp(zName, "sqlite_", 7)==0 ){ + if( i==1 ){ + if( sqlite3StrICmp(zName+7, &ALT_TEMP_SCHEMA_TABLE[7])==0 + || sqlite3StrICmp(zName+7, &ALT_SCHEMA_TABLE[7])==0 + || sqlite3StrICmp(zName+7, &DFLT_SCHEMA_TABLE[7])==0 + ){ + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, + DFLT_TEMP_SCHEMA_TABLE); + } + }else{ + if( sqlite3StrICmp(zName+7, &ALT_SCHEMA_TABLE[7])==0 ){ + p = sqlite3HashFind(&db->aDb[i].pSchema->tblHash, + DFLT_SCHEMA_TABLE); + } + } + } + }else{ + /* Match against TEMP first */ + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, zName); + if( p ) return p; + /* The main database is second */ + p = sqlite3HashFind(&db->aDb[0].pSchema->tblHash, zName); + if( p ) return p; + /* Attached databases are in order of attachment */ + for(i=2; inDb; i++){ + assert( sqlite3SchemaMutexHeld(db, i, 0) ); + p = sqlite3HashFind(&db->aDb[i].pSchema->tblHash, zName); + if( p ) break; + } + if( p==0 && sqlite3StrNICmp(zName, "sqlite_", 7)==0 ){ + if( sqlite3StrICmp(zName+7, &ALT_SCHEMA_TABLE[7])==0 ){ + p = sqlite3HashFind(&db->aDb[0].pSchema->tblHash, DFLT_SCHEMA_TABLE); + }else if( sqlite3StrICmp(zName+7, &ALT_TEMP_SCHEMA_TABLE[7])==0 ){ + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, + DFLT_TEMP_SCHEMA_TABLE); + } + } + } + return p; +} + +/* +** Locate the in-memory structure that describes a particular database +** table given the name of that table and (optionally) the name of the +** database containing the table. Return NULL if not found. Also leave an +** error message in pParse->zErrMsg. +** +** The difference between this routine and sqlite3FindTable() is that this +** routine leaves an error message in pParse->zErrMsg where +** sqlite3FindTable() does not. +*/ +SQLITE_PRIVATE Table *sqlite3LocateTable( + Parse *pParse, /* context in which to report errors */ + u32 flags, /* LOCATE_VIEW or LOCATE_NOERR */ + const char *zName, /* Name of the table we are looking for */ + const char *zDbase /* Name of the database. Might be NULL */ +){ + Table *p; + sqlite3 *db = pParse->db; + + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + if( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 + && SQLITE_OK!=sqlite3ReadSchema(pParse) + ){ + return 0; + } + + p = sqlite3FindTable(db, zName, zDbase); + if( p==0 ){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* If zName is the not the name of a table in the schema created using + ** CREATE, then check to see if it is the name of an virtual table that + ** can be an eponymous virtual table. */ + if( pParse->disableVtab==0 && db->init.busy==0 ){ + Module *pMod = (Module*)sqlite3HashFind(&db->aModule, zName); + if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ + pMod = sqlite3PragmaVtabRegister(db, zName); + } + if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ + testcase( pMod->pEpoTab==0 ); + return pMod->pEpoTab; + } + } +#endif + if( flags & LOCATE_NOERR ) return 0; + pParse->checkSchema = 1; + }else if( IsVirtual(p) && pParse->disableVtab ){ + p = 0; + } + + if( p==0 ){ + const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table"; + if( zDbase ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); + } + }else{ + assert( HasRowid(p) || p->iPKey<0 ); + } + + return p; +} + +/* +** Locate the table identified by *p. +** +** This is a wrapper around sqlite3LocateTable(). The difference between +** sqlite3LocateTable() and this function is that this function restricts +** the search to schema (p->pSchema) if it is not NULL. p->pSchema may be +** non-NULL if it is part of a view or trigger program definition. See +** sqlite3FixSrcList() for details. +*/ +SQLITE_PRIVATE Table *sqlite3LocateTableItem( + Parse *pParse, + u32 flags, + SrcItem *p +){ + const char *zDb; + assert( p->pSchema==0 || p->zDatabase==0 ); + if( p->pSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); + zDb = pParse->db->aDb[iDb].zDbSName; + }else{ + zDb = p->zDatabase; + } + return sqlite3LocateTable(pParse, flags, p->zName, zDb); +} + +/* +** Locate the in-memory structure that describes +** a particular index given the name of that index +** and the name of the database that contains the index. +** Return NULL if not found. +** +** If zDatabase is 0, all databases are searched for the +** table and the first matching index is returned. (No checking +** for duplicate index names is done.) The search order is +** TEMP first, then MAIN, then any auxiliary databases added +** using the ATTACH command. +*/ +SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ + Index *p = 0; + int i; + /* All mutexes are required for schema access. Make sure we hold them. */ + assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); + for(i=OMIT_TEMPDB; inDb; i++){ + int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ + Schema *pSchema = db->aDb[j].pSchema; + assert( pSchema ); + if( zDb && sqlite3DbIsNamed(db, j, zDb)==0 ) continue; + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + p = sqlite3HashFind(&pSchema->idxHash, zName); + if( p ) break; + } + return p; +} + +/* +** Reclaim the memory used by an index +*/ +SQLITE_PRIVATE void sqlite3FreeIndex(sqlite3 *db, Index *p){ +#ifndef SQLITE_OMIT_ANALYZE + sqlite3DeleteIndexSamples(db, p); +#endif + sqlite3ExprDelete(db, p->pPartIdxWhere); + sqlite3ExprListDelete(db, p->aColExpr); + sqlite3DbFree(db, p->zColAff); + if( p->isResized ) sqlite3DbFree(db, (void *)p->azColl); +#ifdef SQLITE_ENABLE_STAT4 + sqlite3_free(p->aiRowEst); +#endif + sqlite3DbFree(db, p); +} + +/* +** For the index called zIdxName which is found in the database iDb, +** unlike that index from its Table then remove the index from +** the index hash table and free all memory structures associated +** with the index. +*/ +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ + Index *pIndex; + Hash *pHash; + + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pHash = &db->aDb[iDb].pSchema->idxHash; + pIndex = sqlite3HashInsert(pHash, zIdxName, 0); + if( ALWAYS(pIndex) ){ + if( pIndex->pTable->pIndex==pIndex ){ + pIndex->pTable->pIndex = pIndex->pNext; + }else{ + Index *p; + /* Justification of ALWAYS(); The index must be on the list of + ** indices. */ + p = pIndex->pTable->pIndex; + while( ALWAYS(p) && p->pNext!=pIndex ){ p = p->pNext; } + if( ALWAYS(p && p->pNext==pIndex) ){ + p->pNext = pIndex->pNext; + } + } + sqlite3FreeIndex(db, pIndex); + } + db->mDbFlags |= DBFLAG_SchemaChange; +} + +/* +** Look through the list of open database files in db->aDb[] and if +** any have been closed, remove them from the list. Reallocate the +** db->aDb[] structure to a smaller size, if possible. +** +** Entry 0 (the "main" database) and entry 1 (the "temp" database) +** are never candidates for being collapsed. +*/ +SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3 *db){ + int i, j; + for(i=j=2; inDb; i++){ + struct Db *pDb = &db->aDb[i]; + if( pDb->pBt==0 ){ + sqlite3DbFree(db, pDb->zDbSName); + pDb->zDbSName = 0; + continue; + } + if( jaDb[j] = db->aDb[i]; + } + j++; + } + db->nDb = j; + if( db->nDb<=2 && db->aDb!=db->aDbStatic ){ + memcpy(db->aDbStatic, db->aDb, 2*sizeof(db->aDb[0])); + sqlite3DbFree(db, db->aDb); + db->aDb = db->aDbStatic; + } +} + +/* +** Reset the schema for the database at index iDb. Also reset the +** TEMP schema. The reset is deferred if db->nSchemaLock is not zero. +** Deferred resets may be run by calling with iDb<0. +*/ +SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3 *db, int iDb){ + int i; + assert( iDbnDb ); + + if( iDb>=0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + DbSetProperty(db, iDb, DB_ResetWanted); + DbSetProperty(db, 1, DB_ResetWanted); + db->mDbFlags &= ~DBFLAG_SchemaKnownOk; + } + + if( db->nSchemaLock==0 ){ + for(i=0; inDb; i++){ + if( DbHasProperty(db, i, DB_ResetWanted) ){ + sqlite3SchemaClear(db->aDb[i].pSchema); + } + } + } +} + +/* +** Erase all schema information from all attached databases (including +** "main" and "temp") for a single database connection. +*/ +SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){ + int i; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Db *pDb = &db->aDb[i]; + if( pDb->pSchema ){ + if( db->nSchemaLock==0 ){ + sqlite3SchemaClear(pDb->pSchema); + }else{ + DbSetProperty(db, i, DB_ResetWanted); + } + } + } + db->mDbFlags &= ~(DBFLAG_SchemaChange|DBFLAG_SchemaKnownOk); + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); + if( db->nSchemaLock==0 ){ + sqlite3CollapseDatabaseArray(db); + } +} + +/* +** This routine is called when a commit occurs. +*/ +SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ + db->mDbFlags &= ~DBFLAG_SchemaChange; +} + +/* +** Set the expression associated with a column. This is usually +** the DEFAULT value, but might also be the expression that computes +** the value for a generated column. +*/ +SQLITE_PRIVATE void sqlite3ColumnSetExpr( + Parse *pParse, /* Parsing context */ + Table *pTab, /* The table containing the column */ + Column *pCol, /* The column to receive the new DEFAULT expression */ + Expr *pExpr /* The new default expression */ +){ + ExprList *pList; + assert( !IsVirtual(pTab) ); + pList = pTab->u.tab.pDfltList; + if( pCol->iDflt==0 + || NEVER(pList==0) + || NEVER(pList->nExpriDflt) + ){ + pCol->iDflt = pList==0 ? 1 : pList->nExpr+1; + pTab->u.tab.pDfltList = sqlite3ExprListAppend(pParse, pList, pExpr); + }else{ + sqlite3ExprDelete(pParse->db, pList->a[pCol->iDflt-1].pExpr); + pList->a[pCol->iDflt-1].pExpr = pExpr; + } +} + +/* +** Return the expression associated with a column. The expression might be +** the DEFAULT clause or the AS clause of a generated column. +** Return NULL if the column has no associated expression. +*/ +SQLITE_PRIVATE Expr *sqlite3ColumnExpr(Table *pTab, Column *pCol){ + if( pCol->iDflt==0 ) return 0; + if( NEVER(IsVirtual(pTab)) ) return 0; + if( NEVER(pTab->u.tab.pDfltList==0) ) return 0; + if( NEVER(pTab->u.tab.pDfltList->nExpriDflt) ) return 0; + return pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr; +} + +/* +** Set the collating sequence name for a column. +*/ +SQLITE_PRIVATE void sqlite3ColumnSetColl( + sqlite3 *db, + Column *pCol, + const char *zColl +){ + int nColl; + int n; + char *zNew; + assert( zColl!=0 ); + n = sqlite3Strlen30(pCol->zCnName) + 1; + if( pCol->colFlags & COLFLAG_HASTYPE ){ + n += sqlite3Strlen30(pCol->zCnName+n) + 1; + } + nColl = sqlite3Strlen30(zColl) + 1; + zNew = sqlite3DbRealloc(db, pCol->zCnName, nColl+n); + if( zNew ){ + pCol->zCnName = zNew; + memcpy(pCol->zCnName + n, zColl, nColl); + pCol->colFlags |= COLFLAG_HASCOLL; + } +} + +/* +** Return the collating squence name for a column +*/ +SQLITE_PRIVATE const char *sqlite3ColumnColl(Column *pCol){ + const char *z; + if( (pCol->colFlags & COLFLAG_HASCOLL)==0 ) return 0; + z = pCol->zCnName; + while( *z ){ z++; } + if( pCol->colFlags & COLFLAG_HASTYPE ){ + do{ z++; }while( *z ); + } + return z+1; +} + +/* +** Delete memory allocated for the column names of a table or view (the +** Table.aCol[] array). +*/ +SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ + int i; + Column *pCol; + assert( pTable!=0 ); + if( (pCol = pTable->aCol)!=0 ){ + for(i=0; inCol; i++, pCol++){ + assert( pCol->zCnName==0 || pCol->hName==sqlite3StrIHash(pCol->zCnName) ); + sqlite3DbFree(db, pCol->zCnName); + } + sqlite3DbFree(db, pTable->aCol); + if( !IsVirtual(pTable) ){ + sqlite3ExprListDelete(db, pTable->u.tab.pDfltList); + } + if( db==0 || db->pnBytesFreed==0 ){ + pTable->aCol = 0; + pTable->nCol = 0; + if( !IsVirtual(pTable) ){ + pTable->u.tab.pDfltList = 0; + } + } + } +} + +/* +** Remove the memory data structures associated with the given +** Table. No changes are made to disk by this routine. +** +** This routine just deletes the data structure. It does not unlink +** the table data structure from the hash table. But it does destroy +** memory structures of the indices and foreign keys associated with +** the table. +** +** The db parameter is optional. It is needed if the Table object +** contains lookaside memory. (Table objects in the schema do not use +** lookaside memory, but some ephemeral Table objects do.) Or the +** db parameter can be used with db->pnBytesFreed to measure the memory +** used by the Table object. +*/ +static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ + Index *pIndex, *pNext; + +#ifdef SQLITE_DEBUG + /* Record the number of outstanding lookaside allocations in schema Tables + ** prior to doing any free() operations. Since schema Tables do not use + ** lookaside, this number should not change. + ** + ** If malloc has already failed, it may be that it failed while allocating + ** a Table object that was going to be marked ephemeral. So do not check + ** that no lookaside memory is used in this case either. */ + int nLookaside = 0; + if( db && !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ + nLookaside = sqlite3LookasideUsed(db, 0); + } +#endif + + /* Delete all indices associated with this table. */ + for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ + pNext = pIndex->pNext; + assert( pIndex->pSchema==pTable->pSchema + || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); + if( (db==0 || db->pnBytesFreed==0) && !IsVirtual(pTable) ){ + char *zName = pIndex->zName; + TESTONLY ( Index *pOld = ) sqlite3HashInsert( + &pIndex->pSchema->idxHash, zName, 0 + ); + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); + assert( pOld==pIndex || pOld==0 ); + } + sqlite3FreeIndex(db, pIndex); + } + + if( IsOrdinaryTable(pTable) ){ + sqlite3FkDelete(db, pTable); + } +#ifndef SQLITE_OMIT_VIRTUAL_TABLE + else if( IsVirtual(pTable) ){ + sqlite3VtabClear(db, pTable); + } +#endif + else{ + assert( IsView(pTable) ); + sqlite3SelectDelete(db, pTable->u.view.pSelect); + } + + /* Delete the Table structure itself. + */ + sqlite3DeleteColumnNames(db, pTable); + sqlite3DbFree(db, pTable->zName); + sqlite3DbFree(db, pTable->zColAff); + sqlite3ExprListDelete(db, pTable->pCheck); + sqlite3DbFree(db, pTable); + + /* Verify that no lookaside memory was used by schema tables */ + assert( nLookaside==0 || nLookaside==sqlite3LookasideUsed(db,0) ); +} +SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ + /* Do not delete the table until the reference count reaches zero. */ + if( !pTable ) return; + if( ((!db || db->pnBytesFreed==0) && (--pTable->nTabRef)>0) ) return; + deleteTable(db, pTable); +} + + +/* +** Unlink the given table from the hash tables and the delete the +** table structure with all its indices and foreign keys. +*/ +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){ + Table *p; + Db *pDb; + + assert( db!=0 ); + assert( iDb>=0 && iDbnDb ); + assert( zTabName ); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ + pDb = &db->aDb[iDb]; + p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0); + sqlite3DeleteTable(db, p); + db->mDbFlags |= DBFLAG_SchemaChange; +} + +/* +** Given a token, return a string that consists of the text of that +** token. Space to hold the returned string +** is obtained from sqliteMalloc() and must be freed by the calling +** function. +** +** Any quotation marks (ex: "name", 'name', [name], or `name`) that +** surround the body of the token are removed. +** +** Tokens are often just pointers into the original SQL text and so +** are not \000 terminated and are not persistent. The returned string +** is \000 terminated and is persistent. +*/ +SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, const Token *pName){ + char *zName; + if( pName ){ + zName = sqlite3DbStrNDup(db, (const char*)pName->z, pName->n); + sqlite3Dequote(zName); + }else{ + zName = 0; + } + return zName; +} + +/* +** Open the sqlite_schema table stored in database number iDb for +** writing. The table is opened using cursor 0. +*/ +SQLITE_PRIVATE void sqlite3OpenSchemaTable(Parse *p, int iDb){ + Vdbe *v = sqlite3GetVdbe(p); + sqlite3TableLock(p, iDb, SCHEMA_ROOT, 1, DFLT_SCHEMA_TABLE); + sqlite3VdbeAddOp4Int(v, OP_OpenWrite, 0, SCHEMA_ROOT, iDb, 5); + if( p->nTab==0 ){ + p->nTab = 1; + } +} + +/* +** Parameter zName points to a nul-terminated buffer containing the name +** of a database ("main", "temp" or the name of an attached db). This +** function returns the index of the named database in db->aDb[], or +** -1 if the named db cannot be found. +*/ +SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *db, const char *zName){ + int i = -1; /* Database number */ + if( zName ){ + Db *pDb; + for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ + if( 0==sqlite3_stricmp(pDb->zDbSName, zName) ) break; + /* "main" is always an acceptable alias for the primary database + ** even if it has been renamed using SQLITE_DBCONFIG_MAINDBNAME. */ + if( i==0 && 0==sqlite3_stricmp("main", zName) ) break; + } + } + return i; +} + +/* +** The token *pName contains the name of a database (either "main" or +** "temp" or the name of an attached db). This routine returns the +** index of the named database in db->aDb[], or -1 if the named db +** does not exist. +*/ +SQLITE_PRIVATE int sqlite3FindDb(sqlite3 *db, Token *pName){ + int i; /* Database number */ + char *zName; /* Name we are searching for */ + zName = sqlite3NameFromToken(db, pName); + i = sqlite3FindDbName(db, zName); + sqlite3DbFree(db, zName); + return i; +} + +/* The table or view or trigger name is passed to this routine via tokens +** pName1 and pName2. If the table name was fully qualified, for example: +** +** CREATE TABLE xxx.yyy (...); +** +** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if +** the table name is not fully qualified, i.e.: +** +** CREATE TABLE yyy(...); +** +** Then pName1 is set to "yyy" and pName2 is "". +** +** This routine sets the *ppUnqual pointer to point at the token (pName1 or +** pName2) that stores the unqualified table name. The index of the +** database "xxx" is returned. +*/ +SQLITE_PRIVATE int sqlite3TwoPartName( + Parse *pParse, /* Parsing and code generating context */ + Token *pName1, /* The "xxx" in the name "xxx.yyy" or "xxx" */ + Token *pName2, /* The "yyy" in the name "xxx.yyy" */ + Token **pUnqual /* Write the unqualified object name here */ +){ + int iDb; /* Database holding the object */ + sqlite3 *db = pParse->db; + + assert( pName2!=0 ); + if( pName2->n>0 ){ + if( db->init.busy ) { + sqlite3ErrorMsg(pParse, "corrupt database"); + return -1; + } + *pUnqual = pName2; + iDb = sqlite3FindDb(db, pName1); + if( iDb<0 ){ + sqlite3ErrorMsg(pParse, "unknown database %T", pName1); + return -1; + } + }else{ + assert( db->init.iDb==0 || db->init.busy || IN_SPECIAL_PARSE + || (db->mDbFlags & DBFLAG_Vacuum)!=0); + iDb = db->init.iDb; + *pUnqual = pName1; + } + return iDb; +} + +/* +** True if PRAGMA writable_schema is ON +*/ +SQLITE_PRIVATE int sqlite3WritableSchema(sqlite3 *db){ + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))==0 ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + SQLITE_WriteSchema ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + SQLITE_Defensive ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + (SQLITE_WriteSchema|SQLITE_Defensive) ); + return (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))==SQLITE_WriteSchema; +} + +/* +** This routine is used to check if the UTF-8 string zName is a legal +** unqualified name for a new schema object (table, index, view or +** trigger). All names are legal except those that begin with the string +** "sqlite_" (in upper, lower or mixed case). This portion of the namespace +** is reserved for internal use. +** +** When parsing the sqlite_schema table, this routine also checks to +** make sure the "type", "name", and "tbl_name" columns are consistent +** with the SQL. +*/ +SQLITE_PRIVATE int sqlite3CheckObjectName( + Parse *pParse, /* Parsing context */ + const char *zName, /* Name of the object to check */ + const char *zType, /* Type of this object */ + const char *zTblName /* Parent table name for triggers and indexes */ +){ + sqlite3 *db = pParse->db; + if( sqlite3WritableSchema(db) + || db->init.imposterTable + || !sqlite3Config.bExtraSchemaChecks + ){ + /* Skip these error checks for writable_schema=ON */ + return SQLITE_OK; + } + if( db->init.busy ){ + if( sqlite3_stricmp(zType, db->init.azInit[0]) + || sqlite3_stricmp(zName, db->init.azInit[1]) + || sqlite3_stricmp(zTblName, db->init.azInit[2]) + ){ + sqlite3ErrorMsg(pParse, ""); /* corruptSchema() will supply the error */ + return SQLITE_ERROR; + } + }else{ + if( (pParse->nested==0 && 0==sqlite3StrNICmp(zName, "sqlite_", 7)) + || (sqlite3ReadOnlyShadowTables(db) && sqlite3ShadowTableName(db, zName)) + ){ + sqlite3ErrorMsg(pParse, "object name reserved for internal use: %s", + zName); + return SQLITE_ERROR; + } + + } + return SQLITE_OK; +} + +/* +** Return the PRIMARY KEY index of a table +*/ +SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table *pTab){ + Index *p; + for(p=pTab->pIndex; p && !IsPrimaryKeyIndex(p); p=p->pNext){} + return p; +} + +/* +** Convert an table column number into a index column number. That is, +** for the column iCol in the table (as defined by the CREATE TABLE statement) +** find the (first) offset of that column in index pIdx. Or return -1 +** if column iCol is not used in index pIdx. +*/ +SQLITE_PRIVATE i16 sqlite3TableColumnToIndex(Index *pIdx, i16 iCol){ + int i; + for(i=0; inColumn; i++){ + if( iCol==pIdx->aiColumn[i] ) return i; + } + return -1; +} + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* Convert a storage column number into a table column number. +** +** The storage column number (0,1,2,....) is the index of the value +** as it appears in the record on disk. The true column number +** is the index (0,1,2,...) of the column in the CREATE TABLE statement. +** +** The storage column number is less than the table column number if +** and only there are VIRTUAL columns to the left. +** +** If SQLITE_OMIT_GENERATED_COLUMNS, this routine is a no-op macro. +*/ +SQLITE_PRIVATE i16 sqlite3StorageColumnToTable(Table *pTab, i16 iCol){ + if( pTab->tabFlags & TF_HasVirtual ){ + int i; + for(i=0; i<=iCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) iCol++; + } + } + return iCol; +} +#endif + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* Convert a table column number into a storage column number. +** +** The storage column number (0,1,2,....) is the index of the value +** as it appears in the record on disk. Or, if the input column is +** the N-th virtual column (zero-based) then the storage number is +** the number of non-virtual columns in the table plus N. +** +** The true column number is the index (0,1,2,...) of the column in +** the CREATE TABLE statement. +** +** If the input column is a VIRTUAL column, then it should not appear +** in storage. But the value sometimes is cached in registers that +** follow the range of registers used to construct storage. This +** avoids computing the same VIRTUAL column multiple times, and provides +** values for use by OP_Param opcodes in triggers. Hence, if the +** input column is a VIRTUAL table, put it after all the other columns. +** +** In the following, N means "normal column", S means STORED, and +** V means VIRTUAL. Suppose the CREATE TABLE has columns like this: +** +** CREATE TABLE ex(N,S,V,N,S,V,N,S,V); +** -- 0 1 2 3 4 5 6 7 8 +** +** Then the mapping from this function is as follows: +** +** INPUTS: 0 1 2 3 4 5 6 7 8 +** OUTPUTS: 0 1 6 2 3 7 4 5 8 +** +** So, in other words, this routine shifts all the virtual columns to +** the end. +** +** If SQLITE_OMIT_GENERATED_COLUMNS then there are no virtual columns and +** this routine is a no-op macro. If the pTab does not have any virtual +** columns, then this routine is no-op that always return iCol. If iCol +** is negative (indicating the ROWID column) then this routine return iCol. +*/ +SQLITE_PRIVATE i16 sqlite3TableColumnToStorage(Table *pTab, i16 iCol){ + int i; + i16 n; + assert( iColnCol ); + if( (pTab->tabFlags & TF_HasVirtual)==0 || iCol<0 ) return iCol; + for(i=0, n=0; iaCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) n++; + } + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ){ + /* iCol is a virtual column itself */ + return pTab->nNVCol + i - n; + }else{ + /* iCol is a normal or stored column */ + return n; + } +} +#endif + +/* +** Insert a single OP_JournalMode query opcode in order to force the +** prepared statement to return false for sqlite3_stmt_readonly(). This +** is used by CREATE TABLE IF NOT EXISTS and similar if the table already +** exists, so that the prepared statement for CREATE TABLE IF NOT EXISTS +** will return false for sqlite3_stmt_readonly() even if that statement +** is a read-only no-op. +*/ +static void sqlite3ForceNotReadOnly(Parse *pParse){ + int iReg = ++pParse->nMem; + Vdbe *v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp3(v, OP_JournalMode, 0, iReg, PAGER_JOURNALMODE_QUERY); + sqlite3VdbeUsesBtree(v, 0); + } +} + +/* +** Begin constructing a new table representation in memory. This is +** the first of several action routines that get called in response +** to a CREATE TABLE statement. In particular, this routine is called +** after seeing tokens "CREATE" and "TABLE" and the table name. The isTemp +** flag is true if the table should be stored in the auxiliary database +** file instead of in the main database file. This is normally the case +** when the "TEMP" or "TEMPORARY" keyword occurs in between +** CREATE and TABLE. +** +** The new table record is initialized and put in pParse->pNewTable. +** As more of the CREATE TABLE statement is parsed, additional action +** routines will be called to add more information to this record. +** At the end of the CREATE TABLE statement, the sqlite3EndTable() routine +** is called to complete the construction of the new table record. +*/ +SQLITE_PRIVATE void sqlite3StartTable( + Parse *pParse, /* Parser context */ + Token *pName1, /* First part of the name of the table or view */ + Token *pName2, /* Second part of the name of the table or view */ + int isTemp, /* True if this is a TEMP table */ + int isView, /* True if this is a VIEW */ + int isVirtual, /* True if this is a VIRTUAL table */ + int noErr /* Do nothing if table already exists */ +){ + Table *pTable; + char *zName = 0; /* The name of the new table */ + sqlite3 *db = pParse->db; + Vdbe *v; + int iDb; /* Database number to create the table in */ + Token *pName; /* Unqualified name of the table to create */ + + if( db->init.busy && db->init.newTnum==1 ){ + /* Special case: Parsing the sqlite_schema or sqlite_temp_schema schema */ + iDb = db->init.iDb; + zName = sqlite3DbStrDup(db, SCHEMA_TABLE(iDb)); + pName = pName1; + }else{ + /* The common case */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) return; + if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ + /* If creating a temp table, the name may not be qualified. Unless + ** the database name is "temp" anyway. */ + sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); + return; + } + if( !OMIT_TEMPDB && isTemp ) iDb = 1; + zName = sqlite3NameFromToken(db, pName); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)zName, pName); + } + } + pParse->sNameToken = *pName; + if( zName==0 ) return; + if( sqlite3CheckObjectName(pParse, zName, isView?"view":"table", zName) ){ + goto begin_table_error; + } + if( db->init.iDb==1 ) isTemp = 1; +#ifndef SQLITE_OMIT_AUTHORIZATION + assert( isTemp==0 || isTemp==1 ); + assert( isView==0 || isView==1 ); + { + static const u8 aCode[] = { + SQLITE_CREATE_TABLE, + SQLITE_CREATE_TEMP_TABLE, + SQLITE_CREATE_VIEW, + SQLITE_CREATE_TEMP_VIEW + }; + char *zDb = db->aDb[iDb].zDbSName; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ + goto begin_table_error; + } + if( !isVirtual && sqlite3AuthCheck(pParse, (int)aCode[isTemp+2*isView], + zName, 0, zDb) ){ + goto begin_table_error; + } + } +#endif + + /* Make sure the new table name does not collide with an existing + ** index or table name in the same database. Issue an error message if + ** it does. The exception is if the statement being parsed was passed + ** to an sqlite3_declare_vtab() call. In that case only the column names + ** and types will be used, so there is no need to test for namespace + ** collisions. + */ + if( !IN_SPECIAL_PARSE ){ + char *zDb = db->aDb[iDb].zDbSName; + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto begin_table_error; + } + pTable = sqlite3FindTable(db, zName, zDb); + if( pTable ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "table %T already exists", pName); + }else{ + assert( !db->init.busy || CORRUPT_DB ); + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3ForceNotReadOnly(pParse); + } + goto begin_table_error; + } + if( sqlite3FindIndex(db, zName, zDb)!=0 ){ + sqlite3ErrorMsg(pParse, "there is already an index named %s", zName); + goto begin_table_error; + } + } + + pTable = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTable==0 ){ + assert( db->mallocFailed ); + pParse->rc = SQLITE_NOMEM_BKPT; + pParse->nErr++; + goto begin_table_error; + } + pTable->zName = zName; + pTable->iPKey = -1; + pTable->pSchema = db->aDb[iDb].pSchema; + pTable->nTabRef = 1; +#ifdef SQLITE_DEFAULT_ROWEST + pTable->nRowLogEst = sqlite3LogEst(SQLITE_DEFAULT_ROWEST); +#else + pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); +#endif + assert( pParse->pNewTable==0 ); + pParse->pNewTable = pTable; + + /* Begin generating the code that will insert the table record into + ** the schema table. Note in particular that we must go ahead + ** and allocate the record number for the table entry now. Before any + ** PRIMARY KEY or UNIQUE keywords are parsed. Those keywords will cause + ** indices to be created and the table record must come before the + ** indices. Hence, the record number for the table must be allocated + ** now. + */ + if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ + int addr1; + int fileFormat; + int reg1, reg2, reg3; + /* nullRow[] is an OP_Record encoding of a row containing 5 NULLs */ + static const char nullRow[] = { 6, 0, 0, 0, 0, 0 }; + sqlite3BeginWriteOperation(pParse, 1, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( isVirtual ){ + sqlite3VdbeAddOp0(v, OP_VBegin); + } +#endif + + /* If the file format and encoding in the database have not been set, + ** set them now. + */ + reg1 = pParse->regRowid = ++pParse->nMem; + reg2 = pParse->regRoot = ++pParse->nMem; + reg3 = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + addr1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); + fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? + 1 : SQLITE_MAX_FILE_FORMAT; + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, fileFormat); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, ENC(db)); + sqlite3VdbeJumpHere(v, addr1); + + /* This just creates a place-holder record in the sqlite_schema table. + ** The record created does not contain anything yet. It will be replaced + ** by the real entry in code generated at sqlite3EndTable(). + ** + ** The rowid for the new entry is left in register pParse->regRowid. + ** The root page number of the new table is left in reg pParse->regRoot. + ** The rowid and root page number values are needed by the code that + ** sqlite3EndTable will generate. + */ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) + if( isView || isVirtual ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, reg2); + }else +#endif + { + assert( !pParse->bReturning ); + pParse->u1.addrCrTab = + sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY); + } + sqlite3OpenSchemaTable(pParse, iDb); + sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); + sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC); + sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3VdbeAddOp0(v, OP_Close); + } + + /* Normal (non-error) return. */ + return; + + /* If an error occurs, we jump here */ +begin_table_error: + pParse->checkSchema = 1; + sqlite3DbFree(db, zName); + return; +} + +/* Set properties of a table column based on the (magical) +** name of the column. +*/ +#if SQLITE_ENABLE_HIDDEN_COLUMNS +SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ + if( sqlite3_strnicmp(pCol->zCnName, "__hidden__", 10)==0 ){ + pCol->colFlags |= COLFLAG_HIDDEN; + if( pTab ) pTab->tabFlags |= TF_HasHidden; + }else if( pTab && pCol!=pTab->aCol && (pCol[-1].colFlags & COLFLAG_HIDDEN) ){ + pTab->tabFlags |= TF_OOOHidden; + } +} +#endif + +/* +** Name of the special TEMP trigger used to implement RETURNING. The +** name begins with "sqlite_" so that it is guaranteed not to collide +** with any application-generated triggers. +*/ +#define RETURNING_TRIGGER_NAME "sqlite_returning" + +/* +** Clean up the data structures associated with the RETURNING clause. +*/ +static void sqlite3DeleteReturning(sqlite3 *db, Returning *pRet){ + Hash *pHash; + pHash = &(db->aDb[1].pSchema->trigHash); + sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, 0); + sqlite3ExprListDelete(db, pRet->pReturnEL); + sqlite3DbFree(db, pRet); +} + +/* +** Add the RETURNING clause to the parse currently underway. +** +** This routine creates a special TEMP trigger that will fire for each row +** of the DML statement. That TEMP trigger contains a single SELECT +** statement with a result set that is the argument of the RETURNING clause. +** The trigger has the Trigger.bReturning flag and an opcode of +** TK_RETURNING instead of TK_SELECT, so that the trigger code generator +** knows to handle it specially. The TEMP trigger is automatically +** removed at the end of the parse. +** +** When this routine is called, we do not yet know if the RETURNING clause +** is attached to a DELETE, INSERT, or UPDATE, so construct it as a +** RETURNING trigger instead. It will then be converted into the appropriate +** type on the first call to sqlite3TriggersExist(). +*/ +SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ + Returning *pRet; + Hash *pHash; + sqlite3 *db = pParse->db; + if( pParse->pNewTrigger ){ + sqlite3ErrorMsg(pParse, "cannot use RETURNING in a trigger"); + }else{ + assert( pParse->bReturning==0 ); + } + pParse->bReturning = 1; + pRet = sqlite3DbMallocZero(db, sizeof(*pRet)); + if( pRet==0 ){ + sqlite3ExprListDelete(db, pList); + return; + } + pParse->u1.pReturning = pRet; + pRet->pParse = pParse; + pRet->pReturnEL = pList; + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3DeleteReturning, pRet); + testcase( pParse->earlyCleanup ); + if( db->mallocFailed ) return; + pRet->retTrig.zName = RETURNING_TRIGGER_NAME; + pRet->retTrig.op = TK_RETURNING; + pRet->retTrig.tr_tm = TRIGGER_AFTER; + pRet->retTrig.bReturning = 1; + pRet->retTrig.pSchema = db->aDb[1].pSchema; + pRet->retTrig.pTabSchema = db->aDb[1].pSchema; + pRet->retTrig.step_list = &pRet->retTStep; + pRet->retTStep.op = TK_RETURNING; + pRet->retTStep.pTrig = &pRet->retTrig; + pRet->retTStep.pExprList = pList; + pHash = &(db->aDb[1].pSchema->trigHash); + assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 || pParse->nErr ); + if( sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, &pRet->retTrig) + ==&pRet->retTrig ){ + sqlite3OomFault(db); + } +} + +/* +** Add a new column to the table currently being constructed. +** +** The parser calls this routine once for each column declaration +** in a CREATE TABLE statement. sqlite3StartTable() gets called +** first to get things going. Then this routine is called for each +** column. +*/ +SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ + Table *p; + int i; + char *z; + char *zType; + Column *pCol; + sqlite3 *db = pParse->db; + u8 hName; + Column *aNew; + u8 eType = COLTYPE_CUSTOM; + u8 szEst = 1; + char affinity = SQLITE_AFF_BLOB; + + if( (p = pParse->pNewTable)==0 ) return; + if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName); + return; + } + if( !IN_RENAME_OBJECT ) sqlite3DequoteToken(&sName); + + /* Because keywords GENERATE ALWAYS can be converted into indentifiers + ** by the parser, we can sometimes end up with a typename that ends + ** with "generated always". Check for this case and omit the surplus + ** text. */ + if( sType.n>=16 + && sqlite3_strnicmp(sType.z+(sType.n-6),"always",6)==0 + ){ + sType.n -= 6; + while( ALWAYS(sType.n>0) && sqlite3Isspace(sType.z[sType.n-1]) ) sType.n--; + if( sType.n>=9 + && sqlite3_strnicmp(sType.z+(sType.n-9),"generated",9)==0 + ){ + sType.n -= 9; + while( sType.n>0 && sqlite3Isspace(sType.z[sType.n-1]) ) sType.n--; + } + } + + /* Check for standard typenames. For standard typenames we will + ** set the Column.eType field rather than storing the typename after + ** the column name, in order to save space. */ + if( sType.n>=3 ){ + sqlite3DequoteToken(&sType); + for(i=0; i0) ); + if( z==0 ) return; + if( IN_RENAME_OBJECT ) sqlite3RenameTokenMap(pParse, (void*)z, &sName); + memcpy(z, sName.z, sName.n); + z[sName.n] = 0; + sqlite3Dequote(z); + hName = sqlite3StrIHash(z); + for(i=0; inCol; i++){ + if( p->aCol[i].hName==hName && sqlite3StrICmp(z, p->aCol[i].zCnName)==0 ){ + sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); + sqlite3DbFree(db, z); + return; + } + } + aNew = sqlite3DbRealloc(db,p->aCol,(p->nCol+1)*sizeof(p->aCol[0])); + if( aNew==0 ){ + sqlite3DbFree(db, z); + return; + } + p->aCol = aNew; + pCol = &p->aCol[p->nCol]; + memset(pCol, 0, sizeof(p->aCol[0])); + pCol->zCnName = z; + pCol->hName = hName; + sqlite3ColumnPropertiesFromName(p, pCol); + + if( sType.n==0 ){ + /* If there is no type specified, columns have the default affinity + ** 'BLOB' with a default size of 4 bytes. */ + pCol->affinity = affinity; + pCol->eCType = eType; + pCol->szEst = szEst; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( affinity==SQLITE_AFF_BLOB ){ + if( 4>=sqlite3GlobalConfig.szSorterRef ){ + pCol->colFlags |= COLFLAG_SORTERREF; + } + } +#endif + }else{ + zType = z + sqlite3Strlen30(z) + 1; + memcpy(zType, sType.z, sType.n); + zType[sType.n] = 0; + sqlite3Dequote(zType); + pCol->affinity = sqlite3AffinityType(zType, pCol); + pCol->colFlags |= COLFLAG_HASTYPE; + } + p->nCol++; + p->nNVCol++; + pParse->constraintName.n = 0; +} + +/* +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. A "NOT NULL" constraint has +** been seen on a column. This routine sets the notNull flag on +** the column currently under construction. +*/ +SQLITE_PRIVATE void sqlite3AddNotNull(Parse *pParse, int onError){ + Table *p; + Column *pCol; + p = pParse->pNewTable; + if( p==0 || NEVER(p->nCol<1) ) return; + pCol = &p->aCol[p->nCol-1]; + pCol->notNull = (u8)onError; + p->tabFlags |= TF_HasNotNull; + + /* Set the uniqNotNull flag on any UNIQUE or PK indexes already created + ** on this column. */ + if( pCol->colFlags & COLFLAG_UNIQUE ){ + Index *pIdx; + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->nKeyCol==1 && pIdx->onError!=OE_None ); + if( pIdx->aiColumn[0]==p->nCol-1 ){ + pIdx->uniqNotNull = 1; + } + } + } +} + +/* +** Scan the column type name zType (length nType) and return the +** associated affinity type. +** +** This routine does a case-independent search of zType for the +** substrings in the following table. If one of the substrings is +** found, the corresponding affinity is returned. If zType contains +** more than one of the substrings, entries toward the top of +** the table take priority. For example, if zType is 'BLOBINT', +** SQLITE_AFF_INTEGER is returned. +** +** Substring | Affinity +** -------------------------------- +** 'INT' | SQLITE_AFF_INTEGER +** 'CHAR' | SQLITE_AFF_TEXT +** 'CLOB' | SQLITE_AFF_TEXT +** 'TEXT' | SQLITE_AFF_TEXT +** 'BLOB' | SQLITE_AFF_BLOB +** 'REAL' | SQLITE_AFF_REAL +** 'FLOA' | SQLITE_AFF_REAL +** 'DOUB' | SQLITE_AFF_REAL +** +** If none of the substrings in the above table are found, +** SQLITE_AFF_NUMERIC is returned. +*/ +SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, Column *pCol){ + u32 h = 0; + char aff = SQLITE_AFF_NUMERIC; + const char *zChar = 0; + + assert( zIn!=0 ); + while( zIn[0] ){ + h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; + zIn++; + if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){ /* CHAR */ + aff = SQLITE_AFF_TEXT; + zChar = zIn; + }else if( h==(('c'<<24)+('l'<<16)+('o'<<8)+'b') ){ /* CLOB */ + aff = SQLITE_AFF_TEXT; + }else if( h==(('t'<<24)+('e'<<16)+('x'<<8)+'t') ){ /* TEXT */ + aff = SQLITE_AFF_TEXT; + }else if( h==(('b'<<24)+('l'<<16)+('o'<<8)+'b') /* BLOB */ + && (aff==SQLITE_AFF_NUMERIC || aff==SQLITE_AFF_REAL) ){ + aff = SQLITE_AFF_BLOB; + if( zIn[0]=='(' ) zChar = zIn; +#ifndef SQLITE_OMIT_FLOATING_POINT + }else if( h==(('r'<<24)+('e'<<16)+('a'<<8)+'l') /* REAL */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; + }else if( h==(('f'<<24)+('l'<<16)+('o'<<8)+'a') /* FLOA */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; + }else if( h==(('d'<<24)+('o'<<16)+('u'<<8)+'b') /* DOUB */ + && aff==SQLITE_AFF_NUMERIC ){ + aff = SQLITE_AFF_REAL; +#endif + }else if( (h&0x00FFFFFF)==(('i'<<16)+('n'<<8)+'t') ){ /* INT */ + aff = SQLITE_AFF_INTEGER; + break; + } + } + + /* If pCol is not NULL, store an estimate of the field size. The + ** estimate is scaled so that the size of an integer is 1. */ + if( pCol ){ + int v = 0; /* default size is approx 4 bytes */ + if( aff r=(k/4+1) */ + sqlite3GetInt32(zChar, &v); + break; + } + zChar++; + } + }else{ + v = 16; /* BLOB, TEXT, CLOB -> r=5 (approx 20 bytes)*/ + } + } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( v>=sqlite3GlobalConfig.szSorterRef ){ + pCol->colFlags |= COLFLAG_SORTERREF; + } +#endif + v = v/4 + 1; + if( v>255 ) v = 255; + pCol->szEst = v; + } + return aff; +} + +/* +** The expression is the default value for the most recently added column +** of the table currently under construction. +** +** Default value expressions must be constant. Raise an exception if this +** is not the case. +** +** This routine is called by the parser while in the middle of +** parsing a CREATE TABLE statement. +*/ +SQLITE_PRIVATE void sqlite3AddDefaultValue( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The parsed expression of the default value */ + const char *zStart, /* Start of the default value text */ + const char *zEnd /* First character past end of defaut value text */ +){ + Table *p; + Column *pCol; + sqlite3 *db = pParse->db; + p = pParse->pNewTable; + if( p!=0 ){ + int isInit = db->init.busy && db->init.iDb!=1; + pCol = &(p->aCol[p->nCol-1]); + if( !sqlite3ExprIsConstantOrFunction(pExpr, isInit) ){ + sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", + pCol->zCnName); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, "cannot use DEFAULT on a generated column"); +#endif + }else{ + /* A copy of pExpr is used instead of the original, as pExpr contains + ** tokens that point to volatile memory. + */ + Expr x, *pDfltExpr; + memset(&x, 0, sizeof(x)); + x.op = TK_SPAN; + x.u.zToken = sqlite3DbSpanDup(db, zStart, zEnd); + x.pLeft = pExpr; + x.flags = EP_Skip; + pDfltExpr = sqlite3ExprDup(db, &x, EXPRDUP_REDUCE); + sqlite3DbFree(db, x.u.zToken); + sqlite3ColumnSetExpr(pParse, p, pCol, pDfltExpr); + } + } + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprUnmap(pParse, pExpr); + } + sqlite3ExprDelete(db, pExpr); +} + +/* +** Backwards Compatibility Hack: +** +** Historical versions of SQLite accepted strings as column names in +** indexes and PRIMARY KEY constraints and in UNIQUE constraints. Example: +** +** CREATE TABLE xyz(a,b,c,d,e,PRIMARY KEY('a'),UNIQUE('b','c' COLLATE trim) +** CREATE INDEX abc ON xyz('c','d' DESC,'e' COLLATE nocase DESC); +** +** This is goofy. But to preserve backwards compatibility we continue to +** accept it. This routine does the necessary conversion. It converts +** the expression given in its argument from a TK_STRING into a TK_ID +** if the expression is just a TK_STRING with an optional COLLATE clause. +** If the expression is anything other than TK_STRING, the expression is +** unchanged. +*/ +static void sqlite3StringToId(Expr *p){ + if( p->op==TK_STRING ){ + p->op = TK_ID; + }else if( p->op==TK_COLLATE && p->pLeft->op==TK_STRING ){ + p->pLeft->op = TK_ID; + } +} + +/* +** Tag the given column as being part of the PRIMARY KEY +*/ +static void makeColumnPartOfPrimaryKey(Parse *pParse, Column *pCol){ + pCol->colFlags |= COLFLAG_PRIMKEY; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, + "generated columns cannot be part of the PRIMARY KEY"); + } +#endif +} + +/* +** Designate the PRIMARY KEY for the table. pList is a list of names +** of columns that form the primary key. If pList is NULL, then the +** most recently added column of the table is the primary key. +** +** A table can have at most one primary key. If the table already has +** a primary key (and this is the second primary key) then create an +** error. +** +** If the PRIMARY KEY is on a single column whose datatype is INTEGER, +** then we will try to use that column as the rowid. Set the Table.iPKey +** field of the table under construction to be the index of the +** INTEGER PRIMARY KEY column. Table.iPKey is set to -1 if there is +** no INTEGER PRIMARY KEY. +** +** If the key is not an INTEGER PRIMARY KEY, then create a unique +** index for the key. No index is created for INTEGER PRIMARY KEYs. +*/ +SQLITE_PRIVATE void sqlite3AddPrimaryKey( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List of field names to be indexed */ + int onError, /* What to do with a uniqueness conflict */ + int autoInc, /* True if the AUTOINCREMENT keyword is present */ + int sortOrder /* SQLITE_SO_ASC or SQLITE_SO_DESC */ +){ + Table *pTab = pParse->pNewTable; + Column *pCol = 0; + int iCol = -1, i; + int nTerm; + if( pTab==0 ) goto primary_key_exit; + if( pTab->tabFlags & TF_HasPrimaryKey ){ + sqlite3ErrorMsg(pParse, + "table \"%s\" has more than one primary key", pTab->zName); + goto primary_key_exit; + } + pTab->tabFlags |= TF_HasPrimaryKey; + if( pList==0 ){ + iCol = pTab->nCol - 1; + pCol = &pTab->aCol[iCol]; + makeColumnPartOfPrimaryKey(pParse, pCol); + nTerm = 1; + }else{ + nTerm = pList->nExpr; + for(i=0; ia[i].pExpr); + assert( pCExpr!=0 ); + sqlite3StringToId(pCExpr); + if( pCExpr->op==TK_ID ){ + const char *zCName = pCExpr->u.zToken; + for(iCol=0; iColnCol; iCol++){ + if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zCnName)==0 ){ + pCol = &pTab->aCol[iCol]; + makeColumnPartOfPrimaryKey(pParse, pCol); + break; + } + } + } + } + } + if( nTerm==1 + && pCol + && pCol->eCType==COLTYPE_INTEGER + && sortOrder!=SQLITE_SO_DESC + ){ + if( IN_RENAME_OBJECT && pList ){ + Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[0].pExpr); + sqlite3RenameTokenRemap(pParse, &pTab->iPKey, pCExpr); + } + pTab->iPKey = iCol; + pTab->keyConf = (u8)onError; + assert( autoInc==0 || autoInc==1 ); + pTab->tabFlags |= autoInc*TF_Autoincrement; + if( pList ) pParse->iPkSortOrder = pList->a[0].sortFlags; + (void)sqlite3HasExplicitNulls(pParse, pList); + }else if( autoInc ){ +#ifndef SQLITE_OMIT_AUTOINCREMENT + sqlite3ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " + "INTEGER PRIMARY KEY"); +#endif + }else{ + sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, + 0, sortOrder, 0, SQLITE_IDXTYPE_PRIMARYKEY); + pList = 0; + } + +primary_key_exit: + sqlite3ExprListDelete(pParse->db, pList); + return; +} + +/* +** Add a new CHECK constraint to the table currently under construction. +*/ +SQLITE_PRIVATE void sqlite3AddCheckConstraint( + Parse *pParse, /* Parsing context */ + Expr *pCheckExpr, /* The check expression */ + const char *zStart, /* Opening "(" */ + const char *zEnd /* Closing ")" */ +){ +#ifndef SQLITE_OMIT_CHECK + Table *pTab = pParse->pNewTable; + sqlite3 *db = pParse->db; + if( pTab && !IN_DECLARE_VTAB + && !sqlite3BtreeIsReadonly(db->aDb[db->init.iDb].pBt) + ){ + pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); + if( pParse->constraintName.n ){ + sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + }else{ + Token t; + for(zStart++; sqlite3Isspace(zStart[0]); zStart++){} + while( sqlite3Isspace(zEnd[-1]) ){ zEnd--; } + t.z = zStart; + t.n = (int)(zEnd - t.z); + sqlite3ExprListSetName(pParse, pTab->pCheck, &t, 1); + } + }else +#endif + { + sqlite3ExprDelete(pParse->db, pCheckExpr); + } +} + +/* +** Set the collation function of the most recently parsed table column +** to the CollSeq given. +*/ +SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){ + Table *p; + int i; + char *zColl; /* Dequoted name of collation sequence */ + sqlite3 *db; + + if( (p = pParse->pNewTable)==0 || IN_RENAME_OBJECT ) return; + i = p->nCol-1; + db = pParse->db; + zColl = sqlite3NameFromToken(db, pToken); + if( !zColl ) return; + + if( sqlite3LocateCollSeq(pParse, zColl) ){ + Index *pIdx; + sqlite3ColumnSetColl(db, &p->aCol[i], zColl); + + /* If the column is declared as " PRIMARY KEY COLLATE ", + ** then an index may have been created on this column before the + ** collation type was added. Correct this if it is the case. + */ + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->nKeyCol==1 ); + if( pIdx->aiColumn[0]==i ){ + pIdx->azColl[0] = sqlite3ColumnColl(&p->aCol[i]); + } + } + } + sqlite3DbFree(db, zColl); +} + +/* Change the most recently parsed column to be a GENERATED ALWAYS AS +** column. +*/ +SQLITE_PRIVATE void sqlite3AddGenerated(Parse *pParse, Expr *pExpr, Token *pType){ +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + u8 eType = COLFLAG_VIRTUAL; + Table *pTab = pParse->pNewTable; + Column *pCol; + if( pTab==0 ){ + /* generated column in an CREATE TABLE IF NOT EXISTS that already exists */ + goto generated_done; + } + pCol = &(pTab->aCol[pTab->nCol-1]); + if( IN_DECLARE_VTAB ){ + sqlite3ErrorMsg(pParse, "virtual tables cannot use computed columns"); + goto generated_done; + } + if( pCol->iDflt>0 ) goto generated_error; + if( pType ){ + if( pType->n==7 && sqlite3StrNICmp("virtual",pType->z,7)==0 ){ + /* no-op */ + }else if( pType->n==6 && sqlite3StrNICmp("stored",pType->z,6)==0 ){ + eType = COLFLAG_STORED; + }else{ + goto generated_error; + } + } + if( eType==COLFLAG_VIRTUAL ) pTab->nNVCol--; + pCol->colFlags |= eType; + assert( TF_HasVirtual==COLFLAG_VIRTUAL ); + assert( TF_HasStored==COLFLAG_STORED ); + pTab->tabFlags |= eType; + if( pCol->colFlags & COLFLAG_PRIMKEY ){ + makeColumnPartOfPrimaryKey(pParse, pCol); /* For the error message */ + } + sqlite3ColumnSetExpr(pParse, pTab, pCol, pExpr); + pExpr = 0; + goto generated_done; + +generated_error: + sqlite3ErrorMsg(pParse, "error in generated column \"%s\"", + pCol->zCnName); +generated_done: + sqlite3ExprDelete(pParse->db, pExpr); +#else + /* Throw and error for the GENERATED ALWAYS AS clause if the + ** SQLITE_OMIT_GENERATED_COLUMNS compile-time option is used. */ + sqlite3ErrorMsg(pParse, "generated columns not supported"); + sqlite3ExprDelete(pParse->db, pExpr); +#endif +} + +/* +** Generate code that will increment the schema cookie. +** +** The schema cookie is used to determine when the schema for the +** database changes. After each schema change, the cookie value +** changes. When a process first reads the schema it records the +** cookie. Thereafter, whenever it goes to access the database, +** it checks the cookie to make sure the schema has not changed +** since it was last read. +** +** This plan is not completely bullet-proof. It is possible for +** the schema to change multiple times and for the cookie to be +** set back to prior value. But schema changes are infrequent +** and the probability of hitting the same cookie value is only +** 1 chance in 2^32. So we're safe enough. +** +** IMPLEMENTATION-OF: R-34230-56049 SQLite automatically increments +** the schema-version whenever the schema changes. +*/ +SQLITE_PRIVATE void sqlite3ChangeCookie(Parse *pParse, int iDb){ + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, + (int)(1+(unsigned)db->aDb[iDb].pSchema->schema_cookie)); +} + +/* +** Measure the number of characters needed to output the given +** identifier. The number returned includes any quotes used +** but does not include the null terminator. +** +** The estimate is conservative. It might be larger that what is +** really needed. +*/ +static int identLength(const char *z){ + int n; + for(n=0; *z; n++, z++){ + if( *z=='"' ){ n++; } + } + return n + 2; +} + +/* +** The first parameter is a pointer to an output buffer. The second +** parameter is a pointer to an integer that contains the offset at +** which to write into the output buffer. This function copies the +** nul-terminated string pointed to by the third parameter, zSignedIdent, +** to the specified offset in the buffer and updates *pIdx to refer +** to the first byte after the last byte written before returning. +** +** If the string zSignedIdent consists entirely of alpha-numeric +** characters, does not begin with a digit and is not an SQL keyword, +** then it is copied to the output buffer exactly as it is. Otherwise, +** it is quoted using double-quotes. +*/ +static void identPut(char *z, int *pIdx, char *zSignedIdent){ + unsigned char *zIdent = (unsigned char*)zSignedIdent; + int i, j, needQuote; + i = *pIdx; + + for(j=0; zIdent[j]; j++){ + if( !sqlite3Isalnum(zIdent[j]) && zIdent[j]!='_' ) break; + } + needQuote = sqlite3Isdigit(zIdent[0]) + || sqlite3KeywordCode(zIdent, j)!=TK_ID + || zIdent[j]!=0 + || j==0; + + if( needQuote ) z[i++] = '"'; + for(j=0; zIdent[j]; j++){ + z[i++] = zIdent[j]; + if( zIdent[j]=='"' ) z[i++] = '"'; + } + if( needQuote ) z[i++] = '"'; + z[i] = 0; + *pIdx = i; +} + +/* +** Generate a CREATE TABLE statement appropriate for the given +** table. Memory to hold the text of the statement is obtained +** from sqliteMalloc() and must be freed by the calling function. +*/ +static char *createTableStmt(sqlite3 *db, Table *p){ + int i, k, n; + char *zStmt; + char *zSep, *zSep2, *zEnd; + Column *pCol; + n = 0; + for(pCol = p->aCol, i=0; inCol; i++, pCol++){ + n += identLength(pCol->zCnName) + 5; + } + n += identLength(p->zName); + if( n<50 ){ + zSep = ""; + zSep2 = ","; + zEnd = ")"; + }else{ + zSep = "\n "; + zSep2 = ",\n "; + zEnd = "\n)"; + } + n += 35 + 6*p->nCol; + zStmt = sqlite3DbMallocRaw(0, n); + if( zStmt==0 ){ + sqlite3OomFault(db); + return 0; + } + sqlite3_snprintf(n, zStmt, "CREATE TABLE "); + k = sqlite3Strlen30(zStmt); + identPut(zStmt, &k, p->zName); + zStmt[k++] = '('; + for(pCol=p->aCol, i=0; inCol; i++, pCol++){ + static const char * const azType[] = { + /* SQLITE_AFF_BLOB */ "", + /* SQLITE_AFF_TEXT */ " TEXT", + /* SQLITE_AFF_NUMERIC */ " NUM", + /* SQLITE_AFF_INTEGER */ " INT", + /* SQLITE_AFF_REAL */ " REAL" + }; + int len; + const char *zType; + + sqlite3_snprintf(n-k, &zStmt[k], zSep); + k += sqlite3Strlen30(&zStmt[k]); + zSep = zSep2; + identPut(zStmt, &k, pCol->zCnName); + assert( pCol->affinity-SQLITE_AFF_BLOB >= 0 ); + assert( pCol->affinity-SQLITE_AFF_BLOB < ArraySize(azType) ); + testcase( pCol->affinity==SQLITE_AFF_BLOB ); + testcase( pCol->affinity==SQLITE_AFF_TEXT ); + testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); + testcase( pCol->affinity==SQLITE_AFF_INTEGER ); + testcase( pCol->affinity==SQLITE_AFF_REAL ); + + zType = azType[pCol->affinity - SQLITE_AFF_BLOB]; + len = sqlite3Strlen30(zType); + assert( pCol->affinity==SQLITE_AFF_BLOB + || pCol->affinity==sqlite3AffinityType(zType, 0) ); + memcpy(&zStmt[k], zType, len); + k += len; + assert( k<=n ); + } + sqlite3_snprintf(n-k, &zStmt[k], "%s", zEnd); + return zStmt; +} + +/* +** Resize an Index object to hold N columns total. Return SQLITE_OK +** on success and SQLITE_NOMEM on an OOM error. +*/ +static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ + char *zExtra; + int nByte; + if( pIdx->nColumn>=N ) return SQLITE_OK; + assert( pIdx->isResized==0 ); + nByte = (sizeof(char*) + sizeof(LogEst) + sizeof(i16) + 1)*N; + zExtra = sqlite3DbMallocZero(db, nByte); + if( zExtra==0 ) return SQLITE_NOMEM_BKPT; + memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); + pIdx->azColl = (const char**)zExtra; + zExtra += sizeof(char*)*N; + memcpy(zExtra, pIdx->aiRowLogEst, sizeof(LogEst)*(pIdx->nKeyCol+1)); + pIdx->aiRowLogEst = (LogEst*)zExtra; + zExtra += sizeof(LogEst)*N; + memcpy(zExtra, pIdx->aiColumn, sizeof(i16)*pIdx->nColumn); + pIdx->aiColumn = (i16*)zExtra; + zExtra += sizeof(i16)*N; + memcpy(zExtra, pIdx->aSortOrder, pIdx->nColumn); + pIdx->aSortOrder = (u8*)zExtra; + pIdx->nColumn = N; + pIdx->isResized = 1; + return SQLITE_OK; +} + +/* +** Estimate the total row width for a table. +*/ +static void estimateTableWidth(Table *pTab){ + unsigned wTable = 0; + const Column *pTabCol; + int i; + for(i=pTab->nCol, pTabCol=pTab->aCol; i>0; i--, pTabCol++){ + wTable += pTabCol->szEst; + } + if( pTab->iPKey<0 ) wTable++; + pTab->szTabRow = sqlite3LogEst(wTable*4); +} + +/* +** Estimate the average size of a row for an index. +*/ +static void estimateIndexWidth(Index *pIdx){ + unsigned wIndex = 0; + int i; + const Column *aCol = pIdx->pTable->aCol; + for(i=0; inColumn; i++){ + i16 x = pIdx->aiColumn[i]; + assert( xpTable->nCol ); + wIndex += x<0 ? 1 : aCol[pIdx->aiColumn[i]].szEst; + } + pIdx->szIdxRow = sqlite3LogEst(wIndex*4); +} + +/* Return true if column number x is any of the first nCol entries of aiCol[]. +** This is used to determine if the column number x appears in any of the +** first nCol entries of an index. +*/ +static int hasColumn(const i16 *aiCol, int nCol, int x){ + while( nCol-- > 0 ){ + assert( aiCol[0]>=0 ); + if( x==*(aiCol++) ){ + return 1; + } + } + return 0; +} + +/* +** Return true if any of the first nKey entries of index pIdx exactly +** match the iCol-th entry of pPk. pPk is always a WITHOUT ROWID +** PRIMARY KEY index. pIdx is an index on the same table. pIdx may +** or may not be the same index as pPk. +** +** The first nKey entries of pIdx are guaranteed to be ordinary columns, +** not a rowid or expression. +** +** This routine differs from hasColumn() in that both the column and the +** collating sequence must match for this routine, but for hasColumn() only +** the column name must match. +*/ +static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ + int i, j; + assert( nKey<=pIdx->nColumn ); + assert( iColnColumn,pPk->nKeyCol) ); + assert( pPk->idxType==SQLITE_IDXTYPE_PRIMARYKEY ); + assert( pPk->pTable->tabFlags & TF_WithoutRowid ); + assert( pPk->pTable==pIdx->pTable ); + testcase( pPk==pIdx ); + j = pPk->aiColumn[iCol]; + assert( j!=XN_ROWID && j!=XN_EXPR ); + for(i=0; iaiColumn[i]>=0 || j>=0 ); + if( pIdx->aiColumn[i]==j + && sqlite3StrICmp(pIdx->azColl[i], pPk->azColl[iCol])==0 + ){ + return 1; + } + } + return 0; +} + +/* Recompute the colNotIdxed field of the Index. +** +** colNotIdxed is a bitmask that has a 0 bit representing each indexed +** columns that are within the first 63 columns of the table. The +** high-order bit of colNotIdxed is always 1. All unindexed columns +** of the table have a 1. +** +** 2019-10-24: For the purpose of this computation, virtual columns are +** not considered to be covered by the index, even if they are in the +** index, because we do not trust the logic in whereIndexExprTrans() to be +** able to find all instances of a reference to the indexed table column +** and convert them into references to the index. Hence we always want +** the actual table at hand in order to recompute the virtual column, if +** necessary. +** +** The colNotIdxed mask is AND-ed with the SrcList.a[].colUsed mask +** to determine if the index is covering index. +*/ +static void recomputeColumnsNotIndexed(Index *pIdx){ + Bitmask m = 0; + int j; + Table *pTab = pIdx->pTable; + for(j=pIdx->nColumn-1; j>=0; j--){ + int x = pIdx->aiColumn[j]; + if( x>=0 && (pTab->aCol[x].colFlags & COLFLAG_VIRTUAL)==0 ){ + testcase( x==BMS-1 ); + testcase( x==BMS-2 ); + if( xcolNotIdxed = ~m; + assert( (pIdx->colNotIdxed>>63)==1 ); +} + +/* +** This routine runs at the end of parsing a CREATE TABLE statement that +** has a WITHOUT ROWID clause. The job of this routine is to convert both +** internal schema data structures and the generated VDBE code so that they +** are appropriate for a WITHOUT ROWID table instead of a rowid table. +** Changes include: +** +** (1) Set all columns of the PRIMARY KEY schema object to be NOT NULL. +** (2) Convert P3 parameter of the OP_CreateBtree from BTREE_INTKEY +** into BTREE_BLOBKEY. +** (3) Bypass the creation of the sqlite_schema table entry +** for the PRIMARY KEY as the primary key index is now +** identified by the sqlite_schema table entry of the table itself. +** (4) Set the Index.tnum of the PRIMARY KEY Index object in the +** schema to the rootpage from the main table. +** (5) Add all table columns to the PRIMARY KEY Index object +** so that the PRIMARY KEY is a covering index. The surplus +** columns are part of KeyInfo.nAllField and are not used for +** sorting or lookup or uniqueness checks. +** (6) Replace the rowid tail on all automatically generated UNIQUE +** indices with the PRIMARY KEY columns. +** +** For virtual tables, only (1) is performed. +*/ +static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ + Index *pIdx; + Index *pPk; + int nPk; + int nExtra; + int i, j; + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + + /* Mark every PRIMARY KEY column as NOT NULL (except for imposter tables) + */ + if( !db->init.imposterTable ){ + for(i=0; inCol; i++){ + if( (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 + && (pTab->aCol[i].notNull==OE_None) + ){ + pTab->aCol[i].notNull = OE_Abort; + } + } + pTab->tabFlags |= TF_HasNotNull; + } + + /* Convert the P3 operand of the OP_CreateBtree opcode from BTREE_INTKEY + ** into BTREE_BLOBKEY. + */ + assert( !pParse->bReturning ); + if( pParse->u1.addrCrTab ){ + assert( v ); + sqlite3VdbeChangeP3(v, pParse->u1.addrCrTab, BTREE_BLOBKEY); + } + + /* Locate the PRIMARY KEY index. Or, if this table was originally + ** an INTEGER PRIMARY KEY table, create a new PRIMARY KEY index. + */ + if( pTab->iPKey>=0 ){ + ExprList *pList; + Token ipkToken; + sqlite3TokenInit(&ipkToken, pTab->aCol[pTab->iPKey].zCnName); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); + if( pList==0 ){ + pTab->tabFlags &= ~TF_WithoutRowid; + return; + } + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, pList->a[0].pExpr, &pTab->iPKey); + } + pList->a[0].sortFlags = pParse->iPkSortOrder; + assert( pParse->pNewTable==pTab ); + pTab->iPKey = -1; + sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0, + SQLITE_IDXTYPE_PRIMARYKEY); + if( db->mallocFailed || pParse->nErr ){ + pTab->tabFlags &= ~TF_WithoutRowid; + return; + } + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk->nKeyCol==1 ); + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + + /* + ** Remove all redundant columns from the PRIMARY KEY. For example, change + ** "PRIMARY KEY(a,b,a,b,c,b,c,d)" into just "PRIMARY KEY(a,b,c,d)". Later + ** code assumes the PRIMARY KEY contains no repeated columns. + */ + for(i=j=1; inKeyCol; i++){ + if( isDupColumn(pPk, j, pPk, i) ){ + pPk->nColumn--; + }else{ + testcase( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ); + pPk->azColl[j] = pPk->azColl[i]; + pPk->aSortOrder[j] = pPk->aSortOrder[i]; + pPk->aiColumn[j++] = pPk->aiColumn[i]; + } + } + pPk->nKeyCol = j; + } + assert( pPk!=0 ); + pPk->isCovering = 1; + if( !db->init.imposterTable ) pPk->uniqNotNull = 1; + nPk = pPk->nColumn = pPk->nKeyCol; + + /* Bypass the creation of the PRIMARY KEY btree and the sqlite_schema + ** table entry. This is only required if currently generating VDBE + ** code for a CREATE TABLE (not when parsing one as part of reading + ** a database schema). */ + if( v && pPk->tnum>0 ){ + assert( db->init.busy==0 ); + sqlite3VdbeChangeOpcode(v, (int)pPk->tnum, OP_Goto); + } + + /* The root page of the PRIMARY KEY is the table root page */ + pPk->tnum = pTab->tnum; + + /* Update the in-memory representation of all UNIQUE indices by converting + ** the final rowid column into one or more columns of the PRIMARY KEY. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int n; + if( IsPrimaryKeyIndex(pIdx) ) continue; + for(i=n=0; inKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); + n++; + } + } + if( n==0 ){ + /* This index is a superset of the primary key */ + pIdx->nColumn = pIdx->nKeyCol; + continue; + } + if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; + for(i=0, j=pIdx->nKeyCol; inKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); + pIdx->aiColumn[j] = pPk->aiColumn[i]; + pIdx->azColl[j] = pPk->azColl[i]; + if( pPk->aSortOrder[i] ){ + /* See ticket https://www.sqlite.org/src/info/bba7b69f9849b5bf */ + pIdx->bAscKeyBug = 1; + } + j++; + } + } + assert( pIdx->nColumn>=pIdx->nKeyCol+n ); + assert( pIdx->nColumn>=j ); + } + + /* Add all table columns to the PRIMARY KEY index + */ + nExtra = 0; + for(i=0; inCol; i++){ + if( !hasColumn(pPk->aiColumn, nPk, i) + && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) nExtra++; + } + if( resizeIndexObject(db, pPk, nPk+nExtra) ) return; + for(i=0, j=nPk; inCol; i++){ + if( !hasColumn(pPk->aiColumn, j, i) + && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 + ){ + assert( jnColumn ); + pPk->aiColumn[j] = i; + pPk->azColl[j] = sqlite3StrBINARY; + j++; + } + } + assert( pPk->nColumn==j ); + assert( pTab->nNVCol<=j ); + recomputeColumnsNotIndexed(pPk); +} + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return true if pTab is a virtual table and zName is a shadow table name +** for that virtual table. +*/ +SQLITE_PRIVATE int sqlite3IsShadowTableOf(sqlite3 *db, Table *pTab, const char *zName){ + int nName; /* Length of zName */ + Module *pMod; /* Module for the virtual table */ + + if( !IsVirtual(pTab) ) return 0; + nName = sqlite3Strlen30(pTab->zName); + if( sqlite3_strnicmp(zName, pTab->zName, nName)!=0 ) return 0; + if( zName[nName]!='_' ) return 0; + pMod = (Module*)sqlite3HashFind(&db->aModule, pTab->u.vtab.azArg[0]); + if( pMod==0 ) return 0; + if( pMod->pModule->iVersion<3 ) return 0; + if( pMod->pModule->xShadowName==0 ) return 0; + return pMod->pModule->xShadowName(zName+nName+1); +} +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return true if zName is a shadow table name in the current database +** connection. +** +** zName is temporarily modified while this routine is running, but is +** restored to its original value prior to this routine returning. +*/ +SQLITE_PRIVATE int sqlite3ShadowTableName(sqlite3 *db, const char *zName){ + char *zTail; /* Pointer to the last "_" in zName */ + Table *pTab; /* Table that zName is a shadow of */ + zTail = strrchr(zName, '_'); + if( zTail==0 ) return 0; + *zTail = 0; + pTab = sqlite3FindTable(db, zName, 0); + *zTail = '_'; + if( pTab==0 ) return 0; + if( !IsVirtual(pTab) ) return 0; + return sqlite3IsShadowTableOf(db, pTab, zName); +} +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + + +#ifdef SQLITE_DEBUG +/* +** Mark all nodes of an expression as EP_Immutable, indicating that +** they should not be changed. Expressions attached to a table or +** index definition are tagged this way to help ensure that we do +** not pass them into code generator routines by mistake. +*/ +static int markImmutableExprStep(Walker *pWalker, Expr *pExpr){ + ExprSetVVAProperty(pExpr, EP_Immutable); + return WRC_Continue; +} +static void markExprListImmutable(ExprList *pList){ + if( pList ){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = markImmutableExprStep; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.xSelectCallback2 = 0; + sqlite3WalkExprList(&w, pList); + } +} +#else +#define markExprListImmutable(X) /* no-op */ +#endif /* SQLITE_DEBUG */ + + +/* +** This routine is called to report the final ")" that terminates +** a CREATE TABLE statement. +** +** The table structure that other action routines have been building +** is added to the internal hash tables, assuming no errors have +** occurred. +** +** An entry for the table is made in the schema table on disk, unless +** this is a temporary table or db->init.busy==1. When db->init.busy==1 +** it means we are reading the sqlite_schema table because we just +** connected to the database or because the sqlite_schema table has +** recently changed, so the entry for this table already exists in +** the sqlite_schema table. We do not want to create it again. +** +** If the pSelect argument is not NULL, it means that this routine +** was called to create a table generated from a +** "CREATE TABLE ... AS SELECT ..." statement. The column names of +** the new table will match the result set of the SELECT. +*/ +SQLITE_PRIVATE void sqlite3EndTable( + Parse *pParse, /* Parse context */ + Token *pCons, /* The ',' token after the last column defn. */ + Token *pEnd, /* The ')' before options in the CREATE TABLE */ + u32 tabOpts, /* Extra table options. Usually 0. */ + Select *pSelect /* Select from a "CREATE ... AS SELECT" */ +){ + Table *p; /* The new table */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb; /* Database in which the table lives */ + Index *pIdx; /* An implied index of the table */ + + if( pEnd==0 && pSelect==0 ){ + return; + } + p = pParse->pNewTable; + if( p==0 ) return; + + if( pSelect==0 && sqlite3ShadowTableName(db, p->zName) ){ + p->tabFlags |= TF_Shadow; + } + + /* If the db->init.busy is 1 it means we are reading the SQL off the + ** "sqlite_schema" or "sqlite_temp_schema" table on the disk. + ** So do not write to the disk again. Extract the root page number + ** for the table from the db->init.newTnum field. (The page number + ** should have been put there by the sqliteOpenCb routine.) + ** + ** If the root page number is 1, that means this is the sqlite_schema + ** table itself. So mark it read-only. + */ + if( db->init.busy ){ + if( pSelect ){ + sqlite3ErrorMsg(pParse, ""); + return; + } + p->tnum = db->init.newTnum; + if( p->tnum==1 ) p->tabFlags |= TF_Readonly; + } + + /* Special processing for tables that include the STRICT keyword: + ** + ** * Do not allow custom column datatypes. Every column must have + ** a datatype that is one of INT, INTEGER, REAL, TEXT, or BLOB. + ** + ** * If a PRIMARY KEY is defined, other than the INTEGER PRIMARY KEY, + ** then all columns of the PRIMARY KEY must have a NOT NULL + ** constraint. + */ + if( tabOpts & TF_Strict ){ + int ii; + p->tabFlags |= TF_Strict; + for(ii=0; iinCol; ii++){ + Column *pCol = &p->aCol[ii]; + if( pCol->eCType==COLTYPE_CUSTOM ){ + if( pCol->colFlags & COLFLAG_HASTYPE ){ + sqlite3ErrorMsg(pParse, + "unknown datatype for %s.%s: \"%s\"", + p->zName, pCol->zCnName, sqlite3ColumnType(pCol, "") + ); + }else{ + sqlite3ErrorMsg(pParse, "missing datatype for %s.%s", + p->zName, pCol->zCnName); + } + return; + }else if( pCol->eCType==COLTYPE_ANY ){ + pCol->affinity = SQLITE_AFF_BLOB; + } + if( (pCol->colFlags & COLFLAG_PRIMKEY)!=0 + && p->iPKey!=ii + && pCol->notNull == OE_None + ){ + pCol->notNull = OE_Abort; + p->tabFlags |= TF_HasNotNull; + } + } + } + + assert( (p->tabFlags & TF_HasPrimaryKey)==0 + || p->iPKey>=0 || sqlite3PrimaryKeyIndex(p)!=0 ); + assert( (p->tabFlags & TF_HasPrimaryKey)!=0 + || (p->iPKey<0 && sqlite3PrimaryKeyIndex(p)==0) ); + + /* Special processing for WITHOUT ROWID Tables */ + if( tabOpts & TF_WithoutRowid ){ + if( (p->tabFlags & TF_Autoincrement) ){ + sqlite3ErrorMsg(pParse, + "AUTOINCREMENT not allowed on WITHOUT ROWID tables"); + return; + } + if( (p->tabFlags & TF_HasPrimaryKey)==0 ){ + sqlite3ErrorMsg(pParse, "PRIMARY KEY missing on table %s", p->zName); + return; + } + p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; + convertToWithoutRowidTable(pParse, p); + } + iDb = sqlite3SchemaToIndex(db, p->pSchema); + +#ifndef SQLITE_OMIT_CHECK + /* Resolve names in all CHECK constraint expressions. + */ + if( p->pCheck ){ + sqlite3ResolveSelfReference(pParse, p, NC_IsCheck, 0, p->pCheck); + if( pParse->nErr ){ + /* If errors are seen, delete the CHECK constraints now, else they might + ** actually be used if PRAGMA writable_schema=ON is set. */ + sqlite3ExprListDelete(db, p->pCheck); + p->pCheck = 0; + }else{ + markExprListImmutable(p->pCheck); + } + } +#endif /* !defined(SQLITE_OMIT_CHECK) */ +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( p->tabFlags & TF_HasGenerated ){ + int ii, nNG = 0; + testcase( p->tabFlags & TF_HasVirtual ); + testcase( p->tabFlags & TF_HasStored ); + for(ii=0; iinCol; ii++){ + u32 colFlags = p->aCol[ii].colFlags; + if( (colFlags & COLFLAG_GENERATED)!=0 ){ + Expr *pX = sqlite3ColumnExpr(p, &p->aCol[ii]); + testcase( colFlags & COLFLAG_VIRTUAL ); + testcase( colFlags & COLFLAG_STORED ); + if( sqlite3ResolveSelfReference(pParse, p, NC_GenCol, pX, 0) ){ + /* If there are errors in resolving the expression, change the + ** expression to a NULL. This prevents code generators that operate + ** on the expression from inserting extra parts into the expression + ** tree that have been allocated from lookaside memory, which is + ** illegal in a schema and will lead to errors or heap corruption + ** when the database connection closes. */ + sqlite3ColumnSetExpr(pParse, p, &p->aCol[ii], + sqlite3ExprAlloc(db, TK_NULL, 0, 0)); + } + }else{ + nNG++; + } + } + if( nNG==0 ){ + sqlite3ErrorMsg(pParse, "must have at least one non-generated column"); + return; + } + } +#endif + + /* Estimate the average row size for the table and for all implied indices */ + estimateTableWidth(p); + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + estimateIndexWidth(pIdx); + } + + /* If not initializing, then create a record for the new table + ** in the schema table of the database. + ** + ** If this is a TEMPORARY table, write the entry into the auxiliary + ** file instead of into the main database file. + */ + if( !db->init.busy ){ + int n; + Vdbe *v; + char *zType; /* "view" or "table" */ + char *zType2; /* "VIEW" or "TABLE" */ + char *zStmt; /* Text of the CREATE TABLE or CREATE VIEW statement */ + + v = sqlite3GetVdbe(pParse); + if( NEVER(v==0) ) return; + + sqlite3VdbeAddOp1(v, OP_Close, 0); + + /* + ** Initialize zType for the new view or table. + */ + if( IsOrdinaryTable(p) ){ + /* A regular table */ + zType = "table"; + zType2 = "TABLE"; +#ifndef SQLITE_OMIT_VIEW + }else{ + /* A view */ + zType = "view"; + zType2 = "VIEW"; +#endif + } + + /* If this is a CREATE TABLE xx AS SELECT ..., execute the SELECT + ** statement to populate the new table. The root-page number for the + ** new table is in register pParse->regRoot. + ** + ** Once the SELECT has been coded by sqlite3Select(), it is in a + ** suitable state to query for the column names and types to be used + ** by the new table. + ** + ** A shared-cache write-lock is not required to write to the new table, + ** as a schema-lock must have already been obtained to create it. Since + ** a schema-lock excludes all other database users, the write-lock would + ** be redundant. + */ + if( pSelect ){ + SelectDest dest; /* Where the SELECT should store results */ + int regYield; /* Register holding co-routine entry-point */ + int addrTop; /* Top of the co-routine */ + int regRec; /* A record to be insert into the new table */ + int regRowid; /* Rowid of the next row to insert */ + int addrInsLoop; /* Top of the loop for inserting rows */ + Table *pSelTab; /* A table that describes the SELECT results */ + + regYield = ++pParse->nMem; + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; + assert(pParse->nTab==1); + sqlite3MayAbort(pParse); + sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); + pParse->nTab = 2; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + if( pParse->nErr ) return; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect, SQLITE_AFF_BLOB); + if( pSelTab==0 ) return; + assert( p->aCol==0 ); + p->nCol = p->nNVCol = pSelTab->nCol; + p->aCol = pSelTab->aCol; + pSelTab->nCol = 0; + pSelTab->aCol = 0; + sqlite3DeleteTable(db, pSelTab); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + sqlite3Select(pParse, pSelect, &dest); + if( pParse->nErr ) return; + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); + addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); + sqlite3TableAffinity(v, p, 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeGoto(v, addrInsLoop); + sqlite3VdbeJumpHere(v, addrInsLoop); + sqlite3VdbeAddOp1(v, OP_Close, 1); + } + + /* Compute the complete text of the CREATE statement */ + if( pSelect ){ + zStmt = createTableStmt(db, p); + }else{ + Token *pEnd2 = tabOpts ? &pParse->sLastToken : pEnd; + n = (int)(pEnd2->z - pParse->sNameToken.z); + if( pEnd2->z[0]!=';' ) n += pEnd2->n; + zStmt = sqlite3MPrintf(db, + "CREATE %s %.*s", zType2, n, pParse->sNameToken.z + ); + } + + /* A slot for the record has already been allocated in the + ** schema table. We just need to update that slot with all + ** the information we've collected. + */ + sqlite3NestedParse(pParse, + "UPDATE %Q." DFLT_SCHEMA_TABLE + " SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q" + " WHERE rowid=#%d", + db->aDb[iDb].zDbSName, + zType, + p->zName, + p->zName, + pParse->regRoot, + zStmt, + pParse->regRowid + ); + sqlite3DbFree(db, zStmt); + sqlite3ChangeCookie(pParse, iDb); + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* Check to see if we need to create an sqlite_sequence table for + ** keeping track of autoincrement keys. + */ + if( (p->tabFlags & TF_Autoincrement)!=0 && !IN_SPECIAL_PARSE ){ + Db *pDb = &db->aDb[iDb]; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( pDb->pSchema->pSeqTab==0 ){ + sqlite3NestedParse(pParse, + "CREATE TABLE %Q.sqlite_sequence(name,seq)", + pDb->zDbSName + ); + } + } +#endif + + /* Reparse everything to update our internal data structures */ + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName),0); + } + + /* Add the table to the in-memory representation of the database. + */ + if( db->init.busy ){ + Table *pOld; + Schema *pSchema = p->pSchema; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + assert( HasRowid(p) || p->iPKey<0 ); + pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, p); + if( pOld ){ + assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ + sqlite3OomFault(db); + return; + } + pParse->pNewTable = 0; + db->mDbFlags |= DBFLAG_SchemaChange; + + /* If this is the magic sqlite_sequence table used by autoincrement, + ** then record a pointer to this table in the main database structure + ** so that INSERT can find the table easily. */ + assert( !pParse->nested ); +#ifndef SQLITE_OMIT_AUTOINCREMENT + if( strcmp(p->zName, "sqlite_sequence")==0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + p->pSchema->pSeqTab = p; + } +#endif + } + +#ifndef SQLITE_OMIT_ALTERTABLE + if( !pSelect && IsOrdinaryTable(p) ){ + assert( pCons && pEnd ); + if( pCons->z==0 ){ + pCons = pEnd; + } + p->u.tab.addColOffset = 13 + (int)(pCons->z - pParse->sNameToken.z); + } +#endif +} + +#ifndef SQLITE_OMIT_VIEW +/* +** The parser calls this routine in order to create a new VIEW +*/ +SQLITE_PRIVATE void sqlite3CreateView( + Parse *pParse, /* The parsing context */ + Token *pBegin, /* The CREATE token that begins the statement */ + Token *pName1, /* The token that holds the name of the view */ + Token *pName2, /* The token that holds the name of the view */ + ExprList *pCNames, /* Optional list of view column names */ + Select *pSelect, /* A SELECT statement that will become the new view */ + int isTemp, /* TRUE for a TEMPORARY view */ + int noErr /* Suppress error messages if VIEW already exists */ +){ + Table *p; + int n; + const char *z; + Token sEnd; + DbFixer sFix; + Token *pName = 0; + int iDb; + sqlite3 *db = pParse->db; + + if( pParse->nVar>0 ){ + sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); + goto create_view_fail; + } + sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); + p = pParse->pNewTable; + if( p==0 || pParse->nErr ) goto create_view_fail; + + /* Legacy versions of SQLite allowed the use of the magic "rowid" column + ** on a view, even though views do not have rowids. The following flag + ** setting fixes this problem. But the fix can be disabled by compiling + ** with -DSQLITE_ALLOW_ROWID_IN_VIEW in case there are legacy apps that + ** depend upon the old buggy behavior. */ +#ifndef SQLITE_ALLOW_ROWID_IN_VIEW + p->tabFlags |= TF_NoVisibleRowid; +#endif + + sqlite3TwoPartName(pParse, pName1, pName2, &pName); + iDb = sqlite3SchemaToIndex(db, p->pSchema); + sqlite3FixInit(&sFix, pParse, iDb, "view", pName); + if( sqlite3FixSelect(&sFix, pSelect) ) goto create_view_fail; + + /* Make a copy of the entire SELECT statement that defines the view. + ** This will force all the Expr.token.z values to be dynamically + ** allocated rather than point to the input string - which means that + ** they will persist after the current sqlite3_exec() call returns. + */ + pSelect->selFlags |= SF_View; + if( IN_RENAME_OBJECT ){ + p->u.view.pSelect = pSelect; + pSelect = 0; + }else{ + p->u.view.pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + } + p->pCheck = sqlite3ExprListDup(db, pCNames, EXPRDUP_REDUCE); + p->eTabType = TABTYP_VIEW; + if( db->mallocFailed ) goto create_view_fail; + + /* Locate the end of the CREATE VIEW statement. Make sEnd point to + ** the end. + */ + sEnd = pParse->sLastToken; + assert( sEnd.z[0]!=0 || sEnd.n==0 ); + if( sEnd.z[0]!=';' ){ + sEnd.z += sEnd.n; + } + sEnd.n = 0; + n = (int)(sEnd.z - pBegin->z); + assert( n>0 ); + z = pBegin->z; + while( sqlite3Isspace(z[n-1]) ){ n--; } + sEnd.z = &z[n-1]; + sEnd.n = 1; + + /* Use sqlite3EndTable() to add the view to the schema table */ + sqlite3EndTable(pParse, 0, &sEnd, 0, 0); + +create_view_fail: + sqlite3SelectDelete(db, pSelect); + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprlistUnmap(pParse, pCNames); + } + sqlite3ExprListDelete(db, pCNames); + return; +} +#endif /* SQLITE_OMIT_VIEW */ + +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) +/* +** The Table structure pTable is really a VIEW. Fill in the names of +** the columns of the view in the pTable structure. Return the number +** of errors. If an error is seen leave an error message in pParse->zErrMsg. +*/ +SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ + Table *pSelTab; /* A fake table from which we get the result set */ + Select *pSel; /* Copy of the SELECT that implements the view */ + int nErr = 0; /* Number of errors encountered */ + int n; /* Temporarily holds the number of cursors assigned */ + sqlite3 *db = pParse->db; /* Database connection for malloc errors */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int rc; +#endif +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth; /* Saved xAuth pointer */ +#endif + + assert( pTable ); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + db->nSchemaLock++; + rc = sqlite3VtabCallConnect(pParse, pTable); + db->nSchemaLock--; + if( rc ){ + return 1; + } + if( IsVirtual(pTable) ) return 0; +#endif + +#ifndef SQLITE_OMIT_VIEW + /* A positive nCol means the columns names for this view are + ** already known. + */ + if( pTable->nCol>0 ) return 0; + + /* A negative nCol is a special marker meaning that we are currently + ** trying to compute the column names. If we enter this routine with + ** a negative nCol, it means two or more views form a loop, like this: + ** + ** CREATE VIEW one AS SELECT * FROM two; + ** CREATE VIEW two AS SELECT * FROM one; + ** + ** Actually, the error above is now caught prior to reaching this point. + ** But the following test is still important as it does come up + ** in the following: + ** + ** CREATE TABLE main.ex1(a); + ** CREATE TEMP VIEW ex1 AS SELECT a FROM ex1; + ** SELECT * FROM temp.ex1; + */ + if( pTable->nCol<0 ){ + sqlite3ErrorMsg(pParse, "view %s is circularly defined", pTable->zName); + return 1; + } + assert( pTable->nCol>=0 ); + + /* If we get this far, it means we need to compute the table names. + ** Note that the call to sqlite3ResultSetOfSelect() will expand any + ** "*" elements in the results set of the view and will assign cursors + ** to the elements of the FROM clause. But we do not want these changes + ** to be permanent. So the computation is done on a copy of the SELECT + ** statement that defines the view. + */ + assert( IsView(pTable) ); + pSel = sqlite3SelectDup(db, pTable->u.view.pSelect, 0); + if( pSel ){ + u8 eParseMode = pParse->eParseMode; + pParse->eParseMode = PARSE_MODE_NORMAL; + n = pParse->nTab; + sqlite3SrcListAssignCursors(pParse, pSel->pSrc); + pTable->nCol = -1; + DisableLookaside; +#ifndef SQLITE_OMIT_AUTHORIZATION + xAuth = db->xAuth; + db->xAuth = 0; + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE); + db->xAuth = xAuth; +#else + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE); +#endif + pParse->nTab = n; + if( pSelTab==0 ){ + pTable->nCol = 0; + nErr++; + }else if( pTable->pCheck ){ + /* CREATE VIEW name(arglist) AS ... + ** The names of the columns in the table are taken from + ** arglist which is stored in pTable->pCheck. The pCheck field + ** normally holds CHECK constraints on an ordinary table, but for + ** a VIEW it holds the list of column names. + */ + sqlite3ColumnsFromExprList(pParse, pTable->pCheck, + &pTable->nCol, &pTable->aCol); + if( db->mallocFailed==0 + && pParse->nErr==0 + && pTable->nCol==pSel->pEList->nExpr + ){ + sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel, + SQLITE_AFF_NONE); + } + }else{ + /* CREATE VIEW name AS... without an argument list. Construct + ** the column names from the SELECT statement that defines the view. + */ + assert( pTable->aCol==0 ); + pTable->nCol = pSelTab->nCol; + pTable->aCol = pSelTab->aCol; + pTable->tabFlags |= (pSelTab->tabFlags & COLFLAG_NOINSERT); + pSelTab->nCol = 0; + pSelTab->aCol = 0; + assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); + } + pTable->nNVCol = pTable->nCol; + sqlite3DeleteTable(db, pSelTab); + sqlite3SelectDelete(db, pSel); + EnableLookaside; + pParse->eParseMode = eParseMode; + } else { + nErr++; + } + pTable->pSchema->schemaFlags |= DB_UnresetViews; + if( db->mallocFailed ){ + sqlite3DeleteColumnNames(db, pTable); + } +#endif /* SQLITE_OMIT_VIEW */ + return nErr; +} +#endif /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ + +#ifndef SQLITE_OMIT_VIEW +/* +** Clear the column names from every VIEW in database idx. +*/ +static void sqliteViewResetAll(sqlite3 *db, int idx){ + HashElem *i; + assert( sqlite3SchemaMutexHeld(db, idx, 0) ); + if( !DbHasProperty(db, idx, DB_UnresetViews) ) return; + for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + if( IsView(pTab) ){ + sqlite3DeleteColumnNames(db, pTab); + } + } + DbClearProperty(db, idx, DB_UnresetViews); +} +#else +# define sqliteViewResetAll(A,B) +#endif /* SQLITE_OMIT_VIEW */ + +/* +** This function is called by the VDBE to adjust the internal schema +** used by SQLite when the btree layer moves a table root page. The +** root-page of a table or index in database iDb has changed from iFrom +** to iTo. +** +** Ticket #1728: The symbol table might still contain information +** on tables and/or indices that are the process of being deleted. +** If you are unlucky, one of those deleted indices or tables might +** have the same rootpage number as the real table or index that is +** being moved. So we cannot stop searching after the first match +** because the first match might be for one of the deleted indices +** or tables and not the table/index that is actually being moved. +** We must continue looping until all tables and indices with +** rootpage==iFrom have been converted to have a rootpage of iTo +** in order to be certain that we got the right one. +*/ +#ifndef SQLITE_OMIT_AUTOVACUUM +SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3 *db, int iDb, Pgno iFrom, Pgno iTo){ + HashElem *pElem; + Hash *pHash; + Db *pDb; + + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pDb = &db->aDb[iDb]; + pHash = &pDb->pSchema->tblHash; + for(pElem=sqliteHashFirst(pHash); pElem; pElem=sqliteHashNext(pElem)){ + Table *pTab = sqliteHashData(pElem); + if( pTab->tnum==iFrom ){ + pTab->tnum = iTo; + } + } + pHash = &pDb->pSchema->idxHash; + for(pElem=sqliteHashFirst(pHash); pElem; pElem=sqliteHashNext(pElem)){ + Index *pIdx = sqliteHashData(pElem); + if( pIdx->tnum==iFrom ){ + pIdx->tnum = iTo; + } + } +} +#endif + +/* +** Write code to erase the table with root-page iTable from database iDb. +** Also write code to modify the sqlite_schema table and internal schema +** if a root-page of another table is moved by the btree-layer whilst +** erasing iTable (this can happen with an auto-vacuum database). +*/ +static void destroyRootPage(Parse *pParse, int iTable, int iDb){ + Vdbe *v = sqlite3GetVdbe(pParse); + int r1 = sqlite3GetTempReg(pParse); + if( iTable<2 ) sqlite3ErrorMsg(pParse, "corrupt schema"); + sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); + sqlite3MayAbort(pParse); +#ifndef SQLITE_OMIT_AUTOVACUUM + /* OP_Destroy stores an in integer r1. If this integer + ** is non-zero, then it is the root page number of a table moved to + ** location iTable. The following code modifies the sqlite_schema table to + ** reflect this. + ** + ** The "#NNN" in the SQL is a special constant that means whatever value + ** is in register NNN. See grammar rules associated with the TK_REGISTER + ** token for additional information. + */ + sqlite3NestedParse(pParse, + "UPDATE %Q." DFLT_SCHEMA_TABLE + " SET rootpage=%d WHERE #%d AND rootpage=#%d", + pParse->db->aDb[iDb].zDbSName, iTable, r1, r1); +#endif + sqlite3ReleaseTempReg(pParse, r1); +} + +/* +** Write VDBE code to erase table pTab and all associated indices on disk. +** Code to update the sqlite_schema tables and internal schema definitions +** in case a root-page belonging to another table is moved by the btree layer +** is also added (this can happen with an auto-vacuum database). +*/ +static void destroyTable(Parse *pParse, Table *pTab){ + /* If the database may be auto-vacuum capable (if SQLITE_OMIT_AUTOVACUUM + ** is not defined), then it is important to call OP_Destroy on the + ** table and index root-pages in order, starting with the numerically + ** largest root-page number. This guarantees that none of the root-pages + ** to be destroyed is relocated by an earlier OP_Destroy. i.e. if the + ** following were coded: + ** + ** OP_Destroy 4 0 + ** ... + ** OP_Destroy 5 0 + ** + ** and root page 5 happened to be the largest root-page number in the + ** database, then root page 5 would be moved to page 4 by the + ** "OP_Destroy 4 0" opcode. The subsequent "OP_Destroy 5 0" would hit + ** a free-list page. + */ + Pgno iTab = pTab->tnum; + Pgno iDestroyed = 0; + + while( 1 ){ + Index *pIdx; + Pgno iLargest = 0; + + if( iDestroyed==0 || iTabpIndex; pIdx; pIdx=pIdx->pNext){ + Pgno iIdx = pIdx->tnum; + assert( pIdx->pSchema==pTab->pSchema ); + if( (iDestroyed==0 || (iIdxiLargest ){ + iLargest = iIdx; + } + } + if( iLargest==0 ){ + return; + }else{ + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + assert( iDb>=0 && iDbdb->nDb ); + destroyRootPage(pParse, iLargest, iDb); + iDestroyed = iLargest; + } + } +} + +/* +** Remove entries from the sqlite_statN tables (for N in (1,2,3)) +** after a DROP INDEX or DROP TABLE command. +*/ +static void sqlite3ClearStatTables( + Parse *pParse, /* The parsing context */ + int iDb, /* The database number */ + const char *zType, /* "idx" or "tbl" */ + const char *zName /* Name of index or table */ +){ + int i; + const char *zDbName = pParse->db->aDb[iDb].zDbSName; + for(i=1; i<=4; i++){ + char zTab[24]; + sqlite3_snprintf(sizeof(zTab),zTab,"sqlite_stat%d",i); + if( sqlite3FindTable(pParse->db, zTab, zDbName) ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.%s WHERE %s=%Q", + zDbName, zTab, zType, zName + ); + } + } +} + +/* +** Generate code to drop a table. +*/ +SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ + Vdbe *v; + sqlite3 *db = pParse->db; + Trigger *pTrigger; + Db *pDb = &db->aDb[iDb]; + + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + sqlite3BeginWriteOperation(pParse, 1, iDb); + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp0(v, OP_VBegin); + } +#endif + + /* Drop all triggers associated with the table being dropped. Code + ** is generated to remove entries from sqlite_schema and/or + ** sqlite_temp_schema if required. + */ + pTrigger = sqlite3TriggerList(pParse, pTab); + while( pTrigger ){ + assert( pTrigger->pSchema==pTab->pSchema || + pTrigger->pSchema==db->aDb[1].pSchema ); + sqlite3DropTriggerPtr(pParse, pTrigger); + pTrigger = pTrigger->pNext; + } + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* Remove any entries of the sqlite_sequence table associated with + ** the table being dropped. This is done before the table is dropped + ** at the btree level, in case the sqlite_sequence table needs to + ** move as a result of the drop (can happen in auto-vacuum mode). + */ + if( pTab->tabFlags & TF_Autoincrement ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q.sqlite_sequence WHERE name=%Q", + pDb->zDbSName, pTab->zName + ); + } +#endif + + /* Drop all entries in the schema table that refer to the + ** table. The program name loops through the schema table and deletes + ** every row that refers to a table of the same name as the one being + ** dropped. Triggers are handled separately because a trigger can be + ** created in the temp database that refers to a table in another + ** database. + */ + sqlite3NestedParse(pParse, + "DELETE FROM %Q." DFLT_SCHEMA_TABLE + " WHERE tbl_name=%Q and type!='trigger'", + pDb->zDbSName, pTab->zName); + if( !isView && !IsVirtual(pTab) ){ + destroyTable(pParse, pTab); + } + + /* Remove the table entry from SQLite's internal schema and modify + ** the schema cookie. + */ + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); + sqlite3MayAbort(pParse); + } + sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); + sqlite3ChangeCookie(pParse, iDb); + sqliteViewResetAll(db, iDb); +} + +/* +** Return TRUE if shadow tables should be read-only in the current +** context. +*/ +SQLITE_PRIVATE int sqlite3ReadOnlyShadowTables(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (db->flags & SQLITE_Defensive)!=0 + && db->pVtabCtx==0 + && db->nVdbeExec==0 + && !sqlite3VtabInSync(db) + ){ + return 1; + } +#endif + return 0; +} + +/* +** Return true if it is not allowed to drop the given table +*/ +static int tableMayNotBeDropped(sqlite3 *db, Table *pTab){ + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3StrNICmp(pTab->zName+7, "stat", 4)==0 ) return 0; + if( sqlite3StrNICmp(pTab->zName+7, "parameters", 10)==0 ) return 0; + return 1; + } + if( (pTab->tabFlags & TF_Shadow)!=0 && sqlite3ReadOnlyShadowTables(db) ){ + return 1; + } + return 0; +} + +/* +** This routine is called to do the work of a DROP TABLE statement. +** pName is the name of the table to be dropped. +*/ +SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){ + Table *pTab; + Vdbe *v; + sqlite3 *db = pParse->db; + int iDb; + + if( db->mallocFailed ){ + goto exit_drop_table; + } + assert( pParse->nErr==0 ); + assert( pName->nSrc==1 ); + if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; + if( noErr ) db->suppressErr++; + assert( isView==0 || isView==LOCATE_VIEW ); + pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); + if( noErr ) db->suppressErr--; + + if( pTab==0 ){ + if( noErr ){ + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + sqlite3ForceNotReadOnly(pParse); + } + goto exit_drop_table; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 && iDbnDb ); + + /* If pTab is a virtual table, call ViewGetColumnNames() to ensure + ** it is initialized. + */ + if( IsVirtual(pTab) && sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto exit_drop_table; + } +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code; + const char *zTab = SCHEMA_TABLE(iDb); + const char *zDb = db->aDb[iDb].zDbSName; + const char *zArg2 = 0; + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb)){ + goto exit_drop_table; + } + if( isView ){ + if( !OMIT_TEMPDB && iDb==1 ){ + code = SQLITE_DROP_TEMP_VIEW; + }else{ + code = SQLITE_DROP_VIEW; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + }else if( IsVirtual(pTab) ){ + code = SQLITE_DROP_VTABLE; + zArg2 = sqlite3GetVTable(db, pTab)->pMod->zName; +#endif + }else{ + if( !OMIT_TEMPDB && iDb==1 ){ + code = SQLITE_DROP_TEMP_TABLE; + }else{ + code = SQLITE_DROP_TABLE; + } + } + if( sqlite3AuthCheck(pParse, code, pTab->zName, zArg2, zDb) ){ + goto exit_drop_table; + } + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb) ){ + goto exit_drop_table; + } + } +#endif + if( tableMayNotBeDropped(db, pTab) ){ + sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); + goto exit_drop_table; + } + +#ifndef SQLITE_OMIT_VIEW + /* Ensure DROP TABLE is not used on a view, and DROP VIEW is not used + ** on a table. + */ + if( isView && !IsView(pTab) ){ + sqlite3ErrorMsg(pParse, "use DROP TABLE to delete table %s", pTab->zName); + goto exit_drop_table; + } + if( !isView && IsView(pTab) ){ + sqlite3ErrorMsg(pParse, "use DROP VIEW to delete view %s", pTab->zName); + goto exit_drop_table; + } +#endif + + /* Generate code to remove the table from the schema table + ** on disk. + */ + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3BeginWriteOperation(pParse, 1, iDb); + if( !isView ){ + sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName); + sqlite3FkDropTable(pParse, pName, pTab); + } + sqlite3CodeDropTable(pParse, pTab, iDb, isView); + } + +exit_drop_table: + sqlite3SrcListDelete(db, pName); +} + +/* +** This routine is called to create a new foreign key on the table +** currently under construction. pFromCol determines which columns +** in the current table point to the foreign key. If pFromCol==0 then +** connect the key to the last column inserted. pTo is the name of +** the table referred to (a.k.a the "parent" table). pToCol is a list +** of tables in the parent pTo table. flags contains all +** information about the conflict resolution algorithms specified +** in the ON DELETE, ON UPDATE and ON INSERT clauses. +** +** An FKey structure is created and added to the table currently +** under construction in the pParse->pNewTable field. +** +** The foreign key is set for IMMEDIATE processing. A subsequent call +** to sqlite3DeferForeignKey() might change this to DEFERRED. +*/ +SQLITE_PRIVATE void sqlite3CreateForeignKey( + Parse *pParse, /* Parsing context */ + ExprList *pFromCol, /* Columns in this table that point to other table */ + Token *pTo, /* Name of the other table */ + ExprList *pToCol, /* Columns in the other table */ + int flags /* Conflict resolution algorithms. */ +){ + sqlite3 *db = pParse->db; +#ifndef SQLITE_OMIT_FOREIGN_KEY + FKey *pFKey = 0; + FKey *pNextTo; + Table *p = pParse->pNewTable; + int nByte; + int i; + int nCol; + char *z; + + assert( pTo!=0 ); + if( p==0 || IN_DECLARE_VTAB ) goto fk_end; + if( pFromCol==0 ){ + int iCol = p->nCol-1; + if( NEVER(iCol<0) ) goto fk_end; + if( pToCol && pToCol->nExpr!=1 ){ + sqlite3ErrorMsg(pParse, "foreign key on %s" + " should reference only one column of table %T", + p->aCol[iCol].zCnName, pTo); + goto fk_end; + } + nCol = 1; + }else if( pToCol && pToCol->nExpr!=pFromCol->nExpr ){ + sqlite3ErrorMsg(pParse, + "number of columns in foreign key does not match the number of " + "columns in the referenced table"); + goto fk_end; + }else{ + nCol = pFromCol->nExpr; + } + nByte = sizeof(*pFKey) + (nCol-1)*sizeof(pFKey->aCol[0]) + pTo->n + 1; + if( pToCol ){ + for(i=0; inExpr; i++){ + nByte += sqlite3Strlen30(pToCol->a[i].zEName) + 1; + } + } + pFKey = sqlite3DbMallocZero(db, nByte ); + if( pFKey==0 ){ + goto fk_end; + } + pFKey->pFrom = p; + pFKey->pNextFrom = p->u.tab.pFKey; + z = (char*)&pFKey->aCol[nCol]; + pFKey->zTo = z; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)z, pTo); + } + memcpy(z, pTo->z, pTo->n); + z[pTo->n] = 0; + sqlite3Dequote(z); + z += pTo->n+1; + pFKey->nCol = nCol; + if( pFromCol==0 ){ + pFKey->aCol[0].iFrom = p->nCol-1; + }else{ + for(i=0; inCol; j++){ + if( sqlite3StrICmp(p->aCol[j].zCnName, pFromCol->a[i].zEName)==0 ){ + pFKey->aCol[i].iFrom = j; + break; + } + } + if( j>=p->nCol ){ + sqlite3ErrorMsg(pParse, + "unknown column \"%s\" in foreign key definition", + pFromCol->a[i].zEName); + goto fk_end; + } + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, &pFKey->aCol[i], pFromCol->a[i].zEName); + } + } + } + if( pToCol ){ + for(i=0; ia[i].zEName); + pFKey->aCol[i].zCol = z; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, z, pToCol->a[i].zEName); + } + memcpy(z, pToCol->a[i].zEName, n); + z[n] = 0; + z += n+1; + } + } + pFKey->isDeferred = 0; + pFKey->aAction[0] = (u8)(flags & 0xff); /* ON DELETE action */ + pFKey->aAction[1] = (u8)((flags >> 8 ) & 0xff); /* ON UPDATE action */ + + assert( sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); + pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, + pFKey->zTo, (void *)pFKey + ); + if( pNextTo==pFKey ){ + sqlite3OomFault(db); + goto fk_end; + } + if( pNextTo ){ + assert( pNextTo->pPrevTo==0 ); + pFKey->pNextTo = pNextTo; + pNextTo->pPrevTo = pFKey; + } + + /* Link the foreign key to the table as the last step. + */ + assert( !IsVirtual(p) ); + p->u.tab.pFKey = pFKey; + pFKey = 0; + +fk_end: + sqlite3DbFree(db, pFKey); +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + sqlite3ExprListDelete(db, pFromCol); + sqlite3ExprListDelete(db, pToCol); +} + +/* +** This routine is called when an INITIALLY IMMEDIATE or INITIALLY DEFERRED +** clause is seen as part of a foreign key definition. The isDeferred +** parameter is 1 for INITIALLY DEFERRED and 0 for INITIALLY IMMEDIATE. +** The behavior of the most recently created foreign key is adjusted +** accordingly. +*/ +SQLITE_PRIVATE void sqlite3DeferForeignKey(Parse *pParse, int isDeferred){ +#ifndef SQLITE_OMIT_FOREIGN_KEY + Table *pTab; + FKey *pFKey; + if( (pTab = pParse->pNewTable)==0 ) return; + if( NEVER(IsVirtual(pTab)) ) return; + if( (pFKey = pTab->u.tab.pFKey)==0 ) return; + assert( isDeferred==0 || isDeferred==1 ); /* EV: R-30323-21917 */ + pFKey->isDeferred = (u8)isDeferred; +#endif +} + +/* +** Generate code that will erase and refill index *pIdx. This is +** used to initialize a newly created index or to recompute the +** content of an index in response to a REINDEX command. +** +** if memRootPage is not negative, it means that the index is newly +** created. The register specified by memRootPage contains the +** root page number of the index. If memRootPage is negative, then +** the index already exists and must be cleared before being refilled and +** the root page number of the index is taken from pIndex->tnum. +*/ +static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ + Table *pTab = pIndex->pTable; /* The table that is indexed */ + int iTab = pParse->nTab++; /* Btree cursor used for pTab */ + int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */ + int iSorter; /* Cursor opened by OpenSorter (if in use) */ + int addr1; /* Address of top of loop */ + int addr2; /* Address to jump to for next iteration */ + Pgno tnum; /* Root page of index */ + int iPartIdxLabel; /* Jump to this label to skip a row */ + Vdbe *v; /* Generate code into this virtual machine */ + KeyInfo *pKey; /* KeyInfo for index */ + int regRecord; /* Register holding assembled index record */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, + db->aDb[iDb].zDbSName ) ){ + return; + } +#endif + + /* Require a write-lock on the table to perform this operation */ + sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); + + v = sqlite3GetVdbe(pParse); + if( v==0 ) return; + if( memRootPage>=0 ){ + tnum = (Pgno)memRootPage; + }else{ + tnum = pIndex->tnum; + } + pKey = sqlite3KeyInfoOfIndex(pParse, pIndex); + assert( pKey!=0 || db->mallocFailed || pParse->nErr ); + + /* Open the sorter cursor if we are to use one. */ + iSorter = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, pIndex->nKeyCol, (char*) + sqlite3KeyInfoRef(pKey), P4_KEYINFO); + + /* Open the table. Loop through all rows of the table, inserting index + ** records into the sorter. */ + sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); VdbeCoverage(v); + regRecord = sqlite3GetTempReg(pParse); + sqlite3MultiWrite(pParse); + + sqlite3GenerateIndexKey(pParse,pIndex,iTab,regRecord,0,&iPartIdxLabel,0,0); + sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + if( memRootPage<0 ) sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); + sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, (int)tnum, iDb, + (char *)pKey, P4_KEYINFO); + sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); + + addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); + if( IsUniqueIndex(pIndex) ){ + int j2 = sqlite3VdbeGoto(v, 1); + addr2 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeVerifyAbortable(v, OE_Abort); + sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, + pIndex->nKeyCol); VdbeCoverage(v); + sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); + sqlite3VdbeJumpHere(v, j2); + }else{ + /* Most CREATE INDEX and REINDEX statements that are not UNIQUE can not + ** abort. The exception is if one of the indexed expressions contains a + ** user function that throws an exception when it is evaluated. But the + ** overhead of adding a statement journal to a CREATE INDEX statement is + ** very small (since most of the pages written do not contain content that + ** needs to be restored if the statement aborts), so we call + ** sqlite3MayAbort() for all CREATE INDEX statements. */ + sqlite3MayAbort(pParse); + addr2 = sqlite3VdbeCurrentAddr(v); + } + sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); + if( !pIndex->bAscKeyBug ){ + /* This OP_SeekEnd opcode makes index insert for a REINDEX go much + ** faster by avoiding unnecessary seeks. But the optimization does + ** not work for UNIQUE constraint indexes on WITHOUT ROWID tables + ** with DESC primary keys, since those indexes have there keys in + ** a different order from the main table. + ** See ticket: https://www.sqlite.org/src/info/bba7b69f9849b5bf + */ + sqlite3VdbeAddOp1(v, OP_SeekEnd, iIdx); + } + sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + + sqlite3VdbeAddOp1(v, OP_Close, iTab); + sqlite3VdbeAddOp1(v, OP_Close, iIdx); + sqlite3VdbeAddOp1(v, OP_Close, iSorter); +} + +/* +** Allocate heap space to hold an Index object with nCol columns. +** +** Increase the allocation size to provide an extra nExtra bytes +** of 8-byte aligned space after the Index object and return a +** pointer to this extra space in *ppExtra. +*/ +SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( + sqlite3 *db, /* Database connection */ + i16 nCol, /* Total number of columns in the index */ + int nExtra, /* Number of bytes of extra space to alloc */ + char **ppExtra /* Pointer to the "extra" space */ +){ + Index *p; /* Allocated index object */ + int nByte; /* Bytes of space for Index object + arrays */ + + nByte = ROUND8(sizeof(Index)) + /* Index structure */ + ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ + ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ + sizeof(i16)*nCol + /* Index.aiColumn */ + sizeof(u8)*nCol); /* Index.aSortOrder */ + p = sqlite3DbMallocZero(db, nByte + nExtra); + if( p ){ + char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); + p->azColl = (const char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); + p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; + p->aSortOrder = (u8*)pExtra; + p->nColumn = nCol; + p->nKeyCol = nCol - 1; + *ppExtra = ((char*)p) + nByte; + } + return p; +} + +/* +** If expression list pList contains an expression that was parsed with +** an explicit "NULLS FIRST" or "NULLS LAST" clause, leave an error in +** pParse and return non-zero. Otherwise, return zero. +*/ +SQLITE_PRIVATE int sqlite3HasExplicitNulls(Parse *pParse, ExprList *pList){ + if( pList ){ + int i; + for(i=0; inExpr; i++){ + if( pList->a[i].bNulls ){ + u8 sf = pList->a[i].sortFlags; + sqlite3ErrorMsg(pParse, "unsupported use of NULLS %s", + (sf==0 || sf==3) ? "FIRST" : "LAST" + ); + return 1; + } + } + } + return 0; +} + +/* +** Create a new index for an SQL table. pName1.pName2 is the name of the index +** and pTblList is the name of the table that is to be indexed. Both will +** be NULL for a primary key or an index that is created to satisfy a +** UNIQUE constraint. If pTable and pIndex are NULL, use pParse->pNewTable +** as the table to be indexed. pParse->pNewTable is a table that is +** currently being constructed by a CREATE TABLE statement. +** +** pList is a list of columns to be indexed. pList will be NULL if this +** is a primary key or unique-constraint on the most recent column added +** to the table currently under construction. +*/ +SQLITE_PRIVATE void sqlite3CreateIndex( + Parse *pParse, /* All information about this parse */ + Token *pName1, /* First part of index name. May be NULL */ + Token *pName2, /* Second part of index name. May be NULL */ + SrcList *pTblName, /* Table to index. Use pParse->pNewTable if 0 */ + ExprList *pList, /* A list of columns to be indexed */ + int onError, /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ + Token *pStart, /* The CREATE token that begins this statement */ + Expr *pPIWhere, /* WHERE clause for partial indices */ + int sortOrder, /* Sort order of primary key when pList==NULL */ + int ifNotExist, /* Omit error if index already exists */ + u8 idxType /* The index type */ +){ + Table *pTab = 0; /* Table to be indexed */ + Index *pIndex = 0; /* The index to be created */ + char *zName = 0; /* Name of the index */ + int nName; /* Number of characters in zName */ + int i, j; + DbFixer sFix; /* For assigning database names to pTable */ + int sortOrderMask; /* 1 to honor DESC in index. 0 to ignore. */ + sqlite3 *db = pParse->db; + Db *pDb; /* The specific table containing the indexed database */ + int iDb; /* Index of the database that is being written */ + Token *pName = 0; /* Unqualified name of the index to create */ + struct ExprList_item *pListItem; /* For looping over pList */ + int nExtra = 0; /* Space allocated for zExtra[] */ + int nExtraCol; /* Number of extra columns needed */ + char *zExtra = 0; /* Extra space after the Index object */ + Index *pPk = 0; /* PRIMARY KEY index for WITHOUT ROWID tables */ + + if( db->mallocFailed || pParse->nErr>0 ){ + goto exit_create_index; + } + if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){ + goto exit_create_index; + } + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto exit_create_index; + } + if( sqlite3HasExplicitNulls(pParse, pList) ){ + goto exit_create_index; + } + + /* + ** Find the table that is to be indexed. Return early if not found. + */ + if( pTblName!=0 ){ + + /* Use the two-part index name to determine the database + ** to search for the table. 'Fix' the table name to this db + ** before looking up the table. + */ + assert( pName1 && pName2 ); + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ) goto exit_create_index; + assert( pName && pName->z ); + +#ifndef SQLITE_OMIT_TEMPDB + /* If the index name was unqualified, check if the table + ** is a temp table. If so, set the database to 1. Do not do this + ** if initialising a database schema. + */ + if( !db->init.busy ){ + pTab = sqlite3SrcListLookup(pParse, pTblName); + if( pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ + iDb = 1; + } + } +#endif + + sqlite3FixInit(&sFix, pParse, iDb, "index", pName); + if( sqlite3FixSrcList(&sFix, pTblName) ){ + /* Because the parser constructs pTblName from a single identifier, + ** sqlite3FixSrcList can never fail. */ + assert(0); + } + pTab = sqlite3LocateTableItem(pParse, 0, &pTblName->a[0]); + assert( db->mallocFailed==0 || pTab==0 ); + if( pTab==0 ) goto exit_create_index; + if( iDb==1 && db->aDb[iDb].pSchema!=pTab->pSchema ){ + sqlite3ErrorMsg(pParse, + "cannot create a TEMP index on non-TEMP table \"%s\"", + pTab->zName); + goto exit_create_index; + } + if( !HasRowid(pTab) ) pPk = sqlite3PrimaryKeyIndex(pTab); + }else{ + assert( pName==0 ); + assert( pStart==0 ); + pTab = pParse->pNewTable; + if( !pTab ) goto exit_create_index; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + } + pDb = &db->aDb[iDb]; + + assert( pTab!=0 ); + assert( pParse->nErr==0 ); + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + && db->init.busy==0 + && pTblName!=0 +#if SQLITE_USER_AUTHENTICATION + && sqlite3UserAuthTable(pTab->zName)==0 +#endif + ){ + sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); + goto exit_create_index; + } +#ifndef SQLITE_OMIT_VIEW + if( IsView(pTab) ){ + sqlite3ErrorMsg(pParse, "views may not be indexed"); + goto exit_create_index; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "virtual tables may not be indexed"); + goto exit_create_index; + } +#endif + + /* + ** Find the name of the index. Make sure there is not already another + ** index or table with the same name. + ** + ** Exception: If we are reading the names of permanent indices from the + ** sqlite_schema table (because some other process changed the schema) and + ** one of the index names collides with the name of a temporary table or + ** index, then we will continue to process this index. + ** + ** If pName==0 it means that we are + ** dealing with a primary key or UNIQUE constraint. We have to invent our + ** own name. + */ + if( pName ){ + zName = sqlite3NameFromToken(db, pName); + if( zName==0 ) goto exit_create_index; + assert( pName->z!=0 ); + if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName,"index",pTab->zName) ){ + goto exit_create_index; + } + if( !IN_RENAME_OBJECT ){ + if( !db->init.busy ){ + if( sqlite3FindTable(db, zName, 0)!=0 ){ + sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); + goto exit_create_index; + } + } + if( sqlite3FindIndex(db, zName, pDb->zDbSName)!=0 ){ + if( !ifNotExist ){ + sqlite3ErrorMsg(pParse, "index %s already exists", zName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3ForceNotReadOnly(pParse); + } + goto exit_create_index; + } + } + }else{ + int n; + Index *pLoop; + for(pLoop=pTab->pIndex, n=1; pLoop; pLoop=pLoop->pNext, n++){} + zName = sqlite3MPrintf(db, "sqlite_autoindex_%s_%d", pTab->zName, n); + if( zName==0 ){ + goto exit_create_index; + } + + /* Automatic index names generated from within sqlite3_declare_vtab() + ** must have names that are distinct from normal automatic index names. + ** The following statement converts "sqlite3_autoindex..." into + ** "sqlite3_butoindex..." in order to make the names distinct. + ** The "vtab_err.test" test demonstrates the need of this statement. */ + if( IN_SPECIAL_PARSE ) zName[7]++; + } + + /* Check for authorization to create an index. + */ +#ifndef SQLITE_OMIT_AUTHORIZATION + if( !IN_RENAME_OBJECT ){ + const char *zDb = pDb->zDbSName; + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ + goto exit_create_index; + } + i = SQLITE_CREATE_INDEX; + if( !OMIT_TEMPDB && iDb==1 ) i = SQLITE_CREATE_TEMP_INDEX; + if( sqlite3AuthCheck(pParse, i, zName, pTab->zName, zDb) ){ + goto exit_create_index; + } + } +#endif + + /* If pList==0, it means this routine was called to make a primary + ** key out of the last column added to the table under construction. + ** So create a fake list to simulate this. + */ + if( pList==0 ){ + Token prevCol; + Column *pCol = &pTab->aCol[pTab->nCol-1]; + pCol->colFlags |= COLFLAG_UNIQUE; + sqlite3TokenInit(&prevCol, pCol->zCnName); + pList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); + if( pList==0 ) goto exit_create_index; + assert( pList->nExpr==1 ); + sqlite3ExprListSetSortOrder(pList, sortOrder, SQLITE_SO_UNDEFINED); + }else{ + sqlite3ExprListCheckLength(pParse, pList, "index"); + if( pParse->nErr ) goto exit_create_index; + } + + /* Figure out how many bytes of space are required to store explicitly + ** specified collation sequence names. + */ + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + assert( pExpr!=0 ); + if( pExpr->op==TK_COLLATE ){ + nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); + } + } + + /* + ** Allocate the index structure. + */ + nName = sqlite3Strlen30(zName); + nExtraCol = pPk ? pPk->nKeyCol : 1; + assert( pList->nExpr + nExtraCol <= 32767 /* Fits in i16 */ ); + pIndex = sqlite3AllocateIndexObject(db, pList->nExpr + nExtraCol, + nName + nExtra + 1, &zExtra); + if( db->mallocFailed ){ + goto exit_create_index; + } + assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowLogEst) ); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->azColl) ); + pIndex->zName = zExtra; + zExtra += nName + 1; + memcpy(pIndex->zName, zName, nName+1); + pIndex->pTable = pTab; + pIndex->onError = (u8)onError; + pIndex->uniqNotNull = onError!=OE_None; + pIndex->idxType = idxType; + pIndex->pSchema = db->aDb[iDb].pSchema; + pIndex->nKeyCol = pList->nExpr; + if( pPIWhere ){ + sqlite3ResolveSelfReference(pParse, pTab, NC_PartIdx, pPIWhere, 0); + pIndex->pPartIdxWhere = pPIWhere; + pPIWhere = 0; + } + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + + /* Check to see if we should honor DESC requests on index columns + */ + if( pDb->pSchema->file_format>=4 ){ + sortOrderMask = -1; /* Honor DESC */ + }else{ + sortOrderMask = 0; /* Ignore DESC */ + } + + /* Analyze the list of expressions that form the terms of the index and + ** report any errors. In the common case where the expression is exactly + ** a table column, store that column in aiColumn[]. For general expressions, + ** populate pIndex->aColExpr and store XN_EXPR (-2) in aiColumn[]. + ** + ** TODO: Issue a warning if two or more columns of the index are identical. + ** TODO: Issue a warning if the table primary key is used as part of the + ** index key. + */ + pListItem = pList->a; + if( IN_RENAME_OBJECT ){ + pIndex->aColExpr = pList; + pList = 0; + } + for(i=0; inKeyCol; i++, pListItem++){ + Expr *pCExpr; /* The i-th index expression */ + int requestedSortOrder; /* ASC or DESC on the i-th expression */ + const char *zColl; /* Collation sequence name */ + + sqlite3StringToId(pListItem->pExpr); + sqlite3ResolveSelfReference(pParse, pTab, NC_IdxExpr, pListItem->pExpr, 0); + if( pParse->nErr ) goto exit_create_index; + pCExpr = sqlite3ExprSkipCollate(pListItem->pExpr); + if( pCExpr->op!=TK_COLUMN ){ + if( pTab==pParse->pNewTable ){ + sqlite3ErrorMsg(pParse, "expressions prohibited in PRIMARY KEY and " + "UNIQUE constraints"); + goto exit_create_index; + } + if( pIndex->aColExpr==0 ){ + pIndex->aColExpr = pList; + pList = 0; + } + j = XN_EXPR; + pIndex->aiColumn[i] = XN_EXPR; + pIndex->uniqNotNull = 0; + }else{ + j = pCExpr->iColumn; + assert( j<=0x7fff ); + if( j<0 ){ + j = pTab->iPKey; + }else{ + if( pTab->aCol[j].notNull==0 ){ + pIndex->uniqNotNull = 0; + } + if( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ){ + pIndex->bHasVCol = 1; + } + } + pIndex->aiColumn[i] = (i16)j; + } + zColl = 0; + if( pListItem->pExpr->op==TK_COLLATE ){ + int nColl; + zColl = pListItem->pExpr->u.zToken; + nColl = sqlite3Strlen30(zColl) + 1; + assert( nExtra>=nColl ); + memcpy(zExtra, zColl, nColl); + zColl = zExtra; + zExtra += nColl; + nExtra -= nColl; + }else if( j>=0 ){ + zColl = sqlite3ColumnColl(&pTab->aCol[j]); + } + if( !zColl ) zColl = sqlite3StrBINARY; + if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ + goto exit_create_index; + } + pIndex->azColl[i] = zColl; + requestedSortOrder = pListItem->sortFlags & sortOrderMask; + pIndex->aSortOrder[i] = (u8)requestedSortOrder; + } + + /* Append the table key to the end of the index. For WITHOUT ROWID + ** tables (when pPk!=0) this will be the declared PRIMARY KEY. For + ** normal tables (when pPk==0) this will be the rowid. + */ + if( pPk ){ + for(j=0; jnKeyCol; j++){ + int x = pPk->aiColumn[j]; + assert( x>=0 ); + if( isDupColumn(pIndex, pIndex->nKeyCol, pPk, j) ){ + pIndex->nColumn--; + }else{ + testcase( hasColumn(pIndex->aiColumn,pIndex->nKeyCol,x) ); + pIndex->aiColumn[i] = x; + pIndex->azColl[i] = pPk->azColl[j]; + pIndex->aSortOrder[i] = pPk->aSortOrder[j]; + i++; + } + } + assert( i==pIndex->nColumn ); + }else{ + pIndex->aiColumn[i] = XN_ROWID; + pIndex->azColl[i] = sqlite3StrBINARY; + } + sqlite3DefaultRowEst(pIndex); + if( pParse->pNewTable==0 ) estimateIndexWidth(pIndex); + + /* If this index contains every column of its table, then mark + ** it as a covering index */ + assert( HasRowid(pTab) + || pTab->iPKey<0 || sqlite3TableColumnToIndex(pIndex, pTab->iPKey)>=0 ); + recomputeColumnsNotIndexed(pIndex); + if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ + pIndex->isCovering = 1; + for(j=0; jnCol; j++){ + if( j==pTab->iPKey ) continue; + if( sqlite3TableColumnToIndex(pIndex,j)>=0 ) continue; + pIndex->isCovering = 0; + break; + } + } + + if( pTab==pParse->pNewTable ){ + /* This routine has been called to create an automatic index as a + ** result of a PRIMARY KEY or UNIQUE clause on a column definition, or + ** a PRIMARY KEY or UNIQUE clause following the column definitions. + ** i.e. one of: + ** + ** CREATE TABLE t(x PRIMARY KEY, y); + ** CREATE TABLE t(x, y, UNIQUE(x, y)); + ** + ** Either way, check to see if the table already has such an index. If + ** so, don't bother creating this one. This only applies to + ** automatically created indices. Users can do as they wish with + ** explicit indices. + ** + ** Two UNIQUE or PRIMARY KEY constraints are considered equivalent + ** (and thus suppressing the second one) even if they have different + ** sort orders. + ** + ** If there are different collating sequences or if the columns of + ** the constraint occur in different orders, then the constraints are + ** considered distinct and both result in separate indices. + */ + Index *pIdx; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int k; + assert( IsUniqueIndex(pIdx) ); + assert( pIdx->idxType!=SQLITE_IDXTYPE_APPDEF ); + assert( IsUniqueIndex(pIndex) ); + + if( pIdx->nKeyCol!=pIndex->nKeyCol ) continue; + for(k=0; knKeyCol; k++){ + const char *z1; + const char *z2; + assert( pIdx->aiColumn[k]>=0 ); + if( pIdx->aiColumn[k]!=pIndex->aiColumn[k] ) break; + z1 = pIdx->azColl[k]; + z2 = pIndex->azColl[k]; + if( sqlite3StrICmp(z1, z2) ) break; + } + if( k==pIdx->nKeyCol ){ + if( pIdx->onError!=pIndex->onError ){ + /* This constraint creates the same index as a previous + ** constraint specified somewhere in the CREATE TABLE statement. + ** However the ON CONFLICT clauses are different. If both this + ** constraint and the previous equivalent constraint have explicit + ** ON CONFLICT clauses this is an error. Otherwise, use the + ** explicitly specified behavior for the index. + */ + if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ + sqlite3ErrorMsg(pParse, + "conflicting ON CONFLICT clauses specified", 0); + } + if( pIdx->onError==OE_Default ){ + pIdx->onError = pIndex->onError; + } + } + if( idxType==SQLITE_IDXTYPE_PRIMARYKEY ) pIdx->idxType = idxType; + if( IN_RENAME_OBJECT ){ + pIndex->pNext = pParse->pNewIndex; + pParse->pNewIndex = pIndex; + pIndex = 0; + } + goto exit_create_index; + } + } + } + + if( !IN_RENAME_OBJECT ){ + + /* Link the new Index structure to its table and to the other + ** in-memory database structures. + */ + assert( pParse->nErr==0 ); + if( db->init.busy ){ + Index *p; + assert( !IN_SPECIAL_PARSE ); + assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); + if( pTblName!=0 ){ + pIndex->tnum = db->init.newTnum; + if( sqlite3IndexHasDuplicateRootPage(pIndex) ){ + sqlite3ErrorMsg(pParse, "invalid rootpage"); + pParse->rc = SQLITE_CORRUPT_BKPT; + goto exit_create_index; + } + } + p = sqlite3HashInsert(&pIndex->pSchema->idxHash, + pIndex->zName, pIndex); + if( p ){ + assert( p==pIndex ); /* Malloc must have failed */ + sqlite3OomFault(db); + goto exit_create_index; + } + db->mDbFlags |= DBFLAG_SchemaChange; + } + + /* If this is the initial CREATE INDEX statement (or CREATE TABLE if the + ** index is an implied index for a UNIQUE or PRIMARY KEY constraint) then + ** emit code to allocate the index rootpage on disk and make an entry for + ** the index in the sqlite_schema table and populate the index with + ** content. But, do not do this if we are simply reading the sqlite_schema + ** table to parse the schema, or if this index is the PRIMARY KEY index + ** of a WITHOUT ROWID table. + ** + ** If pTblName==0 it means this index is generated as an implied PRIMARY KEY + ** or UNIQUE index in a CREATE TABLE statement. Since the table + ** has just been created, it contains no data and the index initialization + ** step can be skipped. + */ + else if( HasRowid(pTab) || pTblName!=0 ){ + Vdbe *v; + char *zStmt; + int iMem = ++pParse->nMem; + + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto exit_create_index; + + sqlite3BeginWriteOperation(pParse, 1, iDb); + + /* Create the rootpage for the index using CreateIndex. But before + ** doing so, code a Noop instruction and store its address in + ** Index.tnum. This is required in case this index is actually a + ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In + ** that case the convertToWithoutRowidTable() routine will replace + ** the Noop with a Goto to jump over the VDBE code generated below. */ + pIndex->tnum = (Pgno)sqlite3VdbeAddOp0(v, OP_Noop); + sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, iMem, BTREE_BLOBKEY); + + /* Gather the complete text of the CREATE INDEX statement into + ** the zStmt variable + */ + assert( pName!=0 || pStart==0 ); + if( pStart ){ + int n = (int)(pParse->sLastToken.z - pName->z) + pParse->sLastToken.n; + if( pName->z[n-1]==';' ) n--; + /* A named index with an explicit CREATE INDEX statement */ + zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", + onError==OE_None ? "" : " UNIQUE", n, pName->z); + }else{ + /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ + /* zStmt = sqlite3MPrintf(""); */ + zStmt = 0; + } + + /* Add an entry in sqlite_schema for this index + */ + sqlite3NestedParse(pParse, + "INSERT INTO %Q." DFLT_SCHEMA_TABLE " VALUES('index',%Q,%Q,#%d,%Q);", + db->aDb[iDb].zDbSName, + pIndex->zName, + pTab->zName, + iMem, + zStmt + ); + sqlite3DbFree(db, zStmt); + + /* Fill the index with data and reparse the schema. Code an OP_Expire + ** to invalidate all pre-compiled statements. + */ + if( pTblName ){ + sqlite3RefillIndex(pParse, pIndex, iMem); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), 0); + sqlite3VdbeAddOp2(v, OP_Expire, 0, 1); + } + + sqlite3VdbeJumpHere(v, (int)pIndex->tnum); + } + } + if( db->init.busy || pTblName==0 ){ + pIndex->pNext = pTab->pIndex; + pTab->pIndex = pIndex; + pIndex = 0; + } + else if( IN_RENAME_OBJECT ){ + assert( pParse->pNewIndex==0 ); + pParse->pNewIndex = pIndex; + pIndex = 0; + } + + /* Clean up before exiting */ +exit_create_index: + if( pIndex ) sqlite3FreeIndex(db, pIndex); + if( pTab ){ + /* Ensure all REPLACE indexes on pTab are at the end of the pIndex list. + ** The list was already ordered when this routine was entered, so at this + ** point at most a single index (the newly added index) will be out of + ** order. So we have to reorder at most one index. */ + Index **ppFrom; + Index *pThis; + for(ppFrom=&pTab->pIndex; (pThis = *ppFrom)!=0; ppFrom=&pThis->pNext){ + Index *pNext; + if( pThis->onError!=OE_Replace ) continue; + while( (pNext = pThis->pNext)!=0 && pNext->onError!=OE_Replace ){ + *ppFrom = pNext; + pThis->pNext = pNext->pNext; + pNext->pNext = pThis; + ppFrom = &pNext->pNext; + } + break; + } +#ifdef SQLITE_DEBUG + /* Verify that all REPLACE indexes really are now at the end + ** of the index list. In other words, no other index type ever + ** comes after a REPLACE index on the list. */ + for(pThis = pTab->pIndex; pThis; pThis=pThis->pNext){ + assert( pThis->onError!=OE_Replace + || pThis->pNext==0 + || pThis->pNext->onError==OE_Replace ); + } +#endif + } + sqlite3ExprDelete(db, pPIWhere); + sqlite3ExprListDelete(db, pList); + sqlite3SrcListDelete(db, pTblName); + sqlite3DbFree(db, zName); +} + +/* +** Fill the Index.aiRowEst[] array with default information - information +** to be used when we have not run the ANALYZE command. +** +** aiRowEst[0] is supposed to contain the number of elements in the index. +** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the +** number of rows in the table that match any particular value of the +** first column of the index. aiRowEst[2] is an estimate of the number +** of rows that match any particular combination of the first 2 columns +** of the index. And so forth. It must always be the case that +* +** aiRowEst[N]<=aiRowEst[N-1] +** aiRowEst[N]>=1 +** +** Apart from that, we have little to go on besides intuition as to +** how aiRowEst[] should be initialized. The numbers generated here +** are based on typical values found in actual indices. +*/ +SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ + /* 10, 9, 8, 7, 6 */ + static const LogEst aVal[] = { 33, 32, 30, 28, 26 }; + LogEst *a = pIdx->aiRowLogEst; + LogEst x; + int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); + int i; + + /* Indexes with default row estimates should not have stat1 data */ + assert( !pIdx->hasStat1 ); + + /* Set the first entry (number of rows in the index) to the estimated + ** number of rows in the table, or half the number of rows in the table + ** for a partial index. + ** + ** 2020-05-27: If some of the stat data is coming from the sqlite_stat1 + ** table but other parts we are having to guess at, then do not let the + ** estimated number of rows in the table be less than 1000 (LogEst 99). + ** Failure to do this can cause the indexes for which we do not have + ** stat1 data to be ignored by the query planner. + */ + x = pIdx->pTable->nRowLogEst; + assert( 99==sqlite3LogEst(1000) ); + if( x<99 ){ + pIdx->pTable->nRowLogEst = x = 99; + } + if( pIdx->pPartIdxWhere!=0 ){ x -= 10; assert( 10==sqlite3LogEst(2) ); } + a[0] = x; + + /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is + ** 6 and each subsequent value (if any) is 5. */ + memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); + for(i=nCopy+1; i<=pIdx->nKeyCol; i++){ + a[i] = 23; assert( 23==sqlite3LogEst(5) ); + } + + assert( 0==sqlite3LogEst(1) ); + if( IsUniqueIndex(pIdx) ) a[pIdx->nKeyCol] = 0; +} + +/* +** This routine will drop an existing named index. This routine +** implements the DROP INDEX statement. +*/ +SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists){ + Index *pIndex; + Vdbe *v; + sqlite3 *db = pParse->db; + int iDb; + + assert( pParse->nErr==0 ); /* Never called with prior errors */ + if( db->mallocFailed ){ + goto exit_drop_index; + } + assert( pName->nSrc==1 ); + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto exit_drop_index; + } + pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].zDatabase); + if( pIndex==0 ){ + if( !ifExists ){ + sqlite3ErrorMsg(pParse, "no such index: %S", pName->a); + }else{ + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + sqlite3ForceNotReadOnly(pParse); + } + pParse->checkSchema = 1; + goto exit_drop_index; + } + if( pIndex->idxType!=SQLITE_IDXTYPE_APPDEF ){ + sqlite3ErrorMsg(pParse, "index associated with UNIQUE " + "or PRIMARY KEY constraint cannot be dropped", 0); + goto exit_drop_index; + } + iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int code = SQLITE_DROP_INDEX; + Table *pTab = pIndex->pTable; + const char *zDb = db->aDb[iDb].zDbSName; + const char *zTab = SCHEMA_TABLE(iDb); + if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ + goto exit_drop_index; + } + if( !OMIT_TEMPDB && iDb==1 ) code = SQLITE_DROP_TEMP_INDEX; + if( sqlite3AuthCheck(pParse, code, pIndex->zName, pTab->zName, zDb) ){ + goto exit_drop_index; + } + } +#endif + + /* Generate code to remove the index and from the schema table */ + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3BeginWriteOperation(pParse, 1, iDb); + sqlite3NestedParse(pParse, + "DELETE FROM %Q." DFLT_SCHEMA_TABLE " WHERE name=%Q AND type='index'", + db->aDb[iDb].zDbSName, pIndex->zName + ); + sqlite3ClearStatTables(pParse, iDb, "idx", pIndex->zName); + sqlite3ChangeCookie(pParse, iDb); + destroyRootPage(pParse, pIndex->tnum, iDb); + sqlite3VdbeAddOp4(v, OP_DropIndex, iDb, 0, 0, pIndex->zName, 0); + } + +exit_drop_index: + sqlite3SrcListDelete(db, pName); +} + +/* +** pArray is a pointer to an array of objects. Each object in the +** array is szEntry bytes in size. This routine uses sqlite3DbRealloc() +** to extend the array so that there is space for a new object at the end. +** +** When this function is called, *pnEntry contains the current size of +** the array (in entries - so the allocation is ((*pnEntry) * szEntry) bytes +** in total). +** +** If the realloc() is successful (i.e. if no OOM condition occurs), the +** space allocated for the new object is zeroed, *pnEntry updated to +** reflect the new size of the array and a pointer to the new allocation +** returned. *pIdx is set to the index of the new array entry in this case. +** +** Otherwise, if the realloc() fails, *pIdx is set to -1, *pnEntry remains +** unchanged and a copy of pArray returned. +*/ +SQLITE_PRIVATE void *sqlite3ArrayAllocate( + sqlite3 *db, /* Connection to notify of malloc failures */ + void *pArray, /* Array of objects. Might be reallocated */ + int szEntry, /* Size of each object in the array */ + int *pnEntry, /* Number of objects currently in use */ + int *pIdx /* Write the index of a new slot here */ +){ + char *z; + sqlite3_int64 n = *pIdx = *pnEntry; + if( (n & (n-1))==0 ){ + sqlite3_int64 sz = (n==0) ? 1 : 2*n; + void *pNew = sqlite3DbRealloc(db, pArray, sz*szEntry); + if( pNew==0 ){ + *pIdx = -1; + return pArray; + } + pArray = pNew; + } + z = (char*)pArray; + memset(&z[n * szEntry], 0, szEntry); + ++*pnEntry; + return pArray; +} + +/* +** Append a new element to the given IdList. Create a new IdList if +** need be. +** +** A new IdList is returned, or NULL if malloc() fails. +*/ +SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){ + sqlite3 *db = pParse->db; + int i; + if( pList==0 ){ + pList = sqlite3DbMallocZero(db, sizeof(IdList) ); + if( pList==0 ) return 0; + } + pList->a = sqlite3ArrayAllocate( + db, + pList->a, + sizeof(pList->a[0]), + &pList->nId, + &i + ); + if( i<0 ){ + sqlite3IdListDelete(db, pList); + return 0; + } + pList->a[i].zName = sqlite3NameFromToken(db, pToken); + if( IN_RENAME_OBJECT && pList->a[i].zName ){ + sqlite3RenameTokenMap(pParse, (void*)pList->a[i].zName, pToken); + } + return pList; +} + +/* +** Delete an IdList. +*/ +SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ + int i; + if( pList==0 ) return; + for(i=0; inId; i++){ + sqlite3DbFree(db, pList->a[i].zName); + } + sqlite3DbFree(db, pList->a); + sqlite3DbFreeNN(db, pList); +} + +/* +** Return the index in pList of the identifier named zId. Return -1 +** if not found. +*/ +SQLITE_PRIVATE int sqlite3IdListIndex(IdList *pList, const char *zName){ + int i; + if( pList==0 ) return -1; + for(i=0; inId; i++){ + if( sqlite3StrICmp(pList->a[i].zName, zName)==0 ) return i; + } + return -1; +} + +/* +** Maximum size of a SrcList object. +** The SrcList object is used to represent the FROM clause of a +** SELECT statement, and the query planner cannot deal with more +** than 64 tables in a join. So any value larger than 64 here +** is sufficient for most uses. Smaller values, like say 10, are +** appropriate for small and memory-limited applications. +*/ +#ifndef SQLITE_MAX_SRCLIST +# define SQLITE_MAX_SRCLIST 200 +#endif + +/* +** Expand the space allocated for the given SrcList object by +** creating nExtra new slots beginning at iStart. iStart is zero based. +** New slots are zeroed. +** +** For example, suppose a SrcList initially contains two entries: A,B. +** To append 3 new entries onto the end, do this: +** +** sqlite3SrcListEnlarge(db, pSrclist, 3, 2); +** +** After the call above it would contain: A, B, nil, nil, nil. +** If the iStart argument had been 1 instead of 2, then the result +** would have been: A, nil, nil, nil, B. To prepend the new slots, +** the iStart value would be 0. The result then would +** be: nil, nil, nil, A, B. +** +** If a memory allocation fails or the SrcList becomes too large, leave +** the original SrcList unchanged, return NULL, and leave an error message +** in pParse. +*/ +SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( + Parse *pParse, /* Parsing context into which errors are reported */ + SrcList *pSrc, /* The SrcList to be enlarged */ + int nExtra, /* Number of new slots to add to pSrc->a[] */ + int iStart /* Index in pSrc->a[] of first new slot */ +){ + int i; + + /* Sanity checking on calling parameters */ + assert( iStart>=0 ); + assert( nExtra>=1 ); + assert( pSrc!=0 ); + assert( iStart<=pSrc->nSrc ); + + /* Allocate additional space if needed */ + if( (u32)pSrc->nSrc+nExtra>pSrc->nAlloc ){ + SrcList *pNew; + sqlite3_int64 nAlloc = 2*(sqlite3_int64)pSrc->nSrc+nExtra; + sqlite3 *db = pParse->db; + + if( pSrc->nSrc+nExtra>=SQLITE_MAX_SRCLIST ){ + sqlite3ErrorMsg(pParse, "too many FROM clause terms, max: %d", + SQLITE_MAX_SRCLIST); + return 0; + } + if( nAlloc>SQLITE_MAX_SRCLIST ) nAlloc = SQLITE_MAX_SRCLIST; + pNew = sqlite3DbRealloc(db, pSrc, + sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); + if( pNew==0 ){ + assert( db->mallocFailed ); + return 0; + } + pSrc = pNew; + pSrc->nAlloc = nAlloc; + } + + /* Move existing slots that come after the newly inserted slots + ** out of the way */ + for(i=pSrc->nSrc-1; i>=iStart; i--){ + pSrc->a[i+nExtra] = pSrc->a[i]; + } + pSrc->nSrc += nExtra; + + /* Zero the newly allocated slots */ + memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra); + for(i=iStart; ia[i].iCursor = -1; + } + + /* Return a pointer to the enlarged SrcList */ + return pSrc; +} + + +/* +** Append a new table name to the given SrcList. Create a new SrcList if +** need be. A new entry is created in the SrcList even if pTable is NULL. +** +** A SrcList is returned, or NULL if there is an OOM error or if the +** SrcList grows to large. The returned +** SrcList might be the same as the SrcList that was input or it might be +** a new one. If an OOM error does occurs, then the prior value of pList +** that is input to this routine is automatically freed. +** +** If pDatabase is not null, it means that the table has an optional +** database name prefix. Like this: "database.table". The pDatabase +** points to the table name and the pTable points to the database name. +** The SrcList.a[].zName field is filled with the table name which might +** come from pTable (if pDatabase is NULL) or from pDatabase. +** SrcList.a[].zDatabase is filled with the database name from pTable, +** or with NULL if no database is specified. +** +** In other words, if call like this: +** +** sqlite3SrcListAppend(D,A,B,0); +** +** Then B is a table name and the database name is unspecified. If called +** like this: +** +** sqlite3SrcListAppend(D,A,B,C); +** +** Then C is the table name and B is the database name. If C is defined +** then so is B. In other words, we never have a case where: +** +** sqlite3SrcListAppend(D,A,0,C); +** +** Both pTable and pDatabase are assumed to be quoted. They are dequoted +** before being added to the SrcList. +*/ +SQLITE_PRIVATE SrcList *sqlite3SrcListAppend( + Parse *pParse, /* Parsing context, in which errors are reported */ + SrcList *pList, /* Append to this SrcList. NULL creates a new SrcList */ + Token *pTable, /* Table to append */ + Token *pDatabase /* Database of the table */ +){ + SrcItem *pItem; + sqlite3 *db; + assert( pDatabase==0 || pTable!=0 ); /* Cannot have C without B */ + assert( pParse!=0 ); + assert( pParse->db!=0 ); + db = pParse->db; + if( pList==0 ){ + pList = sqlite3DbMallocRawNN(pParse->db, sizeof(SrcList) ); + if( pList==0 ) return 0; + pList->nAlloc = 1; + pList->nSrc = 1; + memset(&pList->a[0], 0, sizeof(pList->a[0])); + pList->a[0].iCursor = -1; + }else{ + SrcList *pNew = sqlite3SrcListEnlarge(pParse, pList, 1, pList->nSrc); + if( pNew==0 ){ + sqlite3SrcListDelete(db, pList); + return 0; + }else{ + pList = pNew; + } + } + pItem = &pList->a[pList->nSrc-1]; + if( pDatabase && pDatabase->z==0 ){ + pDatabase = 0; + } + if( pDatabase ){ + pItem->zName = sqlite3NameFromToken(db, pDatabase); + pItem->zDatabase = sqlite3NameFromToken(db, pTable); + }else{ + pItem->zName = sqlite3NameFromToken(db, pTable); + pItem->zDatabase = 0; + } + return pList; +} + +/* +** Assign VdbeCursor index numbers to all tables in a SrcList +*/ +SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ + int i; + SrcItem *pItem; + assert( pList || pParse->db->mallocFailed ); + if( ALWAYS(pList) ){ + for(i=0, pItem=pList->a; inSrc; i++, pItem++){ + if( pItem->iCursor>=0 ) continue; + pItem->iCursor = pParse->nTab++; + if( pItem->pSelect ){ + sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc); + } + } + } +} + +/* +** Delete an entire SrcList including all its substructure. +*/ +SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ + int i; + SrcItem *pItem; + if( pList==0 ) return; + for(pItem=pList->a, i=0; inSrc; i++, pItem++){ + if( pItem->zDatabase ) sqlite3DbFreeNN(db, pItem->zDatabase); + sqlite3DbFree(db, pItem->zName); + if( pItem->zAlias ) sqlite3DbFreeNN(db, pItem->zAlias); + if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); + if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); + sqlite3DeleteTable(db, pItem->pTab); + if( pItem->pSelect ) sqlite3SelectDelete(db, pItem->pSelect); + if( pItem->pOn ) sqlite3ExprDelete(db, pItem->pOn); + if( pItem->pUsing ) sqlite3IdListDelete(db, pItem->pUsing); + } + sqlite3DbFreeNN(db, pList); +} + +/* +** This routine is called by the parser to add a new term to the +** end of a growing FROM clause. The "p" parameter is the part of +** the FROM clause that has already been constructed. "p" is NULL +** if this is the first term of the FROM clause. pTable and pDatabase +** are the name of the table and database named in the FROM clause term. +** pDatabase is NULL if the database name qualifier is missing - the +** usual case. If the term has an alias, then pAlias points to the +** alias token. If the term is a subquery, then pSubquery is the +** SELECT statement that the subquery encodes. The pTable and +** pDatabase parameters are NULL for subqueries. The pOn and pUsing +** parameters are the content of the ON and USING clauses. +** +** Return a new SrcList which encodes is the FROM with the new +** term added. +*/ +SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm( + Parse *pParse, /* Parsing context */ + SrcList *p, /* The left part of the FROM clause already seen */ + Token *pTable, /* Name of the table to add to the FROM clause */ + Token *pDatabase, /* Name of the database containing pTable */ + Token *pAlias, /* The right-hand side of the AS subexpression */ + Select *pSubquery, /* A subquery used in place of a table name */ + Expr *pOn, /* The ON clause of a join */ + IdList *pUsing /* The USING clause of a join */ +){ + SrcItem *pItem; + sqlite3 *db = pParse->db; + if( !p && (pOn || pUsing) ){ + sqlite3ErrorMsg(pParse, "a JOIN clause is required before %s", + (pOn ? "ON" : "USING") + ); + goto append_from_error; + } + p = sqlite3SrcListAppend(pParse, p, pTable, pDatabase); + if( p==0 ){ + goto append_from_error; + } + assert( p->nSrc>0 ); + pItem = &p->a[p->nSrc-1]; + assert( (pTable==0)==(pDatabase==0) ); + assert( pItem->zName==0 || pDatabase!=0 ); + if( IN_RENAME_OBJECT && pItem->zName ){ + Token *pToken = (ALWAYS(pDatabase) && pDatabase->z) ? pDatabase : pTable; + sqlite3RenameTokenMap(pParse, pItem->zName, pToken); + } + assert( pAlias!=0 ); + if( pAlias->n ){ + pItem->zAlias = sqlite3NameFromToken(db, pAlias); + } + pItem->pSelect = pSubquery; + pItem->pOn = pOn; + pItem->pUsing = pUsing; + return p; + + append_from_error: + assert( p==0 ); + sqlite3ExprDelete(db, pOn); + sqlite3IdListDelete(db, pUsing); + sqlite3SelectDelete(db, pSubquery); + return 0; +} + +/* +** Add an INDEXED BY or NOT INDEXED clause to the most recently added +** element of the source-list passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ + assert( pIndexedBy!=0 ); + if( p && pIndexedBy->n>0 ){ + SrcItem *pItem; + assert( p->nSrc>0 ); + pItem = &p->a[p->nSrc-1]; + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); + if( pIndexedBy->n==1 && !pIndexedBy->z ){ + /* A "NOT INDEXED" clause was supplied. See parse.y + ** construct "indexed_opt" for details. */ + pItem->fg.notIndexed = 1; + }else{ + pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); + pItem->fg.isIndexedBy = 1; + } + } +} + +/* +** Append the contents of SrcList p2 to SrcList p1 and return the resulting +** SrcList. Or, if an error occurs, return NULL. In all cases, p1 and p2 +** are deleted by this function. +*/ +SQLITE_PRIVATE SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2){ + assert( p1 && p1->nSrc==1 ); + if( p2 ){ + SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, 1); + if( pNew==0 ){ + sqlite3SrcListDelete(pParse->db, p2); + }else{ + p1 = pNew; + memcpy(&p1->a[1], p2->a, p2->nSrc*sizeof(SrcItem)); + sqlite3DbFree(pParse->db, p2); + } + } + return p1; +} + +/* +** Add the list of function arguments to the SrcList entry for a +** table-valued-function. +*/ +SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList *pList){ + if( p ){ + SrcItem *pItem = &p->a[p->nSrc-1]; + assert( pItem->fg.notIndexed==0 ); + assert( pItem->fg.isIndexedBy==0 ); + assert( pItem->fg.isTabFunc==0 ); + pItem->u1.pFuncArg = pList; + pItem->fg.isTabFunc = 1; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + +/* +** When building up a FROM clause in the parser, the join operator +** is initially attached to the left operand. But the code generator +** expects the join operator to be on the right operand. This routine +** Shifts all join operators from left to right for an entire FROM +** clause. +** +** Example: Suppose the join is like this: +** +** A natural cross join B +** +** The operator is "natural cross join". The A and B operands are stored +** in p->a[0] and p->a[1], respectively. The parser initially stores the +** operator with A. This routine shifts that operator over to B. +*/ +SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ + if( p ){ + int i; + for(i=p->nSrc-1; i>0; i--){ + p->a[i].fg.jointype = p->a[i-1].fg.jointype; + } + p->a[0].fg.jointype = 0; + } +} + +/* +** Generate VDBE code for a BEGIN statement. +*/ +SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ + sqlite3 *db; + Vdbe *v; + int i; + + assert( pParse!=0 ); + db = pParse->db; + assert( db!=0 ); + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "BEGIN", 0, 0) ){ + return; + } + v = sqlite3GetVdbe(pParse); + if( !v ) return; + if( type!=TK_DEFERRED ){ + for(i=0; inDb; i++){ + int eTxnType; + Btree *pBt = db->aDb[i].pBt; + if( pBt && sqlite3BtreeIsReadonly(pBt) ){ + eTxnType = 0; /* Read txn */ + }else if( type==TK_EXCLUSIVE ){ + eTxnType = 2; /* Exclusive txn */ + }else{ + eTxnType = 1; /* Write txn */ + } + sqlite3VdbeAddOp2(v, OP_Transaction, i, eTxnType); + sqlite3VdbeUsesBtree(v, i); + } + } + sqlite3VdbeAddOp0(v, OP_AutoCommit); +} + +/* +** Generate VDBE code for a COMMIT or ROLLBACK statement. +** Code for ROLLBACK is generated if eType==TK_ROLLBACK. Otherwise +** code is generated for a COMMIT. +*/ +SQLITE_PRIVATE void sqlite3EndTransaction(Parse *pParse, int eType){ + Vdbe *v; + int isRollback; + + assert( pParse!=0 ); + assert( pParse->db!=0 ); + assert( eType==TK_COMMIT || eType==TK_END || eType==TK_ROLLBACK ); + isRollback = eType==TK_ROLLBACK; + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, + isRollback ? "ROLLBACK" : "COMMIT", 0, 0) ){ + return; + } + v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, isRollback); + } +} + +/* +** This function is called by the parser when it parses a command to create, +** release or rollback an SQL savepoint. +*/ +SQLITE_PRIVATE void sqlite3Savepoint(Parse *pParse, int op, Token *pName){ + char *zName = sqlite3NameFromToken(pParse->db, pName); + if( zName ){ + Vdbe *v = sqlite3GetVdbe(pParse); +#ifndef SQLITE_OMIT_AUTHORIZATION + static const char * const az[] = { "BEGIN", "RELEASE", "ROLLBACK" }; + assert( !SAVEPOINT_BEGIN && SAVEPOINT_RELEASE==1 && SAVEPOINT_ROLLBACK==2 ); +#endif + if( !v || sqlite3AuthCheck(pParse, SQLITE_SAVEPOINT, az[op], zName, 0) ){ + sqlite3DbFree(pParse->db, zName); + return; + } + sqlite3VdbeAddOp4(v, OP_Savepoint, op, 0, 0, zName, P4_DYNAMIC); + } +} + +/* +** Make sure the TEMP database is open and available for use. Return +** the number of errors. Leave any error messages in the pParse structure. +*/ +SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ + sqlite3 *db = pParse->db; + if( db->aDb[1].pBt==0 && !pParse->explain ){ + int rc; + Btree *pBt; + static const int flags = + SQLITE_OPEN_READWRITE | + SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_TEMP_DB; + + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pBt, 0, flags); + if( rc!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "unable to open a temporary database " + "file for storing temporary tables"); + pParse->rc = rc; + return 1; + } + db->aDb[1].pBt = pBt; + assert( db->aDb[1].pSchema ); + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, 0, 0) ){ + sqlite3OomFault(db); + return 1; + } + } + return 0; +} + +/* +** Record the fact that the schema cookie will need to be verified +** for database iDb. The code to actually verify the schema cookie +** will occur at the end of the top-level VDBE and will be generated +** later, by sqlite3FinishCoding(). +*/ +static void sqlite3CodeVerifySchemaAtToplevel(Parse *pToplevel, int iDb){ + assert( iDb>=0 && iDbdb->nDb ); + assert( pToplevel->db->aDb[iDb].pBt!=0 || iDb==1 ); + assert( iDbdb, iDb, 0) ); + if( DbMaskTest(pToplevel->cookieMask, iDb)==0 ){ + DbMaskSet(pToplevel->cookieMask, iDb); + if( !OMIT_TEMPDB && iDb==1 ){ + sqlite3OpenTempDatabase(pToplevel); + } + } +} +SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ + sqlite3CodeVerifySchemaAtToplevel(sqlite3ParseToplevel(pParse), iDb); +} + + +/* +** If argument zDb is NULL, then call sqlite3CodeVerifySchema() for each +** attached database. Otherwise, invoke it for the database named zDb only. +*/ +SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ + sqlite3 *db = pParse->db; + int i; + for(i=0; inDb; i++){ + Db *pDb = &db->aDb[i]; + if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zDbSName)) ){ + sqlite3CodeVerifySchema(pParse, i); + } + } +} + +/* +** Generate VDBE code that prepares for doing an operation that +** might change the database. +** +** This routine starts a new transaction if we are not already within +** a transaction. If we are already within a transaction, then a checkpoint +** is set if the setStatement parameter is true. A checkpoint should +** be set for operations that might fail (due to a constraint) part of +** the way through and which will need to undo some writes without having to +** rollback the whole transaction. For operations where all constraints +** can be checked before any changes are made to the database, it is never +** necessary to undo a write and the checkpoint should not be set. +*/ +SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + sqlite3CodeVerifySchemaAtToplevel(pToplevel, iDb); + DbMaskSet(pToplevel->writeMask, iDb); + pToplevel->isMultiWrite |= setStatement; +} + +/* +** Indicate that the statement currently under construction might write +** more than one entry (example: deleting one row then inserting another, +** inserting multiple rows in a table, or inserting a row and index entries.) +** If an abort occurs after some of these writes have completed, then it will +** be necessary to undo the completed writes. +*/ +SQLITE_PRIVATE void sqlite3MultiWrite(Parse *pParse){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + pToplevel->isMultiWrite = 1; +} + +/* +** The code generator calls this routine if is discovers that it is +** possible to abort a statement prior to completion. In order to +** perform this abort without corrupting the database, we need to make +** sure that the statement is protected by a statement transaction. +** +** Technically, we only need to set the mayAbort flag if the +** isMultiWrite flag was previously set. There is a time dependency +** such that the abort must occur after the multiwrite. This makes +** some statements involving the REPLACE conflict resolution algorithm +** go a little faster. But taking advantage of this time dependency +** makes it more difficult to prove that the code is correct (in +** particular, it prevents us from writing an effective +** implementation of sqlite3AssertMayAbort()) and so we have chosen +** to take the safe route and skip the optimization. +*/ +SQLITE_PRIVATE void sqlite3MayAbort(Parse *pParse){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + pToplevel->mayAbort = 1; +} + +/* +** Code an OP_Halt that causes the vdbe to return an SQLITE_CONSTRAINT +** error. The onError parameter determines which (if any) of the statement +** and/or current transaction is rolled back. +*/ +SQLITE_PRIVATE void sqlite3HaltConstraint( + Parse *pParse, /* Parsing context */ + int errCode, /* extended error code */ + int onError, /* Constraint type */ + char *p4, /* Error message */ + i8 p4type, /* P4_STATIC or P4_TRANSIENT */ + u8 p5Errmsg /* P5_ErrMsg type */ +){ + Vdbe *v; + assert( pParse->pVdbe!=0 ); + v = sqlite3GetVdbe(pParse); + assert( (errCode&0xff)==SQLITE_CONSTRAINT || pParse->nested ); + if( onError==OE_Abort ){ + sqlite3MayAbort(pParse); + } + sqlite3VdbeAddOp4(v, OP_Halt, errCode, onError, 0, p4, p4type); + sqlite3VdbeChangeP5(v, p5Errmsg); +} + +/* +** Code an OP_Halt due to UNIQUE or PRIMARY KEY constraint violation. +*/ +SQLITE_PRIVATE void sqlite3UniqueConstraint( + Parse *pParse, /* Parsing context */ + int onError, /* Constraint type */ + Index *pIdx /* The index that triggers the constraint */ +){ + char *zErr; + int j; + StrAccum errMsg; + Table *pTab = pIdx->pTable; + + sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, + pParse->db->aLimit[SQLITE_LIMIT_LENGTH]); + if( pIdx->aColExpr ){ + sqlite3_str_appendf(&errMsg, "index '%q'", pIdx->zName); + }else{ + for(j=0; jnKeyCol; j++){ + char *zCol; + assert( pIdx->aiColumn[j]>=0 ); + zCol = pTab->aCol[pIdx->aiColumn[j]].zCnName; + if( j ) sqlite3_str_append(&errMsg, ", ", 2); + sqlite3_str_appendall(&errMsg, pTab->zName); + sqlite3_str_append(&errMsg, ".", 1); + sqlite3_str_appendall(&errMsg, zCol); + } + } + zErr = sqlite3StrAccumFinish(&errMsg); + sqlite3HaltConstraint(pParse, + IsPrimaryKeyIndex(pIdx) ? SQLITE_CONSTRAINT_PRIMARYKEY + : SQLITE_CONSTRAINT_UNIQUE, + onError, zErr, P4_DYNAMIC, P5_ConstraintUnique); +} + + +/* +** Code an OP_Halt due to non-unique rowid. +*/ +SQLITE_PRIVATE void sqlite3RowidConstraint( + Parse *pParse, /* Parsing context */ + int onError, /* Conflict resolution algorithm */ + Table *pTab /* The table with the non-unique rowid */ +){ + char *zMsg; + int rc; + if( pTab->iPKey>=0 ){ + zMsg = sqlite3MPrintf(pParse->db, "%s.%s", pTab->zName, + pTab->aCol[pTab->iPKey].zCnName); + rc = SQLITE_CONSTRAINT_PRIMARYKEY; + }else{ + zMsg = sqlite3MPrintf(pParse->db, "%s.rowid", pTab->zName); + rc = SQLITE_CONSTRAINT_ROWID; + } + sqlite3HaltConstraint(pParse, rc, onError, zMsg, P4_DYNAMIC, + P5_ConstraintUnique); +} + +/* +** Check to see if pIndex uses the collating sequence pColl. Return +** true if it does and false if it does not. +*/ +#ifndef SQLITE_OMIT_REINDEX +static int collationMatch(const char *zColl, Index *pIndex){ + int i; + assert( zColl!=0 ); + for(i=0; inColumn; i++){ + const char *z = pIndex->azColl[i]; + assert( z!=0 || pIndex->aiColumn[i]<0 ); + if( pIndex->aiColumn[i]>=0 && 0==sqlite3StrICmp(z, zColl) ){ + return 1; + } + } + return 0; +} +#endif + +/* +** Recompute all indices of pTab that use the collating sequence pColl. +** If pColl==0 then recompute all indices of pTab. +*/ +#ifndef SQLITE_OMIT_REINDEX +static void reindexTable(Parse *pParse, Table *pTab, char const *zColl){ + if( !IsVirtual(pTab) ){ + Index *pIndex; /* An index associated with pTab */ + + for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ + if( zColl==0 || collationMatch(zColl, pIndex) ){ + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIndex, -1); + } + } + } +} +#endif + +/* +** Recompute all indices of all tables in all databases where the +** indices use the collating sequence pColl. If pColl==0 then recompute +** all indices everywhere. +*/ +#ifndef SQLITE_OMIT_REINDEX +static void reindexDatabases(Parse *pParse, char const *zColl){ + Db *pDb; /* A single database */ + int iDb; /* The database index number */ + sqlite3 *db = pParse->db; /* The database connection */ + HashElem *k; /* For looping over tables in pDb */ + Table *pTab; /* A table in the database */ + + assert( sqlite3BtreeHoldsAllMutexes(db) ); /* Needed for schema access */ + for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ + assert( pDb!=0 ); + for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + reindexTable(pParse, pTab, zColl); + } + } +} +#endif + +/* +** Generate code for the REINDEX command. +** +** REINDEX -- 1 +** REINDEX -- 2 +** REINDEX ?.? -- 3 +** REINDEX ?.? -- 4 +** +** Form 1 causes all indices in all attached databases to be rebuilt. +** Form 2 rebuilds all indices in all databases that use the named +** collating function. Forms 3 and 4 rebuild the named index or all +** indices associated with the named table. +*/ +#ifndef SQLITE_OMIT_REINDEX +SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ + CollSeq *pColl; /* Collating sequence to be reindexed, or NULL */ + char *z; /* Name of a table or index */ + const char *zDb; /* Name of the database */ + Table *pTab; /* A table in the database */ + Index *pIndex; /* An index associated with pTab */ + int iDb; /* The database index number */ + sqlite3 *db = pParse->db; /* The database connection */ + Token *pObjName; /* Name of the table or index to be reindexed */ + + /* Read the database schema. If an error occurs, leave an error message + ** and code in pParse and return NULL. */ + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + return; + } + + if( pName1==0 ){ + reindexDatabases(pParse, 0); + return; + }else if( NEVER(pName2==0) || pName2->z==0 ){ + char *zColl; + assert( pName1->z ); + zColl = sqlite3NameFromToken(pParse->db, pName1); + if( !zColl ) return; + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); + if( pColl ){ + reindexDatabases(pParse, zColl); + sqlite3DbFree(db, zColl); + return; + } + sqlite3DbFree(db, zColl); + } + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pObjName); + if( iDb<0 ) return; + z = sqlite3NameFromToken(db, pObjName); + if( z==0 ) return; + zDb = db->aDb[iDb].zDbSName; + pTab = sqlite3FindTable(db, z, zDb); + if( pTab ){ + reindexTable(pParse, pTab, 0); + sqlite3DbFree(db, z); + return; + } + pIndex = sqlite3FindIndex(db, z, zDb); + sqlite3DbFree(db, z); + if( pIndex ){ + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIndex, -1); + return; + } + sqlite3ErrorMsg(pParse, "unable to identify the object to be reindexed"); +} +#endif + +/* +** Return a KeyInfo structure that is appropriate for the given Index. +** +** The caller should invoke sqlite3KeyInfoUnref() on the returned object +** when it has finished using it. +*/ +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ + int i; + int nCol = pIdx->nColumn; + int nKey = pIdx->nKeyCol; + KeyInfo *pKey; + if( pParse->nErr ) return 0; + if( pIdx->uniqNotNull ){ + pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); + }else{ + pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); + } + if( pKey ){ + assert( sqlite3KeyInfoIsWriteable(pKey) ); + for(i=0; iazColl[i]; + pKey->aColl[i] = zColl==sqlite3StrBINARY ? 0 : + sqlite3LocateCollSeq(pParse, zColl); + pKey->aSortFlags[i] = pIdx->aSortOrder[i]; + assert( 0==(pKey->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) ); + } + if( pParse->nErr ){ + assert( pParse->rc==SQLITE_ERROR_MISSING_COLLSEQ ); + if( pIdx->bNoQuery==0 ){ + /* Deactivate the index because it contains an unknown collating + ** sequence. The only way to reactive the index is to reload the + ** schema. Adding the missing collating sequence later does not + ** reactive the index. The application had the chance to register + ** the missing index using the collation-needed callback. For + ** simplicity, SQLite will not give the application a second chance. + */ + pIdx->bNoQuery = 1; + pParse->rc = SQLITE_ERROR_RETRY; + } + sqlite3KeyInfoUnref(pKey); + pKey = 0; + } + } + return pKey; +} + +#ifndef SQLITE_OMIT_CTE +/* +** Create a new CTE object +*/ +SQLITE_PRIVATE Cte *sqlite3CteNew( + Parse *pParse, /* Parsing context */ + Token *pName, /* Name of the common-table */ + ExprList *pArglist, /* Optional column name list for the table */ + Select *pQuery, /* Query used to initialize the table */ + u8 eM10d /* The MATERIALIZED flag */ +){ + Cte *pNew; + sqlite3 *db = pParse->db; + + pNew = sqlite3DbMallocZero(db, sizeof(*pNew)); + assert( pNew!=0 || db->mallocFailed ); + + if( db->mallocFailed ){ + sqlite3ExprListDelete(db, pArglist); + sqlite3SelectDelete(db, pQuery); + }else{ + pNew->pSelect = pQuery; + pNew->pCols = pArglist; + pNew->zName = sqlite3NameFromToken(pParse->db, pName); + pNew->eM10d = eM10d; + } + return pNew; +} + +/* +** Clear information from a Cte object, but do not deallocate storage +** for the object itself. +*/ +static void cteClear(sqlite3 *db, Cte *pCte){ + assert( pCte!=0 ); + sqlite3ExprListDelete(db, pCte->pCols); + sqlite3SelectDelete(db, pCte->pSelect); + sqlite3DbFree(db, pCte->zName); +} + +/* +** Free the contents of the CTE object passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3CteDelete(sqlite3 *db, Cte *pCte){ + assert( pCte!=0 ); + cteClear(db, pCte); + sqlite3DbFree(db, pCte); +} + +/* +** This routine is invoked once per CTE by the parser while parsing a +** WITH clause. The CTE described by teh third argument is added to +** the WITH clause of the second argument. If the second argument is +** NULL, then a new WITH argument is created. +*/ +SQLITE_PRIVATE With *sqlite3WithAdd( + Parse *pParse, /* Parsing context */ + With *pWith, /* Existing WITH clause, or NULL */ + Cte *pCte /* CTE to add to the WITH clause */ +){ + sqlite3 *db = pParse->db; + With *pNew; + char *zName; + + if( pCte==0 ){ + return pWith; + } + + /* Check that the CTE name is unique within this WITH clause. If + ** not, store an error in the Parse structure. */ + zName = pCte->zName; + if( zName && pWith ){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){ + sqlite3ErrorMsg(pParse, "duplicate WITH table name: %s", zName); + } + } + } + + if( pWith ){ + sqlite3_int64 nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); + pNew = sqlite3DbRealloc(db, pWith, nByte); + }else{ + pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); + } + assert( (pNew!=0 && zName!=0) || db->mallocFailed ); + + if( db->mallocFailed ){ + sqlite3CteDelete(db, pCte); + pNew = pWith; + }else{ + pNew->a[pNew->nCte++] = *pCte; + sqlite3DbFree(db, pCte); + } + + return pNew; +} + +/* +** Free the contents of the With object passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3WithDelete(sqlite3 *db, With *pWith){ + if( pWith ){ + int i; + for(i=0; inCte; i++){ + cteClear(db, &pWith->a[i]); + } + sqlite3DbFree(db, pWith); + } +} +#endif /* !defined(SQLITE_OMIT_CTE) */ + +/************** End of build.c ***********************************************/ +/************** Begin file callback.c ****************************************/ +/* +** 2005 May 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains functions used to access the internal hash tables +** of user defined functions and collation sequences. +*/ + +/* #include "sqliteInt.h" */ + +/* +** Invoke the 'collation needed' callback to request a collation sequence +** in the encoding enc of name zName, length nName. +*/ +static void callCollNeeded(sqlite3 *db, int enc, const char *zName){ + assert( !db->xCollNeeded || !db->xCollNeeded16 ); + if( db->xCollNeeded ){ + char *zExternal = sqlite3DbStrDup(db, zName); + if( !zExternal ) return; + db->xCollNeeded(db->pCollNeededArg, db, enc, zExternal); + sqlite3DbFree(db, zExternal); + } +#ifndef SQLITE_OMIT_UTF16 + if( db->xCollNeeded16 ){ + char const *zExternal; + sqlite3_value *pTmp = sqlite3ValueNew(db); + sqlite3ValueSetStr(pTmp, -1, zName, SQLITE_UTF8, SQLITE_STATIC); + zExternal = sqlite3ValueText(pTmp, SQLITE_UTF16NATIVE); + if( zExternal ){ + db->xCollNeeded16(db->pCollNeededArg, db, (int)ENC(db), zExternal); + } + sqlite3ValueFree(pTmp); + } +#endif +} + +/* +** This routine is called if the collation factory fails to deliver a +** collation function in the best encoding but there may be other versions +** of this collation function (for other text encodings) available. Use one +** of these instead if they exist. Avoid a UTF-8 <-> UTF-16 conversion if +** possible. +*/ +static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ + CollSeq *pColl2; + char *z = pColl->zName; + int i; + static const u8 aEnc[] = { SQLITE_UTF16BE, SQLITE_UTF16LE, SQLITE_UTF8 }; + for(i=0; i<3; i++){ + pColl2 = sqlite3FindCollSeq(db, aEnc[i], z, 0); + if( pColl2->xCmp!=0 ){ + memcpy(pColl, pColl2, sizeof(CollSeq)); + pColl->xDel = 0; /* Do not copy the destructor */ + return SQLITE_OK; + } + } + return SQLITE_ERROR; +} + +/* +** This routine is called on a collation sequence before it is used to +** check that it is defined. An undefined collation sequence exists when +** a database is loaded that contains references to collation sequences +** that have not been defined by sqlite3_create_collation() etc. +** +** If required, this routine calls the 'collation needed' callback to +** request a definition of the collating sequence. If this doesn't work, +** an equivalent collating sequence that uses a text encoding different +** from the main database is substituted, if one is available. +*/ +SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ + if( pColl && pColl->xCmp==0 ){ + const char *zName = pColl->zName; + sqlite3 *db = pParse->db; + CollSeq *p = sqlite3GetCollSeq(pParse, ENC(db), pColl, zName); + if( !p ){ + return SQLITE_ERROR; + } + assert( p==pColl ); + } + return SQLITE_OK; +} + + + +/* +** Locate and return an entry from the db.aCollSeq hash table. If the entry +** specified by zName and nName is not found and parameter 'create' is +** true, then create a new entry. Otherwise return NULL. +** +** Each pointer stored in the sqlite3.aCollSeq hash table contains an +** array of three CollSeq structures. The first is the collation sequence +** preferred for UTF-8, the second UTF-16le, and the third UTF-16be. +** +** Stored immediately after the three collation sequences is a copy of +** the collation sequence name. A pointer to this string is stored in +** each collation sequence structure. +*/ +static CollSeq *findCollSeqEntry( + sqlite3 *db, /* Database connection */ + const char *zName, /* Name of the collating sequence */ + int create /* Create a new entry if true */ +){ + CollSeq *pColl; + pColl = sqlite3HashFind(&db->aCollSeq, zName); + + if( 0==pColl && create ){ + int nName = sqlite3Strlen30(zName) + 1; + pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName); + if( pColl ){ + CollSeq *pDel = 0; + pColl[0].zName = (char*)&pColl[3]; + pColl[0].enc = SQLITE_UTF8; + pColl[1].zName = (char*)&pColl[3]; + pColl[1].enc = SQLITE_UTF16LE; + pColl[2].zName = (char*)&pColl[3]; + pColl[2].enc = SQLITE_UTF16BE; + memcpy(pColl[0].zName, zName, nName); + pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, pColl); + + /* If a malloc() failure occurred in sqlite3HashInsert(), it will + ** return the pColl pointer to be deleted (because it wasn't added + ** to the hash table). + */ + assert( pDel==0 || pDel==pColl ); + if( pDel!=0 ){ + sqlite3OomFault(db); + sqlite3DbFree(db, pDel); + pColl = 0; + } + } + } + return pColl; +} + +/* +** Parameter zName points to a UTF-8 encoded string nName bytes long. +** Return the CollSeq* pointer for the collation sequence named zName +** for the encoding 'enc' from the database 'db'. +** +** If the entry specified is not found and 'create' is true, then create a +** new entry. Otherwise return NULL. +** +** A separate function sqlite3LocateCollSeq() is a wrapper around +** this routine. sqlite3LocateCollSeq() invokes the collation factory +** if necessary and generates an error message if the collating sequence +** cannot be found. +** +** See also: sqlite3LocateCollSeq(), sqlite3GetCollSeq() +*/ +SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( + sqlite3 *db, /* Database connection to search */ + u8 enc, /* Desired text encoding */ + const char *zName, /* Name of the collating sequence. Might be NULL */ + int create /* True to create CollSeq if doesn't already exist */ +){ + CollSeq *pColl; + assert( SQLITE_UTF8==1 && SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + assert( enc>=SQLITE_UTF8 && enc<=SQLITE_UTF16BE ); + if( zName ){ + pColl = findCollSeqEntry(db, zName, create); + if( pColl ) pColl += enc-1; + }else{ + pColl = db->pDfltColl; + } + return pColl; +} + +/* +** Change the text encoding for a database connection. This means that +** the pDfltColl must change as well. +*/ +SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8 enc){ + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + db->enc = enc; + /* EVIDENCE-OF: R-08308-17224 The default collating function for all + ** strings is BINARY. + */ + db->pDfltColl = sqlite3FindCollSeq(db, enc, sqlite3StrBINARY, 0); +} + +/* +** This function is responsible for invoking the collation factory callback +** or substituting a collation sequence of a different encoding when the +** requested collation sequence is not available in the desired encoding. +** +** If it is not NULL, then pColl must point to the database native encoding +** collation sequence with name zName, length nName. +** +** The return value is either the collation sequence to be used in database +** db for collation type name zName, length nName, or NULL, if no collation +** sequence can be found. If no collation is found, leave an error message. +** +** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() +*/ +SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( + Parse *pParse, /* Parsing context */ + u8 enc, /* The desired encoding for the collating sequence */ + CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ + const char *zName /* Collating sequence name */ +){ + CollSeq *p; + sqlite3 *db = pParse->db; + + p = pColl; + if( !p ){ + p = sqlite3FindCollSeq(db, enc, zName, 0); + } + if( !p || !p->xCmp ){ + /* No collation sequence of this type for this encoding is registered. + ** Call the collation factory to see if it can supply us with one. + */ + callCollNeeded(db, enc, zName); + p = sqlite3FindCollSeq(db, enc, zName, 0); + } + if( p && !p->xCmp && synthCollSeq(db, p) ){ + p = 0; + } + assert( !p || p->xCmp ); + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + pParse->rc = SQLITE_ERROR_MISSING_COLLSEQ; + } + return p; +} + +/* +** This function returns the collation sequence for database native text +** encoding identified by the string zName. +** +** If the requested collation sequence is not available, or not available +** in the database native encoding, the collation factory is invoked to +** request it. If the collation factory does not supply such a sequence, +** and the sequence is available in another text encoding, then that is +** returned instead. +** +** If no versions of the requested collations sequence are available, or +** another error occurs, NULL is returned and an error message written into +** pParse. +** +** This routine is a wrapper around sqlite3FindCollSeq(). This routine +** invokes the collation factory if the named collation cannot be found +** and generates an error message. +** +** See also: sqlite3FindCollSeq(), sqlite3GetCollSeq() +*/ +SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ + sqlite3 *db = pParse->db; + u8 enc = ENC(db); + u8 initbusy = db->init.busy; + CollSeq *pColl; + + pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); + if( !initbusy && (!pColl || !pColl->xCmp) ){ + pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); + } + + return pColl; +} + +/* During the search for the best function definition, this procedure +** is called to test how well the function passed as the first argument +** matches the request for a function with nArg arguments in a system +** that uses encoding enc. The value returned indicates how well the +** request is matched. A higher value indicates a better match. +** +** If nArg is -1 that means to only return a match (non-zero) if p->nArg +** is also -1. In other words, we are searching for a function that +** takes a variable number of arguments. +** +** If nArg is -2 that means that we are searching for any function +** regardless of the number of arguments it uses, so return a positive +** match score for any +** +** The returned value is always between 0 and 6, as follows: +** +** 0: Not a match. +** 1: UTF8/16 conversion required and function takes any number of arguments. +** 2: UTF16 byte order change required and function takes any number of args. +** 3: encoding matches and function takes any number of arguments +** 4: UTF8/16 conversion required - argument count matches exactly +** 5: UTF16 byte order conversion required - argument count matches exactly +** 6: Perfect match: encoding and argument count match exactly. +** +** If nArg==(-2) then any function with a non-null xSFunc is +** a perfect match and any function with xSFunc NULL is +** a non-match. +*/ +#define FUNC_PERFECT_MATCH 6 /* The score for a perfect match */ +static int matchQuality( + FuncDef *p, /* The function we are evaluating for match quality */ + int nArg, /* Desired number of arguments. (-1)==any */ + u8 enc /* Desired text encoding */ +){ + int match; + assert( p->nArg>=-1 ); + + /* Wrong number of arguments means "no match" */ + if( p->nArg!=nArg ){ + if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; + if( p->nArg>=0 ) return 0; + } + + /* Give a better score to a function with a specific number of arguments + ** than to function that accepts any number of arguments. */ + if( p->nArg==nArg ){ + match = 4; + }else{ + match = 1; + } + + /* Bonus points if the text encoding matches */ + if( enc==(p->funcFlags & SQLITE_FUNC_ENCMASK) ){ + match += 2; /* Exact encoding match */ + }else if( (enc & p->funcFlags & 2)!=0 ){ + match += 1; /* Both are UTF16, but with different byte orders */ + } + + return match; +} + +/* +** Search a FuncDefHash for a function with the given name. Return +** a pointer to the matching FuncDef if found, or 0 if there is no match. +*/ +SQLITE_PRIVATE FuncDef *sqlite3FunctionSearch( + int h, /* Hash of the name */ + const char *zFunc /* Name of function */ +){ + FuncDef *p; + for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){ + if( sqlite3StrICmp(p->zName, zFunc)==0 ){ + return p; + } + } + return 0; +} + +/* +** Insert a new FuncDef into a FuncDefHash hash table. +*/ +SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs( + FuncDef *aDef, /* List of global functions to be inserted */ + int nDef /* Length of the apDef[] list */ +){ + int i; + for(i=0; i='a' && zName[0]<='z' ); + pOther = sqlite3FunctionSearch(h, zName); + if( pOther ){ + assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] ); + aDef[i].pNext = pOther->pNext; + pOther->pNext = &aDef[i]; + }else{ + aDef[i].pNext = 0; + aDef[i].u.pHash = sqlite3BuiltinFunctions.a[h]; + sqlite3BuiltinFunctions.a[h] = &aDef[i]; + } + } +} + + + +/* +** Locate a user function given a name, a number of arguments and a flag +** indicating whether the function prefers UTF-16 over UTF-8. Return a +** pointer to the FuncDef structure that defines that function, or return +** NULL if the function does not exist. +** +** If the createFlag argument is true, then a new (blank) FuncDef +** structure is created and liked into the "db" structure if a +** no matching function previously existed. +** +** If nArg is -2, then the first valid function found is returned. A +** function is valid if xSFunc is non-zero. The nArg==(-2) +** case is used to see if zName is a valid function name for some number +** of arguments. If nArg is -2, then createFlag must be 0. +** +** If createFlag is false, then a function with the required name and +** number of arguments may be returned even if the eTextRep flag does not +** match that requested. +*/ +SQLITE_PRIVATE FuncDef *sqlite3FindFunction( + sqlite3 *db, /* An open database */ + const char *zName, /* Name of the function. zero-terminated */ + int nArg, /* Number of arguments. -1 means any number */ + u8 enc, /* Preferred text encoding */ + u8 createFlag /* Create new entry if true and does not otherwise exist */ +){ + FuncDef *p; /* Iterator variable */ + FuncDef *pBest = 0; /* Best match found so far */ + int bestScore = 0; /* Score of best match */ + int h; /* Hash value */ + int nName; /* Length of the name */ + + assert( nArg>=(-2) ); + assert( nArg>=(-1) || createFlag==0 ); + nName = sqlite3Strlen30(zName); + + /* First search for a match amongst the application-defined functions. + */ + p = (FuncDef*)sqlite3HashFind(&db->aFunc, zName); + while( p ){ + int score = matchQuality(p, nArg, enc); + if( score>bestScore ){ + pBest = p; + bestScore = score; + } + p = p->pNext; + } + + /* If no match is found, search the built-in functions. + ** + ** If the DBFLAG_PreferBuiltin flag is set, then search the built-in + ** functions even if a prior app-defined function was found. And give + ** priority to built-in functions. + ** + ** Except, if createFlag is true, that means that we are trying to + ** install a new function. Whatever FuncDef structure is returned it will + ** have fields overwritten with new information appropriate for the + ** new function. But the FuncDefs for built-in functions are read-only. + ** So we must not search for built-ins when creating a new function. + */ + if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){ + bestScore = 0; + h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName); + p = sqlite3FunctionSearch(h, zName); + while( p ){ + int score = matchQuality(p, nArg, enc); + if( score>bestScore ){ + pBest = p; + bestScore = score; + } + p = p->pNext; + } + } + + /* If the createFlag parameter is true and the search did not reveal an + ** exact match for the name, number of arguments and encoding, then add a + ** new entry to the hash table and return it. + */ + if( createFlag && bestScorezName = (const char*)&pBest[1]; + pBest->nArg = (u16)nArg; + pBest->funcFlags = enc; + memcpy((char*)&pBest[1], zName, nName+1); + for(z=(u8*)pBest->zName; *z; z++) *z = sqlite3UpperToLower[*z]; + pOther = (FuncDef*)sqlite3HashInsert(&db->aFunc, pBest->zName, pBest); + if( pOther==pBest ){ + sqlite3DbFree(db, pBest); + sqlite3OomFault(db); + return 0; + }else{ + pBest->pNext = pOther; + } + } + + if( pBest && (pBest->xSFunc || createFlag) ){ + return pBest; + } + return 0; +} + +/* +** Free all resources held by the schema structure. The void* argument points +** at a Schema struct. This function does not call sqlite3DbFree(db, ) on the +** pointer itself, it just cleans up subsidiary resources (i.e. the contents +** of the schema hash tables). +** +** The Schema.cache_size variable is not cleared. +*/ +SQLITE_PRIVATE void sqlite3SchemaClear(void *p){ + Hash temp1; + Hash temp2; + HashElem *pElem; + Schema *pSchema = (Schema *)p; + + temp1 = pSchema->tblHash; + temp2 = pSchema->trigHash; + sqlite3HashInit(&pSchema->trigHash); + sqlite3HashClear(&pSchema->idxHash); + for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ + sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); + } + sqlite3HashClear(&temp2); + sqlite3HashInit(&pSchema->tblHash); + for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ + Table *pTab = sqliteHashData(pElem); + sqlite3DeleteTable(0, pTab); + } + sqlite3HashClear(&temp1); + sqlite3HashClear(&pSchema->fkeyHash); + pSchema->pSeqTab = 0; + if( pSchema->schemaFlags & DB_SchemaLoaded ){ + pSchema->iGeneration++; + } + pSchema->schemaFlags &= ~(DB_SchemaLoaded|DB_ResetWanted); +} + +/* +** Find and return the schema associated with a BTree. Create +** a new one if necessary. +*/ +SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ + Schema * p; + if( pBt ){ + p = (Schema *)sqlite3BtreeSchema(pBt, sizeof(Schema), sqlite3SchemaClear); + }else{ + p = (Schema *)sqlite3DbMallocZero(0, sizeof(Schema)); + } + if( !p ){ + sqlite3OomFault(db); + }else if ( 0==p->file_format ){ + sqlite3HashInit(&p->tblHash); + sqlite3HashInit(&p->idxHash); + sqlite3HashInit(&p->trigHash); + sqlite3HashInit(&p->fkeyHash); + p->enc = SQLITE_UTF8; + } + return p; +} + +/************** End of callback.c ********************************************/ +/************** Begin file delete.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** in order to generate code for DELETE FROM statements. +*/ +/* #include "sqliteInt.h" */ + +/* +** While a SrcList can in general represent multiple tables and subqueries +** (as in the FROM clause of a SELECT statement) in this case it contains +** the name of a single table, as one might find in an INSERT, DELETE, +** or UPDATE statement. Look up that table in the symbol table and +** return a pointer. Set an error message and return NULL if the table +** name is not found or if any other error occurs. +** +** The following fields are initialized appropriate in pSrc: +** +** pSrc->a[0].pTab Pointer to the Table object +** pSrc->a[0].pIndex Pointer to the INDEXED BY index, if there is one +** +*/ +SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ + SrcItem *pItem = pSrc->a; + Table *pTab; + assert( pItem && pSrc->nSrc>=1 ); + pTab = sqlite3LocateTableItem(pParse, 0, pItem); + sqlite3DeleteTable(pParse->db, pItem->pTab); + pItem->pTab = pTab; + if( pTab ){ + pTab->nTabRef++; + if( pItem->fg.isIndexedBy && sqlite3IndexedByLookup(pParse, pItem) ){ + pTab = 0; + } + } + return pTab; +} + +/* Return true if table pTab is read-only. +** +** A table is read-only if any of the following are true: +** +** 1) It is a virtual table and no implementation of the xUpdate method +** has been provided +** +** 2) It is a system table (i.e. sqlite_schema), this call is not +** part of a nested parse and writable_schema pragma has not +** been specified +** +** 3) The table is a shadow table, the database connection is in +** defensive mode, and the current sqlite3_prepare() +** is for a top-level SQL statement. +*/ +static int tabIsReadOnly(Parse *pParse, Table *pTab){ + sqlite3 *db; + if( IsVirtual(pTab) ){ + return sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0; + } + if( (pTab->tabFlags & (TF_Readonly|TF_Shadow))==0 ) return 0; + db = pParse->db; + if( (pTab->tabFlags & TF_Readonly)!=0 ){ + return sqlite3WritableSchema(db)==0 && pParse->nested==0; + } + assert( pTab->tabFlags & TF_Shadow ); + return sqlite3ReadOnlyShadowTables(db); +} + +/* +** Check to make sure the given table is writable. If it is not +** writable, generate an error message and return 1. If it is +** writable return 0; +*/ +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ + if( tabIsReadOnly(pParse, pTab) ){ + sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); + return 1; + } +#ifndef SQLITE_OMIT_VIEW + if( !viewOk && IsView(pTab) ){ + sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); + return 1; + } +#endif + return 0; +} + + +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) +/* +** Evaluate a view and store its result in an ephemeral table. The +** pWhere argument is an optional WHERE clause that restricts the +** set of rows in the view that are to be added to the ephemeral table. +*/ +SQLITE_PRIVATE void sqlite3MaterializeView( + Parse *pParse, /* Parsing context */ + Table *pView, /* View definition */ + Expr *pWhere, /* Optional WHERE clause to be added */ + ExprList *pOrderBy, /* Optional ORDER BY clause */ + Expr *pLimit, /* Optional LIMIT clause */ + int iCur /* Cursor number for ephemeral table */ +){ + SelectDest dest; + Select *pSel; + SrcList *pFrom; + sqlite3 *db = pParse->db; + int iDb = sqlite3SchemaToIndex(db, pView->pSchema); + pWhere = sqlite3ExprDup(db, pWhere, 0); + pFrom = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pFrom ){ + assert( pFrom->nSrc==1 ); + pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); + pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + assert( pFrom->a[0].pOn==0 ); + assert( pFrom->a[0].pUsing==0 ); + } + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, pOrderBy, + SF_IncludeHidden, pLimit); + sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); + sqlite3Select(pParse, pSel, &dest); + sqlite3SelectDelete(db, pSel); +} +#endif /* !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) */ + +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) +/* +** Generate an expression tree to implement the WHERE, ORDER BY, +** and LIMIT/OFFSET portion of DELETE and UPDATE statements. +** +** DELETE FROM table_wxyz WHERE a<5 ORDER BY a LIMIT 1; +** \__________________________/ +** pLimitWhere (pInClause) +*/ +SQLITE_PRIVATE Expr *sqlite3LimitWhere( + Parse *pParse, /* The parser context */ + SrcList *pSrc, /* the FROM clause -- which tables to scan */ + Expr *pWhere, /* The WHERE clause. May be null */ + ExprList *pOrderBy, /* The ORDER BY clause. May be null */ + Expr *pLimit, /* The LIMIT clause. May be null */ + char *zStmtType /* Either DELETE or UPDATE. For err msgs. */ +){ + sqlite3 *db = pParse->db; + Expr *pLhs = NULL; /* LHS of IN(SELECT...) operator */ + Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ + ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ + SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ + Select *pSelect = NULL; /* Complete SELECT tree */ + Table *pTab; + + /* Check that there isn't an ORDER BY without a LIMIT clause. + */ + if( pOrderBy && pLimit==0 ) { + sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); + sqlite3ExprDelete(pParse->db, pWhere); + sqlite3ExprListDelete(pParse->db, pOrderBy); + return 0; + } + + /* We only need to generate a select expression if there + ** is a limit/offset term to enforce. + */ + if( pLimit == 0 ) { + return pWhere; + } + + /* Generate a select expression tree to enforce the limit/offset + ** term for the DELETE or UPDATE statement. For example: + ** DELETE FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 + ** becomes: + ** DELETE FROM table_a WHERE rowid IN ( + ** SELECT rowid FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 + ** ); + */ + + pTab = pSrc->a[0].pTab; + if( HasRowid(pTab) ){ + pLhs = sqlite3PExpr(pParse, TK_ROW, 0, 0); + pEList = sqlite3ExprListAppend( + pParse, 0, sqlite3PExpr(pParse, TK_ROW, 0, 0) + ); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + if( pPk->nKeyCol==1 ){ + const char *zName = pTab->aCol[pPk->aiColumn[0]].zCnName; + pLhs = sqlite3Expr(db, TK_ID, zName); + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, zName)); + }else{ + int i; + for(i=0; inKeyCol; i++){ + Expr *p = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[i]].zCnName); + pEList = sqlite3ExprListAppend(pParse, pEList, p); + } + pLhs = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( pLhs ){ + pLhs->x.pList = sqlite3ExprListDup(db, pEList, 0); + } + } + } + + /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree + ** and the SELECT subtree. */ + pSrc->a[0].pTab = 0; + pSelectSrc = sqlite3SrcListDup(db, pSrc, 0); + pSrc->a[0].pTab = pTab; + if( pSrc->a[0].fg.isIndexedBy ){ + pSrc->a[0].u2.pIBIndex = 0; + pSrc->a[0].fg.isIndexedBy = 0; + sqlite3DbFree(db, pSrc->a[0].u1.zIndexedBy); + }else if( pSrc->a[0].fg.isCte ){ + pSrc->a[0].u2.pCteUse->nUse++; + } + + /* generate the SELECT expression tree. */ + pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, + pOrderBy,0,pLimit + ); + + /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ + pInClause = sqlite3PExpr(pParse, TK_IN, pLhs, 0); + sqlite3PExprAddSelect(pParse, pInClause, pSelect); + return pInClause; +} +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) */ + /* && !defined(SQLITE_OMIT_SUBQUERY) */ + +/* +** Generate code for a DELETE FROM statement. +** +** DELETE FROM table_wxyz WHERE a<5 AND b NOT NULL; +** \________/ \________________/ +** pTabList pWhere +*/ +SQLITE_PRIVATE void sqlite3DeleteFrom( + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* The table from which we should delete things */ + Expr *pWhere, /* The WHERE clause. May be null */ + ExprList *pOrderBy, /* ORDER BY clause. May be null */ + Expr *pLimit /* LIMIT clause. May be null */ +){ + Vdbe *v; /* The virtual database engine */ + Table *pTab; /* The table from which records will be deleted */ + int i; /* Loop counter */ + WhereInfo *pWInfo; /* Information about the WHERE clause */ + Index *pIdx; /* For looping over indices of the table */ + int iTabCur; /* Cursor number for the table */ + int iDataCur = 0; /* VDBE cursor for the canonical data source */ + int iIdxCur = 0; /* Cursor number of the first index */ + int nIdx; /* Number of indices */ + sqlite3 *db; /* Main database structure */ + AuthContext sContext; /* Authorization context */ + NameContext sNC; /* Name context to resolve expressions in */ + int iDb; /* Database number */ + int memCnt = 0; /* Memory cell used for change counting */ + int rcauth; /* Value returned by authorization callback */ + int eOnePass; /* ONEPASS_OFF or _SINGLE or _MULTI */ + int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ + u8 *aToOpen = 0; /* Open cursor iTabCur+j if aToOpen[j] is true */ + Index *pPk; /* The PRIMARY KEY index on the table */ + int iPk = 0; /* First of nPk registers holding PRIMARY KEY value */ + i16 nPk = 1; /* Number of columns in the PRIMARY KEY */ + int iKey; /* Memory cell holding key of row to be deleted */ + i16 nKey; /* Number of memory cells in the row key */ + int iEphCur = 0; /* Ephemeral table holding all primary key values */ + int iRowSet = 0; /* Register for rowset of rows to delete */ + int addrBypass = 0; /* Address of jump over the delete logic */ + int addrLoop = 0; /* Top of the delete loop */ + int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ + int bComplex; /* True if there are triggers or FKs or + ** subqueries in the WHERE clause */ + +#ifndef SQLITE_OMIT_TRIGGER + int isView; /* True if attempting to delete from a view */ + Trigger *pTrigger; /* List of table triggers, if required */ +#endif + + memset(&sContext, 0, sizeof(sContext)); + db = pParse->db; + if( pParse->nErr || db->mallocFailed ){ + goto delete_from_cleanup; + } + assert( pTabList->nSrc==1 ); + + + /* Locate the table which we want to delete. This table has to be + ** put in an SrcList structure because some of the subroutines we + ** will be calling are designed to work with multiple tables and expect + ** an SrcList* parameter instead of just a Table* parameter. + */ + pTab = sqlite3SrcListLookup(pParse, pTabList); + if( pTab==0 ) goto delete_from_cleanup; + + /* Figure out if we have any triggers and if the table being + ** deleted from is a view + */ +#ifndef SQLITE_OMIT_TRIGGER + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + isView = IsView(pTab); +#else +# define pTrigger 0 +# define isView 0 +#endif + bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); +#ifdef SQLITE_OMIT_VIEW +# undef isView +# define isView 0 +#endif + +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( !isView ){ + pWhere = sqlite3LimitWhere( + pParse, pTabList, pWhere, pOrderBy, pLimit, "DELETE" + ); + pOrderBy = 0; + pLimit = 0; + } +#endif + + /* If pTab is really a view, make sure it has been initialized. + */ + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto delete_from_cleanup; + } + + if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){ + goto delete_from_cleanup; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDbnDb ); + rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, + db->aDb[iDb].zDbSName); + assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); + if( rcauth==SQLITE_DENY ){ + goto delete_from_cleanup; + } + assert(!isView || pTrigger); + + /* Assign cursor numbers to the table and all its indices. + */ + assert( pTabList->nSrc==1 ); + iTabCur = pTabList->a[0].iCursor = pParse->nTab++; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ + pParse->nTab++; + } + + /* Start the view context + */ + if( isView ){ + sqlite3AuthContextPush(pParse, &sContext, pTab->zName); + } + + /* Begin generating code. + */ + v = sqlite3GetVdbe(pParse); + if( v==0 ){ + goto delete_from_cleanup; + } + if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); + sqlite3BeginWriteOperation(pParse, bComplex, iDb); + + /* If we are trying to delete from a view, realize that view into + ** an ephemeral table. + */ +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) + if( isView ){ + sqlite3MaterializeView(pParse, pTab, + pWhere, pOrderBy, pLimit, iTabCur + ); + iDataCur = iIdxCur = iTabCur; + pOrderBy = 0; + pLimit = 0; + } +#endif + + /* Resolve the column names in the WHERE clause. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + if( sqlite3ResolveExprNames(&sNC, pWhere) ){ + goto delete_from_cleanup; + } + + /* Initialize the counter of the number of rows deleted, if + ** we are counting rows. + */ + if( (db->flags & SQLITE_CountRows)!=0 + && !pParse->nested + && !pParse->pTriggerTab + && !pParse->bReturning + ){ + memCnt = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, memCnt); + } + +#ifndef SQLITE_OMIT_TRUNCATE_OPTIMIZATION + /* Special case: A DELETE without a WHERE clause deletes everything. + ** It is easier just to erase the whole table. Prior to version 3.6.5, + ** this optimization caused the row change count (the value returned by + ** API function sqlite3_count_changes) to be set incorrectly. + ** + ** The "rcauth==SQLITE_OK" terms is the + ** IMPLEMENTATION-OF: R-17228-37124 If the action code is SQLITE_DELETE and + ** the callback returns SQLITE_IGNORE then the DELETE operation proceeds but + ** the truncate optimization is disabled and all rows are deleted + ** individually. + */ + if( rcauth==SQLITE_OK + && pWhere==0 + && !bComplex + && !IsVirtual(pTab) +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + && db->xPreUpdateCallback==0 +#endif + ){ + assert( !isView ); + sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt ? memCnt : -1, + pTab->zName, P4_STATIC); + } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->pSchema==pTab->pSchema ); + sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + sqlite3VdbeChangeP3(v, -1, memCnt ? memCnt : -1); + } + } + }else +#endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ + { + u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; + if( sNC.ncFlags & NC_VarSelect ) bComplex = 1; + wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); + if( HasRowid(pTab) ){ + /* For a rowid table, initialize the RowSet to an empty set */ + pPk = 0; + nPk = 1; + iRowSet = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); + }else{ + /* For a WITHOUT ROWID table, create an ephemeral table used to + ** hold all primary keys for rows to be deleted. */ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + nPk = pPk->nKeyCol; + iPk = pParse->nMem+1; + pParse->nMem += nPk; + iEphCur = pParse->nTab++; + addrEphOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEphCur, nPk); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } + + /* Construct a query to find the rowid or primary key for every row + ** to be deleted, based on the WHERE clause. Set variable eOnePass + ** to indicate the strategy used to implement this delete: + ** + ** ONEPASS_OFF: Two-pass approach - use a FIFO for rowids/PK values. + ** ONEPASS_SINGLE: One-pass approach - at most one row deleted. + ** ONEPASS_MULTI: One-pass approach - any number of rows may be deleted. + */ + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); + if( pWInfo==0 ) goto delete_from_cleanup; + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); + assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); + if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse); + if( sqlite3WhereUsesDeferredSeek(pWInfo) ){ + sqlite3VdbeAddOp1(v, OP_FinishSeek, iTabCur); + } + + /* Keep track of the number of rows to be deleted */ + if( memCnt ){ + sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); + } + + /* Extract the rowid or primary key for the current row */ + if( pPk ){ + for(i=0; iaiColumn[i]>=0 ); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, + pPk->aiColumn[i], iPk+i); + } + iKey = iPk; + }else{ + iKey = ++pParse->nMem; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, -1, iKey); + } + + if( eOnePass!=ONEPASS_OFF ){ + /* For ONEPASS, no need to store the rowid/primary-key. There is only + ** one, so just keep it in its register(s) and fall through to the + ** delete code. */ + nKey = nPk; /* OP_Found will use an unpacked key */ + aToOpen = sqlite3DbMallocRawNN(db, nIdx+2); + if( aToOpen==0 ){ + sqlite3WhereEnd(pWInfo); + goto delete_from_cleanup; + } + memset(aToOpen, 1, nIdx+1); + aToOpen[nIdx+1] = 0; + if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; + if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; + if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); + addrBypass = sqlite3VdbeMakeLabel(pParse); + }else{ + if( pPk ){ + /* Add the PK key for this row to the temporary table */ + iKey = ++pParse->nMem; + nKey = 0; /* Zero tells OP_Found to use a composite key */ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, + sqlite3IndexAffinityStr(pParse->db, pPk), nPk); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEphCur, iKey, iPk, nPk); + }else{ + /* Add the rowid of the row to be deleted to the RowSet */ + nKey = 1; /* OP_DeferredSeek always uses a single rowid */ + sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); + } + sqlite3WhereEnd(pWInfo); + } + + /* Unless this is a view, open cursors for the table we are + ** deleting from and all its indices. If this is a view, then the + ** only effect this statement has is to fire the INSTEAD OF + ** triggers. + */ + if( !isView ){ + int iAddrOnce = 0; + if( eOnePass==ONEPASS_MULTI ){ + iAddrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + testcase( IsVirtual(pTab) ); + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, OPFLAG_FORDELETE, + iTabCur, aToOpen, &iDataCur, &iIdxCur); + assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); + assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); + if( eOnePass==ONEPASS_MULTI ){ + sqlite3VdbeJumpHereOrPopInst(v, iAddrOnce); + } + } + + /* Set up a loop over the rowids/primary-keys that were found in the + ** where-clause loop above. + */ + if( eOnePass!=ONEPASS_OFF ){ + assert( nKey==nPk ); /* OP_Found will use an unpacked key */ + if( !IsVirtual(pTab) && aToOpen[iDataCur-iTabCur] ){ + assert( pPk!=0 || IsView(pTab) ); + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); + VdbeCoverage(v); + } + }else if( pPk ){ + addrLoop = sqlite3VdbeAddOp1(v, OP_Rewind, iEphCur); VdbeCoverage(v); + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Column, iEphCur, 0, iKey); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iEphCur, iKey); + } + assert( nKey==0 ); /* OP_Found will use a composite key */ + }else{ + addrLoop = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, 0, iKey); + VdbeCoverage(v); + assert( nKey==1 ); + } + + /* Delete the row */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + sqlite3VtabMakeWritable(pParse, pTab); + assert( eOnePass==ONEPASS_OFF || eOnePass==ONEPASS_SINGLE ); + sqlite3MayAbort(pParse); + if( eOnePass==ONEPASS_SINGLE ){ + sqlite3VdbeAddOp1(v, OP_Close, iTabCur); + if( sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } + } + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, OE_Abort); + }else +#endif + { + int count = (pParse->nested==0); /* True to count changes */ + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + iKey, nKey, count, OE_Default, eOnePass, aiCurOnePass[1]); + } + + /* End of the loop over all rowids/primary-keys. */ + if( eOnePass!=ONEPASS_OFF ){ + sqlite3VdbeResolveLabel(v, addrBypass); + sqlite3WhereEnd(pWInfo); + }else if( pPk ){ + sqlite3VdbeAddOp2(v, OP_Next, iEphCur, addrLoop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrLoop); + }else{ + sqlite3VdbeGoto(v, addrLoop); + sqlite3VdbeJumpHere(v, addrLoop); + } + } /* End non-truncate path */ + + /* Update the sqlite_sequence table by storing the content of the + ** maximum rowid counter values recorded while inserting into + ** autoincrement tables. + */ + if( pParse->nested==0 && pParse->pTriggerTab==0 ){ + sqlite3AutoincrementEnd(pParse); + } + + /* Return the number of rows that were deleted. If this routine is + ** generating code because of a call to sqlite3NestedParse(), do not + ** invoke the callback function. + */ + if( memCnt ){ + sqlite3VdbeAddOp2(v, OP_ChngCntRow, memCnt, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", SQLITE_STATIC); + } + +delete_from_cleanup: + sqlite3AuthContextPop(&sContext); + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprDelete(db, pWhere); +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) + sqlite3ExprListDelete(db, pOrderBy); + sqlite3ExprDelete(db, pLimit); +#endif + sqlite3DbFree(db, aToOpen); + return; +} +/* Make sure "isView" and other macros defined above are undefined. Otherwise +** they may interfere with compilation of other functions in this file +** (or in another file, if this file becomes part of the amalgamation). */ +#ifdef isView + #undef isView +#endif +#ifdef pTrigger + #undef pTrigger +#endif + +/* +** This routine generates VDBE code that causes a single row of a +** single table to be deleted. Both the original table entry and +** all indices are removed. +** +** Preconditions: +** +** 1. iDataCur is an open cursor on the btree that is the canonical data +** store for the table. (This will be either the table itself, +** in the case of a rowid table, or the PRIMARY KEY index in the case +** of a WITHOUT ROWID table.) +** +** 2. Read/write cursors for all indices of pTab must be open as +** cursor number iIdxCur+i for the i-th index. +** +** 3. The primary key for the row to be deleted must be stored in a +** sequence of nPk memory cells starting at iPk. If nPk==0 that means +** that a search record formed from OP_MakeRecord is contained in the +** single memory location iPk. +** +** eMode: +** Parameter eMode may be passed either ONEPASS_OFF (0), ONEPASS_SINGLE, or +** ONEPASS_MULTI. If eMode is not ONEPASS_OFF, then the cursor +** iDataCur already points to the row to delete. If eMode is ONEPASS_OFF +** then this function must seek iDataCur to the entry identified by iPk +** and nPk before reading from it. +** +** If eMode is ONEPASS_MULTI, then this call is being made as part +** of a ONEPASS delete that affects multiple rows. In this case, if +** iIdxNoSeek is a valid cursor number (>=0) and is not the same as +** iDataCur, then its position should be preserved following the delete +** operation. Or, if iIdxNoSeek is not a valid cursor number, the +** position of iDataCur should be preserved instead. +** +** iIdxNoSeek: +** If iIdxNoSeek is a valid cursor number (>=0) not equal to iDataCur, +** then it identifies an index cursor (from within array of cursors +** starting at iIdxCur) that already points to the index entry to be deleted. +** Except, this optimization is disabled if there are BEFORE triggers since +** the trigger body might have moved the cursor. +*/ +SQLITE_PRIVATE void sqlite3GenerateRowDelete( + Parse *pParse, /* Parsing context */ + Table *pTab, /* Table containing the row to be deleted */ + Trigger *pTrigger, /* List of triggers to (potentially) fire */ + int iDataCur, /* Cursor from which column data is extracted */ + int iIdxCur, /* First index cursor */ + int iPk, /* First memory cell containing the PRIMARY KEY */ + i16 nPk, /* Number of PRIMARY KEY memory cells */ + u8 count, /* If non-zero, increment the row change counter */ + u8 onconf, /* Default ON CONFLICT policy for triggers */ + u8 eMode, /* ONEPASS_OFF, _SINGLE, or _MULTI. See above */ + int iIdxNoSeek /* Cursor number of cursor that does not need seeking */ +){ + Vdbe *v = pParse->pVdbe; /* Vdbe */ + int iOld = 0; /* First register in OLD.* array */ + int iLabel; /* Label resolved to end of generated code */ + u8 opSeek; /* Seek opcode */ + + /* Vdbe is guaranteed to have been allocated by this stage. */ + assert( v ); + VdbeModuleComment((v, "BEGIN: GenRowDel(%d,%d,%d,%d)", + iDataCur, iIdxCur, iPk, (int)nPk)); + + /* Seek cursor iCur to the row to delete. If this row no longer exists + ** (this can happen if a trigger program has already deleted it), do + ** not attempt to delete it or fire any DELETE triggers. */ + iLabel = sqlite3VdbeMakeLabel(pParse); + opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; + if( eMode==ONEPASS_OFF ){ + sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); + VdbeCoverageIf(v, opSeek==OP_NotExists); + VdbeCoverageIf(v, opSeek==OP_NotFound); + } + + /* If there are any triggers to fire, allocate a range of registers to + ** use for the old.* references in the triggers. */ + if( sqlite3FkRequired(pParse, pTab, 0, 0) || pTrigger ){ + u32 mask; /* Mask of OLD.* columns in use */ + int iCol; /* Iterator used while populating OLD.* */ + int addrStart; /* Start of BEFORE trigger programs */ + + /* TODO: Could use temporary registers here. Also could attempt to + ** avoid copying the contents of the rowid register. */ + mask = sqlite3TriggerColmask( + pParse, pTrigger, 0, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onconf + ); + mask |= sqlite3FkOldmask(pParse, pTab); + iOld = pParse->nMem+1; + pParse->nMem += (1 + pTab->nCol); + + /* Populate the OLD.* pseudo-table register array. These values will be + ** used by any BEFORE and AFTER triggers that exist. */ + sqlite3VdbeAddOp2(v, OP_Copy, iPk, iOld); + for(iCol=0; iColnCol; iCol++){ + testcase( mask!=0xffffffff && iCol==31 ); + testcase( mask!=0xffffffff && iCol==32 ); + if( mask==0xffffffff || (iCol<=31 && (mask & MASKBIT32(iCol))!=0) ){ + int kk = sqlite3TableColumnToStorage(pTab, iCol); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, iCol, iOld+kk+1); + } + } + + /* Invoke BEFORE DELETE trigger programs. */ + addrStart = sqlite3VdbeCurrentAddr(v); + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_BEFORE, pTab, iOld, onconf, iLabel + ); + + /* If any BEFORE triggers were coded, then seek the cursor to the + ** row to be deleted again. It may be that the BEFORE triggers moved + ** the cursor or already deleted the row that the cursor was + ** pointing to. + ** + ** Also disable the iIdxNoSeek optimization since the BEFORE trigger + ** may have moved that cursor. + */ + if( addrStart=0 ); + iIdxNoSeek = -1; + } + + /* Do FK processing. This call checks that any FK constraints that + ** refer to this table (i.e. constraints attached to other tables) + ** are not violated by deleting this row. */ + sqlite3FkCheck(pParse, pTab, iOld, 0, 0, 0); + } + + /* Delete the index and table entries. Skip this step if pTab is really + ** a view (in which case the only effect of the DELETE statement is to + ** fire the INSTEAD OF triggers). + ** + ** If variable 'count' is non-zero, then this OP_Delete instruction should + ** invoke the update-hook. The pre-update-hook, on the other hand should + ** be invoked unless table pTab is a system table. The difference is that + ** the update-hook is not invoked for rows removed by REPLACE, but the + ** pre-update-hook is. + */ + if( !IsView(pTab) ){ + u8 p5 = 0; + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); + if( pParse->nested==0 || 0==sqlite3_stricmp(pTab->zName, "sqlite_stat1") ){ + sqlite3VdbeAppendP4(v, (char*)pTab, P4_TABLE); + } + if( eMode!=ONEPASS_OFF ){ + sqlite3VdbeChangeP5(v, OPFLAG_AUXDELETE); + } + if( iIdxNoSeek>=0 && iIdxNoSeek!=iDataCur ){ + sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); + } + if( eMode==ONEPASS_MULTI ) p5 |= OPFLAG_SAVEPOSITION; + sqlite3VdbeChangeP5(v, p5); + } + + /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to + ** handle rows (possibly in other tables) that refer via a foreign key + ** to the row just deleted. */ + sqlite3FkActions(pParse, pTab, 0, iOld, 0, 0); + + /* Invoke AFTER DELETE trigger programs. */ + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel + ); + + /* Jump here if the row had already been deleted before any BEFORE + ** trigger programs were invoked. Or if a trigger program throws a + ** RAISE(IGNORE) exception. */ + sqlite3VdbeResolveLabel(v, iLabel); + VdbeModuleComment((v, "END: GenRowDel()")); +} + +/* +** This routine generates VDBE code that causes the deletion of all +** index entries associated with a single row of a single table, pTab +** +** Preconditions: +** +** 1. A read/write cursor "iDataCur" must be open on the canonical storage +** btree for the table pTab. (This will be either the table itself +** for rowid tables or to the primary key index for WITHOUT ROWID +** tables.) +** +** 2. Read/write cursors for all indices of pTab must be open as +** cursor number iIdxCur+i for the i-th index. (The pTab->pIndex +** index is the 0-th index.) +** +** 3. The "iDataCur" cursor must be already be positioned on the row +** that is to be deleted. +*/ +SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Table containing the row to be deleted */ + int iDataCur, /* Cursor of table holding data. */ + int iIdxCur, /* First index cursor */ + int *aRegIdx, /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ + int iIdxNoSeek /* Do not delete from this cursor */ +){ + int i; /* Index loop counter */ + int r1 = -1; /* Register holding an index key */ + int iPartIdxLabel; /* Jump destination for skipping partial index entries */ + Index *pIdx; /* Current index */ + Index *pPrior = 0; /* Prior index */ + Vdbe *v; /* The prepared statement under construction */ + Index *pPk; /* PRIMARY KEY index, or NULL for rowid tables */ + + v = pParse->pVdbe; + pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ + assert( iIdxCur+i!=iDataCur || pPk==pIdx ); + if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; + if( pIdx==pPk ) continue; + if( iIdxCur+i==iIdxNoSeek ) continue; + VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, + &iPartIdxLabel, pPrior, r1); + sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, + pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + sqlite3VdbeChangeP5(v, 1); /* Cause IdxDelete to error if no entry found */ + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); + pPrior = pIdx; + } +} + +/* +** Generate code that will assemble an index key and stores it in register +** regOut. The key with be for index pIdx which is an index on pTab. +** iCur is the index of a cursor open on the pTab table and pointing to +** the entry that needs indexing. If pTab is a WITHOUT ROWID table, then +** iCur must be the cursor of the PRIMARY KEY index. +** +** Return a register number which is the first in a block of +** registers that holds the elements of the index key. The +** block of registers has already been deallocated by the time +** this routine returns. +** +** If *piPartIdxLabel is not NULL, fill it in with a label and jump +** to that label if pIdx is a partial index that should be skipped. +** The label should be resolved using sqlite3ResolvePartIdxLabel(). +** A partial index should be skipped if its WHERE clause evaluates +** to false or null. If pIdx is not a partial index, *piPartIdxLabel +** will be set to zero which is an empty label that is ignored by +** sqlite3ResolvePartIdxLabel(). +** +** The pPrior and regPrior parameters are used to implement a cache to +** avoid unnecessary register loads. If pPrior is not NULL, then it is +** a pointer to a different index for which an index key has just been +** computed into register regPrior. If the current pIdx index is generating +** its key into the same sequence of registers and if pPrior and pIdx share +** a column in common, then the register corresponding to that column already +** holds the correct value and the loading of that register is skipped. +** This optimization is helpful when doing a DELETE or an INTEGRITY_CHECK +** on a table with multiple indices, and especially with the ROWID or +** PRIMARY KEY columns of the index. +*/ +SQLITE_PRIVATE int sqlite3GenerateIndexKey( + Parse *pParse, /* Parsing context */ + Index *pIdx, /* The index for which to generate a key */ + int iDataCur, /* Cursor number from which to take column data */ + int regOut, /* Put the new key into this register if not 0 */ + int prefixOnly, /* Compute only a unique prefix of the key */ + int *piPartIdxLabel, /* OUT: Jump to this label to skip partial index */ + Index *pPrior, /* Previously generated index key */ + int regPrior /* Register holding previous generated key */ +){ + Vdbe *v = pParse->pVdbe; + int j; + int regBase; + int nCol; + + if( piPartIdxLabel ){ + if( pIdx->pPartIdxWhere ){ + *piPartIdxLabel = sqlite3VdbeMakeLabel(pParse); + pParse->iSelfTab = iDataCur + 1; + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, + SQLITE_JUMPIFNULL); + pParse->iSelfTab = 0; + pPrior = 0; /* Ticket a9efb42811fa41ee 2019-11-02; + ** pPartIdxWhere may have corrupted regPrior registers */ + }else{ + *piPartIdxLabel = 0; + } + } + nCol = (prefixOnly && pIdx->uniqNotNull) ? pIdx->nKeyCol : pIdx->nColumn; + regBase = sqlite3GetTempRange(pParse, nCol); + if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; + for(j=0; jaiColumn[j]==pIdx->aiColumn[j] + && pPrior->aiColumn[j]!=XN_EXPR + ){ + /* This column was already computed by the previous index */ + continue; + } + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); + if( pIdx->aiColumn[j]>=0 ){ + /* If the column affinity is REAL but the number is an integer, then it + ** might be stored in the table as an integer (using a compact + ** representation) then converted to REAL by an OP_RealAffinity opcode. + ** But we are getting ready to store this value back into an index, where + ** it should be converted by to INTEGER again. So omit the + ** OP_RealAffinity opcode if it is present */ + sqlite3VdbeDeletePriorOpcode(v, OP_RealAffinity); + } + } + if( regOut ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut); + } + sqlite3ReleaseTempRange(pParse, regBase, nCol); + return regBase; +} + +/* +** If a prior call to sqlite3GenerateIndexKey() generated a jump-over label +** because it was a partial index, then this routine should be called to +** resolve that label. +*/ +SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ + if( iLabel ){ + sqlite3VdbeResolveLabel(pParse->pVdbe, iLabel); + } +} + +/************** End of delete.c **********************************************/ +/************** Begin file func.c ********************************************/ +/* +** 2002 February 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the C-language implementations for many of the SQL +** functions of SQLite. (Some function, and in particular the date and +** time functions, are implemented separately.) +*/ +/* #include "sqliteInt.h" */ +/* #include */ +/* #include */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* #include */ +#endif +/* #include "vdbeInt.h" */ + +/* +** Return the collating function associated with a function. +*/ +static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ + VdbeOp *pOp; + assert( context->pVdbe!=0 ); + pOp = &context->pVdbe->aOp[context->iOp-1]; + assert( pOp->opcode==OP_CollSeq ); + assert( pOp->p4type==P4_COLLSEQ ); + return pOp->p4.pColl; +} + +/* +** Indicate that the accumulator load should be skipped on this +** iteration of the aggregate loop. +*/ +static void sqlite3SkipAccumulatorLoad(sqlite3_context *context){ + assert( context->isError<=0 ); + context->isError = -1; + context->skipFlag = 1; +} + +/* +** Implementation of the non-aggregate min() and max() functions +*/ +static void minmaxFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + int mask; /* 0 for min() or 0xffffffff for max() */ + int iBest; + CollSeq *pColl; + + assert( argc>1 ); + mask = sqlite3_user_data(context)==0 ? 0 : -1; + pColl = sqlite3GetFuncCollSeq(context); + assert( pColl ); + assert( mask==-1 || mask==0 ); + iBest = 0; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + for(i=1; i=0 ){ + testcase( mask==0 ); + iBest = i; + } + } + sqlite3_result_value(context, argv[iBest]); +} + +/* +** Return the type of the argument. +*/ +static void typeofFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + static const char *azType[] = { "integer", "real", "text", "blob", "null" }; + int i = sqlite3_value_type(argv[0]) - 1; + UNUSED_PARAMETER(NotUsed); + assert( i>=0 && i=0xc0 ){ + while( (*z & 0xc0)==0x80 ){ z++; z0++; } + } + } + sqlite3_result_int(context, (int)(z-z0)); + break; + } + default: { + sqlite3_result_null(context); + break; + } + } +} + +/* +** Implementation of the abs() function. +** +** IMP: R-23979-26855 The abs(X) function returns the absolute value of +** the numeric argument X. +*/ +static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_INTEGER: { + i64 iVal = sqlite3_value_int64(argv[0]); + if( iVal<0 ){ + if( iVal==SMALLEST_INT64 ){ + /* IMP: R-31676-45509 If X is the integer -9223372036854775808 + ** then abs(X) throws an integer overflow error since there is no + ** equivalent positive 64-bit two complement value. */ + sqlite3_result_error(context, "integer overflow", -1); + return; + } + iVal = -iVal; + } + sqlite3_result_int64(context, iVal); + break; + } + case SQLITE_NULL: { + /* IMP: R-37434-19929 Abs(X) returns NULL if X is NULL. */ + sqlite3_result_null(context); + break; + } + default: { + /* Because sqlite3_value_double() returns 0.0 if the argument is not + ** something that can be converted into a number, we have: + ** IMP: R-01992-00519 Abs(X) returns 0.0 if X is a string or blob + ** that cannot be converted to a numeric value. + */ + double rVal = sqlite3_value_double(argv[0]); + if( rVal<0 ) rVal = -rVal; + sqlite3_result_double(context, rVal); + break; + } + } +} + +/* +** Implementation of the instr() function. +** +** instr(haystack,needle) finds the first occurrence of needle +** in haystack and returns the number of previous characters plus 1, +** or 0 if needle does not occur within haystack. +** +** If both haystack and needle are BLOBs, then the result is one more than +** the number of bytes in haystack prior to the first occurrence of needle, +** or 0 if needle never occurs in haystack. +*/ +static void instrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zHaystack; + const unsigned char *zNeedle; + int nHaystack; + int nNeedle; + int typeHaystack, typeNeedle; + int N = 1; + int isText; + unsigned char firstChar; + sqlite3_value *pC1 = 0; + sqlite3_value *pC2 = 0; + + UNUSED_PARAMETER(argc); + typeHaystack = sqlite3_value_type(argv[0]); + typeNeedle = sqlite3_value_type(argv[1]); + if( typeHaystack==SQLITE_NULL || typeNeedle==SQLITE_NULL ) return; + nHaystack = sqlite3_value_bytes(argv[0]); + nNeedle = sqlite3_value_bytes(argv[1]); + if( nNeedle>0 ){ + if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ + zHaystack = sqlite3_value_blob(argv[0]); + zNeedle = sqlite3_value_blob(argv[1]); + isText = 0; + }else if( typeHaystack!=SQLITE_BLOB && typeNeedle!=SQLITE_BLOB ){ + zHaystack = sqlite3_value_text(argv[0]); + zNeedle = sqlite3_value_text(argv[1]); + isText = 1; + }else{ + pC1 = sqlite3_value_dup(argv[0]); + zHaystack = sqlite3_value_text(pC1); + if( zHaystack==0 ) goto endInstrOOM; + nHaystack = sqlite3_value_bytes(pC1); + pC2 = sqlite3_value_dup(argv[1]); + zNeedle = sqlite3_value_text(pC2); + if( zNeedle==0 ) goto endInstrOOM; + nNeedle = sqlite3_value_bytes(pC2); + isText = 1; + } + if( zNeedle==0 || (nHaystack && zHaystack==0) ) goto endInstrOOM; + firstChar = zNeedle[0]; + while( nNeedle<=nHaystack + && (zHaystack[0]!=firstChar || memcmp(zHaystack, zNeedle, nNeedle)!=0) + ){ + N++; + do{ + nHaystack--; + zHaystack++; + }while( isText && (zHaystack[0]&0xc0)==0x80 ); + } + if( nNeedle>nHaystack ) N = 0; + } + sqlite3_result_int(context, N); +endInstr: + sqlite3_value_free(pC1); + sqlite3_value_free(pC2); + return; +endInstrOOM: + sqlite3_result_error_nomem(context); + goto endInstr; +} + +/* +** Implementation of the printf() function. +*/ +static void printfFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + PrintfArguments x; + StrAccum str; + const char *zFormat; + int n; + sqlite3 *db = sqlite3_context_db_handle(context); + + if( argc>=1 && (zFormat = (const char*)sqlite3_value_text(argv[0]))!=0 ){ + x.nArg = argc-1; + x.nUsed = 0; + x.apArg = argv+1; + sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); + str.printfFlags = SQLITE_PRINTF_SQLFUNC; + sqlite3_str_appendf(&str, zFormat, &x); + n = str.nChar; + sqlite3_result_text(context, sqlite3StrAccumFinish(&str), n, + SQLITE_DYNAMIC); + } +} + +/* +** Implementation of the substr() function. +** +** substr(x,p1,p2) returns p2 characters of x[] beginning with p1. +** p1 is 1-indexed. So substr(x,1,1) returns the first character +** of x. If x is text, then we actually count UTF-8 characters. +** If x is a blob, then we count bytes. +** +** If p1 is negative, then we begin abs(p1) from the end of x[]. +** +** If p2 is negative, return the p2 characters preceding p1. +*/ +static void substrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *z; + const unsigned char *z2; + int len; + int p0type; + i64 p1, p2; + int negP2 = 0; + + assert( argc==3 || argc==2 ); + if( sqlite3_value_type(argv[1])==SQLITE_NULL + || (argc==3 && sqlite3_value_type(argv[2])==SQLITE_NULL) + ){ + return; + } + p0type = sqlite3_value_type(argv[0]); + p1 = sqlite3_value_int(argv[1]); + if( p0type==SQLITE_BLOB ){ + len = sqlite3_value_bytes(argv[0]); + z = sqlite3_value_blob(argv[0]); + if( z==0 ) return; + assert( len==sqlite3_value_bytes(argv[0]) ); + }else{ + z = sqlite3_value_text(argv[0]); + if( z==0 ) return; + len = 0; + if( p1<0 ){ + for(z2=z; *z2; len++){ + SQLITE_SKIP_UTF8(z2); + } + } + } +#ifdef SQLITE_SUBSTR_COMPATIBILITY + /* If SUBSTR_COMPATIBILITY is defined then substr(X,0,N) work the same as + ** as substr(X,1,N) - it returns the first N characters of X. This + ** is essentially a back-out of the bug-fix in check-in [5fc125d362df4b8] + ** from 2009-02-02 for compatibility of applications that exploited the + ** old buggy behavior. */ + if( p1==0 ) p1 = 1; /* */ +#endif + if( argc==3 ){ + p2 = sqlite3_value_int(argv[2]); + if( p2<0 ){ + p2 = -p2; + negP2 = 1; + } + }else{ + p2 = sqlite3_context_db_handle(context)->aLimit[SQLITE_LIMIT_LENGTH]; + } + if( p1<0 ){ + p1 += len; + if( p1<0 ){ + p2 += p1; + if( p2<0 ) p2 = 0; + p1 = 0; + } + }else if( p1>0 ){ + p1--; + }else if( p2>0 ){ + p2--; + } + if( negP2 ){ + p1 -= p2; + if( p1<0 ){ + p2 += p1; + p1 = 0; + } + } + assert( p1>=0 && p2>=0 ); + if( p0type!=SQLITE_BLOB ){ + while( *z && p1 ){ + SQLITE_SKIP_UTF8(z); + p1--; + } + for(z2=z; *z2 && p2; p2--){ + SQLITE_SKIP_UTF8(z2); + } + sqlite3_result_text64(context, (char*)z, z2-z, SQLITE_TRANSIENT, + SQLITE_UTF8); + }else{ + if( p1+p2>len ){ + p2 = len-p1; + if( p2<0 ) p2 = 0; + } + sqlite3_result_blob64(context, (char*)&z[p1], (u64)p2, SQLITE_TRANSIENT); + } +} + +/* +** Implementation of the round() function +*/ +#ifndef SQLITE_OMIT_FLOATING_POINT +static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + int n = 0; + double r; + char *zBuf; + assert( argc==1 || argc==2 ); + if( argc==2 ){ + if( SQLITE_NULL==sqlite3_value_type(argv[1]) ) return; + n = sqlite3_value_int(argv[1]); + if( n>30 ) n = 30; + if( n<0 ) n = 0; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + r = sqlite3_value_double(argv[0]); + /* If Y==0 and X will fit in a 64-bit int, + ** handle the rounding directly, + ** otherwise use printf. + */ + if( r<-4503599627370496.0 || r>+4503599627370496.0 ){ + /* The value has no fractional part so there is nothing to round */ + }else if( n==0 ){ + r = (double)((sqlite_int64)(r+(r<0?-0.5:+0.5))); + }else{ + zBuf = sqlite3_mprintf("%.*f",n,r); + if( zBuf==0 ){ + sqlite3_result_error_nomem(context); + return; + } + sqlite3AtoF(zBuf, &r, sqlite3Strlen30(zBuf), SQLITE_UTF8); + sqlite3_free(zBuf); + } + sqlite3_result_double(context, r); +} +#endif + +/* +** Allocate nByte bytes of space using sqlite3Malloc(). If the +** allocation fails, call sqlite3_result_error_nomem() to notify +** the database handle that malloc() has failed and return NULL. +** If nByte is larger than the maximum string or blob length, then +** raise an SQLITE_TOOBIG exception and return NULL. +*/ +static void *contextMalloc(sqlite3_context *context, i64 nByte){ + char *z; + sqlite3 *db = sqlite3_context_db_handle(context); + assert( nByte>0 ); + testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nByte==db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); + if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + z = 0; + }else{ + z = sqlite3Malloc(nByte); + if( !z ){ + sqlite3_result_error_nomem(context); + } + } + return z; +} + +/* +** Implementation of the upper() and lower() SQL functions. +*/ +static void upperFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + char *z1; + const char *z2; + int i, n; + UNUSED_PARAMETER(argc); + z2 = (char*)sqlite3_value_text(argv[0]); + n = sqlite3_value_bytes(argv[0]); + /* Verify that the call to _bytes() does not invalidate the _text() pointer */ + assert( z2==(char*)sqlite3_value_text(argv[0]) ); + if( z2 ){ + z1 = contextMalloc(context, ((i64)n)+1); + if( z1 ){ + for(i=0; imatchOne; /* "?" or "_" */ + u32 matchAll = pInfo->matchAll; /* "*" or "%" */ + u8 noCase = pInfo->noCase; /* True if uppercase==lowercase */ + const u8 *zEscaped = 0; /* One past the last escaped input char */ + + while( (c = Utf8Read(zPattern))!=0 ){ + if( c==matchAll ){ /* Match "*" */ + /* Skip over multiple "*" characters in the pattern. If there + ** are also "?" characters, skip those as well, but consume a + ** single character of the input string for each "?" skipped */ + while( (c=Utf8Read(zPattern)) == matchAll + || (c == matchOne && matchOne!=0) ){ + if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ + return SQLITE_NOWILDCARDMATCH; + } + } + if( c==0 ){ + return SQLITE_MATCH; /* "*" at the end of the pattern matches */ + }else if( c==matchOther ){ + if( pInfo->matchSet==0 ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return SQLITE_NOWILDCARDMATCH; + }else{ + /* "[...]" immediately follows the "*". We have to do a slow + ** recursive search in this case, but it is an unusual case. */ + assert( matchOther<0x80 ); /* '[' is a single-byte character */ + while( *zString ){ + int bMatch = patternCompare(&zPattern[-1],zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; + SQLITE_SKIP_UTF8(zString); + } + return SQLITE_NOWILDCARDMATCH; + } + } + + /* At this point variable c contains the first character of the + ** pattern string past the "*". Search in the input string for the + ** first matching character and recursively continue the match from + ** that point. + ** + ** For a case-insensitive search, set variable cx to be the same as + ** c but in the other case and search the input string for either + ** c or cx. + */ + if( c<=0x80 ){ + char zStop[3]; + int bMatch; + if( noCase ){ + zStop[0] = sqlite3Toupper(c); + zStop[1] = sqlite3Tolower(c); + zStop[2] = 0; + }else{ + zStop[0] = c; + zStop[1] = 0; + } + while(1){ + zString += strcspn((const char*)zString, zStop); + if( zString[0]==0 ) break; + zString++; + bMatch = patternCompare(zPattern,zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; + } + }else{ + int bMatch; + while( (c2 = Utf8Read(zString))!=0 ){ + if( c2!=c ) continue; + bMatch = patternCompare(zPattern,zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; + } + } + return SQLITE_NOWILDCARDMATCH; + } + if( c==matchOther ){ + if( pInfo->matchSet==0 ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return SQLITE_NOMATCH; + zEscaped = zPattern; + }else{ + u32 prior_c = 0; + int seen = 0; + int invert = 0; + c = sqlite3Utf8Read(&zString); + if( c==0 ) return SQLITE_NOMATCH; + c2 = sqlite3Utf8Read(&zPattern); + if( c2=='^' ){ + invert = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ + c2 = sqlite3Utf8Read(&zPattern); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==0 || (seen ^ invert)==0 ){ + return SQLITE_NOMATCH; + } + continue; + } + } + c2 = Utf8Read(zString); + if( c==c2 ) continue; + if( noCase && sqlite3Tolower(c)==sqlite3Tolower(c2) && c<0x80 && c2<0x80 ){ + continue; + } + if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue; + return SQLITE_NOMATCH; + } + return *zString==0 ? SQLITE_MATCH : SQLITE_NOMATCH; +} + +/* +** The sqlite3_strglob() interface. Return 0 on a match (like strcmp()) and +** non-zero if there is no match. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); +} + +/* +** The sqlite3_strlike() interface. Return 0 on a match and non-zero for +** a miss - like strcmp(). +*/ +SQLITE_API int sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ + return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); +} + +/* +** Count the number of times that the LIKE operator (or GLOB which is +** just a variation of LIKE) gets called. This is used for testing +** only. +*/ +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_like_count = 0; +#endif + + +/* +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A LIKE B +** +** is implemented as like(B,A). +** +** This same function (with a different compareInfo structure) computes +** the GLOB operator. +*/ +static void likeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zA, *zB; + u32 escape; + int nPat; + sqlite3 *db = sqlite3_context_db_handle(context); + struct compareInfo *pInfo = sqlite3_user_data(context); + struct compareInfo backupInfo; + +#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + || sqlite3_value_type(argv[1])==SQLITE_BLOB + ){ +#ifdef SQLITE_TEST + sqlite3_like_count++; +#endif + sqlite3_result_int(context, 0); + return; + } +#endif + + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + nPat = sqlite3_value_bytes(argv[0]); + testcase( nPat==db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH] ); + testcase( nPat==db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]+1 ); + if( nPat > db->aLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH] ){ + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; + } + if( argc==3 ){ + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + const unsigned char *zEsc = sqlite3_value_text(argv[2]); + if( zEsc==0 ) return; + if( sqlite3Utf8CharLen((char*)zEsc, -1)!=1 ){ + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } + escape = sqlite3Utf8Read(&zEsc); + if( escape==pInfo->matchAll || escape==pInfo->matchOne ){ + memcpy(&backupInfo, pInfo, sizeof(backupInfo)); + pInfo = &backupInfo; + if( escape==pInfo->matchAll ) pInfo->matchAll = 0; + if( escape==pInfo->matchOne ) pInfo->matchOne = 0; + } + }else{ + escape = pInfo->matchSet; + } + zB = sqlite3_value_text(argv[0]); + zA = sqlite3_value_text(argv[1]); + if( zA && zB ){ +#ifdef SQLITE_TEST + sqlite3_like_count++; +#endif + sqlite3_result_int(context, + patternCompare(zB, zA, pInfo, escape)==SQLITE_MATCH); + } +} + +/* +** Implementation of the NULLIF(x,y) function. The result is the first +** argument if the arguments are different. The result is NULL if the +** arguments are equal to each other. +*/ +static void nullifFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + CollSeq *pColl = sqlite3GetFuncCollSeq(context); + UNUSED_PARAMETER(NotUsed); + if( sqlite3MemCompare(argv[0], argv[1], pColl)!=0 ){ + sqlite3_result_value(context, argv[0]); + } +} + +/* +** Implementation of the sqlite_version() function. The result is the version +** of the SQLite library that is running. +*/ +static void versionFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + /* IMP: R-48699-48617 This function is an SQL wrapper around the + ** sqlite3_libversion() C-interface. */ + sqlite3_result_text(context, sqlite3_libversion(), -1, SQLITE_STATIC); +} + +/* +** Implementation of the sqlite_source_id() function. The result is a string +** that identifies the particular version of the source code used to build +** SQLite. +*/ +static void sourceidFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **NotUsed2 +){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + /* IMP: R-24470-31136 This function is an SQL wrapper around the + ** sqlite3_sourceid() C interface. */ + sqlite3_result_text(context, sqlite3_sourceid(), -1, SQLITE_STATIC); +} + +/* +** Implementation of the sqlite_log() function. This is a wrapper around +** sqlite3_log(). The return value is NULL. The function exists purely for +** its side-effects. +*/ +static void errlogFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(context); + sqlite3_log(sqlite3_value_int(argv[0]), "%s", sqlite3_value_text(argv[1])); +} + +/* +** Implementation of the sqlite_compileoption_used() function. +** The result is an integer that identifies if the compiler option +** was used to build SQLite. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +static void compileoptionusedFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zOptName; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + /* IMP: R-39564-36305 The sqlite_compileoption_used() SQL + ** function is a wrapper around the sqlite3_compileoption_used() C/C++ + ** function. + */ + if( (zOptName = (const char*)sqlite3_value_text(argv[0]))!=0 ){ + sqlite3_result_int(context, sqlite3_compileoption_used(zOptName)); + } +} +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +/* +** Implementation of the sqlite_compileoption_get() function. +** The result is a string that identifies the compiler options +** used to build SQLite. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +static void compileoptiongetFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int n; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + /* IMP: R-04922-24076 The sqlite_compileoption_get() SQL function + ** is a wrapper around the sqlite3_compileoption_get() C/C++ function. + */ + n = sqlite3_value_int(argv[0]); + sqlite3_result_text(context, sqlite3_compileoption_get(n), -1, SQLITE_STATIC); +} +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +/* Array for converting from half-bytes (nybbles) into ASCII hex +** digits. */ +static const char hexdigits[] = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' +}; + +/* +** Implementation of the QUOTE() function. This function takes a single +** argument. If the argument is numeric, the return value is the same as +** the argument. If the argument is NULL, the return value is the string +** "NULL". Otherwise, the argument is enclosed in single quotes with +** single-quote escapes. +*/ +static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_FLOAT: { + double r1, r2; + char zBuf[50]; + r1 = sqlite3_value_double(argv[0]); + sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1); + sqlite3AtoF(zBuf, &r2, 20, SQLITE_UTF8); + if( r1!=r2 ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.20e", r1); + } + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + break; + } + case SQLITE_INTEGER: { + sqlite3_result_value(context, argv[0]); + break; + } + case SQLITE_BLOB: { + char *zText = 0; + char const *zBlob = sqlite3_value_blob(argv[0]); + int nBlob = sqlite3_value_bytes(argv[0]); + assert( zBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ + zText = (char *)contextMalloc(context, (2*(i64)nBlob)+4); + if( zText ){ + int i; + for(i=0; i>4)&0x0F]; + zText[(i*2)+3] = hexdigits[(zBlob[i])&0x0F]; + } + zText[(nBlob*2)+2] = '\''; + zText[(nBlob*2)+3] = '\0'; + zText[0] = 'X'; + zText[1] = '\''; + sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT); + sqlite3_free(zText); + } + break; + } + case SQLITE_TEXT: { + int i,j; + u64 n; + const unsigned char *zArg = sqlite3_value_text(argv[0]); + char *z; + + if( zArg==0 ) return; + for(i=0, n=0; zArg[i]; i++){ if( zArg[i]=='\'' ) n++; } + z = contextMalloc(context, ((i64)i)+((i64)n)+3); + if( z ){ + z[0] = '\''; + for(i=0, j=1; zArg[i]; i++){ + z[j++] = zArg[i]; + if( zArg[i]=='\'' ){ + z[j++] = '\''; + } + } + z[j++] = '\''; + z[j] = 0; + sqlite3_result_text(context, z, j, sqlite3_free); + } + break; + } + default: { + assert( sqlite3_value_type(argv[0])==SQLITE_NULL ); + sqlite3_result_text(context, "NULL", 4, SQLITE_STATIC); + break; + } + } +} + +/* +** The unicode() function. Return the integer unicode code-point value +** for the first character of the input string. +*/ +static void unicodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *z = sqlite3_value_text(argv[0]); + (void)argc; + if( z && z[0] ) sqlite3_result_int(context, sqlite3Utf8Read(&z)); +} + +/* +** The char() function takes zero or more arguments, each of which is +** an integer. It constructs a string where each character of the string +** is the unicode character for the corresponding integer argument. +*/ +static void charFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *z, *zOut; + int i; + zOut = z = sqlite3_malloc64( argc*4+1 ); + if( z==0 ){ + sqlite3_result_error_nomem(context); + return; + } + for(i=0; i0x10ffff ) x = 0xfffd; + c = (unsigned)(x & 0x1fffff); + if( c<0x00080 ){ + *zOut++ = (u8)(c&0xFF); + }else if( c<0x00800 ){ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else if( c<0x10000 ){ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + }else{ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); + *zOut++ = 0x80 + (u8)(c & 0x3F); + } \ + } + sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); +} + +/* +** The hex() function. Interpret the argument as a blob. Return +** a hexadecimal rendering as text. +*/ +static void hexFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i, n; + const unsigned char *pBlob; + char *zHex, *z; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + pBlob = sqlite3_value_blob(argv[0]); + n = sqlite3_value_bytes(argv[0]); + assert( pBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ + z = zHex = contextMalloc(context, ((i64)n)*2 + 1); + if( zHex ){ + for(i=0; i>4)&0xf]; + *(z++) = hexdigits[c&0xf]; + } + *z = 0; + sqlite3_result_text(context, zHex, n*2, sqlite3_free); + } +} + +/* +** The zeroblob(N) function returns a zero-filled blob of size N bytes. +*/ +static void zeroblobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + i64 n; + int rc; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + n = sqlite3_value_int64(argv[0]); + if( n<0 ) n = 0; + rc = sqlite3_result_zeroblob64(context, n); /* IMP: R-00293-64994 */ + if( rc ){ + sqlite3_result_error_code(context, rc); + } +} + +/* +** The replace() function. Three arguments are all strings: call +** them A, B, and C. The result is also a string which is derived +** from A by replacing every occurrence of B with C. The match +** must be exact. Collating sequences are not used. +*/ +static void replaceFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zStr; /* The input string A */ + const unsigned char *zPattern; /* The pattern string B */ + const unsigned char *zRep; /* The replacement string C */ + unsigned char *zOut; /* The output */ + int nStr; /* Size of zStr */ + int nPattern; /* Size of zPattern */ + int nRep; /* Size of zRep */ + i64 nOut; /* Maximum size of zOut */ + int loopLimit; /* Last zStr[] that might match zPattern[] */ + int i, j; /* Loop counters */ + unsigned cntExpand; /* Number zOut expansions */ + sqlite3 *db = sqlite3_context_db_handle(context); + + assert( argc==3 ); + UNUSED_PARAMETER(argc); + zStr = sqlite3_value_text(argv[0]); + if( zStr==0 ) return; + nStr = sqlite3_value_bytes(argv[0]); + assert( zStr==sqlite3_value_text(argv[0]) ); /* No encoding change */ + zPattern = sqlite3_value_text(argv[1]); + if( zPattern==0 ){ + assert( sqlite3_value_type(argv[1])==SQLITE_NULL + || sqlite3_context_db_handle(context)->mallocFailed ); + return; + } + if( zPattern[0]==0 ){ + assert( sqlite3_value_type(argv[1])!=SQLITE_NULL ); + sqlite3_result_value(context, argv[0]); + return; + } + nPattern = sqlite3_value_bytes(argv[1]); + assert( zPattern==sqlite3_value_text(argv[1]) ); /* No encoding change */ + zRep = sqlite3_value_text(argv[2]); + if( zRep==0 ) return; + nRep = sqlite3_value_bytes(argv[2]); + assert( zRep==sqlite3_value_text(argv[2]) ); + nOut = nStr + 1; + assert( nOutnPattern ){ + nOut += nRep - nPattern; + testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); + if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + sqlite3_free(zOut); + return; + } + cntExpand++; + if( (cntExpand&(cntExpand-1))==0 ){ + /* Grow the size of the output buffer only on substitutions + ** whose index is a power of two: 1, 2, 4, 8, 16, 32, ... */ + u8 *zOld; + zOld = zOut; + zOut = sqlite3Realloc(zOut, (int)nOut + (nOut - nStr - 1)); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(zOld); + return; + } + } + } + memcpy(&zOut[j], zRep, nRep); + j += nRep; + i += nPattern-1; + } + } + assert( j+nStr-i+1<=nOut ); + memcpy(&zOut[j], &zStr[i], nStr-i); + j += nStr - i; + assert( j<=nOut ); + zOut[j] = 0; + sqlite3_result_text(context, (char*)zOut, j, sqlite3_free); +} + +/* +** Implementation of the TRIM(), LTRIM(), and RTRIM() functions. +** The userdata is 0x1 for left trim, 0x2 for right trim, 0x3 for both. +*/ +static void trimFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zIn; /* Input string */ + const unsigned char *zCharSet; /* Set of characters to trim */ + unsigned int nIn; /* Number of bytes in input */ + int flags; /* 1: trimleft 2: trimright 3: trim */ + int i; /* Loop counter */ + unsigned int *aLen = 0; /* Length of each character in zCharSet */ + unsigned char **azChar = 0; /* Individual characters in zCharSet */ + int nChar; /* Number of characters in zCharSet */ + + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + return; + } + zIn = sqlite3_value_text(argv[0]); + if( zIn==0 ) return; + nIn = (unsigned)sqlite3_value_bytes(argv[0]); + assert( zIn==sqlite3_value_text(argv[0]) ); + if( argc==1 ){ + static const unsigned lenOne[] = { 1 }; + static unsigned char * const azOne[] = { (u8*)" " }; + nChar = 1; + aLen = (unsigned*)lenOne; + azChar = (unsigned char **)azOne; + zCharSet = 0; + }else if( (zCharSet = sqlite3_value_text(argv[1]))==0 ){ + return; + }else{ + const unsigned char *z; + for(z=zCharSet, nChar=0; *z; nChar++){ + SQLITE_SKIP_UTF8(z); + } + if( nChar>0 ){ + azChar = contextMalloc(context, + ((i64)nChar)*(sizeof(char*)+sizeof(unsigned))); + if( azChar==0 ){ + return; + } + aLen = (unsigned*)&azChar[nChar]; + for(z=zCharSet, nChar=0; *z; nChar++){ + azChar[nChar] = (unsigned char *)z; + SQLITE_SKIP_UTF8(z); + aLen[nChar] = (unsigned)(z - azChar[nChar]); + } + } + } + if( nChar>0 ){ + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(context)); + if( flags & 1 ){ + while( nIn>0 ){ + unsigned int len = 0; + for(i=0; i=nChar ) break; + zIn += len; + nIn -= len; + } + } + if( flags & 2 ){ + while( nIn>0 ){ + unsigned int len = 0; + for(i=0; i=nChar ) break; + nIn -= len; + } + } + if( zCharSet ){ + sqlite3_free(azChar); + } + } + sqlite3_result_text(context, (char*)zIn, nIn, SQLITE_TRANSIENT); +} + + +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +/* +** The "unknown" function is automatically substituted in place of +** any unrecognized function name when doing an EXPLAIN or EXPLAIN QUERY PLAN +** when the SQLITE_ENABLE_UNKNOWN_FUNCTION compile-time option is used. +** When the "sqlite3" command-line shell is built using this functionality, +** that allows an EXPLAIN or EXPLAIN QUERY PLAN for complex queries +** involving application-defined functions to be examined in a generic +** sqlite3 shell. +*/ +static void unknownFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + /* no-op */ +} +#endif /*SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION*/ + + +/* IMP: R-25361-16150 This function is omitted from SQLite by default. It +** is only available if the SQLITE_SOUNDEX compile-time option is used +** when SQLite is built. +*/ +#ifdef SQLITE_SOUNDEX +/* +** Compute the soundex encoding of a word. +** +** IMP: R-59782-00072 The soundex(X) function returns a string that is the +** soundex encoding of the string X. +*/ +static void soundexFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + char zResult[8]; + const u8 *zIn; + int i, j; + static const unsigned char iCode[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, + 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 0, 1, 2, 0, 0, 2, 2, 4, 5, 5, 0, + 1, 2, 6, 2, 3, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, + }; + assert( argc==1 ); + zIn = (u8*)sqlite3_value_text(argv[0]); + if( zIn==0 ) zIn = (u8*)""; + for(i=0; zIn[i] && !sqlite3Isalpha(zIn[i]); i++){} + if( zIn[i] ){ + u8 prevcode = iCode[zIn[i]&0x7f]; + zResult[0] = sqlite3Toupper(zIn[i]); + for(j=1; j<4 && zIn[i]; i++){ + int code = iCode[zIn[i]&0x7f]; + if( code>0 ){ + if( code!=prevcode ){ + prevcode = code; + zResult[j++] = code + '0'; + } + }else{ + prevcode = 0; + } + } + while( j<4 ){ + zResult[j++] = '0'; + } + zResult[j] = 0; + sqlite3_result_text(context, zResult, 4, SQLITE_TRANSIENT); + }else{ + /* IMP: R-64894-50321 The string "?000" is returned if the argument + ** is NULL or contains no ASCII alphabetic characters. */ + sqlite3_result_text(context, "?000", 4, SQLITE_STATIC); + } +} +#endif /* SQLITE_SOUNDEX */ + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** A function that loads a shared-library extension then returns NULL. +*/ +static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ + const char *zFile = (const char *)sqlite3_value_text(argv[0]); + const char *zProc; + sqlite3 *db = sqlite3_context_db_handle(context); + char *zErrMsg = 0; + + /* Disallow the load_extension() SQL function unless the SQLITE_LoadExtFunc + ** flag is set. See the sqlite3_enable_load_extension() API. + */ + if( (db->flags & SQLITE_LoadExtFunc)==0 ){ + sqlite3_result_error(context, "not authorized", -1); + return; + } + + if( argc==2 ){ + zProc = (const char *)sqlite3_value_text(argv[1]); + }else{ + zProc = 0; + } + if( zFile && sqlite3_load_extension(db, zFile, zProc, &zErrMsg) ){ + sqlite3_result_error(context, zErrMsg, -1); + sqlite3_free(zErrMsg); + } +} +#endif + + +/* +** An instance of the following structure holds the context of a +** sum() or avg() aggregate computation. +*/ +typedef struct SumCtx SumCtx; +struct SumCtx { + double rSum; /* Floating point sum */ + i64 iSum; /* Integer sum */ + i64 cnt; /* Number of elements summed */ + u8 overflow; /* True if integer overflow seen */ + u8 approx; /* True if non-integer value was input to the sum */ +}; + +/* +** Routines used to compute the sum, average, and total. +** +** The SUM() function follows the (broken) SQL standard which means +** that it returns NULL if it sums over no inputs. TOTAL returns +** 0.0 in that case. In addition, TOTAL always returns a float where +** SUM might return an integer if it never encounters a floating point +** value. TOTAL never fails, but SUM might through an exception if +** it overflows an integer. +*/ +static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ + SumCtx *p; + int type; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + type = sqlite3_value_numeric_type(argv[0]); + if( p && type!=SQLITE_NULL ){ + p->cnt++; + if( type==SQLITE_INTEGER ){ + i64 v = sqlite3_value_int64(argv[0]); + p->rSum += v; + if( (p->approx|p->overflow)==0 && sqlite3AddInt64(&p->iSum, v) ){ + p->approx = p->overflow = 1; + } + }else{ + p->rSum += sqlite3_value_double(argv[0]); + p->approx = 1; + } + } +} +#ifndef SQLITE_OMIT_WINDOWFUNC +static void sumInverse(sqlite3_context *context, int argc, sqlite3_value**argv){ + SumCtx *p; + int type; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + type = sqlite3_value_numeric_type(argv[0]); + /* p is always non-NULL because sumStep() will have been called first + ** to initialize it */ + if( ALWAYS(p) && type!=SQLITE_NULL ){ + assert( p->cnt>0 ); + p->cnt--; + assert( type==SQLITE_INTEGER || p->approx ); + if( type==SQLITE_INTEGER && p->approx==0 ){ + i64 v = sqlite3_value_int64(argv[0]); + p->rSum -= v; + p->iSum -= v; + }else{ + p->rSum -= sqlite3_value_double(argv[0]); + } + } +} +#else +# define sumInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ +static void sumFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + if( p && p->cnt>0 ){ + if( p->overflow ){ + sqlite3_result_error(context,"integer overflow",-1); + }else if( p->approx ){ + sqlite3_result_double(context, p->rSum); + }else{ + sqlite3_result_int64(context, p->iSum); + } + } +} +static void avgFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + if( p && p->cnt>0 ){ + sqlite3_result_double(context, p->rSum/(double)p->cnt); + } +} +static void totalFinalize(sqlite3_context *context){ + SumCtx *p; + p = sqlite3_aggregate_context(context, 0); + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + sqlite3_result_double(context, p ? p->rSum : (double)0); +} + +/* +** The following structure keeps track of state information for the +** count() aggregate function. +*/ +typedef struct CountCtx CountCtx; +struct CountCtx { + i64 n; +#ifdef SQLITE_DEBUG + int bInverse; /* True if xInverse() ever called */ +#endif +}; + +/* +** Routines to implement the count() aggregate function. +*/ +static void countStep(sqlite3_context *context, int argc, sqlite3_value **argv){ + CountCtx *p; + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && p ){ + p->n++; + } + +#ifndef SQLITE_OMIT_DEPRECATED + /* The sqlite3_aggregate_count() function is deprecated. But just to make + ** sure it still operates correctly, verify that its count agrees with our + ** internal count when using count(*) and when the total count can be + ** expressed as a 32-bit integer. */ + assert( argc==1 || p==0 || p->n>0x7fffffff || p->bInverse + || p->n==sqlite3_aggregate_count(context) ); +#endif +} +static void countFinalize(sqlite3_context *context){ + CountCtx *p; + p = sqlite3_aggregate_context(context, 0); + sqlite3_result_int64(context, p ? p->n : 0); +} +#ifndef SQLITE_OMIT_WINDOWFUNC +static void countInverse(sqlite3_context *ctx, int argc, sqlite3_value **argv){ + CountCtx *p; + p = sqlite3_aggregate_context(ctx, sizeof(*p)); + /* p is always non-NULL since countStep() will have been called first */ + if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && ALWAYS(p) ){ + p->n--; +#ifdef SQLITE_DEBUG + p->bInverse = 1; +#endif + } +} +#else +# define countInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** Routines to implement min() and max() aggregate functions. +*/ +static void minmaxStep( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + Mem *pArg = (Mem *)argv[0]; + Mem *pBest; + UNUSED_PARAMETER(NotUsed); + + pBest = (Mem *)sqlite3_aggregate_context(context, sizeof(*pBest)); + if( !pBest ) return; + + if( sqlite3_value_type(pArg)==SQLITE_NULL ){ + if( pBest->flags ) sqlite3SkipAccumulatorLoad(context); + }else if( pBest->flags ){ + int max; + int cmp; + CollSeq *pColl = sqlite3GetFuncCollSeq(context); + /* This step function is used for both the min() and max() aggregates, + ** the only difference between the two being that the sense of the + ** comparison is inverted. For the max() aggregate, the + ** sqlite3_user_data() function returns (void *)-1. For min() it + ** returns (void *)db, where db is the sqlite3* database pointer. + ** Therefore the next statement sets variable 'max' to 1 for the max() + ** aggregate, or 0 for min(). + */ + max = sqlite3_user_data(context)!=0; + cmp = sqlite3MemCompare(pBest, pArg, pColl); + if( (max && cmp<0) || (!max && cmp>0) ){ + sqlite3VdbeMemCopy(pBest, pArg); + }else{ + sqlite3SkipAccumulatorLoad(context); + } + }else{ + pBest->db = sqlite3_context_db_handle(context); + sqlite3VdbeMemCopy(pBest, pArg); + } +} +static void minMaxValueFinalize(sqlite3_context *context, int bValue){ + sqlite3_value *pRes; + pRes = (sqlite3_value *)sqlite3_aggregate_context(context, 0); + if( pRes ){ + if( pRes->flags ){ + sqlite3_result_value(context, pRes); + } + if( bValue==0 ) sqlite3VdbeMemRelease(pRes); + } +} +#ifndef SQLITE_OMIT_WINDOWFUNC +static void minMaxValue(sqlite3_context *context){ + minMaxValueFinalize(context, 1); +} +#else +# define minMaxValue 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ +static void minMaxFinalize(sqlite3_context *context){ + minMaxValueFinalize(context, 0); +} + +/* +** group_concat(EXPR, ?SEPARATOR?) +** +** The SEPARATOR goes before the EXPR string. This is tragic. The +** groupConcatInverse() implementation would have been easier if the +** SEPARATOR were appended after EXPR. And the order is undocumented, +** so we could change it, in theory. But the old behavior has been +** around for so long that we dare not, for fear of breaking something. +*/ +typedef struct { + StrAccum str; /* The accumulated concatenation */ +#ifndef SQLITE_OMIT_WINDOWFUNC + int nAccum; /* Number of strings presently concatenated */ + int nFirstSepLength; /* Used to detect separator length change */ + /* If pnSepLengths!=0, refs an array of inter-string separator lengths, + ** stored as actually incorporated into presently accumulated result. + ** (Hence, its slots in use number nAccum-1 between method calls.) + ** If pnSepLengths==0, nFirstSepLength is the length used throughout. + */ + int *pnSepLengths; +#endif +} GroupConcatCtx; + +static void groupConcatStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zVal; + GroupConcatCtx *pGCC; + const char *zSep; + int nVal, nSep; + assert( argc==1 || argc==2 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pGCC = (GroupConcatCtx*)sqlite3_aggregate_context(context, sizeof(*pGCC)); + if( pGCC ){ + sqlite3 *db = sqlite3_context_db_handle(context); + int firstTerm = pGCC->str.mxAlloc==0; + pGCC->str.mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; + if( argc==1 ){ + if( !firstTerm ){ + sqlite3_str_appendchar(&pGCC->str, 1, ','); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + else{ + pGCC->nFirstSepLength = 1; + } +#endif + }else if( !firstTerm ){ + zSep = (char*)sqlite3_value_text(argv[1]); + nSep = sqlite3_value_bytes(argv[1]); + if( zSep ){ + sqlite3_str_append(&pGCC->str, zSep, nSep); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + else{ + nSep = 0; + } + if( nSep != pGCC->nFirstSepLength || pGCC->pnSepLengths != 0 ){ + int *pnsl = pGCC->pnSepLengths; + if( pnsl == 0 ){ + /* First separator length variation seen, start tracking them. */ + pnsl = (int*)sqlite3_malloc64((pGCC->nAccum+1) * sizeof(int)); + if( pnsl!=0 ){ + int i = 0, nA = pGCC->nAccum-1; + while( inFirstSepLength; + } + }else{ + pnsl = (int*)sqlite3_realloc64(pnsl, pGCC->nAccum * sizeof(int)); + } + if( pnsl!=0 ){ + if( ALWAYS(pGCC->nAccum>0) ){ + pnsl[pGCC->nAccum-1] = nSep; + } + pGCC->pnSepLengths = pnsl; + }else{ + sqlite3StrAccumSetError(&pGCC->str, SQLITE_NOMEM); + } + } +#endif + } +#ifndef SQLITE_OMIT_WINDOWFUNC + else{ + pGCC->nFirstSepLength = sqlite3_value_bytes(argv[1]); + } + pGCC->nAccum += 1; +#endif + zVal = (char*)sqlite3_value_text(argv[0]); + nVal = sqlite3_value_bytes(argv[0]); + if( zVal ) sqlite3_str_append(&pGCC->str, zVal, nVal); + } +} + +#ifndef SQLITE_OMIT_WINDOWFUNC +static void groupConcatInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GroupConcatCtx *pGCC; + assert( argc==1 || argc==2 ); + (void)argc; /* Suppress unused parameter warning */ + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pGCC = (GroupConcatCtx*)sqlite3_aggregate_context(context, sizeof(*pGCC)); + /* pGCC is always non-NULL since groupConcatStep() will have always + ** run frist to initialize it */ + if( ALWAYS(pGCC) ){ + int nVS = sqlite3_value_bytes(argv[0]); + pGCC->nAccum -= 1; + if( pGCC->pnSepLengths!=0 ){ + assert(pGCC->nAccum >= 0); + if( pGCC->nAccum>0 ){ + nVS += *pGCC->pnSepLengths; + memmove(pGCC->pnSepLengths, pGCC->pnSepLengths+1, + (pGCC->nAccum-1)*sizeof(int)); + } + }else{ + /* If removing single accumulated string, harmlessly over-do. */ + nVS += pGCC->nFirstSepLength; + } + if( nVS>=(int)pGCC->str.nChar ){ + pGCC->str.nChar = 0; + }else{ + pGCC->str.nChar -= nVS; + memmove(pGCC->str.zText, &pGCC->str.zText[nVS], pGCC->str.nChar); + } + if( pGCC->str.nChar==0 ){ + pGCC->str.mxAlloc = 0; + sqlite3_free(pGCC->pnSepLengths); + pGCC->pnSepLengths = 0; + } + } +} +#else +# define groupConcatInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ +static void groupConcatFinalize(sqlite3_context *context){ + GroupConcatCtx *pGCC + = (GroupConcatCtx*)sqlite3_aggregate_context(context, 0); + if( pGCC ){ + sqlite3ResultStrAccum(context, &pGCC->str); +#ifndef SQLITE_OMIT_WINDOWFUNC + sqlite3_free(pGCC->pnSepLengths); +#endif + } +} +#ifndef SQLITE_OMIT_WINDOWFUNC +static void groupConcatValue(sqlite3_context *context){ + GroupConcatCtx *pGCC + = (GroupConcatCtx*)sqlite3_aggregate_context(context, 0); + if( pGCC ){ + StrAccum *pAccum = &pGCC->str; + if( pAccum->accError==SQLITE_TOOBIG ){ + sqlite3_result_error_toobig(context); + }else if( pAccum->accError==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + }else{ + const char *zText = sqlite3_str_value(pAccum); + sqlite3_result_text(context, zText, pAccum->nChar, SQLITE_TRANSIENT); + } + } +} +#else +# define groupConcatValue 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** This routine does per-connection function registration. Most +** of the built-in functions above are part of the global function set. +** This routine only deals with those that are not global. +*/ +SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ + int rc = sqlite3_overload_function(db, "MATCH", 2); + assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); + if( rc==SQLITE_NOMEM ){ + sqlite3OomFault(db); + } +} + +/* +** Re-register the built-in LIKE functions. The caseSensitive +** parameter determines whether or not the LIKE operator is case +** sensitive. +*/ +SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ + struct compareInfo *pInfo; + int flags; + if( caseSensitive ){ + pInfo = (struct compareInfo*)&likeInfoAlt; + flags = SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE; + }else{ + pInfo = (struct compareInfo*)&likeInfoNorm; + flags = SQLITE_FUNC_LIKE; + } + sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); + sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); + sqlite3FindFunction(db, "like", 2, SQLITE_UTF8, 0)->funcFlags |= flags; + sqlite3FindFunction(db, "like", 3, SQLITE_UTF8, 0)->funcFlags |= flags; +} + +/* +** pExpr points to an expression which implements a function. If +** it is appropriate to apply the LIKE optimization to that function +** then set aWc[0] through aWc[2] to the wildcard characters and the +** escape character and then return TRUE. If the function is not a +** LIKE-style function then return FALSE. +** +** The expression "a LIKE b ESCAPE c" is only considered a valid LIKE +** operator if c is a string literal that is exactly one byte in length. +** That one byte is stored in aWc[3]. aWc[3] is set to zero if there is +** no ESCAPE clause. +** +** *pIsNocase is set to true if uppercase and lowercase are equivalent for +** the function (default for LIKE). If the function makes the distinction +** between uppercase and lowercase (as does GLOB) then *pIsNocase is set to +** false. +*/ +SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ + FuncDef *pDef; + int nExpr; + assert( pExpr!=0 ); + assert( pExpr->op==TK_FUNCTION ); + if( !pExpr->x.pList ){ + return 0; + } + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + nExpr = pExpr->x.pList->nExpr; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, nExpr, SQLITE_UTF8, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 ) return 0; +#endif + if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_FUNC_LIKE)==0 ){ + return 0; + } + + /* The memcpy() statement assumes that the wildcard characters are + ** the first three statements in the compareInfo structure. The + ** asserts() that follow verify that assumption + */ + memcpy(aWc, pDef->pUserData, 3); + assert( (char*)&likeInfoAlt == (char*)&likeInfoAlt.matchAll ); + assert( &((char*)&likeInfoAlt)[1] == (char*)&likeInfoAlt.matchOne ); + assert( &((char*)&likeInfoAlt)[2] == (char*)&likeInfoAlt.matchSet ); + + if( nExpr<3 ){ + aWc[3] = 0; + }else{ + Expr *pEscape = pExpr->x.pList->a[2].pExpr; + char *zEscape; + if( pEscape->op!=TK_STRING ) return 0; + zEscape = pEscape->u.zToken; + if( zEscape[0]==0 || zEscape[1]!=0 ) return 0; + if( zEscape[0]==aWc[0] ) return 0; + if( zEscape[0]==aWc[1] ) return 0; + aWc[3] = zEscape[0]; + } + + *pIsNocase = (pDef->funcFlags & SQLITE_FUNC_CASE)==0; + return 1; +} + +/* Mathematical Constants */ +#ifndef M_PI +# define M_PI 3.141592653589793238462643383279502884 +#endif +#ifndef M_LN10 +# define M_LN10 2.302585092994045684017991454684364208 +#endif +#ifndef M_LN2 +# define M_LN2 0.693147180559945309417232121458176568 +#endif + + +/* Extra math functions that require linking with -lm +*/ +#ifdef SQLITE_ENABLE_MATH_FUNCTIONS +/* +** Implementation SQL functions: +** +** ceil(X) +** ceiling(X) +** floor(X) +** +** The sqlite3_user_data() pointer is a pointer to the libm implementation +** of the underlying C function. +*/ +static void ceilingFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + switch( sqlite3_value_numeric_type(argv[0]) ){ + case SQLITE_INTEGER: { + sqlite3_result_int64(context, sqlite3_value_int64(argv[0])); + break; + } + case SQLITE_FLOAT: { + double (*x)(double) = (double(*)(double))sqlite3_user_data(context); + sqlite3_result_double(context, x(sqlite3_value_double(argv[0]))); + break; + } + default: { + break; + } + } +} + +/* +** On some systems, ceil() and floor() are intrinsic function. You are +** unable to take a pointer to these functions. Hence, we here wrap them +** in our own actual functions. +*/ +static double xCeil(double x){ return ceil(x); } +static double xFloor(double x){ return floor(x); } + +/* +** Implementation of SQL functions: +** +** ln(X) - natural logarithm +** log(X) - log X base 10 +** log10(X) - log X base 10 +** log(B,X) - log X base B +*/ +static void logFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double x, b, ans; + assert( argc==1 || argc==2 ); + switch( sqlite3_value_numeric_type(argv[0]) ){ + case SQLITE_INTEGER: + case SQLITE_FLOAT: + x = sqlite3_value_double(argv[0]); + if( x<=0.0 ) return; + break; + default: + return; + } + if( argc==2 ){ + switch( sqlite3_value_numeric_type(argv[0]) ){ + case SQLITE_INTEGER: + case SQLITE_FLOAT: + b = log(x); + if( b<=0.0 ) return; + x = sqlite3_value_double(argv[1]); + if( x<=0.0 ) return; + break; + default: + return; + } + ans = log(x)/b; + }else{ + ans = log(x); + switch( SQLITE_PTR_TO_INT(sqlite3_user_data(context)) ){ + case 1: + /* Convert from natural logarithm to log base 10 */ + ans *= 1.0/M_LN10; + break; + case 2: + /* Convert from natural logarithm to log base 2 */ + ans *= 1.0/M_LN2; + break; + default: + break; + } + } + sqlite3_result_double(context, ans); +} + +/* +** Functions to converts degrees to radians and radians to degrees. +*/ +static double degToRad(double x){ return x*(M_PI/180.0); } +static double radToDeg(double x){ return x*(180.0/M_PI); } + +/* +** Implementation of 1-argument SQL math functions: +** +** exp(X) - Compute e to the X-th power +*/ +static void math1Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int type0; + double v0, ans; + double (*x)(double); + assert( argc==1 ); + type0 = sqlite3_value_numeric_type(argv[0]); + if( type0!=SQLITE_INTEGER && type0!=SQLITE_FLOAT ) return; + v0 = sqlite3_value_double(argv[0]); + x = (double(*)(double))sqlite3_user_data(context); + ans = x(v0); + sqlite3_result_double(context, ans); +} + +/* +** Implementation of 2-argument SQL math functions: +** +** power(X,Y) - Compute X to the Y-th power +*/ +static void math2Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int type0, type1; + double v0, v1, ans; + double (*x)(double,double); + assert( argc==2 ); + type0 = sqlite3_value_numeric_type(argv[0]); + if( type0!=SQLITE_INTEGER && type0!=SQLITE_FLOAT ) return; + type1 = sqlite3_value_numeric_type(argv[1]); + if( type1!=SQLITE_INTEGER && type1!=SQLITE_FLOAT ) return; + v0 = sqlite3_value_double(argv[0]); + v1 = sqlite3_value_double(argv[1]); + x = (double(*)(double,double))sqlite3_user_data(context); + ans = x(v0, v1); + sqlite3_result_double(context, ans); +} + +/* +** Implementation of 0-argument pi() function. +*/ +static void piFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==0 ); + sqlite3_result_double(context, M_PI); +} + +#endif /* SQLITE_ENABLE_MATH_FUNCTIONS */ + +/* +** Implementation of sign(X) function. +*/ +static void signFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int type0; + double x; + UNUSED_PARAMETER(argc); + assert( argc==1 ); + type0 = sqlite3_value_numeric_type(argv[0]); + if( type0!=SQLITE_INTEGER && type0!=SQLITE_FLOAT ) return; + x = sqlite3_value_double(argv[0]); + sqlite3_result_int(context, x<0.0 ? -1 : x>0.0 ? +1 : 0); +} + +/* +** All of the FuncDef structures in the aBuiltinFunc[] array above +** to the global function hash table. This occurs at start-time (as +** a consequence of calling sqlite3_initialize()). +** +** After this routine runs +*/ +SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ + /* + ** The following array holds FuncDef structures for all of the functions + ** defined in this file. + ** + ** The array cannot be constant since changes are made to the + ** FuncDef.pHash elements at start-time. The elements of this array + ** are read-only after initialization is complete. + ** + ** For peak efficiency, put the most frequently used function last. + */ + static FuncDef aBuiltinFunc[] = { +/***** Functions only available with SQLITE_TESTCTRL_INTERNAL_FUNCTIONS *****/ +#if !defined(SQLITE_UNTESTABLE) + TEST_FUNC(implies_nonnull_row, 2, INLINEFUNC_implies_nonnull_row, 0), + TEST_FUNC(expr_compare, 2, INLINEFUNC_expr_compare, 0), + TEST_FUNC(expr_implies_expr, 2, INLINEFUNC_expr_implies_expr, 0), + TEST_FUNC(affinity, 1, INLINEFUNC_affinity, 0), +#endif /* !defined(SQLITE_UNTESTABLE) */ +/***** Regular functions *****/ +#ifdef SQLITE_SOUNDEX + FUNCTION(soundex, 1, 0, 0, soundexFunc ), +#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION + SFUNCTION(load_extension, 1, 0, 0, loadExt ), + SFUNCTION(load_extension, 2, 0, 0, loadExt ), +#endif +#if SQLITE_USER_AUTHENTICATION + FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), +#endif +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), + DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + INLINE_FUNC(unlikely, 1, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), + INLINE_FUNC(likelihood, 2, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), + INLINE_FUNC(likely, 1, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + FUNCTION2(sqlite_offset, 1, 0, 0, noopFunc, SQLITE_FUNC_OFFSET| + SQLITE_FUNC_TYPEOF), +#endif + FUNCTION(ltrim, 1, 1, 0, trimFunc ), + FUNCTION(ltrim, 2, 1, 0, trimFunc ), + FUNCTION(rtrim, 1, 2, 0, trimFunc ), + FUNCTION(rtrim, 2, 2, 0, trimFunc ), + FUNCTION(trim, 1, 3, 0, trimFunc ), + FUNCTION(trim, 2, 3, 0, trimFunc ), + FUNCTION(min, -1, 0, 1, minmaxFunc ), + FUNCTION(min, 0, 0, 1, 0 ), + WAGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize, minMaxValue, 0, + SQLITE_FUNC_MINMAX|SQLITE_FUNC_ANYORDER ), + FUNCTION(max, -1, 1, 1, minmaxFunc ), + FUNCTION(max, 0, 1, 1, 0 ), + WAGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize, minMaxValue, 0, + SQLITE_FUNC_MINMAX|SQLITE_FUNC_ANYORDER ), + FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), + FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), + FUNCTION(instr, 2, 0, 0, instrFunc ), + FUNCTION(printf, -1, 0, 0, printfFunc ), + FUNCTION(unicode, 1, 0, 0, unicodeFunc ), + FUNCTION(char, -1, 0, 0, charFunc ), + FUNCTION(abs, 1, 0, 0, absFunc ), +#ifndef SQLITE_OMIT_FLOATING_POINT + FUNCTION(round, 1, 0, 0, roundFunc ), + FUNCTION(round, 2, 0, 0, roundFunc ), +#endif + FUNCTION(upper, 1, 0, 0, upperFunc ), + FUNCTION(lower, 1, 0, 0, lowerFunc ), + FUNCTION(hex, 1, 0, 0, hexFunc ), + INLINE_FUNC(ifnull, 2, INLINEFUNC_coalesce, 0 ), + VFUNCTION(random, 0, 0, 0, randomFunc ), + VFUNCTION(randomblob, 1, 0, 0, randomBlob ), + FUNCTION(nullif, 2, 0, 1, nullifFunc ), + DFUNCTION(sqlite_version, 0, 0, 0, versionFunc ), + DFUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), + FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), + FUNCTION(quote, 1, 0, 0, quoteFunc ), + VFUNCTION(last_insert_rowid, 0, 0, 0, last_insert_rowid), + VFUNCTION(changes, 0, 0, 0, changes ), + VFUNCTION(total_changes, 0, 0, 0, total_changes ), + FUNCTION(replace, 3, 0, 0, replaceFunc ), + FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), + FUNCTION(substr, 2, 0, 0, substrFunc ), + FUNCTION(substr, 3, 0, 0, substrFunc ), + FUNCTION(substring, 2, 0, 0, substrFunc ), + FUNCTION(substring, 3, 0, 0, substrFunc ), + WAGGREGATE(sum, 1,0,0, sumStep, sumFinalize, sumFinalize, sumInverse, 0), + WAGGREGATE(total, 1,0,0, sumStep,totalFinalize,totalFinalize,sumInverse, 0), + WAGGREGATE(avg, 1,0,0, sumStep, avgFinalize, avgFinalize, sumInverse, 0), + WAGGREGATE(count, 0,0,0, countStep, + countFinalize, countFinalize, countInverse, + SQLITE_FUNC_COUNT|SQLITE_FUNC_ANYORDER ), + WAGGREGATE(count, 1,0,0, countStep, + countFinalize, countFinalize, countInverse, SQLITE_FUNC_ANYORDER ), + WAGGREGATE(group_concat, 1, 0, 0, groupConcatStep, + groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), + WAGGREGATE(group_concat, 2, 0, 0, groupConcatStep, + groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), + + LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), +#ifdef SQLITE_CASE_SENSITIVE_LIKE + LIKEFUNC(like, 2, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), + LIKEFUNC(like, 3, &likeInfoAlt, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), +#else + LIKEFUNC(like, 2, &likeInfoNorm, SQLITE_FUNC_LIKE), + LIKEFUNC(like, 3, &likeInfoNorm, SQLITE_FUNC_LIKE), +#endif +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + FUNCTION(unknown, -1, 0, 0, unknownFunc ), +#endif + FUNCTION(coalesce, 1, 0, 0, 0 ), + FUNCTION(coalesce, 0, 0, 0, 0 ), +#ifdef SQLITE_ENABLE_MATH_FUNCTIONS + MFUNCTION(ceil, 1, xCeil, ceilingFunc ), + MFUNCTION(ceiling, 1, xCeil, ceilingFunc ), + MFUNCTION(floor, 1, xFloor, ceilingFunc ), +#if SQLITE_HAVE_C99_MATH_FUNCS + MFUNCTION(trunc, 1, trunc, ceilingFunc ), +#endif + FUNCTION(ln, 1, 0, 0, logFunc ), + FUNCTION(log, 1, 1, 0, logFunc ), + FUNCTION(log10, 1, 1, 0, logFunc ), + FUNCTION(log2, 1, 2, 0, logFunc ), + FUNCTION(log, 2, 0, 0, logFunc ), + MFUNCTION(exp, 1, exp, math1Func ), + MFUNCTION(pow, 2, pow, math2Func ), + MFUNCTION(power, 2, pow, math2Func ), + MFUNCTION(mod, 2, fmod, math2Func ), + MFUNCTION(acos, 1, acos, math1Func ), + MFUNCTION(asin, 1, asin, math1Func ), + MFUNCTION(atan, 1, atan, math1Func ), + MFUNCTION(atan2, 2, atan2, math2Func ), + MFUNCTION(cos, 1, cos, math1Func ), + MFUNCTION(sin, 1, sin, math1Func ), + MFUNCTION(tan, 1, tan, math1Func ), + MFUNCTION(cosh, 1, cosh, math1Func ), + MFUNCTION(sinh, 1, sinh, math1Func ), + MFUNCTION(tanh, 1, tanh, math1Func ), +#if SQLITE_HAVE_C99_MATH_FUNCS + MFUNCTION(acosh, 1, acosh, math1Func ), + MFUNCTION(asinh, 1, asinh, math1Func ), + MFUNCTION(atanh, 1, atanh, math1Func ), +#endif + MFUNCTION(sqrt, 1, sqrt, math1Func ), + MFUNCTION(radians, 1, degToRad, math1Func ), + MFUNCTION(degrees, 1, radToDeg, math1Func ), + FUNCTION(pi, 0, 0, 0, piFunc ), +#endif /* SQLITE_ENABLE_MATH_FUNCTIONS */ + FUNCTION(sign, 1, 0, 0, signFunc ), + INLINE_FUNC(coalesce, -1, INLINEFUNC_coalesce, 0 ), + INLINE_FUNC(iif, 3, INLINEFUNC_iif, 0 ), + }; +#ifndef SQLITE_OMIT_ALTERTABLE + sqlite3AlterFunctions(); +#endif + sqlite3WindowFunctions(); + sqlite3RegisterDateTimeFunctions(); + sqlite3InsertBuiltinFuncs(aBuiltinFunc, ArraySize(aBuiltinFunc)); + +#if 0 /* Enable to print out how the built-in functions are hashed */ + { + int i; + FuncDef *p; + for(i=0; iu.pHash){ + int n = sqlite3Strlen30(p->zName); + int h = p->zName[0] + n; + printf(" %s(%d)", p->zName, h); + } + printf("\n"); + } + } +#endif +} + +/************** End of func.c ************************************************/ +/************** Begin file fkey.c ********************************************/ +/* +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used by the compiler to add foreign key +** support to compiled SQL statements. +*/ +/* #include "sqliteInt.h" */ + +#ifndef SQLITE_OMIT_FOREIGN_KEY +#ifndef SQLITE_OMIT_TRIGGER + +/* +** Deferred and Immediate FKs +** -------------------------- +** +** Foreign keys in SQLite come in two flavours: deferred and immediate. +** If an immediate foreign key constraint is violated, +** SQLITE_CONSTRAINT_FOREIGNKEY is returned and the current +** statement transaction rolled back. If a +** deferred foreign key constraint is violated, no action is taken +** immediately. However if the application attempts to commit the +** transaction before fixing the constraint violation, the attempt fails. +** +** Deferred constraints are implemented using a simple counter associated +** with the database handle. The counter is set to zero each time a +** database transaction is opened. Each time a statement is executed +** that causes a foreign key violation, the counter is incremented. Each +** time a statement is executed that removes an existing violation from +** the database, the counter is decremented. When the transaction is +** committed, the commit fails if the current value of the counter is +** greater than zero. This scheme has two big drawbacks: +** +** * When a commit fails due to a deferred foreign key constraint, +** there is no way to tell which foreign constraint is not satisfied, +** or which row it is not satisfied for. +** +** * If the database contains foreign key violations when the +** transaction is opened, this may cause the mechanism to malfunction. +** +** Despite these problems, this approach is adopted as it seems simpler +** than the alternatives. +** +** INSERT operations: +** +** I.1) For each FK for which the table is the child table, search +** the parent table for a match. If none is found increment the +** constraint counter. +** +** I.2) For each FK for which the table is the parent table, +** search the child table for rows that correspond to the new +** row in the parent table. Decrement the counter for each row +** found (as the constraint is now satisfied). +** +** DELETE operations: +** +** D.1) For each FK for which the table is the child table, +** search the parent table for a row that corresponds to the +** deleted row in the child table. If such a row is not found, +** decrement the counter. +** +** D.2) For each FK for which the table is the parent table, search +** the child table for rows that correspond to the deleted row +** in the parent table. For each found increment the counter. +** +** UPDATE operations: +** +** An UPDATE command requires that all 4 steps above are taken, but only +** for FK constraints for which the affected columns are actually +** modified (values must be compared at runtime). +** +** Note that I.1 and D.1 are very similar operations, as are I.2 and D.2. +** This simplifies the implementation a bit. +** +** For the purposes of immediate FK constraints, the OR REPLACE conflict +** resolution is considered to delete rows before the new row is inserted. +** If a delete caused by OR REPLACE violates an FK constraint, an exception +** is thrown, even if the FK constraint would be satisfied after the new +** row is inserted. +** +** Immediate constraints are usually handled similarly. The only difference +** is that the counter used is stored as part of each individual statement +** object (struct Vdbe). If, after the statement has run, its immediate +** constraint counter is greater than zero, +** it returns SQLITE_CONSTRAINT_FOREIGNKEY +** and the statement transaction is rolled back. An exception is an INSERT +** statement that inserts a single row only (no triggers). In this case, +** instead of using a counter, an exception is thrown immediately if the +** INSERT violates a foreign key constraint. This is necessary as such +** an INSERT does not open a statement transaction. +** +** TODO: How should dropping a table be handled? How should renaming a +** table be handled? +** +** +** Query API Notes +** --------------- +** +** Before coding an UPDATE or DELETE row operation, the code-generator +** for those two operations needs to know whether or not the operation +** requires any FK processing and, if so, which columns of the original +** row are required by the FK processing VDBE code (i.e. if FKs were +** implemented using triggers, which of the old.* columns would be +** accessed). No information is required by the code-generator before +** coding an INSERT operation. The functions used by the UPDATE/DELETE +** generation code to query for this information are: +** +** sqlite3FkRequired() - Test to see if FK processing is required. +** sqlite3FkOldmask() - Query for the set of required old.* columns. +** +** +** Externally accessible module functions +** -------------------------------------- +** +** sqlite3FkCheck() - Check for foreign key violations. +** sqlite3FkActions() - Code triggers for ON UPDATE/ON DELETE actions. +** sqlite3FkDelete() - Delete an FKey structure. +*/ + +/* +** VDBE Calling Convention +** ----------------------- +** +** Example: +** +** For the following INSERT statement: +** +** CREATE TABLE t1(a, b INTEGER PRIMARY KEY, c); +** INSERT INTO t1 VALUES(1, 2, 3.1); +** +** Register (x): 2 (type integer) +** Register (x+1): 1 (type integer) +** Register (x+2): NULL (type NULL) +** Register (x+3): 3.1 (type real) +*/ + +/* +** A foreign key constraint requires that the key columns in the parent +** table are collectively subject to a UNIQUE or PRIMARY KEY constraint. +** Given that pParent is the parent table for foreign key constraint pFKey, +** search the schema for a unique index on the parent key columns. +** +** If successful, zero is returned. If the parent key is an INTEGER PRIMARY +** KEY column, then output variable *ppIdx is set to NULL. Otherwise, *ppIdx +** is set to point to the unique index. +** +** If the parent key consists of a single column (the foreign key constraint +** is not a composite foreign key), output variable *paiCol is set to NULL. +** Otherwise, it is set to point to an allocated array of size N, where +** N is the number of columns in the parent key. The first element of the +** array is the index of the child table column that is mapped by the FK +** constraint to the parent table column stored in the left-most column +** of index *ppIdx. The second element of the array is the index of the +** child table column that corresponds to the second left-most column of +** *ppIdx, and so on. +** +** If the required index cannot be found, either because: +** +** 1) The named parent key columns do not exist, or +** +** 2) The named parent key columns do exist, but are not subject to a +** UNIQUE or PRIMARY KEY constraint, or +** +** 3) No parent key columns were provided explicitly as part of the +** foreign key definition, and the parent table does not have a +** PRIMARY KEY, or +** +** 4) No parent key columns were provided explicitly as part of the +** foreign key definition, and the PRIMARY KEY of the parent table +** consists of a different number of columns to the child key in +** the child table. +** +** then non-zero is returned, and a "foreign key mismatch" error loaded +** into pParse. If an OOM error occurs, non-zero is returned and the +** pParse->db->mallocFailed flag is set. +*/ +SQLITE_PRIVATE int sqlite3FkLocateIndex( + Parse *pParse, /* Parse context to store any error in */ + Table *pParent, /* Parent table of FK constraint pFKey */ + FKey *pFKey, /* Foreign key to find index for */ + Index **ppIdx, /* OUT: Unique index on parent table */ + int **paiCol /* OUT: Map of index columns in pFKey */ +){ + Index *pIdx = 0; /* Value to return via *ppIdx */ + int *aiCol = 0; /* Value to return via *paiCol */ + int nCol = pFKey->nCol; /* Number of columns in parent key */ + char *zKey = pFKey->aCol[0].zCol; /* Name of left-most parent key column */ + + /* The caller is responsible for zeroing output parameters. */ + assert( ppIdx && *ppIdx==0 ); + assert( !paiCol || *paiCol==0 ); + assert( pParse ); + + /* If this is a non-composite (single column) foreign key, check if it + ** maps to the INTEGER PRIMARY KEY of table pParent. If so, leave *ppIdx + ** and *paiCol set to zero and return early. + ** + ** Otherwise, for a composite foreign key (more than one column), allocate + ** space for the aiCol array (returned via output parameter *paiCol). + ** Non-composite foreign keys do not require the aiCol array. + */ + if( nCol==1 ){ + /* The FK maps to the IPK if any of the following are true: + ** + ** 1) There is an INTEGER PRIMARY KEY column and the FK is implicitly + ** mapped to the primary key of table pParent, or + ** 2) The FK is explicitly mapped to a column declared as INTEGER + ** PRIMARY KEY. + */ + if( pParent->iPKey>=0 ){ + if( !zKey ) return 0; + if( !sqlite3StrICmp(pParent->aCol[pParent->iPKey].zCnName, zKey) ){ + return 0; + } + } + }else if( paiCol ){ + assert( nCol>1 ); + aiCol = (int *)sqlite3DbMallocRawNN(pParse->db, nCol*sizeof(int)); + if( !aiCol ) return 1; + *paiCol = aiCol; + } + + for(pIdx=pParent->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->nKeyCol==nCol && IsUniqueIndex(pIdx) && pIdx->pPartIdxWhere==0 ){ + /* pIdx is a UNIQUE index (or a PRIMARY KEY) and has the right number + ** of columns. If each indexed column corresponds to a foreign key + ** column of pFKey, then this index is a winner. */ + + if( zKey==0 ){ + /* If zKey is NULL, then this foreign key is implicitly mapped to + ** the PRIMARY KEY of table pParent. The PRIMARY KEY index may be + ** identified by the test. */ + if( IsPrimaryKeyIndex(pIdx) ){ + if( aiCol ){ + int i; + for(i=0; iaCol[i].iFrom; + } + break; + } + }else{ + /* If zKey is non-NULL, then this foreign key was declared to + ** map to an explicit list of columns in table pParent. Check if this + ** index matches those columns. Also, check that the index uses + ** the default collation sequences for each column. */ + int i, j; + for(i=0; iaiColumn[i]; /* Index of column in parent tbl */ + const char *zDfltColl; /* Def. collation for column */ + char *zIdxCol; /* Name of indexed column */ + + if( iCol<0 ) break; /* No foreign keys against expression indexes */ + + /* If the index uses a collation sequence that is different from + ** the default collation sequence for the column, this index is + ** unusable. Bail out early in this case. */ + zDfltColl = sqlite3ColumnColl(&pParent->aCol[iCol]); + if( !zDfltColl ) zDfltColl = sqlite3StrBINARY; + if( sqlite3StrICmp(pIdx->azColl[i], zDfltColl) ) break; + + zIdxCol = pParent->aCol[iCol].zCnName; + for(j=0; jaCol[j].zCol, zIdxCol)==0 ){ + if( aiCol ) aiCol[i] = pFKey->aCol[j].iFrom; + break; + } + } + if( j==nCol ) break; + } + if( i==nCol ) break; /* pIdx is usable */ + } + } + } + + if( !pIdx ){ + if( !pParse->disableTriggers ){ + sqlite3ErrorMsg(pParse, + "foreign key mismatch - \"%w\" referencing \"%w\"", + pFKey->pFrom->zName, pFKey->zTo); + } + sqlite3DbFree(pParse->db, aiCol); + return 1; + } + + *ppIdx = pIdx; + return 0; +} + +/* +** This function is called when a row is inserted into or deleted from the +** child table of foreign key constraint pFKey. If an SQL UPDATE is executed +** on the child table of pFKey, this function is invoked twice for each row +** affected - once to "delete" the old row, and then again to "insert" the +** new row. +** +** Each time it is called, this function generates VDBE code to locate the +** row in the parent table that corresponds to the row being inserted into +** or deleted from the child table. If the parent row can be found, no +** special action is taken. Otherwise, if the parent row can *not* be +** found in the parent table: +** +** Operation | FK type | Action taken +** -------------------------------------------------------------------------- +** INSERT immediate Increment the "immediate constraint counter". +** +** DELETE immediate Decrement the "immediate constraint counter". +** +** INSERT deferred Increment the "deferred constraint counter". +** +** DELETE deferred Decrement the "deferred constraint counter". +** +** These operations are identified in the comment at the top of this file +** (fkey.c) as "I.1" and "D.1". +*/ +static void fkLookupParent( + Parse *pParse, /* Parse context */ + int iDb, /* Index of database housing pTab */ + Table *pTab, /* Parent table of FK pFKey */ + Index *pIdx, /* Unique index on parent key columns in pTab */ + FKey *pFKey, /* Foreign key constraint */ + int *aiCol, /* Map from parent key columns to child table columns */ + int regData, /* Address of array containing child table row */ + int nIncr, /* Increment constraint counter by this */ + int isIgnore /* If true, pretend pTab contains all NULL values */ +){ + int i; /* Iterator variable */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Vdbe to add code to */ + int iCur = pParse->nTab - 1; /* Cursor number to use */ + int iOk = sqlite3VdbeMakeLabel(pParse); /* jump here if parent key found */ + + sqlite3VdbeVerifyAbortable(v, + (!pFKey->isDeferred + && !(pParse->db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel + && !pParse->isMultiWrite) ? OE_Abort : OE_Ignore); + + /* If nIncr is less than zero, then check at runtime if there are any + ** outstanding constraints to resolve. If there are not, there is no need + ** to check if deleting this row resolves any outstanding violations. + ** + ** Check if any of the key columns in the child table row are NULL. If + ** any are, then the constraint is considered satisfied. No need to + ** search for a matching row in the parent table. */ + if( nIncr<0 ){ + sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, iOk); + VdbeCoverage(v); + } + for(i=0; inCol; i++){ + int iReg = sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[i]) + regData + 1; + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); VdbeCoverage(v); + } + + if( isIgnore==0 ){ + if( pIdx==0 ){ + /* If pIdx is NULL, then the parent key is the INTEGER PRIMARY KEY + ** column of the parent table (table pTab). */ + int iMustBeInt; /* Address of MustBeInt instruction */ + int regTemp = sqlite3GetTempReg(pParse); + + /* Invoke MustBeInt to coerce the child key value to an integer (i.e. + ** apply the affinity of the parent key). If this fails, then there + ** is no matching parent key. Before using MustBeInt, make a copy of + ** the value. Otherwise, the value inserted into the child key column + ** will have INTEGER affinity applied to it, which may not be correct. */ + sqlite3VdbeAddOp2(v, OP_SCopy, + sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[0])+1+regData, regTemp); + iMustBeInt = sqlite3VdbeAddOp2(v, OP_MustBeInt, regTemp, 0); + VdbeCoverage(v); + + /* If the parent table is the same as the child table, and we are about + ** to increment the constraint-counter (i.e. this is an INSERT operation), + ** then check if the row being inserted matches itself. If so, do not + ** increment the constraint-counter. */ + if( pTab==pFKey->pFrom && nIncr==1 ){ + sqlite3VdbeAddOp3(v, OP_Eq, regData, iOk, regTemp); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + } + + sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); VdbeCoverage(v); + sqlite3VdbeGoto(v, iOk); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + sqlite3VdbeJumpHere(v, iMustBeInt); + sqlite3ReleaseTempReg(pParse, regTemp); + }else{ + int nCol = pFKey->nCol; + int regTemp = sqlite3GetTempRange(pParse, nCol); + int regRec = sqlite3GetTempReg(pParse); + + sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + for(i=0; ipFrom, aiCol[i])+1+regData, + regTemp+i); + } + + /* If the parent table is the same as the child table, and we are about + ** to increment the constraint-counter (i.e. this is an INSERT operation), + ** then check if the row being inserted matches itself. If so, do not + ** increment the constraint-counter. + ** + ** If any of the parent-key values are NULL, then the row cannot match + ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any + ** of the parent-key values are NULL (at this point it is known that + ** none of the child key values are). + */ + if( pTab==pFKey->pFrom && nIncr==1 ){ + int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; + for(i=0; ipFrom,aiCol[i]) + +1+regData; + int iParent = 1+regData; + iParent += sqlite3TableColumnToStorage(pIdx->pTable, + pIdx->aiColumn[i]); + assert( pIdx->aiColumn[i]>=0 ); + assert( aiCol[i]!=pTab->iPKey ); + if( pIdx->aiColumn[i]==pTab->iPKey ){ + /* The parent key is a composite key that includes the IPK column */ + iParent = regData; + } + sqlite3VdbeAddOp3(v, OP_Ne, iChild, iJump, iParent); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL); + } + sqlite3VdbeGoto(v, iOk); + } + + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, + sqlite3IndexAffinityStr(pParse->db,pIdx), nCol); + sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); + + sqlite3ReleaseTempReg(pParse, regRec); + sqlite3ReleaseTempRange(pParse, regTemp, nCol); + } + } + + if( !pFKey->isDeferred && !(pParse->db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel + && !pParse->isMultiWrite + ){ + /* Special case: If this is an INSERT statement that will insert exactly + ** one row into the table, raise a constraint immediately instead of + ** incrementing a counter. This is necessary as the VM code is being + ** generated for will not open a statement transaction. */ + assert( nIncr==1 ); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, 0, P4_STATIC, P5_ConstraintFK); + }else{ + if( nIncr>0 && pFKey->isDeferred==0 ){ + sqlite3MayAbort(pParse); + } + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); + } + + sqlite3VdbeResolveLabel(v, iOk); + sqlite3VdbeAddOp1(v, OP_Close, iCur); +} + + +/* +** Return an Expr object that refers to a memory register corresponding +** to column iCol of table pTab. +** +** regBase is the first of an array of register that contains the data +** for pTab. regBase itself holds the rowid. regBase+1 holds the first +** column. regBase+2 holds the second column, and so forth. +*/ +static Expr *exprTableRegister( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* The table whose content is at r[regBase]... */ + int regBase, /* Contents of table pTab */ + i16 iCol /* Which column of pTab is desired */ +){ + Expr *pExpr; + Column *pCol; + const char *zColl; + sqlite3 *db = pParse->db; + + pExpr = sqlite3Expr(db, TK_REGISTER, 0); + if( pExpr ){ + if( iCol>=0 && iCol!=pTab->iPKey ){ + pCol = &pTab->aCol[iCol]; + pExpr->iTable = regBase + sqlite3TableColumnToStorage(pTab,iCol) + 1; + pExpr->affExpr = pCol->affinity; + zColl = sqlite3ColumnColl(pCol); + if( zColl==0 ) zColl = db->pDfltColl->zName; + pExpr = sqlite3ExprAddCollateString(pParse, pExpr, zColl); + }else{ + pExpr->iTable = regBase; + pExpr->affExpr = SQLITE_AFF_INTEGER; + } + } + return pExpr; +} + +/* +** Return an Expr object that refers to column iCol of table pTab which +** has cursor iCur. +*/ +static Expr *exprTableColumn( + sqlite3 *db, /* The database connection */ + Table *pTab, /* The table whose column is desired */ + int iCursor, /* The open cursor on the table */ + i16 iCol /* The column that is wanted */ +){ + Expr *pExpr = sqlite3Expr(db, TK_COLUMN, 0); + if( pExpr ){ + pExpr->y.pTab = pTab; + pExpr->iTable = iCursor; + pExpr->iColumn = iCol; + } + return pExpr; +} + +/* +** This function is called to generate code executed when a row is deleted +** from the parent table of foreign key constraint pFKey and, if pFKey is +** deferred, when a row is inserted into the same table. When generating +** code for an SQL UPDATE operation, this function may be called twice - +** once to "delete" the old row and once to "insert" the new row. +** +** Parameter nIncr is passed -1 when inserting a row (as this may decrease +** the number of FK violations in the db) or +1 when deleting one (as this +** may increase the number of FK constraint problems). +** +** The code generated by this function scans through the rows in the child +** table that correspond to the parent table row being deleted or inserted. +** For each child row found, one of the following actions is taken: +** +** Operation | FK type | Action taken +** -------------------------------------------------------------------------- +** DELETE immediate Increment the "immediate constraint counter". +** Or, if the ON (UPDATE|DELETE) action is RESTRICT, +** throw a "FOREIGN KEY constraint failed" exception. +** +** INSERT immediate Decrement the "immediate constraint counter". +** +** DELETE deferred Increment the "deferred constraint counter". +** Or, if the ON (UPDATE|DELETE) action is RESTRICT, +** throw a "FOREIGN KEY constraint failed" exception. +** +** INSERT deferred Decrement the "deferred constraint counter". +** +** These operations are identified in the comment at the top of this file +** (fkey.c) as "I.2" and "D.2". +*/ +static void fkScanChildren( + Parse *pParse, /* Parse context */ + SrcList *pSrc, /* The child table to be scanned */ + Table *pTab, /* The parent table */ + Index *pIdx, /* Index on parent covering the foreign key */ + FKey *pFKey, /* The foreign key linking pSrc to pTab */ + int *aiCol, /* Map from pIdx cols to child table cols */ + int regData, /* Parent row data starts here */ + int nIncr /* Amount to increment deferred counter by */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + int i; /* Iterator variable */ + Expr *pWhere = 0; /* WHERE clause to scan with */ + NameContext sNameContext; /* Context used to resolve WHERE clause */ + WhereInfo *pWInfo; /* Context used by sqlite3WhereXXX() */ + int iFkIfZero = 0; /* Address of OP_FkIfZero */ + Vdbe *v = sqlite3GetVdbe(pParse); + + assert( pIdx==0 || pIdx->pTable==pTab ); + assert( pIdx==0 || pIdx->nKeyCol==pFKey->nCol ); + assert( pIdx!=0 || pFKey->nCol==1 ); + assert( pIdx!=0 || HasRowid(pTab) ); + + if( nIncr<0 ){ + iFkIfZero = sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, 0); + VdbeCoverage(v); + } + + /* Create an Expr object representing an SQL expression like: + ** + ** = AND = ... + ** + ** The collation sequence used for the comparison should be that of + ** the parent key columns. The affinity of the parent key column should + ** be applied to each child key value before the comparison takes place. + */ + for(i=0; inCol; i++){ + Expr *pLeft; /* Value from parent table row */ + Expr *pRight; /* Column ref to child table */ + Expr *pEq; /* Expression (pLeft = pRight) */ + i16 iCol; /* Index of column in child table */ + const char *zCol; /* Name of column in child table */ + + iCol = pIdx ? pIdx->aiColumn[i] : -1; + pLeft = exprTableRegister(pParse, pTab, regData, iCol); + iCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; + assert( iCol>=0 ); + zCol = pFKey->pFrom->aCol[iCol].zCnName; + pRight = sqlite3Expr(db, TK_ID, zCol); + pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight); + pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); + } + + /* If the child table is the same as the parent table, then add terms + ** to the WHERE clause that prevent this entry from being scanned. + ** The added WHERE clause terms are like this: + ** + ** $current_rowid!=rowid + ** NOT( $current_a==a AND $current_b==b AND ... ) + ** + ** The first form is used for rowid tables. The second form is used + ** for WITHOUT ROWID tables. In the second form, the *parent* key is + ** (a,b,...). Either the parent or primary key could be used to + ** uniquely identify the current row, but the parent key is more convenient + ** as the required values have already been loaded into registers + ** by the caller. + */ + if( pTab==pFKey->pFrom && nIncr>0 ){ + Expr *pNe; /* Expression (pLeft != pRight) */ + Expr *pLeft; /* Value from parent table row */ + Expr *pRight; /* Column ref to child table */ + if( HasRowid(pTab) ){ + pLeft = exprTableRegister(pParse, pTab, regData, -1); + pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, -1); + pNe = sqlite3PExpr(pParse, TK_NE, pLeft, pRight); + }else{ + Expr *pEq, *pAll = 0; + assert( pIdx!=0 ); + for(i=0; inKeyCol; i++){ + i16 iCol = pIdx->aiColumn[i]; + assert( iCol>=0 ); + pLeft = exprTableRegister(pParse, pTab, regData, iCol); + pRight = sqlite3Expr(db, TK_ID, pTab->aCol[iCol].zCnName); + pEq = sqlite3PExpr(pParse, TK_IS, pLeft, pRight); + pAll = sqlite3ExprAnd(pParse, pAll, pEq); + } + pNe = sqlite3PExpr(pParse, TK_NOT, pAll, 0); + } + pWhere = sqlite3ExprAnd(pParse, pWhere, pNe); + } + + /* Resolve the references in the WHERE clause. */ + memset(&sNameContext, 0, sizeof(NameContext)); + sNameContext.pSrcList = pSrc; + sNameContext.pParse = pParse; + sqlite3ResolveExprNames(&sNameContext, pWhere); + + /* Create VDBE to loop through the entries in pSrc that match the WHERE + ** clause. For each row found, increment either the deferred or immediate + ** foreign key constraint counter. */ + if( pParse->nErr==0 ){ + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); + if( pWInfo ){ + sqlite3WhereEnd(pWInfo); + } + } + + /* Clean up the WHERE clause constructed above. */ + sqlite3ExprDelete(db, pWhere); + if( iFkIfZero ){ + sqlite3VdbeJumpHereOrPopInst(v, iFkIfZero); + } +} + +/* +** This function returns a linked list of FKey objects (connected by +** FKey.pNextTo) holding all children of table pTab. For example, +** given the following schema: +** +** CREATE TABLE t1(a PRIMARY KEY); +** CREATE TABLE t2(b REFERENCES t1(a); +** +** Calling this function with table "t1" as an argument returns a pointer +** to the FKey structure representing the foreign key constraint on table +** "t2". Calling this function with "t2" as the argument would return a +** NULL pointer (as there are no FK constraints for which t2 is the parent +** table). +*/ +SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *pTab){ + return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName); +} + +/* +** The second argument is a Trigger structure allocated by the +** fkActionTrigger() routine. This function deletes the Trigger structure +** and all of its sub-components. +** +** The Trigger structure or any of its sub-components may be allocated from +** the lookaside buffer belonging to database handle dbMem. +*/ +static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ + if( p ){ + TriggerStep *pStep = p->step_list; + sqlite3ExprDelete(dbMem, pStep->pWhere); + sqlite3ExprListDelete(dbMem, pStep->pExprList); + sqlite3SelectDelete(dbMem, pStep->pSelect); + sqlite3ExprDelete(dbMem, p->pWhen); + sqlite3DbFree(dbMem, p); + } +} + +/* +** This function is called to generate code that runs when table pTab is +** being dropped from the database. The SrcList passed as the second argument +** to this function contains a single entry guaranteed to resolve to +** table pTab. +** +** Normally, no code is required. However, if either +** +** (a) The table is the parent table of a FK constraint, or +** (b) The table is the child table of a deferred FK constraint and it is +** determined at runtime that there are outstanding deferred FK +** constraint violations in the database, +** +** then the equivalent of "DELETE FROM " is executed before dropping +** the table from the database. Triggers are disabled while running this +** DELETE, but foreign key actions are not. +*/ +SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ + sqlite3 *db = pParse->db; + if( (db->flags&SQLITE_ForeignKeys) && !IsVirtual(pTab) ){ + int iSkip = 0; + Vdbe *v = sqlite3GetVdbe(pParse); + + assert( v ); /* VDBE has already been allocated */ + assert( !IsView(pTab) ); /* Not a view */ + assert( !IsVirtual(pTab) ); + if( sqlite3FkReferences(pTab)==0 ){ + /* Search for a deferred foreign key constraint for which this table + ** is the child table. If one cannot be found, return without + ** generating any VDBE code. If one can be found, then jump over + ** the entire DELETE if there are no outstanding deferred constraints + ** when this statement is run. */ + FKey *p; + for(p=pTab->u.tab.pFKey; p; p=p->pNextFrom){ + if( p->isDeferred || (db->flags & SQLITE_DeferFKs) ) break; + } + if( !p ) return; + iSkip = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeAddOp2(v, OP_FkIfZero, 1, iSkip); VdbeCoverage(v); + } + + pParse->disableTriggers = 1; + sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0, 0, 0); + pParse->disableTriggers = 0; + + /* If the DELETE has generated immediate foreign key constraint + ** violations, halt the VDBE and return an error at this point, before + ** any modifications to the schema are made. This is because statement + ** transactions are not able to rollback schema changes. + ** + ** If the SQLITE_DeferFKs flag is set, then this is not required, as + ** the statement transaction will not be rolled back even if FK + ** constraints are violated. + */ + if( (db->flags & SQLITE_DeferFKs)==0 ){ + sqlite3VdbeVerifyAbortable(v, OE_Abort); + sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, + OE_Abort, 0, P4_STATIC, P5_ConstraintFK); + } + + if( iSkip ){ + sqlite3VdbeResolveLabel(v, iSkip); + } + } +} + + +/* +** The second argument points to an FKey object representing a foreign key +** for which pTab is the child table. An UPDATE statement against pTab +** is currently being processed. For each column of the table that is +** actually updated, the corresponding element in the aChange[] array +** is zero or greater (if a column is unmodified the corresponding element +** is set to -1). If the rowid column is modified by the UPDATE statement +** the bChngRowid argument is non-zero. +** +** This function returns true if any of the columns that are part of the +** child key for FK constraint *p are modified. +*/ +static int fkChildIsModified( + Table *pTab, /* Table being updated */ + FKey *p, /* Foreign key for which pTab is the child */ + int *aChange, /* Array indicating modified columns */ + int bChngRowid /* True if rowid is modified by this update */ +){ + int i; + for(i=0; inCol; i++){ + int iChildKey = p->aCol[i].iFrom; + if( aChange[iChildKey]>=0 ) return 1; + if( iChildKey==pTab->iPKey && bChngRowid ) return 1; + } + return 0; +} + +/* +** The second argument points to an FKey object representing a foreign key +** for which pTab is the parent table. An UPDATE statement against pTab +** is currently being processed. For each column of the table that is +** actually updated, the corresponding element in the aChange[] array +** is zero or greater (if a column is unmodified the corresponding element +** is set to -1). If the rowid column is modified by the UPDATE statement +** the bChngRowid argument is non-zero. +** +** This function returns true if any of the columns that are part of the +** parent key for FK constraint *p are modified. +*/ +static int fkParentIsModified( + Table *pTab, + FKey *p, + int *aChange, + int bChngRowid +){ + int i; + for(i=0; inCol; i++){ + char *zKey = p->aCol[i].zCol; + int iKey; + for(iKey=0; iKeynCol; iKey++){ + if( aChange[iKey]>=0 || (iKey==pTab->iPKey && bChngRowid) ){ + Column *pCol = &pTab->aCol[iKey]; + if( zKey ){ + if( 0==sqlite3StrICmp(pCol->zCnName, zKey) ) return 1; + }else if( pCol->colFlags & COLFLAG_PRIMKEY ){ + return 1; + } + } + } + } + return 0; +} + +/* +** Return true if the parser passed as the first argument is being +** used to code a trigger that is really a "SET NULL" action belonging +** to trigger pFKey. +*/ +static int isSetNullAction(Parse *pParse, FKey *pFKey){ + Parse *pTop = sqlite3ParseToplevel(pParse); + if( pTop->pTriggerPrg ){ + Trigger *p = pTop->pTriggerPrg->pTrigger; + if( (p==pFKey->apTrigger[0] && pFKey->aAction[0]==OE_SetNull) + || (p==pFKey->apTrigger[1] && pFKey->aAction[1]==OE_SetNull) + ){ + return 1; + } + } + return 0; +} + +/* +** This function is called when inserting, deleting or updating a row of +** table pTab to generate VDBE code to perform foreign key constraint +** processing for the operation. +** +** For a DELETE operation, parameter regOld is passed the index of the +** first register in an array of (pTab->nCol+1) registers containing the +** rowid of the row being deleted, followed by each of the column values +** of the row being deleted, from left to right. Parameter regNew is passed +** zero in this case. +** +** For an INSERT operation, regOld is passed zero and regNew is passed the +** first register of an array of (pTab->nCol+1) registers containing the new +** row data. +** +** For an UPDATE operation, this function is called twice. Once before +** the original record is deleted from the table using the calling convention +** described for DELETE. Then again after the original record is deleted +** but before the new record is inserted using the INSERT convention. +*/ +SQLITE_PRIVATE void sqlite3FkCheck( + Parse *pParse, /* Parse context */ + Table *pTab, /* Row is being deleted from this table */ + int regOld, /* Previous row data is stored here */ + int regNew, /* New row data is stored here */ + int *aChange, /* Array indicating UPDATEd columns (or 0) */ + int bChngRowid /* True if rowid is UPDATEd */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + FKey *pFKey; /* Used to iterate through FKs */ + int iDb; /* Index of database containing pTab */ + const char *zDb; /* Name of database containing pTab */ + int isIgnoreErrors = pParse->disableTriggers; + + /* Exactly one of regOld and regNew should be non-zero. */ + assert( (regOld==0)!=(regNew==0) ); + + /* If foreign-keys are disabled, this function is a no-op. */ + if( (db->flags&SQLITE_ForeignKeys)==0 ) return; + + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zDb = db->aDb[iDb].zDbSName; + + /* Loop through all the foreign key constraints for which pTab is the + ** child table (the table that the foreign key definition is part of). */ + assert( !IsVirtual(pTab) ); + for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pFKey->pNextFrom){ + Table *pTo; /* Parent table of foreign key pFKey */ + Index *pIdx = 0; /* Index on key columns in pTo */ + int *aiFree = 0; + int *aiCol; + int iCol; + int i; + int bIgnore = 0; + + if( aChange + && sqlite3_stricmp(pTab->zName, pFKey->zTo)!=0 + && fkChildIsModified(pTab, pFKey, aChange, bChngRowid)==0 + ){ + continue; + } + + /* Find the parent table of this foreign key. Also find a unique index + ** on the parent key columns in the parent table. If either of these + ** schema items cannot be located, set an error in pParse and return + ** early. */ + if( pParse->disableTriggers ){ + pTo = sqlite3FindTable(db, pFKey->zTo, zDb); + }else{ + pTo = sqlite3LocateTable(pParse, 0, pFKey->zTo, zDb); + } + if( !pTo || sqlite3FkLocateIndex(pParse, pTo, pFKey, &pIdx, &aiFree) ){ + assert( isIgnoreErrors==0 || (regOld!=0 && regNew==0) ); + if( !isIgnoreErrors || db->mallocFailed ) return; + if( pTo==0 ){ + /* If isIgnoreErrors is true, then a table is being dropped. In this + ** case SQLite runs a "DELETE FROM xxx" on the table being dropped + ** before actually dropping it in order to check FK constraints. + ** If the parent table of an FK constraint on the current table is + ** missing, behave as if it is empty. i.e. decrement the relevant + ** FK counter for each row of the current table with non-NULL keys. + */ + Vdbe *v = sqlite3GetVdbe(pParse); + int iJump = sqlite3VdbeCurrentAddr(v) + pFKey->nCol + 1; + for(i=0; inCol; i++){ + int iFromCol, iReg; + iFromCol = pFKey->aCol[i].iFrom; + iReg = sqlite3TableColumnToStorage(pFKey->pFrom,iFromCol) + regOld+1; + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iJump); VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, -1); + } + continue; + } + assert( pFKey->nCol==1 || (aiFree && pIdx) ); + + if( aiFree ){ + aiCol = aiFree; + }else{ + iCol = pFKey->aCol[0].iFrom; + aiCol = &iCol; + } + for(i=0; inCol; i++){ + if( aiCol[i]==pTab->iPKey ){ + aiCol[i] = -1; + } + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Request permission to read the parent key columns. If the + ** authorization callback returns SQLITE_IGNORE, behave as if any + ** values read from the parent table are NULL. */ + if( db->xAuth ){ + int rcauth; + char *zCol = pTo->aCol[pIdx ? pIdx->aiColumn[i] : pTo->iPKey].zCnName; + rcauth = sqlite3AuthReadCol(pParse, pTo->zName, zCol, iDb); + bIgnore = (rcauth==SQLITE_IGNORE); + } +#endif + } + + /* Take a shared-cache advisory read-lock on the parent table. Allocate + ** a cursor to use to search the unique index on the parent key columns + ** in the parent table. */ + sqlite3TableLock(pParse, iDb, pTo->tnum, 0, pTo->zName); + pParse->nTab++; + + if( regOld!=0 ){ + /* A row is being removed from the child table. Search for the parent. + ** If the parent does not exist, removing the child row resolves an + ** outstanding foreign key constraint violation. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1, bIgnore); + } + if( regNew!=0 && !isSetNullAction(pParse, pFKey) ){ + /* A row is being added to the child table. If a parent row cannot + ** be found, adding the child row has violated the FK constraint. + ** + ** If this operation is being performed as part of a trigger program + ** that is actually a "SET NULL" action belonging to this very + ** foreign key, then omit this scan altogether. As all child key + ** values are guaranteed to be NULL, it is not possible for adding + ** this row to cause an FK violation. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1, bIgnore); + } + + sqlite3DbFree(db, aiFree); + } + + /* Loop through all the foreign key constraints that refer to this table. + ** (the "child" constraints) */ + for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ + Index *pIdx = 0; /* Foreign key index for pFKey */ + SrcList *pSrc; + int *aiCol = 0; + + if( aChange && fkParentIsModified(pTab, pFKey, aChange, bChngRowid)==0 ){ + continue; + } + + if( !pFKey->isDeferred && !(db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel && !pParse->isMultiWrite + ){ + assert( regOld==0 && regNew!=0 ); + /* Inserting a single row into a parent table cannot cause (or fix) + ** an immediate foreign key violation. So do nothing in this case. */ + continue; + } + + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ){ + if( !isIgnoreErrors || db->mallocFailed ) return; + continue; + } + assert( aiCol || pFKey->nCol==1 ); + + /* Create a SrcList structure containing the child table. We need the + ** child table as a SrcList for sqlite3WhereBegin() */ + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pSrc ){ + SrcItem *pItem = pSrc->a; + pItem->pTab = pFKey->pFrom; + pItem->zName = pFKey->pFrom->zName; + pItem->pTab->nTabRef++; + pItem->iCursor = pParse->nTab++; + + if( regNew!=0 ){ + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regNew, -1); + } + if( regOld!=0 ){ + int eAction = pFKey->aAction[aChange!=0]; + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); + /* If this is a deferred FK constraint, or a CASCADE or SET NULL + ** action applies, then any foreign key violations caused by + ** removing the parent key will be rectified by the action trigger. + ** So do not set the "may-abort" flag in this case. + ** + ** Note 1: If the FK is declared "ON UPDATE CASCADE", then the + ** may-abort flag will eventually be set on this statement anyway + ** (when this function is called as part of processing the UPDATE + ** within the action trigger). + ** + ** Note 2: At first glance it may seem like SQLite could simply omit + ** all OP_FkCounter related scans when either CASCADE or SET NULL + ** applies. The trouble starts if the CASCADE or SET NULL action + ** trigger causes other triggers or action rules attached to the + ** child table to fire. In these cases the fk constraint counters + ** might be set incorrectly if any OP_FkCounter related scans are + ** omitted. */ + if( !pFKey->isDeferred && eAction!=OE_Cascade && eAction!=OE_SetNull ){ + sqlite3MayAbort(pParse); + } + } + pItem->zName = 0; + sqlite3SrcListDelete(db, pSrc); + } + sqlite3DbFree(db, aiCol); + } +} + +#define COLUMN_MASK(x) (((x)>31) ? 0xffffffff : ((u32)1<<(x))) + +/* +** This function is called before generating code to update or delete a +** row contained in table pTab. +*/ +SQLITE_PRIVATE u32 sqlite3FkOldmask( + Parse *pParse, /* Parse context */ + Table *pTab /* Table being modified */ +){ + u32 mask = 0; + if( pParse->db->flags&SQLITE_ForeignKeys ){ + FKey *p; + int i; + assert( !IsVirtual(pTab) ); + for(p=pTab->u.tab.pFKey; p; p=p->pNextFrom){ + for(i=0; inCol; i++) mask |= COLUMN_MASK(p->aCol[i].iFrom); + } + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + Index *pIdx = 0; + sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); + if( pIdx ){ + for(i=0; inKeyCol; i++){ + assert( pIdx->aiColumn[i]>=0 ); + mask |= COLUMN_MASK(pIdx->aiColumn[i]); + } + } + } + } + return mask; +} + + +/* +** This function is called before generating code to update or delete a +** row contained in table pTab. If the operation is a DELETE, then +** parameter aChange is passed a NULL value. For an UPDATE, aChange points +** to an array of size N, where N is the number of columns in table pTab. +** If the i'th column is not modified by the UPDATE, then the corresponding +** entry in the aChange[] array is set to -1. If the column is modified, +** the value is 0 or greater. Parameter chngRowid is set to true if the +** UPDATE statement modifies the rowid fields of the table. +** +** If any foreign key processing will be required, this function returns +** non-zero. If there is no foreign key related processing, this function +** returns zero. +** +** For an UPDATE, this function returns 2 if: +** +** * There are any FKs for which pTab is the child and the parent table +** and any FK processing at all is required (even of a different FK), or +** +** * the UPDATE modifies one or more parent keys for which the action is +** not "NO ACTION" (i.e. is CASCADE, SET DEFAULT or SET NULL). +** +** Or, assuming some other foreign key processing is required, 1. +*/ +SQLITE_PRIVATE int sqlite3FkRequired( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being modified */ + int *aChange, /* Non-NULL for UPDATE operations */ + int chngRowid /* True for UPDATE that affects rowid */ +){ + int eRet = 1; /* Value to return if bHaveFK is true */ + int bHaveFK = 0; /* If FK processing is required */ + if( pParse->db->flags&SQLITE_ForeignKeys && !IsVirtual(pTab) ){ + if( !aChange ){ + /* A DELETE operation. Foreign key processing is required if the + ** table in question is either the child or parent table for any + ** foreign key constraint. */ + bHaveFK = (sqlite3FkReferences(pTab) || pTab->u.tab.pFKey); + }else{ + /* This is an UPDATE. Foreign key processing is only required if the + ** operation modifies one or more child or parent key columns. */ + FKey *p; + + /* Check if any child key columns are being modified. */ + for(p=pTab->u.tab.pFKey; p; p=p->pNextFrom){ + if( fkChildIsModified(pTab, p, aChange, chngRowid) ){ + if( 0==sqlite3_stricmp(pTab->zName, p->zTo) ) eRet = 2; + bHaveFK = 1; + } + } + + /* Check if any parent key columns are being modified. */ + for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ + if( fkParentIsModified(pTab, p, aChange, chngRowid) ){ + if( p->aAction[1]!=OE_None ) return 2; + bHaveFK = 1; + } + } + } + } + return bHaveFK ? eRet : 0; +} + +/* +** This function is called when an UPDATE or DELETE operation is being +** compiled on table pTab, which is the parent table of foreign-key pFKey. +** If the current operation is an UPDATE, then the pChanges parameter is +** passed a pointer to the list of columns being modified. If it is a +** DELETE, pChanges is passed a NULL pointer. +** +** It returns a pointer to a Trigger structure containing a trigger +** equivalent to the ON UPDATE or ON DELETE action specified by pFKey. +** If the action is "NO ACTION" or "RESTRICT", then a NULL pointer is +** returned (these actions require no special handling by the triggers +** sub-system, code for them is created by fkScanChildren()). +** +** For example, if pFKey is the foreign key and pTab is table "p" in +** the following schema: +** +** CREATE TABLE p(pk PRIMARY KEY); +** CREATE TABLE c(ck REFERENCES p ON DELETE CASCADE); +** +** then the returned trigger structure is equivalent to: +** +** CREATE TRIGGER ... DELETE ON p BEGIN +** DELETE FROM c WHERE ck = old.pk; +** END; +** +** The returned pointer is cached as part of the foreign key object. It +** is eventually freed along with the rest of the foreign key object by +** sqlite3FkDelete(). +*/ +static Trigger *fkActionTrigger( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated or deleted from */ + FKey *pFKey, /* Foreign key to get action for */ + ExprList *pChanges /* Change-list for UPDATE, NULL for DELETE */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + int action; /* One of OE_None, OE_Cascade etc. */ + Trigger *pTrigger; /* Trigger definition to return */ + int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ + + action = pFKey->aAction[iAction]; + if( action==OE_Restrict && (db->flags & SQLITE_DeferFKs) ){ + return 0; + } + pTrigger = pFKey->apTrigger[iAction]; + + if( action!=OE_None && !pTrigger ){ + char const *zFrom; /* Name of child table */ + int nFrom; /* Length in bytes of zFrom */ + Index *pIdx = 0; /* Parent key index for this FK */ + int *aiCol = 0; /* child table cols -> parent key cols */ + TriggerStep *pStep = 0; /* First (only) step of trigger program */ + Expr *pWhere = 0; /* WHERE clause of trigger step */ + ExprList *pList = 0; /* Changes list if ON UPDATE CASCADE */ + Select *pSelect = 0; /* If RESTRICT, "SELECT RAISE(...)" */ + int i; /* Iterator variable */ + Expr *pWhen = 0; /* WHEN clause for the trigger */ + + if( sqlite3FkLocateIndex(pParse, pTab, pFKey, &pIdx, &aiCol) ) return 0; + assert( aiCol || pFKey->nCol==1 ); + + for(i=0; inCol; i++){ + Token tOld = { "old", 3 }; /* Literal "old" token */ + Token tNew = { "new", 3 }; /* Literal "new" token */ + Token tFromCol; /* Name of column in child table */ + Token tToCol; /* Name of column in parent table */ + int iFromCol; /* Idx of column in child table */ + Expr *pEq; /* tFromCol = OLD.tToCol */ + + iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; + assert( iFromCol>=0 ); + assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKeynCol) ); + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); + sqlite3TokenInit(&tToCol, + pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zCnName); + sqlite3TokenInit(&tFromCol, pFKey->pFrom->aCol[iFromCol].zCnName); + + /* Create the expression "OLD.zToCol = zFromCol". It is important + ** that the "OLD.zToCol" term is on the LHS of the = operator, so + ** that the affinity and collation sequence associated with the + ** parent table are used for the comparison. */ + pEq = sqlite3PExpr(pParse, TK_EQ, + sqlite3PExpr(pParse, TK_DOT, + sqlite3ExprAlloc(db, TK_ID, &tOld, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)), + sqlite3ExprAlloc(db, TK_ID, &tFromCol, 0) + ); + pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); + + /* For ON UPDATE, construct the next term of the WHEN clause. + ** The final WHEN clause will be like this: + ** + ** WHEN NOT(old.col1 IS new.col1 AND ... AND old.colN IS new.colN) + */ + if( pChanges ){ + pEq = sqlite3PExpr(pParse, TK_IS, + sqlite3PExpr(pParse, TK_DOT, + sqlite3ExprAlloc(db, TK_ID, &tOld, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)), + sqlite3PExpr(pParse, TK_DOT, + sqlite3ExprAlloc(db, TK_ID, &tNew, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)) + ); + pWhen = sqlite3ExprAnd(pParse, pWhen, pEq); + } + + if( action!=OE_Restrict && (action!=OE_Cascade || pChanges) ){ + Expr *pNew; + if( action==OE_Cascade ){ + pNew = sqlite3PExpr(pParse, TK_DOT, + sqlite3ExprAlloc(db, TK_ID, &tNew, 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)); + }else if( action==OE_SetDflt ){ + Column *pCol = pFKey->pFrom->aCol + iFromCol; + Expr *pDflt; + if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + pDflt = 0; + }else{ + pDflt = sqlite3ColumnExpr(pFKey->pFrom, pCol); + } + if( pDflt ){ + pNew = sqlite3ExprDup(db, pDflt, 0); + }else{ + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); + } + }else{ + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); + } + pList = sqlite3ExprListAppend(pParse, pList, pNew); + sqlite3ExprListSetName(pParse, pList, &tFromCol, 0); + } + } + sqlite3DbFree(db, aiCol); + + zFrom = pFKey->pFrom->zName; + nFrom = sqlite3Strlen30(zFrom); + + if( action==OE_Restrict ){ + Token tFrom; + Expr *pRaise; + + tFrom.z = zFrom; + tFrom.n = nFrom; + pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); + if( pRaise ){ + pRaise->affExpr = OE_Abort; + } + pSelect = sqlite3SelectNew(pParse, + sqlite3ExprListAppend(pParse, 0, pRaise), + sqlite3SrcListAppend(pParse, 0, &tFrom, 0), + pWhere, + 0, 0, 0, 0, 0 + ); + pWhere = 0; + } + + /* Disable lookaside memory allocation */ + DisableLookaside; + + pTrigger = (Trigger *)sqlite3DbMallocZero(db, + sizeof(Trigger) + /* struct Trigger */ + sizeof(TriggerStep) + /* Single step in trigger program */ + nFrom + 1 /* Space for pStep->zTarget */ + ); + if( pTrigger ){ + pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; + pStep->zTarget = (char *)&pStep[1]; + memcpy((char *)pStep->zTarget, zFrom, nFrom); + + pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); + pStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + if( pWhen ){ + pWhen = sqlite3PExpr(pParse, TK_NOT, pWhen, 0); + pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); + } + } + + /* Re-enable the lookaside buffer, if it was disabled earlier. */ + EnableLookaside; + + sqlite3ExprDelete(db, pWhere); + sqlite3ExprDelete(db, pWhen); + sqlite3ExprListDelete(db, pList); + sqlite3SelectDelete(db, pSelect); + if( db->mallocFailed==1 ){ + fkTriggerDelete(db, pTrigger); + return 0; + } + assert( pStep!=0 ); + assert( pTrigger!=0 ); + + switch( action ){ + case OE_Restrict: + pStep->op = TK_SELECT; + break; + case OE_Cascade: + if( !pChanges ){ + pStep->op = TK_DELETE; + break; + } + /* no break */ deliberate_fall_through + default: + pStep->op = TK_UPDATE; + } + pStep->pTrig = pTrigger; + pTrigger->pSchema = pTab->pSchema; + pTrigger->pTabSchema = pTab->pSchema; + pFKey->apTrigger[iAction] = pTrigger; + pTrigger->op = (pChanges ? TK_UPDATE : TK_DELETE); + } + + return pTrigger; +} + +/* +** This function is called when deleting or updating a row to implement +** any required CASCADE, SET NULL or SET DEFAULT actions. +*/ +SQLITE_PRIVATE void sqlite3FkActions( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated or deleted from */ + ExprList *pChanges, /* Change-list for UPDATE, NULL for DELETE */ + int regOld, /* Address of array containing old row */ + int *aChange, /* Array indicating UPDATEd columns (or 0) */ + int bChngRowid /* True if rowid is UPDATEd */ +){ + /* If foreign-key support is enabled, iterate through all FKs that + ** refer to table pTab. If there is an action associated with the FK + ** for this operation (either update or delete), invoke the associated + ** trigger sub-program. */ + if( pParse->db->flags&SQLITE_ForeignKeys ){ + FKey *pFKey; /* Iterator variable */ + for(pFKey = sqlite3FkReferences(pTab); pFKey; pFKey=pFKey->pNextTo){ + if( aChange==0 || fkParentIsModified(pTab, pFKey, aChange, bChngRowid) ){ + Trigger *pAct = fkActionTrigger(pParse, pTab, pFKey, pChanges); + if( pAct ){ + sqlite3CodeRowTriggerDirect(pParse, pAct, pTab, regOld, OE_Abort, 0); + } + } + } + } +} + +#endif /* ifndef SQLITE_OMIT_TRIGGER */ + +/* +** Free all memory associated with foreign key definitions attached to +** table pTab. Remove the deleted foreign keys from the Schema.fkeyHash +** hash table. +*/ +SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ + FKey *pFKey; /* Iterator variable */ + FKey *pNext; /* Copy of pFKey->pNextFrom */ + + assert( !IsVirtual(pTab) ); + for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pNext){ + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); + + /* Remove the FK from the fkeyHash hash table. */ + if( !db || db->pnBytesFreed==0 ){ + if( pFKey->pPrevTo ){ + pFKey->pPrevTo->pNextTo = pFKey->pNextTo; + }else{ + void *p = (void *)pFKey->pNextTo; + const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, p); + } + if( pFKey->pNextTo ){ + pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; + } + } + + /* EV: R-30323-21917 Each foreign key constraint in SQLite is + ** classified as either immediate or deferred. + */ + assert( pFKey->isDeferred==0 || pFKey->isDeferred==1 ); + + /* Delete any triggers created to implement actions for this FK. */ +#ifndef SQLITE_OMIT_TRIGGER + fkTriggerDelete(db, pFKey->apTrigger[0]); + fkTriggerDelete(db, pFKey->apTrigger[1]); +#endif + + pNext = pFKey->pNextFrom; + sqlite3DbFree(db, pFKey); + } +} +#endif /* ifndef SQLITE_OMIT_FOREIGN_KEY */ + +/************** End of fkey.c ************************************************/ +/************** Begin file insert.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** to handle INSERT statements in SQLite. +*/ +/* #include "sqliteInt.h" */ + +/* +** Generate code that will +** +** (1) acquire a lock for table pTab then +** (2) open pTab as cursor iCur. +** +** If pTab is a WITHOUT ROWID table, then it is the PRIMARY KEY index +** for that table that is actually opened. +*/ +SQLITE_PRIVATE void sqlite3OpenTable( + Parse *pParse, /* Generate code into this VDBE */ + int iCur, /* The cursor number of the table */ + int iDb, /* The database index in sqlite3.aDb[] */ + Table *pTab, /* The table to be opened */ + int opcode /* OP_OpenRead or OP_OpenWrite */ +){ + Vdbe *v; + assert( !IsVirtual(pTab) ); + assert( pParse->pVdbe!=0 ); + v = pParse->pVdbe; + assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); + sqlite3TableLock(pParse, iDb, pTab->tnum, + (opcode==OP_OpenWrite)?1:0, pTab->zName); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nNVCol); + VdbeComment((v, "%s", pTab->zName)); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->tnum==pTab->tnum ); + sqlite3VdbeAddOp3(v, opcode, iCur, pPk->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + VdbeComment((v, "%s", pTab->zName)); + } +} + +/* +** Return a pointer to the column affinity string associated with index +** pIdx. A column affinity string has one character for each column in +** the table, according to the affinity of the column: +** +** Character Column affinity +** ------------------------------ +** 'A' BLOB +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'F' REAL +** +** An extra 'D' is appended to the end of the string to cover the +** rowid that appears as the last column in every index. +** +** Memory for the buffer containing the column index affinity string +** is managed along with the rest of the Index structure. It will be +** released when sqlite3DeleteIndex() is called. +*/ +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ + if( !pIdx->zColAff ){ + /* The first time a column affinity string for a particular index is + ** required, it is allocated and populated here. It is then stored as + ** a member of the Index structure for subsequent use. + ** + ** The column affinity string will eventually be deleted by + ** sqliteDeleteIndex() when the Index structure itself is cleaned + ** up. + */ + int n; + Table *pTab = pIdx->pTable; + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); + if( !pIdx->zColAff ){ + sqlite3OomFault(db); + return 0; + } + for(n=0; nnColumn; n++){ + i16 x = pIdx->aiColumn[n]; + char aff; + if( x>=0 ){ + aff = pTab->aCol[x].affinity; + }else if( x==XN_ROWID ){ + aff = SQLITE_AFF_INTEGER; + }else{ + assert( x==XN_EXPR ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + } + if( affSQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; + pIdx->zColAff[n] = aff; + } + pIdx->zColAff[n] = 0; + } + + return pIdx->zColAff; +} + +/* +** Make changes to the evolving bytecode to do affinity transformations +** of values that are about to be gathered into a row for table pTab. +** +** For ordinary (legacy, non-strict) tables: +** ----------------------------------------- +** +** Compute the affinity string for table pTab, if it has not already been +** computed. As an optimization, omit trailing SQLITE_AFF_BLOB affinities. +** +** If the affinity string is empty (because it was all SQLITE_AFF_BLOB entries +** which were then optimized out) then this routine becomes a no-op. +** +** Otherwise if iReg>0 then code an OP_Affinity opcode that will set the +** affinities for register iReg and following. Or if iReg==0, +** then just set the P4 operand of the previous opcode (which should be +** an OP_MakeRecord) to the affinity string. +** +** A column affinity string has one character per column: +** +** Character Column affinity +** --------- --------------- +** 'A' BLOB +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'E' REAL +** +** For STRICT tables: +** ------------------ +** +** Generate an appropropriate OP_TypeCheck opcode that will verify the +** datatypes against the column definitions in pTab. If iReg==0, that +** means an OP_MakeRecord opcode has already been generated and should be +** the last opcode generated. The new OP_TypeCheck needs to be inserted +** before the OP_MakeRecord. The new OP_TypeCheck should use the same +** register set as the OP_MakeRecord. If iReg>0 then register iReg is +** the first of a series of registers that will form the new record. +** Apply the type checking to that array of registers. +*/ +SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ + int i, j; + char *zColAff; + if( pTab->tabFlags & TF_Strict ){ + if( iReg==0 ){ + /* Move the previous opcode (which should be OP_MakeRecord) forward + ** by one slot and insert a new OP_TypeCheck where the current + ** OP_MakeRecord is found */ + VdbeOp *pPrev; + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); + pPrev = sqlite3VdbeGetOp(v, -1); + assert( pPrev!=0 ); + assert( pPrev->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); + pPrev->opcode = OP_TypeCheck; + sqlite3VdbeAddOp3(v, OP_MakeRecord, pPrev->p1, pPrev->p2, pPrev->p3); + }else{ + /* Insert an isolated OP_Typecheck */ + sqlite3VdbeAddOp2(v, OP_TypeCheck, iReg, pTab->nNVCol); + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); + } + return; + } + zColAff = pTab->zColAff; + if( zColAff==0 ){ + sqlite3 *db = sqlite3VdbeDb(v); + zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); + if( !zColAff ){ + sqlite3OomFault(db); + return; + } + + for(i=j=0; inCol; i++){ + assert( pTab->aCol[i].affinity!=0 ); + if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + zColAff[j++] = pTab->aCol[i].affinity; + } + } + do{ + zColAff[j--] = 0; + }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); + pTab->zColAff = zColAff; + } + assert( zColAff!=0 ); + i = sqlite3Strlen30NN(zColAff); + if( i ){ + if( iReg ){ + sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i); + }else{ + assert( sqlite3VdbeGetOp(v, -1)->opcode==OP_MakeRecord + || sqlite3VdbeDb(v)->mallocFailed ); + sqlite3VdbeChangeP4(v, -1, zColAff, i); + } + } +} + +/* +** Return non-zero if the table pTab in database iDb or any of its indices +** have been opened at any point in the VDBE program. This is used to see if +** a statement of the form "INSERT INTO SELECT ..." can +** run without using a temporary table for the results of the SELECT. +*/ +static int readsTable(Parse *p, int iDb, Table *pTab){ + Vdbe *v = sqlite3GetVdbe(p); + int i; + int iEnd = sqlite3VdbeCurrentAddr(v); +#ifndef SQLITE_OMIT_VIRTUALTABLE + VTable *pVTab = IsVirtual(pTab) ? sqlite3GetVTable(p->db, pTab) : 0; +#endif + + for(i=1; iopcode==OP_OpenRead && pOp->p3==iDb ){ + Index *pIndex; + Pgno tnum = pOp->p2; + if( tnum==pTab->tnum ){ + return 1; + } + for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ + if( tnum==pIndex->tnum ){ + return 1; + } + } + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pOp->opcode==OP_VOpen && pOp->p4.pVtab==pVTab ){ + assert( pOp->p4.pVtab!=0 ); + assert( pOp->p4type==P4_VTAB ); + return 1; + } +#endif + } + return 0; +} + +/* This walker callback will compute the union of colFlags flags for all +** referenced columns in a CHECK constraint or generated column expression. +*/ +static int exprColumnFlagUnion(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN && pExpr->iColumn>=0 ){ + assert( pExpr->iColumn < pWalker->u.pTab->nCol ); + pWalker->eCode |= pWalker->u.pTab->aCol[pExpr->iColumn].colFlags; + } + return WRC_Continue; +} + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* +** All regular columns for table pTab have been puts into registers +** starting with iRegStore. The registers that correspond to STORED +** or VIRTUAL columns have not yet been initialized. This routine goes +** back and computes the values for those columns based on the previously +** computed normal columns. +*/ +SQLITE_PRIVATE void sqlite3ComputeGeneratedColumns( + Parse *pParse, /* Parsing context */ + int iRegStore, /* Register holding the first column */ + Table *pTab /* The table */ +){ + int i; + Walker w; + Column *pRedo; + int eProgress; + VdbeOp *pOp; + + assert( pTab->tabFlags & TF_HasGenerated ); + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + + /* Before computing generated columns, first go through and make sure + ** that appropriate affinity has been applied to the regular columns + */ + sqlite3TableAffinity(pParse->pVdbe, pTab, iRegStore); + if( (pTab->tabFlags & TF_HasStored)!=0 + && (pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1))->opcode==OP_Affinity + ){ + /* Change the OP_Affinity argument to '@' (NONE) for all stored + ** columns. '@' is the no-op affinity and those columns have not + ** yet been computed. */ + int ii, jj; + char *zP4 = pOp->p4.z; + assert( zP4!=0 ); + assert( pOp->p4type==P4_DYNAMIC ); + for(ii=jj=0; zP4[jj]; ii++){ + if( pTab->aCol[ii].colFlags & COLFLAG_VIRTUAL ){ + continue; + } + if( pTab->aCol[ii].colFlags & COLFLAG_STORED ){ + zP4[jj] = SQLITE_AFF_NONE; + } + jj++; + } + } + + /* Because there can be multiple generated columns that refer to one another, + ** this is a two-pass algorithm. On the first pass, mark all generated + ** columns as "not available". + */ + for(i=0; inCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[i].colFlags & COLFLAG_STORED ); + pTab->aCol[i].colFlags |= COLFLAG_NOTAVAIL; + } + } + + w.u.pTab = pTab; + w.xExprCallback = exprColumnFlagUnion; + w.xSelectCallback = 0; + w.xSelectCallback2 = 0; + + /* On the second pass, compute the value of each NOT-AVAILABLE column. + ** Companion code in the TK_COLUMN case of sqlite3ExprCodeTarget() will + ** compute dependencies and mark remove the COLSPAN_NOTAVAIL mark, as + ** they are needed. + */ + pParse->iSelfTab = -iRegStore; + do{ + eProgress = 0; + pRedo = 0; + for(i=0; inCol; i++){ + Column *pCol = pTab->aCol + i; + if( (pCol->colFlags & COLFLAG_NOTAVAIL)!=0 ){ + int x; + pCol->colFlags |= COLFLAG_BUSY; + w.eCode = 0; + sqlite3WalkExpr(&w, sqlite3ColumnExpr(pTab, pCol)); + pCol->colFlags &= ~COLFLAG_BUSY; + if( w.eCode & COLFLAG_NOTAVAIL ){ + pRedo = pCol; + continue; + } + eProgress = 1; + assert( pCol->colFlags & COLFLAG_GENERATED ); + x = sqlite3TableColumnToStorage(pTab, i) + iRegStore; + sqlite3ExprCodeGeneratedColumn(pParse, pTab, pCol, x); + pCol->colFlags &= ~COLFLAG_NOTAVAIL; + } + } + }while( pRedo && eProgress ); + if( pRedo ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", pRedo->zCnName); + } + pParse->iSelfTab = 0; +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + + +#ifndef SQLITE_OMIT_AUTOINCREMENT +/* +** Locate or create an AutoincInfo structure associated with table pTab +** which is in database iDb. Return the register number for the register +** that holds the maximum rowid. Return zero if pTab is not an AUTOINCREMENT +** table. (Also return zero when doing a VACUUM since we do not want to +** update the AUTOINCREMENT counters during a VACUUM.) +** +** There is at most one AutoincInfo structure per table even if the +** same table is autoincremented multiple times due to inserts within +** triggers. A new AutoincInfo structure is created if this is the +** first use of table pTab. On 2nd and subsequent uses, the original +** AutoincInfo structure is used. +** +** Four consecutive registers are allocated: +** +** (1) The name of the pTab table. +** (2) The maximum ROWID of pTab. +** (3) The rowid in sqlite_sequence of pTab +** (4) The original value of the max ROWID in pTab, or NULL if none +** +** The 2nd register is the one that is returned. That is all the +** insert routine needs to know about. +*/ +static int autoIncBegin( + Parse *pParse, /* Parsing context */ + int iDb, /* Index of the database holding pTab */ + Table *pTab /* The table we are writing to */ +){ + int memId = 0; /* Register holding maximum rowid */ + assert( pParse->db->aDb[iDb].pSchema!=0 ); + if( (pTab->tabFlags & TF_Autoincrement)!=0 + && (pParse->db->mDbFlags & DBFLAG_Vacuum)==0 + ){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + AutoincInfo *pInfo; + Table *pSeqTab = pParse->db->aDb[iDb].pSchema->pSeqTab; + + /* Verify that the sqlite_sequence table exists and is an ordinary + ** rowid table with exactly two columns. + ** Ticket d8dc2b3a58cd5dc2918a1d4acb 2018-05-23 */ + if( pSeqTab==0 + || !HasRowid(pSeqTab) + || NEVER(IsVirtual(pSeqTab)) + || pSeqTab->nCol!=2 + ){ + pParse->nErr++; + pParse->rc = SQLITE_CORRUPT_SEQUENCE; + return 0; + } + + pInfo = pToplevel->pAinc; + while( pInfo && pInfo->pTab!=pTab ){ pInfo = pInfo->pNext; } + if( pInfo==0 ){ + pInfo = sqlite3DbMallocRawNN(pParse->db, sizeof(*pInfo)); + sqlite3ParserAddCleanup(pToplevel, sqlite3DbFree, pInfo); + testcase( pParse->earlyCleanup ); + if( pParse->db->mallocFailed ) return 0; + pInfo->pNext = pToplevel->pAinc; + pToplevel->pAinc = pInfo; + pInfo->pTab = pTab; + pInfo->iDb = iDb; + pToplevel->nMem++; /* Register to hold name of table */ + pInfo->regCtr = ++pToplevel->nMem; /* Max rowid register */ + pToplevel->nMem +=2; /* Rowid in sqlite_sequence + orig max val */ + } + memId = pInfo->regCtr; + } + return memId; +} + +/* +** This routine generates code that will initialize all of the +** register used by the autoincrement tracker. +*/ +SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ + AutoincInfo *p; /* Information about an AUTOINCREMENT */ + sqlite3 *db = pParse->db; /* The database connection */ + Db *pDb; /* Database only autoinc table */ + int memId; /* Register holding max rowid */ + Vdbe *v = pParse->pVdbe; /* VDBE under construction */ + + /* This routine is never called during trigger-generation. It is + ** only called from the top-level */ + assert( pParse->pTriggerTab==0 ); + assert( sqlite3IsToplevel(pParse) ); + + assert( v ); /* We failed long ago if this is not so */ + for(p = pParse->pAinc; p; p = p->pNext){ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList autoInc[] = { + /* 0 */ {OP_Null, 0, 0, 0}, + /* 1 */ {OP_Rewind, 0, 10, 0}, + /* 2 */ {OP_Column, 0, 0, 0}, + /* 3 */ {OP_Ne, 0, 9, 0}, + /* 4 */ {OP_Rowid, 0, 0, 0}, + /* 5 */ {OP_Column, 0, 1, 0}, + /* 6 */ {OP_AddImm, 0, 0, 0}, + /* 7 */ {OP_Copy, 0, 0, 0}, + /* 8 */ {OP_Goto, 0, 11, 0}, + /* 9 */ {OP_Next, 0, 2, 0}, + /* 10 */ {OP_Integer, 0, 0, 0}, + /* 11 */ {OP_Close, 0, 0, 0} + }; + VdbeOp *aOp; + pDb = &db->aDb[p->iDb]; + memId = p->regCtr; + assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); + sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenRead); + sqlite3VdbeLoadString(v, memId-1, p->pTab->zName); + aOp = sqlite3VdbeAddOpList(v, ArraySize(autoInc), autoInc, iLn); + if( aOp==0 ) break; + aOp[0].p2 = memId; + aOp[0].p3 = memId+2; + aOp[2].p3 = memId; + aOp[3].p1 = memId-1; + aOp[3].p3 = memId; + aOp[3].p5 = SQLITE_JUMPIFNULL; + aOp[4].p2 = memId+1; + aOp[5].p3 = memId; + aOp[6].p1 = memId; + aOp[7].p2 = memId+2; + aOp[7].p1 = memId; + aOp[10].p2 = memId; + if( pParse->nTab==0 ) pParse->nTab = 1; + } +} + +/* +** Update the maximum rowid for an autoincrement calculation. +** +** This routine should be called when the regRowid register holds a +** new rowid that is about to be inserted. If that new rowid is +** larger than the maximum rowid in the memId memory cell, then the +** memory cell is updated. +*/ +static void autoIncStep(Parse *pParse, int memId, int regRowid){ + if( memId>0 ){ + sqlite3VdbeAddOp2(pParse->pVdbe, OP_MemMax, memId, regRowid); + } +} + +/* +** This routine generates the code needed to write autoincrement +** maximum rowid values back into the sqlite_sequence register. +** Every statement that might do an INSERT into an autoincrement +** table (either directly or through triggers) needs to call this +** routine just before the "exit" code. +*/ +static SQLITE_NOINLINE void autoIncrementEnd(Parse *pParse){ + AutoincInfo *p; + Vdbe *v = pParse->pVdbe; + sqlite3 *db = pParse->db; + + assert( v ); + for(p = pParse->pAinc; p; p = p->pNext){ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList autoIncEnd[] = { + /* 0 */ {OP_NotNull, 0, 2, 0}, + /* 1 */ {OP_NewRowid, 0, 0, 0}, + /* 2 */ {OP_MakeRecord, 0, 2, 0}, + /* 3 */ {OP_Insert, 0, 0, 0}, + /* 4 */ {OP_Close, 0, 0, 0} + }; + VdbeOp *aOp; + Db *pDb = &db->aDb[p->iDb]; + int iRec; + int memId = p->regCtr; + + iRec = sqlite3GetTempReg(pParse); + assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); + sqlite3VdbeAddOp3(v, OP_Le, memId+2, sqlite3VdbeCurrentAddr(v)+7, memId); + VdbeCoverage(v); + sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); + aOp = sqlite3VdbeAddOpList(v, ArraySize(autoIncEnd), autoIncEnd, iLn); + if( aOp==0 ) break; + aOp[0].p1 = memId+1; + aOp[1].p2 = memId+1; + aOp[2].p1 = memId-1; + aOp[2].p3 = iRec; + aOp[3].p2 = iRec; + aOp[3].p3 = memId+1; + aOp[3].p5 = OPFLAG_APPEND; + sqlite3ReleaseTempReg(pParse, iRec); + } +} +SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ + if( pParse->pAinc ) autoIncrementEnd(pParse); +} +#else +/* +** If SQLITE_OMIT_AUTOINCREMENT is defined, then the three routines +** above are all no-ops +*/ +# define autoIncBegin(A,B,C) (0) +# define autoIncStep(A,B,C) +#endif /* SQLITE_OMIT_AUTOINCREMENT */ + + +/* Forward declaration */ +static int xferOptimization( + Parse *pParse, /* Parser context */ + Table *pDest, /* The table we are inserting into */ + Select *pSelect, /* A SELECT statement to use as the data source */ + int onError, /* How to handle constraint errors */ + int iDbDest /* The database of pDest */ +); + +/* +** This routine is called to handle SQL of the following forms: +** +** insert into TABLE (IDLIST) values(EXPRLIST),(EXPRLIST),... +** insert into TABLE (IDLIST) select +** insert into TABLE (IDLIST) default values +** +** The IDLIST following the table name is always optional. If omitted, +** then a list of all (non-hidden) columns for the table is substituted. +** The IDLIST appears in the pColumn parameter. pColumn is NULL if IDLIST +** is omitted. +** +** For the pSelect parameter holds the values to be inserted for the +** first two forms shown above. A VALUES clause is really just short-hand +** for a SELECT statement that omits the FROM clause and everything else +** that follows. If the pSelect parameter is NULL, that means that the +** DEFAULT VALUES form of the INSERT statement is intended. +** +** The code generated follows one of four templates. For a simple +** insert with data coming from a single-row VALUES clause, the code executes +** once straight down through. Pseudo-code follows (we call this +** the "1st template"): +** +** open write cursor to
    and its indices +** put VALUES clause expressions into registers +** write the resulting record into
    +** cleanup +** +** The three remaining templates assume the statement is of the form +** +** INSERT INTO
    SELECT ... +** +** If the SELECT clause is of the restricted form "SELECT * FROM " - +** in other words if the SELECT pulls all columns from a single table +** and there is no WHERE or LIMIT or GROUP BY or ORDER BY clauses, and +** if and are distinct tables but have identical +** schemas, including all the same indices, then a special optimization +** is invoked that copies raw records from over to . +** See the xferOptimization() function for the implementation of this +** template. This is the 2nd template. +** +** open a write cursor to
    +** open read cursor on +** transfer all records in over to
    +** close cursors +** foreach index on
    +** open a write cursor on the
    index +** open a read cursor on the corresponding index +** transfer all records from the read to the write cursors +** close cursors +** end foreach +** +** The 3rd template is for when the second template does not apply +** and the SELECT clause does not read from
    at any time. +** The generated code follows this template: +** +** X <- A +** goto B +** A: setup for the SELECT +** loop over the rows in the SELECT +** load values into registers R..R+n +** yield X +** end loop +** cleanup after the SELECT +** end-coroutine X +** B: open write cursor to
    and its indices +** C: yield X, at EOF goto D +** insert the select result into
    from R..R+n +** goto C +** D: cleanup +** +** The 4th template is used if the insert statement takes its +** values from a SELECT but the data is being inserted into a table +** that is also read as part of the SELECT. In the third form, +** we have to use an intermediate table to store the results of +** the select. The template is like this: +** +** X <- A +** goto B +** A: setup for the SELECT +** loop over the tables in the SELECT +** load value into register R..R+n +** yield X +** end loop +** cleanup after the SELECT +** end co-routine R +** B: open temp table +** L: yield X, at EOF goto M +** insert row from R..R+n into temp table +** goto L +** M: open write cursor to
    and its indices +** rewind temp table +** C: loop over rows of intermediate table +** transfer values form intermediate table into
    +** end loop +** D: cleanup +*/ +SQLITE_PRIVATE void sqlite3Insert( + Parse *pParse, /* Parser context */ + SrcList *pTabList, /* Name of table into which we are inserting */ + Select *pSelect, /* A SELECT statement to use as the data source */ + IdList *pColumn, /* Column names corresponding to IDLIST, or NULL. */ + int onError, /* How to handle constraint errors */ + Upsert *pUpsert /* ON CONFLICT clauses for upsert, or NULL */ +){ + sqlite3 *db; /* The main database structure */ + Table *pTab; /* The table to insert into. aka TABLE */ + int i, j; /* Loop counters */ + Vdbe *v; /* Generate code into this virtual machine */ + Index *pIdx; /* For looping over indices of the table */ + int nColumn; /* Number of columns in the data */ + int nHidden = 0; /* Number of hidden columns if TABLE is virtual */ + int iDataCur = 0; /* VDBE cursor that is the main data repository */ + int iIdxCur = 0; /* First index cursor */ + int ipkColumn = -1; /* Column that is the INTEGER PRIMARY KEY */ + int endOfLoop; /* Label for the end of the insertion loop */ + int srcTab = 0; /* Data comes from this temporary cursor if >=0 */ + int addrInsTop = 0; /* Jump to label "D" */ + int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ + SelectDest dest; /* Destination for SELECT on rhs of INSERT */ + int iDb; /* Index of database holding TABLE */ + u8 useTempTable = 0; /* Store SELECT results in intermediate table */ + u8 appendFlag = 0; /* True if the insert is likely to be an append */ + u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ + u8 bIdListInOrder; /* True if IDLIST is in table order */ + ExprList *pList = 0; /* List of VALUES() to be inserted */ + int iRegStore; /* Register in which to store next column */ + + /* Register allocations */ + int regFromSelect = 0;/* Base register for data coming from SELECT */ + int regAutoinc = 0; /* Register holding the AUTOINCREMENT counter */ + int regRowCount = 0; /* Memory cell used for the row counter */ + int regIns; /* Block of regs holding rowid+data being inserted */ + int regRowid; /* registers holding insert rowid */ + int regData; /* register holding first column to insert */ + int *aRegIdx = 0; /* One register allocated to each index */ + +#ifndef SQLITE_OMIT_TRIGGER + int isView; /* True if attempting to insert into a view */ + Trigger *pTrigger; /* List of triggers on pTab, if required */ + int tmask; /* Mask of trigger times */ +#endif + + db = pParse->db; + if( pParse->nErr || db->mallocFailed ){ + goto insert_cleanup; + } + dest.iSDParm = 0; /* Suppress a harmless compiler warning */ + + /* If the Select object is really just a simple VALUES() list with a + ** single row (the common case) then keep that one row of values + ** and discard the other (unused) parts of the pSelect object + */ + if( pSelect && (pSelect->selFlags & SF_Values)!=0 && pSelect->pPrior==0 ){ + pList = pSelect->pEList; + pSelect->pEList = 0; + sqlite3SelectDelete(db, pSelect); + pSelect = 0; + } + + /* Locate the table into which we will be inserting new information. + */ + assert( pTabList->nSrc==1 ); + pTab = sqlite3SrcListLookup(pParse, pTabList); + if( pTab==0 ){ + goto insert_cleanup; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDbnDb ); + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, + db->aDb[iDb].zDbSName) ){ + goto insert_cleanup; + } + withoutRowid = !HasRowid(pTab); + + /* Figure out if we have any triggers and if the table being + ** inserted into is a view + */ +#ifndef SQLITE_OMIT_TRIGGER + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_INSERT, 0, &tmask); + isView = IsView(pTab); +#else +# define pTrigger 0 +# define tmask 0 +# define isView 0 +#endif +#ifdef SQLITE_OMIT_VIEW +# undef isView +# define isView 0 +#endif + assert( (pTrigger && tmask) || (pTrigger==0 && tmask==0) ); + + /* If pTab is really a view, make sure it has been initialized. + ** ViewGetColumnNames() is a no-op if pTab is not a view. + */ + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto insert_cleanup; + } + + /* Cannot insert into a read-only table. + */ + if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + goto insert_cleanup; + } + + /* Allocate a VDBE + */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto insert_cleanup; + if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); + sqlite3BeginWriteOperation(pParse, pSelect || pTrigger, iDb); + +#ifndef SQLITE_OMIT_XFER_OPT + /* If the statement is of the form + ** + ** INSERT INTO SELECT * FROM ; + ** + ** Then special optimizations can be applied that make the transfer + ** very fast and which reduce fragmentation of indices. + ** + ** This is the 2nd template. + */ + if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ + assert( !pTrigger ); + assert( pList==0 ); + goto insert_end; + } +#endif /* SQLITE_OMIT_XFER_OPT */ + + /* If this is an AUTOINCREMENT table, look up the sequence number in the + ** sqlite_sequence table and store it in memory cell regAutoinc. + */ + regAutoinc = autoIncBegin(pParse, iDb, pTab); + + /* Allocate a block registers to hold the rowid and the values + ** for all columns of the new row. + */ + regRowid = regIns = pParse->nMem+1; + pParse->nMem += pTab->nCol + 1; + if( IsVirtual(pTab) ){ + regRowid++; + pParse->nMem++; + } + regData = regRowid+1; + + /* If the INSERT statement included an IDLIST term, then make sure + ** all elements of the IDLIST really are columns of the table and + ** remember the column indices. + ** + ** If the table has an INTEGER PRIMARY KEY column and that column + ** is named in the IDLIST, then record in the ipkColumn variable + ** the index into IDLIST of the primary key column. ipkColumn is + ** the index of the primary key as it appears in IDLIST, not as + ** is appears in the original table. (The index of the INTEGER + ** PRIMARY KEY in the original table is pTab->iPKey.) After this + ** loop, if ipkColumn==(-1), that means that integer primary key + ** is unspecified, and hence the table is either WITHOUT ROWID or + ** it will automatically generated an integer primary key. + ** + ** bIdListInOrder is true if the columns in IDLIST are in storage + ** order. This enables an optimization that avoids shuffling the + ** columns into storage order. False negatives are harmless, + ** but false positives will cause database corruption. + */ + bIdListInOrder = (pTab->tabFlags & (TF_OOOHidden|TF_HasStored))==0; + if( pColumn ){ + for(i=0; inId; i++){ + pColumn->a[i].idx = -1; + } + for(i=0; inId; i++){ + for(j=0; jnCol; j++){ + if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zCnName)==0 ){ + pColumn->a[i].idx = j; + if( i!=j ) bIdListInOrder = 0; + if( j==pTab->iPKey ){ + ipkColumn = i; assert( !withoutRowid ); + } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->aCol[j].colFlags & (COLFLAG_STORED|COLFLAG_VIRTUAL) ){ + sqlite3ErrorMsg(pParse, + "cannot INSERT into generated column \"%s\"", + pTab->aCol[j].zCnName); + goto insert_cleanup; + } +#endif + break; + } + } + if( j>=pTab->nCol ){ + if( sqlite3IsRowid(pColumn->a[i].zName) && !withoutRowid ){ + ipkColumn = i; + bIdListInOrder = 0; + }else{ + sqlite3ErrorMsg(pParse, "table %S has no column named %s", + pTabList->a, pColumn->a[i].zName); + pParse->checkSchema = 1; + goto insert_cleanup; + } + } + } + } + + /* Figure out how many columns of data are supplied. If the data + ** is coming from a SELECT statement, then generate a co-routine that + ** produces a single row of the SELECT on each invocation. The + ** co-routine is the common header to the 3rd and 4th templates. + */ + if( pSelect ){ + /* Data is coming from a SELECT or from a multi-row VALUES clause. + ** Generate a co-routine to run the SELECT. */ + int regYield; /* Register holding co-routine entry-point */ + int addrTop; /* Top of the co-routine */ + int rc; /* Result code */ + + regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; + rc = sqlite3Select(pParse, pSelect, &dest); + regFromSelect = dest.iSdst; + if( rc || db->mallocFailed || pParse->nErr ) goto insert_cleanup; + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ + assert( pSelect->pEList ); + nColumn = pSelect->pEList->nExpr; + + /* Set useTempTable to TRUE if the result of the SELECT statement + ** should be written into a temporary table (template 4). Set to + ** FALSE if each output row of the SELECT can be written directly into + ** the destination table (template 3). + ** + ** A temp table must be used if the table being updated is also one + ** of the tables being read by the SELECT statement. Also use a + ** temp table in the case of row triggers. + */ + if( pTrigger || readsTable(pParse, iDb, pTab) ){ + useTempTable = 1; + } + + if( useTempTable ){ + /* Invoke the coroutine to extract information from the SELECT + ** and add it to a transient table srcTab. The code generated + ** here is from the 4th template: + ** + ** B: open temp table + ** L: yield X, goto M at EOF + ** insert row from R..R+n into temp table + ** goto L + ** M: ... + */ + int regRec; /* Register to hold packed record */ + int regTempRowid; /* Register to hold temp table ROWID */ + int addrL; /* Label "L" */ + + srcTab = pParse->nTab++; + regRec = sqlite3GetTempReg(pParse); + regTempRowid = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, srcTab, nColumn); + addrL = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regFromSelect, nColumn, regRec); + sqlite3VdbeAddOp2(v, OP_NewRowid, srcTab, regTempRowid); + sqlite3VdbeAddOp3(v, OP_Insert, srcTab, regRec, regTempRowid); + sqlite3VdbeGoto(v, addrL); + sqlite3VdbeJumpHere(v, addrL); + sqlite3ReleaseTempReg(pParse, regRec); + sqlite3ReleaseTempReg(pParse, regTempRowid); + } + }else{ + /* This is the case if the data for the INSERT is coming from a + ** single-row VALUES clause + */ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + srcTab = -1; + assert( useTempTable==0 ); + if( pList ){ + nColumn = pList->nExpr; + if( sqlite3ResolveExprListNames(&sNC, pList) ){ + goto insert_cleanup; + } + }else{ + nColumn = 0; + } + } + + /* If there is no IDLIST term but the table has an integer primary + ** key, the set the ipkColumn variable to the integer primary key + ** column index in the original table definition. + */ + if( pColumn==0 && nColumn>0 ){ + ipkColumn = pTab->iPKey; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( ipkColumn>=0 && (pTab->tabFlags & TF_HasGenerated)!=0 ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + for(i=ipkColumn-1; i>=0; i--){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[i].colFlags & COLFLAG_STORED ); + ipkColumn--; + } + } + } +#endif + + /* Make sure the number of columns in the source data matches the number + ** of columns to be inserted into the table. + */ + assert( TF_HasHidden==COLFLAG_HIDDEN ); + assert( TF_HasGenerated==COLFLAG_GENERATED ); + assert( COLFLAG_NOINSERT==(COLFLAG_GENERATED|COLFLAG_HIDDEN) ); + if( (pTab->tabFlags & (TF_HasGenerated|TF_HasHidden))!=0 ){ + for(i=0; inCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_NOINSERT ) nHidden++; + } + } + if( nColumn!=(pTab->nCol-nHidden) ){ + sqlite3ErrorMsg(pParse, + "table %S has %d columns but %d values were supplied", + pTabList->a, pTab->nCol-nHidden, nColumn); + goto insert_cleanup; + } + } + if( pColumn!=0 && nColumn!=pColumn->nId ){ + sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId); + goto insert_cleanup; + } + + /* Initialize the count of rows to be inserted + */ + if( (db->flags & SQLITE_CountRows)!=0 + && !pParse->nested + && !pParse->pTriggerTab + && !pParse->bReturning + ){ + regRowCount = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + } + + /* If this is not a view, open the table and and all indices */ + if( !isView ){ + int nIdx; + nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, -1, 0, + &iDataCur, &iIdxCur); + aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+2)); + if( aRegIdx==0 ){ + goto insert_cleanup; + } + for(i=0, pIdx=pTab->pIndex; ipNext, i++){ + assert( pIdx ); + aRegIdx[i] = ++pParse->nMem; + pParse->nMem += pIdx->nColumn; + } + aRegIdx[i] = ++pParse->nMem; /* Register to store the table record */ + } +#ifndef SQLITE_OMIT_UPSERT + if( pUpsert ){ + Upsert *pNx; + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "UPSERT not implemented for virtual table \"%s\"", + pTab->zName); + goto insert_cleanup; + } + if( IsView(pTab) ){ + sqlite3ErrorMsg(pParse, "cannot UPSERT a view"); + goto insert_cleanup; + } + if( sqlite3HasExplicitNulls(pParse, pUpsert->pUpsertTarget) ){ + goto insert_cleanup; + } + pTabList->a[0].iCursor = iDataCur; + pNx = pUpsert; + do{ + pNx->pUpsertSrc = pTabList; + pNx->regData = regData; + pNx->iDataCur = iDataCur; + pNx->iIdxCur = iIdxCur; + if( pNx->pUpsertTarget ){ + if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx) ){ + goto insert_cleanup; + } + } + pNx = pNx->pNextUpsert; + }while( pNx!=0 ); + } +#endif + + + /* This is the top of the main insertion loop */ + if( useTempTable ){ + /* This block codes the top of loop only. The complete loop is the + ** following pseudocode (template 4): + ** + ** rewind temp table, if empty goto D + ** C: loop over rows of intermediate table + ** transfer values form intermediate table into
    + ** end loop + ** D: ... + */ + addrInsTop = sqlite3VdbeAddOp1(v, OP_Rewind, srcTab); VdbeCoverage(v); + addrCont = sqlite3VdbeCurrentAddr(v); + }else if( pSelect ){ + /* This block codes the top of loop only. The complete loop is the + ** following pseudocode (template 3): + ** + ** C: yield X, at EOF goto D + ** insert the select result into
    from R..R+n + ** goto C + ** D: ... + */ + sqlite3VdbeReleaseRegisters(pParse, regData, pTab->nCol, 0, 0); + addrInsTop = addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); + VdbeCoverage(v); + if( ipkColumn>=0 ){ + /* tag-20191021-001: If the INTEGER PRIMARY KEY is being generated by the + ** SELECT, go ahead and copy the value into the rowid slot now, so that + ** the value does not get overwritten by a NULL at tag-20191021-002. */ + sqlite3VdbeAddOp2(v, OP_Copy, regFromSelect+ipkColumn, regRowid); + } + } + + /* Compute data for ordinary columns of the new entry. Values + ** are written in storage order into registers starting with regData. + ** Only ordinary columns are computed in this loop. The rowid + ** (if there is one) is computed later and generated columns are + ** computed after the rowid since they might depend on the value + ** of the rowid. + */ + nHidden = 0; + iRegStore = regData; assert( regData==regRowid+1 ); + for(i=0; inCol; i++, iRegStore++){ + int k; + u32 colFlags; + assert( i>=nHidden ); + if( i==pTab->iPKey ){ + /* tag-20191021-002: References to the INTEGER PRIMARY KEY are filled + ** using the rowid. So put a NULL in the IPK slot of the record to avoid + ** using excess space. The file format definition requires this extra + ** NULL - we cannot optimize further by skipping the column completely */ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); + continue; + } + if( ((colFlags = pTab->aCol[i].colFlags) & COLFLAG_NOINSERT)!=0 ){ + nHidden++; + if( (colFlags & COLFLAG_VIRTUAL)!=0 ){ + /* Virtual columns do not participate in OP_MakeRecord. So back up + ** iRegStore by one slot to compensate for the iRegStore++ in the + ** outer for() loop */ + iRegStore--; + continue; + }else if( (colFlags & COLFLAG_STORED)!=0 ){ + /* Stored columns are computed later. But if there are BEFORE + ** triggers, the slots used for stored columns will be OP_Copy-ed + ** to a second block of registers, so the register needs to be + ** initialized to NULL to avoid an uninitialized register read */ + if( tmask & TRIGGER_BEFORE ){ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); + } + continue; + }else if( pColumn==0 ){ + /* Hidden columns that are not explicitly named in the INSERT + ** get there default value */ + sqlite3ExprCodeFactorable(pParse, + sqlite3ColumnExpr(pTab, &pTab->aCol[i]), + iRegStore); + continue; + } + } + if( pColumn ){ + for(j=0; jnId && pColumn->a[j].idx!=i; j++){} + if( j>=pColumn->nId ){ + /* A column not named in the insert column list gets its + ** default value */ + sqlite3ExprCodeFactorable(pParse, + sqlite3ColumnExpr(pTab, &pTab->aCol[i]), + iRegStore); + continue; + } + k = j; + }else if( nColumn==0 ){ + /* This is INSERT INTO ... DEFAULT VALUES. Load the default value. */ + sqlite3ExprCodeFactorable(pParse, + sqlite3ColumnExpr(pTab, &pTab->aCol[i]), + iRegStore); + continue; + }else{ + k = i - nHidden; + } + + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, k, iRegStore); + }else if( pSelect ){ + if( regFromSelect!=regData ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+k, iRegStore); + } + }else{ + sqlite3ExprCode(pParse, pList->a[k].pExpr, iRegStore); + } + } + + + /* Run the BEFORE and INSTEAD OF triggers, if there are any + */ + endOfLoop = sqlite3VdbeMakeLabel(pParse); + if( tmask & TRIGGER_BEFORE ){ + int regCols = sqlite3GetTempRange(pParse, pTab->nCol+1); + + /* build the NEW.* reference row. Note that if there is an INTEGER + ** PRIMARY KEY into which a NULL is being inserted, that NULL will be + ** translated into a unique ID for the row. But on a BEFORE trigger, + ** we do not know what the unique ID will be (because the insert has + ** not happened yet) so we substitute a rowid of -1 + */ + if( ipkColumn<0 ){ + sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); + }else{ + int addr1; + assert( !withoutRowid ); + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regCols); + }else{ + assert( pSelect==0 ); /* Otherwise useTempTable is true */ + sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regCols); + } + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); + } + + /* Copy the new data already generated. */ + assert( pTab->nNVCol>0 ); + sqlite3VdbeAddOp3(v, OP_Copy, regRowid+1, regCols+1, pTab->nNVCol-1); + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Compute the new value for generated columns after all other + ** columns have already been computed. This must be done after + ** computing the ROWID in case one of the generated columns + ** refers to the ROWID. */ + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regCols+1, pTab); + } +#endif + + /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger, + ** do not attempt any conversions before assembling the record. + ** If this is a real table, attempt conversions as required by the + ** table column affinities. + */ + if( !isView ){ + sqlite3TableAffinity(v, pTab, regCols+1); + } + + /* Fire BEFORE or INSTEAD OF triggers */ + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_BEFORE, + pTab, regCols-pTab->nCol-1, onError, endOfLoop); + + sqlite3ReleaseTempRange(pParse, regCols, pTab->nCol+1); + } + + if( !isView ){ + if( IsVirtual(pTab) ){ + /* The row that the VUpdate opcode will delete: none */ + sqlite3VdbeAddOp2(v, OP_Null, 0, regIns); + } + if( ipkColumn>=0 ){ + /* Compute the new rowid */ + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regRowid); + }else if( pSelect ){ + /* Rowid already initialized at tag-20191021-001 */ + }else{ + Expr *pIpk = pList->a[ipkColumn].pExpr; + if( pIpk->op==TK_NULL && !IsVirtual(pTab) ){ + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); + appendFlag = 1; + }else{ + sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regRowid); + } + } + /* If the PRIMARY KEY expression is NULL, then use OP_NewRowid + ** to generate a unique primary key value. + */ + if( !appendFlag ){ + int addr1; + if( !IsVirtual(pTab) ){ + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); + sqlite3VdbeJumpHere(v, addr1); + }else{ + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, addr1+2); VdbeCoverage(v); + } + sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); + } + }else if( IsVirtual(pTab) || withoutRowid ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowid); + }else{ + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); + appendFlag = 1; + } + autoIncStep(pParse, regAutoinc, regRowid); + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Compute the new value for generated columns after all other + ** columns have already been computed. This must be done after + ** computing the ROWID in case one of the generated columns + ** is derived from the INTEGER PRIMARY KEY. */ + if( pTab->tabFlags & TF_HasGenerated ){ + sqlite3ComputeGeneratedColumns(pParse, regRowid+1, pTab); + } +#endif + + /* Generate code to check constraints and generate index keys and + ** do the insertion. + */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + sqlite3VtabMakeWritable(pParse, pTab); + sqlite3VdbeAddOp4(v, OP_VUpdate, 1, pTab->nCol+2, regIns, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); + sqlite3MayAbort(pParse); + }else +#endif + { + int isReplace = 0;/* Set to true if constraints may cause a replace */ + int bUseSeek; /* True to use OPFLAG_SEEKRESULT */ + sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, + regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0, pUpsert + ); + sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + + /* Set the OPFLAG_USESEEKRESULT flag if either (a) there are no REPLACE + ** constraints or (b) there are no triggers and this table is not a + ** parent table in a foreign key constraint. It is safe to set the + ** flag in the second case as if any REPLACE constraint is hit, an + ** OP_Delete or OP_IdxDelete instruction will be executed on each + ** cursor that is disturbed. And these instructions both clear the + ** VdbeCursor.seekResult variable, disabling the OPFLAG_USESEEKRESULT + ** functionality. */ + bUseSeek = (isReplace==0 || !sqlite3VdbeHasSubProgram(v)); + sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, + regIns, aRegIdx, 0, appendFlag, bUseSeek + ); + } +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + }else if( pParse->bReturning ){ + /* If there is a RETURNING clause, populate the rowid register with + ** constant value -1, in case one or more of the returned expressions + ** refer to the "rowid" of the view. */ + sqlite3VdbeAddOp2(v, OP_Integer, -1, regRowid); +#endif + } + + /* Update the count of rows that are inserted + */ + if( regRowCount ){ + sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); + } + + if( pTrigger ){ + /* Code AFTER triggers */ + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_AFTER, + pTab, regData-2-pTab->nCol, onError, endOfLoop); + } + + /* The bottom of the main insertion loop, if the data source + ** is a SELECT statement. + */ + sqlite3VdbeResolveLabel(v, endOfLoop); + if( useTempTable ){ + sqlite3VdbeAddOp2(v, OP_Next, srcTab, addrCont); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrInsTop); + sqlite3VdbeAddOp1(v, OP_Close, srcTab); + }else if( pSelect ){ + sqlite3VdbeGoto(v, addrCont); +#ifdef SQLITE_DEBUG + /* If we are jumping back to an OP_Yield that is preceded by an + ** OP_ReleaseReg, set the p5 flag on the OP_Goto so that the + ** OP_ReleaseReg will be included in the loop. */ + if( sqlite3VdbeGetOp(v, addrCont-1)->opcode==OP_ReleaseReg ){ + assert( sqlite3VdbeGetOp(v, addrCont)->opcode==OP_Yield ); + sqlite3VdbeChangeP5(v, 1); + } +#endif + sqlite3VdbeJumpHere(v, addrInsTop); + } + +#ifndef SQLITE_OMIT_XFER_OPT +insert_end: +#endif /* SQLITE_OMIT_XFER_OPT */ + /* Update the sqlite_sequence table by storing the content of the + ** maximum rowid counter values recorded while inserting into + ** autoincrement tables. + */ + if( pParse->nested==0 && pParse->pTriggerTab==0 ){ + sqlite3AutoincrementEnd(pParse); + } + + /* + ** Return the number of rows inserted. If this routine is + ** generating code because of a call to sqlite3NestedParse(), do not + ** invoke the callback function. + */ + if( regRowCount ){ + sqlite3VdbeAddOp2(v, OP_ChngCntRow, regRowCount, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", SQLITE_STATIC); + } + +insert_cleanup: + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprListDelete(db, pList); + sqlite3UpsertDelete(db, pUpsert); + sqlite3SelectDelete(db, pSelect); + sqlite3IdListDelete(db, pColumn); + sqlite3DbFree(db, aRegIdx); +} + +/* Make sure "isView" and other macros defined above are undefined. Otherwise +** they may interfere with compilation of other functions in this file +** (or in another file, if this file becomes part of the amalgamation). */ +#ifdef isView + #undef isView +#endif +#ifdef pTrigger + #undef pTrigger +#endif +#ifdef tmask + #undef tmask +#endif + +/* +** Meanings of bits in of pWalker->eCode for +** sqlite3ExprReferencesUpdatedColumn() +*/ +#define CKCNSTRNT_COLUMN 0x01 /* CHECK constraint uses a changing column */ +#define CKCNSTRNT_ROWID 0x02 /* CHECK constraint references the ROWID */ + +/* This is the Walker callback from sqlite3ExprReferencesUpdatedColumn(). +* Set bit 0x01 of pWalker->eCode if pWalker->eCode to 0 and if this +** expression node references any of the +** columns that are being modifed by an UPDATE statement. +*/ +static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN ){ + assert( pExpr->iColumn>=0 || pExpr->iColumn==-1 ); + if( pExpr->iColumn>=0 ){ + if( pWalker->u.aiCol[pExpr->iColumn]>=0 ){ + pWalker->eCode |= CKCNSTRNT_COLUMN; + } + }else{ + pWalker->eCode |= CKCNSTRNT_ROWID; + } + } + return WRC_Continue; +} + +/* +** pExpr is a CHECK constraint on a row that is being UPDATE-ed. The +** only columns that are modified by the UPDATE are those for which +** aiChng[i]>=0, and also the ROWID is modified if chngRowid is true. +** +** Return true if CHECK constraint pExpr uses any of the +** changing columns (or the rowid if it is changing). In other words, +** return true if this CHECK constraint must be validated for +** the new row in the UPDATE statement. +** +** 2018-09-15: pExpr might also be an expression for an index-on-expressions. +** The operation of this routine is the same - return true if an only if +** the expression uses one or more of columns identified by the second and +** third arguments. +*/ +SQLITE_PRIVATE int sqlite3ExprReferencesUpdatedColumn( + Expr *pExpr, /* The expression to be checked */ + int *aiChng, /* aiChng[x]>=0 if column x changed by the UPDATE */ + int chngRowid /* True if UPDATE changes the rowid */ +){ + Walker w; + memset(&w, 0, sizeof(w)); + w.eCode = 0; + w.xExprCallback = checkConstraintExprNode; + w.u.aiCol = aiChng; + sqlite3WalkExpr(&w, pExpr); + if( !chngRowid ){ + testcase( (w.eCode & CKCNSTRNT_ROWID)!=0 ); + w.eCode &= ~CKCNSTRNT_ROWID; + } + testcase( w.eCode==0 ); + testcase( w.eCode==CKCNSTRNT_COLUMN ); + testcase( w.eCode==CKCNSTRNT_ROWID ); + testcase( w.eCode==(CKCNSTRNT_ROWID|CKCNSTRNT_COLUMN) ); + return w.eCode!=0; +} + +/* +** The sqlite3GenerateConstraintChecks() routine usually wants to visit +** the indexes of a table in the order provided in the Table->pIndex list. +** However, sometimes (rarely - when there is an upsert) it wants to visit +** the indexes in a different order. The following data structures accomplish +** this. +** +** The IndexIterator object is used to walk through all of the indexes +** of a table in either Index.pNext order, or in some other order established +** by an array of IndexListTerm objects. +*/ +typedef struct IndexListTerm IndexListTerm; +typedef struct IndexIterator IndexIterator; +struct IndexIterator { + int eType; /* 0 for Index.pNext list. 1 for an array of IndexListTerm */ + int i; /* Index of the current item from the list */ + union { + struct { /* Use this object for eType==0: A Index.pNext list */ + Index *pIdx; /* The current Index */ + } lx; + struct { /* Use this object for eType==1; Array of IndexListTerm */ + int nIdx; /* Size of the array */ + IndexListTerm *aIdx; /* Array of IndexListTerms */ + } ax; + } u; +}; + +/* When IndexIterator.eType==1, then each index is an array of instances +** of the following object +*/ +struct IndexListTerm { + Index *p; /* The index */ + int ix; /* Which entry in the original Table.pIndex list is this index*/ +}; + +/* Return the first index on the list */ +static Index *indexIteratorFirst(IndexIterator *pIter, int *pIx){ + assert( pIter->i==0 ); + if( pIter->eType ){ + *pIx = pIter->u.ax.aIdx[0].ix; + return pIter->u.ax.aIdx[0].p; + }else{ + *pIx = 0; + return pIter->u.lx.pIdx; + } +} + +/* Return the next index from the list. Return NULL when out of indexes */ +static Index *indexIteratorNext(IndexIterator *pIter, int *pIx){ + if( pIter->eType ){ + int i = ++pIter->i; + if( i>=pIter->u.ax.nIdx ){ + *pIx = i; + return 0; + } + *pIx = pIter->u.ax.aIdx[i].ix; + return pIter->u.ax.aIdx[i].p; + }else{ + ++(*pIx); + pIter->u.lx.pIdx = pIter->u.lx.pIdx->pNext; + return pIter->u.lx.pIdx; + } +} + +/* +** Generate code to do constraint checks prior to an INSERT or an UPDATE +** on table pTab. +** +** The regNewData parameter is the first register in a range that contains +** the data to be inserted or the data after the update. There will be +** pTab->nCol+1 registers in this range. The first register (the one +** that regNewData points to) will contain the new rowid, or NULL in the +** case of a WITHOUT ROWID table. The second register in the range will +** contain the content of the first table column. The third register will +** contain the content of the second table column. And so forth. +** +** The regOldData parameter is similar to regNewData except that it contains +** the data prior to an UPDATE rather than afterwards. regOldData is zero +** for an INSERT. This routine can distinguish between UPDATE and INSERT by +** checking regOldData for zero. +** +** For an UPDATE, the pkChng boolean is true if the true primary key (the +** rowid for a normal table or the PRIMARY KEY for a WITHOUT ROWID table) +** might be modified by the UPDATE. If pkChng is false, then the key of +** the iDataCur content table is guaranteed to be unchanged by the UPDATE. +** +** For an INSERT, the pkChng boolean indicates whether or not the rowid +** was explicitly specified as part of the INSERT statement. If pkChng +** is zero, it means that the either rowid is computed automatically or +** that the table is a WITHOUT ROWID table and has no rowid. On an INSERT, +** pkChng will only be true if the INSERT statement provides an integer +** value for either the rowid column or its INTEGER PRIMARY KEY alias. +** +** The code generated by this routine will store new index entries into +** registers identified by aRegIdx[]. No index entry is created for +** indices where aRegIdx[i]==0. The order of indices in aRegIdx[] is +** the same as the order of indices on the linked list of indices +** at pTab->pIndex. +** +** (2019-05-07) The generated code also creates a new record for the +** main table, if pTab is a rowid table, and stores that record in the +** register identified by aRegIdx[nIdx] - in other words in the first +** entry of aRegIdx[] past the last index. It is important that the +** record be generated during constraint checks to avoid affinity changes +** to the register content that occur after constraint checks but before +** the new record is inserted. +** +** The caller must have already opened writeable cursors on the main +** table and all applicable indices (that is to say, all indices for which +** aRegIdx[] is not zero). iDataCur is the cursor for the main table when +** inserting or updating a rowid table, or the cursor for the PRIMARY KEY +** index when operating on a WITHOUT ROWID table. iIdxCur is the cursor +** for the first index in the pTab->pIndex list. Cursors for other indices +** are at iIdxCur+N for the N-th element of the pTab->pIndex list. +** +** This routine also generates code to check constraints. NOT NULL, +** CHECK, and UNIQUE constraints are all checked. If a constraint fails, +** then the appropriate action is performed. There are five possible +** actions: ROLLBACK, ABORT, FAIL, REPLACE, and IGNORE. +** +** Constraint type Action What Happens +** --------------- ---------- ---------------------------------------- +** any ROLLBACK The current transaction is rolled back and +** sqlite3_step() returns immediately with a +** return code of SQLITE_CONSTRAINT. +** +** any ABORT Back out changes from the current command +** only (do not do a complete rollback) then +** cause sqlite3_step() to return immediately +** with SQLITE_CONSTRAINT. +** +** any FAIL Sqlite3_step() returns immediately with a +** return code of SQLITE_CONSTRAINT. The +** transaction is not rolled back and any +** changes to prior rows are retained. +** +** any IGNORE The attempt in insert or update the current +** row is skipped, without throwing an error. +** Processing continues with the next row. +** (There is an immediate jump to ignoreDest.) +** +** NOT NULL REPLACE The NULL value is replace by the default +** value for that column. If the default value +** is NULL, the action is the same as ABORT. +** +** UNIQUE REPLACE The other row that conflicts with the row +** being inserted is removed. +** +** CHECK REPLACE Illegal. The results in an exception. +** +** Which action to take is determined by the overrideError parameter. +** Or if overrideError==OE_Default, then the pParse->onError parameter +** is used. Or if pParse->onError==OE_Default then the onError value +** for the constraint is used. +*/ +SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( + Parse *pParse, /* The parser context */ + Table *pTab, /* The table being inserted or updated */ + int *aRegIdx, /* Use register aRegIdx[i] for index i. 0 for unused */ + int iDataCur, /* Canonical data cursor (main table or PK index) */ + int iIdxCur, /* First index cursor */ + int regNewData, /* First register in a range holding values to insert */ + int regOldData, /* Previous content. 0 for INSERTs */ + u8 pkChng, /* Non-zero if the rowid or PRIMARY KEY changed */ + u8 overrideError, /* Override onError to this if not OE_Default */ + int ignoreDest, /* Jump to this label on an OE_Ignore resolution */ + int *pbMayReplace, /* OUT: Set to true if constraint may cause a replace */ + int *aiChng, /* column i is unchanged if aiChng[i]<0 */ + Upsert *pUpsert /* ON CONFLICT clauses, if any. NULL otherwise */ +){ + Vdbe *v; /* VDBE under constrution */ + Index *pIdx; /* Pointer to one of the indices */ + Index *pPk = 0; /* The PRIMARY KEY index for WITHOUT ROWID tables */ + sqlite3 *db; /* Database connection */ + int i; /* loop counter */ + int ix; /* Index loop counter */ + int nCol; /* Number of columns */ + int onError; /* Conflict resolution strategy */ + int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ + int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ + Upsert *pUpsertClause = 0; /* The specific ON CONFLICT clause for pIdx */ + u8 isUpdate; /* True if this is an UPDATE operation */ + u8 bAffinityDone = 0; /* True if the OP_Affinity operation has been run */ + int upsertIpkReturn = 0; /* Address of Goto at end of IPK uniqueness check */ + int upsertIpkDelay = 0; /* Address of Goto to bypass initial IPK check */ + int ipkTop = 0; /* Top of the IPK uniqueness check */ + int ipkBottom = 0; /* OP_Goto at the end of the IPK uniqueness check */ + /* Variables associated with retesting uniqueness constraints after + ** replace triggers fire have run */ + int regTrigCnt; /* Register used to count replace trigger invocations */ + int addrRecheck = 0; /* Jump here to recheck all uniqueness constraints */ + int lblRecheckOk = 0; /* Each recheck jumps to this label if it passes */ + Trigger *pTrigger; /* List of DELETE triggers on the table pTab */ + int nReplaceTrig = 0; /* Number of replace triggers coded */ + IndexIterator sIdxIter; /* Index iterator */ + + isUpdate = regOldData!=0; + db = pParse->db; + v = pParse->pVdbe; + assert( v!=0 ); + assert( !IsView(pTab) ); /* This table is not a VIEW */ + nCol = pTab->nCol; + + /* pPk is the PRIMARY KEY index for WITHOUT ROWID tables and NULL for + ** normal rowid tables. nPkField is the number of key fields in the + ** pPk index or 1 for a rowid table. In other words, nPkField is the + ** number of fields in the true primary key of the table. */ + if( HasRowid(pTab) ){ + pPk = 0; + nPkField = 1; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + nPkField = pPk->nKeyCol; + } + + /* Record that this module has started */ + VdbeModuleComment((v, "BEGIN: GenCnstCks(%d,%d,%d,%d,%d)", + iDataCur, iIdxCur, regNewData, regOldData, pkChng)); + + /* Test all NOT NULL constraints. + */ + if( pTab->tabFlags & TF_HasNotNull ){ + int b2ndPass = 0; /* True if currently running 2nd pass */ + int nSeenReplace = 0; /* Number of ON CONFLICT REPLACE operations */ + int nGenerated = 0; /* Number of generated columns with NOT NULL */ + while(1){ /* Make 2 passes over columns. Exit loop via "break" */ + for(i=0; iaCol[i]; /* The column to check for NOT NULL */ + int isGenerated; /* non-zero if column is generated */ + onError = pCol->notNull; + if( onError==OE_None ) continue; /* No NOT NULL on this column */ + if( i==pTab->iPKey ){ + continue; /* ROWID is never NULL */ + } + isGenerated = pCol->colFlags & COLFLAG_GENERATED; + if( isGenerated && !b2ndPass ){ + nGenerated++; + continue; /* Generated columns processed on 2nd pass */ + } + if( aiChng && aiChng[i]<0 && !isGenerated ){ + /* Do not check NOT NULL on columns that do not change */ + continue; + } + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + if( onError==OE_Replace ){ + if( b2ndPass /* REPLACE becomes ABORT on the 2nd pass */ + || pCol->iDflt==0 /* REPLACE is ABORT if no DEFAULT value */ + ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + testcase( pCol->colFlags & COLFLAG_GENERATED ); + onError = OE_Abort; + }else{ + assert( !isGenerated ); + } + }else if( b2ndPass && !isGenerated ){ + continue; + } + assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail + || onError==OE_Ignore || onError==OE_Replace ); + testcase( i!=sqlite3TableColumnToStorage(pTab, i) ); + iReg = sqlite3TableColumnToStorage(pTab, i) + regNewData + 1; + switch( onError ){ + case OE_Replace: { + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, iReg); + VdbeCoverage(v); + assert( (pCol->colFlags & COLFLAG_GENERATED)==0 ); + nSeenReplace++; + sqlite3ExprCodeCopy(pParse, + sqlite3ColumnExpr(pTab, pCol), iReg); + sqlite3VdbeJumpHere(v, addr1); + break; + } + case OE_Abort: + sqlite3MayAbort(pParse); + /* no break */ deliberate_fall_through + case OE_Rollback: + case OE_Fail: { + char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, + pCol->zCnName); + sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, + onError, iReg); + sqlite3VdbeAppendP4(v, zMsg, P4_DYNAMIC); + sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); + VdbeCoverage(v); + break; + } + default: { + assert( onError==OE_Ignore ); + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, ignoreDest); + VdbeCoverage(v); + break; + } + } /* end switch(onError) */ + } /* end loop i over columns */ + if( nGenerated==0 && nSeenReplace==0 ){ + /* If there are no generated columns with NOT NULL constraints + ** and no NOT NULL ON CONFLICT REPLACE constraints, then a single + ** pass is sufficient */ + break; + } + if( b2ndPass ) break; /* Never need more than 2 passes */ + b2ndPass = 1; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( nSeenReplace>0 && (pTab->tabFlags & TF_HasGenerated)!=0 ){ + /* If any NOT NULL ON CONFLICT REPLACE constraints fired on the + ** first pass, recomputed values for all generated columns, as + ** those values might depend on columns affected by the REPLACE. + */ + sqlite3ComputeGeneratedColumns(pParse, regNewData+1, pTab); + } +#endif + } /* end of 2-pass loop */ + } /* end if( has-not-null-constraints ) */ + + /* Test all CHECK constraints + */ +#ifndef SQLITE_OMIT_CHECK + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + ExprList *pCheck = pTab->pCheck; + pParse->iSelfTab = -(regNewData+1); + onError = overrideError!=OE_Default ? overrideError : OE_Abort; + for(i=0; inExpr; i++){ + int allOk; + Expr *pCopy; + Expr *pExpr = pCheck->a[i].pExpr; + if( aiChng + && !sqlite3ExprReferencesUpdatedColumn(pExpr, aiChng, pkChng) + ){ + /* The check constraints do not reference any of the columns being + ** updated so there is no point it verifying the check constraint */ + continue; + } + if( bAffinityDone==0 ){ + sqlite3TableAffinity(v, pTab, regNewData+1); + bAffinityDone = 1; + } + allOk = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeVerifyAbortable(v, onError); + pCopy = sqlite3ExprDup(db, pExpr, 0); + if( !db->mallocFailed ){ + sqlite3ExprIfTrue(pParse, pCopy, allOk, SQLITE_JUMPIFNULL); + } + sqlite3ExprDelete(db, pCopy); + if( onError==OE_Ignore ){ + sqlite3VdbeGoto(v, ignoreDest); + }else{ + char *zName = pCheck->a[i].zEName; + assert( zName!=0 || pParse->db->mallocFailed ); + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-26383-51744 */ + sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK, + onError, zName, P4_TRANSIENT, + P5_ConstraintCheck); + } + sqlite3VdbeResolveLabel(v, allOk); + } + pParse->iSelfTab = 0; + } +#endif /* !defined(SQLITE_OMIT_CHECK) */ + + /* UNIQUE and PRIMARY KEY constraints should be handled in the following + ** order: + ** + ** (1) OE_Update + ** (2) OE_Abort, OE_Fail, OE_Rollback, OE_Ignore + ** (3) OE_Replace + ** + ** OE_Fail and OE_Ignore must happen before any changes are made. + ** OE_Update guarantees that only a single row will change, so it + ** must happen before OE_Replace. Technically, OE_Abort and OE_Rollback + ** could happen in any order, but they are grouped up front for + ** convenience. + ** + ** 2018-08-14: Ticket https://www.sqlite.org/src/info/908f001483982c43 + ** The order of constraints used to have OE_Update as (2) and OE_Abort + ** and so forth as (1). But apparently PostgreSQL checks the OE_Update + ** constraint before any others, so it had to be moved. + ** + ** Constraint checking code is generated in this order: + ** (A) The rowid constraint + ** (B) Unique index constraints that do not have OE_Replace as their + ** default conflict resolution strategy + ** (C) Unique index that do use OE_Replace by default. + ** + ** The ordering of (2) and (3) is accomplished by making sure the linked + ** list of indexes attached to a table puts all OE_Replace indexes last + ** in the list. See sqlite3CreateIndex() for where that happens. + */ + sIdxIter.eType = 0; + sIdxIter.i = 0; + sIdxIter.u.ax.aIdx = 0; /* Silence harmless compiler warning */ + sIdxIter.u.lx.pIdx = pTab->pIndex; + if( pUpsert ){ + if( pUpsert->pUpsertTarget==0 ){ + /* There is just on ON CONFLICT clause and it has no constraint-target */ + assert( pUpsert->pNextUpsert==0 ); + if( pUpsert->isDoUpdate==0 ){ + /* A single ON CONFLICT DO NOTHING clause, without a constraint-target. + ** Make all unique constraint resolution be OE_Ignore */ + overrideError = OE_Ignore; + pUpsert = 0; + }else{ + /* A single ON CONFLICT DO UPDATE. Make all resolutions OE_Update */ + overrideError = OE_Update; + } + }else if( pTab->pIndex!=0 ){ + /* Otherwise, we'll need to run the IndexListTerm array version of the + ** iterator to ensure that all of the ON CONFLICT conditions are + ** checked first and in order. */ + int nIdx, jj; + u64 nByte; + Upsert *pTerm; + u8 *bUsed; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ + assert( aRegIdx[nIdx]>0 ); + } + sIdxIter.eType = 1; + sIdxIter.u.ax.nIdx = nIdx; + nByte = (sizeof(IndexListTerm)+1)*nIdx + nIdx; + sIdxIter.u.ax.aIdx = sqlite3DbMallocZero(db, nByte); + if( sIdxIter.u.ax.aIdx==0 ) return; /* OOM */ + bUsed = (u8*)&sIdxIter.u.ax.aIdx[nIdx]; + pUpsert->pToFree = sIdxIter.u.ax.aIdx; + for(i=0, pTerm=pUpsert; pTerm; pTerm=pTerm->pNextUpsert){ + if( pTerm->pUpsertTarget==0 ) break; + if( pTerm->pUpsertIdx==0 ) continue; /* Skip ON CONFLICT for the IPK */ + jj = 0; + pIdx = pTab->pIndex; + while( ALWAYS(pIdx!=0) && pIdx!=pTerm->pUpsertIdx ){ + pIdx = pIdx->pNext; + jj++; + } + if( bUsed[jj] ) continue; /* Duplicate ON CONFLICT clause ignored */ + bUsed[jj] = 1; + sIdxIter.u.ax.aIdx[i].p = pIdx; + sIdxIter.u.ax.aIdx[i].ix = jj; + i++; + } + for(jj=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, jj++){ + if( bUsed[jj] ) continue; + sIdxIter.u.ax.aIdx[i].p = pIdx; + sIdxIter.u.ax.aIdx[i].ix = jj; + i++; + } + assert( i==nIdx ); + } + } + + /* Determine if it is possible that triggers (either explicitly coded + ** triggers or FK resolution actions) might run as a result of deletes + ** that happen when OE_Replace conflict resolution occurs. (Call these + ** "replace triggers".) If any replace triggers run, we will need to + ** recheck all of the uniqueness constraints after they have all run. + ** But on the recheck, the resolution is OE_Abort instead of OE_Replace. + ** + ** If replace triggers are a possibility, then + ** + ** (1) Allocate register regTrigCnt and initialize it to zero. + ** That register will count the number of replace triggers that + ** fire. Constraint recheck only occurs if the number is positive. + ** (2) Initialize pTrigger to the list of all DELETE triggers on pTab. + ** (3) Initialize addrRecheck and lblRecheckOk + ** + ** The uniqueness rechecking code will create a series of tests to run + ** in a second pass. The addrRecheck and lblRecheckOk variables are + ** used to link together these tests which are separated from each other + ** in the generate bytecode. + */ + if( (db->flags & (SQLITE_RecTriggers|SQLITE_ForeignKeys))==0 ){ + /* There are not DELETE triggers nor FK constraints. No constraint + ** rechecks are needed. */ + pTrigger = 0; + regTrigCnt = 0; + }else{ + if( db->flags&SQLITE_RecTriggers ){ + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + regTrigCnt = pTrigger!=0 || sqlite3FkRequired(pParse, pTab, 0, 0); + }else{ + pTrigger = 0; + regTrigCnt = sqlite3FkRequired(pParse, pTab, 0, 0); + } + if( regTrigCnt ){ + /* Replace triggers might exist. Allocate the counter and + ** initialize it to zero. */ + regTrigCnt = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regTrigCnt); + VdbeComment((v, "trigger count")); + lblRecheckOk = sqlite3VdbeMakeLabel(pParse); + addrRecheck = lblRecheckOk; + } + } + + /* If rowid is changing, make sure the new rowid does not previously + ** exist in the table. + */ + if( pkChng && pPk==0 ){ + int addrRowidOk = sqlite3VdbeMakeLabel(pParse); + + /* Figure out what action to take in case of a rowid collision */ + onError = pTab->keyConf; + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + + /* figure out whether or not upsert applies in this case */ + if( pUpsert ){ + pUpsertClause = sqlite3UpsertOfIndex(pUpsert,0); + if( pUpsertClause!=0 ){ + if( pUpsertClause->isDoUpdate==0 ){ + onError = OE_Ignore; /* DO NOTHING is the same as INSERT OR IGNORE */ + }else{ + onError = OE_Update; /* DO UPDATE */ + } + } + if( pUpsertClause!=pUpsert ){ + /* The first ON CONFLICT clause has a conflict target other than + ** the IPK. We have to jump ahead to that first ON CONFLICT clause + ** and then come back here and deal with the IPK afterwards */ + upsertIpkDelay = sqlite3VdbeAddOp0(v, OP_Goto); + } + } + + /* If the response to a rowid conflict is REPLACE but the response + ** to some other UNIQUE constraint is FAIL or IGNORE, then we need + ** to defer the running of the rowid conflict checking until after + ** the UNIQUE constraints have run. + */ + if( onError==OE_Replace /* IPK rule is REPLACE */ + && onError!=overrideError /* Rules for other constraints are different */ + && pTab->pIndex /* There exist other constraints */ + ){ + ipkTop = sqlite3VdbeAddOp0(v, OP_Goto)+1; + VdbeComment((v, "defer IPK REPLACE until last")); + } + + if( isUpdate ){ + /* pkChng!=0 does not mean that the rowid has changed, only that + ** it might have changed. Skip the conflict logic below if the rowid + ** is unchanged. */ + sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRowidOk, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); + } + + /* Check to see if the new rowid already exists in the table. Skip + ** the following conflict logic if it does not. */ + VdbeNoopComment((v, "uniqueness check for ROWID")); + sqlite3VdbeVerifyAbortable(v, onError); + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRowidOk, regNewData); + VdbeCoverage(v); + + switch( onError ){ + default: { + onError = OE_Abort; + /* no break */ deliberate_fall_through + } + case OE_Rollback: + case OE_Abort: + case OE_Fail: { + testcase( onError==OE_Rollback ); + testcase( onError==OE_Abort ); + testcase( onError==OE_Fail ); + sqlite3RowidConstraint(pParse, onError, pTab); + break; + } + case OE_Replace: { + /* If there are DELETE triggers on this table and the + ** recursive-triggers flag is set, call GenerateRowDelete() to + ** remove the conflicting row from the table. This will fire + ** the triggers and remove both the table and index b-tree entries. + ** + ** Otherwise, if there are no triggers or the recursive-triggers + ** flag is not set, but the table has one or more indexes, call + ** GenerateRowIndexDelete(). This removes the index b-tree entries + ** only. The table b-tree entry will be replaced by the new entry + ** when it is inserted. + ** + ** If either GenerateRowDelete() or GenerateRowIndexDelete() is called, + ** also invoke MultiWrite() to indicate that this VDBE may require + ** statement rollback (if the statement is aborted after the delete + ** takes place). Earlier versions called sqlite3MultiWrite() regardless, + ** but being more selective here allows statements like: + ** + ** REPLACE INTO t(rowid) VALUES($newrowid) + ** + ** to run without a statement journal if there are no indexes on the + ** table. + */ + if( regTrigCnt ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + regNewData, 1, 0, OE_Replace, 1, -1); + sqlite3VdbeAddOp2(v, OP_AddImm, regTrigCnt, 1); /* incr trigger cnt */ + nReplaceTrig++; + }else{ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + assert( HasRowid(pTab) ); + /* This OP_Delete opcode fires the pre-update-hook only. It does + ** not modify the b-tree. It is more efficient to let the coming + ** OP_Insert replace the existing entry than it is to delete the + ** existing entry and then insert a new one. */ + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP); + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + if( pTab->pIndex ){ + sqlite3MultiWrite(pParse); + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,-1); + } + } + seenReplace = 1; + break; + } +#ifndef SQLITE_OMIT_UPSERT + case OE_Update: { + sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, 0, iDataCur); + /* no break */ deliberate_fall_through + } +#endif + case OE_Ignore: { + testcase( onError==OE_Ignore ); + sqlite3VdbeGoto(v, ignoreDest); + break; + } + } + sqlite3VdbeResolveLabel(v, addrRowidOk); + if( pUpsert && pUpsertClause!=pUpsert ){ + upsertIpkReturn = sqlite3VdbeAddOp0(v, OP_Goto); + }else if( ipkTop ){ + ipkBottom = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, ipkTop-1); + } + } + + /* Test all UNIQUE constraints by creating entries for each UNIQUE + ** index and making sure that duplicate entries do not already exist. + ** Compute the revised record entries for indices as we go. + ** + ** This loop also handles the case of the PRIMARY KEY index for a + ** WITHOUT ROWID table. + */ + for(pIdx = indexIteratorFirst(&sIdxIter, &ix); + pIdx; + pIdx = indexIteratorNext(&sIdxIter, &ix) + ){ + int regIdx; /* Range of registers hold conent for pIdx */ + int regR; /* Range of registers holding conflicting PK */ + int iThisCur; /* Cursor for this UNIQUE index */ + int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ + int addrConflictCk; /* First opcode in the conflict check logic */ + + if( aRegIdx[ix]==0 ) continue; /* Skip indices that do not change */ + if( pUpsert ){ + pUpsertClause = sqlite3UpsertOfIndex(pUpsert, pIdx); + if( upsertIpkDelay && pUpsertClause==pUpsert ){ + sqlite3VdbeJumpHere(v, upsertIpkDelay); + } + } + addrUniqueOk = sqlite3VdbeMakeLabel(pParse); + if( bAffinityDone==0 ){ + sqlite3TableAffinity(v, pTab, regNewData+1); + bAffinityDone = 1; + } + VdbeNoopComment((v, "prep index %s", pIdx->zName)); + iThisCur = iIdxCur+ix; + + + /* Skip partial indices for which the WHERE clause is not true */ + if( pIdx->pPartIdxWhere ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]); + pParse->iSelfTab = -(regNewData+1); + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, addrUniqueOk, + SQLITE_JUMPIFNULL); + pParse->iSelfTab = 0; + } + + /* Create a record for this index entry as it should appear after + ** the insert or update. Store that record in the aRegIdx[ix] register + */ + regIdx = aRegIdx[ix]+1; + for(i=0; inColumn; i++){ + int iField = pIdx->aiColumn[i]; + int x; + if( iField==XN_EXPR ){ + pParse->iSelfTab = -(regNewData+1); + sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); + pParse->iSelfTab = 0; + VdbeComment((v, "%s column %d", pIdx->zName, i)); + }else if( iField==XN_ROWID || iField==pTab->iPKey ){ + x = regNewData; + sqlite3VdbeAddOp2(v, OP_IntCopy, x, regIdx+i); + VdbeComment((v, "rowid")); + }else{ + testcase( sqlite3TableColumnToStorage(pTab, iField)!=iField ); + x = sqlite3TableColumnToStorage(pTab, iField) + regNewData + 1; + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", pTab->aCol[iField].zCnName)); + } + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); + VdbeComment((v, "for %s", pIdx->zName)); +#ifdef SQLITE_ENABLE_NULL_TRIM + if( pIdx->idxType==SQLITE_IDXTYPE_PRIMARYKEY ){ + sqlite3SetMakeRecordP5(v, pIdx->pTable); + } +#endif + sqlite3VdbeReleaseRegisters(pParse, regIdx, pIdx->nColumn, 0, 0); + + /* In an UPDATE operation, if this index is the PRIMARY KEY index + ** of a WITHOUT ROWID table and there has been no change the + ** primary key, then no collision is possible. The collision detection + ** logic below can all be skipped. */ + if( isUpdate && pPk==pIdx && pkChng==0 ){ + sqlite3VdbeResolveLabel(v, addrUniqueOk); + continue; + } + + /* Find out what action to take in case there is a uniqueness conflict */ + onError = pIdx->onError; + if( onError==OE_None ){ + sqlite3VdbeResolveLabel(v, addrUniqueOk); + continue; /* pIdx is not a UNIQUE index */ + } + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + + /* Figure out if the upsert clause applies to this index */ + if( pUpsertClause ){ + if( pUpsertClause->isDoUpdate==0 ){ + onError = OE_Ignore; /* DO NOTHING is the same as INSERT OR IGNORE */ + }else{ + onError = OE_Update; /* DO UPDATE */ + } + } + + /* Collision detection may be omitted if all of the following are true: + ** (1) The conflict resolution algorithm is REPLACE + ** (2) The table is a WITHOUT ROWID table + ** (3) There are no secondary indexes on the table + ** (4) No delete triggers need to be fired if there is a conflict + ** (5) No FK constraint counters need to be updated if a conflict occurs. + ** + ** This is not possible for ENABLE_PREUPDATE_HOOK builds, as the row + ** must be explicitly deleted in order to ensure any pre-update hook + ** is invoked. */ +#ifndef SQLITE_ENABLE_PREUPDATE_HOOK + if( (ix==0 && pIdx->pNext==0) /* Condition 3 */ + && pPk==pIdx /* Condition 2 */ + && onError==OE_Replace /* Condition 1 */ + && ( 0==(db->flags&SQLITE_RecTriggers) || /* Condition 4 */ + 0==sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0)) + && ( 0==(db->flags&SQLITE_ForeignKeys) || /* Condition 5 */ + (0==pTab->u.tab.pFKey && 0==sqlite3FkReferences(pTab))) + ){ + sqlite3VdbeResolveLabel(v, addrUniqueOk); + continue; + } +#endif /* ifndef SQLITE_ENABLE_PREUPDATE_HOOK */ + + /* Check to see if the new index entry will be unique */ + sqlite3VdbeVerifyAbortable(v, onError); + addrConflictCk = + sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, + regIdx, pIdx->nKeyCol); VdbeCoverage(v); + + /* Generate code to handle collisions */ + regR = pIdx==pPk ? regIdx : sqlite3GetTempRange(pParse, nPkField); + if( isUpdate || onError==OE_Replace ){ + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_IdxRowid, iThisCur, regR); + /* Conflict only if the rowid of the existing index entry + ** is different from old-rowid */ + if( isUpdate ){ + sqlite3VdbeAddOp3(v, OP_Eq, regR, addrUniqueOk, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); + } + }else{ + int x; + /* Extract the PRIMARY KEY from the end of the index entry and + ** store it in registers regR..regR+nPk-1 */ + if( pIdx!=pPk ){ + for(i=0; inKeyCol; i++){ + assert( pPk->aiColumn[i]>=0 ); + x = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[i]); + sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); + VdbeComment((v, "%s.%s", pTab->zName, + pTab->aCol[pPk->aiColumn[i]].zCnName)); + } + } + if( isUpdate ){ + /* If currently processing the PRIMARY KEY of a WITHOUT ROWID + ** table, only conflict if the new PRIMARY KEY values are actually + ** different from the old. + ** + ** For a UNIQUE index, only conflict if the PRIMARY KEY values + ** of the matched index row are different from the original PRIMARY + ** KEY values of this row before the update. */ + int addrJump = sqlite3VdbeCurrentAddr(v)+pPk->nKeyCol; + int op = OP_Ne; + int regCmp = (IsPrimaryKeyIndex(pIdx) ? regIdx : regR); + + for(i=0; inKeyCol; i++){ + char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); + x = pPk->aiColumn[i]; + assert( x>=0 ); + if( i==(pPk->nKeyCol-1) ){ + addrJump = addrUniqueOk; + op = OP_Eq; + } + x = sqlite3TableColumnToStorage(pTab, x); + sqlite3VdbeAddOp4(v, op, + regOldData+1+x, addrJump, regCmp+i, p4, P4_COLLSEQ + ); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverageIf(v, op==OP_Eq); + VdbeCoverageIf(v, op==OP_Ne); + } + } + } + } + + /* Generate code that executes if the new index entry is not unique */ + assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail + || onError==OE_Ignore || onError==OE_Replace || onError==OE_Update ); + switch( onError ){ + case OE_Rollback: + case OE_Abort: + case OE_Fail: { + testcase( onError==OE_Rollback ); + testcase( onError==OE_Abort ); + testcase( onError==OE_Fail ); + sqlite3UniqueConstraint(pParse, onError, pIdx); + break; + } +#ifndef SQLITE_OMIT_UPSERT + case OE_Update: { + sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, pIdx, iIdxCur+ix); + /* no break */ deliberate_fall_through + } +#endif + case OE_Ignore: { + testcase( onError==OE_Ignore ); + sqlite3VdbeGoto(v, ignoreDest); + break; + } + default: { + int nConflictCk; /* Number of opcodes in conflict check logic */ + + assert( onError==OE_Replace ); + nConflictCk = sqlite3VdbeCurrentAddr(v) - addrConflictCk; + assert( nConflictCk>0 ); + testcase( nConflictCk>1 ); + if( regTrigCnt ){ + sqlite3MultiWrite(pParse); + nReplaceTrig++; + } + if( pTrigger && isUpdate ){ + sqlite3VdbeAddOp1(v, OP_CursorLock, iDataCur); + } + sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, + regR, nPkField, 0, OE_Replace, + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), iThisCur); + if( pTrigger && isUpdate ){ + sqlite3VdbeAddOp1(v, OP_CursorUnlock, iDataCur); + } + if( regTrigCnt ){ + int addrBypass; /* Jump destination to bypass recheck logic */ + + sqlite3VdbeAddOp2(v, OP_AddImm, regTrigCnt, 1); /* incr trigger cnt */ + addrBypass = sqlite3VdbeAddOp0(v, OP_Goto); /* Bypass recheck */ + VdbeComment((v, "bypass recheck")); + + /* Here we insert code that will be invoked after all constraint + ** checks have run, if and only if one or more replace triggers + ** fired. */ + sqlite3VdbeResolveLabel(v, lblRecheckOk); + lblRecheckOk = sqlite3VdbeMakeLabel(pParse); + if( pIdx->pPartIdxWhere ){ + /* Bypass the recheck if this partial index is not defined + ** for the current row */ + sqlite3VdbeAddOp2(v, OP_IsNull, regIdx-1, lblRecheckOk); + VdbeCoverage(v); + } + /* Copy the constraint check code from above, except change + ** the constraint-ok jump destination to be the address of + ** the next retest block */ + while( nConflictCk>0 ){ + VdbeOp x; /* Conflict check opcode to copy */ + /* The sqlite3VdbeAddOp4() call might reallocate the opcode array. + ** Hence, make a complete copy of the opcode, rather than using + ** a pointer to the opcode. */ + x = *sqlite3VdbeGetOp(v, addrConflictCk); + if( x.opcode!=OP_IdxRowid ){ + int p2; /* New P2 value for copied conflict check opcode */ + const char *zP4; + if( sqlite3OpcodeProperty[x.opcode]&OPFLG_JUMP ){ + p2 = lblRecheckOk; + }else{ + p2 = x.p2; + } + zP4 = x.p4type==P4_INT32 ? SQLITE_INT_TO_PTR(x.p4.i) : x.p4.z; + sqlite3VdbeAddOp4(v, x.opcode, x.p1, p2, x.p3, zP4, x.p4type); + sqlite3VdbeChangeP5(v, x.p5); + VdbeCoverageIf(v, p2!=x.p2); + } + nConflictCk--; + addrConflictCk++; + } + /* If the retest fails, issue an abort */ + sqlite3UniqueConstraint(pParse, OE_Abort, pIdx); + + sqlite3VdbeJumpHere(v, addrBypass); /* Terminate the recheck bypass */ + } + seenReplace = 1; + break; + } + } + sqlite3VdbeResolveLabel(v, addrUniqueOk); + if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); + if( pUpsertClause + && upsertIpkReturn + && sqlite3UpsertNextIsIPK(pUpsertClause) + ){ + sqlite3VdbeGoto(v, upsertIpkDelay+1); + sqlite3VdbeJumpHere(v, upsertIpkReturn); + upsertIpkReturn = 0; + } + } + + /* If the IPK constraint is a REPLACE, run it last */ + if( ipkTop ){ + sqlite3VdbeGoto(v, ipkTop); + VdbeComment((v, "Do IPK REPLACE")); + sqlite3VdbeJumpHere(v, ipkBottom); + } + + /* Recheck all uniqueness constraints after replace triggers have run */ + testcase( regTrigCnt!=0 && nReplaceTrig==0 ); + assert( regTrigCnt!=0 || nReplaceTrig==0 ); + if( nReplaceTrig ){ + sqlite3VdbeAddOp2(v, OP_IfNot, regTrigCnt, lblRecheckOk);VdbeCoverage(v); + if( !pPk ){ + if( isUpdate ){ + sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRecheck, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRecheck, regNewData); + VdbeCoverage(v); + sqlite3RowidConstraint(pParse, OE_Abort, pTab); + }else{ + sqlite3VdbeGoto(v, addrRecheck); + } + sqlite3VdbeResolveLabel(v, lblRecheckOk); + } + + /* Generate the table record */ + if( HasRowid(pTab) ){ + int regRec = aRegIdx[ix]; + sqlite3VdbeAddOp3(v, OP_MakeRecord, regNewData+1, pTab->nNVCol, regRec); + sqlite3SetMakeRecordP5(v, pTab); + if( !bAffinityDone ){ + sqlite3TableAffinity(v, pTab, 0); + } + } + + *pbMayReplace = seenReplace; + VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace)); +} + +#ifdef SQLITE_ENABLE_NULL_TRIM +/* +** Change the P5 operand on the last opcode (which should be an OP_MakeRecord) +** to be the number of columns in table pTab that must not be NULL-trimmed. +** +** Or if no columns of pTab may be NULL-trimmed, leave P5 at zero. +*/ +SQLITE_PRIVATE void sqlite3SetMakeRecordP5(Vdbe *v, Table *pTab){ + u16 i; + + /* Records with omitted columns are only allowed for schema format + ** version 2 and later (SQLite version 3.1.4, 2005-02-20). */ + if( pTab->pSchema->file_format<2 ) return; + + for(i=pTab->nCol-1; i>0; i--){ + if( pTab->aCol[i].iDflt!=0 ) break; + if( pTab->aCol[i].colFlags & COLFLAG_PRIMKEY ) break; + } + sqlite3VdbeChangeP5(v, i+1); +} +#endif + +/* +** Table pTab is a WITHOUT ROWID table that is being written to. The cursor +** number is iCur, and register regData contains the new record for the +** PK index. This function adds code to invoke the pre-update hook, +** if one is registered. +*/ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +static void codeWithoutRowidPreupdate( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated */ + int iCur, /* Cursor number for table */ + int regData /* Data containing new record */ +){ + Vdbe *v = pParse->pVdbe; + int r = sqlite3GetTempReg(pParse); + assert( !HasRowid(pTab) ); + assert( 0==(pParse->db->mDbFlags & DBFLAG_Vacuum) || CORRUPT_DB ); + sqlite3VdbeAddOp2(v, OP_Integer, 0, r); + sqlite3VdbeAddOp4(v, OP_Insert, iCur, regData, r, (char*)pTab, P4_TABLE); + sqlite3VdbeChangeP5(v, OPFLAG_ISNOOP); + sqlite3ReleaseTempReg(pParse, r); +} +#else +# define codeWithoutRowidPreupdate(a,b,c,d) +#endif + +/* +** This routine generates code to finish the INSERT or UPDATE operation +** that was started by a prior call to sqlite3GenerateConstraintChecks. +** A consecutive range of registers starting at regNewData contains the +** rowid and the content to be inserted. +** +** The arguments to this routine should be the same as the first six +** arguments to sqlite3GenerateConstraintChecks. +*/ +SQLITE_PRIVATE void sqlite3CompleteInsertion( + Parse *pParse, /* The parser context */ + Table *pTab, /* the table into which we are inserting */ + int iDataCur, /* Cursor of the canonical data source */ + int iIdxCur, /* First index cursor */ + int regNewData, /* Range of content */ + int *aRegIdx, /* Register used by each index. 0 for unused indices */ + int update_flags, /* True for UPDATE, False for INSERT */ + int appendBias, /* True if this is likely to be an append */ + int useSeekResult /* True to set the USESEEKRESULT flag on OP_[Idx]Insert */ +){ + Vdbe *v; /* Prepared statements under construction */ + Index *pIdx; /* An index being inserted or updated */ + u8 pik_flags; /* flag values passed to the btree insert */ + int i; /* Loop counter */ + + assert( update_flags==0 + || update_flags==OPFLAG_ISUPDATE + || update_flags==(OPFLAG_ISUPDATE|OPFLAG_SAVEPOSITION) + ); + + v = pParse->pVdbe; + assert( v!=0 ); + assert( !IsView(pTab) ); /* This table is not a VIEW */ + for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + /* All REPLACE indexes are at the end of the list */ + assert( pIdx->onError!=OE_Replace + || pIdx->pNext==0 + || pIdx->pNext->onError==OE_Replace ); + if( aRegIdx[i]==0 ) continue; + if( pIdx->pPartIdxWhere ){ + sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + } + pik_flags = (useSeekResult ? OPFLAG_USESEEKRESULT : 0); + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + assert( pParse->nested==0 ); + pik_flags |= OPFLAG_NCHANGE; + pik_flags |= (update_flags & OPFLAG_SAVEPOSITION); + if( update_flags==0 ){ + codeWithoutRowidPreupdate(pParse, pTab, iIdxCur+i, aRegIdx[i]); + } + } + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i], + aRegIdx[i]+1, + pIdx->uniqNotNull ? pIdx->nKeyCol: pIdx->nColumn); + sqlite3VdbeChangeP5(v, pik_flags); + } + if( !HasRowid(pTab) ) return; + if( pParse->nested ){ + pik_flags = 0; + }else{ + pik_flags = OPFLAG_NCHANGE; + pik_flags |= (update_flags?update_flags:OPFLAG_LASTROWID); + } + if( appendBias ){ + pik_flags |= OPFLAG_APPEND; + } + if( useSeekResult ){ + pik_flags |= OPFLAG_USESEEKRESULT; + } + sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, aRegIdx[i], regNewData); + if( !pParse->nested ){ + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); + } + sqlite3VdbeChangeP5(v, pik_flags); +} + +/* +** Allocate cursors for the pTab table and all its indices and generate +** code to open and initialized those cursors. +** +** The cursor for the object that contains the complete data (normally +** the table itself, but the PRIMARY KEY index in the case of a WITHOUT +** ROWID table) is returned in *piDataCur. The first index cursor is +** returned in *piIdxCur. The number of indices is returned. +** +** Use iBase as the first cursor (either the *piDataCur for rowid tables +** or the first index for WITHOUT ROWID tables) if it is non-negative. +** If iBase is negative, then allocate the next available cursor. +** +** For a rowid table, *piDataCur will be exactly one less than *piIdxCur. +** For a WITHOUT ROWID table, *piDataCur will be somewhere in the range +** of *piIdxCurs, depending on where the PRIMARY KEY index appears on the +** pTab->pIndex list. +** +** If pTab is a virtual table, then this routine is a no-op and the +** *piDataCur and *piIdxCur values are left uninitialized. +*/ +SQLITE_PRIVATE int sqlite3OpenTableAndIndices( + Parse *pParse, /* Parsing context */ + Table *pTab, /* Table to be opened */ + int op, /* OP_OpenRead or OP_OpenWrite */ + u8 p5, /* P5 value for OP_Open* opcodes (except on WITHOUT ROWID) */ + int iBase, /* Use this for the table cursor, if there is one */ + u8 *aToOpen, /* If not NULL: boolean for each table and index */ + int *piDataCur, /* Write the database source cursor number here */ + int *piIdxCur /* Write the first index cursor number here */ +){ + int i; + int iDb; + int iDataCur; + Index *pIdx; + Vdbe *v; + + assert( op==OP_OpenRead || op==OP_OpenWrite ); + assert( op==OP_OpenWrite || p5==0 ); + if( IsVirtual(pTab) ){ + /* This routine is a no-op for virtual tables. Leave the output + ** variables *piDataCur and *piIdxCur uninitialized so that valgrind + ** can detect if they are used by mistake in the caller. */ + return 0; + } + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + v = pParse->pVdbe; + assert( v!=0 ); + if( iBase<0 ) iBase = pParse->nTab; + iDataCur = iBase++; + if( piDataCur ) *piDataCur = iDataCur; + if( HasRowid(pTab) && (aToOpen==0 || aToOpen[0]) ){ + sqlite3OpenTable(pParse, iDataCur, iDb, pTab, op); + }else{ + sqlite3TableLock(pParse, iDb, pTab->tnum, op==OP_OpenWrite, pTab->zName); + } + if( piIdxCur ) *piIdxCur = iBase; + for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + int iIdxCur = iBase++; + assert( pIdx->pSchema==pTab->pSchema ); + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + if( piDataCur ) *piDataCur = iIdxCur; + p5 = 0; + } + if( aToOpen==0 || aToOpen[i+1] ){ + sqlite3VdbeAddOp3(v, op, iIdxCur, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + sqlite3VdbeChangeP5(v, p5); + VdbeComment((v, "%s", pIdx->zName)); + } + } + if( iBase>pParse->nTab ) pParse->nTab = iBase; + return i; +} + + +#ifdef SQLITE_TEST +/* +** The following global variable is incremented whenever the +** transfer optimization is used. This is used for testing +** purposes only - to make sure the transfer optimization really +** is happening when it is supposed to. +*/ +SQLITE_API int sqlite3_xferopt_count; +#endif /* SQLITE_TEST */ + + +#ifndef SQLITE_OMIT_XFER_OPT +/* +** Check to see if index pSrc is compatible as a source of data +** for index pDest in an insert transfer optimization. The rules +** for a compatible index: +** +** * The index is over the same set of columns +** * The same DESC and ASC markings occurs on all columns +** * The same onError processing (OE_Abort, OE_Ignore, etc) +** * The same collating sequence on each column +** * The index has the exact same WHERE clause +*/ +static int xferCompatibleIndex(Index *pDest, Index *pSrc){ + int i; + assert( pDest && pSrc ); + assert( pDest->pTable!=pSrc->pTable ); + if( pDest->nKeyCol!=pSrc->nKeyCol || pDest->nColumn!=pSrc->nColumn ){ + return 0; /* Different number of columns */ + } + if( pDest->onError!=pSrc->onError ){ + return 0; /* Different conflict resolution strategies */ + } + for(i=0; inKeyCol; i++){ + if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ + return 0; /* Different columns indexed */ + } + if( pSrc->aiColumn[i]==XN_EXPR ){ + assert( pSrc->aColExpr!=0 && pDest->aColExpr!=0 ); + if( sqlite3ExprCompare(0, pSrc->aColExpr->a[i].pExpr, + pDest->aColExpr->a[i].pExpr, -1)!=0 ){ + return 0; /* Different expressions in the index */ + } + } + if( pSrc->aSortOrder[i]!=pDest->aSortOrder[i] ){ + return 0; /* Different sort orders */ + } + if( sqlite3_stricmp(pSrc->azColl[i],pDest->azColl[i])!=0 ){ + return 0; /* Different collating sequences */ + } + } + if( sqlite3ExprCompare(0, pSrc->pPartIdxWhere, pDest->pPartIdxWhere, -1) ){ + return 0; /* Different WHERE clauses */ + } + + /* If no test above fails then the indices must be compatible */ + return 1; +} + +/* +** Attempt the transfer optimization on INSERTs of the form +** +** INSERT INTO tab1 SELECT * FROM tab2; +** +** The xfer optimization transfers raw records from tab2 over to tab1. +** Columns are not decoded and reassembled, which greatly improves +** performance. Raw index records are transferred in the same way. +** +** The xfer optimization is only attempted if tab1 and tab2 are compatible. +** There are lots of rules for determining compatibility - see comments +** embedded in the code for details. +** +** This routine returns TRUE if the optimization is guaranteed to be used. +** Sometimes the xfer optimization will only work if the destination table +** is empty - a factor that can only be determined at run-time. In that +** case, this routine generates code for the xfer optimization but also +** does a test to see if the destination table is empty and jumps over the +** xfer optimization code if the test fails. In that case, this routine +** returns FALSE so that the caller will know to go ahead and generate +** an unoptimized transfer. This routine also returns FALSE if there +** is no chance that the xfer optimization can be applied. +** +** This optimization is particularly useful at making VACUUM run faster. +*/ +static int xferOptimization( + Parse *pParse, /* Parser context */ + Table *pDest, /* The table we are inserting into */ + Select *pSelect, /* A SELECT statement to use as the data source */ + int onError, /* How to handle constraint errors */ + int iDbDest /* The database of pDest */ +){ + sqlite3 *db = pParse->db; + ExprList *pEList; /* The result set of the SELECT */ + Table *pSrc; /* The table in the FROM clause of SELECT */ + Index *pSrcIdx, *pDestIdx; /* Source and destination indices */ + SrcItem *pItem; /* An element of pSelect->pSrc */ + int i; /* Loop counter */ + int iDbSrc; /* The database of pSrc */ + int iSrc, iDest; /* Cursors from source and destination */ + int addr1, addr2; /* Loop addresses */ + int emptyDestTest = 0; /* Address of test for empty pDest */ + int emptySrcTest = 0; /* Address of test for empty pSrc */ + Vdbe *v; /* The VDBE we are building */ + int regAutoinc; /* Memory register used by AUTOINC */ + int destHasUniqueIdx = 0; /* True if pDest has a UNIQUE index */ + int regData, regRowid; /* Registers holding data and rowid */ + + if( pSelect==0 ){ + return 0; /* Must be of the form INSERT INTO ... SELECT ... */ + } + if( pParse->pWith || pSelect->pWith ){ + /* Do not attempt to process this query if there are an WITH clauses + ** attached to it. Proceeding may generate a false "no such table: xxx" + ** error if pSelect reads from a CTE named "xxx". */ + return 0; + } + if( sqlite3TriggerList(pParse, pDest) ){ + return 0; /* tab1 must not have triggers */ + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pDest) ){ + return 0; /* tab1 must not be a virtual table */ + } +#endif + if( onError==OE_Default ){ + if( pDest->iPKey>=0 ) onError = pDest->keyConf; + if( onError==OE_Default ) onError = OE_Abort; + } + assert(pSelect->pSrc); /* allocated even if there is no FROM clause */ + if( pSelect->pSrc->nSrc!=1 ){ + return 0; /* FROM clause must have exactly one term */ + } + if( pSelect->pSrc->a[0].pSelect ){ + return 0; /* FROM clause cannot contain a subquery */ + } + if( pSelect->pWhere ){ + return 0; /* SELECT may not have a WHERE clause */ + } + if( pSelect->pOrderBy ){ + return 0; /* SELECT may not have an ORDER BY clause */ + } + /* Do not need to test for a HAVING clause. If HAVING is present but + ** there is no ORDER BY, we will get an error. */ + if( pSelect->pGroupBy ){ + return 0; /* SELECT may not have a GROUP BY clause */ + } + if( pSelect->pLimit ){ + return 0; /* SELECT may not have a LIMIT clause */ + } + if( pSelect->pPrior ){ + return 0; /* SELECT may not be a compound query */ + } + if( pSelect->selFlags & SF_Distinct ){ + return 0; /* SELECT may not be DISTINCT */ + } + pEList = pSelect->pEList; + assert( pEList!=0 ); + if( pEList->nExpr!=1 ){ + return 0; /* The result set must have exactly one column */ + } + assert( pEList->a[0].pExpr ); + if( pEList->a[0].pExpr->op!=TK_ASTERISK ){ + return 0; /* The result set must be the special operator "*" */ + } + + /* At this point we have established that the statement is of the + ** correct syntactic form to participate in this optimization. Now + ** we have to check the semantics. + */ + pItem = pSelect->pSrc->a; + pSrc = sqlite3LocateTableItem(pParse, 0, pItem); + if( pSrc==0 ){ + return 0; /* FROM clause does not contain a real table */ + } + if( pSrc->tnum==pDest->tnum && pSrc->pSchema==pDest->pSchema ){ + testcase( pSrc!=pDest ); /* Possible due to bad sqlite_schema.rootpage */ + return 0; /* tab1 and tab2 may not be the same table */ + } + if( HasRowid(pDest)!=HasRowid(pSrc) ){ + return 0; /* source and destination must both be WITHOUT ROWID or not */ + } + if( !IsOrdinaryTable(pSrc) ){ + return 0; /* tab2 may not be a view or virtual table */ + } + if( pDest->nCol!=pSrc->nCol ){ + return 0; /* Number of columns must be the same in tab1 and tab2 */ + } + if( pDest->iPKey!=pSrc->iPKey ){ + return 0; /* Both tables must have the same INTEGER PRIMARY KEY */ + } + if( (pDest->tabFlags & TF_Strict)!=0 && (pSrc->tabFlags & TF_Strict)==0 ){ + return 0; /* Cannot feed from a non-strict into a strict table */ + } + for(i=0; inCol; i++){ + Column *pDestCol = &pDest->aCol[i]; + Column *pSrcCol = &pSrc->aCol[i]; +#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS + if( (db->mDbFlags & DBFLAG_Vacuum)==0 + && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN + ){ + return 0; /* Neither table may have __hidden__ columns */ + } +#endif +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Even if tables t1 and t2 have identical schemas, if they contain + ** generated columns, then this statement is semantically incorrect: + ** + ** INSERT INTO t2 SELECT * FROM t1; + ** + ** The reason is that generated column values are returned by the + ** the SELECT statement on the right but the INSERT statement on the + ** left wants them to be omitted. + ** + ** Nevertheless, this is a useful notational shorthand to tell SQLite + ** to do a bulk transfer all of the content from t1 over to t2. + ** + ** We could, in theory, disable this (except for internal use by the + ** VACUUM command where it is actually needed). But why do that? It + ** seems harmless enough, and provides a useful service. + */ + if( (pDestCol->colFlags & COLFLAG_GENERATED) != + (pSrcCol->colFlags & COLFLAG_GENERATED) ){ + return 0; /* Both columns have the same generated-column type */ + } + /* But the transfer is only allowed if both the source and destination + ** tables have the exact same expressions for generated columns. + ** This requirement could be relaxed for VIRTUAL columns, I suppose. + */ + if( (pDestCol->colFlags & COLFLAG_GENERATED)!=0 ){ + if( sqlite3ExprCompare(0, + sqlite3ColumnExpr(pSrc, pSrcCol), + sqlite3ColumnExpr(pDest, pDestCol), -1)!=0 ){ + testcase( pDestCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pDestCol->colFlags & COLFLAG_STORED ); + return 0; /* Different generator expressions */ + } + } +#endif + if( pDestCol->affinity!=pSrcCol->affinity ){ + return 0; /* Affinity must be the same on all columns */ + } + if( sqlite3_stricmp(sqlite3ColumnColl(pDestCol), + sqlite3ColumnColl(pSrcCol))!=0 ){ + return 0; /* Collating sequence must be the same on all columns */ + } + if( pDestCol->notNull && !pSrcCol->notNull ){ + return 0; /* tab2 must be NOT NULL if tab1 is */ + } + /* Default values for second and subsequent columns need to match. */ + if( (pDestCol->colFlags & COLFLAG_GENERATED)==0 && i>0 ){ + Expr *pDestExpr = sqlite3ColumnExpr(pDest, pDestCol); + Expr *pSrcExpr = sqlite3ColumnExpr(pSrc, pSrcCol); + assert( pDestExpr==0 || pDestExpr->op==TK_SPAN ); + assert( pSrcExpr==0 || pSrcExpr->op==TK_SPAN ); + if( (pDestExpr==0)!=(pSrcExpr==0) + || (pDestExpr!=0 && strcmp(pDestExpr->u.zToken, + pSrcExpr->u.zToken)!=0) + ){ + return 0; /* Default values must be the same for all columns */ + } + } + } + for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ + if( IsUniqueIndex(pDestIdx) ){ + destHasUniqueIdx = 1; + } + for(pSrcIdx=pSrc->pIndex; pSrcIdx; pSrcIdx=pSrcIdx->pNext){ + if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; + } + if( pSrcIdx==0 ){ + return 0; /* pDestIdx has no corresponding index in pSrc */ + } + if( pSrcIdx->tnum==pDestIdx->tnum && pSrc->pSchema==pDest->pSchema + && sqlite3FaultSim(411)==SQLITE_OK ){ + /* The sqlite3FaultSim() call allows this corruption test to be + ** bypassed during testing, in order to exercise other corruption tests + ** further downstream. */ + return 0; /* Corrupt schema - two indexes on the same btree */ + } + } +#ifndef SQLITE_OMIT_CHECK + if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + return 0; /* Tables have different CHECK constraints. Ticket #2252 */ + } +#endif +#ifndef SQLITE_OMIT_FOREIGN_KEY + /* Disallow the transfer optimization if the destination table constains + ** any foreign key constraints. This is more restrictive than necessary. + ** But the main beneficiary of the transfer optimization is the VACUUM + ** command, and the VACUUM command disables foreign key constraints. So + ** the extra complication to make this rule less restrictive is probably + ** not worth the effort. Ticket [6284df89debdfa61db8073e062908af0c9b6118e] + */ + if( (db->flags & SQLITE_ForeignKeys)!=0 && pDest->u.tab.pFKey!=0 ){ + return 0; + } +#endif + if( (db->flags & SQLITE_CountRows)!=0 ){ + return 0; /* xfer opt does not play well with PRAGMA count_changes */ + } + + /* If we get this far, it means that the xfer optimization is at + ** least a possibility, though it might only work if the destination + ** table (tab1) is initially empty. + */ +#ifdef SQLITE_TEST + sqlite3_xferopt_count++; +#endif + iDbSrc = sqlite3SchemaToIndex(db, pSrc->pSchema); + v = sqlite3GetVdbe(pParse); + sqlite3CodeVerifySchema(pParse, iDbSrc); + iSrc = pParse->nTab++; + iDest = pParse->nTab++; + regAutoinc = autoIncBegin(pParse, iDbDest, pDest); + regData = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Null, 0, regData); + regRowid = sqlite3GetTempReg(pParse); + sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); + assert( HasRowid(pDest) || destHasUniqueIdx ); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 && ( + (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ + || destHasUniqueIdx /* (2) */ + || (onError!=OE_Abort && onError!=OE_Rollback) /* (3) */ + )){ + /* In some circumstances, we are able to run the xfer optimization + ** only if the destination table is initially empty. Unless the + ** DBFLAG_Vacuum flag is set, this block generates code to make + ** that determination. If DBFLAG_Vacuum is set, then the destination + ** table is always empty. + ** + ** Conditions under which the destination must be empty: + ** + ** (1) There is no INTEGER PRIMARY KEY but there are indices. + ** (If the destination is not initially empty, the rowid fields + ** of index entries might need to change.) + ** + ** (2) The destination has a unique index. (The xfer optimization + ** is unable to test uniqueness.) + ** + ** (3) onError is something other than OE_Abort and OE_Rollback. + */ + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iDest, 0); VdbeCoverage(v); + emptyDestTest = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, addr1); + } + if( HasRowid(pSrc) ){ + u8 insFlags; + sqlite3OpenTable(pParse, iSrc, iDbSrc, pSrc, OP_OpenRead); + emptySrcTest = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); + if( pDest->iPKey>=0 ){ + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 ){ + sqlite3VdbeVerifyAbortable(v, onError); + addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); + VdbeCoverage(v); + sqlite3RowidConstraint(pParse, onError, pDest); + sqlite3VdbeJumpHere(v, addr2); + } + autoIncStep(pParse, regAutoinc, regRowid); + }else if( pDest->pIndex==0 && !(db->mDbFlags & DBFLAG_VacuumInto) ){ + addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid); + }else{ + addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); + assert( (pDest->tabFlags & TF_Autoincrement)==0 ); + } + + if( db->mDbFlags & DBFLAG_Vacuum ){ + sqlite3VdbeAddOp1(v, OP_SeekEnd, iDest); + insFlags = OPFLAG_APPEND|OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT; + }else{ + insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND|OPFLAG_PREFORMAT; + } +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( (db->mDbFlags & DBFLAG_Vacuum)==0 ){ + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); + insFlags &= ~OPFLAG_PREFORMAT; + }else +#endif + { + sqlite3VdbeAddOp3(v, OP_RowCell, iDest, iSrc, regRowid); + } + sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 ){ + sqlite3VdbeChangeP4(v, -1, (char*)pDest, P4_TABLE); + } + sqlite3VdbeChangeP5(v, insFlags); + + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + }else{ + sqlite3TableLock(pParse, iDbDest, pDest->tnum, 1, pDest->zName); + sqlite3TableLock(pParse, iDbSrc, pSrc->tnum, 0, pSrc->zName); + } + for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ + u8 idxInsFlags = 0; + for(pSrcIdx=pSrc->pIndex; ALWAYS(pSrcIdx); pSrcIdx=pSrcIdx->pNext){ + if( xferCompatibleIndex(pDestIdx, pSrcIdx) ) break; + } + assert( pSrcIdx ); + sqlite3VdbeAddOp3(v, OP_OpenRead, iSrc, pSrcIdx->tnum, iDbSrc); + sqlite3VdbeSetP4KeyInfo(pParse, pSrcIdx); + VdbeComment((v, "%s", pSrcIdx->zName)); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iDest, pDestIdx->tnum, iDbDest); + sqlite3VdbeSetP4KeyInfo(pParse, pDestIdx); + sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); + VdbeComment((v, "%s", pDestIdx->zName)); + addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); + if( db->mDbFlags & DBFLAG_Vacuum ){ + /* This INSERT command is part of a VACUUM operation, which guarantees + ** that the destination table is empty. If all indexed columns use + ** collation sequence BINARY, then it can also be assumed that the + ** index will be populated by inserting keys in strictly sorted + ** order. In this case, instead of seeking within the b-tree as part + ** of every OP_IdxInsert opcode, an OP_SeekEnd is added before the + ** OP_IdxInsert to seek to the point within the b-tree where each key + ** should be inserted. This is faster. + ** + ** If any of the indexed columns use a collation sequence other than + ** BINARY, this optimization is disabled. This is because the user + ** might change the definition of a collation sequence and then run + ** a VACUUM command. In that case keys may not be written in strictly + ** sorted order. */ + for(i=0; inColumn; i++){ + const char *zColl = pSrcIdx->azColl[i]; + if( sqlite3_stricmp(sqlite3StrBINARY, zColl) ) break; + } + if( i==pSrcIdx->nColumn ){ + idxInsFlags = OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT; + sqlite3VdbeAddOp1(v, OP_SeekEnd, iDest); + sqlite3VdbeAddOp2(v, OP_RowCell, iDest, iSrc); + } + }else if( !HasRowid(pSrc) && pDestIdx->idxType==SQLITE_IDXTYPE_PRIMARYKEY ){ + idxInsFlags |= OPFLAG_NCHANGE; + } + if( idxInsFlags!=(OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT) ){ + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 + && !HasRowid(pDest) + && IsPrimaryKeyIndex(pDestIdx) + ){ + codeWithoutRowidPreupdate(pParse, pDest, iDest, regData); + } + } + sqlite3VdbeAddOp2(v, OP_IdxInsert, iDest, regData); + sqlite3VdbeChangeP5(v, idxInsFlags|OPFLAG_APPEND); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + } + if( emptySrcTest ) sqlite3VdbeJumpHere(v, emptySrcTest); + sqlite3ReleaseTempReg(pParse, regRowid); + sqlite3ReleaseTempReg(pParse, regData); + if( emptyDestTest ){ + sqlite3AutoincrementEnd(pParse); + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); + sqlite3VdbeJumpHere(v, emptyDestTest); + sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); + return 0; + }else{ + return 1; + } +} +#endif /* SQLITE_OMIT_XFER_OPT */ + +/************** End of insert.c **********************************************/ +/************** Begin file legacy.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. +*/ + +/* #include "sqliteInt.h" */ + +/* +** Execute SQL code. Return one of the SQLITE_ success/failure +** codes. Also write an error message into memory obtained from +** malloc() and make *pzErrMsg point to that message. +** +** If the SQL is a query, then for each row in the query result +** the xCallback() function is called. pArg becomes the first +** argument to xCallback(). If xCallback=NULL then no callback +** is invoked, even for queries. +*/ +SQLITE_API int sqlite3_exec( + sqlite3 *db, /* The database on which the SQL executes */ + const char *zSql, /* The SQL to be executed */ + sqlite3_callback xCallback, /* Invoke this callback routine */ + void *pArg, /* First argument to xCallback() */ + char **pzErrMsg /* Write error messages here */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zLeftover; /* Tail of unprocessed SQL */ + sqlite3_stmt *pStmt = 0; /* The current SQL statement */ + char **azCols = 0; /* Names of result columns */ + int callbackIsInit; /* True if callback data is initialized */ + + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; + if( zSql==0 ) zSql = ""; + + sqlite3_mutex_enter(db->mutex); + sqlite3Error(db, SQLITE_OK); + while( rc==SQLITE_OK && zSql[0] ){ + int nCol = 0; + char **azVals = 0; + + pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); + assert( rc==SQLITE_OK || pStmt==0 ); + if( rc!=SQLITE_OK ){ + continue; + } + if( !pStmt ){ + /* this happens for a comment or white-space */ + zSql = zLeftover; + continue; + } + callbackIsInit = 0; + + while( 1 ){ + int i; + rc = sqlite3_step(pStmt); + + /* Invoke the callback function if required */ + if( xCallback && (SQLITE_ROW==rc || + (SQLITE_DONE==rc && !callbackIsInit + && db->flags&SQLITE_NullCallback)) ){ + if( !callbackIsInit ){ + nCol = sqlite3_column_count(pStmt); + azCols = sqlite3DbMallocRaw(db, (2*nCol+1)*sizeof(const char*)); + if( azCols==0 ){ + goto exec_out; + } + for(i=0; ierrMask)==rc ); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/************** End of legacy.c **********************************************/ +/************** Begin file loadext.c *****************************************/ +/* +** 2006 June 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to dynamically load extensions into +** the SQLite library. +*/ + +#ifndef SQLITE_CORE + #define SQLITE_CORE 1 /* Disable the API redefinition in sqlite3ext.h */ +#endif +/************** Include sqlite3ext.h in the middle of loadext.c **************/ +/************** Begin file sqlite3ext.h **************************************/ +/* +** 2006 June 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the SQLite interface for use by +** shared libraries that want to be imported as extensions into +** an SQLite instance. Shared libraries that intend to be loaded +** as extensions by SQLite should #include this file instead of +** sqlite3.h. +*/ +#ifndef SQLITE3EXT_H +#define SQLITE3EXT_H +/* #include "sqlite3.h" */ + +/* +** The following structure holds pointers to all of the SQLite API +** routines. +** +** WARNING: In order to maintain backwards compatibility, add new +** interfaces to the end of this structure only. If you insert new +** interfaces in the middle of this structure, then older different +** versions of SQLite will not be able to load each other's shared +** libraries! +*/ +struct sqlite3_api_routines { + void * (*aggregate_context)(sqlite3_context*,int nBytes); + int (*aggregate_count)(sqlite3_context*); + int (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*)); + int (*bind_double)(sqlite3_stmt*,int,double); + int (*bind_int)(sqlite3_stmt*,int,int); + int (*bind_int64)(sqlite3_stmt*,int,sqlite_int64); + int (*bind_null)(sqlite3_stmt*,int); + int (*bind_parameter_count)(sqlite3_stmt*); + int (*bind_parameter_index)(sqlite3_stmt*,const char*zName); + const char * (*bind_parameter_name)(sqlite3_stmt*,int); + int (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*)); + int (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*)); + int (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*); + int (*busy_handler)(sqlite3*,int(*)(void*,int),void*); + int (*busy_timeout)(sqlite3*,int ms); + int (*changes)(sqlite3*); + int (*close)(sqlite3*); + int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const char*)); + int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*, + int eTextRep,const void*)); + const void * (*column_blob)(sqlite3_stmt*,int iCol); + int (*column_bytes)(sqlite3_stmt*,int iCol); + int (*column_bytes16)(sqlite3_stmt*,int iCol); + int (*column_count)(sqlite3_stmt*pStmt); + const char * (*column_database_name)(sqlite3_stmt*,int); + const void * (*column_database_name16)(sqlite3_stmt*,int); + const char * (*column_decltype)(sqlite3_stmt*,int i); + const void * (*column_decltype16)(sqlite3_stmt*,int); + double (*column_double)(sqlite3_stmt*,int iCol); + int (*column_int)(sqlite3_stmt*,int iCol); + sqlite_int64 (*column_int64)(sqlite3_stmt*,int iCol); + const char * (*column_name)(sqlite3_stmt*,int); + const void * (*column_name16)(sqlite3_stmt*,int); + const char * (*column_origin_name)(sqlite3_stmt*,int); + const void * (*column_origin_name16)(sqlite3_stmt*,int); + const char * (*column_table_name)(sqlite3_stmt*,int); + const void * (*column_table_name16)(sqlite3_stmt*,int); + const unsigned char * (*column_text)(sqlite3_stmt*,int iCol); + const void * (*column_text16)(sqlite3_stmt*,int iCol); + int (*column_type)(sqlite3_stmt*,int iCol); + sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol); + void * (*commit_hook)(sqlite3*,int(*)(void*),void*); + int (*complete)(const char*sql); + int (*complete16)(const void*sql); + int (*create_collation)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_collation16)(sqlite3*,const void*,int,void*, + int(*)(void*,int,const void*,int,const void*)); + int (*create_function)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); + int (*create_function16)(sqlite3*,const void*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*)); + int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*); + int (*data_count)(sqlite3_stmt*pStmt); + sqlite3 * (*db_handle)(sqlite3_stmt*); + int (*declare_vtab)(sqlite3*,const char*); + int (*enable_shared_cache)(int); + int (*errcode)(sqlite3*db); + const char * (*errmsg)(sqlite3*); + const void * (*errmsg16)(sqlite3*); + int (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**); + int (*expired)(sqlite3_stmt*); + int (*finalize)(sqlite3_stmt*pStmt); + void (*free)(void*); + void (*free_table)(char**result); + int (*get_autocommit)(sqlite3*); + void * (*get_auxdata)(sqlite3_context*,int); + int (*get_table)(sqlite3*,const char*,char***,int*,int*,char**); + int (*global_recover)(void); + void (*interruptx)(sqlite3*); + sqlite_int64 (*last_insert_rowid)(sqlite3*); + const char * (*libversion)(void); + int (*libversion_number)(void); + void *(*malloc)(int); + char * (*mprintf)(const char*,...); + int (*open)(const char*,sqlite3**); + int (*open16)(const void*,sqlite3**); + int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); + int (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); + void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*); + void (*progress_handler)(sqlite3*,int,int(*)(void*),void*); + void *(*realloc)(void*,int); + int (*reset)(sqlite3_stmt*pStmt); + void (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_double)(sqlite3_context*,double); + void (*result_error)(sqlite3_context*,const char*,int); + void (*result_error16)(sqlite3_context*,const void*,int); + void (*result_int)(sqlite3_context*,int); + void (*result_int64)(sqlite3_context*,sqlite_int64); + void (*result_null)(sqlite3_context*); + void (*result_text)(sqlite3_context*,const char*,int,void(*)(void*)); + void (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*)); + void (*result_value)(sqlite3_context*,sqlite3_value*); + void * (*rollback_hook)(sqlite3*,void(*)(void*),void*); + int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, + const char*,const char*),void*); + void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); + char * (*xsnprintf)(int,char*,const char*,...); + int (*step)(sqlite3_stmt*); + int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, + char const**,char const**,int*,int*,int*); + void (*thread_cleanup)(void); + int (*total_changes)(sqlite3*); + void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*); + int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*); + void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*, + sqlite_int64),void*); + void * (*user_data)(sqlite3_context*); + const void * (*value_blob)(sqlite3_value*); + int (*value_bytes)(sqlite3_value*); + int (*value_bytes16)(sqlite3_value*); + double (*value_double)(sqlite3_value*); + int (*value_int)(sqlite3_value*); + sqlite_int64 (*value_int64)(sqlite3_value*); + int (*value_numeric_type)(sqlite3_value*); + const unsigned char * (*value_text)(sqlite3_value*); + const void * (*value_text16)(sqlite3_value*); + const void * (*value_text16be)(sqlite3_value*); + const void * (*value_text16le)(sqlite3_value*); + int (*value_type)(sqlite3_value*); + char *(*vmprintf)(const char*,va_list); + /* Added ??? */ + int (*overload_function)(sqlite3*, const char *zFuncName, int nArg); + /* Added by 3.3.13 */ + int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**); + int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**); + int (*clear_bindings)(sqlite3_stmt*); + /* Added by 3.4.1 */ + int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*, + void (*xDestroy)(void *)); + /* Added by 3.5.0 */ + int (*bind_zeroblob)(sqlite3_stmt*,int,int); + int (*blob_bytes)(sqlite3_blob*); + int (*blob_close)(sqlite3_blob*); + int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64, + int,sqlite3_blob**); + int (*blob_read)(sqlite3_blob*,void*,int,int); + int (*blob_write)(sqlite3_blob*,const void*,int,int); + int (*create_collation_v2)(sqlite3*,const char*,int,void*, + int(*)(void*,int,const void*,int,const void*), + void(*)(void*)); + int (*file_control)(sqlite3*,const char*,int,void*); + sqlite3_int64 (*memory_highwater)(int); + sqlite3_int64 (*memory_used)(void); + sqlite3_mutex *(*mutex_alloc)(int); + void (*mutex_enter)(sqlite3_mutex*); + void (*mutex_free)(sqlite3_mutex*); + void (*mutex_leave)(sqlite3_mutex*); + int (*mutex_try)(sqlite3_mutex*); + int (*open_v2)(const char*,sqlite3**,int,const char*); + int (*release_memory)(int); + void (*result_error_nomem)(sqlite3_context*); + void (*result_error_toobig)(sqlite3_context*); + int (*sleep)(int); + void (*soft_heap_limit)(int); + sqlite3_vfs *(*vfs_find)(const char*); + int (*vfs_register)(sqlite3_vfs*,int); + int (*vfs_unregister)(sqlite3_vfs*); + int (*xthreadsafe)(void); + void (*result_zeroblob)(sqlite3_context*,int); + void (*result_error_code)(sqlite3_context*,int); + int (*test_control)(int, ...); + void (*randomness)(int,void*); + sqlite3 *(*context_db_handle)(sqlite3_context*); + int (*extended_result_codes)(sqlite3*,int); + int (*limit)(sqlite3*,int,int); + sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*); + const char *(*sql)(sqlite3_stmt*); + int (*status)(int,int*,int*,int); + int (*backup_finish)(sqlite3_backup*); + sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*); + int (*backup_pagecount)(sqlite3_backup*); + int (*backup_remaining)(sqlite3_backup*); + int (*backup_step)(sqlite3_backup*,int); + const char *(*compileoption_get)(int); + int (*compileoption_used)(const char*); + int (*create_function_v2)(sqlite3*,const char*,int,int,void*, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*)); + int (*db_config)(sqlite3*,int,...); + sqlite3_mutex *(*db_mutex)(sqlite3*); + int (*db_status)(sqlite3*,int,int*,int*,int); + int (*extended_errcode)(sqlite3*); + void (*log)(int,const char*,...); + sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64); + const char *(*sourceid)(void); + int (*stmt_status)(sqlite3_stmt*,int,int); + int (*strnicmp)(const char*,const char*,int); + int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*); + int (*wal_autocheckpoint)(sqlite3*,int); + int (*wal_checkpoint)(sqlite3*,const char*); + void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); + int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); + int (*vtab_config)(sqlite3*,int op,...); + int (*vtab_on_conflict)(sqlite3*); + /* Version 3.7.16 and later */ + int (*close_v2)(sqlite3*); + const char *(*db_filename)(sqlite3*,const char*); + int (*db_readonly)(sqlite3*,const char*); + int (*db_release_memory)(sqlite3*); + const char *(*errstr)(int); + int (*stmt_busy)(sqlite3_stmt*); + int (*stmt_readonly)(sqlite3_stmt*); + int (*stricmp)(const char*,const char*); + int (*uri_boolean)(const char*,const char*,int); + sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); + const char *(*uri_parameter)(const char*,const char*); + char *(*xvsnprintf)(int,char*,const char*,va_list); + int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); + /* Version 3.8.7 and later */ + int (*auto_extension)(void(*)(void)); + int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64, + void(*)(void*)); + int (*bind_text64)(sqlite3_stmt*,int,const char*,sqlite3_uint64, + void(*)(void*),unsigned char); + int (*cancel_auto_extension)(void(*)(void)); + int (*load_extension)(sqlite3*,const char*,const char*,char**); + void *(*malloc64)(sqlite3_uint64); + sqlite3_uint64 (*msize)(void*); + void *(*realloc64)(void*,sqlite3_uint64); + void (*reset_auto_extension)(void); + void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64, + void(*)(void*)); + void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64, + void(*)(void*), unsigned char); + int (*strglob)(const char*,const char*); + /* Version 3.8.11 and later */ + sqlite3_value *(*value_dup)(const sqlite3_value*); + void (*value_free)(sqlite3_value*); + int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); + int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); + /* Version 3.9.0 and later */ + unsigned int (*value_subtype)(sqlite3_value*); + void (*result_subtype)(sqlite3_context*,unsigned int); + /* Version 3.10.0 and later */ + int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int); + int (*strlike)(const char*,const char*,unsigned int); + int (*db_cacheflush)(sqlite3*); + /* Version 3.12.0 and later */ + int (*system_errno)(sqlite3*); + /* Version 3.14.0 and later */ + int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*); + char *(*expanded_sql)(sqlite3_stmt*); + /* Version 3.18.0 and later */ + void (*set_last_insert_rowid)(sqlite3*,sqlite3_int64); + /* Version 3.20.0 and later */ + int (*prepare_v3)(sqlite3*,const char*,int,unsigned int, + sqlite3_stmt**,const char**); + int (*prepare16_v3)(sqlite3*,const void*,int,unsigned int, + sqlite3_stmt**,const void**); + int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*,void(*)(void*)); + void (*result_pointer)(sqlite3_context*,void*,const char*,void(*)(void*)); + void *(*value_pointer)(sqlite3_value*,const char*); + int (*vtab_nochange)(sqlite3_context*); + int (*value_nochange)(sqlite3_value*); + const char *(*vtab_collation)(sqlite3_index_info*,int); + /* Version 3.24.0 and later */ + int (*keyword_count)(void); + int (*keyword_name)(int,const char**,int*); + int (*keyword_check)(const char*,int); + sqlite3_str *(*str_new)(sqlite3*); + char *(*str_finish)(sqlite3_str*); + void (*str_appendf)(sqlite3_str*, const char *zFormat, ...); + void (*str_vappendf)(sqlite3_str*, const char *zFormat, va_list); + void (*str_append)(sqlite3_str*, const char *zIn, int N); + void (*str_appendall)(sqlite3_str*, const char *zIn); + void (*str_appendchar)(sqlite3_str*, int N, char C); + void (*str_reset)(sqlite3_str*); + int (*str_errcode)(sqlite3_str*); + int (*str_length)(sqlite3_str*); + char *(*str_value)(sqlite3_str*); + /* Version 3.25.0 and later */ + int (*create_window_function)(sqlite3*,const char*,int,int,void*, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInv)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*)); + /* Version 3.26.0 and later */ + const char *(*normalized_sql)(sqlite3_stmt*); + /* Version 3.28.0 and later */ + int (*stmt_isexplain)(sqlite3_stmt*); + int (*value_frombind)(sqlite3_value*); + /* Version 3.30.0 and later */ + int (*drop_modules)(sqlite3*,const char**); + /* Version 3.31.0 and later */ + sqlite3_int64 (*hard_heap_limit64)(sqlite3_int64); + const char *(*uri_key)(const char*,int); + const char *(*filename_database)(const char*); + const char *(*filename_journal)(const char*); + const char *(*filename_wal)(const char*); + /* Version 3.32.0 and later */ + char *(*create_filename)(const char*,const char*,const char*, + int,const char**); + void (*free_filename)(char*); + sqlite3_file *(*database_file_object)(const char*); + /* Version 3.34.0 and later */ + int (*txn_state)(sqlite3*,const char*); + /* Version 3.36.1 and later */ + sqlite3_int64 (*changes64)(sqlite3*); + sqlite3_int64 (*total_changes64)(sqlite3*); +}; + +/* +** This is the function signature used for all extension entry points. It +** is also defined in the file "loadext.c". +*/ +typedef int (*sqlite3_loadext_entry)( + sqlite3 *db, /* Handle to the database. */ + char **pzErrMsg, /* Used to set error string on failure. */ + const sqlite3_api_routines *pThunk /* Extension API function pointers. */ +); + +/* +** The following macros redefine the API routines so that they are +** redirected through the global sqlite3_api structure. +** +** This header file is also used by the loadext.c source file +** (part of the main SQLite library - not an extension) so that +** it can get access to the sqlite3_api_routines structure +** definition. But the main library does not want to redefine +** the API. So the redefinition macros are only valid if the +** SQLITE_CORE macros is undefined. +*/ +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) +#define sqlite3_aggregate_context sqlite3_api->aggregate_context +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_aggregate_count sqlite3_api->aggregate_count +#endif +#define sqlite3_bind_blob sqlite3_api->bind_blob +#define sqlite3_bind_double sqlite3_api->bind_double +#define sqlite3_bind_int sqlite3_api->bind_int +#define sqlite3_bind_int64 sqlite3_api->bind_int64 +#define sqlite3_bind_null sqlite3_api->bind_null +#define sqlite3_bind_parameter_count sqlite3_api->bind_parameter_count +#define sqlite3_bind_parameter_index sqlite3_api->bind_parameter_index +#define sqlite3_bind_parameter_name sqlite3_api->bind_parameter_name +#define sqlite3_bind_text sqlite3_api->bind_text +#define sqlite3_bind_text16 sqlite3_api->bind_text16 +#define sqlite3_bind_value sqlite3_api->bind_value +#define sqlite3_busy_handler sqlite3_api->busy_handler +#define sqlite3_busy_timeout sqlite3_api->busy_timeout +#define sqlite3_changes sqlite3_api->changes +#define sqlite3_close sqlite3_api->close +#define sqlite3_collation_needed sqlite3_api->collation_needed +#define sqlite3_collation_needed16 sqlite3_api->collation_needed16 +#define sqlite3_column_blob sqlite3_api->column_blob +#define sqlite3_column_bytes sqlite3_api->column_bytes +#define sqlite3_column_bytes16 sqlite3_api->column_bytes16 +#define sqlite3_column_count sqlite3_api->column_count +#define sqlite3_column_database_name sqlite3_api->column_database_name +#define sqlite3_column_database_name16 sqlite3_api->column_database_name16 +#define sqlite3_column_decltype sqlite3_api->column_decltype +#define sqlite3_column_decltype16 sqlite3_api->column_decltype16 +#define sqlite3_column_double sqlite3_api->column_double +#define sqlite3_column_int sqlite3_api->column_int +#define sqlite3_column_int64 sqlite3_api->column_int64 +#define sqlite3_column_name sqlite3_api->column_name +#define sqlite3_column_name16 sqlite3_api->column_name16 +#define sqlite3_column_origin_name sqlite3_api->column_origin_name +#define sqlite3_column_origin_name16 sqlite3_api->column_origin_name16 +#define sqlite3_column_table_name sqlite3_api->column_table_name +#define sqlite3_column_table_name16 sqlite3_api->column_table_name16 +#define sqlite3_column_text sqlite3_api->column_text +#define sqlite3_column_text16 sqlite3_api->column_text16 +#define sqlite3_column_type sqlite3_api->column_type +#define sqlite3_column_value sqlite3_api->column_value +#define sqlite3_commit_hook sqlite3_api->commit_hook +#define sqlite3_complete sqlite3_api->complete +#define sqlite3_complete16 sqlite3_api->complete16 +#define sqlite3_create_collation sqlite3_api->create_collation +#define sqlite3_create_collation16 sqlite3_api->create_collation16 +#define sqlite3_create_function sqlite3_api->create_function +#define sqlite3_create_function16 sqlite3_api->create_function16 +#define sqlite3_create_module sqlite3_api->create_module +#define sqlite3_create_module_v2 sqlite3_api->create_module_v2 +#define sqlite3_data_count sqlite3_api->data_count +#define sqlite3_db_handle sqlite3_api->db_handle +#define sqlite3_declare_vtab sqlite3_api->declare_vtab +#define sqlite3_enable_shared_cache sqlite3_api->enable_shared_cache +#define sqlite3_errcode sqlite3_api->errcode +#define sqlite3_errmsg sqlite3_api->errmsg +#define sqlite3_errmsg16 sqlite3_api->errmsg16 +#define sqlite3_exec sqlite3_api->exec +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_expired sqlite3_api->expired +#endif +#define sqlite3_finalize sqlite3_api->finalize +#define sqlite3_free sqlite3_api->free +#define sqlite3_free_table sqlite3_api->free_table +#define sqlite3_get_autocommit sqlite3_api->get_autocommit +#define sqlite3_get_auxdata sqlite3_api->get_auxdata +#define sqlite3_get_table sqlite3_api->get_table +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_global_recover sqlite3_api->global_recover +#endif +#define sqlite3_interrupt sqlite3_api->interruptx +#define sqlite3_last_insert_rowid sqlite3_api->last_insert_rowid +#define sqlite3_libversion sqlite3_api->libversion +#define sqlite3_libversion_number sqlite3_api->libversion_number +#define sqlite3_malloc sqlite3_api->malloc +#define sqlite3_mprintf sqlite3_api->mprintf +#define sqlite3_open sqlite3_api->open +#define sqlite3_open16 sqlite3_api->open16 +#define sqlite3_prepare sqlite3_api->prepare +#define sqlite3_prepare16 sqlite3_api->prepare16 +#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 +#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 +#define sqlite3_profile sqlite3_api->profile +#define sqlite3_progress_handler sqlite3_api->progress_handler +#define sqlite3_realloc sqlite3_api->realloc +#define sqlite3_reset sqlite3_api->reset +#define sqlite3_result_blob sqlite3_api->result_blob +#define sqlite3_result_double sqlite3_api->result_double +#define sqlite3_result_error sqlite3_api->result_error +#define sqlite3_result_error16 sqlite3_api->result_error16 +#define sqlite3_result_int sqlite3_api->result_int +#define sqlite3_result_int64 sqlite3_api->result_int64 +#define sqlite3_result_null sqlite3_api->result_null +#define sqlite3_result_text sqlite3_api->result_text +#define sqlite3_result_text16 sqlite3_api->result_text16 +#define sqlite3_result_text16be sqlite3_api->result_text16be +#define sqlite3_result_text16le sqlite3_api->result_text16le +#define sqlite3_result_value sqlite3_api->result_value +#define sqlite3_rollback_hook sqlite3_api->rollback_hook +#define sqlite3_set_authorizer sqlite3_api->set_authorizer +#define sqlite3_set_auxdata sqlite3_api->set_auxdata +#define sqlite3_snprintf sqlite3_api->xsnprintf +#define sqlite3_step sqlite3_api->step +#define sqlite3_table_column_metadata sqlite3_api->table_column_metadata +#define sqlite3_thread_cleanup sqlite3_api->thread_cleanup +#define sqlite3_total_changes sqlite3_api->total_changes +#define sqlite3_trace sqlite3_api->trace +#ifndef SQLITE_OMIT_DEPRECATED +#define sqlite3_transfer_bindings sqlite3_api->transfer_bindings +#endif +#define sqlite3_update_hook sqlite3_api->update_hook +#define sqlite3_user_data sqlite3_api->user_data +#define sqlite3_value_blob sqlite3_api->value_blob +#define sqlite3_value_bytes sqlite3_api->value_bytes +#define sqlite3_value_bytes16 sqlite3_api->value_bytes16 +#define sqlite3_value_double sqlite3_api->value_double +#define sqlite3_value_int sqlite3_api->value_int +#define sqlite3_value_int64 sqlite3_api->value_int64 +#define sqlite3_value_numeric_type sqlite3_api->value_numeric_type +#define sqlite3_value_text sqlite3_api->value_text +#define sqlite3_value_text16 sqlite3_api->value_text16 +#define sqlite3_value_text16be sqlite3_api->value_text16be +#define sqlite3_value_text16le sqlite3_api->value_text16le +#define sqlite3_value_type sqlite3_api->value_type +#define sqlite3_vmprintf sqlite3_api->vmprintf +#define sqlite3_vsnprintf sqlite3_api->xvsnprintf +#define sqlite3_overload_function sqlite3_api->overload_function +#define sqlite3_prepare_v2 sqlite3_api->prepare_v2 +#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 +#define sqlite3_clear_bindings sqlite3_api->clear_bindings +#define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob +#define sqlite3_blob_bytes sqlite3_api->blob_bytes +#define sqlite3_blob_close sqlite3_api->blob_close +#define sqlite3_blob_open sqlite3_api->blob_open +#define sqlite3_blob_read sqlite3_api->blob_read +#define sqlite3_blob_write sqlite3_api->blob_write +#define sqlite3_create_collation_v2 sqlite3_api->create_collation_v2 +#define sqlite3_file_control sqlite3_api->file_control +#define sqlite3_memory_highwater sqlite3_api->memory_highwater +#define sqlite3_memory_used sqlite3_api->memory_used +#define sqlite3_mutex_alloc sqlite3_api->mutex_alloc +#define sqlite3_mutex_enter sqlite3_api->mutex_enter +#define sqlite3_mutex_free sqlite3_api->mutex_free +#define sqlite3_mutex_leave sqlite3_api->mutex_leave +#define sqlite3_mutex_try sqlite3_api->mutex_try +#define sqlite3_open_v2 sqlite3_api->open_v2 +#define sqlite3_release_memory sqlite3_api->release_memory +#define sqlite3_result_error_nomem sqlite3_api->result_error_nomem +#define sqlite3_result_error_toobig sqlite3_api->result_error_toobig +#define sqlite3_sleep sqlite3_api->sleep +#define sqlite3_soft_heap_limit sqlite3_api->soft_heap_limit +#define sqlite3_vfs_find sqlite3_api->vfs_find +#define sqlite3_vfs_register sqlite3_api->vfs_register +#define sqlite3_vfs_unregister sqlite3_api->vfs_unregister +#define sqlite3_threadsafe sqlite3_api->xthreadsafe +#define sqlite3_result_zeroblob sqlite3_api->result_zeroblob +#define sqlite3_result_error_code sqlite3_api->result_error_code +#define sqlite3_test_control sqlite3_api->test_control +#define sqlite3_randomness sqlite3_api->randomness +#define sqlite3_context_db_handle sqlite3_api->context_db_handle +#define sqlite3_extended_result_codes sqlite3_api->extended_result_codes +#define sqlite3_limit sqlite3_api->limit +#define sqlite3_next_stmt sqlite3_api->next_stmt +#define sqlite3_sql sqlite3_api->sql +#define sqlite3_status sqlite3_api->status +#define sqlite3_backup_finish sqlite3_api->backup_finish +#define sqlite3_backup_init sqlite3_api->backup_init +#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount +#define sqlite3_backup_remaining sqlite3_api->backup_remaining +#define sqlite3_backup_step sqlite3_api->backup_step +#define sqlite3_compileoption_get sqlite3_api->compileoption_get +#define sqlite3_compileoption_used sqlite3_api->compileoption_used +#define sqlite3_create_function_v2 sqlite3_api->create_function_v2 +#define sqlite3_db_config sqlite3_api->db_config +#define sqlite3_db_mutex sqlite3_api->db_mutex +#define sqlite3_db_status sqlite3_api->db_status +#define sqlite3_extended_errcode sqlite3_api->extended_errcode +#define sqlite3_log sqlite3_api->log +#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64 +#define sqlite3_sourceid sqlite3_api->sourceid +#define sqlite3_stmt_status sqlite3_api->stmt_status +#define sqlite3_strnicmp sqlite3_api->strnicmp +#define sqlite3_unlock_notify sqlite3_api->unlock_notify +#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint +#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint +#define sqlite3_wal_hook sqlite3_api->wal_hook +#define sqlite3_blob_reopen sqlite3_api->blob_reopen +#define sqlite3_vtab_config sqlite3_api->vtab_config +#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict +/* Version 3.7.16 and later */ +#define sqlite3_close_v2 sqlite3_api->close_v2 +#define sqlite3_db_filename sqlite3_api->db_filename +#define sqlite3_db_readonly sqlite3_api->db_readonly +#define sqlite3_db_release_memory sqlite3_api->db_release_memory +#define sqlite3_errstr sqlite3_api->errstr +#define sqlite3_stmt_busy sqlite3_api->stmt_busy +#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly +#define sqlite3_stricmp sqlite3_api->stricmp +#define sqlite3_uri_boolean sqlite3_api->uri_boolean +#define sqlite3_uri_int64 sqlite3_api->uri_int64 +#define sqlite3_uri_parameter sqlite3_api->uri_parameter +#define sqlite3_uri_vsnprintf sqlite3_api->xvsnprintf +#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 +/* Version 3.8.7 and later */ +#define sqlite3_auto_extension sqlite3_api->auto_extension +#define sqlite3_bind_blob64 sqlite3_api->bind_blob64 +#define sqlite3_bind_text64 sqlite3_api->bind_text64 +#define sqlite3_cancel_auto_extension sqlite3_api->cancel_auto_extension +#define sqlite3_load_extension sqlite3_api->load_extension +#define sqlite3_malloc64 sqlite3_api->malloc64 +#define sqlite3_msize sqlite3_api->msize +#define sqlite3_realloc64 sqlite3_api->realloc64 +#define sqlite3_reset_auto_extension sqlite3_api->reset_auto_extension +#define sqlite3_result_blob64 sqlite3_api->result_blob64 +#define sqlite3_result_text64 sqlite3_api->result_text64 +#define sqlite3_strglob sqlite3_api->strglob +/* Version 3.8.11 and later */ +#define sqlite3_value_dup sqlite3_api->value_dup +#define sqlite3_value_free sqlite3_api->value_free +#define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 +#define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 +/* Version 3.9.0 and later */ +#define sqlite3_value_subtype sqlite3_api->value_subtype +#define sqlite3_result_subtype sqlite3_api->result_subtype +/* Version 3.10.0 and later */ +#define sqlite3_status64 sqlite3_api->status64 +#define sqlite3_strlike sqlite3_api->strlike +#define sqlite3_db_cacheflush sqlite3_api->db_cacheflush +/* Version 3.12.0 and later */ +#define sqlite3_system_errno sqlite3_api->system_errno +/* Version 3.14.0 and later */ +#define sqlite3_trace_v2 sqlite3_api->trace_v2 +#define sqlite3_expanded_sql sqlite3_api->expanded_sql +/* Version 3.18.0 and later */ +#define sqlite3_set_last_insert_rowid sqlite3_api->set_last_insert_rowid +/* Version 3.20.0 and later */ +#define sqlite3_prepare_v3 sqlite3_api->prepare_v3 +#define sqlite3_prepare16_v3 sqlite3_api->prepare16_v3 +#define sqlite3_bind_pointer sqlite3_api->bind_pointer +#define sqlite3_result_pointer sqlite3_api->result_pointer +#define sqlite3_value_pointer sqlite3_api->value_pointer +/* Version 3.22.0 and later */ +#define sqlite3_vtab_nochange sqlite3_api->vtab_nochange +#define sqlite3_value_nochange sqlite3_api->value_nochange +#define sqlite3_vtab_collation sqlite3_api->vtab_collation +/* Version 3.24.0 and later */ +#define sqlite3_keyword_count sqlite3_api->keyword_count +#define sqlite3_keyword_name sqlite3_api->keyword_name +#define sqlite3_keyword_check sqlite3_api->keyword_check +#define sqlite3_str_new sqlite3_api->str_new +#define sqlite3_str_finish sqlite3_api->str_finish +#define sqlite3_str_appendf sqlite3_api->str_appendf +#define sqlite3_str_vappendf sqlite3_api->str_vappendf +#define sqlite3_str_append sqlite3_api->str_append +#define sqlite3_str_appendall sqlite3_api->str_appendall +#define sqlite3_str_appendchar sqlite3_api->str_appendchar +#define sqlite3_str_reset sqlite3_api->str_reset +#define sqlite3_str_errcode sqlite3_api->str_errcode +#define sqlite3_str_length sqlite3_api->str_length +#define sqlite3_str_value sqlite3_api->str_value +/* Version 3.25.0 and later */ +#define sqlite3_create_window_function sqlite3_api->create_window_function +/* Version 3.26.0 and later */ +#define sqlite3_normalized_sql sqlite3_api->normalized_sql +/* Version 3.28.0 and later */ +#define sqlite3_stmt_isexplain sqlite3_api->stmt_isexplain +#define sqlite3_value_frombind sqlite3_api->value_frombind +/* Version 3.30.0 and later */ +#define sqlite3_drop_modules sqlite3_api->drop_modules +/* Version 3.31.0 and later */ +#define sqlite3_hard_heap_limit64 sqlite3_api->hard_heap_limit64 +#define sqlite3_uri_key sqlite3_api->uri_key +#define sqlite3_filename_database sqlite3_api->filename_database +#define sqlite3_filename_journal sqlite3_api->filename_journal +#define sqlite3_filename_wal sqlite3_api->filename_wal +/* Version 3.32.0 and later */ +#define sqlite3_create_filename sqlite3_api->create_filename +#define sqlite3_free_filename sqlite3_api->free_filename +#define sqlite3_database_file_object sqlite3_api->database_file_object +/* Version 3.34.0 and later */ +#define sqlite3_txn_state sqlite3_api->txn_state +#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ + +#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +# define SQLITE_EXTENSION_INIT3 \ + extern const sqlite3_api_routines *sqlite3_api; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +# define SQLITE_EXTENSION_INIT3 /*no-op*/ +#endif + +#endif /* SQLITE3EXT_H */ + +/************** End of sqlite3ext.h ******************************************/ +/************** Continuing where we left off in loadext.c ********************/ +/* #include "sqliteInt.h" */ + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** Some API routines are omitted when various features are +** excluded from a build of SQLite. Substitute a NULL pointer +** for any missing APIs. +*/ +#ifndef SQLITE_ENABLE_COLUMN_METADATA +# define sqlite3_column_database_name 0 +# define sqlite3_column_database_name16 0 +# define sqlite3_column_table_name 0 +# define sqlite3_column_table_name16 0 +# define sqlite3_column_origin_name 0 +# define sqlite3_column_origin_name16 0 +#endif + +#ifdef SQLITE_OMIT_AUTHORIZATION +# define sqlite3_set_authorizer 0 +#endif + +#ifdef SQLITE_OMIT_UTF16 +# define sqlite3_bind_text16 0 +# define sqlite3_collation_needed16 0 +# define sqlite3_column_decltype16 0 +# define sqlite3_column_name16 0 +# define sqlite3_column_text16 0 +# define sqlite3_complete16 0 +# define sqlite3_create_collation16 0 +# define sqlite3_create_function16 0 +# define sqlite3_errmsg16 0 +# define sqlite3_open16 0 +# define sqlite3_prepare16 0 +# define sqlite3_prepare16_v2 0 +# define sqlite3_prepare16_v3 0 +# define sqlite3_result_error16 0 +# define sqlite3_result_text16 0 +# define sqlite3_result_text16be 0 +# define sqlite3_result_text16le 0 +# define sqlite3_value_text16 0 +# define sqlite3_value_text16be 0 +# define sqlite3_value_text16le 0 +# define sqlite3_column_database_name16 0 +# define sqlite3_column_table_name16 0 +# define sqlite3_column_origin_name16 0 +#endif + +#ifdef SQLITE_OMIT_COMPLETE +# define sqlite3_complete 0 +# define sqlite3_complete16 0 +#endif + +#ifdef SQLITE_OMIT_DECLTYPE +# define sqlite3_column_decltype16 0 +# define sqlite3_column_decltype 0 +#endif + +#ifdef SQLITE_OMIT_PROGRESS_CALLBACK +# define sqlite3_progress_handler 0 +#endif + +#ifdef SQLITE_OMIT_VIRTUALTABLE +# define sqlite3_create_module 0 +# define sqlite3_create_module_v2 0 +# define sqlite3_declare_vtab 0 +# define sqlite3_vtab_config 0 +# define sqlite3_vtab_on_conflict 0 +# define sqlite3_vtab_collation 0 +#endif + +#ifdef SQLITE_OMIT_SHARED_CACHE +# define sqlite3_enable_shared_cache 0 +#endif + +#if defined(SQLITE_OMIT_TRACE) || defined(SQLITE_OMIT_DEPRECATED) +# define sqlite3_profile 0 +# define sqlite3_trace 0 +#endif + +#ifdef SQLITE_OMIT_GET_TABLE +# define sqlite3_free_table 0 +# define sqlite3_get_table 0 +#endif + +#ifdef SQLITE_OMIT_INCRBLOB +#define sqlite3_bind_zeroblob 0 +#define sqlite3_blob_bytes 0 +#define sqlite3_blob_close 0 +#define sqlite3_blob_open 0 +#define sqlite3_blob_read 0 +#define sqlite3_blob_write 0 +#define sqlite3_blob_reopen 0 +#endif + +#if defined(SQLITE_OMIT_TRACE) +# define sqlite3_trace_v2 0 +#endif + +/* +** The following structure contains pointers to all SQLite API routines. +** A pointer to this structure is passed into extensions when they are +** loaded so that the extension can make calls back into the SQLite +** library. +** +** When adding new APIs, add them to the bottom of this structure +** in order to preserve backwards compatibility. +** +** Extensions that use newer APIs should first call the +** sqlite3_libversion_number() to make sure that the API they +** intend to use is supported by the library. Extensions should +** also check to make sure that the pointer to the function is +** not NULL before calling it. +*/ +static const sqlite3_api_routines sqlite3Apis = { + sqlite3_aggregate_context, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_aggregate_count, +#else + 0, +#endif + sqlite3_bind_blob, + sqlite3_bind_double, + sqlite3_bind_int, + sqlite3_bind_int64, + sqlite3_bind_null, + sqlite3_bind_parameter_count, + sqlite3_bind_parameter_index, + sqlite3_bind_parameter_name, + sqlite3_bind_text, + sqlite3_bind_text16, + sqlite3_bind_value, + sqlite3_busy_handler, + sqlite3_busy_timeout, + sqlite3_changes, + sqlite3_close, + sqlite3_collation_needed, + sqlite3_collation_needed16, + sqlite3_column_blob, + sqlite3_column_bytes, + sqlite3_column_bytes16, + sqlite3_column_count, + sqlite3_column_database_name, + sqlite3_column_database_name16, + sqlite3_column_decltype, + sqlite3_column_decltype16, + sqlite3_column_double, + sqlite3_column_int, + sqlite3_column_int64, + sqlite3_column_name, + sqlite3_column_name16, + sqlite3_column_origin_name, + sqlite3_column_origin_name16, + sqlite3_column_table_name, + sqlite3_column_table_name16, + sqlite3_column_text, + sqlite3_column_text16, + sqlite3_column_type, + sqlite3_column_value, + sqlite3_commit_hook, + sqlite3_complete, + sqlite3_complete16, + sqlite3_create_collation, + sqlite3_create_collation16, + sqlite3_create_function, + sqlite3_create_function16, + sqlite3_create_module, + sqlite3_data_count, + sqlite3_db_handle, + sqlite3_declare_vtab, + sqlite3_enable_shared_cache, + sqlite3_errcode, + sqlite3_errmsg, + sqlite3_errmsg16, + sqlite3_exec, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_expired, +#else + 0, +#endif + sqlite3_finalize, + sqlite3_free, + sqlite3_free_table, + sqlite3_get_autocommit, + sqlite3_get_auxdata, + sqlite3_get_table, + 0, /* Was sqlite3_global_recover(), but that function is deprecated */ + sqlite3_interrupt, + sqlite3_last_insert_rowid, + sqlite3_libversion, + sqlite3_libversion_number, + sqlite3_malloc, + sqlite3_mprintf, + sqlite3_open, + sqlite3_open16, + sqlite3_prepare, + sqlite3_prepare16, + sqlite3_profile, + sqlite3_progress_handler, + sqlite3_realloc, + sqlite3_reset, + sqlite3_result_blob, + sqlite3_result_double, + sqlite3_result_error, + sqlite3_result_error16, + sqlite3_result_int, + sqlite3_result_int64, + sqlite3_result_null, + sqlite3_result_text, + sqlite3_result_text16, + sqlite3_result_text16be, + sqlite3_result_text16le, + sqlite3_result_value, + sqlite3_rollback_hook, + sqlite3_set_authorizer, + sqlite3_set_auxdata, + sqlite3_snprintf, + sqlite3_step, + sqlite3_table_column_metadata, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_thread_cleanup, +#else + 0, +#endif + sqlite3_total_changes, + sqlite3_trace, +#ifndef SQLITE_OMIT_DEPRECATED + sqlite3_transfer_bindings, +#else + 0, +#endif + sqlite3_update_hook, + sqlite3_user_data, + sqlite3_value_blob, + sqlite3_value_bytes, + sqlite3_value_bytes16, + sqlite3_value_double, + sqlite3_value_int, + sqlite3_value_int64, + sqlite3_value_numeric_type, + sqlite3_value_text, + sqlite3_value_text16, + sqlite3_value_text16be, + sqlite3_value_text16le, + sqlite3_value_type, + sqlite3_vmprintf, + /* + ** The original API set ends here. All extensions can call any + ** of the APIs above provided that the pointer is not NULL. But + ** before calling APIs that follow, extension should check the + ** sqlite3_libversion_number() to make sure they are dealing with + ** a library that is new enough to support that API. + ************************************************************************* + */ + sqlite3_overload_function, + + /* + ** Added after 3.3.13 + */ + sqlite3_prepare_v2, + sqlite3_prepare16_v2, + sqlite3_clear_bindings, + + /* + ** Added for 3.4.1 + */ + sqlite3_create_module_v2, + + /* + ** Added for 3.5.0 + */ + sqlite3_bind_zeroblob, + sqlite3_blob_bytes, + sqlite3_blob_close, + sqlite3_blob_open, + sqlite3_blob_read, + sqlite3_blob_write, + sqlite3_create_collation_v2, + sqlite3_file_control, + sqlite3_memory_highwater, + sqlite3_memory_used, +#ifdef SQLITE_MUTEX_OMIT + 0, + 0, + 0, + 0, + 0, +#else + sqlite3_mutex_alloc, + sqlite3_mutex_enter, + sqlite3_mutex_free, + sqlite3_mutex_leave, + sqlite3_mutex_try, +#endif + sqlite3_open_v2, + sqlite3_release_memory, + sqlite3_result_error_nomem, + sqlite3_result_error_toobig, + sqlite3_sleep, + sqlite3_soft_heap_limit, + sqlite3_vfs_find, + sqlite3_vfs_register, + sqlite3_vfs_unregister, + + /* + ** Added for 3.5.8 + */ + sqlite3_threadsafe, + sqlite3_result_zeroblob, + sqlite3_result_error_code, + sqlite3_test_control, + sqlite3_randomness, + sqlite3_context_db_handle, + + /* + ** Added for 3.6.0 + */ + sqlite3_extended_result_codes, + sqlite3_limit, + sqlite3_next_stmt, + sqlite3_sql, + sqlite3_status, + + /* + ** Added for 3.7.4 + */ + sqlite3_backup_finish, + sqlite3_backup_init, + sqlite3_backup_pagecount, + sqlite3_backup_remaining, + sqlite3_backup_step, +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + sqlite3_compileoption_get, + sqlite3_compileoption_used, +#else + 0, + 0, +#endif + sqlite3_create_function_v2, + sqlite3_db_config, + sqlite3_db_mutex, + sqlite3_db_status, + sqlite3_extended_errcode, + sqlite3_log, + sqlite3_soft_heap_limit64, + sqlite3_sourceid, + sqlite3_stmt_status, + sqlite3_strnicmp, +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + sqlite3_unlock_notify, +#else + 0, +#endif +#ifndef SQLITE_OMIT_WAL + sqlite3_wal_autocheckpoint, + sqlite3_wal_checkpoint, + sqlite3_wal_hook, +#else + 0, + 0, + 0, +#endif + sqlite3_blob_reopen, + sqlite3_vtab_config, + sqlite3_vtab_on_conflict, + sqlite3_close_v2, + sqlite3_db_filename, + sqlite3_db_readonly, + sqlite3_db_release_memory, + sqlite3_errstr, + sqlite3_stmt_busy, + sqlite3_stmt_readonly, + sqlite3_stricmp, + sqlite3_uri_boolean, + sqlite3_uri_int64, + sqlite3_uri_parameter, + sqlite3_vsnprintf, + sqlite3_wal_checkpoint_v2, + /* Version 3.8.7 and later */ + sqlite3_auto_extension, + sqlite3_bind_blob64, + sqlite3_bind_text64, + sqlite3_cancel_auto_extension, + sqlite3_load_extension, + sqlite3_malloc64, + sqlite3_msize, + sqlite3_realloc64, + sqlite3_reset_auto_extension, + sqlite3_result_blob64, + sqlite3_result_text64, + sqlite3_strglob, + /* Version 3.8.11 and later */ + (sqlite3_value*(*)(const sqlite3_value*))sqlite3_value_dup, + sqlite3_value_free, + sqlite3_result_zeroblob64, + sqlite3_bind_zeroblob64, + /* Version 3.9.0 and later */ + sqlite3_value_subtype, + sqlite3_result_subtype, + /* Version 3.10.0 and later */ + sqlite3_status64, + sqlite3_strlike, + sqlite3_db_cacheflush, + /* Version 3.12.0 and later */ + sqlite3_system_errno, + /* Version 3.14.0 and later */ + sqlite3_trace_v2, + sqlite3_expanded_sql, + /* Version 3.18.0 and later */ + sqlite3_set_last_insert_rowid, + /* Version 3.20.0 and later */ + sqlite3_prepare_v3, + sqlite3_prepare16_v3, + sqlite3_bind_pointer, + sqlite3_result_pointer, + sqlite3_value_pointer, + /* Version 3.22.0 and later */ + sqlite3_vtab_nochange, + sqlite3_value_nochange, + sqlite3_vtab_collation, + /* Version 3.24.0 and later */ + sqlite3_keyword_count, + sqlite3_keyword_name, + sqlite3_keyword_check, + sqlite3_str_new, + sqlite3_str_finish, + sqlite3_str_appendf, + sqlite3_str_vappendf, + sqlite3_str_append, + sqlite3_str_appendall, + sqlite3_str_appendchar, + sqlite3_str_reset, + sqlite3_str_errcode, + sqlite3_str_length, + sqlite3_str_value, + /* Version 3.25.0 and later */ + sqlite3_create_window_function, + /* Version 3.26.0 and later */ +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3_normalized_sql, +#else + 0, +#endif + /* Version 3.28.0 and later */ + sqlite3_stmt_isexplain, + sqlite3_value_frombind, + /* Version 3.30.0 and later */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_drop_modules, +#else + 0, +#endif + /* Version 3.31.0 and later */ + sqlite3_hard_heap_limit64, + sqlite3_uri_key, + sqlite3_filename_database, + sqlite3_filename_journal, + sqlite3_filename_wal, + /* Version 3.32.0 and later */ + sqlite3_create_filename, + sqlite3_free_filename, + sqlite3_database_file_object, + /* Version 3.34.0 and later */ + sqlite3_txn_state, + /* Version 3.36.1 and later */ + sqlite3_changes64, + sqlite3_total_changes64, +}; + +/* True if x is the directory separator character +*/ +#if SQLITE_OS_WIN +# define DirSep(X) ((X)=='/'||(X)=='\\') +#else +# define DirSep(X) ((X)=='/') +#endif + +/* +** Attempt to load an SQLite extension library contained in the file +** zFile. The entry point is zProc. zProc may be 0 in which case a +** default entry point name (sqlite3_extension_init) is used. Use +** of the default name is recommended. +** +** Return SQLITE_OK on success and SQLITE_ERROR if something goes wrong. +** +** If an error occurs and pzErrMsg is not 0, then fill *pzErrMsg with +** error message text. The calling function should free this memory +** by calling sqlite3DbFree(db, ). +*/ +static int sqlite3LoadExtension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ +){ + sqlite3_vfs *pVfs = db->pVfs; + void *handle; + sqlite3_loadext_entry xInit; + char *zErrmsg = 0; + const char *zEntry; + char *zAltEntry = 0; + void **aHandle; + u64 nMsg = strlen(zFile); + int ii; + int rc; + + /* Shared library endings to try if zFile cannot be loaded as written */ + static const char *azEndings[] = { +#if SQLITE_OS_WIN + "dll" +#elif defined(__APPLE__) + "dylib" +#else + "so" +#endif + }; + + + if( pzErrMsg ) *pzErrMsg = 0; + + /* Ticket #1863. To avoid a creating security problems for older + ** applications that relink against newer versions of SQLite, the + ** ability to run load_extension is turned off by default. One + ** must call either sqlite3_enable_load_extension(db) or + ** sqlite3_db_config(db, SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, 1, 0) + ** to turn on extension loading. + */ + if( (db->flags & SQLITE_LoadExtension)==0 ){ + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("not authorized"); + } + return SQLITE_ERROR; + } + + zEntry = zProc ? zProc : "sqlite3_extension_init"; + + /* tag-20210611-1. Some dlopen() implementations will segfault if given + ** an oversize filename. Most filesystems have a pathname limit of 4K, + ** so limit the extension filename length to about twice that. + ** https://sqlite.org/forum/forumpost/08a0d6d9bf */ + if( nMsg>SQLITE_MAX_PATHLEN ) goto extension_not_found; + + handle = sqlite3OsDlOpen(pVfs, zFile); +#if SQLITE_OS_UNIX || SQLITE_OS_WIN + for(ii=0; ii sqlite3_example_init + ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + */ + if( xInit==0 && zProc==0 ){ + int iFile, iEntry, c; + int ncFile = sqlite3Strlen30(zFile); + zAltEntry = sqlite3_malloc64(ncFile+30); + if( zAltEntry==0 ){ + sqlite3OsDlClose(pVfs, handle); + return SQLITE_NOMEM_BKPT; + } + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } + } + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); + } + if( xInit==0 ){ + if( pzErrMsg ){ + nMsg += strlen(zEntry) + 300; + *pzErrMsg = zErrmsg = sqlite3_malloc64(nMsg); + if( zErrmsg ){ + assert( nMsg<0x7fffffff ); /* zErrmsg would be NULL if not so */ + sqlite3_snprintf((int)nMsg, zErrmsg, + "no entry point [%s] in shared library [%s]", zEntry, zFile); + sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); + } + } + sqlite3OsDlClose(pVfs, handle); + sqlite3_free(zAltEntry); + return SQLITE_ERROR; + } + sqlite3_free(zAltEntry); + rc = xInit(db, &zErrmsg, &sqlite3Apis); + if( rc ){ + if( rc==SQLITE_OK_LOAD_PERMANENTLY ) return SQLITE_OK; + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); + } + sqlite3_free(zErrmsg); + sqlite3OsDlClose(pVfs, handle); + return SQLITE_ERROR; + } + + /* Append the new shared library handle to the db->aExtension array. */ + aHandle = sqlite3DbMallocZero(db, sizeof(handle)*(db->nExtension+1)); + if( aHandle==0 ){ + return SQLITE_NOMEM_BKPT; + } + if( db->nExtension>0 ){ + memcpy(aHandle, db->aExtension, sizeof(handle)*db->nExtension); + } + sqlite3DbFree(db, db->aExtension); + db->aExtension = aHandle; + + db->aExtension[db->nExtension++] = handle; + return SQLITE_OK; + +extension_not_found: + if( pzErrMsg ){ + nMsg += 300; + *pzErrMsg = zErrmsg = sqlite3_malloc64(nMsg); + if( zErrmsg ){ + assert( nMsg<0x7fffffff ); /* zErrmsg would be NULL if not so */ + sqlite3_snprintf((int)nMsg, zErrmsg, + "unable to open shared library [%.*s]", SQLITE_MAX_PATHLEN, zFile); + sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); + } + } + return SQLITE_ERROR; +} +SQLITE_API int sqlite3_load_extension( + sqlite3 *db, /* Load the extension into this database connection */ + const char *zFile, /* Name of the shared library containing extension */ + const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ + char **pzErrMsg /* Put error message here if not 0 */ +){ + int rc; + sqlite3_mutex_enter(db->mutex); + rc = sqlite3LoadExtension(db, zFile, zProc, pzErrMsg); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Call this routine when the database connection is closing in order +** to clean up loaded extensions +*/ +SQLITE_PRIVATE void sqlite3CloseExtensions(sqlite3 *db){ + int i; + assert( sqlite3_mutex_held(db->mutex) ); + for(i=0; inExtension; i++){ + sqlite3OsDlClose(db->pVfs, db->aExtension[i]); + } + sqlite3DbFree(db, db->aExtension); +} + +/* +** Enable or disable extension loading. Extension loading is disabled by +** default so as not to open security holes in older applications. +*/ +SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ + sqlite3_mutex_enter(db->mutex); + if( onoff ){ + db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; + }else{ + db->flags &= ~(u64)(SQLITE_LoadExtension|SQLITE_LoadExtFunc); + } + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */ + +/* +** The following object holds the list of automatically loaded +** extensions. +** +** This list is shared across threads. The SQLITE_MUTEX_STATIC_MAIN +** mutex must be held while accessing this list. +*/ +typedef struct sqlite3AutoExtList sqlite3AutoExtList; +static SQLITE_WSD struct sqlite3AutoExtList { + u32 nExt; /* Number of entries in aExt[] */ + void (**aExt)(void); /* Pointers to the extension init functions */ +} sqlite3Autoext = { 0, 0 }; + +/* The "wsdAutoext" macro will resolve to the autoextension +** state vector. If writable static data is unsupported on the target, +** we have to locate the state vector at run-time. In the more common +** case where writable static data is supported, wsdStat can refer directly +** to the "sqlite3Autoext" state vector declared above. +*/ +#ifdef SQLITE_OMIT_WSD +# define wsdAutoextInit \ + sqlite3AutoExtList *x = &GLOBAL(sqlite3AutoExtList,sqlite3Autoext) +# define wsdAutoext x[0] +#else +# define wsdAutoextInit +# define wsdAutoext sqlite3Autoext +#endif + + +/* +** Register a statically linked extension that is automatically +** loaded by every new database connection. +*/ +SQLITE_API int sqlite3_auto_extension( + void (*xInit)(void) +){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ){ + return rc; + }else +#endif + { + u32 i; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); +#endif + wsdAutoextInit; + sqlite3_mutex_enter(mutex); + for(i=0; i=0; i--){ + if( wsdAutoext.aExt[i]==xInit ){ + wsdAutoext.nExt--; + wsdAutoext.aExt[i] = wsdAutoext.aExt[wsdAutoext.nExt]; + n++; + break; + } + } + sqlite3_mutex_leave(mutex); + return n; +} + +/* +** Reset the automatic extension loading mechanism. +*/ +SQLITE_API void sqlite3_reset_auto_extension(void){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize()==SQLITE_OK ) +#endif + { +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); +#endif + wsdAutoextInit; + sqlite3_mutex_enter(mutex); + sqlite3_free(wsdAutoext.aExt); + wsdAutoext.aExt = 0; + wsdAutoext.nExt = 0; + sqlite3_mutex_leave(mutex); + } +} + +/* +** Load all automatic extensions. +** +** If anything goes wrong, set an error in the database connection. +*/ +SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ + u32 i; + int go = 1; + int rc; + sqlite3_loadext_entry xInit; + + wsdAutoextInit; + if( wsdAutoext.nExt==0 ){ + /* Common case: early out without every having to acquire a mutex */ + return; + } + for(i=0; go; i++){ + char *zErrmsg; +#if SQLITE_THREADSAFE + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); +#endif +#ifdef SQLITE_OMIT_LOAD_EXTENSION + const sqlite3_api_routines *pThunk = 0; +#else + const sqlite3_api_routines *pThunk = &sqlite3Apis; +#endif + sqlite3_mutex_enter(mutex); + if( i>=wsdAutoext.nExt ){ + xInit = 0; + go = 0; + }else{ + xInit = (sqlite3_loadext_entry)wsdAutoext.aExt[i]; + } + sqlite3_mutex_leave(mutex); + zErrmsg = 0; + if( xInit && (rc = xInit(db, &zErrmsg, pThunk))!=0 ){ + sqlite3ErrorWithMsg(db, rc, + "automatic extension loading failed: %s", zErrmsg); + go = 0; + } + sqlite3_free(zErrmsg); + } +} + +/************** End of loadext.c *********************************************/ +/************** Begin file pragma.c ******************************************/ +/* +** 2003 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the PRAGMA command. +*/ +/* #include "sqliteInt.h" */ + +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif + +/*************************************************************************** +** The "pragma.h" include file is an automatically generated file that +** that includes the PragType_XXXX macro definitions and the aPragmaName[] +** object. This ensures that the aPragmaName[] table is arranged in +** lexicographical order to facility a binary search of the pragma name. +** Do not edit pragma.h directly. Edit and rerun the script in at +** ../tool/mkpragmatab.tcl. */ +/************** Include pragma.h in the middle of pragma.c *******************/ +/************** Begin file pragma.h ******************************************/ +/* DO NOT EDIT! +** This file is automatically generated by the script at +** ../tool/mkpragmatab.tcl. To update the set of pragmas, edit +** that script and rerun it. +*/ + +/* The various pragma types */ +#define PragTyp_ACTIVATE_EXTENSIONS 0 +#define PragTyp_ANALYSIS_LIMIT 1 +#define PragTyp_HEADER_VALUE 2 +#define PragTyp_AUTO_VACUUM 3 +#define PragTyp_FLAG 4 +#define PragTyp_BUSY_TIMEOUT 5 +#define PragTyp_CACHE_SIZE 6 +#define PragTyp_CACHE_SPILL 7 +#define PragTyp_CASE_SENSITIVE_LIKE 8 +#define PragTyp_COLLATION_LIST 9 +#define PragTyp_COMPILE_OPTIONS 10 +#define PragTyp_DATA_STORE_DIRECTORY 11 +#define PragTyp_DATABASE_LIST 12 +#define PragTyp_DEFAULT_CACHE_SIZE 13 +#define PragTyp_ENCODING 14 +#define PragTyp_FOREIGN_KEY_CHECK 15 +#define PragTyp_FOREIGN_KEY_LIST 16 +#define PragTyp_FUNCTION_LIST 17 +#define PragTyp_HARD_HEAP_LIMIT 18 +#define PragTyp_INCREMENTAL_VACUUM 19 +#define PragTyp_INDEX_INFO 20 +#define PragTyp_INDEX_LIST 21 +#define PragTyp_INTEGRITY_CHECK 22 +#define PragTyp_JOURNAL_MODE 23 +#define PragTyp_JOURNAL_SIZE_LIMIT 24 +#define PragTyp_LOCK_PROXY_FILE 25 +#define PragTyp_LOCKING_MODE 26 +#define PragTyp_PAGE_COUNT 27 +#define PragTyp_MMAP_SIZE 28 +#define PragTyp_MODULE_LIST 29 +#define PragTyp_OPTIMIZE 30 +#define PragTyp_PAGE_SIZE 31 +#define PragTyp_PRAGMA_LIST 32 +#define PragTyp_SECURE_DELETE 33 +#define PragTyp_SHRINK_MEMORY 34 +#define PragTyp_SOFT_HEAP_LIMIT 35 +#define PragTyp_SYNCHRONOUS 36 +#define PragTyp_TABLE_INFO 37 +#define PragTyp_TABLE_LIST 38 +#define PragTyp_TEMP_STORE 39 +#define PragTyp_TEMP_STORE_DIRECTORY 40 +#define PragTyp_THREADS 41 +#define PragTyp_WAL_AUTOCHECKPOINT 42 +#define PragTyp_WAL_CHECKPOINT 43 +#define PragTyp_LOCK_STATUS 44 +#define PragTyp_STATS 45 + +/* Property flags associated with various pragma. */ +#define PragFlg_NeedSchema 0x01 /* Force schema load before running */ +#define PragFlg_NoColumns 0x02 /* OP_ResultRow called with zero columns */ +#define PragFlg_NoColumns1 0x04 /* zero columns if RHS argument is present */ +#define PragFlg_ReadOnly 0x08 /* Read-only HEADER_VALUE */ +#define PragFlg_Result0 0x10 /* Acts as query when no argument */ +#define PragFlg_Result1 0x20 /* Acts as query when has one argument */ +#define PragFlg_SchemaOpt 0x40 /* Schema restricts name search if present */ +#define PragFlg_SchemaReq 0x80 /* Schema required - "main" is default */ + +/* Names of columns for pragmas that return multi-column result +** or that return single-column results where the name of the +** result column is different from the name of the pragma +*/ +static const char *const pragCName[] = { + /* 0 */ "id", /* Used by: foreign_key_list */ + /* 1 */ "seq", + /* 2 */ "table", + /* 3 */ "from", + /* 4 */ "to", + /* 5 */ "on_update", + /* 6 */ "on_delete", + /* 7 */ "match", + /* 8 */ "cid", /* Used by: table_xinfo */ + /* 9 */ "name", + /* 10 */ "type", + /* 11 */ "notnull", + /* 12 */ "dflt_value", + /* 13 */ "pk", + /* 14 */ "hidden", + /* table_info reuses 8 */ + /* 15 */ "schema", /* Used by: table_list */ + /* 16 */ "name", + /* 17 */ "type", + /* 18 */ "ncol", + /* 19 */ "wr", + /* 20 */ "strict", + /* 21 */ "seqno", /* Used by: index_xinfo */ + /* 22 */ "cid", + /* 23 */ "name", + /* 24 */ "desc", + /* 25 */ "coll", + /* 26 */ "key", + /* 27 */ "name", /* Used by: function_list */ + /* 28 */ "builtin", + /* 29 */ "type", + /* 30 */ "enc", + /* 31 */ "narg", + /* 32 */ "flags", + /* 33 */ "tbl", /* Used by: stats */ + /* 34 */ "idx", + /* 35 */ "wdth", + /* 36 */ "hght", + /* 37 */ "flgs", + /* 38 */ "seq", /* Used by: index_list */ + /* 39 */ "name", + /* 40 */ "unique", + /* 41 */ "origin", + /* 42 */ "partial", + /* 43 */ "table", /* Used by: foreign_key_check */ + /* 44 */ "rowid", + /* 45 */ "parent", + /* 46 */ "fkid", + /* index_info reuses 21 */ + /* 47 */ "seq", /* Used by: database_list */ + /* 48 */ "name", + /* 49 */ "file", + /* 50 */ "busy", /* Used by: wal_checkpoint */ + /* 51 */ "log", + /* 52 */ "checkpointed", + /* collation_list reuses 38 */ + /* 53 */ "database", /* Used by: lock_status */ + /* 54 */ "status", + /* 55 */ "cache_size", /* Used by: default_cache_size */ + /* module_list pragma_list reuses 9 */ + /* 56 */ "timeout", /* Used by: busy_timeout */ +}; + +/* Definitions of all built-in pragmas */ +typedef struct PragmaName { + const char *const zName; /* Name of pragma */ + u8 ePragTyp; /* PragTyp_XXX value */ + u8 mPragFlg; /* Zero or more PragFlg_XXX values */ + u8 iPragCName; /* Start of column names in pragCName[] */ + u8 nPragCName; /* Num of col names. 0 means use pragma name */ + u64 iArg; /* Extra argument */ +} PragmaName; +static const PragmaName aPragmaName[] = { +#if defined(SQLITE_ENABLE_CEROD) + {/* zName: */ "activate_extensions", + /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif + {/* zName: */ "analysis_limit", + /* ePragTyp: */ PragTyp_ANALYSIS_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + {/* zName: */ "application_id", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_APPLICATION_ID }, +#endif +#if !defined(SQLITE_OMIT_AUTOVACUUM) + {/* zName: */ "auto_vacuum", + /* ePragTyp: */ PragTyp_AUTO_VACUUM, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_AUTOMATIC_INDEX) + {/* zName: */ "automatic_index", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_AutoIndex }, +#endif +#endif + {/* zName: */ "busy_timeout", + /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 56, 1, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "cache_size", + /* ePragTyp: */ PragTyp_CACHE_SIZE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "cache_spill", + /* ePragTyp: */ PragTyp_CACHE_SPILL, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA) + {/* zName: */ "case_sensitive_like", + /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif + {/* zName: */ "cell_size_check", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CellSizeCk }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "checkpoint_fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CkptFullFSync }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + {/* zName: */ "collation_list", + /* ePragTyp: */ PragTyp_COLLATION_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 38, 2, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) + {/* zName: */ "compile_options", + /* ePragTyp: */ PragTyp_COMPILE_OPTIONS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "count_changes", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CountRows }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_OS_WIN + {/* zName: */ "data_store_directory", + /* ePragTyp: */ PragTyp_DATA_STORE_DIRECTORY, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + {/* zName: */ "data_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_DATA_VERSION }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + {/* zName: */ "database_list", + /* ePragTyp: */ PragTyp_DATABASE_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0, + /* ColNames: */ 47, 3, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) + {/* zName: */ "default_cache_size", + /* ePragTyp: */ PragTyp_DEFAULT_CACHE_SIZE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 55, 1, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + {/* zName: */ "defer_foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_DeferFKs }, +#endif +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "empty_result_callbacks", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_NullCallback }, +#endif +#if !defined(SQLITE_OMIT_UTF16) + {/* zName: */ "encoding", + /* ePragTyp: */ PragTyp_ENCODING, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + {/* zName: */ "foreign_key_check", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 43, 4, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FOREIGN_KEY) + {/* zName: */ "foreign_key_list", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 0, 8, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) + {/* zName: */ "foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ForeignKeys }, +#endif +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + {/* zName: */ "freelist_count", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_FREE_PAGE_COUNT }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "full_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_FullColNames }, + {/* zName: */ "fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_FullFSync }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + {/* zName: */ "function_list", + /* ePragTyp: */ PragTyp_FUNCTION_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 27, 6, + /* iArg: */ 0 }, +#endif +#endif + {/* zName: */ "hard_heap_limit", + /* ePragTyp: */ PragTyp_HARD_HEAP_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if !defined(SQLITE_OMIT_CHECK) + {/* zName: */ "ignore_check_constraints", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_IgnoreChecks }, +#endif +#endif +#if !defined(SQLITE_OMIT_AUTOVACUUM) + {/* zName: */ "incremental_vacuum", + /* ePragTyp: */ PragTyp_INCREMENTAL_VACUUM, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + {/* zName: */ "index_info", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 21, 3, + /* iArg: */ 0 }, + {/* zName: */ "index_list", + /* ePragTyp: */ PragTyp_INDEX_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 38, 5, + /* iArg: */ 0 }, + {/* zName: */ "index_xinfo", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 21, 6, + /* iArg: */ 1 }, +#endif +#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) + {/* zName: */ "integrity_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "journal_mode", + /* ePragTyp: */ PragTyp_JOURNAL_MODE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "journal_size_limit", + /* ePragTyp: */ PragTyp_JOURNAL_SIZE_LIMIT, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "legacy_alter_table", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_LegacyAlter }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_ENABLE_LOCKING_STYLE + {/* zName: */ "lock_proxy_file", + /* ePragTyp: */ PragTyp_LOCK_PROXY_FILE, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + {/* zName: */ "lock_status", + /* ePragTyp: */ PragTyp_LOCK_STATUS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 53, 2, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "locking_mode", + /* ePragTyp: */ PragTyp_LOCKING_MODE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "max_page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "mmap_size", + /* ePragTyp: */ PragTyp_MMAP_SIZE, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) +#if !defined(SQLITE_OMIT_VIRTUALTABLE) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + {/* zName: */ "module_list", + /* ePragTyp: */ PragTyp_MODULE_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 9, 1, + /* iArg: */ 0 }, +#endif +#endif +#endif + {/* zName: */ "optimize", + /* ePragTyp: */ PragTyp_OPTIMIZE, + /* ePragFlg: */ PragFlg_Result1|PragFlg_NeedSchema, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "page_size", + /* ePragTyp: */ PragTyp_PAGE_SIZE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) + {/* zName: */ "parser_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ParserTrace }, +#endif +#endif +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + {/* zName: */ "pragma_list", + /* ePragTyp: */ PragTyp_PRAGMA_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 9, 1, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "query_only", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_QueryOnly }, +#endif +#if !defined(SQLITE_OMIT_INTEGRITY_CHECK) + {/* zName: */ "quick_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "read_uncommitted", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ReadUncommit }, + {/* zName: */ "recursive_triggers", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_RecTriggers }, + {/* zName: */ "reverse_unordered_selects", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ReverseOrder }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + {/* zName: */ "schema_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_SCHEMA_VERSION }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "secure_delete", + /* ePragTyp: */ PragTyp_SECURE_DELETE, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "short_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ShortColNames }, +#endif + {/* zName: */ "shrink_memory", + /* ePragTyp: */ PragTyp_SHRINK_MEMORY, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "soft_heap_limit", + /* ePragTyp: */ PragTyp_SOFT_HEAP_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) + {/* zName: */ "sql_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_SqlTrace }, +#endif +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) + {/* zName: */ "stats", + /* ePragTyp: */ PragTyp_STATS, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 33, 5, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "synchronous", + /* ePragTyp: */ PragTyp_SYNCHRONOUS, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) + {/* zName: */ "table_info", + /* ePragTyp: */ PragTyp_TABLE_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 8, 6, + /* iArg: */ 0 }, + {/* zName: */ "table_list", + /* ePragTyp: */ PragTyp_TABLE_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1, + /* ColNames: */ 15, 6, + /* iArg: */ 1 }, + {/* zName: */ "table_xinfo", + /* ePragTyp: */ PragTyp_TABLE_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 8, 7, + /* iArg: */ 1 }, +#endif +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "temp_store", + /* ePragTyp: */ PragTyp_TEMP_STORE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "temp_store_directory", + /* ePragTyp: */ PragTyp_TEMP_STORE_DIRECTORY, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif + {/* zName: */ "threads", + /* ePragTyp: */ PragTyp_THREADS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "trusted_schema", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_TrustedSchema }, +#endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + {/* zName: */ "user_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_USER_VERSION }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) + {/* zName: */ "vdbe_addoptrace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeAddopTrace }, + {/* zName: */ "vdbe_debug", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace }, + {/* zName: */ "vdbe_eqp", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeEQP }, + {/* zName: */ "vdbe_listing", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeListing }, + {/* zName: */ "vdbe_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeTrace }, +#endif +#endif +#if !defined(SQLITE_OMIT_WAL) + {/* zName: */ "wal_autocheckpoint", + /* ePragTyp: */ PragTyp_WAL_AUTOCHECKPOINT, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "wal_checkpoint", + /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, + /* ePragFlg: */ PragFlg_NeedSchema, + /* ColNames: */ 50, 3, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "writable_schema", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_WriteSchema|SQLITE_NoSchemaError }, +#endif +}; +/* Number of pragmas: 68 on by default, 78 total. */ + +/************** End of pragma.h **********************************************/ +/************** Continuing where we left off in pragma.c *********************/ + +/* +** Interpret the given string as a safety level. Return 0 for OFF, +** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or +** unrecognized string argument. The FULL and EXTRA option is disallowed +** if the omitFull parameter it 1. +** +** Note that the values returned are one less that the values that +** should be passed into sqlite3BtreeSetSafetyLevel(). The is done +** to support legacy SQL code. The safety level used to be boolean +** and older scripts may have used numbers 0 for OFF and 1 for ON. +*/ +static u8 getSafetyLevel(const char *z, int omitFull, u8 dflt){ + /* 123456789 123456789 123 */ + static const char zText[] = "onoffalseyestruextrafull"; + static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 15, 20}; + static const u8 iLength[] = {2, 2, 3, 5, 3, 4, 5, 4}; + static const u8 iValue[] = {1, 0, 0, 0, 1, 1, 3, 2}; + /* on no off false yes true extra full */ + int i, n; + if( sqlite3Isdigit(*z) ){ + return (u8)sqlite3Atoi(z); + } + n = sqlite3Strlen30(z); + for(i=0; i=0&&i<=2)?i:0); +} +#endif /* ifndef SQLITE_OMIT_AUTOVACUUM */ + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* +** Interpret the given string as a temp db location. Return 1 for file +** backed temporary databases, 2 for the Red-Black tree in memory database +** and 0 to use the compile-time default. +*/ +static int getTempStore(const char *z){ + if( z[0]>='0' && z[0]<='2' ){ + return z[0] - '0'; + }else if( sqlite3StrICmp(z, "file")==0 ){ + return 1; + }else if( sqlite3StrICmp(z, "memory")==0 ){ + return 2; + }else{ + return 0; + } +} +#endif /* SQLITE_PAGER_PRAGMAS */ + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* +** Invalidate temp storage, either when the temp storage is changed +** from default, or when 'file' and the temp_store_directory has changed +*/ +static int invalidateTempStorage(Parse *pParse){ + sqlite3 *db = pParse->db; + if( db->aDb[1].pBt!=0 ){ + if( !db->autoCommit + || sqlite3BtreeTxnState(db->aDb[1].pBt)!=SQLITE_TXN_NONE + ){ + sqlite3ErrorMsg(pParse, "temporary storage cannot be changed " + "from within a transaction"); + return SQLITE_ERROR; + } + sqlite3BtreeClose(db->aDb[1].pBt); + db->aDb[1].pBt = 0; + sqlite3ResetAllSchemasOfConnection(db); + } + return SQLITE_OK; +} +#endif /* SQLITE_PAGER_PRAGMAS */ + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS +/* +** If the TEMP database is open, close it and mark the database schema +** as needing reloading. This must be done when using the SQLITE_TEMP_STORE +** or DEFAULT_TEMP_STORE pragmas. +*/ +static int changeTempStorage(Parse *pParse, const char *zStorageType){ + int ts = getTempStore(zStorageType); + sqlite3 *db = pParse->db; + if( db->temp_store==ts ) return SQLITE_OK; + if( invalidateTempStorage( pParse ) != SQLITE_OK ){ + return SQLITE_ERROR; + } + db->temp_store = (u8)ts; + return SQLITE_OK; +} +#endif /* SQLITE_PAGER_PRAGMAS */ + +/* +** Set result column names for a pragma. +*/ +static void setPragmaResultColumnNames( + Vdbe *v, /* The query under construction */ + const PragmaName *pPragma /* The pragma */ +){ + u8 n = pPragma->nPragCName; + sqlite3VdbeSetNumCols(v, n==0 ? 1 : n); + if( n==0 ){ + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, pPragma->zName, SQLITE_STATIC); + }else{ + int i, j; + for(i=0, j=pPragma->iPragCName; iautoCommit ){ + Db *pDb = db->aDb; + int n = db->nDb; + assert( SQLITE_FullFSync==PAGER_FULLFSYNC ); + assert( SQLITE_CkptFullFSync==PAGER_CKPT_FULLFSYNC ); + assert( SQLITE_CacheSpill==PAGER_CACHESPILL ); + assert( (PAGER_FULLFSYNC | PAGER_CKPT_FULLFSYNC | PAGER_CACHESPILL) + == PAGER_FLAGS_MASK ); + assert( (pDb->safety_level & PAGER_SYNCHRONOUS_MASK)==pDb->safety_level ); + while( (n--) > 0 ){ + if( pDb->pBt ){ + sqlite3BtreeSetPagerFlags(pDb->pBt, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK) ); + } + pDb++; + } + } +} +#else +# define setAllPagerFlags(X) /* no-op */ +#endif + + +/* +** Return a human-readable name for a constraint resolution action. +*/ +#ifndef SQLITE_OMIT_FOREIGN_KEY +static const char *actionName(u8 action){ + const char *zName; + switch( action ){ + case OE_SetNull: zName = "SET NULL"; break; + case OE_SetDflt: zName = "SET DEFAULT"; break; + case OE_Cascade: zName = "CASCADE"; break; + case OE_Restrict: zName = "RESTRICT"; break; + default: zName = "NO ACTION"; + assert( action==OE_None ); break; + } + return zName; +} +#endif + + +/* +** Parameter eMode must be one of the PAGER_JOURNALMODE_XXX constants +** defined in pager.h. This function returns the associated lowercase +** journal-mode name. +*/ +SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){ + static char * const azModeName[] = { + "delete", "persist", "off", "truncate", "memory" +#ifndef SQLITE_OMIT_WAL + , "wal" +#endif + }; + assert( PAGER_JOURNALMODE_DELETE==0 ); + assert( PAGER_JOURNALMODE_PERSIST==1 ); + assert( PAGER_JOURNALMODE_OFF==2 ); + assert( PAGER_JOURNALMODE_TRUNCATE==3 ); + assert( PAGER_JOURNALMODE_MEMORY==4 ); + assert( PAGER_JOURNALMODE_WAL==5 ); + assert( eMode>=0 && eMode<=ArraySize(azModeName) ); + + if( eMode==ArraySize(azModeName) ) return 0; + return azModeName[eMode]; +} + +/* +** Locate a pragma in the aPragmaName[] array. +*/ +static const PragmaName *pragmaLocate(const char *zName){ + int upr, lwr, mid = 0, rc; + lwr = 0; + upr = ArraySize(aPragmaName)-1; + while( lwr<=upr ){ + mid = (lwr+upr)/2; + rc = sqlite3_stricmp(zName, aPragmaName[mid].zName); + if( rc==0 ) break; + if( rc<0 ){ + upr = mid - 1; + }else{ + lwr = mid + 1; + } + } + return lwr>upr ? 0 : &aPragmaName[mid]; +} + +/* +** Create zero or more entries in the output for the SQL functions +** defined by FuncDef p. +*/ +static void pragmaFunclistLine( + Vdbe *v, /* The prepared statement being created */ + FuncDef *p, /* A particular function definition */ + int isBuiltin, /* True if this is a built-in function */ + int showInternFuncs /* True if showing internal functions */ +){ + for(; p; p=p->pNext){ + const char *zType; + static const u32 mask = + SQLITE_DETERMINISTIC | + SQLITE_DIRECTONLY | + SQLITE_SUBTYPE | + SQLITE_INNOCUOUS | + SQLITE_FUNC_INTERNAL + ; + static const char *azEnc[] = { 0, "utf8", "utf16le", "utf16be" }; + + assert( SQLITE_FUNC_ENCMASK==0x3 ); + assert( strcmp(azEnc[SQLITE_UTF8],"utf8")==0 ); + assert( strcmp(azEnc[SQLITE_UTF16LE],"utf16le")==0 ); + assert( strcmp(azEnc[SQLITE_UTF16BE],"utf16be")==0 ); + + if( p->xSFunc==0 ) continue; + if( (p->funcFlags & SQLITE_FUNC_INTERNAL)!=0 + && showInternFuncs==0 + ){ + continue; + } + if( p->xValue!=0 ){ + zType = "w"; + }else if( p->xFinalize!=0 ){ + zType = "a"; + }else{ + zType = "s"; + } + sqlite3VdbeMultiLoad(v, 1, "sissii", + p->zName, isBuiltin, + zType, azEnc[p->funcFlags&SQLITE_FUNC_ENCMASK], + p->nArg, + (p->funcFlags & mask) ^ SQLITE_INNOCUOUS + ); + } +} + + +/* +** Helper subroutine for PRAGMA integrity_check: +** +** Generate code to output a single-column result row with a value of the +** string held in register 3. Decrement the result count in register 1 +** and halt if the maximum number of result rows have been issued. +*/ +static int integrityCheckResultRow(Vdbe *v){ + int addr; + sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); + addr = sqlite3VdbeAddOp3(v, OP_IfPos, 1, sqlite3VdbeCurrentAddr(v)+2, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp0(v, OP_Halt); + return addr; +} + +/* +** Process a pragma statement. +** +** Pragmas are of this form: +** +** PRAGMA [schema.]id [= value] +** +** The identifier might also be a string. The value is a string, and +** identifier, or a number. If minusFlag is true, then the value is +** a number that was preceded by a minus sign. +** +** If the left side is "database.id" then pId1 is the database name +** and pId2 is the id. If the left side is just "id" then pId1 is the +** id and pId2 is any empty string. +*/ +SQLITE_PRIVATE void sqlite3Pragma( + Parse *pParse, + Token *pId1, /* First part of [schema.]id field */ + Token *pId2, /* Second part of [schema.]id field, or NULL */ + Token *pValue, /* Token for , or NULL */ + int minusFlag /* True if a '-' sign preceded */ +){ + char *zLeft = 0; /* Nul-terminated UTF-8 string */ + char *zRight = 0; /* Nul-terminated UTF-8 string , or NULL */ + const char *zDb = 0; /* The database name */ + Token *pId; /* Pointer to token */ + char *aFcntl[4]; /* Argument to SQLITE_FCNTL_PRAGMA */ + int iDb; /* Database index for */ + int rc; /* return value form SQLITE_FCNTL_PRAGMA */ + sqlite3 *db = pParse->db; /* The database connection */ + Db *pDb; /* The specific database being pragmaed */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ + const PragmaName *pPragma; /* The pragma */ + + if( v==0 ) return; + sqlite3VdbeRunOnlyOnce(v); + pParse->nMem = 2; + + /* Interpret the [schema.] part of the pragma statement. iDb is the + ** index of the database this pragma is being applied to in db.aDb[]. */ + iDb = sqlite3TwoPartName(pParse, pId1, pId2, &pId); + if( iDb<0 ) return; + pDb = &db->aDb[iDb]; + + /* If the temp database has been explicitly named as part of the + ** pragma, make sure it is open. + */ + if( iDb==1 && sqlite3OpenTempDatabase(pParse) ){ + return; + } + + zLeft = sqlite3NameFromToken(db, pId); + if( !zLeft ) return; + if( minusFlag ){ + zRight = sqlite3MPrintf(db, "-%T", pValue); + }else{ + zRight = sqlite3NameFromToken(db, pValue); + } + + assert( pId2 ); + zDb = pId2->n>0 ? pDb->zDbSName : 0; + if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ + goto pragma_out; + } + + /* Send an SQLITE_FCNTL_PRAGMA file-control to the underlying VFS + ** connection. If it returns SQLITE_OK, then assume that the VFS + ** handled the pragma and generate a no-op prepared statement. + ** + ** IMPLEMENTATION-OF: R-12238-55120 Whenever a PRAGMA statement is parsed, + ** an SQLITE_FCNTL_PRAGMA file control is sent to the open sqlite3_file + ** object corresponding to the database file to which the pragma + ** statement refers. + ** + ** IMPLEMENTATION-OF: R-29875-31678 The argument to the SQLITE_FCNTL_PRAGMA + ** file control is an array of pointers to strings (char**) in which the + ** second element of the array is the name of the pragma and the third + ** element is the argument to the pragma or NULL if the pragma has no + ** argument. + */ + aFcntl[0] = 0; + aFcntl[1] = zLeft; + aFcntl[2] = zRight; + aFcntl[3] = 0; + db->busyHandler.nBusy = 0; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); + if( rc==SQLITE_OK ){ + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, aFcntl[0], SQLITE_TRANSIENT); + returnSingleText(v, aFcntl[0]); + sqlite3_free(aFcntl[0]); + goto pragma_out; + } + if( rc!=SQLITE_NOTFOUND ){ + if( aFcntl[0] ){ + sqlite3ErrorMsg(pParse, "%s", aFcntl[0]); + sqlite3_free(aFcntl[0]); + } + pParse->nErr++; + pParse->rc = rc; + goto pragma_out; + } + + /* Locate the pragma in the lookup table */ + pPragma = pragmaLocate(zLeft); + if( pPragma==0 ) goto pragma_out; + + /* Make sure the database schema is loaded if the pragma requires that */ + if( (pPragma->mPragFlg & PragFlg_NeedSchema)!=0 ){ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + } + + /* Register the result column names for pragmas that return results */ + if( (pPragma->mPragFlg & PragFlg_NoColumns)==0 + && ((pPragma->mPragFlg & PragFlg_NoColumns1)==0 || zRight==0) + ){ + setPragmaResultColumnNames(v, pPragma); + } + + /* Jump to the appropriate pragma handler */ + switch( pPragma->ePragTyp ){ + +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) + /* + ** PRAGMA [schema.]default_cache_size + ** PRAGMA [schema.]default_cache_size=N + ** + ** The first form reports the current persistent setting for the + ** page cache size. The value returned is the maximum number of + ** pages in the page cache. The second form sets both the current + ** page cache size value and the persistent page cache size value + ** stored in the database file. + ** + ** Older versions of SQLite would set the default cache size to a + ** negative number to indicate synchronous=OFF. These days, synchronous + ** is always on by default regardless of the sign of the default cache + ** size. But continue to take the absolute value of the default cache + ** size of historical compatibility. + */ + case PragTyp_DEFAULT_CACHE_SIZE: { + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList getCacheSize[] = { + { OP_Transaction, 0, 0, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ + { OP_IfPos, 1, 8, 0}, + { OP_Integer, 0, 2, 0}, + { OP_Subtract, 1, 2, 1}, + { OP_IfPos, 1, 8, 0}, + { OP_Integer, 0, 1, 0}, /* 6 */ + { OP_Noop, 0, 0, 0}, + { OP_ResultRow, 1, 1, 0}, + }; + VdbeOp *aOp; + sqlite3VdbeUsesBtree(v, iDb); + if( !zRight ){ + pParse->nMem += 2; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(getCacheSize)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize, iLn); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[6].p1 = SQLITE_DEFAULT_CACHE_SIZE; + }else{ + int size = sqlite3AbsInt32(sqlite3Atoi(zRight)); + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_DEFAULT_CACHE_SIZE, size); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pDb->pSchema->cache_size = size; + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } + break; + } +#endif /* !SQLITE_OMIT_PAGER_PRAGMAS && !SQLITE_OMIT_DEPRECATED */ + +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + /* + ** PRAGMA [schema.]page_size + ** PRAGMA [schema.]page_size=N + ** + ** The first form reports the current setting for the + ** database page size in bytes. The second form sets the + ** database page size value. The value can only be set if + ** the database has not yet been created. + */ + case PragTyp_PAGE_SIZE: { + Btree *pBt = pDb->pBt; + assert( pBt!=0 ); + if( !zRight ){ + int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; + returnSingleInt(v, size); + }else{ + /* Malloc may fail when setting the page-size, as there is an internal + ** buffer that the pager module resizes using sqlite3_realloc(). + */ + db->nextPagesize = sqlite3Atoi(zRight); + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,0,0) ){ + sqlite3OomFault(db); + } + } + break; + } + + /* + ** PRAGMA [schema.]secure_delete + ** PRAGMA [schema.]secure_delete=ON/OFF/FAST + ** + ** The first form reports the current setting for the + ** secure_delete flag. The second form changes the secure_delete + ** flag setting and reports the new value. + */ + case PragTyp_SECURE_DELETE: { + Btree *pBt = pDb->pBt; + int b = -1; + assert( pBt!=0 ); + if( zRight ){ + if( sqlite3_stricmp(zRight, "fast")==0 ){ + b = 2; + }else{ + b = sqlite3GetBoolean(zRight, 0); + } + } + if( pId2->n==0 && b>=0 ){ + int ii; + for(ii=0; iinDb; ii++){ + sqlite3BtreeSecureDelete(db->aDb[ii].pBt, b); + } + } + b = sqlite3BtreeSecureDelete(pBt, b); + returnSingleInt(v, b); + break; + } + + /* + ** PRAGMA [schema.]max_page_count + ** PRAGMA [schema.]max_page_count=N + ** + ** The first form reports the current setting for the + ** maximum number of pages in the database file. The + ** second form attempts to change this setting. Both + ** forms return the current setting. + ** + ** The absolute value of N is used. This is undocumented and might + ** change. The only purpose is to provide an easy way to test + ** the sqlite3AbsInt32() function. + ** + ** PRAGMA [schema.]page_count + ** + ** Return the number of pages in the specified database. + */ + case PragTyp_PAGE_COUNT: { + int iReg; + i64 x = 0; + sqlite3CodeVerifySchema(pParse, iDb); + iReg = ++pParse->nMem; + if( sqlite3Tolower(zLeft[0])=='p' ){ + sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); + }else{ + if( zRight && sqlite3DecOrHexToI64(zRight,&x)==0 ){ + if( x<0 ) x = 0; + else if( x>0xfffffffe ) x = 0xfffffffe; + }else{ + x = 0; + } + sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, (int)x); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); + break; + } + + /* + ** PRAGMA [schema.]locking_mode + ** PRAGMA [schema.]locking_mode = (normal|exclusive) + */ + case PragTyp_LOCKING_MODE: { + const char *zRet = "normal"; + int eMode = getLockingMode(zRight); + + if( pId2->n==0 && eMode==PAGER_LOCKINGMODE_QUERY ){ + /* Simple "PRAGMA locking_mode;" statement. This is a query for + ** the current default locking mode (which may be different to + ** the locking-mode of the main database). + */ + eMode = db->dfltLockMode; + }else{ + Pager *pPager; + if( pId2->n==0 ){ + /* This indicates that no database name was specified as part + ** of the PRAGMA command. In this case the locking-mode must be + ** set on all attached databases, as well as the main db file. + ** + ** Also, the sqlite3.dfltLockMode variable is set so that + ** any subsequently attached databases also use the specified + ** locking mode. + */ + int ii; + assert(pDb==&db->aDb[0]); + for(ii=2; iinDb; ii++){ + pPager = sqlite3BtreePager(db->aDb[ii].pBt); + sqlite3PagerLockingMode(pPager, eMode); + } + db->dfltLockMode = (u8)eMode; + } + pPager = sqlite3BtreePager(pDb->pBt); + eMode = sqlite3PagerLockingMode(pPager, eMode); + } + + assert( eMode==PAGER_LOCKINGMODE_NORMAL + || eMode==PAGER_LOCKINGMODE_EXCLUSIVE ); + if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ + zRet = "exclusive"; + } + returnSingleText(v, zRet); + break; + } + + /* + ** PRAGMA [schema.]journal_mode + ** PRAGMA [schema.]journal_mode = + ** (delete|persist|off|truncate|memory|wal|off) + */ + case PragTyp_JOURNAL_MODE: { + int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ + int ii; /* Loop counter */ + + if( zRight==0 ){ + /* If there is no "=MODE" part of the pragma, do a query for the + ** current mode */ + eMode = PAGER_JOURNALMODE_QUERY; + }else{ + const char *zMode; + int n = sqlite3Strlen30(zRight); + for(eMode=0; (zMode = sqlite3JournalModename(eMode))!=0; eMode++){ + if( sqlite3StrNICmp(zRight, zMode, n)==0 ) break; + } + if( !zMode ){ + /* If the "=MODE" part does not match any known journal mode, + ** then do a query */ + eMode = PAGER_JOURNALMODE_QUERY; + } + if( eMode==PAGER_JOURNALMODE_OFF && (db->flags & SQLITE_Defensive)!=0 ){ + /* Do not allow journal-mode "OFF" in defensive since the database + ** can become corrupted using ordinary SQL when the journal is off */ + eMode = PAGER_JOURNALMODE_QUERY; + } + } + if( eMode==PAGER_JOURNALMODE_QUERY && pId2->n==0 ){ + /* Convert "PRAGMA journal_mode" into "PRAGMA main.journal_mode" */ + iDb = 0; + pId2->n = 1; + } + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3VdbeUsesBtree(v, ii); + sqlite3VdbeAddOp3(v, OP_JournalMode, ii, 1, eMode); + } + } + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + break; + } + + /* + ** PRAGMA [schema.]journal_size_limit + ** PRAGMA [schema.]journal_size_limit=N + ** + ** Get or set the size limit on rollback journal files. + */ + case PragTyp_JOURNAL_SIZE_LIMIT: { + Pager *pPager = sqlite3BtreePager(pDb->pBt); + i64 iLimit = -2; + if( zRight ){ + sqlite3DecOrHexToI64(zRight, &iLimit); + if( iLimit<-1 ) iLimit = -1; + } + iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); + returnSingleInt(v, iLimit); + break; + } + +#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ + + /* + ** PRAGMA [schema.]auto_vacuum + ** PRAGMA [schema.]auto_vacuum=N + ** + ** Get or set the value of the database 'auto-vacuum' parameter. + ** The value is one of: 0 NONE 1 FULL 2 INCREMENTAL + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + case PragTyp_AUTO_VACUUM: { + Btree *pBt = pDb->pBt; + assert( pBt!=0 ); + if( !zRight ){ + returnSingleInt(v, sqlite3BtreeGetAutoVacuum(pBt)); + }else{ + int eAuto = getAutoVacuum(zRight); + assert( eAuto>=0 && eAuto<=2 ); + db->nextAutovac = (u8)eAuto; + /* Call SetAutoVacuum() to set initialize the internal auto and + ** incr-vacuum flags. This is required in case this connection + ** creates the database file. It is important that it is created + ** as an auto-vacuum capable db. + */ + rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); + if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ + /* When setting the auto_vacuum mode to either "full" or + ** "incremental", write the value of meta[6] in the database + ** file. Before writing to meta[6], check that meta[3] indicates + ** that this really is an auto-vacuum capable database. + */ + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList setMeta6[] = { + { OP_Transaction, 0, 1, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, + { OP_If, 1, 0, 0}, /* 2 */ + { OP_Halt, SQLITE_OK, OE_Abort, 0}, /* 3 */ + { OP_SetCookie, 0, BTREE_INCR_VACUUM, 0}, /* 4 */ + }; + VdbeOp *aOp; + int iAddr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setMeta6)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(setMeta6), setMeta6, iLn); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[2].p2 = iAddr+4; + aOp[4].p1 = iDb; + aOp[4].p3 = eAuto - 1; + sqlite3VdbeUsesBtree(v, iDb); + } + } + break; + } +#endif + + /* + ** PRAGMA [schema.]incremental_vacuum(N) + ** + ** Do N steps of incremental vacuuming on a database. + */ +#ifndef SQLITE_OMIT_AUTOVACUUM + case PragTyp_INCREMENTAL_VACUUM: { + int iLimit, addr; + if( zRight==0 || !sqlite3GetInt32(zRight, &iLimit) || iLimit<=0 ){ + iLimit = 0x7fffffff; + } + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3VdbeAddOp2(v, OP_Integer, iLimit, 1); + addr = sqlite3VdbeAddOp1(v, OP_IncrVacuum, iDb); VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_ResultRow, 1); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); + sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + break; + } +#endif + +#ifndef SQLITE_OMIT_PAGER_PRAGMAS + /* + ** PRAGMA [schema.]cache_size + ** PRAGMA [schema.]cache_size=N + ** + ** The first form reports the current local setting for the + ** page cache size. The second form sets the local + ** page cache size value. If N is positive then that is the + ** number of pages in the cache. If N is negative, then the + ** number of pages is adjusted so that the cache uses -N kibibytes + ** of memory. + */ + case PragTyp_CACHE_SIZE: { + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(v, pDb->pSchema->cache_size); + }else{ + int size = sqlite3Atoi(zRight); + pDb->pSchema->cache_size = size; + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } + break; + } + + /* + ** PRAGMA [schema.]cache_spill + ** PRAGMA cache_spill=BOOLEAN + ** PRAGMA [schema.]cache_spill=N + ** + ** The first form reports the current local setting for the + ** page cache spill size. The second form turns cache spill on + ** or off. When turnning cache spill on, the size is set to the + ** current cache_size. The third form sets a spill size that + ** may be different form the cache size. + ** If N is positive then that is the + ** number of pages in the cache. If N is negative, then the + ** number of pages is adjusted so that the cache uses -N kibibytes + ** of memory. + ** + ** If the number of cache_spill pages is less then the number of + ** cache_size pages, no spilling occurs until the page count exceeds + ** the number of cache_size pages. + ** + ** The cache_spill=BOOLEAN setting applies to all attached schemas, + ** not just the schema specified. + */ + case PragTyp_CACHE_SPILL: { + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(v, + (db->flags & SQLITE_CacheSpill)==0 ? 0 : + sqlite3BtreeSetSpillSize(pDb->pBt,0)); + }else{ + int size = 1; + if( sqlite3GetInt32(zRight, &size) ){ + sqlite3BtreeSetSpillSize(pDb->pBt, size); + } + if( sqlite3GetBoolean(zRight, size!=0) ){ + db->flags |= SQLITE_CacheSpill; + }else{ + db->flags &= ~(u64)SQLITE_CacheSpill; + } + setAllPagerFlags(db); + } + break; + } + + /* + ** PRAGMA [schema.]mmap_size(N) + ** + ** Used to set mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_SIZE) is set. + ** The parameter N is measured in bytes. + ** + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. + */ + case PragTyp_MMAP_SIZE: { + sqlite3_int64 sz; +#if SQLITE_MAX_MMAP_SIZE>0 + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( zRight ){ + int ii; + sqlite3DecOrHexToI64(zRight, &sz); + if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; + if( pId2->n==0 ) db->szMmap = sz; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, sz); + } + } + } + sz = -1; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_MMAP_SIZE, &sz); +#else + sz = 0; + rc = SQLITE_OK; +#endif + if( rc==SQLITE_OK ){ + returnSingleInt(v, sz); + }else if( rc!=SQLITE_NOTFOUND ){ + pParse->nErr++; + pParse->rc = rc; + } + break; + } + + /* + ** PRAGMA temp_store + ** PRAGMA temp_store = "default"|"memory"|"file" + ** + ** Return or set the local value of the temp_store flag. Changing + ** the local value does not make changes to the disk file and the default + ** value will be restored the next time the database is opened. + ** + ** Note that it is possible for the library compile-time options to + ** override this setting + */ + case PragTyp_TEMP_STORE: { + if( !zRight ){ + returnSingleInt(v, db->temp_store); + }else{ + changeTempStorage(pParse, zRight); + } + break; + } + + /* + ** PRAGMA temp_store_directory + ** PRAGMA temp_store_directory = ""|"directory_name" + ** + ** Return or set the local value of the temp_store_directory flag. Changing + ** the value sets a specific directory to be used for temporary files. + ** Setting to a null string reverts to the default temporary directory search. + ** If temporary directory is changed, then invalidateTempStorage. + ** + */ + case PragTyp_TEMP_STORE_DIRECTORY: { + if( !zRight ){ + returnSingleText(v, sqlite3_temp_directory); + }else{ +#ifndef SQLITE_OMIT_WSD + if( zRight[0] ){ + int res; + rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); + if( rc!=SQLITE_OK || res==0 ){ + sqlite3ErrorMsg(pParse, "not a writable directory"); + goto pragma_out; + } + } + if( SQLITE_TEMP_STORE==0 + || (SQLITE_TEMP_STORE==1 && db->temp_store<=1) + || (SQLITE_TEMP_STORE==2 && db->temp_store==1) + ){ + invalidateTempStorage(pParse); + } + sqlite3_free(sqlite3_temp_directory); + if( zRight[0] ){ + sqlite3_temp_directory = sqlite3_mprintf("%s", zRight); + }else{ + sqlite3_temp_directory = 0; + } +#endif /* SQLITE_OMIT_WSD */ + } + break; + } + +#if SQLITE_OS_WIN + /* + ** PRAGMA data_store_directory + ** PRAGMA data_store_directory = ""|"directory_name" + ** + ** Return or set the local value of the data_store_directory flag. Changing + ** the value sets a specific directory to be used for database files that + ** were specified with a relative pathname. Setting to a null string reverts + ** to the default database directory, which for database files specified with + ** a relative path will probably be based on the current directory for the + ** process. Database file specified with an absolute path are not impacted + ** by this setting, regardless of its value. + ** + */ + case PragTyp_DATA_STORE_DIRECTORY: { + if( !zRight ){ + returnSingleText(v, sqlite3_data_directory); + }else{ +#ifndef SQLITE_OMIT_WSD + if( zRight[0] ){ + int res; + rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); + if( rc!=SQLITE_OK || res==0 ){ + sqlite3ErrorMsg(pParse, "not a writable directory"); + goto pragma_out; + } + } + sqlite3_free(sqlite3_data_directory); + if( zRight[0] ){ + sqlite3_data_directory = sqlite3_mprintf("%s", zRight); + }else{ + sqlite3_data_directory = 0; + } +#endif /* SQLITE_OMIT_WSD */ + } + break; + } +#endif + +#if SQLITE_ENABLE_LOCKING_STYLE + /* + ** PRAGMA [schema.]lock_proxy_file + ** PRAGMA [schema.]lock_proxy_file = ":auto:"|"lock_file_path" + ** + ** Return or set the value of the lock_proxy_file flag. Changing + ** the value sets a specific file to be used for database access locks. + ** + */ + case PragTyp_LOCK_PROXY_FILE: { + if( !zRight ){ + Pager *pPager = sqlite3BtreePager(pDb->pBt); + char *proxy_file_path = NULL; + sqlite3_file *pFile = sqlite3PagerFile(pPager); + sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, + &proxy_file_path); + returnSingleText(v, proxy_file_path); + }else{ + Pager *pPager = sqlite3BtreePager(pDb->pBt); + sqlite3_file *pFile = sqlite3PagerFile(pPager); + int res; + if( zRight[0] ){ + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + zRight); + } else { + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + NULL); + } + if( res!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "failed to set lock proxy file"); + goto pragma_out; + } + } + break; + } +#endif /* SQLITE_ENABLE_LOCKING_STYLE */ + + /* + ** PRAGMA [schema.]synchronous + ** PRAGMA [schema.]synchronous=OFF|ON|NORMAL|FULL|EXTRA + ** + ** Return or set the local value of the synchronous flag. Changing + ** the local value does not make changes to the disk file and the + ** default value will be restored the next time the database is + ** opened. + */ + case PragTyp_SYNCHRONOUS: { + if( !zRight ){ + returnSingleInt(v, pDb->safety_level-1); + }else{ + if( !db->autoCommit ){ + sqlite3ErrorMsg(pParse, + "Safety level may not be changed inside a transaction"); + }else if( iDb!=1 ){ + int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; + if( iLevel==0 ) iLevel = 1; + pDb->safety_level = iLevel; + pDb->bSyncSet = 1; + setAllPagerFlags(db); + } + } + break; + } +#endif /* SQLITE_OMIT_PAGER_PRAGMAS */ + +#ifndef SQLITE_OMIT_FLAG_PRAGMAS + case PragTyp_FLAG: { + if( zRight==0 ){ + setPragmaResultColumnNames(v, pPragma); + returnSingleInt(v, (db->flags & pPragma->iArg)!=0 ); + }else{ + u64 mask = pPragma->iArg; /* Mask of bits to set or clear. */ + if( db->autoCommit==0 ){ + /* Foreign key support may not be enabled or disabled while not + ** in auto-commit mode. */ + mask &= ~(SQLITE_ForeignKeys); + } +#if SQLITE_USER_AUTHENTICATION + if( db->auth.authLevel==UAUTH_User ){ + /* Do not allow non-admin users to modify the schema arbitrarily */ + mask &= ~(SQLITE_WriteSchema); + } +#endif + + if( sqlite3GetBoolean(zRight, 0) ){ + db->flags |= mask; + }else{ + db->flags &= ~mask; + if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0; + } + + /* Many of the flag-pragmas modify the code generated by the SQL + ** compiler (eg. count_changes). So add an opcode to expire all + ** compiled SQL statements after modifying a pragma value. + */ + sqlite3VdbeAddOp0(v, OP_Expire); + setAllPagerFlags(db); + } + break; + } +#endif /* SQLITE_OMIT_FLAG_PRAGMAS */ + +#ifndef SQLITE_OMIT_SCHEMA_PRAGMAS + /* + ** PRAGMA table_info(
    ) + ** + ** Return a single row for each column of the named table. The columns of + ** the returned data set are: + ** + ** cid: Column id (numbered from left to right, starting at 0) + ** name: Column name + ** type: Column declaration type. + ** notnull: True if 'NOT NULL' is part of column declaration + ** dflt_value: The default value for the column, if any. + ** pk: Non-zero for PK fields. + */ + case PragTyp_TABLE_INFO: if( zRight ){ + Table *pTab; + sqlite3CodeVerifyNamedSchema(pParse, zDb); + pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); + if( pTab ){ + int i, k; + int nHidden = 0; + Column *pCol; + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + pParse->nMem = 7; + sqlite3ViewGetColumnNames(pParse, pTab); + for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ + int isHidden = 0; + if( pCol->colFlags & COLFLAG_NOINSERT ){ + if( pPragma->iArg==0 ){ + nHidden++; + continue; + } + if( pCol->colFlags & COLFLAG_VIRTUAL ){ + isHidden = 2; /* GENERATED ALWAYS AS ... VIRTUAL */ + }else if( pCol->colFlags & COLFLAG_STORED ){ + isHidden = 3; /* GENERATED ALWAYS AS ... STORED */ + }else{ assert( pCol->colFlags & COLFLAG_HIDDEN ); + isHidden = 1; /* HIDDEN */ + } + } + if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ + k = 0; + }else if( pPk==0 ){ + k = 1; + }else{ + for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} + } + assert( sqlite3ColumnExpr(pTab,pCol)==0 + || sqlite3ColumnExpr(pTab,pCol)->op==TK_SPAN + || isHidden>=2 ); + sqlite3VdbeMultiLoad(v, 1, pPragma->iArg ? "issisii" : "issisi", + i-nHidden, + pCol->zCnName, + sqlite3ColumnType(pCol,""), + pCol->notNull ? 1 : 0, + isHidden>=2 || sqlite3ColumnExpr(pTab,pCol)==0 ? 0 : + sqlite3ColumnExpr(pTab,pCol)->u.zToken, + k, + isHidden); + } + } + } + break; + + /* + ** PRAGMA table_list + ** + ** Return a single row for each table, virtual table, or view in the + ** entire schema. + ** + ** schema: Name of attached database hold this table + ** name: Name of the table itself + ** type: "table", "view", "virtual", "shadow" + ** ncol: Number of columns + ** wr: True for a WITHOUT ROWID table + ** strict: True for a STRICT table + */ + case PragTyp_TABLE_LIST: { + int ii; + pParse->nMem = 6; + sqlite3CodeVerifyNamedSchema(pParse, zDb); + for(ii=0; iinDb; ii++){ + HashElem *k; + Hash *pHash; + if( zDb && sqlite3_stricmp(zDb, db->aDb[ii].zDbSName)!=0 ) continue; + pHash = &db->aDb[ii].pSchema->tblHash; + for(k=sqliteHashFirst(pHash); k; k=sqliteHashNext(k) ){ + Table *pTab = sqliteHashData(k); + const char *zType; + if( zRight && sqlite3_stricmp(zRight, pTab->zName)!=0 ) continue; + if( IsView(pTab) ){ + zType = "view"; + }else if( IsVirtual(pTab) ){ + zType = "virtual"; + }else if( pTab->tabFlags & TF_Shadow ){ + zType = "shadow"; + }else{ + zType = "table"; + } + sqlite3VdbeMultiLoad(v, 1, "sssiii", + db->aDb[ii].zDbSName, + pTab->zName, + zType, + pTab->nCol, + (pTab->tabFlags & TF_WithoutRowid)!=0, + (pTab->tabFlags & TF_Strict)!=0 + ); + } + } + } + break; + +#ifdef SQLITE_DEBUG + case PragTyp_STATS: { + Index *pIdx; + HashElem *i; + pParse->nMem = 5; + sqlite3CodeVerifySchema(pParse, iDb); + for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + sqlite3VdbeMultiLoad(v, 1, "ssiii", + pTab->zName, + 0, + pTab->szTabRow, + pTab->nRowLogEst, + pTab->tabFlags); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3VdbeMultiLoad(v, 2, "siiiX", + pIdx->zName, + pIdx->szIdxRow, + pIdx->aiRowLogEst[0], + pIdx->hasStat1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); + } + } + } + break; +#endif + + case PragTyp_INDEX_INFO: if( zRight ){ + Index *pIdx; + Table *pTab; + pIdx = sqlite3FindIndex(db, zRight, zDb); + if( pIdx==0 ){ + /* If there is no index named zRight, check to see if there is a + ** WITHOUT ROWID table named zRight, and if there is, show the + ** structure of the PRIMARY KEY index for that table. */ + pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); + if( pTab && !HasRowid(pTab) ){ + pIdx = sqlite3PrimaryKeyIndex(pTab); + } + } + if( pIdx ){ + int iIdxDb = sqlite3SchemaToIndex(db, pIdx->pSchema); + int i; + int mx; + if( pPragma->iArg ){ + /* PRAGMA index_xinfo (newer version with more rows and columns) */ + mx = pIdx->nColumn; + pParse->nMem = 6; + }else{ + /* PRAGMA index_info (legacy version) */ + mx = pIdx->nKeyCol; + pParse->nMem = 3; + } + pTab = pIdx->pTable; + sqlite3CodeVerifySchema(pParse, iIdxDb); + assert( pParse->nMem<=pPragma->nPragCName ); + for(i=0; iaiColumn[i]; + sqlite3VdbeMultiLoad(v, 1, "iisX", i, cnum, + cnum<0 ? 0 : pTab->aCol[cnum].zCnName); + if( pPragma->iArg ){ + sqlite3VdbeMultiLoad(v, 4, "isiX", + pIdx->aSortOrder[i], + pIdx->azColl[i], + inKeyCol); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, pParse->nMem); + } + } + } + break; + + case PragTyp_INDEX_LIST: if( zRight ){ + Index *pIdx; + Table *pTab; + int i; + pTab = sqlite3FindTable(db, zRight, zDb); + if( pTab ){ + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pParse->nMem = 5; + sqlite3CodeVerifySchema(pParse, iTabDb); + for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ + const char *azOrigin[] = { "c", "u", "pk" }; + sqlite3VdbeMultiLoad(v, 1, "isisi", + i, + pIdx->zName, + IsUniqueIndex(pIdx), + azOrigin[pIdx->idxType], + pIdx->pPartIdxWhere!=0); + } + } + } + break; + + case PragTyp_DATABASE_LIST: { + int i; + pParse->nMem = 3; + for(i=0; inDb; i++){ + if( db->aDb[i].pBt==0 ) continue; + assert( db->aDb[i].zDbSName!=0 ); + sqlite3VdbeMultiLoad(v, 1, "iss", + i, + db->aDb[i].zDbSName, + sqlite3BtreeGetFilename(db->aDb[i].pBt)); + } + } + break; + + case PragTyp_COLLATION_LIST: { + int i = 0; + HashElem *p; + pParse->nMem = 2; + for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(p); + sqlite3VdbeMultiLoad(v, 1, "is", i++, pColl->zName); + } + } + break; + +#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS + case PragTyp_FUNCTION_LIST: { + int i; + HashElem *j; + FuncDef *p; + int showInternFunc = (db->mDbFlags & DBFLAG_InternalFunc)!=0; + pParse->nMem = 6; + for(i=0; iu.pHash ){ + pragmaFunclistLine(v, p, 1, showInternFunc); + } + } + for(j=sqliteHashFirst(&db->aFunc); j; j=sqliteHashNext(j)){ + p = (FuncDef*)sqliteHashData(j); + pragmaFunclistLine(v, p, 0, showInternFunc); + } + } + break; + +#ifndef SQLITE_OMIT_VIRTUALTABLE + case PragTyp_MODULE_LIST: { + HashElem *j; + pParse->nMem = 1; + for(j=sqliteHashFirst(&db->aModule); j; j=sqliteHashNext(j)){ + Module *pMod = (Module*)sqliteHashData(j); + sqlite3VdbeMultiLoad(v, 1, "s", pMod->zName); + } + } + break; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + case PragTyp_PRAGMA_LIST: { + int i; + for(i=0; iu.tab.pFKey; + if( pFK ){ + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); + int i = 0; + pParse->nMem = 8; + sqlite3CodeVerifySchema(pParse, iTabDb); + while(pFK){ + int j; + for(j=0; jnCol; j++){ + sqlite3VdbeMultiLoad(v, 1, "iissssss", + i, + j, + pFK->zTo, + pTab->aCol[pFK->aCol[j].iFrom].zCnName, + pFK->aCol[j].zCol, + actionName(pFK->aAction[1]), /* ON UPDATE */ + actionName(pFK->aAction[0]), /* ON DELETE */ + "NONE"); + } + ++i; + pFK = pFK->pNextFrom; + } + } + } + } + break; +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + +#ifndef SQLITE_OMIT_FOREIGN_KEY +#ifndef SQLITE_OMIT_TRIGGER + case PragTyp_FOREIGN_KEY_CHECK: { + FKey *pFK; /* A foreign key constraint */ + Table *pTab; /* Child table contain "REFERENCES" keyword */ + Table *pParent; /* Parent table that child points to */ + Index *pIdx; /* Index in the parent table */ + int i; /* Loop counter: Foreign key number for pTab */ + int j; /* Loop counter: Field of the foreign key */ + HashElem *k; /* Loop counter: Next table in schema */ + int x; /* result variable */ + int regResult; /* 3 registers to hold a result row */ + int regKey; /* Register to hold key for checking the FK */ + int regRow; /* Registers to hold a row from pTab */ + int addrTop; /* Top of a loop checking foreign keys */ + int addrOk; /* Jump here if the key is OK */ + int *aiCols; /* child to parent column mapping */ + + regResult = pParse->nMem+1; + pParse->nMem += 4; + regKey = ++pParse->nMem; + regRow = ++pParse->nMem; + k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); + while( k ){ + if( zRight ){ + pTab = sqlite3LocateTable(pParse, 0, zRight, zDb); + k = 0; + }else{ + pTab = (Table*)sqliteHashData(k); + k = sqliteHashNext(k); + } + if( pTab==0 || IsVirtual(pTab) || pTab->u.tab.pFKey==0 ) continue; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zDb = db->aDb[iDb].zDbSName; + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); + sqlite3VdbeLoadString(v, regResult, pTab->zName); + assert( !IsVirtual(pTab) ); + for(i=1, pFK=pTab->u.tab.pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3FindTable(db, pFK->zTo, zDb); + if( pParent==0 ) continue; + pIdx = 0; + sqlite3TableLock(pParse, iDb, pParent->tnum, 0, pParent->zName); + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, 0); + if( x==0 ){ + if( pIdx==0 ){ + sqlite3OpenTable(pParse, i, iDb, pParent, OP_OpenRead); + }else{ + sqlite3VdbeAddOp3(v, OP_OpenRead, i, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + } + }else{ + k = 0; + break; + } + } + assert( pParse->nErr>0 || pFK==0 ); + if( pFK ) break; + if( pParse->nTabnTab = i; + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, 0); VdbeCoverage(v); + assert( !IsVirtual(pTab) ); + for(i=1, pFK=pTab->u.tab.pFKey; pFK; i++, pFK=pFK->pNextFrom){ + pParent = sqlite3FindTable(db, pFK->zTo, zDb); + pIdx = 0; + aiCols = 0; + if( pParent ){ + x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols); + assert( x==0 || db->mallocFailed ); + } + addrOk = sqlite3VdbeMakeLabel(pParse); + + /* Generate code to read the child key values into registers + ** regRow..regRow+n. If any of the child key values are NULL, this + ** row cannot cause an FK violation. Jump directly to addrOk in + ** this case. */ + if( regRow+pFK->nCol>pParse->nMem ) pParse->nMem = regRow+pFK->nCol; + for(j=0; jnCol; j++){ + int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom; + sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v); + } + + /* Generate code to query the parent index for a matching parent + ** key. If a match is found, jump to addrOk. */ + if( pIdx ){ + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, + sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); + sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); + VdbeCoverage(v); + }else if( pParent ){ + int jmp = sqlite3VdbeCurrentAddr(v)+2; + sqlite3VdbeAddOp3(v, OP_SeekRowid, i, jmp, regRow); VdbeCoverage(v); + sqlite3VdbeGoto(v, addrOk); + assert( pFK->nCol==1 || db->mallocFailed ); + } + + /* Generate code to report an FK violation to the caller. */ + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, regResult+1); + } + sqlite3VdbeMultiLoad(v, regResult+2, "siX", pFK->zTo, i-1); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); + sqlite3VdbeResolveLabel(v, addrOk); + sqlite3DbFree(db, aiCols); + } + sqlite3VdbeAddOp2(v, OP_Next, 0, addrTop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrTop); + } + } + break; +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ +#endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ + +#ifndef SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA + /* Reinstall the LIKE and GLOB functions. The variant of LIKE + ** used will be case sensitive or not depending on the RHS. + */ + case PragTyp_CASE_SENSITIVE_LIKE: { + if( zRight ){ + sqlite3RegisterLikeFunctions(db, sqlite3GetBoolean(zRight, 0)); + } + } + break; +#endif /* SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA */ + +#ifndef SQLITE_INTEGRITY_CHECK_ERROR_MAX +# define SQLITE_INTEGRITY_CHECK_ERROR_MAX 100 +#endif + +#ifndef SQLITE_OMIT_INTEGRITY_CHECK + /* PRAGMA integrity_check + ** PRAGMA integrity_check(N) + ** PRAGMA quick_check + ** PRAGMA quick_check(N) + ** + ** Verify the integrity of the database. + ** + ** The "quick_check" is reduced version of + ** integrity_check designed to detect most database corruption + ** without the overhead of cross-checking indexes. Quick_check + ** is linear time wherease integrity_check is O(NlogN). + ** + ** The maximum nubmer of errors is 100 by default. A different default + ** can be specified using a numeric parameter N. + ** + ** Or, the parameter N can be the name of a table. In that case, only + ** the one table named is verified. The freelist is only verified if + ** the named table is "sqlite_schema" (or one of its aliases). + ** + ** All schemas are checked by default. To check just a single + ** schema, use the form: + ** + ** PRAGMA schema.integrity_check; + */ + case PragTyp_INTEGRITY_CHECK: { + int i, j, addr, mxErr; + Table *pObjTab = 0; /* Check only this one table, if not NULL */ + + int isQuick = (sqlite3Tolower(zLeft[0])=='q'); + + /* If the PRAGMA command was of the form "PRAGMA .integrity_check", + ** then iDb is set to the index of the database identified by . + ** In this case, the integrity of database iDb only is verified by + ** the VDBE created below. + ** + ** Otherwise, if the command was simply "PRAGMA integrity_check" (or + ** "PRAGMA quick_check"), then iDb is set to 0. In this case, set iDb + ** to -1 here, to indicate that the VDBE should verify the integrity + ** of all attached databases. */ + assert( iDb>=0 ); + assert( iDb==0 || pId2->z ); + if( pId2->z==0 ) iDb = -1; + + /* Initialize the VDBE program */ + pParse->nMem = 6; + + /* Set the maximum error count */ + mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; + if( zRight ){ + if( sqlite3GetInt32(zRight, &mxErr) ){ + if( mxErr<=0 ){ + mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; + } + }else{ + pObjTab = sqlite3LocateTable(pParse, 0, zRight, + iDb>=0 ? db->aDb[iDb].zDbSName : 0); + } + } + sqlite3VdbeAddOp2(v, OP_Integer, mxErr-1, 1); /* reg[1] holds errors left */ + + /* Do an integrity check on each database file */ + for(i=0; inDb; i++){ + HashElem *x; /* For looping over tables in the schema */ + Hash *pTbls; /* Set of all tables in the schema */ + int *aRoot; /* Array of root page numbers of all btrees */ + int cnt = 0; /* Number of entries in aRoot[] */ + int mxIdx = 0; /* Maximum number of indexes for any table */ + + if( OMIT_TEMPDB && i==1 ) continue; + if( iDb>=0 && i!=iDb ) continue; + + sqlite3CodeVerifySchema(pParse, i); + + /* Do an integrity check of the B-Tree + ** + ** Begin by finding the root pages numbers + ** for all tables and indices in the database. + */ + assert( sqlite3SchemaMutexHeld(db, i, 0) ); + pTbls = &db->aDb[i].pSchema->tblHash; + for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); /* Current table */ + Index *pIdx; /* An index on pTab */ + int nIdx; /* Number of indexes on pTab */ + if( pObjTab && pObjTab!=pTab ) continue; + if( HasRowid(pTab) ) cnt++; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } + if( nIdx>mxIdx ) mxIdx = nIdx; + } + if( cnt==0 ) continue; + if( pObjTab ) cnt++; + aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(cnt+1)); + if( aRoot==0 ) break; + cnt = 0; + if( pObjTab ) aRoot[++cnt] = 0; + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( pObjTab && pObjTab!=pTab ) continue; + if( HasRowid(pTab) ) aRoot[++cnt] = pTab->tnum; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + aRoot[++cnt] = pIdx->tnum; + } + } + aRoot[0] = cnt; + + /* Make sure sufficient number of registers have been allocated */ + pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); + sqlite3ClearTempRegCache(pParse); + + /* Do the b-tree integrity checks */ + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); + sqlite3VdbeChangeP5(v, (u8)i); + addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName), + P4_DYNAMIC); + sqlite3VdbeAddOp3(v, OP_Concat, 2, 3, 3); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, addr); + + /* Make sure all the indices are constructed correctly. + */ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + Index *pIdx, *pPk; + Index *pPrior = 0; + int loopTop; + int iDataCur, iIdxCur; + int r1 = -1; + int bStrict; + + if( pTab->tnum<1 ) continue; /* Skip VIEWs or VIRTUAL TABLEs */ + if( pObjTab && pObjTab!=pTab ) continue; + pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, + 1, 0, &iDataCur, &iIdxCur); + /* reg[7] counts the number of entries in the table. + ** reg[8+i] counts the number of entries in the i-th index + */ + sqlite3VdbeAddOp2(v, OP_Integer, 0, 7); + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ + } + assert( pParse->nMem>=8+j ); + assert( sqlite3NoTempsInRange(pParse,1,7+j) ); + sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); + loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); + if( !isQuick ){ + /* Sanity check on record header decoding */ + sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nNVCol-1,3); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + VdbeComment((v, "(right-most column)")); + } + /* Verify that all NOT NULL columns really are NOT NULL. At the + ** same time verify the type of the content of STRICT tables */ + bStrict = (pTab->tabFlags & TF_Strict)!=0; + for(j=0; jnCol; j++){ + char *zErr; + Column *pCol = pTab->aCol + j; + int doError, jmp2; + if( j==pTab->iPKey ) continue; + if( pCol->notNull==0 && !bStrict ) continue; + doError = bStrict ? sqlite3VdbeMakeLabel(pParse) : 0; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + if( sqlite3VdbeGetOp(v,-1)->opcode==OP_Column ){ + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + } + if( pCol->notNull ){ + jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, + pCol->zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + if( bStrict ){ + sqlite3VdbeGoto(v, doError); + }else{ + integrityCheckResultRow(v); + } + sqlite3VdbeJumpHere(v, jmp2); + } + if( (pTab->tabFlags & TF_Strict)!=0 + && pCol->eCType!=COLTYPE_ANY + ){ + jmp2 = sqlite3VdbeAddOp3(v, OP_IsNullOrType, 3, 0, + sqlite3StdTypeMap[pCol->eCType-1]); + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "non-%s value in %s.%s", + sqlite3StdType[pCol->eCType-1], + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + sqlite3VdbeResolveLabel(v, doError); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, jmp2); + } + } + /* Verify CHECK constraints */ + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + ExprList *pCheck = sqlite3ExprListDup(db, pTab->pCheck, 0); + if( db->mallocFailed==0 ){ + int addrCkFault = sqlite3VdbeMakeLabel(pParse); + int addrCkOk = sqlite3VdbeMakeLabel(pParse); + char *zErr; + int k; + pParse->iSelfTab = iDataCur + 1; + for(k=pCheck->nExpr-1; k>0; k--){ + sqlite3ExprIfFalse(pParse, pCheck->a[k].pExpr, addrCkFault, 0); + } + sqlite3ExprIfTrue(pParse, pCheck->a[0].pExpr, addrCkOk, + SQLITE_JUMPIFNULL); + sqlite3VdbeResolveLabel(v, addrCkFault); + pParse->iSelfTab = 0; + zErr = sqlite3MPrintf(db, "CHECK constraint failed in %s", + pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + integrityCheckResultRow(v); + sqlite3VdbeResolveLabel(v, addrCkOk); + } + sqlite3ExprListDelete(db, pCheck); + } + if( !isQuick ){ /* Omit the remaining tests for quick_check */ + /* Validate index entries for the current row */ + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + int jmp2, jmp3, jmp4, jmp5; + int ckUniq = sqlite3VdbeMakeLabel(pParse); + if( pPk==pIdx ) continue; + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, + pPrior, r1); + pPrior = pIdx; + sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1);/* increment entry count */ + /* Verify that an index entry exists for the current table row */ + jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, + pIdx->nColumn); VdbeCoverage(v); + sqlite3VdbeLoadString(v, 3, "row "); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeLoadString(v, 4, " missing from index "); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + jmp4 = integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, jmp2); + /* For UNIQUE indexes, verify that only one entry exists with the + ** current key. The entry is unique if (1) any column is NULL + ** or (2) the next entry has a different key */ + if( IsUniqueIndex(pIdx) ){ + int uniqOk = sqlite3VdbeMakeLabel(pParse); + int jmp6; + int kk; + for(kk=0; kknKeyCol; kk++){ + int iCol = pIdx->aiColumn[kk]; + assert( iCol!=XN_ROWID && iColnCol ); + if( iCol>=0 && pTab->aCol[iCol].notNull ) continue; + sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); + VdbeCoverage(v); + } + jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); + sqlite3VdbeGoto(v, uniqOk); + sqlite3VdbeJumpHere(v, jmp6); + sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, + pIdx->nKeyCol); VdbeCoverage(v); + sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); + sqlite3VdbeGoto(v, jmp5); + sqlite3VdbeResolveLabel(v, uniqOk); + } + sqlite3VdbeJumpHere(v, jmp4); + sqlite3ResolvePartIdxLabel(pParse, jmp3); + } + } + sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, loopTop-1); + if( !isQuick ){ + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + if( pPk==pIdx ) continue; + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+j, 0, 3); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, addr); + } + } + } + } + { + static const int iLn = VDBE_OFFSET_LINENO(2); + static const VdbeOpList endCode[] = { + { OP_AddImm, 1, 0, 0}, /* 0 */ + { OP_IfNotZero, 1, 4, 0}, /* 1 */ + { OP_String8, 0, 3, 0}, /* 2 */ + { OP_ResultRow, 3, 1, 0}, /* 3 */ + { OP_Halt, 0, 0, 0}, /* 4 */ + { OP_String8, 0, 3, 0}, /* 5 */ + { OP_Goto, 0, 3, 0}, /* 6 */ + }; + VdbeOp *aOp; + + aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); + if( aOp ){ + aOp[0].p2 = 1-mxErr; + aOp[2].p4type = P4_STATIC; + aOp[2].p4.z = "ok"; + aOp[5].p4type = P4_STATIC; + aOp[5].p4.z = (char*)sqlite3ErrStr(SQLITE_CORRUPT); + } + sqlite3VdbeChangeP3(v, 0, sqlite3VdbeCurrentAddr(v)-2); + } + } + break; +#endif /* SQLITE_OMIT_INTEGRITY_CHECK */ + +#ifndef SQLITE_OMIT_UTF16 + /* + ** PRAGMA encoding + ** PRAGMA encoding = "utf-8"|"utf-16"|"utf-16le"|"utf-16be" + ** + ** In its first form, this pragma returns the encoding of the main + ** database. If the database is not initialized, it is initialized now. + ** + ** The second form of this pragma is a no-op if the main database file + ** has not already been initialized. In this case it sets the default + ** encoding that will be used for the main database file if a new file + ** is created. If an existing main database file is opened, then the + ** default text encoding for the existing database is used. + ** + ** In all cases new databases created using the ATTACH command are + ** created to use the same default text encoding as the main database. If + ** the main database has not been initialized and/or created when ATTACH + ** is executed, this is done before the ATTACH operation. + ** + ** In the second form this pragma sets the text encoding to be used in + ** new database files created using this database handle. It is only + ** useful if invoked immediately after the main database i + */ + case PragTyp_ENCODING: { + static const struct EncName { + char *zName; + u8 enc; + } encnames[] = { + { "UTF8", SQLITE_UTF8 }, + { "UTF-8", SQLITE_UTF8 }, /* Must be element [1] */ + { "UTF-16le", SQLITE_UTF16LE }, /* Must be element [2] */ + { "UTF-16be", SQLITE_UTF16BE }, /* Must be element [3] */ + { "UTF16le", SQLITE_UTF16LE }, + { "UTF16be", SQLITE_UTF16BE }, + { "UTF-16", 0 }, /* SQLITE_UTF16NATIVE */ + { "UTF16", 0 }, /* SQLITE_UTF16NATIVE */ + { 0, 0 } + }; + const struct EncName *pEnc; + if( !zRight ){ /* "PRAGMA encoding" */ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; + assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); + assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); + assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); + returnSingleText(v, encnames[ENC(pParse->db)].zName); + }else{ /* "PRAGMA encoding = XXX" */ + /* Only change the value of sqlite.enc if the database handle is not + ** initialized. If the main database exists, the new sqlite.enc value + ** will be overwritten when the schema is next loaded. If it does not + ** already exists, it will be created to use the new encoding value. + */ + if( (db->mDbFlags & DBFLAG_EncodingFixed)==0 ){ + for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ + if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){ + u8 enc = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + SCHEMA_ENC(db) = enc; + sqlite3SetTextEncoding(db, enc); + break; + } + } + if( !pEnc->zName ){ + sqlite3ErrorMsg(pParse, "unsupported encoding: %s", zRight); + } + } + } + } + break; +#endif /* SQLITE_OMIT_UTF16 */ + +#ifndef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS + /* + ** PRAGMA [schema.]schema_version + ** PRAGMA [schema.]schema_version = + ** + ** PRAGMA [schema.]user_version + ** PRAGMA [schema.]user_version = + ** + ** PRAGMA [schema.]freelist_count + ** + ** PRAGMA [schema.]data_version + ** + ** PRAGMA [schema.]application_id + ** PRAGMA [schema.]application_id = + ** + ** The pragma's schema_version and user_version are used to set or get + ** the value of the schema-version and user-version, respectively. Both + ** the schema-version and the user-version are 32-bit signed integers + ** stored in the database header. + ** + ** The schema-cookie is usually only manipulated internally by SQLite. It + ** is incremented by SQLite whenever the database schema is modified (by + ** creating or dropping a table or index). The schema version is used by + ** SQLite each time a query is executed to ensure that the internal cache + ** of the schema used when compiling the SQL query matches the schema of + ** the database against which the compiled query is actually executed. + ** Subverting this mechanism by using "PRAGMA schema_version" to modify + ** the schema-version is potentially dangerous and may lead to program + ** crashes or database corruption. Use with caution! + ** + ** The user-version is not used internally by SQLite. It may be used by + ** applications for any purpose. + */ + case PragTyp_HEADER_VALUE: { + int iCookie = pPragma->iArg; /* Which cookie to read or write */ + sqlite3VdbeUsesBtree(v, iDb); + if( zRight && (pPragma->mPragFlg & PragFlg_ReadOnly)==0 ){ + /* Write the specified cookie value */ + static const VdbeOpList setCookie[] = { + { OP_Transaction, 0, 1, 0}, /* 0 */ + { OP_SetCookie, 0, 0, 0}, /* 1 */ + }; + VdbeOp *aOp; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(setCookie)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(setCookie), setCookie, 0); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[1].p2 = iCookie; + aOp[1].p3 = sqlite3Atoi(zRight); + aOp[1].p5 = 1; + }else{ + /* Read the specified cookie value */ + static const VdbeOpList readCookie[] = { + { OP_Transaction, 0, 0, 0}, /* 0 */ + { OP_ReadCookie, 0, 1, 0}, /* 1 */ + { OP_ResultRow, 1, 1, 0} + }; + VdbeOp *aOp; + sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(readCookie)); + aOp = sqlite3VdbeAddOpList(v, ArraySize(readCookie),readCookie,0); + if( ONLY_IF_REALLOC_STRESS(aOp==0) ) break; + aOp[0].p1 = iDb; + aOp[1].p1 = iDb; + aOp[1].p3 = iCookie; + sqlite3VdbeReusable(v); + } + } + break; +#endif /* SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS */ + +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS + /* + ** PRAGMA compile_options + ** + ** Return the names of all compile-time options used in this build, + ** one option per row. + */ + case PragTyp_COMPILE_OPTIONS: { + int i = 0; + const char *zOpt; + pParse->nMem = 1; + while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ + sqlite3VdbeLoadString(v, 1, zOpt); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); + } + sqlite3VdbeReusable(v); + } + break; +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +#ifndef SQLITE_OMIT_WAL + /* + ** PRAGMA [schema.]wal_checkpoint = passive|full|restart|truncate + ** + ** Checkpoint the database. + */ + case PragTyp_WAL_CHECKPOINT: { + int iBt = (pId2->z?iDb:SQLITE_MAX_DB); + int eMode = SQLITE_CHECKPOINT_PASSIVE; + if( zRight ){ + if( sqlite3StrICmp(zRight, "full")==0 ){ + eMode = SQLITE_CHECKPOINT_FULL; + }else if( sqlite3StrICmp(zRight, "restart")==0 ){ + eMode = SQLITE_CHECKPOINT_RESTART; + }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ + eMode = SQLITE_CHECKPOINT_TRUNCATE; + } + } + pParse->nMem = 3; + sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); + } + break; + + /* + ** PRAGMA wal_autocheckpoint + ** PRAGMA wal_autocheckpoint = N + ** + ** Configure a database connection to automatically checkpoint a database + ** after accumulating N frames in the log. Or query for the current value + ** of N. + */ + case PragTyp_WAL_AUTOCHECKPOINT: { + if( zRight ){ + sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); + } + returnSingleInt(v, + db->xWalCallback==sqlite3WalDefaultHook ? + SQLITE_PTR_TO_INT(db->pWalArg) : 0); + } + break; +#endif + + /* + ** PRAGMA shrink_memory + ** + ** IMPLEMENTATION-OF: R-23445-46109 This pragma causes the database + ** connection on which it is invoked to free up as much memory as it + ** can, by calling sqlite3_db_release_memory(). + */ + case PragTyp_SHRINK_MEMORY: { + sqlite3_db_release_memory(db); + break; + } + + /* + ** PRAGMA optimize + ** PRAGMA optimize(MASK) + ** PRAGMA schema.optimize + ** PRAGMA schema.optimize(MASK) + ** + ** Attempt to optimize the database. All schemas are optimized in the first + ** two forms, and only the specified schema is optimized in the latter two. + ** + ** The details of optimizations performed by this pragma are expected + ** to change and improve over time. Applications should anticipate that + ** this pragma will perform new optimizations in future releases. + ** + ** The optional argument is a bitmask of optimizations to perform: + ** + ** 0x0001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. + ** + ** 0x0002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. + ** + ** 0x0004 (Not yet implemented) Record usage and performance + ** information from the current session in the + ** database file so that it will be available to "optimize" + ** pragmas run by future database connections. + ** + ** 0x0008 (Not yet implemented) Create indexes that might have + ** been helpful to recent queries + ** + ** The default MASK is and always shall be 0xfffe. 0xfffe means perform all + ** of the optimizations listed above except Debug Mode, including new + ** optimizations that have not yet been invented. If new optimizations are + ** ever added that should be off by default, those off-by-default + ** optimizations will have bitmasks of 0x10000 or larger. + ** + ** DETERMINATION OF WHEN TO RUN ANALYZE + ** + ** In the current implementation, a table is analyzed if only if all of + ** the following are true: + ** + ** (1) MASK bit 0x02 is set. + ** + ** (2) The query planner used sqlite_stat1-style statistics for one or + ** more indexes of the table at some point during the lifetime of + ** the current connection. + ** + ** (3) One or more indexes of the table are currently unanalyzed OR + ** the number of rows in the table has increased by 25 times or more + ** since the last time ANALYZE was run. + ** + ** The rules for when tables are analyzed are likely to change in + ** future releases. + */ + case PragTyp_OPTIMIZE: { + int iDbLast; /* Loop termination point for the schema loop */ + int iTabCur; /* Cursor for a table whose size needs checking */ + HashElem *k; /* Loop over tables of a schema */ + Schema *pSchema; /* The current schema */ + Table *pTab; /* A table in the schema */ + Index *pIdx; /* An index of the table */ + LogEst szThreshold; /* Size threshold above which reanalysis is needd */ + char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ + u32 opMask; /* Mask of operations to perform */ + + if( zRight ){ + opMask = (u32)sqlite3Atoi(zRight); + if( (opMask & 0x02)==0 ) break; + }else{ + opMask = 0xfffe; + } + iTabCur = pParse->nTab++; + for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ + if( iDb==1 ) continue; + sqlite3CodeVerifySchema(pParse, iDb); + pSchema = db->aDb[iDb].pSchema; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it. + ** This also has the effect of skipping virtual tables and views */ + if( (pTab->tabFlags & TF_StatsUsed)==0 ) continue; + + /* Reanalyze if the table is 25 times larger than the last analysis */ + szThreshold = pTab->nRowLogEst + 46; assert( sqlite3LogEst(25)==46 ); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !pIdx->hasStat1 ){ + szThreshold = 0; /* Always analyze if any index lacks statistics */ + break; + } + } + if( szThreshold ){ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + sqlite3VdbeAddOp3(v, OP_IfSmaller, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), szThreshold); + VdbeCoverage(v); + } + zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", + db->aDb[iDb].zDbSName, pTab->zName); + if( opMask & 0x01 ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); + }else{ + sqlite3VdbeAddOp4(v, OP_SqlExec, 0, 0, 0, zSubSql, P4_DYNAMIC); + } + } + } + sqlite3VdbeAddOp0(v, OP_Expire); + break; + } + + /* + ** PRAGMA busy_timeout + ** PRAGMA busy_timeout = N + ** + ** Call sqlite3_busy_timeout(db, N). Return the current timeout value + ** if one is set. If no busy handler or a different busy handler is set + ** then 0 is returned. Setting the busy_timeout to 0 or negative + ** disables the timeout. + */ + /*case PragTyp_BUSY_TIMEOUT*/ default: { + assert( pPragma->ePragTyp==PragTyp_BUSY_TIMEOUT ); + if( zRight ){ + sqlite3_busy_timeout(db, sqlite3Atoi(zRight)); + } + returnSingleInt(v, db->busyTimeout); + break; + } + + /* + ** PRAGMA soft_heap_limit + ** PRAGMA soft_heap_limit = N + ** + ** IMPLEMENTATION-OF: R-26343-45930 This pragma invokes the + ** sqlite3_soft_heap_limit64() interface with the argument N, if N is + ** specified and is a non-negative integer. + ** IMPLEMENTATION-OF: R-64451-07163 The soft_heap_limit pragma always + ** returns the same integer that would be returned by the + ** sqlite3_soft_heap_limit64(-1) C-language function. + */ + case PragTyp_SOFT_HEAP_LIMIT: { + sqlite3_int64 N; + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ + sqlite3_soft_heap_limit64(N); + } + returnSingleInt(v, sqlite3_soft_heap_limit64(-1)); + break; + } + + /* + ** PRAGMA hard_heap_limit + ** PRAGMA hard_heap_limit = N + ** + ** Invoke sqlite3_hard_heap_limit64() to query or set the hard heap + ** limit. The hard heap limit can be activated or lowered by this + ** pragma, but not raised or deactivated. Only the + ** sqlite3_hard_heap_limit64() C-language API can raise or deactivate + ** the hard heap limit. This allows an application to set a heap limit + ** constraint that cannot be relaxed by an untrusted SQL script. + */ + case PragTyp_HARD_HEAP_LIMIT: { + sqlite3_int64 N; + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ + sqlite3_int64 iPrior = sqlite3_hard_heap_limit64(-1); + if( N>0 && (iPrior==0 || iPrior>N) ) sqlite3_hard_heap_limit64(N); + } + returnSingleInt(v, sqlite3_hard_heap_limit64(-1)); + break; + } + + /* + ** PRAGMA threads + ** PRAGMA threads = N + ** + ** Configure the maximum number of worker threads. Return the new + ** maximum, which might be less than requested. + */ + case PragTyp_THREADS: { + sqlite3_int64 N; + if( zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK + && N>=0 + ){ + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); + } + returnSingleInt(v, sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); + break; + } + + /* + ** PRAGMA analysis_limit + ** PRAGMA analysis_limit = N + ** + ** Configure the maximum number of rows that ANALYZE will examine + ** in each index that it looks at. Return the new limit. + */ + case PragTyp_ANALYSIS_LIMIT: { + sqlite3_int64 N; + if( zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK + && N>=0 + ){ + db->nAnalysisLimit = (int)(N&0x7fffffff); + } + returnSingleInt(v, db->nAnalysisLimit); + break; + } + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* + ** Report the current state of file logs for all databases + */ + case PragTyp_LOCK_STATUS: { + static const char *const azLockName[] = { + "unlocked", "shared", "reserved", "pending", "exclusive" + }; + int i; + pParse->nMem = 2; + for(i=0; inDb; i++){ + Btree *pBt; + const char *zState = "unknown"; + int j; + if( db->aDb[i].zDbSName==0 ) continue; + pBt = db->aDb[i].pBt; + if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ + zState = "closed"; + }else if( sqlite3_file_control(db, i ? db->aDb[i].zDbSName : 0, + SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ + zState = azLockName[j]; + } + sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zDbSName, zState); + } + break; + } +#endif + +#if defined(SQLITE_ENABLE_CEROD) + case PragTyp_ACTIVATE_EXTENSIONS: if( zRight ){ + if( sqlite3StrNICmp(zRight, "cerod-", 6)==0 ){ + sqlite3_activate_cerod(&zRight[6]); + } + } + break; +#endif + + } /* End of the PRAGMA switch */ + + /* The following block is a no-op unless SQLITE_DEBUG is defined. Its only + ** purpose is to execute assert() statements to verify that if the + ** PragFlg_NoColumns1 flag is set and the caller specified an argument + ** to the PRAGMA, the implementation has not added any OP_ResultRow + ** instructions to the VM. */ + if( (pPragma->mPragFlg & PragFlg_NoColumns1) && zRight ){ + sqlite3VdbeVerifyNoResultRow(v); + } + +pragma_out: + sqlite3DbFree(db, zLeft); + sqlite3DbFree(db, zRight); +} +#ifndef SQLITE_OMIT_VIRTUALTABLE +/***************************************************************************** +** Implementation of an eponymous virtual table that runs a pragma. +** +*/ +typedef struct PragmaVtab PragmaVtab; +typedef struct PragmaVtabCursor PragmaVtabCursor; +struct PragmaVtab { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database connection to which it belongs */ + const PragmaName *pName; /* Name of the pragma */ + u8 nHidden; /* Number of hidden columns */ + u8 iHidden; /* Index of the first hidden column */ +}; +struct PragmaVtabCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pPragma; /* The pragma statement to run */ + sqlite_int64 iRowid; /* Current rowid */ + char *azArg[2]; /* Value of the argument and schema */ +}; + +/* +** Pragma virtual table module xConnect method. +*/ +static int pragmaVtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + const PragmaName *pPragma = (const PragmaName*)pAux; + PragmaVtab *pTab = 0; + int rc; + int i, j; + char cSep = '('; + StrAccum acc; + char zBuf[200]; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3_str_appendall(&acc, "CREATE TABLE x"); + for(i=0, j=pPragma->iPragCName; inPragCName; i++, j++){ + sqlite3_str_appendf(&acc, "%c\"%s\"", cSep, pragCName[j]); + cSep = ','; + } + if( i==0 ){ + sqlite3_str_appendf(&acc, "(\"%s\"", pPragma->zName); + i++; + } + j = 0; + if( pPragma->mPragFlg & PragFlg_Result1 ){ + sqlite3_str_appendall(&acc, ",arg HIDDEN"); + j++; + } + if( pPragma->mPragFlg & (PragFlg_SchemaOpt|PragFlg_SchemaReq) ){ + sqlite3_str_appendall(&acc, ",schema HIDDEN"); + j++; + } + sqlite3_str_append(&acc, ")", 1); + sqlite3StrAccumFinish(&acc); + assert( strlen(zBuf) < sizeof(zBuf)-1 ); + rc = sqlite3_declare_vtab(db, zBuf); + if( rc==SQLITE_OK ){ + pTab = (PragmaVtab*)sqlite3_malloc(sizeof(PragmaVtab)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pTab, 0, sizeof(PragmaVtab)); + pTab->pName = pPragma; + pTab->db = db; + pTab->iHidden = i; + pTab->nHidden = j; + } + }else{ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Pragma virtual table module xDisconnect method. +*/ +static int pragmaVtabDisconnect(sqlite3_vtab *pVtab){ + PragmaVtab *pTab = (PragmaVtab*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* Figure out the best index to use to search a pragma virtual table. +** +** There are not really any index choices. But we want to encourage the +** query planner to give == constraints on as many hidden parameters as +** possible, and especially on the first hidden parameter. So return a +** high cost if hidden parameters are unconstrained. +*/ +static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + PragmaVtab *pTab = (PragmaVtab*)tab; + const struct sqlite3_index_constraint *pConstraint; + int i, j; + int seen[2]; + + pIdxInfo->estimatedCost = (double)1; + if( pTab->nHidden==0 ){ return SQLITE_OK; } + pConstraint = pIdxInfo->aConstraint; + seen[0] = 0; + seen[1] = 0; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->iColumn < pTab->iHidden ) continue; + j = pConstraint->iColumn - pTab->iHidden; + assert( j < 2 ); + seen[j] = i+1; + } + if( seen[0]==0 ){ + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + return SQLITE_OK; + } + j = seen[0]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 1; + pIdxInfo->aConstraintUsage[j].omit = 1; + if( seen[1]==0 ) return SQLITE_OK; + pIdxInfo->estimatedCost = (double)20; + pIdxInfo->estimatedRows = 20; + j = seen[1]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 2; + pIdxInfo->aConstraintUsage[j].omit = 1; + return SQLITE_OK; +} + +/* Create a new cursor for the pragma virtual table */ +static int pragmaVtabOpen(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCursor){ + PragmaVtabCursor *pCsr; + pCsr = (PragmaVtabCursor*)sqlite3_malloc(sizeof(*pCsr)); + if( pCsr==0 ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(PragmaVtabCursor)); + pCsr->base.pVtab = pVtab; + *ppCursor = &pCsr->base; + return SQLITE_OK; +} + +/* Clear all content from pragma virtual table cursor. */ +static void pragmaVtabCursorClear(PragmaVtabCursor *pCsr){ + int i; + sqlite3_finalize(pCsr->pPragma); + pCsr->pPragma = 0; + for(i=0; iazArg); i++){ + sqlite3_free(pCsr->azArg[i]); + pCsr->azArg[i] = 0; + } +} + +/* Close a pragma virtual table cursor */ +static int pragmaVtabClose(sqlite3_vtab_cursor *cur){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)cur; + pragmaVtabCursorClear(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* Advance the pragma virtual table cursor to the next row */ +static int pragmaVtabNext(sqlite3_vtab_cursor *pVtabCursor){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + int rc = SQLITE_OK; + + /* Increment the xRowid value */ + pCsr->iRowid++; + assert( pCsr->pPragma ); + if( SQLITE_ROW!=sqlite3_step(pCsr->pPragma) ){ + rc = sqlite3_finalize(pCsr->pPragma); + pCsr->pPragma = 0; + pragmaVtabCursorClear(pCsr); + } + return rc; +} + +/* +** Pragma virtual table module xFilter method. +*/ +static int pragmaVtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + PragmaVtab *pTab = (PragmaVtab*)(pVtabCursor->pVtab); + int rc; + int i, j; + StrAccum acc; + char *zSql; + + UNUSED_PARAMETER(idxNum); + UNUSED_PARAMETER(idxStr); + pragmaVtabCursorClear(pCsr); + j = (pTab->pName->mPragFlg & PragFlg_Result1)!=0 ? 0 : 1; + for(i=0; iazArg) ); + assert( pCsr->azArg[j]==0 ); + if( zText ){ + pCsr->azArg[j] = sqlite3_mprintf("%s", zText); + if( pCsr->azArg[j]==0 ){ + return SQLITE_NOMEM; + } + } + } + sqlite3StrAccumInit(&acc, 0, 0, 0, pTab->db->aLimit[SQLITE_LIMIT_SQL_LENGTH]); + sqlite3_str_appendall(&acc, "PRAGMA "); + if( pCsr->azArg[1] ){ + sqlite3_str_appendf(&acc, "%Q.", pCsr->azArg[1]); + } + sqlite3_str_appendall(&acc, pTab->pName->zName); + if( pCsr->azArg[0] ){ + sqlite3_str_appendf(&acc, "=%Q", pCsr->azArg[0]); + } + zSql = sqlite3StrAccumFinish(&acc); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pPragma, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK ){ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); + return rc; + } + return pragmaVtabNext(pVtabCursor); +} + +/* +** Pragma virtual table module xEof method. +*/ +static int pragmaVtabEof(sqlite3_vtab_cursor *pVtabCursor){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + return (pCsr->pPragma==0); +} + +/* The xColumn method simply returns the corresponding column from +** the PRAGMA. +*/ +static int pragmaVtabColumn( + sqlite3_vtab_cursor *pVtabCursor, + sqlite3_context *ctx, + int i +){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + PragmaVtab *pTab = (PragmaVtab*)(pVtabCursor->pVtab); + if( iiHidden ){ + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pPragma, i)); + }else{ + sqlite3_result_text(ctx, pCsr->azArg[i-pTab->iHidden],-1,SQLITE_TRANSIENT); + } + return SQLITE_OK; +} + +/* +** Pragma virtual table module xRowid method. +*/ +static int pragmaVtabRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *p){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + *p = pCsr->iRowid; + return SQLITE_OK; +} + +/* The pragma virtual table object */ +static const sqlite3_module pragmaVtabModule = { + 0, /* iVersion */ + 0, /* xCreate - create a table */ + pragmaVtabConnect, /* xConnect - connect to an existing table */ + pragmaVtabBestIndex, /* xBestIndex - Determine search strategy */ + pragmaVtabDisconnect, /* xDisconnect - Disconnect from a table */ + 0, /* xDestroy - Drop a table */ + pragmaVtabOpen, /* xOpen - open a cursor */ + pragmaVtabClose, /* xClose - close a cursor */ + pragmaVtabFilter, /* xFilter - configure scan constraints */ + pragmaVtabNext, /* xNext - advance a cursor */ + pragmaVtabEof, /* xEof */ + pragmaVtabColumn, /* xColumn - read data */ + pragmaVtabRowid, /* xRowid - read data */ + 0, /* xUpdate - write data */ + 0, /* xBegin - begin transaction */ + 0, /* xSync - sync transaction */ + 0, /* xCommit - commit transaction */ + 0, /* xRollback - rollback transaction */ + 0, /* xFindFunction - function overloading */ + 0, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ +}; + +/* +** Check to see if zTabName is really the name of a pragma. If it is, +** then register an eponymous virtual table for that pragma and return +** a pointer to the Module object for the new virtual table. +*/ +SQLITE_PRIVATE Module *sqlite3PragmaVtabRegister(sqlite3 *db, const char *zName){ + const PragmaName *pName; + assert( sqlite3_strnicmp(zName, "pragma_", 7)==0 ); + pName = pragmaLocate(zName+7); + if( pName==0 ) return 0; + if( (pName->mPragFlg & (PragFlg_Result0|PragFlg_Result1))==0 ) return 0; + assert( sqlite3HashFind(&db->aModule, zName)==0 ); + return sqlite3VtabCreateModule(db, zName, &pragmaVtabModule, (void*)pName, 0); +} + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#endif /* SQLITE_OMIT_PRAGMA */ + +/************** End of pragma.c **********************************************/ +/************** Begin file prepare.c *****************************************/ +/* +** 2005 May 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the implementation of the sqlite3_prepare() +** interface, and routines that contribute to loading the database schema +** from disk. +*/ +/* #include "sqliteInt.h" */ + +/* +** Fill the InitData structure with an error message that indicates +** that the database is corrupt. +*/ +static void corruptSchema( + InitData *pData, /* Initialization context */ + char **azObj, /* Type and name of object being parsed */ + const char *zExtra /* Error information */ +){ + sqlite3 *db = pData->db; + if( db->mallocFailed ){ + pData->rc = SQLITE_NOMEM_BKPT; + }else if( pData->pzErrMsg[0]!=0 ){ + /* A error message has already been generated. Do not overwrite it */ + }else if( pData->mInitFlags & (INITFLAG_AlterRename|INITFLAG_AlterDrop) ){ + *pData->pzErrMsg = sqlite3MPrintf(db, + "error in %s %s after %s: %s", azObj[0], azObj[1], + (pData->mInitFlags & INITFLAG_AlterRename) ? "rename" : "drop column", + zExtra + ); + pData->rc = SQLITE_ERROR; + }else if( db->flags & SQLITE_WriteSchema ){ + pData->rc = SQLITE_CORRUPT_BKPT; + }else{ + char *z; + const char *zObj = azObj[1] ? azObj[1] : "?"; + z = sqlite3MPrintf(db, "malformed database schema (%s)", zObj); + if( zExtra && zExtra[0] ) z = sqlite3MPrintf(db, "%z - %s", z, zExtra); + *pData->pzErrMsg = z; + pData->rc = SQLITE_CORRUPT_BKPT; + } +} + +/* +** Check to see if any sibling index (another index on the same table) +** of pIndex has the same root page number, and if it does, return true. +** This would indicate a corrupt schema. +*/ +SQLITE_PRIVATE int sqlite3IndexHasDuplicateRootPage(Index *pIndex){ + Index *p; + for(p=pIndex->pTable->pIndex; p; p=p->pNext){ + if( p->tnum==pIndex->tnum && p!=pIndex ) return 1; + } + return 0; +} + +/* forward declaration */ +static int sqlite3Prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + Vdbe *pReprepare, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +); + + +/* +** This is the callback routine for the code that initializes the +** database. See sqlite3Init() below for additional information. +** This routine is also called from the OP_ParseSchema opcode of the VDBE. +** +** Each callback contains the following information: +** +** argv[0] = type of object: "table", "index", "trigger", or "view". +** argv[1] = name of thing being created +** argv[2] = associated table if an index or trigger +** argv[3] = root page number for table or index. 0 for trigger or view. +** argv[4] = SQL text for the CREATE statement. +** +*/ +SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ + InitData *pData = (InitData*)pInit; + sqlite3 *db = pData->db; + int iDb = pData->iDb; + + assert( argc==5 ); + UNUSED_PARAMETER2(NotUsed, argc); + assert( sqlite3_mutex_held(db->mutex) ); + db->mDbFlags |= DBFLAG_EncodingFixed; + if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ + pData->nInitRow++; + if( db->mallocFailed ){ + corruptSchema(pData, argv, 0); + return 1; + } + + assert( iDb>=0 && iDbnDb ); + if( argv[3]==0 ){ + corruptSchema(pData, argv, 0); + }else if( argv[4] + && 'c'==sqlite3UpperToLower[(unsigned char)argv[4][0]] + && 'r'==sqlite3UpperToLower[(unsigned char)argv[4][1]] ){ + /* Call the parser to process a CREATE TABLE, INDEX or VIEW. + ** But because db->init.busy is set to 1, no VDBE code is generated + ** or executed. All the parser does is build the internal data + ** structures that describe the table, index, or view. + ** + ** No other valid SQL statement, other than the variable CREATE statements, + ** can begin with the letters "C" and "R". Thus, it is not possible run + ** any other kind of statement while parsing the schema, even a corrupt + ** schema. + */ + int rc; + u8 saved_iDb = db->init.iDb; + sqlite3_stmt *pStmt; + TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ + + assert( db->init.busy ); + db->init.iDb = iDb; + if( sqlite3GetUInt32(argv[3], &db->init.newTnum)==0 + || (db->init.newTnum>pData->mxPage && pData->mxPage>0) + ){ + if( sqlite3Config.bExtraSchemaChecks ){ + corruptSchema(pData, argv, "invalid rootpage"); + } + } + db->init.orphanTrigger = 0; + db->init.azInit = (const char**)argv; + pStmt = 0; + TESTONLY(rcp = ) sqlite3Prepare(db, argv[4], -1, 0, 0, &pStmt, 0); + rc = db->errCode; + assert( (rc&0xFF)==(rcp&0xFF) ); + db->init.iDb = saved_iDb; + /* assert( saved_iDb==0 || (db->mDbFlags & DBFLAG_Vacuum)!=0 ); */ + if( SQLITE_OK!=rc ){ + if( db->init.orphanTrigger ){ + assert( iDb==1 ); + }else{ + if( rc > pData->rc ) pData->rc = rc; + if( rc==SQLITE_NOMEM ){ + sqlite3OomFault(db); + }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ + corruptSchema(pData, argv, sqlite3_errmsg(db)); + } + } + } + db->init.azInit = sqlite3StdType; /* Any array of string ptrs will do */ + sqlite3_finalize(pStmt); + }else if( argv[1]==0 || (argv[4]!=0 && argv[4][0]!=0) ){ + corruptSchema(pData, argv, 0); + }else{ + /* If the SQL column is blank it means this is an index that + ** was created to be the PRIMARY KEY or to fulfill a UNIQUE + ** constraint for a CREATE TABLE. The index should have already + ** been created when we processed the CREATE TABLE. All we have + ** to do here is record the root page number for that index. + */ + Index *pIndex; + pIndex = sqlite3FindIndex(db, argv[1], db->aDb[iDb].zDbSName); + if( pIndex==0 ){ + corruptSchema(pData, argv, "orphan index"); + }else + if( sqlite3GetUInt32(argv[3],&pIndex->tnum)==0 + || pIndex->tnum<2 + || pIndex->tnum>pData->mxPage + || sqlite3IndexHasDuplicateRootPage(pIndex) + ){ + if( sqlite3Config.bExtraSchemaChecks ){ + corruptSchema(pData, argv, "invalid rootpage"); + } + } + } + return 0; +} + +/* +** Attempt to read the database schema and initialize internal +** data structures for a single database file. The index of the +** database file is given by iDb. iDb==0 is used for the main +** database. iDb==1 should never be used. iDb>=2 is used for +** auxiliary databases. Return one of the SQLITE_ error codes to +** indicate success or failure. +*/ +SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFlags){ + int rc; + int i; +#ifndef SQLITE_OMIT_DEPRECATED + int size; +#endif + Db *pDb; + char const *azArg[6]; + int meta[5]; + InitData initData; + const char *zSchemaTabName; + int openedTransaction = 0; + int mask = ((db->mDbFlags & DBFLAG_EncodingFixed) | ~DBFLAG_EncodingFixed); + + assert( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 ); + assert( iDb>=0 && iDbnDb ); + assert( db->aDb[iDb].pSchema ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); + + db->init.busy = 1; + + /* Construct the in-memory representation schema tables (sqlite_schema or + ** sqlite_temp_schema) by invoking the parser directly. The appropriate + ** table name will be inserted automatically by the parser so we can just + ** use the abbreviation "x" here. The parser will also automatically tag + ** the schema table as read-only. */ + azArg[0] = "table"; + azArg[1] = zSchemaTabName = SCHEMA_TABLE(iDb); + azArg[2] = azArg[1]; + azArg[3] = "1"; + azArg[4] = "CREATE TABLE x(type text,name text,tbl_name text," + "rootpage int,sql text)"; + azArg[5] = 0; + initData.db = db; + initData.iDb = iDb; + initData.rc = SQLITE_OK; + initData.pzErrMsg = pzErrMsg; + initData.mInitFlags = mFlags; + initData.nInitRow = 0; + initData.mxPage = 0; + sqlite3InitCallback(&initData, 5, (char **)azArg, 0); + db->mDbFlags &= mask; + if( initData.rc ){ + rc = initData.rc; + goto error_out; + } + + /* Create a cursor to hold the database open + */ + pDb = &db->aDb[iDb]; + if( pDb->pBt==0 ){ + assert( iDb==1 ); + DbSetProperty(db, 1, DB_SchemaLoaded); + rc = SQLITE_OK; + goto error_out; + } + + /* If there is not already a read-only (or read-write) transaction opened + ** on the b-tree database, open one now. If a transaction is opened, it + ** will be closed before this function returns. */ + sqlite3BtreeEnter(pDb->pBt); + if( sqlite3BtreeTxnState(pDb->pBt)==SQLITE_TXN_NONE ){ + rc = sqlite3BtreeBeginTrans(pDb->pBt, 0, 0); + if( rc!=SQLITE_OK ){ + sqlite3SetString(pzErrMsg, db, sqlite3ErrStr(rc)); + goto initone_error_out; + } + openedTransaction = 1; + } + + /* Get the database meta information. + ** + ** Meta values are as follows: + ** meta[0] Schema cookie. Changes with each schema change. + ** meta[1] File format of schema layer. + ** meta[2] Size of the page cache. + ** meta[3] Largest rootpage (auto/incr_vacuum mode) + ** meta[4] Db text encoding. 1:UTF-8 2:UTF-16LE 3:UTF-16BE + ** meta[5] User version + ** meta[6] Incremental vacuum mode + ** meta[7] unused + ** meta[8] unused + ** meta[9] unused + ** + ** Note: The #defined SQLITE_UTF* symbols in sqliteInt.h correspond to + ** the possible values of meta[4]. + */ + for(i=0; ipBt, i+1, (u32 *)&meta[i]); + } + if( (db->flags & SQLITE_ResetDatabase)!=0 ){ + memset(meta, 0, sizeof(meta)); + } + pDb->pSchema->schema_cookie = meta[BTREE_SCHEMA_VERSION-1]; + + /* If opening a non-empty database, check the text encoding. For the + ** main database, set sqlite3.enc to the encoding of the main database. + ** For an attached db, it is an error if the encoding is not the same + ** as sqlite3.enc. + */ + if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ + if( iDb==0 && (db->mDbFlags & DBFLAG_EncodingFixed)==0 ){ + u8 encoding; +#ifndef SQLITE_OMIT_UTF16 + /* If opening the main database, set ENC(db). */ + encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; + if( encoding==0 ) encoding = SQLITE_UTF8; +#else + encoding = SQLITE_UTF8; +#endif + sqlite3SetTextEncoding(db, encoding); + }else{ + /* If opening an attached database, the encoding much match ENC(db) */ + if( (meta[BTREE_TEXT_ENCODING-1] & 3)!=ENC(db) ){ + sqlite3SetString(pzErrMsg, db, "attached databases must use the same" + " text encoding as main database"); + rc = SQLITE_ERROR; + goto initone_error_out; + } + } + } + pDb->pSchema->enc = ENC(db); + + if( pDb->pSchema->cache_size==0 ){ +#ifndef SQLITE_OMIT_DEPRECATED + size = sqlite3AbsInt32(meta[BTREE_DEFAULT_CACHE_SIZE-1]); + if( size==0 ){ size = SQLITE_DEFAULT_CACHE_SIZE; } + pDb->pSchema->cache_size = size; +#else + pDb->pSchema->cache_size = SQLITE_DEFAULT_CACHE_SIZE; +#endif + sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size); + } + + /* + ** file_format==1 Version 3.0.0. + ** file_format==2 Version 3.1.3. // ALTER TABLE ADD COLUMN + ** file_format==3 Version 3.1.4. // ditto but with non-NULL defaults + ** file_format==4 Version 3.3.0. // DESC indices. Boolean constants + */ + pDb->pSchema->file_format = (u8)meta[BTREE_FILE_FORMAT-1]; + if( pDb->pSchema->file_format==0 ){ + pDb->pSchema->file_format = 1; + } + if( pDb->pSchema->file_format>SQLITE_MAX_FILE_FORMAT ){ + sqlite3SetString(pzErrMsg, db, "unsupported file format"); + rc = SQLITE_ERROR; + goto initone_error_out; + } + + /* Ticket #2804: When we open a database in the newer file format, + ** clear the legacy_file_format pragma flag so that a VACUUM will + ** not downgrade the database and thus invalidate any descending + ** indices that the user might have created. + */ + if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){ + db->flags &= ~(u64)SQLITE_LegacyFileFmt; + } + + /* Read the schema information out of the schema tables + */ + assert( db->init.busy ); + initData.mxPage = sqlite3BtreeLastPage(pDb->pBt); + { + char *zSql; + zSql = sqlite3MPrintf(db, + "SELECT*FROM\"%w\".%s ORDER BY rowid", + db->aDb[iDb].zDbSName, zSchemaTabName); +#ifndef SQLITE_OMIT_AUTHORIZATION + { + sqlite3_xauth xAuth; + xAuth = db->xAuth; + db->xAuth = 0; +#endif + rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; + } +#endif + if( rc==SQLITE_OK ) rc = initData.rc; + sqlite3DbFree(db, zSql); +#ifndef SQLITE_OMIT_ANALYZE + if( rc==SQLITE_OK ){ + sqlite3AnalysisLoad(db, iDb); + } +#endif + } + assert( pDb == &(db->aDb[iDb]) ); + if( db->mallocFailed ){ + rc = SQLITE_NOMEM_BKPT; + sqlite3ResetAllSchemasOfConnection(db); + pDb = &db->aDb[iDb]; + }else + if( rc==SQLITE_OK || (db->flags&SQLITE_NoSchemaError)){ + /* Hack: If the SQLITE_NoSchemaError flag is set, then consider + ** the schema loaded, even if errors (other than OOM) occurred. In + ** this situation the current sqlite3_prepare() operation will fail, + ** but the following one will attempt to compile the supplied statement + ** against whatever subset of the schema was loaded before the error + ** occurred. + ** + ** The primary purpose of this is to allow access to the sqlite_schema + ** table even when its contents have been corrupted. + */ + DbSetProperty(db, iDb, DB_SchemaLoaded); + rc = SQLITE_OK; + } + + /* Jump here for an error that occurs after successfully allocating + ** curMain and calling sqlite3BtreeEnter(). For an error that occurs + ** before that point, jump to error_out. + */ +initone_error_out: + if( openedTransaction ){ + sqlite3BtreeCommit(pDb->pBt); + } + sqlite3BtreeLeave(pDb->pBt); + +error_out: + if( rc ){ + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomFault(db); + } + sqlite3ResetOneSchema(db, iDb); + } + db->init.busy = 0; + return rc; +} + +/* +** Initialize all database files - the main database file, the file +** used to store temporary tables, and any additional database files +** created using ATTACH statements. Return a success code. If an +** error occurs, write an error message into *pzErrMsg. +** +** After a database is initialized, the DB_SchemaLoaded bit is set +** bit is set in the flags field of the Db structure. +*/ +SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ + int i, rc; + int commit_internal = !(db->mDbFlags&DBFLAG_SchemaChange); + + assert( sqlite3_mutex_held(db->mutex) ); + assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) ); + assert( db->init.busy==0 ); + ENC(db) = SCHEMA_ENC(db); + assert( db->nDb>0 ); + /* Do the main schema first */ + if( !DbHasProperty(db, 0, DB_SchemaLoaded) ){ + rc = sqlite3InitOne(db, 0, pzErrMsg, 0); + if( rc ) return rc; + } + /* All other schemas after the main schema. The "temp" schema must be last */ + for(i=db->nDb-1; i>0; i--){ + assert( i==1 || sqlite3BtreeHoldsMutex(db->aDb[i].pBt) ); + if( !DbHasProperty(db, i, DB_SchemaLoaded) ){ + rc = sqlite3InitOne(db, i, pzErrMsg, 0); + if( rc ) return rc; + } + } + if( commit_internal ){ + sqlite3CommitInternalChanges(db); + } + return SQLITE_OK; +} + +/* +** This routine is a no-op if the database schema is already initialized. +** Otherwise, the schema is loaded. An error code is returned. +*/ +SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse){ + int rc = SQLITE_OK; + sqlite3 *db = pParse->db; + assert( sqlite3_mutex_held(db->mutex) ); + if( !db->init.busy ){ + rc = sqlite3Init(db, &pParse->zErrMsg); + if( rc!=SQLITE_OK ){ + pParse->rc = rc; + pParse->nErr++; + }else if( db->noSharedCache ){ + db->mDbFlags |= DBFLAG_SchemaKnownOk; + } + } + return rc; +} + + +/* +** Check schema cookies in all databases. If any cookie is out +** of date set pParse->rc to SQLITE_SCHEMA. If all schema cookies +** make no changes to pParse->rc. +*/ +static void schemaIsValid(Parse *pParse){ + sqlite3 *db = pParse->db; + int iDb; + int rc; + int cookie; + + assert( pParse->checkSchema ); + assert( sqlite3_mutex_held(db->mutex) ); + for(iDb=0; iDbnDb; iDb++){ + int openedTransaction = 0; /* True if a transaction is opened */ + Btree *pBt = db->aDb[iDb].pBt; /* Btree database to read cookie from */ + if( pBt==0 ) continue; + + /* If there is not already a read-only (or read-write) transaction opened + ** on the b-tree database, open one now. If a transaction is opened, it + ** will be closed immediately after reading the meta-value. */ + if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_NONE ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomFault(db); + pParse->rc = SQLITE_NOMEM; + } + if( rc!=SQLITE_OK ) return; + openedTransaction = 1; + } + + /* Read the schema cookie from the database. If it does not match the + ** value stored as part of the in-memory schema representation, + ** set Parse.rc to SQLITE_SCHEMA. */ + sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ + sqlite3ResetOneSchema(db, iDb); + pParse->rc = SQLITE_SCHEMA; + } + + /* Close the transaction, if one was opened. */ + if( openedTransaction ){ + sqlite3BtreeCommit(pBt); + } + } +} + +/* +** Convert a schema pointer into the iDb index that indicates +** which database file in db->aDb[] the schema refers to. +** +** If the same database is attached more than once, the first +** attached database is returned. +*/ +SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ + int i = -32768; + + /* If pSchema is NULL, then return -32768. This happens when code in + ** expr.c is trying to resolve a reference to a transient table (i.e. one + ** created by a sub-select). In this case the return value of this + ** function should never be used. + ** + ** We return -32768 instead of the more usual -1 simply because using + ** -32768 as the incorrect index into db->aDb[] is much + ** more likely to cause a segfault than -1 (of course there are assert() + ** statements too, but it never hurts to play the odds) and + ** -32768 will still fit into a 16-bit signed integer. + */ + assert( sqlite3_mutex_held(db->mutex) ); + if( pSchema ){ + for(i=0; 1; i++){ + assert( inDb ); + if( db->aDb[i].pSchema==pSchema ){ + break; + } + } + assert( i>=0 && inDb ); + } + return i; +} + +/* +** Free all memory allocations in the pParse object +*/ +SQLITE_PRIVATE void sqlite3ParserReset(Parse *pParse){ + sqlite3 *db = pParse->db; + while( pParse->pCleanup ){ + ParseCleanup *pCleanup = pParse->pCleanup; + pParse->pCleanup = pCleanup->pNext; + pCleanup->xCleanup(db, pCleanup->pPtr); + sqlite3DbFreeNN(db, pCleanup); + } + sqlite3DbFree(db, pParse->aLabel); + if( pParse->pConstExpr ){ + sqlite3ExprListDelete(db, pParse->pConstExpr); + } + if( db ){ + assert( db->lookaside.bDisable >= pParse->disableLookaside ); + db->lookaside.bDisable -= pParse->disableLookaside; + db->lookaside.sz = db->lookaside.bDisable ? 0 : db->lookaside.szTrue; + } + pParse->disableLookaside = 0; +} + +/* +** Add a new cleanup operation to a Parser. The cleanup should happen when +** the parser object is destroyed. But, beware: the cleanup might happen +** immediately. +** +** Use this mechanism for uncommon cleanups. There is a higher setup +** cost for this mechansim (an extra malloc), so it should not be used +** for common cleanups that happen on most calls. But for less +** common cleanups, we save a single NULL-pointer comparison in +** sqlite3ParserReset(), which reduces the total CPU cycle count. +** +** If a memory allocation error occurs, then the cleanup happens immediately. +** When either SQLITE_DEBUG or SQLITE_COVERAGE_TEST are defined, the +** pParse->earlyCleanup flag is set in that case. Calling code show verify +** that test cases exist for which this happens, to guard against possible +** use-after-free errors following an OOM. The preferred way to do this is +** to immediately follow the call to this routine with: +** +** testcase( pParse->earlyCleanup ); +** +** This routine returns a copy of its pPtr input (the third parameter) +** except if an early cleanup occurs, in which case it returns NULL. So +** another way to check for early cleanup is to check the return value. +** Or, stop using the pPtr parameter with this call and use only its +** return value thereafter. Something like this: +** +** pObj = sqlite3ParserAddCleanup(pParse, destructor, pObj); +*/ +SQLITE_PRIVATE void *sqlite3ParserAddCleanup( + Parse *pParse, /* Destroy when this Parser finishes */ + void (*xCleanup)(sqlite3*,void*), /* The cleanup routine */ + void *pPtr /* Pointer to object to be cleaned up */ +){ + ParseCleanup *pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + if( pCleanup ){ + pCleanup->pNext = pParse->pCleanup; + pParse->pCleanup = pCleanup; + pCleanup->pPtr = pPtr; + pCleanup->xCleanup = xCleanup; + }else{ + xCleanup(pParse->db, pPtr); + pPtr = 0; +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + pParse->earlyCleanup = 1; +#endif + } + return pPtr; +} + +/* +** Compile the UTF-8 encoded SQL statement zSql into a statement handle. +*/ +static int sqlite3Prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + Vdbe *pReprepare, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + char *zErrMsg = 0; /* Error message */ + int rc = SQLITE_OK; /* Result code */ + int i; /* Loop counter */ + Parse sParse; /* Parsing context */ + + memset(&sParse, 0, PARSE_HDR_SZ); + memset(PARSE_TAIL(&sParse), 0, PARSE_TAIL_SZ); + sParse.pReprepare = pReprepare; + assert( ppStmt && *ppStmt==0 ); + /* assert( !db->mallocFailed ); // not true with SQLITE_USE_ALLOCA */ + assert( sqlite3_mutex_held(db->mutex) ); + + /* For a long-term use prepared statement avoid the use of + ** lookaside memory. + */ + if( prepFlags & SQLITE_PREPARE_PERSISTENT ){ + sParse.disableLookaside++; + DisableLookaside; + } + sParse.disableVtab = (prepFlags & SQLITE_PREPARE_NO_VTAB)!=0; + + /* Check to verify that it is possible to get a read lock on all + ** database schemas. The inability to get a read lock indicates that + ** some other database connection is holding a write-lock, which in + ** turn means that the other connection has made uncommitted changes + ** to the schema. + ** + ** Were we to proceed and prepare the statement against the uncommitted + ** schema changes and if those schema changes are subsequently rolled + ** back and different changes are made in their place, then when this + ** prepared statement goes to run the schema cookie would fail to detect + ** the schema change. Disaster would follow. + ** + ** This thread is currently holding mutexes on all Btrees (because + ** of the sqlite3BtreeEnterAll() in sqlite3LockAndPrepare()) so it + ** is not possible for another thread to start a new schema change + ** while this routine is running. Hence, we do not need to hold + ** locks on the schema, we just need to make sure nobody else is + ** holding them. + ** + ** Note that setting READ_UNCOMMITTED overrides most lock detection, + ** but it does *not* override schema lock detection, so this all still + ** works even if READ_UNCOMMITTED is set. + */ + if( !db->noSharedCache ){ + for(i=0; inDb; i++) { + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + assert( sqlite3BtreeHoldsMutex(pBt) ); + rc = sqlite3BtreeSchemaLocked(pBt); + if( rc ){ + const char *zDb = db->aDb[i].zDbSName; + sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); + testcase( db->flags & SQLITE_ReadUncommit ); + goto end_prepare; + } + } + } + } + + sqlite3VtabUnlockList(db); + + sParse.db = db; + if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ + char *zSqlCopy; + int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + testcase( nBytes==mxLen ); + testcase( nBytes==mxLen+1 ); + if( nBytes>mxLen ){ + sqlite3ErrorWithMsg(db, SQLITE_TOOBIG, "statement too long"); + rc = sqlite3ApiExit(db, SQLITE_TOOBIG); + goto end_prepare; + } + zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); + if( zSqlCopy ){ + sqlite3RunParser(&sParse, zSqlCopy, &zErrMsg); + sParse.zTail = &zSql[sParse.zTail-zSqlCopy]; + sqlite3DbFree(db, zSqlCopy); + }else{ + sParse.zTail = &zSql[nBytes]; + } + }else{ + sqlite3RunParser(&sParse, zSql, &zErrMsg); + } + assert( 0==sParse.nQueryLoop ); + + if( pzTail ){ + *pzTail = sParse.zTail; + } + + if( db->init.busy==0 ){ + sqlite3VdbeSetSql(sParse.pVdbe, zSql, (int)(sParse.zTail-zSql), prepFlags); + } + if( db->mallocFailed ){ + sParse.rc = SQLITE_NOMEM_BKPT; + sParse.checkSchema = 0; + } + if( sParse.rc!=SQLITE_OK && sParse.rc!=SQLITE_DONE ){ + if( sParse.checkSchema ){ + schemaIsValid(&sParse); + } + if( sParse.pVdbe ){ + sqlite3VdbeFinalize(sParse.pVdbe); + } + assert( 0==(*ppStmt) ); + rc = sParse.rc; + if( zErrMsg ){ + sqlite3ErrorWithMsg(db, rc, "%s", zErrMsg); + sqlite3DbFree(db, zErrMsg); + }else{ + sqlite3Error(db, rc); + } + }else{ + assert( zErrMsg==0 ); + *ppStmt = (sqlite3_stmt*)sParse.pVdbe; + rc = SQLITE_OK; + sqlite3ErrorClear(db); + } + + + /* Delete any TriggerPrg structures allocated while parsing this statement. */ + while( sParse.pTriggerPrg ){ + TriggerPrg *pT = sParse.pTriggerPrg; + sParse.pTriggerPrg = pT->pNext; + sqlite3DbFree(db, pT); + } + +end_prepare: + + sqlite3ParserReset(&sParse); + return rc; +} +static int sqlite3LockAndPrepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + Vdbe *pOld, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + int cnt = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppStmt = 0; + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + do{ + /* Make multiple attempts to compile the SQL, until it either succeeds + ** or encounters a permanent error. A schema problem after one schema + ** reset is considered a permanent error. */ + rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); + assert( rc==SQLITE_OK || *ppStmt==0 ); + }while( rc==SQLITE_ERROR_RETRY + || (rc==SQLITE_SCHEMA && (sqlite3ResetOneSchema(db,-1), cnt++)==0) ); + sqlite3BtreeLeaveAll(db); + rc = sqlite3ApiExit(db, rc); + assert( (rc&db->errMask)==rc ); + db->busyHandler.nBusy = 0; + sqlite3_mutex_leave(db->mutex); + return rc; +} + + +/* +** Rerun the compilation of a statement after a schema change. +** +** If the statement is successfully recompiled, return SQLITE_OK. Otherwise, +** if the statement cannot be recompiled because another connection has +** locked the sqlite3_schema table, return SQLITE_LOCKED. If any other error +** occurs, return SQLITE_SCHEMA. +*/ +SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ + int rc; + sqlite3_stmt *pNew; + const char *zSql; + sqlite3 *db; + u8 prepFlags; + + assert( sqlite3_mutex_held(sqlite3VdbeDb(p)->mutex) ); + zSql = sqlite3_sql((sqlite3_stmt *)p); + assert( zSql!=0 ); /* Reprepare only called for prepare_v2() statements */ + db = sqlite3VdbeDb(p); + assert( sqlite3_mutex_held(db->mutex) ); + prepFlags = sqlite3VdbePrepareFlags(p); + rc = sqlite3LockAndPrepare(db, zSql, -1, prepFlags, p, &pNew, 0); + if( rc ){ + if( rc==SQLITE_NOMEM ){ + sqlite3OomFault(db); + } + assert( pNew==0 ); + return rc; + }else{ + assert( pNew!=0 ); + } + sqlite3VdbeSwap((Vdbe*)pNew, p); + sqlite3TransferBindings(pNew, (sqlite3_stmt*)p); + sqlite3VdbeResetStepResult((Vdbe*)pNew); + sqlite3VdbeFinalize((Vdbe*)pNew); + return SQLITE_OK; +} + + +/* +** Two versions of the official API. Legacy and new use. In the legacy +** version, the original SQL text is not saved in the prepared statement +** and so if a schema change occurs, SQLITE_SCHEMA is returned by +** sqlite3_step(). In the new version, the original SQL text is retained +** and the statement is automatically recompiled if an schema change +** occurs. +*/ +SQLITE_API int sqlite3_prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3LockAndPrepare(db,zSql,nBytes,0,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} +SQLITE_API int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + /* EVIDENCE-OF: R-37923-12173 The sqlite3_prepare_v2() interface works + ** exactly the same as sqlite3_prepare_v3() with a zero prepFlags + ** parameter. + ** + ** Proof in that the 5th parameter to sqlite3LockAndPrepare is 0 */ + rc = sqlite3LockAndPrepare(db,zSql,nBytes,SQLITE_PREPARE_SAVESQL,0, + ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); + return rc; +} +SQLITE_API int sqlite3_prepare_v3( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + /* EVIDENCE-OF: R-56861-42673 sqlite3_prepare_v3() differs from + ** sqlite3_prepare_v2() only in having the extra prepFlags parameter, + ** which is a bit array consisting of zero or more of the + ** SQLITE_PREPARE_* flags. + ** + ** Proof by comparison to the implementation of sqlite3_prepare_v2() + ** directly above. */ + rc = sqlite3LockAndPrepare(db,zSql,nBytes, + SQLITE_PREPARE_SAVESQL|(prepFlags&SQLITE_PREPARE_MASK), + 0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); + return rc; +} + + +#ifndef SQLITE_OMIT_UTF16 +/* +** Compile the UTF-16 encoded SQL statement zSql into a statement handle. +*/ +static int sqlite3Prepare16( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + /* This function currently works by first transforming the UTF-16 + ** encoded string to UTF-8, then invoking sqlite3_prepare(). The + ** tricky bit is figuring out the pointer to return in *pzTail. + */ + char *zSql8; + const char *zTail8 = 0; + int rc = SQLITE_OK; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppStmt = 0; + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ + return SQLITE_MISUSE_BKPT; + } + if( nBytes>=0 ){ + int sz; + const char *z = (const char*)zSql; + for(sz=0; szmutex); + zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); + if( zSql8 ){ + rc = sqlite3LockAndPrepare(db, zSql8, -1, prepFlags, 0, ppStmt, &zTail8); + } + + if( zTail8 && pzTail ){ + /* If sqlite3_prepare returns a tail pointer, we calculate the + ** equivalent pointer into the UTF-16 string by counting the unicode + ** characters between zSql8 and zTail8, and then returning a pointer + ** the same number of characters into the UTF-16 string. + */ + int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8)); + *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed); + } + sqlite3DbFree(db, zSql8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Two versions of the official API. Legacy and new use. In the legacy +** version, the original SQL text is not saved in the prepared statement +** and so if a schema change occurs, SQLITE_SCHEMA is returned by +** sqlite3_step(). In the new version, the original SQL text is retained +** and the statement is automatically recompiled if an schema change +** occurs. +*/ +SQLITE_API int sqlite3_prepare16( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes,0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} +SQLITE_API int sqlite3_prepare16_v2( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes,SQLITE_PREPARE_SAVESQL,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} +SQLITE_API int sqlite3_prepare16_v3( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes, + SQLITE_PREPARE_SAVESQL|(prepFlags&SQLITE_PREPARE_MASK), + ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} + +#endif /* SQLITE_OMIT_UTF16 */ + +/************** End of prepare.c *********************************************/ +/************** Begin file select.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** to handle SELECT statements in SQLite. +*/ +/* #include "sqliteInt.h" */ + +/* +** An instance of the following object is used to record information about +** how to process the DISTINCT keyword, to simplify passing that information +** into the selectInnerLoop() routine. +*/ +typedef struct DistinctCtx DistinctCtx; +struct DistinctCtx { + u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ + int tabTnct; /* Ephemeral table used for DISTINCT processing */ + int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ +}; + +/* +** An instance of the following object is used to record information about +** the ORDER BY (or GROUP BY) clause of query is being coded. +** +** The aDefer[] array is used by the sorter-references optimization. For +** example, assuming there is no index that can be used for the ORDER BY, +** for the query: +** +** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10; +** +** it may be more efficient to add just the "a" values to the sorter, and +** retrieve the associated "bigblob" values directly from table t1 as the +** 10 smallest "a" values are extracted from the sorter. +** +** When the sorter-reference optimization is used, there is one entry in the +** aDefer[] array for each database table that may be read as values are +** extracted from the sorter. +*/ +typedef struct SortCtx SortCtx; +struct SortCtx { + ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */ + int nOBSat; /* Number of ORDER BY terms satisfied by indices */ + int iECursor; /* Cursor number for the sorter */ + int regReturn; /* Register holding block-output return address */ + int labelBkOut; /* Start label for the block-output subroutine */ + int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ + int labelDone; /* Jump here when done, ex: LIMIT reached */ + int labelOBLopt; /* Jump here when sorter is full */ + u8 sortFlags; /* Zero or more SORTFLAG_* bits */ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + u8 nDefer; /* Number of valid entries in aDefer[] */ + struct DeferredCsr { + Table *pTab; /* Table definition */ + int iCsr; /* Cursor number for table */ + int nKey; /* Number of PK columns for table pTab (>=1) */ + } aDefer[4]; +#endif + struct RowLoadInfo *pDeferredRowLoad; /* Deferred row loading info or NULL */ +}; +#define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ + +/* +** Delete all the content of a Select structure. Deallocate the structure +** itself depending on the value of bFree +** +** If bFree==1, call sqlite3DbFree() on the p object. +** If bFree==0, Leave the first Select object unfreed +*/ +static void clearSelect(sqlite3 *db, Select *p, int bFree){ + while( p ){ + Select *pPrior = p->pPrior; + sqlite3ExprListDelete(db, p->pEList); + sqlite3SrcListDelete(db, p->pSrc); + sqlite3ExprDelete(db, p->pWhere); + sqlite3ExprListDelete(db, p->pGroupBy); + sqlite3ExprDelete(db, p->pHaving); + sqlite3ExprListDelete(db, p->pOrderBy); + sqlite3ExprDelete(db, p->pLimit); + if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( OK_IF_ALWAYS_TRUE(p->pWinDefn) ){ + sqlite3WindowListDelete(db, p->pWinDefn); + } + while( p->pWin ){ + assert( p->pWin->ppThis==&p->pWin ); + sqlite3WindowUnlinkFromSelect(p->pWin); + } +#endif + if( bFree ) sqlite3DbFreeNN(db, p); + p = pPrior; + bFree = 1; + } +} + +/* +** Initialize a SelectDest structure. +*/ +SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ + pDest->eDest = (u8)eDest; + pDest->iSDParm = iParm; + pDest->iSDParm2 = 0; + pDest->zAffSdst = 0; + pDest->iSdst = 0; + pDest->nSdst = 0; +} + + +/* +** Allocate a new Select structure and return a pointer to that +** structure. +*/ +SQLITE_PRIVATE Select *sqlite3SelectNew( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* which columns to include in the result */ + SrcList *pSrc, /* the FROM clause -- which tables to scan */ + Expr *pWhere, /* the WHERE clause */ + ExprList *pGroupBy, /* the GROUP BY clause */ + Expr *pHaving, /* the HAVING clause */ + ExprList *pOrderBy, /* the ORDER BY clause */ + u32 selFlags, /* Flag parameters, such as SF_Distinct */ + Expr *pLimit /* LIMIT value. NULL means not used */ +){ + Select *pNew, *pAllocated; + Select standin; + pAllocated = pNew = sqlite3DbMallocRawNN(pParse->db, sizeof(*pNew) ); + if( pNew==0 ){ + assert( pParse->db->mallocFailed ); + pNew = &standin; + } + if( pEList==0 ){ + pEList = sqlite3ExprListAppend(pParse, 0, + sqlite3Expr(pParse->db,TK_ASTERISK,0)); + } + pNew->pEList = pEList; + pNew->op = TK_SELECT; + pNew->selFlags = selFlags; + pNew->iLimit = 0; + pNew->iOffset = 0; + pNew->selId = ++pParse->nSelect; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->nSelectRow = 0; + if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc)); + pNew->pSrc = pSrc; + pNew->pWhere = pWhere; + pNew->pGroupBy = pGroupBy; + pNew->pHaving = pHaving; + pNew->pOrderBy = pOrderBy; + pNew->pPrior = 0; + pNew->pNext = 0; + pNew->pLimit = pLimit; + pNew->pWith = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + pNew->pWin = 0; + pNew->pWinDefn = 0; +#endif + if( pParse->db->mallocFailed ) { + clearSelect(pParse->db, pNew, pNew!=&standin); + pAllocated = 0; + }else{ + assert( pNew->pSrc!=0 || pParse->nErr>0 ); + } + return pAllocated; +} + + +/* +** Delete the given Select structure and all of its substructures. +*/ +SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ + if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1); +} + +/* +** Return a pointer to the right-most SELECT statement in a compound. +*/ +static Select *findRightmost(Select *p){ + while( p->pNext ) p = p->pNext; + return p; +} + +/* +** Given 1 to 3 identifiers preceding the JOIN keyword, determine the +** type of join. Return an integer constant that expresses that type +** in terms of the following bit values: +** +** JT_INNER +** JT_CROSS +** JT_OUTER +** JT_NATURAL +** JT_LEFT +** JT_RIGHT +** +** A full outer join is the combination of JT_LEFT and JT_RIGHT. +** +** If an illegal or unsupported join type is seen, then still return +** a join type, but put an error in the pParse structure. +*/ +SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ + int jointype = 0; + Token *apAll[3]; + Token *p; + /* 0123456789 123456789 123456789 123 */ + static const char zKeyText[] = "naturaleftouterightfullinnercross"; + static const struct { + u8 i; /* Beginning of keyword text in zKeyText[] */ + u8 nChar; /* Length of the keyword in characters */ + u8 code; /* Join type mask */ + } aKeyword[] = { + /* natural */ { 0, 7, JT_NATURAL }, + /* left */ { 6, 4, JT_LEFT|JT_OUTER }, + /* outer */ { 10, 5, JT_OUTER }, + /* right */ { 14, 5, JT_RIGHT|JT_OUTER }, + /* full */ { 19, 4, JT_LEFT|JT_RIGHT|JT_OUTER }, + /* inner */ { 23, 5, JT_INNER }, + /* cross */ { 28, 5, JT_INNER|JT_CROSS }, + }; + int i, j; + apAll[0] = pA; + apAll[1] = pB; + apAll[2] = pC; + for(i=0; i<3 && apAll[i]; i++){ + p = apAll[i]; + for(j=0; jn==aKeyword[j].nChar + && sqlite3StrNICmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){ + jointype |= aKeyword[j].code; + break; + } + } + testcase( j==0 || j==1 || j==2 || j==3 || j==4 || j==5 || j==6 ); + if( j>=ArraySize(aKeyword) ){ + jointype |= JT_ERROR; + break; + } + } + if( + (jointype & (JT_INNER|JT_OUTER))==(JT_INNER|JT_OUTER) || + (jointype & JT_ERROR)!=0 + ){ + const char *zSp = " "; + assert( pB!=0 ); + if( pC==0 ){ zSp++; } + sqlite3ErrorMsg(pParse, "unknown or unsupported join type: " + "%T %T%s%T", pA, pB, zSp, pC); + jointype = JT_INNER; + }else if( (jointype & JT_OUTER)!=0 + && (jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ){ + sqlite3ErrorMsg(pParse, + "RIGHT and FULL OUTER JOINs are not currently supported"); + jointype = JT_INNER; + } + return jointype; +} + +/* +** Return the index of a column in a table. Return -1 if the column +** is not contained in the table. +*/ +SQLITE_PRIVATE int sqlite3ColumnIndex(Table *pTab, const char *zCol){ + int i; + u8 h = sqlite3StrIHash(zCol); + Column *pCol; + for(pCol=pTab->aCol, i=0; inCol; pCol++, i++){ + if( pCol->hName==h && sqlite3StrICmp(pCol->zCnName, zCol)==0 ) return i; + } + return -1; +} + +/* +** Search the first N tables in pSrc, from left to right, looking for a +** table that has a column named zCol. +** +** When found, set *piTab and *piCol to the table index and column index +** of the matching column and return TRUE. +** +** If not found, return FALSE. +*/ +static int tableAndColumnIndex( + SrcList *pSrc, /* Array of tables to search */ + int N, /* Number of tables in pSrc->a[] to search */ + const char *zCol, /* Name of the column we are looking for */ + int *piTab, /* Write index of pSrc->a[] here */ + int *piCol, /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ + int bIgnoreHidden /* True to ignore hidden columns */ +){ + int i; /* For looping over tables in pSrc */ + int iCol; /* Index of column matching zCol */ + + assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ + for(i=0; ia[i].pTab, zCol); + if( iCol>=0 + && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pTab->aCol[iCol])==0) + ){ + if( piTab ){ + *piTab = i; + *piCol = iCol; + } + return 1; + } + } + return 0; +} + +/* +** This function is used to add terms implied by JOIN syntax to the +** WHERE clause expression of a SELECT statement. The new term, which +** is ANDed with the existing WHERE clause, is of the form: +** +** (tab1.col1 = tab2.col2) +** +** where tab1 is the iSrc'th table in SrcList pSrc and tab2 is the +** (iSrc+1)'th. Column col1 is column iColLeft of tab1, and col2 is +** column iColRight of tab2. +*/ +static void addWhereTerm( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* List of tables in FROM clause */ + int iLeft, /* Index of first table to join in pSrc */ + int iColLeft, /* Index of column in first table */ + int iRight, /* Index of second table in pSrc */ + int iColRight, /* Index of column in second table */ + int isOuterJoin, /* True if this is an OUTER join */ + Expr **ppWhere /* IN/OUT: The WHERE clause to add to */ +){ + sqlite3 *db = pParse->db; + Expr *pE1; + Expr *pE2; + Expr *pEq; + + assert( iLeftnSrc>iRight ); + assert( pSrc->a[iLeft].pTab ); + assert( pSrc->a[iRight].pTab ); + + pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iColLeft); + pE2 = sqlite3CreateColumnExpr(db, pSrc, iRight, iColRight); + + pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2); + assert( pE2!=0 || pEq==0 ); /* Due to db->mallocFailed test + ** in sqlite3DbMallocRawNN() called from + ** sqlite3PExpr(). */ + if( pEq && isOuterJoin ){ + ExprSetProperty(pEq, EP_FromJoin); + assert( !ExprHasProperty(pEq, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(pEq, EP_NoReduce); + pEq->iRightJoinTable = pE2->iTable; + } + *ppWhere = sqlite3ExprAnd(pParse, *ppWhere, pEq); +} + +/* +** Set the EP_FromJoin property on all terms of the given expression. +** And set the Expr.iRightJoinTable to iTable for every term in the +** expression. +** +** The EP_FromJoin property is used on terms of an expression to tell +** the LEFT OUTER JOIN processing logic that this term is part of the +** join restriction specified in the ON or USING clause and not a part +** of the more general WHERE clause. These terms are moved over to the +** WHERE clause during join processing but we need to remember that they +** originated in the ON or USING clause. +** +** The Expr.iRightJoinTable tells the WHERE clause processing that the +** expression depends on table iRightJoinTable even if that table is not +** explicitly mentioned in the expression. That information is needed +** for cases like this: +** +** SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.b AND t1.x=5 +** +** The where clause needs to defer the handling of the t1.x=5 +** term until after the t2 loop of the join. In that way, a +** NULL t2 row will be inserted whenever t1.x!=5. If we do not +** defer the handling of t1.x=5, it will be processed immediately +** after the t1 loop and rows with t1.x!=5 will never appear in +** the output, which is incorrect. +*/ +SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr *p, int iTable){ + while( p ){ + ExprSetProperty(p, EP_FromJoin); + assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(p, EP_NoReduce); + p->iRightJoinTable = iTable; + if( p->op==TK_FUNCTION && p->x.pList ){ + int i; + for(i=0; ix.pList->nExpr; i++){ + sqlite3SetJoinExpr(p->x.pList->a[i].pExpr, iTable); + } + } + sqlite3SetJoinExpr(p->pLeft, iTable); + p = p->pRight; + } +} + +/* Undo the work of sqlite3SetJoinExpr(). In the expression p, convert every +** term that is marked with EP_FromJoin and iRightJoinTable==iTable into +** an ordinary term that omits the EP_FromJoin mark. +** +** This happens when a LEFT JOIN is simplified into an ordinary JOIN. +*/ +static void unsetJoinExpr(Expr *p, int iTable){ + while( p ){ + if( ExprHasProperty(p, EP_FromJoin) + && (iTable<0 || p->iRightJoinTable==iTable) ){ + ExprClearProperty(p, EP_FromJoin); + } + if( p->op==TK_COLUMN && p->iTable==iTable ){ + ExprClearProperty(p, EP_CanBeNull); + } + if( p->op==TK_FUNCTION && p->x.pList ){ + int i; + for(i=0; ix.pList->nExpr; i++){ + unsetJoinExpr(p->x.pList->a[i].pExpr, iTable); + } + } + unsetJoinExpr(p->pLeft, iTable); + p = p->pRight; + } +} + +/* +** This routine processes the join information for a SELECT statement. +** ON and USING clauses are converted into extra terms of the WHERE clause. +** NATURAL joins also create extra WHERE clause terms. +** +** The terms of a FROM clause are contained in the Select.pSrc structure. +** The left most table is the first entry in Select.pSrc. The right-most +** table is the last entry. The join operator is held in the entry to +** the left. Thus entry 0 contains the join operator for the join between +** entries 0 and 1. Any ON or USING clauses associated with the join are +** also attached to the left entry. +** +** This routine returns the number of errors encountered. +*/ +static int sqliteProcessJoin(Parse *pParse, Select *p){ + SrcList *pSrc; /* All tables in the FROM clause */ + int i, j; /* Loop counters */ + SrcItem *pLeft; /* Left table being joined */ + SrcItem *pRight; /* Right table being joined */ + + pSrc = p->pSrc; + pLeft = &pSrc->a[0]; + pRight = &pLeft[1]; + for(i=0; inSrc-1; i++, pRight++, pLeft++){ + Table *pRightTab = pRight->pTab; + int isOuter; + + if( NEVER(pLeft->pTab==0 || pRightTab==0) ) continue; + isOuter = (pRight->fg.jointype & JT_OUTER)!=0; + + /* When the NATURAL keyword is present, add WHERE clause terms for + ** every column that the two tables have in common. + */ + if( pRight->fg.jointype & JT_NATURAL ){ + if( pRight->pOn || pRight->pUsing ){ + sqlite3ErrorMsg(pParse, "a NATURAL join may not have " + "an ON or USING clause", 0); + return 1; + } + for(j=0; jnCol; j++){ + char *zName; /* Name of column in the right table */ + int iLeft; /* Matching left table */ + int iLeftCol; /* Matching column in the left table */ + + if( IsHiddenColumn(&pRightTab->aCol[j]) ) continue; + zName = pRightTab->aCol[j].zCnName; + if( tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol, 1) ){ + addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, j, + isOuter, &p->pWhere); + } + } + } + + /* Disallow both ON and USING clauses in the same join + */ + if( pRight->pOn && pRight->pUsing ){ + sqlite3ErrorMsg(pParse, "cannot have both ON and USING " + "clauses in the same join"); + return 1; + } + + /* Add the ON clause to the end of the WHERE clause, connected by + ** an AND operator. + */ + if( pRight->pOn ){ + if( isOuter ) sqlite3SetJoinExpr(pRight->pOn, pRight->iCursor); + p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->pOn); + pRight->pOn = 0; + } + + /* Create extra terms on the WHERE clause for each column named + ** in the USING clause. Example: If the two tables to be joined are + ** A and B and the USING clause names X, Y, and Z, then add this + ** to the WHERE clause: A.X=B.X AND A.Y=B.Y AND A.Z=B.Z + ** Report an error if any column mentioned in the USING clause is + ** not contained in both tables to be joined. + */ + if( pRight->pUsing ){ + IdList *pList = pRight->pUsing; + for(j=0; jnId; j++){ + char *zName; /* Name of the term in the USING clause */ + int iLeft; /* Table on the left with matching column name */ + int iLeftCol; /* Column number of matching column on the left */ + int iRightCol; /* Column number of matching column on the right */ + + zName = pList->a[j].zName; + iRightCol = sqlite3ColumnIndex(pRightTab, zName); + if( iRightCol<0 + || !tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol, 0) + ){ + sqlite3ErrorMsg(pParse, "cannot join using column %s - column " + "not present in both tables", zName); + return 1; + } + addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, iRightCol, + isOuter, &p->pWhere); + } + } + } + return 0; +} + +/* +** An instance of this object holds information (beyond pParse and pSelect) +** needed to load the next result row that is to be added to the sorter. +*/ +typedef struct RowLoadInfo RowLoadInfo; +struct RowLoadInfo { + int regResult; /* Store results in array of registers here */ + u8 ecelFlags; /* Flag argument to ExprCodeExprList() */ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + ExprList *pExtra; /* Extra columns needed by sorter refs */ + int regExtraResult; /* Where to load the extra columns */ +#endif +}; + +/* +** This routine does the work of loading query data into an array of +** registers so that it can be added to the sorter. +*/ +static void innerLoopLoadRow( + Parse *pParse, /* Statement under construction */ + Select *pSelect, /* The query being coded */ + RowLoadInfo *pInfo /* Info needed to complete the row load */ +){ + sqlite3ExprCodeExprList(pParse, pSelect->pEList, pInfo->regResult, + 0, pInfo->ecelFlags); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pInfo->pExtra ){ + sqlite3ExprCodeExprList(pParse, pInfo->pExtra, pInfo->regExtraResult, 0, 0); + sqlite3ExprListDelete(pParse->db, pInfo->pExtra); + } +#endif +} + +/* +** Code the OP_MakeRecord instruction that generates the entry to be +** added into the sorter. +** +** Return the register in which the result is stored. +*/ +static int makeSorterRecord( + Parse *pParse, + SortCtx *pSort, + Select *pSelect, + int regBase, + int nBase +){ + int nOBSat = pSort->nOBSat; + Vdbe *v = pParse->pVdbe; + int regOut = ++pParse->nMem; + if( pSort->pDeferredRowLoad ){ + innerLoopLoadRow(pParse, pSelect, pSort->pDeferredRowLoad); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regOut); + return regOut; +} + +/* +** Generate code that will push the record in registers regData +** through regData+nData-1 onto the sorter. +*/ +static void pushOntoSorter( + Parse *pParse, /* Parser context */ + SortCtx *pSort, /* Information about the ORDER BY clause */ + Select *pSelect, /* The whole SELECT statement */ + int regData, /* First register holding data to be sorted */ + int regOrigData, /* First register holding data before packing */ + int nData, /* Number of elements in the regData data array */ + int nPrefixReg /* No. of reg prior to regData available for use */ +){ + Vdbe *v = pParse->pVdbe; /* Stmt under construction */ + int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0); + int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ + int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ + int regBase; /* Regs for sorter record */ + int regRecord = 0; /* Assembled sorter record */ + int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ + int op; /* Opcode to add sorter record to sorter */ + int iLimit; /* LIMIT counter */ + int iSkip = 0; /* End of the sorter insert loop */ + + assert( bSeq==0 || bSeq==1 ); + + /* Three cases: + ** (1) The data to be sorted has already been packed into a Record + ** by a prior OP_MakeRecord. In this case nData==1 and regData + ** will be completely unrelated to regOrigData. + ** (2) All output columns are included in the sort record. In that + ** case regData==regOrigData. + ** (3) Some output columns are omitted from the sort record due to + ** the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the + ** SQLITE_ECEL_OMITREF optimization, or due to the + ** SortCtx.pDeferredRowLoad optimiation. In any of these cases + ** regOrigData is 0 to prevent this routine from trying to copy + ** values that might not yet exist. + */ + assert( nData==1 || regData==regOrigData || regOrigData==0 ); + + if( nPrefixReg ){ + assert( nPrefixReg==nExpr+bSeq ); + regBase = regData - nPrefixReg; + }else{ + regBase = pParse->nMem + 1; + pParse->nMem += nBase; + } + assert( pSelect->iOffset==0 || pSelect->iLimit!=0 ); + iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; + pSort->labelDone = sqlite3VdbeMakeLabel(pParse); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, + SQLITE_ECEL_DUP | (regOrigData? SQLITE_ECEL_REF : 0)); + if( bSeq ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); + } + if( nPrefixReg==0 && nData>0 ){ + sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); + } + if( nOBSat>0 ){ + int regPrevKey; /* The first nOBSat columns of the previous row */ + int addrFirst; /* Address of the OP_IfNot opcode */ + int addrJmp; /* Address of the OP_Jump opcode */ + VdbeOp *pOp; /* Opcode that opens the sorter */ + int nKey; /* Number of sorting key columns, including OP_Sequence */ + KeyInfo *pKI; /* Original KeyInfo on the sorter table */ + + regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase); + regPrevKey = pParse->nMem+1; + pParse->nMem += pSort->nOBSat; + nKey = nExpr - pSort->nOBSat + bSeq; + if( bSeq ){ + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); + }else{ + addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor); + } + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); + pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + if( pParse->db->mallocFailed ) return; + pOp->p2 = nKey + nData; + pKI = pOp->p4.pKeyInfo; + memset(pKI->aSortFlags, 0, pKI->nKeyField); /* Makes OP_Jump testable */ + sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); + testcase( pKI->nAllField > pKI->nKeyField+2 ); + pOp->p4.pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pSort->pOrderBy,nOBSat, + pKI->nAllField-pKI->nKeyField-1); + pOp = 0; /* Ensure pOp not used after sqltie3VdbeAddOp3() */ + addrJmp = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); + pSort->labelBkOut = sqlite3VdbeMakeLabel(pParse); + pSort->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); + if( iLimit ){ + sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, pSort->labelDone); + VdbeCoverage(v); + } + sqlite3VdbeJumpHere(v, addrFirst); + sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); + sqlite3VdbeJumpHere(v, addrJmp); + } + if( iLimit ){ + /* At this point the values for the new sorter entry are stored + ** in an array of registers. They need to be composed into a record + ** and inserted into the sorter if either (a) there are currently + ** less than LIMIT+OFFSET items or (b) the new record is smaller than + ** the largest record currently in the sorter. If (b) is true and there + ** are already LIMIT+OFFSET items in the sorter, delete the largest + ** entry before inserting the new one. This way there are never more + ** than LIMIT+OFFSET items in the sorter. + ** + ** If the new record does not need to be inserted into the sorter, + ** jump to the next iteration of the loop. If the pSort->labelOBLopt + ** value is not zero, then it is a label of where to jump. Otherwise, + ** just bypass the row insert logic. See the header comment on the + ** sqlite3WhereOrderByLimitOptLabel() function for additional info. + */ + int iCsr = pSort->iECursor; + sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Last, iCsr, 0); + iSkip = sqlite3VdbeAddOp4Int(v, OP_IdxLE, + iCsr, 0, regBase+nOBSat, nExpr-nOBSat); + VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_Delete, iCsr); + } + if( regRecord==0 ){ + regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase); + } + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + op = OP_SorterInsert; + }else{ + op = OP_IdxInsert; + } + sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord, + regBase+nOBSat, nBase-nOBSat); + if( iSkip ){ + sqlite3VdbeChangeP2(v, iSkip, + pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v)); + } +} + +/* +** Add code to implement the OFFSET +*/ +static void codeOffset( + Vdbe *v, /* Generate code into this VM */ + int iOffset, /* Register holding the offset counter */ + int iContinue /* Jump here to skip the current record */ +){ + if( iOffset>0 ){ + sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v); + VdbeComment((v, "OFFSET")); + } +} + +/* +** Add code that will check to make sure the array of registers starting at +** iMem form a distinct entry. This is used by both "SELECT DISTINCT ..." and +** distinct aggregates ("SELECT count(DISTINCT ) ..."). Three strategies +** are available. Which is used depends on the value of parameter eTnctType, +** as follows: +** +** WHERE_DISTINCT_UNORDERED/WHERE_DISTINCT_NOOP: +** Build an ephemeral table that contains all entries seen before and +** skip entries which have been seen before. +** +** Parameter iTab is the cursor number of an ephemeral table that must +** be opened before the VM code generated by this routine is executed. +** The ephemeral cursor table is queried for a record identical to the +** record formed by the current array of registers. If one is found, +** jump to VM address addrRepeat. Otherwise, insert a new record into +** the ephemeral cursor and proceed. +** +** The returned value in this case is a copy of parameter iTab. +** +** WHERE_DISTINCT_ORDERED: +** In this case rows are being delivered sorted order. The ephermal +** table is not required. Instead, the current set of values +** is compared against previous row. If they match, the new row +** is not distinct and control jumps to VM address addrRepeat. Otherwise, +** the VM program proceeds with processing the new row. +** +** The returned value in this case is the register number of the first +** in an array of registers used to store the previous result row so that +** it can be compared to the next. The caller must ensure that this +** register is initialized to NULL. (The fixDistinctOpenEph() routine +** will take care of this initialization.) +** +** WHERE_DISTINCT_UNIQUE: +** In this case it has already been determined that the rows are distinct. +** No special action is required. The return value is zero. +** +** Parameter pEList is the list of expressions used to generated the +** contents of each row. It is used by this routine to determine (a) +** how many elements there are in the array of registers and (b) the +** collation sequences that should be used for the comparisons if +** eTnctType is WHERE_DISTINCT_ORDERED. +*/ +static int codeDistinct( + Parse *pParse, /* Parsing and code generating context */ + int eTnctType, /* WHERE_DISTINCT_* value */ + int iTab, /* A sorting index used to test for distinctness */ + int addrRepeat, /* Jump to here if not distinct */ + ExprList *pEList, /* Expression for each element */ + int regElem /* First element */ +){ + int iRet = 0; + int nResultCol = pEList->nExpr; + Vdbe *v = pParse->pVdbe; + + switch( eTnctType ){ + case WHERE_DISTINCT_ORDERED: { + int i; + int iJump; /* Jump destination */ + int regPrev; /* Previous row content */ + + /* Allocate space for the previous row */ + iRet = regPrev = pParse->nMem+1; + pParse->nMem += nResultCol; + + iJump = sqlite3VdbeCurrentAddr(v) + nResultCol; + for(i=0; ia[i].pExpr); + if( idb->mallocFailed ); + sqlite3VdbeAddOp3(v, OP_Copy, regElem, regPrev, nResultCol-1); + break; + } + + case WHERE_DISTINCT_UNIQUE: { + /* nothing to do */ + break; + } + + default: { + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, regElem, nResultCol); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regElem, nResultCol, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, regElem, nResultCol); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3ReleaseTempReg(pParse, r1); + iRet = iTab; + break; + } + } + + return iRet; +} + +/* +** This routine runs after codeDistinct(). It makes necessary +** adjustments to the OP_OpenEphemeral opcode that the codeDistinct() +** routine made use of. This processing must be done separately since +** sometimes codeDistinct is called before the OP_OpenEphemeral is actually +** laid down. +** +** WHERE_DISTINCT_NOOP: +** WHERE_DISTINCT_UNORDERED: +** +** No adjustments necessary. This function is a no-op. +** +** WHERE_DISTINCT_UNIQUE: +** +** The ephemeral table is not needed. So change the +** OP_OpenEphemeral opcode into an OP_Noop. +** +** WHERE_DISTINCT_ORDERED: +** +** The ephemeral table is not needed. But we do need register +** iVal to be initialized to NULL. So change the OP_OpenEphemeral +** into an OP_Null on the iVal register. +*/ +static void fixDistinctOpenEph( + Parse *pParse, /* Parsing and code generating context */ + int eTnctType, /* WHERE_DISTINCT_* value */ + int iVal, /* Value returned by codeDistinct() */ + int iOpenEphAddr /* Address of OP_OpenEphemeral instruction for iTab */ +){ + if( pParse->nErr==0 + && (eTnctType==WHERE_DISTINCT_UNIQUE || eTnctType==WHERE_DISTINCT_ORDERED) + ){ + Vdbe *v = pParse->pVdbe; + sqlite3VdbeChangeToNoop(v, iOpenEphAddr); + if( sqlite3VdbeGetOp(v, iOpenEphAddr+1)->opcode==OP_Explain ){ + sqlite3VdbeChangeToNoop(v, iOpenEphAddr+1); + } + if( eTnctType==WHERE_DISTINCT_ORDERED ){ + /* Change the OP_OpenEphemeral to an OP_Null that sets the MEM_Cleared + ** bit on the first register of the previous value. This will cause the + ** OP_Ne added in codeDistinct() to always fail on the first iteration of + ** the loop even if the first row is all NULLs. */ + VdbeOp *pOp = sqlite3VdbeGetOp(v, iOpenEphAddr); + pOp->opcode = OP_Null; + pOp->p1 = 1; + pOp->p2 = iVal; + } + } +} + +#ifdef SQLITE_ENABLE_SORTER_REFERENCES +/* +** This function is called as part of inner-loop generation for a SELECT +** statement with an ORDER BY that is not optimized by an index. It +** determines the expressions, if any, that the sorter-reference +** optimization should be used for. The sorter-reference optimization +** is used for SELECT queries like: +** +** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10 +** +** If the optimization is used for expression "bigblob", then instead of +** storing values read from that column in the sorter records, the PK of +** the row from table t1 is stored instead. Then, as records are extracted from +** the sorter to return to the user, the required value of bigblob is +** retrieved directly from table t1. If the values are very large, this +** can be more efficient than storing them directly in the sorter records. +** +** The ExprList_item.bSorterRef flag is set for each expression in pEList +** for which the sorter-reference optimization should be enabled. +** Additionally, the pSort->aDefer[] array is populated with entries +** for all cursors required to evaluate all selected expressions. Finally. +** output variable (*ppExtra) is set to an expression list containing +** expressions for all extra PK values that should be stored in the +** sorter records. +*/ +static void selectExprDefer( + Parse *pParse, /* Leave any error here */ + SortCtx *pSort, /* Sorter context */ + ExprList *pEList, /* Expressions destined for sorter */ + ExprList **ppExtra /* Expressions to append to sorter record */ +){ + int i; + int nDefer = 0; + ExprList *pExtra = 0; + for(i=0; inExpr; i++){ + struct ExprList_item *pItem = &pEList->a[i]; + if( pItem->u.x.iOrderByCol==0 ){ + Expr *pExpr = pItem->pExpr; + Table *pTab = pExpr->y.pTab; + if( pExpr->op==TK_COLUMN && pExpr->iColumn>=0 && pTab && !IsVirtual(pTab) + && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF) + ){ + int j; + for(j=0; jaDefer[j].iCsr==pExpr->iTable ) break; + } + if( j==nDefer ){ + if( nDefer==ArraySize(pSort->aDefer) ){ + continue; + }else{ + int nKey = 1; + int k; + Index *pPk = 0; + if( !HasRowid(pTab) ){ + pPk = sqlite3PrimaryKeyIndex(pTab); + nKey = pPk->nKeyCol; + } + for(k=0; kiTable = pExpr->iTable; + pNew->y.pTab = pExpr->y.pTab; + pNew->iColumn = pPk ? pPk->aiColumn[k] : -1; + pExtra = sqlite3ExprListAppend(pParse, pExtra, pNew); + } + } + pSort->aDefer[nDefer].pTab = pExpr->y.pTab; + pSort->aDefer[nDefer].iCsr = pExpr->iTable; + pSort->aDefer[nDefer].nKey = nKey; + nDefer++; + } + } + pItem->bSorterRef = 1; + } + } + } + pSort->nDefer = (u8)nDefer; + *ppExtra = pExtra; +} +#endif + +/* +** This routine generates the code for the inside of the inner loop +** of a SELECT. +** +** If srcTab is negative, then the p->pEList expressions +** are evaluated in order to get the data for this row. If srcTab is +** zero or more, then data is pulled from srcTab and p->pEList is used only +** to get the number of columns and the collation sequence for each column. +*/ +static void selectInnerLoop( + Parse *pParse, /* The parser context */ + Select *p, /* The complete select statement being coded */ + int srcTab, /* Pull data from this table if non-negative */ + SortCtx *pSort, /* If not NULL, info on how to process ORDER BY */ + DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ + SelectDest *pDest, /* How to dispose of the results */ + int iContinue, /* Jump here to continue with next row */ + int iBreak /* Jump here to break out of the inner loop */ +){ + Vdbe *v = pParse->pVdbe; + int i; + int hasDistinct; /* True if the DISTINCT keyword is present */ + int eDest = pDest->eDest; /* How to dispose of results */ + int iParm = pDest->iSDParm; /* First argument to disposal method */ + int nResultCol; /* Number of result columns */ + int nPrefixReg = 0; /* Number of extra registers before regResult */ + RowLoadInfo sRowLoadInfo; /* Info for deferred row loading */ + + /* Usually, regResult is the first cell in an array of memory cells + ** containing the current result row. In this case regOrig is set to the + ** same value. However, if the results are being sent to the sorter, the + ** values for any expressions that are also part of the sort-key are omitted + ** from this array. In this case regOrig is set to zero. */ + int regResult; /* Start of memory holding current results */ + int regOrig; /* Start of memory holding full result (or 0) */ + + assert( v ); + assert( p->pEList!=0 ); + hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; + if( pSort && pSort->pOrderBy==0 ) pSort = 0; + if( pSort==0 && !hasDistinct ){ + assert( iContinue!=0 ); + codeOffset(v, p->iOffset, iContinue); + } + + /* Pull the requested columns. + */ + nResultCol = p->pEList->nExpr; + + if( pDest->iSdst==0 ){ + if( pSort ){ + nPrefixReg = pSort->pOrderBy->nExpr; + if( !(pSort->sortFlags & SORTFLAG_UseSorter) ) nPrefixReg++; + pParse->nMem += nPrefixReg; + } + pDest->iSdst = pParse->nMem+1; + pParse->nMem += nResultCol; + }else if( pDest->iSdst+nResultCol > pParse->nMem ){ + /* This is an error condition that can result, for example, when a SELECT + ** on the right-hand side of an INSERT contains more result columns than + ** there are columns in the table on the left. The error will be caught + ** and reported later. But we need to make sure enough memory is allocated + ** to avoid other spurious errors in the meantime. */ + pParse->nMem += nResultCol; + } + pDest->nSdst = nResultCol; + regOrig = regResult = pDest->iSdst; + if( srcTab>=0 ){ + for(i=0; ipEList->a[i].zEName)); + } + }else if( eDest!=SRT_Exists ){ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + ExprList *pExtra = 0; +#endif + /* If the destination is an EXISTS(...) expression, the actual + ** values returned by the SELECT are not required. + */ + u8 ecelFlags; /* "ecel" is an abbreviation of "ExprCodeExprList" */ + ExprList *pEList; + if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){ + ecelFlags = SQLITE_ECEL_DUP; + }else{ + ecelFlags = 0; + } + if( pSort && hasDistinct==0 && eDest!=SRT_EphemTab && eDest!=SRT_Table ){ + /* For each expression in p->pEList that is a copy of an expression in + ** the ORDER BY clause (pSort->pOrderBy), set the associated + ** iOrderByCol value to one more than the index of the ORDER BY + ** expression within the sort-key that pushOntoSorter() will generate. + ** This allows the p->pEList field to be omitted from the sorted record, + ** saving space and CPU cycles. */ + ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF); + + for(i=pSort->nOBSat; ipOrderBy->nExpr; i++){ + int j; + if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){ + p->pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat; + } + } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + selectExprDefer(pParse, pSort, p->pEList, &pExtra); + if( pExtra && pParse->db->mallocFailed==0 ){ + /* If there are any extra PK columns to add to the sorter records, + ** allocate extra memory cells and adjust the OpenEphemeral + ** instruction to account for the larger records. This is only + ** required if there are one or more WITHOUT ROWID tables with + ** composite primary keys in the SortCtx.aDefer[] array. */ + VdbeOp *pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + pOp->p2 += (pExtra->nExpr - pSort->nDefer); + pOp->p4.pKeyInfo->nAllField += (pExtra->nExpr - pSort->nDefer); + pParse->nMem += pExtra->nExpr; + } +#endif + + /* Adjust nResultCol to account for columns that are omitted + ** from the sorter by the optimizations in this branch */ + pEList = p->pEList; + for(i=0; inExpr; i++){ + if( pEList->a[i].u.x.iOrderByCol>0 +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + || pEList->a[i].bSorterRef +#endif + ){ + nResultCol--; + regOrig = 0; + } + } + + testcase( regOrig ); + testcase( eDest==SRT_Set ); + testcase( eDest==SRT_Mem ); + testcase( eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); + assert( eDest==SRT_Set || eDest==SRT_Mem + || eDest==SRT_Coroutine || eDest==SRT_Output + || eDest==SRT_Upfrom ); + } + sRowLoadInfo.regResult = regResult; + sRowLoadInfo.ecelFlags = ecelFlags; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + sRowLoadInfo.pExtra = pExtra; + sRowLoadInfo.regExtraResult = regResult + nResultCol; + if( pExtra ) nResultCol += pExtra->nExpr; +#endif + if( p->iLimit + && (ecelFlags & SQLITE_ECEL_OMITREF)!=0 + && nPrefixReg>0 + ){ + assert( pSort!=0 ); + assert( hasDistinct==0 ); + pSort->pDeferredRowLoad = &sRowLoadInfo; + regOrig = 0; + }else{ + innerLoopLoadRow(pParse, p, &sRowLoadInfo); + } + } + + /* If the DISTINCT keyword was present on the SELECT statement + ** and this row has been seen before, then do not make this row + ** part of the result. + */ + if( hasDistinct ){ + int eType = pDistinct->eTnctType; + int iTab = pDistinct->tabTnct; + assert( nResultCol==p->pEList->nExpr ); + iTab = codeDistinct(pParse, eType, iTab, iContinue, p->pEList, regResult); + fixDistinctOpenEph(pParse, eType, iTab, pDistinct->addrTnct); + if( pSort==0 ){ + codeOffset(v, p->iOffset, iContinue); + } + } + + switch( eDest ){ + /* In this mode, write each query result to the key of the temporary + ** table iParm. + */ +#ifndef SQLITE_OMIT_COMPOUND_SELECT + case SRT_Union: { + int r1; + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + + /* Construct a record from the query result, but instead of + ** saving that record, use it as a key to delete elements from + ** the temporary table iParm. + */ + case SRT_Except: { + sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); + break; + } +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ + + /* Store the result as data using a unique key. + */ + case SRT_Fifo: + case SRT_DistFifo: + case SRT_Table: + case SRT_EphemTab: { + int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1); + testcase( eDest==SRT_Table ); + testcase( eDest==SRT_EphemTab ); + testcase( eDest==SRT_Fifo ); + testcase( eDest==SRT_DistFifo ); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); +#ifndef SQLITE_OMIT_CTE + if( eDest==SRT_DistFifo ){ + /* If the destination is DistFifo, then cursor (iParm+1) is open + ** on an ephemeral index. If the current row is already present + ** in the index, do not write it to the output. If not, add the + ** current row to the index and proceed with writing it to the + ** output table as well. */ + int addr = sqlite3VdbeCurrentAddr(v) + 4; + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); + VdbeCoverage(v); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm+1, r1,regResult,nResultCol); + assert( pSort==0 ); + } +#endif + if( pSort ){ + assert( regResult==regOrig ); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, regOrig, 1, nPrefixReg); + }else{ + int r2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, r2); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3ReleaseTempReg(pParse, r2); + } + sqlite3ReleaseTempRange(pParse, r1, nPrefixReg+1); + break; + } + + case SRT_Upfrom: { + if( pSort ){ + pushOntoSorter( + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + }else{ + int i2 = pDest->iSDParm2; + int r1 = sqlite3GetTempReg(pParse); + + /* If the UPDATE FROM join is an aggregate that matches no rows, it + ** might still be trying to return one row, because that is what + ** aggregates do. Don't record that empty row in the output table. */ + sqlite3VdbeAddOp2(v, OP_IsNull, regResult, iBreak); VdbeCoverage(v); + + sqlite3VdbeAddOp3(v, OP_MakeRecord, + regResult+(i2<0), nResultCol-(i2<0), r1); + if( i2<0 ){ + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, regResult); + }else{ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, i2); + } + } + break; + } + +#ifndef SQLITE_OMIT_SUBQUERY + /* If we are creating a set for an "expr IN (SELECT ...)" construct, + ** then there should be a single item on the stack. Write this + ** item into the set table with bogus data. + */ + case SRT_Set: { + if( pSort ){ + /* At first glance you would think we could optimize out the + ** ORDER BY in this case since the order of entries in the set + ** does not matter. But there might be a LIMIT clause, in which + ** case the order does matter */ + pushOntoSorter( + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + }else{ + int r1 = sqlite3GetTempReg(pParse); + assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, + r1, pDest->zAffSdst, nResultCol); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); + sqlite3ReleaseTempReg(pParse, r1); + } + break; + } + + + /* If any row exist in the result set, record that fact and abort. + */ + case SRT_Exists: { + sqlite3VdbeAddOp2(v, OP_Integer, 1, iParm); + /* The LIMIT clause will terminate the loop for us */ + break; + } + + /* If this is a scalar select that is part of an expression, then + ** store the results in the appropriate memory cell or array of + ** memory cells and break out of the scan loop. + */ + case SRT_Mem: { + if( pSort ){ + assert( nResultCol<=pDest->nSdst ); + pushOntoSorter( + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + }else{ + assert( nResultCol==pDest->nSdst ); + assert( regResult==iParm ); + /* The LIMIT clause will jump out of the loop for us */ + } + break; + } +#endif /* #ifndef SQLITE_OMIT_SUBQUERY */ + + case SRT_Coroutine: /* Send data to a co-routine */ + case SRT_Output: { /* Return the results */ + testcase( eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); + if( pSort ){ + pushOntoSorter(pParse, pSort, p, regResult, regOrig, nResultCol, + nPrefixReg); + }else if( eDest==SRT_Coroutine ){ + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); + }else{ + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol); + } + break; + } + +#ifndef SQLITE_OMIT_CTE + /* Write the results into a priority queue that is order according to + ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an + ** index with pSO->nExpr+2 columns. Build a key using pSO for the first + ** pSO->nExpr columns, then make sure all keys are unique by adding a + ** final OP_Sequence column. The last column is the record as a blob. + */ + case SRT_DistQueue: + case SRT_Queue: { + int nKey; + int r1, r2, r3; + int addrTest = 0; + ExprList *pSO; + pSO = pDest->pOrderBy; + assert( pSO ); + nKey = pSO->nExpr; + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempRange(pParse, nKey+2); + r3 = r2+nKey+1; + if( eDest==SRT_DistQueue ){ + /* If the destination is DistQueue, then cursor (iParm+1) is open + ** on a second ephemeral index that holds all values every previously + ** added to the queue. */ + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, + regResult, nResultCol); + VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); + if( eDest==SRT_DistQueue ){ + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + for(i=0; ia[i].u.x.iOrderByCol - 1, + r2+i); + } + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, r2, nKey+2); + if( addrTest ) sqlite3VdbeJumpHere(v, addrTest); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r2, nKey+2); + break; + } +#endif /* SQLITE_OMIT_CTE */ + + + +#if !defined(SQLITE_OMIT_TRIGGER) + /* Discard the results. This is used for SELECT statements inside + ** the body of a TRIGGER. The purpose of such selects is to call + ** user-defined functions that have side effects. We do not care + ** about the actual results of the select. + */ + default: { + assert( eDest==SRT_Discard ); + break; + } +#endif + } + + /* Jump to the end of the loop if the LIMIT is reached. Except, if + ** there is a sorter, in which case the sorter has already limited + ** the output for us. + */ + if( pSort==0 && p->iLimit ){ + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v); + } +} + +/* +** Allocate a KeyInfo object sufficient for an index of N key columns and +** X extra columns. +*/ +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ + int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*); + KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); + if( p ){ + p->aSortFlags = (u8*)&p->aColl[N+X]; + p->nKeyField = (u16)N; + p->nAllField = (u16)(N+X); + p->enc = ENC(db); + p->db = db; + p->nRef = 1; + memset(&p[1], 0, nExtra); + }else{ + sqlite3OomFault(db); + } + return p; +} + +/* +** Deallocate a KeyInfo object +*/ +SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo *p){ + if( p ){ + assert( p->nRef>0 ); + p->nRef--; + if( p->nRef==0 ) sqlite3DbFreeNN(p->db, p); + } +} + +/* +** Make a new pointer to a KeyInfo object +*/ +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoRef(KeyInfo *p){ + if( p ){ + assert( p->nRef>0 ); + p->nRef++; + } + return p; +} + +#ifdef SQLITE_DEBUG +/* +** Return TRUE if a KeyInfo object can be change. The KeyInfo object +** can only be changed if this is just a single reference to the object. +** +** This routine is used only inside of assert() statements. +*/ +SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } +#endif /* SQLITE_DEBUG */ + +/* +** Given an expression list, generate a KeyInfo structure that records +** the collating sequence for each expression in that expression list. +** +** If the ExprList is an ORDER BY or GROUP BY clause then the resulting +** KeyInfo structure is appropriate for initializing a virtual index to +** implement that clause. If the ExprList is the result set of a SELECT +** then the KeyInfo structure is appropriate for initializing a virtual +** index to implement a DISTINCT test. +** +** Space to hold the KeyInfo structure is obtained from malloc. The calling +** function is responsible for seeing that this structure is eventually +** freed. +*/ +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +){ + int nExpr; + KeyInfo *pInfo; + struct ExprList_item *pItem; + sqlite3 *db = pParse->db; + int i; + + nExpr = pList->nExpr; + pInfo = sqlite3KeyInfoAlloc(db, nExpr-iStart, nExtra+1); + if( pInfo ){ + assert( sqlite3KeyInfoIsWriteable(pInfo) ); + for(i=iStart, pItem=pList->a+iStart; iaColl[i-iStart] = sqlite3ExprNNCollSeq(pParse, pItem->pExpr); + pInfo->aSortFlags[i-iStart] = pItem->sortFlags; + } + } + return pInfo; +} + +/* +** Name of the connection operator, used for error messages. +*/ +SQLITE_PRIVATE const char *sqlite3SelectOpName(int id){ + char *z; + switch( id ){ + case TK_ALL: z = "UNION ALL"; break; + case TK_INTERSECT: z = "INTERSECT"; break; + case TK_EXCEPT: z = "EXCEPT"; break; + default: z = "UNION"; break; + } + return z; +} + +#ifndef SQLITE_OMIT_EXPLAIN +/* +** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function +** is a no-op. Otherwise, it adds a single row of output to the EQP result, +** where the caption is of the form: +** +** "USE TEMP B-TREE FOR xxx" +** +** where xxx is one of "DISTINCT", "ORDER BY" or "GROUP BY". Exactly which +** is determined by the zUsage argument. +*/ +static void explainTempTable(Parse *pParse, const char *zUsage){ + ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s", zUsage)); +} + +/* +** Assign expression b to lvalue a. A second, no-op, version of this macro +** is provided when SQLITE_OMIT_EXPLAIN is defined. This allows the code +** in sqlite3Select() to assign values to structure member variables that +** only exist if SQLITE_OMIT_EXPLAIN is not defined without polluting the +** code with #ifndef directives. +*/ +# define explainSetInteger(a, b) a = b + +#else +/* No-op versions of the explainXXX() functions and macros. */ +# define explainTempTable(y,z) +# define explainSetInteger(y,z) +#endif + + +/* +** If the inner loop was generated using a non-null pOrderBy argument, +** then the results were placed in a sorter. After the loop is terminated +** we need to run the sorter and output the results. The following +** routine generates the code needed to do that. +*/ +static void generateSortTail( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement */ + SortCtx *pSort, /* Information on the ORDER BY clause */ + int nColumn, /* Number of columns of data */ + SelectDest *pDest /* Write the sorted results here */ +){ + Vdbe *v = pParse->pVdbe; /* The prepared statement */ + int addrBreak = pSort->labelDone; /* Jump here to exit loop */ + int addrContinue = sqlite3VdbeMakeLabel(pParse);/* Jump here for next cycle */ + int addr; /* Top of output loop. Jump for Next. */ + int addrOnce = 0; + int iTab; + ExprList *pOrderBy = pSort->pOrderBy; + int eDest = pDest->eDest; + int iParm = pDest->iSDParm; + int regRow; + int regRowid; + int iCol; + int nKey; /* Number of key columns in sorter record */ + int iSortTab; /* Sorter cursor to read from */ + int i; + int bSeq; /* True if sorter record includes seq. no. */ + int nRefKey = 0; + struct ExprList_item *aOutEx = p->pEList->a; + + assert( addrBreak<0 ); + if( pSort->labelBkOut ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeGoto(v, addrBreak); + sqlite3VdbeResolveLabel(v, pSort->labelBkOut); + } + +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + /* Open any cursors needed for sorter-reference expressions */ + for(i=0; inDefer; i++){ + Table *pTab = pSort->aDefer[i].pTab; + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3OpenTable(pParse, pSort->aDefer[i].iCsr, iDb, pTab, OP_OpenRead); + nRefKey = MAX(nRefKey, pSort->aDefer[i].nKey); + } +#endif + + iTab = pSort->iECursor; + if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){ + regRowid = 0; + regRow = pDest->iSdst; + }else{ + regRowid = sqlite3GetTempReg(pParse); + if( eDest==SRT_EphemTab || eDest==SRT_Table ){ + regRow = sqlite3GetTempReg(pParse); + nColumn = 0; + }else{ + regRow = sqlite3GetTempRange(pParse, nColumn); + } + } + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + int regSortOut = ++pParse->nMem; + iSortTab = pParse->nTab++; + if( pSort->labelBkOut ){ + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, + nKey+1+nColumn+nRefKey); + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); + addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); + VdbeCoverage(v); + codeOffset(v, p->iOffset, addrContinue); + sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); + bSeq = 0; + }else{ + addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); + codeOffset(v, p->iOffset, addrContinue); + iSortTab = iTab; + bSeq = 1; + } + for(i=0, iCol=nKey+bSeq-1; inDefer ){ + int iKey = iCol+1; + int regKey = sqlite3GetTempRange(pParse, nRefKey); + + for(i=0; inDefer; i++){ + int iCsr = pSort->aDefer[i].iCsr; + Table *pTab = pSort->aDefer[i].pTab; + int nKey = pSort->aDefer[i].nKey; + + sqlite3VdbeAddOp1(v, OP_NullRow, iCsr); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey); + sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr, + sqlite3VdbeCurrentAddr(v)+1, regKey); + }else{ + int k; + int iJmp; + assert( sqlite3PrimaryKeyIndex(pTab)->nKeyCol==nKey ); + for(k=0; k=0; i--){ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( aOutEx[i].bSorterRef ){ + sqlite3ExprCode(pParse, aOutEx[i].pExpr, regRow+i); + }else +#endif + { + int iRead; + if( aOutEx[i].u.x.iOrderByCol ){ + iRead = aOutEx[i].u.x.iOrderByCol-1; + }else{ + iRead = iCol--; + } + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i); + VdbeComment((v, "%s", aOutEx[i].zEName)); + } + } + switch( eDest ){ + case SRT_Table: + case SRT_EphemTab: { + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, nKey+bSeq, regRow); + sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case SRT_Set: { + assert( nColumn==sqlite3Strlen30(pDest->zAffSdst) ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, nColumn, regRowid, + pDest->zAffSdst, nColumn); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, regRowid, regRow, nColumn); + break; + } + case SRT_Mem: { + /* The LIMIT clause will terminate the loop for us */ + break; + } +#endif + case SRT_Upfrom: { + int i2 = pDest->iSDParm2; + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord,regRow+(i2<0),nColumn-(i2<0),r1); + if( i2<0 ){ + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, regRow); + }else{ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regRow, i2); + } + break; + } + default: { + assert( eDest==SRT_Output || eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); + testcase( eDest==SRT_Coroutine ); + if( eDest==SRT_Output ){ + sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); + }else{ + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); + } + break; + } + } + if( regRowid ){ + if( eDest==SRT_Set ){ + sqlite3ReleaseTempRange(pParse, regRow, nColumn); + }else{ + sqlite3ReleaseTempReg(pParse, regRow); + } + sqlite3ReleaseTempReg(pParse, regRowid); + } + /* The bottom of the loop + */ + sqlite3VdbeResolveLabel(v, addrContinue); + if( pSort->sortFlags & SORTFLAG_UseSorter ){ + sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); + } + if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); + sqlite3VdbeResolveLabel(v, addrBreak); +} + +/* +** Return a pointer to a string containing the 'declaration type' of the +** expression pExpr. The string may be treated as static by the caller. +** +** Also try to estimate the size of the returned value and return that +** result in *pEstWidth. +** +** The declaration type is the exact datatype definition extracted from the +** original CREATE TABLE statement if the expression is a column. The +** declaration type for a ROWID field is INTEGER. Exactly when an expression +** is considered a column can be complex in the presence of subqueries. The +** result-set expression in all of the following SELECT statements is +** considered a column by this function. +** +** SELECT col FROM tbl; +** SELECT (SELECT col FROM tbl; +** SELECT (SELECT col FROM tbl); +** SELECT abc FROM (SELECT col AS abc FROM tbl); +** +** The declaration type for any expression other than a column is NULL. +** +** This routine has either 3 or 6 parameters depending on whether or not +** the SQLITE_ENABLE_COLUMN_METADATA compile-time option is used. +*/ +#ifdef SQLITE_ENABLE_COLUMN_METADATA +# define columnType(A,B,C,D,E) columnTypeImpl(A,B,C,D,E) +#else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ +# define columnType(A,B,C,D,E) columnTypeImpl(A,B) +#endif +static const char *columnTypeImpl( + NameContext *pNC, +#ifndef SQLITE_ENABLE_COLUMN_METADATA + Expr *pExpr +#else + Expr *pExpr, + const char **pzOrigDb, + const char **pzOrigTab, + const char **pzOrigCol +#endif +){ + char const *zType = 0; + int j; +#ifdef SQLITE_ENABLE_COLUMN_METADATA + char const *zOrigDb = 0; + char const *zOrigTab = 0; + char const *zOrigCol = 0; +#endif + + assert( pExpr!=0 ); + assert( pNC->pSrcList!=0 ); + switch( pExpr->op ){ + case TK_COLUMN: { + /* The expression is a column. Locate the table the column is being + ** extracted from in NameContext.pSrcList. This table may be real + ** database table or a subquery. + */ + Table *pTab = 0; /* Table structure column is extracted from */ + Select *pS = 0; /* Select the column is extracted from */ + int iCol = pExpr->iColumn; /* Index of column in pTab */ + while( pNC && !pTab ){ + SrcList *pTabList = pNC->pSrcList; + for(j=0;jnSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); + if( jnSrc ){ + pTab = pTabList->a[j].pTab; + pS = pTabList->a[j].pSelect; + }else{ + pNC = pNC->pNext; + } + } + + if( pTab==0 ){ + /* At one time, code such as "SELECT new.x" within a trigger would + ** cause this condition to run. Since then, we have restructured how + ** trigger code is generated and so this condition is no longer + ** possible. However, it can still be true for statements like + ** the following: + ** + ** CREATE TABLE t1(col INTEGER); + ** SELECT (SELECT t1.col) FROM FROM t1; + ** + ** when columnType() is called on the expression "t1.col" in the + ** sub-select. In this case, set the column type to NULL, even + ** though it should really be "INTEGER". + ** + ** This is not a problem, as the column type of "t1.col" is never + ** used. When columnType() is called on the expression + ** "(SELECT t1.col)", the correct type is returned (see the TK_SELECT + ** branch below. */ + break; + } + + assert( pTab && pExpr->y.pTab==pTab ); + if( pS ){ + /* The "table" is actually a sub-select or a view in the FROM clause + ** of the SELECT statement. Return the declaration type and origin + ** data for the result-set column of the sub-select. + */ + if( iColpEList->nExpr +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + && iCol>=0 +#else + && ALWAYS(iCol>=0) +#endif + ){ + /* If iCol is less than zero, then the expression requests the + ** rowid of the sub-select or view. This expression is legal (see + ** test case misc2.2.2) - it always evaluates to NULL. + */ + NameContext sNC; + Expr *p = pS->pEList->a[iCol].pExpr; + sNC.pSrcList = pS->pSrc; + sNC.pNext = pNC; + sNC.pParse = pNC->pParse; + zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol); + } + }else{ + /* A real table or a CTE table */ + assert( !pS ); +#ifdef SQLITE_ENABLE_COLUMN_METADATA + if( iCol<0 ) iCol = pTab->iPKey; + assert( iCol==XN_ROWID || (iCol>=0 && iColnCol) ); + if( iCol<0 ){ + zType = "INTEGER"; + zOrigCol = "rowid"; + }else{ + zOrigCol = pTab->aCol[iCol].zCnName; + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); + } + zOrigTab = pTab->zName; + if( pNC->pParse && pTab->pSchema ){ + int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema); + zOrigDb = pNC->pParse->db->aDb[iDb].zDbSName; + } +#else + assert( iCol==XN_ROWID || (iCol>=0 && iColnCol) ); + if( iCol<0 ){ + zType = "INTEGER"; + }else{ + zType = sqlite3ColumnType(&pTab->aCol[iCol],0); + } +#endif + } + break; + } +#ifndef SQLITE_OMIT_SUBQUERY + case TK_SELECT: { + /* The expression is a sub-select. Return the declaration type and + ** origin info for the single column in the result set of the SELECT + ** statement. + */ + NameContext sNC; + Select *pS = pExpr->x.pSelect; + Expr *p = pS->pEList->a[0].pExpr; + assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + sNC.pSrcList = pS->pSrc; + sNC.pNext = pNC; + sNC.pParse = pNC->pParse; + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); + break; + } +#endif + } + +#ifdef SQLITE_ENABLE_COLUMN_METADATA + if( pzOrigDb ){ + assert( pzOrigTab && pzOrigCol ); + *pzOrigDb = zOrigDb; + *pzOrigTab = zOrigTab; + *pzOrigCol = zOrigCol; + } +#endif + return zType; +} + +/* +** Generate code that will tell the VDBE the declaration types of columns +** in the result set. +*/ +static void generateColumnTypes( + Parse *pParse, /* Parser context */ + SrcList *pTabList, /* List of tables */ + ExprList *pEList /* Expressions defining the result set */ +){ +#ifndef SQLITE_OMIT_DECLTYPE + Vdbe *v = pParse->pVdbe; + int i; + NameContext sNC; + sNC.pSrcList = pTabList; + sNC.pParse = pParse; + sNC.pNext = 0; + for(i=0; inExpr; i++){ + Expr *p = pEList->a[i].pExpr; + const char *zType; +#ifdef SQLITE_ENABLE_COLUMN_METADATA + const char *zOrigDb = 0; + const char *zOrigTab = 0; + const char *zOrigCol = 0; + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); + + /* The vdbe must make its own copy of the column-type and other + ** column specific strings, in case the schema is reset before this + ** virtual machine is deleted. + */ + sqlite3VdbeSetColName(v, i, COLNAME_DATABASE, zOrigDb, SQLITE_TRANSIENT); + sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, SQLITE_TRANSIENT); + sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, SQLITE_TRANSIENT); +#else + zType = columnType(&sNC, p, 0, 0, 0); +#endif + sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); + } +#endif /* !defined(SQLITE_OMIT_DECLTYPE) */ +} + + +/* +** Compute the column names for a SELECT statement. +** +** The only guarantee that SQLite makes about column names is that if the +** column has an AS clause assigning it a name, that will be the name used. +** That is the only documented guarantee. However, countless applications +** developed over the years have made baseless assumptions about column names +** and will break if those assumptions changes. Hence, use extreme caution +** when modifying this routine to avoid breaking legacy. +** +** See Also: sqlite3ColumnsFromExprList() +** +** The PRAGMA short_column_names and PRAGMA full_column_names settings are +** deprecated. The default setting is short=ON, full=OFF. 99.9% of all +** applications should operate this way. Nevertheless, we need to support the +** other modes for legacy: +** +** short=OFF, full=OFF: Column name is the text of the expression has it +** originally appears in the SELECT statement. In +** other words, the zSpan of the result expression. +** +** short=ON, full=OFF: (This is the default setting). If the result +** refers directly to a table column, then the +** result column name is just the table column +** name: COLUMN. Otherwise use zSpan. +** +** full=ON, short=ANY: If the result refers directly to a table column, +** then the result column name with the table name +** prefix, ex: TABLE.COLUMN. Otherwise use zSpan. +*/ +SQLITE_PRIVATE void sqlite3GenerateColumnNames( + Parse *pParse, /* Parser context */ + Select *pSelect /* Generate column names for this SELECT statement */ +){ + Vdbe *v = pParse->pVdbe; + int i; + Table *pTab; + SrcList *pTabList; + ExprList *pEList; + sqlite3 *db = pParse->db; + int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */ + int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */ + +#ifndef SQLITE_OMIT_EXPLAIN + /* If this is an EXPLAIN, skip this step */ + if( pParse->explain ){ + return; + } +#endif + + if( pParse->colNamesSet ) return; + /* Column names are determined by the left-most term of a compound select */ + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + SELECTTRACE(1,pParse,pSelect,("generating column names\n")); + pTabList = pSelect->pSrc; + pEList = pSelect->pEList; + assert( v!=0 ); + assert( pTabList!=0 ); + pParse->colNamesSet = 1; + fullName = (db->flags & SQLITE_FullColNames)!=0; + srcName = (db->flags & SQLITE_ShortColNames)!=0 || fullName; + sqlite3VdbeSetNumCols(v, pEList->nExpr); + for(i=0; inExpr; i++){ + Expr *p = pEList->a[i].pExpr; + + assert( p!=0 ); + assert( p->op!=TK_AGG_COLUMN ); /* Agg processing has not run yet */ + assert( p->op!=TK_COLUMN || p->y.pTab!=0 ); /* Covering idx not yet coded */ + if( pEList->a[i].zEName && pEList->a[i].eEName==ENAME_NAME ){ + /* An AS clause always takes first priority */ + char *zName = pEList->a[i].zEName; + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); + }else if( srcName && p->op==TK_COLUMN ){ + char *zCol; + int iCol = p->iColumn; + pTab = p->y.pTab; + assert( pTab!=0 ); + if( iCol<0 ) iCol = pTab->iPKey; + assert( iCol==-1 || (iCol>=0 && iColnCol) ); + if( iCol<0 ){ + zCol = "rowid"; + }else{ + zCol = pTab->aCol[iCol].zCnName; + } + if( fullName ){ + char *zName = 0; + zName = sqlite3MPrintf(db, "%s.%s", pTab->zName, zCol); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_DYNAMIC); + }else{ + sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT); + } + }else{ + const char *z = pEList->a[i].zEName; + z = z==0 ? sqlite3MPrintf(db, "column%d", i+1) : sqlite3DbStrDup(db, z); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, z, SQLITE_DYNAMIC); + } + } + generateColumnTypes(pParse, pTabList, pEList); +} + +/* +** Given an expression list (which is really the list of expressions +** that form the result set of a SELECT statement) compute appropriate +** column names for a table that would hold the expression list. +** +** All column names will be unique. +** +** Only the column names are computed. Column.zType, Column.zColl, +** and other fields of Column are zeroed. +** +** Return SQLITE_OK on success. If a memory allocation error occurs, +** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. +** +** The only guarantee that SQLite makes about column names is that if the +** column has an AS clause assigning it a name, that will be the name used. +** That is the only documented guarantee. However, countless applications +** developed over the years have made baseless assumptions about column names +** and will break if those assumptions changes. Hence, use extreme caution +** when modifying this routine to avoid breaking legacy. +** +** See Also: sqlite3GenerateColumnNames() +*/ +SQLITE_PRIVATE int sqlite3ColumnsFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pEList, /* Expr list from which to derive column names */ + i16 *pnCol, /* Write the number of columns here */ + Column **paCol /* Write the new column list here */ +){ + sqlite3 *db = pParse->db; /* Database connection */ + int i, j; /* Loop counters */ + u32 cnt; /* Index added to make the name unique */ + Column *aCol, *pCol; /* For looping over result columns */ + int nCol; /* Number of columns in the result set */ + char *zName; /* Column name */ + int nName; /* Size of name in zName[] */ + Hash ht; /* Hash table of column names */ + Table *pTab; + + sqlite3HashInit(&ht); + if( pEList ){ + nCol = pEList->nExpr; + aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); + testcase( aCol==0 ); + if( NEVER(nCol>32767) ) nCol = 32767; + }else{ + nCol = 0; + aCol = 0; + } + assert( nCol==(i16)nCol ); + *pnCol = nCol; + *paCol = aCol; + + for(i=0, pCol=aCol; imallocFailed; i++, pCol++){ + /* Get an appropriate name for the column + */ + if( (zName = pEList->a[i].zEName)!=0 && pEList->a[i].eEName==ENAME_NAME ){ + /* If the column contains an "AS " phrase, use as the name */ + }else{ + Expr *pColExpr = sqlite3ExprSkipCollateAndLikely(pEList->a[i].pExpr); + while( ALWAYS(pColExpr!=0) && pColExpr->op==TK_DOT ){ + pColExpr = pColExpr->pRight; + assert( pColExpr!=0 ); + } + if( pColExpr->op==TK_COLUMN && (pTab = pColExpr->y.pTab)!=0 ){ + /* For columns use the column name name */ + int iCol = pColExpr->iColumn; + if( iCol<0 ) iCol = pTab->iPKey; + zName = iCol>=0 ? pTab->aCol[iCol].zCnName : "rowid"; + }else if( pColExpr->op==TK_ID ){ + assert( !ExprHasProperty(pColExpr, EP_IntValue) ); + zName = pColExpr->u.zToken; + }else{ + /* Use the original text of the column expression as its name */ + zName = pEList->a[i].zEName; + } + } + if( zName && !sqlite3IsTrueOrFalse(zName) ){ + zName = sqlite3DbStrDup(db, zName); + }else{ + zName = sqlite3MPrintf(db,"column%d",i+1); + } + + /* Make sure the column name is unique. If the name is not unique, + ** append an integer to the name so that it becomes unique. + */ + cnt = 0; + while( zName && sqlite3HashFind(&ht, zName)!=0 ){ + nName = sqlite3Strlen30(zName); + if( nName>0 ){ + for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){} + if( zName[j]==':' ) nName = j; + } + zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); + if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt); + } + pCol->zCnName = zName; + pCol->hName = sqlite3StrIHash(zName); + sqlite3ColumnPropertiesFromName(0, pCol); + if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){ + sqlite3OomFault(db); + } + } + sqlite3HashClear(&ht); + if( db->mallocFailed ){ + for(j=0; jdb; + NameContext sNC; + Column *pCol; + CollSeq *pColl; + int i; + Expr *p; + struct ExprList_item *a; + + assert( pSelect!=0 ); + assert( (pSelect->selFlags & SF_Resolved)!=0 ); + assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed ); + if( db->mallocFailed ) return; + memset(&sNC, 0, sizeof(sNC)); + sNC.pSrcList = pSelect->pSrc; + a = pSelect->pEList->a; + for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ + const char *zType; + int n, m; + pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); + p = a[i].pExpr; + zType = columnType(&sNC, p, 0, 0, 0); + /* pCol->szEst = ... // Column size est for SELECT tables never used */ + pCol->affinity = sqlite3ExprAffinity(p); + if( zType ){ + m = sqlite3Strlen30(zType); + n = sqlite3Strlen30(pCol->zCnName); + pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2); + if( pCol->zCnName ){ + memcpy(&pCol->zCnName[n+1], zType, m+1); + pCol->colFlags |= COLFLAG_HASTYPE; + } + } + if( pCol->affinity<=SQLITE_AFF_NONE ) pCol->affinity = aff; + pColl = sqlite3ExprCollSeq(pParse, p); + if( pColl ){ + assert( pTab->pIndex==0 ); + sqlite3ColumnSetColl(db, pCol, pColl->zName); + } + } + pTab->szTabRow = 1; /* Any non-zero value works */ +} + +/* +** Given a SELECT statement, generate a Table structure that describes +** the result set of that SELECT. +*/ +SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect, char aff){ + Table *pTab; + sqlite3 *db = pParse->db; + u64 savedFlags; + + savedFlags = db->flags; + db->flags &= ~(u64)SQLITE_FullColNames; + db->flags |= SQLITE_ShortColNames; + sqlite3SelectPrep(pParse, pSelect, 0); + db->flags = savedFlags; + if( pParse->nErr ) return 0; + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + pTab = sqlite3DbMallocZero(db, sizeof(Table) ); + if( pTab==0 ){ + return 0; + } + pTab->nTabRef = 1; + pTab->zName = 0; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect, aff); + pTab->iPKey = -1; + if( db->mallocFailed ){ + sqlite3DeleteTable(db, pTab); + return 0; + } + return pTab; +} + +/* +** Get a VDBE for the given parser context. Create a new one if necessary. +** If an error occurs, return NULL and leave a message in pParse. +*/ +SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ + if( pParse->pVdbe ){ + return pParse->pVdbe; + } + if( pParse->pToplevel==0 + && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) + ){ + pParse->okConstFactor = 1; + } + return sqlite3VdbeCreate(pParse); +} + + +/* +** Compute the iLimit and iOffset fields of the SELECT based on the +** pLimit expressions. pLimit->pLeft and pLimit->pRight hold the expressions +** that appear in the original SQL statement after the LIMIT and OFFSET +** keywords. Or NULL if those keywords are omitted. iLimit and iOffset +** are the integer memory register numbers for counters used to compute +** the limit and offset. If there is no limit and/or offset, then +** iLimit and iOffset are negative. +** +** This routine changes the values of iLimit and iOffset only if +** a limit or offset is defined by pLimit->pLeft and pLimit->pRight. iLimit +** and iOffset should have been preset to appropriate default values (zero) +** prior to calling this routine. +** +** The iOffset register (if it exists) is initialized to the value +** of the OFFSET. The iLimit register is initialized to LIMIT. Register +** iOffset+1 is initialized to LIMIT+OFFSET. +** +** Only if pLimit->pLeft!=0 do the limit registers get +** redefined. The UNION ALL operator uses this property to force +** the reuse of the same limit and offset registers across multiple +** SELECT statements. +*/ +static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ + Vdbe *v = 0; + int iLimit = 0; + int iOffset; + int n; + Expr *pLimit = p->pLimit; + + if( p->iLimit ) return; + + /* + ** "LIMIT -1" always shows all rows. There is some + ** controversy about what the correct behavior should be. + ** The current implementation interprets "LIMIT 0" to mean + ** no rows. + */ + if( pLimit ){ + assert( pLimit->op==TK_LIMIT ); + assert( pLimit->pLeft!=0 ); + p->iLimit = iLimit = ++pParse->nMem; + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); + if( sqlite3ExprIsInteger(pLimit->pLeft, &n) ){ + sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); + VdbeComment((v, "LIMIT counter")); + if( n==0 ){ + sqlite3VdbeGoto(v, iBreak); + }else if( n>=0 && p->nSelectRow>sqlite3LogEst((u64)n) ){ + p->nSelectRow = sqlite3LogEst((u64)n); + p->selFlags |= SF_FixedLimit; + } + }else{ + sqlite3ExprCode(pParse, pLimit->pLeft, iLimit); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v); + VdbeComment((v, "LIMIT counter")); + sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, iBreak); VdbeCoverage(v); + } + if( pLimit->pRight ){ + p->iOffset = iOffset = ++pParse->nMem; + pParse->nMem++; /* Allocate an extra register for limit+offset */ + sqlite3ExprCode(pParse, pLimit->pRight, iOffset); + sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); + VdbeComment((v, "OFFSET counter")); + sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset); + VdbeComment((v, "LIMIT+OFFSET")); + } + } +} + +#ifndef SQLITE_OMIT_COMPOUND_SELECT +/* +** Return the appropriate collating sequence for the iCol-th column of +** the result set for the compound-select statement "p". Return NULL if +** the column has no default collating sequence. +** +** The collating sequence for the compound select is taken from the +** left-most term of the select that has a collating sequence. +*/ +static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ + CollSeq *pRet; + if( p->pPrior ){ + pRet = multiSelectCollSeq(pParse, p->pPrior, iCol); + }else{ + pRet = 0; + } + assert( iCol>=0 ); + /* iCol must be less than p->pEList->nExpr. Otherwise an error would + ** have been thrown during name resolution and we would not have gotten + ** this far */ + if( pRet==0 && ALWAYS(iColpEList->nExpr) ){ + pRet = sqlite3ExprCollSeq(pParse, p->pEList->a[iCol].pExpr); + } + return pRet; +} + +/* +** The select statement passed as the second parameter is a compound SELECT +** with an ORDER BY clause. This function allocates and returns a KeyInfo +** structure suitable for implementing the ORDER BY. +** +** Space to hold the KeyInfo structure is obtained from malloc. The calling +** function is responsible for ensuring that this structure is eventually +** freed. +*/ +static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ + ExprList *pOrderBy = p->pOrderBy; + int nOrderBy = p->pOrderBy->nExpr; + sqlite3 *db = pParse->db; + KeyInfo *pRet = sqlite3KeyInfoAlloc(db, nOrderBy+nExtra, 1); + if( pRet ){ + int i; + for(i=0; ia[i]; + Expr *pTerm = pItem->pExpr; + CollSeq *pColl; + + if( pTerm->flags & EP_Collate ){ + pColl = sqlite3ExprCollSeq(pParse, pTerm); + }else{ + pColl = multiSelectCollSeq(pParse, p, pItem->u.x.iOrderByCol-1); + if( pColl==0 ) pColl = db->pDfltColl; + pOrderBy->a[i].pExpr = + sqlite3ExprAddCollateString(pParse, pTerm, pColl->zName); + } + assert( sqlite3KeyInfoIsWriteable(pRet) ); + pRet->aColl[i] = pColl; + pRet->aSortFlags[i] = pOrderBy->a[i].sortFlags; + } + } + + return pRet; +} + +#ifndef SQLITE_OMIT_CTE +/* +** This routine generates VDBE code to compute the content of a WITH RECURSIVE +** query of the form: +** +** AS ( UNION [ALL] ) +** \___________/ \_______________/ +** p->pPrior p +** +** +** There is exactly one reference to the recursive-table in the FROM clause +** of recursive-query, marked with the SrcList->a[].fg.isRecursive flag. +** +** The setup-query runs once to generate an initial set of rows that go +** into a Queue table. Rows are extracted from the Queue table one by +** one. Each row extracted from Queue is output to pDest. Then the single +** extracted row (now in the iCurrent table) becomes the content of the +** recursive-table for a recursive-query run. The output of the recursive-query +** is added back into the Queue table. Then another row is extracted from Queue +** and the iteration continues until the Queue table is empty. +** +** If the compound query operator is UNION then no duplicate rows are ever +** inserted into the Queue table. The iDistinct table keeps a copy of all rows +** that have ever been inserted into Queue and causes duplicates to be +** discarded. If the operator is UNION ALL, then duplicates are allowed. +** +** If the query has an ORDER BY, then entries in the Queue table are kept in +** ORDER BY order and the first entry is extracted for each cycle. Without +** an ORDER BY, the Queue table is just a FIFO. +** +** If a LIMIT clause is provided, then the iteration stops after LIMIT rows +** have been output to pDest. A LIMIT of zero means to output no rows and a +** negative LIMIT means to output all rows. If there is also an OFFSET clause +** with a positive value, then the first OFFSET outputs are discarded rather +** than being sent to pDest. The LIMIT count does not begin until after OFFSET +** rows have been skipped. +*/ +static void generateWithRecursiveQuery( + Parse *pParse, /* Parsing context */ + Select *p, /* The recursive SELECT to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */ + int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */ + Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + Select *pSetup; /* The setup query */ + Select *pFirstRec; /* Left-most recursive term */ + int addrTop; /* Top of the loop */ + int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ + int iCurrent = 0; /* The Current table */ + int regCurrent; /* Register holding Current table */ + int iQueue; /* The Queue table */ + int iDistinct = 0; /* To ensure unique results if UNION */ + int eDest = SRT_Fifo; /* How to write to Queue */ + SelectDest destQueue; /* SelectDest targetting the Queue table */ + int i; /* Loop counter */ + int rc; /* Result code */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Expr *pLimit; /* Saved LIMIT and OFFSET */ + int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */ + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ){ + sqlite3ErrorMsg(pParse, "cannot use window functions in recursive queries"); + return; + } +#endif + + /* Obtain authorization to do a recursive query */ + if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return; + + /* Process the LIMIT and OFFSET clauses, if they exist */ + addrBreak = sqlite3VdbeMakeLabel(pParse); + p->nSelectRow = 320; /* 4 billion rows */ + computeLimitRegisters(pParse, p, addrBreak); + pLimit = p->pLimit; + regLimit = p->iLimit; + regOffset = p->iOffset; + p->pLimit = 0; + p->iLimit = p->iOffset = 0; + pOrderBy = p->pOrderBy; + + /* Locate the cursor number of the Current table */ + for(i=0; ALWAYS(inSrc); i++){ + if( pSrc->a[i].fg.isRecursive ){ + iCurrent = pSrc->a[i].iCursor; + break; + } + } + + /* Allocate cursors numbers for Queue and Distinct. The cursor number for + ** the Distinct table must be exactly one greater than Queue in order + ** for the SRT_DistFifo and SRT_DistQueue destinations to work. */ + iQueue = pParse->nTab++; + if( p->op==TK_UNION ){ + eDest = pOrderBy ? SRT_DistQueue : SRT_DistFifo; + iDistinct = pParse->nTab++; + }else{ + eDest = pOrderBy ? SRT_Queue : SRT_Fifo; + } + sqlite3SelectDestInit(&destQueue, eDest, iQueue); + + /* Allocate cursors for Current, Queue, and Distinct. */ + regCurrent = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); + if( pOrderBy ){ + KeyInfo *pKeyInfo = multiSelectOrderByKeyInfo(pParse, p, 1); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, + (char*)pKeyInfo, P4_KEYINFO); + destQueue.pOrderBy = pOrderBy; + }else{ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); + } + VdbeComment((v, "Queue table")); + if( iDistinct ){ + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); + p->selFlags |= SF_UsesEphemeral; + } + + /* Detach the ORDER BY clause from the compound SELECT */ + p->pOrderBy = 0; + + /* Figure out how many elements of the compound SELECT are part of the + ** recursive query. Make sure no recursive elements use aggregate + ** functions. Mark the recursive elements as UNION ALL even if they + ** are really UNION because the distinctness will be enforced by the + ** iDistinct table. pFirstRec is left pointing to the left-most + ** recursive term of the CTE. + */ + for(pFirstRec=p; ALWAYS(pFirstRec!=0); pFirstRec=pFirstRec->pPrior){ + if( pFirstRec->selFlags & SF_Aggregate ){ + sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported"); + goto end_of_recursive_query; + } + pFirstRec->op = TK_ALL; + if( (pFirstRec->pPrior->selFlags & SF_Recursive)==0 ) break; + } + + /* Store the results of the setup-query in Queue. */ + pSetup = pFirstRec->pPrior; + pSetup->pNext = 0; + ExplainQueryPlan((pParse, 1, "SETUP")); + rc = sqlite3Select(pParse, pSetup, &destQueue); + pSetup->pNext = p; + if( rc ) goto end_of_recursive_query; + + /* Find the next row in the Queue and output that row */ + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); VdbeCoverage(v); + + /* Transfer the next row in Queue over to Current */ + sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ + if( pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Column, iQueue, pOrderBy->nExpr+1, regCurrent); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); + } + sqlite3VdbeAddOp1(v, OP_Delete, iQueue); + + /* Output the single row in Current */ + addrCont = sqlite3VdbeMakeLabel(pParse); + codeOffset(v, regOffset, addrCont); + selectInnerLoop(pParse, p, iCurrent, + 0, 0, pDest, addrCont, addrBreak); + if( regLimit ){ + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, regLimit, addrBreak); + VdbeCoverage(v); + } + sqlite3VdbeResolveLabel(v, addrCont); + + /* Execute the recursive SELECT taking the single row in Current as + ** the value for the recursive-table. Store the results in the Queue. + */ + pFirstRec->pPrior = 0; + ExplainQueryPlan((pParse, 1, "RECURSIVE STEP")); + sqlite3Select(pParse, p, &destQueue); + assert( pFirstRec->pPrior==0 ); + pFirstRec->pPrior = pSetup; + + /* Keep running the loop until the Queue is empty */ + sqlite3VdbeGoto(v, addrTop); + sqlite3VdbeResolveLabel(v, addrBreak); + +end_of_recursive_query: + sqlite3ExprListDelete(pParse->db, p->pOrderBy); + p->pOrderBy = pOrderBy; + p->pLimit = pLimit; + return; +} +#endif /* SQLITE_OMIT_CTE */ + +/* Forward references */ +static int multiSelectOrderBy( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +); + +/* +** Handle the special case of a compound-select that originates from a +** VALUES clause. By handling this as a special case, we avoid deep +** recursion, and thus do not need to enforce the SQLITE_LIMIT_COMPOUND_SELECT +** on a VALUES clause. +** +** Because the Select object originates from a VALUES clause: +** (1) There is no LIMIT or OFFSET or else there is a LIMIT of exactly 1 +** (2) All terms are UNION ALL +** (3) There is no ORDER BY clause +** +** The "LIMIT of exactly 1" case of condition (1) comes about when a VALUES +** clause occurs within scalar expression (ex: "SELECT (VALUES(1),(2),(3))"). +** The sqlite3CodeSubselect will have added the LIMIT 1 clause in tht case. +** Since the limit is exactly 1, we only need to evalutes the left-most VALUES. +*/ +static int multiSelectValues( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + int nRow = 1; + int rc = 0; + int bShowAll = p->pLimit==0; + assert( p->selFlags & SF_MultiValue ); + do{ + assert( p->selFlags & SF_Values ); + assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) ); + assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ) return -1; +#endif + if( p->pPrior==0 ) break; + assert( p->pPrior->pNext==p ); + p = p->pPrior; + nRow += bShowAll; + }while(1); + ExplainQueryPlan((pParse, 0, "SCAN %d CONSTANT ROW%s", nRow, + nRow==1 ? "" : "S")); + while( p ){ + selectInnerLoop(pParse, p, -1, 0, 0, pDest, 1, 1); + if( !bShowAll ) break; + p->nSelectRow = nRow; + p = p->pNext; + } + return rc; +} + +/* +** Return true if the SELECT statement which is known to be the recursive +** part of a recursive CTE still has its anchor terms attached. If the +** anchor terms have already been removed, then return false. +*/ +static int hasAnchor(Select *p){ + while( p && (p->selFlags & SF_Recursive)!=0 ){ p = p->pPrior; } + return p!=0; +} + +/* +** This routine is called to process a compound query form from +** two or more separate queries using UNION, UNION ALL, EXCEPT, or +** INTERSECT +** +** "p" points to the right-most of the two queries. the query on the +** left is p->pPrior. The left query could also be a compound query +** in which case this routine will be called recursively. +** +** The results of the total query are to be written into a destination +** of type eDest with parameter iParm. +** +** Example 1: Consider a three-way compound SQL statement. +** +** SELECT a FROM t1 UNION SELECT b FROM t2 UNION SELECT c FROM t3 +** +** This statement is parsed up as follows: +** +** SELECT c FROM t3 +** | +** `-----> SELECT b FROM t2 +** | +** `------> SELECT a FROM t1 +** +** The arrows in the diagram above represent the Select.pPrior pointer. +** So if this routine is called with p equal to the t3 query, then +** pPrior will be the t2 query. p->op will be TK_UNION in this case. +** +** Notice that because of the way SQLite parses compound SELECTs, the +** individual selects always group from left to right. +*/ +static int multiSelect( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + int rc = SQLITE_OK; /* Success code from a subroutine */ + Select *pPrior; /* Another SELECT immediately to our left */ + Vdbe *v; /* Generate code to this VDBE */ + SelectDest dest; /* Alternative data destination */ + Select *pDelete = 0; /* Chain of simple selects to delete */ + sqlite3 *db; /* Database connection */ + + /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only + ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. + */ + assert( p && p->pPrior ); /* Calling function guarantees this much */ + assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION ); + assert( p->selFlags & SF_Compound ); + db = pParse->db; + pPrior = p->pPrior; + dest = *pDest; + assert( pPrior->pOrderBy==0 ); + assert( pPrior->pLimit==0 ); + + v = sqlite3GetVdbe(pParse); + assert( v!=0 ); /* The VDBE already created by calling function */ + + /* Create the destination temporary table if necessary + */ + if( dest.eDest==SRT_EphemTab ){ + assert( p->pEList ); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, dest.iSDParm, p->pEList->nExpr); + dest.eDest = SRT_Table; + } + + /* Special handling for a compound-select that originates as a VALUES clause. + */ + if( p->selFlags & SF_MultiValue ){ + rc = multiSelectValues(pParse, p, &dest); + if( rc>=0 ) goto multi_select_end; + rc = SQLITE_OK; + } + + /* Make sure all SELECTs in the statement have the same number of elements + ** in their result sets. + */ + assert( p->pEList && pPrior->pEList ); + assert( p->pEList->nExpr==pPrior->pEList->nExpr ); + +#ifndef SQLITE_OMIT_CTE + if( (p->selFlags & SF_Recursive)!=0 && hasAnchor(p) ){ + generateWithRecursiveQuery(pParse, p, &dest); + }else +#endif + + /* Compound SELECTs that have an ORDER BY clause are handled separately. + */ + if( p->pOrderBy ){ + return multiSelectOrderBy(pParse, p, pDest); + }else{ + +#ifndef SQLITE_OMIT_EXPLAIN + if( pPrior->pPrior==0 ){ + ExplainQueryPlan((pParse, 1, "COMPOUND QUERY")); + ExplainQueryPlan((pParse, 1, "LEFT-MOST SUBQUERY")); + } +#endif + + /* Generate code for the left and right SELECT statements. + */ + switch( p->op ){ + case TK_ALL: { + int addr = 0; + int nLimit = 0; /* Initialize to suppress harmless compiler warning */ + assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; + pPrior->pLimit = p->pLimit; + SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL left...\n")); + rc = sqlite3Select(pParse, pPrior, &dest); + pPrior->pLimit = 0; + if( rc ){ + goto multi_select_end; + } + p->pPrior = 0; + p->iLimit = pPrior->iLimit; + p->iOffset = pPrior->iOffset; + if( p->iLimit ){ + addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); + VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_OffsetLimit, + p->iLimit, p->iOffset+1, p->iOffset); + } + } + ExplainQueryPlan((pParse, 1, "UNION ALL")); + SELECTTRACE(1, pParse, p, ("multiSelect UNION ALL right...\n")); + rc = sqlite3Select(pParse, p, &dest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + if( p->pLimit + && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit) + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) + ){ + p->nSelectRow = sqlite3LogEst((u64)nLimit); + } + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + } + break; + } + case TK_EXCEPT: + case TK_UNION: { + int unionTab; /* Cursor number of the temp table holding result */ + u8 op = 0; /* One of the SRT_ operations to apply to self */ + int priorOp; /* The SRT_ operation to apply to prior selects */ + Expr *pLimit; /* Saved values of p->nLimit */ + int addr; + SelectDest uniondest; + + testcase( p->op==TK_EXCEPT ); + testcase( p->op==TK_UNION ); + priorOp = SRT_Union; + if( dest.eDest==priorOp ){ + /* We can reuse a temporary table generated by a SELECT to our + ** right. + */ + assert( p->pLimit==0 ); /* Not allowed on leftward elements */ + unionTab = dest.iSDParm; + }else{ + /* We will need to create our own temporary table to hold the + ** intermediate results. + */ + unionTab = pParse->nTab++; + assert( p->pOrderBy==0 ); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + findRightmost(p)->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + } + + + /* Code the SELECT statements to our left + */ + assert( !pPrior->pOrderBy ); + sqlite3SelectDestInit(&uniondest, priorOp, unionTab); + SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); + rc = sqlite3Select(pParse, pPrior, &uniondest); + if( rc ){ + goto multi_select_end; + } + + /* Code the current SELECT statement + */ + if( p->op==TK_EXCEPT ){ + op = SRT_Except; + }else{ + assert( p->op==TK_UNION ); + op = SRT_Union; + } + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + uniondest.eDest = op; + ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", + sqlite3SelectOpName(p->op))); + SELECTTRACE(1, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); + rc = sqlite3Select(pParse, p, &uniondest); + testcase( rc!=SQLITE_OK ); + assert( p->pOrderBy==0 ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->pOrderBy = 0; + if( p->op==TK_UNION ){ + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + } + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + p->iLimit = 0; + p->iOffset = 0; + + /* Convert the data in the temporary table into whatever form + ** it is that we currently need. + */ + assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); + assert( p->pEList || db->mallocFailed ); + if( dest.eDest!=priorOp && db->mallocFailed==0 ){ + int iCont, iBreak, iStart; + iBreak = sqlite3VdbeMakeLabel(pParse); + iCont = sqlite3VdbeMakeLabel(pParse); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); + iStart = sqlite3VdbeCurrentAddr(v); + selectInnerLoop(pParse, p, unionTab, + 0, 0, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); + } + break; + } + default: assert( p->op==TK_INTERSECT ); { + int tab1, tab2; + int iCont, iBreak, iStart; + Expr *pLimit; + int addr; + SelectDest intersectdest; + int r1; + + /* INTERSECT is different from the others since it requires + ** two temporary tables. Hence it has its own case. Begin + ** by allocating the tables we will need. + */ + tab1 = pParse->nTab++; + tab2 = pParse->nTab++; + assert( p->pOrderBy==0 ); + + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + findRightmost(p)->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + + /* Code the SELECTs to our left into temporary table "tab1". + */ + sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); + SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT left...\n")); + rc = sqlite3Select(pParse, pPrior, &intersectdest); + if( rc ){ + goto multi_select_end; + } + + /* Code the current SELECT into temporary table "tab2" + */ + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); + assert( p->addrOpenEphm[1] == -1 ); + p->addrOpenEphm[1] = addr; + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + intersectdest.iSDParm = tab2; + ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", + sqlite3SelectOpName(p->op))); + SELECTTRACE(1, pParse, p, ("multiSelect INTERSECT right...\n")); + rc = sqlite3Select(pParse, p, &intersectdest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + if( p->nSelectRow>pPrior->nSelectRow ){ + p->nSelectRow = pPrior->nSelectRow; + } + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + + /* Generate code to take the intersection of the two temporary + ** tables. + */ + if( rc ) break; + assert( p->pEList ); + iBreak = sqlite3VdbeMakeLabel(pParse); + iCont = sqlite3VdbeMakeLabel(pParse); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); + r1 = sqlite3GetTempReg(pParse); + iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1); + sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, r1); + selectInnerLoop(pParse, p, tab1, + 0, 0, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); + sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); + break; + } + } + + #ifndef SQLITE_OMIT_EXPLAIN + if( p->pNext==0 ){ + ExplainQueryPlanPop(pParse); + } + #endif + } + if( pParse->nErr ) goto multi_select_end; + + /* Compute collating sequences used by + ** temporary tables needed to implement the compound select. + ** Attach the KeyInfo structure to all temporary tables. + ** + ** This section is run by the right-most SELECT statement only. + ** SELECT statements to the left always skip this part. The right-most + ** SELECT might also skip this part if it has no ORDER BY clause and + ** no temp tables are required. + */ + if( p->selFlags & SF_UsesEphemeral ){ + int i; /* Loop counter */ + KeyInfo *pKeyInfo; /* Collating sequence for the result set */ + Select *pLoop; /* For looping through SELECT statements */ + CollSeq **apColl; /* For looping through pKeyInfo->aColl[] */ + int nCol; /* Number of columns in result set */ + + assert( p->pNext==0 ); + nCol = p->pEList->nExpr; + pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); + if( !pKeyInfo ){ + rc = SQLITE_NOMEM_BKPT; + goto multi_select_end; + } + for(i=0, apColl=pKeyInfo->aColl; ipDfltColl; + } + } + + for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ + for(i=0; i<2; i++){ + int addr = pLoop->addrOpenEphm[i]; + if( addr<0 ){ + /* If [0] is unused then [1] is also unused. So we can + ** always safely abort as soon as the first unused slot is found */ + assert( pLoop->addrOpenEphm[1]<0 ); + break; + } + sqlite3VdbeChangeP2(v, addr, nCol); + sqlite3VdbeChangeP4(v, addr, (char*)sqlite3KeyInfoRef(pKeyInfo), + P4_KEYINFO); + pLoop->addrOpenEphm[i] = -1; + } + } + sqlite3KeyInfoUnref(pKeyInfo); + } + +multi_select_end: + pDest->iSdst = dest.iSdst; + pDest->nSdst = dest.nSdst; + if( pDelete ){ + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3SelectDelete, + pDelete); + } + return rc; +} +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ + +/* +** Error message for when two or more terms of a compound select have different +** size result sets. +*/ +SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ + if( p->selFlags & SF_Values ){ + sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); + }else{ + sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" + " do not have the same number of result columns", + sqlite3SelectOpName(p->op)); + } +} + +/* +** Code an output subroutine for a coroutine implementation of a +** SELECT statment. +** +** The data to be output is contained in pIn->iSdst. There are +** pIn->nSdst columns to be output. pDest is where the output should +** be sent. +** +** regReturn is the number of the register holding the subroutine +** return address. +** +** If regPrev>0 then it is the first register in a vector that +** records the previous output. mem[regPrev] is a flag that is false +** if there has been no previous output. If regPrev>0 then code is +** generated to suppress duplicates. pKeyInfo is used for comparing +** keys. +** +** If the LIMIT found in p->iLimit is reached, jump immediately to +** iBreak. +*/ +static int generateOutputSubroutine( + Parse *pParse, /* Parsing context */ + Select *p, /* The SELECT statement */ + SelectDest *pIn, /* Coroutine supplying data */ + SelectDest *pDest, /* Where to send the data */ + int regReturn, /* The return address register */ + int regPrev, /* Previous result register. No uniqueness if 0 */ + KeyInfo *pKeyInfo, /* For comparing with previous entry */ + int iBreak /* Jump here if we hit the LIMIT */ +){ + Vdbe *v = pParse->pVdbe; + int iContinue; + int addr; + + addr = sqlite3VdbeCurrentAddr(v); + iContinue = sqlite3VdbeMakeLabel(pParse); + + /* Suppress duplicates for UNION, EXCEPT, and INTERSECT + */ + if( regPrev ){ + int addr1, addr2; + addr1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); + addr2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, + (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, addr2+2, iContinue, addr2+2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); + sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); + } + if( pParse->db->mallocFailed ) return 0; + + /* Suppress the first OFFSET entries if there is an OFFSET clause + */ + codeOffset(v, p->iOffset, iContinue); + + assert( pDest->eDest!=SRT_Exists ); + assert( pDest->eDest!=SRT_Table ); + switch( pDest->eDest ){ + /* Store the result as data using a unique key. + */ + case SRT_EphemTab: { + int r1 = sqlite3GetTempReg(pParse); + int r2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1); + sqlite3VdbeAddOp2(v, OP_NewRowid, pDest->iSDParm, r2); + sqlite3VdbeAddOp3(v, OP_Insert, pDest->iSDParm, r1, r2); + sqlite3VdbeChangeP5(v, OPFLAG_APPEND); + sqlite3ReleaseTempReg(pParse, r2); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + +#ifndef SQLITE_OMIT_SUBQUERY + /* If we are creating a set for an "expr IN (SELECT ...)". + */ + case SRT_Set: { + int r1; + testcase( pIn->nSdst>1 ); + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, + r1, pDest->zAffSdst, pIn->nSdst); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1, + pIn->iSdst, pIn->nSdst); + sqlite3ReleaseTempReg(pParse, r1); + break; + } + + /* If this is a scalar select that is part of an expression, then + ** store the results in the appropriate memory cell and break out + ** of the scan loop. Note that the select might return multiple columns + ** if it is the RHS of a row-value IN operator. + */ + case SRT_Mem: { + testcase( pIn->nSdst>1 ); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, pIn->nSdst); + /* The LIMIT clause will jump out of the loop for us */ + break; + } +#endif /* #ifndef SQLITE_OMIT_SUBQUERY */ + + /* The results are stored in a sequence of registers + ** starting at pDest->iSdst. Then the co-routine yields. + */ + case SRT_Coroutine: { + if( pDest->iSdst==0 ){ + pDest->iSdst = sqlite3GetTempRange(pParse, pIn->nSdst); + pDest->nSdst = pIn->nSdst; + } + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSdst, pIn->nSdst); + sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); + break; + } + + /* If none of the above, then the result destination must be + ** SRT_Output. This routine is never called with any other + ** destination other than the ones handled above or SRT_Output. + ** + ** For SRT_Output, results are stored in a sequence of registers. + ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to + ** return the next row of result. + */ + default: { + assert( pDest->eDest==SRT_Output ); + sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iSdst, pIn->nSdst); + break; + } + } + + /* Jump to the end of the loop if the LIMIT is reached. + */ + if( p->iLimit ){ + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, p->iLimit, iBreak); VdbeCoverage(v); + } + + /* Generate the subroutine return + */ + sqlite3VdbeResolveLabel(v, iContinue); + sqlite3VdbeAddOp1(v, OP_Return, regReturn); + + return addr; +} + +/* +** Alternative compound select code generator for cases when there +** is an ORDER BY clause. +** +** We assume a query of the following form: +** +** ORDER BY +** +** is one of UNION ALL, UNION, EXCEPT, or INTERSECT. The idea +** is to code both and with the ORDER BY clause as +** co-routines. Then run the co-routines in parallel and merge the results +** into the output. In addition to the two coroutines (called selectA and +** selectB) there are 7 subroutines: +** +** outA: Move the output of the selectA coroutine into the output +** of the compound query. +** +** outB: Move the output of the selectB coroutine into the output +** of the compound query. (Only generated for UNION and +** UNION ALL. EXCEPT and INSERTSECT never output a row that +** appears only in B.) +** +** AltB: Called when there is data from both coroutines and AB. +** +** EofA: Called when data is exhausted from selectA. +** +** EofB: Called when data is exhausted from selectB. +** +** The implementation of the latter five subroutines depend on which +** is used: +** +** +** UNION ALL UNION EXCEPT INTERSECT +** ------------- ----------------- -------------- ----------------- +** AltB: outA, nextA outA, nextA outA, nextA nextA +** +** AeqB: outA, nextA nextA nextA outA, nextA +** +** AgtB: outB, nextB outB, nextB nextB nextB +** +** EofA: outB, nextB outB, nextB halt halt +** +** EofB: outA, nextA outA, nextA outA, nextA halt +** +** In the AltB, AeqB, and AgtB subroutines, an EOF on A following nextA +** causes an immediate jump to EofA and an EOF on B following nextB causes +** an immediate jump to EofB. Within EofA and EofB, and EOF on entry or +** following nextX causes a jump to the end of the select processing. +** +** Duplicate removal in the UNION, EXCEPT, and INTERSECT cases is handled +** within the output subroutine. The regPrev register set holds the previously +** output value. A comparison is made against this value and the output +** is skipped if the next results would be the same as the previous. +** +** The implementation plan is to implement the two coroutines and seven +** subroutines first, then put the control logic at the bottom. Like this: +** +** goto Init +** coA: coroutine for left query (A) +** coB: coroutine for right query (B) +** outA: output one row of A +** outB: output one row of B (UNION and UNION ALL only) +** EofA: ... +** EofB: ... +** AltB: ... +** AeqB: ... +** AgtB: ... +** Init: initialize coroutine registers +** yield coA +** if eof(A) goto EofA +** yield coB +** if eof(B) goto EofB +** Cmpr: Compare A, B +** Jump AltB, AeqB, AgtB +** End: ... +** +** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not +** actually called using Gosub and they do not Return. EofA and EofB loop +** until all data is exhausted then jump to the "end" labe. AltB, AeqB, +** and AgtB jump to either L2 or to one of EofA or EofB. +*/ +#ifndef SQLITE_OMIT_COMPOUND_SELECT +static int multiSelectOrderBy( + Parse *pParse, /* Parsing context */ + Select *p, /* The right-most of SELECTs to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + int i, j; /* Loop counters */ + Select *pPrior; /* Another SELECT immediately to our left */ + Vdbe *v; /* Generate code to this VDBE */ + SelectDest destA; /* Destination for coroutine A */ + SelectDest destB; /* Destination for coroutine B */ + int regAddrA; /* Address register for select-A coroutine */ + int regAddrB; /* Address register for select-B coroutine */ + int addrSelectA; /* Address of the select-A coroutine */ + int addrSelectB; /* Address of the select-B coroutine */ + int regOutA; /* Address register for the output-A subroutine */ + int regOutB; /* Address register for the output-B subroutine */ + int addrOutA; /* Address of the output-A subroutine */ + int addrOutB = 0; /* Address of the output-B subroutine */ + int addrEofA; /* Address of the select-A-exhausted subroutine */ + int addrEofA_noB; /* Alternate addrEofA if B is uninitialized */ + int addrEofB; /* Address of the select-B-exhausted subroutine */ + int addrAltB; /* Address of the AB subroutine */ + int regLimitA; /* Limit register for select-A */ + int regLimitB; /* Limit register for select-A */ + int regPrev; /* A range of registers to hold previous output */ + int savedLimit; /* Saved value of p->iLimit */ + int savedOffset; /* Saved value of p->iOffset */ + int labelCmpr; /* Label for the start of the merge algorithm */ + int labelEnd; /* Label for the end of the overall SELECT stmt */ + int addr1; /* Jump instructions that get retargetted */ + int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ + KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ + KeyInfo *pKeyMerge; /* Comparison information for merging rows */ + sqlite3 *db; /* Database connection */ + ExprList *pOrderBy; /* The ORDER BY clause */ + int nOrderBy; /* Number of terms in the ORDER BY clause */ + u32 *aPermute; /* Mapping from ORDER BY terms to result set columns */ + + assert( p->pOrderBy!=0 ); + assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */ + db = pParse->db; + v = pParse->pVdbe; + assert( v!=0 ); /* Already thrown the error if VDBE alloc failed */ + labelEnd = sqlite3VdbeMakeLabel(pParse); + labelCmpr = sqlite3VdbeMakeLabel(pParse); + + + /* Patch up the ORDER BY clause + */ + op = p->op; + pPrior = p->pPrior; + assert( pPrior->pOrderBy==0 ); + pOrderBy = p->pOrderBy; + assert( pOrderBy ); + nOrderBy = pOrderBy->nExpr; + + /* For operators other than UNION ALL we have to make sure that + ** the ORDER BY clause covers every term of the result set. Add + ** terms to the ORDER BY clause as necessary. + */ + if( op!=TK_ALL ){ + for(i=1; db->mallocFailed==0 && i<=p->pEList->nExpr; i++){ + struct ExprList_item *pItem; + for(j=0, pItem=pOrderBy->a; ju.x.iOrderByCol>0 ); + if( pItem->u.x.iOrderByCol==i ) break; + } + if( j==nOrderBy ){ + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + if( pNew==0 ) return SQLITE_NOMEM_BKPT; + pNew->flags |= EP_IntValue; + pNew->u.iValue = i; + p->pOrderBy = pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); + if( pOrderBy ) pOrderBy->a[nOrderBy++].u.x.iOrderByCol = (u16)i; + } + } + } + + /* Compute the comparison permutation and keyinfo that is used with + ** the permutation used to determine if the next + ** row of results comes from selectA or selectB. Also add explicit + ** collations to the ORDER BY clause terms so that when the subqueries + ** to the right and the left are evaluated, they use the correct + ** collation. + */ + aPermute = sqlite3DbMallocRawNN(db, sizeof(u32)*(nOrderBy + 1)); + if( aPermute ){ + struct ExprList_item *pItem; + aPermute[0] = nOrderBy; + for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){ + assert( pItem->u.x.iOrderByCol>0 ); + assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); + aPermute[i] = pItem->u.x.iOrderByCol - 1; + } + pKeyMerge = multiSelectOrderByKeyInfo(pParse, p, 1); + }else{ + pKeyMerge = 0; + } + + /* Reattach the ORDER BY clause to the query. + */ + p->pOrderBy = pOrderBy; + pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy, 0); + + /* Allocate a range of temporary registers and the KeyInfo needed + ** for the logic that removes duplicate result rows when the + ** operator is UNION, EXCEPT, or INTERSECT (but not UNION ALL). + */ + if( op==TK_ALL ){ + regPrev = 0; + }else{ + int nExpr = p->pEList->nExpr; + assert( nOrderBy>=nExpr || db->mallocFailed ); + regPrev = pParse->nMem+1; + pParse->nMem += nExpr+1; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regPrev); + pKeyDup = sqlite3KeyInfoAlloc(db, nExpr, 1); + if( pKeyDup ){ + assert( sqlite3KeyInfoIsWriteable(pKeyDup) ); + for(i=0; iaColl[i] = multiSelectCollSeq(pParse, p, i); + pKeyDup->aSortFlags[i] = 0; + } + } + } + + /* Separate the left and the right query from one another + */ + p->pPrior = 0; + pPrior->pNext = 0; + sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); + if( pPrior->pPrior==0 ){ + sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); + } + + /* Compute the limit registers */ + computeLimitRegisters(pParse, p, labelEnd); + if( p->iLimit && op==TK_ALL ){ + regLimitA = ++pParse->nMem; + regLimitB = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Copy, p->iOffset ? p->iOffset+1 : p->iLimit, + regLimitA); + sqlite3VdbeAddOp2(v, OP_Copy, regLimitA, regLimitB); + }else{ + regLimitA = regLimitB = 0; + } + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = 0; + + regAddrA = ++pParse->nMem; + regAddrB = ++pParse->nMem; + regOutA = ++pParse->nMem; + regOutB = ++pParse->nMem; + sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA); + sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB); + + ExplainQueryPlan((pParse, 1, "MERGE (%s)", sqlite3SelectOpName(p->op))); + + /* Generate a coroutine to evaluate the SELECT statement to the + ** left of the compound operator - the "A" select. + */ + addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); + VdbeComment((v, "left SELECT")); + pPrior->iLimit = regLimitA; + ExplainQueryPlan((pParse, 1, "LEFT")); + sqlite3Select(pParse, pPrior, &destA); + sqlite3VdbeEndCoroutine(v, regAddrA); + sqlite3VdbeJumpHere(v, addr1); + + /* Generate a coroutine to evaluate the SELECT statement on + ** the right - the "B" select + */ + addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); + VdbeComment((v, "right SELECT")); + savedLimit = p->iLimit; + savedOffset = p->iOffset; + p->iLimit = regLimitB; + p->iOffset = 0; + ExplainQueryPlan((pParse, 1, "RIGHT")); + sqlite3Select(pParse, p, &destB); + p->iLimit = savedLimit; + p->iOffset = savedOffset; + sqlite3VdbeEndCoroutine(v, regAddrB); + + /* Generate a subroutine that outputs the current row of the A + ** select as the next output row of the compound select. + */ + VdbeNoopComment((v, "Output routine for A")); + addrOutA = generateOutputSubroutine(pParse, + p, &destA, pDest, regOutA, + regPrev, pKeyDup, labelEnd); + + /* Generate a subroutine that outputs the current row of the B + ** select as the next output row of the compound select. + */ + if( op==TK_ALL || op==TK_UNION ){ + VdbeNoopComment((v, "Output routine for B")); + addrOutB = generateOutputSubroutine(pParse, + p, &destB, pDest, regOutB, + regPrev, pKeyDup, labelEnd); + } + sqlite3KeyInfoUnref(pKeyDup); + + /* Generate a subroutine to run when the results from select A + ** are exhausted and only data in select B remains. + */ + if( op==TK_EXCEPT || op==TK_INTERSECT ){ + addrEofA_noB = addrEofA = labelEnd; + }else{ + VdbeNoopComment((v, "eof-A subroutine")); + addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); + VdbeCoverage(v); + sqlite3VdbeGoto(v, addrEofA); + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + } + + /* Generate a subroutine to run when the results from select B + ** are exhausted and only data in select A remains. + */ + if( op==TK_INTERSECT ){ + addrEofB = addrEofA; + if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; + }else{ + VdbeNoopComment((v, "eof-B subroutine")); + addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); + sqlite3VdbeGoto(v, addrEofB); + } + + /* Generate code to handle the case of AB + */ + VdbeNoopComment((v, "A-gt-B subroutine")); + addrAgtB = sqlite3VdbeCurrentAddr(v); + if( op==TK_ALL || op==TK_UNION ){ + sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); + } + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + sqlite3VdbeGoto(v, labelCmpr); + + /* This code runs once to initialize everything. + */ + sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); + + /* Implement the main merge loop + */ + sqlite3VdbeResolveLabel(v, labelCmpr); + sqlite3VdbeAddOp4(v, OP_Permutation, 0, 0, 0, (char*)aPermute, P4_INTARRAY); + sqlite3VdbeAddOp4(v, OP_Compare, destA.iSdst, destB.iSdst, nOrderBy, + (char*)pKeyMerge, P4_KEYINFO); + sqlite3VdbeChangeP5(v, OPFLAG_PERMUTE); + sqlite3VdbeAddOp3(v, OP_Jump, addrAltB, addrAeqB, addrAgtB); VdbeCoverage(v); + + /* Jump to the this point in order to terminate the query. + */ + sqlite3VdbeResolveLabel(v, labelEnd); + + /* Reassembly the compound query so that it will be freed correctly + ** by the calling function */ + if( p->pPrior ){ + sqlite3SelectDelete(db, p->pPrior); + } + p->pPrior = pPrior; + pPrior->pNext = p; + + sqlite3ExprListDelete(db, pPrior->pOrderBy); + pPrior->pOrderBy = 0; + + /*** TBD: Insert subroutine calls to close cursors on incomplete + **** subqueries ****/ + ExplainQueryPlanPop(pParse); + return pParse->nErr!=0; +} +#endif + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + +/* An instance of the SubstContext object describes an substitution edit +** to be performed on a parse tree. +** +** All references to columns in table iTable are to be replaced by corresponding +** expressions in pEList. +*/ +typedef struct SubstContext { + Parse *pParse; /* The parsing context */ + int iTable; /* Replace references to this table */ + int iNewTable; /* New table number */ + int isLeftJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ + ExprList *pEList; /* Replacement expressions */ +} SubstContext; + +/* Forward Declarations */ +static void substExprList(SubstContext*, ExprList*); +static void substSelect(SubstContext*, Select*, int); + +/* +** Scan through the expression pExpr. Replace every reference to +** a column in table number iTable with a copy of the iColumn-th +** entry in pEList. (But leave references to the ROWID column +** unchanged.) +** +** This routine is part of the flattening procedure. A subquery +** whose result set is defined by pEList appears as entry in the +** FROM clause of a SELECT such that the VDBE cursor assigned to that +** FORM clause entry is iTable. This routine makes the necessary +** changes to pExpr so that it refers directly to the source table +** of the subquery rather the result set of the subquery. +*/ +static Expr *substExpr( + SubstContext *pSubst, /* Description of the substitution */ + Expr *pExpr /* Expr in which substitution occurs */ +){ + if( pExpr==0 ) return 0; + if( ExprHasProperty(pExpr, EP_FromJoin) + && pExpr->iRightJoinTable==pSubst->iTable + ){ + pExpr->iRightJoinTable = pSubst->iNewTable; + } + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pSubst->iTable + && !ExprHasProperty(pExpr, EP_FixedCol) + ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( pExpr->iColumn<0 ){ + pExpr->op = TK_NULL; + }else +#endif + { + Expr *pNew; + Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr; + Expr ifNullRow; + assert( pSubst->pEList!=0 && pExpr->iColumnpEList->nExpr ); + assert( pExpr->pRight==0 ); + if( sqlite3ExprIsVector(pCopy) ){ + sqlite3VectorErrorMsg(pSubst->pParse, pCopy); + }else{ + sqlite3 *db = pSubst->pParse->db; + if( pSubst->isLeftJoin && pCopy->op!=TK_COLUMN ){ + memset(&ifNullRow, 0, sizeof(ifNullRow)); + ifNullRow.op = TK_IF_NULL_ROW; + ifNullRow.pLeft = pCopy; + ifNullRow.iTable = pSubst->iNewTable; + ifNullRow.flags = EP_IfNullRow; + pCopy = &ifNullRow; + } + testcase( ExprHasProperty(pCopy, EP_Subquery) ); + pNew = sqlite3ExprDup(db, pCopy, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pNew); + return pExpr; + } + if( pSubst->isLeftJoin ){ + ExprSetProperty(pNew, EP_CanBeNull); + } + if( ExprHasProperty(pExpr,EP_FromJoin) ){ + sqlite3SetJoinExpr(pNew, pExpr->iRightJoinTable); + } + sqlite3ExprDelete(db, pExpr); + pExpr = pNew; + + /* Ensure that the expression now has an implicit collation sequence, + ** just as it did when it was a column of a view or sub-query. */ + if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){ + CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + (pColl ? pColl->zName : "BINARY") + ); + } + ExprClearProperty(pExpr, EP_Collate); + } + } + }else{ + if( pExpr->op==TK_IF_NULL_ROW && pExpr->iTable==pSubst->iTable ){ + pExpr->iTable = pSubst->iNewTable; + } + pExpr->pLeft = substExpr(pSubst, pExpr->pLeft); + pExpr->pRight = substExpr(pSubst, pExpr->pRight); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + substSelect(pSubst, pExpr->x.pSelect, 1); + }else{ + substExprList(pSubst, pExpr->x.pList); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + Window *pWin = pExpr->y.pWin; + pWin->pFilter = substExpr(pSubst, pWin->pFilter); + substExprList(pSubst, pWin->pPartition); + substExprList(pSubst, pWin->pOrderBy); + } +#endif + } + return pExpr; +} +static void substExprList( + SubstContext *pSubst, /* Description of the substitution */ + ExprList *pList /* List to scan and in which to make substitutes */ +){ + int i; + if( pList==0 ) return; + for(i=0; inExpr; i++){ + pList->a[i].pExpr = substExpr(pSubst, pList->a[i].pExpr); + } +} +static void substSelect( + SubstContext *pSubst, /* Description of the substitution */ + Select *p, /* SELECT statement in which to make substitutions */ + int doPrior /* Do substitutes on p->pPrior too */ +){ + SrcList *pSrc; + SrcItem *pItem; + int i; + if( !p ) return; + do{ + substExprList(pSubst, p->pEList); + substExprList(pSubst, p->pGroupBy); + substExprList(pSubst, p->pOrderBy); + p->pHaving = substExpr(pSubst, p->pHaving); + p->pWhere = substExpr(pSubst, p->pWhere); + pSrc = p->pSrc; + assert( pSrc!=0 ); + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + substSelect(pSubst, pItem->pSelect, 1); + if( pItem->fg.isTabFunc ){ + substExprList(pSubst, pItem->u1.pFuncArg); + } + } + }while( doPrior && (p = p->pPrior)!=0 ); +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** pSelect is a SELECT statement and pSrcItem is one item in the FROM +** clause of that SELECT. +** +** This routine scans the entire SELECT statement and recomputes the +** pSrcItem->colUsed mask. +*/ +static int recomputeColumnsUsedExpr(Walker *pWalker, Expr *pExpr){ + SrcItem *pItem; + if( pExpr->op!=TK_COLUMN ) return WRC_Continue; + pItem = pWalker->u.pSrcItem; + if( pItem->iCursor!=pExpr->iTable ) return WRC_Continue; + if( pExpr->iColumn<0 ) return WRC_Continue; + pItem->colUsed |= sqlite3ExprColUsed(pExpr); + return WRC_Continue; +} +static void recomputeColumnsUsed( + Select *pSelect, /* The complete SELECT statement */ + SrcItem *pSrcItem /* Which FROM clause item to recompute */ +){ + Walker w; + if( NEVER(pSrcItem->pTab==0) ) return; + memset(&w, 0, sizeof(w)); + w.xExprCallback = recomputeColumnsUsedExpr; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.u.pSrcItem = pSrcItem; + pSrcItem->colUsed = 0; + sqlite3WalkSelect(&w, pSelect); +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** Assign new cursor numbers to each of the items in pSrc. For each +** new cursor number assigned, set an entry in the aCsrMap[] array +** to map the old cursor number to the new: +** +** aCsrMap[iOld+1] = iNew; +** +** The array is guaranteed by the caller to be large enough for all +** existing cursor numbers in pSrc. aCsrMap[0] is the array size. +** +** If pSrc contains any sub-selects, call this routine recursively +** on the FROM clause of each such sub-select, with iExcept set to -1. +*/ +static void srclistRenumberCursors( + Parse *pParse, /* Parse context */ + int *aCsrMap, /* Array to store cursor mappings in */ + SrcList *pSrc, /* FROM clause to renumber */ + int iExcept /* FROM clause item to skip */ +){ + int i; + SrcItem *pItem; + for(i=0, pItem=pSrc->a; inSrc; i++, pItem++){ + if( i!=iExcept ){ + Select *p; + assert( pItem->iCursor < aCsrMap[0] ); + if( !pItem->fg.isRecursive || aCsrMap[pItem->iCursor+1]==0 ){ + aCsrMap[pItem->iCursor+1] = pParse->nTab++; + } + pItem->iCursor = aCsrMap[pItem->iCursor+1]; + for(p=pItem->pSelect; p; p=p->pPrior){ + srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); + } + } + } +} + +/* +** *piCursor is a cursor number. Change it if it needs to be mapped. +*/ +static void renumberCursorDoMapping(Walker *pWalker, int *piCursor){ + int *aCsrMap = pWalker->u.aiCol; + int iCsr = *piCursor; + if( iCsr < aCsrMap[0] && aCsrMap[iCsr+1]>0 ){ + *piCursor = aCsrMap[iCsr+1]; + } +} + +/* +** Expression walker callback used by renumberCursors() to update +** Expr objects to match newly assigned cursor numbers. +*/ +static int renumberCursorsCb(Walker *pWalker, Expr *pExpr){ + int op = pExpr->op; + if( op==TK_COLUMN || op==TK_IF_NULL_ROW ){ + renumberCursorDoMapping(pWalker, &pExpr->iTable); + } + if( ExprHasProperty(pExpr, EP_FromJoin) ){ + renumberCursorDoMapping(pWalker, &pExpr->iRightJoinTable); + } + return WRC_Continue; +} + +/* +** Assign a new cursor number to each cursor in the FROM clause (Select.pSrc) +** of the SELECT statement passed as the second argument, and to each +** cursor in the FROM clause of any FROM clause sub-selects, recursively. +** Except, do not assign a new cursor number to the iExcept'th element in +** the FROM clause of (*p). Update all expressions and other references +** to refer to the new cursor numbers. +** +** Argument aCsrMap is an array that may be used for temporary working +** space. Two guarantees are made by the caller: +** +** * the array is larger than the largest cursor number used within the +** select statement passed as an argument, and +** +** * the array entries for all cursor numbers that do *not* appear in +** FROM clauses of the select statement as described above are +** initialized to zero. +*/ +static void renumberCursors( + Parse *pParse, /* Parse context */ + Select *p, /* Select to renumber cursors within */ + int iExcept, /* FROM clause item to skip */ + int *aCsrMap /* Working space */ +){ + Walker w; + srclistRenumberCursors(pParse, aCsrMap, p->pSrc, iExcept); + memset(&w, 0, sizeof(w)); + w.u.aiCol = aCsrMap; + w.xExprCallback = renumberCursorsCb; + w.xSelectCallback = sqlite3SelectWalkNoop; + sqlite3WalkSelect(&w, p); +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** This routine attempts to flatten subqueries as a performance optimization. +** This routine returns 1 if it makes changes and 0 if no flattening occurs. +** +** To understand the concept of flattening, consider the following +** query: +** +** SELECT a FROM (SELECT x+y AS a FROM t1 WHERE z<100) WHERE a>5 +** +** The default way of implementing this query is to execute the +** subquery first and store the results in a temporary table, then +** run the outer query on that temporary table. This requires two +** passes over the data. Furthermore, because the temporary table +** has no indices, the WHERE clause on the outer query cannot be +** optimized. +** +** This routine attempts to rewrite queries such as the above into +** a single flat select, like this: +** +** SELECT x+y AS a FROM t1 WHERE z<100 AND a>5 +** +** The code generated for this simplification gives the same result +** but only has to scan the data once. And because indices might +** exist on the table t1, a complete scan of the data might be +** avoided. +** +** Flattening is subject to the following constraints: +** +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** The subquery and the outer query cannot both be aggregates. +** +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** (2) If the subquery is an aggregate then +** (2a) the outer query must not be a join and +** (2b) the outer query must not use subqueries +** other than the one FROM-clause subquery that is a candidate +** for flattening. (This is due to ticket [2f7170d73bf9abf80] +** from 2015-02-09.) +** +** (3) If the subquery is the right operand of a LEFT JOIN then +** (3a) the subquery may not be a join and +** (3b) the FROM clause of the subquery may not contain a virtual +** table and +** (3c) the outer query may not be an aggregate. +** (3d) the outer query may not be DISTINCT. +** +** (4) The subquery can not be DISTINCT. +** +** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT +** sub-queries that were excluded from this optimization. Restriction +** (4) has since been expanded to exclude all DISTINCT subqueries. +** +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** If the subquery is aggregate, the outer query may not be DISTINCT. +** +** (7) The subquery must have a FROM clause. TODO: For subqueries without +** A FROM clause, consider adding a FROM clause with the special +** table sqlite_once that consists of a single row containing a +** single NULL. +** +** (8) If the subquery uses LIMIT then the outer query may not be a join. +** +** (9) If the subquery uses LIMIT then the outer query may not be aggregate. +** +** (**) Restriction (10) was removed from the code on 2005-02-05 but we +** accidently carried the comment forward until 2014-09-15. Original +** constraint: "If the subquery is aggregate then the outer query +** may not use LIMIT." +** +** (11) The subquery and the outer query may not both have ORDER BY clauses. +** +** (**) Not implemented. Subsumed into restriction (3). Was previously +** a separate restriction deriving from ticket #350. +** +** (13) The subquery and outer query may not both use LIMIT. +** +** (14) The subquery may not use OFFSET. +** +** (15) If the outer query is part of a compound select, then the +** subquery may not use LIMIT. +** (See ticket #2339 and ticket [02a8e81d44]). +** +** (16) If the outer query is aggregate, then the subquery may not +** use ORDER BY. (Ticket #2942) This used to not matter +** until we introduced the group_concat() function. +** +** (17) If the subquery is a compound select, then +** (17a) all compound operators must be a UNION ALL, and +** (17b) no terms within the subquery compound may be aggregate +** or DISTINCT, and +** (17c) every term within the subquery compound must have a FROM clause +** (17d) the outer query may not be +** (17d1) aggregate, or +** (17d2) DISTINCT +** (17e) the subquery may not contain window functions, and +** (17f) the subquery must not be the RHS of a LEFT JOIN. +** +** The parent and sub-query may contain WHERE clauses. Subject to +** rules (11), (13) and (14), they may also contain ORDER BY, +** LIMIT and OFFSET clauses. The subquery cannot use any compound +** operator other than UNION ALL because all the other compound +** operators have an implied DISTINCT which is disallowed by +** restriction (4). +** +** Also, each component of the sub-query must return the same number +** of result columns. This is actually a requirement for any compound +** SELECT statement, but all the code here does is make sure that no +** such (illegal) sub-query is flattened. The caller will detect the +** syntax error and return a detailed message. +** +** (18) If the sub-query is a compound select, then all terms of the +** ORDER BY clause of the parent must be copies of a term returned +** by the parent query. +** +** (19) If the subquery uses LIMIT then the outer query may not +** have a WHERE clause. +** +** (20) If the sub-query is a compound select, then it must not use +** an ORDER BY clause. Ticket #3773. We could relax this constraint +** somewhat by saying that the terms of the ORDER BY clause must +** appear as unmodified result columns in the outer query. But we +** have other optimizations in mind to deal with that case. +** +** (21) If the subquery uses LIMIT then the outer query may not be +** DISTINCT. (See ticket [752e1646fc]). +** +** (22) The subquery may not be a recursive CTE. +** +** (23) If the outer query is a recursive CTE, then the sub-query may not be +** a compound query. This restriction is because transforming the +** parent to a compound query confuses the code that handles +** recursive queries in multiSelect(). +** +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** The subquery may not be an aggregate that uses the built-in min() or +** or max() functions. (Without this restriction, a query like: +** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily +** return the value X for which Y was maximal.) +** +** (25) If either the subquery or the parent query contains a window +** function in the select list or ORDER BY clause, flattening +** is not attempted. +** +** +** In this routine, the "p" parameter is a pointer to the outer query. +** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query +** uses aggregates. +** +** If flattening is not attempted, this routine is a no-op and returns 0. +** If flattening is attempted this routine returns 1. +** +** All of the expression analysis must occur on both the outer query and +** the subquery before this routine runs. +*/ +static int flattenSubquery( + Parse *pParse, /* Parsing context */ + Select *p, /* The parent or outer SELECT statement */ + int iFrom, /* Index in p->pSrc->a[] of the inner subquery */ + int isAgg /* True if outer SELECT uses aggregate functions */ +){ + const char *zSavedAuthContext = pParse->zAuthContext; + Select *pParent; /* Current UNION ALL term of the other query */ + Select *pSub; /* The inner query or "subquery" */ + Select *pSub1; /* Pointer to the rightmost select in sub-query */ + SrcList *pSrc; /* The FROM clause of the outer query */ + SrcList *pSubSrc; /* The FROM clause of the subquery */ + int iParent; /* VDBE cursor number of the pSub result set temp table */ + int iNewParent = -1;/* Replacement table for iParent */ + int isLeftJoin = 0; /* True if pSub is the right side of a LEFT JOIN */ + int i; /* Loop counter */ + Expr *pWhere; /* The WHERE clause */ + SrcItem *pSubitem; /* The subquery */ + sqlite3 *db = pParse->db; + Walker w; /* Walker to persist agginfo data */ + int *aCsrMap = 0; + + /* Check to see if flattening is permitted. Return 0 if not. + */ + assert( p!=0 ); + assert( p->pPrior==0 ); + if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0; + pSrc = p->pSrc; + assert( pSrc && iFrom>=0 && iFromnSrc ); + pSubitem = &pSrc->a[iFrom]; + iParent = pSubitem->iCursor; + pSub = pSubitem->pSelect; + assert( pSub!=0 ); + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin || pSub->pWin ) return 0; /* Restriction (25) */ +#endif + + pSubSrc = pSub->pSrc; + assert( pSubSrc ); + /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, + ** not arbitrary expressions, we allowed some combining of LIMIT and OFFSET + ** because they could be computed at compile-time. But when LIMIT and OFFSET + ** became arbitrary expressions, we were forced to add restrictions (13) + ** and (14). */ + if( pSub->pLimit && p->pLimit ) return 0; /* Restriction (13) */ + if( pSub->pLimit && pSub->pLimit->pRight ) return 0; /* Restriction (14) */ + if( (p->selFlags & SF_Compound)!=0 && pSub->pLimit ){ + return 0; /* Restriction (15) */ + } + if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ + if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (4) */ + if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){ + return 0; /* Restrictions (8)(9) */ + } + if( p->pOrderBy && pSub->pOrderBy ){ + return 0; /* Restriction (11) */ + } + if( isAgg && pSub->pOrderBy ) return 0; /* Restriction (16) */ + if( pSub->pLimit && p->pWhere ) return 0; /* Restriction (19) */ + if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ + return 0; /* Restriction (21) */ + } + if( pSub->selFlags & (SF_Recursive) ){ + return 0; /* Restrictions (22) */ + } + + /* + ** If the subquery is the right operand of a LEFT JOIN, then the + ** subquery may not be a join itself (3a). Example of why this is not + ** allowed: + ** + ** t1 LEFT OUTER JOIN (t2 JOIN t3) + ** + ** If we flatten the above, we would get + ** + ** (t1 LEFT OUTER JOIN t2) JOIN t3 + ** + ** which is not at all the same thing. + ** + ** If the subquery is the right operand of a LEFT JOIN, then the outer + ** query cannot be an aggregate. (3c) This is an artifact of the way + ** aggregates are processed - there is no mechanism to determine if + ** the LEFT JOIN table should be all-NULL. + ** + ** See also tickets #306, #350, and #3300. + */ + if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ + isLeftJoin = 1; + if( pSubSrc->nSrc>1 /* (3a) */ + || isAgg /* (3b) */ + || IsVirtual(pSubSrc->a[0].pTab) /* (3c) */ + || (p->selFlags & SF_Distinct)!=0 /* (3d) */ + ){ + return 0; + } + } +#ifdef SQLITE_EXTRA_IFNULLROW + else if( iFrom>0 && !isAgg ){ + /* Setting isLeftJoin to -1 causes OP_IfNullRow opcodes to be generated for + ** every reference to any result column from subquery in a join, even + ** though they are not necessary. This will stress-test the OP_IfNullRow + ** opcode. */ + isLeftJoin = -1; + } +#endif + + /* Restriction (17): If the sub-query is a compound SELECT, then it must + ** use only the UNION ALL operator. And none of the simple select queries + ** that make up the compound SELECT are allowed to be aggregate or distinct + ** queries. + */ + if( pSub->pPrior ){ + if( pSub->pOrderBy ){ + return 0; /* Restriction (20) */ + } + if( isAgg || (p->selFlags & SF_Distinct)!=0 || isLeftJoin>0 ){ + return 0; /* (17d1), (17d2), or (17f) */ + } + for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){ + testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); + testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); + assert( pSub->pSrc!=0 ); + assert( (pSub->selFlags & SF_Recursive)==0 ); + assert( pSub->pEList->nExpr==pSub1->pEList->nExpr ); + if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 /* (17b) */ + || (pSub1->pPrior && pSub1->op!=TK_ALL) /* (17a) */ + || pSub1->pSrc->nSrc<1 /* (17c) */ +#ifndef SQLITE_OMIT_WINDOWFUNC + || pSub1->pWin /* (17e) */ +#endif + ){ + return 0; + } + testcase( pSub1->pSrc->nSrc>1 ); + } + + /* Restriction (18). */ + if( p->pOrderBy ){ + int ii; + for(ii=0; iipOrderBy->nExpr; ii++){ + if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0; + } + } + + /* Restriction (23) */ + if( (p->selFlags & SF_Recursive) ) return 0; + + if( pSrc->nSrc>1 ){ + if( pParse->nSelect>500 ) return 0; + aCsrMap = sqlite3DbMallocZero(db, (pParse->nTab+1)*sizeof(int)); + if( aCsrMap ) aCsrMap[0] = pParse->nTab; + } + } + + /***** If we reach this point, flattening is permitted. *****/ + SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n", + pSub->selId, pSub, iFrom)); + + /* Authorize the subquery */ + pParse->zAuthContext = pSubitem->zName; + TESTONLY(i =) sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0); + testcase( i==SQLITE_DENY ); + pParse->zAuthContext = zSavedAuthContext; + + /* Delete the transient structures associated with thesubquery */ + pSub1 = pSubitem->pSelect; + sqlite3DbFree(db, pSubitem->zDatabase); + sqlite3DbFree(db, pSubitem->zName); + sqlite3DbFree(db, pSubitem->zAlias); + pSubitem->zDatabase = 0; + pSubitem->zName = 0; + pSubitem->zAlias = 0; + pSubitem->pSelect = 0; + assert( pSubitem->pOn==0 ); + + /* If the sub-query is a compound SELECT statement, then (by restrictions + ** 17 and 18 above) it must be a UNION ALL and the parent query must + ** be of the form: + ** + ** SELECT FROM () + ** + ** followed by any ORDER BY, LIMIT and/or OFFSET clauses. This block + ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or + ** OFFSET clauses and joins them to the left-hand-side of the original + ** using UNION ALL operators. In this case N is the number of simple + ** select statements in the compound sub-query. + ** + ** Example: + ** + ** SELECT a+1 FROM ( + ** SELECT x FROM tab + ** UNION ALL + ** SELECT y FROM tab + ** UNION ALL + ** SELECT abs(z*2) FROM tab2 + ** ) WHERE a!=5 ORDER BY 1 + ** + ** Transformed into: + ** + ** SELECT x+1 FROM tab WHERE x+1!=5 + ** UNION ALL + ** SELECT y+1 FROM tab WHERE y+1!=5 + ** UNION ALL + ** SELECT abs(z*2)+1 FROM tab2 WHERE abs(z*2)+1!=5 + ** ORDER BY 1 + ** + ** We call this the "compound-subquery flattening". + */ + for(pSub=pSub->pPrior; pSub; pSub=pSub->pPrior){ + Select *pNew; + ExprList *pOrderBy = p->pOrderBy; + Expr *pLimit = p->pLimit; + Select *pPrior = p->pPrior; + Table *pItemTab = pSubitem->pTab; + pSubitem->pTab = 0; + p->pOrderBy = 0; + p->pPrior = 0; + p->pLimit = 0; + pNew = sqlite3SelectDup(db, p, 0); + p->pLimit = pLimit; + p->pOrderBy = pOrderBy; + p->op = TK_ALL; + pSubitem->pTab = pItemTab; + if( pNew==0 ){ + p->pPrior = pPrior; + }else{ + pNew->selId = ++pParse->nSelect; + if( aCsrMap && ALWAYS(db->mallocFailed==0) ){ + renumberCursors(pParse, pNew, iFrom, aCsrMap); + } + pNew->pPrior = pPrior; + if( pPrior ) pPrior->pNext = pNew; + pNew->pNext = p; + p->pPrior = pNew; + SELECTTRACE(2,pParse,p,("compound-subquery flattener" + " creates %u as peer\n",pNew->selId)); + } + assert( pSubitem->pSelect==0 ); + } + sqlite3DbFree(db, aCsrMap); + if( db->mallocFailed ){ + pSubitem->pSelect = pSub1; + return 1; + } + + /* Defer deleting the Table object associated with the + ** subquery until code generation is + ** complete, since there may still exist Expr.pTab entries that + ** refer to the subquery even after flattening. Ticket #3346. + ** + ** pSubitem->pTab is always non-NULL by test restrictions and tests above. + */ + if( ALWAYS(pSubitem->pTab!=0) ){ + Table *pTabToDel = pSubitem->pTab; + if( pTabToDel->nTabRef==1 ){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + sqlite3ParserAddCleanup(pToplevel, + (void(*)(sqlite3*,void*))sqlite3DeleteTable, + pTabToDel); + testcase( pToplevel->earlyCleanup ); + }else{ + pTabToDel->nTabRef--; + } + pSubitem->pTab = 0; + } + + /* The following loop runs once for each term in a compound-subquery + ** flattening (as described above). If we are doing a different kind + ** of flattening - a flattening other than a compound-subquery flattening - + ** then this loop only runs once. + ** + ** This loop moves all of the FROM elements of the subquery into the + ** the FROM clause of the outer query. Before doing this, remember + ** the cursor number for the original outer query FROM element in + ** iParent. The iParent cursor will never be used. Subsequent code + ** will scan expressions looking for iParent references and replace + ** those references with expressions that resolve to the subquery FROM + ** elements we are now copying in. + */ + pSub = pSub1; + for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ + int nSubSrc; + u8 jointype = 0; + assert( pSub!=0 ); + pSubSrc = pSub->pSrc; /* FROM clause of subquery */ + nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ + pSrc = pParent->pSrc; /* FROM clause of the outer query */ + + if( pParent==p ){ + jointype = pSubitem->fg.jointype; /* First time through the loop */ + } + + /* The subquery uses a single slot of the FROM clause of the outer + ** query. If the subquery has more than one element in its FROM clause, + ** then expand the outer query to make space for it to hold all elements + ** of the subquery. + ** + ** Example: + ** + ** SELECT * FROM tabA, (SELECT * FROM sub1, sub2), tabB; + ** + ** The outer query has 3 slots in its FROM clause. One slot of the + ** outer query (the middle slot) is used by the subquery. The next + ** block of code will expand the outer query FROM clause to 4 slots. + ** The middle slot is expanded to two slots in order to make space + ** for the two elements in the FROM clause of the subquery. + */ + if( nSubSrc>1 ){ + pSrc = sqlite3SrcListEnlarge(pParse, pSrc, nSubSrc-1,iFrom+1); + if( pSrc==0 ) break; + pParent->pSrc = pSrc; + } + + /* Transfer the FROM clause terms from the subquery into the + ** outer query. + */ + for(i=0; ia[i+iFrom].pUsing); + assert( pSrc->a[i+iFrom].fg.isTabFunc==0 ); + pSrc->a[i+iFrom] = pSubSrc->a[i]; + iNewParent = pSubSrc->a[i].iCursor; + memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); + } + pSrc->a[iFrom].fg.jointype = jointype; + + /* Now begin substituting subquery result set expressions for + ** references to the iParent in the outer query. + ** + ** Example: + ** + ** SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b; + ** \ \_____________ subquery __________/ / + ** \_____________________ outer query ______________________________/ + ** + ** We look at every expression in the outer query and every place we see + ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". + */ + if( pSub->pOrderBy && (pParent->selFlags & SF_NoopOrderBy)==0 ){ + /* At this point, any non-zero iOrderByCol values indicate that the + ** ORDER BY column expression is identical to the iOrderByCol'th + ** expression returned by SELECT statement pSub. Since these values + ** do not necessarily correspond to columns in SELECT statement pParent, + ** zero them before transfering the ORDER BY clause. + ** + ** Not doing this may cause an error if a subsequent call to this + ** function attempts to flatten a compound sub-query into pParent + ** (the only way this can happen is if the compound sub-query is + ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */ + ExprList *pOrderBy = pSub->pOrderBy; + for(i=0; inExpr; i++){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } + assert( pParent->pOrderBy==0 ); + pParent->pOrderBy = pOrderBy; + pSub->pOrderBy = 0; + } + pWhere = pSub->pWhere; + pSub->pWhere = 0; + if( isLeftJoin>0 ){ + sqlite3SetJoinExpr(pWhere, iNewParent); + } + if( pWhere ){ + if( pParent->pWhere ){ + pParent->pWhere = sqlite3PExpr(pParse, TK_AND, pWhere, pParent->pWhere); + }else{ + pParent->pWhere = pWhere; + } + } + if( db->mallocFailed==0 ){ + SubstContext x; + x.pParse = pParse; + x.iTable = iParent; + x.iNewTable = iNewParent; + x.isLeftJoin = isLeftJoin; + x.pEList = pSub->pEList; + substSelect(&x, pParent, 0); + } + + /* The flattened query is a compound if either the inner or the + ** outer query is a compound. */ + pParent->selFlags |= pSub->selFlags & SF_Compound; + assert( (pSub->selFlags & SF_Distinct)==0 ); /* restriction (17b) */ + + /* + ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y; + ** + ** One is tempted to try to add a and b to combine the limits. But this + ** does not work if either limit is negative. + */ + if( pSub->pLimit ){ + pParent->pLimit = pSub->pLimit; + pSub->pLimit = 0; + } + + /* Recompute the SrcList_item.colUsed masks for the flattened + ** tables. */ + for(i=0; ia[i+iFrom]); + } + } + + /* Finially, delete what is left of the subquery and return + ** success. + */ + sqlite3AggInfoPersistWalkerInit(&w, pParse); + sqlite3WalkSelect(&w,pSub1); + sqlite3SelectDelete(db, pSub1); + +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p,("After flattening:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + + return 1; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +/* +** A structure to keep track of all of the column values that are fixed to +** a known value due to WHERE clause constraints of the form COLUMN=VALUE. +*/ +typedef struct WhereConst WhereConst; +struct WhereConst { + Parse *pParse; /* Parsing context */ + u8 *pOomFault; /* Pointer to pParse->db->mallocFailed */ + int nConst; /* Number for COLUMN=CONSTANT terms */ + int nChng; /* Number of times a constant is propagated */ + int bHasAffBlob; /* At least one column in apExpr[] as affinity BLOB */ + Expr **apExpr; /* [i*2] is COLUMN and [i*2+1] is VALUE */ +}; + +/* +** Add a new entry to the pConst object. Except, do not add duplicate +** pColumn entires. Also, do not add if doing so would not be appropriate. +** +** The caller guarantees the pColumn is a column and pValue is a constant. +** This routine has to do some additional checks before completing the +** insert. +*/ +static void constInsert( + WhereConst *pConst, /* The WhereConst into which we are inserting */ + Expr *pColumn, /* The COLUMN part of the constraint */ + Expr *pValue, /* The VALUE part of the constraint */ + Expr *pExpr /* Overall expression: COLUMN=VALUE or VALUE=COLUMN */ +){ + int i; + assert( pColumn->op==TK_COLUMN ); + assert( sqlite3ExprIsConstant(pValue) ); + + if( ExprHasProperty(pColumn, EP_FixedCol) ) return; + if( sqlite3ExprAffinity(pValue)!=0 ) return; + if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pConst->pParse,pExpr)) ){ + return; + } + + /* 2018-10-25 ticket [cf5ed20f] + ** Make sure the same pColumn is not inserted more than once */ + for(i=0; inConst; i++){ + const Expr *pE2 = pConst->apExpr[i*2]; + assert( pE2->op==TK_COLUMN ); + if( pE2->iTable==pColumn->iTable + && pE2->iColumn==pColumn->iColumn + ){ + return; /* Already present. Return without doing anything. */ + } + } + if( sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){ + pConst->bHasAffBlob = 1; + } + + pConst->nConst++; + pConst->apExpr = sqlite3DbReallocOrFree(pConst->pParse->db, pConst->apExpr, + pConst->nConst*2*sizeof(Expr*)); + if( pConst->apExpr==0 ){ + pConst->nConst = 0; + }else{ + pConst->apExpr[pConst->nConst*2-2] = pColumn; + pConst->apExpr[pConst->nConst*2-1] = pValue; + } +} + +/* +** Find all terms of COLUMN=VALUE or VALUE=COLUMN in pExpr where VALUE +** is a constant expression and where the term must be true because it +** is part of the AND-connected terms of the expression. For each term +** found, add it to the pConst structure. +*/ +static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ + Expr *pRight, *pLeft; + if( NEVER(pExpr==0) ) return; + if( ExprHasProperty(pExpr, EP_FromJoin) ) return; + if( pExpr->op==TK_AND ){ + findConstInWhere(pConst, pExpr->pRight); + findConstInWhere(pConst, pExpr->pLeft); + return; + } + if( pExpr->op!=TK_EQ ) return; + pRight = pExpr->pRight; + pLeft = pExpr->pLeft; + assert( pRight!=0 ); + assert( pLeft!=0 ); + if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pLeft) ){ + constInsert(pConst,pRight,pLeft,pExpr); + } + if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pRight) ){ + constInsert(pConst,pLeft,pRight,pExpr); + } +} + +/* +** This is a helper function for Walker callback propagateConstantExprRewrite(). +** +** Argument pExpr is a candidate expression to be replaced by a value. If +** pExpr is equivalent to one of the columns named in pWalker->u.pConst, +** then overwrite it with the corresponding value. Except, do not do so +** if argument bIgnoreAffBlob is non-zero and the affinity of pExpr +** is SQLITE_AFF_BLOB. +*/ +static int propagateConstantExprRewriteOne( + WhereConst *pConst, + Expr *pExpr, + int bIgnoreAffBlob +){ + int i; + if( pConst->pOomFault[0] ) return WRC_Prune; + if( pExpr->op!=TK_COLUMN ) return WRC_Continue; + if( ExprHasProperty(pExpr, EP_FixedCol|EP_FromJoin) ){ + testcase( ExprHasProperty(pExpr, EP_FixedCol) ); + testcase( ExprHasProperty(pExpr, EP_FromJoin) ); + return WRC_Continue; + } + for(i=0; inConst; i++){ + Expr *pColumn = pConst->apExpr[i*2]; + if( pColumn==pExpr ) continue; + if( pColumn->iTable!=pExpr->iTable ) continue; + if( pColumn->iColumn!=pExpr->iColumn ) continue; + if( bIgnoreAffBlob && sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){ + break; + } + /* A match is found. Add the EP_FixedCol property */ + pConst->nChng++; + ExprClearProperty(pExpr, EP_Leaf); + ExprSetProperty(pExpr, EP_FixedCol); + assert( pExpr->pLeft==0 ); + pExpr->pLeft = sqlite3ExprDup(pConst->pParse->db, pConst->apExpr[i*2+1], 0); + if( pConst->pParse->db->mallocFailed ) return WRC_Prune; + break; + } + return WRC_Prune; +} + +/* +** This is a Walker expression callback. pExpr is a node from the WHERE +** clause of a SELECT statement. This function examines pExpr to see if +** any substitutions based on the contents of pWalker->u.pConst should +** be made to pExpr or its immediate children. +** +** A substitution is made if: +** +** + pExpr is a column with an affinity other than BLOB that matches +** one of the columns in pWalker->u.pConst, or +** +** + pExpr is a binary comparison operator (=, <=, >=, <, >) that +** uses an affinity other than TEXT and one of its immediate +** children is a column that matches one of the columns in +** pWalker->u.pConst. +*/ +static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){ + WhereConst *pConst = pWalker->u.pConst; + assert( TK_GT==TK_EQ+1 ); + assert( TK_LE==TK_EQ+2 ); + assert( TK_LT==TK_EQ+3 ); + assert( TK_GE==TK_EQ+4 ); + if( pConst->bHasAffBlob ){ + if( (pExpr->op>=TK_EQ && pExpr->op<=TK_GE) + || pExpr->op==TK_IS + ){ + propagateConstantExprRewriteOne(pConst, pExpr->pLeft, 0); + if( pConst->pOomFault[0] ) return WRC_Prune; + if( sqlite3ExprAffinity(pExpr->pLeft)!=SQLITE_AFF_TEXT ){ + propagateConstantExprRewriteOne(pConst, pExpr->pRight, 0); + } + } + } + return propagateConstantExprRewriteOne(pConst, pExpr, pConst->bHasAffBlob); +} + +/* +** The WHERE-clause constant propagation optimization. +** +** If the WHERE clause contains terms of the form COLUMN=CONSTANT or +** CONSTANT=COLUMN that are top-level AND-connected terms that are not +** part of a ON clause from a LEFT JOIN, then throughout the query +** replace all other occurrences of COLUMN with CONSTANT. +** +** For example, the query: +** +** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=t1.a AND t3.c=t2.b +** +** Is transformed into +** +** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=39 AND t3.c=39 +** +** Return true if any transformations where made and false if not. +** +** Implementation note: Constant propagation is tricky due to affinity +** and collating sequence interactions. Consider this example: +** +** CREATE TABLE t1(a INT,b TEXT); +** INSERT INTO t1 VALUES(123,'0123'); +** SELECT * FROM t1 WHERE a=123 AND b=a; +** SELECT * FROM t1 WHERE a=123 AND b=123; +** +** The two SELECT statements above should return different answers. b=a +** is alway true because the comparison uses numeric affinity, but b=123 +** is false because it uses text affinity and '0123' is not the same as '123'. +** To work around this, the expression tree is not actually changed from +** "b=a" to "b=123" but rather the "a" in "b=a" is tagged with EP_FixedCol +** and the "123" value is hung off of the pLeft pointer. Code generator +** routines know to generate the constant "123" instead of looking up the +** column value. Also, to avoid collation problems, this optimization is +** only attempted if the "a=123" term uses the default BINARY collation. +** +** 2021-05-25 forum post 6a06202608: Another troublesome case is... +** +** CREATE TABLE t1(x); +** INSERT INTO t1 VALUES(10.0); +** SELECT 1 FROM t1 WHERE x=10 AND x LIKE 10; +** +** The query should return no rows, because the t1.x value is '10.0' not '10' +** and '10.0' is not LIKE '10'. But if we are not careful, the first WHERE +** term "x=10" will cause the second WHERE term to become "10 LIKE 10", +** resulting in a false positive. To avoid this, constant propagation for +** columns with BLOB affinity is only allowed if the constant is used with +** operators ==, <=, <, >=, >, or IS in a way that will cause the correct +** type conversions to occur. See logic associated with the bHasAffBlob flag +** for details. +*/ +static int propagateConstants( + Parse *pParse, /* The parsing context */ + Select *p /* The query in which to propagate constants */ +){ + WhereConst x; + Walker w; + int nChng = 0; + x.pParse = pParse; + x.pOomFault = &pParse->db->mallocFailed; + do{ + x.nConst = 0; + x.nChng = 0; + x.apExpr = 0; + x.bHasAffBlob = 0; + findConstInWhere(&x, p->pWhere); + if( x.nConst ){ + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = propagateConstantExprRewrite; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.xSelectCallback2 = 0; + w.walkerDepth = 0; + w.u.pConst = &x; + sqlite3WalkExpr(&w, p->pWhere); + sqlite3DbFree(x.pParse->db, x.apExpr); + nChng += x.nChng; + } + }while( x.nChng ); + return nChng; +} + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +# if !defined(SQLITE_OMIT_WINDOWFUNC) +/* +** This function is called to determine whether or not it is safe to +** push WHERE clause expression pExpr down to FROM clause sub-query +** pSubq, which contains at least one window function. Return 1 +** if it is safe and the expression should be pushed down, or 0 +** otherwise. +** +** It is only safe to push the expression down if it consists only +** of constants and copies of expressions that appear in the PARTITION +** BY clause of all window function used by the sub-query. It is safe +** to filter out entire partitions, but not rows within partitions, as +** this may change the results of the window functions. +** +** At the time this function is called it is guaranteed that +** +** * the sub-query uses only one distinct window frame, and +** * that the window frame has a PARTITION BY clase. +*/ +static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ + assert( pSubq->pWin->pPartition ); + assert( (pSubq->selFlags & SF_MultiPart)==0 ); + assert( pSubq->pPrior==0 ); + return sqlite3ExprIsConstantOrGroupBy(pParse, pExpr, pSubq->pWin->pPartition); +} +# endif /* SQLITE_OMIT_WINDOWFUNC */ +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** Make copies of relevant WHERE clause terms of the outer query into +** the WHERE clause of subquery. Example: +** +** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1) WHERE x=5 AND y=10; +** +** Transformed into: +** +** SELECT * FROM (SELECT a AS x, c-d AS y FROM t1 WHERE a=5 AND c-d=10) +** WHERE x=5 AND y=10; +** +** The hope is that the terms added to the inner query will make it more +** efficient. +** +** Do not attempt this optimization if: +** +** (1) (** This restriction was removed on 2017-09-29. We used to +** disallow this optimization for aggregate subqueries, but now +** it is allowed by putting the extra terms on the HAVING clause. +** The added HAVING clause is pointless if the subquery lacks +** a GROUP BY clause. But such a HAVING clause is also harmless +** so there does not appear to be any reason to add extra logic +** to suppress it. **) +** +** (2) The inner query is the recursive part of a common table expression. +** +** (3) The inner query has a LIMIT clause (since the changes to the WHERE +** clause would change the meaning of the LIMIT). +** +** (4) The inner query is the right operand of a LEFT JOIN and the +** expression to be pushed down does not come from the ON clause +** on that LEFT JOIN. +** +** (5) The WHERE clause expression originates in the ON or USING clause +** of a LEFT JOIN where iCursor is not the right-hand table of that +** left join. An example: +** +** SELECT * +** FROM (SELECT 1 AS a1 UNION ALL SELECT 2) AS aa +** JOIN (SELECT 1 AS b2 UNION ALL SELECT 2) AS bb ON (a1=b2) +** LEFT JOIN (SELECT 8 AS c3 UNION ALL SELECT 9) AS cc ON (b2=2); +** +** The correct answer is three rows: (1,1,NULL),(2,2,8),(2,2,9). +** But if the (b2=2) term were to be pushed down into the bb subquery, +** then the (1,1,NULL) row would be suppressed. +** +** (6) Window functions make things tricky as changes to the WHERE clause +** of the inner query could change the window over which window +** functions are calculated. Therefore, do not attempt the optimization +** if: +** +** (6a) The inner query uses multiple incompatible window partitions. +** +** (6b) The inner query is a compound and uses window-functions. +** +** (6c) The WHERE clause does not consist entirely of constants and +** copies of expressions found in the PARTITION BY clause of +** all window-functions used by the sub-query. It is safe to +** filter out entire partitions, as this does not change the +** window over which any window-function is calculated. +** +** (7) The inner query is a Common Table Expression (CTE) that should +** be materialized. (This restriction is implemented in the calling +** routine.) +** +** Return 0 if no changes are made and non-zero if one or more WHERE clause +** terms are duplicated into the subquery. +*/ +static int pushDownWhereTerms( + Parse *pParse, /* Parse context (for malloc() and error reporting) */ + Select *pSubq, /* The subquery whose WHERE clause is to be augmented */ + Expr *pWhere, /* The WHERE clause of the outer query */ + int iCursor, /* Cursor number of the subquery */ + int isLeftJoin /* True if pSubq is the right term of a LEFT JOIN */ +){ + Expr *pNew; + int nChng = 0; + if( pWhere==0 ) return 0; + if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0; + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pSubq->pPrior ){ + Select *pSel; + for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + if( pSel->pWin ) return 0; /* restriction (6b) */ + } + }else{ + if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; + } +#endif + +#ifdef SQLITE_DEBUG + /* Only the first term of a compound can have a WITH clause. But make + ** sure no other terms are marked SF_Recursive in case something changes + ** in the future. + */ + { + Select *pX; + for(pX=pSubq; pX; pX=pX->pPrior){ + assert( (pX->selFlags & (SF_Recursive))==0 ); + } + } +#endif + + if( pSubq->pLimit!=0 ){ + return 0; /* restriction (3) */ + } + while( pWhere->op==TK_AND ){ + nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, + iCursor, isLeftJoin); + pWhere = pWhere->pLeft; + } + if( isLeftJoin + && (ExprHasProperty(pWhere,EP_FromJoin)==0 + || pWhere->iRightJoinTable!=iCursor) + ){ + return 0; /* restriction (4) */ + } + if( ExprHasProperty(pWhere,EP_FromJoin) && pWhere->iRightJoinTable!=iCursor ){ + return 0; /* restriction (5) */ + } + if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ + nChng++; + pSubq->selFlags |= SF_PushDown; + while( pSubq ){ + SubstContext x; + pNew = sqlite3ExprDup(pParse->db, pWhere, 0); + unsetJoinExpr(pNew, -1); + x.pParse = pParse; + x.iTable = iCursor; + x.iNewTable = iCursor; + x.isLeftJoin = 0; + x.pEList = pSubq->pEList; + pNew = substExpr(&x, pNew); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ + /* Restriction 6c has prevented push-down in this case */ + sqlite3ExprDelete(pParse->db, pNew); + nChng--; + break; + } +#endif + if( pSubq->selFlags & SF_Aggregate ){ + pSubq->pHaving = sqlite3ExprAnd(pParse, pSubq->pHaving, pNew); + }else{ + pSubq->pWhere = sqlite3ExprAnd(pParse, pSubq->pWhere, pNew); + } + pSubq = pSubq->pPrior; + } + } + return nChng; +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +/* +** The pFunc is the only aggregate function in the query. Check to see +** if the query is a candidate for the min/max optimization. +** +** If the query is a candidate for the min/max optimization, then set +** *ppMinMax to be an ORDER BY clause to be used for the optimization +** and return either WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX depending on +** whether pFunc is a min() or max() function. +** +** If the query is not a candidate for the min/max optimization, return +** WHERE_ORDERBY_NORMAL (which must be zero). +** +** This routine must be called after aggregate functions have been +** located but before their arguments have been subjected to aggregate +** analysis. +*/ +static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){ + int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ + ExprList *pEList = pFunc->x.pList; /* Arguments to agg function */ + const char *zFunc; /* Name of aggregate function pFunc */ + ExprList *pOrderBy; + u8 sortFlags = 0; + + assert( *ppMinMax==0 ); + assert( pFunc->op==TK_AGG_FUNCTION ); + assert( !IsWindowFunc(pFunc) ); + if( pEList==0 + || pEList->nExpr!=1 + || ExprHasProperty(pFunc, EP_WinFunc) + || OptimizationDisabled(db, SQLITE_MinMaxOpt) + ){ + return eRet; + } + zFunc = pFunc->u.zToken; + if( sqlite3StrICmp(zFunc, "min")==0 ){ + eRet = WHERE_ORDERBY_MIN; + if( sqlite3ExprCanBeNull(pEList->a[0].pExpr) ){ + sortFlags = KEYINFO_ORDER_BIGNULL; + } + }else if( sqlite3StrICmp(zFunc, "max")==0 ){ + eRet = WHERE_ORDERBY_MAX; + sortFlags = KEYINFO_ORDER_DESC; + }else{ + return eRet; + } + *ppMinMax = pOrderBy = sqlite3ExprListDup(db, pEList, 0); + assert( pOrderBy!=0 || db->mallocFailed ); + if( pOrderBy ) pOrderBy->a[0].sortFlags = sortFlags; + return eRet; +} + +/* +** The select statement passed as the first argument is an aggregate query. +** The second argument is the associated aggregate-info object. This +** function tests if the SELECT is of the form: +** +** SELECT count(*) FROM +** +** where table is a database table, not a sub-select or view. If the query +** does match this pattern, then a pointer to the Table object representing +** is returned. Otherwise, 0 is returned. +*/ +static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ + Table *pTab; + Expr *pExpr; + + assert( !p->pGroupBy ); + + if( p->pWhere || p->pEList->nExpr!=1 + || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect + ){ + return 0; + } + pTab = p->pSrc->a[0].pTab; + pExpr = p->pEList->a[0].pExpr; + assert( pTab && !IsView(pTab) && pExpr ); + + if( IsVirtual(pTab) ) return 0; + if( pExpr->op!=TK_AGG_FUNCTION ) return 0; + if( NEVER(pAggInfo->nFunc==0) ) return 0; + if( (pAggInfo->aFunc[0].pFunc->funcFlags&SQLITE_FUNC_COUNT)==0 ) return 0; + if( ExprHasProperty(pExpr, EP_Distinct|EP_WinFunc) ) return 0; + + return pTab; +} + +/* +** If the source-list item passed as an argument was augmented with an +** INDEXED BY clause, then try to locate the specified index. If there +** was such a clause and the named index cannot be found, return +** SQLITE_ERROR and leave an error in pParse. Otherwise, populate +** pFrom->pIndex and return SQLITE_OK. +*/ +SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ + Table *pTab = pFrom->pTab; + char *zIndexedBy = pFrom->u1.zIndexedBy; + Index *pIdx; + assert( pTab!=0 ); + assert( pFrom->fg.isIndexedBy!=0 ); + + for(pIdx=pTab->pIndex; + pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); + pIdx=pIdx->pNext + ); + if( !pIdx ){ + sqlite3ErrorMsg(pParse, "no such index: %s", zIndexedBy, 0); + pParse->checkSchema = 1; + return SQLITE_ERROR; + } + pFrom->u2.pIBIndex = pIdx; + return SQLITE_OK; +} + +/* +** Detect compound SELECT statements that use an ORDER BY clause with +** an alternative collating sequence. +** +** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ... +** +** These are rewritten as a subquery: +** +** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) +** ORDER BY ... COLLATE ... +** +** This transformation is necessary because the multiSelectOrderBy() routine +** above that generates the code for a compound SELECT with an ORDER BY clause +** uses a merge algorithm that requires the same collating sequence on the +** result columns as on the ORDER BY clause. See ticket +** http://www.sqlite.org/src/info/6709574d2a +** +** This transformation is only needed for EXCEPT, INTERSECT, and UNION. +** The UNION ALL operator works fine with multiSelectOrderBy() even when +** there are COLLATE terms in the ORDER BY. +*/ +static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ + int i; + Select *pNew; + Select *pX; + sqlite3 *db; + struct ExprList_item *a; + SrcList *pNewSrc; + Parse *pParse; + Token dummy; + + if( p->pPrior==0 ) return WRC_Continue; + if( p->pOrderBy==0 ) return WRC_Continue; + for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){} + if( pX==0 ) return WRC_Continue; + a = p->pOrderBy->a; +#ifndef SQLITE_OMIT_WINDOWFUNC + /* If iOrderByCol is already non-zero, then it has already been matched + ** to a result column of the SELECT statement. This occurs when the + ** SELECT is rewritten for window-functions processing and then passed + ** to sqlite3SelectPrep() and similar a second time. The rewriting done + ** by this function is not required in this case. */ + if( a[0].u.x.iOrderByCol ) return WRC_Continue; +#endif + for(i=p->pOrderBy->nExpr-1; i>=0; i--){ + if( a[i].pExpr->flags & EP_Collate ) break; + } + if( i<0 ) return WRC_Continue; + + /* If we reach this point, that means the transformation is required. */ + + pParse = pWalker->pParse; + db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + if( pNew==0 ) return WRC_Abort; + memset(&dummy, 0, sizeof(dummy)); + pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0); + if( pNewSrc==0 ) return WRC_Abort; + *pNew = *p; + p->pSrc = pNewSrc; + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0)); + p->op = TK_SELECT; + p->pWhere = 0; + pNew->pGroupBy = 0; + pNew->pHaving = 0; + pNew->pOrderBy = 0; + p->pPrior = 0; + p->pNext = 0; + p->pWith = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + p->pWinDefn = 0; +#endif + p->selFlags &= ~SF_Compound; + assert( (p->selFlags & SF_Converted)==0 ); + p->selFlags |= SF_Converted; + assert( pNew->pPrior!=0 ); + pNew->pPrior->pNext = pNew; + pNew->pLimit = 0; + return WRC_Continue; +} + +/* +** Check to see if the FROM clause term pFrom has table-valued function +** arguments. If it does, leave an error message in pParse and return +** non-zero, since pFrom is not allowed to be a table-valued function. +*/ +static int cannotBeFunction(Parse *pParse, SrcItem *pFrom){ + if( pFrom->fg.isTabFunc ){ + sqlite3ErrorMsg(pParse, "'%s' is not a function", pFrom->zName); + return 1; + } + return 0; +} + +#ifndef SQLITE_OMIT_CTE +/* +** Argument pWith (which may be NULL) points to a linked list of nested +** WITH contexts, from inner to outermost. If the table identified by +** FROM clause element pItem is really a common-table-expression (CTE) +** then return a pointer to the CTE definition for that table. Otherwise +** return NULL. +** +** If a non-NULL value is returned, set *ppContext to point to the With +** object that the returned CTE belongs to. +*/ +static struct Cte *searchWith( + With *pWith, /* Current innermost WITH clause */ + SrcItem *pItem, /* FROM clause element to resolve */ + With **ppContext /* OUT: WITH clause return value belongs to */ +){ + const char *zName = pItem->zName; + With *p; + assert( pItem->zDatabase==0 ); + assert( zName!=0 ); + for(p=pWith; p; p=p->pOuter){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){ + *ppContext = p; + return &p->a[i]; + } + } + if( p->bView ) break; + } + return 0; +} + +/* The code generator maintains a stack of active WITH clauses +** with the inner-most WITH clause being at the top of the stack. +** +** This routine pushes the WITH clause passed as the second argument +** onto the top of the stack. If argument bFree is true, then this +** WITH clause will never be popped from the stack but should instead +** be freed along with the Parse object. In other cases, when +** bFree==0, the With object will be freed along with the SELECT +** statement with which it is associated. +** +** This routine returns a copy of pWith. Or, if bFree is true and +** the pWith object is destroyed immediately due to an OOM condition, +** then this routine return NULL. +** +** If bFree is true, do not continue to use the pWith pointer after +** calling this routine, Instead, use only the return value. +*/ +SQLITE_PRIVATE With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ + if( pWith ){ + if( bFree ){ + pWith = (With*)sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3WithDelete, + pWith); + if( pWith==0 ) return 0; + } + if( pParse->nErr==0 ){ + assert( pParse->pWith!=pWith ); + pWith->pOuter = pParse->pWith; + pParse->pWith = pWith; + } + } + return pWith; +} + +/* +** This function checks if argument pFrom refers to a CTE declared by +** a WITH clause on the stack currently maintained by the parser (on the +** pParse->pWith linked list). And if currently processing a CTE +** CTE expression, through routine checks to see if the reference is +** a recursive reference to the CTE. +** +** If pFrom matches a CTE according to either of these two above, pFrom->pTab +** and other fields are populated accordingly. +** +** Return 0 if no match is found. +** Return 1 if a match is found. +** Return 2 if an error condition is detected. +*/ +static int resolveFromTermToCte( + Parse *pParse, /* The parsing context */ + Walker *pWalker, /* Current tree walker */ + SrcItem *pFrom /* The FROM clause term to check */ +){ + Cte *pCte; /* Matched CTE (or NULL if no match) */ + With *pWith; /* The matching WITH */ + + assert( pFrom->pTab==0 ); + if( pParse->pWith==0 ){ + /* There are no WITH clauses in the stack. No match is possible */ + return 0; + } + if( pParse->nErr ){ + /* Prior errors might have left pParse->pWith in a goofy state, so + ** go no further. */ + return 0; + } + if( pFrom->zDatabase!=0 ){ + /* The FROM term contains a schema qualifier (ex: main.t1) and so + ** it cannot possibly be a CTE reference. */ + return 0; + } + if( pFrom->fg.notCte ){ + /* The FROM term is specifically excluded from matching a CTE. + ** (1) It is part of a trigger that used to have zDatabase but had + ** zDatabase removed by sqlite3FixTriggerStep(). + ** (2) This is the first term in the FROM clause of an UPDATE. + */ + return 0; + } + pCte = searchWith(pParse->pWith, pFrom, &pWith); + if( pCte ){ + sqlite3 *db = pParse->db; + Table *pTab; + ExprList *pEList; + Select *pSel; + Select *pLeft; /* Left-most SELECT statement */ + Select *pRecTerm; /* Left-most recursive term */ + int bMayRecursive; /* True if compound joined by UNION [ALL] */ + With *pSavedWith; /* Initial value of pParse->pWith */ + int iRecTab = -1; /* Cursor for recursive table */ + CteUse *pCteUse; + + /* If pCte->zCteErr is non-NULL at this point, then this is an illegal + ** recursive reference to CTE pCte. Leave an error in pParse and return + ** early. If pCte->zCteErr is NULL, then this is not a recursive reference. + ** In this case, proceed. */ + if( pCte->zCteErr ){ + sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName); + return 2; + } + if( cannotBeFunction(pParse, pFrom) ) return 2; + + assert( pFrom->pTab==0 ); + pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return 2; + pCteUse = pCte->pUse; + if( pCteUse==0 ){ + pCte->pUse = pCteUse = sqlite3DbMallocZero(db, sizeof(pCteUse[0])); + if( pCteUse==0 + || sqlite3ParserAddCleanup(pParse,sqlite3DbFree,pCteUse)==0 + ){ + sqlite3DbFree(db, pTab); + return 2; + } + pCteUse->eM10d = pCte->eM10d; + } + pFrom->pTab = pTab; + pTab->nTabRef = 1; + pTab->zName = sqlite3DbStrDup(db, pCte->zName); + pTab->iPKey = -1; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; + pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + if( db->mallocFailed ) return 2; + pFrom->pSelect->selFlags |= SF_CopyCte; + assert( pFrom->pSelect ); + pFrom->fg.isCte = 1; + pFrom->u2.pCteUse = pCteUse; + pCteUse->nUse++; + if( pCteUse->nUse>=2 && pCteUse->eM10d==M10d_Any ){ + pCteUse->eM10d = M10d_Yes; + } + + /* Check if this is a recursive CTE. */ + pRecTerm = pSel = pFrom->pSelect; + bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); + while( bMayRecursive && pRecTerm->op==pSel->op ){ + int i; + SrcList *pSrc = pRecTerm->pSrc; + assert( pRecTerm->pPrior!=0 ); + for(i=0; inSrc; i++){ + SrcItem *pItem = &pSrc->a[i]; + if( pItem->zDatabase==0 + && pItem->zName!=0 + && 0==sqlite3StrICmp(pItem->zName, pCte->zName) + ){ + pItem->pTab = pTab; + pTab->nTabRef++; + pItem->fg.isRecursive = 1; + if( pRecTerm->selFlags & SF_Recursive ){ + sqlite3ErrorMsg(pParse, + "multiple references to recursive table: %s", pCte->zName + ); + return 2; + } + pRecTerm->selFlags |= SF_Recursive; + if( iRecTab<0 ) iRecTab = pParse->nTab++; + pItem->iCursor = iRecTab; + } + } + if( (pRecTerm->selFlags & SF_Recursive)==0 ) break; + pRecTerm = pRecTerm->pPrior; + } + + pCte->zCteErr = "circular reference: %s"; + pSavedWith = pParse->pWith; + pParse->pWith = pWith; + if( pSel->selFlags & SF_Recursive ){ + int rc; + assert( pRecTerm!=0 ); + assert( (pRecTerm->selFlags & SF_Recursive)==0 ); + assert( pRecTerm->pNext!=0 ); + assert( (pRecTerm->pNext->selFlags & SF_Recursive)!=0 ); + assert( pRecTerm->pWith==0 ); + pRecTerm->pWith = pSel->pWith; + rc = sqlite3WalkSelect(pWalker, pRecTerm); + pRecTerm->pWith = 0; + if( rc ){ + pParse->pWith = pSavedWith; + return 2; + } + }else{ + if( sqlite3WalkSelect(pWalker, pSel) ){ + pParse->pWith = pSavedWith; + return 2; + } + } + pParse->pWith = pWith; + + for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); + pEList = pLeft->pEList; + if( pCte->pCols ){ + if( pEList && pEList->nExpr!=pCte->pCols->nExpr ){ + sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns", + pCte->zName, pEList->nExpr, pCte->pCols->nExpr + ); + pParse->pWith = pSavedWith; + return 2; + } + pEList = pCte->pCols; + } + + sqlite3ColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + if( bMayRecursive ){ + if( pSel->selFlags & SF_Recursive ){ + pCte->zCteErr = "multiple recursive references: %s"; + }else{ + pCte->zCteErr = "recursive reference in a subquery: %s"; + } + sqlite3WalkSelect(pWalker, pSel); + } + pCte->zCteErr = 0; + pParse->pWith = pSavedWith; + return 1; /* Success */ + } + return 0; /* No match */ +} +#endif + +#ifndef SQLITE_OMIT_CTE +/* +** If the SELECT passed as the second argument has an associated WITH +** clause, pop it from the stack stored as part of the Parse object. +** +** This function is used as the xSelectCallback2() callback by +** sqlite3SelectExpand() when walking a SELECT tree to resolve table +** names and other FROM clause elements. +*/ +SQLITE_PRIVATE void sqlite3SelectPopWith(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + if( OK_IF_ALWAYS_TRUE(pParse->pWith) && p->pPrior==0 ){ + With *pWith = findRightmost(p)->pWith; + if( pWith!=0 ){ + assert( pParse->pWith==pWith || pParse->nErr ); + pParse->pWith = pWith->pOuter; + } + } +} +#endif + +/* +** The SrcList_item structure passed as the second argument represents a +** sub-query in the FROM clause of a SELECT statement. This function +** allocates and populates the SrcList_item.pTab object. If successful, +** SQLITE_OK is returned. Otherwise, if an OOM error is encountered, +** SQLITE_NOMEM. +*/ +SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ + Select *pSel = pFrom->pSelect; + Table *pTab; + + assert( pSel ); + pFrom->pTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); + if( pTab==0 ) return SQLITE_NOMEM; + pTab->nTabRef = 1; + if( pFrom->zAlias ){ + pTab->zName = sqlite3DbStrDup(pParse->db, pFrom->zAlias); + }else{ + pTab->zName = sqlite3MPrintf(pParse->db, "subquery_%u", pSel->selId); + } + while( pSel->pPrior ){ pSel = pSel->pPrior; } + sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); + pTab->iPKey = -1; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); +#ifndef SQLITE_ALLOW_ROWID_IN_VIEW + /* The usual case - do not allow ROWID on a subquery */ + pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; +#else + pTab->tabFlags |= TF_Ephemeral; /* Legacy compatibility mode */ +#endif + + + return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; +} + +/* +** This routine is a Walker callback for "expanding" a SELECT statement. +** "Expanding" means to do the following: +** +** (1) Make sure VDBE cursor numbers have been assigned to every +** element of the FROM clause. +** +** (2) Fill in the pTabList->a[].pTab fields in the SrcList that +** defines FROM clause. When views appear in the FROM clause, +** fill pTabList->a[].pSelect with a copy of the SELECT statement +** that implements the view. A copy is made of the view's SELECT +** statement so that we can freely modify or delete that statement +** without worrying about messing up the persistent representation +** of the view. +** +** (3) Add terms to the WHERE clause to accommodate the NATURAL keyword +** on joins and the ON and USING clause of joins. +** +** (4) Scan the list of columns in the result set (pEList) looking +** for instances of the "*" operator or the TABLE.* operator. +** If found, expand each "*" to be every column in every table +** and TABLE.* to be every column in TABLE. +** +*/ +static int selectExpander(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + int i, j, k, rc; + SrcList *pTabList; + ExprList *pEList; + SrcItem *pFrom; + sqlite3 *db = pParse->db; + Expr *pE, *pRight, *pExpr; + u16 selFlags = p->selFlags; + u32 elistFlags = 0; + + p->selFlags |= SF_Expanded; + if( db->mallocFailed ){ + return WRC_Abort; + } + assert( p->pSrc!=0 ); + if( (selFlags & SF_Expanded)!=0 ){ + return WRC_Prune; + } + if( pWalker->eCode ){ + /* Renumber selId because it has been copied from a view */ + p->selId = ++pParse->nSelect; + } + pTabList = p->pSrc; + pEList = p->pEList; + if( pParse->pWith && (p->selFlags & SF_View) ){ + if( p->pWith==0 ){ + p->pWith = (With*)sqlite3DbMallocZero(db, sizeof(With)); + if( p->pWith==0 ){ + return WRC_Abort; + } + } + p->pWith->bView = 1; + } + sqlite3WithPush(pParse, p->pWith, 0); + + /* Make sure cursor numbers have been assigned to all entries in + ** the FROM clause of the SELECT statement. + */ + sqlite3SrcListAssignCursors(pParse, pTabList); + + /* Look up every table named in the FROM clause of the select. If + ** an entry of the FROM clause is a subquery instead of a table or view, + ** then create a transient table structure to describe the subquery. + */ + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab; + assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); + if( pFrom->pTab ) continue; + assert( pFrom->fg.isRecursive==0 ); + if( pFrom->zName==0 ){ +#ifndef SQLITE_OMIT_SUBQUERY + Select *pSel = pFrom->pSelect; + /* A sub-query in the FROM clause of a SELECT */ + assert( pSel!=0 ); + assert( pFrom->pTab==0 ); + if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; + if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort; +#endif +#ifndef SQLITE_OMIT_CTE + }else if( (rc = resolveFromTermToCte(pParse, pWalker, pFrom))!=0 ){ + if( rc>1 ) return WRC_Abort; + pTab = pFrom->pTab; + assert( pTab!=0 ); +#endif + }else{ + /* An ordinary table or view name in the FROM clause */ + assert( pFrom->pTab==0 ); + pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); + if( pTab==0 ) return WRC_Abort; + if( pTab->nTabRef>=0xffff ){ + sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", + pTab->zName); + pFrom->pTab = 0; + return WRC_Abort; + } + pTab->nTabRef++; + if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){ + return WRC_Abort; + } +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) + if( !IsOrdinaryTable(pTab) ){ + i16 nCol; + u8 eCodeOrig = pWalker->eCode; + if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; + assert( pFrom->pSelect==0 ); + if( IsView(pTab) ){ + if( (db->flags & SQLITE_EnableView)==0 + && pTab->pSchema!=db->aDb[1].pSchema + ){ + sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited", + pTab->zName); + } + pFrom->pSelect = sqlite3SelectDup(db, pTab->u.view.pSelect, 0); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + else if( ALWAYS(IsVirtual(pTab)) + && pFrom->fg.fromDDL + && ALWAYS(pTab->u.vtab.p!=0) + && pTab->u.vtab.p->eVtabRisk > ((db->flags & SQLITE_TrustedSchema)!=0) + ){ + sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"", + pTab->zName); + } + assert( SQLITE_VTABRISK_Normal==1 && SQLITE_VTABRISK_High==2 ); +#endif + nCol = pTab->nCol; + pTab->nCol = -1; + pWalker->eCode = 1; /* Turn on Select.selId renumbering */ + sqlite3WalkSelect(pWalker, pFrom->pSelect); + pWalker->eCode = eCodeOrig; + pTab->nCol = nCol; + } +#endif + } + + /* Locate the index named by the INDEXED BY clause, if any. */ + if( pFrom->fg.isIndexedBy && sqlite3IndexedByLookup(pParse, pFrom) ){ + return WRC_Abort; + } + } + + /* Process NATURAL keywords, and ON and USING clauses of joins. + */ + if( pParse->nErr || db->mallocFailed || sqliteProcessJoin(pParse, p) ){ + return WRC_Abort; + } + + /* For every "*" that occurs in the column list, insert the names of + ** all columns in all tables. And for every TABLE.* insert the names + ** of all columns in TABLE. The parser inserted a special expression + ** with the TK_ASTERISK operator for each "*" that it found in the column + ** list. The following code just has to locate the TK_ASTERISK + ** expressions and expand each one to the list of all columns in + ** all tables. + ** + ** The first loop just checks to see if there are any "*" operators + ** that need expanding. + */ + for(k=0; knExpr; k++){ + pE = pEList->a[k].pExpr; + if( pE->op==TK_ASTERISK ) break; + assert( pE->op!=TK_DOT || pE->pRight!=0 ); + assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); + if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break; + elistFlags |= pE->flags; + } + if( knExpr ){ + /* + ** If we get here it means the result set contains one or more "*" + ** operators that need to be expanded. Loop through each expression + ** in the result set and expand them one by one. + */ + struct ExprList_item *a = pEList->a; + ExprList *pNew = 0; + int flags = pParse->db->flags; + int longNames = (flags & SQLITE_FullColNames)!=0 + && (flags & SQLITE_ShortColNames)==0; + + for(k=0; knExpr; k++){ + pE = a[k].pExpr; + elistFlags |= pE->flags; + pRight = pE->pRight; + assert( pE->op!=TK_DOT || pRight!=0 ); + if( pE->op!=TK_ASTERISK + && (pE->op!=TK_DOT || pRight->op!=TK_ASTERISK) + ){ + /* This particular expression does not need to be expanded. + */ + pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); + if( pNew ){ + pNew->a[pNew->nExpr-1].zEName = a[k].zEName; + pNew->a[pNew->nExpr-1].eEName = a[k].eEName; + a[k].zEName = 0; + } + a[k].pExpr = 0; + }else{ + /* This expression is a "*" or a "TABLE.*" and needs to be + ** expanded. */ + int tableSeen = 0; /* Set to 1 when TABLE matches */ + char *zTName = 0; /* text of name of TABLE */ + if( pE->op==TK_DOT ){ + assert( pE->pLeft!=0 ); + assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); + zTName = pE->pLeft->u.zToken; + } + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + Select *pSub = pFrom->pSelect; + char *zTabName = pFrom->zAlias; + const char *zSchemaName = 0; + int iDb; + if( zTabName==0 ){ + zTabName = pTab->zName; + } + if( db->mallocFailed ) break; + if( pSub==0 || (pSub->selFlags & SF_NestedFrom)==0 ){ + pSub = 0; + if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ + continue; + } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zSchemaName = iDb>=0 ? db->aDb[iDb].zDbSName : "*"; + } + for(j=0; jnCol; j++){ + char *zName = pTab->aCol[j].zCnName; + char *zColname; /* The computed column name */ + char *zToFree; /* Malloced string that needs to be freed */ + Token sColname; /* Computed column name as a token */ + + assert( zName ); + if( zTName && pSub + && sqlite3MatchEName(&pSub->pEList->a[j], 0, zTName, 0)==0 + ){ + continue; + } + + /* If a column is marked as 'hidden', omit it from the expanded + ** result-set list unless the SELECT has the SF_IncludeHidden + ** bit set. + */ + if( (p->selFlags & SF_IncludeHidden)==0 + && IsHiddenColumn(&pTab->aCol[j]) + ){ + continue; + } + tableSeen = 1; + + if( i>0 && zTName==0 ){ + if( (pFrom->fg.jointype & JT_NATURAL)!=0 + && tableAndColumnIndex(pTabList, i, zName, 0, 0, 1) + ){ + /* In a NATURAL join, omit the join columns from the + ** table to the right of the join */ + continue; + } + if( sqlite3IdListIndex(pFrom->pUsing, zName)>=0 ){ + /* In a join with a USING clause, omit columns in the + ** using clause from the table on the right. */ + continue; + } + } + pRight = sqlite3Expr(db, TK_ID, zName); + zColname = zName; + zToFree = 0; + if( longNames || pTabList->nSrc>1 ){ + Expr *pLeft; + pLeft = sqlite3Expr(db, TK_ID, zTabName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); + if( zSchemaName ){ + pLeft = sqlite3Expr(db, TK_ID, zSchemaName); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr); + } + if( longNames ){ + zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); + zToFree = zColname; + } + }else{ + pExpr = pRight; + } + pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); + sqlite3TokenInit(&sColname, zColname); + sqlite3ExprListSetName(pParse, pNew, &sColname, 0); + if( pNew && (p->selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ + struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; + sqlite3DbFree(db, pX->zEName); + if( pSub ){ + pX->zEName = sqlite3DbStrDup(db, pSub->pEList->a[j].zEName); + testcase( pX->zEName==0 ); + }else{ + pX->zEName = sqlite3MPrintf(db, "%s.%s.%s", + zSchemaName, zTabName, zColname); + testcase( pX->zEName==0 ); + } + pX->eEName = ENAME_TAB; + } + sqlite3DbFree(db, zToFree); + } + } + if( !tableSeen ){ + if( zTName ){ + sqlite3ErrorMsg(pParse, "no such table: %s", zTName); + }else{ + sqlite3ErrorMsg(pParse, "no tables specified"); + } + } + } + } + sqlite3ExprListDelete(db, pEList); + p->pEList = pNew; + } + if( p->pEList ){ + if( p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns in result set"); + return WRC_Abort; + } + if( (elistFlags & (EP_HasFunc|EP_Subquery))!=0 ){ + p->selFlags |= SF_ComplexResult; + } + } + return WRC_Continue; +} + +#if SQLITE_DEBUG +/* +** Always assert. This xSelectCallback2 implementation proves that the +** xSelectCallback2 is never invoked. +*/ +SQLITE_PRIVATE void sqlite3SelectWalkAssert2(Walker *NotUsed, Select *NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + assert( 0 ); +} +#endif +/* +** This routine "expands" a SELECT statement and all of its subqueries. +** For additional information on what it means to "expand" a SELECT +** statement, see the comment on the selectExpand worker callback above. +** +** Expanding a SELECT statement is the first step in processing a +** SELECT statement. The SELECT statement must be expanded before +** name resolution is performed. +** +** If anything goes wrong, an error message is written into pParse. +** The calling function can detect the problem by looking at pParse->nErr +** and/or pParse->db->mallocFailed. +*/ +static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ + Walker w; + w.xExprCallback = sqlite3ExprWalkNoop; + w.pParse = pParse; + if( OK_IF_ALWAYS_TRUE(pParse->hasCompound) ){ + w.xSelectCallback = convertCompoundSelectToSubquery; + w.xSelectCallback2 = 0; + sqlite3WalkSelect(&w, pSelect); + } + w.xSelectCallback = selectExpander; + w.xSelectCallback2 = sqlite3SelectPopWith; + w.eCode = 0; + sqlite3WalkSelect(&w, pSelect); +} + + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo() +** interface. +** +** For each FROM-clause subquery, add Column.zType and Column.zColl +** information to the Table structure that represents the result set +** of that subquery. +** +** The Table structure that represents the result set was constructed +** by selectExpander() but the type and collation information was omitted +** at that point because identifiers had not yet been resolved. This +** routine is called after identifier resolution. +*/ +static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ + Parse *pParse; + int i; + SrcList *pTabList; + SrcItem *pFrom; + + assert( p->selFlags & SF_Resolved ); + if( p->selFlags & SF_HasTypeInfo ) return; + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + assert( pTab!=0 ); + if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel, + SQLITE_AFF_NONE); + } + } + } +} +#endif + + +/* +** This routine adds datatype and collating sequence information to +** the Table structures of all FROM-clause subqueries in a +** SELECT statement. +** +** Use this routine after name resolution. +*/ +static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ +#ifndef SQLITE_OMIT_SUBQUERY + Walker w; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.xSelectCallback2 = selectAddSubqueryTypeInfo; + w.xExprCallback = sqlite3ExprWalkNoop; + w.pParse = pParse; + sqlite3WalkSelect(&w, pSelect); +#endif +} + + +/* +** This routine sets up a SELECT statement for processing. The +** following is accomplished: +** +** * VDBE Cursor numbers are assigned to all FROM-clause terms. +** * Ephemeral Table objects are created for all FROM-clause subqueries. +** * ON and USING clauses are shifted into WHERE statements +** * Wildcards "*" and "TABLE.*" in result sets are expanded. +** * Identifiers in expression are matched to tables. +** +** This routine acts recursively on all subqueries within the SELECT. +*/ +SQLITE_PRIVATE void sqlite3SelectPrep( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + NameContext *pOuterNC /* Name context for container */ +){ + assert( p!=0 || pParse->db->mallocFailed ); + if( pParse->db->mallocFailed ) return; + if( p->selFlags & SF_HasTypeInfo ) return; + sqlite3SelectExpand(pParse, p); + if( pParse->nErr || pParse->db->mallocFailed ) return; + sqlite3ResolveSelectNames(pParse, p, pOuterNC); + if( pParse->nErr || pParse->db->mallocFailed ) return; + sqlite3SelectAddTypeInfo(pParse, p); +} + +/* +** Reset the aggregate accumulator. +** +** The aggregate accumulator is a set of memory cells that hold +** intermediate results while calculating an aggregate. This +** routine generates code that stores NULLs in all of those memory +** cells. +*/ +static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ + Vdbe *v = pParse->pVdbe; + int i; + struct AggInfo_func *pFunc; + int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + if( nReg==0 ) return; + if( pParse->nErr || pParse->db->mallocFailed ) return; +#ifdef SQLITE_DEBUG + /* Verify that all AggInfo registers are within the range specified by + ** AggInfo.mnReg..AggInfo.mxReg */ + assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); + for(i=0; inColumn; i++){ + assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg + && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); + } + for(i=0; inFunc; i++){ + assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg + && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); + } +#endif + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); + for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ + if( pFunc->iDistinct>=0 ){ + Expr *pE = pFunc->pFExpr; + assert( !ExprHasProperty(pE, EP_xIsSelect) ); + if( pE->x.pList==0 || pE->x.pList->nExpr!=1 ){ + sqlite3ErrorMsg(pParse, "DISTINCT aggregates must have exactly one " + "argument"); + pFunc->iDistinct = -1; + }else{ + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pE->x.pList,0,0); + pFunc->iDistAddr = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); + ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s(DISTINCT)", + pFunc->pFunc->zName)); + } + } + } +} + +/* +** Invoke the OP_AggFinalize opcode for every aggregate function +** in the AggInfo structure. +*/ +static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ + Vdbe *v = pParse->pVdbe; + int i; + struct AggInfo_func *pF; + for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ + ExprList *pList = pF->pFExpr->x.pList; + assert( !ExprHasProperty(pF->pFExpr, EP_xIsSelect) ); + sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + } +} + + +/* +** Update the accumulator memory cells for an aggregate based on +** the current cursor position. +** +** If regAcc is non-zero and there are no min() or max() aggregates +** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator +** registers if register regAcc contains 0. The caller will take care +** of setting and clearing regAcc. +*/ +static void updateAccumulator( + Parse *pParse, + int regAcc, + AggInfo *pAggInfo, + int eDistinctType +){ + Vdbe *v = pParse->pVdbe; + int i; + int regHit = 0; + int addrHitTest = 0; + struct AggInfo_func *pF; + struct AggInfo_col *pC; + + pAggInfo->directMode = 1; + for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ + int nArg; + int addrNext = 0; + int regAgg; + ExprList *pList = pF->pFExpr->x.pList; + assert( !ExprHasProperty(pF->pFExpr, EP_xIsSelect) ); + assert( !IsWindowFunc(pF->pFExpr) ); + if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){ + Expr *pFilter = pF->pFExpr->y.pWin->pFilter; + if( pAggInfo->nAccumulator + && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + && regAcc + ){ + /* If regAcc==0, there there exists some min() or max() function + ** without a FILTER clause that will ensure the magnet registers + ** are populated. */ + if( regHit==0 ) regHit = ++pParse->nMem; + /* If this is the first row of the group (regAcc contains 0), clear the + ** "magnet" register regHit so that the accumulator registers + ** are populated if the FILTER clause jumps over the the + ** invocation of min() or max() altogether. Or, if this is not + ** the first row (regAcc contains 1), set the magnet register so that + ** the accumulators are not populated unless the min()/max() is invoked + ** and indicates that they should be. */ + sqlite3VdbeAddOp2(v, OP_Copy, regAcc, regHit); + } + addrNext = sqlite3VdbeMakeLabel(pParse); + sqlite3ExprIfFalse(pParse, pFilter, addrNext, SQLITE_JUMPIFNULL); + } + if( pList ){ + nArg = pList->nExpr; + regAgg = sqlite3GetTempRange(pParse, nArg); + sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); + }else{ + nArg = 0; + regAgg = 0; + } + if( pF->iDistinct>=0 && pList ){ + if( addrNext==0 ){ + addrNext = sqlite3VdbeMakeLabel(pParse); + } + pF->iDistinct = codeDistinct(pParse, eDistinctType, + pF->iDistinct, addrNext, pList, regAgg); + } + if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl = 0; + struct ExprList_item *pItem; + int j; + assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ + for(j=0, pItem=pList->a; !pColl && jpExpr); + } + if( !pColl ){ + pColl = pParse->db->pDfltColl; + } + if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); + } + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); + if( addrNext ){ + sqlite3VdbeResolveLabel(v, addrNext); + } + } + if( regHit==0 && pAggInfo->nAccumulator ){ + regHit = regAcc; + } + if( regHit ){ + addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); + } + for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ + sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem); + } + + pAggInfo->directMode = 0; + if( addrHitTest ){ + sqlite3VdbeJumpHereOrPopInst(v, addrHitTest); + } +} + +/* +** Add a single OP_Explain instruction to the VDBE to explain a simple +** count(*) query ("SELECT count(*) FROM pTab"). +*/ +#ifndef SQLITE_OMIT_EXPLAIN +static void explainSimpleCount( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being queried */ + Index *pIdx /* Index used to optimize scan, or NULL */ +){ + if( pParse->explain==2 ){ + int bCover = (pIdx!=0 && (HasRowid(pTab) || !IsPrimaryKeyIndex(pIdx))); + sqlite3VdbeExplain(pParse, 0, "SCAN %s%s%s", + pTab->zName, + bCover ? " USING COVERING INDEX " : "", + bCover ? pIdx->zName : "" + ); + } +} +#else +# define explainSimpleCount(a,b,c) +#endif + +/* +** sqlite3WalkExpr() callback used by havingToWhere(). +** +** If the node passed to the callback is a TK_AND node, return +** WRC_Continue to tell sqlite3WalkExpr() to iterate through child nodes. +** +** Otherwise, return WRC_Prune. In this case, also check if the +** sub-expression matches the criteria for being moved to the WHERE +** clause. If so, add it to the WHERE clause and replace the sub-expression +** within the HAVING expression with a constant "1". +*/ +static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ + if( pExpr->op!=TK_AND ){ + Select *pS = pWalker->u.pSelect; + /* This routine is called before the HAVING clause of the current + ** SELECT is analyzed for aggregates. So if pExpr->pAggInfo is set + ** here, it indicates that the expression is a correlated reference to a + ** column from an outer aggregate query, or an aggregate function that + ** belongs to an outer query. Do not move the expression to the WHERE + ** clause in this obscure case, as doing so may corrupt the outer Select + ** statements AggInfo structure. */ + if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy) + && ExprAlwaysFalse(pExpr)==0 + && pExpr->pAggInfo==0 + ){ + sqlite3 *db = pWalker->pParse->db; + Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1"); + if( pNew ){ + Expr *pWhere = pS->pWhere; + SWAP(Expr, *pNew, *pExpr); + pNew = sqlite3ExprAnd(pWalker->pParse, pWhere, pNew); + pS->pWhere = pNew; + pWalker->eCode = 1; + } + } + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Transfer eligible terms from the HAVING clause of a query, which is +** processed after grouping, to the WHERE clause, which is processed before +** grouping. For example, the query: +** +** SELECT * FROM WHERE a=? GROUP BY b HAVING b=? AND c=? +** +** can be rewritten as: +** +** SELECT * FROM WHERE a=? AND b=? GROUP BY b HAVING c=? +** +** A term of the HAVING expression is eligible for transfer if it consists +** entirely of constants and expressions that are also GROUP BY terms that +** use the "BINARY" collation sequence. +*/ +static void havingToWhere(Parse *pParse, Select *p){ + Walker sWalker; + memset(&sWalker, 0, sizeof(sWalker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = havingToWhereExprCb; + sWalker.u.pSelect = p; + sqlite3WalkExpr(&sWalker, p->pHaving); +#if SELECTTRACE_ENABLED + if( sWalker.eCode && (sqlite3SelectTrace & 0x100)!=0 ){ + SELECTTRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif +} + +/* +** Check to see if the pThis entry of pTabList is a self-join of a prior view. +** If it is, then return the SrcList_item for the prior view. If it is not, +** then return 0. +*/ +static SrcItem *isSelfJoinView( + SrcList *pTabList, /* Search for self-joins in this FROM clause */ + SrcItem *pThis /* Search for prior reference to this subquery */ +){ + SrcItem *pItem; + assert( pThis->pSelect!=0 ); + if( pThis->pSelect->selFlags & SF_PushDown ) return 0; + for(pItem = pTabList->a; pItempSelect==0 ) continue; + if( pItem->fg.viaCoroutine ) continue; + if( pItem->zName==0 ) continue; + assert( pItem->pTab!=0 ); + assert( pThis->pTab!=0 ); + if( pItem->pTab->pSchema!=pThis->pTab->pSchema ) continue; + if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue; + pS1 = pItem->pSelect; + if( pItem->pTab->pSchema==0 && pThis->pSelect->selId!=pS1->selId ){ + /* The query flattener left two different CTE tables with identical + ** names in the same FROM clause. */ + continue; + } + if( pItem->pSelect->selFlags & SF_PushDown ){ + /* The view was modified by some other optimization such as + ** pushDownWhereTerms() */ + continue; + } + return pItem; + } + return 0; +} + +/* +** Deallocate a single AggInfo object +*/ +static void agginfoFree(sqlite3 *db, AggInfo *p){ + sqlite3DbFree(db, p->aCol); + sqlite3DbFree(db, p->aFunc); + sqlite3DbFreeNN(db, p); +} + +#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION +/* +** Attempt to transform a query of the form +** +** SELECT count(*) FROM (SELECT x FROM t1 UNION ALL SELECT y FROM t2) +** +** Into this: +** +** SELECT (SELECT count(*) FROM t1)+(SELECT count(*) FROM t2) +** +** The transformation only works if all of the following are true: +** +** * The subquery is a UNION ALL of two or more terms +** * The subquery does not have a LIMIT clause +** * There is no WHERE or GROUP BY or HAVING clauses on the subqueries +** * The outer query is a simple count(*) with no WHERE clause or other +** extraneous syntax. +** +** Return TRUE if the optimization is undertaken. +*/ +static int countOfViewOptimization(Parse *pParse, Select *p){ + Select *pSub, *pPrior; + Expr *pExpr; + Expr *pCount; + sqlite3 *db; + if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ + if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ + if( p->pWhere ) return 0; + if( p->pGroupBy ) return 0; + pExpr = p->pEList->a[0].pExpr; + if( pExpr->op!=TK_AGG_FUNCTION ) return 0; /* Result is an aggregate */ + if( sqlite3_stricmp(pExpr->u.zToken,"count") ) return 0; /* Is count() */ + if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */ + if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */ + pSub = p->pSrc->a[0].pSelect; + if( pSub==0 ) return 0; /* The FROM is a subquery */ + if( pSub->pPrior==0 ) return 0; /* Must be a compound ry */ + do{ + if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */ + if( pSub->pWhere ) return 0; /* No WHERE clause */ + if( pSub->pLimit ) return 0; /* No LIMIT clause */ + if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ + pSub = pSub->pPrior; /* Repeat over compound */ + }while( pSub ); + + /* If we reach this point then it is OK to perform the transformation */ + + db = pParse->db; + pCount = pExpr; + pExpr = 0; + pSub = p->pSrc->a[0].pSelect; + p->pSrc->a[0].pSelect = 0; + sqlite3SrcListDelete(db, p->pSrc); + p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc)); + while( pSub ){ + Expr *pTerm; + pPrior = pSub->pPrior; + pSub->pPrior = 0; + pSub->pNext = 0; + pSub->selFlags |= SF_Aggregate; + pSub->selFlags &= ~SF_Compound; + pSub->nSelectRow = 0; + sqlite3ExprListDelete(db, pSub->pEList); + pTerm = pPrior ? sqlite3ExprDup(db, pCount, 0) : pCount; + pSub->pEList = sqlite3ExprListAppend(pParse, 0, pTerm); + pTerm = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, pTerm, pSub); + if( pExpr==0 ){ + pExpr = pTerm; + }else{ + pExpr = sqlite3PExpr(pParse, TK_PLUS, pTerm, pExpr); + } + pSub = pPrior; + } + p->pEList->a[0].pExpr = pExpr; + p->selFlags &= ~SF_Aggregate; + +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("After count-of-view optimization:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + return 1; +} +#endif /* SQLITE_COUNTOFVIEW_OPTIMIZATION */ + +/* +** Generate code for the SELECT statement given in the p argument. +** +** The results are returned according to the SelectDest structure. +** See comments in sqliteInt.h for further information. +** +** This routine returns the number of errors. If any errors are +** encountered, then an appropriate error message is left in +** pParse->zErrMsg. +** +** This routine does NOT free the Select structure passed in. The +** calling function needs to do that. +*/ +SQLITE_PRIVATE int sqlite3Select( + Parse *pParse, /* The parser context */ + Select *p, /* The SELECT statement being coded. */ + SelectDest *pDest /* What to do with the query results */ +){ + int i, j; /* Loop counters */ + WhereInfo *pWInfo; /* Return from sqlite3WhereBegin() */ + Vdbe *v; /* The virtual machine under construction */ + int isAgg; /* True for select lists like "count(*)" */ + ExprList *pEList = 0; /* List of columns to extract. */ + SrcList *pTabList; /* List of tables to select from */ + Expr *pWhere; /* The WHERE clause. May be NULL */ + ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ + Expr *pHaving; /* The HAVING clause. May be NULL */ + AggInfo *pAggInfo = 0; /* Aggregate information */ + int rc = 1; /* Value to return from this function */ + DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ + SortCtx sSort; /* Info on how to code the ORDER BY clause */ + int iEnd; /* Address of the end of the query */ + sqlite3 *db; /* The database connection */ + ExprList *pMinMaxOrderBy = 0; /* Added ORDER BY for min/max queries */ + u8 minMaxFlag; /* Flag for min/max queries */ + + db = pParse->db; + v = sqlite3GetVdbe(pParse); + if( p==0 || db->mallocFailed || pParse->nErr ){ + return 1; + } + if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p, ("begin processing:\n", pParse->addrExplain)); + if( sqlite3SelectTrace & 0x100 ){ + sqlite3TreeViewSelect(0, p, 0); + } +#endif + + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); + if( IgnorableDistinct(pDest) ){ + assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || + pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || + pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo ); + /* All of these destinations are also able to ignore the ORDER BY clause */ + if( p->pOrderBy ){ +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p, ("dropping superfluous ORDER BY:\n")); + if( sqlite3SelectTrace & 0x100 ){ + sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY"); + } +#endif + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))sqlite3ExprListDelete, + p->pOrderBy); + testcase( pParse->earlyCleanup ); + p->pOrderBy = 0; + } + p->selFlags &= ~SF_Distinct; + p->selFlags |= SF_NoopOrderBy; + } + sqlite3SelectPrep(pParse, p, 0); + if( pParse->nErr || db->mallocFailed ){ + goto select_end; + } + assert( p->pEList!=0 ); +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x104 ){ + SELECTTRACE(0x104,pParse,p, ("after name resolution:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + + /* If the SF_UFSrcCheck flag is set, then this function is being called + ** as part of populating the temp table for an UPDATE...FROM statement. + ** In this case, it is an error if the target object (pSrc->a[0]) name + ** or alias is duplicated within FROM clause (pSrc->a[1..n]). + ** + ** Postgres disallows this case too. The reason is that some other + ** systems handle this case differently, and not all the same way, + ** which is just confusing. To avoid this, we follow PG's lead and + ** disallow it altogether. */ + if( p->selFlags & SF_UFSrcCheck ){ + SrcItem *p0 = &p->pSrc->a[0]; + for(i=1; ipSrc->nSrc; i++){ + SrcItem *p1 = &p->pSrc->a[i]; + if( p0->pTab==p1->pTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ + sqlite3ErrorMsg(pParse, + "target object/alias may not appear in FROM clause: %s", + p0->zAlias ? p0->zAlias : p0->pTab->zName + ); + goto select_end; + } + } + + /* Clear the SF_UFSrcCheck flag. The check has already been performed, + ** and leaving this flag set can cause errors if a compound sub-query + ** in p->pSrc is flattened into this query and this function called + ** again as part of compound SELECT processing. */ + p->selFlags &= ~SF_UFSrcCheck; + } + + if( pDest->eDest==SRT_Output ){ + sqlite3GenerateColumnNames(pParse, p); + } + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( sqlite3WindowRewrite(pParse, p) ){ + assert( db->mallocFailed || pParse->nErr>0 ); + goto select_end; + } +#if SELECTTRACE_ENABLED + if( p->pWin && (sqlite3SelectTrace & 0x108)!=0 ){ + SELECTTRACE(0x104,pParse,p, ("after window rewrite:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif +#endif /* SQLITE_OMIT_WINDOWFUNC */ + pTabList = p->pSrc; + isAgg = (p->selFlags & SF_Aggregate)!=0; + memset(&sSort, 0, sizeof(sSort)); + sSort.pOrderBy = p->pOrderBy; + + /* Try to do various optimizations (flattening subqueries, and strength + ** reduction of join operators) in the FROM clause up into the main query + */ +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + for(i=0; !p->pPrior && inSrc; i++){ + SrcItem *pItem = &pTabList->a[i]; + Select *pSub = pItem->pSelect; + Table *pTab = pItem->pTab; + + /* The expander should have already created transient Table objects + ** even for FROM clause elements such as subqueries that do not correspond + ** to a real table */ + assert( pTab!=0 ); + + /* Convert LEFT JOIN into JOIN if there are terms of the right table + ** of the LEFT JOIN used in the WHERE clause. + */ + if( (pItem->fg.jointype & JT_LEFT)!=0 + && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor) + && OptimizationEnabled(db, SQLITE_SimplifyJoin) + ){ + SELECTTRACE(0x100,pParse,p, + ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); + pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); + unsetJoinExpr(p->pWhere, pItem->iCursor); + } + + /* No futher action if this term of the FROM clause is no a subquery */ + if( pSub==0 ) continue; + + /* Catch mismatch in the declared columns of a view and the number of + ** columns in the SELECT on the RHS */ + if( pTab->nCol!=pSub->pEList->nExpr ){ + sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d", + pTab->nCol, pTab->zName, pSub->pEList->nExpr); + goto select_end; + } + + /* Do not try to flatten an aggregate subquery. + ** + ** Flattening an aggregate subquery is only possible if the outer query + ** is not a join. But if the outer query is not a join, then the subquery + ** will be implemented as a co-routine and there is no advantage to + ** flattening in that case. + */ + if( (pSub->selFlags & SF_Aggregate)!=0 ) continue; + assert( pSub->pGroupBy==0 ); + + /* If a FROM-clause subquery has an ORDER BY clause that is not + ** really doing anything, then delete it now so that it does not + ** interfere with query flattening. See the discussion at + ** https://sqlite.org/forum/forumpost/2d76f2bcf65d256a + ** + ** Beware of these cases where the ORDER BY clause may not be safely + ** omitted: + ** + ** (1) There is also a LIMIT clause + ** (2) The subquery was added to help with window-function + ** processing + ** (3) The subquery is in the FROM clause of an UPDATE + ** (4) The outer query uses an aggregate function other than + ** the built-in count(), min(), or max(). + ** (5) The ORDER BY isn't going to accomplish anything because + ** one of: + ** (a) The outer query has a different ORDER BY clause + ** (b) The subquery is part of a join + ** See forum post 062d576715d277c8 + */ + if( pSub->pOrderBy!=0 + && (p->pOrderBy!=0 || pTabList->nSrc>1) /* Condition (5) */ + && pSub->pLimit==0 /* Condition (1) */ + && (pSub->selFlags & SF_OrderByReqd)==0 /* Condition (2) */ + && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */ + && OptimizationEnabled(db, SQLITE_OmitOrderBy) + ){ + SELECTTRACE(0x100,pParse,p, + ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1)); + sqlite3ExprListDelete(db, pSub->pOrderBy); + pSub->pOrderBy = 0; + } + + /* If the outer query contains a "complex" result set (that is, + ** if the result set of the outer query uses functions or subqueries) + ** and if the subquery contains an ORDER BY clause and if + ** it will be implemented as a co-routine, then do not flatten. This + ** restriction allows SQL constructs like this: + ** + ** SELECT expensive_function(x) + ** FROM (SELECT x FROM tab ORDER BY y LIMIT 10); + ** + ** The expensive_function() is only computed on the 10 rows that + ** are output, rather than every row of the table. + ** + ** The requirement that the outer query have a complex result set + ** means that flattening does occur on simpler SQL constraints without + ** the expensive_function() like: + ** + ** SELECT x FROM (SELECT x FROM tab ORDER BY y LIMIT 10); + */ + if( pSub->pOrderBy!=0 + && i==0 + && (p->selFlags & SF_ComplexResult)!=0 + && (pTabList->nSrc==1 + || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) + ){ + continue; + } + + if( flattenSubquery(pParse, p, i, isAgg) ){ + if( pParse->nErr ) goto select_end; + /* This subquery can be absorbed into its parent. */ + i = -1; + } + pTabList = p->pSrc; + if( db->mallocFailed ) goto select_end; + if( !IgnorableOrderby(pDest) ){ + sSort.pOrderBy = p->pOrderBy; + } + } +#endif + +#ifndef SQLITE_OMIT_COMPOUND_SELECT + /* Handle compound SELECT statements using the separate multiSelect() + ** procedure. + */ + if( p->pPrior ){ + rc = multiSelect(pParse, p, pDest); +#if SELECTTRACE_ENABLED + SELECTTRACE(0x1,pParse,p,("end compound-select processing\n")); + if( (sqlite3SelectTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + sqlite3TreeViewSelect(0, p, 0); + } +#endif + if( p->pNext==0 ) ExplainQueryPlanPop(pParse); + return rc; + } +#endif + + /* Do the WHERE-clause constant propagation optimization if this is + ** a join. No need to speed time on this operation for non-join queries + ** as the equivalent optimization will be handled by query planner in + ** sqlite3WhereBegin(). + */ + if( p->pWhere!=0 + && p->pWhere->op==TK_AND + && OptimizationEnabled(db, SQLITE_PropagateConst) + && propagateConstants(pParse, p) + ){ +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p,("After constant propagation:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + }else{ + SELECTTRACE(0x100,pParse,p,("Constant propagation not helpful\n")); + } + +#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION + if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) + && countOfViewOptimization(pParse, p) + ){ + if( db->mallocFailed ) goto select_end; + pEList = p->pEList; + pTabList = p->pSrc; + } +#endif + + /* For each term in the FROM clause, do two things: + ** (1) Authorized unreferenced tables + ** (2) Generate code for all sub-queries + */ + for(i=0; inSrc; i++){ + SrcItem *pItem = &pTabList->a[i]; + SrcItem *pPrior; + SelectDest dest; + Select *pSub; +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + const char *zSavedAuthContext; +#endif + + /* Issue SQLITE_READ authorizations with a fake column name for any + ** tables that are referenced but from which no values are extracted. + ** Examples of where these kinds of null SQLITE_READ authorizations + ** would occur: + ** + ** SELECT count(*) FROM t1; -- SQLITE_READ t1."" + ** SELECT t1.* FROM t1, t2; -- SQLITE_READ t2."" + ** + ** The fake column name is an empty string. It is possible for a table to + ** have a column named by the empty string, in which case there is no way to + ** distinguish between an unreferenced table and an actual reference to the + ** "" column. The original design was for the fake column name to be a NULL, + ** which would be unambiguous. But legacy authorization callbacks might + ** assume the column name is non-NULL and segfault. The use of an empty + ** string for the fake column name seems safer. + */ + if( pItem->colUsed==0 && pItem->zName!=0 ){ + sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase); + } + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + /* Generate code for all sub-queries in the FROM clause + */ + pSub = pItem->pSelect; + if( pSub==0 ) continue; + + /* The code for a subquery should only be generated once. */ + assert( pItem->addrFillSub==0 ); + + /* Increment Parse.nHeight by the height of the largest expression + ** tree referred to by this, the parent select. The child select + ** may contain expression trees of at most + ** (SQLITE_MAX_EXPR_DEPTH-Parse.nHeight) height. This is a bit + ** more conservative than necessary, but much easier than enforcing + ** an exact limit. + */ + pParse->nHeight += sqlite3SelectExprHeight(p); + + /* Make copies of constant WHERE-clause terms in the outer query down + ** inside the subquery. This can help the subquery to run more efficiently. + */ + if( OptimizationEnabled(db, SQLITE_PushDown) + && (pItem->fg.isCte==0 + || (pItem->u2.pCteUse->eM10d!=M10d_Yes && pItem->u2.pCteUse->nUse<2)) + && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem->iCursor, + (pItem->fg.jointype & JT_OUTER)!=0) + ){ +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p, + ("After WHERE-clause push-down into subquery %d:\n", pSub->selId)); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + assert( pItem->pSelect && (pItem->pSelect->selFlags & SF_PushDown)!=0 ); + }else{ + SELECTTRACE(0x100,pParse,p,("Push-down not possible\n")); + } + + zSavedAuthContext = pParse->zAuthContext; + pParse->zAuthContext = pItem->zName; + + /* Generate code to implement the subquery + ** + ** The subquery is implemented as a co-routine if: + ** (1) the subquery is guaranteed to be the outer loop (so that + ** it does not need to be computed more than once), and + ** (2) the subquery is not a CTE that should be materialized + ** + ** TODO: Are there other reasons beside (1) and (2) to use a co-routine + ** implementation? + */ + if( i==0 + && (pTabList->nSrc==1 + || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ + && (pItem->fg.isCte==0 || pItem->u2.pCteUse->eM10d!=M10d_Yes) /* (2) */ + ){ + /* Implement a co-routine that will return a single row of the result + ** set on each invocation. + */ + int addrTop = sqlite3VdbeCurrentAddr(v)+1; + + pItem->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); + VdbeComment((v, "%!S", pItem)); + pItem->addrFillSub = addrTop; + sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + ExplainQueryPlan((pParse, 1, "CO-ROUTINE %!S", pItem)); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->fg.viaCoroutine = 1; + pItem->regResult = dest.iSdst; + sqlite3VdbeEndCoroutine(v, pItem->regReturn); + sqlite3VdbeJumpHere(v, addrTop-1); + sqlite3ClearTempRegCache(pParse); + }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ + /* This is a CTE for which materialization code has already been + ** generated. Invoke the subroutine to compute the materialization, + ** the make the pItem->iCursor be a copy of the ephemerial table that + ** holds the result of the materialization. */ + CteUse *pCteUse = pItem->u2.pCteUse; + sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e); + if( pItem->iCursor!=pCteUse->iCur ){ + sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pCteUse->iCur); + VdbeComment((v, "%!S", pItem)); + } + pSub->nSelectRow = pCteUse->nRowEst; + }else if( (pPrior = isSelfJoinView(pTabList, pItem))!=0 ){ + /* This view has already been materialized by a prior entry in + ** this same FROM clause. Reuse it. */ + if( pPrior->addrFillSub ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pPrior->regReturn, pPrior->addrFillSub); + } + sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); + pSub->nSelectRow = pPrior->pSelect->nSelectRow; + }else{ + /* Materialize the view. If the view is not correlated, generate a + ** subroutine to do the materialization so that subsequent uses of + ** the same view can reuse the materialization. */ + int topAddr; + int onceAddr = 0; + int retAddr; + + pItem->regReturn = ++pParse->nMem; + topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); + pItem->addrFillSub = topAddr+1; + if( pItem->fg.isCorrelated==0 ){ + /* If the subquery is not correlated and if we are not inside of + ** a trigger, then we only need to compute the value of the subquery + ** once. */ + onceAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + VdbeComment((v, "materialize %!S", pItem)); + }else{ + VdbeNoopComment((v, "materialize %!S", pItem)); + } + sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); + ExplainQueryPlan((pParse, 1, "MATERIALIZE %!S", pItem)); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowLogEst = pSub->nSelectRow; + if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); + retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); + VdbeComment((v, "end %!S", pItem)); + sqlite3VdbeChangeP1(v, topAddr, retAddr); + sqlite3ClearTempRegCache(pParse); + if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ + CteUse *pCteUse = pItem->u2.pCteUse; + pCteUse->addrM9e = pItem->addrFillSub; + pCteUse->regRtn = pItem->regReturn; + pCteUse->iCur = pItem->iCursor; + pCteUse->nRowEst = pSub->nSelectRow; + } + } + if( db->mallocFailed ) goto select_end; + pParse->nHeight -= sqlite3SelectExprHeight(p); + pParse->zAuthContext = zSavedAuthContext; +#endif + } + + /* Various elements of the SELECT copied into local variables for + ** convenience */ + pEList = p->pEList; + pWhere = p->pWhere; + pGroupBy = p->pGroupBy; + pHaving = p->pHaving; + sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; + +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + + /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + ** if the select-list is the same as the ORDER BY list, then this query + ** can be rewritten as a GROUP BY. In other words, this: + ** + ** SELECT DISTINCT xyz FROM ... ORDER BY xyz + ** + ** is transformed to: + ** + ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz + ** + ** The second form is preferred as a single index (or temp-table) may be + ** used for both the ORDER BY and DISTINCT processing. As originally + ** written the query must use a temp-table for at least one of the ORDER + ** BY and DISTINCT, and an index or separate temp-table for the other. + */ + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct + && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 +#ifndef SQLITE_OMIT_WINDOWFUNC + && p->pWin==0 +#endif + ){ + p->selFlags &= ~SF_Distinct; + pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); + p->selFlags |= SF_Aggregate; + /* Notice that even thought SF_Distinct has been cleared from p->selFlags, + ** the sDistinct.isTnct is still set. Hence, isTnct represents the + ** original setting of the SF_Distinct flag, not the current setting */ + assert( sDistinct.isTnct ); + +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + } + + /* If there is an ORDER BY clause, then create an ephemeral index to + ** do the sorting. But this sorting ephemeral index might end up + ** being unused if the data can be extracted in pre-sorted order. + ** If that is the case, then the OP_OpenEphemeral instruction will be + ** changed to an OP_Noop once we figure out that the sorting index is + ** not needed. The sSort.addrSortIndex variable is used to facilitate + ** that change. + */ + if( sSort.pOrderBy ){ + KeyInfo *pKeyInfo; + pKeyInfo = sqlite3KeyInfoFromExprList( + pParse, sSort.pOrderBy, 0, pEList->nExpr); + sSort.iECursor = pParse->nTab++; + sSort.addrSortIndex = + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sSort.iECursor, sSort.pOrderBy->nExpr+1+pEList->nExpr, 0, + (char*)pKeyInfo, P4_KEYINFO + ); + }else{ + sSort.addrSortIndex = -1; + } + + /* If the output is destined for a temporary table, open that table. + */ + if( pDest->eDest==SRT_EphemTab ){ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr); + } + + /* Set the limiter. + */ + iEnd = sqlite3VdbeMakeLabel(pParse); + if( (p->selFlags & SF_FixedLimit)==0 ){ + p->nSelectRow = 320; /* 4 billion rows */ + } + computeLimitRegisters(pParse, p, iEnd); + if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ + sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); + sSort.sortFlags |= SORTFLAG_UseSorter; + } + + /* Open an ephemeral index to use for the distinct set. + */ + if( p->selFlags & SF_Distinct ){ + sDistinct.tabTnct = pParse->nTab++; + sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + sDistinct.tabTnct, 0, 0, + (char*)sqlite3KeyInfoFromExprList(pParse, p->pEList,0,0), + P4_KEYINFO); + sqlite3VdbeChangeP5(v, BTREE_UNORDERED); + sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; + }else{ + sDistinct.eTnctType = WHERE_DISTINCT_NOOP; + } + + if( !isAgg && pGroupBy==0 ){ + /* No aggregate functions and no GROUP BY clause */ + u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0) + | (p->selFlags & SF_FixedLimit); +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin = p->pWin; /* Main window object (or NULL) */ + if( pWin ){ + sqlite3WindowCodeInit(pParse, p); + } +#endif + assert( WHERE_USE_LIMIT==SF_FixedLimit ); + + + /* Begin the database scan. */ + SELECTTRACE(1,pParse,p,("WhereBegin\n")); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, + p->pEList, wctrlFlags, p->nSelectRow); + if( pWInfo==0 ) goto select_end; + if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ + p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); + } + if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ + sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); + } + if( sSort.pOrderBy ){ + sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + sSort.labelOBLopt = sqlite3WhereOrderByLimitOptLabel(pWInfo); + if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ + sSort.pOrderBy = 0; + } + } + SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + + /* If sorting index that was created by a prior OP_OpenEphemeral + ** instruction ended up not being needed, then change the OP_OpenEphemeral + ** into an OP_Noop. + */ + if( sSort.addrSortIndex>=0 && sSort.pOrderBy==0 ){ + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); + } + + assert( p->pEList==pEList ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + int addrGosub = sqlite3VdbeMakeLabel(pParse); + int iCont = sqlite3VdbeMakeLabel(pParse); + int iBreak = sqlite3VdbeMakeLabel(pParse); + int regGosub = ++pParse->nMem; + + sqlite3WindowCodeStep(pParse, p, pWInfo, regGosub, addrGosub); + + sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + sqlite3VdbeResolveLabel(v, addrGosub); + VdbeNoopComment((v, "inner-loop subroutine")); + sSort.labelOBLopt = 0; + selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp1(v, OP_Return, regGosub); + VdbeComment((v, "end inner-loop subroutine")); + sqlite3VdbeResolveLabel(v, iBreak); + }else +#endif /* SQLITE_OMIT_WINDOWFUNC */ + { + /* Use the standard inner loop. */ + selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, + sqlite3WhereContinueLabel(pWInfo), + sqlite3WhereBreakLabel(pWInfo)); + + /* End the database scan loop. + */ + SELECTTRACE(1,pParse,p,("WhereEnd\n")); + sqlite3WhereEnd(pWInfo); + } + }else{ + /* This case when there exist aggregate functions or a GROUP BY clause + ** or both */ + NameContext sNC; /* Name context for processing aggregate information */ + int iAMem; /* First Mem address for storing current GROUP BY */ + int iBMem; /* First Mem address for previous GROUP BY */ + int iUseFlag; /* Mem address holding flag indicating that at least + ** one row of the input to the aggregator has been + ** processed */ + int iAbortFlag; /* Mem address which causes query abort if positive */ + int groupBySort; /* Rows come from source in GROUP BY order */ + int addrEnd; /* End of processing for this SELECT */ + int sortPTab = 0; /* Pseudotable used to decode sorting results */ + int sortOut = 0; /* Output register from the sorter */ + int orderByGrp = 0; /* True if the GROUP BY and ORDER BY are the same */ + + /* Remove any and all aliases between the result set and the + ** GROUP BY clause. + */ + if( pGroupBy ){ + int k; /* Loop counter */ + struct ExprList_item *pItem; /* For looping over expression in a list */ + + for(k=p->pEList->nExpr, pItem=p->pEList->a; k>0; k--, pItem++){ + pItem->u.x.iAlias = 0; + } + for(k=pGroupBy->nExpr, pItem=pGroupBy->a; k>0; k--, pItem++){ + pItem->u.x.iAlias = 0; + } + assert( 66==sqlite3LogEst(100) ); + if( p->nSelectRow>66 ) p->nSelectRow = 66; + + /* If there is both a GROUP BY and an ORDER BY clause and they are + ** identical, then it may be possible to disable the ORDER BY clause + ** on the grounds that the GROUP BY will cause elements to come out + ** in the correct order. It also may not - the GROUP BY might use a + ** database index that causes rows to be grouped together as required + ** but not actually sorted. Either way, record the fact that the + ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp + ** variable. */ + if( sSort.pOrderBy && pGroupBy->nExpr==sSort.pOrderBy->nExpr ){ + int ii; + /* The GROUP BY processing doesn't care whether rows are delivered in + ** ASC or DESC order - only that each group is returned contiguously. + ** So set the ASC/DESC flags in the GROUP BY to match those in the + ** ORDER BY to maximize the chances of rows being delivered in an + ** order that makes the ORDER BY redundant. */ + for(ii=0; iinExpr; ii++){ + u8 sortFlags = sSort.pOrderBy->a[ii].sortFlags & KEYINFO_ORDER_DESC; + pGroupBy->a[ii].sortFlags = sortFlags; + } + if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ + orderByGrp = 1; + } + } + }else{ + assert( 0==sqlite3LogEst(1) ); + p->nSelectRow = 0; + } + + /* Create a label to jump to when we want to abort the query */ + addrEnd = sqlite3VdbeMakeLabel(pParse); + + /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in + ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the + ** SELECT statement. + */ + pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) ); + if( pAggInfo ){ + sqlite3ParserAddCleanup(pParse, + (void(*)(sqlite3*,void*))agginfoFree, pAggInfo); + testcase( pParse->earlyCleanup ); + } + if( db->mallocFailed ){ + goto select_end; + } + pAggInfo->selId = p->selId; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + sNC.uNC.pAggInfo = pAggInfo; + VVA_ONLY( sNC.ncFlags = NC_UAggInfo; ) + pAggInfo->mnReg = pParse->nMem+1; + pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; + pAggInfo->pGroupBy = pGroupBy; + sqlite3ExprAnalyzeAggList(&sNC, pEList); + sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); + if( pHaving ){ + if( pGroupBy ){ + assert( pWhere==p->pWhere ); + assert( pHaving==p->pHaving ); + assert( pGroupBy==p->pGroupBy ); + havingToWhere(pParse, p); + pWhere = p->pWhere; + } + sqlite3ExprAnalyzeAggregates(&sNC, pHaving); + } + pAggInfo->nAccumulator = pAggInfo->nColumn; + if( p->pGroupBy==0 && p->pHaving==0 && pAggInfo->nFunc==1 ){ + minMaxFlag = minMaxQuery(db, pAggInfo->aFunc[0].pFExpr, &pMinMaxOrderBy); + }else{ + minMaxFlag = WHERE_ORDERBY_NORMAL; + } + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + sNC.ncFlags |= NC_InAggFunc; + sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter); + } +#endif + sNC.ncFlags &= ~NC_InAggFunc; + } + pAggInfo->mxReg = pParse->nMem; + if( db->mallocFailed ) goto select_end; +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x400 ){ + int ii; + SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); + sqlite3TreeViewSelect(0, p, 0); + if( minMaxFlag ){ + sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag); + sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY"); + } + for(ii=0; iinColumn; ii++){ + sqlite3DebugPrintf("agg-column[%d] iMem=%d\n", + ii, pAggInfo->aCol[ii].iMem); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; iinFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", + ii, pAggInfo->aFunc[ii].iMem); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } + } +#endif + + + /* Processing for aggregates with GROUP BY is very different and + ** much more complex than aggregates without a GROUP BY. + */ + if( pGroupBy ){ + KeyInfo *pKeyInfo; /* Keying information for the group by clause */ + int addr1; /* A-vs-B comparision jump */ + int addrOutputRow; /* Start of subroutine that outputs a result row */ + int regOutputRow; /* Return address register for output subroutine */ + int addrSetAbort; /* Set the abort flag and return */ + int addrTopOfLoop; /* Top of the input loop */ + int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */ + int addrReset; /* Subroutine for resetting the accumulator */ + int regReset; /* Return address register for reset subroutine */ + ExprList *pDistinct = 0; + u16 distFlag = 0; + int eDist = WHERE_DISTINCT_NOOP; + + if( pAggInfo->nFunc==1 + && pAggInfo->aFunc[0].iDistinct>=0 + && pAggInfo->aFunc[0].pFExpr->x.pList + ){ + Expr *pExpr = pAggInfo->aFunc[0].pFExpr->x.pList->a[0].pExpr; + pExpr = sqlite3ExprDup(db, pExpr, 0); + pDistinct = sqlite3ExprListDup(db, pGroupBy, 0); + pDistinct = sqlite3ExprListAppend(pParse, pDistinct, pExpr); + distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0; + } + + /* If there is a GROUP BY clause we might need a sorting index to + ** implement it. Allocate that sorting index now. If it turns out + ** that we do not need it after all, the OP_SorterOpen instruction + ** will be converted into a Noop. + */ + pAggInfo->sortingIdx = pParse->nTab++; + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pGroupBy, + 0, pAggInfo->nColumn); + addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, + pAggInfo->sortingIdx, pAggInfo->nSortingColumn, + 0, (char*)pKeyInfo, P4_KEYINFO); + + /* Initialize memory locations used by GROUP BY aggregate processing + */ + iUseFlag = ++pParse->nMem; + iAbortFlag = ++pParse->nMem; + regOutputRow = ++pParse->nMem; + addrOutputRow = sqlite3VdbeMakeLabel(pParse); + regReset = ++pParse->nMem; + addrReset = sqlite3VdbeMakeLabel(pParse); + iAMem = pParse->nMem + 1; + pParse->nMem += pGroupBy->nExpr; + iBMem = pParse->nMem + 1; + pParse->nMem += pGroupBy->nExpr; + sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); + VdbeComment((v, "clear abort flag")); + sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); + + /* Begin a loop that will extract all source rows in GROUP BY order. + ** This might involve two separate loops with an OP_Sort in between, or + ** it might be a single loop that uses an index to extract information + ** in the right order to begin with. + */ + sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); + SELECTTRACE(1,pParse,p,("WhereBegin\n")); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, + WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 + ); + if( pWInfo==0 ){ + sqlite3ExprListDelete(db, pDistinct); + goto select_end; + } + eDist = sqlite3WhereIsDistinct(pWInfo); + SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ + /* The optimizer is able to deliver rows in group by order so + ** we do not have to sort. The OP_OpenEphemeral table will be + ** cancelled later because we still need to use the pKeyInfo + */ + groupBySort = 0; + }else{ + /* Rows are coming out in undetermined order. We have to push + ** each row into a sorting index, terminate the first loop, + ** then loop over the sorting index in order to get the output + ** in sorted order + */ + int regBase; + int regRecord; + int nCol; + int nGroupBy; + + explainTempTable(pParse, + (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ? + "DISTINCT" : "GROUP BY"); + + groupBySort = 1; + nGroupBy = pGroupBy->nExpr; + nCol = nGroupBy; + j = nGroupBy; + for(i=0; inColumn; i++){ + if( pAggInfo->aCol[i].iSorterColumn>=j ){ + nCol++; + j++; + } + } + regBase = sqlite3GetTempRange(pParse, nCol); + sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); + j = nGroupBy; + for(i=0; inColumn; i++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[i]; + if( pCol->iSorterColumn>=j ){ + int r1 = j + regBase; + sqlite3ExprCodeGetColumnOfTable(v, + pCol->pTab, pCol->iTable, pCol->iColumn, r1); + j++; + } + } + regRecord = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ReleaseTempRange(pParse, regBase, nCol); + SELECTTRACE(1,pParse,p,("WhereEnd\n")); + sqlite3WhereEnd(pWInfo); + pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; + sortOut = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); + sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd); + VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); + pAggInfo->useSortingIdx = 1; + } + + /* If the index or temporary table used by the GROUP BY sort + ** will naturally deliver rows in the order required by the ORDER BY + ** clause, cancel the ephemeral table open coded earlier. + ** + ** This is an optimization - the correct answer should result regardless. + ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to + ** disable this optimization for testing purposes. */ + if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder) + && (groupBySort || sqlite3WhereIsSorted(pWInfo)) + ){ + sSort.pOrderBy = 0; + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); + } + + /* Evaluate the current GROUP BY terms and store in b0, b1, b2... + ** (b0 is memory location iBMem+0, b1 is iBMem+1, and so forth) + ** Then compare the current GROUP BY terms against the GROUP BY terms + ** from the previous row currently stored in a0, a1, a2... + */ + addrTopOfLoop = sqlite3VdbeCurrentAddr(v); + if( groupBySort ){ + sqlite3VdbeAddOp3(v, OP_SorterData, pAggInfo->sortingIdx, + sortOut, sortPTab); + } + for(j=0; jnExpr; j++){ + if( groupBySort ){ + sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); + }else{ + pAggInfo->directMode = 1; + sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); + } + } + sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, + (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addr1+1, 0, addr1+1); VdbeCoverage(v); + + /* Generate code that runs whenever the GROUP BY changes. + ** Changes in the GROUP BY are detected by the previous code + ** block. If there were no changes, this block is skipped. + ** + ** This code copies current group by terms in b0,b1,b2,... + ** over to a0,a1,a2. It then calls the output subroutine + ** and resets the aggregate accumulator registers in preparation + ** for the next GROUP BY batch. + */ + sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); + sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); + VdbeComment((v, "output one row")); + sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v); + VdbeComment((v, "check abort flag")); + sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); + VdbeComment((v, "reset accumulator")); + + /* Update the aggregate accumulators based on the content of + ** the current row + */ + sqlite3VdbeJumpHere(v, addr1); + updateAccumulator(pParse, iUseFlag, pAggInfo, eDist); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); + VdbeComment((v, "indicate data in accumulator")); + + /* End of the loop + */ + if( groupBySort ){ + sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop); + VdbeCoverage(v); + }else{ + SELECTTRACE(1,pParse,p,("WhereEnd\n")); + sqlite3WhereEnd(pWInfo); + sqlite3VdbeChangeToNoop(v, addrSortingIdx); + } + sqlite3ExprListDelete(db, pDistinct); + + /* Output the final row of result + */ + sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); + VdbeComment((v, "output final row")); + + /* Jump over the subroutines + */ + sqlite3VdbeGoto(v, addrEnd); + + /* Generate a subroutine that outputs a single row of the result + ** set. This subroutine first looks at the iUseFlag. If iUseFlag + ** is less than or equal to zero, the subroutine is a no-op. If + ** the processing calls for the query to abort, this subroutine + ** increments the iAbortFlag memory location before returning in + ** order to signal the caller to abort. + */ + addrSetAbort = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, iAbortFlag); + VdbeComment((v, "set abort flag")); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + sqlite3VdbeResolveLabel(v, addrOutputRow); + addrOutputRow = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); + VdbeCoverage(v); + VdbeComment((v, "Groupby result generator entry point")); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + finalizeAggFunctions(pParse, pAggInfo); + sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); + selectInnerLoop(pParse, p, -1, &sSort, + &sDistinct, pDest, + addrOutputRow+1, addrSetAbort); + sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); + VdbeComment((v, "end groupby result generator")); + + /* Generate a subroutine that will reset the group-by accumulator + */ + sqlite3VdbeResolveLabel(v, addrReset); + resetAccumulator(pParse, pAggInfo); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); + VdbeComment((v, "indicate accumulator empty")); + sqlite3VdbeAddOp1(v, OP_Return, regReset); + + if( eDist!=WHERE_DISTINCT_NOOP ){ + struct AggInfo_func *pF = &pAggInfo->aFunc[0]; + fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); + } + } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ + else { + Table *pTab; + if( (pTab = isSimpleCount(p, pAggInfo))!=0 ){ + /* If isSimpleCount() returns a pointer to a Table structure, then + ** the SQL statement is of the form: + ** + ** SELECT count(*) FROM + ** + ** where the Table structure returned represents table . + ** + ** This statement is so common that it is optimized specially. The + ** OP_Count instruction is executed either on the intkey table that + ** contains the data for table or on one of its indexes. It + ** is better to execute the op on an index, as indexes are almost + ** always spread across less pages than their corresponding tables. + */ + const int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + const int iCsr = pParse->nTab++; /* Cursor to scan b-tree */ + Index *pIdx; /* Iterator variable */ + KeyInfo *pKeyInfo = 0; /* Keyinfo for scanned index */ + Index *pBest = 0; /* Best index found so far */ + Pgno iRoot = pTab->tnum; /* Root page of scanned b-tree */ + + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + + /* Search for the index that has the lowest scan cost. + ** + ** (2011-04-15) Do not do a full scan of an unordered index. + ** + ** (2013-10-03) Do not count the entries in a partial index. + ** + ** In practice the KeyInfo structure will not be used. It is only + ** passed to keep OP_OpenRead happy. + */ + if( !HasRowid(pTab) ) pBest = sqlite3PrimaryKeyIndex(pTab); + if( !p->pSrc->a[0].fg.notIndexed ){ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->bUnordered==0 + && pIdx->szIdxRowszTabRow + && pIdx->pPartIdxWhere==0 + && (!pBest || pIdx->szIdxRowszIdxRow) + ){ + pBest = pIdx; + } + } + } + if( pBest ){ + iRoot = pBest->tnum; + pKeyInfo = sqlite3KeyInfoOfIndex(pParse, pBest); + } + + /* Open a read-only cursor, execute the OP_Count, close the cursor. */ + sqlite3VdbeAddOp4Int(v, OP_OpenRead, iCsr, (int)iRoot, iDb, 1); + if( pKeyInfo ){ + sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); + } + sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); + explainSimpleCount(pParse, pTab, pBest); + }else{ + int regAcc = 0; /* "populate accumulators" flag */ + ExprList *pDistinct = 0; + u16 distFlag = 0; + int eDist; + + /* If there are accumulator registers but no min() or max() functions + ** without FILTER clauses, allocate register regAcc. Register regAcc + ** will contain 0 the first time the inner loop runs, and 1 thereafter. + ** The code generated by updateAccumulator() uses this to ensure + ** that the accumulator registers are (a) updated only once if + ** there are no min() or max functions or (b) always updated for the + ** first row visited by the aggregate, so that they are updated at + ** least once even if the FILTER clause means the min() or max() + ** function visits zero rows. */ + if( pAggInfo->nAccumulator ){ + for(i=0; inFunc; i++){ + if( ExprHasProperty(pAggInfo->aFunc[i].pFExpr, EP_WinFunc) ){ + continue; + } + if( pAggInfo->aFunc[i].pFunc->funcFlags&SQLITE_FUNC_NEEDCOLL ){ + break; + } + } + if( i==pAggInfo->nFunc ){ + regAcc = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regAcc); + } + }else if( pAggInfo->nFunc==1 && pAggInfo->aFunc[0].iDistinct>=0 ){ + pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList; + distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0; + } + + /* This case runs if the aggregate has no GROUP BY clause. The + ** processing is much simpler since there is only a single row + ** of output. + */ + assert( p->pGroupBy==0 ); + resetAccumulator(pParse, pAggInfo); + + /* If this query is a candidate for the min/max optimization, then + ** minMaxFlag will have been previously set to either + ** WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX and pMinMaxOrderBy will + ** be an appropriate ORDER BY expression for the optimization. + */ + assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 ); + assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 ); + + SELECTTRACE(1,pParse,p,("WhereBegin\n")); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, + pDistinct, minMaxFlag|distFlag, 0); + if( pWInfo==0 ){ + goto select_end; + } + SELECTTRACE(1,pParse,p,("WhereBegin returns\n")); + eDist = sqlite3WhereIsDistinct(pWInfo); + updateAccumulator(pParse, regAcc, pAggInfo, eDist); + if( eDist!=WHERE_DISTINCT_NOOP ){ + struct AggInfo_func *pF = &pAggInfo->aFunc[0]; + fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); + } + + if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); + if( minMaxFlag ){ + sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); + } + SELECTTRACE(1,pParse,p,("WhereEnd\n")); + sqlite3WhereEnd(pWInfo); + finalizeAggFunctions(pParse, pAggInfo); + } + + sSort.pOrderBy = 0; + sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); + selectInnerLoop(pParse, p, -1, 0, 0, + pDest, addrEnd, addrEnd); + } + sqlite3VdbeResolveLabel(v, addrEnd); + + } /* endif aggregate query */ + + if( sDistinct.eTnctType==WHERE_DISTINCT_UNORDERED ){ + explainTempTable(pParse, "DISTINCT"); + } + + /* If there is an ORDER BY clause, then we need to sort the results + ** and send them to the callback one by one. + */ + if( sSort.pOrderBy ){ + explainTempTable(pParse, + sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + assert( p->pEList==pEList ); + generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); + } + + /* Jump here to skip this query + */ + sqlite3VdbeResolveLabel(v, iEnd); + + /* The SELECT has been coded. If there is an error in the Parse structure, + ** set the return code to 1. Otherwise 0. */ + rc = (pParse->nErr>0); + + /* Control jumps to here if an error is encountered above, or upon + ** successful coding of the SELECT. + */ +select_end: + assert( db->mallocFailed==0 || db->mallocFailed==1 ); + pParse->nErr += db->mallocFailed; + sqlite3ExprListDelete(db, pMinMaxOrderBy); +#ifdef SQLITE_DEBUG + if( pAggInfo && !db->mallocFailed ){ + for(i=0; inColumn; i++){ + Expr *pExpr = pAggInfo->aCol[i].pCExpr; + assert( pExpr!=0 ); + assert( pExpr->pAggInfo==pAggInfo ); + assert( pExpr->iAgg==i ); + } + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( pExpr!=0 ); + assert( pExpr->pAggInfo==pAggInfo ); + assert( pExpr->iAgg==i ); + } + } +#endif + +#if SELECTTRACE_ENABLED + SELECTTRACE(0x1,pParse,p,("end processing\n")); + if( (sqlite3SelectTrace & 0x2000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + sqlite3TreeViewSelect(0, p, 0); + } +#endif + ExplainQueryPlanPop(pParse); + return rc; +} + +/************** End of select.c **********************************************/ +/************** Begin file table.c *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains the sqlite3_get_table() and sqlite3_free_table() +** interface routines. These are just wrappers around the main +** interface routine of sqlite3_exec(). +** +** These routines are in a separate files so that they will not be linked +** if they are not used. +*/ +/* #include "sqliteInt.h" */ + +#ifndef SQLITE_OMIT_GET_TABLE + +/* +** This structure is used to pass data from sqlite3_get_table() through +** to the callback function is uses to build the result. +*/ +typedef struct TabResult { + char **azResult; /* Accumulated output */ + char *zErrMsg; /* Error message text, if an error occurs */ + u32 nAlloc; /* Slots allocated for azResult[] */ + u32 nRow; /* Number of rows in the result */ + u32 nColumn; /* Number of columns in the result */ + u32 nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ + int rc; /* Return code from sqlite3_exec() */ +} TabResult; + +/* +** This routine is called once for each row in the result table. Its job +** is to fill in the TabResult structure appropriately, allocating new +** memory as necessary. +*/ +static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ + TabResult *p = (TabResult*)pArg; /* Result accumulator */ + int need; /* Slots needed in p->azResult[] */ + int i; /* Loop counter */ + char *z; /* A single column of result */ + + /* Make sure there is enough space in p->azResult to hold everything + ** we need to remember from this invocation of the callback. + */ + if( p->nRow==0 && argv!=0 ){ + need = nCol*2; + }else{ + need = nCol; + } + if( p->nData + need > p->nAlloc ){ + char **azNew; + p->nAlloc = p->nAlloc*2 + need; + azNew = sqlite3Realloc( p->azResult, sizeof(char*)*p->nAlloc ); + if( azNew==0 ) goto malloc_failed; + p->azResult = azNew; + } + + /* If this is the first row, then generate an extra row containing + ** the names of all columns. + */ + if( p->nRow==0 ){ + p->nColumn = nCol; + for(i=0; iazResult[p->nData++] = z; + } + }else if( (int)p->nColumn!=nCol ){ + sqlite3_free(p->zErrMsg); + p->zErrMsg = sqlite3_mprintf( + "sqlite3_get_table() called with two or more incompatible queries" + ); + p->rc = SQLITE_ERROR; + return 1; + } + + /* Copy over the row data + */ + if( argv!=0 ){ + for(i=0; iazResult[p->nData++] = z; + } + p->nRow++; + } + return 0; + +malloc_failed: + p->rc = SQLITE_NOMEM_BKPT; + return 1; +} + +/* +** Query the database. But instead of invoking a callback for each row, +** malloc() for space to hold the result and return the entire results +** at the conclusion of the call. +** +** The result that is written to ***pazResult is held in memory obtained +** from malloc(). But the caller cannot free this memory directly. +** Instead, the entire table should be passed to sqlite3_free_table() when +** the calling procedure is finished using it. +*/ +SQLITE_API int sqlite3_get_table( + sqlite3 *db, /* The database on which the SQL executes */ + const char *zSql, /* The SQL to be executed */ + char ***pazResult, /* Write the result table here */ + int *pnRow, /* Write the number of rows in the result here */ + int *pnColumn, /* Write the number of columns of result here */ + char **pzErrMsg /* Write error messages here */ +){ + int rc; + TabResult res; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pazResult==0 ) return SQLITE_MISUSE_BKPT; +#endif + *pazResult = 0; + if( pnColumn ) *pnColumn = 0; + if( pnRow ) *pnRow = 0; + if( pzErrMsg ) *pzErrMsg = 0; + res.zErrMsg = 0; + res.nRow = 0; + res.nColumn = 0; + res.nData = 1; + res.nAlloc = 20; + res.rc = SQLITE_OK; + res.azResult = sqlite3_malloc64(sizeof(char*)*res.nAlloc ); + if( res.azResult==0 ){ + db->errCode = SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; + } + res.azResult[0] = 0; + rc = sqlite3_exec(db, zSql, sqlite3_get_table_cb, &res, pzErrMsg); + assert( sizeof(res.azResult[0])>= sizeof(res.nData) ); + res.azResult[0] = SQLITE_INT_TO_PTR(res.nData); + if( (rc&0xff)==SQLITE_ABORT ){ + sqlite3_free_table(&res.azResult[1]); + if( res.zErrMsg ){ + if( pzErrMsg ){ + sqlite3_free(*pzErrMsg); + *pzErrMsg = sqlite3_mprintf("%s",res.zErrMsg); + } + sqlite3_free(res.zErrMsg); + } + db->errCode = res.rc; /* Assume 32-bit assignment is atomic */ + return res.rc; + } + sqlite3_free(res.zErrMsg); + if( rc!=SQLITE_OK ){ + sqlite3_free_table(&res.azResult[1]); + return rc; + } + if( res.nAlloc>res.nData ){ + char **azNew; + azNew = sqlite3Realloc( res.azResult, sizeof(char*)*res.nData ); + if( azNew==0 ){ + sqlite3_free_table(&res.azResult[1]); + db->errCode = SQLITE_NOMEM; + return SQLITE_NOMEM_BKPT; + } + res.azResult = azNew; + } + *pazResult = &res.azResult[1]; + if( pnColumn ) *pnColumn = res.nColumn; + if( pnRow ) *pnRow = res.nRow; + return rc; +} + +/* +** This routine frees the space the sqlite3_get_table() malloced. +*/ +SQLITE_API void sqlite3_free_table( + char **azResult /* Result returned from sqlite3_get_table() */ +){ + if( azResult ){ + int i, n; + azResult--; + assert( azResult!=0 ); + n = SQLITE_PTR_TO_INT(azResult[0]); + for(i=1; ipNext; + + sqlite3ExprDelete(db, pTmp->pWhere); + sqlite3ExprListDelete(db, pTmp->pExprList); + sqlite3SelectDelete(db, pTmp->pSelect); + sqlite3IdListDelete(db, pTmp->pIdList); + sqlite3UpsertDelete(db, pTmp->pUpsert); + sqlite3SrcListDelete(db, pTmp->pFrom); + sqlite3DbFree(db, pTmp->zSpan); + + sqlite3DbFree(db, pTmp); + } +} + +/* +** Given table pTab, return a list of all the triggers attached to +** the table. The list is connected by Trigger.pNext pointers. +** +** All of the triggers on pTab that are in the same database as pTab +** are already attached to pTab->pTrigger. But there might be additional +** triggers on pTab in the TEMP schema. This routine prepends all +** TEMP triggers on pTab to the beginning of the pTab->pTrigger list +** and returns the combined list. +** +** To state it another way: This routine returns a list of all triggers +** that fire off of pTab. The list will include any TEMP triggers on +** pTab as well as the triggers lised in pTab->pTrigger. +*/ +SQLITE_PRIVATE Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ + Schema *pTmpSchema; /* Schema of the pTab table */ + Trigger *pList; /* List of triggers to return */ + HashElem *p; /* Loop variable for TEMP triggers */ + + if( pParse->disableTriggers ){ + return 0; + } + pTmpSchema = pParse->db->aDb[1].pSchema; + p = sqliteHashFirst(&pTmpSchema->trigHash); + pList = pTab->pTrigger; + while( p ){ + Trigger *pTrig = (Trigger *)sqliteHashData(p); + if( pTrig->pTabSchema==pTab->pSchema + && pTrig->table + && 0==sqlite3StrICmp(pTrig->table, pTab->zName) + && pTrig->pTabSchema!=pTmpSchema + ){ + pTrig->pNext = pList; + pList = pTrig; + }else if( pTrig->op==TK_RETURNING +#ifndef SQLITE_OMIT_VIRTUALTABLE + && pParse->db->pVtabCtx==0 +#endif + ){ + assert( pParse->bReturning ); + assert( &(pParse->u1.pReturning->retTrig) == pTrig ); + pTrig->table = pTab->zName; + pTrig->pTabSchema = pTab->pSchema; + pTrig->pNext = pList; + pList = pTrig; + } + p = sqliteHashNext(p); + } +#if 0 + if( pList ){ + Trigger *pX; + printf("Triggers for %s:", pTab->zName); + for(pX=pList; pX; pX=pX->pNext){ + printf(" %s", pX->zName); + } + printf("\n"); + fflush(stdout); + } +#endif + return pList; +} + +/* +** This is called by the parser when it sees a CREATE TRIGGER statement +** up to the point of the BEGIN before the trigger actions. A Trigger +** structure is generated based on the information available and stored +** in pParse->pNewTrigger. After the trigger actions have been parsed, the +** sqlite3FinishTrigger() function is called to complete the trigger +** construction process. +*/ +SQLITE_PRIVATE void sqlite3BeginTrigger( + Parse *pParse, /* The parse context of the CREATE TRIGGER statement */ + Token *pName1, /* The name of the trigger */ + Token *pName2, /* The name of the trigger */ + int tr_tm, /* One of TK_BEFORE, TK_AFTER, TK_INSTEAD */ + int op, /* One of TK_INSERT, TK_UPDATE, TK_DELETE */ + IdList *pColumns, /* column list if this is an UPDATE OF trigger */ + SrcList *pTableName,/* The name of the table/view the trigger applies to */ + Expr *pWhen, /* WHEN clause */ + int isTemp, /* True if the TEMPORARY keyword is present */ + int noErr /* Suppress errors if the trigger already exists */ +){ + Trigger *pTrigger = 0; /* The new trigger */ + Table *pTab; /* Table that the trigger fires off of */ + char *zName = 0; /* Name of the trigger */ + sqlite3 *db = pParse->db; /* The database connection */ + int iDb; /* The database to store the trigger in */ + Token *pName; /* The unqualified db name */ + DbFixer sFix; /* State vector for the DB fixer */ + + assert( pName1!=0 ); /* pName1->z might be NULL, but not pName1 itself */ + assert( pName2!=0 ); + assert( op==TK_INSERT || op==TK_UPDATE || op==TK_DELETE ); + assert( op>0 && op<0xff ); + if( isTemp ){ + /* If TEMP was specified, then the trigger name may not be qualified. */ + if( pName2->n>0 ){ + sqlite3ErrorMsg(pParse, "temporary trigger may not have qualified name"); + goto trigger_cleanup; + } + iDb = 1; + pName = pName1; + }else{ + /* Figure out the db that the trigger will be created in */ + iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); + if( iDb<0 ){ + goto trigger_cleanup; + } + } + if( !pTableName || db->mallocFailed ){ + goto trigger_cleanup; + } + + /* A long-standing parser bug is that this syntax was allowed: + ** + ** CREATE TRIGGER attached.demo AFTER INSERT ON attached.tab .... + ** ^^^^^^^^ + ** + ** To maintain backwards compatibility, ignore the database + ** name on pTableName if we are reparsing out of the schema table + */ + if( db->init.busy && iDb!=1 ){ + sqlite3DbFree(db, pTableName->a[0].zDatabase); + pTableName->a[0].zDatabase = 0; + } + + /* If the trigger name was unqualified, and the table is a temp table, + ** then set iDb to 1 to create the trigger in the temporary database. + ** If sqlite3SrcListLookup() returns 0, indicating the table does not + ** exist, the error is caught by the block below. + */ + pTab = sqlite3SrcListLookup(pParse, pTableName); + if( db->init.busy==0 && pName2->n==0 && pTab + && pTab->pSchema==db->aDb[1].pSchema ){ + iDb = 1; + } + + /* Ensure the table name matches database name and that the table exists */ + if( db->mallocFailed ) goto trigger_cleanup; + assert( pTableName->nSrc==1 ); + sqlite3FixInit(&sFix, pParse, iDb, "trigger", pName); + if( sqlite3FixSrcList(&sFix, pTableName) ){ + goto trigger_cleanup; + } + pTab = sqlite3SrcListLookup(pParse, pTableName); + if( !pTab ){ + /* The table does not exist. */ + goto trigger_orphan_error; + } + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "cannot create triggers on virtual tables"); + goto trigger_orphan_error; + } + + /* Check that the trigger name is not reserved and that no trigger of the + ** specified name exists */ + zName = sqlite3NameFromToken(db, pName); + if( zName==0 ){ + assert( db->mallocFailed ); + goto trigger_cleanup; + } + if( sqlite3CheckObjectName(pParse, zName, "trigger", pTab->zName) ){ + goto trigger_cleanup; + } + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !IN_RENAME_OBJECT ){ + if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash),zName) ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + } + goto trigger_cleanup; + } + } + + /* Do not create a trigger on a system table */ + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ + sqlite3ErrorMsg(pParse, "cannot create trigger on system table"); + goto trigger_cleanup; + } + + /* INSTEAD of triggers are only for views and views only support INSTEAD + ** of triggers. + */ + if( IsView(pTab) && tr_tm!=TK_INSTEAD ){ + sqlite3ErrorMsg(pParse, "cannot create %s trigger on view: %S", + (tr_tm == TK_BEFORE)?"BEFORE":"AFTER", pTableName->a); + goto trigger_orphan_error; + } + if( !IsView(pTab) && tr_tm==TK_INSTEAD ){ + sqlite3ErrorMsg(pParse, "cannot create INSTEAD OF" + " trigger on table: %S", pTableName->a); + goto trigger_orphan_error; + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + if( !IN_RENAME_OBJECT ){ + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); + int code = SQLITE_CREATE_TRIGGER; + const char *zDb = db->aDb[iTabDb].zDbSName; + const char *zDbTrig = isTemp ? db->aDb[1].zDbSName : zDb; + if( iTabDb==1 || isTemp ) code = SQLITE_CREATE_TEMP_TRIGGER; + if( sqlite3AuthCheck(pParse, code, zName, pTab->zName, zDbTrig) ){ + goto trigger_cleanup; + } + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iTabDb),0,zDb)){ + goto trigger_cleanup; + } + } +#endif + + /* INSTEAD OF triggers can only appear on views and BEFORE triggers + ** cannot appear on views. So we might as well translate every + ** INSTEAD OF trigger into a BEFORE trigger. It simplifies code + ** elsewhere. + */ + if (tr_tm == TK_INSTEAD){ + tr_tm = TK_BEFORE; + } + + /* Build the Trigger object */ + pTrigger = (Trigger*)sqlite3DbMallocZero(db, sizeof(Trigger)); + if( pTrigger==0 ) goto trigger_cleanup; + pTrigger->zName = zName; + zName = 0; + pTrigger->table = sqlite3DbStrDup(db, pTableName->a[0].zName); + pTrigger->pSchema = db->aDb[iDb].pSchema; + pTrigger->pTabSchema = pTab->pSchema; + pTrigger->op = (u8)op; + pTrigger->tr_tm = tr_tm==TK_BEFORE ? TRIGGER_BEFORE : TRIGGER_AFTER; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, pTrigger->table, pTableName->a[0].zName); + pTrigger->pWhen = pWhen; + pWhen = 0; + }else{ + pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); + } + pTrigger->pColumns = pColumns; + pColumns = 0; + assert( pParse->pNewTrigger==0 ); + pParse->pNewTrigger = pTrigger; + +trigger_cleanup: + sqlite3DbFree(db, zName); + sqlite3SrcListDelete(db, pTableName); + sqlite3IdListDelete(db, pColumns); + sqlite3ExprDelete(db, pWhen); + if( !pParse->pNewTrigger ){ + sqlite3DeleteTrigger(db, pTrigger); + }else{ + assert( pParse->pNewTrigger==pTrigger ); + } + return; + +trigger_orphan_error: + if( db->init.iDb==1 ){ + /* Ticket #3810. + ** Normally, whenever a table is dropped, all associated triggers are + ** dropped too. But if a TEMP trigger is created on a non-TEMP table + ** and the table is dropped by a different database connection, the + ** trigger is not visible to the database connection that does the + ** drop so the trigger cannot be dropped. This results in an + ** "orphaned trigger" - a trigger whose associated table is missing. + ** + ** 2020-11-05 see also https://sqlite.org/forum/forumpost/157dc791df + */ + db->init.orphanTrigger = 1; + } + goto trigger_cleanup; +} + +/* +** This routine is called after all of the trigger actions have been parsed +** in order to complete the process of building the trigger. +*/ +SQLITE_PRIVATE void sqlite3FinishTrigger( + Parse *pParse, /* Parser context */ + TriggerStep *pStepList, /* The triggered program */ + Token *pAll /* Token that describes the complete CREATE TRIGGER */ +){ + Trigger *pTrig = pParse->pNewTrigger; /* Trigger being finished */ + char *zName; /* Name of trigger */ + sqlite3 *db = pParse->db; /* The database */ + DbFixer sFix; /* Fixer object */ + int iDb; /* Database containing the trigger */ + Token nameToken; /* Trigger name for error reporting */ + + pParse->pNewTrigger = 0; + if( NEVER(pParse->nErr) || !pTrig ) goto triggerfinish_cleanup; + zName = pTrig->zName; + iDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); + pTrig->step_list = pStepList; + while( pStepList ){ + pStepList->pTrig = pTrig; + pStepList = pStepList->pNext; + } + sqlite3TokenInit(&nameToken, pTrig->zName); + sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken); + if( sqlite3FixTriggerStep(&sFix, pTrig->step_list) + || sqlite3FixExpr(&sFix, pTrig->pWhen) + ){ + goto triggerfinish_cleanup; + } + +#ifndef SQLITE_OMIT_ALTERTABLE + if( IN_RENAME_OBJECT ){ + assert( !db->init.busy ); + pParse->pNewTrigger = pTrig; + pTrig = 0; + }else +#endif + + /* if we are not initializing, + ** build the sqlite_schema entry + */ + if( !db->init.busy ){ + Vdbe *v; + char *z; + + /* Make an entry in the sqlite_schema table */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto triggerfinish_cleanup; + sqlite3BeginWriteOperation(pParse, 0, iDb); + z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n); + testcase( z==0 ); + sqlite3NestedParse(pParse, + "INSERT INTO %Q." DFLT_SCHEMA_TABLE + " VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')", + db->aDb[iDb].zDbSName, zName, + pTrig->table, z); + sqlite3DbFree(db, z); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName), 0); + } + + if( db->init.busy ){ + Trigger *pLink = pTrig; + Hash *pHash = &db->aDb[iDb].pSchema->trigHash; + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + assert( pLink!=0 ); + pTrig = sqlite3HashInsert(pHash, zName, pTrig); + if( pTrig ){ + sqlite3OomFault(db); + }else if( pLink->pSchema==pLink->pTabSchema ){ + Table *pTab; + pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table); + assert( pTab!=0 ); + pLink->pNext = pTab->pTrigger; + pTab->pTrigger = pLink; + } + } + +triggerfinish_cleanup: + sqlite3DeleteTrigger(db, pTrig); + assert( IN_RENAME_OBJECT || !pParse->pNewTrigger ); + sqlite3DeleteTriggerStep(db, pStepList); +} + +/* +** Duplicate a range of text from an SQL statement, then convert all +** whitespace characters into ordinary space characters. +*/ +static char *triggerSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){ + char *z = sqlite3DbSpanDup(db, zStart, zEnd); + int i; + if( z ) for(i=0; z[i]; i++) if( sqlite3Isspace(z[i]) ) z[i] = ' '; + return z; +} + +/* +** Turn a SELECT statement (that the pSelect parameter points to) into +** a trigger step. Return a pointer to a TriggerStep structure. +** +** The parser calls this routine when it finds a SELECT statement in +** body of a TRIGGER. +*/ +SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep( + sqlite3 *db, /* Database connection */ + Select *pSelect, /* The SELECT statement */ + const char *zStart, /* Start of SQL text */ + const char *zEnd /* End of SQL text */ +){ + TriggerStep *pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep)); + if( pTriggerStep==0 ) { + sqlite3SelectDelete(db, pSelect); + return 0; + } + pTriggerStep->op = TK_SELECT; + pTriggerStep->pSelect = pSelect; + pTriggerStep->orconf = OE_Default; + pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); + return pTriggerStep; +} + +/* +** Allocate space to hold a new trigger step. The allocated space +** holds both the TriggerStep object and the TriggerStep.target.z string. +** +** If an OOM error occurs, NULL is returned and db->mallocFailed is set. +*/ +static TriggerStep *triggerStepAllocate( + Parse *pParse, /* Parser context */ + u8 op, /* Trigger opcode */ + Token *pName, /* The target name */ + const char *zStart, /* Start of SQL text */ + const char *zEnd /* End of SQL text */ +){ + sqlite3 *db = pParse->db; + TriggerStep *pTriggerStep; + + pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); + if( pTriggerStep ){ + char *z = (char*)&pTriggerStep[1]; + memcpy(z, pName->z, pName->n); + sqlite3Dequote(z); + pTriggerStep->zTarget = z; + pTriggerStep->op = op; + pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, pTriggerStep->zTarget, pName); + } + } + return pTriggerStep; +} + +/* +** Build a trigger step out of an INSERT statement. Return a pointer +** to the new trigger step. +** +** The parser calls this routine when it sees an INSERT inside the +** body of a trigger. +*/ +SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( + Parse *pParse, /* Parser */ + Token *pTableName, /* Name of the table into which we insert */ + IdList *pColumn, /* List of columns in pTableName to insert into */ + Select *pSelect, /* A SELECT statement that supplies values */ + u8 orconf, /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ + Upsert *pUpsert, /* ON CONFLICT clauses for upsert */ + const char *zStart, /* Start of SQL text */ + const char *zEnd /* End of SQL text */ +){ + sqlite3 *db = pParse->db; + TriggerStep *pTriggerStep; + + assert(pSelect != 0 || db->mallocFailed); + + pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTableName,zStart,zEnd); + if( pTriggerStep ){ + if( IN_RENAME_OBJECT ){ + pTriggerStep->pSelect = pSelect; + pSelect = 0; + }else{ + pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + } + pTriggerStep->pIdList = pColumn; + pTriggerStep->pUpsert = pUpsert; + pTriggerStep->orconf = orconf; + if( pUpsert ){ + sqlite3HasExplicitNulls(pParse, pUpsert->pUpsertTarget); + } + }else{ + testcase( pColumn ); + sqlite3IdListDelete(db, pColumn); + testcase( pUpsert ); + sqlite3UpsertDelete(db, pUpsert); + } + sqlite3SelectDelete(db, pSelect); + + return pTriggerStep; +} + +/* +** Construct a trigger step that implements an UPDATE statement and return +** a pointer to that trigger step. The parser calls this routine when it +** sees an UPDATE statement inside the body of a CREATE TRIGGER. +*/ +SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( + Parse *pParse, /* Parser */ + Token *pTableName, /* Name of the table to be updated */ + SrcList *pFrom, + ExprList *pEList, /* The SET clause: list of column and new values */ + Expr *pWhere, /* The WHERE clause */ + u8 orconf, /* The conflict algorithm. (OE_Abort, OE_Ignore, etc) */ + const char *zStart, /* Start of SQL text */ + const char *zEnd /* End of SQL text */ +){ + sqlite3 *db = pParse->db; + TriggerStep *pTriggerStep; + + pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTableName,zStart,zEnd); + if( pTriggerStep ){ + if( IN_RENAME_OBJECT ){ + pTriggerStep->pExprList = pEList; + pTriggerStep->pWhere = pWhere; + pTriggerStep->pFrom = pFrom; + pEList = 0; + pWhere = 0; + pFrom = 0; + }else{ + pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + pTriggerStep->pFrom = sqlite3SrcListDup(db, pFrom, EXPRDUP_REDUCE); + } + pTriggerStep->orconf = orconf; + } + sqlite3ExprListDelete(db, pEList); + sqlite3ExprDelete(db, pWhere); + sqlite3SrcListDelete(db, pFrom); + return pTriggerStep; +} + +/* +** Construct a trigger step that implements a DELETE statement and return +** a pointer to that trigger step. The parser calls this routine when it +** sees a DELETE statement inside the body of a CREATE TRIGGER. +*/ +SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep( + Parse *pParse, /* Parser */ + Token *pTableName, /* The table from which rows are deleted */ + Expr *pWhere, /* The WHERE clause */ + const char *zStart, /* Start of SQL text */ + const char *zEnd /* End of SQL text */ +){ + sqlite3 *db = pParse->db; + TriggerStep *pTriggerStep; + + pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTableName,zStart,zEnd); + if( pTriggerStep ){ + if( IN_RENAME_OBJECT ){ + pTriggerStep->pWhere = pWhere; + pWhere = 0; + }else{ + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + } + pTriggerStep->orconf = OE_Default; + } + sqlite3ExprDelete(db, pWhere); + return pTriggerStep; +} + +/* +** Recursively delete a Trigger structure +*/ +SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3 *db, Trigger *pTrigger){ + if( pTrigger==0 || pTrigger->bReturning ) return; + sqlite3DeleteTriggerStep(db, pTrigger->step_list); + sqlite3DbFree(db, pTrigger->zName); + sqlite3DbFree(db, pTrigger->table); + sqlite3ExprDelete(db, pTrigger->pWhen); + sqlite3IdListDelete(db, pTrigger->pColumns); + sqlite3DbFree(db, pTrigger); +} + +/* +** This function is called to drop a trigger from the database schema. +** +** This may be called directly from the parser and therefore identifies +** the trigger by name. The sqlite3DropTriggerPtr() routine does the +** same job as this routine except it takes a pointer to the trigger +** instead of the trigger name. +**/ +SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){ + Trigger *pTrigger = 0; + int i; + const char *zDb; + const char *zName; + sqlite3 *db = pParse->db; + + if( db->mallocFailed ) goto drop_trigger_cleanup; + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + goto drop_trigger_cleanup; + } + + assert( pName->nSrc==1 ); + zDb = pName->a[0].zDatabase; + zName = pName->a[0].zName; + assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); + for(i=OMIT_TEMPDB; inDb; i++){ + int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ + if( zDb && sqlite3DbIsNamed(db, j, zDb)==0 ) continue; + assert( sqlite3SchemaMutexHeld(db, j, 0) ); + pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName); + if( pTrigger ) break; + } + if( !pTrigger ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "no such trigger: %S", pName->a); + }else{ + sqlite3CodeVerifyNamedSchema(pParse, zDb); + } + pParse->checkSchema = 1; + goto drop_trigger_cleanup; + } + sqlite3DropTriggerPtr(pParse, pTrigger); + +drop_trigger_cleanup: + sqlite3SrcListDelete(db, pName); +} + +/* +** Return a pointer to the Table structure for the table that a trigger +** is set on. +*/ +static Table *tableOfTrigger(Trigger *pTrigger){ + return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table); +} + + +/* +** Drop a trigger given a pointer to that trigger. +*/ +SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ + Table *pTable; + Vdbe *v; + sqlite3 *db = pParse->db; + int iDb; + + iDb = sqlite3SchemaToIndex(pParse->db, pTrigger->pSchema); + assert( iDb>=0 && iDbnDb ); + pTable = tableOfTrigger(pTrigger); + assert( (pTable && pTable->pSchema==pTrigger->pSchema) || iDb==1 ); +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pTable ){ + int code = SQLITE_DROP_TRIGGER; + const char *zDb = db->aDb[iDb].zDbSName; + const char *zTab = SCHEMA_TABLE(iDb); + if( iDb==1 ) code = SQLITE_DROP_TEMP_TRIGGER; + if( sqlite3AuthCheck(pParse, code, pTrigger->zName, pTable->zName, zDb) || + sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ + return; + } + } +#endif + + /* Generate code to destroy the database record of the trigger. + */ + if( (v = sqlite3GetVdbe(pParse))!=0 ){ + sqlite3NestedParse(pParse, + "DELETE FROM %Q." DFLT_SCHEMA_TABLE " WHERE name=%Q AND type='trigger'", + db->aDb[iDb].zDbSName, pTrigger->zName + ); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddOp4(v, OP_DropTrigger, iDb, 0, 0, pTrigger->zName, 0); + } +} + +/* +** Remove a trigger from the hash tables of the sqlite* pointer. +*/ +SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const char *zName){ + Trigger *pTrigger; + Hash *pHash; + + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + pHash = &(db->aDb[iDb].pSchema->trigHash); + pTrigger = sqlite3HashInsert(pHash, zName, 0); + if( ALWAYS(pTrigger) ){ + if( pTrigger->pSchema==pTrigger->pTabSchema ){ + Table *pTab = tableOfTrigger(pTrigger); + if( pTab ){ + Trigger **pp; + for(pp=&pTab->pTrigger; *pp; pp=&((*pp)->pNext)){ + if( *pp==pTrigger ){ + *pp = (*pp)->pNext; + break; + } + } + } + } + sqlite3DeleteTrigger(db, pTrigger); + db->mDbFlags |= DBFLAG_SchemaChange; + } +} + +/* +** pEList is the SET clause of an UPDATE statement. Each entry +** in pEList is of the format =. If any of the entries +** in pEList have an which matches an identifier in pIdList, +** then return TRUE. If pIdList==NULL, then it is considered a +** wildcard that matches anything. Likewise if pEList==NULL then +** it matches anything so always return true. Return false only +** if there is no match. +*/ +static int checkColumnOverlap(IdList *pIdList, ExprList *pEList){ + int e; + if( pIdList==0 || NEVER(pEList==0) ) return 1; + for(e=0; enExpr; e++){ + if( sqlite3IdListIndex(pIdList, pEList->a[e].zEName)>=0 ) return 1; + } + return 0; +} + +/* +** Return a list of all triggers on table pTab if there exists at least +** one trigger that must be fired when an operation of type 'op' is +** performed on the table, and, if that operation is an UPDATE, if at +** least one of the columns in pChanges is being modified. +*/ +SQLITE_PRIVATE Trigger *sqlite3TriggersExist( + Parse *pParse, /* Parse context */ + Table *pTab, /* The table the contains the triggers */ + int op, /* one of TK_DELETE, TK_INSERT, TK_UPDATE */ + ExprList *pChanges, /* Columns that change in an UPDATE statement */ + int *pMask /* OUT: Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ +){ + int mask = 0; + Trigger *pList = 0; + Trigger *p; + + pList = sqlite3TriggerList(pParse, pTab); + assert( pList==0 || IsVirtual(pTab)==0 + || (pList->bReturning && pList->pNext==0) ); + if( pList!=0 ){ + p = pList; + if( (pParse->db->flags & SQLITE_EnableTrigger)==0 + && pTab->pTrigger!=0 + ){ + /* The SQLITE_DBCONFIG_ENABLE_TRIGGER setting is off. That means that + ** only TEMP triggers are allowed. Truncate the pList so that it + ** includes only TEMP triggers */ + if( pList==pTab->pTrigger ){ + pList = 0; + goto exit_triggers_exist; + } + while( ALWAYS(p->pNext) && p->pNext!=pTab->pTrigger ) p = p->pNext; + p->pNext = 0; + p = pList; + } + do{ + if( p->op==op && checkColumnOverlap(p->pColumns, pChanges) ){ + mask |= p->tr_tm; + }else if( p->op==TK_RETURNING ){ + /* The first time a RETURNING trigger is seen, the "op" value tells + ** us what time of trigger it should be. */ + assert( sqlite3IsToplevel(pParse) ); + p->op = op; + if( IsVirtual(pTab) ){ + if( op!=TK_INSERT ){ + sqlite3ErrorMsg(pParse, + "%s RETURNING is not available on virtual tables", + op==TK_DELETE ? "DELETE" : "UPDATE"); + } + p->tr_tm = TRIGGER_BEFORE; + }else{ + p->tr_tm = TRIGGER_AFTER; + } + mask |= p->tr_tm; + }else if( p->bReturning && p->op==TK_INSERT && op==TK_UPDATE + && sqlite3IsToplevel(pParse) ){ + /* Also fire a RETURNING trigger for an UPSERT */ + mask |= p->tr_tm; + } + p = p->pNext; + }while( p ); + } +exit_triggers_exist: + if( pMask ){ + *pMask = mask; + } + return (mask ? pList : 0); +} + +/* +** Convert the pStep->zTarget string into a SrcList and return a pointer +** to that SrcList. +** +** This routine adds a specific database name, if needed, to the target when +** forming the SrcList. This prevents a trigger in one database from +** referring to a target in another database. An exception is when the +** trigger is in TEMP in which case it can refer to any other database it +** wants. +*/ +SQLITE_PRIVATE SrcList *sqlite3TriggerStepSrc( + Parse *pParse, /* The parsing context */ + TriggerStep *pStep /* The trigger containing the target token */ +){ + sqlite3 *db = pParse->db; + SrcList *pSrc; /* SrcList to be returned */ + char *zName = sqlite3DbStrDup(db, pStep->zTarget); + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + assert( pSrc==0 || pSrc->nSrc==1 ); + assert( zName || pSrc==0 ); + if( pSrc ){ + Schema *pSchema = pStep->pTrig->pSchema; + pSrc->a[0].zName = zName; + if( pSchema!=db->aDb[1].pSchema ){ + pSrc->a[0].pSchema = pSchema; + } + if( pStep->pFrom ){ + SrcList *pDup = sqlite3SrcListDup(db, pStep->pFrom, 0); + pSrc = sqlite3SrcListAppendList(pParse, pSrc, pDup); + } + }else{ + sqlite3DbFree(db, zName); + } + return pSrc; +} + +/* +** Return true if the pExpr term from the RETURNING clause argument +** list is of the form "*". Raise an error if the terms if of the +** form "table.*". +*/ +static int isAsteriskTerm( + Parse *pParse, /* Parsing context */ + Expr *pTerm /* A term in the RETURNING clause */ +){ + assert( pTerm!=0 ); + if( pTerm->op==TK_ASTERISK ) return 1; + if( pTerm->op!=TK_DOT ) return 0; + assert( pTerm->pRight!=0 ); + assert( pTerm->pLeft!=0 ); + if( pTerm->pRight->op!=TK_ASTERISK ) return 0; + sqlite3ErrorMsg(pParse, "RETURNING may not use \"TABLE.*\" wildcards"); + return 1; +} + +/* The input list pList is the list of result set terms from a RETURNING +** clause. The table that we are returning from is pTab. +** +** This routine makes a copy of the pList, and at the same time expands +** any "*" wildcards to be the complete set of columns from pTab. +*/ +static ExprList *sqlite3ExpandReturning( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* The arguments to RETURNING */ + Table *pTab /* The table being updated */ +){ + ExprList *pNew = 0; + sqlite3 *db = pParse->db; + int i; + + for(i=0; inExpr; i++){ + Expr *pOldExpr = pList->a[i].pExpr; + if( NEVER(pOldExpr==0) ) continue; + if( isAsteriskTerm(pParse, pOldExpr) ){ + int jj; + for(jj=0; jjnCol; jj++){ + Expr *pNewExpr; + if( IsHiddenColumn(pTab->aCol+jj) ) continue; + pNewExpr = sqlite3Expr(db, TK_ID, pTab->aCol[jj].zCnName); + pNew = sqlite3ExprListAppend(pParse, pNew, pNewExpr); + if( !db->mallocFailed ){ + struct ExprList_item *pItem = &pNew->a[pNew->nExpr-1]; + pItem->zEName = sqlite3DbStrDup(db, pTab->aCol[jj].zCnName); + pItem->eEName = ENAME_NAME; + } + } + }else{ + Expr *pNewExpr = sqlite3ExprDup(db, pOldExpr, 0); + pNew = sqlite3ExprListAppend(pParse, pNew, pNewExpr); + if( !db->mallocFailed && ALWAYS(pList->a[i].zEName!=0) ){ + struct ExprList_item *pItem = &pNew->a[pNew->nExpr-1]; + pItem->zEName = sqlite3DbStrDup(db, pList->a[i].zEName); + pItem->eEName = pList->a[i].eEName; + } + } + } + return pNew; +} + +/* +** Generate code for the RETURNING trigger. Unlike other triggers +** that invoke a subprogram in the bytecode, the code for RETURNING +** is generated in-line. +*/ +static void codeReturningTrigger( + Parse *pParse, /* Parse context */ + Trigger *pTrigger, /* The trigger step that defines the RETURNING */ + Table *pTab, /* The table to code triggers from */ + int regIn /* The first in an array of registers */ +){ + Vdbe *v = pParse->pVdbe; + sqlite3 *db = pParse->db; + ExprList *pNew; + Returning *pReturning; + Select sSelect; + SrcList sFrom; + + assert( v!=0 ); + assert( pParse->bReturning ); + pReturning = pParse->u1.pReturning; + assert( pTrigger == &(pReturning->retTrig) ); + memset(&sSelect, 0, sizeof(sSelect)); + memset(&sFrom, 0, sizeof(sFrom)); + sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); + sSelect.pSrc = &sFrom; + sFrom.nSrc = 1; + sFrom.a[0].pTab = pTab; + sFrom.a[0].iCursor = -1; + sqlite3SelectPrep(pParse, &sSelect, 0); + if( db->mallocFailed==0 && pParse->nErr==0 ){ + sqlite3GenerateColumnNames(pParse, &sSelect); + } + sqlite3ExprListDelete(db, sSelect.pEList); + pNew = sqlite3ExpandReturning(pParse, pReturning->pReturnEL, pTab); + if( pNew ){ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + if( pReturning->nRetCol==0 ){ + pReturning->nRetCol = pNew->nExpr; + pReturning->iRetCur = pParse->nTab++; + } + sNC.pParse = pParse; + sNC.uNC.iBaseReg = regIn; + sNC.ncFlags = NC_UBaseReg; + pParse->eTriggerOp = pTrigger->op; + pParse->pTriggerTab = pTab; + if( sqlite3ResolveExprListNames(&sNC, pNew)==SQLITE_OK ){ + int i; + int nCol = pNew->nExpr; + int reg = pParse->nMem+1; + pParse->nMem += nCol+2; + pReturning->iRetReg = reg; + for(i=0; ia[i].pExpr; + sqlite3ExprCodeFactorable(pParse, pCol, reg+i); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, reg, i, reg+i); + sqlite3VdbeAddOp2(v, OP_NewRowid, pReturning->iRetCur, reg+i+1); + sqlite3VdbeAddOp3(v, OP_Insert, pReturning->iRetCur, reg+i, reg+i+1); + } + sqlite3ExprListDelete(db, pNew); + pParse->eTriggerOp = 0; + pParse->pTriggerTab = 0; + } +} + + + +/* +** Generate VDBE code for the statements inside the body of a single +** trigger. +*/ +static int codeTriggerProgram( + Parse *pParse, /* The parser context */ + TriggerStep *pStepList, /* List of statements inside the trigger body */ + int orconf /* Conflict algorithm. (OE_Abort, etc) */ +){ + TriggerStep *pStep; + Vdbe *v = pParse->pVdbe; + sqlite3 *db = pParse->db; + + assert( pParse->pTriggerTab && pParse->pToplevel ); + assert( pStepList ); + assert( v!=0 ); + for(pStep=pStepList; pStep; pStep=pStep->pNext){ + /* Figure out the ON CONFLICT policy that will be used for this step + ** of the trigger program. If the statement that caused this trigger + ** to fire had an explicit ON CONFLICT, then use it. Otherwise, use + ** the ON CONFLICT policy that was specified as part of the trigger + ** step statement. Example: + ** + ** CREATE TRIGGER AFTER INSERT ON t1 BEGIN; + ** INSERT OR REPLACE INTO t2 VALUES(new.a, new.b); + ** END; + ** + ** INSERT INTO t1 ... ; -- insert into t2 uses REPLACE policy + ** INSERT OR IGNORE INTO t1 ... ; -- insert into t2 uses IGNORE policy + */ + pParse->eOrconf = (orconf==OE_Default)?pStep->orconf:(u8)orconf; + assert( pParse->okConstFactor==0 ); + +#ifndef SQLITE_OMIT_TRACE + if( pStep->zSpan ){ + sqlite3VdbeAddOp4(v, OP_Trace, 0x7fffffff, 1, 0, + sqlite3MPrintf(db, "-- %s", pStep->zSpan), + P4_DYNAMIC); + } +#endif + + switch( pStep->op ){ + case TK_UPDATE: { + sqlite3Update(pParse, + sqlite3TriggerStepSrc(pParse, pStep), + sqlite3ExprListDup(db, pStep->pExprList, 0), + sqlite3ExprDup(db, pStep->pWhere, 0), + pParse->eOrconf, 0, 0, 0 + ); + sqlite3VdbeAddOp0(v, OP_ResetCount); + break; + } + case TK_INSERT: { + sqlite3Insert(pParse, + sqlite3TriggerStepSrc(pParse, pStep), + sqlite3SelectDup(db, pStep->pSelect, 0), + sqlite3IdListDup(db, pStep->pIdList), + pParse->eOrconf, + sqlite3UpsertDup(db, pStep->pUpsert) + ); + sqlite3VdbeAddOp0(v, OP_ResetCount); + break; + } + case TK_DELETE: { + sqlite3DeleteFrom(pParse, + sqlite3TriggerStepSrc(pParse, pStep), + sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0 + ); + sqlite3VdbeAddOp0(v, OP_ResetCount); + break; + } + default: assert( pStep->op==TK_SELECT ); { + SelectDest sDest; + Select *pSelect = sqlite3SelectDup(db, pStep->pSelect, 0); + sqlite3SelectDestInit(&sDest, SRT_Discard, 0); + sqlite3Select(pParse, pSelect, &sDest); + sqlite3SelectDelete(db, pSelect); + break; + } + } + } + + return 0; +} + +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +/* +** This function is used to add VdbeComment() annotations to a VDBE +** program. It is not used in production code, only for debugging. +*/ +static const char *onErrorText(int onError){ + switch( onError ){ + case OE_Abort: return "abort"; + case OE_Rollback: return "rollback"; + case OE_Fail: return "fail"; + case OE_Replace: return "replace"; + case OE_Ignore: return "ignore"; + case OE_Default: return "default"; + } + return "n/a"; +} +#endif + +/* +** Parse context structure pFrom has just been used to create a sub-vdbe +** (trigger program). If an error has occurred, transfer error information +** from pFrom to pTo. +*/ +static void transferParseError(Parse *pTo, Parse *pFrom){ + assert( pFrom->zErrMsg==0 || pFrom->nErr ); + assert( pTo->zErrMsg==0 || pTo->nErr ); + if( pTo->nErr==0 ){ + pTo->zErrMsg = pFrom->zErrMsg; + pTo->nErr = pFrom->nErr; + pTo->rc = pFrom->rc; + }else{ + sqlite3DbFree(pFrom->db, pFrom->zErrMsg); + } +} + +/* +** Create and populate a new TriggerPrg object with a sub-program +** implementing trigger pTrigger with ON CONFLICT policy orconf. +*/ +static TriggerPrg *codeRowTrigger( + Parse *pParse, /* Current parse context */ + Trigger *pTrigger, /* Trigger to code */ + Table *pTab, /* The table pTrigger is attached to */ + int orconf /* ON CONFLICT policy to code trigger program with */ +){ + Parse *pTop = sqlite3ParseToplevel(pParse); + sqlite3 *db = pParse->db; /* Database handle */ + TriggerPrg *pPrg; /* Value to return */ + Expr *pWhen = 0; /* Duplicate of trigger WHEN expression */ + Vdbe *v; /* Temporary VM */ + NameContext sNC; /* Name context for sub-vdbe */ + SubProgram *pProgram = 0; /* Sub-vdbe for trigger program */ + Parse *pSubParse; /* Parse context for sub-vdbe */ + int iEndTrigger = 0; /* Label to jump to if WHEN is false */ + + assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); + assert( pTop->pVdbe ); + + /* Allocate the TriggerPrg and SubProgram objects. To ensure that they + ** are freed if an error occurs, link them into the Parse.pTriggerPrg + ** list of the top-level Parse object sooner rather than later. */ + pPrg = sqlite3DbMallocZero(db, sizeof(TriggerPrg)); + if( !pPrg ) return 0; + pPrg->pNext = pTop->pTriggerPrg; + pTop->pTriggerPrg = pPrg; + pPrg->pProgram = pProgram = sqlite3DbMallocZero(db, sizeof(SubProgram)); + if( !pProgram ) return 0; + sqlite3VdbeLinkSubProgram(pTop->pVdbe, pProgram); + pPrg->pTrigger = pTrigger; + pPrg->orconf = orconf; + pPrg->aColmask[0] = 0xffffffff; + pPrg->aColmask[1] = 0xffffffff; + + /* Allocate and populate a new Parse context to use for coding the + ** trigger sub-program. */ + pSubParse = sqlite3StackAllocZero(db, sizeof(Parse)); + if( !pSubParse ) return 0; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pSubParse; + pSubParse->db = db; + pSubParse->pTriggerTab = pTab; + pSubParse->pToplevel = pTop; + pSubParse->zAuthContext = pTrigger->zName; + pSubParse->eTriggerOp = pTrigger->op; + pSubParse->nQueryLoop = pParse->nQueryLoop; + pSubParse->disableVtab = pParse->disableVtab; + + v = sqlite3GetVdbe(pSubParse); + if( v ){ + VdbeComment((v, "Start: %s.%s (%s %s%s%s ON %s)", + pTrigger->zName, onErrorText(orconf), + (pTrigger->tr_tm==TRIGGER_BEFORE ? "BEFORE" : "AFTER"), + (pTrigger->op==TK_UPDATE ? "UPDATE" : ""), + (pTrigger->op==TK_INSERT ? "INSERT" : ""), + (pTrigger->op==TK_DELETE ? "DELETE" : ""), + pTab->zName + )); +#ifndef SQLITE_OMIT_TRACE + if( pTrigger->zName ){ + sqlite3VdbeChangeP4(v, -1, + sqlite3MPrintf(db, "-- TRIGGER %s", pTrigger->zName), P4_DYNAMIC + ); + } +#endif + + /* If one was specified, code the WHEN clause. If it evaluates to false + ** (or NULL) the sub-vdbe is immediately halted by jumping to the + ** OP_Halt inserted at the end of the program. */ + if( pTrigger->pWhen ){ + pWhen = sqlite3ExprDup(db, pTrigger->pWhen, 0); + if( db->mallocFailed==0 + && SQLITE_OK==sqlite3ResolveExprNames(&sNC, pWhen) + ){ + iEndTrigger = sqlite3VdbeMakeLabel(pSubParse); + sqlite3ExprIfFalse(pSubParse, pWhen, iEndTrigger, SQLITE_JUMPIFNULL); + } + sqlite3ExprDelete(db, pWhen); + } + + /* Code the trigger program into the sub-vdbe. */ + codeTriggerProgram(pSubParse, pTrigger->step_list, orconf); + + /* Insert an OP_Halt at the end of the sub-program. */ + if( iEndTrigger ){ + sqlite3VdbeResolveLabel(v, iEndTrigger); + } + sqlite3VdbeAddOp0(v, OP_Halt); + VdbeComment((v, "End: %s.%s", pTrigger->zName, onErrorText(orconf))); + + transferParseError(pParse, pSubParse); + if( db->mallocFailed==0 && pParse->nErr==0 ){ + pProgram->aOp = sqlite3VdbeTakeOpArray(v, &pProgram->nOp, &pTop->nMaxArg); + } + pProgram->nMem = pSubParse->nMem; + pProgram->nCsr = pSubParse->nTab; + pProgram->token = (void *)pTrigger; + pPrg->aColmask[0] = pSubParse->oldmask; + pPrg->aColmask[1] = pSubParse->newmask; + sqlite3VdbeDelete(v); + } + + assert( !pSubParse->pTriggerPrg && !pSubParse->nMaxArg ); + sqlite3ParserReset(pSubParse); + sqlite3StackFree(db, pSubParse); + + return pPrg; +} + +/* +** Return a pointer to a TriggerPrg object containing the sub-program for +** trigger pTrigger with default ON CONFLICT algorithm orconf. If no such +** TriggerPrg object exists, a new object is allocated and populated before +** being returned. +*/ +static TriggerPrg *getRowTrigger( + Parse *pParse, /* Current parse context */ + Trigger *pTrigger, /* Trigger to code */ + Table *pTab, /* The table trigger pTrigger is attached to */ + int orconf /* ON CONFLICT algorithm. */ +){ + Parse *pRoot = sqlite3ParseToplevel(pParse); + TriggerPrg *pPrg; + + assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); + + /* It may be that this trigger has already been coded (or is in the + ** process of being coded). If this is the case, then an entry with + ** a matching TriggerPrg.pTrigger field will be present somewhere + ** in the Parse.pTriggerPrg list. Search for such an entry. */ + for(pPrg=pRoot->pTriggerPrg; + pPrg && (pPrg->pTrigger!=pTrigger || pPrg->orconf!=orconf); + pPrg=pPrg->pNext + ); + + /* If an existing TriggerPrg could not be located, create a new one. */ + if( !pPrg ){ + pPrg = codeRowTrigger(pParse, pTrigger, pTab, orconf); + } + + return pPrg; +} + +/* +** Generate code for the trigger program associated with trigger p on +** table pTab. The reg, orconf and ignoreJump parameters passed to this +** function are the same as those described in the header function for +** sqlite3CodeRowTrigger() +*/ +SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( + Parse *pParse, /* Parse context */ + Trigger *p, /* Trigger to code */ + Table *pTab, /* The table to code triggers from */ + int reg, /* Reg array containing OLD.* and NEW.* values */ + int orconf, /* ON CONFLICT policy */ + int ignoreJump /* Instruction to jump to for RAISE(IGNORE) */ +){ + Vdbe *v = sqlite3GetVdbe(pParse); /* Main VM */ + TriggerPrg *pPrg; + pPrg = getRowTrigger(pParse, p, pTab, orconf); + assert( pPrg || pParse->nErr || pParse->db->mallocFailed ); + + /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program + ** is a pointer to the sub-vdbe containing the trigger program. */ + if( pPrg ){ + int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); + + sqlite3VdbeAddOp4(v, OP_Program, reg, ignoreJump, ++pParse->nMem, + (const char *)pPrg->pProgram, P4_SUBPROGRAM); + VdbeComment( + (v, "Call: %s.%s", (p->zName?p->zName:"fkey"), onErrorText(orconf))); + + /* Set the P5 operand of the OP_Program instruction to non-zero if + ** recursive invocation of this trigger program is disallowed. Recursive + ** invocation is disallowed if (a) the sub-program is really a trigger, + ** not a foreign key action, and (b) the flag to enable recursive triggers + ** is clear. */ + sqlite3VdbeChangeP5(v, (u8)bRecursive); + } +} + +/* +** This is called to code the required FOR EACH ROW triggers for an operation +** on table pTab. The operation to code triggers for (INSERT, UPDATE or DELETE) +** is given by the op parameter. The tr_tm parameter determines whether the +** BEFORE or AFTER triggers are coded. If the operation is an UPDATE, then +** parameter pChanges is passed the list of columns being modified. +** +** If there are no triggers that fire at the specified time for the specified +** operation on pTab, this function is a no-op. +** +** The reg argument is the address of the first in an array of registers +** that contain the values substituted for the new.* and old.* references +** in the trigger program. If N is the number of columns in table pTab +** (a copy of pTab->nCol), then registers are populated as follows: +** +** Register Contains +** ------------------------------------------------------ +** reg+0 OLD.rowid +** reg+1 OLD.* value of left-most column of pTab +** ... ... +** reg+N OLD.* value of right-most column of pTab +** reg+N+1 NEW.rowid +** reg+N+2 NEW.* value of left-most column of pTab +** ... ... +** reg+N+N+1 NEW.* value of right-most column of pTab +** +** For ON DELETE triggers, the registers containing the NEW.* values will +** never be accessed by the trigger program, so they are not allocated or +** populated by the caller (there is no data to populate them with anyway). +** Similarly, for ON INSERT triggers the values stored in the OLD.* registers +** are never accessed, and so are not allocated by the caller. So, for an +** ON INSERT trigger, the value passed to this function as parameter reg +** is not a readable register, although registers (reg+N) through +** (reg+N+N+1) are. +** +** Parameter orconf is the default conflict resolution algorithm for the +** trigger program to use (REPLACE, IGNORE etc.). Parameter ignoreJump +** is the instruction that control should jump to if a trigger program +** raises an IGNORE exception. +*/ +SQLITE_PRIVATE void sqlite3CodeRowTrigger( + Parse *pParse, /* Parse context */ + Trigger *pTrigger, /* List of triggers on table pTab */ + int op, /* One of TK_UPDATE, TK_INSERT, TK_DELETE */ + ExprList *pChanges, /* Changes list for any UPDATE OF triggers */ + int tr_tm, /* One of TRIGGER_BEFORE, TRIGGER_AFTER */ + Table *pTab, /* The table to code triggers from */ + int reg, /* The first in an array of registers (see above) */ + int orconf, /* ON CONFLICT policy */ + int ignoreJump /* Instruction to jump to for RAISE(IGNORE) */ +){ + Trigger *p; /* Used to iterate through pTrigger list */ + + assert( op==TK_UPDATE || op==TK_INSERT || op==TK_DELETE ); + assert( tr_tm==TRIGGER_BEFORE || tr_tm==TRIGGER_AFTER ); + assert( (op==TK_UPDATE)==(pChanges!=0) ); + + for(p=pTrigger; p; p=p->pNext){ + + /* Sanity checking: The schema for the trigger and for the table are + ** always defined. The trigger must be in the same schema as the table + ** or else it must be a TEMP trigger. */ + assert( p->pSchema!=0 ); + assert( p->pTabSchema!=0 ); + assert( p->pSchema==p->pTabSchema + || p->pSchema==pParse->db->aDb[1].pSchema ); + + /* Determine whether we should code this trigger. One of two choices: + ** 1. The trigger is an exact match to the current DML statement + ** 2. This is a RETURNING trigger for INSERT but we are currently + ** doing the UPDATE part of an UPSERT. + */ + if( (p->op==op || (p->bReturning && p->op==TK_INSERT && op==TK_UPDATE)) + && p->tr_tm==tr_tm + && checkColumnOverlap(p->pColumns, pChanges) + ){ + if( !p->bReturning ){ + sqlite3CodeRowTriggerDirect(pParse, p, pTab, reg, orconf, ignoreJump); + }else if( sqlite3IsToplevel(pParse) ){ + codeReturningTrigger(pParse, p, pTab, reg); + } + } + } +} + +/* +** Triggers may access values stored in the old.* or new.* pseudo-table. +** This function returns a 32-bit bitmask indicating which columns of the +** old.* or new.* tables actually are used by triggers. This information +** may be used by the caller, for example, to avoid having to load the entire +** old.* record into memory when executing an UPDATE or DELETE command. +** +** Bit 0 of the returned mask is set if the left-most column of the +** table may be accessed using an [old|new].reference. Bit 1 is set if +** the second leftmost column value is required, and so on. If there +** are more than 32 columns in the table, and at least one of the columns +** with an index greater than 32 may be accessed, 0xffffffff is returned. +** +** It is not possible to determine if the old.rowid or new.rowid column is +** accessed by triggers. The caller must always assume that it is. +** +** Parameter isNew must be either 1 or 0. If it is 0, then the mask returned +** applies to the old.* table. If 1, the new.* table. +** +** Parameter tr_tm must be a mask with one or both of the TRIGGER_BEFORE +** and TRIGGER_AFTER bits set. Values accessed by BEFORE triggers are only +** included in the returned mask if the TRIGGER_BEFORE bit is set in the +** tr_tm parameter. Similarly, values accessed by AFTER triggers are only +** included in the returned mask if the TRIGGER_AFTER bit is set in tr_tm. +*/ +SQLITE_PRIVATE u32 sqlite3TriggerColmask( + Parse *pParse, /* Parse context */ + Trigger *pTrigger, /* List of triggers on table pTab */ + ExprList *pChanges, /* Changes list for any UPDATE OF triggers */ + int isNew, /* 1 for new.* ref mask, 0 for old.* ref mask */ + int tr_tm, /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ + Table *pTab, /* The table to code triggers from */ + int orconf /* Default ON CONFLICT policy for trigger steps */ +){ + const int op = pChanges ? TK_UPDATE : TK_DELETE; + u32 mask = 0; + Trigger *p; + + assert( isNew==1 || isNew==0 ); + for(p=pTrigger; p; p=p->pNext){ + if( p->op==op + && (tr_tm&p->tr_tm) + && checkColumnOverlap(p->pColumns,pChanges) + ){ + if( p->bReturning ){ + mask = 0xffffffff; + }else{ + TriggerPrg *pPrg; + pPrg = getRowTrigger(pParse, p, pTab, orconf); + if( pPrg ){ + mask |= pPrg->aColmask[isNew]; + } + } + } + } + + return mask; +} + +#endif /* !defined(SQLITE_OMIT_TRIGGER) */ + +/************** End of trigger.c *********************************************/ +/************** Begin file update.c ******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C code routines that are called by the parser +** to handle UPDATE statements. +*/ +/* #include "sqliteInt.h" */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Forward declaration */ +static void updateVirtualTable( + Parse *pParse, /* The parsing context */ + SrcList *pSrc, /* The virtual table to be modified */ + Table *pTab, /* The virtual table */ + ExprList *pChanges, /* The columns to change in the UPDATE statement */ + Expr *pRowidExpr, /* Expression used to recompute the rowid */ + int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ +); +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** The most recently coded instruction was an OP_Column to retrieve the +** i-th column of table pTab. This routine sets the P4 parameter of the +** OP_Column to the default value, if any. +** +** The default value of a column is specified by a DEFAULT clause in the +** column definition. This was either supplied by the user when the table +** was created, or added later to the table definition by an ALTER TABLE +** command. If the latter, then the row-records in the table btree on disk +** may not contain a value for the column and the default value, taken +** from the P4 parameter of the OP_Column instruction, is returned instead. +** If the former, then all row-records are guaranteed to include a value +** for the column and the P4 value is not required. +** +** Column definitions created by an ALTER TABLE command may only have +** literal default values specified: a number, null or a string. (If a more +** complicated default expression value was provided, it is evaluated +** when the ALTER TABLE is executed and one of the literal values written +** into the sqlite_schema table.) +** +** Therefore, the P4 parameter is only required if the default value for +** the column is a literal number, string or null. The sqlite3ValueFromExpr() +** function is capable of transforming these types of expressions into +** sqlite3_value objects. +** +** If column as REAL affinity and the table is an ordinary b-tree table +** (not a virtual table) then the value might have been stored as an +** integer. In that case, add an OP_RealAffinity opcode to make sure +** it has been converted into REAL. +*/ +SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ + assert( pTab!=0 ); + if( !IsView(pTab) ){ + sqlite3_value *pValue = 0; + u8 enc = ENC(sqlite3VdbeDb(v)); + Column *pCol = &pTab->aCol[i]; + VdbeComment((v, "%s.%s", pTab->zName, pCol->zCnName)); + assert( inCol ); + sqlite3ValueFromExpr(sqlite3VdbeDb(v), + sqlite3ColumnExpr(pTab,pCol), enc, + pCol->affinity, &pValue); + if( pValue ){ + sqlite3VdbeAppendP4(v, pValue, P4_MEM); + } + } +#ifndef SQLITE_OMIT_FLOATING_POINT + if( pTab->aCol[i].affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); + } +#endif +} + +/* +** Check to see if column iCol of index pIdx references any of the +** columns defined by aXRef and chngRowid. Return true if it does +** and false if not. This is an optimization. False-positives are a +** performance degradation, but false-negatives can result in a corrupt +** index and incorrect answers. +** +** aXRef[j] will be non-negative if column j of the original table is +** being updated. chngRowid will be true if the rowid of the table is +** being updated. +*/ +static int indexColumnIsBeingUpdated( + Index *pIdx, /* The index to check */ + int iCol, /* Which column of the index to check */ + int *aXRef, /* aXRef[j]>=0 if column j is being updated */ + int chngRowid /* true if the rowid is being updated */ +){ + i16 iIdxCol = pIdx->aiColumn[iCol]; + assert( iIdxCol!=XN_ROWID ); /* Cannot index rowid */ + if( iIdxCol>=0 ){ + return aXRef[iIdxCol]>=0; + } + assert( iIdxCol==XN_EXPR ); + assert( pIdx->aColExpr!=0 ); + assert( pIdx->aColExpr->a[iCol].pExpr!=0 ); + return sqlite3ExprReferencesUpdatedColumn(pIdx->aColExpr->a[iCol].pExpr, + aXRef,chngRowid); +} + +/* +** Check to see if index pIdx is a partial index whose conditional +** expression might change values due to an UPDATE. Return true if +** the index is subject to change and false if the index is guaranteed +** to be unchanged. This is an optimization. False-positives are a +** performance degradation, but false-negatives can result in a corrupt +** index and incorrect answers. +** +** aXRef[j] will be non-negative if column j of the original table is +** being updated. chngRowid will be true if the rowid of the table is +** being updated. +*/ +static int indexWhereClauseMightChange( + Index *pIdx, /* The index to check */ + int *aXRef, /* aXRef[j]>=0 if column j is being updated */ + int chngRowid /* true if the rowid is being updated */ +){ + if( pIdx->pPartIdxWhere==0 ) return 0; + return sqlite3ExprReferencesUpdatedColumn(pIdx->pPartIdxWhere, + aXRef, chngRowid); +} + +/* +** Allocate and return a pointer to an expression of type TK_ROW with +** Expr.iColumn set to value (iCol+1). The resolver will modify the +** expression to be a TK_COLUMN reading column iCol of the first +** table in the source-list (pSrc->a[0]). +*/ +static Expr *exprRowColumn(Parse *pParse, int iCol){ + Expr *pRet = sqlite3PExpr(pParse, TK_ROW, 0, 0); + if( pRet ) pRet->iColumn = iCol+1; + return pRet; +} + +/* +** Assuming both the pLimit and pOrderBy parameters are NULL, this function +** generates VM code to run the query: +** +** SELECT , pChanges FROM pTabList WHERE pWhere +** +** and write the results to the ephemeral table already opened as cursor +** iEph. None of pChanges, pTabList or pWhere are modified or consumed by +** this function, they must be deleted by the caller. +** +** Or, if pLimit and pOrderBy are not NULL, and pTab is not a view: +** +** SELECT , pChanges FROM pTabList +** WHERE pWhere +** GROUP BY +** ORDER BY pOrderBy LIMIT pLimit +** +** If pTab is a view, the GROUP BY clause is omitted. +** +** Exactly how results are written to table iEph, and exactly what +** the in the query above are is determined by the type +** of table pTabList->a[0].pTab. +** +** If the table is a WITHOUT ROWID table, then argument pPk must be its +** PRIMARY KEY. In this case are the primary key columns +** of the table, in order. The results of the query are written to ephemeral +** table iEph as index keys, using OP_IdxInsert. +** +** If the table is actually a view, then are all columns of +** the view. The results are written to the ephemeral table iEph as records +** with automatically assigned integer keys. +** +** If the table is a virtual or ordinary intkey table, then +** is its rowid. For a virtual table, the results are written to iEph as +** records with automatically assigned integer keys For intkey tables, the +** rowid value in is used as the integer key, and the +** remaining fields make up the table record. +*/ +static void updateFromSelect( + Parse *pParse, /* Parse context */ + int iEph, /* Cursor for open eph. table */ + Index *pPk, /* PK if table 0 is WITHOUT ROWID */ + ExprList *pChanges, /* List of expressions to return */ + SrcList *pTabList, /* List of tables to select from */ + Expr *pWhere, /* WHERE clause for query */ + ExprList *pOrderBy, /* ORDER BY clause */ + Expr *pLimit /* LIMIT clause */ +){ + int i; + SelectDest dest; + Select *pSelect = 0; + ExprList *pList = 0; + ExprList *pGrp = 0; + Expr *pLimit2 = 0; + ExprList *pOrderBy2 = 0; + sqlite3 *db = pParse->db; + Table *pTab = pTabList->a[0].pTab; + SrcList *pSrc; + Expr *pWhere2; + int eDest; + +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( pOrderBy && pLimit==0 ) { + sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on UPDATE"); + return; + } + pOrderBy2 = sqlite3ExprListDup(db, pOrderBy, 0); + pLimit2 = sqlite3ExprDup(db, pLimit, 0); +#else + UNUSED_PARAMETER(pOrderBy); + UNUSED_PARAMETER(pLimit); +#endif + + pSrc = sqlite3SrcListDup(db, pTabList, 0); + pWhere2 = sqlite3ExprDup(db, pWhere, 0); + + assert( pTabList->nSrc>1 ); + if( pSrc ){ + pSrc->a[0].fg.notCte = 1; + pSrc->a[0].iCursor = -1; + pSrc->a[0].pTab->nTabRef--; + pSrc->a[0].pTab = 0; + } + if( pPk ){ + for(i=0; inKeyCol; i++){ + Expr *pNew = exprRowColumn(pParse, pPk->aiColumn[i]); +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( pLimit ){ + pGrp = sqlite3ExprListAppend(pParse, pGrp, sqlite3ExprDup(db, pNew, 0)); + } +#endif + pList = sqlite3ExprListAppend(pParse, pList, pNew); + } + eDest = IsVirtual(pTab) ? SRT_Table : SRT_Upfrom; + }else if( IsView(pTab) ){ + for(i=0; inCol; i++){ + pList = sqlite3ExprListAppend(pParse, pList, exprRowColumn(pParse, i)); + } + eDest = SRT_Table; + }else{ + eDest = IsVirtual(pTab) ? SRT_Table : SRT_Upfrom; + pList = sqlite3ExprListAppend(pParse, 0, sqlite3PExpr(pParse,TK_ROW,0,0)); +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( pLimit ){ + pGrp = sqlite3ExprListAppend(pParse, 0, sqlite3PExpr(pParse,TK_ROW,0,0)); + } +#endif + } + assert( pChanges!=0 || pParse->db->mallocFailed ); + if( pChanges ){ + for(i=0; inExpr; i++){ + pList = sqlite3ExprListAppend(pParse, pList, + sqlite3ExprDup(db, pChanges->a[i].pExpr, 0) + ); + } + } + pSelect = sqlite3SelectNew(pParse, pList, + pSrc, pWhere2, pGrp, 0, pOrderBy2, SF_UFSrcCheck|SF_IncludeHidden, pLimit2 + ); + if( pSelect ) pSelect->selFlags |= SF_OrderByReqd; + sqlite3SelectDestInit(&dest, eDest, iEph); + dest.iSDParm2 = (pPk ? pPk->nKeyCol : -1); + sqlite3Select(pParse, pSelect, &dest); + sqlite3SelectDelete(db, pSelect); +} + +/* +** Process an UPDATE statement. +** +** UPDATE OR IGNORE tbl SET a=b, c=d FROM tbl2... WHERE e<5 AND f NOT NULL; +** \_______/ \_/ \______/ \_____/ \________________/ +** onError | pChanges | pWhere +** \_______________________/ +** pTabList +*/ +SQLITE_PRIVATE void sqlite3Update( + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* The table in which we should change things */ + ExprList *pChanges, /* Things to be changed */ + Expr *pWhere, /* The WHERE clause. May be null */ + int onError, /* How to handle constraint errors */ + ExprList *pOrderBy, /* ORDER BY clause. May be null */ + Expr *pLimit, /* LIMIT clause. May be null */ + Upsert *pUpsert /* ON CONFLICT clause, or null */ +){ + int i, j, k; /* Loop counters */ + Table *pTab; /* The table to be updated */ + int addrTop = 0; /* VDBE instruction address of the start of the loop */ + WhereInfo *pWInfo = 0; /* Information about the WHERE clause */ + Vdbe *v; /* The virtual database engine */ + Index *pIdx; /* For looping over indices */ + Index *pPk; /* The PRIMARY KEY index for WITHOUT ROWID tables */ + int nIdx; /* Number of indices that need updating */ + int nAllIdx; /* Total number of indexes */ + int iBaseCur; /* Base cursor number */ + int iDataCur; /* Cursor for the canonical data btree */ + int iIdxCur; /* Cursor for the first index */ + sqlite3 *db; /* The database structure */ + int *aRegIdx = 0; /* Registers for to each index and the main table */ + int *aXRef = 0; /* aXRef[i] is the index in pChanges->a[] of the + ** an expression for the i-th column of the table. + ** aXRef[i]==-1 if the i-th column is not changed. */ + u8 *aToOpen; /* 1 for tables and indices to be opened */ + u8 chngPk; /* PRIMARY KEY changed in a WITHOUT ROWID table */ + u8 chngRowid; /* Rowid changed in a normal table */ + u8 chngKey; /* Either chngPk or chngRowid */ + Expr *pRowidExpr = 0; /* Expression defining the new record number */ + int iRowidExpr = -1; /* Index of "rowid=" (or IPK) assignment in pChanges */ + AuthContext sContext; /* The authorization context */ + NameContext sNC; /* The name-context to resolve expressions in */ + int iDb; /* Database containing the table being updated */ + int eOnePass; /* ONEPASS_XXX value from where.c */ + int hasFK; /* True if foreign key processing is required */ + int labelBreak; /* Jump here to break out of UPDATE loop */ + int labelContinue; /* Jump here to continue next step of UPDATE loop */ + int flags; /* Flags for sqlite3WhereBegin() */ + +#ifndef SQLITE_OMIT_TRIGGER + int isView; /* True when updating a view (INSTEAD OF trigger) */ + Trigger *pTrigger; /* List of triggers on pTab, if required */ + int tmask; /* Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ +#endif + int newmask; /* Mask of NEW.* columns accessed by BEFORE triggers */ + int iEph = 0; /* Ephemeral table holding all primary key values */ + int nKey = 0; /* Number of elements in regKey for WITHOUT ROWID */ + int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ + int addrOpen = 0; /* Address of OP_OpenEphemeral */ + int iPk = 0; /* First of nPk cells holding PRIMARY KEY value */ + i16 nPk = 0; /* Number of components of the PRIMARY KEY */ + int bReplace = 0; /* True if REPLACE conflict resolution might happen */ + int bFinishSeek = 1; /* The OP_FinishSeek opcode is needed */ + int nChangeFrom = 0; /* If there is a FROM, pChanges->nExpr, else 0 */ + + /* Register Allocations */ + int regRowCount = 0; /* A count of rows changed */ + int regOldRowid = 0; /* The old rowid */ + int regNewRowid = 0; /* The new rowid */ + int regNew = 0; /* Content of the NEW.* table in triggers */ + int regOld = 0; /* Content of OLD.* table in triggers */ + int regRowSet = 0; /* Rowset of rows to be updated */ + int regKey = 0; /* composite PRIMARY KEY value */ + + memset(&sContext, 0, sizeof(sContext)); + db = pParse->db; + if( pParse->nErr || db->mallocFailed ){ + goto update_cleanup; + } + + /* Locate the table which we want to update. + */ + pTab = sqlite3SrcListLookup(pParse, pTabList); + if( pTab==0 ) goto update_cleanup; + iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + + /* Figure out if we have any triggers and if the table being + ** updated is a view. + */ +#ifndef SQLITE_OMIT_TRIGGER + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_UPDATE, pChanges, &tmask); + isView = IsView(pTab); + assert( pTrigger || tmask==0 ); +#else +# define pTrigger 0 +# define isView 0 +# define tmask 0 +#endif +#ifdef SQLITE_OMIT_VIEW +# undef isView +# define isView 0 +#endif + + /* If there was a FROM clause, set nChangeFrom to the number of expressions + ** in the change-list. Otherwise, set it to 0. There cannot be a FROM + ** clause if this function is being called to generate code for part of + ** an UPSERT statement. */ + nChangeFrom = (pTabList->nSrc>1) ? pChanges->nExpr : 0; + assert( nChangeFrom==0 || pUpsert==0 ); + +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( !isView && nChangeFrom==0 ){ + pWhere = sqlite3LimitWhere( + pParse, pTabList, pWhere, pOrderBy, pLimit, "UPDATE" + ); + pOrderBy = 0; + pLimit = 0; + } +#endif + + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ + goto update_cleanup; + } + if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + goto update_cleanup; + } + + /* Allocate a cursors for the main database table and for all indices. + ** The index cursors might not be used, but if they are used they + ** need to occur right after the database cursor. So go ahead and + ** allocate enough space, just in case. + */ + iBaseCur = iDataCur = pParse->nTab++; + iIdxCur = iDataCur+1; + pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + testcase( pPk!=0 && pPk!=pTab->pIndex ); + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ + if( pPk==pIdx ){ + iDataCur = pParse->nTab; + } + pParse->nTab++; + } + if( pUpsert ){ + /* On an UPSERT, reuse the same cursors already opened by INSERT */ + iDataCur = pUpsert->iDataCur; + iIdxCur = pUpsert->iIdxCur; + pParse->nTab = iBaseCur; + } + pTabList->a[0].iCursor = iDataCur; + + /* Allocate space for aXRef[], aRegIdx[], and aToOpen[]. + ** Initialize aXRef[] and aToOpen[] to their default values. + */ + aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx+1) + nIdx+2 ); + if( aXRef==0 ) goto update_cleanup; + aRegIdx = aXRef+pTab->nCol; + aToOpen = (u8*)(aRegIdx+nIdx+1); + memset(aToOpen, 1, nIdx+1); + aToOpen[nIdx+1] = 0; + for(i=0; inCol; i++) aXRef[i] = -1; + + /* Initialize the name-context */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + sNC.uNC.pUpsert = pUpsert; + sNC.ncFlags = NC_UUpsert; + + /* Begin generating code. */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto update_cleanup; + + /* Resolve the column names in all the expressions of the + ** of the UPDATE statement. Also find the column index + ** for each column to be updated in the pChanges array. For each + ** column to be updated, make sure we have authorization to change + ** that column. + */ + chngRowid = chngPk = 0; + for(i=0; inExpr; i++){ + u8 hCol = sqlite3StrIHash(pChanges->a[i].zEName); + /* If this is an UPDATE with a FROM clause, do not resolve expressions + ** here. The call to sqlite3Select() below will do that. */ + if( nChangeFrom==0 && sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){ + goto update_cleanup; + } + for(j=0; jnCol; j++){ + if( pTab->aCol[j].hName==hCol + && sqlite3StrICmp(pTab->aCol[j].zCnName, pChanges->a[i].zEName)==0 + ){ + if( j==pTab->iPKey ){ + chngRowid = 1; + pRowidExpr = pChanges->a[i].pExpr; + iRowidExpr = i; + }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){ + chngPk = 1; + } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + else if( pTab->aCol[j].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[j].colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, + "cannot UPDATE generated column \"%s\"", + pTab->aCol[j].zCnName); + goto update_cleanup; + } +#endif + aXRef[j] = i; + break; + } + } + if( j>=pTab->nCol ){ + if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zEName) ){ + j = -1; + chngRowid = 1; + pRowidExpr = pChanges->a[i].pExpr; + iRowidExpr = i; + }else{ + sqlite3ErrorMsg(pParse, "no such column: %s", pChanges->a[i].zEName); + pParse->checkSchema = 1; + goto update_cleanup; + } + } +#ifndef SQLITE_OMIT_AUTHORIZATION + { + int rc; + rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, + j<0 ? "ROWID" : pTab->aCol[j].zCnName, + db->aDb[iDb].zDbSName); + if( rc==SQLITE_DENY ){ + goto update_cleanup; + }else if( rc==SQLITE_IGNORE ){ + aXRef[j] = -1; + } + } +#endif + } + assert( (chngRowid & chngPk)==0 ); + assert( chngRowid==0 || chngRowid==1 ); + assert( chngPk==0 || chngPk==1 ); + chngKey = chngRowid + chngPk; + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Mark generated columns as changing if their generator expressions + ** reference any changing column. The actual aXRef[] value for + ** generated expressions is not used, other than to check to see that it + ** is non-negative, so the value of aXRef[] for generated columns can be + ** set to any non-negative number. We use 99999 so that the value is + ** obvious when looking at aXRef[] in a symbolic debugger. + */ + if( pTab->tabFlags & TF_HasGenerated ){ + int bProgress; + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + do{ + bProgress = 0; + for(i=0; inCol; i++){ + if( aXRef[i]>=0 ) continue; + if( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)==0 ) continue; + if( sqlite3ExprReferencesUpdatedColumn( + sqlite3ColumnExpr(pTab, &pTab->aCol[i]), + aXRef, chngRowid) + ){ + aXRef[i] = 99999; + bProgress = 1; + } + } + }while( bProgress ); + } +#endif + + /* The SET expressions are not actually used inside the WHERE loop. + ** So reset the colUsed mask. Unless this is a virtual table. In that + ** case, set all bits of the colUsed mask (to ensure that the virtual + ** table implementation makes all columns available). + */ + pTabList->a[0].colUsed = IsVirtual(pTab) ? ALLBITS : 0; + + hasFK = sqlite3FkRequired(pParse, pTab, aXRef, chngKey); + + /* There is one entry in the aRegIdx[] array for each index on the table + ** being updated. Fill in aRegIdx[] with a register number that will hold + ** the key for accessing each index. + */ + if( onError==OE_Replace ) bReplace = 1; + for(nAllIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nAllIdx++){ + int reg; + if( chngKey || hasFK>1 || pIdx==pPk + || indexWhereClauseMightChange(pIdx,aXRef,chngRowid) + ){ + reg = ++pParse->nMem; + pParse->nMem += pIdx->nColumn; + }else{ + reg = 0; + for(i=0; inKeyCol; i++){ + if( indexColumnIsBeingUpdated(pIdx, i, aXRef, chngRowid) ){ + reg = ++pParse->nMem; + pParse->nMem += pIdx->nColumn; + if( onError==OE_Default && pIdx->onError==OE_Replace ){ + bReplace = 1; + } + break; + } + } + } + if( reg==0 ) aToOpen[nAllIdx+1] = 0; + aRegIdx[nAllIdx] = reg; + } + aRegIdx[nAllIdx] = ++pParse->nMem; /* Register storing the table record */ + if( bReplace ){ + /* If REPLACE conflict resolution might be invoked, open cursors on all + ** indexes in case they are needed to delete records. */ + memset(aToOpen, 1, nIdx+1); + } + + if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); + sqlite3BeginWriteOperation(pParse, pTrigger || hasFK, iDb); + + /* Allocate required registers. */ + if( !IsVirtual(pTab) ){ + /* For now, regRowSet and aRegIdx[nAllIdx] share the same register. + ** If regRowSet turns out to be needed, then aRegIdx[nAllIdx] will be + ** reallocated. aRegIdx[nAllIdx] is the register in which the main + ** table record is written. regRowSet holds the RowSet for the + ** two-pass update algorithm. */ + assert( aRegIdx[nAllIdx]==pParse->nMem ); + regRowSet = aRegIdx[nAllIdx]; + regOldRowid = regNewRowid = ++pParse->nMem; + if( chngPk || pTrigger || hasFK ){ + regOld = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + } + if( chngKey || pTrigger || hasFK ){ + regNewRowid = ++pParse->nMem; + } + regNew = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + } + + /* Start the view context. */ + if( isView ){ + sqlite3AuthContextPush(pParse, &sContext, pTab->zName); + } + + /* If we are trying to update a view, realize that view into + ** an ephemeral table. + */ +#if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) + if( nChangeFrom==0 && isView ){ + sqlite3MaterializeView(pParse, pTab, + pWhere, pOrderBy, pLimit, iDataCur + ); + pOrderBy = 0; + pLimit = 0; + } +#endif + + /* Resolve the column names in all the expressions in the + ** WHERE clause. + */ + if( nChangeFrom==0 && sqlite3ResolveExprNames(&sNC, pWhere) ){ + goto update_cleanup; + } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Virtual tables must be handled separately */ + if( IsVirtual(pTab) ){ + updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, + pWhere, onError); + goto update_cleanup; + } +#endif + + /* Jump to labelBreak to abandon further processing of this UPDATE */ + labelContinue = labelBreak = sqlite3VdbeMakeLabel(pParse); + + /* Not an UPSERT. Normal processing. Begin by + ** initialize the count of updated rows */ + if( (db->flags&SQLITE_CountRows)!=0 + && !pParse->pTriggerTab + && !pParse->nested + && !pParse->bReturning + && pUpsert==0 + ){ + regRowCount = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); + } + + if( nChangeFrom==0 && HasRowid(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); + iEph = pParse->nTab++; + addrOpen = sqlite3VdbeAddOp3(v, OP_OpenEphemeral, iEph, 0, regRowSet); + }else{ + assert( pPk!=0 || HasRowid(pTab) ); + nPk = pPk ? pPk->nKeyCol : 0; + iPk = pParse->nMem+1; + pParse->nMem += nPk; + pParse->nMem += nChangeFrom; + regKey = ++pParse->nMem; + if( pUpsert==0 ){ + int nEphCol = nPk + nChangeFrom + (isView ? pTab->nCol : 0); + iEph = pParse->nTab++; + if( pPk ) sqlite3VdbeAddOp3(v, OP_Null, 0, iPk, iPk+nPk-1); + addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nEphCol); + if( pPk ){ + KeyInfo *pKeyInfo = sqlite3KeyInfoOfIndex(pParse, pPk); + if( pKeyInfo ){ + pKeyInfo->nAllField = nEphCol; + sqlite3VdbeAppendP4(v, pKeyInfo, P4_KEYINFO); + } + } + if( nChangeFrom ){ + updateFromSelect( + pParse, iEph, pPk, pChanges, pTabList, pWhere, pOrderBy, pLimit + ); +#ifndef SQLITE_OMIT_SUBQUERY + if( isView ) iDataCur = iEph; +#endif + } + } + } + + if( nChangeFrom ){ + sqlite3MultiWrite(pParse); + eOnePass = ONEPASS_OFF; + nKey = nPk; + regKey = iPk; + }else{ + if( pUpsert ){ + /* If this is an UPSERT, then all cursors have already been opened by + ** the outer INSERT and the data cursor should be pointing at the row + ** that is to be updated. So bypass the code that searches for the + ** row(s) to be updated. + */ + pWInfo = 0; + eOnePass = ONEPASS_SINGLE; + sqlite3ExprIfFalse(pParse, pWhere, labelBreak, SQLITE_JUMPIFNULL); + bFinishSeek = 0; + }else{ + /* Begin the database scan. + ** + ** Do not consider a single-pass strategy for a multi-row update if + ** there are any triggers or foreign keys to process, or rows may + ** be deleted as a result of REPLACE conflict handling. Any of these + ** things might disturb a cursor being used to scan through the table + ** or index, causing a single-pass approach to malfunction. */ + flags = WHERE_ONEPASS_DESIRED; + if( !pParse->nested && !pTrigger && !hasFK && !chngKey && !bReplace ){ + flags |= WHERE_ONEPASS_MULTIROW; + } + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, flags,iIdxCur); + if( pWInfo==0 ) goto update_cleanup; + + /* A one-pass strategy that might update more than one row may not + ** be used if any column of the index used for the scan is being + ** updated. Otherwise, if there is an index on "b", statements like + ** the following could create an infinite loop: + ** + ** UPDATE t1 SET b=b+1 WHERE b>? + ** + ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI + ** strategy that uses an index for which one or more columns are being + ** updated. */ + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + bFinishSeek = sqlite3WhereUsesDeferredSeek(pWInfo); + if( eOnePass!=ONEPASS_SINGLE ){ + sqlite3MultiWrite(pParse); + if( eOnePass==ONEPASS_MULTI ){ + int iCur = aiCurOnePass[1]; + if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){ + eOnePass = ONEPASS_OFF; + } + assert( iCur!=iDataCur || !HasRowid(pTab) ); + } + } + } + + if( HasRowid(pTab) ){ + /* Read the rowid of the current row of the WHERE scan. In ONEPASS_OFF + ** mode, write the rowid into the FIFO. In either of the one-pass modes, + ** leave it in register regOldRowid. */ + sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid); + if( eOnePass==ONEPASS_OFF ){ + aRegIdx[nAllIdx] = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_Insert, iEph, regRowSet, regOldRowid); + }else{ + if( ALWAYS(addrOpen) ) sqlite3VdbeChangeToNoop(v, addrOpen); + } + }else{ + /* Read the PK of the current row into an array of registers. In + ** ONEPASS_OFF mode, serialize the array into a record and store it in + ** the ephemeral table. Or, in ONEPASS_SINGLE or MULTI mode, change + ** the OP_OpenEphemeral instruction to a Noop (the ephemeral table + ** is not required) and leave the PK fields in the array of registers. */ + for(i=0; iaiColumn[i]>=0 ); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, + pPk->aiColumn[i], iPk+i); + } + if( eOnePass ){ + if( addrOpen ) sqlite3VdbeChangeToNoop(v, addrOpen); + nKey = nPk; + regKey = iPk; + }else{ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, + sqlite3IndexAffinityStr(db, pPk), nPk); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEph, regKey, iPk, nPk); + } + } + } + + if( pUpsert==0 ){ + if( nChangeFrom==0 && eOnePass!=ONEPASS_MULTI ){ + sqlite3WhereEnd(pWInfo); + } + + if( !isView ){ + int addrOnce = 0; + + /* Open every index that needs updating. */ + if( eOnePass!=ONEPASS_OFF ){ + if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; + if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; + } + + if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){ + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, + aToOpen, 0, 0); + if( addrOnce ){ + sqlite3VdbeJumpHereOrPopInst(v, addrOnce); + } + } + + /* Top of the update loop */ + if( eOnePass!=ONEPASS_OFF ){ + if( aiCurOnePass[0]!=iDataCur + && aiCurOnePass[1]!=iDataCur +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + && !isView +#endif + ){ + assert( pPk ); + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey,nKey); + VdbeCoverage(v); + } + if( eOnePass!=ONEPASS_SINGLE ){ + labelContinue = sqlite3VdbeMakeLabel(pParse); + } + sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak); + VdbeCoverageIf(v, pPk==0); + VdbeCoverageIf(v, pPk!=0); + }else if( pPk || nChangeFrom ){ + labelContinue = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v); + addrTop = sqlite3VdbeCurrentAddr(v); + if( nChangeFrom ){ + if( !isView ){ + if( pPk ){ + for(i=0; i=0 ); + if( nChangeFrom==0 ){ + sqlite3ExprCode(pParse, pRowidExpr, regNewRowid); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, iEph, iRowidExpr, regNewRowid); + } + sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); VdbeCoverage(v); + } + + /* Compute the old pre-UPDATE content of the row being changed, if that + ** information is needed */ + if( chngPk || hasFK || pTrigger ){ + u32 oldmask = (hasFK ? sqlite3FkOldmask(pParse, pTab) : 0); + oldmask |= sqlite3TriggerColmask(pParse, + pTrigger, pChanges, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onError + ); + for(i=0; inCol; i++){ + u32 colFlags = pTab->aCol[i].colFlags; + k = sqlite3TableColumnToStorage(pTab, i) + regOld; + if( oldmask==0xffffffff + || (i<32 && (oldmask & MASKBIT32(i))!=0) + || (colFlags & COLFLAG_PRIMKEY)!=0 + ){ + testcase( oldmask!=0xffffffff && i==31 ); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, k); + } + } + if( chngRowid==0 && pPk==0 ){ + sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); + } + } + + /* Populate the array of registers beginning at regNew with the new + ** row data. This array is used to check constants, create the new + ** table and index records, and as the values for any new.* references + ** made by triggers. + ** + ** If there are one or more BEFORE triggers, then do not populate the + ** registers associated with columns that are (a) not modified by + ** this UPDATE statement and (b) not accessed by new.* references. The + ** values for registers not modified by the UPDATE must be reloaded from + ** the database after the BEFORE triggers are fired anyway (as the trigger + ** may have modified them). So not loading those that are not going to + ** be used eliminates some redundant opcodes. + */ + newmask = sqlite3TriggerColmask( + pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError + ); + for(i=0, k=regNew; inCol; i++, k++){ + if( i==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, k); + }else if( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)!=0 ){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) k--; + }else{ + j = aXRef[i]; + if( j>=0 ){ + if( nChangeFrom ){ + int nOff = (isView ? pTab->nCol : nPk); + assert( eOnePass==ONEPASS_OFF ); + sqlite3VdbeAddOp3(v, OP_Column, iEph, nOff+j, k); + }else{ + sqlite3ExprCode(pParse, pChanges->a[j].pExpr, k); + } + }else if( 0==(tmask&TRIGGER_BEFORE) || i>31 || (newmask & MASKBIT32(i)) ){ + /* This branch loads the value of a column that will not be changed + ** into a register. This is done if there are no BEFORE triggers, or + ** if there are one or more BEFORE triggers that use this value via + ** a new.* reference in a trigger program. + */ + testcase( i==31 ); + testcase( i==32 ); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); + bFinishSeek = 0; + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, k); + } + } + } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regNew, pTab); + } +#endif + + /* Fire any BEFORE UPDATE triggers. This happens before constraints are + ** verified. One could argue that this is wrong. + */ + if( tmask&TRIGGER_BEFORE ){ + sqlite3TableAffinity(v, pTab, regNew); + sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, + TRIGGER_BEFORE, pTab, regOldRowid, onError, labelContinue); + + if( !isView ){ + /* The row-trigger may have deleted the row being updated. In this + ** case, jump to the next row. No updates or AFTER triggers are + ** required. This behavior - what happens when the row being updated + ** is deleted or renamed by a BEFORE trigger - is left undefined in the + ** documentation. + */ + if( pPk ){ + sqlite3VdbeAddOp4Int(v, OP_NotFound,iDataCur,labelContinue,regKey,nKey); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue,regOldRowid); + VdbeCoverage(v); + } + + /* After-BEFORE-trigger-reload-loop: + ** If it did not delete it, the BEFORE trigger may still have modified + ** some of the columns of the row being updated. Load the values for + ** all columns not modified by the update statement into their registers + ** in case this has happened. Only unmodified columns are reloaded. + ** The values computed for modified columns use the values before the + ** BEFORE trigger runs. See test case trigger1-18.0 (added 2018-04-26) + ** for an example. + */ + for(i=0, k=regNew; inCol; i++, k++){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) k--; + }else if( aXRef[i]<0 && i!=pTab->iPKey ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); + } + } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regNew, pTab); + } +#endif + } + } + + if( !isView ){ + /* Do constraint checks. */ + assert( regOldRowid>0 ); + sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, + regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace, + aXRef, 0); + + /* If REPLACE conflict handling may have been used, or if the PK of the + ** row is changing, then the GenerateConstraintChecks() above may have + ** moved cursor iDataCur. Reseek it. */ + if( bReplace || chngKey ){ + if( pPk ){ + sqlite3VdbeAddOp4Int(v, OP_NotFound,iDataCur,labelContinue,regKey,nKey); + }else{ + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue,regOldRowid); + } + VdbeCoverageNeverTaken(v); + } + + /* Do FK constraint checks. */ + if( hasFK ){ + sqlite3FkCheck(pParse, pTab, regOldRowid, 0, aXRef, chngKey); + } + + /* Delete the index entries associated with the current record. */ + sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1); + + /* We must run the OP_FinishSeek opcode to resolve a prior + ** OP_DeferredSeek if there is any possibility that there have been + ** no OP_Column opcodes since the OP_DeferredSeek was issued. But + ** we want to avoid the OP_FinishSeek if possible, as running it + ** costs CPU cycles. */ + if( bFinishSeek ){ + sqlite3VdbeAddOp1(v, OP_FinishSeek, iDataCur); + } + + /* If changing the rowid value, or if there are foreign key constraints + ** to process, delete the old record. Otherwise, add a noop OP_Delete + ** to invoke the pre-update hook. + ** + ** That (regNew==regnewRowid+1) is true is also important for the + ** pre-update hook. If the caller invokes preupdate_new(), the returned + ** value is copied from memory cell (regNewRowid+1+iCol), where iCol + ** is the column index supplied by the user. + */ + assert( regNew==regNewRowid+1 ); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + sqlite3VdbeAddOp3(v, OP_Delete, iDataCur, + OPFLAG_ISUPDATE | ((hasFK>1 || chngKey) ? 0 : OPFLAG_ISNOOP), + regNewRowid + ); + if( eOnePass==ONEPASS_MULTI ){ + assert( hasFK==0 && chngKey==0 ); + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); + } + if( !pParse->nested ){ + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); + } +#else + if( hasFK>1 || chngKey ){ + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); + } +#endif + + if( hasFK ){ + sqlite3FkCheck(pParse, pTab, 0, regNewRowid, aXRef, chngKey); + } + + /* Insert the new index entries and the new record. */ + sqlite3CompleteInsertion( + pParse, pTab, iDataCur, iIdxCur, regNewRowid, aRegIdx, + OPFLAG_ISUPDATE | (eOnePass==ONEPASS_MULTI ? OPFLAG_SAVEPOSITION : 0), + 0, 0 + ); + + /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to + ** handle rows (possibly in other tables) that refer via a foreign key + ** to the row just updated. */ + if( hasFK ){ + sqlite3FkActions(pParse, pTab, pChanges, regOldRowid, aXRef, chngKey); + } + } + + /* Increment the row counter + */ + if( regRowCount ){ + sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); + } + + sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, + TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue); + + /* Repeat the above with the next record to be updated, until + ** all record selected by the WHERE clause have been updated. + */ + if( eOnePass==ONEPASS_SINGLE ){ + /* Nothing to do at end-of-loop for a single-pass */ + }else if( eOnePass==ONEPASS_MULTI ){ + sqlite3VdbeResolveLabel(v, labelContinue); + sqlite3WhereEnd(pWInfo); + }else{ + sqlite3VdbeResolveLabel(v, labelContinue); + sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); + } + sqlite3VdbeResolveLabel(v, labelBreak); + + /* Update the sqlite_sequence table by storing the content of the + ** maximum rowid counter values recorded while inserting into + ** autoincrement tables. + */ + if( pParse->nested==0 && pParse->pTriggerTab==0 && pUpsert==0 ){ + sqlite3AutoincrementEnd(pParse); + } + + /* + ** Return the number of rows that were changed, if we are tracking + ** that information. + */ + if( regRowCount ){ + sqlite3VdbeAddOp2(v, OP_ChngCntRow, regRowCount, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows updated", SQLITE_STATIC); + } + +update_cleanup: + sqlite3AuthContextPop(&sContext); + sqlite3DbFree(db, aXRef); /* Also frees aRegIdx[] and aToOpen[] */ + sqlite3SrcListDelete(db, pTabList); + sqlite3ExprListDelete(db, pChanges); + sqlite3ExprDelete(db, pWhere); +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) + sqlite3ExprListDelete(db, pOrderBy); + sqlite3ExprDelete(db, pLimit); +#endif + return; +} +/* Make sure "isView" and other macros defined above are undefined. Otherwise +** they may interfere with compilation of other functions in this file +** (or in another file, if this file becomes part of the amalgamation). */ +#ifdef isView + #undef isView +#endif +#ifdef pTrigger + #undef pTrigger +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Generate code for an UPDATE of a virtual table. +** +** There are two possible strategies - the default and the special +** "onepass" strategy. Onepass is only used if the virtual table +** implementation indicates that pWhere may match at most one row. +** +** The default strategy is to create an ephemeral table that contains +** for each row to be changed: +** +** (A) The original rowid of that row. +** (B) The revised rowid for the row. +** (C) The content of every column in the row. +** +** Then loop through the contents of this ephemeral table executing a +** VUpdate for each row. When finished, drop the ephemeral table. +** +** The "onepass" strategy does not use an ephemeral table. Instead, it +** stores the same values (A, B and C above) in a register array and +** makes a single invocation of VUpdate. +*/ +static void updateVirtualTable( + Parse *pParse, /* The parsing context */ + SrcList *pSrc, /* The virtual table to be modified */ + Table *pTab, /* The virtual table */ + ExprList *pChanges, /* The columns to change in the UPDATE statement */ + Expr *pRowid, /* Expression used to recompute the rowid */ + int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ + Expr *pWhere, /* WHERE clause of the UPDATE statement */ + int onError /* ON CONFLICT strategy */ +){ + Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ + int ephemTab; /* Table holding the result of the SELECT */ + int i; /* Loop counter */ + sqlite3 *db = pParse->db; /* Database connection */ + const char *pVTab = (const char*)sqlite3GetVTable(db, pTab); + WhereInfo *pWInfo = 0; + int nArg = 2 + pTab->nCol; /* Number of arguments to VUpdate */ + int regArg; /* First register in VUpdate arg array */ + int regRec; /* Register in which to assemble record */ + int regRowid; /* Register for ephem table rowid */ + int iCsr = pSrc->a[0].iCursor; /* Cursor used for virtual table scan */ + int aDummy[2]; /* Unused arg for sqlite3WhereOkOnePass() */ + int eOnePass; /* True to use onepass strategy */ + int addr; /* Address of OP_OpenEphemeral */ + + /* Allocate nArg registers in which to gather the arguments for VUpdate. Then + ** create and open the ephemeral table in which the records created from + ** these arguments will be temporarily stored. */ + assert( v ); + ephemTab = pParse->nTab++; + addr= sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, nArg); + regArg = pParse->nMem + 1; + pParse->nMem += nArg; + if( pSrc->nSrc>1 ){ + Index *pPk = 0; + Expr *pRow; + ExprList *pList; + if( HasRowid(pTab) ){ + if( pRowid ){ + pRow = sqlite3ExprDup(db, pRowid, 0); + }else{ + pRow = sqlite3PExpr(pParse, TK_ROW, 0, 0); + } + }else{ + i16 iPk; /* PRIMARY KEY column */ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->nKeyCol==1 ); + iPk = pPk->aiColumn[0]; + if( aXRef[iPk]>=0 ){ + pRow = sqlite3ExprDup(db, pChanges->a[aXRef[iPk]].pExpr, 0); + }else{ + pRow = exprRowColumn(pParse, iPk); + } + } + pList = sqlite3ExprListAppend(pParse, 0, pRow); + + for(i=0; inCol; i++){ + if( aXRef[i]>=0 ){ + pList = sqlite3ExprListAppend(pParse, pList, + sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0) + ); + }else{ + pList = sqlite3ExprListAppend(pParse, pList, exprRowColumn(pParse, i)); + } + } + + updateFromSelect(pParse, ephemTab, pPk, pList, pSrc, pWhere, 0, 0); + sqlite3ExprListDelete(db, pList); + eOnePass = ONEPASS_OFF; + }else{ + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; + + /* Start scanning the virtual table */ + pWInfo = sqlite3WhereBegin(pParse, pSrc,pWhere,0,0,WHERE_ONEPASS_DESIRED,0); + if( pWInfo==0 ) return; + + /* Populate the argument registers. */ + for(i=0; inCol; i++){ + assert( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)==0 ); + if( aXRef[i]>=0 ){ + sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); + }else{ + sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG);/* For sqlite3_vtab_nochange() */ + } + } + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); + if( pRowid ){ + sqlite3ExprCode(pParse, pRowid, regArg+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); + } + }else{ + Index *pPk; /* PRIMARY KEY index */ + i16 iPk; /* PRIMARY KEY column */ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->nKeyCol==1 ); + iPk = pPk->aiColumn[0]; + sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, iPk, regArg); + sqlite3VdbeAddOp2(v, OP_SCopy, regArg+2+iPk, regArg+1); + } + + eOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy); + + /* There is no ONEPASS_MULTI on virtual tables */ + assert( eOnePass==ONEPASS_OFF || eOnePass==ONEPASS_SINGLE ); + + if( eOnePass ){ + /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded + ** above. */ + sqlite3VdbeChangeToNoop(v, addr); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); + }else{ + /* Create a record from the argument register contents and insert it into + ** the ephemeral table. */ + sqlite3MultiWrite(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec); +#if defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_NULL_TRIM) + /* Signal an assert() within OP_MakeRecord that it is allowed to + ** accept no-change records with serial_type 10 */ + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC); +#endif + sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, ephemTab, regRec, regRowid); + } + } + + + if( eOnePass==ONEPASS_OFF ){ + /* End the virtual table scan */ + if( pSrc->nSrc==1 ){ + sqlite3WhereEnd(pWInfo); + } + + /* Begin scannning through the ephemeral table. */ + addr = sqlite3VdbeAddOp1(v, OP_Rewind, ephemTab); VdbeCoverage(v); + + /* Extract arguments from the current row of the ephemeral table and + ** invoke the VUpdate method. */ + for(i=0; ipNextUpsert; + sqlite3ExprListDelete(db, p->pUpsertTarget); + sqlite3ExprDelete(db, p->pUpsertTargetWhere); + sqlite3ExprListDelete(db, p->pUpsertSet); + sqlite3ExprDelete(db, p->pUpsertWhere); + sqlite3DbFree(db, p->pToFree); + sqlite3DbFree(db, p); + p = pNext; + }while( p ); +} +SQLITE_PRIVATE void sqlite3UpsertDelete(sqlite3 *db, Upsert *p){ + if( p ) upsertDelete(db, p); +} + + +/* +** Duplicate an Upsert object. +*/ +SQLITE_PRIVATE Upsert *sqlite3UpsertDup(sqlite3 *db, Upsert *p){ + if( p==0 ) return 0; + return sqlite3UpsertNew(db, + sqlite3ExprListDup(db, p->pUpsertTarget, 0), + sqlite3ExprDup(db, p->pUpsertTargetWhere, 0), + sqlite3ExprListDup(db, p->pUpsertSet, 0), + sqlite3ExprDup(db, p->pUpsertWhere, 0), + sqlite3UpsertDup(db, p->pNextUpsert) + ); +} + +/* +** Create a new Upsert object. +*/ +SQLITE_PRIVATE Upsert *sqlite3UpsertNew( + sqlite3 *db, /* Determines which memory allocator to use */ + ExprList *pTarget, /* Target argument to ON CONFLICT, or NULL */ + Expr *pTargetWhere, /* Optional WHERE clause on the target */ + ExprList *pSet, /* UPDATE columns, or NULL for a DO NOTHING */ + Expr *pWhere, /* WHERE clause for the ON CONFLICT UPDATE */ + Upsert *pNext /* Next ON CONFLICT clause in the list */ +){ + Upsert *pNew; + pNew = sqlite3DbMallocZero(db, sizeof(Upsert)); + if( pNew==0 ){ + sqlite3ExprListDelete(db, pTarget); + sqlite3ExprDelete(db, pTargetWhere); + sqlite3ExprListDelete(db, pSet); + sqlite3ExprDelete(db, pWhere); + sqlite3UpsertDelete(db, pNext); + return 0; + }else{ + pNew->pUpsertTarget = pTarget; + pNew->pUpsertTargetWhere = pTargetWhere; + pNew->pUpsertSet = pSet; + pNew->pUpsertWhere = pWhere; + pNew->isDoUpdate = pSet!=0; + pNew->pNextUpsert = pNext; + } + return pNew; +} + +/* +** Analyze the ON CONFLICT clause described by pUpsert. Resolve all +** symbols in the conflict-target. +** +** Return SQLITE_OK if everything works, or an error code is something +** is wrong. +*/ +SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( + Parse *pParse, /* The parsing context */ + SrcList *pTabList, /* Table into which we are inserting */ + Upsert *pUpsert /* The ON CONFLICT clauses */ +){ + Table *pTab; /* That table into which we are inserting */ + int rc; /* Result code */ + int iCursor; /* Cursor used by pTab */ + Index *pIdx; /* One of the indexes of pTab */ + ExprList *pTarget; /* The conflict-target clause */ + Expr *pTerm; /* One term of the conflict-target clause */ + NameContext sNC; /* Context for resolving symbolic names */ + Expr sCol[2]; /* Index column converted into an Expr */ + int nClause = 0; /* Counter of ON CONFLICT clauses */ + + assert( pTabList->nSrc==1 ); + assert( pTabList->a[0].pTab!=0 ); + assert( pUpsert!=0 ); + assert( pUpsert->pUpsertTarget!=0 ); + + /* Resolve all symbolic names in the conflict-target clause, which + ** includes both the list of columns and the optional partial-index + ** WHERE clause. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + for(; pUpsert && pUpsert->pUpsertTarget; + pUpsert=pUpsert->pNextUpsert, nClause++){ + rc = sqlite3ResolveExprListNames(&sNC, pUpsert->pUpsertTarget); + if( rc ) return rc; + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertTargetWhere); + if( rc ) return rc; + + /* Check to see if the conflict target matches the rowid. */ + pTab = pTabList->a[0].pTab; + pTarget = pUpsert->pUpsertTarget; + iCursor = pTabList->a[0].iCursor; + if( HasRowid(pTab) + && pTarget->nExpr==1 + && (pTerm = pTarget->a[0].pExpr)->op==TK_COLUMN + && pTerm->iColumn==XN_ROWID + ){ + /* The conflict-target is the rowid of the primary table */ + assert( pUpsert->pUpsertIdx==0 ); + continue; + } + + /* Initialize sCol[0..1] to be an expression parse tree for a + ** single column of an index. The sCol[0] node will be the TK_COLLATE + ** operator and sCol[1] will be the TK_COLUMN operator. Code below + ** will populate the specific collation and column number values + ** prior to comparing against the conflict-target expression. + */ + memset(sCol, 0, sizeof(sCol)); + sCol[0].op = TK_COLLATE; + sCol[0].pLeft = &sCol[1]; + sCol[1].op = TK_COLUMN; + sCol[1].iTable = pTabList->a[0].iCursor; + + /* Check for matches against other indexes */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int ii, jj, nn; + if( !IsUniqueIndex(pIdx) ) continue; + if( pTarget->nExpr!=pIdx->nKeyCol ) continue; + if( pIdx->pPartIdxWhere ){ + if( pUpsert->pUpsertTargetWhere==0 ) continue; + if( sqlite3ExprCompare(pParse, pUpsert->pUpsertTargetWhere, + pIdx->pPartIdxWhere, iCursor)!=0 ){ + continue; + } + } + nn = pIdx->nKeyCol; + for(ii=0; iiazColl[ii]; + if( pIdx->aiColumn[ii]==XN_EXPR ){ + assert( pIdx->aColExpr!=0 ); + assert( pIdx->aColExpr->nExpr>ii ); + pExpr = pIdx->aColExpr->a[ii].pExpr; + if( pExpr->op!=TK_COLLATE ){ + sCol[0].pLeft = pExpr; + pExpr = &sCol[0]; + } + }else{ + sCol[0].pLeft = &sCol[1]; + sCol[1].iColumn = pIdx->aiColumn[ii]; + pExpr = &sCol[0]; + } + for(jj=0; jja[jj].pExpr,pExpr,iCursor)<2 ){ + break; /* Column ii of the index matches column jj of target */ + } + } + if( jj>=nn ){ + /* The target contains no match for column jj of the index */ + break; + } + } + if( iipUpsertIdx = pIdx; + break; + } + if( pUpsert->pUpsertIdx==0 ){ + char zWhich[16]; + if( nClause==0 && pUpsert->pNextUpsert==0 ){ + zWhich[0] = 0; + }else{ + sqlite3_snprintf(sizeof(zWhich),zWhich,"%r ", nClause+1); + } + sqlite3ErrorMsg(pParse, "%sON CONFLICT clause does not match any " + "PRIMARY KEY or UNIQUE constraint", zWhich); + return SQLITE_ERROR; + } + } + return SQLITE_OK; +} + +/* +** Return true if pUpsert is the last ON CONFLICT clause with a +** conflict target, or if pUpsert is followed by another ON CONFLICT +** clause that targets the INTEGER PRIMARY KEY. +*/ +SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert *pUpsert){ + Upsert *pNext; + if( NEVER(pUpsert==0) ) return 0; + pNext = pUpsert->pNextUpsert; + if( pNext==0 ) return 1; + if( pNext->pUpsertTarget==0 ) return 1; + if( pNext->pUpsertIdx==0 ) return 1; + return 0; +} + +/* +** Given the list of ON CONFLICT clauses described by pUpsert, and +** a particular index pIdx, return a pointer to the particular ON CONFLICT +** clause that applies to the index. Or, if the index is not subject to +** any ON CONFLICT clause, return NULL. +*/ +SQLITE_PRIVATE Upsert *sqlite3UpsertOfIndex(Upsert *pUpsert, Index *pIdx){ + while( + pUpsert + && pUpsert->pUpsertTarget!=0 + && pUpsert->pUpsertIdx!=pIdx + ){ + pUpsert = pUpsert->pNextUpsert; + } + return pUpsert; +} + +/* +** Generate bytecode that does an UPDATE as part of an upsert. +** +** If pIdx is NULL, then the UNIQUE constraint that failed was the IPK. +** In this case parameter iCur is a cursor open on the table b-tree that +** currently points to the conflicting table row. Otherwise, if pIdx +** is not NULL, then pIdx is the constraint that failed and iCur is a +** cursor points to the conflicting row. +*/ +SQLITE_PRIVATE void sqlite3UpsertDoUpdate( + Parse *pParse, /* The parsing and code-generating context */ + Upsert *pUpsert, /* The ON CONFLICT clause for the upsert */ + Table *pTab, /* The table being updated */ + Index *pIdx, /* The UNIQUE constraint that failed */ + int iCur /* Cursor for pIdx (or pTab if pIdx==NULL) */ +){ + Vdbe *v = pParse->pVdbe; + sqlite3 *db = pParse->db; + SrcList *pSrc; /* FROM clause for the UPDATE */ + int iDataCur; + int i; + Upsert *pTop = pUpsert; + + assert( v!=0 ); + assert( pUpsert!=0 ); + iDataCur = pUpsert->iDataCur; + pUpsert = sqlite3UpsertOfIndex(pTop, pIdx); + VdbeNoopComment((v, "Begin DO UPDATE of UPSERT")); + if( pIdx && iCur!=iDataCur ){ + if( HasRowid(pTab) ){ + int regRowid = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_IdxRowid, iCur, regRowid); + sqlite3VdbeAddOp3(v, OP_SeekRowid, iDataCur, 0, regRowid); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, regRowid); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + int nPk = pPk->nKeyCol; + int iPk = pParse->nMem+1; + pParse->nMem += nPk; + for(i=0; iaiColumn[i]>=0 ); + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[i]); + sqlite3VdbeAddOp3(v, OP_Column, iCur, k, iPk+i); + VdbeComment((v, "%s.%s", pIdx->zName, + pTab->aCol[pPk->aiColumn[i]].zCnName)); + } + sqlite3VdbeVerifyAbortable(v, OE_Abort); + i = sqlite3VdbeAddOp4Int(v, OP_Found, iDataCur, 0, iPk, nPk); + VdbeCoverage(v); + sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CORRUPT, OE_Abort, 0, + "corrupt database", P4_STATIC); + sqlite3MayAbort(pParse); + sqlite3VdbeJumpHere(v, i); + } + } + /* pUpsert does not own pTop->pUpsertSrc - the outer INSERT statement does. + ** So we have to make a copy before passing it down into sqlite3Update() */ + pSrc = sqlite3SrcListDup(db, pTop->pUpsertSrc, 0); + /* excluded.* columns of type REAL need to be converted to a hard real */ + for(i=0; inCol; i++){ + if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, pTop->regData+i); + } + } + sqlite3Update(pParse, pSrc, sqlite3ExprListDup(db,pUpsert->pUpsertSet,0), + sqlite3ExprDup(db,pUpsert->pUpsertWhere,0), OE_Abort, 0, 0, pUpsert); + VdbeNoopComment((v, "End DO UPDATE of UPSERT")); +} + +#endif /* SQLITE_OMIT_UPSERT */ + +/************** End of upsert.c **********************************************/ +/************** Begin file vacuum.c ******************************************/ +/* +** 2003 April 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to implement the VACUUM command. +** +** Most of the code in this file may be omitted by defining the +** SQLITE_OMIT_VACUUM macro. +*/ +/* #include "sqliteInt.h" */ +/* #include "vdbeInt.h" */ + +#if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) + +/* +** Execute zSql on database db. +** +** If zSql returns rows, then each row will have exactly one +** column. (This will only happen if zSql begins with "SELECT".) +** Take each row of result and call execSql() again recursively. +** +** The execSqlF() routine does the same thing, except it accepts +** a format string as its third argument +*/ +static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ + sqlite3_stmt *pStmt; + int rc; + + /* printf("SQL: [%s]\n", zSql); fflush(stdout); */ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + const char *zSubSql = (const char*)sqlite3_column_text(pStmt,0); + assert( sqlite3_strnicmp(zSql,"SELECT",6)==0 ); + /* The secondary SQL must be one of CREATE TABLE, CREATE INDEX, + ** or INSERT. Historically there have been attacks that first + ** corrupt the sqlite_schema.sql field with other kinds of statements + ** then run VACUUM to get those statements to execute at inappropriate + ** times. */ + if( zSubSql + && (strncmp(zSubSql,"CRE",3)==0 || strncmp(zSubSql,"INS",3)==0) + ){ + rc = execSql(db, pzErrMsg, zSubSql); + if( rc!=SQLITE_OK ) break; + } + } + assert( rc!=SQLITE_ROW ); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + if( rc ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + } + (void)sqlite3_finalize(pStmt); + return rc; +} +static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){ + char *z; + va_list ap; + int rc; + va_start(ap, zSql); + z = sqlite3VMPrintf(db, zSql, ap); + va_end(ap); + if( z==0 ) return SQLITE_NOMEM; + rc = execSql(db, pzErrMsg, z); + sqlite3DbFree(db, z); + return rc; +} + +/* +** The VACUUM command is used to clean up the database, +** collapse free space, etc. It is modelled after the VACUUM command +** in PostgreSQL. The VACUUM command works as follows: +** +** (1) Create a new transient database file +** (2) Copy all content from the database being vacuumed into +** the new transient database file +** (3) Copy content from the transient database back into the +** original database. +** +** The transient database requires temporary disk space approximately +** equal to the size of the original database. The copy operation of +** step (3) requires additional temporary disk space approximately equal +** to the size of the original database for the rollback journal. +** Hence, temporary disk space that is approximately 2x the size of the +** original database is required. Every page of the database is written +** approximately 3 times: Once for step (2) and twice for step (3). +** Two writes per page are required in step (3) because the original +** database content must be written into the rollback journal prior to +** overwriting the database with the vacuumed content. +** +** Only 1x temporary space and only 1x writes would be required if +** the copy of step (3) were replaced by deleting the original database +** and renaming the transient database as the original. But that will +** not work if other processes are attached to the original database. +** And a power loss in between deleting the original and renaming the +** transient would cause the database file to appear to be deleted +** following reboot. +*/ +SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){ + Vdbe *v = sqlite3GetVdbe(pParse); + int iDb = 0; + if( v==0 ) goto build_vacuum_end; + if( pParse->nErr ) goto build_vacuum_end; + if( pNm ){ +#ifndef SQLITE_BUG_COMPATIBLE_20160819 + /* Default behavior: Report an error if the argument to VACUUM is + ** not recognized */ + iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm); + if( iDb<0 ) goto build_vacuum_end; +#else + /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments + ** to VACUUM are silently ignored. This is a back-out of a bug fix that + ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270). + ** The buggy behavior is required for binary compatibility with some + ** legacy applications. */ + iDb = sqlite3FindDb(pParse->db, pNm); + if( iDb<0 ) iDb = 0; +#endif + } + if( iDb!=1 ){ + int iIntoReg = 0; + if( pInto && sqlite3ResolveSelfReference(pParse,0,0,pInto,0)==0 ){ + iIntoReg = ++pParse->nMem; + sqlite3ExprCode(pParse, pInto, iIntoReg); + } + sqlite3VdbeAddOp2(v, OP_Vacuum, iDb, iIntoReg); + sqlite3VdbeUsesBtree(v, iDb); + } +build_vacuum_end: + sqlite3ExprDelete(pParse->db, pInto); + return; +} + +/* +** This routine implements the OP_Vacuum opcode of the VDBE. +*/ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( + char **pzErrMsg, /* Write error message here */ + sqlite3 *db, /* Database connection */ + int iDb, /* Which attached DB to vacuum */ + sqlite3_value *pOut /* Write results here, if not NULL. VACUUM INTO */ +){ + int rc = SQLITE_OK; /* Return code from service routines */ + Btree *pMain; /* The database being vacuumed */ + Btree *pTemp; /* The temporary database we vacuum into */ + u32 saved_mDbFlags; /* Saved value of db->mDbFlags */ + u64 saved_flags; /* Saved value of db->flags */ + i64 saved_nChange; /* Saved value of db->nChange */ + i64 saved_nTotalChange; /* Saved value of db->nTotalChange */ + u32 saved_openFlags; /* Saved value of db->openFlags */ + u8 saved_mTrace; /* Saved trace settings */ + Db *pDb = 0; /* Database to detach at end of vacuum */ + int isMemDb; /* True if vacuuming a :memory: database */ + int nRes; /* Bytes of reserved space at the end of each page */ + int nDb; /* Number of attached databases */ + const char *zDbMain; /* Schema name of database to vacuum */ + const char *zOut; /* Name of output file */ + + if( !db->autoCommit ){ + sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); + return SQLITE_ERROR; /* IMP: R-12218-18073 */ + } + if( db->nVdbeActive>1 ){ + sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); + return SQLITE_ERROR; /* IMP: R-15610-35227 */ + } + saved_openFlags = db->openFlags; + if( pOut ){ + if( sqlite3_value_type(pOut)!=SQLITE_TEXT ){ + sqlite3SetString(pzErrMsg, db, "non-text filename"); + return SQLITE_ERROR; + } + zOut = (const char*)sqlite3_value_text(pOut); + db->openFlags &= ~SQLITE_OPEN_READONLY; + db->openFlags |= SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; + }else{ + zOut = ""; + } + + /* Save the current value of the database flags so that it can be + ** restored before returning. Then set the writable-schema flag, and + ** disable CHECK and foreign key constraints. */ + saved_flags = db->flags; + saved_mDbFlags = db->mDbFlags; + saved_nChange = db->nChange; + saved_nTotalChange = db->nTotalChange; + saved_mTrace = db->mTrace; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; + db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; + db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder + | SQLITE_Defensive | SQLITE_CountRows); + db->mTrace = 0; + + zDbMain = db->aDb[iDb].zDbSName; + pMain = db->aDb[iDb].pBt; + isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain)); + + /* Attach the temporary database as 'vacuum_db'. The synchronous pragma + ** can be set to 'off' for this file, as it is not recovered if a crash + ** occurs anyway. The integrity of the database is maintained by a + ** (possibly synchronous) transaction opened on the main database before + ** sqlite3BtreeCopyFile() is called. + ** + ** An optimisation would be to use a non-journaled pager. + ** (Later:) I tried setting "PRAGMA vacuum_db.journal_mode=OFF" but + ** that actually made the VACUUM run slower. Very little journalling + ** actually occurs when doing a vacuum since the vacuum_db is initially + ** empty. Only the journal header is written. Apparently it takes more + ** time to parse and run the PRAGMA to turn journalling off than it does + ** to write the journal header file. + */ + nDb = db->nDb; + rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut); + db->openFlags = saved_openFlags; + if( rc!=SQLITE_OK ) goto end_of_vacuum; + assert( (db->nDb-1)==nDb ); + pDb = &db->aDb[nDb]; + assert( strcmp(pDb->zDbSName,"vacuum_db")==0 ); + pTemp = pDb->pBt; + if( pOut ){ + sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); + i64 sz = 0; + if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){ + rc = SQLITE_ERROR; + sqlite3SetString(pzErrMsg, db, "output file already exists"); + goto end_of_vacuum; + } + db->mDbFlags |= DBFLAG_VacuumInto; + } + nRes = sqlite3BtreeGetRequestedReserve(pMain); + + sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); + sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); + sqlite3BtreeSetPagerFlags(pTemp, PAGER_SYNCHRONOUS_OFF|PAGER_CACHESPILL); + + /* Begin a transaction and take an exclusive lock on the main database + ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, + ** to ensure that we do not try to change the page-size on a WAL database. + */ + rc = execSql(db, pzErrMsg, "BEGIN"); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = sqlite3BtreeBeginTrans(pMain, pOut==0 ? 2 : 0, 0); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Do not attempt to change the page size for a WAL database */ + if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain)) + ==PAGER_JOURNALMODE_WAL ){ + db->nextPagesize = 0; + } + + if( sqlite3BtreeSetPageSize(pTemp, sqlite3BtreeGetPageSize(pMain), nRes, 0) + || (!isMemDb && sqlite3BtreeSetPageSize(pTemp, db->nextPagesize, nRes, 0)) + || NEVER(db->mallocFailed) + ){ + rc = SQLITE_NOMEM_BKPT; + goto end_of_vacuum; + } + +#ifndef SQLITE_OMIT_AUTOVACUUM + sqlite3BtreeSetAutoVacuum(pTemp, db->nextAutovac>=0 ? db->nextAutovac : + sqlite3BtreeGetAutoVacuum(pMain)); +#endif + + /* Query the schema of the main database. Create a mirror schema + ** in the temporary database. + */ + db->init.iDb = nDb; /* force new CREATE statements into vacuum_db */ + rc = execSqlF(db, pzErrMsg, + "SELECT sql FROM \"%w\".sqlite_schema" + " WHERE type='table'AND name<>'sqlite_sequence'" + " AND coalesce(rootpage,1)>0", + zDbMain + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + rc = execSqlF(db, pzErrMsg, + "SELECT sql FROM \"%w\".sqlite_schema" + " WHERE type='index'", + zDbMain + ); + if( rc!=SQLITE_OK ) goto end_of_vacuum; + db->init.iDb = 0; + + /* Loop through the tables in the main database. For each, do + ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy + ** the contents to the temporary database. + */ + rc = execSqlF(db, pzErrMsg, + "SELECT'INSERT INTO vacuum_db.'||quote(name)" + "||' SELECT*FROM\"%w\".'||quote(name)" + "FROM vacuum_db.sqlite_schema " + "WHERE type='table'AND coalesce(rootpage,1)>0", + zDbMain + ); + assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 ); + db->mDbFlags &= ~DBFLAG_Vacuum; + if( rc!=SQLITE_OK ) goto end_of_vacuum; + + /* Copy the triggers, views, and virtual tables from the main database + ** over to the temporary database. None of these objects has any + ** associated storage, so all we have to do is copy their entries + ** from the schema table. + */ + rc = execSqlF(db, pzErrMsg, + "INSERT INTO vacuum_db.sqlite_schema" + " SELECT*FROM \"%w\".sqlite_schema" + " WHERE type IN('view','trigger')" + " OR(type='table'AND rootpage=0)", + zDbMain + ); + if( rc ) goto end_of_vacuum; + + /* At this point, there is a write transaction open on both the + ** vacuum database and the main database. Assuming no error occurs, + ** both transactions are closed by this block - the main database + ** transaction by sqlite3BtreeCopyFile() and the other by an explicit + ** call to sqlite3BtreeCommit(). + */ + { + u32 meta; + int i; + + /* This array determines which meta meta values are preserved in the + ** vacuum. Even entries are the meta value number and odd entries + ** are an increment to apply to the meta value after the vacuum. + ** The increment is used to increase the schema cookie so that other + ** connections to the same database will know to reread the schema. + */ + static const unsigned char aCopy[] = { + BTREE_SCHEMA_VERSION, 1, /* Add one to the old schema cookie */ + BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ + BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ + BTREE_USER_VERSION, 0, /* Preserve the user version */ + BTREE_APPLICATION_ID, 0, /* Preserve the application id */ + }; + + assert( SQLITE_TXN_WRITE==sqlite3BtreeTxnState(pTemp) ); + assert( pOut!=0 || SQLITE_TXN_WRITE==sqlite3BtreeTxnState(pMain) ); + + /* Copy Btree meta values */ + for(i=0; iflags */ + db->init.iDb = 0; + db->mDbFlags = saved_mDbFlags; + db->flags = saved_flags; + db->nChange = saved_nChange; + db->nTotalChange = saved_nTotalChange; + db->mTrace = saved_mTrace; + sqlite3BtreeSetPageSize(pMain, -1, 0, 1); + + /* Currently there is an SQL level transaction open on the vacuum + ** database. No locks are held on any other files (since the main file + ** was committed at the btree level). So it safe to end the transaction + ** by manually setting the autoCommit flag to true and detaching the + ** vacuum database. The vacuum_db journal file is deleted when the pager + ** is closed by the DETACH. + */ + db->autoCommit = 1; + + if( pDb ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + pDb->pSchema = 0; + } + + /* This both clears the schemas and reduces the size of the db->aDb[] + ** array. */ + sqlite3ResetAllSchemasOfConnection(db); + + return rc; +} + +#endif /* SQLITE_OMIT_VACUUM && SQLITE_OMIT_ATTACH */ + +/************** End of vacuum.c **********************************************/ +/************** Begin file vtab.c ********************************************/ +/* +** 2006 June 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains code used to help implement virtual tables. +*/ +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* #include "sqliteInt.h" */ + +/* +** Before a virtual table xCreate() or xConnect() method is invoked, the +** sqlite3.pVtabCtx member variable is set to point to an instance of +** this struct allocated on the stack. It is used by the implementation of +** the sqlite3_declare_vtab() and sqlite3_vtab_config() APIs, both of which +** are invoked only from within xCreate and xConnect methods. +*/ +struct VtabCtx { + VTable *pVTable; /* The virtual table being constructed */ + Table *pTab; /* The Table object to which the virtual table belongs */ + VtabCtx *pPrior; /* Parent context (if any) */ + int bDeclared; /* True after sqlite3_declare_vtab() is called */ +}; + +/* +** Construct and install a Module object for a virtual table. When this +** routine is called, it is guaranteed that all appropriate locks are held +** and the module is not already part of the connection. +** +** If there already exists a module with zName, replace it with the new one. +** If pModule==0, then delete the module zName if it exists. +*/ +SQLITE_PRIVATE Module *sqlite3VtabCreateModule( + sqlite3 *db, /* Database in which module is registered */ + const char *zName, /* Name assigned to this module */ + const sqlite3_module *pModule, /* The definition of the module */ + void *pAux, /* Context pointer for xCreate/xConnect */ + void (*xDestroy)(void *) /* Module destructor function */ +){ + Module *pMod; + Module *pDel; + char *zCopy; + if( pModule==0 ){ + zCopy = (char*)zName; + pMod = 0; + }else{ + int nName = sqlite3Strlen30(zName); + pMod = (Module *)sqlite3Malloc(sizeof(Module) + nName + 1); + if( pMod==0 ){ + sqlite3OomFault(db); + return 0; + } + zCopy = (char *)(&pMod[1]); + memcpy(zCopy, zName, nName+1); + pMod->zName = zCopy; + pMod->pModule = pModule; + pMod->pAux = pAux; + pMod->xDestroy = xDestroy; + pMod->pEpoTab = 0; + pMod->nRefModule = 1; + } + pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); + if( pDel ){ + if( pDel==pMod ){ + sqlite3OomFault(db); + sqlite3DbFree(db, pDel); + pMod = 0; + }else{ + sqlite3VtabEponymousTableClear(db, pDel); + sqlite3VtabModuleUnref(db, pDel); + } + } + return pMod; +} + +/* +** The actual function that does the work of creating a new module. +** This function implements the sqlite3_create_module() and +** sqlite3_create_module_v2() interfaces. +*/ +static int createModule( + sqlite3 *db, /* Database in which module is registered */ + const char *zName, /* Name assigned to this module */ + const sqlite3_module *pModule, /* The definition of the module */ + void *pAux, /* Context pointer for xCreate/xConnect */ + void (*xDestroy)(void *) /* Module destructor function */ +){ + int rc = SQLITE_OK; + + sqlite3_mutex_enter(db->mutex); + (void)sqlite3VtabCreateModule(db, zName, pModule, pAux, xDestroy); + rc = sqlite3ApiExit(db, rc); + if( rc!=SQLITE_OK && xDestroy ) xDestroy(pAux); + sqlite3_mutex_leave(db->mutex); + return rc; +} + + +/* +** External API function used to create a new virtual-table module. +*/ +SQLITE_API int sqlite3_create_module( + sqlite3 *db, /* Database in which module is registered */ + const char *zName, /* Name assigned to this module */ + const sqlite3_module *pModule, /* The definition of the module */ + void *pAux /* Context pointer for xCreate/xConnect */ +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif + return createModule(db, zName, pModule, pAux, 0); +} + +/* +** External API function used to create a new virtual-table module. +*/ +SQLITE_API int sqlite3_create_module_v2( + sqlite3 *db, /* Database in which module is registered */ + const char *zName, /* Name assigned to this module */ + const sqlite3_module *pModule, /* The definition of the module */ + void *pAux, /* Context pointer for xCreate/xConnect */ + void (*xDestroy)(void *) /* Module destructor function */ +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif + return createModule(db, zName, pModule, pAux, xDestroy); +} + +/* +** External API to drop all virtual-table modules, except those named +** on the azNames list. +*/ +SQLITE_API int sqlite3_drop_modules(sqlite3 *db, const char** azNames){ + HashElem *pThis, *pNext; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + for(pThis=sqliteHashFirst(&db->aModule); pThis; pThis=pNext){ + Module *pMod = (Module*)sqliteHashData(pThis); + pNext = sqliteHashNext(pThis); + if( azNames ){ + int ii; + for(ii=0; azNames[ii]!=0 && strcmp(azNames[ii],pMod->zName)!=0; ii++){} + if( azNames[ii]!=0 ) continue; + } + createModule(db, pMod->zName, 0, 0, 0); + } + return SQLITE_OK; +} + +/* +** Decrement the reference count on a Module object. Destroy the +** module when the reference count reaches zero. +*/ +SQLITE_PRIVATE void sqlite3VtabModuleUnref(sqlite3 *db, Module *pMod){ + assert( pMod->nRefModule>0 ); + pMod->nRefModule--; + if( pMod->nRefModule==0 ){ + if( pMod->xDestroy ){ + pMod->xDestroy(pMod->pAux); + } + assert( pMod->pEpoTab==0 ); + sqlite3DbFree(db, pMod); + } +} + +/* +** Lock the virtual table so that it cannot be disconnected. +** Locks nest. Every lock should have a corresponding unlock. +** If an unlock is omitted, resources leaks will occur. +** +** If a disconnect is attempted while a virtual table is locked, +** the disconnect is deferred until all locks have been removed. +*/ +SQLITE_PRIVATE void sqlite3VtabLock(VTable *pVTab){ + pVTab->nRef++; +} + + +/* +** pTab is a pointer to a Table structure representing a virtual-table. +** Return a pointer to the VTable object used by connection db to access +** this virtual-table, if one has been created, or NULL otherwise. +*/ +SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3 *db, Table *pTab){ + VTable *pVtab; + assert( IsVirtual(pTab) ); + for(pVtab=pTab->u.vtab.p; pVtab && pVtab->db!=db; pVtab=pVtab->pNext); + return pVtab; +} + +/* +** Decrement the ref-count on a virtual table object. When the ref-count +** reaches zero, call the xDisconnect() method to delete the object. +*/ +SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *pVTab){ + sqlite3 *db = pVTab->db; + + assert( db ); + assert( pVTab->nRef>0 ); + assert( db->eOpenState==SQLITE_STATE_OPEN + || db->eOpenState==SQLITE_STATE_ZOMBIE ); + + pVTab->nRef--; + if( pVTab->nRef==0 ){ + sqlite3_vtab *p = pVTab->pVtab; + sqlite3VtabModuleUnref(pVTab->db, pVTab->pMod); + if( p ){ + p->pModule->xDisconnect(p); + } + sqlite3DbFree(db, pVTab); + } +} + +/* +** Table p is a virtual table. This function moves all elements in the +** p->u.vtab.p list to the sqlite3.pDisconnect lists of their associated +** database connections to be disconnected at the next opportunity. +** Except, if argument db is not NULL, then the entry associated with +** connection db is left in the p->u.vtab.p list. +*/ +static VTable *vtabDisconnectAll(sqlite3 *db, Table *p){ + VTable *pRet = 0; + VTable *pVTable = p->u.vtab.p; + p->u.vtab.p = 0; + + /* Assert that the mutex (if any) associated with the BtShared database + ** that contains table p is held by the caller. See header comments + ** above function sqlite3VtabUnlockList() for an explanation of why + ** this makes it safe to access the sqlite3.pDisconnect list of any + ** database connection that may have an entry in the p->u.vtab.p list. + */ + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); + + while( pVTable ){ + sqlite3 *db2 = pVTable->db; + VTable *pNext = pVTable->pNext; + assert( db2 ); + if( db2==db ){ + pRet = pVTable; + p->u.vtab.p = pRet; + pRet->pNext = 0; + }else{ + pVTable->pNext = db2->pDisconnect; + db2->pDisconnect = pVTable; + } + pVTable = pNext; + } + + assert( !db || pRet ); + return pRet; +} + +/* +** Table *p is a virtual table. This function removes the VTable object +** for table *p associated with database connection db from the linked +** list in p->pVTab. It also decrements the VTable ref count. This is +** used when closing database connection db to free all of its VTable +** objects without disturbing the rest of the Schema object (which may +** be being used by other shared-cache connections). +*/ +SQLITE_PRIVATE void sqlite3VtabDisconnect(sqlite3 *db, Table *p){ + VTable **ppVTab; + + assert( IsVirtual(p) ); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + assert( sqlite3_mutex_held(db->mutex) ); + + for(ppVTab=&p->u.vtab.p; *ppVTab; ppVTab=&(*ppVTab)->pNext){ + if( (*ppVTab)->db==db ){ + VTable *pVTab = *ppVTab; + *ppVTab = pVTab->pNext; + sqlite3VtabUnlock(pVTab); + break; + } + } +} + + +/* +** Disconnect all the virtual table objects in the sqlite3.pDisconnect list. +** +** This function may only be called when the mutexes associated with all +** shared b-tree databases opened using connection db are held by the +** caller. This is done to protect the sqlite3.pDisconnect list. The +** sqlite3.pDisconnect list is accessed only as follows: +** +** 1) By this function. In this case, all BtShared mutexes and the mutex +** associated with the database handle itself must be held. +** +** 2) By function vtabDisconnectAll(), when it adds a VTable entry to +** the sqlite3.pDisconnect list. In this case either the BtShared mutex +** associated with the database the virtual table is stored in is held +** or, if the virtual table is stored in a non-sharable database, then +** the database handle mutex is held. +** +** As a result, a sqlite3.pDisconnect cannot be accessed simultaneously +** by multiple threads. It is thread-safe. +*/ +SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ + VTable *p = db->pDisconnect; + + assert( sqlite3BtreeHoldsAllMutexes(db) ); + assert( sqlite3_mutex_held(db->mutex) ); + + if( p ){ + db->pDisconnect = 0; + sqlite3ExpirePreparedStatements(db, 0); + do { + VTable *pNext = p->pNext; + sqlite3VtabUnlock(p); + p = pNext; + }while( p ); + } +} + +/* +** Clear any and all virtual-table information from the Table record. +** This routine is called, for example, just before deleting the Table +** record. +** +** Since it is a virtual-table, the Table structure contains a pointer +** to the head of a linked list of VTable structures. Each VTable +** structure is associated with a single sqlite3* user of the schema. +** The reference count of the VTable structure associated with database +** connection db is decremented immediately (which may lead to the +** structure being xDisconnected and free). Any other VTable structures +** in the list are moved to the sqlite3.pDisconnect list of the associated +** database connection. +*/ +SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ + if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); + if( p->u.vtab.azArg ){ + int i; + for(i=0; iu.vtab.nArg; i++){ + if( i!=1 ) sqlite3DbFree(db, p->u.vtab.azArg[i]); + } + sqlite3DbFree(db, p->u.vtab.azArg); + } +} + +/* +** Add a new module argument to pTable->u.vtab.azArg[]. +** The string is not copied - the pointer is stored. The +** string will be freed automatically when the table is +** deleted. +*/ +static void addModuleArgument(Parse *pParse, Table *pTable, char *zArg){ + sqlite3_int64 nBytes = sizeof(char *)*(2+pTable->u.vtab.nArg); + char **azModuleArg; + sqlite3 *db = pParse->db; + if( pTable->u.vtab.nArg+3>=db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns on %s", pTable->zName); + } + azModuleArg = sqlite3DbRealloc(db, pTable->u.vtab.azArg, nBytes); + if( azModuleArg==0 ){ + sqlite3DbFree(db, zArg); + }else{ + int i = pTable->u.vtab.nArg++; + azModuleArg[i] = zArg; + azModuleArg[i+1] = 0; + pTable->u.vtab.azArg = azModuleArg; + } +} + +/* +** The parser calls this routine when it first sees a CREATE VIRTUAL TABLE +** statement. The module name has been parsed, but the optional list +** of parameters that follow the module name are still pending. +*/ +SQLITE_PRIVATE void sqlite3VtabBeginParse( + Parse *pParse, /* Parsing context */ + Token *pName1, /* Name of new table, or database name */ + Token *pName2, /* Name of new table or NULL */ + Token *pModuleName, /* Name of the module for the virtual table */ + int ifNotExists /* No error if the table already exists */ +){ + Table *pTable; /* The new virtual table */ + sqlite3 *db; /* Database connection */ + + sqlite3StartTable(pParse, pName1, pName2, 0, 0, 1, ifNotExists); + pTable = pParse->pNewTable; + if( pTable==0 ) return; + assert( 0==pTable->pIndex ); + pTable->eTabType = TABTYP_VTAB; + + db = pParse->db; + + assert( pTable->u.vtab.nArg==0 ); + addModuleArgument(pParse, pTable, sqlite3NameFromToken(db, pModuleName)); + addModuleArgument(pParse, pTable, 0); + addModuleArgument(pParse, pTable, sqlite3DbStrDup(db, pTable->zName)); + assert( (pParse->sNameToken.z==pName2->z && pName2->z!=0) + || (pParse->sNameToken.z==pName1->z && pName2->z==0) + ); + pParse->sNameToken.n = (int)( + &pModuleName->z[pModuleName->n] - pParse->sNameToken.z + ); + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Creating a virtual table invokes the authorization callback twice. + ** The first invocation, to obtain permission to INSERT a row into the + ** sqlite_schema table, has already been made by sqlite3StartTable(). + ** The second call, to obtain permission to create the table, is made now. + */ + if( pTable->u.vtab.azArg ){ + int iDb = sqlite3SchemaToIndex(db, pTable->pSchema); + assert( iDb>=0 ); /* The database the table is being created in */ + sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, + pTable->u.vtab.azArg[0], pParse->db->aDb[iDb].zDbSName); + } +#endif +} + +/* +** This routine takes the module argument that has been accumulating +** in pParse->zArg[] and appends it to the list of arguments on the +** virtual table currently under construction in pParse->pTable. +*/ +static void addArgumentToVtab(Parse *pParse){ + if( pParse->sArg.z && pParse->pNewTable ){ + const char *z = (const char*)pParse->sArg.z; + int n = pParse->sArg.n; + sqlite3 *db = pParse->db; + addModuleArgument(pParse, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); + } +} + +/* +** The parser calls this routine after the CREATE VIRTUAL TABLE statement +** has been completely parsed. +*/ +SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ + Table *pTab = pParse->pNewTable; /* The table being constructed */ + sqlite3 *db = pParse->db; /* The database connection */ + + if( pTab==0 ) return; + addArgumentToVtab(pParse); + pParse->sArg.z = 0; + if( pTab->u.vtab.nArg<1 ) return; + + /* If the CREATE VIRTUAL TABLE statement is being entered for the + ** first time (in other words if the virtual table is actually being + ** created now instead of just being read out of sqlite_schema) then + ** do additional initialization work and store the statement text + ** in the sqlite_schema table. + */ + if( !db->init.busy ){ + char *zStmt; + char *zWhere; + int iDb; + int iReg; + Vdbe *v; + + sqlite3MayAbort(pParse); + + /* Compute the complete text of the CREATE VIRTUAL TABLE statement */ + if( pEnd ){ + pParse->sNameToken.n = (int)(pEnd->z - pParse->sNameToken.z) + pEnd->n; + } + zStmt = sqlite3MPrintf(db, "CREATE VIRTUAL TABLE %T", &pParse->sNameToken); + + /* A slot for the record has already been allocated in the + ** schema table. We just need to update that slot with all + ** the information we've collected. + ** + ** The VM register number pParse->regRowid holds the rowid of an + ** entry in the sqlite_schema table tht was created for this vtab + ** by sqlite3StartTable(). + */ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3NestedParse(pParse, + "UPDATE %Q." DFLT_SCHEMA_TABLE " " + "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q " + "WHERE rowid=#%d", + db->aDb[iDb].zDbSName, + pTab->zName, + pTab->zName, + zStmt, + pParse->regRowid + ); + v = sqlite3GetVdbe(pParse); + sqlite3ChangeCookie(pParse, iDb); + + sqlite3VdbeAddOp0(v, OP_Expire); + zWhere = sqlite3MPrintf(db, "name=%Q AND sql=%Q", pTab->zName, zStmt); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere, 0); + sqlite3DbFree(db, zStmt); + + iReg = ++pParse->nMem; + sqlite3VdbeLoadString(v, iReg, pTab->zName); + sqlite3VdbeAddOp2(v, OP_VCreate, iDb, iReg); + } + + /* If we are rereading the sqlite_schema table create the in-memory + ** record of the table. The xConnect() method is not called until + ** the first time the virtual table is used in an SQL statement. This + ** allows a schema that contains virtual tables to be loaded before + ** the required virtual table implementations are registered. */ + else { + Table *pOld; + Schema *pSchema = pTab->pSchema; + const char *zName = pTab->zName; + assert( sqlite3SchemaMutexHeld(db, 0, pSchema) ); + pOld = sqlite3HashInsert(&pSchema->tblHash, zName, pTab); + if( pOld ){ + sqlite3OomFault(db); + assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ + return; + } + pParse->pNewTable = 0; + } +} + +/* +** The parser calls this routine when it sees the first token +** of an argument to the module name in a CREATE VIRTUAL TABLE statement. +*/ +SQLITE_PRIVATE void sqlite3VtabArgInit(Parse *pParse){ + addArgumentToVtab(pParse); + pParse->sArg.z = 0; + pParse->sArg.n = 0; +} + +/* +** The parser calls this routine for each token after the first token +** in an argument to the module name in a CREATE VIRTUAL TABLE statement. +*/ +SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse *pParse, Token *p){ + Token *pArg = &pParse->sArg; + if( pArg->z==0 ){ + pArg->z = p->z; + pArg->n = p->n; + }else{ + assert(pArg->z <= p->z); + pArg->n = (int)(&p->z[p->n] - pArg->z); + } +} + +/* +** Invoke a virtual table constructor (either xCreate or xConnect). The +** pointer to the function to invoke is passed as the fourth parameter +** to this procedure. +*/ +static int vtabCallConstructor( + sqlite3 *db, + Table *pTab, + Module *pMod, + int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), + char **pzErr +){ + VtabCtx sCtx; + VTable *pVTable; + int rc; + const char *const*azArg = (const char *const*)pTab->u.vtab.azArg; + int nArg = pTab->u.vtab.nArg; + char *zErr = 0; + char *zModuleName; + int iDb; + VtabCtx *pCtx; + + /* Check that the virtual-table is not already being initialized */ + for(pCtx=db->pVtabCtx; pCtx; pCtx=pCtx->pPrior){ + if( pCtx->pTab==pTab ){ + *pzErr = sqlite3MPrintf(db, + "vtable constructor called recursively: %s", pTab->zName + ); + return SQLITE_LOCKED; + } + } + + zModuleName = sqlite3DbStrDup(db, pTab->zName); + if( !zModuleName ){ + return SQLITE_NOMEM_BKPT; + } + + pVTable = sqlite3MallocZero(sizeof(VTable)); + if( !pVTable ){ + sqlite3OomFault(db); + sqlite3DbFree(db, zModuleName); + return SQLITE_NOMEM_BKPT; + } + pVTable->db = db; + pVTable->pMod = pMod; + pVTable->eVtabRisk = SQLITE_VTABRISK_Normal; + + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pTab->u.vtab.azArg[1] = db->aDb[iDb].zDbSName; + + /* Invoke the virtual table constructor */ + assert( &db->pVtabCtx ); + assert( xConstruct ); + sCtx.pTab = pTab; + sCtx.pVTable = pVTable; + sCtx.pPrior = db->pVtabCtx; + sCtx.bDeclared = 0; + db->pVtabCtx = &sCtx; + rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + db->pVtabCtx = sCtx.pPrior; + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); + assert( sCtx.pTab==pTab ); + + if( SQLITE_OK!=rc ){ + if( zErr==0 ){ + *pzErr = sqlite3MPrintf(db, "vtable constructor failed: %s", zModuleName); + }else { + *pzErr = sqlite3MPrintf(db, "%s", zErr); + sqlite3_free(zErr); + } + sqlite3DbFree(db, pVTable); + }else if( ALWAYS(pVTable->pVtab) ){ + /* Justification of ALWAYS(): A correct vtab constructor must allocate + ** the sqlite3_vtab object if successful. */ + memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0])); + pVTable->pVtab->pModule = pMod->pModule; + pMod->nRefModule++; + pVTable->nRef = 1; + if( sCtx.bDeclared==0 ){ + const char *zFormat = "vtable constructor did not declare schema: %s"; + *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); + sqlite3VtabUnlock(pVTable); + rc = SQLITE_ERROR; + }else{ + int iCol; + u16 oooHidden = 0; + /* If everything went according to plan, link the new VTable structure + ** into the linked list headed by pTab->u.vtab.p. Then loop through the + ** columns of the table to see if any of them contain the token "hidden". + ** If so, set the Column COLFLAG_HIDDEN flag and remove the token from + ** the type string. */ + pVTable->pNext = pTab->u.vtab.p; + pTab->u.vtab.p = pVTable; + + for(iCol=0; iColnCol; iCol++){ + char *zType = sqlite3ColumnType(&pTab->aCol[iCol], ""); + int nType; + int i = 0; + nType = sqlite3Strlen30(zType); + for(i=0; i0 ){ + assert(zType[i-1]==' '); + zType[i-1] = '\0'; + } + pTab->aCol[iCol].colFlags |= COLFLAG_HIDDEN; + pTab->tabFlags |= TF_HasHidden; + oooHidden = TF_OOOHidden; + }else{ + pTab->tabFlags |= oooHidden; + } + } + } + } + + sqlite3DbFree(db, zModuleName); + return rc; +} + +/* +** This function is invoked by the parser to call the xConnect() method +** of the virtual table pTab. If an error occurs, an error code is returned +** and an error left in pParse. +** +** This call is a no-op if table pTab is not a virtual table. +*/ +SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ + sqlite3 *db = pParse->db; + const char *zMod; + Module *pMod; + int rc; + + assert( pTab ); + if( !IsVirtual(pTab) || sqlite3GetVTable(db, pTab) ){ + return SQLITE_OK; + } + + /* Locate the required virtual table module */ + zMod = pTab->u.vtab.azArg[0]; + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); + + if( !pMod ){ + const char *zModule = pTab->u.vtab.azArg[0]; + sqlite3ErrorMsg(pParse, "no such module: %s", zModule); + rc = SQLITE_ERROR; + }else{ + char *zErr = 0; + rc = vtabCallConstructor(db, pTab, pMod, pMod->pModule->xConnect, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3ErrorMsg(pParse, "%s", zErr); + pParse->rc = rc; + } + sqlite3DbFree(db, zErr); + } + + return rc; +} +/* +** Grow the db->aVTrans[] array so that there is room for at least one +** more v-table. Return SQLITE_NOMEM if a malloc fails, or SQLITE_OK otherwise. +*/ +static int growVTrans(sqlite3 *db){ + const int ARRAY_INCR = 5; + + /* Grow the sqlite3.aVTrans array if required */ + if( (db->nVTrans%ARRAY_INCR)==0 ){ + VTable **aVTrans; + sqlite3_int64 nBytes = sizeof(sqlite3_vtab*)* + ((sqlite3_int64)db->nVTrans + ARRAY_INCR); + aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes); + if( !aVTrans ){ + return SQLITE_NOMEM_BKPT; + } + memset(&aVTrans[db->nVTrans], 0, sizeof(sqlite3_vtab *)*ARRAY_INCR); + db->aVTrans = aVTrans; + } + + return SQLITE_OK; +} + +/* +** Add the virtual table pVTab to the array sqlite3.aVTrans[]. Space should +** have already been reserved using growVTrans(). +*/ +static void addToVTrans(sqlite3 *db, VTable *pVTab){ + /* Add pVtab to the end of sqlite3.aVTrans */ + db->aVTrans[db->nVTrans++] = pVTab; + sqlite3VtabLock(pVTab); +} + +/* +** This function is invoked by the vdbe to call the xCreate method +** of the virtual table named zTab in database iDb. +** +** If an error occurs, *pzErr is set to point to an English language +** description of the error and an SQLITE_XXX error code is returned. +** In this case the caller must call sqlite3DbFree(db, ) on *pzErr. +*/ +SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){ + int rc = SQLITE_OK; + Table *pTab; + Module *pMod; + const char *zMod; + + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName); + assert( pTab && IsVirtual(pTab) && !pTab->u.vtab.p ); + + /* Locate the required virtual table module */ + zMod = pTab->u.vtab.azArg[0]; + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); + + /* If the module has been registered and includes a Create method, + ** invoke it now. If the module has not been registered, return an + ** error. Otherwise, do nothing. + */ + if( pMod==0 || pMod->pModule->xCreate==0 || pMod->pModule->xDestroy==0 ){ + *pzErr = sqlite3MPrintf(db, "no such module: %s", zMod); + rc = SQLITE_ERROR; + }else{ + rc = vtabCallConstructor(db, pTab, pMod, pMod->pModule->xCreate, pzErr); + } + + /* Justification of ALWAYS(): The xConstructor method is required to + ** create a valid sqlite3_vtab if it returns SQLITE_OK. */ + if( rc==SQLITE_OK && ALWAYS(sqlite3GetVTable(db, pTab)) ){ + rc = growVTrans(db); + if( rc==SQLITE_OK ){ + addToVTrans(db, sqlite3GetVTable(db, pTab)); + } + } + + return rc; +} + +/* +** This function is used to set the schema of a virtual table. It is only +** valid to call this function from within the xCreate() or xConnect() of a +** virtual table module. +*/ +SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ + VtabCtx *pCtx; + int rc = SQLITE_OK; + Table *pTab; + char *zErr = 0; + Parse sParse; + int initBusy; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + pCtx = db->pVtabCtx; + if( !pCtx || pCtx->bDeclared ){ + sqlite3Error(db, SQLITE_MISUSE); + sqlite3_mutex_leave(db->mutex); + return SQLITE_MISUSE_BKPT; + } + pTab = pCtx->pTab; + assert( IsVirtual(pTab) ); + + memset(&sParse, 0, sizeof(sParse)); + sParse.eParseMode = PARSE_MODE_DECLARE_VTAB; + sParse.db = db; + /* We should never be able to reach this point while loading the + ** schema. Nevertheless, defend against that (turn off db->init.busy) + ** in case a bug arises. */ + assert( db->init.busy==0 ); + initBusy = db->init.busy; + db->init.busy = 0; + sParse.nQueryLoop = 1; + if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable, &zErr) + && sParse.pNewTable + && !db->mallocFailed + && IsOrdinaryTable(sParse.pNewTable) + ){ + if( !pTab->aCol ){ + Table *pNew = sParse.pNewTable; + Index *pIdx; + pTab->aCol = pNew->aCol; + sqlite3ExprListDelete(db, pNew->u.tab.pDfltList); + pTab->nNVCol = pTab->nCol = pNew->nCol; + pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid); + pNew->nCol = 0; + pNew->aCol = 0; + assert( pTab->pIndex==0 ); + assert( HasRowid(pNew) || sqlite3PrimaryKeyIndex(pNew)!=0 ); + if( !HasRowid(pNew) + && pCtx->pVTable->pMod->pModule->xUpdate!=0 + && sqlite3PrimaryKeyIndex(pNew)->nKeyCol!=1 + ){ + /* WITHOUT ROWID virtual tables must either be read-only (xUpdate==0) + ** or else must have a single-column PRIMARY KEY */ + rc = SQLITE_ERROR; + } + pIdx = pNew->pIndex; + if( pIdx ){ + assert( pIdx->pNext==0 ); + pTab->pIndex = pIdx; + pNew->pIndex = 0; + pIdx->pTable = pTab; + } + } + pCtx->bDeclared = 1; + }else{ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); + sqlite3DbFree(db, zErr); + rc = SQLITE_ERROR; + } + sParse.eParseMode = PARSE_MODE_NORMAL; + + if( sParse.pVdbe ){ + sqlite3VdbeFinalize(sParse.pVdbe); + } + sqlite3DeleteTable(db, sParse.pNewTable); + sqlite3ParserReset(&sParse); + db->init.busy = initBusy; + + assert( (rc&0xff)==rc ); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** This function is invoked by the vdbe to call the xDestroy method +** of the virtual table named zTab in database iDb. This occurs +** when a DROP TABLE is mentioned. +** +** This call is a no-op if zTab is not a virtual table. +*/ +SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){ + int rc = SQLITE_OK; + Table *pTab; + + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName); + if( pTab!=0 && ALWAYS(pTab->u.vtab.p!=0) ){ + VTable *p; + int (*xDestroy)(sqlite3_vtab *); + for(p=pTab->u.vtab.p; p; p=p->pNext){ + assert( p->pVtab ); + if( p->pVtab->nRef>0 ){ + return SQLITE_LOCKED; + } + } + p = vtabDisconnectAll(db, pTab); + xDestroy = p->pMod->pModule->xDestroy; + if( xDestroy==0 ) xDestroy = p->pMod->pModule->xDisconnect; + assert( xDestroy!=0 ); + pTab->nTabRef++; + rc = xDestroy(p->pVtab); + /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ + if( rc==SQLITE_OK ){ + assert( pTab->u.vtab.p==p && p->pNext==0 ); + p->pVtab = 0; + pTab->u.vtab.p = 0; + sqlite3VtabUnlock(p); + } + sqlite3DeleteTable(db, pTab); + } + + return rc; +} + +/* +** This function invokes either the xRollback or xCommit method +** of each of the virtual tables in the sqlite3.aVTrans array. The method +** called is identified by the second argument, "offset", which is +** the offset of the method to call in the sqlite3_module structure. +** +** The array is cleared after invoking the callbacks. +*/ +static void callFinaliser(sqlite3 *db, int offset){ + int i; + if( db->aVTrans ){ + VTable **aVTrans = db->aVTrans; + db->aVTrans = 0; + for(i=0; inVTrans; i++){ + VTable *pVTab = aVTrans[i]; + sqlite3_vtab *p = pVTab->pVtab; + if( p ){ + int (*x)(sqlite3_vtab *); + x = *(int (**)(sqlite3_vtab *))((char *)p->pModule + offset); + if( x ) x(p); + } + pVTab->iSavepoint = 0; + sqlite3VtabUnlock(pVTab); + } + sqlite3DbFree(db, aVTrans); + db->nVTrans = 0; + } +} + +/* +** Invoke the xSync method of all virtual tables in the sqlite3.aVTrans +** array. Return the error code for the first error that occurs, or +** SQLITE_OK if all xSync operations are successful. +** +** If an error message is available, leave it in p->zErrMsg. +*/ +SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, Vdbe *p){ + int i; + int rc = SQLITE_OK; + VTable **aVTrans = db->aVTrans; + + db->aVTrans = 0; + for(i=0; rc==SQLITE_OK && inVTrans; i++){ + int (*x)(sqlite3_vtab *); + sqlite3_vtab *pVtab = aVTrans[i]->pVtab; + if( pVtab && (x = pVtab->pModule->xSync)!=0 ){ + rc = x(pVtab); + sqlite3VtabImportErrmsg(p, pVtab); + } + } + db->aVTrans = aVTrans; + return rc; +} + +/* +** Invoke the xRollback method of all virtual tables in the +** sqlite3.aVTrans array. Then clear the array itself. +*/ +SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db){ + callFinaliser(db, offsetof(sqlite3_module,xRollback)); + return SQLITE_OK; +} + +/* +** Invoke the xCommit method of all virtual tables in the +** sqlite3.aVTrans array. Then clear the array itself. +*/ +SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db){ + callFinaliser(db, offsetof(sqlite3_module,xCommit)); + return SQLITE_OK; +} + +/* +** If the virtual table pVtab supports the transaction interface +** (xBegin/xRollback/xCommit and optionally xSync) and a transaction is +** not currently open, invoke the xBegin method now. +** +** If the xBegin call is successful, place the sqlite3_vtab pointer +** in the sqlite3.aVTrans array. +*/ +SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ + int rc = SQLITE_OK; + const sqlite3_module *pModule; + + /* Special case: If db->aVTrans is NULL and db->nVTrans is greater + ** than zero, then this function is being called from within a + ** virtual module xSync() callback. It is illegal to write to + ** virtual module tables in this case, so return SQLITE_LOCKED. + */ + if( sqlite3VtabInSync(db) ){ + return SQLITE_LOCKED; + } + if( !pVTab ){ + return SQLITE_OK; + } + pModule = pVTab->pVtab->pModule; + + if( pModule->xBegin ){ + int i; + + /* If pVtab is already in the aVTrans array, return early */ + for(i=0; inVTrans; i++){ + if( db->aVTrans[i]==pVTab ){ + return SQLITE_OK; + } + } + + /* Invoke the xBegin method. If successful, add the vtab to the + ** sqlite3.aVTrans[] array. */ + rc = growVTrans(db); + if( rc==SQLITE_OK ){ + rc = pModule->xBegin(pVTab->pVtab); + if( rc==SQLITE_OK ){ + int iSvpt = db->nStatement + db->nSavepoint; + addToVTrans(db, pVTab); + if( iSvpt && pModule->xSavepoint ){ + pVTab->iSavepoint = iSvpt; + rc = pModule->xSavepoint(pVTab->pVtab, iSvpt-1); + } + } + } + } + return rc; +} + +/* +** Invoke either the xSavepoint, xRollbackTo or xRelease method of all +** virtual tables that currently have an open transaction. Pass iSavepoint +** as the second argument to the virtual table method invoked. +** +** If op is SAVEPOINT_BEGIN, the xSavepoint method is invoked. If it is +** SAVEPOINT_ROLLBACK, the xRollbackTo method. Otherwise, if op is +** SAVEPOINT_RELEASE, then the xRelease method of each virtual table with +** an open transaction is invoked. +** +** If any virtual table method returns an error code other than SQLITE_OK, +** processing is abandoned and the error returned to the caller of this +** function immediately. If all calls to virtual table methods are successful, +** SQLITE_OK is returned. +*/ +SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ + int rc = SQLITE_OK; + + assert( op==SAVEPOINT_RELEASE||op==SAVEPOINT_ROLLBACK||op==SAVEPOINT_BEGIN ); + assert( iSavepoint>=-1 ); + if( db->aVTrans ){ + int i; + for(i=0; rc==SQLITE_OK && inVTrans; i++){ + VTable *pVTab = db->aVTrans[i]; + const sqlite3_module *pMod = pVTab->pMod->pModule; + if( pVTab->pVtab && pMod->iVersion>=2 ){ + int (*xMethod)(sqlite3_vtab *, int); + sqlite3VtabLock(pVTab); + switch( op ){ + case SAVEPOINT_BEGIN: + xMethod = pMod->xSavepoint; + pVTab->iSavepoint = iSavepoint+1; + break; + case SAVEPOINT_ROLLBACK: + xMethod = pMod->xRollbackTo; + break; + default: + xMethod = pMod->xRelease; + break; + } + if( xMethod && pVTab->iSavepoint>iSavepoint ){ + rc = xMethod(pVTab->pVtab, iSavepoint); + } + sqlite3VtabUnlock(pVTab); + } + } + } + return rc; +} + +/* +** The first parameter (pDef) is a function implementation. The +** second parameter (pExpr) is the first argument to this function. +** If pExpr is a column in a virtual table, then let the virtual +** table implementation have an opportunity to overload the function. +** +** This routine is used to allow virtual table implementations to +** overload MATCH, LIKE, GLOB, and REGEXP operators. +** +** Return either the pDef argument (indicating no change) or a +** new FuncDef structure that is marked as ephemeral using the +** SQLITE_FUNC_EPHEM flag. +*/ +SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( + sqlite3 *db, /* Database connection for reporting malloc problems */ + FuncDef *pDef, /* Function to possibly overload */ + int nArg, /* Number of arguments to the function */ + Expr *pExpr /* First argument to the function */ +){ + Table *pTab; + sqlite3_vtab *pVtab; + sqlite3_module *pMod; + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**) = 0; + void *pArg = 0; + FuncDef *pNew; + int rc = 0; + + /* Check to see the left operand is a column in a virtual table */ + if( NEVER(pExpr==0) ) return pDef; + if( pExpr->op!=TK_COLUMN ) return pDef; + pTab = pExpr->y.pTab; + if( pTab==0 ) return pDef; + if( !IsVirtual(pTab) ) return pDef; + pVtab = sqlite3GetVTable(db, pTab)->pVtab; + assert( pVtab!=0 ); + assert( pVtab->pModule!=0 ); + pMod = (sqlite3_module *)pVtab->pModule; + if( pMod->xFindFunction==0 ) return pDef; + + /* Call the xFindFunction method on the virtual table implementation + ** to see if the implementation wants to overload this function. + ** + ** Though undocumented, we have historically always invoked xFindFunction + ** with an all lower-case function name. Continue in this tradition to + ** avoid any chance of an incompatibility. + */ +#ifdef SQLITE_DEBUG + { + int i; + for(i=0; pDef->zName[i]; i++){ + unsigned char x = (unsigned char)pDef->zName[i]; + assert( x==sqlite3UpperToLower[x] ); + } + } +#endif + rc = pMod->xFindFunction(pVtab, nArg, pDef->zName, &xSFunc, &pArg); + if( rc==0 ){ + return pDef; + } + + /* Create a new ephemeral function definition for the overloaded + ** function */ + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) + + sqlite3Strlen30(pDef->zName) + 1); + if( pNew==0 ){ + return pDef; + } + *pNew = *pDef; + pNew->zName = (const char*)&pNew[1]; + memcpy((char*)&pNew[1], pDef->zName, sqlite3Strlen30(pDef->zName)+1); + pNew->xSFunc = xSFunc; + pNew->pUserData = pArg; + pNew->funcFlags |= SQLITE_FUNC_EPHEM; + return pNew; +} + +/* +** Make sure virtual table pTab is contained in the pParse->apVirtualLock[] +** array so that an OP_VBegin will get generated for it. Add pTab to the +** array if it is missing. If pTab is already in the array, this routine +** is a no-op. +*/ +SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ + Parse *pToplevel = sqlite3ParseToplevel(pParse); + int i, n; + Table **apVtabLock; + + assert( IsVirtual(pTab) ); + for(i=0; inVtabLock; i++){ + if( pTab==pToplevel->apVtabLock[i] ) return; + } + n = (pToplevel->nVtabLock+1)*sizeof(pToplevel->apVtabLock[0]); + apVtabLock = sqlite3Realloc(pToplevel->apVtabLock, n); + if( apVtabLock ){ + pToplevel->apVtabLock = apVtabLock; + pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; + }else{ + sqlite3OomFault(pToplevel->db); + } +} + +/* +** Check to see if virtual table module pMod can be have an eponymous +** virtual table instance. If it can, create one if one does not already +** exist. Return non-zero if either the eponymous virtual table instance +** exists when this routine returns or if an attempt to create it failed +** and an error message was left in pParse. +** +** An eponymous virtual table instance is one that is named after its +** module, and more importantly, does not require a CREATE VIRTUAL TABLE +** statement in order to come into existance. Eponymous virtual table +** instances always exist. They cannot be DROP-ed. +** +** Any virtual table module for which xConnect and xCreate are the same +** method can have an eponymous virtual table instance. +*/ +SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ + const sqlite3_module *pModule = pMod->pModule; + Table *pTab; + char *zErr = 0; + int rc; + sqlite3 *db = pParse->db; + if( pMod->pEpoTab ) return 1; + if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0; + pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return 0; + pTab->zName = sqlite3DbStrDup(db, pMod->zName); + if( pTab->zName==0 ){ + sqlite3DbFree(db, pTab); + return 0; + } + pMod->pEpoTab = pTab; + pTab->nTabRef = 1; + pTab->eTabType = TABTYP_VTAB; + pTab->pSchema = db->aDb[0].pSchema; + assert( pTab->u.vtab.nArg==0 ); + pTab->iPKey = -1; + pTab->tabFlags |= TF_Eponymous; + addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(pParse, pTab, 0); + addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); + rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); + if( rc ){ + sqlite3ErrorMsg(pParse, "%s", zErr); + sqlite3DbFree(db, zErr); + sqlite3VtabEponymousTableClear(db, pMod); + } + return 1; +} + +/* +** Erase the eponymous virtual table instance associated with +** virtual table module pMod, if it exists. +*/ +SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ + Table *pTab = pMod->pEpoTab; + if( pTab!=0 ){ + /* Mark the table as Ephemeral prior to deleting it, so that the + ** sqlite3DeleteTable() routine will know that it is not stored in + ** the schema. */ + pTab->tabFlags |= TF_Ephemeral; + sqlite3DeleteTable(db, pTab); + pMod->pEpoTab = 0; + } +} + +/* +** Return the ON CONFLICT resolution mode in effect for the virtual +** table update operation currently in progress. +** +** The results of this routine are undefined unless it is called from +** within an xUpdate method. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *db){ + static const unsigned char aMap[] = { + SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE + }; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); + assert( OE_Ignore==4 && OE_Replace==5 ); + assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); + return (int)aMap[db->vtabOnConflict-1]; +} + +/* +** Call from within the xCreate() or xConnect() methods to provide +** the SQLite core with additional information about the behavior +** of the virtual table being implemented. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc = SQLITE_OK; + VtabCtx *p; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + p = db->pVtabCtx; + if( !p ){ + rc = SQLITE_MISUSE_BKPT; + }else{ + assert( p->pTab==0 || IsVirtual(p->pTab) ); + va_start(ap, op); + switch( op ){ + case SQLITE_VTAB_CONSTRAINT_SUPPORT: { + p->pVTable->bConstraint = (u8)va_arg(ap, int); + break; + } + case SQLITE_VTAB_INNOCUOUS: { + p->pVTable->eVtabRisk = SQLITE_VTABRISK_Low; + break; + } + case SQLITE_VTAB_DIRECTONLY: { + p->pVTable->eVtabRisk = SQLITE_VTABRISK_High; + break; + } + default: { + rc = SQLITE_MISUSE_BKPT; + break; + } + } + va_end(ap); + } + + if( rc!=SQLITE_OK ) sqlite3Error(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/************** End of vtab.c ************************************************/ +/************** Begin file wherecode.c ***************************************/ +/* +** 2015-06-06 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This module contains C code that generates VDBE code used to process +** the WHERE clause of SQL statements. +** +** This file was split off from where.c on 2015-06-06 in order to reduce the +** size of where.c and make it easier to edit. This file contains the routines +** that actually generate the bulk of the WHERE loop code. The original where.c +** file retains the code that does query planning and analysis. +*/ +/* #include "sqliteInt.h" */ +/************** Include whereInt.h in the middle of wherecode.c **************/ +/************** Begin file whereInt.h ****************************************/ +/* +** 2013-11-12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains structure and macro definitions for the query +** planner logic in "where.c". These definitions are broken out into +** a separate source file for easier editing. +*/ +#ifndef SQLITE_WHEREINT_H +#define SQLITE_WHEREINT_H + + +/* Forward references +*/ +typedef struct WhereClause WhereClause; +typedef struct WhereMaskSet WhereMaskSet; +typedef struct WhereOrInfo WhereOrInfo; +typedef struct WhereAndInfo WhereAndInfo; +typedef struct WhereLevel WhereLevel; +typedef struct WhereLoop WhereLoop; +typedef struct WherePath WherePath; +typedef struct WhereTerm WhereTerm; +typedef struct WhereLoopBuilder WhereLoopBuilder; +typedef struct WhereScan WhereScan; +typedef struct WhereOrCost WhereOrCost; +typedef struct WhereOrSet WhereOrSet; + +/* +** This object contains information needed to implement a single nested +** loop in WHERE clause. +** +** Contrast this object with WhereLoop. This object describes the +** implementation of the loop. WhereLoop describes the algorithm. +** This object contains a pointer to the WhereLoop algorithm as one of +** its elements. +** +** The WhereInfo object contains a single instance of this object for +** each term in the FROM clause (which is to say, for each of the +** nested loops as implemented). The order of WhereLevel objects determines +** the loop nested order, with WhereInfo.a[0] being the outer loop and +** WhereInfo.a[WhereInfo.nLevel-1] being the inner loop. +*/ +struct WhereLevel { + int iLeftJoin; /* Memory cell used to implement LEFT OUTER JOIN */ + int iTabCur; /* The VDBE cursor used to access the table */ + int iIdxCur; /* The VDBE cursor used to access pIdx */ + int addrBrk; /* Jump here to break out of the loop */ + int addrNxt; /* Jump here to start the next IN combination */ + int addrSkip; /* Jump here for next iteration of skip-scan */ + int addrCont; /* Jump here to continue with the next loop cycle */ + int addrFirst; /* First instruction of interior of the loop */ + int addrBody; /* Beginning of the body of this loop */ + int regBignull; /* big-null flag reg. True if a NULL-scan is needed */ + int addrBignull; /* Jump here for next part of big-null scan */ +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS + u32 iLikeRepCntr; /* LIKE range processing counter register (times 2) */ + int addrLikeRep; /* LIKE range processing address */ +#endif + u8 iFrom; /* Which entry in the FROM clause */ + u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ + int p1, p2; /* Operands of the opcode used to end the loop */ + union { /* Information that depends on pWLoop->wsFlags */ + struct { + int nIn; /* Number of entries in aInLoop[] */ + struct InLoop { + int iCur; /* The VDBE cursor used by this IN operator */ + int addrInTop; /* Top of the IN loop */ + int iBase; /* Base register of multi-key index record */ + int nPrefix; /* Number of prior entires in the key */ + u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ + } *aInLoop; /* Information about each nested IN operator */ + } in; /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */ + Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ + } u; + struct WhereLoop *pWLoop; /* The selected WhereLoop object */ + Bitmask notReady; /* FROM entries not usable at this level */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrVisit; /* Address at which row is visited */ +#endif +}; + +/* +** Each instance of this object represents an algorithm for evaluating one +** term of a join. Every term of the FROM clause will have at least +** one corresponding WhereLoop object (unless INDEXED BY constraints +** prevent a query solution - which is an error) and many terms of the +** FROM clause will have multiple WhereLoop objects, each describing a +** potential way of implementing that FROM-clause term, together with +** dependencies and cost estimates for using the chosen algorithm. +** +** Query planning consists of building up a collection of these WhereLoop +** objects, then computing a particular sequence of WhereLoop objects, with +** one WhereLoop object per FROM clause term, that satisfy all dependencies +** and that minimize the overall cost. +*/ +struct WhereLoop { + Bitmask prereq; /* Bitmask of other loops that must run first */ + Bitmask maskSelf; /* Bitmask identifying table iTab */ +#ifdef SQLITE_DEBUG + char cId; /* Symbolic ID of this loop for debugging use */ +#endif + u8 iTab; /* Position in FROM clause of table for this loop */ + u8 iSortIdx; /* Sorting index number. 0==None */ + LogEst rSetup; /* One-time setup cost (ex: create transient index) */ + LogEst rRun; /* Cost of running each loop */ + LogEst nOut; /* Estimated number of output rows */ + union { + struct { /* Information for internal btree tables */ + u16 nEq; /* Number of equality constraints */ + u16 nBtm; /* Size of BTM vector */ + u16 nTop; /* Size of TOP vector */ + u16 nDistinctCol; /* Index columns used to sort for DISTINCT */ + Index *pIndex; /* Index used, or NULL */ + } btree; + struct { /* Information for virtual tables */ + int idxNum; /* Index number */ + u8 needFree; /* True if sqlite3_free(idxStr) is needed */ + i8 isOrdered; /* True if satisfies ORDER BY */ + u16 omitMask; /* Terms that may be omitted */ + char *idxStr; /* Index identifier string */ + } vtab; + } u; + u32 wsFlags; /* WHERE_* flags describing the plan */ + u16 nLTerm; /* Number of entries in aLTerm[] */ + u16 nSkip; /* Number of NULL aLTerm[] entries */ + /**** whereLoopXfer() copies fields above ***********************/ +# define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) + u16 nLSlot; /* Number of slots allocated for aLTerm[] */ + WhereTerm **aLTerm; /* WhereTerms used */ + WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ + WhereTerm *aLTermSpace[3]; /* Initial aLTerm[] space */ +}; + +/* This object holds the prerequisites and the cost of running a +** subquery on one operand of an OR operator in the WHERE clause. +** See WhereOrSet for additional information +*/ +struct WhereOrCost { + Bitmask prereq; /* Prerequisites */ + LogEst rRun; /* Cost of running this subquery */ + LogEst nOut; /* Number of outputs for this subquery */ +}; + +/* The WhereOrSet object holds a set of possible WhereOrCosts that +** correspond to the subquery(s) of OR-clause processing. Only the +** best N_OR_COST elements are retained. +*/ +#define N_OR_COST 3 +struct WhereOrSet { + u16 n; /* Number of valid a[] entries */ + WhereOrCost a[N_OR_COST]; /* Set of best costs */ +}; + +/* +** Each instance of this object holds a sequence of WhereLoop objects +** that implement some or all of a query plan. +** +** Think of each WhereLoop object as a node in a graph with arcs +** showing dependencies and costs for travelling between nodes. (That is +** not a completely accurate description because WhereLoop costs are a +** vector, not a scalar, and because dependencies are many-to-one, not +** one-to-one as are graph nodes. But it is a useful visualization aid.) +** Then a WherePath object is a path through the graph that visits some +** or all of the WhereLoop objects once. +** +** The "solver" works by creating the N best WherePath objects of length +** 1. Then using those as a basis to compute the N best WherePath objects +** of length 2. And so forth until the length of WherePaths equals the +** number of nodes in the FROM clause. The best (lowest cost) WherePath +** at the end is the chosen query plan. +*/ +struct WherePath { + Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ + Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ + LogEst nRow; /* Estimated number of rows generated by this path */ + LogEst rCost; /* Total cost of this path */ + LogEst rUnsorted; /* Total cost of this path ignoring sorting costs */ + i8 isOrdered; /* No. of ORDER BY terms satisfied. -1 for unknown */ + WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ +}; + +/* +** The query generator uses an array of instances of this structure to +** help it analyze the subexpressions of the WHERE clause. Each WHERE +** clause subexpression is separated from the others by AND operators, +** usually, or sometimes subexpressions separated by OR. +** +** All WhereTerms are collected into a single WhereClause structure. +** The following identity holds: +** +** WhereTerm.pWC->a[WhereTerm.idx] == WhereTerm +** +** When a term is of the form: +** +** X +** +** where X is a column name and is one of certain operators, +** then WhereTerm.leftCursor and WhereTerm.u.leftColumn record the +** cursor number and column number for X. WhereTerm.eOperator records +** the using a bitmask encoding defined by WO_xxx below. The +** use of a bitmask encoding for the operator allows us to search +** quickly for terms that match any of several different operators. +** +** A WhereTerm might also be two or more subterms connected by OR: +** +** (t1.X ) OR (t1.Y ) OR .... +** +** In this second case, wtFlag has the TERM_ORINFO bit set and eOperator==WO_OR +** and the WhereTerm.u.pOrInfo field points to auxiliary information that +** is collected about the OR clause. +** +** If a term in the WHERE clause does not match either of the two previous +** categories, then eOperator==0. The WhereTerm.pExpr field is still set +** to the original subexpression content and wtFlags is set up appropriately +** but no other fields in the WhereTerm object are meaningful. +** +** When eOperator!=0, prereqRight and prereqAll record sets of cursor numbers, +** but they do so indirectly. A single WhereMaskSet structure translates +** cursor number into bits and the translated bit is stored in the prereq +** fields. The translation is used in order to maximize the number of +** bits that will fit in a Bitmask. The VDBE cursor numbers might be +** spread out over the non-negative integers. For example, the cursor +** numbers might be 3, 8, 9, 10, 20, 23, 41, and 45. The WhereMaskSet +** translates these sparse cursor numbers into consecutive integers +** beginning with 0 in order to make the best possible use of the available +** bits in the Bitmask. So, in the example above, the cursor numbers +** would be mapped into integers 0 through 7. +** +** The number of terms in a join is limited by the number of bits +** in prereqRight and prereqAll. The default is 64 bits, hence SQLite +** is only able to process joins with 64 or fewer tables. +*/ +struct WhereTerm { + Expr *pExpr; /* Pointer to the subexpression that is this term */ + WhereClause *pWC; /* The clause this term is part of */ + LogEst truthProb; /* Probability of truth for this expression */ + u16 wtFlags; /* TERM_xxx bit flags. See below */ + u16 eOperator; /* A WO_xx value describing */ + u8 nChild; /* Number of children that must disable us */ + u8 eMatchOp; /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */ + int iParent; /* Disable pWC->a[iParent] when this term disabled */ + int leftCursor; /* Cursor number of X in "X " */ + union { + struct { + int leftColumn; /* Column number of X in "X " */ + int iField; /* Field in (?,?,?) IN (SELECT...) vector */ + } x; /* Opcode other than OP_OR or OP_AND */ + WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ + WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ + } u; + Bitmask prereqRight; /* Bitmask of tables used by pExpr->pRight */ + Bitmask prereqAll; /* Bitmask of tables referenced by pExpr */ +}; + +/* +** Allowed values of WhereTerm.wtFlags +*/ +#define TERM_DYNAMIC 0x0001 /* Need to call sqlite3ExprDelete(db, pExpr) */ +#define TERM_VIRTUAL 0x0002 /* Added by the optimizer. Do not code */ +#define TERM_CODED 0x0004 /* This term is already coded */ +#define TERM_COPIED 0x0008 /* Has a child */ +#define TERM_ORINFO 0x0010 /* Need to free the WhereTerm.u.pOrInfo object */ +#define TERM_ANDINFO 0x0020 /* Need to free the WhereTerm.u.pAndInfo obj */ +#define TERM_OR_OK 0x0040 /* Used during OR-clause processing */ +#define TERM_VNULL 0x0080 /* Manufactured x>NULL or x<=NULL term */ +#define TERM_LIKEOPT 0x0100 /* Virtual terms from the LIKE optimization */ +#define TERM_LIKECOND 0x0200 /* Conditionally this LIKE operator term */ +#define TERM_LIKE 0x0400 /* The original LIKE operator */ +#define TERM_IS 0x0800 /* Term.pExpr is an IS operator */ +#define TERM_VARSELECT 0x1000 /* Term.pExpr contains a correlated sub-query */ +#define TERM_HEURTRUTH 0x2000 /* Heuristic truthProb used */ +#ifdef SQLITE_ENABLE_STAT4 +# define TERM_HIGHTRUTH 0x4000 /* Term excludes few rows */ +#else +# define TERM_HIGHTRUTH 0 /* Only used with STAT4 */ +#endif + +/* +** An instance of the WhereScan object is used as an iterator for locating +** terms in the WHERE clause that are useful to the query planner. +*/ +struct WhereScan { + WhereClause *pOrigWC; /* Original, innermost WhereClause */ + WhereClause *pWC; /* WhereClause currently being scanned */ + const char *zCollName; /* Required collating sequence, if not NULL */ + Expr *pIdxExpr; /* Search for this index expression */ + char idxaff; /* Must match this affinity, if zCollName!=NULL */ + unsigned char nEquiv; /* Number of entries in aiCur[] and aiColumn[] */ + unsigned char iEquiv; /* Next unused slot in aiCur[] and aiColumn[] */ + u32 opMask; /* Acceptable operators */ + int k; /* Resume scanning at this->pWC->a[this->k] */ + int aiCur[11]; /* Cursors in the equivalence class */ + i16 aiColumn[11]; /* Corresponding column number in the eq-class */ +}; + +/* +** An instance of the following structure holds all information about a +** WHERE clause. Mostly this is a container for one or more WhereTerms. +** +** Explanation of pOuter: For a WHERE clause of the form +** +** a AND ((b AND c) OR (d AND e)) AND f +** +** There are separate WhereClause objects for the whole clause and for +** the subclauses "(b AND c)" and "(d AND e)". The pOuter field of the +** subclauses points to the WhereClause object for the whole clause. +*/ +struct WhereClause { + WhereInfo *pWInfo; /* WHERE clause processing context */ + WhereClause *pOuter; /* Outer conjunction */ + u8 op; /* Split operator. TK_AND or TK_OR */ + u8 hasOr; /* True if any a[].eOperator is WO_OR */ + int nTerm; /* Number of terms */ + int nSlot; /* Number of entries in a[] */ + WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ +#if defined(SQLITE_SMALL_STACK) + WhereTerm aStatic[1]; /* Initial static space for a[] */ +#else + WhereTerm aStatic[8]; /* Initial static space for a[] */ +#endif +}; + +/* +** A WhereTerm with eOperator==WO_OR has its u.pOrInfo pointer set to +** a dynamically allocated instance of the following structure. +*/ +struct WhereOrInfo { + WhereClause wc; /* Decomposition into subterms */ + Bitmask indexable; /* Bitmask of all indexable tables in the clause */ +}; + +/* +** A WhereTerm with eOperator==WO_AND has its u.pAndInfo pointer set to +** a dynamically allocated instance of the following structure. +*/ +struct WhereAndInfo { + WhereClause wc; /* The subexpression broken out */ +}; + +/* +** An instance of the following structure keeps track of a mapping +** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. +** +** The VDBE cursor numbers are small integers contained in +** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE +** clause, the cursor numbers might not begin with 0 and they might +** contain gaps in the numbering sequence. But we want to make maximum +** use of the bits in our bitmasks. This structure provides a mapping +** from the sparse cursor numbers into consecutive integers beginning +** with 0. +** +** If WhereMaskSet.ix[A]==B it means that The A-th bit of a Bitmask +** corresponds VDBE cursor number B. The A-th bit of a bitmask is 1<3, 5->1, 8->2, 29->0, +** 57->5, 73->4. Or one of 719 other combinations might be used. It +** does not really matter. What is important is that sparse cursor +** numbers all get mapped into bit numbers that begin with 0 and contain +** no gaps. +*/ +struct WhereMaskSet { + int bVarSelect; /* Used by sqlite3WhereExprUsage() */ + int n; /* Number of assigned cursor values */ + int ix[BMS]; /* Cursor assigned to each bit */ +}; + +/* +** Initialize a WhereMaskSet object +*/ +#define initMaskSet(P) (P)->n=0 + +/* +** This object is a convenience wrapper holding all information needed +** to construct WhereLoop objects for a particular query. +*/ +struct WhereLoopBuilder { + WhereInfo *pWInfo; /* Information about this WHERE */ + WhereClause *pWC; /* WHERE clause terms */ + ExprList *pOrderBy; /* ORDER BY clause */ + WhereLoop *pNew; /* Template WhereLoop */ + WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ +#ifdef SQLITE_ENABLE_STAT4 + UnpackedRecord *pRec; /* Probe for stat4 (if required) */ + int nRecValid; /* Number of valid fields currently in pRec */ +#endif + unsigned char bldFlags1; /* First set of SQLITE_BLDF_* flags */ + unsigned char bldFlags2; /* Second set of SQLITE_BLDF_* flags */ + unsigned int iPlanLimit; /* Search limiter */ +}; + +/* Allowed values for WhereLoopBuider.bldFlags */ +#define SQLITE_BLDF1_INDEXED 0x0001 /* An index is used */ +#define SQLITE_BLDF1_UNIQUE 0x0002 /* All keys of a UNIQUE index used */ + +#define SQLITE_BLDF2_2NDPASS 0x0004 /* Second builder pass needed */ + +/* The WhereLoopBuilder.iPlanLimit is used to limit the number of +** index+constraint combinations the query planner will consider for a +** particular query. If this parameter is unlimited, then certain +** pathological queries can spend excess time in the sqlite3WhereBegin() +** routine. The limit is high enough that is should not impact real-world +** queries. +** +** SQLITE_QUERY_PLANNER_LIMIT is the baseline limit. The limit is +** increased by SQLITE_QUERY_PLANNER_LIMIT_INCR before each term of the FROM +** clause is processed, so that every table in a join is guaranteed to be +** able to propose a some index+constraint combinations even if the initial +** baseline limit was exhausted by prior tables of the join. +*/ +#ifndef SQLITE_QUERY_PLANNER_LIMIT +# define SQLITE_QUERY_PLANNER_LIMIT 20000 +#endif +#ifndef SQLITE_QUERY_PLANNER_LIMIT_INCR +# define SQLITE_QUERY_PLANNER_LIMIT_INCR 1000 +#endif + +/* +** Each instance of this object records a change to a single node +** in an expression tree to cause that node to point to a column +** of an index rather than an expression or a virtual column. All +** such transformations need to be undone at the end of WHERE clause +** processing. +*/ +typedef struct WhereExprMod WhereExprMod; +struct WhereExprMod { + WhereExprMod *pNext; /* Next translation on a list of them all */ + Expr *pExpr; /* The Expr node that was transformed */ + Expr orig; /* Original value of the Expr node */ +}; + +/* +** The WHERE clause processing routine has two halves. The +** first part does the start of the WHERE loop and the second +** half does the tail of the WHERE loop. An instance of +** this structure is returned by the first half and passed +** into the second half to give some continuity. +** +** An instance of this object holds the complete state of the query +** planner. +*/ +struct WhereInfo { + Parse *pParse; /* Parsing and code generating context */ + SrcList *pTabList; /* List of tables in the join */ + ExprList *pOrderBy; /* The ORDER BY clause or NULL */ + ExprList *pResultSet; /* Result set of the query */ + Expr *pWhere; /* The complete WHERE clause */ + int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ + int iContinue; /* Jump here to continue with next record */ + int iBreak; /* Jump here to break out of the loop */ + int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ + u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ + LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ + u8 nLevel; /* Number of nested loop */ + i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ + u8 eOnePass; /* ONEPASS_OFF, or _SINGLE, or _MULTI */ + u8 eDistinct; /* One of the WHERE_DISTINCT_* values */ + unsigned bDeferredSeek :1; /* Uses OP_DeferredSeek */ + unsigned untestedTerms :1; /* Not all WHERE terms resolved by outer loop */ + unsigned bOrderedInnerLoop:1;/* True if only the inner-most loop is ordered */ + unsigned sorted :1; /* True if really sorted (not just grouped) */ + LogEst nRowOut; /* Estimated number of output rows */ + int iTop; /* The very beginning of the WHERE loop */ + int iEndWhere; /* End of the WHERE clause itself */ + WhereLoop *pLoops; /* List of all WhereLoop objects */ + WhereExprMod *pExprMods; /* Expression modifications */ + Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ + WhereClause sWC; /* Decomposition of the WHERE clause */ + WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ + WhereLevel a[1]; /* Information about each nest loop in WHERE */ +}; + +/* +** Private interfaces - callable only by other where.c routines. +** +** where.c: +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet*,int); +#ifdef WHERETRACE_ENABLED +SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC); +SQLITE_PRIVATE void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm); +SQLITE_PRIVATE void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC); +#endif +SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( + WhereClause *pWC, /* The WHERE clause to be searched */ + int iCur, /* Cursor number of LHS */ + int iColumn, /* Column number of LHS */ + Bitmask notReady, /* RHS must not overlap with this mask */ + u32 op, /* Mask of WO_xx values describing operator */ + Index *pIdx /* Must be compatible with this index, if not NULL */ +); + +/* wherecode.c: */ +#ifndef SQLITE_OMIT_EXPLAIN +SQLITE_PRIVATE int sqlite3WhereExplainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +); +#else +# define sqlite3WhereExplainOneScan(u,v,w,x) 0 +#endif /* SQLITE_OMIT_EXPLAIN */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +SQLITE_PRIVATE void sqlite3WhereAddScanStatus( + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ +); +#else +# define sqlite3WhereAddScanStatus(a, b, c, d) ((void)d) +#endif +SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + Parse *pParse, /* Parsing context */ + Vdbe *v, /* Prepared statement under construction */ + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + WhereLevel *pLevel, /* The current level pointer */ + Bitmask notReady /* Which tables are currently available */ +); + +/* whereexpr.c: */ +SQLITE_PRIVATE void sqlite3WhereClauseInit(WhereClause*,WhereInfo*); +SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause*); +SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause*,Expr*,u8); +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet*, Expr*); +SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); +SQLITE_PRIVATE void sqlite3WhereExprAnalyze(SrcList*, WhereClause*); +SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); + + + + + +/* +** Bitmasks for the operators on WhereTerm objects. These are all +** operators that are of interest to the query planner. An +** OR-ed combination of these values can be used when searching for +** particular WhereTerms within a WhereClause. +** +** Value constraints: +** WO_EQ == SQLITE_INDEX_CONSTRAINT_EQ +** WO_LT == SQLITE_INDEX_CONSTRAINT_LT +** WO_LE == SQLITE_INDEX_CONSTRAINT_LE +** WO_GT == SQLITE_INDEX_CONSTRAINT_GT +** WO_GE == SQLITE_INDEX_CONSTRAINT_GE +*/ +#define WO_IN 0x0001 +#define WO_EQ 0x0002 +#define WO_LT (WO_EQ<<(TK_LT-TK_EQ)) +#define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) +#define WO_GT (WO_EQ<<(TK_GT-TK_EQ)) +#define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) +#define WO_AUX 0x0040 /* Op useful to virtual tables only */ +#define WO_IS 0x0080 +#define WO_ISNULL 0x0100 +#define WO_OR 0x0200 /* Two or more OR-connected terms */ +#define WO_AND 0x0400 /* Two or more AND-connected terms */ +#define WO_EQUIV 0x0800 /* Of the form A==B, both columns */ +#define WO_NOOP 0x1000 /* This term does not restrict search space */ + +#define WO_ALL 0x1fff /* Mask of all possible WO_* values */ +#define WO_SINGLE 0x01ff /* Mask of all non-compound WO_* values */ + +/* +** These are definitions of bits in the WhereLoop.wsFlags field. +** The particular combination of bits in each WhereLoop help to +** determine the algorithm that WhereLoop represents. +*/ +#define WHERE_COLUMN_EQ 0x00000001 /* x=EXPR */ +#define WHERE_COLUMN_RANGE 0x00000002 /* xEXPR */ +#define WHERE_COLUMN_IN 0x00000004 /* x IN (...) */ +#define WHERE_COLUMN_NULL 0x00000008 /* x IS NULL */ +#define WHERE_CONSTRAINT 0x0000000f /* Any of the WHERE_COLUMN_xxx values */ +#define WHERE_TOP_LIMIT 0x00000010 /* xEXPR or x>=EXPR constraint */ +#define WHERE_BOTH_LIMIT 0x00000030 /* Both x>EXPR and xaiColumn[i]; + if( i==XN_EXPR ) return ""; + if( i==XN_ROWID ) return "rowid"; + return pIdx->pTable->aCol[i].zCnName; +} + +/* +** This routine is a helper for explainIndexRange() below +** +** pStr holds the text of an expression that we are building up one term +** at a time. This routine adds a new term to the end of the expression. +** Terms are separated by AND so add the "AND" text for second and subsequent +** terms only. +*/ +static void explainAppendTerm( + StrAccum *pStr, /* The text expression being built */ + Index *pIdx, /* Index to read column names from */ + int nTerm, /* Number of terms */ + int iTerm, /* Zero-based index of first term. */ + int bAnd, /* Non-zero to append " AND " */ + const char *zOp /* Name of the operator */ +){ + int i; + + assert( nTerm>=1 ); + if( bAnd ) sqlite3_str_append(pStr, " AND ", 5); + + if( nTerm>1 ) sqlite3_str_append(pStr, "(", 1); + for(i=0; i1 ) sqlite3_str_append(pStr, ")", 1); + + sqlite3_str_append(pStr, zOp, 1); + + if( nTerm>1 ) sqlite3_str_append(pStr, "(", 1); + for(i=0; i1 ) sqlite3_str_append(pStr, ")", 1); +} + +/* +** Argument pLevel describes a strategy for scanning table pTab. This +** function appends text to pStr that describes the subset of table +** rows scanned by the strategy in the form of an SQL expression. +** +** For example, if the query: +** +** SELECT * FROM t1 WHERE a=1 AND b>2; +** +** is run and there is an index on (a, b), then this function returns a +** string similar to: +** +** "a=? AND b>?" +*/ +static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ + Index *pIndex = pLoop->u.btree.pIndex; + u16 nEq = pLoop->u.btree.nEq; + u16 nSkip = pLoop->nSkip; + int i, j; + + if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; + sqlite3_str_append(pStr, " (", 2); + for(i=0; i=nSkip ? "%s=?" : "ANY(%s)", z); + } + + j = i; + if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ + explainAppendTerm(pStr, pIndex, pLoop->u.btree.nBtm, j, i, ">"); + i = 1; + } + if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ + explainAppendTerm(pStr, pIndex, pLoop->u.btree.nTop, j, i, "<"); + } + sqlite3_str_append(pStr, ")", 1); +} + +/* +** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN +** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was +** defined at compile-time. If it is not a no-op, a single OP_Explain opcode +** is added to the output to describe the table scan strategy in pLevel. +** +** If an OP_Explain opcode is added to the VM, its address is returned. +** Otherwise, if no OP_Explain is coded, zero is returned. +*/ +SQLITE_PRIVATE int sqlite3WhereExplainOneScan( + Parse *pParse, /* Parse context */ + SrcList *pTabList, /* Table list this loop refers to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ +){ + int ret = 0; +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( sqlite3ParseToplevel(pParse)->explain==2 ) +#endif + { + SrcItem *pItem = &pTabList->a[pLevel->iFrom]; + Vdbe *v = pParse->pVdbe; /* VM being constructed */ + sqlite3 *db = pParse->db; /* Database handle */ + int isSearch; /* True for a SEARCH. False for SCAN. */ + WhereLoop *pLoop; /* The controlling WhereLoop object */ + u32 flags; /* Flags that describe this loop */ + char *zMsg; /* Text to add to EQP output */ + StrAccum str; /* EQP output string */ + char zBuf[100]; /* Initial space for EQP output string */ + + pLoop = pLevel->pWLoop; + flags = pLoop->wsFlags; + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_OR_SUBCLAUSE) ) return 0; + + isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) + || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + + sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + str.printfFlags = SQLITE_PRINTF_INTERNAL; + sqlite3_str_appendf(&str, "%s %S", isSearch ? "SEARCH" : "SCAN", pItem); + if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + const char *zFmt = 0; + Index *pIdx; + + assert( pLoop->u.btree.pIndex!=0 ); + pIdx = pLoop->u.btree.pIndex; + assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); + if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + if( isSearch ){ + zFmt = "PRIMARY KEY"; + } + }else if( flags & WHERE_PARTIALIDX ){ + zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; + }else if( flags & WHERE_AUTO_INDEX ){ + zFmt = "AUTOMATIC COVERING INDEX"; + }else if( flags & WHERE_IDX_ONLY ){ + zFmt = "COVERING INDEX %s"; + }else{ + zFmt = "INDEX %s"; + } + if( zFmt ){ + sqlite3_str_append(&str, " USING ", 7); + sqlite3_str_appendf(&str, zFmt, pIdx->zName); + explainIndexRange(&str, pLoop); + } + }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ + const char *zRangeOp; + if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ + zRangeOp = "="; + }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ + zRangeOp = ">? AND rowid<"; + }else if( flags&WHERE_BTM_LIMIT ){ + zRangeOp = ">"; + }else{ + assert( flags&WHERE_TOP_LIMIT); + zRangeOp = "<"; + } + sqlite3_str_appendf(&str, + " USING INTEGER PRIMARY KEY (rowid%s?)",zRangeOp); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ + sqlite3_str_appendf(&str, " VIRTUAL TABLE INDEX %d:%s", + pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); + } +#endif +#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS + if( pLoop->nOut>=10 ){ + sqlite3_str_appendf(&str, " (~%llu rows)", + sqlite3LogEstToInt(pLoop->nOut)); + }else{ + sqlite3_str_append(&str, " (~1 row)", 9); + } +#endif + zMsg = sqlite3StrAccumFinish(&str); + sqlite3ExplainBreakpoint("",zMsg); + ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v), + pParse->addrExplain, 0, zMsg,P4_DYNAMIC); + } + return ret; +} +#endif /* SQLITE_OMIT_EXPLAIN */ + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Configure the VM passed as the first argument with an +** sqlite3_stmt_scanstatus() entry corresponding to the scan used to +** implement level pLvl. Argument pSrclist is a pointer to the FROM +** clause that the scan reads data from. +** +** If argument addrExplain is not 0, it must be the address of an +** OP_Explain instruction that describes the same loop. +*/ +SQLITE_PRIVATE void sqlite3WhereAddScanStatus( + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ +){ + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pSrclist->a[pLvl->iFrom].zName; + } + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); +} +#endif + + +/* +** Disable a term in the WHERE clause. Except, do not disable the term +** if it controls a LEFT OUTER JOIN and it did not originate in the ON +** or USING clause of that join. +** +** Consider the term t2.z='ok' in the following queries: +** +** (1) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x WHERE t2.z='ok' +** (2) SELECT * FROM t1 LEFT JOIN t2 ON t1.a=t2.x AND t2.z='ok' +** (3) SELECT * FROM t1, t2 WHERE t1.a=t2.x AND t2.z='ok' +** +** The t2.z='ok' is disabled in the in (2) because it originates +** in the ON clause. The term is disabled in (3) because it is not part +** of a LEFT OUTER JOIN. In (1), the term is not disabled. +** +** Disabling a term causes that term to not be tested in the inner loop +** of the join. Disabling is an optimization. When terms are satisfied +** by indices, we disable them to prevent redundant tests in the inner +** loop. We would get the correct results if nothing were ever disabled, +** but joins might run a little slower. The trick is to disable as much +** as we can without disabling too much. If we disabled in (1), we'd get +** the wrong answer. See ticket #813. +** +** If all the children of a term are disabled, then that term is also +** automatically disabled. In this way, terms get disabled if derived +** virtual terms are tested first. For example: +** +** x GLOB 'abc*' AND x>='abc' AND x<'acd' +** \___________/ \______/ \_____/ +** parent child1 child2 +** +** Only the parent term was in the original WHERE clause. The child1 +** and child2 terms were added by the LIKE optimization. If both of +** the virtual child terms are valid, then testing of the parent can be +** skipped. +** +** Usually the parent term is marked as TERM_CODED. But if the parent +** term was originally TERM_LIKE, then the parent gets TERM_LIKECOND instead. +** The TERM_LIKECOND marking indicates that the term should be coded inside +** a conditional such that is only evaluated on the second pass of a +** LIKE-optimization loop, when scanning BLOBs instead of strings. +*/ +static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ + int nLoop = 0; + assert( pTerm!=0 ); + while( (pTerm->wtFlags & TERM_CODED)==0 + && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + && (pLevel->notReady & pTerm->prereqAll)==0 + ){ + if( nLoop && (pTerm->wtFlags & TERM_LIKE)!=0 ){ + pTerm->wtFlags |= TERM_LIKECOND; + }else{ + pTerm->wtFlags |= TERM_CODED; + } +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x20000 ){ + sqlite3DebugPrintf("DISABLE-"); + sqlite3WhereTermPrint(pTerm, (int)(pTerm - (pTerm->pWC->a))); + } +#endif + if( pTerm->iParent<0 ) break; + pTerm = &pTerm->pWC->a[pTerm->iParent]; + assert( pTerm!=0 ); + pTerm->nChild--; + if( pTerm->nChild!=0 ) break; + nLoop++; + } +} + +/* +** Code an OP_Affinity opcode to apply the column affinity string zAff +** to the n registers starting at base. +** +** As an optimization, SQLITE_AFF_BLOB and SQLITE_AFF_NONE entries (which +** are no-ops) at the beginning and end of zAff are ignored. If all entries +** in zAff are SQLITE_AFF_BLOB or SQLITE_AFF_NONE, then no code gets generated. +** +** This routine makes its own copy of zAff so that the caller is free +** to modify zAff after this routine returns. +*/ +static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ + Vdbe *v = pParse->pVdbe; + if( zAff==0 ){ + assert( pParse->db->mallocFailed ); + return; + } + assert( v!=0 ); + + /* Adjust base and n to skip over SQLITE_AFF_BLOB and SQLITE_AFF_NONE + ** entries at the beginning and end of the affinity string. + */ + assert( SQLITE_AFF_NONE0 && zAff[0]<=SQLITE_AFF_BLOB ){ + n--; + base++; + zAff++; + } + while( n>1 && zAff[n-1]<=SQLITE_AFF_BLOB ){ + n--; + } + + /* Code the OP_Affinity opcode if there is anything left to do. */ + if( n>0 ){ + sqlite3VdbeAddOp4(v, OP_Affinity, base, n, 0, zAff, n); + } +} + +/* +** Expression pRight, which is the RHS of a comparison operation, is +** either a vector of n elements or, if n==1, a scalar expression. +** Before the comparison operation, affinity zAff is to be applied +** to the pRight values. This function modifies characters within the +** affinity string to SQLITE_AFF_BLOB if either: +** +** * the comparison will be performed with no affinity, or +** * the affinity change in zAff is guaranteed not to change the value. +*/ +static void updateRangeAffinityStr( + Expr *pRight, /* RHS of comparison */ + int n, /* Number of vector elements in comparison */ + char *zAff /* Affinity string to modify */ +){ + int i; + for(i=0; idb; + Expr *pNew; + pNew = sqlite3ExprDup(db, pX, 0); + if( db->mallocFailed==0 ){ + ExprList *pOrigRhs = pNew->x.pSelect->pEList; /* Original unmodified RHS */ + ExprList *pOrigLhs = pNew->pLeft->x.pList; /* Original unmodified LHS */ + ExprList *pRhs = 0; /* New RHS after modifications */ + ExprList *pLhs = 0; /* New LHS after mods */ + int i; /* Loop counter */ + Select *pSelect; /* Pointer to the SELECT on the RHS */ + + for(i=iEq; inLTerm; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + int iField = pLoop->aLTerm[i]->u.x.iField - 1; + if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ + pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); + pOrigRhs->a[iField].pExpr = 0; + assert( pOrigLhs->a[iField].pExpr!=0 ); + pLhs = sqlite3ExprListAppend(pParse, pLhs, pOrigLhs->a[iField].pExpr); + pOrigLhs->a[iField].pExpr = 0; + } + } + sqlite3ExprListDelete(db, pOrigRhs); + sqlite3ExprListDelete(db, pOrigLhs); + pNew->pLeft->x.pList = pLhs; + pNew->x.pSelect->pEList = pRhs; + if( pLhs && pLhs->nExpr==1 ){ + /* Take care here not to generate a TK_VECTOR containing only a + ** single value. Since the parser never creates such a vector, some + ** of the subroutines do not handle this case. */ + Expr *p = pLhs->a[0].pExpr; + pLhs->a[0].pExpr = 0; + sqlite3ExprDelete(db, pNew->pLeft); + pNew->pLeft = p; + } + pSelect = pNew->x.pSelect; + if( pSelect->pOrderBy ){ + /* If the SELECT statement has an ORDER BY clause, zero the + ** iOrderByCol variables. These are set to non-zero when an + ** ORDER BY term exactly matches one of the terms of the + ** result-set. Since the result-set of the SELECT statement may + ** have been modified or reordered, these variables are no longer + ** set correctly. Since setting them is just an optimization, + ** it's easiest just to zero them here. */ + ExprList *pOrderBy = pSelect->pOrderBy; + for(i=0; inExpr; i++){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } + } + +#if 0 + printf("For indexing, change the IN expr:\n"); + sqlite3TreeViewExpr(0, pX, 0); + printf("Into:\n"); + sqlite3TreeViewExpr(0, pNew, 0); +#endif + } + return pNew; +} + + +/* +** Generate code for a single equality term of the WHERE clause. An equality +** term can be either X=expr or X IN (...). pTerm is the term to be +** coded. +** +** The current value for the constraint is left in a register, the index +** of which is returned. An attempt is made store the result in iTarget but +** this is only guaranteed for TK_ISNULL and TK_IN constraints. If the +** constraint is a TK_EQ or TK_IS, then the current value might be left in +** some other register and it is the caller's responsibility to compensate. +** +** For a constraint of the form X=expr, the expression is evaluated in +** straight-line code. For constraints of the form X IN (...) +** this routine sets up a loop that will iterate over all values of X. +*/ +static int codeEqualityTerm( + Parse *pParse, /* The parsing context */ + WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ + int bRev, /* True for reverse-order IN operations */ + int iTarget /* Attempt to leave results in this register */ +){ + Expr *pX = pTerm->pExpr; + Vdbe *v = pParse->pVdbe; + int iReg; /* Register holding results */ + + assert( pLevel->pWLoop->aLTerm[iEq]==pTerm ); + assert( iTarget>0 ); + if( pX->op==TK_EQ || pX->op==TK_IS ){ + iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); + }else if( pX->op==TK_ISNULL ){ + iReg = iTarget; + sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); +#ifndef SQLITE_OMIT_SUBQUERY + }else{ + int eType = IN_INDEX_NOOP; + int iTab; + struct InLoop *pIn; + WhereLoop *pLoop = pLevel->pWLoop; + int i; + int nEq = 0; + int *aiMap = 0; + + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( bRev ); + bRev = !bRev; + } + assert( pX->op==TK_IN ); + iReg = iTarget; + + for(i=0; iaLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){ + disableTerm(pLevel, pTerm); + return iTarget; + } + } + for(i=iEq;inLTerm; i++){ + assert( pLoop->aLTerm[i]!=0 ); + if( pLoop->aLTerm[i]->pExpr==pX ) nEq++; + } + + iTab = 0; + if( (pX->flags & EP_xIsSelect)==0 || pX->x.pSelect->pEList->nExpr==1 ){ + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab); + }else{ + sqlite3 *db = pParse->db; + pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); + + if( !db->mallocFailed ){ + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); + pTerm->pExpr->iTable = iTab; + } + sqlite3ExprDelete(db, pX); + pX = pTerm->pExpr; + } + + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + VdbeCoverageIf(v, bRev); + VdbeCoverageIf(v, !bRev); + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + + pLoop->wsFlags |= WHERE_IN_ABLE; + if( pLevel->u.in.nIn==0 ){ + pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + } + if( iEq>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 ){ + pLoop->wsFlags |= WHERE_IN_EARLYOUT; + } + + i = pLevel->u.in.nIn; + pLevel->u.in.nIn += nEq; + pLevel->u.in.aInLoop = + sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); + pIn = pLevel->u.in.aInLoop; + if( pIn ){ + int iMap = 0; /* Index in aiMap[] */ + pIn += i; + for(i=iEq;inLTerm; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + int iOut = iReg + i - iEq; + if( eType==IN_INDEX_ROWID ){ + pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut); + }else{ + int iCol = aiMap ? aiMap[iMap++] : 0; + pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut); + } + sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v); + if( i==iEq ){ + pIn->iCur = iTab; + pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; + if( iEq>0 ){ + pIn->iBase = iReg - i; + pIn->nPrefix = i; + }else{ + pIn->nPrefix = 0; + } + }else{ + pIn->eEndLoopOp = OP_Noop; + } + pIn++; + } + } + testcase( iEq>0 + && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 + && (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ); + if( iEq>0 + && (pLoop->wsFlags & (WHERE_IN_SEEKSCAN|WHERE_VIRTUALTABLE))==0 + ){ + sqlite3VdbeAddOp3(v, OP_SeekHit, pLevel->iIdxCur, 0, iEq); + } + }else{ + pLevel->u.in.nIn = 0; + } + sqlite3DbFree(pParse->db, aiMap); +#endif + } + + /* As an optimization, try to disable the WHERE clause term that is + ** driving the index as it will always be true. The correct answer is + ** obtained regardless, but we might get the answer with fewer CPU cycles + ** by omitting the term. + ** + ** But do not disable the term unless we are certain that the term is + ** not a transitive constraint. For an example of where that does not + ** work, see https://sqlite.org/forum/forumpost/eb8613976a (2021-05-04) + */ + if( (pLevel->pWLoop->wsFlags & WHERE_TRANSCONS)==0 + || (pTerm->eOperator & WO_EQUIV)==0 + ){ + disableTerm(pLevel, pTerm); + } + + return iReg; +} + +/* +** Generate code that will evaluate all == and IN constraints for an +** index scan. +** +** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). +** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 +** The index has as many as three equality constraints, but in this +** example, the third "c" value is an inequality. So only two +** constraints are coded. This routine will generate code to evaluate +** a==5 and b IN (1,2,3). The current values for a and b will be stored +** in consecutive registers and the index of the first register is returned. +** +** In the example above nEq==2. But this subroutine works for any value +** of nEq including 0. If nEq==0, this routine is nearly a no-op. +** The only thing it does is allocate the pLevel->iMem memory cell and +** compute the affinity string. +** +** The nExtraReg parameter is 0 or 1. It is 0 if all WHERE clause constraints +** are == or IN and are covered by the nEq. nExtraReg is 1 if there is +** an inequality constraint (such as the "c>=5 AND c<10" in the example) that +** occurs after the nEq quality constraints. +** +** This routine allocates a range of nEq+nExtraReg memory cells and returns +** the index of the first memory cell in that range. The code that +** calls this routine will use that memory range to store keys for +** start and termination conditions of the loop. +** key value of the loop. If one or more IN operators appear, then +** this routine allocates an additional nEq memory cells for internal +** use. +** +** Before returning, *pzAff is set to point to a buffer containing a +** copy of the column affinity string of the index allocated using +** sqlite3DbMalloc(). Except, entries in the copy of the string associated +** with equality constraints that use BLOB or NONE affinity are set to +** SQLITE_AFF_BLOB. This is to deal with SQL such as the following: +** +** CREATE TABLE t1(a TEXT PRIMARY KEY, b); +** SELECT ... FROM t1 AS t2, t1 WHERE t1.a = t2.b; +** +** In the example above, the index on t1(a) has TEXT affinity. But since +** the right hand side of the equality constraint (t2.b) has BLOB/NONE affinity, +** no conversion should be attempted before using a t2.b value as part of +** a key to search the index. Hence the first byte in the returned affinity +** string in this example would be set to SQLITE_AFF_BLOB. +*/ +static int codeAllEqualityTerms( + Parse *pParse, /* Parsing context */ + WhereLevel *pLevel, /* Which nested loop of the FROM we are coding */ + int bRev, /* Reverse the order of IN operators */ + int nExtraReg, /* Number of extra registers to allocate */ + char **pzAff /* OUT: Set to point to affinity string */ +){ + u16 nEq; /* The number of == or IN constraints to code */ + u16 nSkip; /* Number of left-most columns to skip */ + Vdbe *v = pParse->pVdbe; /* The vm under construction */ + Index *pIdx; /* The index being used for this loop */ + WhereTerm *pTerm; /* A single constraint term */ + WhereLoop *pLoop; /* The WhereLoop object */ + int j; /* Loop counter */ + int regBase; /* Base register */ + int nReg; /* Number of registers to allocate */ + char *zAff; /* Affinity string to return */ + + /* This module is only called on query plans that use an index. */ + pLoop = pLevel->pWLoop; + assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + nEq = pLoop->u.btree.nEq; + nSkip = pLoop->nSkip; + pIdx = pLoop->u.btree.pIndex; + assert( pIdx!=0 ); + + /* Figure out how many memory cells we will need then allocate them. + */ + regBase = pParse->nMem + 1; + nReg = pLoop->u.btree.nEq + nExtraReg; + pParse->nMem += nReg; + + zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx)); + assert( zAff!=0 || pParse->db->mallocFailed ); + + if( nSkip ){ + int iIdxCur = pLevel->iIdxCur; + sqlite3VdbeAddOp3(v, OP_Null, 0, regBase, regBase+nSkip-1); + sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); + j = sqlite3VdbeAddOp0(v, OP_Goto); + pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), + iIdxCur, 0, regBase, nSkip); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + sqlite3VdbeJumpHere(v, j); + for(j=0; jaiColumn[j]==XN_EXPR ); + VdbeComment((v, "%s", explainIndexColumnName(pIdx, j))); + } + } + + /* Evaluate the equality constraints + */ + assert( zAff==0 || (int)strlen(zAff)>=nEq ); + for(j=nSkip; jaLTerm[j]; + assert( pTerm!=0 ); + /* The following testcase is true for indices with redundant columns. + ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ + testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + r1 = codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, regBase+j); + if( r1!=regBase+j ){ + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, regBase); + regBase = r1; + }else{ + sqlite3VdbeAddOp2(v, OP_Copy, r1, regBase+j); + } + } + if( pTerm->eOperator & WO_IN ){ + if( pTerm->pExpr->flags & EP_xIsSelect ){ + /* No affinity ever needs to be (or should be) applied to a value + ** from the RHS of an "? IN (SELECT ...)" expression. The + ** sqlite3FindInIndex() routine has already ensured that the + ** affinity of the comparison has been applied to the value. */ + if( zAff ) zAff[j] = SQLITE_AFF_BLOB; + } + }else if( (pTerm->eOperator & WO_ISNULL)==0 ){ + Expr *pRight = pTerm->pExpr->pRight; + if( (pTerm->wtFlags & TERM_IS)==0 && sqlite3ExprCanBeNull(pRight) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); + VdbeCoverage(v); + } + if( pParse->db->mallocFailed==0 && pParse->nErr==0 ){ + if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_BLOB ){ + zAff[j] = SQLITE_AFF_BLOB; + } + if( sqlite3ExprNeedsNoAffinityChange(pRight, zAff[j]) ){ + zAff[j] = SQLITE_AFF_BLOB; + } + } + } + } + *pzAff = zAff; + return regBase; +} + +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS +/* +** If the most recently coded instruction is a constant range constraint +** (a string literal) that originated from the LIKE optimization, then +** set P3 and P5 on the OP_String opcode so that the string will be cast +** to a BLOB at appropriate times. +** +** The LIKE optimization trys to evaluate "x LIKE 'abc%'" as a range +** expression: "x>='ABC' AND x<'abd'". But this requires that the range +** scan loop run twice, once for strings and a second time for BLOBs. +** The OP_String opcodes on the second pass convert the upper and lower +** bound string constants to blobs. This routine makes the necessary changes +** to the OP_String opcodes for that to happen. +** +** Except, of course, if SQLITE_LIKE_DOESNT_MATCH_BLOBS is defined, then +** only the one pass through the string space is required, so this routine +** becomes a no-op. +*/ +static void whereLikeOptimizationStringFixup( + Vdbe *v, /* prepared statement under construction */ + WhereLevel *pLevel, /* The loop that contains the LIKE operator */ + WhereTerm *pTerm /* The upper or lower bound just coded */ +){ + if( pTerm->wtFlags & TERM_LIKEOPT ){ + VdbeOp *pOp; + assert( pLevel->iLikeRepCntr>0 ); + pOp = sqlite3VdbeGetOp(v, -1); + assert( pOp!=0 ); + assert( pOp->opcode==OP_String8 + || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); + pOp->p3 = (int)(pLevel->iLikeRepCntr>>1); /* Register holding counter */ + pOp->p5 = (u8)(pLevel->iLikeRepCntr&1); /* ASC or DESC */ + } +} +#else +# define whereLikeOptimizationStringFixup(A,B,C) +#endif + +#ifdef SQLITE_ENABLE_CURSOR_HINTS +/* +** Information is passed from codeCursorHint() down to individual nodes of +** the expression tree (by sqlite3WalkExpr()) using an instance of this +** structure. +*/ +struct CCurHint { + int iTabCur; /* Cursor for the main table */ + int iIdxCur; /* Cursor for the index, if pIdx!=0. Unused otherwise */ + Index *pIdx; /* The index used to access the table */ +}; + +/* +** This function is called for every node of an expression that is a candidate +** for a cursor hint on an index cursor. For TK_COLUMN nodes that reference +** the table CCurHint.iTabCur, verify that the same column can be +** accessed through the index. If it cannot, then set pWalker->eCode to 1. +*/ +static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){ + struct CCurHint *pHint = pWalker->u.pCCurHint; + assert( pHint->pIdx!=0 ); + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pHint->iTabCur + && sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn)<0 + ){ + pWalker->eCode = 1; + } + return WRC_Continue; +} + +/* +** Test whether or not expression pExpr, which was part of a WHERE clause, +** should be included in the cursor-hint for a table that is on the rhs +** of a LEFT JOIN. Set Walker.eCode to non-zero before returning if the +** expression is not suitable. +** +** An expression is unsuitable if it might evaluate to non NULL even if +** a TK_COLUMN node that does affect the value of the expression is set +** to NULL. For example: +** +** col IS NULL +** col IS NOT NULL +** coalesce(col, 1) +** CASE WHEN col THEN 0 ELSE 1 END +*/ +static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_IS + || pExpr->op==TK_ISNULL || pExpr->op==TK_ISNOT + || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE + ){ + pWalker->eCode = 1; + }else if( pExpr->op==TK_FUNCTION ){ + int d1; + char d2[4]; + if( 0==sqlite3IsLikeFunction(pWalker->pParse->db, pExpr, &d1, d2) ){ + pWalker->eCode = 1; + } + } + + return WRC_Continue; +} + + +/* +** This function is called on every node of an expression tree used as an +** argument to the OP_CursorHint instruction. If the node is a TK_COLUMN +** that accesses any table other than the one identified by +** CCurHint.iTabCur, then do the following: +** +** 1) allocate a register and code an OP_Column instruction to read +** the specified column into the new register, and +** +** 2) transform the expression node to a TK_REGISTER node that reads +** from the newly populated register. +** +** Also, if the node is a TK_COLUMN that does access the table idenified +** by pCCurHint.iTabCur, and an index is being used (which we will +** know because CCurHint.pIdx!=0) then transform the TK_COLUMN into +** an access of the index rather than the original table. +*/ +static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ + int rc = WRC_Continue; + struct CCurHint *pHint = pWalker->u.pCCurHint; + if( pExpr->op==TK_COLUMN ){ + if( pExpr->iTable!=pHint->iTabCur ){ + int reg = ++pWalker->pParse->nMem; /* Register for column value */ + sqlite3ExprCode(pWalker->pParse, pExpr, reg); + pExpr->op = TK_REGISTER; + pExpr->iTable = reg; + }else if( pHint->pIdx!=0 ){ + pExpr->iTable = pHint->iIdxCur; + pExpr->iColumn = sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn); + assert( pExpr->iColumn>=0 ); + } + }else if( pExpr->op==TK_AGG_FUNCTION ){ + /* An aggregate function in the WHERE clause of a query means this must + ** be a correlated sub-query, and expression pExpr is an aggregate from + ** the parent context. Do not walk the function arguments in this case. + ** + ** todo: It should be possible to replace this node with a TK_REGISTER + ** expression, as the result of the expression must be stored in a + ** register at this point. The same holds for TK_AGG_COLUMN nodes. */ + rc = WRC_Prune; + } + return rc; +} + +/* +** Insert an OP_CursorHint instruction if it is appropriate to do so. +*/ +static void codeCursorHint( + SrcItem *pTabItem, /* FROM clause item */ + WhereInfo *pWInfo, /* The where clause */ + WhereLevel *pLevel, /* Which loop to provide hints for */ + WhereTerm *pEndRange /* Hint this end-of-scan boundary term if not NULL */ +){ + Parse *pParse = pWInfo->pParse; + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + Expr *pExpr = 0; + WhereLoop *pLoop = pLevel->pWLoop; + int iCur; + WhereClause *pWC; + WhereTerm *pTerm; + int i, j; + struct CCurHint sHint; + Walker sWalker; + + if( OptimizationDisabled(db, SQLITE_CursorHints) ) return; + iCur = pLevel->iTabCur; + assert( iCur==pWInfo->pTabList->a[pLevel->iFrom].iCursor ); + sHint.iTabCur = iCur; + sHint.iIdxCur = pLevel->iIdxCur; + sHint.pIdx = pLoop->u.btree.pIndex; + memset(&sWalker, 0, sizeof(sWalker)); + sWalker.pParse = pParse; + sWalker.u.pCCurHint = &sHint; + pWC = &pWInfo->sWC; + for(i=0; inTerm; i++){ + pTerm = &pWC->a[i]; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->prereqAll & pLevel->notReady ) continue; + + /* Any terms specified as part of the ON(...) clause for any LEFT + ** JOIN for which the current table is not the rhs are omitted + ** from the cursor-hint. + ** + ** If this table is the rhs of a LEFT JOIN, "IS" or "IS NULL" terms + ** that were specified as part of the WHERE clause must be excluded. + ** This is to address the following: + ** + ** SELECT ... t1 LEFT JOIN t2 ON (t1.a=t2.b) WHERE t2.c IS NULL; + ** + ** Say there is a single row in t2 that matches (t1.a=t2.b), but its + ** t2.c values is not NULL. If the (t2.c IS NULL) constraint is + ** pushed down to the cursor, this row is filtered out, causing + ** SQLite to synthesize a row of NULL values. Which does match the + ** WHERE clause, and so the query returns a row. Which is incorrect. + ** + ** For the same reason, WHERE terms such as: + ** + ** WHERE 1 = (t2.c IS NULL) + ** + ** are also excluded. See codeCursorHintIsOrFunction() for details. + */ + if( pTabItem->fg.jointype & JT_LEFT ){ + Expr *pExpr = pTerm->pExpr; + if( !ExprHasProperty(pExpr, EP_FromJoin) + || pExpr->iRightJoinTable!=pTabItem->iCursor + ){ + sWalker.eCode = 0; + sWalker.xExprCallback = codeCursorHintIsOrFunction; + sqlite3WalkExpr(&sWalker, pTerm->pExpr); + if( sWalker.eCode ) continue; + } + }else{ + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue; + } + + /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize + ** the cursor. These terms are not needed as hints for a pure range + ** scan (that has no == terms) so omit them. */ + if( pLoop->u.btree.nEq==0 && pTerm!=pEndRange ){ + for(j=0; jnLTerm && pLoop->aLTerm[j]!=pTerm; j++){} + if( jnLTerm ) continue; + } + + /* No subqueries or non-deterministic functions allowed */ + if( sqlite3ExprContainsSubquery(pTerm->pExpr) ) continue; + + /* For an index scan, make sure referenced columns are actually in + ** the index. */ + if( sHint.pIdx!=0 ){ + sWalker.eCode = 0; + sWalker.xExprCallback = codeCursorHintCheckExpr; + sqlite3WalkExpr(&sWalker, pTerm->pExpr); + if( sWalker.eCode ) continue; + } + + /* If we survive all prior tests, that means this term is worth hinting */ + pExpr = sqlite3ExprAnd(pParse, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0)); + } + if( pExpr!=0 ){ + sWalker.xExprCallback = codeCursorHintFixExpr; + sqlite3WalkExpr(&sWalker, pExpr); + sqlite3VdbeAddOp4(v, OP_CursorHint, + (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0, + (const char*)pExpr, P4_EXPR); + } +} +#else +# define codeCursorHint(A,B,C,D) /* No-op */ +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ + +/* +** Cursor iCur is open on an intkey b-tree (a table). Register iRowid contains +** a rowid value just read from cursor iIdxCur, open on index pIdx. This +** function generates code to do a deferred seek of cursor iCur to the +** rowid stored in register iRowid. +** +** Normally, this is just: +** +** OP_DeferredSeek $iCur $iRowid +** +** However, if the scan currently being coded is a branch of an OR-loop and +** the statement currently being coded is a SELECT, then P3 of OP_DeferredSeek +** is set to iIdxCur and P4 is set to point to an array of integers +** containing one entry for each column of the table cursor iCur is open +** on. For each table column, if the column is the i'th column of the +** index, then the corresponding array entry is set to (i+1). If the column +** does not appear in the index at all, the array entry is set to 0. +*/ +static void codeDeferredSeek( + WhereInfo *pWInfo, /* Where clause context */ + Index *pIdx, /* Index scan is using */ + int iCur, /* Cursor for IPK b-tree */ + int iIdxCur /* Index cursor */ +){ + Parse *pParse = pWInfo->pParse; /* Parse context */ + Vdbe *v = pParse->pVdbe; /* Vdbe to generate code within */ + + assert( iIdxCur>0 ); + assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 ); + + pWInfo->bDeferredSeek = 1; + sqlite3VdbeAddOp3(v, OP_DeferredSeek, iIdxCur, 0, iCur); + if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE) + && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask) + ){ + int i; + Table *pTab = pIdx->pTable; + u32 *ai = (u32*)sqlite3DbMallocZero(pParse->db, sizeof(u32)*(pTab->nCol+1)); + if( ai ){ + ai[0] = pTab->nCol; + for(i=0; inColumn-1; i++){ + int x1, x2; + assert( pIdx->aiColumn[i]nCol ); + x1 = pIdx->aiColumn[i]; + x2 = sqlite3TableColumnToStorage(pTab, x1); + testcase( x1!=x2 ); + if( x1>=0 ) ai[x2+1] = i+1; + } + sqlite3VdbeChangeP4(v, -1, (char*)ai, P4_INTARRAY); + } + } +} + +/* +** If the expression passed as the second argument is a vector, generate +** code to write the first nReg elements of the vector into an array +** of registers starting with iReg. +** +** If the expression is not a vector, then nReg must be passed 1. In +** this case, generate code to evaluate the expression and leave the +** result in register iReg. +*/ +static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){ + assert( nReg>0 ); + if( p && sqlite3ExprIsVector(p) ){ +#ifndef SQLITE_OMIT_SUBQUERY + if( (p->flags & EP_xIsSelect) ){ + Vdbe *v = pParse->pVdbe; + int iSelect; + assert( p->op==TK_SELECT ); + iSelect = sqlite3CodeSubselect(pParse, p); + sqlite3VdbeAddOp3(v, OP_Copy, iSelect, iReg, nReg-1); + }else +#endif + { + int i; + ExprList *pList = p->x.pList; + assert( nReg<=pList->nExpr ); + for(i=0; ia[i].pExpr, iReg+i); + } + } + }else{ + assert( nReg==1 || pParse->nErr ); + sqlite3ExprCode(pParse, p, iReg); + } +} + +/* An instance of the IdxExprTrans object carries information about a +** mapping from an expression on table columns into a column in an index +** down through the Walker. +*/ +typedef struct IdxExprTrans { + Expr *pIdxExpr; /* The index expression */ + int iTabCur; /* The cursor of the corresponding table */ + int iIdxCur; /* The cursor for the index */ + int iIdxCol; /* The column for the index */ + int iTabCol; /* The column for the table */ + WhereInfo *pWInfo; /* Complete WHERE clause information */ + sqlite3 *db; /* Database connection (for malloc()) */ +} IdxExprTrans; + +/* +** Preserve pExpr on the WhereETrans list of the WhereInfo. +*/ +static void preserveExpr(IdxExprTrans *pTrans, Expr *pExpr){ + WhereExprMod *pNew; + pNew = sqlite3DbMallocRaw(pTrans->db, sizeof(*pNew)); + if( pNew==0 ) return; + pNew->pNext = pTrans->pWInfo->pExprMods; + pTrans->pWInfo->pExprMods = pNew; + pNew->pExpr = pExpr; + memcpy(&pNew->orig, pExpr, sizeof(*pExpr)); +} + +/* The walker node callback used to transform matching expressions into +** a reference to an index column for an index on an expression. +** +** If pExpr matches, then transform it into a reference to the index column +** that contains the value of pExpr. +*/ +static int whereIndexExprTransNode(Walker *p, Expr *pExpr){ + IdxExprTrans *pX = p->u.pIdxTrans; + if( sqlite3ExprCompare(0, pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){ + preserveExpr(pX, pExpr); + pExpr->affExpr = sqlite3ExprAffinity(pExpr); + pExpr->op = TK_COLUMN; + pExpr->iTable = pX->iIdxCur; + pExpr->iColumn = pX->iIdxCol; + pExpr->y.pTab = 0; + testcase( ExprHasProperty(pExpr, EP_Skip) ); + testcase( ExprHasProperty(pExpr, EP_Unlikely) ); + ExprClearProperty(pExpr, EP_Skip|EP_Unlikely); + return WRC_Prune; + }else{ + return WRC_Continue; + } +} + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* A walker node callback that translates a column reference to a table +** into a corresponding column reference of an index. +*/ +static int whereIndexExprTransColumn(Walker *p, Expr *pExpr){ + if( pExpr->op==TK_COLUMN ){ + IdxExprTrans *pX = p->u.pIdxTrans; + if( pExpr->iTable==pX->iTabCur && pExpr->iColumn==pX->iTabCol ){ + assert( pExpr->y.pTab!=0 ); + preserveExpr(pX, pExpr); + pExpr->affExpr = sqlite3TableColumnAffinity(pExpr->y.pTab,pExpr->iColumn); + pExpr->iTable = pX->iIdxCur; + pExpr->iColumn = pX->iIdxCol; + pExpr->y.pTab = 0; + } + } + return WRC_Continue; +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + +/* +** For an indexes on expression X, locate every instance of expression X +** in pExpr and change that subexpression into a reference to the appropriate +** column of the index. +** +** 2019-10-24: Updated to also translate references to a VIRTUAL column in +** the table into references to the corresponding (stored) column of the +** index. +*/ +static void whereIndexExprTrans( + Index *pIdx, /* The Index */ + int iTabCur, /* Cursor of the table that is being indexed */ + int iIdxCur, /* Cursor of the index itself */ + WhereInfo *pWInfo /* Transform expressions in this WHERE clause */ +){ + int iIdxCol; /* Column number of the index */ + ExprList *aColExpr; /* Expressions that are indexed */ + Table *pTab; + Walker w; + IdxExprTrans x; + aColExpr = pIdx->aColExpr; + if( aColExpr==0 && !pIdx->bHasVCol ){ + /* The index does not reference any expressions or virtual columns + ** so no translations are needed. */ + return; + } + pTab = pIdx->pTable; + memset(&w, 0, sizeof(w)); + w.u.pIdxTrans = &x; + x.iTabCur = iTabCur; + x.iIdxCur = iIdxCur; + x.pWInfo = pWInfo; + x.db = pWInfo->pParse->db; + for(iIdxCol=0; iIdxColnColumn; iIdxCol++){ + i16 iRef = pIdx->aiColumn[iIdxCol]; + if( iRef==XN_EXPR ){ + assert( aColExpr->a[iIdxCol].pExpr!=0 ); + x.pIdxExpr = aColExpr->a[iIdxCol].pExpr; + if( sqlite3ExprIsConstant(x.pIdxExpr) ) continue; + w.xExprCallback = whereIndexExprTransNode; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( iRef>=0 + && (pTab->aCol[iRef].colFlags & COLFLAG_VIRTUAL)!=0 + && ((pTab->aCol[iRef].colFlags & COLFLAG_HASCOLL)==0 + || sqlite3StrICmp(sqlite3ColumnColl(&pTab->aCol[iRef]), + sqlite3StrBINARY)==0) + ){ + /* Check to see if there are direct references to generated columns + ** that are contained in the index. Pulling the generated column + ** out of the index is an optimization only - the main table is always + ** available if the index cannot be used. To avoid unnecessary + ** complication, omit this optimization if the collating sequence for + ** the column is non-standard */ + x.iTabCol = iRef; + w.xExprCallback = whereIndexExprTransColumn; +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + }else{ + continue; + } + x.iIdxCol = iIdxCol; + sqlite3WalkExpr(&w, pWInfo->pWhere); + sqlite3WalkExprList(&w, pWInfo->pOrderBy); + sqlite3WalkExprList(&w, pWInfo->pResultSet); + } +} + +/* +** The pTruth expression is always true because it is the WHERE clause +** a partial index that is driving a query loop. Look through all of the +** WHERE clause terms on the query, and if any of those terms must be +** true because pTruth is true, then mark those WHERE clause terms as +** coded. +*/ +static void whereApplyPartialIndexConstraints( + Expr *pTruth, + int iTabCur, + WhereClause *pWC +){ + int i; + WhereTerm *pTerm; + while( pTruth->op==TK_AND ){ + whereApplyPartialIndexConstraints(pTruth->pLeft, iTabCur, pWC); + pTruth = pTruth->pRight; + } + for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + Expr *pExpr; + if( pTerm->wtFlags & TERM_CODED ) continue; + pExpr = pTerm->pExpr; + if( sqlite3ExprCompare(0, pExpr, pTruth, iTabCur)==0 ){ + pTerm->wtFlags |= TERM_CODED; + } + } +} + +/* +** Generate code for the start of the iLevel-th loop in the WHERE clause +** implementation described by pWInfo. +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + Parse *pParse, /* Parsing context */ + Vdbe *v, /* Prepared statement under construction */ + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + WhereLevel *pLevel, /* The current level pointer */ + Bitmask notReady /* Which tables are currently available */ +){ + int j, k; /* Loop counters */ + int iCur; /* The VDBE cursor for the table */ + int addrNxt; /* Where to jump to continue with the next IN case */ + int bRev; /* True if we need to scan in reverse order */ + WhereLoop *pLoop; /* The WhereLoop object being coded */ + WhereClause *pWC; /* Decomposition of the entire WHERE clause */ + WhereTerm *pTerm; /* A WHERE clause term */ + sqlite3 *db; /* Database connection */ + SrcItem *pTabItem; /* FROM clause term being coded */ + int addrBrk; /* Jump here to break out of the loop */ + int addrHalt; /* addrBrk for the outermost loop */ + int addrCont; /* Jump here to continue with next cycle */ + int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ + int iReleaseReg = 0; /* Temp register to free before returning */ + Index *pIdx = 0; /* Index used by loop (if any) */ + int iLoop; /* Iteration of constraint generator loop */ + + pWC = &pWInfo->sWC; + db = pParse->db; + pLoop = pLevel->pWLoop; + pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + iCur = pTabItem->iCursor; + pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); + bRev = (pWInfo->revMask>>iLevel)&1; + VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); +#if WHERETRACE_ENABLED /* 0x20800 */ + if( sqlite3WhereTrace & 0x800 ){ + sqlite3DebugPrintf("Coding level %d of %d: notReady=%llx iFrom=%d\n", + iLevel, pWInfo->nLevel, (u64)notReady, pLevel->iFrom); + sqlite3WhereLoopPrint(pLoop, pWC); + } + if( sqlite3WhereTrace & 0x20000 ){ + if( iLevel==0 ){ + sqlite3DebugPrintf("WHERE clause being coded:\n"); + sqlite3TreeViewExpr(0, pWInfo->pWhere, 0); + } + sqlite3DebugPrintf("All WHERE-clause terms before coding:\n"); + sqlite3WhereClausePrint(pWC); + } +#endif + + /* Create labels for the "break" and "continue" instructions + ** for the current loop. Jump to addrBrk to break out of a loop. + ** Jump to cont to go immediately to the next iteration of the + ** loop. + ** + ** When there is an IN operator, we also have a "addrNxt" label that + ** means to continue with the next IN value combination. When + ** there are no IN operators in the constraints, the "addrNxt" label + ** is the same as "addrBrk". + */ + addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(pParse); + + /* If this is the right table of a LEFT OUTER JOIN, allocate and + ** initialize a memory cell that records if this table matches any + ** row of the left table of the join. + */ + assert( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE) + || pLevel->iFrom>0 || (pTabItem[0].fg.jointype & JT_LEFT)==0 + ); + if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ + pLevel->iLeftJoin = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); + VdbeComment((v, "init LEFT JOIN no-match flag")); + } + + /* Compute a safe address to jump to if we discover that the table for + ** this loop is empty and can never contribute content. */ + for(j=iLevel; j>0 && pWInfo->a[j].iLeftJoin==0; j--){} + addrHalt = pWInfo->a[j].addrBrk; + + /* Special case of a FROM clause subquery implemented as a co-routine */ + if( pTabItem->fg.viaCoroutine ){ + int regYield = pTabItem->regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); + VdbeCoverage(v); + VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); + pLevel->op = OP_Goto; + }else + +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + /* Case 1: The table is a virtual-table. Use the VFilter and VNext + ** to access the data. + */ + int iReg; /* P3 Value for OP_VFilter */ + int addrNotFound; + int nConstraint = pLoop->nLTerm; + int iIn; /* Counter for IN constraints */ + + iReg = sqlite3GetTempRange(pParse, nConstraint+2); + addrNotFound = pLevel->addrBrk; + for(j=0; jaLTerm[j]; + if( NEVER(pTerm==0) ) continue; + if( pTerm->eOperator & WO_IN ){ + codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); + addrNotFound = pLevel->addrNxt; + }else{ + Expr *pRight = pTerm->pExpr->pRight; + codeExprOrVector(pParse, pRight, iTarget, 1); + } + } + sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); + sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); + sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, + pLoop->u.vtab.idxStr, + pLoop->u.vtab.needFree ? P4_DYNAMIC : P4_STATIC); + VdbeCoverage(v); + pLoop->u.vtab.needFree = 0; + /* An OOM inside of AddOp4(OP_VFilter) instruction above might have freed + ** the u.vtab.idxStr. NULL it out to prevent a use-after-free */ + if( db->mallocFailed ) pLoop->u.vtab.idxStr = 0; + pLevel->p1 = iCur; + pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext; + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + iIn = pLevel->u.in.nIn; + for(j=nConstraint-1; j>=0; j--){ + pTerm = pLoop->aLTerm[j]; + if( (pTerm->eOperator & WO_IN)!=0 ) iIn--; + if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){ + disableTerm(pLevel, pTerm); + }else if( (pTerm->eOperator & WO_IN)!=0 + && sqlite3ExprVectorSize(pTerm->pExpr->pLeft)==1 + ){ + Expr *pCompare; /* The comparison operator */ + Expr *pRight; /* RHS of the comparison */ + VdbeOp *pOp; /* Opcode to access the value of the IN constraint */ + + /* Reload the constraint value into reg[iReg+j+2]. The same value + ** was loaded into the same register prior to the OP_VFilter, but + ** the xFilter implementation might have changed the datatype or + ** encoding of the value in the register, so it *must* be reloaded. */ + assert( pLevel->u.in.aInLoop!=0 || db->mallocFailed ); + if( !db->mallocFailed ){ + assert( iIn>=0 && iInu.in.nIn ); + pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[iIn].addrInTop); + assert( pOp->opcode==OP_Column || pOp->opcode==OP_Rowid ); + assert( pOp->opcode!=OP_Column || pOp->p3==iReg+j+2 ); + assert( pOp->opcode!=OP_Rowid || pOp->p2==iReg+j+2 ); + testcase( pOp->opcode==OP_Rowid ); + sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3); + } + + /* Generate code that will continue to the next row if + ** the IN constraint is not satisfied */ + pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0); + assert( pCompare!=0 || db->mallocFailed ); + if( pCompare ){ + pCompare->pLeft = pTerm->pExpr->pLeft; + pCompare->pRight = pRight = sqlite3Expr(db, TK_REGISTER, 0); + if( pRight ){ + pRight->iTable = iReg+j+2; + sqlite3ExprIfFalse( + pParse, pCompare, pLevel->addrCont, SQLITE_JUMPIFNULL + ); + } + pCompare->pLeft = 0; + sqlite3ExprDelete(db, pCompare); + } + } + } + assert( iIn==0 || db->mallocFailed ); + /* These registers need to be preserved in case there is an IN operator + ** loop. So we could deallocate the registers here (and potentially + ** reuse them later) if (pLoop->wsFlags & WHERE_IN_ABLE)==0. But it seems + ** simpler and safer to simply not reuse the registers. + ** + ** sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); + */ + }else +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_IN|WHERE_COLUMN_EQ))!=0 + ){ + /* Case 2: We can directly reference a single row using an + ** equality comparison against the ROWID field. Or + ** we reference multiple rows using a "rowid IN (...)" + ** construct. + */ + assert( pLoop->u.btree.nEq==1 ); + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->pExpr!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + iReleaseReg = ++pParse->nMem; + iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); + if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); + addrNxt = pLevel->addrNxt; + sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg); + VdbeCoverage(v); + pLevel->op = OP_Noop; + }else if( (pLoop->wsFlags & WHERE_IPK)!=0 + && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 + ){ + /* Case 3: We have an inequality comparison against the ROWID field. + */ + int testOp = OP_Noop; + int start; + int memEndValue = 0; + WhereTerm *pStart, *pEnd; + + j = 0; + pStart = pEnd = 0; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; + if( pLoop->wsFlags & WHERE_TOP_LIMIT ) pEnd = pLoop->aLTerm[j++]; + assert( pStart!=0 || pEnd!=0 ); + if( bRev ){ + pTerm = pStart; + pStart = pEnd; + pEnd = pTerm; + } + codeCursorHint(pTabItem, pWInfo, pLevel, pEnd); + if( pStart ){ + Expr *pX; /* The expression that defines the start bound */ + int r1, rTemp; /* Registers for holding the start boundary */ + int op; /* Cursor seek operation */ + + /* The following constant maps TK_xx codes into corresponding + ** seek opcodes. It depends on a particular ordering of TK_xx + */ + const u8 aMoveOp[] = { + /* TK_GT */ OP_SeekGT, + /* TK_LE */ OP_SeekLE, + /* TK_LT */ OP_SeekLT, + /* TK_GE */ OP_SeekGE + }; + assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ + assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ + assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ + + assert( (pStart->wtFlags & TERM_VNULL)==0 ); + testcase( pStart->wtFlags & TERM_VIRTUAL ); + pX = pStart->pExpr; + assert( pX!=0 ); + testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ + if( sqlite3ExprIsVector(pX->pRight) ){ + r1 = rTemp = sqlite3GetTempReg(pParse); + codeExprOrVector(pParse, pX->pRight, r1, 1); + testcase( pX->op==TK_GT ); + testcase( pX->op==TK_GE ); + testcase( pX->op==TK_LT ); + testcase( pX->op==TK_LE ); + op = aMoveOp[((pX->op - TK_GT - 1) & 0x3) | 0x1]; + assert( pX->op!=TK_GT || op==OP_SeekGE ); + assert( pX->op!=TK_GE || op==OP_SeekGE ); + assert( pX->op!=TK_LT || op==OP_SeekLE ); + assert( pX->op!=TK_LE || op==OP_SeekLE ); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); + disableTerm(pLevel, pStart); + op = aMoveOp[(pX->op - TK_GT)]; + } + sqlite3VdbeAddOp3(v, op, iCur, addrBrk, r1); + VdbeComment((v, "pk")); + VdbeCoverageIf(v, pX->op==TK_GT); + VdbeCoverageIf(v, pX->op==TK_LE); + VdbeCoverageIf(v, pX->op==TK_LT); + VdbeCoverageIf(v, pX->op==TK_GE); + sqlite3ReleaseTempReg(pParse, rTemp); + }else{ + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + } + if( pEnd ){ + Expr *pX; + pX = pEnd->pExpr; + assert( pX!=0 ); + assert( (pEnd->wtFlags & TERM_VNULL)==0 ); + testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ + testcase( pEnd->wtFlags & TERM_VIRTUAL ); + memEndValue = ++pParse->nMem; + codeExprOrVector(pParse, pX->pRight, memEndValue, 1); + if( 0==sqlite3ExprIsVector(pX->pRight) + && (pX->op==TK_LT || pX->op==TK_GT) + ){ + testOp = bRev ? OP_Le : OP_Ge; + }else{ + testOp = bRev ? OP_Lt : OP_Gt; + } + if( 0==sqlite3ExprIsVector(pX->pRight) ){ + disableTerm(pLevel, pEnd); + } + } + start = sqlite3VdbeCurrentAddr(v); + pLevel->op = bRev ? OP_Prev : OP_Next; + pLevel->p1 = iCur; + pLevel->p2 = start; + assert( pLevel->p5==0 ); + if( testOp!=OP_Noop ){ + iRowidReg = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); + sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); + VdbeCoverageIf(v, testOp==OP_Le); + VdbeCoverageIf(v, testOp==OP_Lt); + VdbeCoverageIf(v, testOp==OP_Ge); + VdbeCoverageIf(v, testOp==OP_Gt); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL); + } + }else if( pLoop->wsFlags & WHERE_INDEXED ){ + /* Case 4: A scan using an index. + ** + ** The WHERE clause may contain zero or more equality + ** terms ("==" or "IN" operators) that refer to the N + ** left-most columns of the index. It may also contain + ** inequality constraints (>, <, >= or <=) on the indexed + ** column that immediately follows the N equalities. Only + ** the right-most column can be an inequality - the rest must + ** use the "==" and "IN" operators. For example, if the + ** index is on (x,y,z), then the following clauses are all + ** optimized: + ** + ** x=5 + ** x=5 AND y=10 + ** x=5 AND y<10 + ** x=5 AND y>5 AND y<10 + ** x=5 AND y=5 AND z<=10 + ** + ** The z<10 term of the following cannot be used, only + ** the x=5 term: + ** + ** x=5 AND z<10 + ** + ** N may be zero if there are inequality constraints. + ** If there are no inequality constraints, then N is at + ** least one. + ** + ** This case is also used when there are no WHERE clause + ** constraints but an index is selected anyway, in order + ** to force the output order to conform to an ORDER BY. + */ + static const u8 aStartOp[] = { + 0, + 0, + OP_Rewind, /* 2: (!start_constraints && startEq && !bRev) */ + OP_Last, /* 3: (!start_constraints && startEq && bRev) */ + OP_SeekGT, /* 4: (start_constraints && !startEq && !bRev) */ + OP_SeekLT, /* 5: (start_constraints && !startEq && bRev) */ + OP_SeekGE, /* 6: (start_constraints && startEq && !bRev) */ + OP_SeekLE /* 7: (start_constraints && startEq && bRev) */ + }; + static const u8 aEndOp[] = { + OP_IdxGE, /* 0: (end_constraints && !bRev && !endEq) */ + OP_IdxGT, /* 1: (end_constraints && !bRev && endEq) */ + OP_IdxLE, /* 2: (end_constraints && bRev && !endEq) */ + OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ + }; + u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ + u16 nBtm = pLoop->u.btree.nBtm; /* Length of BTM vector */ + u16 nTop = pLoop->u.btree.nTop; /* Length of TOP vector */ + int regBase; /* Base register holding constraint values */ + WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ + WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ + int startEq; /* True if range start uses ==, >= or <= */ + int endEq; /* True if range end uses ==, >= or <= */ + int start_constraints; /* Start of range is constrained */ + int nConstraint; /* Number of constraint terms */ + int iIdxCur; /* The VDBE cursor for the index */ + int nExtraReg = 0; /* Number of extra registers needed */ + int op; /* Instruction opcode */ + char *zStartAff; /* Affinity for start of range constraint */ + char *zEndAff = 0; /* Affinity for end of range constraint */ + u8 bSeekPastNull = 0; /* True to seek past initial nulls */ + u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ + int omitTable; /* True if we use the index only */ + int regBignull = 0; /* big-null flag register */ + int addrSeekScan = 0; /* Opcode of the OP_SeekScan, if any */ + + pIdx = pLoop->u.btree.pIndex; + iIdxCur = pLevel->iIdxCur; + assert( nEq>=pLoop->nSkip ); + + /* Find any inequality constraint terms for the start and end + ** of the range. + */ + j = nEq; + if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ + pRangeStart = pLoop->aLTerm[j++]; + nExtraReg = MAX(nExtraReg, pLoop->u.btree.nBtm); + /* Like optimization range constraints always occur in pairs */ + assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || + (pLoop->wsFlags & WHERE_TOP_LIMIT)!=0 ); + } + if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ + pRangeEnd = pLoop->aLTerm[j++]; + nExtraReg = MAX(nExtraReg, pLoop->u.btree.nTop); +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( (pRangeEnd->wtFlags & TERM_LIKEOPT)!=0 ){ + assert( pRangeStart!=0 ); /* LIKE opt constraints */ + assert( pRangeStart->wtFlags & TERM_LIKEOPT ); /* occur in pairs */ + pLevel->iLikeRepCntr = (u32)++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 1, (int)pLevel->iLikeRepCntr); + VdbeComment((v, "LIKE loop counter")); + pLevel->addrLikeRep = sqlite3VdbeCurrentAddr(v); + /* iLikeRepCntr actually stores 2x the counter register number. The + ** bottom bit indicates whether the search order is ASC or DESC. */ + testcase( bRev ); + testcase( pIdx->aSortOrder[nEq]==SQLITE_SO_DESC ); + assert( (bRev & ~1)==0 ); + pLevel->iLikeRepCntr <<=1; + pLevel->iLikeRepCntr |= bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC); + } +#endif + if( pRangeStart==0 ){ + j = pIdx->aiColumn[nEq]; + if( (j>=0 && pIdx->pTable->aCol[j].notNull==0) || j==XN_EXPR ){ + bSeekPastNull = 1; + } + } + } + assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); + + /* If the WHERE_BIGNULL_SORT flag is set, then index column nEq uses + ** a non-default "big-null" sort (either ASC NULLS LAST or DESC NULLS + ** FIRST). In both cases separate ordered scans are made of those + ** index entries for which the column is null and for those for which + ** it is not. For an ASC sort, the non-NULL entries are scanned first. + ** For DESC, NULL entries are scanned first. + */ + if( (pLoop->wsFlags & (WHERE_TOP_LIMIT|WHERE_BTM_LIMIT))==0 + && (pLoop->wsFlags & WHERE_BIGNULL_SORT)!=0 + ){ + assert( bSeekPastNull==0 && nExtraReg==0 && nBtm==0 && nTop==0 ); + assert( pRangeEnd==0 && pRangeStart==0 ); + testcase( pLoop->nSkip>0 ); + nExtraReg = 1; + bSeekPastNull = 1; + pLevel->regBignull = regBignull = ++pParse->nMem; + if( pLevel->iLeftJoin ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, regBignull); + } + pLevel->addrBignull = sqlite3VdbeMakeLabel(pParse); + } + + /* If we are doing a reverse order scan on an ascending index, or + ** a forward order scan on a descending index, interchange the + ** start and end terms (pRangeStart and pRangeEnd). + */ + if( (nEqnColumn && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) ){ + SWAP(WhereTerm *, pRangeEnd, pRangeStart); + SWAP(u8, bSeekPastNull, bStopAtNull); + SWAP(u8, nBtm, nTop); + } + + if( iLevel>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)!=0 ){ + /* In case OP_SeekScan is used, ensure that the index cursor does not + ** point to a valid row for the first iteration of this loop. */ + sqlite3VdbeAddOp1(v, OP_NullRow, iIdxCur); + } + + /* Generate code to evaluate all constraint terms using == or IN + ** and store the values of those terms in an array of registers + ** starting at regBase. + */ + codeCursorHint(pTabItem, pWInfo, pLevel, pRangeEnd); + regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); + assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); + if( zStartAff && nTop ){ + zEndAff = sqlite3DbStrDup(db, &zStartAff[nEq]); + } + addrNxt = (regBignull ? pLevel->addrBignull : pLevel->addrNxt); + + testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); + testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_LE)!=0 ); + testcase( pRangeEnd && (pRangeEnd->eOperator & WO_GE)!=0 ); + startEq = !pRangeStart || pRangeStart->eOperator & (WO_LE|WO_GE); + endEq = !pRangeEnd || pRangeEnd->eOperator & (WO_LE|WO_GE); + start_constraints = pRangeStart || nEq>0; + + /* Seek the index cursor to the start of the range. */ + nConstraint = nEq; + if( pRangeStart ){ + Expr *pRight = pRangeStart->pExpr->pRight; + codeExprOrVector(pParse, pRight, regBase+nEq, nBtm); + whereLikeOptimizationStringFixup(v, pLevel, pRangeStart); + if( (pRangeStart->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); + } + if( zStartAff ){ + updateRangeAffinityStr(pRight, nBtm, &zStartAff[nEq]); + } + nConstraint += nBtm; + testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); + if( sqlite3ExprIsVector(pRight)==0 ){ + disableTerm(pLevel, pRangeStart); + }else{ + startEq = 1; + } + bSeekPastNull = 0; + }else if( bSeekPastNull ){ + startEq = 0; + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + start_constraints = 1; + nConstraint++; + }else if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + start_constraints = 1; + nConstraint++; + } + codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); + if( pLoop->nSkip>0 && nConstraint==pLoop->nSkip ){ + /* The skip-scan logic inside the call to codeAllEqualityConstraints() + ** above has already left the cursor sitting on the correct row, + ** so no further seeking is needed */ + }else{ + if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_Integer, 1, regBignull); + VdbeComment((v, "NULL-scan pass ctr")); + } + + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; + assert( op!=0 ); + if( (pLoop->wsFlags & WHERE_IN_SEEKSCAN)!=0 && op==OP_SeekGE ){ + assert( regBignull==0 ); + /* TUNING: The OP_SeekScan opcode seeks to reduce the number + ** of expensive seek operations by replacing a single seek with + ** 1 or more step operations. The question is, how many steps + ** should we try before giving up and going with a seek. The cost + ** of a seek is proportional to the logarithm of the of the number + ** of entries in the tree, so basing the number of steps to try + ** on the estimated number of rows in the btree seems like a good + ** guess. */ + addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan, + (pIdx->aiRowLogEst[0]+9)/10); + VdbeCoverage(v); + } + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGT); testcase( op==OP_SeekGT ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); + + assert( bSeekPastNull==0 || bStopAtNull==0 ); + if( regBignull ){ + assert( bSeekPastNull==1 || bStopAtNull==1 ); + assert( bSeekPastNull==!bStopAtNull ); + assert( bStopAtNull==startEq ); + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + op = aStartOp[(nConstraint>1)*4 + 2 + bRev]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, + nConstraint-startEq); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + assert( op==OP_Rewind || op==OP_Last || op==OP_SeekGE || op==OP_SeekLE); + } + } + + /* Load the value for the inequality constraint at the end of the + ** range (if any). + */ + nConstraint = nEq; + if( pRangeEnd ){ + Expr *pRight = pRangeEnd->pExpr->pRight; + codeExprOrVector(pParse, pRight, regBase+nEq, nTop); + whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); + if( (pRangeEnd->wtFlags & TERM_VNULL)==0 + && sqlite3ExprCanBeNull(pRight) + ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); + VdbeCoverage(v); + } + if( zEndAff ){ + updateRangeAffinityStr(pRight, nTop, zEndAff); + codeApplyAffinity(pParse, regBase+nEq, nTop, zEndAff); + }else{ + assert( pParse->db->mallocFailed ); + } + nConstraint += nTop; + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); + + if( sqlite3ExprIsVector(pRight)==0 ){ + disableTerm(pLevel, pRangeEnd); + }else{ + endEq = 1; + } + }else if( bStopAtNull ){ + if( regBignull==0 ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; + } + nConstraint++; + } + sqlite3DbFree(db, zStartAff); + sqlite3DbFree(db, zEndAff); + + /* Top of the loop body */ + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + + /* Check if the index cursor is past the end of the range. */ + if( nConstraint ){ + if( regBignull ){ + /* Except, skip the end-of-range check while doing the NULL-scan */ + sqlite3VdbeAddOp2(v, OP_IfNot, regBignull, sqlite3VdbeCurrentAddr(v)+3); + VdbeComment((v, "If NULL-scan 2nd pass")); + VdbeCoverage(v); + } + op = aEndOp[bRev*2 + endEq]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); + if( addrSeekScan ) sqlite3VdbeJumpHere(v, addrSeekScan); + } + if( regBignull ){ + /* During a NULL-scan, check to see if we have reached the end of + ** the NULLs */ + assert( bSeekPastNull==!bStopAtNull ); + assert( bSeekPastNull+bStopAtNull==1 ); + assert( nConstraint+bSeekPastNull>0 ); + sqlite3VdbeAddOp2(v, OP_If, regBignull, sqlite3VdbeCurrentAddr(v)+2); + VdbeComment((v, "If NULL-scan 1st pass")); + VdbeCoverage(v); + op = aEndOp[bRev*2 + bSeekPastNull]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, + nConstraint+bSeekPastNull); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); + } + + if( (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0 ){ + sqlite3VdbeAddOp3(v, OP_SeekHit, iIdxCur, nEq, nEq); + } + + /* Seek the table cursor, if required */ + omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0; + if( omitTable ){ + /* pIdx is a covering index. No need to access the main table. */ + }else if( HasRowid(pIdx->pTable) ){ + codeDeferredSeek(pWInfo, pIdx, iCur, iIdxCur); + }else if( iCur!=iIdxCur ){ + Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); + iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); + } + sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, + iRowidReg, pPk->nKeyCol); VdbeCoverage(v); + } + + if( pLevel->iLeftJoin==0 ){ + /* If pIdx is an index on one or more expressions, then look through + ** all the expressions in pWInfo and try to transform matching expressions + ** into reference to index columns. Also attempt to translate references + ** to virtual columns in the table into references to (stored) columns + ** of the index. + ** + ** Do not do this for the RHS of a LEFT JOIN. This is because the + ** expression may be evaluated after OP_NullRow has been executed on + ** the cursor. In this case it is important to do the full evaluation, + ** as the result of the expression may not be NULL, even if all table + ** column values are. https://www.sqlite.org/src/info/7fa8049685b50b5a + ** + ** Also, do not do this when processing one index an a multi-index + ** OR clause, since the transformation will become invalid once we + ** move forward to the next index. + ** https://sqlite.org/src/info/4e8e4857d32d401f + */ + if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){ + whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo); + } + + /* If a partial index is driving the loop, try to eliminate WHERE clause + ** terms from the query that must be true due to the WHERE clause of + ** the partial index. + ** + ** 2019-11-02 ticket 623eff57e76d45f6: This optimization does not work + ** for a LEFT JOIN. + */ + if( pIdx->pPartIdxWhere ){ + whereApplyPartialIndexConstraints(pIdx->pPartIdxWhere, iCur, pWC); + } + }else{ + testcase( pIdx->pPartIdxWhere ); + /* The following assert() is not a requirement, merely an observation: + ** The OR-optimization doesn't work for the right hand table of + ** a LEFT JOIN: */ + assert( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ); + } + + /* Record the instruction used to terminate the loop. */ + if( pLoop->wsFlags & WHERE_ONEROW ){ + pLevel->op = OP_Noop; + }else if( bRev ){ + pLevel->op = OP_Prev; + }else{ + pLevel->op = OP_Next; + } + pLevel->p1 = iIdxCur; + pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; + if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + }else{ + assert( pLevel->p5==0 ); + } + if( omitTable ) pIdx = 0; + }else + +#ifndef SQLITE_OMIT_OR_OPTIMIZATION + if( pLoop->wsFlags & WHERE_MULTI_OR ){ + /* Case 5: Two or more separately indexed terms connected by OR + ** + ** Example: + ** + ** CREATE TABLE t1(a,b,c,d); + ** CREATE INDEX i1 ON t1(a); + ** CREATE INDEX i2 ON t1(b); + ** CREATE INDEX i3 ON t1(c); + ** + ** SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13) + ** + ** In the example, there are three indexed terms connected by OR. + ** The top of the loop looks like this: + ** + ** Null 1 # Zero the rowset in reg 1 + ** + ** Then, for each indexed term, the following. The arguments to + ** RowSetTest are such that the rowid of the current row is inserted + ** into the RowSet. If it is already present, control skips the + ** Gosub opcode and jumps straight to the code generated by WhereEnd(). + ** + ** sqlite3WhereBegin() + ** RowSetTest # Insert rowid into rowset + ** Gosub 2 A + ** sqlite3WhereEnd() + ** + ** Following the above, code to terminate the loop. Label A, the target + ** of the Gosub above, jumps to the instruction right after the Goto. + ** + ** Null 1 # Zero the rowset in reg 1 + ** Goto B # The loop is finished. + ** + ** A: # Return data, whatever. + ** + ** Return 2 # Jump back to the Gosub + ** + ** B: + ** + ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then + ** use an ephemeral index instead of a RowSet to record the primary + ** keys of the rows we have already seen. + ** + */ + WhereClause *pOrWc; /* The OR-clause broken out into subterms */ + SrcList *pOrTab; /* Shortened table list or OR-clause generation */ + Index *pCov = 0; /* Potential covering index (or NULL) */ + int iCovCur = pParse->nTab++; /* Cursor used for index scans (if any) */ + + int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ + int regRowset = 0; /* Register for RowSet object */ + int regRowid = 0; /* Register holding rowid */ + int iLoopBody = sqlite3VdbeMakeLabel(pParse);/* Start of loop body */ + int iRetInit; /* Address of regReturn init */ + int untestedTerms = 0; /* Some terms not completely tested */ + int ii; /* Loop counter */ + Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ + Table *pTab = pTabItem->pTab; + + pTerm = pLoop->aLTerm[0]; + assert( pTerm!=0 ); + assert( pTerm->eOperator & WO_OR ); + assert( (pTerm->wtFlags & TERM_ORINFO)!=0 ); + pOrWc = &pTerm->u.pOrInfo->wc; + pLevel->op = OP_Return; + pLevel->p1 = regReturn; + + /* Set up a new SrcList in pOrTab containing the table being scanned + ** by this loop in the a[0] slot and all notReady tables in a[1..] slots. + ** This becomes the SrcList in the recursive call to sqlite3WhereBegin(). + */ + if( pWInfo->nLevel>1 ){ + int nNotReady; /* The number of notReady tables */ + SrcItem *origSrc; /* Original list of tables */ + nNotReady = pWInfo->nLevel - iLevel - 1; + pOrTab = sqlite3StackAllocRaw(db, + sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); + if( pOrTab==0 ) return notReady; + pOrTab->nAlloc = (u8)(nNotReady + 1); + pOrTab->nSrc = pOrTab->nAlloc; + memcpy(pOrTab->a, pTabItem, sizeof(*pTabItem)); + origSrc = pWInfo->pTabList->a; + for(k=1; k<=nNotReady; k++){ + memcpy(&pOrTab->a[k], &origSrc[pLevel[k].iFrom], sizeof(pOrTab->a[k])); + } + }else{ + pOrTab = pWInfo->pTabList; + } + + /* Initialize the rowset register to contain NULL. An SQL NULL is + ** equivalent to an empty rowset. Or, create an ephemeral index + ** capable of holding primary keys in the case of a WITHOUT ROWID. + ** + ** Also initialize regReturn to contain the address of the instruction + ** immediately following the OP_Return at the bottom of the loop. This + ** is required in a few obscure LEFT JOIN cases where control jumps + ** over the top of the loop into the body of it. In this case the + ** correct response for the end-of-loop code (the OP_Return) is to + ** fall through to the next instruction, just as an OP_Next does if + ** called on an uninitialized cursor. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + if( HasRowid(pTab) ){ + regRowset = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, regRowset); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + regRowset = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, regRowset, pPk->nKeyCol); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } + regRowid = ++pParse->nMem; + } + iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); + + /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y + ** Then for every term xN, evaluate as the subexpression: xN AND z + ** That way, terms in y that are factored into the disjunction will + ** be picked up by the recursive calls to sqlite3WhereBegin() below. + ** + ** Actually, each subexpression is converted to "xN AND w" where w is + ** the "interesting" terms of z - terms that did not originate in the + ** ON or USING clause of a LEFT JOIN, and terms that are usable as + ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 + */ + if( pWC->nTerm>1 ){ + int iTerm; + for(iTerm=0; iTermnTerm; iTerm++){ + Expr *pExpr = pWC->a[iTerm].pExpr; + if( &pWC->a[iTerm] == pTerm ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); + testcase( pWC->a[iTerm].wtFlags & TERM_CODED ); + if( (pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_CODED))!=0 ) continue; + if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); + pExpr = sqlite3ExprDup(db, pExpr, 0); + pAndExpr = sqlite3ExprAnd(pParse, pAndExpr, pExpr); + } + if( pAndExpr ){ + /* The extra 0x10000 bit on the opcode is masked off and does not + ** become part of the new Expr.op. However, it does make the + ** op==TK_AND comparison inside of sqlite3PExpr() false, and this + ** prevents sqlite3PExpr() from applying the AND short-circuit + ** optimization, which we do not want here. */ + pAndExpr = sqlite3PExpr(pParse, TK_AND|0x10000, 0, pAndExpr); + } + } + + /* Run a separate WHERE clause for each term of the OR clause. After + ** eliminating duplicates from other WHERE clauses, the action for each + ** sub-WHERE clause is to to invoke the main loop body as a subroutine. + */ + ExplainQueryPlan((pParse, 1, "MULTI-INDEX OR")); + for(ii=0; iinTerm; ii++){ + WhereTerm *pOrTerm = &pOrWc->a[ii]; + if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ + WhereInfo *pSubWInfo; /* Info for single OR-term scan */ + Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ + Expr *pDelete; /* Local copy of OR clause term */ + int jmp1 = 0; /* Address of jump operation */ + testcase( (pTabItem[0].fg.jointype & JT_LEFT)!=0 + && !ExprHasProperty(pOrExpr, EP_FromJoin) + ); /* See TH3 vtab25.400 and ticket 614b25314c766238 */ + pDelete = pOrExpr = sqlite3ExprDup(db, pOrExpr, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDelete); + continue; + } + if( pAndExpr ){ + pAndExpr->pLeft = pOrExpr; + pOrExpr = pAndExpr; + } + /* Loop through table entries that match term pOrTerm. */ + ExplainQueryPlan((pParse, 1, "INDEX %d", ii+1)); + WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, + WHERE_OR_SUBCLAUSE, iCovCur); + assert( pSubWInfo || pParse->nErr || db->mallocFailed ); + if( pSubWInfo ){ + WhereLoop *pSubLoop; + int addrExplain = sqlite3WhereExplainOneScan( + pParse, pOrTab, &pSubWInfo->a[0], 0 + ); + sqlite3WhereAddScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain); + + /* This is the sub-WHERE clause body. First skip over + ** duplicate rows from prior sub-WHERE clauses, and record the + ** rowid (or PRIMARY KEY) for the current row so that the same + ** row will be skipped in subsequent sub-WHERE clauses. + */ + if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ + int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); + if( HasRowid(pTab) ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, -1, regRowid); + jmp1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, + regRowid, iSet); + VdbeCoverage(v); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + int nPk = pPk->nKeyCol; + int iPk; + int r; + + /* Read the PK into an array of temp registers. */ + r = sqlite3GetTempRange(pParse, nPk); + for(iPk=0; iPkaiColumn[iPk]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, iCol,r+iPk); + } + + /* Check if the temp table already contains this key. If so, + ** the row has already been included in the result set and + ** can be ignored (by jumping past the Gosub below). Otherwise, + ** insert the key into the temp table and proceed with processing + ** the row. + ** + ** Use some of the same optimizations as OP_RowSetTest: If iSet + ** is zero, assume that the key cannot already be present in + ** the temp table. And if iSet is -1, assume that there is no + ** need to insert the key into the temp table, as it will never + ** be tested for. */ + if( iSet ){ + jmp1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); + VdbeCoverage(v); + } + if( iSet>=0 ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, regRowset, regRowid, + r, nPk); + if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + } + + /* Release the array of temp registers */ + sqlite3ReleaseTempRange(pParse, r, nPk); + } + } + + /* Invoke the main loop body as a subroutine */ + sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); + + /* Jump here (skipping the main loop body subroutine) if the + ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ + if( jmp1 ) sqlite3VdbeJumpHere(v, jmp1); + + /* The pSubWInfo->untestedTerms flag means that this OR term + ** contained one or more AND term from a notReady table. The + ** terms from the notReady table could not be tested and will + ** need to be tested later. + */ + if( pSubWInfo->untestedTerms ) untestedTerms = 1; + + /* If all of the OR-connected terms are optimized using the same + ** index, and the index is opened using the same cursor number + ** by each call to sqlite3WhereBegin() made by this loop, it may + ** be possible to use that index as a covering index. + ** + ** If the call to sqlite3WhereBegin() above resulted in a scan that + ** uses an index, and this is either the first OR-connected term + ** processed or the index is the same as that used by all previous + ** terms, set pCov to the candidate covering index. Otherwise, set + ** pCov to NULL to indicate that no candidate covering index will + ** be available. + */ + pSubLoop = pSubWInfo->a[0].pWLoop; + assert( (pSubLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); + if( (pSubLoop->wsFlags & WHERE_INDEXED)!=0 + && (ii==0 || pSubLoop->u.btree.pIndex==pCov) + && (HasRowid(pTab) || !IsPrimaryKeyIndex(pSubLoop->u.btree.pIndex)) + ){ + assert( pSubWInfo->a[0].iIdxCur==iCovCur ); + pCov = pSubLoop->u.btree.pIndex; + }else{ + pCov = 0; + } + if( sqlite3WhereUsesDeferredSeek(pSubWInfo) ){ + pWInfo->bDeferredSeek = 1; + } + + /* Finish the loop through table entries that match term pOrTerm. */ + sqlite3WhereEnd(pSubWInfo); + ExplainQueryPlanPop(pParse); + } + sqlite3ExprDelete(db, pDelete); + } + } + ExplainQueryPlanPop(pParse); + pLevel->u.pCovidx = pCov; + if( pCov ) pLevel->iIdxCur = iCovCur; + if( pAndExpr ){ + pAndExpr->pLeft = 0; + sqlite3ExprDelete(db, pAndExpr); + } + sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v)); + sqlite3VdbeGoto(v, pLevel->addrBrk); + sqlite3VdbeResolveLabel(v, iLoopBody); + + if( pWInfo->nLevel>1 ){ sqlite3StackFree(db, pOrTab); } + if( !untestedTerms ) disableTerm(pLevel, pTerm); + }else +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + + { + /* Case 6: There is no usable index. We must do a complete + ** scan of the entire table. + */ + static const u8 aStep[] = { OP_Next, OP_Prev }; + static const u8 aStart[] = { OP_Rewind, OP_Last }; + assert( bRev==0 || bRev==1 ); + if( pTabItem->fg.isRecursive ){ + /* Tables marked isRecursive have only a single row that is stored in + ** a pseudo-cursor. No need to Rewind or Next such cursors. */ + pLevel->op = OP_Noop; + }else{ + codeCursorHint(pTabItem, pWInfo, pLevel, 0); + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrHalt); + VdbeCoverageIf(v, bRev==0); + VdbeCoverageIf(v, bRev!=0); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + } + } + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pLevel->addrVisit = sqlite3VdbeCurrentAddr(v); +#endif + + /* Insert code to test every subexpression that can be completely + ** computed using the current set of tables. + ** + ** This loop may run between one and three times, depending on the + ** constraints to be generated. The value of stack variable iLoop + ** determines the constraints coded by each iteration, as follows: + ** + ** iLoop==1: Code only expressions that are entirely covered by pIdx. + ** iLoop==2: Code remaining expressions that do not contain correlated + ** sub-queries. + ** iLoop==3: Code all remaining expressions. + ** + ** An effort is made to skip unnecessary iterations of the loop. + */ + iLoop = (pIdx ? 1 : 2); + do{ + int iNext = 0; /* Next value for iLoop */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + int skipLikeAddr = 0; + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + testcase( pWInfo->untestedTerms==0 + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ); + pWInfo->untestedTerms = 1; + continue; + } + pE = pTerm->pExpr; + assert( pE!=0 ); + if( (pTabItem->fg.jointype&JT_LEFT) && !ExprHasProperty(pE,EP_FromJoin) ){ + continue; + } + + if( iLoop==1 && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){ + iNext = 2; + continue; + } + if( iLoop<3 && (pTerm->wtFlags & TERM_VARSELECT) ){ + if( iNext==0 ) iNext = 3; + continue; + } + + if( (pTerm->wtFlags & TERM_LIKECOND)!=0 ){ + /* If the TERM_LIKECOND flag is set, that means that the range search + ** is sufficient to guarantee that the LIKE operator is true, so we + ** can skip the call to the like(A,B) function. But this only works + ** for strings. So do not skip the call to the function on the pass + ** that compares BLOBs. */ +#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS + continue; +#else + u32 x = pLevel->iLikeRepCntr; + if( x>0 ){ + skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)?OP_IfNot:OP_If,(int)(x>>1)); + VdbeCoverageIf(v, (x&1)==1); + VdbeCoverageIf(v, (x&1)==0); + } +#endif + } +#ifdef WHERETRACE_ENABLED /* 0xffff */ + if( sqlite3WhereTrace ){ + VdbeNoopComment((v, "WhereTerm[%d] (%p) priority=%d", + pWC->nTerm-j, pTerm, iLoop)); + } + if( sqlite3WhereTrace & 0x800 ){ + sqlite3DebugPrintf("Coding auxiliary constraint:\n"); + sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); + } +#endif + sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); + if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); + pTerm->wtFlags |= TERM_CODED; + } + iLoop = iNext; + }while( iLoop>0 ); + + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE, sEAlt; + WhereTerm *pAlt; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) continue; + if( (pTerm->eOperator & WO_EQUIV)==0 ) continue; + if( pTerm->leftCursor!=iCur ) continue; + if( pTabItem->fg.jointype & JT_LEFT ) continue; + pE = pTerm->pExpr; +#ifdef WHERETRACE_ENABLED /* 0x800 */ + if( sqlite3WhereTrace & 0x800 ){ + sqlite3DebugPrintf("Coding transitive constraint:\n"); + sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); + } +#endif + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); + pAlt = sqlite3WhereFindTerm(pWC, iCur, pTerm->u.x.leftColumn, notReady, + WO_EQ|WO_IN|WO_IS, 0); + if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; + if( (pAlt->eOperator & WO_IN) + && (pAlt->pExpr->flags & EP_xIsSelect) + && (pAlt->pExpr->x.pSelect->pEList->nExpr>1) + ){ + continue; + } + testcase( pAlt->eOperator & WO_EQ ); + testcase( pAlt->eOperator & WO_IS ); + testcase( pAlt->eOperator & WO_IN ); + VdbeModuleComment((v, "begin transitive constraint")); + sEAlt = *pAlt->pExpr; + sEAlt.pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, &sEAlt, addrCont, SQLITE_JUMPIFNULL); + pAlt->wtFlags |= TERM_CODED; + } + + /* For a LEFT OUTER JOIN, generate code that will record the fact that + ** at least one row of the right table has matched the left table. + */ + if( pLevel->iLeftJoin ){ + pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); + VdbeComment((v, "record LEFT JOIN hit")); + for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + assert( pWInfo->untestedTerms ); + continue; + } + assert( pTerm->pExpr ); + sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); + pTerm->wtFlags |= TERM_CODED; + } + } + +#if WHERETRACE_ENABLED /* 0x20800 */ + if( sqlite3WhereTrace & 0x20000 ){ + sqlite3DebugPrintf("All WHERE-clause terms after coding level %d:\n", + iLevel); + sqlite3WhereClausePrint(pWC); + } + if( sqlite3WhereTrace & 0x800 ){ + sqlite3DebugPrintf("End Coding level %d: notReady=%llx\n", + iLevel, (u64)pLevel->notReady); + } +#endif + return pLevel->notReady; +} + +/************** End of wherecode.c *******************************************/ +/************** Begin file whereexpr.c ***************************************/ +/* +** 2015-06-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This module contains C code that generates VDBE code used to process +** the WHERE clause of SQL statements. +** +** This file was originally part of where.c but was split out to improve +** readability and editabiliity. This file contains utility routines for +** analyzing Expr objects in the WHERE clause. +*/ +/* #include "sqliteInt.h" */ +/* #include "whereInt.h" */ + +/* Forward declarations */ +static void exprAnalyze(SrcList*, WhereClause*, int); + +/* +** Deallocate all memory associated with a WhereOrInfo object. +*/ +static void whereOrInfoDelete(sqlite3 *db, WhereOrInfo *p){ + sqlite3WhereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} + +/* +** Deallocate all memory associated with a WhereAndInfo object. +*/ +static void whereAndInfoDelete(sqlite3 *db, WhereAndInfo *p){ + sqlite3WhereClauseClear(&p->wc); + sqlite3DbFree(db, p); +} + +/* +** Add a single new WhereTerm entry to the WhereClause object pWC. +** The new WhereTerm object is constructed from Expr p and with wtFlags. +** The index in pWC->a[] of the new WhereTerm is returned on success. +** 0 is returned if the new WhereTerm could not be added due to a memory +** allocation error. The memory allocation failure will be recorded in +** the db->mallocFailed flag so that higher-level functions can detect it. +** +** This routine will increase the size of the pWC->a[] array as necessary. +** +** If the wtFlags argument includes TERM_DYNAMIC, then responsibility +** for freeing the expression p is assumed by the WhereClause object pWC. +** This is true even if this routine fails to allocate a new WhereTerm. +** +** WARNING: This routine might reallocate the space used to store +** WhereTerms. All pointers to WhereTerms should be invalidated after +** calling this routine. Such pointers may be reinitialized by referencing +** the pWC->a[] array. +*/ +static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ + WhereTerm *pTerm; + int idx; + testcase( wtFlags & TERM_VIRTUAL ); + if( pWC->nTerm>=pWC->nSlot ){ + WhereTerm *pOld = pWC->a; + sqlite3 *db = pWC->pWInfo->pParse->db; + pWC->a = sqlite3DbMallocRawNN(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); + if( pWC->a==0 ){ + if( wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, p); + } + pWC->a = pOld; + return 0; + } + memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); + if( pOld!=pWC->aStatic ){ + sqlite3DbFree(db, pOld); + } + pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); + } + pTerm = &pWC->a[idx = pWC->nTerm++]; + if( p && ExprHasProperty(p, EP_Unlikely) ){ + pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; + }else{ + pTerm->truthProb = 1; + } + pTerm->pExpr = sqlite3ExprSkipCollateAndLikely(p); + pTerm->wtFlags = wtFlags; + pTerm->pWC = pWC; + pTerm->iParent = -1; + memset(&pTerm->eOperator, 0, + sizeof(WhereTerm) - offsetof(WhereTerm,eOperator)); + return idx; +} + +/* +** Return TRUE if the given operator is one of the operators that is +** allowed for an indexable WHERE clause term. The allowed operators are +** "=", "<", ">", "<=", ">=", "IN", "IS", and "IS NULL" +*/ +static int allowedOp(int op){ + assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL || op==TK_IS; +} + +/* +** Commute a comparison operator. Expressions of the form "X op Y" +** are converted into "Y op X". +*/ +static u16 exprCommute(Parse *pParse, Expr *pExpr){ + if( pExpr->pLeft->op==TK_VECTOR + || pExpr->pRight->op==TK_VECTOR + || sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight) != + sqlite3BinaryCompareCollSeq(pParse, pExpr->pRight, pExpr->pLeft) + ){ + pExpr->flags ^= EP_Commuted; + } + SWAP(Expr*,pExpr->pRight,pExpr->pLeft); + if( pExpr->op>=TK_GT ){ + assert( TK_LT==TK_GT+2 ); + assert( TK_GE==TK_LE+2 ); + assert( TK_GT>TK_EQ ); + assert( TK_GTop>=TK_GT && pExpr->op<=TK_GE ); + pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; + } + return 0; +} + +/* +** Translate from TK_xx operator to WO_xx bitmask. +*/ +static u16 operatorMask(int op){ + u16 c; + assert( allowedOp(op) ); + if( op==TK_IN ){ + c = WO_IN; + }else if( op==TK_ISNULL ){ + c = WO_ISNULL; + }else if( op==TK_IS ){ + c = WO_IS; + }else{ + assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); + c = (u16)(WO_EQ<<(op-TK_EQ)); + } + assert( op!=TK_ISNULL || c==WO_ISNULL ); + assert( op!=TK_IN || c==WO_IN ); + assert( op!=TK_EQ || c==WO_EQ ); + assert( op!=TK_LT || c==WO_LT ); + assert( op!=TK_LE || c==WO_LE ); + assert( op!=TK_GT || c==WO_GT ); + assert( op!=TK_GE || c==WO_GE ); + assert( op!=TK_IS || c==WO_IS ); + return c; +} + + +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION +/* +** Check to see if the given expression is a LIKE or GLOB operator that +** can be optimized using inequality constraints. Return TRUE if it is +** so and false if not. +** +** In order for the operator to be optimizible, the RHS must be a string +** literal that does not begin with a wildcard. The LHS must be a column +** that may only be NULL, a string, or a BLOB, never a number. (This means +** that virtual tables cannot participate in the LIKE optimization.) The +** collating sequence for the column on the LHS must be appropriate for +** the operator. +*/ +static int isLikeOrGlob( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* Test this expression */ + Expr **ppPrefix, /* Pointer to TK_STRING expression with pattern prefix */ + int *pisComplete, /* True if the only wildcard is % in the last character */ + int *pnoCase /* True if uppercase is equivalent to lowercase */ +){ + const u8 *z = 0; /* String on RHS of LIKE operator */ + Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ + ExprList *pList; /* List of operands to the LIKE operator */ + u8 c; /* One character in z[] */ + int cnt; /* Number of non-wildcard prefix characters */ + u8 wc[4]; /* Wildcard characters */ + sqlite3 *db = pParse->db; /* Database connection */ + sqlite3_value *pVal = 0; + int op; /* Opcode of pRight */ + int rc; /* Result code to return */ + + if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, (char*)wc) ){ + return 0; + } +#ifdef SQLITE_EBCDIC + if( *pnoCase ) return 0; +#endif + pList = pExpr->x.pList; + pLeft = pList->a[1].pExpr; + + pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); + op = pRight->op; + if( op==TK_VARIABLE && (db->flags & SQLITE_EnableQPSG)==0 ){ + Vdbe *pReprepare = pParse->pReprepare; + int iCol = pRight->iColumn; + pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_BLOB); + if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ + z = sqlite3_value_text(pVal); + } + sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); + assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); + }else if( op==TK_STRING ){ + z = (u8*)pRight->u.zToken; + } + if( z ){ + + /* Count the number of prefix characters prior to the first wildcard */ + cnt = 0; + while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ + cnt++; + if( c==wc[3] && z[cnt]!=0 ) cnt++; + } + + /* The optimization is possible only if (1) the pattern does not begin + ** with a wildcard and if (2) the non-wildcard prefix does not end with + ** an (illegal 0xff) character, or (3) the pattern does not consist of + ** a single escape character. The second condition is necessary so + ** that we can increment the prefix key to find an upper bound for the + ** range search. The third is because the caller assumes that the pattern + ** consists of at least one character after all escapes have been + ** removed. */ + if( cnt!=0 && 255!=(u8)z[cnt-1] && (cnt>1 || z[0]!=wc[3]) ){ + Expr *pPrefix; + + /* A "complete" match if the pattern ends with "*" or "%" */ + *pisComplete = c==wc[0] && z[cnt+1]==0; + + /* Get the pattern prefix. Remove all escapes from the prefix. */ + pPrefix = sqlite3Expr(db, TK_STRING, (char*)z); + if( pPrefix ){ + int iFrom, iTo; + char *zNew = pPrefix->u.zToken; + zNew[cnt] = 0; + for(iFrom=iTo=0; iFrom0 ); + + /* If the LHS is not an ordinary column with TEXT affinity, then the + ** pattern prefix boundaries (both the start and end boundaries) must + ** not look like a number. Otherwise the pattern might be treated as + ** a number, which will invalidate the LIKE optimization. + ** + ** Getting this right has been a persistent source of bugs in the + ** LIKE optimization. See, for example: + ** 2018-09-10 https://sqlite.org/src/info/c94369cae9b561b1 + ** 2019-05-02 https://sqlite.org/src/info/b043a54c3de54b28 + ** 2019-06-10 https://sqlite.org/src/info/fd76310a5e843e07 + ** 2019-06-14 https://sqlite.org/src/info/ce8717f0885af975 + ** 2019-09-03 https://sqlite.org/src/info/0f0428096f17252a + */ + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || (pLeft->y.pTab && IsVirtual(pLeft->y.pTab)) /* Might be numeric */ + ){ + int isNum; + double rDummy; + isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + if( isNum<=0 ){ + if( iTo==1 && zNew[0]=='-' ){ + isNum = +1; + }else{ + zNew[iTo-1]++; + isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + zNew[iTo-1]--; + } + } + if( isNum>0 ){ + sqlite3ExprDelete(db, pPrefix); + sqlite3ValueFree(pVal); + return 0; + } + } + } + *ppPrefix = pPrefix; + + /* If the RHS pattern is a bound parameter, make arrangements to + ** reprepare the statement when that parameter is rebound */ + if( op==TK_VARIABLE ){ + Vdbe *v = pParse->pVdbe; + sqlite3VdbeSetVarmask(v, pRight->iColumn); + if( *pisComplete && pRight->u.zToken[1] ){ + /* If the rhs of the LIKE expression is a variable, and the current + ** value of the variable means there is no need to invoke the LIKE + ** function, then no OP_Variable will be added to the program. + ** This causes problems for the sqlite3_bind_parameter_name() + ** API. To work around them, add a dummy OP_Variable here. + */ + int r1 = sqlite3GetTempReg(pParse); + sqlite3ExprCodeTarget(pParse, pRight, r1); + sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0); + sqlite3ReleaseTempReg(pParse, r1); + } + } + }else{ + z = 0; + } + } + + rc = (z!=0); + sqlite3ValueFree(pVal); + return rc; +} +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Check to see if the pExpr expression is a form that needs to be passed +** to the xBestIndex method of virtual tables. Forms of interest include: +** +** Expression Virtual Table Operator +** ----------------------- --------------------------------- +** 1. column MATCH expr SQLITE_INDEX_CONSTRAINT_MATCH +** 2. column GLOB expr SQLITE_INDEX_CONSTRAINT_GLOB +** 3. column LIKE expr SQLITE_INDEX_CONSTRAINT_LIKE +** 4. column REGEXP expr SQLITE_INDEX_CONSTRAINT_REGEXP +** 5. column != expr SQLITE_INDEX_CONSTRAINT_NE +** 6. expr != column SQLITE_INDEX_CONSTRAINT_NE +** 7. column IS NOT expr SQLITE_INDEX_CONSTRAINT_ISNOT +** 8. expr IS NOT column SQLITE_INDEX_CONSTRAINT_ISNOT +** 9. column IS NOT NULL SQLITE_INDEX_CONSTRAINT_ISNOTNULL +** +** In every case, "column" must be a column of a virtual table. If there +** is a match, set *ppLeft to the "column" expression, set *ppRight to the +** "expr" expression (even though in forms (6) and (8) the column is on the +** right and the expression is on the left). Also set *peOp2 to the +** appropriate virtual table operator. The return value is 1 or 2 if there +** is a match. The usual return is 1, but if the RHS is also a column +** of virtual table in forms (5) or (7) then return 2. +** +** If the expression matches none of the patterns above, return 0. +*/ +static int isAuxiliaryVtabOperator( + sqlite3 *db, /* Parsing context */ + Expr *pExpr, /* Test this expression */ + unsigned char *peOp2, /* OUT: 0 for MATCH, or else an op2 value */ + Expr **ppLeft, /* Column expression to left of MATCH/op2 */ + Expr **ppRight /* Expression to left of MATCH/op2 */ +){ + if( pExpr->op==TK_FUNCTION ){ + static const struct Op2 { + const char *zOp; + unsigned char eOp2; + } aOp[] = { + { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, + { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, + { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, + { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } + }; + ExprList *pList; + Expr *pCol; /* Column reference */ + int i; + + pList = pExpr->x.pList; + if( pList==0 || pList->nExpr!=2 ){ + return 0; + } + + /* Built-in operators MATCH, GLOB, LIKE, and REGEXP attach to a + ** virtual table on their second argument, which is the same as + ** the left-hand side operand in their in-fix form. + ** + ** vtab_column MATCH expression + ** MATCH(expression,vtab_column) + */ + pCol = pList->a[1].pExpr; + testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + if( ExprIsVtab(pCol) ){ + for(i=0; iu.zToken, aOp[i].zOp)==0 ){ + *peOp2 = aOp[i].eOp2; + *ppRight = pList->a[0].pExpr; + *ppLeft = pCol; + return 1; + } + } + } + + /* We can also match against the first column of overloaded + ** functions where xFindFunction returns a value of at least + ** SQLITE_INDEX_CONSTRAINT_FUNCTION. + ** + ** OVERLOADED(vtab_column,expression) + ** + ** Historically, xFindFunction expected to see lower-case function + ** names. But for this use case, xFindFunction is expected to deal + ** with function names in an arbitrary case. + */ + pCol = pList->a[0].pExpr; + testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + if( ExprIsVtab(pCol) ){ + sqlite3_vtab *pVtab; + sqlite3_module *pMod; + void (*xNotUsed)(sqlite3_context*,int,sqlite3_value**); + void *pNotUsed; + pVtab = sqlite3GetVTable(db, pCol->y.pTab)->pVtab; + assert( pVtab!=0 ); + assert( pVtab->pModule!=0 ); + pMod = (sqlite3_module *)pVtab->pModule; + if( pMod->xFindFunction!=0 ){ + i = pMod->xFindFunction(pVtab,2, pExpr->u.zToken, &xNotUsed, &pNotUsed); + if( i>=SQLITE_INDEX_CONSTRAINT_FUNCTION ){ + *peOp2 = i; + *ppRight = pList->a[1].pExpr; + *ppLeft = pCol; + return 1; + } + } + } + }else if( pExpr->op==TK_NE || pExpr->op==TK_ISNOT || pExpr->op==TK_NOTNULL ){ + int res = 0; + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + testcase( pLeft->op==TK_COLUMN && pLeft->y.pTab==0 ); + if( ExprIsVtab(pLeft) ){ + res++; + } + testcase( pRight && pRight->op==TK_COLUMN && pRight->y.pTab==0 ); + if( pRight && ExprIsVtab(pRight) ){ + res++; + SWAP(Expr*, pLeft, pRight); + } + *ppLeft = pLeft; + *ppRight = pRight; + if( pExpr->op==TK_NE ) *peOp2 = SQLITE_INDEX_CONSTRAINT_NE; + if( pExpr->op==TK_ISNOT ) *peOp2 = SQLITE_INDEX_CONSTRAINT_ISNOT; + if( pExpr->op==TK_NOTNULL ) *peOp2 = SQLITE_INDEX_CONSTRAINT_ISNOTNULL; + return res; + } + return 0; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** If the pBase expression originated in the ON or USING clause of +** a join, then transfer the appropriate markings over to derived. +*/ +static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ + if( pDerived ){ + pDerived->flags |= pBase->flags & EP_FromJoin; + pDerived->iRightJoinTable = pBase->iRightJoinTable; + } +} + +/* +** Mark term iChild as being a child of term iParent +*/ +static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ + pWC->a[iChild].iParent = iParent; + pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; + pWC->a[iParent].nChild++; +} + +/* +** Return the N-th AND-connected subterm of pTerm. Or if pTerm is not +** a conjunction, then return just pTerm when N==0. If N is exceeds +** the number of available subterms, return NULL. +*/ +static WhereTerm *whereNthSubterm(WhereTerm *pTerm, int N){ + if( pTerm->eOperator!=WO_AND ){ + return N==0 ? pTerm : 0; + } + if( Nu.pAndInfo->wc.nTerm ){ + return &pTerm->u.pAndInfo->wc.a[N]; + } + return 0; +} + +/* +** Subterms pOne and pTwo are contained within WHERE clause pWC. The +** two subterms are in disjunction - they are OR-ed together. +** +** If these two terms are both of the form: "A op B" with the same +** A and B values but different operators and if the operators are +** compatible (if one is = and the other is <, for example) then +** add a new virtual AND term to pWC that is the combination of the +** two. +** +** Some examples: +** +** x x<=y +** x=y OR x=y --> x=y +** x<=y OR x x<=y +** +** The following is NOT generated: +** +** xy --> x!=y +*/ +static void whereCombineDisjuncts( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* The complete WHERE clause */ + WhereTerm *pOne, /* First disjunct */ + WhereTerm *pTwo /* Second disjunct */ +){ + u16 eOp = pOne->eOperator | pTwo->eOperator; + sqlite3 *db; /* Database connection (for malloc) */ + Expr *pNew; /* New virtual expression */ + int op; /* Operator for the combined expression */ + int idxNew; /* Index in pWC of the next virtual term */ + + if( (pOne->wtFlags | pTwo->wtFlags) & TERM_VNULL ) return; + if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; + if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; + if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp + && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; + assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); + assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); + if( sqlite3ExprCompare(0,pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; + if( sqlite3ExprCompare(0,pOne->pExpr->pRight, pTwo->pExpr->pRight,-1) )return; + /* If we reach this point, it means the two subterms can be combined */ + if( (eOp & (eOp-1))!=0 ){ + if( eOp & (WO_LT|WO_LE) ){ + eOp = WO_LE; + }else{ + assert( eOp & (WO_GT|WO_GE) ); + eOp = WO_GE; + } + } + db = pWC->pWInfo->pParse->db; + pNew = sqlite3ExprDup(db, pOne->pExpr, 0); + if( pNew==0 ) return; + for(op=TK_EQ; eOp!=(WO_EQ<<(op-TK_EQ)); op++){ assert( opop = op; + idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); + exprAnalyze(pSrc, pWC, idxNew); +} + +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) +/* +** Analyze a term that consists of two or more OR-connected +** subterms. So in: +** +** ... WHERE (a=5) AND (b=7 OR c=9 OR d=13) AND (d=13) +** ^^^^^^^^^^^^^^^^^^^^ +** +** This routine analyzes terms such as the middle term in the above example. +** A WhereOrTerm object is computed and attached to the term under +** analysis, regardless of the outcome of the analysis. Hence: +** +** WhereTerm.wtFlags |= TERM_ORINFO +** WhereTerm.u.pOrInfo = a dynamically allocated WhereOrTerm object +** +** The term being analyzed must have two or more of OR-connected subterms. +** A single subterm might be a set of AND-connected sub-subterms. +** Examples of terms under analysis: +** +** (A) t1.x=t2.y OR t1.x=t2.z OR t1.y=15 OR t1.z=t3.a+5 +** (B) x=expr1 OR expr2=x OR x=expr3 +** (C) t1.x=t2.y OR (t1.x=t2.z AND t1.y=15) +** (D) x=expr1 OR (y>11 AND y<22 AND z LIKE '*hello*') +** (E) (p.a=1 AND q.b=2 AND r.c=3) OR (p.x=4 AND q.y=5 AND r.z=6) +** (F) x>A OR (x=A AND y>=B) +** +** CASE 1: +** +** If all subterms are of the form T.C=expr for some single column of C and +** a single table T (as shown in example B above) then create a new virtual +** term that is an equivalent IN expression. In other words, if the term +** being analyzed is: +** +** x = expr1 OR expr2 = x OR x = expr3 +** +** then create a new virtual term like this: +** +** x IN (expr1,expr2,expr3) +** +** CASE 2: +** +** If there are exactly two disjuncts and one side has x>A and the other side +** has x=A (for the same x and A) then add a new virtual conjunct term to the +** WHERE clause of the form "x>=A". Example: +** +** x>A OR (x=A AND y>B) adds: x>=A +** +** The added conjunct can sometimes be helpful in query planning. +** +** CASE 3: +** +** If all subterms are indexable by a single table T, then set +** +** WhereTerm.eOperator = WO_OR +** WhereTerm.u.pOrInfo->indexable |= the cursor number for table T +** +** A subterm is "indexable" if it is of the form +** "T.C " where C is any column of table T and +** is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN". +** A subterm is also indexable if it is an AND of two or more +** subsubterms at least one of which is indexable. Indexable AND +** subterms have their eOperator set to WO_AND and they have +** u.pAndInfo set to a dynamically allocated WhereAndTerm object. +** +** From another point of view, "indexable" means that the subterm could +** potentially be used with an index if an appropriate index exists. +** This analysis does not consider whether or not the index exists; that +** is decided elsewhere. This analysis only looks at whether subterms +** appropriate for indexing exist. +** +** All examples A through E above satisfy case 3. But if a term +** also satisfies case 1 (such as B) we know that the optimizer will +** always prefer case 1, so in that case we pretend that case 3 is not +** satisfied. +** +** It might be the case that multiple tables are indexable. For example, +** (E) above is indexable on tables P, Q, and R. +** +** Terms that satisfy case 3 are candidates for lookup by using +** separate indices to find rowids for each subterm and composing +** the union of all rowids using a RowSet object. This is similar +** to "bitmap indices" in other database engines. +** +** OTHERWISE: +** +** If none of cases 1, 2, or 3 apply, then leave the eOperator set to +** zero. This term is not useful for search. +*/ +static void exprAnalyzeOrTerm( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the complete WHERE clause */ + int idxTerm /* Index of the OR-term to be analyzed */ +){ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + Parse *pParse = pWInfo->pParse; /* Parser context */ + sqlite3 *db = pParse->db; /* Database connection */ + WhereTerm *pTerm = &pWC->a[idxTerm]; /* The term to be analyzed */ + Expr *pExpr = pTerm->pExpr; /* The expression of the term */ + int i; /* Loop counters */ + WhereClause *pOrWc; /* Breakup of pTerm into subterms */ + WhereTerm *pOrTerm; /* A Sub-term within the pOrWc */ + WhereOrInfo *pOrInfo; /* Additional information associated with pTerm */ + Bitmask chngToIN; /* Tables that might satisfy case 1 */ + Bitmask indexable; /* Tables that are indexable, satisfying case 2 */ + + /* + ** Break the OR clause into its separate subterms. The subterms are + ** stored in a WhereClause structure containing within the WhereOrInfo + ** object that is attached to the original OR clause term. + */ + assert( (pTerm->wtFlags & (TERM_DYNAMIC|TERM_ORINFO|TERM_ANDINFO))==0 ); + assert( pExpr->op==TK_OR ); + pTerm->u.pOrInfo = pOrInfo = sqlite3DbMallocZero(db, sizeof(*pOrInfo)); + if( pOrInfo==0 ) return; + pTerm->wtFlags |= TERM_ORINFO; + pOrWc = &pOrInfo->wc; + memset(pOrWc->aStatic, 0, sizeof(pOrWc->aStatic)); + sqlite3WhereClauseInit(pOrWc, pWInfo); + sqlite3WhereSplit(pOrWc, pExpr, TK_OR); + sqlite3WhereExprAnalyze(pSrc, pOrWc); + if( db->mallocFailed ) return; + assert( pOrWc->nTerm>=2 ); + + /* + ** Compute the set of tables that might satisfy cases 1 or 3. + */ + indexable = ~(Bitmask)0; + chngToIN = ~(Bitmask)0; + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){ + if( (pOrTerm->eOperator & WO_SINGLE)==0 ){ + WhereAndInfo *pAndInfo; + assert( (pOrTerm->wtFlags & (TERM_ANDINFO|TERM_ORINFO))==0 ); + chngToIN = 0; + pAndInfo = sqlite3DbMallocRawNN(db, sizeof(*pAndInfo)); + if( pAndInfo ){ + WhereClause *pAndWC; + WhereTerm *pAndTerm; + int j; + Bitmask b = 0; + pOrTerm->u.pAndInfo = pAndInfo; + pOrTerm->wtFlags |= TERM_ANDINFO; + pOrTerm->eOperator = WO_AND; + pAndWC = &pAndInfo->wc; + memset(pAndWC->aStatic, 0, sizeof(pAndWC->aStatic)); + sqlite3WhereClauseInit(pAndWC, pWC->pWInfo); + sqlite3WhereSplit(pAndWC, pOrTerm->pExpr, TK_AND); + sqlite3WhereExprAnalyze(pSrc, pAndWC); + pAndWC->pOuter = pWC; + if( !db->mallocFailed ){ + for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ + assert( pAndTerm->pExpr ); + if( allowedOp(pAndTerm->pExpr->op) + || pAndTerm->eOperator==WO_AUX + ){ + b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); + } + } + } + indexable &= b; + } + }else if( pOrTerm->wtFlags & TERM_COPIED ){ + /* Skip this term for now. We revisit it when we process the + ** corresponding TERM_VIRTUAL term */ + }else{ + Bitmask b; + b = sqlite3WhereGetMask(&pWInfo->sMaskSet, pOrTerm->leftCursor); + if( pOrTerm->wtFlags & TERM_VIRTUAL ){ + WhereTerm *pOther = &pOrWc->a[pOrTerm->iParent]; + b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pOther->leftCursor); + } + indexable &= b; + if( (pOrTerm->eOperator & WO_EQ)==0 ){ + chngToIN = 0; + }else{ + chngToIN &= b; + } + } + } + + /* + ** Record the set of tables that satisfy case 3. The set might be + ** empty. + */ + pOrInfo->indexable = indexable; + if( indexable ){ + pTerm->eOperator = WO_OR; + pWC->hasOr = 1; + }else{ + pTerm->eOperator = WO_OR; + } + + /* For a two-way OR, attempt to implementation case 2. + */ + if( indexable && pOrWc->nTerm==2 ){ + int iOne = 0; + WhereTerm *pOne; + while( (pOne = whereNthSubterm(&pOrWc->a[0],iOne++))!=0 ){ + int iTwo = 0; + WhereTerm *pTwo; + while( (pTwo = whereNthSubterm(&pOrWc->a[1],iTwo++))!=0 ){ + whereCombineDisjuncts(pSrc, pWC, pOne, pTwo); + } + } + } + + /* + ** chngToIN holds a set of tables that *might* satisfy case 1. But + ** we have to do some additional checking to see if case 1 really + ** is satisfied. + ** + ** chngToIN will hold either 0, 1, or 2 bits. The 0-bit case means + ** that there is no possibility of transforming the OR clause into an + ** IN operator because one or more terms in the OR clause contain + ** something other than == on a column in the single table. The 1-bit + ** case means that every term of the OR clause is of the form + ** "table.column=expr" for some single table. The one bit that is set + ** will correspond to the common table. We still need to check to make + ** sure the same column is used on all terms. The 2-bit case is when + ** the all terms are of the form "table1.column=table2.column". It + ** might be possible to form an IN operator with either table1.column + ** or table2.column as the LHS if either is common to every term of + ** the OR clause. + ** + ** Note that terms of the form "table.column1=table.column2" (the + ** same table on both sizes of the ==) cannot be optimized. + */ + if( chngToIN ){ + int okToChngToIN = 0; /* True if the conversion to IN is valid */ + int iColumn = -1; /* Column index on lhs of IN operator */ + int iCursor = -1; /* Table cursor common to all terms */ + int j = 0; /* Loop counter */ + + /* Search for a table and column that appears on one side or the + ** other of the == operator in every subterm. That table and column + ** will be recorded in iCursor and iColumn. There might not be any + ** such table and column. Set okToChngToIN if an appropriate table + ** and column is found but leave okToChngToIN false if not found. + */ + for(j=0; j<2 && !okToChngToIN; j++){ + Expr *pLeft = 0; + pOrTerm = pOrWc->a; + for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ + assert( pOrTerm->eOperator & WO_EQ ); + pOrTerm->wtFlags &= ~TERM_OR_OK; + if( pOrTerm->leftCursor==iCursor ){ + /* This is the 2-bit case and we are on the second iteration and + ** current term is from the first iteration. So skip this term. */ + assert( j==1 ); + continue; + } + if( (chngToIN & sqlite3WhereGetMask(&pWInfo->sMaskSet, + pOrTerm->leftCursor))==0 ){ + /* This term must be of the form t1.a==t2.b where t2 is in the + ** chngToIN set but t1 is not. This term will be either preceded + ** or follwed by an inverted copy (t2.b==t1.a). Skip this term + ** and use its inversion. */ + testcase( pOrTerm->wtFlags & TERM_COPIED ); + testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); + assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) ); + continue; + } + iColumn = pOrTerm->u.x.leftColumn; + iCursor = pOrTerm->leftCursor; + pLeft = pOrTerm->pExpr->pLeft; + break; + } + if( i<0 ){ + /* No candidate table+column was found. This can only occur + ** on the second iteration */ + assert( j==1 ); + assert( IsPowerOfTwo(chngToIN) ); + assert( chngToIN==sqlite3WhereGetMask(&pWInfo->sMaskSet, iCursor) ); + break; + } + testcase( j==1 ); + + /* We have found a candidate table and column. Check to see if that + ** table and column is common to every term in the OR clause */ + okToChngToIN = 1; + for(; i>=0 && okToChngToIN; i--, pOrTerm++){ + assert( pOrTerm->eOperator & WO_EQ ); + if( pOrTerm->leftCursor!=iCursor ){ + pOrTerm->wtFlags &= ~TERM_OR_OK; + }else if( pOrTerm->u.x.leftColumn!=iColumn || (iColumn==XN_EXPR + && sqlite3ExprCompare(pParse, pOrTerm->pExpr->pLeft, pLeft, -1) + )){ + okToChngToIN = 0; + }else{ + int affLeft, affRight; + /* If the right-hand side is also a column, then the affinities + ** of both right and left sides must be such that no type + ** conversions are required on the right. (Ticket #2249) + */ + affRight = sqlite3ExprAffinity(pOrTerm->pExpr->pRight); + affLeft = sqlite3ExprAffinity(pOrTerm->pExpr->pLeft); + if( affRight!=0 && affRight!=affLeft ){ + okToChngToIN = 0; + }else{ + pOrTerm->wtFlags |= TERM_OR_OK; + } + } + } + } + + /* At this point, okToChngToIN is true if original pTerm satisfies + ** case 1. In that case, construct a new virtual term that is + ** pTerm converted into an IN operator. + */ + if( okToChngToIN ){ + Expr *pDup; /* A transient duplicate expression */ + ExprList *pList = 0; /* The RHS of the IN operator */ + Expr *pLeft = 0; /* The LHS of the IN operator */ + Expr *pNew; /* The complete IN operator */ + + for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ + if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; + assert( pOrTerm->eOperator & WO_EQ ); + assert( pOrTerm->leftCursor==iCursor ); + assert( pOrTerm->u.x.leftColumn==iColumn ); + pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); + pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup); + pLeft = pOrTerm->pExpr->pLeft; + } + assert( pLeft!=0 ); + pDup = sqlite3ExprDup(db, pLeft, 0); + pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0); + if( pNew ){ + int idxNew; + transferJoinMarkings(pNew, pExpr); + assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + pNew->x.pList = pList; + idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + /* pTerm = &pWC->a[idxTerm]; // would be needed if pTerm where reused */ + markTermAsChild(pWC, idxNew, idxTerm); + }else{ + sqlite3ExprListDelete(db, pList); + } + } + } +} +#endif /* !SQLITE_OMIT_OR_OPTIMIZATION && !SQLITE_OMIT_SUBQUERY */ + +/* +** We already know that pExpr is a binary operator where both operands are +** column references. This routine checks to see if pExpr is an equivalence +** relation: +** 1. The SQLITE_Transitive optimization must be enabled +** 2. Must be either an == or an IS operator +** 3. Not originating in the ON clause of an OUTER JOIN +** 4. The affinities of A and B must be compatible +** 5a. Both operands use the same collating sequence OR +** 5b. The overall collating sequence is BINARY +** If this routine returns TRUE, that means that the RHS can be substituted +** for the LHS anyplace else in the WHERE clause where the LHS column occurs. +** This is an optimization. No harm comes from returning 0. But if 1 is +** returned when it should not be, then incorrect answers might result. +*/ +static int termIsEquivalence(Parse *pParse, Expr *pExpr){ + char aff1, aff2; + CollSeq *pColl; + if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0; + if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0; + if( ExprHasProperty(pExpr, EP_FromJoin) ) return 0; + aff1 = sqlite3ExprAffinity(pExpr->pLeft); + aff2 = sqlite3ExprAffinity(pExpr->pRight); + if( aff1!=aff2 + && (!sqlite3IsNumericAffinity(aff1) || !sqlite3IsNumericAffinity(aff2)) + ){ + return 0; + } + pColl = sqlite3ExprCompareCollSeq(pParse, pExpr); + if( sqlite3IsBinary(pColl) ) return 1; + return sqlite3ExprCollSeqMatch(pParse, pExpr->pLeft, pExpr->pRight); +} + +/* +** Recursively walk the expressions of a SELECT statement and generate +** a bitmask indicating which tables are used in that expression +** tree. +*/ +static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ + Bitmask mask = 0; + while( pS ){ + SrcList *pSrc = pS->pSrc; + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pEList); + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pGroupBy); + mask |= sqlite3WhereExprListUsage(pMaskSet, pS->pOrderBy); + mask |= sqlite3WhereExprUsage(pMaskSet, pS->pWhere); + mask |= sqlite3WhereExprUsage(pMaskSet, pS->pHaving); + if( ALWAYS(pSrc!=0) ){ + int i; + for(i=0; inSrc; i++){ + mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); + mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].pOn); + if( pSrc->a[i].fg.isTabFunc ){ + mask |= sqlite3WhereExprListUsage(pMaskSet, pSrc->a[i].u1.pFuncArg); + } + } + } + pS = pS->pPrior; + } + return mask; +} + +/* +** Expression pExpr is one operand of a comparison operator that might +** be useful for indexing. This routine checks to see if pExpr appears +** in any index. Return TRUE (1) if pExpr is an indexed term and return +** FALSE (0) if not. If TRUE is returned, also set aiCurCol[0] to the cursor +** number of the table that is indexed and aiCurCol[1] to the column number +** of the column that is indexed, or XN_EXPR (-2) if an expression is being +** indexed. +** +** If pExpr is a TK_COLUMN column reference, then this routine always returns +** true even if that particular column is not indexed, because the column +** might be added to an automatic index later. +*/ +static SQLITE_NOINLINE int exprMightBeIndexed2( + SrcList *pFrom, /* The FROM clause */ + Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ + int *aiCurCol, /* Write the referenced table cursor and column here */ + Expr *pExpr /* An operand of a comparison operator */ +){ + Index *pIdx; + int i; + int iCur; + for(i=0; mPrereq>1; i++, mPrereq>>=1){} + iCur = pFrom->a[i].iCursor; + for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr==0 ) continue; + for(i=0; inKeyCol; i++){ + if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + if( sqlite3ExprCompareSkip(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ + aiCurCol[0] = iCur; + aiCurCol[1] = XN_EXPR; + return 1; + } + } + } + return 0; +} +static int exprMightBeIndexed( + SrcList *pFrom, /* The FROM clause */ + Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ + int *aiCurCol, /* Write the referenced table cursor & column here */ + Expr *pExpr, /* An operand of a comparison operator */ + int op /* The specific comparison operator */ +){ + /* If this expression is a vector to the left or right of a + ** inequality constraint (>, <, >= or <=), perform the processing + ** on the first element of the vector. */ + assert( TK_GT+1==TK_LE && TK_GT+2==TK_LT && TK_GT+3==TK_GE ); + assert( TK_ISop==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){ + pExpr = pExpr->x.pList->a[0].pExpr; + + } + + if( pExpr->op==TK_COLUMN ){ + aiCurCol[0] = pExpr->iTable; + aiCurCol[1] = pExpr->iColumn; + return 1; + } + if( mPrereq==0 ) return 0; /* No table references */ + if( (mPrereq&(mPrereq-1))!=0 ) return 0; /* Refs more than one table */ + return exprMightBeIndexed2(pFrom,mPrereq,aiCurCol,pExpr); +} + + +/* +** The input to this routine is an WhereTerm structure with only the +** "pExpr" field filled in. The job of this routine is to analyze the +** subexpression and populate all the other fields of the WhereTerm +** structure. +** +** If the expression is of the form " X" it gets commuted +** to the standard form of "X ". +** +** If the expression is of the form "X Y" where both X and Y are +** columns, then the original expression is unchanged and a new virtual +** term of the form "Y X" is added to the WHERE clause and +** analyzed separately. The original term is marked with TERM_COPIED +** and the new term is marked with TERM_DYNAMIC (because it's pExpr +** needs to be freed with the WhereClause) and TERM_VIRTUAL (because it +** is a commuted copy of a prior term.) The original term has nChild=1 +** and the copy has idxParent set to the index of the original term. +*/ +static void exprAnalyze( + SrcList *pSrc, /* the FROM clause */ + WhereClause *pWC, /* the WHERE clause */ + int idxTerm /* Index of the term to be analyzed */ +){ + WhereInfo *pWInfo = pWC->pWInfo; /* WHERE clause processing context */ + WhereTerm *pTerm; /* The term to be analyzed */ + WhereMaskSet *pMaskSet; /* Set of table index masks */ + Expr *pExpr; /* The expression to be analyzed */ + Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ + Bitmask prereqAll; /* Prerequesites of pExpr */ + Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ + Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ + int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ + int noCase = 0; /* uppercase equivalent to lowercase */ + int op; /* Top-level operator. pExpr->op */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + sqlite3 *db = pParse->db; /* Database connection */ + unsigned char eOp2 = 0; /* op2 value for LIKE/REGEXP/GLOB */ + int nLeft; /* Number of elements on left side vector */ + + if( db->mallocFailed ){ + return; + } + pTerm = &pWC->a[idxTerm]; + pMaskSet = &pWInfo->sMaskSet; + pExpr = pTerm->pExpr; + assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); + prereqLeft = sqlite3WhereExprUsage(pMaskSet, pExpr->pLeft); + op = pExpr->op; + if( op==TK_IN ){ + assert( pExpr->pRight==0 ); + if( sqlite3ExprCheckIN(pParse, pExpr) ) return; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + pTerm->prereqRight = exprSelectUsage(pMaskSet, pExpr->x.pSelect); + }else{ + pTerm->prereqRight = sqlite3WhereExprListUsage(pMaskSet, pExpr->x.pList); + } + }else if( op==TK_ISNULL ){ + pTerm->prereqRight = 0; + }else{ + pTerm->prereqRight = sqlite3WhereExprUsage(pMaskSet, pExpr->pRight); + } + pMaskSet->bVarSelect = 0; + prereqAll = sqlite3WhereExprUsageNN(pMaskSet, pExpr); + if( pMaskSet->bVarSelect ) pTerm->wtFlags |= TERM_VARSELECT; + if( ExprHasProperty(pExpr, EP_FromJoin) ){ + Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->iRightJoinTable); + prereqAll |= x; + extraRight = x-1; /* ON clause terms may not be used with an index + ** on left table of a LEFT JOIN. Ticket #3015 */ + if( (prereqAll>>1)>=x ){ + sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); + return; + } + } + pTerm->prereqAll = prereqAll; + pTerm->leftCursor = -1; + pTerm->iParent = -1; + pTerm->eOperator = 0; + if( allowedOp(op) ){ + int aiCurCol[2]; + Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); + Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); + u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; + + if( pTerm->u.x.iField>0 ){ + assert( op==TK_IN ); + assert( pLeft->op==TK_VECTOR ); + pLeft = pLeft->x.pList->a[pTerm->u.x.iField-1].pExpr; + } + + if( exprMightBeIndexed(pSrc, prereqLeft, aiCurCol, pLeft, op) ){ + pTerm->leftCursor = aiCurCol[0]; + pTerm->u.x.leftColumn = aiCurCol[1]; + pTerm->eOperator = operatorMask(op) & opMask; + } + if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; + if( pRight + && exprMightBeIndexed(pSrc, pTerm->prereqRight, aiCurCol, pRight, op) + && !ExprHasProperty(pRight, EP_FixedCol) + ){ + WhereTerm *pNew; + Expr *pDup; + u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ + assert( pTerm->u.x.iField==0 ); + if( pTerm->leftCursor>=0 ){ + int idxNew; + pDup = sqlite3ExprDup(db, pExpr, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDup); + return; + } + idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); + if( idxNew==0 ) return; + pNew = &pWC->a[idxNew]; + markTermAsChild(pWC, idxNew, idxTerm); + if( op==TK_IS ) pNew->wtFlags |= TERM_IS; + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + + if( termIsEquivalence(pParse, pDup) ){ + pTerm->eOperator |= WO_EQUIV; + eExtraOp = WO_EQUIV; + } + }else{ + pDup = pExpr; + pNew = pTerm; + } + pNew->wtFlags |= exprCommute(pParse, pDup); + pNew->leftCursor = aiCurCol[0]; + pNew->u.x.leftColumn = aiCurCol[1]; + testcase( (prereqLeft | extraRight) != prereqLeft ); + pNew->prereqRight = prereqLeft | extraRight; + pNew->prereqAll = prereqAll; + pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; + }else + if( op==TK_ISNULL + && !ExprHasProperty(pExpr,EP_FromJoin) + && 0==sqlite3ExprCanBeNull(pLeft) + ){ + pExpr->op = TK_TRUEFALSE; + pExpr->u.zToken = "false"; + ExprSetProperty(pExpr, EP_IsFalse); + pTerm->prereqAll = 0; + pTerm->eOperator = 0; + } + } + +#ifndef SQLITE_OMIT_BETWEEN_OPTIMIZATION + /* If a term is the BETWEEN operator, create two new virtual terms + ** that define the range that the BETWEEN implements. For example: + ** + ** a BETWEEN b AND c + ** + ** is converted into: + ** + ** (a BETWEEN b AND c) AND (a>=b) AND (a<=c) + ** + ** The two new terms are added onto the end of the WhereClause object. + ** The new terms are "dynamic" and are children of the original BETWEEN + ** term. That means that if the BETWEEN term is coded, the children are + ** skipped. Or, if the children are satisfied by an index, the original + ** BETWEEN term is skipped. + */ + else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){ + ExprList *pList = pExpr->x.pList; + int i; + static const u8 ops[] = {TK_GE, TK_LE}; + assert( pList!=0 ); + assert( pList->nExpr==2 ); + for(i=0; i<2; i++){ + Expr *pNewExpr; + int idxNew; + pNewExpr = sqlite3PExpr(pParse, ops[i], + sqlite3ExprDup(db, pExpr->pLeft, 0), + sqlite3ExprDup(db, pList->a[i].pExpr, 0)); + transferJoinMarkings(pNewExpr, pExpr); + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + exprAnalyze(pSrc, pWC, idxNew); + pTerm = &pWC->a[idxTerm]; + markTermAsChild(pWC, idxNew, idxTerm); + } + } +#endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ + +#if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) + /* Analyze a term that is composed of two or more subterms connected by + ** an OR operator. + */ + else if( pExpr->op==TK_OR ){ + assert( pWC->op==TK_AND ); + exprAnalyzeOrTerm(pSrc, pWC, idxTerm); + pTerm = &pWC->a[idxTerm]; + } +#endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + /* The form "x IS NOT NULL" can sometimes be evaluated more efficiently + ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a + ** virtual term of that form. + ** + ** The virtual term must be tagged with TERM_VNULL. + */ + else if( pExpr->op==TK_NOTNULL ){ + if( pExpr->pLeft->op==TK_COLUMN + && pExpr->pLeft->iColumn>=0 + && !ExprHasProperty(pExpr, EP_FromJoin) + ){ + Expr *pNewExpr; + Expr *pLeft = pExpr->pLeft; + int idxNew; + WhereTerm *pNewTerm; + + pNewExpr = sqlite3PExpr(pParse, TK_GT, + sqlite3ExprDup(db, pLeft, 0), + sqlite3ExprAlloc(db, TK_NULL, 0, 0)); + + idxNew = whereClauseInsert(pWC, pNewExpr, + TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); + if( idxNew ){ + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = 0; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.x.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_GT; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; + } + } + } + + +#ifndef SQLITE_OMIT_LIKE_OPTIMIZATION + /* Add constraints to reduce the search space on a LIKE or GLOB + ** operator. + ** + ** A like pattern of the form "x LIKE 'aBc%'" is changed into constraints + ** + ** x>='ABC' AND x<'abd' AND x LIKE 'aBc%' + ** + ** The last character of the prefix "abc" is incremented to form the + ** termination condition "abd". If case is not significant (the default + ** for LIKE) then the lower-bound is made all uppercase and the upper- + ** bound is made all lowercase so that the bounds also work when comparing + ** BLOBs. + */ + else if( pExpr->op==TK_FUNCTION + && pWC->op==TK_AND + && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) + ){ + Expr *pLeft; /* LHS of LIKE/GLOB operator */ + Expr *pStr2; /* Copy of pStr1 - RHS of LIKE/GLOB operator */ + Expr *pNewExpr1; + Expr *pNewExpr2; + int idxNew1; + int idxNew2; + const char *zCollSeqName; /* Name of collating sequence */ + const u16 wtFlags = TERM_LIKEOPT | TERM_VIRTUAL | TERM_DYNAMIC; + + pLeft = pExpr->x.pList->a[1].pExpr; + pStr2 = sqlite3ExprDup(db, pStr1, 0); + + /* Convert the lower bound to upper-case and the upper bound to + ** lower-case (upper-case is less than lower-case in ASCII) so that + ** the range constraints also work for BLOBs + */ + if( noCase && !pParse->db->mallocFailed ){ + int i; + char c; + pTerm->wtFlags |= TERM_LIKE; + for(i=0; (c = pStr1->u.zToken[i])!=0; i++){ + pStr1->u.zToken[i] = sqlite3Toupper(c); + pStr2->u.zToken[i] = sqlite3Tolower(c); + } + } + + if( !db->mallocFailed ){ + u8 c, *pC; /* Last character before the first wildcard */ + pC = (u8*)&pStr2->u.zToken[sqlite3Strlen30(pStr2->u.zToken)-1]; + c = *pC; + if( noCase ){ + /* The point is to increment the last character before the first + ** wildcard. But if we increment '@', that will push it into the + ** alphabetic range where case conversions will mess up the + ** inequality. To avoid this, make sure to also run the full + ** LIKE on all candidate expressions by clearing the isComplete flag + */ + if( c=='A'-1 ) isComplete = 0; + c = sqlite3UpperToLower[c]; + } + *pC = c + 1; + } + zCollSeqName = noCase ? "NOCASE" : sqlite3StrBINARY; + pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); + pNewExpr1 = sqlite3PExpr(pParse, TK_GE, + sqlite3ExprAddCollateString(pParse,pNewExpr1,zCollSeqName), + pStr1); + transferJoinMarkings(pNewExpr1, pExpr); + idxNew1 = whereClauseInsert(pWC, pNewExpr1, wtFlags); + testcase( idxNew1==0 ); + exprAnalyze(pSrc, pWC, idxNew1); + pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); + pNewExpr2 = sqlite3PExpr(pParse, TK_LT, + sqlite3ExprAddCollateString(pParse,pNewExpr2,zCollSeqName), + pStr2); + transferJoinMarkings(pNewExpr2, pExpr); + idxNew2 = whereClauseInsert(pWC, pNewExpr2, wtFlags); + testcase( idxNew2==0 ); + exprAnalyze(pSrc, pWC, idxNew2); + pTerm = &pWC->a[idxTerm]; + if( isComplete ){ + markTermAsChild(pWC, idxNew1, idxTerm); + markTermAsChild(pWC, idxNew2, idxTerm); + } + } +#endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + + /* If there is a vector == or IS term - e.g. "(a, b) == (?, ?)" - create + ** new terms for each component comparison - "a = ?" and "b = ?". The + ** new terms completely replace the original vector comparison, which is + ** no longer used. + ** + ** This is only required if at least one side of the comparison operation + ** is not a sub-select. */ + if( (pExpr->op==TK_EQ || pExpr->op==TK_IS) + && (nLeft = sqlite3ExprVectorSize(pExpr->pLeft))>1 + && sqlite3ExprVectorSize(pExpr->pRight)==nLeft + && ( (pExpr->pLeft->flags & EP_xIsSelect)==0 + || (pExpr->pRight->flags & EP_xIsSelect)==0) + && pWC->op==TK_AND + ){ + int i; + for(i=0; ipLeft, i, nLeft); + Expr *pRight = sqlite3ExprForVectorField(pParse, pExpr->pRight, i, nLeft); + + pNew = sqlite3PExpr(pParse, pExpr->op, pLeft, pRight); + transferJoinMarkings(pNew, pExpr); + idxNew = whereClauseInsert(pWC, pNew, TERM_DYNAMIC); + exprAnalyze(pSrc, pWC, idxNew); + } + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_CODED|TERM_VIRTUAL; /* Disable the original */ + pTerm->eOperator = 0; + } + + /* If there is a vector IN term - e.g. "(a, b) IN (SELECT ...)" - create + ** a virtual term for each vector component. The expression object + ** used by each such virtual term is pExpr (the full vector IN(...) + ** expression). The WhereTerm.u.x.iField variable identifies the index within + ** the vector on the LHS that the virtual term represents. + ** + ** This only works if the RHS is a simple SELECT (not a compound) that does + ** not use window functions. + */ + else if( pExpr->op==TK_IN + && pTerm->u.x.iField==0 + && pExpr->pLeft->op==TK_VECTOR + && pExpr->x.pSelect->pPrior==0 +#ifndef SQLITE_OMIT_WINDOWFUNC + && pExpr->x.pSelect->pWin==0 +#endif + && pWC->op==TK_AND + ){ + int i; + for(i=0; ipLeft); i++){ + int idxNew; + idxNew = whereClauseInsert(pWC, pExpr, TERM_VIRTUAL); + pWC->a[idxNew].u.x.iField = i+1; + exprAnalyze(pSrc, pWC, idxNew); + markTermAsChild(pWC, idxNew, idxTerm); + } + } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Add a WO_AUX auxiliary term to the constraint set if the + ** current expression is of the form "column OP expr" where OP + ** is an operator that gets passed into virtual tables but which is + ** not normally optimized for ordinary tables. In other words, OP + ** is one of MATCH, LIKE, GLOB, REGEXP, !=, IS, IS NOT, or NOT NULL. + ** This information is used by the xBestIndex methods of + ** virtual tables. The native query optimizer does not attempt + ** to do anything with MATCH functions. + */ + else if( pWC->op==TK_AND ){ + Expr *pRight = 0, *pLeft = 0; + int res = isAuxiliaryVtabOperator(db, pExpr, &eOp2, &pLeft, &pRight); + while( res-- > 0 ){ + int idxNew; + WhereTerm *pNewTerm; + Bitmask prereqColumn, prereqExpr; + + prereqExpr = sqlite3WhereExprUsage(pMaskSet, pRight); + prereqColumn = sqlite3WhereExprUsage(pMaskSet, pLeft); + if( (prereqExpr & prereqColumn)==0 ){ + Expr *pNewExpr; + pNewExpr = sqlite3PExpr(pParse, TK_MATCH, + 0, sqlite3ExprDup(db, pRight, 0)); + if( ExprHasProperty(pExpr, EP_FromJoin) && pNewExpr ){ + ExprSetProperty(pNewExpr, EP_FromJoin); + pNewExpr->iRightJoinTable = pExpr->iRightJoinTable; + } + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = prereqExpr; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.x.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_AUX; + pNewTerm->eMatchOp = eOp2; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; + } + SWAP(Expr*, pLeft, pRight); + } + } +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + /* Prevent ON clause terms of a LEFT JOIN from being used to drive + ** an index for tables to the left of the join. + */ + testcase( pTerm!=&pWC->a[idxTerm] ); + pTerm = &pWC->a[idxTerm]; + pTerm->prereqRight |= extraRight; +} + +/*************************************************************************** +** Routines with file scope above. Interface to the rest of the where.c +** subsystem follows. +***************************************************************************/ + +/* +** This routine identifies subexpressions in the WHERE clause where +** each subexpression is separated by the AND operator or some other +** operator specified in the op parameter. The WhereClause structure +** is filled with pointers to subexpressions. For example: +** +** WHERE a=='hello' AND coalesce(b,11)<10 AND (c+12!=d OR c==22) +** \________/ \_______________/ \________________/ +** slot[0] slot[1] slot[2] +** +** The original WHERE clause in pExpr is unaltered. All this routine +** does is make slot[] entries point to substructure within pExpr. +** +** In the previous sentence and in the diagram, "slot[]" refers to +** the WhereClause.a[] array. The slot[] array grows as needed to contain +** all terms of the WHERE clause. +*/ +SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ + Expr *pE2 = sqlite3ExprSkipCollateAndLikely(pExpr); + pWC->op = op; + assert( pE2!=0 || pExpr==0 ); + if( pE2==0 ) return; + if( pE2->op!=op ){ + whereClauseInsert(pWC, pExpr, 0); + }else{ + sqlite3WhereSplit(pWC, pE2->pLeft, op); + sqlite3WhereSplit(pWC, pE2->pRight, op); + } +} + +/* +** Initialize a preallocated WhereClause structure. +*/ +SQLITE_PRIVATE void sqlite3WhereClauseInit( + WhereClause *pWC, /* The WhereClause to be initialized */ + WhereInfo *pWInfo /* The WHERE processing context */ +){ + pWC->pWInfo = pWInfo; + pWC->hasOr = 0; + pWC->pOuter = 0; + pWC->nTerm = 0; + pWC->nSlot = ArraySize(pWC->aStatic); + pWC->a = pWC->aStatic; +} + +/* +** Deallocate a WhereClause structure. The WhereClause structure +** itself is not freed. This routine is the inverse of +** sqlite3WhereClauseInit(). +*/ +SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ + int i; + WhereTerm *a; + sqlite3 *db = pWC->pWInfo->pParse->db; + for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ + if( a->wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, a->pExpr); + } + if( a->wtFlags & TERM_ORINFO ){ + whereOrInfoDelete(db, a->u.pOrInfo); + }else if( a->wtFlags & TERM_ANDINFO ){ + whereAndInfoDelete(db, a->u.pAndInfo); + } + } + if( pWC->a!=pWC->aStatic ){ + sqlite3DbFree(db, pWC->a); + } +} + + +/* +** These routines walk (recursively) an expression tree and generate +** a bitmask indicating which tables are used in that expression +** tree. +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet *pMaskSet, Expr *p){ + Bitmask mask; + if( p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){ + return sqlite3WhereGetMask(pMaskSet, p->iTable); + }else if( ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ + assert( p->op!=TK_IF_NULL_ROW ); + return 0; + } + mask = (p->op==TK_IF_NULL_ROW) ? sqlite3WhereGetMask(pMaskSet, p->iTable) : 0; + if( p->pLeft ) mask |= sqlite3WhereExprUsageNN(pMaskSet, p->pLeft); + if( p->pRight ){ + mask |= sqlite3WhereExprUsageNN(pMaskSet, p->pRight); + assert( p->x.pList==0 ); + }else if( ExprHasProperty(p, EP_xIsSelect) ){ + if( ExprHasProperty(p, EP_VarSelect) ) pMaskSet->bVarSelect = 1; + mask |= exprSelectUsage(pMaskSet, p->x.pSelect); + }else if( p->x.pList ){ + mask |= sqlite3WhereExprListUsage(pMaskSet, p->x.pList); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( (p->op==TK_FUNCTION || p->op==TK_AGG_FUNCTION) && p->y.pWin ){ + mask |= sqlite3WhereExprListUsage(pMaskSet, p->y.pWin->pPartition); + mask |= sqlite3WhereExprListUsage(pMaskSet, p->y.pWin->pOrderBy); + mask |= sqlite3WhereExprUsage(pMaskSet, p->y.pWin->pFilter); + } +#endif + return mask; +} +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ + return p ? sqlite3WhereExprUsageNN(pMaskSet,p) : 0; +} +SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet *pMaskSet, ExprList *pList){ + int i; + Bitmask mask = 0; + if( pList ){ + for(i=0; inExpr; i++){ + mask |= sqlite3WhereExprUsage(pMaskSet, pList->a[i].pExpr); + } + } + return mask; +} + + +/* +** Call exprAnalyze on all terms in a WHERE clause. +** +** Note that exprAnalyze() might add new virtual terms onto the +** end of the WHERE clause. We do not want to analyze these new +** virtual terms, so start analyzing at the end and work forward +** so that the added virtual terms are never processed. +*/ +SQLITE_PRIVATE void sqlite3WhereExprAnalyze( + SrcList *pTabList, /* the FROM clause */ + WhereClause *pWC /* the WHERE clause to be analyzed */ +){ + int i; + for(i=pWC->nTerm-1; i>=0; i--){ + exprAnalyze(pTabList, pWC, i); + } +} + +/* +** For table-valued-functions, transform the function arguments into +** new WHERE clause terms. +** +** Each function argument translates into an equality constraint against +** a HIDDEN column in the table. +*/ +SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( + Parse *pParse, /* Parsing context */ + SrcItem *pItem, /* The FROM clause term to process */ + WhereClause *pWC /* Xfer function arguments to here */ +){ + Table *pTab; + int j, k; + ExprList *pArgs; + Expr *pColRef; + Expr *pTerm; + if( pItem->fg.isTabFunc==0 ) return; + pTab = pItem->pTab; + assert( pTab!=0 ); + pArgs = pItem->u1.pFuncArg; + if( pArgs==0 ) return; + for(j=k=0; jnExpr; j++){ + Expr *pRhs; + while( knCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){k++;} + if( k>=pTab->nCol ){ + sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d", + pTab->zName, j); + return; + } + pColRef = sqlite3ExprAlloc(pParse->db, TK_COLUMN, 0, 0); + if( pColRef==0 ) return; + pColRef->iTable = pItem->iCursor; + pColRef->iColumn = k++; + pColRef->y.pTab = pTab; + pRhs = sqlite3PExpr(pParse, TK_UPLUS, + sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); + pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs); + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3SetJoinExpr(pTerm, pItem->iCursor); + } + whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); + } +} + +/************** End of whereexpr.c *******************************************/ +/************** Begin file where.c *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This module contains C code that generates VDBE code used to process +** the WHERE clause of SQL statements. This module is responsible for +** generating the code that loops through a table looking for applicable +** rows. Indices are selected and used to speed the search when doing +** so is applicable. Because this module is responsible for selecting +** indices, you might also think of this module as the "query optimizer". +*/ +/* #include "sqliteInt.h" */ +/* #include "whereInt.h" */ + +/* +** Extra information appended to the end of sqlite3_index_info but not +** visible to the xBestIndex function, at least not directly. The +** sqlite3_vtab_collation() interface knows how to reach it, however. +** +** This object is not an API and can be changed from one release to the +** next. As long as allocateIndexInfo() and sqlite3_vtab_collation() +** agree on the structure, all will be well. +*/ +typedef struct HiddenIndexInfo HiddenIndexInfo; +struct HiddenIndexInfo { + WhereClause *pWC; /* The Where clause being analyzed */ + Parse *pParse; /* The parsing context */ +}; + +/* Forward declaration of methods */ +static int whereLoopResize(sqlite3*, WhereLoop*, int); + +/* +** Return the estimated number of output rows from a WHERE clause +*/ +SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo *pWInfo){ + return pWInfo->nRowOut; +} + +/* +** Return one of the WHERE_DISTINCT_xxxxx values to indicate how this +** WHERE clause returns outputs for DISTINCT processing. +*/ +SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ + return pWInfo->eDistinct; +} + +/* +** Return the number of ORDER BY terms that are satisfied by the +** WHERE clause. A return of 0 means that the output must be +** completely sorted. A return equal to the number of ORDER BY +** terms means that no sorting is needed at all. A return that +** is positive but less than the number of ORDER BY terms means that +** block sorting is required. +*/ +SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ + return pWInfo->nOBSat; +} + +/* +** In the ORDER BY LIMIT optimization, if the inner-most loop is known +** to emit rows in increasing order, and if the last row emitted by the +** inner-most loop did not fit within the sorter, then we can skip all +** subsequent rows for the current iteration of the inner loop (because they +** will not fit in the sorter either) and continue with the second inner +** loop - the loop immediately outside the inner-most. +** +** When a row does not fit in the sorter (because the sorter already +** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the +** label returned by this function. +** +** If the ORDER BY LIMIT optimization applies, the jump destination should +** be the continuation for the second-inner-most loop. If the ORDER BY +** LIMIT optimization does not apply, then the jump destination should +** be the continuation for the inner-most loop. +** +** It is always safe for this routine to return the continuation of the +** inner-most loop, in the sense that a correct answer will result. +** Returning the continuation the second inner loop is an optimization +** that might make the code run a little faster, but should not change +** the final answer. +*/ +SQLITE_PRIVATE int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){ + WhereLevel *pInner; + if( !pWInfo->bOrderedInnerLoop ){ + /* The ORDER BY LIMIT optimization does not apply. Jump to the + ** continuation of the inner-most loop. */ + return pWInfo->iContinue; + } + pInner = &pWInfo->a[pWInfo->nLevel-1]; + assert( pInner->addrNxt!=0 ); + return pInner->addrNxt; +} + +/* +** While generating code for the min/max optimization, after handling +** the aggregate-step call to min() or max(), check to see if any +** additional looping is required. If the output order is such that +** we are certain that the correct answer has already been found, then +** code an OP_Goto to by pass subsequent processing. +** +** Any extra OP_Goto that is coded here is an optimization. The +** correct answer should be obtained regardless. This OP_Goto just +** makes the answer appear faster. +*/ +SQLITE_PRIVATE void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){ + WhereLevel *pInner; + int i; + if( !pWInfo->bOrderedInnerLoop ) return; + if( pWInfo->nOBSat==0 ) return; + for(i=pWInfo->nLevel-1; i>=0; i--){ + pInner = &pWInfo->a[i]; + if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)!=0 ){ + sqlite3VdbeGoto(v, pInner->addrNxt); + return; + } + } + sqlite3VdbeGoto(v, pWInfo->iBreak); +} + +/* +** Return the VDBE address or label to jump to in order to continue +** immediately with the next row of a WHERE clause. +*/ +SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ + assert( pWInfo->iContinue!=0 ); + return pWInfo->iContinue; +} + +/* +** Return the VDBE address or label to jump to in order to break +** out of a WHERE loop. +*/ +SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ + return pWInfo->iBreak; +} + +/* +** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to +** operate directly on the rowids returned by a WHERE clause. Return +** ONEPASS_SINGLE (1) if the statement can operation directly because only +** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass +** optimization can be used on multiple +** +** If the ONEPASS optimization is used (if this routine returns true) +** then also write the indices of open cursors used by ONEPASS +** into aiCur[0] and aiCur[1]. iaCur[0] gets the cursor of the data +** table and iaCur[1] gets the cursor used by an auxiliary index. +** Either value may be -1, indicating that cursor is not used. +** Any cursors returned will have been opened for writing. +** +** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is +** unable to use the ONEPASS optimization. +*/ +SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ + memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){ + sqlite3DebugPrintf("%s cursors: %d %d\n", + pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI", + aiCur[0], aiCur[1]); + } +#endif + return pWInfo->eOnePass; +} + +/* +** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move +** the data cursor to the row selected by the index cursor. +*/ +SQLITE_PRIVATE int sqlite3WhereUsesDeferredSeek(WhereInfo *pWInfo){ + return pWInfo->bDeferredSeek; +} + +/* +** Move the content of pSrc into pDest +*/ +static void whereOrMove(WhereOrSet *pDest, WhereOrSet *pSrc){ + pDest->n = pSrc->n; + memcpy(pDest->a, pSrc->a, pDest->n*sizeof(pDest->a[0])); +} + +/* +** Try to insert a new prerequisite/cost entry into the WhereOrSet pSet. +** +** The new entry might overwrite an existing entry, or it might be +** appended, or it might be discarded. Do whatever is the right thing +** so that pSet keeps the N_OR_COST best entries seen so far. +*/ +static int whereOrInsert( + WhereOrSet *pSet, /* The WhereOrSet to be updated */ + Bitmask prereq, /* Prerequisites of the new entry */ + LogEst rRun, /* Run-cost of the new entry */ + LogEst nOut /* Number of outputs for the new entry */ +){ + u16 i; + WhereOrCost *p; + for(i=pSet->n, p=pSet->a; i>0; i--, p++){ + if( rRun<=p->rRun && (prereq & p->prereq)==prereq ){ + goto whereOrInsert_done; + } + if( p->rRun<=rRun && (p->prereq & prereq)==p->prereq ){ + return 0; + } + } + if( pSet->na[pSet->n++]; + p->nOut = nOut; + }else{ + p = pSet->a; + for(i=1; in; i++){ + if( p->rRun>pSet->a[i].rRun ) p = pSet->a + i; + } + if( p->rRun<=rRun ) return 0; + } +whereOrInsert_done: + p->prereq = prereq; + p->rRun = rRun; + if( p->nOut>nOut ) p->nOut = nOut; + return 1; +} + +/* +** Return the bitmask for the given cursor number. Return 0 if +** iCursor is not in the set. +*/ +SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){ + int i; + assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); + for(i=0; in; i++){ + if( pMaskSet->ix[i]==iCursor ){ + return MASKBIT(i); + } + } + return 0; +} + +/* +** Create a new mask for cursor iCursor. +** +** There is one cursor per table in the FROM clause. The number of +** tables in the FROM clause is limited by a test early in the +** sqlite3WhereBegin() routine. So we know that the pMaskSet->ix[] +** array will never overflow. +*/ +static void createMask(WhereMaskSet *pMaskSet, int iCursor){ + assert( pMaskSet->n < ArraySize(pMaskSet->ix) ); + pMaskSet->ix[pMaskSet->n++] = iCursor; +} + +/* +** If the right-hand branch of the expression is a TK_COLUMN, then return +** a pointer to the right-hand branch. Otherwise, return NULL. +*/ +static Expr *whereRightSubexprIsColumn(Expr *p){ + p = sqlite3ExprSkipCollateAndLikely(p->pRight); + if( ALWAYS(p!=0) && p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){ + return p; + } + return 0; +} + +/* +** Advance to the next WhereTerm that matches according to the criteria +** established when the pScan object was initialized by whereScanInit(). +** Return NULL if there are no more matching WhereTerms. +*/ +static WhereTerm *whereScanNext(WhereScan *pScan){ + int iCur; /* The cursor on the LHS of the term */ + i16 iColumn; /* The column on the LHS of the term. -1 for IPK */ + Expr *pX; /* An expression being tested */ + WhereClause *pWC; /* Shorthand for pScan->pWC */ + WhereTerm *pTerm; /* The term being tested */ + int k = pScan->k; /* Where to start scanning */ + + assert( pScan->iEquiv<=pScan->nEquiv ); + pWC = pScan->pWC; + while(1){ + iColumn = pScan->aiColumn[pScan->iEquiv-1]; + iCur = pScan->aiCur[pScan->iEquiv-1]; + assert( pWC!=0 ); + do{ + for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ + if( pTerm->leftCursor==iCur + && pTerm->u.x.leftColumn==iColumn + && (iColumn!=XN_EXPR + || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft, + pScan->pIdxExpr,iCur)==0) + && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + ){ + if( (pTerm->eOperator & WO_EQUIV)!=0 + && pScan->nEquivaiCur) + && (pX = whereRightSubexprIsColumn(pTerm->pExpr))!=0 + ){ + int j; + for(j=0; jnEquiv; j++){ + if( pScan->aiCur[j]==pX->iTable + && pScan->aiColumn[j]==pX->iColumn ){ + break; + } + } + if( j==pScan->nEquiv ){ + pScan->aiCur[j] = pX->iTable; + pScan->aiColumn[j] = pX->iColumn; + pScan->nEquiv++; + } + } + if( (pTerm->eOperator & pScan->opMask)!=0 ){ + /* Verify the affinity and collating sequence match */ + if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ + CollSeq *pColl; + Parse *pParse = pWC->pWInfo->pParse; + pX = pTerm->pExpr; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3ExprCompareCollSeq(pParse, pX); + if( pColl==0 ) pColl = pParse->db->pDfltColl; + if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + continue; + } + } + if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0 + && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN + && pX->iTable==pScan->aiCur[0] + && pX->iColumn==pScan->aiColumn[0] + ){ + testcase( pTerm->eOperator & WO_IS ); + continue; + } + pScan->pWC = pWC; + pScan->k = k+1; +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x20000 ){ + int ii; + sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d", + pTerm, pScan->nEquiv); + for(ii=0; iinEquiv; ii++){ + sqlite3DebugPrintf(" {%d:%d}", + pScan->aiCur[ii], pScan->aiColumn[ii]); + } + sqlite3DebugPrintf("\n"); + } +#endif + return pTerm; + } + } + } + pWC = pWC->pOuter; + k = 0; + }while( pWC!=0 ); + if( pScan->iEquiv>=pScan->nEquiv ) break; + pWC = pScan->pOrigWC; + k = 0; + pScan->iEquiv++; + } + return 0; +} + +/* +** This is whereScanInit() for the case of an index on an expression. +** It is factored out into a separate tail-recursion subroutine so that +** the normal whereScanInit() routine, which is a high-runner, does not +** need to push registers onto the stack as part of its prologue. +*/ +static SQLITE_NOINLINE WhereTerm *whereScanInitIndexExpr(WhereScan *pScan){ + pScan->idxaff = sqlite3ExprAffinity(pScan->pIdxExpr); + return whereScanNext(pScan); +} + +/* +** Initialize a WHERE clause scanner object. Return a pointer to the +** first match. Return NULL if there are no matches. +** +** The scanner will be searching the WHERE clause pWC. It will look +** for terms of the form "X " where X is column iColumn of table +** iCur. Or if pIdx!=0 then X is column iColumn of index pIdx. pIdx +** must be one of the indexes of table iCur. +** +** The must be one of the operators described by opMask. +** +** If the search is for X and the WHERE clause contains terms of the +** form X=Y then this routine might also return terms of the form +** "Y ". The number of levels of transitivity is limited, +** but is enough to handle most commonly occurring SQL statements. +** +** If X is not the INTEGER PRIMARY KEY then X must be compatible with +** index pIdx. +*/ +static WhereTerm *whereScanInit( + WhereScan *pScan, /* The WhereScan object being initialized */ + WhereClause *pWC, /* The WHERE clause to be scanned */ + int iCur, /* Cursor to scan for */ + int iColumn, /* Column to scan for */ + u32 opMask, /* Operator(s) to scan for */ + Index *pIdx /* Must be compatible with this index */ +){ + pScan->pOrigWC = pWC; + pScan->pWC = pWC; + pScan->pIdxExpr = 0; + pScan->idxaff = 0; + pScan->zCollName = 0; + pScan->opMask = opMask; + pScan->k = 0; + pScan->aiCur[0] = iCur; + pScan->nEquiv = 1; + pScan->iEquiv = 1; + if( pIdx ){ + int j = iColumn; + iColumn = pIdx->aiColumn[j]; + if( iColumn==XN_EXPR ){ + pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; + pScan->zCollName = pIdx->azColl[j]; + pScan->aiColumn[0] = XN_EXPR; + return whereScanInitIndexExpr(pScan); + }else if( iColumn==pIdx->pTable->iPKey ){ + iColumn = XN_ROWID; + }else if( iColumn>=0 ){ + pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; + pScan->zCollName = pIdx->azColl[j]; + } + }else if( iColumn==XN_EXPR ){ + return 0; + } + pScan->aiColumn[0] = iColumn; + return whereScanNext(pScan); +} + +/* +** Search for a term in the WHERE clause that is of the form "X " +** where X is a reference to the iColumn of table iCur or of index pIdx +** if pIdx!=0 and is one of the WO_xx operator codes specified by +** the op parameter. Return a pointer to the term. Return 0 if not found. +** +** If pIdx!=0 then it must be one of the indexes of table iCur. +** Search for terms matching the iColumn-th column of pIdx +** rather than the iColumn-th column of table iCur. +** +** The term returned might by Y= if there is another constraint in +** the WHERE clause that specifies that X=Y. Any such constraints will be +** identified by the WO_EQUIV bit in the pTerm->eOperator field. The +** aiCur[]/iaColumn[] arrays hold X and all its equivalents. There are 11 +** slots in aiCur[]/aiColumn[] so that means we can look for X plus up to 10 +** other equivalent values. Hence a search for X will return if X=A1 +** and A1=A2 and A2=A3 and ... and A9=A10 and A10=. +** +** If there are multiple terms in the WHERE clause of the form "X " +** then try for the one with no dependencies on - in other words where +** is a constant expression of some kind. Only return entries of +** the form "X Y" where Y is a column in another table if no terms of +** the form "X " exist. If no terms with a constant RHS +** exist, try to return a term that does not use WO_EQUIV. +*/ +SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( + WhereClause *pWC, /* The WHERE clause to be searched */ + int iCur, /* Cursor number of LHS */ + int iColumn, /* Column number of LHS */ + Bitmask notReady, /* RHS must not overlap with this mask */ + u32 op, /* Mask of WO_xx values describing operator */ + Index *pIdx /* Must be compatible with this index, if not NULL */ +){ + WhereTerm *pResult = 0; + WhereTerm *p; + WhereScan scan; + + p = whereScanInit(&scan, pWC, iCur, iColumn, op, pIdx); + op &= WO_EQ|WO_IS; + while( p ){ + if( (p->prereqRight & notReady)==0 ){ + if( p->prereqRight==0 && (p->eOperator&op)!=0 ){ + testcase( p->eOperator & WO_IS ); + return p; + } + if( pResult==0 ) pResult = p; + } + p = whereScanNext(&scan); + } + return pResult; +} + +/* +** This function searches pList for an entry that matches the iCol-th column +** of index pIdx. +** +** If such an expression is found, its index in pList->a[] is returned. If +** no expression is found, -1 is returned. +*/ +static int findIndexCol( + Parse *pParse, /* Parse context */ + ExprList *pList, /* Expression list to search */ + int iBase, /* Cursor for table associated with pIdx */ + Index *pIdx, /* Index to match column of */ + int iCol /* Column of index to match */ +){ + int i; + const char *zColl = pIdx->azColl[iCol]; + + for(i=0; inExpr; i++){ + Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr); + if( ALWAYS(p!=0) + && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) + && p->iColumn==pIdx->aiColumn[iCol] + && p->iTable==iBase + ){ + CollSeq *pColl = sqlite3ExprNNCollSeq(pParse, pList->a[i].pExpr); + if( 0==sqlite3StrICmp(pColl->zName, zColl) ){ + return i; + } + } + } + + return -1; +} + +/* +** Return TRUE if the iCol-th column of index pIdx is NOT NULL +*/ +static int indexColumnNotNull(Index *pIdx, int iCol){ + int j; + assert( pIdx!=0 ); + assert( iCol>=0 && iColnColumn ); + j = pIdx->aiColumn[iCol]; + if( j>=0 ){ + return pIdx->pTable->aCol[j].notNull; + }else if( j==(-1) ){ + return 1; + }else{ + assert( j==(-2) ); + return 0; /* Assume an indexed expression can always yield a NULL */ + + } +} + +/* +** Return true if the DISTINCT expression-list passed as the third argument +** is redundant. +** +** A DISTINCT list is redundant if any subset of the columns in the +** DISTINCT list are collectively unique and individually non-null. +*/ +static int isDistinctRedundant( + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* The FROM clause */ + WhereClause *pWC, /* The WHERE clause */ + ExprList *pDistinct /* The result set that needs to be DISTINCT */ +){ + Table *pTab; + Index *pIdx; + int i; + int iBase; + + /* If there is more than one table or sub-select in the FROM clause of + ** this query, then it will not be possible to show that the DISTINCT + ** clause is redundant. */ + if( pTabList->nSrc!=1 ) return 0; + iBase = pTabList->a[0].iCursor; + pTab = pTabList->a[0].pTab; + + /* If any of the expressions is an IPK column on table iBase, then return + ** true. Note: The (p->iTable==iBase) part of this test may be false if the + ** current SELECT is a correlated sub-query. + */ + for(i=0; inExpr; i++){ + Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr); + if( NEVER(p==0) ) continue; + if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue; + if( p->iTable==iBase && p->iColumn<0 ) return 1; + } + + /* Loop through all indices on the table, checking each to see if it makes + ** the DISTINCT qualifier redundant. It does so if: + ** + ** 1. The index is itself UNIQUE, and + ** + ** 2. All of the columns in the index are either part of the pDistinct + ** list, or else the WHERE clause contains a term of the form "col=X", + ** where X is a constant value. The collation sequences of the + ** comparison and select-list expressions must match those of the index. + ** + ** 3. All of those index columns for which the WHERE clause does not + ** contain a "col=X" term are subject to a NOT NULL constraint. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( !IsUniqueIndex(pIdx) ) continue; + if( pIdx->pPartIdxWhere ) continue; + for(i=0; inKeyCol; i++){ + if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){ + if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break; + if( indexColumnNotNull(pIdx, i)==0 ) break; + } + } + if( i==pIdx->nKeyCol ){ + /* This index implies that the DISTINCT qualifier is redundant. */ + return 1; + } + } + + return 0; +} + + +/* +** Estimate the logarithm of the input value to base 2. +*/ +static LogEst estLog(LogEst N){ + return N<=10 ? 0 : sqlite3LogEst(N) - 33; +} + +/* +** Convert OP_Column opcodes to OP_Copy in previously generated code. +** +** This routine runs over generated VDBE code and translates OP_Column +** opcodes into OP_Copy when the table is being accessed via co-routine +** instead of via table lookup. +** +** If the iAutoidxCur is not zero, then any OP_Rowid instructions on +** cursor iTabCur are transformed into OP_Sequence opcode for the +** iAutoidxCur cursor, in order to generate unique rowids for the +** automatic index being generated. +*/ +static void translateColumnToCopy( + Parse *pParse, /* Parsing context */ + int iStart, /* Translate from this opcode to the end */ + int iTabCur, /* OP_Column/OP_Rowid references to this table */ + int iRegister, /* The first column is in this register */ + int iAutoidxCur /* If non-zero, cursor of autoindex being generated */ +){ + Vdbe *v = pParse->pVdbe; + VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); + int iEnd = sqlite3VdbeCurrentAddr(v); + if( pParse->db->mallocFailed ) return; + for(; iStartp1!=iTabCur ) continue; + if( pOp->opcode==OP_Column ){ + pOp->opcode = OP_Copy; + pOp->p1 = pOp->p2 + iRegister; + pOp->p2 = pOp->p3; + pOp->p3 = 0; + }else if( pOp->opcode==OP_Rowid ){ + pOp->opcode = OP_Sequence; + pOp->p1 = iAutoidxCur; +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( iAutoidxCur==0 ){ + pOp->opcode = OP_Null; + pOp->p3 = 0; + } +#endif + } + } +} + +/* +** Two routines for printing the content of an sqlite3_index_info +** structure. Used for testing and debugging only. If neither +** SQLITE_TEST or SQLITE_DEBUG are defined, then these routines +** are no-ops. +*/ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) +static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", + i, + p->aConstraint[i].iColumn, + p->aConstraint[i].iTermOffset, + p->aConstraint[i].op, + p->aConstraint[i].usable); + } + for(i=0; inOrderBy; i++){ + sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", + i, + p->aOrderBy[i].iColumn, + p->aOrderBy[i].desc); + } +} +static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ + int i; + if( !sqlite3WhereTrace ) return; + for(i=0; inConstraint; i++){ + sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", + i, + p->aConstraintUsage[i].argvIndex, + p->aConstraintUsage[i].omit); + } + sqlite3DebugPrintf(" idxNum=%d\n", p->idxNum); + sqlite3DebugPrintf(" idxStr=%s\n", p->idxStr); + sqlite3DebugPrintf(" orderByConsumed=%d\n", p->orderByConsumed); + sqlite3DebugPrintf(" estimatedCost=%g\n", p->estimatedCost); + sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); +} +#else +#define whereTraceIndexInfoInputs(A) +#define whereTraceIndexInfoOutputs(A) +#endif + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Return TRUE if the WHERE clause term pTerm is of a form where it +** could be used with an index to access pSrc, assuming an appropriate +** index existed. +*/ +static int termCanDriveIndex( + WhereTerm *pTerm, /* WHERE clause term to check */ + SrcItem *pSrc, /* Table we are trying to access */ + Bitmask notReady /* Tables in outer loops of the join */ +){ + char aff; + if( pTerm->leftCursor!=pSrc->iCursor ) return 0; + if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; + if( (pSrc->fg.jointype & JT_LEFT) + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + && (pTerm->eOperator & WO_IS) + ){ + /* Cannot use an IS term from the WHERE clause as an index driver for + ** the RHS of a LEFT JOIN. Such a term can only be used if it is from + ** the ON clause. */ + return 0; + } + if( (pTerm->prereqRight & notReady)!=0 ) return 0; + if( pTerm->u.x.leftColumn<0 ) return 0; + aff = pSrc->pTab->aCol[pTerm->u.x.leftColumn].affinity; + if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; + testcase( pTerm->pExpr->op==TK_IS ); + return 1; +} +#endif + + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Generate code to construct the Index object for an automatic index +** and to set up the WhereLevel object pLevel so that the code generator +** makes use of the automatic index. +*/ +static void constructAutomaticIndex( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause */ + SrcItem *pSrc, /* The FROM clause term to get the next index */ + Bitmask notReady, /* Mask of cursors that are not available */ + WhereLevel *pLevel /* Write new index here */ +){ + int nKeyCol; /* Number of columns in the constructed index */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + WhereTerm *pWCEnd; /* End of pWC->a[] */ + Index *pIdx; /* Object describing the transient index */ + Vdbe *v; /* Prepared statement under construction */ + int addrInit; /* Address of the initialization bypass jump */ + Table *pTable; /* The table being indexed */ + int addrTop; /* Top of the index fill loop */ + int regRecord; /* Register holding an index record */ + int n; /* Column counter */ + int i; /* Loop counter */ + int mxBitCol; /* Maximum column in pSrc->colUsed */ + CollSeq *pColl; /* Collating sequence to on a column */ + WhereLoop *pLoop; /* The Loop object */ + char *zNotUsed; /* Extra space on the end of pIdx */ + Bitmask idxCols; /* Bitmap of columns used for indexing */ + Bitmask extraCols; /* Bitmap of additional columns */ + u8 sentWarning = 0; /* True if a warnning has been issued */ + Expr *pPartial = 0; /* Partial Index Expression */ + int iContinue = 0; /* Jump here to skip excluded rows */ + SrcItem *pTabItem; /* FROM clause term being indexed */ + int addrCounter = 0; /* Address where integer counter is initialized */ + int regBase; /* Array of registers where record is assembled */ + + /* Generate code to skip over the creation and initialization of the + ** transient index on 2nd and subsequent iterations of the loop. */ + v = pParse->pVdbe; + assert( v!=0 ); + addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + + /* Count the number of columns that will be added to the index + ** and used to match WHERE clause constraints */ + nKeyCol = 0; + pTable = pSrc->pTab; + pWCEnd = &pWC->a[pWC->nTerm]; + pLoop = pLevel->pWLoop; + idxCols = 0; + for(pTerm=pWC->a; pTermpExpr; + assert( !ExprHasProperty(pExpr, EP_FromJoin) /* prereq always non-zero */ + || pExpr->iRightJoinTable!=pSrc->iCursor /* for the right-hand */ + || pLoop->prereq!=0 ); /* table of a LEFT JOIN */ + if( pLoop->prereq==0 + && (pTerm->wtFlags & TERM_VIRTUAL)==0 + && !ExprHasProperty(pExpr, EP_FromJoin) + && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){ + pPartial = sqlite3ExprAnd(pParse, pPartial, + sqlite3ExprDup(pParse->db, pExpr, 0)); + } + if( termCanDriveIndex(pTerm, pSrc, notReady) ){ + int iCol = pTerm->u.x.leftColumn; + Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS ); + testcase( iCol==BMS-1 ); + if( !sentWarning ){ + sqlite3_log(SQLITE_WARNING_AUTOINDEX, + "automatic index on %s(%s)", pTable->zName, + pTable->aCol[iCol].zCnName); + sentWarning = 1; + } + if( (idxCols & cMask)==0 ){ + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ + goto end_auto_index_create; + } + pLoop->aLTerm[nKeyCol++] = pTerm; + idxCols |= cMask; + } + } + } + assert( nKeyCol>0 || pParse->db->mallocFailed ); + pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; + pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED + | WHERE_AUTO_INDEX; + + /* Count the number of additional columns needed to create a + ** covering index. A "covering index" is an index that contains all + ** columns that are needed by the query. With a covering index, the + ** original table never needs to be accessed. Automatic indices must + ** be a covering index because the index will not be updated if the + ** original table changes and the index and table cannot both be used + ** if they go out of sync. + */ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + mxBitCol = MIN(BMS-1,pTable->nCol); + testcase( pTable->nCol==BMS-1 ); + testcase( pTable->nCol==BMS-2 ); + for(i=0; icolUsed & MASKBIT(BMS-1) ){ + nKeyCol += pTable->nCol - BMS + 1; + } + + /* Construct the Index object to describe this index */ + pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); + if( pIdx==0 ) goto end_auto_index_create; + pLoop->u.btree.pIndex = pIdx; + pIdx->zName = "auto-index"; + pIdx->pTable = pTable; + n = 0; + idxCols = 0; + for(pTerm=pWC->a; pTermu.x.leftColumn; + Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + testcase( iCol==BMS-1 ); + testcase( iCol==BMS ); + if( (idxCols & cMask)==0 ){ + Expr *pX = pTerm->pExpr; + idxCols |= cMask; + pIdx->aiColumn[n] = pTerm->u.x.leftColumn; + pColl = sqlite3ExprCompareCollSeq(pParse, pX); + assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */ + pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; + n++; + } + } + } + assert( (u32)n==pLoop->u.btree.nEq ); + + /* Add additional columns needed to make the automatic index into + ** a covering index */ + for(i=0; iaiColumn[n] = i; + pIdx->azColl[n] = sqlite3StrBINARY; + n++; + } + } + if( pSrc->colUsed & MASKBIT(BMS-1) ){ + for(i=BMS-1; inCol; i++){ + pIdx->aiColumn[n] = i; + pIdx->azColl[n] = sqlite3StrBINARY; + n++; + } + } + assert( n==nKeyCol ); + pIdx->aiColumn[n] = XN_ROWID; + pIdx->azColl[n] = sqlite3StrBINARY; + + /* Create the automatic index */ + assert( pLevel->iIdxCur>=0 ); + pLevel->iIdxCur = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "for %s", pTable->zName)); + + /* Fill the automatic index with content */ + pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; + if( pTabItem->fg.viaCoroutine ){ + int regYield = pTabItem->regReturn; + addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); + VdbeCoverage(v); + VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); + }else{ + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + } + if( pPartial ){ + iContinue = sqlite3VdbeMakeLabel(pParse); + sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + pLoop->wsFlags |= WHERE_PARTIALIDX; + } + regRecord = sqlite3GetTempReg(pParse); + regBase = sqlite3GenerateIndexKey( + pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0 + ); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); + if( pTabItem->fg.viaCoroutine ){ + sqlite3VdbeChangeP2(v, addrCounter, regBase+n); + testcase( pParse->db->mallocFailed ); + assert( pLevel->iIdxCur>0 ); + translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, + pTabItem->regResult, pLevel->iIdxCur); + sqlite3VdbeGoto(v, addrTop); + pTabItem->fg.viaCoroutine = 0; + }else{ + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); + } + sqlite3VdbeJumpHere(v, addrTop); + sqlite3ReleaseTempReg(pParse, regRecord); + + /* Jump here when skipping the initialization */ + sqlite3VdbeJumpHere(v, addrInit); + +end_auto_index_create: + sqlite3ExprDelete(pParse->db, pPartial); +} +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Allocate and populate an sqlite3_index_info structure. It is the +** responsibility of the caller to eventually release the structure +** by passing the pointer returned by this function to sqlite3_free(). +*/ +static sqlite3_index_info *allocateIndexInfo( + Parse *pParse, /* The parsing context */ + WhereClause *pWC, /* The WHERE clause being analyzed */ + Bitmask mUnusable, /* Ignore terms with these prereqs */ + SrcItem *pSrc, /* The FROM clause term that is the vtab */ + ExprList *pOrderBy, /* The ORDER BY clause */ + u16 *pmNoOmit /* Mask of terms not to omit */ +){ + int i, j; + int nTerm; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_orderby *pIdxOrderBy; + struct sqlite3_index_constraint_usage *pUsage; + struct HiddenIndexInfo *pHidden; + WhereTerm *pTerm; + int nOrderBy; + sqlite3_index_info *pIdxInfo; + u16 mNoOmit = 0; + + /* Count the number of possible WHERE clause constraints referring + ** to this virtual table */ + for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + if( pTerm->leftCursor != pSrc->iCursor ) continue; + if( pTerm->prereqRight & mUnusable ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IS ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; + assert( pTerm->u.x.leftColumn>=(-1) ); + nTerm++; + } + + /* If the ORDER BY clause contains only columns in the current + ** virtual table then allocate space for the aOrderBy part of + ** the sqlite3_index_info structure. + */ + nOrderBy = 0; + if( pOrderBy ){ + int n = pOrderBy->nExpr; + for(i=0; ia[i].pExpr; + if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; + if( pOrderBy->a[i].sortFlags & KEYINFO_ORDER_BIGNULL ) break; + } + if( i==n){ + nOrderBy = n; + } + } + + /* Allocate the sqlite3_index_info structure + */ + pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) + + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm + + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden) ); + if( pIdxInfo==0 ){ + sqlite3ErrorMsg(pParse, "out of memory"); + return 0; + } + pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1]; + pIdxCons = (struct sqlite3_index_constraint*)&pHidden[1]; + pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; + pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; + pIdxInfo->nOrderBy = nOrderBy; + pIdxInfo->aConstraint = pIdxCons; + pIdxInfo->aOrderBy = pIdxOrderBy; + pIdxInfo->aConstraintUsage = pUsage; + pHidden->pWC = pWC; + pHidden->pParse = pParse; + for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + u16 op; + if( pTerm->leftCursor != pSrc->iCursor ) continue; + if( pTerm->prereqRight & mUnusable ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_IS ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; + + /* tag-20191211-002: WHERE-clause constraints are not useful to the + ** right-hand table of a LEFT JOIN. See tag-20191211-001 for the + ** equivalent restriction for ordinary tables. */ + if( (pSrc->fg.jointype & JT_LEFT)!=0 + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + ){ + continue; + } + assert( pTerm->u.x.leftColumn>=(-1) ); + pIdxCons[j].iColumn = pTerm->u.x.leftColumn; + pIdxCons[j].iTermOffset = i; + op = pTerm->eOperator & WO_ALL; + if( op==WO_IN ) op = WO_EQ; + if( op==WO_AUX ){ + pIdxCons[j].op = pTerm->eMatchOp; + }else if( op & (WO_ISNULL|WO_IS) ){ + if( op==WO_ISNULL ){ + pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL; + }else{ + pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS; + } + }else{ + pIdxCons[j].op = (u8)op; + /* The direct assignment in the previous line is possible only because + ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The + ** following asserts verify this fact. */ + assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); + assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); + assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); + assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); + assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); + assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) ); + + if( op & (WO_LT|WO_LE|WO_GT|WO_GE) + && sqlite3ExprIsVector(pTerm->pExpr->pRight) + ){ + testcase( j!=i ); + if( j<16 ) mNoOmit |= (1 << j); + if( op==WO_LT ) pIdxCons[j].op = WO_LE; + if( op==WO_GT ) pIdxCons[j].op = WO_GE; + } + } + + j++; + } + pIdxInfo->nConstraint = j; + for(i=0; ia[i].pExpr; + pIdxOrderBy[i].iColumn = pExpr->iColumn; + pIdxOrderBy[i].desc = pOrderBy->a[i].sortFlags & KEYINFO_ORDER_DESC; + } + + *pmNoOmit = mNoOmit; + return pIdxInfo; +} + +/* +** The table object reference passed as the second argument to this function +** must represent a virtual table. This function invokes the xBestIndex() +** method of the virtual table with the sqlite3_index_info object that +** comes in as the 3rd argument to this function. +** +** If an error occurs, pParse is populated with an error message and an +** appropriate error code is returned. A return of SQLITE_CONSTRAINT from +** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that +** the current configuration of "unusable" flags in sqlite3_index_info can +** not result in a valid plan. +** +** Whether or not an error is returned, it is the responsibility of the +** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates +** that this is required. +*/ +static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ + sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; + int rc; + + whereTraceIndexInfoInputs(p); + rc = pVtab->pModule->xBestIndex(pVtab, p); + whereTraceIndexInfoOutputs(p); + + if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ + if( rc==SQLITE_NOMEM ){ + sqlite3OomFault(pParse->db); + }else if( !pVtab->zErrMsg ){ + sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc)); + }else{ + sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); + } + } + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = 0; + return rc; +} +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ + +#ifdef SQLITE_ENABLE_STAT4 +/* +** Estimate the location of a particular key among all keys in an +** index. Store the results in aStat as follows: +** +** aStat[0] Est. number of rows less than pRec +** aStat[1] Est. number of rows equal to pRec +** +** Return the index of the sample that is the smallest sample that +** is greater than or equal to pRec. Note that this index is not an index +** into the aSample[] array - it is an index into a virtual set of samples +** based on the contents of aSample[] and the number of fields in record +** pRec. +*/ +static int whereKeyStats( + Parse *pParse, /* Database connection */ + Index *pIdx, /* Index to consider domain of */ + UnpackedRecord *pRec, /* Vector of values to consider */ + int roundUp, /* Round up if true. Round down if false */ + tRowcnt *aStat /* OUT: stats written here */ +){ + IndexSample *aSample = pIdx->aSample; + int iCol; /* Index of required stats in anEq[] etc. */ + int i; /* Index of first sample >= pRec */ + int iSample; /* Smallest sample larger than or equal to pRec */ + int iMin = 0; /* Smallest sample not yet tested */ + int iTest; /* Next sample to test */ + int res; /* Result of comparison operation */ + int nField; /* Number of fields in pRec */ + tRowcnt iLower = 0; /* anLt[] + anEq[] of largest sample pRec is > */ + +#ifndef SQLITE_DEBUG + UNUSED_PARAMETER( pParse ); +#endif + assert( pRec!=0 ); + assert( pIdx->nSample>0 ); + assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol ); + + /* Do a binary search to find the first sample greater than or equal + ** to pRec. If pRec contains a single field, the set of samples to search + ** is simply the aSample[] array. If the samples in aSample[] contain more + ** than one fields, all fields following the first are ignored. + ** + ** If pRec contains N fields, where N is more than one, then as well as the + ** samples in aSample[] (truncated to N fields), the search also has to + ** consider prefixes of those samples. For example, if the set of samples + ** in aSample is: + ** + ** aSample[0] = (a, 5) + ** aSample[1] = (a, 10) + ** aSample[2] = (b, 5) + ** aSample[3] = (c, 100) + ** aSample[4] = (c, 105) + ** + ** Then the search space should ideally be the samples above and the + ** unique prefixes [a], [b] and [c]. But since that is hard to organize, + ** the code actually searches this set: + ** + ** 0: (a) + ** 1: (a, 5) + ** 2: (a, 10) + ** 3: (a, 10) + ** 4: (b) + ** 5: (b, 5) + ** 6: (c) + ** 7: (c, 100) + ** 8: (c, 105) + ** 9: (c, 105) + ** + ** For each sample in the aSample[] array, N samples are present in the + ** effective sample array. In the above, samples 0 and 1 are based on + ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc. + ** + ** Often, sample i of each block of N effective samples has (i+1) fields. + ** Except, each sample may be extended to ensure that it is greater than or + ** equal to the previous sample in the array. For example, in the above, + ** sample 2 is the first sample of a block of N samples, so at first it + ** appears that it should be 1 field in size. However, that would make it + ** smaller than sample 1, so the binary search would not work. As a result, + ** it is extended to two fields. The duplicates that this creates do not + ** cause any problems. + */ + nField = pRec->nField; + iCol = 0; + iSample = pIdx->nSample * nField; + do{ + int iSamp; /* Index in aSample[] of test sample */ + int n; /* Number of fields in test sample */ + + iTest = (iMin+iSample)/2; + iSamp = iTest / nField; + if( iSamp>0 ){ + /* The proposed effective sample is a prefix of sample aSample[iSamp]. + ** Specifically, the shortest prefix of at least (1 + iTest%nField) + ** fields that is greater than the previous effective sample. */ + for(n=(iTest % nField) + 1; nnField = n; + res = sqlite3VdbeRecordCompare(aSample[iSamp].n, aSample[iSamp].p, pRec); + if( res<0 ){ + iLower = aSample[iSamp].anLt[n-1] + aSample[iSamp].anEq[n-1]; + iMin = iTest+1; + }else if( res==0 && ndb->mallocFailed==0 ){ + if( res==0 ){ + /* If (res==0) is true, then pRec must be equal to sample i. */ + assert( inSample ); + assert( iCol==nField-1 ); + pRec->nField = nField; + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) + || pParse->db->mallocFailed + ); + }else{ + /* Unless i==pIdx->nSample, indicating that pRec is larger than + ** all samples in the aSample[] array, pRec must be smaller than the + ** (iCol+1) field prefix of sample i. */ + assert( i<=pIdx->nSample && i>=0 ); + pRec->nField = iCol+1; + assert( i==pIdx->nSample + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 + || pParse->db->mallocFailed ); + + /* if i==0 and iCol==0, then record pRec is smaller than all samples + ** in the aSample[] array. Otherwise, if (iCol>0) then pRec must + ** be greater than or equal to the (iCol) field prefix of sample i. + ** If (i>0), then pRec must also be greater than sample (i-1). */ + if( iCol>0 ){ + pRec->nField = iCol; + assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 + || pParse->db->mallocFailed ); + } + if( i>0 ){ + pRec->nField = nField; + assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 + || pParse->db->mallocFailed ); + } + } + } +#endif /* ifdef SQLITE_DEBUG */ + + if( res==0 ){ + /* Record pRec is equal to sample i */ + assert( iCol==nField-1 ); + aStat[0] = aSample[i].anLt[iCol]; + aStat[1] = aSample[i].anEq[iCol]; + }else{ + /* At this point, the (iCol+1) field prefix of aSample[i] is the first + ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec + ** is larger than all samples in the array. */ + tRowcnt iUpper, iGap; + if( i>=pIdx->nSample ){ + iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + }else{ + iUpper = aSample[i].anLt[iCol]; + } + + if( iLower>=iUpper ){ + iGap = 0; + }else{ + iGap = iUpper - iLower; + } + if( roundUp ){ + iGap = (iGap*2)/3; + }else{ + iGap = iGap/3; + } + aStat[0] = iLower + iGap; + aStat[1] = pIdx->aAvgEq[nField-1]; + } + + /* Restore the pRec->nField value before returning. */ + pRec->nField = nField; + return i; +} +#endif /* SQLITE_ENABLE_STAT4 */ + +/* +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. +** +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. +*/ +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); + } + } + return nRet; +} + + +#ifdef SQLITE_ENABLE_STAT4 +/* +** Return the affinity for a single column of an index. +*/ +SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ + assert( iCol>=0 && iColnColumn ); + if( !pIdx->zColAff ){ + if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; + } + assert( pIdx->zColAff[iCol]!=0 ); + return pIdx->zColAff[iCol]; +} +#endif + + +#ifdef SQLITE_ENABLE_STAT4 +/* +** This function is called to estimate the number of rows visited by a +** range-scan on a skip-scan index. For example: +** +** CREATE INDEX i1 ON t1(a, b, c); +** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; +** +** Value pLoop->nOut is currently set to the estimated number of rows +** visited for scanning (a=? AND b=?). This function reduces that estimate +** by some factor to account for the (c BETWEEN ? AND ?) expression based +** on the stat4 data for the index. this scan will be peformed multiple +** times (once for each (a,b) combination that matches a=?) is dealt with +** by the caller. +** +** It does this by scanning through all stat4 samples, comparing values +** extracted from pLower and pUpper with the corresponding column in each +** sample. If L and U are the number of samples found to be less than or +** equal to the values extracted from pLower and pUpper respectively, and +** N is the total number of samples, the pLoop->nOut value is adjusted +** as follows: +** +** nOut = nOut * ( min(U - L, 1) / N ) +** +** If pLower is NULL, or a value cannot be extracted from the term, L is +** set to zero. If pUpper is NULL, or a value cannot be extracted from it, +** U is set to N. +** +** Normally, this function sets *pbDone to 1 before returning. However, +** if no value can be extracted from either pLower or pUpper (and so the +** estimate of the number of rows delivered remains unchanged), *pbDone +** is left as is. +** +** If an error occurs, an SQLite error code is returned. Otherwise, +** SQLITE_OK. +*/ +static int whereRangeSkipScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop, /* Update the .nOut value of this loop */ + int *pbDone /* Set to true if at least one expr. value extracted */ +){ + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + sqlite3 *db = pParse->db; + int nLower = -1; + int nUpper = p->nSample+1; + int rc = SQLITE_OK; + u8 aff = sqlite3IndexColumnAffinity(db, p, nEq); + CollSeq *pColl; + + sqlite3_value *p1 = 0; /* Value extracted from pLower */ + sqlite3_value *p2 = 0; /* Value extracted from pUpper */ + sqlite3_value *pVal = 0; /* Value extracted from record */ + + pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); + if( pLower ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); + nLower = 0; + } + if( pUpper && rc==SQLITE_OK ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); + nUpper = p2 ? 0 : p->nSample; + } + + if( p1 || p2 ){ + int i; + int nDiff; + for(i=0; rc==SQLITE_OK && inSample; i++){ + rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); + if( rc==SQLITE_OK && p1 ){ + int res = sqlite3MemCompare(p1, pVal, pColl); + if( res>=0 ) nLower++; + } + if( rc==SQLITE_OK && p2 ){ + int res = sqlite3MemCompare(p2, pVal, pColl); + if( res>=0 ) nUpper++; + } + } + nDiff = (nUpper - nLower); + if( nDiff<=0 ) nDiff = 1; + + /* If there is both an upper and lower bound specified, and the + ** comparisons indicate that they are close together, use the fallback + ** method (assume that the scan visits 1/64 of the rows) for estimating + ** the number of rows visited. Otherwise, estimate the number of rows + ** using the method described in the header comment for this function. */ + if( nDiff!=1 || pUpper==0 || pLower==0 ){ + int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); + pLoop->nOut -= nAdjust; + *pbDone = 1; + WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + nLower, nUpper, nAdjust*-1, pLoop->nOut)); + } + + }else{ + assert( *pbDone==0 ); + } + + sqlite3ValueFree(p1); + sqlite3ValueFree(p2); + sqlite3ValueFree(pVal); + + return rc; +} +#endif /* SQLITE_ENABLE_STAT4 */ + +/* +** This function is used to estimate the number of rows that will be visited +** by scanning an index for a range of values. The range may have an upper +** bound, a lower bound, or both. The WHERE clause terms that set the upper +** and lower bounds are represented by pLower and pUpper respectively. For +** example, assuming that index p is on t1(a): +** +** ... FROM t1 WHERE a > ? AND a < ? ... +** |_____| |_____| +** | | +** pLower pUpper +** +** If either of the upper or lower bound is not present, then NULL is passed in +** place of the corresponding WhereTerm. +** +** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index +** column subject to the range constraint. Or, equivalently, the number of +** equality constraints optimized by the proposed index scan. For example, +** assuming index p is on t1(a, b), and the SQL query is: +** +** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... +** +** then nEq is set to 1 (as the range restricted column, b, is the second +** left-most column of the index). Or, if the query is: +** +** ... FROM t1 WHERE a > ? AND a < ? ... +** +** then nEq is set to 0. +** +** When this function is called, *pnOut is set to the sqlite3LogEst() of the +** number of rows that the index scan is expected to visit without +** considering the range constraints. If nEq is 0, then *pnOut is the number of +** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) +** to account for the range constraints pLower and pUpper. +** +** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be +** used, a single range inequality reduces the search space by a factor of 4. +** and a pair of constraints (x>? AND x123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop /* Modify the .nOut and maybe .rRun fields */ +){ + int rc = SQLITE_OK; + int nOut = pLoop->nOut; + LogEst nNew; + +#ifdef SQLITE_ENABLE_STAT4 + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + + if( p->nSample>0 && ALWAYS(nEqnSampleCol) + && OptimizationEnabled(pParse->db, SQLITE_Stat4) + ){ + if( nEq==pBuilder->nRecValid ){ + UnpackedRecord *pRec = pBuilder->pRec; + tRowcnt a[2]; + int nBtm = pLoop->u.btree.nBtm; + int nTop = pLoop->u.btree.nTop; + + /* Variable iLower will be set to the estimate of the number of rows in + ** the index that are less than the lower bound of the range query. The + ** lower bound being the concatenation of $P and $L, where $P is the + ** key-prefix formed by the nEq values matched against the nEq left-most + ** columns of the index, and $L is the value in pLower. + ** + ** Or, if pLower is NULL or $L cannot be extracted from it (because it + ** is not a simple variable or literal value), the lower bound of the + ** range is $P. Due to a quirk in the way whereKeyStats() works, even + ** if $L is available, whereKeyStats() is called for both ($P) and + ** ($P:$L) and the larger of the two returned values is used. + ** + ** Similarly, iUpper is to be set to the estimate of the number of rows + ** less than the upper bound of the range query. Where the upper bound + ** is either ($P) or ($P:$U). Again, even if $U is available, both values + ** of iUpper are requested of whereKeyStats() and the smaller used. + ** + ** The number of rows between the two bounds is then just iUpper-iLower. + */ + tRowcnt iLower; /* Rows less than the lower bound */ + tRowcnt iUpper; /* Rows less than the upper bound */ + int iLwrIdx = -2; /* aSample[] for the lower bound */ + int iUprIdx = -1; /* aSample[] for the upper bound */ + + if( pRec ){ + testcase( pRec->nField!=pBuilder->nRecValid ); + pRec->nField = pBuilder->nRecValid; + } + /* Determine iLower and iUpper using ($P) only. */ + if( nEq==0 ){ + iLower = 0; + iUpper = p->nRowEst0; + }else{ + /* Note: this call could be optimized away - since the same values must + ** have been requested when testing key $P in whereEqualScanEst(). */ + whereKeyStats(pParse, p, pRec, 0, a); + iLower = a[0]; + iUpper = a[0] + a[1]; + } + + assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + assert( p->aSortOrder!=0 ); + if( p->aSortOrder[nEq] ){ + /* The roles of pLower and pUpper are swapped for a DESC index */ + SWAP(WhereTerm*, pLower, pUpper); + SWAP(int, nBtm, nTop); + } + + /* If possible, improve on the iLower estimate using ($P:$L). */ + if( pLower ){ + int n; /* Values extracted from pExpr */ + Expr *pExpr = pLower->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n); + if( rc==SQLITE_OK && n ){ + tRowcnt iNew; + u16 mask = WO_GT|WO_LE; + if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); + iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); + iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0); + if( iNew>iLower ) iLower = iNew; + nOut--; + pLower = 0; + } + } + + /* If possible, improve on the iUpper estimate using ($P:$U). */ + if( pUpper ){ + int n; /* Values extracted from pExpr */ + Expr *pExpr = pUpper->pExpr->pRight; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n); + if( rc==SQLITE_OK && n ){ + tRowcnt iNew; + u16 mask = WO_GT|WO_LE; + if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); + iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); + iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0); + if( iNewpRec = pRec; + if( rc==SQLITE_OK ){ + if( iUpper>iLower ){ + nNew = sqlite3LogEst(iUpper - iLower); + /* TUNING: If both iUpper and iLower are derived from the same + ** sample, then assume they are 4x more selective. This brings + ** the estimated selectivity more in line with what it would be + ** if estimated without the use of STAT4 tables. */ + if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); + }else{ + nNew = 10; assert( 10==sqlite3LogEst(2) ); + } + if( nNewwtFlags & TERM_VNULL)==0 ); + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); + + /* TUNING: If there is both an upper and lower limit and neither limit + ** has an application-defined likelihood(), assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ + nNew -= 20; + } + + nOut -= (pLower!=0) + (pUpper!=0); + if( nNew<10 ) nNew = 10; + if( nNewnOut>nOut ){ + WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + pLoop->nOut, nOut)); + } +#endif + pLoop->nOut = (LogEst)nOut; + return rc; +} + +#ifdef SQLITE_ENABLE_STAT4 +/* +** Estimate the number of rows that will be returned based on +** an equality constraint x=VALUE and where that VALUE occurs in +** the histogram data. This only works when x is the left-most +** column of an index and sqlite_stat4 histogram data is available +** for that index. When pExpr==NULL that means the constraint is +** "x IS NULL" instead of "x=VALUE". +** +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. +** +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. +*/ +static int whereEqualScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereLoopBuilder *pBuilder, + Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ + tRowcnt *pnRow /* Write the revised row estimate here */ +){ + Index *p = pBuilder->pNew->u.btree.pIndex; + int nEq = pBuilder->pNew->u.btree.nEq; + UnpackedRecord *pRec = pBuilder->pRec; + int rc; /* Subfunction return code */ + tRowcnt a[2]; /* Statistics */ + int bOk; + + assert( nEq>=1 ); + assert( nEq<=p->nColumn ); + assert( p->aSample!=0 ); + assert( p->nSample>0 ); + assert( pBuilder->nRecValidnRecValid<(nEq-1) ){ + return SQLITE_NOTFOUND; + } + + /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() + ** below would return the same value. */ + if( nEq>=p->nColumn ){ + *pnRow = 1; + return SQLITE_OK; + } + + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk); + pBuilder->pRec = pRec; + if( rc!=SQLITE_OK ) return rc; + if( bOk==0 ) return SQLITE_NOTFOUND; + pBuilder->nRecValid = nEq; + + whereKeyStats(pParse, p, pRec, 0, a); + WHERETRACE(0x10,("equality scan regions %s(%d): %d\n", + p->zName, nEq-1, (int)a[1])); + *pnRow = a[1]; + + return rc; +} +#endif /* SQLITE_ENABLE_STAT4 */ + +#ifdef SQLITE_ENABLE_STAT4 +/* +** Estimate the number of rows that will be returned based on +** an IN constraint where the right-hand side of the IN operator +** is a list of values. Example: +** +** WHERE x IN (1,2,3,4) +** +** Write the estimated row count into *pnRow and return SQLITE_OK. +** If unable to make an estimate, leave *pnRow unchanged and return +** non-zero. +** +** This routine can fail if it is unable to load a collating sequence +** required for string comparison, or if unable to allocate memory +** for a UTF conversion required for comparison. The error is stored +** in the pParse structure. +*/ +static int whereInScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereLoopBuilder *pBuilder, + ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ + tRowcnt *pnRow /* Write the revised row estimate here */ +){ + Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); + int nRecValid = pBuilder->nRecValid; + int rc = SQLITE_OK; /* Subfunction return code */ + tRowcnt nEst; /* Number of rows for a single term */ + tRowcnt nRowEst = 0; /* New estimate of the number of rows */ + int i; /* Loop counter */ + + assert( p->aSample!=0 ); + for(i=0; rc==SQLITE_OK && inExpr; i++){ + nEst = nRow0; + rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); + nRowEst += nEst; + pBuilder->nRecValid = nRecValid; + } + + if( rc==SQLITE_OK ){ + if( nRowEst > nRow0 ) nRowEst = nRow0; + *pnRow = nRowEst; + WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); + } + assert( pBuilder->nRecValid==nRecValid ); + return rc; +} +#endif /* SQLITE_ENABLE_STAT4 */ + + +#ifdef WHERETRACE_ENABLED +/* +** Print the content of a WhereTerm object +*/ +SQLITE_PRIVATE void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){ + if( pTerm==0 ){ + sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); + }else{ + char zType[8]; + char zLeft[50]; + memcpy(zType, "....", 5); + if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; + if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C'; + if( pTerm->eOperator & WO_SINGLE ){ + sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}", + pTerm->leftCursor, pTerm->u.x.leftColumn); + }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){ + sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx", + pTerm->u.pOrInfo->indexable); + }else{ + sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor); + } + sqlite3DebugPrintf( + "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x", + iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags); + /* The 0x10000 .wheretrace flag causes extra information to be + ** shown about each Term */ + if( sqlite3WhereTrace & 0x10000 ){ + sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx", + pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight); + } + if( pTerm->u.x.iField ){ + sqlite3DebugPrintf(" iField=%d", pTerm->u.x.iField); + } + if( pTerm->iParent>=0 ){ + sqlite3DebugPrintf(" iParent=%d", pTerm->iParent); + } + sqlite3DebugPrintf("\n"); + sqlite3TreeViewExpr(0, pTerm->pExpr, 0); + } +} +#endif + +#ifdef WHERETRACE_ENABLED +/* +** Show the complete content of a WhereClause +*/ +SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC){ + int i; + for(i=0; inTerm; i++){ + sqlite3WhereTermPrint(&pWC->a[i], i); + } +} +#endif + +#ifdef WHERETRACE_ENABLED +/* +** Print a WhereLoop object for debugging purposes +*/ +SQLITE_PRIVATE void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){ + WhereInfo *pWInfo = pWC->pWInfo; + int nb = 1+(pWInfo->pTabList->nSrc+3)/4; + SrcItem *pItem = pWInfo->pTabList->a + p->iTab; + Table *pTab = pItem->pTab; + Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; + sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, + p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); + sqlite3DebugPrintf(" %12s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + const char *zName; + if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ + if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ + int i = sqlite3Strlen30(zName) - 1; + while( zName[i]!='_' ) i--; + zName += i; + } + sqlite3DebugPrintf(".%-16s %2d", zName, p->u.btree.nEq); + }else{ + sqlite3DebugPrintf("%20s",""); + } + }else{ + char *z; + if( p->u.vtab.idxStr ){ + z = sqlite3_mprintf("(%d,\"%s\",%#x)", + p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); + }else{ + z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); + } + sqlite3DebugPrintf(" %-19s", z); + sqlite3_free(z); + } + if( p->wsFlags & WHERE_SKIPSCAN ){ + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); + }else{ + sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); + } + sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); + if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + int i; + for(i=0; inLTerm; i++){ + sqlite3WhereTermPrint(p->aLTerm[i], i); + } + } +} +#endif + +/* +** Convert bulk memory into a valid WhereLoop that can be passed +** to whereLoopClear harmlessly. +*/ +static void whereLoopInit(WhereLoop *p){ + p->aLTerm = p->aLTermSpace; + p->nLTerm = 0; + p->nLSlot = ArraySize(p->aLTermSpace); + p->wsFlags = 0; +} + +/* +** Clear the WhereLoop.u union. Leave WhereLoop.pLTerm intact. +*/ +static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ + if( p->wsFlags & (WHERE_VIRTUALTABLE|WHERE_AUTO_INDEX) ){ + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 && p->u.vtab.needFree ){ + sqlite3_free(p->u.vtab.idxStr); + p->u.vtab.needFree = 0; + p->u.vtab.idxStr = 0; + }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ + sqlite3DbFree(db, p->u.btree.pIndex->zColAff); + sqlite3DbFreeNN(db, p->u.btree.pIndex); + p->u.btree.pIndex = 0; + } + } +} + +/* +** Deallocate internal memory used by a WhereLoop object +*/ +static void whereLoopClear(sqlite3 *db, WhereLoop *p){ + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm); + whereLoopClearUnion(db, p); + whereLoopInit(p); +} + +/* +** Increase the memory allocation for pLoop->aLTerm[] to be at least n. +*/ +static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ + WhereTerm **paNew; + if( p->nLSlot>=n ) return SQLITE_OK; + n = (n+7)&~7; + paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n); + if( paNew==0 ) return SQLITE_NOMEM_BKPT; + memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm); + p->aLTerm = paNew; + p->nLSlot = n; + return SQLITE_OK; +} + +/* +** Transfer content from the second pLoop into the first. +*/ +static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ + whereLoopClearUnion(db, pTo); + if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ + memset(pTo, 0, WHERE_LOOP_XFER_SZ); + return SQLITE_NOMEM_BKPT; + } + memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); + memcpy(pTo->aLTerm, pFrom->aLTerm, pTo->nLTerm*sizeof(pTo->aLTerm[0])); + if( pFrom->wsFlags & WHERE_VIRTUALTABLE ){ + pFrom->u.vtab.needFree = 0; + }else if( (pFrom->wsFlags & WHERE_AUTO_INDEX)!=0 ){ + pFrom->u.btree.pIndex = 0; + } + return SQLITE_OK; +} + +/* +** Delete a WhereLoop object +*/ +static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ + whereLoopClear(db, p); + sqlite3DbFreeNN(db, p); +} + +/* +** Free a WhereInfo structure +*/ +static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ + int i; + assert( pWInfo!=0 ); + for(i=0; inLevel; i++){ + WhereLevel *pLevel = &pWInfo->a[i]; + if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){ + sqlite3DbFree(db, pLevel->u.in.aInLoop); + } + } + sqlite3WhereClauseClear(&pWInfo->sWC); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); + } + assert( pWInfo->pExprMods==0 ); + sqlite3DbFreeNN(db, pWInfo); +} + +/* Undo all Expr node modifications +*/ +static void whereUndoExprMods(WhereInfo *pWInfo){ + while( pWInfo->pExprMods ){ + WhereExprMod *p = pWInfo->pExprMods; + pWInfo->pExprMods = p->pNext; + memcpy(p->pExpr, &p->orig, sizeof(p->orig)); + sqlite3DbFree(pWInfo->pParse->db, p); + } +} + +/* +** Return TRUE if all of the following are true: +** +** (1) X has the same or lower cost, or returns the same or fewer rows, +** than Y. +** (2) X uses fewer WHERE clause terms than Y +** (3) Every WHERE clause term used by X is also used by Y +** (4) X skips at least as many columns as Y +** (5) If X is a covering index, than Y is too +** +** Conditions (2) and (3) mean that X is a "proper subset" of Y. +** If X is a proper subset of Y then Y is a better choice and ought +** to have a lower cost. This routine returns TRUE when that cost +** relationship is inverted and needs to be adjusted. Constraint (4) +** was added because if X uses skip-scan less than Y it still might +** deserve a lower cost even if it is a proper subset of Y. Constraint (5) +** was added because a covering index probably deserves to have a lower cost +** than a non-covering index even if it is a proper subset. +*/ +static int whereLoopCheaperProperSubset( + const WhereLoop *pX, /* First WhereLoop to compare */ + const WhereLoop *pY /* Compare against this WhereLoop */ +){ + int i, j; + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* X is not a subset of Y */ + } + if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; + if( pY->nSkip > pX->nSkip ) return 0; + for(i=pX->nLTerm-1; i>=0; i--){ + if( pX->aLTerm[i]==0 ) continue; + for(j=pY->nLTerm-1; j>=0; j--){ + if( pY->aLTerm[j]==pX->aLTerm[i] ) break; + } + if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ + } + if( (pX->wsFlags&WHERE_IDX_ONLY)!=0 + && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){ + return 0; /* Constraint (5) */ + } + return 1; /* All conditions meet */ +} + +/* +** Try to adjust the cost and number of output rows of WhereLoop pTemplate +** upwards or downwards so that: +** +** (1) pTemplate costs less than any other WhereLoops that are a proper +** subset of pTemplate +** +** (2) pTemplate costs more than any other WhereLoops for which pTemplate +** is a proper subset. +** +** To say "WhereLoop X is a proper subset of Y" means that X uses fewer +** WHERE clause terms than Y and that every WHERE clause term used by X is +** also used by Y. +*/ +static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ + if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + for(; p; p=p->pNextLoop){ + if( p->iTab!=pTemplate->iTab ) continue; + if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( whereLoopCheaperProperSubset(p, pTemplate) ){ + /* Adjust pTemplate cost downward so that it is cheaper than its + ** subset p. */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, + MIN(p->rRun, pTemplate->rRun), + MIN(p->nOut - 1, pTemplate->nOut))); + pTemplate->rRun = MIN(p->rRun, pTemplate->rRun); + pTemplate->nOut = MIN(p->nOut - 1, pTemplate->nOut); + }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ + /* Adjust pTemplate cost upward so that it is costlier than p since + ** pTemplate is a proper subset of p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, + MAX(p->rRun, pTemplate->rRun), + MAX(p->nOut + 1, pTemplate->nOut))); + pTemplate->rRun = MAX(p->rRun, pTemplate->rRun); + pTemplate->nOut = MAX(p->nOut + 1, pTemplate->nOut); + } + } +} + +/* +** Search the list of WhereLoops in *ppPrev looking for one that can be +** replaced by pTemplate. +** +** Return NULL if pTemplate does not belong on the WhereLoop list. +** In other words if pTemplate ought to be dropped from further consideration. +** +** If pX is a WhereLoop that pTemplate can replace, then return the +** link that points to pX. +** +** If pTemplate cannot replace any existing element of the list but needs +** to be added to the list as a new entry, then return a pointer to the +** tail of the list. +*/ +static WhereLoop **whereLoopFindLesser( + WhereLoop **ppPrev, + const WhereLoop *pTemplate +){ + WhereLoop *p; + for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ + if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ + /* If either the iTab or iSortIdx values for two WhereLoop are different + ** then those WhereLoops need to be considered separately. Neither is + ** a candidate to replace the other. */ + continue; + } + /* In the current implementation, the rSetup value is either zero + ** or the cost of building an automatic index (NlogN) and the NlogN + ** is the same for compatible WhereLoops. */ + assert( p->rSetup==0 || pTemplate->rSetup==0 + || p->rSetup==pTemplate->rSetup ); + + /* whereLoopAddBtree() always generates and inserts the automatic index + ** case first. Hence compatible candidate WhereLoops never have a larger + ** rSetup. Call this SETUP-INVARIANT */ + assert( p->rSetup>=pTemplate->rSetup ); + + /* Any loop using an appliation-defined index (or PRIMARY KEY or + ** UNIQUE constraint) with one or more == constraints is better + ** than an automatic index. Unless it is a skip-scan. */ + if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 + && (pTemplate->nSkip)==0 + && (pTemplate->wsFlags & WHERE_INDEXED)!=0 + && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 + && (p->prereq & pTemplate->prereq)==pTemplate->prereq + ){ + break; + } + + /* If existing WhereLoop p is better than pTemplate, pTemplate can be + ** discarded. WhereLoop p is better if: + ** (1) p has no more dependencies than pTemplate, and + ** (2) p has an equal or lower cost than pTemplate + */ + if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */ + && p->rSetup<=pTemplate->rSetup /* (2a) */ + && p->rRun<=pTemplate->rRun /* (2b) */ + && p->nOut<=pTemplate->nOut /* (2c) */ + ){ + return 0; /* Discard pTemplate */ + } + + /* If pTemplate is always better than p, then cause p to be overwritten + ** with pTemplate. pTemplate is better than p if: + ** (1) pTemplate has no more dependences than p, and + ** (2) pTemplate has an equal or lower cost than p. + */ + if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ + && p->rRun>=pTemplate->rRun /* (2a) */ + && p->nOut>=pTemplate->nOut /* (2b) */ + ){ + assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ + break; /* Cause p to be overwritten by pTemplate */ + } + } + return ppPrev; +} + +/* +** Insert or replace a WhereLoop entry using the template supplied. +** +** An existing WhereLoop entry might be overwritten if the new template +** is better and has fewer dependencies. Or the template will be ignored +** and no insert will occur if an existing WhereLoop is faster and has +** fewer dependencies than the template. Otherwise a new WhereLoop is +** added based on the template. +** +** If pBuilder->pOrSet is not NULL then we care about only the +** prerequisites and rRun and nOut costs of the N best loops. That +** information is gathered in the pBuilder->pOrSet object. This special +** processing mode is used only for OR clause processing. +** +** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we +** still might overwrite similar loops with the new template if the +** new template is better. Loops may be overwritten if the following +** conditions are met: +** +** (1) They have the same iTab. +** (2) They have the same iSortIdx. +** (3) The template has same or fewer dependencies than the current loop +** (4) The template has the same or lower cost than the current loop +*/ +static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ + WhereLoop **ppPrev, *p; + WhereInfo *pWInfo = pBuilder->pWInfo; + sqlite3 *db = pWInfo->pParse->db; + int rc; + + /* Stop the search once we hit the query planner search limit */ + if( pBuilder->iPlanLimit==0 ){ + WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n")); + if( pBuilder->pOrSet ) pBuilder->pOrSet->n = 0; + return SQLITE_DONE; + } + pBuilder->iPlanLimit--; + + whereLoopAdjustCost(pWInfo->pLoops, pTemplate); + + /* If pBuilder->pOrSet is defined, then only keep track of the costs + ** and prereqs. + */ + if( pBuilder->pOrSet!=0 ){ + if( pTemplate->nLTerm ){ +#if WHERETRACE_ENABLED + u16 n = pBuilder->pOrSet->n; + int x = +#endif + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + pTemplate->nOut); +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); + sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + } + return SQLITE_OK; + } + + /* Look for an existing WhereLoop to replace with pTemplate + */ + ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); + + if( ppPrev==0 ){ + /* There already exists a WhereLoop on the list that is better + ** than pTemplate, so just ignore pTemplate */ +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(" skip: "); + sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + return SQLITE_OK; + }else{ + p = *ppPrev; + } + + /* If we reach this point it means that either p[] should be overwritten + ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new + ** WhereLoop and insert it. + */ +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + if( p!=0 ){ + sqlite3DebugPrintf("replace: "); + sqlite3WhereLoopPrint(p, pBuilder->pWC); + sqlite3DebugPrintf(" with: "); + }else{ + sqlite3DebugPrintf(" add: "); + } + sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + if( p==0 ){ + /* Allocate a new WhereLoop to add to the end of the list */ + *ppPrev = p = sqlite3DbMallocRawNN(db, sizeof(WhereLoop)); + if( p==0 ) return SQLITE_NOMEM_BKPT; + whereLoopInit(p); + p->pNextLoop = 0; + }else{ + /* We will be overwriting WhereLoop p[]. But before we do, first + ** go through the rest of the list and delete any other entries besides + ** p[] that are also supplated by pTemplate */ + WhereLoop **ppTail = &p->pNextLoop; + WhereLoop *pToDel; + while( *ppTail ){ + ppTail = whereLoopFindLesser(ppTail, pTemplate); + if( ppTail==0 ) break; + pToDel = *ppTail; + if( pToDel==0 ) break; + *ppTail = pToDel->pNextLoop; +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(" delete: "); + sqlite3WhereLoopPrint(pToDel, pBuilder->pWC); + } +#endif + whereLoopDelete(db, pToDel); + } + } + rc = whereLoopXfer(db, p, pTemplate); + if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + Index *pIndex = p->u.btree.pIndex; + if( pIndex && pIndex->idxType==SQLITE_IDXTYPE_IPK ){ + p->u.btree.pIndex = 0; + } + } + return rc; +} + +/* +** Adjust the WhereLoop.nOut value downward to account for terms of the +** WHERE clause that reference the loop but which are not used by an +** index. +* +** For every WHERE clause term that is not used by the index +** and which has a truth probability assigned by one of the likelihood(), +** likely(), or unlikely() SQL functions, reduce the estimated number +** of output rows by the probability specified. +** +** TUNING: For every WHERE clause term that is not used by the index +** and which does not have an assigned truth probability, heuristics +** described below are used to try to estimate the truth probability. +** TODO --> Perhaps this is something that could be improved by better +** table statistics. +** +** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75% +** value corresponds to -1 in LogEst notation, so this means decrement +** the WhereLoop.nOut field for every such WHERE clause term. +** +** Heuristic 2: If there exists one or more WHERE clause terms of the +** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the +** final output row estimate is no greater than 1/4 of the total number +** of rows in the table. In other words, assume that x==EXPR will filter +** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the +** "x" column is boolean or else -1 or 0 or 1 is a common default value +** on the "x" column and so in that case only cap the output row estimate +** at 1/2 instead of 1/4. +*/ +static void whereLoopOutputAdjust( + WhereClause *pWC, /* The WHERE clause */ + WhereLoop *pLoop, /* The loop to adjust downward */ + LogEst nRow /* Number of rows in the entire table */ +){ + WhereTerm *pTerm, *pX; + Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); + int i, j; + LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ + + assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); + for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ + assert( pTerm!=0 ); + if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; + if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; + if( (pTerm->prereqAll & notAllowed)!=0 ) continue; + for(j=pLoop->nLTerm-1; j>=0; j--){ + pX = pLoop->aLTerm[j]; + if( pX==0 ) continue; + if( pX==pTerm ) break; + if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; + } + if( j<0 ){ + if( pTerm->truthProb<=0 ){ + /* If a truth probability is specified using the likelihood() hints, + ** then use the probability provided by the application. */ + pLoop->nOut += pTerm->truthProb; + }else{ + /* In the absence of explicit truth probabilities, use heuristics to + ** guess a reasonable truth probability. */ + pLoop->nOut--; + if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 + && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */ + ){ + Expr *pRight = pTerm->pExpr->pRight; + int k = 0; + testcase( pTerm->pExpr->op==TK_IS ); + if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + k = 10; + }else{ + k = 20; + } + if( iReducewtFlags |= TERM_HEURTRUTH; + iReduce = k; + } + } + } + } + } + if( pLoop->nOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce; +} + +/* +** Term pTerm is a vector range comparison operation. The first comparison +** in the vector can be optimized using column nEq of the index. This +** function returns the total number of vector elements that can be used +** as part of the range comparison. +** +** For example, if the query is: +** +** WHERE a = ? AND (b, c, d) > (?, ?, ?) +** +** and the index: +** +** CREATE INDEX ... ON (a, b, c, d, e) +** +** then this function would be invoked with nEq=1. The value returned in +** this case is 3. +*/ +static int whereRangeVectorLen( + Parse *pParse, /* Parsing context */ + int iCur, /* Cursor open on pIdx */ + Index *pIdx, /* The index to be used for a inequality constraint */ + int nEq, /* Number of prior equality constraints on same index */ + WhereTerm *pTerm /* The vector inequality constraint */ +){ + int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft); + int i; + + nCmp = MIN(nCmp, (pIdx->nColumn - nEq)); + for(i=1; ipExpr->pLeft->x.pList->a[i].pExpr; + Expr *pRhs = pTerm->pExpr->pRight; + if( pRhs->flags & EP_xIsSelect ){ + pRhs = pRhs->x.pSelect->pEList->a[i].pExpr; + }else{ + pRhs = pRhs->x.pList->a[i].pExpr; + } + + /* Check that the LHS of the comparison is a column reference to + ** the right column of the right source table. And that the sort + ** order of the index column is the same as the sort order of the + ** leftmost index column. */ + if( pLhs->op!=TK_COLUMN + || pLhs->iTable!=iCur + || pLhs->iColumn!=pIdx->aiColumn[i+nEq] + || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq] + ){ + break; + } + + testcase( pLhs->iColumn==XN_ROWID ); + aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs)); + idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn); + if( aff!=idxaff ) break; + + pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + if( pColl==0 ) break; + if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break; + } + return i; +} + +/* +** Adjust the cost C by the costMult facter T. This only occurs if +** compiled with -DSQLITE_ENABLE_COSTMULT +*/ +#ifdef SQLITE_ENABLE_COSTMULT +# define ApplyCostMultiplier(C,T) C += T +#else +# define ApplyCostMultiplier(C,T) +#endif + +/* +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the +** index pIndex. Try to match one more. +** +** When this function is called, pBuilder->pNew->nOut contains the +** number of rows expected to be visited by filtering using the nEq +** terms only. If it is modified, this value is restored before this +** function returns. +** +** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is +** a fake index used for the INTEGER PRIMARY KEY. +*/ +static int whereLoopAddBtreeIndex( + WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ + SrcItem *pSrc, /* FROM clause term being analyzed */ + Index *pProbe, /* An index on pSrc */ + LogEst nInMul /* log(Number of iterations due to IN) */ +){ + WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + sqlite3 *db = pParse->db; /* Database connection malloc context */ + WhereLoop *pNew; /* Template WhereLoop under construction */ + WhereTerm *pTerm; /* A WhereTerm under consideration */ + int opMask; /* Valid operators for constraints */ + WhereScan scan; /* Iterator for WHERE terms */ + Bitmask saved_prereq; /* Original value of pNew->prereq */ + u16 saved_nLTerm; /* Original value of pNew->nLTerm */ + u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ + u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */ + u16 saved_nTop; /* Original value of pNew->u.btree.nTop */ + u16 saved_nSkip; /* Original value of pNew->nSkip */ + u32 saved_wsFlags; /* Original value of pNew->wsFlags */ + LogEst saved_nOut; /* Original value of pNew->nOut */ + int rc = SQLITE_OK; /* Return code */ + LogEst rSize; /* Number of rows in the table */ + LogEst rLogSize; /* Logarithm of table size */ + WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ + + pNew = pBuilder->pNew; + if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; + WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n", + pProbe->pTable->zName,pProbe->zName, + pNew->u.btree.nEq, pNew->nSkip, pNew->rRun)); + + assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); + assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); + if( pNew->wsFlags & WHERE_BTM_LIMIT ){ + opMask = WO_LT|WO_LE; + }else{ + assert( pNew->u.btree.nBtm==0 ); + opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; + } + if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); + + assert( pNew->u.btree.nEqnColumn ); + assert( pNew->u.btree.nEqnKeyCol + || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY ); + + saved_nEq = pNew->u.btree.nEq; + saved_nBtm = pNew->u.btree.nBtm; + saved_nTop = pNew->u.btree.nTop; + saved_nSkip = pNew->nSkip; + saved_nLTerm = pNew->nLTerm; + saved_wsFlags = pNew->wsFlags; + saved_prereq = pNew->prereq; + saved_nOut = pNew->nOut; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, saved_nEq, + opMask, pProbe); + pNew->rSetup = 0; + rSize = pProbe->aiRowLogEst[0]; + rLogSize = estLog(rSize); + for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ + LogEst rCostIdx; + LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ + int nIn = 0; +#ifdef SQLITE_ENABLE_STAT4 + int nRecValid = pBuilder->nRecValid; +#endif + if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) + && indexColumnNotNull(pProbe, saved_nEq) + ){ + continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ + } + if( pTerm->prereqRight & pNew->maskSelf ) continue; + + /* Do not allow the upper bound of a LIKE optimization range constraint + ** to mix with a lower range bound from some other source */ + if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; + + /* tag-20191211-001: Do not allow constraints from the WHERE clause to + ** be used by the right table of a LEFT JOIN. Only constraints in the + ** ON clause are allowed. See tag-20191211-002 for the vtab equivalent. */ + if( (pSrc->fg.jointype & JT_LEFT)!=0 + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + ){ + continue; + } + + if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ + pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE; + }else{ + pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED; + } + pNew->wsFlags = saved_wsFlags; + pNew->u.btree.nEq = saved_nEq; + pNew->u.btree.nBtm = saved_nBtm; + pNew->u.btree.nTop = saved_nTop; + pNew->nLTerm = saved_nLTerm; + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTerm; + pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; + + assert( nInMul==0 + || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 + || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 + || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 + ); + + if( eOp & WO_IN ){ + Expr *pExpr = pTerm->pExpr; + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ + int i; + nIn = 46; assert( 46==sqlite3LogEst(25) ); + + /* The expression may actually be of the form (x, y) IN (SELECT...). + ** In this case there is a separate term for each of (x) and (y). + ** However, the nIn multiplier should only be applied once, not once + ** for each such term. The following loop checks that pTerm is the + ** first such term in use, and sets nIn back to 0 if it is not. */ + for(i=0; inLTerm-1; i++){ + if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0; + } + }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ + /* "x IN (value, value, ...)" */ + nIn = sqlite3LogEst(pExpr->x.pList->nExpr); + } + if( pProbe->hasStat1 && rLogSize>=10 ){ + LogEst M, logK, x; + /* Let: + ** N = the total number of rows in the table + ** K = the number of entries on the RHS of the IN operator + ** M = the number of rows in the table that match terms to the + ** to the left in the same index. If the IN operator is on + ** the left-most index column, M==N. + ** + ** Given the definitions above, it is better to omit the IN operator + ** from the index lookup and instead do a scan of the M elements, + ** testing each scanned row against the IN operator separately, if: + ** + ** M*log(K) < K*log(N) + ** + ** Our estimates for M, K, and N might be inaccurate, so we build in + ** a safety margin of 2 (LogEst: 10) that favors using the IN operator + ** with the index, as using an index has better worst-case behavior. + ** If we do not have real sqlite_stat1 data, always prefer to use + ** the index. Do not bother with this optimization on very small + ** tables (less than 2 rows) as it is pointless in that case. + */ + M = pProbe->aiRowLogEst[saved_nEq]; + logK = estLog(nIn); + /* TUNING v----- 10 to bias toward indexed IN */ + x = M + logK + 10 - (nIn + rLogSize); + if( x>=0 ){ + WHERETRACE(0x40, + ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) " + "prefers indexed lookup\n", + saved_nEq, M, logK, nIn, rLogSize, x)); + }else if( nInMul<2 && OptimizationEnabled(db, SQLITE_SeekScan) ){ + WHERETRACE(0x40, + ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d" + " nInMul=%d) prefers skip-scan\n", + saved_nEq, M, logK, nIn, rLogSize, x, nInMul)); + pNew->wsFlags |= WHERE_IN_SEEKSCAN; + }else{ + WHERETRACE(0x40, + ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d" + " nInMul=%d) prefers normal scan\n", + saved_nEq, M, logK, nIn, rLogSize, x, nInMul)); + continue; + } + } + pNew->wsFlags |= WHERE_COLUMN_IN; + }else if( eOp & (WO_EQ|WO_IS) ){ + int iCol = pProbe->aiColumn[saved_nEq]; + pNew->wsFlags |= WHERE_COLUMN_EQ; + assert( saved_nEq==pNew->u.btree.nEq ); + if( iCol==XN_ROWID + || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) + ){ + if( iCol==XN_ROWID || pProbe->uniqNotNull + || (pProbe->nKeyCol==1 && pProbe->onError && eOp==WO_EQ) + ){ + pNew->wsFlags |= WHERE_ONEROW; + }else{ + pNew->wsFlags |= WHERE_UNQ_WANTED; + } + } + if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS; + }else if( eOp & WO_ISNULL ){ + pNew->wsFlags |= WHERE_COLUMN_NULL; + }else if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pNew->u.btree.nBtm = whereRangeVectorLen( + pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm + ); + pBtm = pTerm; + pTop = 0; + if( pTerm->wtFlags & TERM_LIKEOPT ){ + /* Range constraints that come from the LIKE optimization are + ** always used in pairs. */ + pTop = &pTerm[1]; + assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); + assert( pTop->wtFlags & TERM_LIKEOPT ); + assert( pTop->eOperator==WO_LT ); + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTop; + pNew->wsFlags |= WHERE_TOP_LIMIT; + pNew->u.btree.nTop = 1; + } + }else{ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pNew->u.btree.nTop = whereRangeVectorLen( + pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm + ); + pTop = pTerm; + pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? + pNew->aLTerm[pNew->nLTerm-2] : 0; + } + + /* At this point pNew->nOut is set to the number of rows expected to + ** be visited by the index scan before considering term pTerm, or the + ** values of nIn and nInMul. In other words, assuming that all + ** "x IN(...)" terms are replaced with "x = ?". This block updates + ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ + assert( pNew->nOut==saved_nOut ); + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + /* Adjust nOut using stat4 data. Or, if there is no stat4 + ** data, using some other estimate. */ + whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); + }else{ + int nEq = ++pNew->u.btree.nEq; + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN|WO_IS) ); + + assert( pNew->nOut==saved_nOut ); + if( pTerm->truthProb<=0 && pProbe->aiColumn[saved_nEq]>=0 ){ + assert( (eOp & WO_IN) || nIn==0 ); + testcase( eOp & WO_IN ); + pNew->nOut += pTerm->truthProb; + pNew->nOut -= nIn; + }else{ +#ifdef SQLITE_ENABLE_STAT4 + tRowcnt nOut = 0; + if( nInMul==0 + && pProbe->nSample + && ALWAYS(pNew->u.btree.nEq<=pProbe->nSampleCol) + && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + && OptimizationEnabled(db, SQLITE_Stat4) + ){ + Expr *pExpr = pTerm->pExpr; + if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ + testcase( eOp & WO_EQ ); + testcase( eOp & WO_IS ); + testcase( eOp & WO_ISNULL ); + rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); + }else{ + rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); + } + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ + if( nOut ){ + pNew->nOut = sqlite3LogEst(nOut); + if( nEq==1 + /* TUNING: Mark terms as "low selectivity" if they seem likely + ** to be true for half or more of the rows in the table. + ** See tag-202002240-1 */ + && pNew->nOut+10 > pProbe->aiRowLogEst[0] + ){ +#if WHERETRACE_ENABLED /* 0x01 */ + if( sqlite3WhereTrace & 0x01 ){ + sqlite3DebugPrintf( + "STAT4 determines term has low selectivity:\n"); + sqlite3WhereTermPrint(pTerm, 999); + } +#endif + pTerm->wtFlags |= TERM_HIGHTRUTH; + if( pTerm->wtFlags & TERM_HEURTRUTH ){ + /* If the term has previously been used with an assumption of + ** higher selectivity, then set the flag to rerun the + ** loop computations. */ + pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS; + } + } + if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + pNew->nOut -= nIn; + } + } + if( nOut==0 ) +#endif + { + pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); + if( eOp & WO_ISNULL ){ + /* TUNING: If there is no likelihood() value, assume that a + ** "col IS NULL" expression matches twice as many rows + ** as (col=?). */ + pNew->nOut += 10; + } + } + } + } + + /* Set rCostIdx to the cost of visiting selected rows in index. Add + ** it to pNew->rRun, which is currently set to the cost of the index + ** seek only. Then, if this is a non-covering index, add the cost of + ** visiting the rows in the main table. */ + assert( pSrc->pTab->szTabRow>0 ); + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); + } + ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); + + nOutUnadjusted = pNew->nOut; + pNew->rRun += nInMul + nIn; + pNew->nOut += nInMul + nIn; + whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); + rc = whereLoopInsert(pBuilder, pNew); + + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + pNew->nOut = saved_nOut; + }else{ + pNew->nOut = nOutUnadjusted; + } + + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 + && pNew->u.btree.nEqnColumn + && (pNew->u.btree.nEqnKeyCol || + pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) + ){ + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); + } + pNew->nOut = saved_nOut; +#ifdef SQLITE_ENABLE_STAT4 + pBuilder->nRecValid = nRecValid; +#endif + } + pNew->prereq = saved_prereq; + pNew->u.btree.nEq = saved_nEq; + pNew->u.btree.nBtm = saved_nBtm; + pNew->u.btree.nTop = saved_nTop; + pNew->nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + pNew->nOut = saved_nOut; + pNew->nLTerm = saved_nLTerm; + + /* Consider using a skip-scan if there are no WHERE clause constraints + ** available for the left-most terms of the index, and if the average + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); + if( saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && saved_nEq==pNew->nLTerm + && pProbe->noSkipScan==0 + && pProbe->hasStat1!=0 + && OptimizationEnabled(db, SQLITE_SkipScan) + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK + ){ + LogEst nIter; + pNew->u.btree.nEq++; + pNew->nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; + pNew->u.btree.nEq = saved_nEq; + pNew->nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + } + + WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n", + pProbe->pTable->zName, pProbe->zName, saved_nEq, rc)); + return rc; +} + +/* +** Return True if it is possible that pIndex might be useful in +** implementing the ORDER BY clause in pBuilder. +** +** Return False if pBuilder does not contain an ORDER BY clause or +** if there is no way for pIndex to be useful in implementing that +** ORDER BY clause. +*/ +static int indexMightHelpWithOrderBy( + WhereLoopBuilder *pBuilder, + Index *pIndex, + int iCursor +){ + ExprList *pOB; + ExprList *aColExpr; + int ii, jj; + + if( pIndex->bUnordered ) return 0; + if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr); + if( NEVER(pExpr==0) ) continue; + if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ + if( pExpr->iColumn<0 ) return 1; + for(jj=0; jjnKeyCol; jj++){ + if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; + } + }else if( (aColExpr = pIndex->aColExpr)!=0 ){ + for(jj=0; jjnKeyCol; jj++){ + if( pIndex->aiColumn[jj]!=XN_EXPR ) continue; + if( sqlite3ExprCompareSkip(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){ + return 1; + } + } + } + } + return 0; +} + +/* Check to see if a partial index with pPartIndexWhere can be used +** in the current query. Return true if it can be and false if not. +*/ +static int whereUsablePartialIndex( + int iTab, /* The table for which we want an index */ + int isLeft, /* True if iTab is the right table of a LEFT JOIN */ + WhereClause *pWC, /* The WHERE clause of the query */ + Expr *pWhere /* The WHERE clause from the partial index */ +){ + int i; + WhereTerm *pTerm; + Parse *pParse = pWC->pWInfo->pParse; + while( pWhere->op==TK_AND ){ + if( !whereUsablePartialIndex(iTab,isLeft,pWC,pWhere->pLeft) ) return 0; + pWhere = pWhere->pRight; + } + if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0; + for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + Expr *pExpr; + pExpr = pTerm->pExpr; + if( (!ExprHasProperty(pExpr, EP_FromJoin) || pExpr->iRightJoinTable==iTab) + && (isLeft==0 || ExprHasProperty(pExpr, EP_FromJoin)) + && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab) + && (pTerm->wtFlags & TERM_VNULL)==0 + ){ + return 1; + } + } + return 0; +} + +/* +** Add all WhereLoop objects for a single table of the join where the table +** is identified by pBuilder->pNew->iTab. That table is guaranteed to be +** a b-tree table, not a virtual table. +** +** The costs (WhereLoop.rRun) of the b-tree loops added by this function +** are calculated as follows: +** +** For a full scan, assuming the table (or index) contains nRow rows: +** +** cost = nRow * 3.0 // full-table scan +** cost = nRow * K // scan of covering index +** cost = nRow * (K+3.0) // scan of non-covering index +** +** where K is a value between 1.1 and 3.0 set based on the relative +** estimated average size of the index and table records. +** +** For an index scan, where nVisit is the number of index rows visited +** by the scan, and nSeek is the number of seek operations required on +** the index b-tree: +** +** cost = nSeek * (log(nRow) + K * nVisit) // covering index +** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index +** +** Normally, nSeek is 1. nSeek values greater than 1 come about if the +** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when +** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. +** +** The estimated values (nRow, nVisit, nSeek) often contain a large amount +** of uncertainty. For this reason, scoring is designed to pick plans that +** "do the least harm" if the estimates are inaccurate. For example, a +** log(nRow) factor is omitted from a non-covering index scan in order to +** bias the scoring in favor of using an index, since the worst-case +** performance of using an index is far better than the worst-case performance +** of a full table scan. +*/ +static int whereLoopAddBtree( + WhereLoopBuilder *pBuilder, /* WHERE clause information */ + Bitmask mPrereq /* Extra prerequesites for using this table */ +){ + WhereInfo *pWInfo; /* WHERE analysis context */ + Index *pProbe; /* An index we are evaluating */ + Index sPk; /* A fake index object for the primary key */ + LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ + i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ + SrcList *pTabList; /* The FROM clause */ + SrcItem *pSrc; /* The FROM clause btree term to add */ + WhereLoop *pNew; /* Template WhereLoop object */ + int rc = SQLITE_OK; /* Return code */ + int iSortIdx = 1; /* Index number */ + int b; /* A boolean value */ + LogEst rSize; /* number of rows in the table */ + WhereClause *pWC; /* The parsed WHERE clause */ + Table *pTab; /* Table being queried */ + + pNew = pBuilder->pNew; + pWInfo = pBuilder->pWInfo; + pTabList = pWInfo->pTabList; + pSrc = pTabList->a + pNew->iTab; + pTab = pSrc->pTab; + pWC = pBuilder->pWC; + assert( !IsVirtual(pSrc->pTab) ); + + if( pSrc->fg.isIndexedBy ){ + /* An INDEXED BY clause specifies a particular index to use */ + pProbe = pSrc->u2.pIBIndex; + }else if( !HasRowid(pTab) ){ + pProbe = pTab->pIndex; + }else{ + /* There is no INDEXED BY clause. Create a fake Index object in local + ** variable sPk to represent the rowid primary key index. Make this + ** fake index the first in a chain of Index objects with all of the real + ** indices to follow */ + Index *pFirst; /* First of real indices on the table */ + memset(&sPk, 0, sizeof(Index)); + sPk.nKeyCol = 1; + sPk.nColumn = 1; + sPk.aiColumn = &aiColumnPk; + sPk.aiRowLogEst = aiRowEstPk; + sPk.onError = OE_Replace; + sPk.pTable = pTab; + sPk.szIdxRow = pTab->szTabRow; + sPk.idxType = SQLITE_IDXTYPE_IPK; + aiRowEstPk[0] = pTab->nRowLogEst; + aiRowEstPk[1] = 0; + pFirst = pSrc->pTab->pIndex; + if( pSrc->fg.notIndexed==0 ){ + /* The real indices of the table are only considered if the + ** NOT INDEXED qualifier is omitted from the FROM clause */ + sPk.pNext = pFirst; + } + pProbe = &sPk; + } + rSize = pTab->nRowLogEst; + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX + /* Automatic indexes */ + if( !pBuilder->pOrSet /* Not part of an OR optimization */ + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 + && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 + && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */ + && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ + && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ + && !pSrc->fg.isCorrelated /* Not a correlated subquery */ + && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ + ){ + /* Generate auto-index WhereLoops */ + LogEst rLogSize; /* Logarithm of the number of rows in the table */ + WhereTerm *pTerm; + WhereTerm *pWCEnd = pWC->a + pWC->nTerm; + rLogSize = estLog(rSize); + for(pTerm=pWC->a; rc==SQLITE_OK && pTermprereqRight & pNew->maskSelf ) continue; + if( termCanDriveIndex(pTerm, pSrc, 0) ){ + pNew->u.btree.nEq = 1; + pNew->nSkip = 0; + pNew->u.btree.pIndex = 0; + pNew->nLTerm = 1; + pNew->aLTerm[0] = pTerm; + /* TUNING: One-time cost for computing the automatic index is + ** estimated to be X*N*log2(N) where N is the number of rows in + ** the table being indexed and where X is 7 (LogEst=28) for normal + ** tables or 0.5 (LogEst=-10) for views and subqueries. The value + ** of X is smaller for views and subqueries so that the query planner + ** will be more aggressive about generating automatic indexes for + ** those objects, since there is no opportunity to add schema + ** indexes on subqueries and views. */ + pNew->rSetup = rLogSize + rSize; + if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){ + pNew->rSetup += 28; + }else{ + pNew->rSetup -= 10; + } + ApplyCostMultiplier(pNew->rSetup, pTab->costMult); + if( pNew->rSetup<0 ) pNew->rSetup = 0; + /* TUNING: Each index lookup yields 20 rows in the table. This + ** is more than the usual guess of 10 rows, since we have no way + ** of knowing how selective the index will ultimately be. It would + ** not be unreasonable to make this value much larger. */ + pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); + pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); + pNew->wsFlags = WHERE_AUTO_INDEX; + pNew->prereq = mPrereq | pTerm->prereqRight; + rc = whereLoopInsert(pBuilder, pNew); + } + } + } +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + + /* Loop over all indices. If there was an INDEXED BY clause, then only + ** consider index pProbe. */ + for(; rc==SQLITE_OK && pProbe; + pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++ + ){ + int isLeft = (pSrc->fg.jointype & JT_OUTER)!=0; + if( pProbe->pPartIdxWhere!=0 + && !whereUsablePartialIndex(pSrc->iCursor, isLeft, pWC, + pProbe->pPartIdxWhere) + ){ + testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ + continue; /* Partial index inappropriate for this query */ + } + if( pProbe->bNoQuery ) continue; + rSize = pProbe->aiRowLogEst[0]; + pNew->u.btree.nEq = 0; + pNew->u.btree.nBtm = 0; + pNew->u.btree.nTop = 0; + pNew->nSkip = 0; + pNew->nLTerm = 0; + pNew->iSortIdx = 0; + pNew->rSetup = 0; + pNew->prereq = mPrereq; + pNew->nOut = rSize; + pNew->u.btree.pIndex = pProbe; + b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); + + /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ + assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 ); + if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ + /* Integer primary key index */ + pNew->wsFlags = WHERE_IPK; + + /* Full table scan */ + pNew->iSortIdx = b ? iSortIdx : 0; + /* TUNING: Cost of full table scan is 3.0*N. The 3.0 factor is an + ** extra cost designed to discourage the use of full table scans, + ** since index lookups have better worst-case performance if our + ** stat guesses are wrong. Reduce the 3.0 penalty slightly + ** (to 2.75) if we have valid STAT4 information for the table. + ** At 2.75, a full table scan is preferred over using an index on + ** a column with just two distinct values where each value has about + ** an equal number of appearances. Without STAT4 data, we still want + ** to use an index in that case, since the constraint might be for + ** the scarcer of the two values, and in that case an index lookup is + ** better. + */ +#ifdef SQLITE_ENABLE_STAT4 + pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0); +#else + pNew->rRun = rSize + 16; +#endif + ApplyCostMultiplier(pNew->rRun, pTab->costMult); + whereLoopOutputAdjust(pWC, pNew, rSize); + rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = rSize; + if( rc ) break; + }else{ + Bitmask m; + if( pProbe->isCovering ){ + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; + m = 0; + }else{ + m = pSrc->colUsed & pProbe->colNotIdxed; + pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; + } + + /* Full scan via index */ + if( b + || !HasRowid(pTab) + || pProbe->pPartIdxWhere!=0 + || pSrc->fg.isIndexedBy + || ( m==0 + && pProbe->bUnordered==0 + && (pProbe->szIdxRowszTabRow) + && (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 + && sqlite3GlobalConfig.bUseCis + && OptimizationEnabled(pWInfo->pParse->db, SQLITE_CoverIdxScan) + ) + ){ + pNew->iSortIdx = b ? iSortIdx : 0; + + /* The cost of visiting the index rows is N*K, where K is + ** between 1.1 and 3.0, depending on the relative sizes of the + ** index and table rows. */ + pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; + if( m!=0 ){ + /* If this is a non-covering index scan, add in the cost of + ** doing table lookups. The cost will be 3x the number of + ** lookups. Take into account WHERE clause terms that can be + ** satisfied using just the index, and that do not require a + ** table lookup. */ + LogEst nLookup = rSize + 16; /* Base cost: N*3 */ + int ii; + int iCur = pSrc->iCursor; + WhereClause *pWC2 = &pWInfo->sWC; + for(ii=0; iinTerm; ii++){ + WhereTerm *pTerm = &pWC2->a[ii]; + if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){ + break; + } + /* pTerm can be evaluated using just the index. So reduce + ** the expected number of table lookups accordingly */ + if( pTerm->truthProb<=0 ){ + nLookup += pTerm->truthProb; + }else{ + nLookup--; + if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19; + } + } + + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup); + } + ApplyCostMultiplier(pNew->rRun, pTab->costMult); + whereLoopOutputAdjust(pWC, pNew, rSize); + rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = rSize; + if( rc ) break; + } + } + + pBuilder->bldFlags1 = 0; + rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); + if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){ + /* If a non-unique index is used, or if a prefix of the key for + ** unique index is used (making the index functionally non-unique) + ** then the sqlite_stat1 data becomes important for scoring the + ** plan */ + pTab->tabFlags |= TF_StatsUsed; + } +#ifdef SQLITE_ENABLE_STAT4 + sqlite3Stat4ProbeFree(pBuilder->pRec); + pBuilder->nRecValid = 0; + pBuilder->pRec = 0; +#endif + } + return rc; +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Argument pIdxInfo is already populated with all constraints that may +** be used by the virtual table identified by pBuilder->pNew->iTab. This +** function marks a subset of those constraints usable, invokes the +** xBestIndex method and adds the returned plan to pBuilder. +** +** A constraint is marked usable if: +** +** * Argument mUsable indicates that its prerequisites are available, and +** +** * It is not one of the operators specified in the mExclude mask passed +** as the fourth argument (which in practice is either WO_IN or 0). +** +** Argument mPrereq is a mask of tables that must be scanned before the +** virtual table in question. These are added to the plans prerequisites +** before it is added to pBuilder. +** +** Output parameter *pbIn is set to true if the plan added to pBuilder +** uses one or more WO_IN terms, or false otherwise. +*/ +static int whereLoopAddVirtualOne( + WhereLoopBuilder *pBuilder, + Bitmask mPrereq, /* Mask of tables that must be used. */ + Bitmask mUsable, /* Mask of usable tables */ + u16 mExclude, /* Exclude terms using these operators */ + sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */ + u16 mNoOmit, /* Do not omit these constraints */ + int *pbIn /* OUT: True if plan uses an IN(...) op */ +){ + WhereClause *pWC = pBuilder->pWC; + struct sqlite3_index_constraint *pIdxCons; + struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage; + int i; + int mxTerm; + int rc = SQLITE_OK; + WhereLoop *pNew = pBuilder->pNew; + Parse *pParse = pBuilder->pWInfo->pParse; + SrcItem *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab]; + int nConstraint = pIdxInfo->nConstraint; + + assert( (mUsable & mPrereq)==mPrereq ); + *pbIn = 0; + pNew->prereq = mPrereq; + + /* Set the usable flag on the subset of constraints identified by + ** arguments mUsable and mExclude. */ + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; ia[pIdxCons->iTermOffset]; + pIdxCons->usable = 0; + if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight + && (pTerm->eOperator & mExclude)==0 + ){ + pIdxCons->usable = 1; + } + } + + /* Initialize the output fields of the sqlite3_index_info structure */ + memset(pUsage, 0, sizeof(pUsage[0])*nConstraint); + assert( pIdxInfo->needToFreeIdxStr==0 ); + pIdxInfo->idxStr = 0; + pIdxInfo->idxNum = 0; + pIdxInfo->orderByConsumed = 0; + pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; + pIdxInfo->estimatedRows = 25; + pIdxInfo->idxFlags = 0; + pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; + + /* Invoke the virtual table xBestIndex() method */ + rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); + if( rc ){ + if( rc==SQLITE_CONSTRAINT ){ + /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means + ** that the particular combination of parameters provided is unusable. + ** Make no entries in the loop table. + */ + WHERETRACE(0xffff, (" ^^^^--- non-viable plan rejected!\n")); + return SQLITE_OK; + } + return rc; + } + + mxTerm = -1; + assert( pNew->nLSlot>=nConstraint ); + for(i=0; iaLTerm[i] = 0; + pNew->u.vtab.omitMask = 0; + pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; + for(i=0; i=0 ){ + WhereTerm *pTerm; + int j = pIdxCons->iTermOffset; + if( iTerm>=nConstraint + || j<0 + || j>=pWC->nTerm + || pNew->aLTerm[iTerm]!=0 + || pIdxCons->usable==0 + ){ + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); + testcase( pIdxInfo->needToFreeIdxStr ); + return SQLITE_ERROR; + } + testcase( iTerm==nConstraint-1 ); + testcase( j==0 ); + testcase( j==pWC->nTerm-1 ); + pTerm = &pWC->a[j]; + pNew->prereq |= pTerm->prereqRight; + assert( iTermnLSlot ); + pNew->aLTerm[iTerm] = pTerm; + if( iTerm>mxTerm ) mxTerm = iTerm; + testcase( iTerm==15 ); + testcase( iTerm==16 ); + if( pUsage[i].omit ){ + if( i<16 && ((1<u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; + pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE; + *pbIn = 1; assert( (mExclude & WO_IN)==0 ); + } + } + } + + pNew->nLTerm = mxTerm+1; + for(i=0; i<=mxTerm; i++){ + if( pNew->aLTerm[i]==0 ){ + /* The non-zero argvIdx values must be contiguous. Raise an + ** error if they are not */ + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); + testcase( pIdxInfo->needToFreeIdxStr ); + return SQLITE_ERROR; + } + } + assert( pNew->nLTerm<=pNew->nLSlot ); + pNew->u.vtab.idxNum = pIdxInfo->idxNum; + pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; + pIdxInfo->needToFreeIdxStr = 0; + pNew->u.vtab.idxStr = pIdxInfo->idxStr; + pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? + pIdxInfo->nOrderBy : 0); + pNew->rSetup = 0; + pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); + pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); + + /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated + ** that the scan will visit at most one row. Clear it otherwise. */ + if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){ + pNew->wsFlags |= WHERE_ONEROW; + }else{ + pNew->wsFlags &= ~WHERE_ONEROW; + } + rc = whereLoopInsert(pBuilder, pNew); + if( pNew->u.vtab.needFree ){ + sqlite3_free(pNew->u.vtab.idxStr); + pNew->u.vtab.needFree = 0; + } + WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", + *pbIn, (sqlite3_uint64)mPrereq, + (sqlite3_uint64)(pNew->prereq & ~mPrereq))); + + return rc; +} + +/* +** If this function is invoked from within an xBestIndex() callback, it +** returns a pointer to a buffer containing the name of the collation +** sequence associated with element iCons of the sqlite3_index_info.aConstraint +** array. Or, if iCons is out of range or there is no active xBestIndex +** call, return NULL. +*/ +SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){ + HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; + const char *zRet = 0; + if( iCons>=0 && iConsnConstraint ){ + CollSeq *pC = 0; + int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset; + Expr *pX = pHidden->pWC->a[iTerm].pExpr; + if( pX->pLeft ){ + pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX); + } + zRet = (pC ? pC->zName : sqlite3StrBINARY); + } + return zRet; +} + +/* +** Add all WhereLoop objects for a table of the join identified by +** pBuilder->pNew->iTab. That table is guaranteed to be a virtual table. +** +** If there are no LEFT or CROSS JOIN joins in the query, both mPrereq and +** mUnusable are set to 0. Otherwise, mPrereq is a mask of all FROM clause +** entries that occur before the virtual table in the FROM clause and are +** separated from it by at least one LEFT or CROSS JOIN. Similarly, the +** mUnusable mask contains all FROM clause entries that occur after the +** virtual table and are separated from it by at least one LEFT or +** CROSS JOIN. +** +** For example, if the query were: +** +** ... FROM t1, t2 LEFT JOIN t3, t4, vt CROSS JOIN t5, t6; +** +** then mPrereq corresponds to (t1, t2) and mUnusable to (t5, t6). +** +** All the tables in mPrereq must be scanned before the current virtual +** table. So any terms for which all prerequisites are satisfied by +** mPrereq may be specified as "usable" in all calls to xBestIndex. +** Conversely, all tables in mUnusable must be scanned after the current +** virtual table, so any terms for which the prerequisites overlap with +** mUnusable should always be configured as "not-usable" for xBestIndex. +*/ +static int whereLoopAddVirtual( + WhereLoopBuilder *pBuilder, /* WHERE clause information */ + Bitmask mPrereq, /* Tables that must be scanned before this one */ + Bitmask mUnusable /* Tables that must be scanned after this one */ +){ + int rc = SQLITE_OK; /* Return code */ + WhereInfo *pWInfo; /* WHERE analysis context */ + Parse *pParse; /* The parsing context */ + WhereClause *pWC; /* The WHERE clause */ + SrcItem *pSrc; /* The FROM clause term to search */ + sqlite3_index_info *p; /* Object to pass to xBestIndex() */ + int nConstraint; /* Number of constraints in p */ + int bIn; /* True if plan uses IN(...) operator */ + WhereLoop *pNew; + Bitmask mBest; /* Tables used by best possible plan */ + u16 mNoOmit; + + assert( (mPrereq & mUnusable)==0 ); + pWInfo = pBuilder->pWInfo; + pParse = pWInfo->pParse; + pWC = pBuilder->pWC; + pNew = pBuilder->pNew; + pSrc = &pWInfo->pTabList->a[pNew->iTab]; + assert( IsVirtual(pSrc->pTab) ); + p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy, + &mNoOmit); + if( p==0 ) return SQLITE_NOMEM_BKPT; + pNew->rSetup = 0; + pNew->wsFlags = WHERE_VIRTUALTABLE; + pNew->nLTerm = 0; + pNew->u.vtab.needFree = 0; + nConstraint = p->nConstraint; + if( whereLoopResize(pParse->db, pNew, nConstraint) ){ + sqlite3DbFree(pParse->db, p); + return SQLITE_NOMEM_BKPT; + } + + /* First call xBestIndex() with all constraints usable. */ + WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pTab->zName)); + WHERETRACE(0x40, (" VirtualOne: all usable\n")); + rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn); + + /* If the call to xBestIndex() with all terms enabled produced a plan + ** that does not require any source tables (IOW: a plan with mBest==0) + ** and does not use an IN(...) operator, then there is no point in making + ** any further calls to xBestIndex() since they will all return the same + ** result (if the xBestIndex() implementation is sane). */ + if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){ + int seenZero = 0; /* True if a plan with no prereqs seen */ + int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */ + Bitmask mPrev = 0; + Bitmask mBestNoIn = 0; + + /* If the plan produced by the earlier call uses an IN(...) term, call + ** xBestIndex again, this time with IN(...) terms disabled. */ + if( bIn ){ + WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n")); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn); + assert( bIn==0 ); + mBestNoIn = pNew->prereq & ~mPrereq; + if( mBestNoIn==0 ){ + seenZero = 1; + seenZeroNoIN = 1; + } + } + + /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq) + ** in the set of terms that apply to the current virtual table. */ + while( rc==SQLITE_OK ){ + int i; + Bitmask mNext = ALLBITS; + assert( mNext>0 ); + for(i=0; ia[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq + ); + if( mThis>mPrev && mThisprereq==mPrereq ){ + seenZero = 1; + if( bIn==0 ) seenZeroNoIN = 1; + } + } + + /* If the calls to xBestIndex() in the above loop did not find a plan + ** that requires no source tables at all (i.e. one guaranteed to be + ** usable), make a call here with all source tables disabled */ + if( rc==SQLITE_OK && seenZero==0 ){ + WHERETRACE(0x40, (" VirtualOne: all disabled\n")); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn); + if( bIn==0 ) seenZeroNoIN = 1; + } + + /* If the calls to xBestIndex() have so far failed to find a plan + ** that requires no source tables at all and does not use an IN(...) + ** operator, make a final call to obtain one here. */ + if( rc==SQLITE_OK && seenZeroNoIN==0 ){ + WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n")); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn); + } + } + + if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr); + sqlite3DbFreeNN(pParse->db, p); + WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pTab->zName, rc)); + return rc; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Add WhereLoop entries to handle OR terms. This works for either +** btrees or virtual tables. +*/ +static int whereLoopAddOr( + WhereLoopBuilder *pBuilder, + Bitmask mPrereq, + Bitmask mUnusable +){ + WhereInfo *pWInfo = pBuilder->pWInfo; + WhereClause *pWC; + WhereLoop *pNew; + WhereTerm *pTerm, *pWCEnd; + int rc = SQLITE_OK; + int iCur; + WhereClause tempWC; + WhereLoopBuilder sSubBuild; + WhereOrSet sSum, sCur; + SrcItem *pItem; + + pWC = pBuilder->pWC; + pWCEnd = pWC->a + pWC->nTerm; + pNew = pBuilder->pNew; + memset(&sSum, 0, sizeof(sSum)); + pItem = pWInfo->pTabList->a + pNew->iTab; + iCur = pItem->iCursor; + + for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 + && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 + ){ + WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; + WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; + WhereTerm *pOrTerm; + int once = 1; + int i, j; + + sSubBuild = *pBuilder; + sSubBuild.pOrderBy = 0; + sSubBuild.pOrSet = &sCur; + + WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); + for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ + sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; + }else if( pOrTerm->leftCursor==iCur ){ + tempWC.pWInfo = pWC->pWInfo; + tempWC.pOuter = pWC; + tempWC.op = TK_AND; + tempWC.nTerm = 1; + tempWC.a = pOrTerm; + sSubBuild.pWC = &tempWC; + }else{ + continue; + } + sCur.n = 0; +#ifdef WHERETRACE_ENABLED + WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", + (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); + if( sqlite3WhereTrace & 0x400 ){ + sqlite3WhereClausePrint(sSubBuild.pWC); + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pItem->pTab) ){ + rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable); + }else +#endif + { + rc = whereLoopAddBtree(&sSubBuild, mPrereq); + } + if( rc==SQLITE_OK ){ + rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); + } + assert( rc==SQLITE_OK || rc==SQLITE_DONE || sCur.n==0 + || rc==SQLITE_NOMEM ); + testcase( rc==SQLITE_NOMEM && sCur.n>0 ); + testcase( rc==SQLITE_DONE ); + if( sCur.n==0 ){ + sSum.n = 0; + break; + }else if( once ){ + whereOrMove(&sSum, &sCur); + once = 0; + }else{ + WhereOrSet sPrev; + whereOrMove(&sPrev, &sSum); + sSum.n = 0; + for(i=0; inLTerm = 1; + pNew->aLTerm[0] = pTerm; + pNew->wsFlags = WHERE_MULTI_OR; + pNew->rSetup = 0; + pNew->iSortIdx = 0; + memset(&pNew->u, 0, sizeof(pNew->u)); + for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun + 1; + pNew->nOut = sSum.a[i].nOut; + pNew->prereq = sSum.a[i].prereq; + rc = whereLoopInsert(pBuilder, pNew); + } + WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); + } + } + return rc; +} + +/* +** Add all WhereLoop objects for all tables +*/ +static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo = pBuilder->pWInfo; + Bitmask mPrereq = 0; + Bitmask mPrior = 0; + int iTab; + SrcList *pTabList = pWInfo->pTabList; + SrcItem *pItem; + SrcItem *pEnd = &pTabList->a[pWInfo->nLevel]; + sqlite3 *db = pWInfo->pParse->db; + int rc = SQLITE_OK; + WhereLoop *pNew; + + /* Loop over the tables in the join, from left to right */ + pNew = pBuilder->pNew; + whereLoopInit(pNew); + pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT; + for(iTab=0, pItem=pTabList->a; pItemiTab = iTab; + pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR; + pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); + if( (pItem->fg.jointype & (JT_LEFT|JT_CROSS))!=0 ){ + /* This condition is true when pItem is the FROM clause term on the + ** right-hand-side of a LEFT or CROSS JOIN. */ + mPrereq = mPrior; + }else{ + mPrereq = 0; + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pItem->pTab) ){ + SrcItem *p; + for(p=&pItem[1]; pfg.jointype & (JT_LEFT|JT_CROSS)) ){ + mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor); + } + } + rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable); + }else +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + { + rc = whereLoopAddBtree(pBuilder, mPrereq); + } + if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){ + rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable); + } + mPrior |= pNew->maskSelf; + if( rc || db->mallocFailed ){ + if( rc==SQLITE_DONE ){ + /* We hit the query planner search limit set by iPlanLimit */ + sqlite3_log(SQLITE_WARNING, "abbreviated query algorithm search"); + rc = SQLITE_OK; + }else{ + break; + } + } + } + + whereLoopClear(db, pNew); + return rc; +} + +/* +** Examine a WherePath (with the addition of the extra WhereLoop of the 6th +** parameters) to see if it outputs rows in the requested ORDER BY +** (or GROUP BY) without requiring a separate sort operation. Return N: +** +** N>0: N terms of the ORDER BY clause are satisfied +** N==0: No terms of the ORDER BY clause are satisfied +** N<0: Unknown yet how many terms of ORDER BY might be satisfied. +** +** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as +** strict. With GROUP BY and DISTINCT the only requirement is that +** equivalent rows appear immediately adjacent to one another. GROUP BY +** and DISTINCT do not require rows to appear in any particular order as long +** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT +** the pOrderBy terms can be matched in any order. With ORDER BY, the +** pOrderBy terms must be matched in strict left-to-right order. +*/ +static i8 wherePathSatisfiesOrderBy( + WhereInfo *pWInfo, /* The WHERE clause */ + ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ + WherePath *pPath, /* The WherePath to check */ + u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */ + u16 nLoop, /* Number of entries in pPath->aLoop[] */ + WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ + Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ +){ + u8 revSet; /* True if rev is known */ + u8 rev; /* Composite sort order */ + u8 revIdx; /* Index sort order */ + u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ + u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */ + u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ + u16 eqOpMask; /* Allowed equality operators */ + u16 nKeyCol; /* Number of key columns in pIndex */ + u16 nColumn; /* Total number of ordered columns in the index */ + u16 nOrderBy; /* Number terms in the ORDER BY clause */ + int iLoop; /* Index of WhereLoop in pPath being processed */ + int i, j; /* Loop counters */ + int iCur; /* Cursor number for current WhereLoop */ + int iColumn; /* A column number within table iCur */ + WhereLoop *pLoop = 0; /* Current WhereLoop being processed. */ + WhereTerm *pTerm; /* A single term of the WHERE clause */ + Expr *pOBExpr; /* An expression from the ORDER BY clause */ + CollSeq *pColl; /* COLLATE function from an ORDER BY clause term */ + Index *pIndex; /* The index associated with pLoop */ + sqlite3 *db = pWInfo->pParse->db; /* Database connection */ + Bitmask obSat = 0; /* Mask of ORDER BY terms satisfied so far */ + Bitmask obDone; /* Mask of all ORDER BY terms */ + Bitmask orderDistinctMask; /* Mask of all well-ordered loops */ + Bitmask ready; /* Mask of inner loops */ + + /* + ** We say the WhereLoop is "one-row" if it generates no more than one + ** row of output. A WhereLoop is one-row if all of the following are true: + ** (a) All index columns match with WHERE_COLUMN_EQ. + ** (b) The index is unique + ** Any WhereLoop with an WHERE_COLUMN_EQ constraint on the rowid is one-row. + ** Every one-row WhereLoop will have the WHERE_ONEROW bit set in wsFlags. + ** + ** We say the WhereLoop is "order-distinct" if the set of columns from + ** that WhereLoop that are in the ORDER BY clause are different for every + ** row of the WhereLoop. Every one-row WhereLoop is automatically + ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause + ** is not order-distinct. To be order-distinct is not quite the same as being + ** UNIQUE since a UNIQUE column or index can have multiple rows that + ** are NULL and NULL values are equivalent for the purpose of order-distinct. + ** To be order-distinct, the columns must be UNIQUE and NOT NULL. + ** + ** The rowid for a table is always UNIQUE and NOT NULL so whenever the + ** rowid appears in the ORDER BY clause, the corresponding WhereLoop is + ** automatically order-distinct. + */ + + assert( pOrderBy!=0 ); + if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; + + nOrderBy = pOrderBy->nExpr; + testcase( nOrderBy==BMS-1 ); + if( nOrderBy>BMS-1 ) return 0; /* Cannot optimize overly large ORDER BYs */ + isOrderDistinct = 1; + obDone = MASKBIT(nOrderBy)-1; + orderDistinctMask = 0; + ready = 0; + eqOpMask = WO_EQ | WO_IS | WO_ISNULL; + if( wctrlFlags & (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MAX|WHERE_ORDERBY_MIN) ){ + eqOpMask |= WO_IN; + } + for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; + if( iLoopaLoop[iLoop]; + if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue; + }else{ + pLoop = pLast; + } + if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ + if( pLoop->u.vtab.isOrdered && (wctrlFlags & WHERE_DISTINCTBY)==0 ){ + obSat = obDone; + } + break; + }else if( wctrlFlags & WHERE_DISTINCTBY ){ + pLoop->u.btree.nDistinctCol = 0; + } + iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; + + /* Mark off any ORDER BY term X that is a column in the table of + ** the current loop for which there is term in the WHERE + ** clause of the form X IS NULL or X=? that reference only outer + ** loops. + */ + for(i=0; ia[i].pExpr); + if( NEVER(pOBExpr==0) ) continue; + if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, + ~ready, eqOpMask, 0); + if( pTerm==0 ) continue; + if( pTerm->eOperator==WO_IN ){ + /* IN terms are only valid for sorting in the ORDER BY LIMIT + ** optimization, and then only if they are actually used + ** by the query plan */ + assert( wctrlFlags & + (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ); + for(j=0; jnLTerm && pTerm!=pLoop->aLTerm[j]; j++){} + if( j>=pLoop->nLTerm ) continue; + } + if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){ + Parse *pParse = pWInfo->pParse; + CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr); + CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr); + assert( pColl1 ); + if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){ + continue; + } + testcase( pTerm->pExpr->op==TK_IS ); + } + obSat |= MASKBIT(i); + } + + if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ + if( pLoop->wsFlags & WHERE_IPK ){ + pIndex = 0; + nKeyCol = 0; + nColumn = 1; + }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ + return 0; + }else{ + nKeyCol = pIndex->nKeyCol; + nColumn = pIndex->nColumn; + assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); + assert( pIndex->aiColumn[nColumn-1]==XN_ROWID + || !HasRowid(pIndex->pTable)); + /* All relevant terms of the index must also be non-NULL in order + ** for isOrderDistinct to be true. So the isOrderDistint value + ** computed here might be a false positive. Corrections will be + ** made at tag-20210426-1 below */ + isOrderDistinct = IsUniqueIndex(pIndex) + && (pLoop->wsFlags & WHERE_SKIPSCAN)==0; + } + + /* Loop through all columns of the index and deal with the ones + ** that are not constrained by == or IN. + */ + rev = revSet = 0; + distinctColumns = 0; + for(j=0; j=pLoop->u.btree.nEq + || (pLoop->aLTerm[j]==0)==(jnSkip) + ); + if( ju.btree.nEq && j>=pLoop->nSkip ){ + u16 eOp = pLoop->aLTerm[j]->eOperator; + + /* Skip over == and IS and ISNULL terms. (Also skip IN terms when + ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL + ** terms imply that the index is not UNIQUE NOT NULL in which case + ** the loop need to be marked as not order-distinct because it can + ** have repeated NULL rows. + ** + ** If the current term is a column of an ((?,?) IN (SELECT...)) + ** expression for which the SELECT returns more than one column, + ** check that it is the only column used by this loop. Otherwise, + ** if it is one of two or more, none of the columns can be + ** considered to match an ORDER BY term. + */ + if( (eOp & eqOpMask)!=0 ){ + if( eOp & (WO_ISNULL|WO_IS) ){ + testcase( eOp & WO_ISNULL ); + testcase( eOp & WO_IS ); + testcase( isOrderDistinct ); + isOrderDistinct = 0; + } + continue; + }else if( ALWAYS(eOp & WO_IN) ){ + /* ALWAYS() justification: eOp is an equality operator due to the + ** ju.btree.nEq constraint above. Any equality other + ** than WO_IN is captured by the previous "if". So this one + ** always has to be WO_IN. */ + Expr *pX = pLoop->aLTerm[j]->pExpr; + for(i=j+1; iu.btree.nEq; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + assert( (pLoop->aLTerm[i]->eOperator & WO_IN) ); + bOnce = 0; + break; + } + } + } + } + + /* Get the column number in the table (iColumn) and sort order + ** (revIdx) for the j-th column of the index. + */ + if( pIndex ){ + iColumn = pIndex->aiColumn[j]; + revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC; + if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID; + }else{ + iColumn = XN_ROWID; + revIdx = 0; + } + + /* An unconstrained column that might be NULL means that this + ** WhereLoop is not well-ordered. tag-20210426-1 + */ + if( isOrderDistinct ){ + if( iColumn>=0 + && j>=pLoop->u.btree.nEq + && pIndex->pTable->aCol[iColumn].notNull==0 + ){ + isOrderDistinct = 0; + } + if( iColumn==XN_EXPR ){ + isOrderDistinct = 0; + } + } + + /* Find the ORDER BY term that corresponds to the j-th column + ** of the index and mark that ORDER BY term off + */ + isMatch = 0; + for(i=0; bOnce && ia[i].pExpr); + testcase( wctrlFlags & WHERE_GROUPBY ); + testcase( wctrlFlags & WHERE_DISTINCTBY ); + if( NEVER(pOBExpr==0) ) continue; + if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; + if( iColumn>=XN_ROWID ){ + if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + }else{ + Expr *pIdxExpr = pIndex->aColExpr->a[j].pExpr; + if( sqlite3ExprCompareSkip(pOBExpr, pIdxExpr, iCur) ){ + continue; + } + } + if( iColumn!=XN_ROWID ){ + pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); + if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; + } + if( wctrlFlags & WHERE_DISTINCTBY ){ + pLoop->u.btree.nDistinctCol = j+1; + } + isMatch = 1; + break; + } + if( isMatch && (wctrlFlags & WHERE_GROUPBY)==0 ){ + /* Make sure the sort order is compatible in an ORDER BY clause. + ** Sort order is irrelevant for a GROUP BY clause. */ + if( revSet ){ + if( (rev ^ revIdx)!=(pOrderBy->a[i].sortFlags&KEYINFO_ORDER_DESC) ){ + isMatch = 0; + } + }else{ + rev = revIdx ^ (pOrderBy->a[i].sortFlags & KEYINFO_ORDER_DESC); + if( rev ) *pRevMask |= MASKBIT(iLoop); + revSet = 1; + } + } + if( isMatch && (pOrderBy->a[i].sortFlags & KEYINFO_ORDER_BIGNULL) ){ + if( j==pLoop->u.btree.nEq ){ + pLoop->wsFlags |= WHERE_BIGNULL_SORT; + }else{ + isMatch = 0; + } + } + if( isMatch ){ + if( iColumn==XN_ROWID ){ + testcase( distinctColumns==0 ); + distinctColumns = 1; + } + obSat |= MASKBIT(i); + }else{ + /* No match found */ + if( j==0 || jmaskSelf; + for(i=0; ia[i].pExpr; + mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); + if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( (mTerm&~orderDistinctMask)==0 ){ + obSat |= MASKBIT(i); + } + } + } + } /* End the loop over all WhereLoops from outer-most down to inner-most */ + if( obSat==obDone ) return (i8)nOrderBy; + if( !isOrderDistinct ){ + for(i=nOrderBy-1; i>0; i--){ + Bitmask m = MASKBIT(i) - 1; + if( (obSat&m)==m ) return i; + } + return 0; + } + return -1; +} + + +/* +** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(), +** the planner assumes that the specified pOrderBy list is actually a GROUP +** BY clause - and so any order that groups rows as required satisfies the +** request. +** +** Normally, in this case it is not possible for the caller to determine +** whether or not the rows are really being delivered in sorted order, or +** just in some other order that provides the required grouping. However, +** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then +** this function may be called on the returned WhereInfo object. It returns +** true if the rows really will be sorted in the specified order, or false +** otherwise. +** +** For example, assuming: +** +** CREATE INDEX i1 ON t1(x, Y); +** +** then +** +** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1 +** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0 +*/ +SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo *pWInfo){ + assert( pWInfo->wctrlFlags & WHERE_GROUPBY ); + assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); + return pWInfo->sorted; +} + +#ifdef WHERETRACE_ENABLED +/* For debugging use only: */ +static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ + static char zName[65]; + int i; + for(i=0; iaLoop[i]->cId; } + if( pLast ) zName[i++] = pLast->cId; + zName[i] = 0; + return zName; +} +#endif + +/* +** Return the cost of sorting nRow rows, assuming that the keys have +** nOrderby columns and that the first nSorted columns are already in +** order. +*/ +static LogEst whereSortingCost( + WhereInfo *pWInfo, + LogEst nRow, + int nOrderBy, + int nSorted +){ + /* TUNING: Estimated cost of a full external sort, where N is + ** the number of rows to sort is: + ** + ** cost = (3.0 * N * log(N)). + ** + ** Or, if the order-by clause has X terms but only the last Y + ** terms are out of order, then block-sorting will reduce the + ** sorting cost to: + ** + ** cost = (3.0 * N * log(N)) * (Y/X) + ** + ** The (Y/X) term is implemented using stack variable rScale + ** below. + */ + LogEst rScale, rSortCost; + assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); + rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; + rSortCost = nRow + rScale + 16; + + /* Multiple by log(M) where M is the number of output rows. + ** Use the LIMIT for M if it is smaller. Or if this sort is for + ** a DISTINCT operator, M will be the number of distinct output + ** rows, so fudge it downwards a bit. + */ + if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimitiLimit; + }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){ + /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT + ** reduces the number of output rows by a factor of 2 */ + if( nRow>10 ){ nRow -= 10; assert( 10==sqlite3LogEst(2) ); } + } + rSortCost += estLog(nRow); + return rSortCost; +} + +/* +** Given the list of WhereLoop objects at pWInfo->pLoops, this routine +** attempts to find the lowest cost path that visits each WhereLoop +** once. This path is then loaded into the pWInfo->a[].pWLoop fields. +** +** Assume that the total number of output rows that will need to be sorted +** will be nRowEst (in the 10*log2 representation). Or, ignore sorting +** costs if nRowEst==0. +** +** Return SQLITE_OK on success or SQLITE_NOMEM of a memory allocation +** error occurs. +*/ +static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ + int mxChoice; /* Maximum number of simultaneous paths tracked */ + int nLoop; /* Number of terms in the join */ + Parse *pParse; /* Parsing context */ + sqlite3 *db; /* The database connection */ + int iLoop; /* Loop counter over the terms of the join */ + int ii, jj; /* Loop counters */ + int mxI = 0; /* Index of next entry to replace */ + int nOrderBy; /* Number of ORDER BY clause terms */ + LogEst mxCost = 0; /* Maximum cost of a set of paths */ + LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */ + int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ + WherePath *aFrom; /* All nFrom paths at the previous level */ + WherePath *aTo; /* The nTo best paths at the current level */ + WherePath *pFrom; /* An element of aFrom[] that we are working on */ + WherePath *pTo; /* An element of aTo[] that we are working on */ + WhereLoop *pWLoop; /* One of the WhereLoop objects */ + WhereLoop **pX; /* Used to divy up the pSpace memory */ + LogEst *aSortCost = 0; /* Sorting and partial sorting costs */ + char *pSpace; /* Temporary memory used by this routine */ + int nSpace; /* Bytes of space allocated at pSpace */ + + pParse = pWInfo->pParse; + db = pParse->db; + nLoop = pWInfo->nLevel; + /* TUNING: For simple queries, only the best path is tracked. + ** For 2-way joins, the 5 best paths are followed. + ** For joins of 3 or more tables, track the 10 best paths */ + mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); + assert( nLoop<=pWInfo->pTabList->nSrc ); + WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); + + /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this + ** case the purpose of this call is to estimate the number of rows returned + ** by the overall query. Once this estimate has been obtained, the caller + ** will invoke this function a second time, passing the estimate as the + ** nRowEst parameter. */ + if( pWInfo->pOrderBy==0 || nRowEst==0 ){ + nOrderBy = 0; + }else{ + nOrderBy = pWInfo->pOrderBy->nExpr; + } + + /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ + nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; + nSpace += sizeof(LogEst) * nOrderBy; + pSpace = sqlite3DbMallocRawNN(db, nSpace); + if( pSpace==0 ) return SQLITE_NOMEM_BKPT; + aTo = (WherePath*)pSpace; + aFrom = aTo+mxChoice; + memset(aFrom, 0, sizeof(aFrom[0])); + pX = (WhereLoop**)(aFrom+mxChoice); + for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){ + pFrom->aLoop = pX; + } + if( nOrderBy ){ + /* If there is an ORDER BY clause and it is not being ignored, set up + ** space for the aSortCost[] array. Each element of the aSortCost array + ** is either zero - meaning it has not yet been initialized - or the + ** cost of sorting nRowEst rows of data where the first X terms of + ** the ORDER BY clause are already in order, where X is the array + ** index. */ + aSortCost = (LogEst*)pX; + memset(aSortCost, 0, sizeof(LogEst) * nOrderBy); + } + assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] ); + assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX ); + + /* Seed the search with a single WherePath containing zero WhereLoops. + ** + ** TUNING: Do not let the number of iterations go above 28. If the cost + ** of computing an automatic index is not paid back within the first 28 + ** rows, then do not use the automatic index. */ + aFrom[0].nRow = MIN(pParse->nQueryLoop, 48); assert( 48==sqlite3LogEst(28) ); + nFrom = 1; + assert( aFrom[0].isOrdered==0 ); + if( nOrderBy ){ + /* If nLoop is zero, then there are no FROM terms in the query. Since + ** in this case the query may return a maximum of one row, the results + ** are already in the requested order. Set isOrdered to nOrderBy to + ** indicate this. Or, if nLoop is greater than zero, set isOrdered to + ** -1, indicating that the result set may or may not be ordered, + ** depending on the loops added to the current plan. */ + aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy; + } + + /* Compute successively longer WherePaths using the previous generation + ** of WherePaths as the basis for the next. Keep track of the mxChoice + ** best paths at each generation */ + for(iLoop=0; iLooppLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ + LogEst rCost; /* Cost of path (pFrom+pWLoop) */ + LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ + i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ + Bitmask maskNew; /* Mask of src visited by (..) */ + Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ + + if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; + if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; + if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<3 ){ + /* Do not use an automatic index if the this loop is expected + ** to run less than 1.25 times. It is tempting to also exclude + ** automatic index usage on an outer loop, but sometimes an automatic + ** index is useful in the outer loop of a correlated subquery. */ + assert( 10==sqlite3LogEst(2) ); + continue; + } + + /* At this point, pWLoop is a candidate to be the next loop. + ** Compute its cost */ + rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); + rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); + nOut = pFrom->nRow + pWLoop->nOut; + maskNew = pFrom->maskLoop | pWLoop->maskSelf; + if( isOrdered<0 ){ + isOrdered = wherePathSatisfiesOrderBy(pWInfo, + pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, + iLoop, pWLoop, &revMask); + }else{ + revMask = pFrom->revLoop; + } + if( isOrdered>=0 && isOrderedisOrdered^isOrdered)&0x80)==0" is equivalent + ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range + ** of legal values for isOrdered, -1..64. + */ + for(jj=0, pTo=aTo; jjmaskLoop==maskNew + && ((pTo->isOrdered^isOrdered)&0x80)==0 + ){ + testcase( jj==nTo-1 ); + break; + } + } + if( jj>=nTo ){ + /* None of the existing best-so-far paths match the candidate. */ + if( nTo>=mxChoice + && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted)) + ){ + /* The current candidate is no better than any of the mxChoice + ** paths currently in the best-so-far buffer. So discard + ** this candidate as not viable. */ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, + isOrdered>=0 ? isOrdered+'0' : '?'); + } +#endif + continue; + } + /* If we reach this points it means that the new candidate path + ** needs to be added to the set of best-so-far paths. */ + if( nTo=0 ? isOrdered+'0' : '?'); + } +#endif + }else{ + /* Control reaches here if best-so-far path pTo=aTo[jj] covers the + ** same set of loops and has the same isOrdered setting as the + ** candidate path. Check to see if the candidate should replace + ** pTo or if the candidate should be skipped. + ** + ** The conditional is an expanded vector comparison equivalent to: + ** (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted) + */ + if( pTo->rCostrCost==rCost + && (pTo->nRownRow==nOut && pTo->rUnsorted<=rUnsorted) + ) + ) + ){ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf( + "Skip %s cost=%-3d,%3d,%3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, + isOrdered>=0 ? isOrdered+'0' : '?'); + sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, + pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + } +#endif + /* Discard the candidate path from further consideration */ + testcase( pTo->rCost==rCost ); + continue; + } + testcase( pTo->rCost==rCost+1 ); + /* Control reaches here if the candidate path is better than the + ** pTo path. Replace pTo with the candidate. */ +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + sqlite3DebugPrintf( + "Update %s cost=%-3d,%3d,%3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, + isOrdered>=0 ? isOrdered+'0' : '?'); + sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, + pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + } +#endif + } + /* pWLoop is a winner. Add it to the set of best so far */ + pTo->maskLoop = pFrom->maskLoop | pWLoop->maskSelf; + pTo->revLoop = revMask; + pTo->nRow = nOut; + pTo->rCost = rCost; + pTo->rUnsorted = rUnsorted; + pTo->isOrdered = isOrdered; + memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); + pTo->aLoop[iLoop] = pWLoop; + if( nTo>=mxChoice ){ + mxI = 0; + mxCost = aTo[0].rCost; + mxUnsorted = aTo[0].nRow; + for(jj=1, pTo=&aTo[1]; jjrCost>mxCost + || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) + ){ + mxCost = pTo->rCost; + mxUnsorted = pTo->rUnsorted; + mxI = jj; + } + } + } + } + } + +#ifdef WHERETRACE_ENABLED /* >=2 */ + if( sqlite3WhereTrace & 0x02 ){ + sqlite3DebugPrintf("---- after round %d ----\n", iLoop); + for(ii=0, pTo=aTo; iirCost, pTo->nRow, + pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); + if( pTo->isOrdered>0 ){ + sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); + }else{ + sqlite3DebugPrintf("\n"); + } + } + } +#endif + + /* Swap the roles of aFrom and aTo for the next generation */ + pFrom = aTo; + aTo = aFrom; + aFrom = pFrom; + nFrom = nTo; + } + + if( nFrom==0 ){ + sqlite3ErrorMsg(pParse, "no query solution"); + sqlite3DbFreeNN(db, pSpace); + return SQLITE_ERROR; + } + + /* Find the lowest cost path. pFrom will be left pointing to that path */ + pFrom = aFrom; + for(ii=1; iirCost>aFrom[ii].rCost ) pFrom = &aFrom[ii]; + } + assert( pWInfo->nLevel==nLoop ); + /* Load the lowest cost path into pWInfo */ + for(iLoop=0; iLoopa + iLoop; + pLevel->pWLoop = pWLoop = pFrom->aLoop[iLoop]; + pLevel->iFrom = pWLoop->iTab; + pLevel->iTabCur = pWInfo->pTabList->a[pLevel->iFrom].iCursor; + } + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 + && (pWInfo->wctrlFlags & WHERE_DISTINCTBY)==0 + && pWInfo->eDistinct==WHERE_DISTINCT_NOOP + && nRowEst + ){ + Bitmask notUsed; + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, + WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); + if( rc==pWInfo->pResultSet->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } + } + pWInfo->bOrderedInnerLoop = 0; + if( pWInfo->pOrderBy ){ + if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ + if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } + }else{ + pWInfo->nOBSat = pFrom->isOrdered; + pWInfo->revMask = pFrom->revLoop; + if( pWInfo->nOBSat<=0 ){ + pWInfo->nOBSat = 0; + if( nLoop>0 ){ + u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags; + if( (wsFlags & WHERE_ONEROW)==0 + && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN) + ){ + Bitmask m = 0; + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, + WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m); + testcase( wsFlags & WHERE_IPK ); + testcase( wsFlags & WHERE_COLUMN_IN ); + if( rc==pWInfo->pOrderBy->nExpr ){ + pWInfo->bOrderedInnerLoop = 1; + pWInfo->revMask = m; + } + } + } + }else if( nLoop + && pWInfo->nOBSat==1 + && (pWInfo->wctrlFlags & (WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX))!=0 + ){ + pWInfo->bOrderedInnerLoop = 1; + } + } + if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) + && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0 + ){ + Bitmask revMask = 0; + int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, + pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask + ); + assert( pWInfo->sorted==0 ); + if( nOrder==pWInfo->pOrderBy->nExpr ){ + pWInfo->sorted = 1; + pWInfo->revMask = revMask; + } + } + } + + + pWInfo->nRowOut = pFrom->nRow; + + /* Free temporary memory and return success */ + sqlite3DbFreeNN(db, pSpace); + return SQLITE_OK; +} + +/* +** Most queries use only a single table (they are not joins) and have +** simple == constraints against indexed fields. This routine attempts +** to plan those simple cases using much less ceremony than the +** general-purpose query planner, and thereby yield faster sqlite3_prepare() +** times for the common case. +** +** Return non-zero on success, if this query can be handled by this +** no-frills query planner. Return zero if this query needs the +** general-purpose query planner. +*/ +static int whereShortCut(WhereLoopBuilder *pBuilder){ + WhereInfo *pWInfo; + SrcItem *pItem; + WhereClause *pWC; + WhereTerm *pTerm; + WhereLoop *pLoop; + int iCur; + int j; + Table *pTab; + Index *pIdx; + WhereScan scan; + + pWInfo = pBuilder->pWInfo; + if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0; + assert( pWInfo->pTabList->nSrc>=1 ); + pItem = pWInfo->pTabList->a; + pTab = pItem->pTab; + if( IsVirtual(pTab) ) return 0; + if( pItem->fg.isIndexedBy ) return 0; + iCur = pItem->iCursor; + pWC = &pWInfo->sWC; + pLoop = pBuilder->pNew; + pLoop->wsFlags = 0; + pLoop->nSkip = 0; + pTerm = whereScanInit(&scan, pWC, iCur, -1, WO_EQ|WO_IS, 0); + if( pTerm ){ + testcase( pTerm->eOperator & WO_IS ); + assert( pTerm->prereqRight==0 ); + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; + pLoop->aLTerm[0] = pTerm; + pLoop->nLTerm = 1; + pLoop->u.btree.nEq = 1; + /* TUNING: Cost of a rowid lookup is 10 */ + pLoop->rRun = 33; /* 33==sqlite3LogEst(10) */ + }else{ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int opMask; + assert( pLoop->aLTermSpace==pLoop->aLTerm ); + if( !IsUniqueIndex(pIdx) + || pIdx->pPartIdxWhere!=0 + || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) + ) continue; + opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ; + for(j=0; jnKeyCol; j++){ + pTerm = whereScanInit(&scan, pWC, iCur, j, opMask, pIdx); + while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan); + if( pTerm==0 ) break; + testcase( pTerm->eOperator & WO_IS ); + pLoop->aLTerm[j] = pTerm; + } + if( j!=pIdx->nKeyCol ) continue; + pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; + if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){ + pLoop->wsFlags |= WHERE_IDX_ONLY; + } + pLoop->nLTerm = j; + pLoop->u.btree.nEq = j; + pLoop->u.btree.pIndex = pIdx; + /* TUNING: Cost of a unique index lookup is 15 */ + pLoop->rRun = 39; /* 39==sqlite3LogEst(15) */ + break; + } + } + if( pLoop->wsFlags ){ + pLoop->nOut = (LogEst)1; + pWInfo->a[0].pWLoop = pLoop; + assert( pWInfo->sMaskSet.n==1 && iCur==pWInfo->sMaskSet.ix[0] ); + pLoop->maskSelf = 1; /* sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); */ + pWInfo->a[0].iTabCur = iCur; + pWInfo->nRowOut = 1; + if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } + if( scan.iEquiv>1 ) pLoop->wsFlags |= WHERE_TRANSCONS; +#ifdef SQLITE_DEBUG + pLoop->cId = '0'; +#endif +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace ){ + sqlite3DebugPrintf("whereShortCut() used to compute solution\n"); + } +#endif + return 1; + } + return 0; +} + +/* +** Helper function for exprIsDeterministic(). +*/ +static int exprNodeIsDeterministic(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_ConstFunc)==0 ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Continue; +} + +/* +** Return true if the expression contains no non-deterministic SQL +** functions. Do not consider non-deterministic SQL functions that are +** part of sub-select statements. +*/ +static int exprIsDeterministic(Expr *p){ + Walker w; + memset(&w, 0, sizeof(w)); + w.eCode = 1; + w.xExprCallback = exprNodeIsDeterministic; + w.xSelectCallback = sqlite3SelectWalkFail; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + + +#ifdef WHERETRACE_ENABLED +/* +** Display all WhereLoops in pWInfo +*/ +static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ + if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ + WhereLoop *p; + int i; + static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; + for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ + p->cId = zLabel[i%(sizeof(zLabel)-1)]; + sqlite3WhereLoopPrint(p, pWC); + } + } +} +# define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C) +#else +# define WHERETRACE_ALL_LOOPS(W,C) +#endif + +/* +** Generate the beginning of the loop used for WHERE clause processing. +** The return value is a pointer to an opaque structure that contains +** information needed to terminate the loop. Later, the calling routine +** should invoke sqlite3WhereEnd() with the return value of this function +** in order to complete the WHERE clause processing. +** +** If an error occurs, this routine returns NULL. +** +** The basic idea is to do a nested loop, one loop for each table in +** the FROM clause of a select. (INSERT and UPDATE statements are the +** same as a SELECT with only a single table in the FROM clause.) For +** example, if the SQL is this: +** +** SELECT * FROM t1, t2, t3 WHERE ...; +** +** Then the code generated is conceptually like the following: +** +** foreach row1 in t1 do \ Code generated +** foreach row2 in t2 do |-- by sqlite3WhereBegin() +** foreach row3 in t3 do / +** ... +** end \ Code generated +** end |-- by sqlite3WhereEnd() +** end / +** +** Note that the loops might not be nested in the order in which they +** appear in the FROM clause if a different order is better able to make +** use of indices. Note also that when the IN operator appears in +** the WHERE clause, it might result in additional nested loops for +** scanning through all values on the right-hand side of the IN. +** +** There are Btree cursors associated with each table. t1 uses cursor +** number pTabList->a[0].iCursor. t2 uses the cursor pTabList->a[1].iCursor. +** And so forth. This routine generates code to open those VDBE cursors +** and sqlite3WhereEnd() generates the code to close them. +** +** The code that sqlite3WhereBegin() generates leaves the cursors named +** in pTabList pointing at their appropriate entries. The [...] code +** can use OP_Column and OP_Rowid opcodes on these cursors to extract +** data from the various tables of the loop. +** +** If the WHERE clause is empty, the foreach loops must each scan their +** entire tables. Thus a three-way join is an O(N^3) operation. But if +** the tables have indices and there are terms in the WHERE clause that +** refer to those indices, a complete table scan can be avoided and the +** code will run much faster. Most of the work of this routine is checking +** to see if there are indices that can be used to speed up the loop. +** +** Terms of the WHERE clause are also used to limit which rows actually +** make it to the "..." in the middle of the loop. After each "foreach", +** terms of the WHERE clause that use only terms in that loop and outer +** loops are evaluated and if false a jump is made around all subsequent +** inner loops (or around the "..." if the test occurs within the inner- +** most loop) +** +** OUTER JOINS +** +** An outer join of tables t1 and t2 is conceptally coded as follows: +** +** foreach row1 in t1 do +** flag = 0 +** foreach row2 in t2 do +** start: +** ... +** flag = 1 +** end +** if flag==0 then +** move the row2 cursor to a null row +** goto start +** fi +** end +** +** ORDER BY CLAUSE PROCESSING +** +** pOrderBy is a pointer to the ORDER BY clause (or the GROUP BY clause +** if the WHERE_GROUPBY flag is set in wctrlFlags) of a SELECT statement +** if there is one. If there is no ORDER BY clause or if this routine +** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. +** +** The iIdxCur parameter is the cursor number of an index. If +** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index +** to use for OR clause processing. The WHERE clause should use this +** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is +** the first cursor in an array of cursors for all indices. iIdxCur should +** be used to compute the appropriate cursor depending on which index is +** used. +*/ +SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( + Parse *pParse, /* The parser context */ + SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ + Expr *pWhere, /* The WHERE clause */ + ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ + ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */ + u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ + int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number + ** If WHERE_USE_LIMIT, then the limit amount */ +){ + int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ + int nTabList; /* Number of elements in pTabList */ + WhereInfo *pWInfo; /* Will become the return value of this function */ + Vdbe *v = pParse->pVdbe; /* The virtual database engine */ + Bitmask notReady; /* Cursors that are not yet positioned */ + WhereLoopBuilder sWLB; /* The WhereLoop builder */ + WhereMaskSet *pMaskSet; /* The expression mask set */ + WhereLevel *pLevel; /* A single level in pWInfo->a[] */ + WhereLoop *pLoop; /* Pointer to a single WhereLoop object */ + int ii; /* Loop counter */ + sqlite3 *db; /* Database connection */ + int rc; /* Return code */ + u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */ + + assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || ( + (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 + )); + + /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */ + assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 + || (wctrlFlags & WHERE_USE_LIMIT)==0 ); + + /* Variable initialization */ + db = pParse->db; + memset(&sWLB, 0, sizeof(sWLB)); + + /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ + testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); + if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; + sWLB.pOrderBy = pOrderBy; + + /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via + ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ + if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ + wctrlFlags &= ~WHERE_WANT_DISTINCT; + } + + /* The number of tables in the FROM clause is limited by the number of + ** bits in a Bitmask + */ + testcase( pTabList->nSrc==BMS ); + if( pTabList->nSrc>BMS ){ + sqlite3ErrorMsg(pParse, "at most %d tables in a join", BMS); + return 0; + } + + /* This function normally generates a nested loop for all tables in + ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should + ** only generate code for the first table in pTabList and assume that + ** any cursors associated with subsequent tables are uninitialized. + */ + nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc; + + /* Allocate and initialize the WhereInfo structure that will become the + ** return value. A single allocation is used to store the WhereInfo + ** struct, the contents of WhereInfo.a[], the WhereClause structure + ** and the WhereMaskSet structure. Since WhereClause contains an 8-byte + ** field (type Bitmask) it must be aligned on an 8-byte boundary on + ** some architectures. Hence the ROUND8() below. + */ + nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); + pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); + if( db->mallocFailed ){ + sqlite3DbFree(db, pWInfo); + pWInfo = 0; + goto whereBeginError; + } + pWInfo->pParse = pParse; + pWInfo->pTabList = pTabList; + pWInfo->pOrderBy = pOrderBy; + pWInfo->pWhere = pWhere; + pWInfo->pResultSet = pResultSet; + pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; + pWInfo->nLevel = nTabList; + pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(pParse); + pWInfo->wctrlFlags = wctrlFlags; + pWInfo->iLimit = iAuxArg; + pWInfo->savedNQueryLoop = pParse->nQueryLoop; + memset(&pWInfo->nOBSat, 0, + offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat)); + memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel)); + assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */ + pMaskSet = &pWInfo->sMaskSet; + sWLB.pWInfo = pWInfo; + sWLB.pWC = &pWInfo->sWC; + sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo); + assert( EIGHT_BYTE_ALIGNMENT(sWLB.pNew) ); + whereLoopInit(sWLB.pNew); +#ifdef SQLITE_DEBUG + sWLB.pNew->cId = '*'; +#endif + + /* Split the WHERE clause into separate subexpressions where each + ** subexpression is separated by an AND operator. + */ + initMaskSet(pMaskSet); + sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo); + sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND); + + /* Special case: No FROM clause + */ + if( nTabList==0 ){ + if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } + ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + }else{ + /* Assign a bit from the bitmask to every term in the FROM clause. + ** + ** The N-th term of the FROM clause is assigned a bitmask of 1<nSrc tables in + ** pTabList, not just the first nTabList tables. nTabList is normally + ** equal to pTabList->nSrc but might be shortened to 1 if the + ** WHERE_OR_SUBCLAUSE flag is set. + */ + ii = 0; + do{ + createMask(pMaskSet, pTabList->a[ii].iCursor); + sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); + }while( (++ii)nSrc ); + #ifdef SQLITE_DEBUG + { + Bitmask mx = 0; + for(ii=0; iinSrc; ii++){ + Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); + assert( m>=mx ); + mx = m; + } + } + #endif + } + + /* Analyze all of the subexpressions. */ + sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); + if( db->mallocFailed ) goto whereBeginError; + + /* Special case: WHERE terms that do not refer to any tables in the join + ** (constant expressions). Evaluate each such term, and jump over all the + ** generated code if the result is not true. + ** + ** Do not do this if the expression contains non-deterministic functions + ** that are not within a sub-select. This is not strictly required, but + ** preserves SQLite's legacy behaviour in the following two cases: + ** + ** FROM ... WHERE random()>0; -- eval random() once per row + ** FROM ... WHERE (SELECT random())>0; -- eval random() once overall + */ + for(ii=0; iinTerm; ii++){ + WhereTerm *pT = &sWLB.pWC->a[ii]; + if( pT->wtFlags & TERM_VIRTUAL ) continue; + if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){ + sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL); + pT->wtFlags |= TERM_CODED; + } + } + + if( wctrlFlags & WHERE_WANT_DISTINCT ){ + if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ + /* The DISTINCT marking is pointless. Ignore it. */ + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + }else if( pOrderBy==0 ){ + /* Try to ORDER BY the result set to make distinct processing easier */ + pWInfo->wctrlFlags |= WHERE_DISTINCTBY; + pWInfo->pOrderBy = pResultSet; + } + } + + /* Construct the WhereLoop objects */ +#if defined(WHERETRACE_ENABLED) + if( sqlite3WhereTrace & 0xffff ){ + sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags); + if( wctrlFlags & WHERE_USE_LIMIT ){ + sqlite3DebugPrintf(", limit: %d", iAuxArg); + } + sqlite3DebugPrintf(")\n"); + if( sqlite3WhereTrace & 0x100 ){ + Select sSelect; + memset(&sSelect, 0, sizeof(sSelect)); + sSelect.selFlags = SF_WhereBegin; + sSelect.pSrc = pTabList; + sSelect.pWhere = pWhere; + sSelect.pOrderBy = pOrderBy; + sSelect.pEList = pResultSet; + sqlite3TreeViewSelect(0, &sSelect, 0); + } + } + if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ + sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); + sqlite3WhereClausePrint(sWLB.pWC); + } +#endif + + if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ + rc = whereLoopAddAll(&sWLB); + if( rc ) goto whereBeginError; + +#ifdef SQLITE_ENABLE_STAT4 + /* If one or more WhereTerm.truthProb values were used in estimating + ** loop parameters, but then those truthProb values were subsequently + ** changed based on STAT4 information while computing subsequent loops, + ** then we need to rerun the whole loop building process so that all + ** loops will be built using the revised truthProb values. */ + if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){ + WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); + WHERETRACE(0xffff, + ("**** Redo all loop computations due to" + " TERM_HIGHTRUTH changes ****\n")); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); + } + rc = whereLoopAddAll(&sWLB); + if( rc ) goto whereBeginError; + } +#endif + WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); + + wherePathSolver(pWInfo, 0); + if( db->mallocFailed ) goto whereBeginError; + if( pWInfo->pOrderBy ){ + wherePathSolver(pWInfo, pWInfo->nRowOut+1); + if( db->mallocFailed ) goto whereBeginError; + } + } + if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ + pWInfo->revMask = ALLBITS; + } + if( pParse->nErr || db->mallocFailed ){ + goto whereBeginError; + } +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace ){ + sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); + if( pWInfo->nOBSat>0 ){ + sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); + } + switch( pWInfo->eDistinct ){ + case WHERE_DISTINCT_UNIQUE: { + sqlite3DebugPrintf(" DISTINCT=unique"); + break; + } + case WHERE_DISTINCT_ORDERED: { + sqlite3DebugPrintf(" DISTINCT=ordered"); + break; + } + case WHERE_DISTINCT_UNORDERED: { + sqlite3DebugPrintf(" DISTINCT=unordered"); + break; + } + } + sqlite3DebugPrintf("\n"); + for(ii=0; iinLevel; ii++){ + sqlite3WhereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC); + } + } +#endif + + /* Attempt to omit tables from the join that do not affect the result. + ** For a table to not affect the result, the following must be true: + ** + ** 1) The query must not be an aggregate. + ** 2) The table must be the RHS of a LEFT JOIN. + ** 3) Either the query must be DISTINCT, or else the ON or USING clause + ** must contain a constraint that limits the scan of the table to + ** at most a single row. + ** 4) The table must not be referenced by any part of the query apart + ** from its own USING or ON clause. + ** + ** For example, given: + ** + ** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1); + ** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2); + ** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3); + ** + ** then table t2 can be omitted from the following: + ** + ** SELECT v1, v3 FROM t1 + ** LEFT JOIN t2 ON (t1.ipk=t2.ipk) + ** LEFT JOIN t3 ON (t1.ipk=t3.ipk) + ** + ** or from: + ** + ** SELECT DISTINCT v1, v3 FROM t1 + ** LEFT JOIN t2 + ** LEFT JOIN t3 ON (t1.ipk=t3.ipk) + */ + notReady = ~(Bitmask)0; + if( pWInfo->nLevel>=2 + && pResultSet!=0 /* these two combine to guarantee */ + && 0==(wctrlFlags & WHERE_AGG_DISTINCT) /* condition (1) above */ + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + ){ + int i; + Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pResultSet); + if( sWLB.pOrderBy ){ + tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy); + } + for(i=pWInfo->nLevel-1; i>=1; i--){ + WhereTerm *pTerm, *pEnd; + SrcItem *pItem; + pLoop = pWInfo->a[i].pWLoop; + pItem = &pWInfo->pTabList->a[pLoop->iTab]; + if( (pItem->fg.jointype & JT_LEFT)==0 ) continue; + if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 + && (pLoop->wsFlags & WHERE_ONEROW)==0 + ){ + continue; + } + if( (tabUsed & pLoop->maskSelf)!=0 ) continue; + pEnd = sWLB.pWC->a + sWLB.pWC->nTerm; + for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 ){ + if( !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + || pTerm->pExpr->iRightJoinTable!=pItem->iCursor + ){ + break; + } + } + } + if( pTerm drop loop %c not used\n", pLoop->cId)); + notReady &= ~pLoop->maskSelf; + for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 ){ + pTerm->wtFlags |= TERM_CODED; + } + } + if( i!=pWInfo->nLevel-1 ){ + int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel); + memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte); + } + pWInfo->nLevel--; + nTabList--; + } + } +#if defined(WHERETRACE_ENABLED) + if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ + sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n"); + sqlite3WhereClausePrint(sWLB.pWC); + } + WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); +#endif + pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; + + /* If the caller is an UPDATE or DELETE statement that is requesting + ** to use a one-pass algorithm, determine if this is appropriate. + ** + ** A one-pass approach can be used if the caller has requested one + ** and either (a) the scan visits at most one row or (b) each + ** of the following are true: + ** + ** * the caller has indicated that a one-pass approach can be used + ** with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and + ** * the table is not a virtual table, and + ** * either the scan does not use the OR optimization or the caller + ** is a DELETE operation (WHERE_DUPLICATES_OK is only specified + ** for DELETE). + ** + ** The last qualification is because an UPDATE statement uses + ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can + ** use a one-pass approach, and this is not set accurately for scans + ** that use the OR optimization. + */ + assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); + if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ + int wsFlags = pWInfo->a[0].pWLoop->wsFlags; + int bOnerow = (wsFlags & WHERE_ONEROW)!=0; + assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) ); + if( bOnerow || ( + 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW) + && !IsVirtual(pTabList->a[0].pTab) + && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK)) + )){ + pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; + if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ + if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ + bFordelete = OPFLAG_FORDELETE; + } + pWInfo->a[0].pWLoop->wsFlags = (wsFlags & ~WHERE_IDX_ONLY); + } + } + } + + /* Open all tables in the pTabList and any indices selected for + ** searching those tables. + */ + for(ii=0, pLevel=pWInfo->a; iia[pLevel->iFrom]; + pTab = pTabItem->pTab; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + pLoop = pLevel->pWLoop; + if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){ + /* Do nothing */ + }else +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ + const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); + int iCur = pTabItem->iCursor; + sqlite3VdbeAddOp4(v, OP_VOpen, iCur, 0, 0, pVTab, P4_VTAB); + }else if( IsVirtual(pTab) ){ + /* noop */ + }else +#endif + if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 + && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){ + int op = OP_OpenRead; + if( pWInfo->eOnePass!=ONEPASS_OFF ){ + op = OP_OpenWrite; + pWInfo->aiCurOnePass[0] = pTabItem->iCursor; + }; + sqlite3OpenTable(pParse, pTabItem->iCursor, iDb, pTab, op); + assert( pTabItem->iCursor==pLevel->iTabCur ); + testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); + testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); + if( pWInfo->eOnePass==ONEPASS_OFF + && pTab->nColtabFlags & (TF_HasGenerated|TF_WithoutRowid))==0 + ){ + /* If we know that only a prefix of the record will be used, + ** it is advantageous to reduce the "column count" field in + ** the P4 operand of the OP_OpenRead/Write opcode. */ + Bitmask b = pTabItem->colUsed; + int n = 0; + for(; b; b=b>>1, n++){} + sqlite3VdbeChangeP4(v, -1, SQLITE_INT_TO_PTR(n), P4_INT32); + assert( n<=pTab->nCol ); + } +#ifdef SQLITE_ENABLE_CURSOR_HINTS + if( pLoop->u.btree.pIndex!=0 ){ + sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); + }else +#endif + { + sqlite3VdbeChangeP5(v, bFordelete); + } +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, pTabItem->iCursor, 0, 0, + (const u8*)&pTabItem->colUsed, P4_INT64); +#endif + }else{ + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + } + if( pLoop->wsFlags & WHERE_INDEXED ){ + Index *pIx = pLoop->u.btree.pIndex; + int iIndexCur; + int op = OP_OpenRead; + /* iAuxArg is always set to a positive value if ONEPASS is possible */ + assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) + && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 + ){ + /* This is one term of an OR-optimization using the PRIMARY KEY of a + ** WITHOUT ROWID table. No need for a separate index */ + iIndexCur = pLevel->iTabCur; + op = 0; + }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ + Index *pJ = pTabItem->pTab->pIndex; + iIndexCur = iAuxArg; + assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); + while( ALWAYS(pJ) && pJ!=pIx ){ + iIndexCur++; + pJ = pJ->pNext; + } + op = OP_OpenWrite; + pWInfo->aiCurOnePass[1] = iIndexCur; + }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){ + iIndexCur = iAuxArg; + op = OP_ReopenIdx; + }else{ + iIndexCur = pParse->nTab++; + } + pLevel->iIdxCur = iIndexCur; + assert( pIx->pSchema==pTab->pSchema ); + assert( iIndexCur>=0 ); + if( op ){ + sqlite3VdbeAddOp3(v, op, iIndexCur, pIx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIx); + if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 + && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 + && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0 + && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 + && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 + && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED + ){ + sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); + } + VdbeComment((v, "%s", pIx->zName)); +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + { + u64 colUsed = 0; + int ii, jj; + for(ii=0; iinColumn; ii++){ + jj = pIx->aiColumn[ii]; + if( jj<0 ) continue; + if( jj>63 ) jj = 63; + if( (pTabItem->colUsed & MASKBIT(jj))==0 ) continue; + colUsed |= ((u64)1)<<(ii<63 ? ii : 63); + } + sqlite3VdbeAddOp4Dup8(v, OP_ColumnsUsed, iIndexCur, 0, 0, + (u8*)&colUsed, P4_INT64); + } +#endif /* SQLITE_ENABLE_COLUMN_USED_MASK */ + } + } + if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); + } + pWInfo->iTop = sqlite3VdbeCurrentAddr(v); + if( db->mallocFailed ) goto whereBeginError; + + /* Generate the code to do the search. Each iteration of the for + ** loop below generates code for a single nested loop of the VM + ** program. + */ + for(ii=0; iia[ii]; + wsFlags = pLevel->pWLoop->wsFlags; +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX + if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ + constructAutomaticIndex(pParse, &pWInfo->sWC, + &pTabList->a[pLevel->iFrom], notReady, pLevel); + if( db->mallocFailed ) goto whereBeginError; + } +#endif + addrExplain = sqlite3WhereExplainOneScan( + pParse, pTabList, pLevel, wctrlFlags + ); + pLevel->addrBody = sqlite3VdbeCurrentAddr(v); + notReady = sqlite3WhereCodeOneLoopStart(pParse,v,pWInfo,ii,pLevel,notReady); + pWInfo->iContinue = pLevel->addrCont; + if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){ + sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain); + } + } + + /* Done. */ + VdbeModuleComment((v, "Begin WHERE-core")); + pWInfo->iEndWhere = sqlite3VdbeCurrentAddr(v); + return pWInfo; + + /* Jump here if malloc fails */ +whereBeginError: + if( pWInfo ){ + testcase( pWInfo->pExprMods!=0 ); + whereUndoExprMods(pWInfo); + pParse->nQueryLoop = pWInfo->savedNQueryLoop; + whereInfoFree(db, pWInfo); + } + return 0; +} + +/* +** Part of sqlite3WhereEnd() will rewrite opcodes to reference the +** index rather than the main table. In SQLITE_DEBUG mode, we want +** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine +** does that. +*/ +#ifndef SQLITE_DEBUG +# define OpcodeRewriteTrace(D,K,P) /* no-op */ +#else +# define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P) + static void sqlite3WhereOpcodeRewriteTrace( + sqlite3 *db, + int pc, + VdbeOp *pOp + ){ + if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return; + sqlite3VdbePrintOp(0, pc, pOp); + } +#endif + +/* +** Generate the end of the WHERE loop. See comments on +** sqlite3WhereBegin() for additional information. +*/ +SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ + Parse *pParse = pWInfo->pParse; + Vdbe *v = pParse->pVdbe; + int i; + WhereLevel *pLevel; + WhereLoop *pLoop; + SrcList *pTabList = pWInfo->pTabList; + sqlite3 *db = pParse->db; + int iEnd = sqlite3VdbeCurrentAddr(v); + + /* Generate loop termination code. + */ + VdbeModuleComment((v, "End WHERE-core")); + for(i=pWInfo->nLevel-1; i>=0; i--){ + int addr; + pLevel = &pWInfo->a[i]; + pLoop = pLevel->pWLoop; + if( pLevel->op!=OP_Noop ){ +#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT + int addrSeek = 0; + Index *pIdx; + int n; + if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED + && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */ + && (pLoop->wsFlags & WHERE_INDEXED)!=0 + && (pIdx = pLoop->u.btree.pIndex)->hasStat1 + && (n = pLoop->u.btree.nDistinctCol)>0 + && pIdx->aiRowLogEst[n]>=36 + ){ + int r1 = pParse->nMem+1; + int j, op; + for(j=0; jiIdxCur, j, r1+j); + } + pParse->nMem += n+1; + op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT; + addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n); + VdbeCoverageIf(v, op==OP_SeekLT); + VdbeCoverageIf(v, op==OP_SeekGT); + sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); + } +#endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ + /* The common case: Advance to the next row */ + sqlite3VdbeResolveLabel(v, pLevel->addrCont); + sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); + sqlite3VdbeChangeP5(v, pLevel->p5); + VdbeCoverage(v); + VdbeCoverageIf(v, pLevel->op==OP_Next); + VdbeCoverageIf(v, pLevel->op==OP_Prev); + VdbeCoverageIf(v, pLevel->op==OP_VNext); + if( pLevel->regBignull ){ + sqlite3VdbeResolveLabel(v, pLevel->addrBignull); + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1); + VdbeCoverage(v); + } +#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT + if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek); +#endif + }else{ + sqlite3VdbeResolveLabel(v, pLevel->addrCont); + } + if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ + struct InLoop *pIn; + int j; + sqlite3VdbeResolveLabel(v, pLevel->addrNxt); + for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ + assert( sqlite3VdbeGetOp(v, pIn->addrInTop+1)->opcode==OP_IsNull + || pParse->db->mallocFailed ); + sqlite3VdbeJumpHere(v, pIn->addrInTop+1); + if( pIn->eEndLoopOp!=OP_Noop ){ + if( pIn->nPrefix ){ + int bEarlyOut = + (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0; + if( pLevel->iLeftJoin ){ + /* For LEFT JOIN queries, cursor pIn->iCur may not have been + ** opened yet. This occurs for WHERE clauses such as + ** "a = ? AND b IN (...)", where the index is on (a, b). If + ** the RHS of the (a=?) is NULL, then the "b IN (...)" may + ** never have been coded, but the body of the loop run to + ** return the null-row. So, if the cursor is not open yet, + ** jump over the OP_Next or OP_Prev instruction about to + ** be coded. */ + sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur, + sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut); + VdbeCoverage(v); + } + if( bEarlyOut ){ + sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur, + sqlite3VdbeCurrentAddr(v)+2, + pIn->iBase, pIn->nPrefix); + VdbeCoverage(v); + /* Retarget the OP_IsNull against the left operand of IN so + ** it jumps past the OP_IfNoHope. This is because the + ** OP_IsNull also bypasses the OP_Affinity opcode that is + ** required by OP_IfNoHope. */ + sqlite3VdbeJumpHere(v, pIn->addrInTop+1); + } + } + sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); + VdbeCoverage(v); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next); + } + sqlite3VdbeJumpHere(v, pIn->addrInTop-1); + } + } + sqlite3VdbeResolveLabel(v, pLevel->addrBrk); + if( pLevel->addrSkip ){ + sqlite3VdbeGoto(v, pLevel->addrSkip); + VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); + sqlite3VdbeJumpHere(v, pLevel->addrSkip); + sqlite3VdbeJumpHere(v, pLevel->addrSkip-2); + } +#ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS + if( pLevel->addrLikeRep ){ + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, (int)(pLevel->iLikeRepCntr>>1), + pLevel->addrLikeRep); + VdbeCoverage(v); + } +#endif + if( pLevel->iLeftJoin ){ + int ws = pLoop->wsFlags; + addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); + assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); + if( (ws & WHERE_IDX_ONLY)==0 ){ + assert( pLevel->iTabCur==pTabList->a[pLevel->iFrom].iCursor ); + sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur); + } + if( (ws & WHERE_INDEXED) + || ((ws & WHERE_MULTI_OR) && pLevel->u.pCovidx) + ){ + if( ws & WHERE_MULTI_OR ){ + Index *pIx = pLevel->u.pCovidx; + int iDb = sqlite3SchemaToIndex(db, pIx->pSchema); + sqlite3VdbeAddOp3(v, OP_ReopenIdx, pLevel->iIdxCur, pIx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIx); + } + sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); + } + if( pLevel->op==OP_Return ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pLevel->p1, pLevel->addrFirst); + }else{ + sqlite3VdbeGoto(v, pLevel->addrFirst); + } + sqlite3VdbeJumpHere(v, addr); + } + VdbeModuleComment((v, "End WHERE-loop%d: %s", i, + pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); + } + + /* The "break" point is here, just past the end of the outer loop. + ** Set it. + */ + sqlite3VdbeResolveLabel(v, pWInfo->iBreak); + + assert( pWInfo->nLevel<=pTabList->nSrc ); + for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ + int k, last; + VdbeOp *pOp, *pLastOp; + Index *pIdx = 0; + SrcItem *pTabItem = &pTabList->a[pLevel->iFrom]; + Table *pTab = pTabItem->pTab; + assert( pTab!=0 ); + pLoop = pLevel->pWLoop; + + /* For a co-routine, change all OP_Column references to the table of + ** the co-routine into OP_Copy of result contained in a register. + ** OP_Rowid becomes OP_Null. + */ + if( pTabItem->fg.viaCoroutine ){ + testcase( pParse->db->mallocFailed ); + translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, + pTabItem->regResult, 0); + continue; + } + +#ifdef SQLITE_ENABLE_EARLY_CURSOR_CLOSE + /* Close all of the cursors that were opened by sqlite3WhereBegin. + ** Except, do not close cursors that will be reused by the OR optimization + ** (WHERE_OR_SUBCLAUSE). And do not close the OP_OpenWrite cursors + ** created for the ONEPASS optimization. + */ + if( (pTab->tabFlags & TF_Ephemeral)==0 + && !IsView(pTab) + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 + ){ + int ws = pLoop->wsFlags; + if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){ + sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); + } + if( (ws & WHERE_INDEXED)!=0 + && (ws & (WHERE_IPK|WHERE_AUTO_INDEX))==0 + && pLevel->iIdxCur!=pWInfo->aiCurOnePass[1] + ){ + sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); + } + } +#endif + + /* If this scan uses an index, make VDBE code substitutions to read data + ** from the index instead of from the table where possible. In some cases + ** this optimization prevents the table from ever being read, which can + ** yield a significant performance boost. + ** + ** Calls to the code generator in between sqlite3WhereBegin and + ** sqlite3WhereEnd will have created code that references the table + ** directly. This loop scans all that code looking for opcodes + ** that reference the table and converts them into opcodes that + ** reference the index. + */ + if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){ + pIdx = pLoop->u.btree.pIndex; + }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ + pIdx = pLevel->u.pCovidx; + } + if( pIdx + && !db->mallocFailed + ){ + if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){ + last = iEnd; + }else{ + last = pWInfo->iEndWhere; + } + k = pLevel->addrBody + 1; +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeAddopTrace ){ + printf("TRANSLATE opcodes in range %d..%d\n", k, last-1); + } + /* Proof that the "+1" on the k value above is safe */ + pOp = sqlite3VdbeGetOp(v, k - 1); + assert( pOp->opcode!=OP_Column || pOp->p1!=pLevel->iTabCur ); + assert( pOp->opcode!=OP_Rowid || pOp->p1!=pLevel->iTabCur ); + assert( pOp->opcode!=OP_IfNullRow || pOp->p1!=pLevel->iTabCur ); +#endif + pOp = sqlite3VdbeGetOp(v, k); + pLastOp = pOp + (last - k); + assert( pOp<=pLastOp ); + do{ + if( pOp->p1!=pLevel->iTabCur ){ + /* no-op */ + }else if( pOp->opcode==OP_Column +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + || pOp->opcode==OP_Offset +#endif + ){ + int x = pOp->p2; + assert( pIdx->pTable==pTab ); + if( !HasRowid(pTab) ){ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + x = pPk->aiColumn[x]; + assert( x>=0 ); + }else{ + testcase( x!=sqlite3StorageColumnToTable(pTab,x) ); + x = sqlite3StorageColumnToTable(pTab,x); + } + x = sqlite3TableColumnToIndex(pIdx, x); + if( x>=0 ){ + pOp->p2 = x; + pOp->p1 = pLevel->iIdxCur; + OpcodeRewriteTrace(db, k, pOp); + } + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 + || pWInfo->eOnePass ); + }else if( pOp->opcode==OP_Rowid ){ + pOp->p1 = pLevel->iIdxCur; + pOp->opcode = OP_IdxRowid; + OpcodeRewriteTrace(db, k, pOp); + }else if( pOp->opcode==OP_IfNullRow ){ + pOp->p1 = pLevel->iIdxCur; + OpcodeRewriteTrace(db, k, pOp); + } +#ifdef SQLITE_DEBUG + k++; +#endif + }while( (++pOp)flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n"); +#endif + } + } + + /* Final cleanup + */ + if( pWInfo->pExprMods ) whereUndoExprMods(pWInfo); + pParse->nQueryLoop = pWInfo->savedNQueryLoop; + whereInfoFree(db, pWInfo); + return; +} + +/************** End of where.c ***********************************************/ +/************** Begin file window.c ******************************************/ +/* +** 2018 May 08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ +/* #include "sqliteInt.h" */ + +#ifndef SQLITE_OMIT_WINDOWFUNC + +/* +** SELECT REWRITING +** +** Any SELECT statement that contains one or more window functions in +** either the select list or ORDER BY clause (the only two places window +** functions may be used) is transformed by function sqlite3WindowRewrite() +** in order to support window function processing. For example, with the +** schema: +** +** CREATE TABLE t1(a, b, c, d, e, f, g); +** +** the statement: +** +** SELECT a+1, max(b) OVER (PARTITION BY c ORDER BY d) FROM t1 ORDER BY e; +** +** is transformed to: +** +** SELECT a+1, max(b) OVER (PARTITION BY c ORDER BY d) FROM ( +** SELECT a, e, c, d, b FROM t1 ORDER BY c, d +** ) ORDER BY e; +** +** The flattening optimization is disabled when processing this transformed +** SELECT statement. This allows the implementation of the window function +** (in this case max()) to process rows sorted in order of (c, d), which +** makes things easier for obvious reasons. More generally: +** +** * FROM, WHERE, GROUP BY and HAVING clauses are all moved to +** the sub-query. +** +** * ORDER BY, LIMIT and OFFSET remain part of the parent query. +** +** * Terminals from each of the expression trees that make up the +** select-list and ORDER BY expressions in the parent query are +** selected by the sub-query. For the purposes of the transformation, +** terminals are column references and aggregate functions. +** +** If there is more than one window function in the SELECT that uses +** the same window declaration (the OVER bit), then a single scan may +** be used to process more than one window function. For example: +** +** SELECT max(b) OVER (PARTITION BY c ORDER BY d), +** min(e) OVER (PARTITION BY c ORDER BY d) +** FROM t1; +** +** is transformed in the same way as the example above. However: +** +** SELECT max(b) OVER (PARTITION BY c ORDER BY d), +** min(e) OVER (PARTITION BY a ORDER BY b) +** FROM t1; +** +** Must be transformed to: +** +** SELECT max(b) OVER (PARTITION BY c ORDER BY d) FROM ( +** SELECT e, min(e) OVER (PARTITION BY a ORDER BY b), c, d, b FROM +** SELECT a, e, c, d, b FROM t1 ORDER BY a, b +** ) ORDER BY c, d +** ) ORDER BY e; +** +** so that both min() and max() may process rows in the order defined by +** their respective window declarations. +** +** INTERFACE WITH SELECT.C +** +** When processing the rewritten SELECT statement, code in select.c calls +** sqlite3WhereBegin() to begin iterating through the results of the +** sub-query, which is always implemented as a co-routine. It then calls +** sqlite3WindowCodeStep() to process rows and finish the scan by calling +** sqlite3WhereEnd(). +** +** sqlite3WindowCodeStep() generates VM code so that, for each row returned +** by the sub-query a sub-routine (OP_Gosub) coded by select.c is invoked. +** When the sub-routine is invoked: +** +** * The results of all window-functions for the row are stored +** in the associated Window.regResult registers. +** +** * The required terminal values are stored in the current row of +** temp table Window.iEphCsr. +** +** In some cases, depending on the window frame and the specific window +** functions invoked, sqlite3WindowCodeStep() caches each entire partition +** in a temp table before returning any rows. In other cases it does not. +** This detail is encapsulated within this file, the code generated by +** select.c is the same in either case. +** +** BUILT-IN WINDOW FUNCTIONS +** +** This implementation features the following built-in window functions: +** +** row_number() +** rank() +** dense_rank() +** percent_rank() +** cume_dist() +** ntile(N) +** lead(expr [, offset [, default]]) +** lag(expr [, offset [, default]]) +** first_value(expr) +** last_value(expr) +** nth_value(expr, N) +** +** These are the same built-in window functions supported by Postgres. +** Although the behaviour of aggregate window functions (functions that +** can be used as either aggregates or window funtions) allows them to +** be implemented using an API, built-in window functions are much more +** esoteric. Additionally, some window functions (e.g. nth_value()) +** may only be implemented by caching the entire partition in memory. +** As such, some built-in window functions use the same API as aggregate +** window functions and some are implemented directly using VDBE +** instructions. Additionally, for those functions that use the API, the +** window frame is sometimes modified before the SELECT statement is +** rewritten. For example, regardless of the specified window frame, the +** row_number() function always uses: +** +** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +** +** See sqlite3WindowUpdate() for details. +** +** As well as some of the built-in window functions, aggregate window +** functions min() and max() are implemented using VDBE instructions if +** the start of the window frame is declared as anything other than +** UNBOUNDED PRECEDING. +*/ + +/* +** Implementation of built-in window function row_number(). Assumes that the +** window frame has been coerced to: +** +** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void row_numberStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + i64 *p = (i64*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ) (*p)++; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void row_numberValueFunc(sqlite3_context *pCtx){ + i64 *p = (i64*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + sqlite3_result_int64(pCtx, (p ? *p : 0)); +} + +/* +** Context object type used by rank(), dense_rank(), percent_rank() and +** cume_dist(). +*/ +struct CallCount { + i64 nValue; + i64 nStep; + i64 nTotal; +}; + +/* +** Implementation of built-in window function dense_rank(). Assumes that +** the window frame has been set to: +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void dense_rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ) p->nStep = 1; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void dense_rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + if( p->nStep ){ + p->nValue++; + p->nStep = 0; + } + sqlite3_result_int64(pCtx, p->nValue); + } +} + +/* +** Implementation of built-in window function nth_value(). This +** implementation is used in "slow mode" only - when the EXCLUDE clause +** is not set to the default value "NO OTHERS". +*/ +struct NthValueCtx { + i64 nStep; + sqlite3_value *pValue; +}; +static void nth_valueStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + i64 iVal; + switch( sqlite3_value_numeric_type(apArg[1]) ){ + case SQLITE_INTEGER: + iVal = sqlite3_value_int64(apArg[1]); + break; + case SQLITE_FLOAT: { + double fVal = sqlite3_value_double(apArg[1]); + if( ((i64)fVal)!=fVal ) goto error_out; + iVal = (i64)fVal; + break; + } + default: + goto error_out; + } + if( iVal<=0 ) goto error_out; + + p->nStep++; + if( iVal==p->nStep ){ + p->pValue = sqlite3_value_dup(apArg[0]); + if( !p->pValue ){ + sqlite3_result_error_nomem(pCtx); + } + } + } + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + return; + + error_out: + sqlite3_result_error( + pCtx, "second argument to nth_value must be a positive integer", -1 + ); +} +static void nth_valueFinalizeFunc(sqlite3_context *pCtx){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, 0); + if( p && p->pValue ){ + sqlite3_result_value(pCtx, p->pValue); + sqlite3_value_free(p->pValue); + p->pValue = 0; + } +} +#define nth_valueInvFunc noopStepFunc +#define nth_valueValueFunc noopValueFunc + +static void first_valueStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pValue==0 ){ + p->pValue = sqlite3_value_dup(apArg[0]); + if( !p->pValue ){ + sqlite3_result_error_nomem(pCtx); + } + } + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void first_valueFinalizeFunc(sqlite3_context *pCtx){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pValue ){ + sqlite3_result_value(pCtx, p->pValue); + sqlite3_value_free(p->pValue); + p->pValue = 0; + } +} +#define first_valueInvFunc noopStepFunc +#define first_valueValueFunc noopValueFunc + +/* +** Implementation of built-in window function rank(). Assumes that +** the window frame has been set to: +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nStep++; + if( p->nValue==0 ){ + p->nValue = p->nStep; + } + } + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + sqlite3_result_int64(pCtx, p->nValue); + p->nValue = 0; + } +} + +/* +** Implementation of built-in window function percent_rank(). Assumes that +** the window frame has been set to: +** +** GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING +*/ +static void percent_rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nTotal++; + } +} +static void percent_rankInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p->nStep++; +} +static void percent_rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nValue = p->nStep; + if( p->nTotal>1 ){ + double r = (double)p->nValue / (double)(p->nTotal-1); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_double(pCtx, 0.0); + } + } +} +#define percent_rankFinalizeFunc percent_rankValueFunc + +/* +** Implementation of built-in window function cume_dist(). Assumes that +** the window frame has been set to: +** +** GROUPS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING +*/ +static void cume_distStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nTotal++; + } +} +static void cume_distInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p->nStep++; +} +static void cume_distValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, 0); + if( p ){ + double r = (double)(p->nStep) / (double)(p->nTotal); + sqlite3_result_double(pCtx, r); + } +} +#define cume_distFinalizeFunc cume_distValueFunc + +/* +** Context object for ntile() window function. +*/ +struct NtileCtx { + i64 nTotal; /* Total rows in partition */ + i64 nParam; /* Parameter passed to ntile(N) */ + i64 iRow; /* Current row */ +}; + +/* +** Implementation of ntile(). This assumes that the window frame has +** been coerced to: +** +** ROWS CURRENT ROW AND UNBOUNDED FOLLOWING +*/ +static void ntileStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NtileCtx *p; + assert( nArg==1 ); UNUSED_PARAMETER(nArg); + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + if( p->nTotal==0 ){ + p->nParam = sqlite3_value_int64(apArg[0]); + if( p->nParam<=0 ){ + sqlite3_result_error( + pCtx, "argument of ntile must be a positive integer", -1 + ); + } + } + p->nTotal++; + } +} +static void ntileInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NtileCtx *p; + assert( nArg==1 ); UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p->iRow++; +} +static void ntileValueFunc(sqlite3_context *pCtx){ + struct NtileCtx *p; + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->nParam>0 ){ + int nSize = (p->nTotal / p->nParam); + if( nSize==0 ){ + sqlite3_result_int64(pCtx, p->iRow+1); + }else{ + i64 nLarge = p->nTotal - p->nParam*nSize; + i64 iSmall = nLarge*(nSize+1); + i64 iRow = p->iRow; + + assert( (nLarge*(nSize+1) + (p->nParam-nLarge)*nSize)==p->nTotal ); + + if( iRowpVal); + p->pVal = sqlite3_value_dup(apArg[0]); + if( p->pVal==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + p->nVal++; + } + } +} +static void last_valueInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct LastValueCtx *p; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( ALWAYS(p) ){ + p->nVal--; + if( p->nVal==0 ){ + sqlite3_value_free(p->pVal); + p->pVal = 0; + } + } +} +static void last_valueValueFunc(sqlite3_context *pCtx){ + struct LastValueCtx *p; + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, 0); + if( p && p->pVal ){ + sqlite3_result_value(pCtx, p->pVal); + } +} +static void last_valueFinalizeFunc(sqlite3_context *pCtx){ + struct LastValueCtx *p; + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pVal ){ + sqlite3_result_value(pCtx, p->pVal); + sqlite3_value_free(p->pVal); + p->pVal = 0; + } +} + +/* +** Static names for the built-in window function names. These static +** names are used, rather than string literals, so that FuncDef objects +** can be associated with a particular window function by direct +** comparison of the zName pointer. Example: +** +** if( pFuncDef->zName==row_valueName ){ ... } +*/ +static const char row_numberName[] = "row_number"; +static const char dense_rankName[] = "dense_rank"; +static const char rankName[] = "rank"; +static const char percent_rankName[] = "percent_rank"; +static const char cume_distName[] = "cume_dist"; +static const char ntileName[] = "ntile"; +static const char last_valueName[] = "last_value"; +static const char nth_valueName[] = "nth_value"; +static const char first_valueName[] = "first_value"; +static const char leadName[] = "lead"; +static const char lagName[] = "lag"; + +/* +** No-op implementations of xStep() and xFinalize(). Used as place-holders +** for built-in window functions that never call those interfaces. +** +** The noopValueFunc() is called but is expected to do nothing. The +** noopStepFunc() is never called, and so it is marked with NO_TEST to +** let the test coverage routine know not to expect this function to be +** invoked. +*/ +static void noopStepFunc( /*NO_TEST*/ + sqlite3_context *p, /*NO_TEST*/ + int n, /*NO_TEST*/ + sqlite3_value **a /*NO_TEST*/ +){ /*NO_TEST*/ + UNUSED_PARAMETER(p); /*NO_TEST*/ + UNUSED_PARAMETER(n); /*NO_TEST*/ + UNUSED_PARAMETER(a); /*NO_TEST*/ + assert(0); /*NO_TEST*/ +} /*NO_TEST*/ +static void noopValueFunc(sqlite3_context *p){ UNUSED_PARAMETER(p); /*no-op*/ } + +/* Window functions that use all window interfaces: xStep, xFinal, +** xValue, and xInverse */ +#define WINDOWFUNCALL(name,nArg,extra) { \ + nArg, (SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + name ## StepFunc, name ## FinalizeFunc, name ## ValueFunc, \ + name ## InvFunc, name ## Name, {0} \ +} + +/* Window functions that are implemented using bytecode and thus have +** no-op routines for their methods */ +#define WINDOWFUNCNOOP(name,nArg,extra) { \ + nArg, (SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + noopStepFunc, noopValueFunc, noopValueFunc, \ + noopStepFunc, name ## Name, {0} \ +} + +/* Window functions that use all window interfaces: xStep, the +** same routine for xFinalize and xValue and which never call +** xInverse. */ +#define WINDOWFUNCX(name,nArg,extra) { \ + nArg, (SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + name ## StepFunc, name ## ValueFunc, name ## ValueFunc, \ + noopStepFunc, name ## Name, {0} \ +} + + +/* +** Register those built-in window functions that are not also aggregates. +*/ +SQLITE_PRIVATE void sqlite3WindowFunctions(void){ + static FuncDef aWindowFuncs[] = { + WINDOWFUNCX(row_number, 0, 0), + WINDOWFUNCX(dense_rank, 0, 0), + WINDOWFUNCX(rank, 0, 0), + WINDOWFUNCALL(percent_rank, 0, 0), + WINDOWFUNCALL(cume_dist, 0, 0), + WINDOWFUNCALL(ntile, 1, 0), + WINDOWFUNCALL(last_value, 1, 0), + WINDOWFUNCALL(nth_value, 2, 0), + WINDOWFUNCALL(first_value, 1, 0), + WINDOWFUNCNOOP(lead, 1, 0), + WINDOWFUNCNOOP(lead, 2, 0), + WINDOWFUNCNOOP(lead, 3, 0), + WINDOWFUNCNOOP(lag, 1, 0), + WINDOWFUNCNOOP(lag, 2, 0), + WINDOWFUNCNOOP(lag, 3, 0), + }; + sqlite3InsertBuiltinFuncs(aWindowFuncs, ArraySize(aWindowFuncs)); +} + +static Window *windowFind(Parse *pParse, Window *pList, const char *zName){ + Window *p; + for(p=pList; p; p=p->pNextWin){ + if( sqlite3StrICmp(p->zName, zName)==0 ) break; + } + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such window: %s", zName); + } + return p; +} + +/* +** This function is called immediately after resolving the function name +** for a window function within a SELECT statement. Argument pList is a +** linked list of WINDOW definitions for the current SELECT statement. +** Argument pFunc is the function definition just resolved and pWin +** is the Window object representing the associated OVER clause. This +** function updates the contents of pWin as follows: +** +** * If the OVER clause refered to a named window (as in "max(x) OVER win"), +** search list pList for a matching WINDOW definition, and update pWin +** accordingly. If no such WINDOW clause can be found, leave an error +** in pParse. +** +** * If the function is a built-in window function that requires the +** window to be coerced (see "BUILT-IN WINDOW FUNCTIONS" at the top +** of this file), pWin is updated here. +*/ +SQLITE_PRIVATE void sqlite3WindowUpdate( + Parse *pParse, + Window *pList, /* List of named windows for this SELECT */ + Window *pWin, /* Window frame to update */ + FuncDef *pFunc /* Window function definition */ +){ + if( pWin->zName && pWin->eFrmType==0 ){ + Window *p = windowFind(pParse, pList, pWin->zName); + if( p==0 ) return; + pWin->pPartition = sqlite3ExprListDup(pParse->db, p->pPartition, 0); + pWin->pOrderBy = sqlite3ExprListDup(pParse->db, p->pOrderBy, 0); + pWin->pStart = sqlite3ExprDup(pParse->db, p->pStart, 0); + pWin->pEnd = sqlite3ExprDup(pParse->db, p->pEnd, 0); + pWin->eStart = p->eStart; + pWin->eEnd = p->eEnd; + pWin->eFrmType = p->eFrmType; + pWin->eExclude = p->eExclude; + }else{ + sqlite3WindowChain(pParse, pWin, pList); + } + if( (pWin->eFrmType==TK_RANGE) + && (pWin->pStart || pWin->pEnd) + && (pWin->pOrderBy==0 || pWin->pOrderBy->nExpr!=1) + ){ + sqlite3ErrorMsg(pParse, + "RANGE with offset PRECEDING/FOLLOWING requires one ORDER BY expression" + ); + }else + if( pFunc->funcFlags & SQLITE_FUNC_WINDOW ){ + sqlite3 *db = pParse->db; + if( pWin->pFilter ){ + sqlite3ErrorMsg(pParse, + "FILTER clause may only be used with aggregate window functions" + ); + }else{ + struct WindowUpdate { + const char *zFunc; + int eFrmType; + int eStart; + int eEnd; + } aUp[] = { + { row_numberName, TK_ROWS, TK_UNBOUNDED, TK_CURRENT }, + { dense_rankName, TK_RANGE, TK_UNBOUNDED, TK_CURRENT }, + { rankName, TK_RANGE, TK_UNBOUNDED, TK_CURRENT }, + { percent_rankName, TK_GROUPS, TK_CURRENT, TK_UNBOUNDED }, + { cume_distName, TK_GROUPS, TK_FOLLOWING, TK_UNBOUNDED }, + { ntileName, TK_ROWS, TK_CURRENT, TK_UNBOUNDED }, + { leadName, TK_ROWS, TK_UNBOUNDED, TK_UNBOUNDED }, + { lagName, TK_ROWS, TK_UNBOUNDED, TK_CURRENT }, + }; + int i; + for(i=0; izName==aUp[i].zFunc ){ + sqlite3ExprDelete(db, pWin->pStart); + sqlite3ExprDelete(db, pWin->pEnd); + pWin->pEnd = pWin->pStart = 0; + pWin->eFrmType = aUp[i].eFrmType; + pWin->eStart = aUp[i].eStart; + pWin->eEnd = aUp[i].eEnd; + pWin->eExclude = 0; + if( pWin->eStart==TK_FOLLOWING ){ + pWin->pStart = sqlite3Expr(db, TK_INTEGER, "1"); + } + break; + } + } + } + } + pWin->pFunc = pFunc; +} + +/* +** Context object passed through sqlite3WalkExprList() to +** selectWindowRewriteExprCb() by selectWindowRewriteEList(). +*/ +typedef struct WindowRewrite WindowRewrite; +struct WindowRewrite { + Window *pWin; + SrcList *pSrc; + ExprList *pSub; + Table *pTab; + Select *pSubSelect; /* Current sub-select, if any */ +}; + +/* +** Callback function used by selectWindowRewriteEList(). If necessary, +** this function appends to the output expression-list and updates +** expression (*ppExpr) in place. +*/ +static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ + struct WindowRewrite *p = pWalker->u.pRewrite; + Parse *pParse = pWalker->pParse; + assert( p!=0 ); + assert( p->pWin!=0 ); + + /* If this function is being called from within a scalar sub-select + ** that used by the SELECT statement being processed, only process + ** TK_COLUMN expressions that refer to it (the outer SELECT). Do + ** not process aggregates or window functions at all, as they belong + ** to the scalar sub-select. */ + if( p->pSubSelect ){ + if( pExpr->op!=TK_COLUMN ){ + return WRC_Continue; + }else{ + int nSrc = p->pSrc->nSrc; + int i; + for(i=0; iiTable==p->pSrc->a[i].iCursor ) break; + } + if( i==nSrc ) return WRC_Continue; + } + } + + switch( pExpr->op ){ + + case TK_FUNCTION: + if( !ExprHasProperty(pExpr, EP_WinFunc) ){ + break; + }else{ + Window *pWin; + for(pWin=p->pWin; pWin; pWin=pWin->pNextWin){ + if( pExpr->y.pWin==pWin ){ + assert( pWin->pOwner==pExpr ); + return WRC_Prune; + } + } + } + /* no break */ deliberate_fall_through + + case TK_AGG_FUNCTION: + case TK_COLUMN: { + int iCol = -1; + if( pParse->db->mallocFailed ) return WRC_Abort; + if( p->pSub ){ + int i; + for(i=0; ipSub->nExpr; i++){ + if( 0==sqlite3ExprCompare(0, p->pSub->a[i].pExpr, pExpr, -1) ){ + iCol = i; + break; + } + } + } + if( iCol<0 ){ + Expr *pDup = sqlite3ExprDup(pParse->db, pExpr, 0); + if( pDup && pDup->op==TK_AGG_FUNCTION ) pDup->op = TK_FUNCTION; + p->pSub = sqlite3ExprListAppend(pParse, p->pSub, pDup); + } + if( p->pSub ){ + int f = pExpr->flags & EP_Collate; + assert( ExprHasProperty(pExpr, EP_Static)==0 ); + ExprSetProperty(pExpr, EP_Static); + sqlite3ExprDelete(pParse->db, pExpr); + ExprClearProperty(pExpr, EP_Static); + memset(pExpr, 0, sizeof(Expr)); + + pExpr->op = TK_COLUMN; + pExpr->iColumn = (iCol<0 ? p->pSub->nExpr-1: iCol); + pExpr->iTable = p->pWin->iEphCsr; + pExpr->y.pTab = p->pTab; + pExpr->flags = f; + } + if( pParse->db->mallocFailed ) return WRC_Abort; + break; + } + + default: /* no-op */ + break; + } + + return WRC_Continue; +} +static int selectWindowRewriteSelectCb(Walker *pWalker, Select *pSelect){ + struct WindowRewrite *p = pWalker->u.pRewrite; + Select *pSave = p->pSubSelect; + if( pSave==pSelect ){ + return WRC_Continue; + }else{ + p->pSubSelect = pSelect; + sqlite3WalkSelect(pWalker, pSelect); + p->pSubSelect = pSave; + } + return WRC_Prune; +} + + +/* +** Iterate through each expression in expression-list pEList. For each: +** +** * TK_COLUMN, +** * aggregate function, or +** * window function with a Window object that is not a member of the +** Window list passed as the second argument (pWin). +** +** Append the node to output expression-list (*ppSub). And replace it +** with a TK_COLUMN that reads the (N-1)th element of table +** pWin->iEphCsr, where N is the number of elements in (*ppSub) after +** appending the new one. +*/ +static void selectWindowRewriteEList( + Parse *pParse, + Window *pWin, + SrcList *pSrc, + ExprList *pEList, /* Rewrite expressions in this list */ + Table *pTab, + ExprList **ppSub /* IN/OUT: Sub-select expression-list */ +){ + Walker sWalker; + WindowRewrite sRewrite; + + assert( pWin!=0 ); + memset(&sWalker, 0, sizeof(Walker)); + memset(&sRewrite, 0, sizeof(WindowRewrite)); + + sRewrite.pSub = *ppSub; + sRewrite.pWin = pWin; + sRewrite.pSrc = pSrc; + sRewrite.pTab = pTab; + + sWalker.pParse = pParse; + sWalker.xExprCallback = selectWindowRewriteExprCb; + sWalker.xSelectCallback = selectWindowRewriteSelectCb; + sWalker.u.pRewrite = &sRewrite; + + (void)sqlite3WalkExprList(&sWalker, pEList); + + *ppSub = sRewrite.pSub; +} + +/* +** Append a copy of each expression in expression-list pAppend to +** expression list pList. Return a pointer to the result list. +*/ +static ExprList *exprListAppendList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + ExprList *pAppend, /* List of values to append. Might be NULL */ + int bIntToNull +){ + if( pAppend ){ + int i; + int nInit = pList ? pList->nExpr : 0; + for(i=0; inExpr; i++){ + sqlite3 *db = pParse->db; + Expr *pDup = sqlite3ExprDup(db, pAppend->a[i].pExpr, 0); + assert( pDup==0 || !ExprHasProperty(pDup, EP_MemToken) ); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDup); + break; + } + if( bIntToNull ){ + int iDummy; + Expr *pSub; + pSub = sqlite3ExprSkipCollateAndLikely(pDup); + if( sqlite3ExprIsInteger(pSub, &iDummy) ){ + pSub->op = TK_NULL; + pSub->flags &= ~(EP_IntValue|EP_IsTrue|EP_IsFalse); + pSub->u.zToken = 0; + } + } + pList = sqlite3ExprListAppend(pParse, pList, pDup); + if( pList ) pList->a[nInit+i].sortFlags = pAppend->a[i].sortFlags; + } + } + return pList; +} + +/* +** When rewriting a query, if the new subquery in the FROM clause +** contains TK_AGG_FUNCTION nodes that refer to an outer query, +** then we have to increase the Expr->op2 values of those nodes +** due to the extra subquery layer that was added. +** +** See also the incrAggDepth() routine in resolve.c +*/ +static int sqlite3WindowExtraAggFuncDepth(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_AGG_FUNCTION + && pExpr->op2>=pWalker->walkerDepth + ){ + pExpr->op2++; + } + return WRC_Continue; +} + +static int disallowAggregatesInOrderByCb(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_AGG_FUNCTION && pExpr->pAggInfo==0 ){ + sqlite3ErrorMsg(pWalker->pParse, + "misuse of aggregate: %s()", pExpr->u.zToken); + } + return WRC_Continue; +} + +/* +** If the SELECT statement passed as the second argument does not invoke +** any SQL window functions, this function is a no-op. Otherwise, it +** rewrites the SELECT statement so that window function xStep functions +** are invoked in the correct order as described under "SELECT REWRITING" +** at the top of this file. +*/ +SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ + int rc = SQLITE_OK; + if( p->pWin && p->pPrior==0 && ALWAYS((p->selFlags & SF_WinRewrite)==0) ){ + Vdbe *v = sqlite3GetVdbe(pParse); + sqlite3 *db = pParse->db; + Select *pSub = 0; /* The subquery */ + SrcList *pSrc = p->pSrc; + Expr *pWhere = p->pWhere; + ExprList *pGroupBy = p->pGroupBy; + Expr *pHaving = p->pHaving; + ExprList *pSort = 0; + + ExprList *pSublist = 0; /* Expression list for sub-query */ + Window *pMWin = p->pWin; /* Main window object */ + Window *pWin; /* Window object iterator */ + Table *pTab; + Walker w; + + u32 selFlags = p->selFlags; + + pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ){ + return sqlite3ErrorToParser(db, SQLITE_NOMEM); + } + sqlite3AggInfoPersistWalkerInit(&w, pParse); + sqlite3WalkSelect(&w, p); + if( (p->selFlags & SF_Aggregate)==0 ){ + w.xExprCallback = disallowAggregatesInOrderByCb; + w.xSelectCallback = 0; + sqlite3WalkExprList(&w, p->pOrderBy); + } + + p->pSrc = 0; + p->pWhere = 0; + p->pGroupBy = 0; + p->pHaving = 0; + p->selFlags &= ~SF_Aggregate; + p->selFlags |= SF_WinRewrite; + + /* Create the ORDER BY clause for the sub-select. This is the concatenation + ** of the window PARTITION and ORDER BY clauses. Then, if this makes it + ** redundant, remove the ORDER BY from the parent SELECT. */ + pSort = exprListAppendList(pParse, 0, pMWin->pPartition, 1); + pSort = exprListAppendList(pParse, pSort, pMWin->pOrderBy, 1); + if( pSort && p->pOrderBy && p->pOrderBy->nExpr<=pSort->nExpr ){ + int nSave = pSort->nExpr; + pSort->nExpr = p->pOrderBy->nExpr; + if( sqlite3ExprListCompare(pSort, p->pOrderBy, -1)==0 ){ + sqlite3ExprListDelete(db, p->pOrderBy); + p->pOrderBy = 0; + } + pSort->nExpr = nSave; + } + + /* Assign a cursor number for the ephemeral table used to buffer rows. + ** The OpenEphemeral instruction is coded later, after it is known how + ** many columns the table will have. */ + pMWin->iEphCsr = pParse->nTab++; + pParse->nTab += 3; + + selectWindowRewriteEList(pParse, pMWin, pSrc, p->pEList, pTab, &pSublist); + selectWindowRewriteEList(pParse, pMWin, pSrc, p->pOrderBy, pTab, &pSublist); + pMWin->nBufferCol = (pSublist ? pSublist->nExpr : 0); + + /* Append the PARTITION BY and ORDER BY expressions to the to the + ** sub-select expression list. They are required to figure out where + ** boundaries for partitions and sets of peer rows lie. */ + pSublist = exprListAppendList(pParse, pSublist, pMWin->pPartition, 0); + pSublist = exprListAppendList(pParse, pSublist, pMWin->pOrderBy, 0); + + /* Append the arguments passed to each window function to the + ** sub-select expression list. Also allocate two registers for each + ** window function - one for the accumulator, another for interim + ** results. */ + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + ExprList *pArgs = pWin->pOwner->x.pList; + if( pWin->pFunc->funcFlags & SQLITE_FUNC_SUBTYPE ){ + selectWindowRewriteEList(pParse, pMWin, pSrc, pArgs, pTab, &pSublist); + pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); + pWin->bExprArgs = 1; + }else{ + pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); + pSublist = exprListAppendList(pParse, pSublist, pArgs, 0); + } + if( pWin->pFilter ){ + Expr *pFilter = sqlite3ExprDup(db, pWin->pFilter, 0); + pSublist = sqlite3ExprListAppend(pParse, pSublist, pFilter); + } + pWin->regAccum = ++pParse->nMem; + pWin->regResult = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + } + + /* If there is no ORDER BY or PARTITION BY clause, and the window + ** function accepts zero arguments, and there are no other columns + ** selected (e.g. "SELECT row_number() OVER () FROM t1"), it is possible + ** that pSublist is still NULL here. Add a constant expression here to + ** keep everything legal in this case. + */ + if( pSublist==0 ){ + pSublist = sqlite3ExprListAppend(pParse, 0, + sqlite3Expr(db, TK_INTEGER, "0") + ); + } + + pSub = sqlite3SelectNew( + pParse, pSublist, pSrc, pWhere, pGroupBy, pHaving, pSort, 0, 0 + ); + SELECTTRACE(1,pParse,pSub, + ("New window-function subquery in FROM clause of (%u/%p)\n", + p->selId, p)); + p->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + assert( pSub!=0 || p->pSrc==0 ); /* Due to db->mallocFailed test inside + ** of sqlite3DbMallocRawNN() called from + ** sqlite3SrcListAppend() */ + if( p->pSrc ){ + Table *pTab2; + p->pSrc->a[0].pSelect = pSub; + sqlite3SrcListAssignCursors(pParse, p->pSrc); + pSub->selFlags |= SF_Expanded|SF_OrderByReqd; + pTab2 = sqlite3ResultSetOfSelect(pParse, pSub, SQLITE_AFF_NONE); + pSub->selFlags |= (selFlags & SF_Aggregate); + if( pTab2==0 ){ + /* Might actually be some other kind of error, but in that case + ** pParse->nErr will be set, so if SQLITE_NOMEM is set, we will get + ** the correct error message regardless. */ + rc = SQLITE_NOMEM; + }else{ + memcpy(pTab, pTab2, sizeof(Table)); + pTab->tabFlags |= TF_Ephemeral; + p->pSrc->a[0].pTab = pTab; + pTab = pTab2; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3WindowExtraAggFuncDepth; + w.xSelectCallback = sqlite3WalkerDepthIncrease; + w.xSelectCallback2 = sqlite3WalkerDepthDecrease; + sqlite3WalkSelect(&w, pSub); + } + }else{ + sqlite3SelectDelete(db, pSub); + } + if( db->mallocFailed ) rc = SQLITE_NOMEM; + sqlite3DbFree(db, pTab); + } + + if( rc ){ + if( pParse->nErr==0 ){ + assert( pParse->db->mallocFailed ); + sqlite3ErrorToParser(pParse->db, SQLITE_NOMEM); + } + } + return rc; +} + +/* +** Unlink the Window object from the Select to which it is attached, +** if it is attached. +*/ +SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window *p){ + if( p->ppThis ){ + *p->ppThis = p->pNextWin; + if( p->pNextWin ) p->pNextWin->ppThis = p->ppThis; + p->ppThis = 0; + } +} + +/* +** Free the Window object passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3 *db, Window *p){ + if( p ){ + sqlite3WindowUnlinkFromSelect(p); + sqlite3ExprDelete(db, p->pFilter); + sqlite3ExprListDelete(db, p->pPartition); + sqlite3ExprListDelete(db, p->pOrderBy); + sqlite3ExprDelete(db, p->pEnd); + sqlite3ExprDelete(db, p->pStart); + sqlite3DbFree(db, p->zName); + sqlite3DbFree(db, p->zBase); + sqlite3DbFree(db, p); + } +} + +/* +** Free the linked list of Window objects starting at the second argument. +*/ +SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p){ + while( p ){ + Window *pNext = p->pNextWin; + sqlite3WindowDelete(db, p); + p = pNext; + } +} + +/* +** The argument expression is an PRECEDING or FOLLOWING offset. The +** value should be a non-negative integer. If the value is not a +** constant, change it to NULL. The fact that it is then a non-negative +** integer will be caught later. But it is important not to leave +** variable values in the expression tree. +*/ +static Expr *sqlite3WindowOffsetExpr(Parse *pParse, Expr *pExpr){ + if( 0==sqlite3ExprIsConstant(pExpr) ){ + if( IN_RENAME_OBJECT ) sqlite3RenameExprUnmap(pParse, pExpr); + sqlite3ExprDelete(pParse->db, pExpr); + pExpr = sqlite3ExprAlloc(pParse->db, TK_NULL, 0, 0); + } + return pExpr; +} + +/* +** Allocate and return a new Window object describing a Window Definition. +*/ +SQLITE_PRIVATE Window *sqlite3WindowAlloc( + Parse *pParse, /* Parsing context */ + int eType, /* Frame type. TK_RANGE, TK_ROWS, TK_GROUPS, or 0 */ + int eStart, /* Start type: CURRENT, PRECEDING, FOLLOWING, UNBOUNDED */ + Expr *pStart, /* Start window size if TK_PRECEDING or FOLLOWING */ + int eEnd, /* End type: CURRENT, FOLLOWING, TK_UNBOUNDED, PRECEDING */ + Expr *pEnd, /* End window size if TK_FOLLOWING or PRECEDING */ + u8 eExclude /* EXCLUDE clause */ +){ + Window *pWin = 0; + int bImplicitFrame = 0; + + /* Parser assures the following: */ + assert( eType==0 || eType==TK_RANGE || eType==TK_ROWS || eType==TK_GROUPS ); + assert( eStart==TK_CURRENT || eStart==TK_PRECEDING + || eStart==TK_UNBOUNDED || eStart==TK_FOLLOWING ); + assert( eEnd==TK_CURRENT || eEnd==TK_FOLLOWING + || eEnd==TK_UNBOUNDED || eEnd==TK_PRECEDING ); + assert( (eStart==TK_PRECEDING || eStart==TK_FOLLOWING)==(pStart!=0) ); + assert( (eEnd==TK_FOLLOWING || eEnd==TK_PRECEDING)==(pEnd!=0) ); + + if( eType==0 ){ + bImplicitFrame = 1; + eType = TK_RANGE; + } + + /* Additionally, the + ** starting boundary type may not occur earlier in the following list than + ** the ending boundary type: + ** + ** UNBOUNDED PRECEDING + ** PRECEDING + ** CURRENT ROW + ** FOLLOWING + ** UNBOUNDED FOLLOWING + ** + ** The parser ensures that "UNBOUNDED PRECEDING" cannot be used as an ending + ** boundary, and than "UNBOUNDED FOLLOWING" cannot be used as a starting + ** frame boundary. + */ + if( (eStart==TK_CURRENT && eEnd==TK_PRECEDING) + || (eStart==TK_FOLLOWING && (eEnd==TK_PRECEDING || eEnd==TK_CURRENT)) + ){ + sqlite3ErrorMsg(pParse, "unsupported frame specification"); + goto windowAllocErr; + } + + pWin = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( pWin==0 ) goto windowAllocErr; + pWin->eFrmType = eType; + pWin->eStart = eStart; + pWin->eEnd = eEnd; + if( eExclude==0 && OptimizationDisabled(pParse->db, SQLITE_WindowFunc) ){ + eExclude = TK_NO; + } + pWin->eExclude = eExclude; + pWin->bImplicitFrame = bImplicitFrame; + pWin->pEnd = sqlite3WindowOffsetExpr(pParse, pEnd); + pWin->pStart = sqlite3WindowOffsetExpr(pParse, pStart); + return pWin; + +windowAllocErr: + sqlite3ExprDelete(pParse->db, pEnd); + sqlite3ExprDelete(pParse->db, pStart); + return 0; +} + +/* +** Attach PARTITION and ORDER BY clauses pPartition and pOrderBy to window +** pWin. Also, if parameter pBase is not NULL, set pWin->zBase to the +** equivalent nul-terminated string. +*/ +SQLITE_PRIVATE Window *sqlite3WindowAssemble( + Parse *pParse, + Window *pWin, + ExprList *pPartition, + ExprList *pOrderBy, + Token *pBase +){ + if( pWin ){ + pWin->pPartition = pPartition; + pWin->pOrderBy = pOrderBy; + if( pBase ){ + pWin->zBase = sqlite3DbStrNDup(pParse->db, pBase->z, pBase->n); + } + }else{ + sqlite3ExprListDelete(pParse->db, pPartition); + sqlite3ExprListDelete(pParse->db, pOrderBy); + } + return pWin; +} + +/* +** Window *pWin has just been created from a WINDOW clause. Tokne pBase +** is the base window. Earlier windows from the same WINDOW clause are +** stored in the linked list starting at pWin->pNextWin. This function +** either updates *pWin according to the base specification, or else +** leaves an error in pParse. +*/ +SQLITE_PRIVATE void sqlite3WindowChain(Parse *pParse, Window *pWin, Window *pList){ + if( pWin->zBase ){ + sqlite3 *db = pParse->db; + Window *pExist = windowFind(pParse, pList, pWin->zBase); + if( pExist ){ + const char *zErr = 0; + /* Check for errors */ + if( pWin->pPartition ){ + zErr = "PARTITION clause"; + }else if( pExist->pOrderBy && pWin->pOrderBy ){ + zErr = "ORDER BY clause"; + }else if( pExist->bImplicitFrame==0 ){ + zErr = "frame specification"; + } + if( zErr ){ + sqlite3ErrorMsg(pParse, + "cannot override %s of window: %s", zErr, pWin->zBase + ); + }else{ + pWin->pPartition = sqlite3ExprListDup(db, pExist->pPartition, 0); + if( pExist->pOrderBy ){ + assert( pWin->pOrderBy==0 ); + pWin->pOrderBy = sqlite3ExprListDup(db, pExist->pOrderBy, 0); + } + sqlite3DbFree(db, pWin->zBase); + pWin->zBase = 0; + } + } + } +} + +/* +** Attach window object pWin to expression p. +*/ +SQLITE_PRIVATE void sqlite3WindowAttach(Parse *pParse, Expr *p, Window *pWin){ + if( p ){ + assert( p->op==TK_FUNCTION ); + assert( pWin ); + p->y.pWin = pWin; + ExprSetProperty(p, EP_WinFunc); + pWin->pOwner = p; + if( (p->flags & EP_Distinct) && pWin->eFrmType!=TK_FILTER ){ + sqlite3ErrorMsg(pParse, + "DISTINCT is not supported for window functions" + ); + } + }else{ + sqlite3WindowDelete(pParse->db, pWin); + } +} + +/* +** Possibly link window pWin into the list at pSel->pWin (window functions +** to be processed as part of SELECT statement pSel). The window is linked +** in if either (a) there are no other windows already linked to this +** SELECT, or (b) the windows already linked use a compatible window frame. +*/ +SQLITE_PRIVATE void sqlite3WindowLink(Select *pSel, Window *pWin){ + if( pSel ){ + if( 0==pSel->pWin || 0==sqlite3WindowCompare(0, pSel->pWin, pWin, 0) ){ + pWin->pNextWin = pSel->pWin; + if( pSel->pWin ){ + pSel->pWin->ppThis = &pWin->pNextWin; + } + pSel->pWin = pWin; + pWin->ppThis = &pSel->pWin; + }else{ + if( sqlite3ExprListCompare(pWin->pPartition, pSel->pWin->pPartition,-1) ){ + pSel->selFlags |= SF_MultiPart; + } + } + } +} + +/* +** Return 0 if the two window objects are identical, 1 if they are +** different, or 2 if it cannot be determined if the objects are identical +** or not. Identical window objects can be processed in a single scan. +*/ +SQLITE_PRIVATE int sqlite3WindowCompare( + const Parse *pParse, + const Window *p1, + const Window *p2, + int bFilter +){ + int res; + if( NEVER(p1==0) || NEVER(p2==0) ) return 1; + if( p1->eFrmType!=p2->eFrmType ) return 1; + if( p1->eStart!=p2->eStart ) return 1; + if( p1->eEnd!=p2->eEnd ) return 1; + if( p1->eExclude!=p2->eExclude ) return 1; + if( sqlite3ExprCompare(pParse, p1->pStart, p2->pStart, -1) ) return 1; + if( sqlite3ExprCompare(pParse, p1->pEnd, p2->pEnd, -1) ) return 1; + if( (res = sqlite3ExprListCompare(p1->pPartition, p2->pPartition, -1)) ){ + return res; + } + if( (res = sqlite3ExprListCompare(p1->pOrderBy, p2->pOrderBy, -1)) ){ + return res; + } + if( bFilter ){ + if( (res = sqlite3ExprCompare(pParse, p1->pFilter, p2->pFilter, -1)) ){ + return res; + } + } + return 0; +} + + +/* +** This is called by code in select.c before it calls sqlite3WhereBegin() +** to begin iterating through the sub-query results. It is used to allocate +** and initialize registers and cursors used by sqlite3WindowCodeStep(). +*/ +SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse *pParse, Select *pSelect){ + int nEphExpr = pSelect->pSrc->a[0].pSelect->pEList->nExpr; + Window *pMWin = pSelect->pWin; + Window *pWin; + Vdbe *v = sqlite3GetVdbe(pParse); + + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pMWin->iEphCsr, nEphExpr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+1, pMWin->iEphCsr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+2, pMWin->iEphCsr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+3, pMWin->iEphCsr); + + /* Allocate registers to use for PARTITION BY values, if any. Initialize + ** said registers to NULL. */ + if( pMWin->pPartition ){ + int nExpr = pMWin->pPartition->nExpr; + pMWin->regPart = pParse->nMem+1; + pParse->nMem += nExpr; + sqlite3VdbeAddOp3(v, OP_Null, 0, pMWin->regPart, pMWin->regPart+nExpr-1); + } + + pMWin->regOne = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regOne); + + if( pMWin->eExclude ){ + pMWin->regStartRowid = ++pParse->nMem; + pMWin->regEndRowid = ++pParse->nMem; + pMWin->csrApp = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regStartRowid); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pMWin->regEndRowid); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->csrApp, pMWin->iEphCsr); + return; + } + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *p = pWin->pFunc; + if( (p->funcFlags & SQLITE_FUNC_MINMAX) && pWin->eStart!=TK_UNBOUNDED ){ + /* The inline versions of min() and max() require a single ephemeral + ** table and 3 registers. The registers are used as follows: + ** + ** regApp+0: slot to copy min()/max() argument to for MakeRecord + ** regApp+1: integer value used to ensure keys are unique + ** regApp+2: output of MakeRecord + */ + ExprList *pList = pWin->pOwner->x.pList; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pList, 0, 0); + pWin->csrApp = pParse->nTab++; + pWin->regApp = pParse->nMem+1; + pParse->nMem += 3; + if( pKeyInfo && pWin->pFunc->zName[1]=='i' ){ + assert( pKeyInfo->aSortFlags[0]==0 ); + pKeyInfo->aSortFlags[0] = KEYINFO_ORDER_DESC; + } + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pWin->csrApp, 2); + sqlite3VdbeAppendP4(v, pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + else if( p->zName==nth_valueName || p->zName==first_valueName ){ + /* Allocate two registers at pWin->regApp. These will be used to + ** store the start and end index of the current frame. */ + pWin->regApp = pParse->nMem+1; + pWin->csrApp = pParse->nTab++; + pParse->nMem += 2; + sqlite3VdbeAddOp2(v, OP_OpenDup, pWin->csrApp, pMWin->iEphCsr); + } + else if( p->zName==leadName || p->zName==lagName ){ + pWin->csrApp = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenDup, pWin->csrApp, pMWin->iEphCsr); + } + } +} + +#define WINDOW_STARTING_INT 0 +#define WINDOW_ENDING_INT 1 +#define WINDOW_NTH_VALUE_INT 2 +#define WINDOW_STARTING_NUM 3 +#define WINDOW_ENDING_NUM 4 + +/* +** A "PRECEDING " (eCond==0) or "FOLLOWING " (eCond==1) or the +** value of the second argument to nth_value() (eCond==2) has just been +** evaluated and the result left in register reg. This function generates VM +** code to check that the value is a non-negative integer and throws an +** exception if it is not. +*/ +static void windowCheckValue(Parse *pParse, int reg, int eCond){ + static const char *azErr[] = { + "frame starting offset must be a non-negative integer", + "frame ending offset must be a non-negative integer", + "second argument to nth_value must be a positive integer", + "frame starting offset must be a non-negative number", + "frame ending offset must be a non-negative number", + }; + static int aOp[] = { OP_Ge, OP_Ge, OP_Gt, OP_Ge, OP_Ge }; + Vdbe *v = sqlite3GetVdbe(pParse); + int regZero = sqlite3GetTempReg(pParse); + assert( eCond>=0 && eCond=WINDOW_STARTING_NUM ){ + int regString = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, regString, 0, "", P4_STATIC); + sqlite3VdbeAddOp3(v, OP_Ge, regString, sqlite3VdbeCurrentAddr(v)+2, reg); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC|SQLITE_JUMPIFNULL); + VdbeCoverage(v); + assert( eCond==3 || eCond==4 ); + VdbeCoverageIf(v, eCond==3); + VdbeCoverageIf(v, eCond==4); + }else{ + sqlite3VdbeAddOp2(v, OP_MustBeInt, reg, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + assert( eCond==0 || eCond==1 || eCond==2 ); + VdbeCoverageIf(v, eCond==0); + VdbeCoverageIf(v, eCond==1); + VdbeCoverageIf(v, eCond==2); + } + sqlite3VdbeAddOp3(v, aOp[eCond], regZero, sqlite3VdbeCurrentAddr(v)+2, reg); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC); + VdbeCoverageNeverNullIf(v, eCond==0); /* NULL case captured by */ + VdbeCoverageNeverNullIf(v, eCond==1); /* the OP_MustBeInt */ + VdbeCoverageNeverNullIf(v, eCond==2); + VdbeCoverageNeverNullIf(v, eCond==3); /* NULL case caught by */ + VdbeCoverageNeverNullIf(v, eCond==4); /* the OP_Ge */ + sqlite3MayAbort(pParse); + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_ERROR, OE_Abort); + sqlite3VdbeAppendP4(v, (void*)azErr[eCond], P4_STATIC); + sqlite3ReleaseTempReg(pParse, regZero); +} + +/* +** Return the number of arguments passed to the window-function associated +** with the object passed as the only argument to this function. +*/ +static int windowArgCount(Window *pWin){ + ExprList *pList = pWin->pOwner->x.pList; + return (pList ? pList->nExpr : 0); +} + +typedef struct WindowCodeArg WindowCodeArg; +typedef struct WindowCsrAndReg WindowCsrAndReg; + +/* +** See comments above struct WindowCodeArg. +*/ +struct WindowCsrAndReg { + int csr; /* Cursor number */ + int reg; /* First in array of peer values */ +}; + +/* +** A single instance of this structure is allocated on the stack by +** sqlite3WindowCodeStep() and a pointer to it passed to the various helper +** routines. This is to reduce the number of arguments required by each +** helper function. +** +** regArg: +** Each window function requires an accumulator register (just as an +** ordinary aggregate function does). This variable is set to the first +** in an array of accumulator registers - one for each window function +** in the WindowCodeArg.pMWin list. +** +** eDelete: +** The window functions implementation sometimes caches the input rows +** that it processes in a temporary table. If it is not zero, this +** variable indicates when rows may be removed from the temp table (in +** order to reduce memory requirements - it would always be safe just +** to leave them there). Possible values for eDelete are: +** +** WINDOW_RETURN_ROW: +** An input row can be discarded after it is returned to the caller. +** +** WINDOW_AGGINVERSE: +** An input row can be discarded after the window functions xInverse() +** callbacks have been invoked in it. +** +** WINDOW_AGGSTEP: +** An input row can be discarded after the window functions xStep() +** callbacks have been invoked in it. +** +** start,current,end +** Consider a window-frame similar to the following: +** +** (ORDER BY a, b GROUPS BETWEEN 2 PRECEDING AND 2 FOLLOWING) +** +** The windows functions implmentation caches the input rows in a temp +** table, sorted by "a, b" (it actually populates the cache lazily, and +** aggressively removes rows once they are no longer required, but that's +** a mere detail). It keeps three cursors open on the temp table. One +** (current) that points to the next row to return to the query engine +** once its window function values have been calculated. Another (end) +** points to the next row to call the xStep() method of each window function +** on (so that it is 2 groups ahead of current). And a third (start) that +** points to the next row to call the xInverse() method of each window +** function on. +** +** Each cursor (start, current and end) consists of a VDBE cursor +** (WindowCsrAndReg.csr) and an array of registers (starting at +** WindowCodeArg.reg) that always contains a copy of the peer values +** read from the corresponding cursor. +** +** Depending on the window-frame in question, all three cursors may not +** be required. In this case both WindowCodeArg.csr and reg are set to +** 0. +*/ +struct WindowCodeArg { + Parse *pParse; /* Parse context */ + Window *pMWin; /* First in list of functions being processed */ + Vdbe *pVdbe; /* VDBE object */ + int addrGosub; /* OP_Gosub to this address to return one row */ + int regGosub; /* Register used with OP_Gosub(addrGosub) */ + int regArg; /* First in array of accumulator registers */ + int eDelete; /* See above */ + int regRowid; + + WindowCsrAndReg start; + WindowCsrAndReg current; + WindowCsrAndReg end; +}; + +/* +** Generate VM code to read the window frames peer values from cursor csr into +** an array of registers starting at reg. +*/ +static void windowReadPeerValues( + WindowCodeArg *p, + int csr, + int reg +){ + Window *pMWin = p->pMWin; + ExprList *pOrderBy = pMWin->pOrderBy; + if( pOrderBy ){ + Vdbe *v = sqlite3GetVdbe(p->pParse); + ExprList *pPart = pMWin->pPartition; + int iColOff = pMWin->nBufferCol + (pPart ? pPart->nExpr : 0); + int i; + for(i=0; inExpr; i++){ + sqlite3VdbeAddOp3(v, OP_Column, csr, iColOff+i, reg+i); + } + } +} + +/* +** Generate VM code to invoke either xStep() (if bInverse is 0) or +** xInverse (if bInverse is non-zero) for each window function in the +** linked list starting at pMWin. Or, for built-in window functions +** that do not use the standard function API, generate the required +** inline VM code. +** +** If argument csr is greater than or equal to 0, then argument reg is +** the first register in an array of registers guaranteed to be large +** enough to hold the array of arguments for each function. In this case +** the arguments are extracted from the current row of csr into the +** array of registers before invoking OP_AggStep or OP_AggInverse +** +** Or, if csr is less than zero, then the array of registers at reg is +** already populated with all columns from the current row of the sub-query. +** +** If argument regPartSize is non-zero, then it is a register containing the +** number of rows in the current partition. +*/ +static void windowAggStep( + WindowCodeArg *p, + Window *pMWin, /* Linked list of window functions */ + int csr, /* Read arguments from this cursor */ + int bInverse, /* True to invoke xInverse instead of xStep */ + int reg /* Array of registers */ +){ + Parse *pParse = p->pParse; + Vdbe *v = sqlite3GetVdbe(pParse); + Window *pWin; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + int regArg; + int nArg = pWin->bExprArgs ? 0 : windowArgCount(pWin); + int i; + + assert( bInverse==0 || pWin->eStart!=TK_UNBOUNDED ); + + /* All OVER clauses in the same window function aggregate step must + ** be the same. */ + assert( pWin==pMWin || sqlite3WindowCompare(pParse,pWin,pMWin,0)!=1 ); + + for(i=0; izName!=nth_valueName ){ + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+i, reg+i); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, pMWin->iEphCsr, pWin->iArgCol+i, reg+i); + } + } + regArg = reg; + + if( pMWin->regStartRowid==0 + && (pFunc->funcFlags & SQLITE_FUNC_MINMAX) + && (pWin->eStart!=TK_UNBOUNDED) + ){ + int addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regArg); + VdbeCoverage(v); + if( bInverse==0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, pWin->regApp+1, 1); + sqlite3VdbeAddOp2(v, OP_SCopy, regArg, pWin->regApp); + sqlite3VdbeAddOp3(v, OP_MakeRecord, pWin->regApp, 2, pWin->regApp+2); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pWin->csrApp, pWin->regApp+2); + }else{ + sqlite3VdbeAddOp4Int(v, OP_SeekGE, pWin->csrApp, 0, regArg, 1); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp1(v, OP_Delete, pWin->csrApp); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + } + sqlite3VdbeJumpHere(v, addrIsNull); + }else if( pWin->regApp ){ + assert( pFunc->zName==nth_valueName + || pFunc->zName==first_valueName + ); + assert( bInverse==0 || bInverse==1 ); + sqlite3VdbeAddOp2(v, OP_AddImm, pWin->regApp+1-bInverse, 1); + }else if( pFunc->xSFunc!=noopStepFunc ){ + int addrIf = 0; + if( pWin->pFilter ){ + int regTmp; + assert( pWin->bExprArgs || !nArg ||nArg==pWin->pOwner->x.pList->nExpr ); + assert( pWin->bExprArgs || nArg ||pWin->pOwner->x.pList==0 ); + regTmp = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+nArg,regTmp); + addrIf = sqlite3VdbeAddOp3(v, OP_IfNot, regTmp, 0, 1); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, regTmp); + } + + if( pWin->bExprArgs ){ + int iOp = sqlite3VdbeCurrentAddr(v); + int iEnd; + + nArg = pWin->pOwner->x.pList->nExpr; + regArg = sqlite3GetTempRange(pParse, nArg); + sqlite3ExprCodeExprList(pParse, pWin->pOwner->x.pList, regArg, 0, 0); + + for(iEnd=sqlite3VdbeCurrentAddr(v); iOpopcode==OP_Column && pOp->p1==pWin->iEphCsr ){ + pOp->p1 = csr; + } + } + } + if( pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl; + assert( nArg>0 ); + pColl = sqlite3ExprNNCollSeq(pParse, pWin->pOwner->x.pList->a[0].pExpr); + sqlite3VdbeAddOp4(v, OP_CollSeq, 0,0,0, (const char*)pColl, P4_COLLSEQ); + } + sqlite3VdbeAddOp3(v, bInverse? OP_AggInverse : OP_AggStep, + bInverse, regArg, pWin->regAccum); + sqlite3VdbeAppendP4(v, pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + if( pWin->bExprArgs ){ + sqlite3ReleaseTempRange(pParse, regArg, nArg); + } + if( addrIf ) sqlite3VdbeJumpHere(v, addrIf); + } + } +} + +/* +** Values that may be passed as the second argument to windowCodeOp(). +*/ +#define WINDOW_RETURN_ROW 1 +#define WINDOW_AGGINVERSE 2 +#define WINDOW_AGGSTEP 3 + +/* +** Generate VM code to invoke either xValue() (bFin==0) or xFinalize() +** (bFin==1) for each window function in the linked list starting at +** pMWin. Or, for built-in window-functions that do not use the standard +** API, generate the equivalent VM code. +*/ +static void windowAggFinal(WindowCodeArg *p, int bFin){ + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; + Vdbe *v = sqlite3GetVdbe(pParse); + Window *pWin; + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + if( pMWin->regStartRowid==0 + && (pWin->pFunc->funcFlags & SQLITE_FUNC_MINMAX) + && (pWin->eStart!=TK_UNBOUNDED) + ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + sqlite3VdbeAddOp1(v, OP_Last, pWin->csrApp); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, pWin->csrApp, 0, pWin->regResult); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + }else if( pWin->regApp ){ + assert( pMWin->regStartRowid==0 ); + }else{ + int nArg = windowArgCount(pWin); + if( bFin ){ + sqlite3VdbeAddOp2(v, OP_AggFinal, pWin->regAccum, nArg); + sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); + sqlite3VdbeAddOp2(v, OP_Copy, pWin->regAccum, pWin->regResult); + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + }else{ + sqlite3VdbeAddOp3(v, OP_AggValue,pWin->regAccum,nArg,pWin->regResult); + sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); + } + } + } +} + +/* +** Generate code to calculate the current values of all window functions in the +** p->pMWin list by doing a full scan of the current window frame. Store the +** results in the Window.regResult registers, ready to return the upper +** layer. +*/ +static void windowFullScan(WindowCodeArg *p){ + Window *pWin; + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; + Vdbe *v = p->pVdbe; + + int regCRowid = 0; /* Current rowid value */ + int regCPeer = 0; /* Current peer values */ + int regRowid = 0; /* AggStep rowid value */ + int regPeer = 0; /* AggStep peer values */ + + int nPeer; + int lblNext; + int lblBrk; + int addrNext; + int csr; + + VdbeModuleComment((v, "windowFullScan begin")); + + assert( pMWin!=0 ); + csr = pMWin->csrApp; + nPeer = (pMWin->pOrderBy ? pMWin->pOrderBy->nExpr : 0); + + lblNext = sqlite3VdbeMakeLabel(pParse); + lblBrk = sqlite3VdbeMakeLabel(pParse); + + regCRowid = sqlite3GetTempReg(pParse); + regRowid = sqlite3GetTempReg(pParse); + if( nPeer ){ + regCPeer = sqlite3GetTempRange(pParse, nPeer); + regPeer = sqlite3GetTempRange(pParse, nPeer); + } + + sqlite3VdbeAddOp2(v, OP_Rowid, pMWin->iEphCsr, regCRowid); + windowReadPeerValues(p, pMWin->iEphCsr, regCPeer); + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + } + + sqlite3VdbeAddOp3(v, OP_SeekGE, csr, lblBrk, pMWin->regStartRowid); + VdbeCoverage(v); + addrNext = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Rowid, csr, regRowid); + sqlite3VdbeAddOp3(v, OP_Gt, pMWin->regEndRowid, lblBrk, regRowid); + VdbeCoverageNeverNull(v); + + if( pMWin->eExclude==TK_CURRENT ){ + sqlite3VdbeAddOp3(v, OP_Eq, regCRowid, lblNext, regRowid); + VdbeCoverageNeverNull(v); + }else if( pMWin->eExclude!=TK_NO ){ + int addr; + int addrEq = 0; + KeyInfo *pKeyInfo = 0; + + if( pMWin->pOrderBy ){ + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pMWin->pOrderBy, 0, 0); + } + if( pMWin->eExclude==TK_TIES ){ + addrEq = sqlite3VdbeAddOp3(v, OP_Eq, regCRowid, 0, regRowid); + VdbeCoverageNeverNull(v); + } + if( pKeyInfo ){ + windowReadPeerValues(p, csr, regPeer); + sqlite3VdbeAddOp3(v, OP_Compare, regPeer, regCPeer, nPeer); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + addr = sqlite3VdbeCurrentAddr(v)+1; + sqlite3VdbeAddOp3(v, OP_Jump, addr, lblNext, addr); + VdbeCoverageEqNe(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblNext); + } + if( addrEq ) sqlite3VdbeJumpHere(v, addrEq); + } + + windowAggStep(p, pMWin, csr, 0, p->regArg); + + sqlite3VdbeResolveLabel(v, lblNext); + sqlite3VdbeAddOp2(v, OP_Next, csr, addrNext); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrNext-1); + sqlite3VdbeJumpHere(v, addrNext+1); + sqlite3ReleaseTempReg(pParse, regRowid); + sqlite3ReleaseTempReg(pParse, regCRowid); + if( nPeer ){ + sqlite3ReleaseTempRange(pParse, regPeer, nPeer); + sqlite3ReleaseTempRange(pParse, regCPeer, nPeer); + } + + windowAggFinal(p, 1); + VdbeModuleComment((v, "windowFullScan end")); +} + +/* +** Invoke the sub-routine at regGosub (generated by code in select.c) to +** return the current row of Window.iEphCsr. If all window functions are +** aggregate window functions that use the standard API, a single +** OP_Gosub instruction is all that this routine generates. Extra VM code +** for per-row processing is only generated for the following built-in window +** functions: +** +** nth_value() +** first_value() +** lag() +** lead() +*/ +static void windowReturnOneRow(WindowCodeArg *p){ + Window *pMWin = p->pMWin; + Vdbe *v = p->pVdbe; + + if( pMWin->regStartRowid ){ + windowFullScan(p); + }else{ + Parse *pParse = p->pParse; + Window *pWin; + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + if( pFunc->zName==nth_valueName + || pFunc->zName==first_valueName + ){ + int csr = pWin->csrApp; + int lbl = sqlite3VdbeMakeLabel(pParse); + int tmpReg = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + + if( pFunc->zName==nth_valueName ){ + sqlite3VdbeAddOp3(v, OP_Column,pMWin->iEphCsr,pWin->iArgCol+1,tmpReg); + windowCheckValue(pParse, tmpReg, 2); + }else{ + sqlite3VdbeAddOp2(v, OP_Integer, 1, tmpReg); + } + sqlite3VdbeAddOp3(v, OP_Add, tmpReg, pWin->regApp, tmpReg); + sqlite3VdbeAddOp3(v, OP_Gt, pWin->regApp+1, lbl, tmpReg); + VdbeCoverageNeverNull(v); + sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, 0, tmpReg); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); + sqlite3VdbeResolveLabel(v, lbl); + sqlite3ReleaseTempReg(pParse, tmpReg); + } + else if( pFunc->zName==leadName || pFunc->zName==lagName ){ + int nArg = pWin->pOwner->x.pList->nExpr; + int csr = pWin->csrApp; + int lbl = sqlite3VdbeMakeLabel(pParse); + int tmpReg = sqlite3GetTempReg(pParse); + int iEph = pMWin->iEphCsr; + + if( nArg<3 ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, iEph,pWin->iArgCol+2,pWin->regResult); + } + sqlite3VdbeAddOp2(v, OP_Rowid, iEph, tmpReg); + if( nArg<2 ){ + int val = (pFunc->zName==leadName ? 1 : -1); + sqlite3VdbeAddOp2(v, OP_AddImm, tmpReg, val); + }else{ + int op = (pFunc->zName==leadName ? OP_Add : OP_Subtract); + int tmpReg2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, iEph, pWin->iArgCol+1, tmpReg2); + sqlite3VdbeAddOp3(v, op, tmpReg2, tmpReg, tmpReg); + sqlite3ReleaseTempReg(pParse, tmpReg2); + } + + sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, lbl, tmpReg); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); + sqlite3VdbeResolveLabel(v, lbl); + sqlite3ReleaseTempReg(pParse, tmpReg); + } + } + } + sqlite3VdbeAddOp2(v, OP_Gosub, p->regGosub, p->addrGosub); +} + +/* +** Generate code to set the accumulator register for each window function +** in the linked list passed as the second argument to NULL. And perform +** any equivalent initialization required by any built-in window functions +** in the list. +*/ +static int windowInitAccum(Parse *pParse, Window *pMWin){ + Vdbe *v = sqlite3GetVdbe(pParse); + int regArg; + int nArg = 0; + Window *pWin; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + assert( pWin->regAccum ); + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + nArg = MAX(nArg, windowArgCount(pWin)); + if( pMWin->regStartRowid==0 ){ + if( pFunc->zName==nth_valueName || pFunc->zName==first_valueName ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + + if( (pFunc->funcFlags & SQLITE_FUNC_MINMAX) && pWin->csrApp ){ + assert( pWin->eStart!=TK_UNBOUNDED ); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pWin->csrApp); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + } + } + regArg = pParse->nMem+1; + pParse->nMem += nArg; + return regArg; +} + +/* +** Return true if the current frame should be cached in the ephemeral table, +** even if there are no xInverse() calls required. +*/ +static int windowCacheFrame(Window *pMWin){ + Window *pWin; + if( pMWin->regStartRowid ) return 1; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + if( (pFunc->zName==nth_valueName) + || (pFunc->zName==first_valueName) + || (pFunc->zName==leadName) + || (pFunc->zName==lagName) + ){ + return 1; + } + } + return 0; +} + +/* +** regOld and regNew are each the first register in an array of size +** pOrderBy->nExpr. This function generates code to compare the two +** arrays of registers using the collation sequences and other comparison +** parameters specified by pOrderBy. +** +** If the two arrays are not equal, the contents of regNew is copied to +** regOld and control falls through. Otherwise, if the contents of the arrays +** are equal, an OP_Goto is executed. The address of the OP_Goto is returned. +*/ +static void windowIfNewPeer( + Parse *pParse, + ExprList *pOrderBy, + int regNew, /* First in array of new values */ + int regOld, /* First in array of old values */ + int addr /* Jump here */ +){ + Vdbe *v = sqlite3GetVdbe(pParse); + if( pOrderBy ){ + int nVal = pOrderBy->nExpr; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOrderBy, 0, 0); + sqlite3VdbeAddOp3(v, OP_Compare, regOld, regNew, nVal); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, + sqlite3VdbeCurrentAddr(v)+1, addr, sqlite3VdbeCurrentAddr(v)+1 + ); + VdbeCoverageEqNe(v); + sqlite3VdbeAddOp3(v, OP_Copy, regNew, regOld, nVal-1); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); + } +} + +/* +** This function is called as part of generating VM programs for RANGE +** offset PRECEDING/FOLLOWING frame boundaries. Assuming "ASC" order for +** the ORDER BY term in the window, and that argument op is OP_Ge, it generates +** code equivalent to: +** +** if( csr1.peerVal + regVal >= csr2.peerVal ) goto lbl; +** +** The value of parameter op may also be OP_Gt or OP_Le. In these cases the +** operator in the above pseudo-code is replaced with ">" or "<=", respectively. +** +** If the sort-order for the ORDER BY term in the window is DESC, then the +** comparison is reversed. Instead of adding regVal to csr1.peerVal, it is +** subtracted. And the comparison operator is inverted to - ">=" becomes "<=", +** ">" becomes "<", and so on. So, with DESC sort order, if the argument op +** is OP_Ge, the generated code is equivalent to: +** +** if( csr1.peerVal - regVal <= csr2.peerVal ) goto lbl; +** +** A special type of arithmetic is used such that if csr1.peerVal is not +** a numeric type (real or integer), then the result of the addition +** or subtraction is a a copy of csr1.peerVal. +*/ +static void windowCodeRangeTest( + WindowCodeArg *p, + int op, /* OP_Ge, OP_Gt, or OP_Le */ + int csr1, /* Cursor number for cursor 1 */ + int regVal, /* Register containing non-negative number */ + int csr2, /* Cursor number for cursor 2 */ + int lbl /* Jump destination if condition is true */ +){ + Parse *pParse = p->pParse; + Vdbe *v = sqlite3GetVdbe(pParse); + ExprList *pOrderBy = p->pMWin->pOrderBy; /* ORDER BY clause for window */ + int reg1 = sqlite3GetTempReg(pParse); /* Reg. for csr1.peerVal+regVal */ + int reg2 = sqlite3GetTempReg(pParse); /* Reg. for csr2.peerVal */ + int regString = ++pParse->nMem; /* Reg. for constant value '' */ + int arith = OP_Add; /* OP_Add or OP_Subtract */ + int addrGe; /* Jump destination */ + int addrDone = sqlite3VdbeMakeLabel(pParse); /* Address past OP_Ge */ + CollSeq *pColl; + + /* Read the peer-value from each cursor into a register */ + windowReadPeerValues(p, csr1, reg1); + windowReadPeerValues(p, csr2, reg2); + + assert( op==OP_Ge || op==OP_Gt || op==OP_Le ); + assert( pOrderBy && pOrderBy->nExpr==1 ); + if( pOrderBy->a[0].sortFlags & KEYINFO_ORDER_DESC ){ + switch( op ){ + case OP_Ge: op = OP_Le; break; + case OP_Gt: op = OP_Lt; break; + default: assert( op==OP_Le ); op = OP_Ge; break; + } + arith = OP_Subtract; + } + + VdbeModuleComment((v, "CodeRangeTest: if( R%d %s R%d %s R%d ) goto lbl", + reg1, (arith==OP_Add ? "+" : "-"), regVal, + ((op==OP_Ge) ? ">=" : (op==OP_Le) ? "<=" : (op==OP_Gt) ? ">" : "<"), reg2 + )); + + /* If the BIGNULL flag is set for the ORDER BY, then it is required to + ** consider NULL values to be larger than all other values, instead of + ** the usual smaller. The VDBE opcodes OP_Ge and so on do not handle this + ** (and adding that capability causes a performance regression), so + ** instead if the BIGNULL flag is set then cases where either reg1 or + ** reg2 are NULL are handled separately in the following block. The code + ** generated is equivalent to: + ** + ** if( reg1 IS NULL ){ + ** if( op==OP_Ge ) goto lbl; + ** if( op==OP_Gt && reg2 IS NOT NULL ) goto lbl; + ** if( op==OP_Le && reg2 IS NULL ) goto lbl; + ** }else if( reg2 IS NULL ){ + ** if( op==OP_Le ) goto lbl; + ** } + ** + ** Additionally, if either reg1 or reg2 are NULL but the jump to lbl is + ** not taken, control jumps over the comparison operator coded below this + ** block. */ + if( pOrderBy->a[0].sortFlags & KEYINFO_ORDER_BIGNULL ){ + /* This block runs if reg1 contains a NULL. */ + int addr = sqlite3VdbeAddOp1(v, OP_NotNull, reg1); VdbeCoverage(v); + switch( op ){ + case OP_Ge: + sqlite3VdbeAddOp2(v, OP_Goto, 0, lbl); + break; + case OP_Gt: + sqlite3VdbeAddOp2(v, OP_NotNull, reg2, lbl); + VdbeCoverage(v); + break; + case OP_Le: + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); + VdbeCoverage(v); + break; + default: assert( op==OP_Lt ); /* no-op */ break; + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrDone); + + /* This block runs if reg1 is not NULL, but reg2 is. */ + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); VdbeCoverage(v); + if( op==OP_Gt || op==OP_Ge ){ + sqlite3VdbeChangeP2(v, -1, addrDone); + } + } + + /* Register reg1 currently contains csr1.peerVal (the peer-value from csr1). + ** This block adds (or subtracts for DESC) the numeric value in regVal + ** from it. Or, if reg1 is not numeric (it is a NULL, a text value or a blob), + ** then leave reg1 as it is. In pseudo-code, this is implemented as: + ** + ** if( reg1>='' ) goto addrGe; + ** reg1 = reg1 +/- regVal + ** addrGe: + ** + ** Since all strings and blobs are greater-than-or-equal-to an empty string, + ** the add/subtract is skipped for these, as required. If reg1 is a NULL, + ** then the arithmetic is performed, but since adding or subtracting from + ** NULL is always NULL anyway, this case is handled as required too. */ + sqlite3VdbeAddOp4(v, OP_String8, 0, regString, 0, "", P4_STATIC); + addrGe = sqlite3VdbeAddOp3(v, OP_Ge, regString, 0, reg1); + VdbeCoverage(v); + if( (op==OP_Ge && arith==OP_Add) || (op==OP_Le && arith==OP_Subtract) ){ + sqlite3VdbeAddOp3(v, op, reg2, lbl, reg1); VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, arith, regVal, reg1, reg1); + sqlite3VdbeJumpHere(v, addrGe); + + /* Compare registers reg2 and reg1, taking the jump if required. Note that + ** control skips over this test if the BIGNULL flag is set and either + ** reg1 or reg2 contain a NULL value. */ + sqlite3VdbeAddOp3(v, op, reg2, lbl, reg1); VdbeCoverage(v); + pColl = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[0].pExpr); + sqlite3VdbeAppendP4(v, (void*)pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + sqlite3VdbeResolveLabel(v, addrDone); + + assert( op==OP_Ge || op==OP_Gt || op==OP_Lt || op==OP_Le ); + testcase(op==OP_Ge); VdbeCoverageIf(v, op==OP_Ge); + testcase(op==OP_Lt); VdbeCoverageIf(v, op==OP_Lt); + testcase(op==OP_Le); VdbeCoverageIf(v, op==OP_Le); + testcase(op==OP_Gt); VdbeCoverageIf(v, op==OP_Gt); + sqlite3ReleaseTempReg(pParse, reg1); + sqlite3ReleaseTempReg(pParse, reg2); + + VdbeModuleComment((v, "CodeRangeTest: end")); +} + +/* +** Helper function for sqlite3WindowCodeStep(). Each call to this function +** generates VM code for a single RETURN_ROW, AGGSTEP or AGGINVERSE +** operation. Refer to the header comment for sqlite3WindowCodeStep() for +** details. +*/ +static int windowCodeOp( + WindowCodeArg *p, /* Context object */ + int op, /* WINDOW_RETURN_ROW, AGGSTEP or AGGINVERSE */ + int regCountdown, /* Register for OP_IfPos countdown */ + int jumpOnEof /* Jump here if stepped cursor reaches EOF */ +){ + int csr, reg; + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; + int ret = 0; + Vdbe *v = p->pVdbe; + int addrContinue = 0; + int bPeer = (pMWin->eFrmType!=TK_ROWS); + + int lblDone = sqlite3VdbeMakeLabel(pParse); + int addrNextRange = 0; + + /* Special case - WINDOW_AGGINVERSE is always a no-op if the frame + ** starts with UNBOUNDED PRECEDING. */ + if( op==WINDOW_AGGINVERSE && pMWin->eStart==TK_UNBOUNDED ){ + assert( regCountdown==0 && jumpOnEof==0 ); + return 0; + } + + if( regCountdown>0 ){ + if( pMWin->eFrmType==TK_RANGE ){ + addrNextRange = sqlite3VdbeCurrentAddr(v); + assert( op==WINDOW_AGGINVERSE || op==WINDOW_AGGSTEP ); + if( op==WINDOW_AGGINVERSE ){ + if( pMWin->eStart==TK_FOLLOWING ){ + windowCodeRangeTest( + p, OP_Le, p->current.csr, regCountdown, p->start.csr, lblDone + ); + }else{ + windowCodeRangeTest( + p, OP_Ge, p->start.csr, regCountdown, p->current.csr, lblDone + ); + } + }else{ + windowCodeRangeTest( + p, OP_Gt, p->end.csr, regCountdown, p->current.csr, lblDone + ); + } + }else{ + sqlite3VdbeAddOp3(v, OP_IfPos, regCountdown, lblDone, 1); + VdbeCoverage(v); + } + } + + if( op==WINDOW_RETURN_ROW && pMWin->regStartRowid==0 ){ + windowAggFinal(p, 0); + } + addrContinue = sqlite3VdbeCurrentAddr(v); + + /* If this is a (RANGE BETWEEN a FOLLOWING AND b FOLLOWING) or + ** (RANGE BETWEEN b PRECEDING AND a PRECEDING) frame, ensure the + ** start cursor does not advance past the end cursor within the + ** temporary table. It otherwise might, if (a>b). Also ensure that, + ** if the input cursor is still finding new rows, that the end + ** cursor does not go past it to EOF. */ + if( pMWin->eStart==pMWin->eEnd && regCountdown + && pMWin->eFrmType==TK_RANGE + ){ + int regRowid1 = sqlite3GetTempReg(pParse); + int regRowid2 = sqlite3GetTempReg(pParse); + if( op==WINDOW_AGGINVERSE ){ + sqlite3VdbeAddOp2(v, OP_Rowid, p->start.csr, regRowid1); + sqlite3VdbeAddOp2(v, OP_Rowid, p->end.csr, regRowid2); + sqlite3VdbeAddOp3(v, OP_Ge, regRowid2, lblDone, regRowid1); + VdbeCoverage(v); + }else if( p->regRowid ){ + sqlite3VdbeAddOp2(v, OP_Rowid, p->end.csr, regRowid1); + sqlite3VdbeAddOp3(v, OP_Ge, p->regRowid, lblDone, regRowid1); + VdbeCoverageNeverNull(v); + } + sqlite3ReleaseTempReg(pParse, regRowid1); + sqlite3ReleaseTempReg(pParse, regRowid2); + assert( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_FOLLOWING ); + } + + switch( op ){ + case WINDOW_RETURN_ROW: + csr = p->current.csr; + reg = p->current.reg; + windowReturnOneRow(p); + break; + + case WINDOW_AGGINVERSE: + csr = p->start.csr; + reg = p->start.reg; + if( pMWin->regStartRowid ){ + assert( pMWin->regEndRowid ); + sqlite3VdbeAddOp2(v, OP_AddImm, pMWin->regStartRowid, 1); + }else{ + windowAggStep(p, pMWin, csr, 1, p->regArg); + } + break; + + default: + assert( op==WINDOW_AGGSTEP ); + csr = p->end.csr; + reg = p->end.reg; + if( pMWin->regStartRowid ){ + assert( pMWin->regEndRowid ); + sqlite3VdbeAddOp2(v, OP_AddImm, pMWin->regEndRowid, 1); + }else{ + windowAggStep(p, pMWin, csr, 0, p->regArg); + } + break; + } + + if( op==p->eDelete ){ + sqlite3VdbeAddOp1(v, OP_Delete, csr); + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); + } + + if( jumpOnEof ){ + sqlite3VdbeAddOp2(v, OP_Next, csr, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + ret = sqlite3VdbeAddOp0(v, OP_Goto); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, csr, sqlite3VdbeCurrentAddr(v)+1+bPeer); + VdbeCoverage(v); + if( bPeer ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblDone); + } + } + + if( bPeer ){ + int nReg = (pMWin->pOrderBy ? pMWin->pOrderBy->nExpr : 0); + int regTmp = (nReg ? sqlite3GetTempRange(pParse, nReg) : 0); + windowReadPeerValues(p, csr, regTmp); + windowIfNewPeer(pParse, pMWin->pOrderBy, regTmp, reg, addrContinue); + sqlite3ReleaseTempRange(pParse, regTmp, nReg); + } + + if( addrNextRange ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNextRange); + } + sqlite3VdbeResolveLabel(v, lblDone); + return ret; +} + + +/* +** Allocate and return a duplicate of the Window object indicated by the +** third argument. Set the Window.pOwner field of the new object to +** pOwner. +*/ +SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){ + Window *pNew = 0; + if( ALWAYS(p) ){ + pNew = sqlite3DbMallocZero(db, sizeof(Window)); + if( pNew ){ + pNew->zName = sqlite3DbStrDup(db, p->zName); + pNew->zBase = sqlite3DbStrDup(db, p->zBase); + pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0); + pNew->pFunc = p->pFunc; + pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0); + pNew->eFrmType = p->eFrmType; + pNew->eEnd = p->eEnd; + pNew->eStart = p->eStart; + pNew->eExclude = p->eExclude; + pNew->regResult = p->regResult; + pNew->regAccum = p->regAccum; + pNew->iArgCol = p->iArgCol; + pNew->iEphCsr = p->iEphCsr; + pNew->bExprArgs = p->bExprArgs; + pNew->pStart = sqlite3ExprDup(db, p->pStart, 0); + pNew->pEnd = sqlite3ExprDup(db, p->pEnd, 0); + pNew->pOwner = pOwner; + pNew->bImplicitFrame = p->bImplicitFrame; + } + } + return pNew; +} + +/* +** Return a copy of the linked list of Window objects passed as the +** second argument. +*/ +SQLITE_PRIVATE Window *sqlite3WindowListDup(sqlite3 *db, Window *p){ + Window *pWin; + Window *pRet = 0; + Window **pp = &pRet; + + for(pWin=p; pWin; pWin=pWin->pNextWin){ + *pp = sqlite3WindowDup(db, 0, pWin); + if( *pp==0 ) break; + pp = &((*pp)->pNextWin); + } + + return pRet; +} + +/* +** Return true if it can be determined at compile time that expression +** pExpr evaluates to a value that, when cast to an integer, is greater +** than zero. False otherwise. +** +** If an OOM error occurs, this function sets the Parse.db.mallocFailed +** flag and returns zero. +*/ +static int windowExprGtZero(Parse *pParse, Expr *pExpr){ + int ret = 0; + sqlite3 *db = pParse->db; + sqlite3_value *pVal = 0; + sqlite3ValueFromExpr(db, pExpr, db->enc, SQLITE_AFF_NUMERIC, &pVal); + if( pVal && sqlite3_value_int(pVal)>0 ){ + ret = 1; + } + sqlite3ValueFree(pVal); + return ret; +} + +/* +** sqlite3WhereBegin() has already been called for the SELECT statement +** passed as the second argument when this function is invoked. It generates +** code to populate the Window.regResult register for each window function +** and invoke the sub-routine at instruction addrGosub once for each row. +** sqlite3WhereEnd() is always called before returning. +** +** This function handles several different types of window frames, which +** require slightly different processing. The following pseudo code is +** used to implement window frames of the form: +** +** ROWS BETWEEN PRECEDING AND FOLLOWING +** +** Other window frame types use variants of the following: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** +** if( first row of partition ){ +** // Rewind three cursors, all open on the eph table. +** Rewind(csrEnd); +** Rewind(csrStart); +** Rewind(csrCurrent); +** +** regEnd = // FOLLOWING expression +** regStart = // PRECEDING expression +** }else{ +** // First time this branch is taken, the eph table contains two +** // rows. The first row in the partition, which all three cursors +** // currently point to, and the following row. +** AGGSTEP +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** RETURN ROW +** if( csrCurrent is EOF ) break; +** if( (regStart--)<=0 ){ +** AggInverse(csrStart) +** Next(csrStart) +** } +** } +** +** The pseudo-code above uses the following shorthand: +** +** AGGSTEP: invoke the aggregate xStep() function for each window function +** with arguments read from the current row of cursor csrEnd, then +** step cursor csrEnd forward one row (i.e. sqlite3BtreeNext()). +** +** RETURN_ROW: return a row to the caller based on the contents of the +** current row of csrCurrent and the current state of all +** aggregates. Then step cursor csrCurrent forward one row. +** +** AGGINVERSE: invoke the aggregate xInverse() function for each window +** functions with arguments read from the current row of cursor +** csrStart. Then step csrStart forward one row. +** +** There are two other ROWS window frames that are handled significantly +** differently from the above - "BETWEEN PRECEDING AND PRECEDING" +** and "BETWEEN FOLLOWING AND FOLLOWING". These are special +** cases because they change the order in which the three cursors (csrStart, +** csrCurrent and csrEnd) iterate through the ephemeral table. Cases that +** use UNBOUNDED or CURRENT ROW are much simpler variations on one of these +** three. +** +** ROWS BETWEEN PRECEDING AND PRECEDING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** if( (regEnd--)<=0 ){ +** AGGSTEP +** } +** RETURN_ROW +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** flush: +** if( (regEnd--)<=0 ){ +** AGGSTEP +** } +** RETURN_ROW +** +** +** ROWS BETWEEN FOLLOWING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = regEnd - +** }else{ +** AGGSTEP +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** } +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** if( eof ) break; +** } +** if( (regStart--)<=0 ){ +** AGGINVERSE +** if( eof ) break +** } +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** For the most part, the patterns above are adapted to support UNBOUNDED by +** assuming that it is equivalent to "infinity PRECEDING/FOLLOWING" and +** CURRENT ROW by assuming that it is equivilent to "0 PRECEDING/FOLLOWING". +** This is optimized of course - branches that will never be taken and +** conditions that are always true are omitted from the VM code. The only +** exceptional case is: +** +** ROWS BETWEEN FOLLOWING AND UNBOUNDED FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regStart = +** }else{ +** AGGSTEP +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** if( (regStart--)<=0 ){ +** AGGINVERSE +** if( eof ) break +** } +** RETURN_ROW +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** Also requiring special handling are the cases: +** +** ROWS BETWEEN PRECEDING AND PRECEDING +** ROWS BETWEEN FOLLOWING AND FOLLOWING +** +** when (expr1 < expr2). This is detected at runtime, not by this function. +** To handle this case, the pseudo-code programs depicted above are modified +** slightly to be: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** if( regEnd < regStart ){ +** RETURN_ROW +** delete eph table contents +** continue +** } +** ... +** +** The new "continue" statement in the above jumps to the next iteration +** of the outer loop - the one started by sqlite3WhereBegin(). +** +** The various GROUPS cases are implemented using the same patterns as +** ROWS. The VM code is modified slightly so that: +** +** 1. The else branch in the main loop is only taken if the row just +** added to the ephemeral table is the start of a new group. In +** other words, it becomes: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else if( new group ){ +** ... +** } +** } +** +** 2. Instead of processing a single row, each RETURN_ROW, AGGSTEP or +** AGGINVERSE step processes the current row of the relevant cursor and +** all subsequent rows belonging to the same group. +** +** RANGE window frames are a little different again. As for GROUPS, the +** main loop runs once per group only. And RETURN_ROW, AGGSTEP and AGGINVERSE +** deal in groups instead of rows. As for ROWS and GROUPS, there are three +** basic cases: +** +** RANGE BETWEEN PRECEDING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** AGGSTEP +** while( (csrCurrent.key + regEnd) < csrEnd.key ){ +** RETURN_ROW +** while( csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** RETURN ROW +** if( csrCurrent is EOF ) break; +** while( csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** } +** } +** +** In the above notation, "csr.key" means the current value of the ORDER BY +** expression (there is only ever 1 for a RANGE that uses an FOLLOWING +** or PRECEDING AND PRECEDING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** while( (csrEnd.key + regEnd) <= csrCurrent.key ){ +** AGGSTEP +** } +** while( (csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** } +** } +** flush: +** while( (csrEnd.key + regEnd) <= csrCurrent.key ){ +** AGGSTEP +** } +** while( (csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** +** RANGE BETWEEN FOLLOWING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** AGGSTEP +** while( (csrCurrent.key + regEnd) < csrEnd.key ){ +** while( (csrCurrent.key + regStart) > csrStart.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** while( (csrCurrent.key + regStart) > csrStart.key ){ +** AGGINVERSE +** if( eof ) break "while( 1 )" loop. +** } +** RETURN_ROW +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** The text above leaves out many details. Refer to the code and comments +** below for a more complete picture. +*/ +SQLITE_PRIVATE void sqlite3WindowCodeStep( + Parse *pParse, /* Parse context */ + Select *p, /* Rewritten SELECT statement */ + WhereInfo *pWInfo, /* Context returned by sqlite3WhereBegin() */ + int regGosub, /* Register for OP_Gosub */ + int addrGosub /* OP_Gosub here to return each row */ +){ + Window *pMWin = p->pWin; + ExprList *pOrderBy = pMWin->pOrderBy; + Vdbe *v = sqlite3GetVdbe(pParse); + int csrWrite; /* Cursor used to write to eph. table */ + int csrInput = p->pSrc->a[0].iCursor; /* Cursor of sub-select */ + int nInput = p->pSrc->a[0].pTab->nCol; /* Number of cols returned by sub */ + int iInput; /* To iterate through sub cols */ + int addrNe; /* Address of OP_Ne */ + int addrGosubFlush = 0; /* Address of OP_Gosub to flush: */ + int addrInteger = 0; /* Address of OP_Integer */ + int addrEmpty; /* Address of OP_Rewind in flush: */ + int regNew; /* Array of registers holding new input row */ + int regRecord; /* regNew array in record form */ + int regNewPeer = 0; /* Peer values for new row (part of regNew) */ + int regPeer = 0; /* Peer values for current row */ + int regFlushPart = 0; /* Register for "Gosub flush_partition" */ + WindowCodeArg s; /* Context object for sub-routines */ + int lblWhereEnd; /* Label just before sqlite3WhereEnd() code */ + int regStart = 0; /* Value of PRECEDING */ + int regEnd = 0; /* Value of FOLLOWING */ + + assert( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_CURRENT + || pMWin->eStart==TK_FOLLOWING || pMWin->eStart==TK_UNBOUNDED + ); + assert( pMWin->eEnd==TK_FOLLOWING || pMWin->eEnd==TK_CURRENT + || pMWin->eEnd==TK_UNBOUNDED || pMWin->eEnd==TK_PRECEDING + ); + assert( pMWin->eExclude==0 || pMWin->eExclude==TK_CURRENT + || pMWin->eExclude==TK_GROUP || pMWin->eExclude==TK_TIES + || pMWin->eExclude==TK_NO + ); + + lblWhereEnd = sqlite3VdbeMakeLabel(pParse); + + /* Fill in the context object */ + memset(&s, 0, sizeof(WindowCodeArg)); + s.pParse = pParse; + s.pMWin = pMWin; + s.pVdbe = v; + s.regGosub = regGosub; + s.addrGosub = addrGosub; + s.current.csr = pMWin->iEphCsr; + csrWrite = s.current.csr+1; + s.start.csr = s.current.csr+2; + s.end.csr = s.current.csr+3; + + /* Figure out when rows may be deleted from the ephemeral table. There + ** are four options - they may never be deleted (eDelete==0), they may + ** be deleted as soon as they are no longer part of the window frame + ** (eDelete==WINDOW_AGGINVERSE), they may be deleted as after the row + ** has been returned to the caller (WINDOW_RETURN_ROW), or they may + ** be deleted after they enter the frame (WINDOW_AGGSTEP). */ + switch( pMWin->eStart ){ + case TK_FOLLOWING: + if( pMWin->eFrmType!=TK_RANGE + && windowExprGtZero(pParse, pMWin->pStart) + ){ + s.eDelete = WINDOW_RETURN_ROW; + } + break; + case TK_UNBOUNDED: + if( windowCacheFrame(pMWin)==0 ){ + if( pMWin->eEnd==TK_PRECEDING ){ + if( pMWin->eFrmType!=TK_RANGE + && windowExprGtZero(pParse, pMWin->pEnd) + ){ + s.eDelete = WINDOW_AGGSTEP; + } + }else{ + s.eDelete = WINDOW_RETURN_ROW; + } + } + break; + default: + s.eDelete = WINDOW_AGGINVERSE; + break; + } + + /* Allocate registers for the array of values from the sub-query, the + ** samve values in record form, and the rowid used to insert said record + ** into the ephemeral table. */ + regNew = pParse->nMem+1; + pParse->nMem += nInput; + regRecord = ++pParse->nMem; + s.regRowid = ++pParse->nMem; + + /* If the window frame contains an " PRECEDING" or " FOLLOWING" + ** clause, allocate registers to store the results of evaluating each + ** . */ + if( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_FOLLOWING ){ + regStart = ++pParse->nMem; + } + if( pMWin->eEnd==TK_PRECEDING || pMWin->eEnd==TK_FOLLOWING ){ + regEnd = ++pParse->nMem; + } + + /* If this is not a "ROWS BETWEEN ..." frame, then allocate arrays of + ** registers to store copies of the ORDER BY expressions (peer values) + ** for the main loop, and for each cursor (start, current and end). */ + if( pMWin->eFrmType!=TK_ROWS ){ + int nPeer = (pOrderBy ? pOrderBy->nExpr : 0); + regNewPeer = regNew + pMWin->nBufferCol; + if( pMWin->pPartition ) regNewPeer += pMWin->pPartition->nExpr; + regPeer = pParse->nMem+1; pParse->nMem += nPeer; + s.start.reg = pParse->nMem+1; pParse->nMem += nPeer; + s.current.reg = pParse->nMem+1; pParse->nMem += nPeer; + s.end.reg = pParse->nMem+1; pParse->nMem += nPeer; + } + + /* Load the column values for the row returned by the sub-select + ** into an array of registers starting at regNew. Assemble them into + ** a record in register regRecord. */ + for(iInput=0; iInputpPartition ){ + int addr; + ExprList *pPart = pMWin->pPartition; + int nPart = pPart->nExpr; + int regNewPart = regNew + pMWin->nBufferCol; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pPart, 0, 0); + + regFlushPart = ++pParse->nMem; + addr = sqlite3VdbeAddOp3(v, OP_Compare, regNewPart, pMWin->regPart, nPart); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, addr+2, addr+4, addr+2); + VdbeCoverageEqNe(v); + addrGosubFlush = sqlite3VdbeAddOp1(v, OP_Gosub, regFlushPart); + VdbeComment((v, "call flush_partition")); + sqlite3VdbeAddOp3(v, OP_Copy, regNewPart, pMWin->regPart, nPart-1); + } + + /* Insert the new row into the ephemeral table */ + sqlite3VdbeAddOp2(v, OP_NewRowid, csrWrite, s.regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, csrWrite, regRecord, s.regRowid); + addrNe = sqlite3VdbeAddOp3(v, OP_Ne, pMWin->regOne, 0, s.regRowid); + VdbeCoverageNeverNull(v); + + /* This block is run for the first row of each partition */ + s.regArg = windowInitAccum(pParse, pMWin); + + if( regStart ){ + sqlite3ExprCode(pParse, pMWin->pStart, regStart); + windowCheckValue(pParse, regStart, 0 + (pMWin->eFrmType==TK_RANGE?3:0)); + } + if( regEnd ){ + sqlite3ExprCode(pParse, pMWin->pEnd, regEnd); + windowCheckValue(pParse, regEnd, 1 + (pMWin->eFrmType==TK_RANGE?3:0)); + } + + if( pMWin->eFrmType!=TK_RANGE && pMWin->eStart==pMWin->eEnd && regStart ){ + int op = ((pMWin->eStart==TK_FOLLOWING) ? OP_Ge : OP_Le); + int addrGe = sqlite3VdbeAddOp3(v, op, regStart, 0, regEnd); + VdbeCoverageNeverNullIf(v, op==OP_Ge); /* NeverNull because bound */ + VdbeCoverageNeverNullIf(v, op==OP_Le); /* values previously checked */ + windowAggFinal(&s, 0); + sqlite3VdbeAddOp2(v, OP_Rewind, s.current.csr, 1); + VdbeCoverageNeverTaken(v); + windowReturnOneRow(&s); + sqlite3VdbeAddOp1(v, OP_ResetSorter, s.current.csr); + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblWhereEnd); + sqlite3VdbeJumpHere(v, addrGe); + } + if( pMWin->eStart==TK_FOLLOWING && pMWin->eFrmType!=TK_RANGE && regEnd ){ + assert( pMWin->eEnd==TK_FOLLOWING ); + sqlite3VdbeAddOp3(v, OP_Subtract, regStart, regEnd, regStart); + } + + if( pMWin->eStart!=TK_UNBOUNDED ){ + sqlite3VdbeAddOp2(v, OP_Rewind, s.start.csr, 1); + VdbeCoverageNeverTaken(v); + } + sqlite3VdbeAddOp2(v, OP_Rewind, s.current.csr, 1); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp2(v, OP_Rewind, s.end.csr, 1); + VdbeCoverageNeverTaken(v); + if( regPeer && pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Copy, regNewPeer, regPeer, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.start.reg, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.current.reg, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.end.reg, pOrderBy->nExpr-1); + } + + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblWhereEnd); + + sqlite3VdbeJumpHere(v, addrNe); + + /* Beginning of the block executed for the second and subsequent rows. */ + if( regPeer ){ + windowIfNewPeer(pParse, pOrderBy, regNewPeer, regPeer, lblWhereEnd); + } + if( pMWin->eStart==TK_FOLLOWING ){ + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eEnd!=TK_UNBOUNDED ){ + if( pMWin->eFrmType==TK_RANGE ){ + int lbl = sqlite3VdbeMakeLabel(pParse); + int addrNext = sqlite3VdbeCurrentAddr(v); + windowCodeRangeTest(&s, OP_Ge, s.current.csr, regEnd, s.end.csr, lbl); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNext); + sqlite3VdbeResolveLabel(v, lbl); + }else{ + windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + } + } + }else + if( pMWin->eEnd==TK_PRECEDING ){ + int bRPS = (pMWin->eStart==TK_PRECEDING && pMWin->eFrmType==TK_RANGE); + windowCodeOp(&s, WINDOW_AGGSTEP, regEnd, 0); + if( bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + if( !bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + }else{ + int addr = 0; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eEnd!=TK_UNBOUNDED ){ + if( pMWin->eFrmType==TK_RANGE ){ + int lbl = 0; + addr = sqlite3VdbeCurrentAddr(v); + if( regEnd ){ + lbl = sqlite3VdbeMakeLabel(pParse); + windowCodeRangeTest(&s, OP_Ge, s.current.csr, regEnd, s.end.csr, lbl); + } + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + if( regEnd ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); + sqlite3VdbeResolveLabel(v, lbl); + } + }else{ + if( regEnd ){ + addr = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0, 1); + VdbeCoverage(v); + } + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + if( regEnd ) sqlite3VdbeJumpHere(v, addr); + } + } + } + + /* End of the main input loop */ + sqlite3VdbeResolveLabel(v, lblWhereEnd); + sqlite3WhereEnd(pWInfo); + + /* Fall through */ + if( pMWin->pPartition ){ + addrInteger = sqlite3VdbeAddOp2(v, OP_Integer, 0, regFlushPart); + sqlite3VdbeJumpHere(v, addrGosubFlush); + } + + s.regRowid = 0; + addrEmpty = sqlite3VdbeAddOp1(v, OP_Rewind, csrWrite); + VdbeCoverage(v); + if( pMWin->eEnd==TK_PRECEDING ){ + int bRPS = (pMWin->eStart==TK_PRECEDING && pMWin->eFrmType==TK_RANGE); + windowCodeOp(&s, WINDOW_AGGSTEP, regEnd, 0); + if( bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + }else if( pMWin->eStart==TK_FOLLOWING ){ + int addrStart; + int addrBreak1; + int addrBreak2; + int addrBreak3; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eFrmType==TK_RANGE ){ + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + }else + if( pMWin->eEnd==TK_UNBOUNDED ){ + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regStart, 1); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, 0, 1); + }else{ + assert( pMWin->eEnd==TK_FOLLOWING ); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 1); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak2); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak3 = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak1); + sqlite3VdbeJumpHere(v, addrBreak3); + }else{ + int addrBreak; + int addrStart; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak); + } + sqlite3VdbeJumpHere(v, addrEmpty); + + sqlite3VdbeAddOp1(v, OP_ResetSorter, s.current.csr); + if( pMWin->pPartition ){ + if( pMWin->regStartRowid ){ + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regStartRowid); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pMWin->regEndRowid); + } + sqlite3VdbeChangeP1(v, addrInteger, sqlite3VdbeCurrentAddr(v)); + sqlite3VdbeAddOp1(v, OP_Return, regFlushPart); + } +} + +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/************** End of window.c **********************************************/ +/************** Begin file parse.c *******************************************/ +/* This file is automatically generated by Lemon from input grammar +** source file "parse.y". */ +/* +** 2001-09-15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains SQLite's SQL parser. +** +** The canonical source code to this file ("parse.y") is a Lemon grammar +** file that specifies the input grammar and actions to take while parsing. +** That input file is processed by Lemon to generate a C-language +** implementation of a parser for the given grammer. You might be reading +** this comment as part of the translated C-code. Edits should be made +** to the original parse.y sources. +*/ + +/* #include "sqliteInt.h" */ + +/* +** Disable all error recovery processing in the parser push-down +** automaton. +*/ +#define YYNOERRORRECOVERY 1 + +/* +** Make yytestcase() the same as testcase() +*/ +#define yytestcase(X) testcase(X) + +/* +** Indicate that sqlite3ParserFree() will never be called with a null +** pointer. +*/ +#define YYPARSEFREENEVERNULL 1 + +/* +** In the amalgamation, the parse.c file generated by lemon and the +** tokenize.c file are concatenated. In that case, sqlite3RunParser() +** has access to the the size of the yyParser object and so the parser +** engine can be allocated from stack. In that case, only the +** sqlite3ParserInit() and sqlite3ParserFinalize() routines are invoked +** and the sqlite3ParserAlloc() and sqlite3ParserFree() routines can be +** omitted. +*/ +#ifdef SQLITE_AMALGAMATION +# define sqlite3Parser_ENGINEALWAYSONSTACK 1 +#endif + +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlite3ParserAlloc(). The default is size_t. +*/ +#define YYMALLOCARGTYPE u64 + +/* +** An instance of the following structure describes the event of a +** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, +** TK_DELETE, or TK_INSTEAD. If the event is of the form +** +** UPDATE ON (a,b,c) +** +** Then the "b" IdList records the list "a,b,c". +*/ +struct TrigEvent { int a; IdList * b; }; + +struct FrameBound { int eType; Expr *pExpr; }; + +/* +** Disable lookaside memory allocation for objects that might be +** shared across database connections. +*/ +static void disableLookaside(Parse *pParse){ + sqlite3 *db = pParse->db; + pParse->disableLookaside++; + DisableLookaside; +} + +#if !defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) \ + && defined(SQLITE_UDL_CAPABLE_PARSER) +/* +** Issue an error message if an ORDER BY or LIMIT clause occurs on an +** UPDATE or DELETE statement. +*/ +static void updateDeleteLimitError( + Parse *pParse, + ExprList *pOrderBy, + Expr *pLimit +){ + if( pOrderBy ){ + sqlite3ErrorMsg(pParse, "syntax error near \"ORDER BY\""); + }else{ + sqlite3ErrorMsg(pParse, "syntax error near \"LIMIT\""); + } + sqlite3ExprListDelete(pParse->db, pOrderBy); + sqlite3ExprDelete(pParse->db, pLimit); +} +#endif /* SQLITE_ENABLE_UPDATE_DELETE_LIMIT */ + + + /* + ** For a compound SELECT statement, make sure p->pPrior->pNext==p for + ** all elements in the list. And make sure list length does not exceed + ** SQLITE_LIMIT_COMPOUND_SELECT. + */ + static void parserDoubleLinkSelect(Parse *pParse, Select *p){ + assert( p!=0 ); + if( p->pPrior ){ + Select *pNext = 0, *pLoop = p; + int mxSelect, cnt = 1; + while(1){ + pLoop->pNext = pNext; + pLoop->selFlags |= SF_Compound; + pNext = pLoop; + pLoop = pLoop->pPrior; + if( pLoop==0 ) break; + cnt++; + if( pLoop->pOrderBy || pLoop->pLimit ){ + sqlite3ErrorMsg(pParse,"%s clause should come after %s not before", + pLoop->pOrderBy!=0 ? "ORDER BY" : "LIMIT", + sqlite3SelectOpName(pNext->op)); + break; + } + } + if( (p->selFlags & SF_MultiValue)==0 && + (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && + cnt>mxSelect + ){ + sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); + } + } + } + + /* Attach a With object describing the WITH clause to a Select + ** object describing the query for which the WITH clause is a prefix. + */ + static Select *attachWithToSelect(Parse *pParse, Select *pSelect, With *pWith){ + if( pSelect ){ + pSelect->pWith = pWith; + parserDoubleLinkSelect(pParse, pSelect); + }else{ + sqlite3WithDelete(pParse->db, pWith); + } + return pSelect; + } + + + /* Construct a new Expr object from a single identifier. Use the + ** new Expr to populate pOut. Set the span of pOut to be the identifier + ** that created the expression. + */ + static Expr *tokenExpr(Parse *pParse, int op, Token t){ + Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1); + if( p ){ + /* memset(p, 0, sizeof(Expr)); */ + p->op = (u8)op; + p->affExpr = 0; + p->flags = EP_Leaf; + ExprClearVVAProperties(p); + p->iAgg = -1; + p->pLeft = p->pRight = 0; + p->x.pList = 0; + p->pAggInfo = 0; + p->y.pTab = 0; + p->op2 = 0; + p->iTable = 0; + p->iColumn = 0; + p->u.zToken = (char*)&p[1]; + memcpy(p->u.zToken, t.z, t.n); + p->u.zToken[t.n] = 0; + if( sqlite3Isquote(p->u.zToken[0]) ){ + sqlite3DequoteExpr(p); + } +#if SQLITE_MAX_EXPR_DEPTH>0 + p->nHeight = 1; +#endif + if( IN_RENAME_OBJECT ){ + return (Expr*)sqlite3RenameTokenMap(pParse, (void*)p, &t); + } + } + return p; + } + + + /* A routine to convert a binary TK_IS or TK_ISNOT expression into a + ** unary TK_ISNULL or TK_NOTNULL expression. */ + static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ + sqlite3 *db = pParse->db; + if( pA && pY && pY->op==TK_NULL && !IN_RENAME_OBJECT ){ + pA->op = (u8)op; + sqlite3ExprDelete(db, pA->pRight); + pA->pRight = 0; + } + } + + /* Add a single new term to an ExprList that is used to store a + ** list of identifiers. Report an error if the ID list contains + ** a COLLATE clause or an ASC or DESC keyword, except ignore the + ** error while parsing a legacy schema. + */ + static ExprList *parserAddExprIdListTerm( + Parse *pParse, + ExprList *pPrior, + Token *pIdToken, + int hasCollate, + int sortOrder + ){ + ExprList *p = sqlite3ExprListAppend(pParse, pPrior, 0); + if( (hasCollate || sortOrder!=SQLITE_SO_UNDEFINED) + && pParse->db->init.busy==0 + ){ + sqlite3ErrorMsg(pParse, "syntax error after column name \"%.*s\"", + pIdToken->n, pIdToken->z); + } + sqlite3ExprListSetName(pParse, p, pIdToken, 1); + return p; + } + +#if TK_SPAN>255 +# error too many tokens in the grammar +#endif +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols. +***************** Begin token definitions *************************************/ +#ifndef TK_SEMI +#define TK_SEMI 1 +#define TK_EXPLAIN 2 +#define TK_QUERY 3 +#define TK_PLAN 4 +#define TK_BEGIN 5 +#define TK_TRANSACTION 6 +#define TK_DEFERRED 7 +#define TK_IMMEDIATE 8 +#define TK_EXCLUSIVE 9 +#define TK_COMMIT 10 +#define TK_END 11 +#define TK_ROLLBACK 12 +#define TK_SAVEPOINT 13 +#define TK_RELEASE 14 +#define TK_TO 15 +#define TK_TABLE 16 +#define TK_CREATE 17 +#define TK_IF 18 +#define TK_NOT 19 +#define TK_EXISTS 20 +#define TK_TEMP 21 +#define TK_LP 22 +#define TK_RP 23 +#define TK_AS 24 +#define TK_COMMA 25 +#define TK_WITHOUT 26 +#define TK_ABORT 27 +#define TK_ACTION 28 +#define TK_AFTER 29 +#define TK_ANALYZE 30 +#define TK_ASC 31 +#define TK_ATTACH 32 +#define TK_BEFORE 33 +#define TK_BY 34 +#define TK_CASCADE 35 +#define TK_CAST 36 +#define TK_CONFLICT 37 +#define TK_DATABASE 38 +#define TK_DESC 39 +#define TK_DETACH 40 +#define TK_EACH 41 +#define TK_FAIL 42 +#define TK_OR 43 +#define TK_AND 44 +#define TK_IS 45 +#define TK_MATCH 46 +#define TK_LIKE_KW 47 +#define TK_BETWEEN 48 +#define TK_IN 49 +#define TK_ISNULL 50 +#define TK_NOTNULL 51 +#define TK_NE 52 +#define TK_EQ 53 +#define TK_GT 54 +#define TK_LE 55 +#define TK_LT 56 +#define TK_GE 57 +#define TK_ESCAPE 58 +#define TK_ID 59 +#define TK_COLUMNKW 60 +#define TK_DO 61 +#define TK_FOR 62 +#define TK_IGNORE 63 +#define TK_INITIALLY 64 +#define TK_INSTEAD 65 +#define TK_NO 66 +#define TK_KEY 67 +#define TK_OF 68 +#define TK_OFFSET 69 +#define TK_PRAGMA 70 +#define TK_RAISE 71 +#define TK_RECURSIVE 72 +#define TK_REPLACE 73 +#define TK_RESTRICT 74 +#define TK_ROW 75 +#define TK_ROWS 76 +#define TK_TRIGGER 77 +#define TK_VACUUM 78 +#define TK_VIEW 79 +#define TK_VIRTUAL 80 +#define TK_WITH 81 +#define TK_NULLS 82 +#define TK_FIRST 83 +#define TK_LAST 84 +#define TK_CURRENT 85 +#define TK_FOLLOWING 86 +#define TK_PARTITION 87 +#define TK_PRECEDING 88 +#define TK_RANGE 89 +#define TK_UNBOUNDED 90 +#define TK_EXCLUDE 91 +#define TK_GROUPS 92 +#define TK_OTHERS 93 +#define TK_TIES 94 +#define TK_GENERATED 95 +#define TK_ALWAYS 96 +#define TK_MATERIALIZED 97 +#define TK_REINDEX 98 +#define TK_RENAME 99 +#define TK_CTIME_KW 100 +#define TK_ANY 101 +#define TK_BITAND 102 +#define TK_BITOR 103 +#define TK_LSHIFT 104 +#define TK_RSHIFT 105 +#define TK_PLUS 106 +#define TK_MINUS 107 +#define TK_STAR 108 +#define TK_SLASH 109 +#define TK_REM 110 +#define TK_CONCAT 111 +#define TK_COLLATE 112 +#define TK_BITNOT 113 +#define TK_ON 114 +#define TK_INDEXED 115 +#define TK_STRING 116 +#define TK_JOIN_KW 117 +#define TK_CONSTRAINT 118 +#define TK_DEFAULT 119 +#define TK_NULL 120 +#define TK_PRIMARY 121 +#define TK_UNIQUE 122 +#define TK_CHECK 123 +#define TK_REFERENCES 124 +#define TK_AUTOINCR 125 +#define TK_INSERT 126 +#define TK_DELETE 127 +#define TK_UPDATE 128 +#define TK_SET 129 +#define TK_DEFERRABLE 130 +#define TK_FOREIGN 131 +#define TK_DROP 132 +#define TK_UNION 133 +#define TK_ALL 134 +#define TK_EXCEPT 135 +#define TK_INTERSECT 136 +#define TK_SELECT 137 +#define TK_VALUES 138 +#define TK_DISTINCT 139 +#define TK_DOT 140 +#define TK_FROM 141 +#define TK_JOIN 142 +#define TK_USING 143 +#define TK_ORDER 144 +#define TK_GROUP 145 +#define TK_HAVING 146 +#define TK_LIMIT 147 +#define TK_WHERE 148 +#define TK_RETURNING 149 +#define TK_INTO 150 +#define TK_NOTHING 151 +#define TK_FLOAT 152 +#define TK_BLOB 153 +#define TK_INTEGER 154 +#define TK_VARIABLE 155 +#define TK_CASE 156 +#define TK_WHEN 157 +#define TK_THEN 158 +#define TK_ELSE 159 +#define TK_INDEX 160 +#define TK_ALTER 161 +#define TK_ADD 162 +#define TK_WINDOW 163 +#define TK_OVER 164 +#define TK_FILTER 165 +#define TK_COLUMN 166 +#define TK_AGG_FUNCTION 167 +#define TK_AGG_COLUMN 168 +#define TK_TRUEFALSE 169 +#define TK_ISNOT 170 +#define TK_FUNCTION 171 +#define TK_UMINUS 172 +#define TK_UPLUS 173 +#define TK_TRUTH 174 +#define TK_REGISTER 175 +#define TK_VECTOR 176 +#define TK_SELECT_COLUMN 177 +#define TK_IF_NULL_ROW 178 +#define TK_ASTERISK 179 +#define TK_SPAN 180 +#define TK_ERROR 181 +#define TK_SPACE 182 +#define TK_ILLEGAL 183 +#endif +/**************** End token definitions ***************************************/ + +/* The next sections is a series of control #defines. +** various aspects of the generated parser. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. +** YYFALLBACK If defined, this indicates that one or more tokens +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** sqlite3ParserTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. +** This is typically a union of many types, one of +** which is sqlite3ParserTOKENTYPE. The entry in the union +** for terminal symbols is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument +** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument +** sqlite3ParserARG_PARAM Code to pass %extra_argument as a subroutine parameter +** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser +** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser +** sqlite3ParserCTX_* As sqlite3ParserARG_ except for %extra_context +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYNTOKEN Number of terminal symbols +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op +** YY_MIN_REDUCE Minimum value for reduce actions +** YY_MAX_REDUCE Maximum value for reduce actions +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ +#define YYCODETYPE unsigned short int +#define YYNOCODE 318 +#define YYACTIONTYPE unsigned short int +#define YYWILDCARD 101 +#define sqlite3ParserTOKENTYPE Token +typedef union { + int yyinit; + sqlite3ParserTOKENTYPE yy0; + With* yy43; + u32 yy51; + int yy64; + struct FrameBound yy81; + struct {int value; int mask;} yy83; + TriggerStep* yy95; + Upsert* yy138; + IdList* yy240; + Cte* yy255; + Select* yy303; + Window* yy375; + u8 yy534; + ExprList* yy562; + struct TrigEvent yy570; + const char* yy600; + SrcList* yy607; + Expr* yy626; +} YYMINORTYPE; +#ifndef YYSTACKDEPTH +#define YYSTACKDEPTH 100 +#endif +#define sqlite3ParserARG_SDECL +#define sqlite3ParserARG_PDECL +#define sqlite3ParserARG_PARAM +#define sqlite3ParserARG_FETCH +#define sqlite3ParserARG_STORE +#define sqlite3ParserCTX_SDECL Parse *pParse; +#define sqlite3ParserCTX_PDECL ,Parse *pParse +#define sqlite3ParserCTX_PARAM ,pParse +#define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; +#define sqlite3ParserCTX_STORE yypParser->pParse=pParse; +#define YYFALLBACK 1 +#define YYNSTATE 572 +#define YYNRULE 401 +#define YYNRULE_WITH_ACTION 339 +#define YYNTOKEN 184 +#define YY_MAX_SHIFT 571 +#define YY_MIN_SHIFTREDUCE 829 +#define YY_MAX_SHIFTREDUCE 1229 +#define YY_ERROR_ACTION 1230 +#define YY_ACCEPT_ACTION 1231 +#define YY_NO_ACTION 1232 +#define YY_MIN_REDUCE 1233 +#define YY_MAX_REDUCE 1633 +/************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) + +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. +** +** N == YY_ERROR_ACTION A syntax error has occurred. +** +** N == YY_ACCEPT_ACTION The parser accepts its input. +** +** N == YY_NO_ACTION No such action. Denotes unused +** slots in the yy_action[] table. +** +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE +** +** The action table is constructed as a single large table named yy_action[]. +** Given state S and lookahead X, the action is computed as either: +** +** (A) N = yy_action[ yy_shift_ofst[S] + X ] +** (B) N = yy_default[S] +** +** The (A) formula is preferred. The B formula is used instead if +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. +** +** The formulas above are for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the yy_reduce_ofst[] array is used in place of +** the yy_shift_ofst[] array. +** +** The following are the tables generated in this section: +** +** yy_action[] A single table containing all actions. +** yy_lookahead[] A table containing the lookahead for each entry in +** yy_action. Used to detect hash collisions. +** yy_shift_ofst[] For each state, the offset into yy_action for +** shifting terminals. +** yy_reduce_ofst[] For each state, the offset into yy_action for +** shifting non-terminals after a reduce. +** yy_default[] Default action for each state. +** +*********** Begin parsing tables **********************************************/ +#define YY_ACTTAB_COUNT (2037) +static const YYACTIONTYPE yy_action[] = { + /* 0 */ 564, 115, 112, 220, 169, 199, 115, 112, 220, 564, + /* 10 */ 375, 1266, 564, 376, 564, 270, 1309, 1309, 406, 407, + /* 20 */ 1084, 199, 1513, 41, 41, 515, 489, 521, 558, 558, + /* 30 */ 558, 965, 41, 41, 395, 41, 41, 51, 51, 966, + /* 40 */ 296, 1269, 296, 122, 123, 113, 1207, 1207, 1041, 1044, + /* 50 */ 1034, 1034, 120, 120, 121, 121, 121, 121, 564, 407, + /* 60 */ 275, 275, 275, 275, 1268, 115, 112, 220, 115, 112, + /* 70 */ 220, 1512, 846, 561, 516, 561, 115, 112, 220, 250, + /* 80 */ 217, 71, 71, 122, 123, 113, 1207, 1207, 1041, 1044, + /* 90 */ 1034, 1034, 120, 120, 121, 121, 121, 121, 440, 440, + /* 100 */ 440, 1149, 119, 119, 119, 119, 118, 118, 117, 117, + /* 110 */ 117, 116, 442, 1183, 1149, 116, 442, 1149, 546, 513, + /* 120 */ 1548, 1554, 374, 442, 6, 1183, 1154, 522, 1154, 407, + /* 130 */ 1556, 461, 373, 1554, 535, 99, 463, 332, 121, 121, + /* 140 */ 121, 121, 119, 119, 119, 119, 118, 118, 117, 117, + /* 150 */ 117, 116, 442, 122, 123, 113, 1207, 1207, 1041, 1044, + /* 160 */ 1034, 1034, 120, 120, 121, 121, 121, 121, 1257, 1183, + /* 170 */ 1184, 1185, 243, 1064, 564, 502, 499, 498, 567, 124, + /* 180 */ 567, 1183, 1184, 1185, 474, 497, 119, 119, 119, 119, + /* 190 */ 118, 118, 117, 117, 117, 116, 442, 70, 70, 407, + /* 200 */ 121, 121, 121, 121, 114, 117, 117, 117, 116, 442, + /* 210 */ 1409, 1469, 119, 119, 119, 119, 118, 118, 117, 117, + /* 220 */ 117, 116, 442, 122, 123, 113, 1207, 1207, 1041, 1044, + /* 230 */ 1034, 1034, 120, 120, 121, 121, 121, 121, 407, 1031, + /* 240 */ 1031, 1042, 1045, 81, 382, 541, 378, 80, 119, 119, + /* 250 */ 119, 119, 118, 118, 117, 117, 117, 116, 442, 381, + /* 260 */ 463, 332, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, + /* 270 */ 1034, 120, 120, 121, 121, 121, 121, 262, 215, 512, + /* 280 */ 1424, 422, 119, 119, 119, 119, 118, 118, 117, 117, + /* 290 */ 117, 116, 442, 1231, 1, 1, 571, 2, 1235, 1573, + /* 300 */ 571, 2, 1235, 307, 1149, 141, 1600, 307, 407, 141, + /* 310 */ 1183, 361, 1317, 1035, 866, 531, 1317, 1149, 359, 1567, + /* 320 */ 1149, 119, 119, 119, 119, 118, 118, 117, 117, 117, + /* 330 */ 116, 442, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, + /* 340 */ 1034, 120, 120, 121, 121, 121, 121, 275, 275, 1001, + /* 350 */ 426, 275, 275, 1128, 1627, 1021, 1627, 137, 542, 1541, + /* 360 */ 561, 272, 950, 950, 561, 1423, 1183, 1184, 1185, 1594, + /* 370 */ 866, 1012, 530, 315, 231, 1011, 468, 1276, 231, 119, + /* 380 */ 119, 119, 119, 118, 118, 117, 117, 117, 116, 442, + /* 390 */ 1570, 119, 119, 119, 119, 118, 118, 117, 117, 117, + /* 400 */ 116, 442, 330, 359, 1567, 564, 446, 1011, 1011, 1013, + /* 410 */ 446, 207, 564, 306, 555, 407, 363, 1021, 363, 346, + /* 420 */ 184, 118, 118, 117, 117, 117, 116, 442, 71, 71, + /* 430 */ 439, 438, 1126, 1012, 472, 71, 71, 1011, 205, 122, + /* 440 */ 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, + /* 450 */ 121, 121, 121, 121, 219, 219, 472, 1183, 407, 570, + /* 460 */ 1183, 1235, 503, 1477, 149, 546, 307, 489, 141, 1011, + /* 470 */ 1011, 1013, 546, 140, 545, 1317, 1214, 191, 1214, 950, + /* 480 */ 950, 514, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, + /* 490 */ 1034, 120, 120, 121, 121, 121, 121, 563, 119, 119, + /* 500 */ 119, 119, 118, 118, 117, 117, 117, 116, 442, 283, + /* 510 */ 275, 275, 415, 1183, 1184, 1185, 1183, 1184, 1185, 372, + /* 520 */ 1183, 243, 344, 561, 502, 499, 498, 1539, 407, 1540, + /* 530 */ 1183, 288, 870, 143, 497, 1549, 185, 231, 9, 6, + /* 540 */ 253, 119, 119, 119, 119, 118, 118, 117, 117, 117, + /* 550 */ 116, 442, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, + /* 560 */ 1034, 120, 120, 121, 121, 121, 121, 407, 137, 446, + /* 570 */ 447, 863, 169, 1183, 397, 1204, 1183, 1184, 1185, 931, + /* 580 */ 526, 1001, 98, 339, 564, 342, 1183, 1184, 1185, 306, + /* 590 */ 555, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, + /* 600 */ 120, 120, 121, 121, 121, 121, 452, 71, 71, 275, + /* 610 */ 275, 119, 119, 119, 119, 118, 118, 117, 117, 117, + /* 620 */ 116, 442, 561, 417, 306, 555, 1183, 1307, 1307, 1183, + /* 630 */ 1184, 1185, 1204, 1149, 330, 458, 318, 407, 363, 470, + /* 640 */ 431, 1167, 32, 541, 527, 350, 1149, 1629, 393, 1149, + /* 650 */ 119, 119, 119, 119, 118, 118, 117, 117, 117, 116, + /* 660 */ 442, 122, 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, + /* 670 */ 120, 120, 121, 121, 121, 121, 407, 199, 472, 1183, + /* 680 */ 1022, 472, 1183, 1184, 1185, 386, 151, 539, 1548, 277, + /* 690 */ 400, 137, 6, 317, 5, 564, 562, 3, 920, 920, + /* 700 */ 122, 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, + /* 710 */ 120, 121, 121, 121, 121, 411, 505, 83, 71, 71, + /* 720 */ 119, 119, 119, 119, 118, 118, 117, 117, 117, 116, + /* 730 */ 442, 1183, 218, 428, 1183, 1183, 1184, 1185, 363, 261, + /* 740 */ 278, 358, 508, 353, 507, 248, 407, 306, 555, 1539, + /* 750 */ 1006, 349, 363, 291, 489, 302, 293, 1542, 281, 119, + /* 760 */ 119, 119, 119, 118, 118, 117, 117, 117, 116, 442, + /* 770 */ 122, 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, + /* 780 */ 120, 121, 121, 121, 121, 407, 148, 1183, 1184, 1185, + /* 790 */ 1183, 1184, 1185, 275, 275, 1304, 1257, 1283, 483, 1476, + /* 800 */ 150, 489, 480, 564, 1187, 1304, 561, 1587, 1255, 122, + /* 810 */ 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, + /* 820 */ 121, 121, 121, 121, 564, 886, 13, 13, 520, 119, + /* 830 */ 119, 119, 119, 118, 118, 117, 117, 117, 116, 442, + /* 840 */ 1183, 420, 417, 564, 269, 269, 1316, 13, 13, 1539, + /* 850 */ 1546, 16, 16, 322, 6, 407, 506, 561, 1089, 1089, + /* 860 */ 486, 1187, 425, 1539, 887, 292, 71, 71, 119, 119, + /* 870 */ 119, 119, 118, 118, 117, 117, 117, 116, 442, 122, + /* 880 */ 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, + /* 890 */ 121, 121, 121, 121, 564, 12, 1183, 1184, 1185, 407, + /* 900 */ 275, 275, 451, 303, 834, 835, 836, 417, 489, 276, + /* 910 */ 276, 1547, 284, 561, 319, 6, 321, 71, 71, 429, + /* 920 */ 451, 450, 561, 952, 101, 113, 1207, 1207, 1041, 1044, + /* 930 */ 1034, 1034, 120, 120, 121, 121, 121, 121, 119, 119, + /* 940 */ 119, 119, 118, 118, 117, 117, 117, 116, 442, 1105, + /* 950 */ 1183, 489, 564, 1312, 437, 455, 478, 564, 246, 245, + /* 960 */ 244, 1409, 1545, 547, 1106, 405, 6, 1544, 196, 1258, + /* 970 */ 413, 6, 105, 462, 103, 71, 71, 286, 564, 1107, + /* 980 */ 13, 13, 119, 119, 119, 119, 118, 118, 117, 117, + /* 990 */ 117, 116, 442, 451, 104, 427, 337, 320, 275, 275, + /* 1000 */ 906, 13, 13, 564, 1482, 1105, 1183, 1184, 1185, 126, + /* 1010 */ 907, 561, 546, 564, 407, 478, 295, 1321, 253, 200, + /* 1020 */ 1106, 548, 1482, 1484, 280, 1409, 55, 55, 1287, 561, + /* 1030 */ 478, 380, 423, 951, 407, 1107, 71, 71, 122, 123, + /* 1040 */ 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, 121, + /* 1050 */ 121, 121, 121, 1204, 407, 287, 552, 309, 122, 123, + /* 1060 */ 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, 121, + /* 1070 */ 121, 121, 121, 441, 1128, 1628, 146, 1628, 122, 111, + /* 1080 */ 113, 1207, 1207, 1041, 1044, 1034, 1034, 120, 120, 121, + /* 1090 */ 121, 121, 121, 404, 403, 1482, 424, 119, 119, 119, + /* 1100 */ 119, 118, 118, 117, 117, 117, 116, 442, 1183, 564, + /* 1110 */ 1204, 544, 1086, 858, 329, 361, 1086, 119, 119, 119, + /* 1120 */ 119, 118, 118, 117, 117, 117, 116, 442, 564, 294, + /* 1130 */ 144, 523, 56, 56, 224, 564, 510, 119, 119, 119, + /* 1140 */ 119, 118, 118, 117, 117, 117, 116, 442, 484, 1409, + /* 1150 */ 537, 15, 15, 1126, 434, 439, 438, 407, 13, 13, + /* 1160 */ 1523, 12, 926, 1211, 1183, 1184, 1185, 925, 1213, 536, + /* 1170 */ 858, 557, 413, 193, 1525, 494, 1212, 448, 1160, 1222, + /* 1180 */ 1183, 564, 123, 113, 1207, 1207, 1041, 1044, 1034, 1034, + /* 1190 */ 120, 120, 121, 121, 121, 121, 1521, 1149, 564, 965, + /* 1200 */ 564, 1214, 247, 1214, 13, 13, 1409, 966, 538, 564, + /* 1210 */ 1149, 108, 556, 1149, 4, 310, 392, 1227, 17, 194, + /* 1220 */ 485, 43, 43, 57, 57, 306, 555, 524, 559, 1160, + /* 1230 */ 464, 564, 44, 44, 392, 1127, 1183, 1184, 1185, 479, + /* 1240 */ 119, 119, 119, 119, 118, 118, 117, 117, 117, 116, + /* 1250 */ 442, 443, 564, 327, 13, 13, 564, 418, 1315, 414, + /* 1260 */ 171, 564, 311, 553, 213, 529, 1253, 564, 517, 543, + /* 1270 */ 412, 108, 556, 137, 4, 58, 58, 435, 314, 59, + /* 1280 */ 59, 274, 217, 549, 60, 60, 349, 476, 559, 1353, + /* 1290 */ 61, 61, 1021, 275, 275, 1228, 213, 564, 106, 106, + /* 1300 */ 8, 275, 275, 275, 275, 107, 561, 443, 566, 565, + /* 1310 */ 564, 443, 1011, 1228, 561, 564, 561, 564, 275, 275, + /* 1320 */ 62, 62, 1352, 553, 247, 456, 564, 98, 110, 306, + /* 1330 */ 555, 561, 564, 45, 45, 405, 1203, 533, 46, 46, + /* 1340 */ 47, 47, 532, 465, 1011, 1011, 1013, 1014, 27, 49, + /* 1350 */ 49, 564, 1021, 405, 469, 50, 50, 564, 106, 106, + /* 1360 */ 305, 564, 84, 204, 405, 107, 564, 443, 566, 565, + /* 1370 */ 405, 564, 1011, 564, 63, 63, 564, 1599, 564, 895, + /* 1380 */ 64, 64, 457, 477, 65, 65, 147, 96, 38, 14, + /* 1390 */ 14, 1528, 412, 564, 66, 66, 128, 128, 926, 67, + /* 1400 */ 67, 52, 52, 925, 1011, 1011, 1013, 1014, 27, 1572, + /* 1410 */ 1171, 445, 208, 1123, 279, 394, 68, 68, 228, 390, + /* 1420 */ 390, 389, 264, 387, 1171, 445, 843, 877, 279, 108, + /* 1430 */ 556, 453, 4, 390, 390, 389, 264, 387, 564, 225, + /* 1440 */ 843, 313, 328, 1003, 98, 252, 559, 544, 471, 312, + /* 1450 */ 252, 564, 208, 225, 564, 313, 473, 30, 252, 279, + /* 1460 */ 466, 69, 69, 312, 390, 390, 389, 264, 387, 443, + /* 1470 */ 333, 843, 98, 564, 53, 53, 323, 157, 157, 227, + /* 1480 */ 495, 553, 249, 289, 225, 564, 313, 162, 31, 1501, + /* 1490 */ 135, 564, 1500, 227, 312, 533, 158, 158, 885, 884, + /* 1500 */ 534, 162, 873, 301, 135, 564, 481, 226, 76, 76, + /* 1510 */ 1021, 347, 1071, 98, 54, 54, 106, 106, 1067, 564, + /* 1520 */ 249, 226, 519, 107, 227, 443, 566, 565, 72, 72, + /* 1530 */ 1011, 334, 162, 564, 230, 135, 108, 556, 959, 4, + /* 1540 */ 252, 408, 129, 129, 564, 1349, 306, 555, 564, 923, + /* 1550 */ 564, 110, 226, 559, 564, 408, 73, 73, 564, 873, + /* 1560 */ 306, 555, 1011, 1011, 1013, 1014, 27, 130, 130, 1071, + /* 1570 */ 449, 131, 131, 127, 127, 357, 443, 156, 156, 892, + /* 1580 */ 893, 155, 155, 338, 449, 356, 408, 564, 553, 968, + /* 1590 */ 969, 306, 555, 1015, 341, 564, 108, 556, 564, 4, + /* 1600 */ 1132, 1286, 533, 564, 856, 343, 145, 532, 345, 1300, + /* 1610 */ 136, 136, 1083, 559, 1083, 449, 564, 1021, 134, 134, + /* 1620 */ 1284, 132, 132, 106, 106, 1285, 133, 133, 564, 352, + /* 1630 */ 107, 564, 443, 566, 565, 1340, 443, 1011, 362, 75, + /* 1640 */ 75, 1082, 564, 1082, 564, 924, 1561, 110, 553, 551, + /* 1650 */ 1015, 77, 77, 1361, 74, 74, 1408, 1336, 1347, 550, + /* 1660 */ 1414, 1265, 1256, 1244, 1243, 42, 42, 48, 48, 1011, + /* 1670 */ 1011, 1013, 1014, 27, 1245, 1580, 490, 1021, 267, 202, + /* 1680 */ 1333, 365, 11, 106, 106, 930, 367, 210, 369, 391, + /* 1690 */ 107, 1395, 443, 566, 565, 223, 1390, 1011, 500, 454, + /* 1700 */ 282, 1400, 285, 108, 556, 214, 4, 325, 1383, 1283, + /* 1710 */ 475, 355, 1473, 1583, 1472, 1399, 371, 1222, 326, 398, + /* 1720 */ 559, 290, 331, 197, 100, 556, 209, 4, 198, 1011, + /* 1730 */ 1011, 1013, 1014, 27, 385, 256, 1520, 1518, 554, 1219, + /* 1740 */ 416, 559, 83, 443, 173, 206, 182, 221, 459, 167, + /* 1750 */ 177, 460, 175, 493, 233, 553, 79, 178, 1396, 179, + /* 1760 */ 35, 180, 96, 1402, 443, 396, 36, 467, 1478, 1401, + /* 1770 */ 482, 237, 1404, 399, 82, 186, 553, 1467, 89, 488, + /* 1780 */ 190, 268, 239, 491, 1021, 340, 240, 401, 1246, 1489, + /* 1790 */ 106, 106, 336, 509, 1294, 241, 1303, 107, 430, 443, + /* 1800 */ 566, 565, 1302, 91, 1011, 1021, 1598, 1301, 1273, 215, + /* 1810 */ 1597, 106, 106, 402, 877, 432, 354, 1272, 107, 1271, + /* 1820 */ 443, 566, 565, 1596, 1566, 1011, 1293, 433, 518, 299, + /* 1830 */ 300, 360, 95, 525, 1344, 364, 1011, 1011, 1013, 1014, + /* 1840 */ 27, 254, 255, 1552, 436, 1551, 125, 544, 10, 379, + /* 1850 */ 1326, 1453, 102, 97, 1345, 528, 304, 1011, 1011, 1013, + /* 1860 */ 1014, 27, 366, 377, 1343, 1342, 368, 370, 1325, 384, + /* 1870 */ 201, 383, 34, 1368, 1367, 568, 1177, 266, 263, 265, + /* 1880 */ 1505, 159, 569, 1241, 1236, 1506, 160, 142, 1504, 1503, + /* 1890 */ 297, 211, 830, 161, 212, 78, 444, 203, 308, 222, + /* 1900 */ 1081, 139, 1079, 316, 174, 163, 1203, 229, 176, 909, + /* 1910 */ 324, 232, 1095, 181, 409, 410, 172, 164, 165, 419, + /* 1920 */ 183, 85, 86, 421, 166, 87, 88, 1098, 1094, 234, + /* 1930 */ 235, 152, 18, 236, 335, 1087, 1216, 252, 187, 487, + /* 1940 */ 238, 188, 37, 845, 492, 356, 242, 496, 351, 501, + /* 1950 */ 189, 90, 19, 504, 348, 20, 875, 92, 298, 168, + /* 1960 */ 888, 153, 93, 511, 94, 1165, 154, 1047, 1134, 39, + /* 1970 */ 216, 1133, 271, 273, 958, 192, 953, 110, 1151, 1155, + /* 1980 */ 251, 7, 21, 1159, 1139, 22, 1153, 33, 23, 24, + /* 1990 */ 25, 540, 1158, 195, 98, 1062, 26, 1048, 1046, 1050, + /* 2000 */ 1104, 1051, 1103, 257, 258, 28, 40, 1173, 1016, 857, + /* 2010 */ 109, 29, 560, 388, 138, 1172, 259, 170, 260, 1232, + /* 2020 */ 1232, 919, 1232, 1232, 1232, 1232, 1232, 1232, 1232, 1232, + /* 2030 */ 1232, 1232, 1589, 1232, 1232, 1232, 1588, +}; +static const YYCODETYPE yy_lookahead[] = { + /* 0 */ 192, 273, 274, 275, 192, 192, 273, 274, 275, 192, + /* 10 */ 218, 215, 192, 218, 192, 212, 234, 235, 205, 19, + /* 20 */ 11, 192, 294, 215, 216, 203, 192, 203, 209, 210, + /* 30 */ 211, 31, 215, 216, 205, 215, 216, 215, 216, 39, + /* 40 */ 227, 215, 229, 43, 44, 45, 46, 47, 48, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 192, 19, + /* 60 */ 238, 239, 238, 239, 215, 273, 274, 275, 273, 274, + /* 70 */ 275, 237, 21, 251, 252, 251, 273, 274, 275, 255, + /* 80 */ 256, 215, 216, 43, 44, 45, 46, 47, 48, 49, + /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 209, 210, + /* 100 */ 211, 76, 102, 103, 104, 105, 106, 107, 108, 109, + /* 110 */ 110, 111, 112, 59, 89, 111, 112, 92, 252, 307, + /* 120 */ 308, 313, 314, 112, 312, 59, 86, 261, 88, 19, + /* 130 */ 313, 80, 315, 313, 314, 25, 127, 128, 54, 55, + /* 140 */ 56, 57, 102, 103, 104, 105, 106, 107, 108, 109, + /* 150 */ 110, 111, 112, 43, 44, 45, 46, 47, 48, 49, + /* 160 */ 50, 51, 52, 53, 54, 55, 56, 57, 192, 115, + /* 170 */ 116, 117, 118, 122, 192, 121, 122, 123, 202, 69, + /* 180 */ 204, 115, 116, 117, 192, 131, 102, 103, 104, 105, + /* 190 */ 106, 107, 108, 109, 110, 111, 112, 215, 216, 19, + /* 200 */ 54, 55, 56, 57, 58, 108, 109, 110, 111, 112, + /* 210 */ 192, 160, 102, 103, 104, 105, 106, 107, 108, 109, + /* 220 */ 110, 111, 112, 43, 44, 45, 46, 47, 48, 49, + /* 230 */ 50, 51, 52, 53, 54, 55, 56, 57, 19, 46, + /* 240 */ 47, 48, 49, 24, 248, 192, 250, 67, 102, 103, + /* 250 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 277, + /* 260 */ 127, 128, 43, 44, 45, 46, 47, 48, 49, 50, + /* 270 */ 51, 52, 53, 54, 55, 56, 57, 26, 164, 165, + /* 280 */ 272, 263, 102, 103, 104, 105, 106, 107, 108, 109, + /* 290 */ 110, 111, 112, 184, 185, 186, 187, 188, 189, 186, + /* 300 */ 187, 188, 189, 194, 76, 196, 229, 194, 19, 196, + /* 310 */ 59, 192, 203, 120, 59, 87, 203, 89, 310, 311, + /* 320 */ 92, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 330 */ 111, 112, 43, 44, 45, 46, 47, 48, 49, 50, + /* 340 */ 51, 52, 53, 54, 55, 56, 57, 238, 239, 73, + /* 350 */ 231, 238, 239, 22, 23, 100, 25, 81, 305, 306, + /* 360 */ 251, 23, 25, 25, 251, 272, 115, 116, 117, 214, + /* 370 */ 115, 116, 144, 192, 265, 120, 114, 222, 265, 102, + /* 380 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 390 */ 192, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 400 */ 111, 112, 126, 310, 311, 192, 297, 152, 153, 154, + /* 410 */ 297, 149, 192, 137, 138, 19, 192, 100, 192, 23, + /* 420 */ 22, 106, 107, 108, 109, 110, 111, 112, 215, 216, + /* 430 */ 106, 107, 101, 116, 192, 215, 216, 120, 149, 43, + /* 440 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 450 */ 54, 55, 56, 57, 117, 117, 192, 59, 19, 187, + /* 460 */ 59, 189, 23, 282, 240, 252, 194, 192, 196, 152, + /* 470 */ 153, 154, 252, 72, 261, 203, 152, 25, 154, 142, + /* 480 */ 142, 261, 43, 44, 45, 46, 47, 48, 49, 50, + /* 490 */ 51, 52, 53, 54, 55, 56, 57, 192, 102, 103, + /* 500 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 267, + /* 510 */ 238, 239, 237, 115, 116, 117, 115, 116, 117, 192, + /* 520 */ 59, 118, 16, 251, 121, 122, 123, 303, 19, 303, + /* 530 */ 59, 267, 23, 72, 131, 308, 22, 265, 22, 312, + /* 540 */ 24, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 550 */ 111, 112, 43, 44, 45, 46, 47, 48, 49, 50, + /* 560 */ 51, 52, 53, 54, 55, 56, 57, 19, 81, 297, + /* 570 */ 295, 23, 192, 59, 203, 59, 115, 116, 117, 108, + /* 580 */ 192, 73, 25, 77, 192, 79, 115, 116, 117, 137, + /* 590 */ 138, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 600 */ 52, 53, 54, 55, 56, 57, 119, 215, 216, 238, + /* 610 */ 239, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 620 */ 111, 112, 251, 192, 137, 138, 59, 234, 235, 115, + /* 630 */ 116, 117, 116, 76, 126, 127, 128, 19, 192, 268, + /* 640 */ 19, 23, 22, 192, 252, 24, 89, 300, 301, 92, + /* 650 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 660 */ 112, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 670 */ 52, 53, 54, 55, 56, 57, 19, 192, 192, 59, + /* 680 */ 23, 192, 115, 116, 117, 200, 240, 307, 308, 22, + /* 690 */ 205, 81, 312, 262, 22, 192, 133, 22, 135, 136, + /* 700 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 710 */ 53, 54, 55, 56, 57, 197, 95, 150, 215, 216, + /* 720 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 730 */ 112, 59, 192, 112, 59, 115, 116, 117, 192, 118, + /* 740 */ 119, 120, 121, 122, 123, 124, 19, 137, 138, 303, + /* 750 */ 23, 130, 192, 267, 192, 252, 267, 306, 203, 102, + /* 760 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 770 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 780 */ 53, 54, 55, 56, 57, 19, 240, 115, 116, 117, + /* 790 */ 115, 116, 117, 238, 239, 222, 192, 224, 280, 237, + /* 800 */ 240, 192, 284, 192, 59, 232, 251, 140, 204, 43, + /* 810 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 820 */ 54, 55, 56, 57, 192, 35, 215, 216, 192, 102, + /* 830 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 840 */ 59, 230, 192, 192, 238, 239, 237, 215, 216, 303, + /* 850 */ 308, 215, 216, 16, 312, 19, 66, 251, 126, 127, + /* 860 */ 128, 116, 230, 303, 74, 203, 215, 216, 102, 103, + /* 870 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 43, + /* 880 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 890 */ 54, 55, 56, 57, 192, 212, 115, 116, 117, 19, + /* 900 */ 238, 239, 192, 252, 7, 8, 9, 192, 192, 238, + /* 910 */ 239, 308, 262, 251, 77, 312, 79, 215, 216, 129, + /* 920 */ 210, 211, 251, 142, 158, 45, 46, 47, 48, 49, + /* 930 */ 50, 51, 52, 53, 54, 55, 56, 57, 102, 103, + /* 940 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 12, + /* 950 */ 59, 192, 192, 237, 252, 243, 192, 192, 126, 127, + /* 960 */ 128, 192, 308, 203, 27, 253, 312, 308, 285, 207, + /* 970 */ 208, 312, 157, 290, 159, 215, 216, 262, 192, 42, + /* 980 */ 215, 216, 102, 103, 104, 105, 106, 107, 108, 109, + /* 990 */ 110, 111, 112, 283, 158, 230, 237, 160, 238, 239, + /* 1000 */ 63, 215, 216, 192, 192, 12, 115, 116, 117, 22, + /* 1010 */ 73, 251, 252, 192, 19, 192, 230, 239, 24, 24, + /* 1020 */ 27, 261, 210, 211, 99, 192, 215, 216, 225, 251, + /* 1030 */ 192, 192, 263, 142, 19, 42, 215, 216, 43, 44, + /* 1040 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + /* 1050 */ 55, 56, 57, 59, 19, 291, 63, 132, 43, 44, + /* 1060 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + /* 1070 */ 55, 56, 57, 252, 22, 23, 22, 25, 43, 44, + /* 1080 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + /* 1090 */ 55, 56, 57, 106, 107, 283, 263, 102, 103, 104, + /* 1100 */ 105, 106, 107, 108, 109, 110, 111, 112, 59, 192, + /* 1110 */ 116, 144, 29, 59, 291, 192, 33, 102, 103, 104, + /* 1120 */ 105, 106, 107, 108, 109, 110, 111, 112, 192, 291, + /* 1130 */ 163, 19, 215, 216, 15, 192, 108, 102, 103, 104, + /* 1140 */ 105, 106, 107, 108, 109, 110, 111, 112, 65, 192, + /* 1150 */ 66, 215, 216, 101, 231, 106, 107, 19, 215, 216, + /* 1160 */ 192, 212, 134, 114, 115, 116, 117, 139, 119, 85, + /* 1170 */ 116, 207, 208, 230, 192, 19, 127, 192, 94, 60, + /* 1180 */ 59, 192, 44, 45, 46, 47, 48, 49, 50, 51, + /* 1190 */ 52, 53, 54, 55, 56, 57, 192, 76, 192, 31, + /* 1200 */ 192, 152, 46, 154, 215, 216, 192, 39, 87, 192, + /* 1210 */ 89, 19, 20, 92, 22, 192, 22, 23, 22, 230, + /* 1220 */ 263, 215, 216, 215, 216, 137, 138, 115, 36, 145, + /* 1230 */ 128, 192, 215, 216, 22, 23, 115, 116, 117, 290, + /* 1240 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 1250 */ 112, 59, 192, 151, 215, 216, 192, 61, 203, 298, + /* 1260 */ 299, 192, 192, 71, 25, 144, 203, 192, 203, 230, + /* 1270 */ 114, 19, 20, 81, 22, 215, 216, 263, 192, 215, + /* 1280 */ 216, 255, 256, 203, 215, 216, 130, 19, 36, 192, + /* 1290 */ 215, 216, 100, 238, 239, 101, 25, 192, 106, 107, + /* 1300 */ 48, 238, 239, 238, 239, 113, 251, 115, 116, 117, + /* 1310 */ 192, 59, 120, 101, 251, 192, 251, 192, 238, 239, + /* 1320 */ 215, 216, 192, 71, 46, 243, 192, 25, 25, 137, + /* 1330 */ 138, 251, 192, 215, 216, 253, 25, 85, 215, 216, + /* 1340 */ 215, 216, 90, 243, 152, 153, 154, 155, 156, 215, + /* 1350 */ 216, 192, 100, 253, 243, 215, 216, 192, 106, 107, + /* 1360 */ 243, 192, 148, 149, 253, 113, 192, 115, 116, 117, + /* 1370 */ 253, 192, 120, 192, 215, 216, 192, 23, 192, 25, + /* 1380 */ 215, 216, 192, 115, 215, 216, 22, 148, 24, 215, + /* 1390 */ 216, 192, 114, 192, 215, 216, 215, 216, 134, 215, + /* 1400 */ 216, 215, 216, 139, 152, 153, 154, 155, 156, 0, + /* 1410 */ 1, 2, 141, 23, 5, 25, 215, 216, 24, 10, + /* 1420 */ 11, 12, 13, 14, 1, 2, 17, 125, 5, 19, + /* 1430 */ 20, 268, 22, 10, 11, 12, 13, 14, 192, 30, + /* 1440 */ 17, 32, 23, 23, 25, 25, 36, 144, 23, 40, + /* 1450 */ 25, 192, 141, 30, 192, 32, 23, 22, 25, 5, + /* 1460 */ 128, 215, 216, 40, 10, 11, 12, 13, 14, 59, + /* 1470 */ 23, 17, 25, 192, 215, 216, 192, 215, 216, 70, + /* 1480 */ 23, 71, 25, 151, 30, 192, 32, 78, 53, 192, + /* 1490 */ 81, 192, 192, 70, 40, 85, 215, 216, 119, 120, + /* 1500 */ 90, 78, 59, 254, 81, 192, 192, 98, 215, 216, + /* 1510 */ 100, 23, 59, 25, 215, 216, 106, 107, 23, 192, + /* 1520 */ 25, 98, 19, 113, 70, 115, 116, 117, 215, 216, + /* 1530 */ 120, 192, 78, 192, 140, 81, 19, 20, 23, 22, + /* 1540 */ 25, 132, 215, 216, 192, 192, 137, 138, 192, 23, + /* 1550 */ 192, 25, 98, 36, 192, 132, 215, 216, 192, 116, + /* 1560 */ 137, 138, 152, 153, 154, 155, 156, 215, 216, 116, + /* 1570 */ 161, 215, 216, 215, 216, 120, 59, 215, 216, 7, + /* 1580 */ 8, 215, 216, 192, 161, 130, 132, 192, 71, 83, + /* 1590 */ 84, 137, 138, 59, 192, 192, 19, 20, 192, 22, + /* 1600 */ 97, 225, 85, 192, 23, 192, 25, 90, 192, 192, + /* 1610 */ 215, 216, 152, 36, 154, 161, 192, 100, 215, 216, + /* 1620 */ 192, 215, 216, 106, 107, 225, 215, 216, 192, 192, + /* 1630 */ 113, 192, 115, 116, 117, 257, 59, 120, 192, 215, + /* 1640 */ 216, 152, 192, 154, 192, 23, 317, 25, 71, 235, + /* 1650 */ 116, 215, 216, 192, 215, 216, 192, 192, 192, 192, + /* 1660 */ 192, 192, 192, 192, 192, 215, 216, 215, 216, 152, + /* 1670 */ 153, 154, 155, 156, 192, 192, 287, 100, 286, 241, + /* 1680 */ 254, 254, 242, 106, 107, 108, 254, 213, 254, 190, + /* 1690 */ 113, 270, 115, 116, 117, 296, 266, 120, 219, 258, + /* 1700 */ 244, 270, 258, 19, 20, 228, 22, 292, 266, 224, + /* 1710 */ 292, 218, 218, 195, 218, 270, 258, 60, 245, 270, + /* 1720 */ 36, 245, 244, 248, 19, 20, 242, 22, 248, 152, + /* 1730 */ 153, 154, 155, 156, 244, 140, 199, 199, 279, 38, + /* 1740 */ 199, 36, 150, 59, 296, 149, 22, 296, 18, 43, + /* 1750 */ 236, 199, 233, 18, 198, 71, 293, 236, 271, 236, + /* 1760 */ 269, 236, 148, 271, 59, 245, 269, 245, 282, 271, + /* 1770 */ 199, 198, 233, 245, 293, 233, 71, 245, 157, 62, + /* 1780 */ 22, 199, 198, 220, 100, 199, 198, 220, 199, 289, + /* 1790 */ 106, 107, 288, 114, 226, 198, 217, 113, 64, 115, + /* 1800 */ 116, 117, 217, 22, 120, 100, 223, 217, 217, 164, + /* 1810 */ 223, 106, 107, 220, 125, 24, 217, 219, 113, 217, + /* 1820 */ 115, 116, 117, 217, 311, 120, 226, 112, 304, 281, + /* 1830 */ 281, 220, 114, 143, 260, 259, 152, 153, 154, 155, + /* 1840 */ 156, 199, 91, 316, 82, 316, 147, 144, 22, 199, + /* 1850 */ 249, 276, 157, 146, 260, 145, 278, 152, 153, 154, + /* 1860 */ 155, 156, 259, 248, 260, 260, 259, 259, 249, 245, + /* 1870 */ 247, 246, 25, 264, 264, 201, 13, 6, 193, 193, + /* 1880 */ 212, 206, 191, 191, 191, 212, 206, 221, 212, 212, + /* 1890 */ 221, 213, 4, 206, 213, 212, 3, 22, 162, 15, + /* 1900 */ 23, 16, 23, 138, 150, 129, 25, 24, 141, 20, + /* 1910 */ 16, 143, 1, 141, 302, 302, 299, 129, 129, 61, + /* 1920 */ 150, 53, 53, 37, 129, 53, 53, 115, 1, 34, + /* 1930 */ 140, 5, 22, 114, 160, 68, 75, 25, 68, 41, + /* 1940 */ 140, 114, 24, 20, 19, 130, 124, 67, 24, 67, + /* 1950 */ 22, 22, 22, 96, 23, 22, 59, 22, 67, 37, + /* 1960 */ 28, 23, 148, 22, 25, 23, 23, 23, 23, 22, + /* 1970 */ 140, 97, 23, 23, 115, 22, 142, 25, 88, 75, + /* 1980 */ 34, 44, 34, 75, 23, 34, 86, 22, 34, 34, + /* 1990 */ 34, 24, 93, 25, 25, 23, 34, 23, 23, 23, + /* 2000 */ 23, 11, 23, 25, 22, 22, 22, 1, 23, 23, + /* 2010 */ 22, 22, 25, 15, 23, 1, 140, 25, 140, 318, + /* 2020 */ 318, 134, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2030 */ 318, 318, 140, 318, 318, 318, 140, 318, 318, 318, + /* 2040 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2050 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2060 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2070 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2080 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2090 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2100 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2110 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2120 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2130 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2140 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2150 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2160 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2170 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2180 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2190 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2200 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2210 */ 318, 318, 318, 318, 318, 318, 318, 318, 318, 318, + /* 2220 */ 318, +}; +#define YY_SHIFT_COUNT (571) +#define YY_SHIFT_MIN (0) +#define YY_SHIFT_MAX (2014) +static const unsigned short int yy_shift_ofst[] = { + /* 0 */ 1423, 1409, 1454, 1192, 1192, 610, 1252, 1410, 1517, 1684, + /* 10 */ 1684, 1684, 276, 0, 0, 180, 1015, 1684, 1684, 1684, + /* 20 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, + /* 30 */ 1049, 1049, 1121, 1121, 54, 487, 610, 610, 610, 610, + /* 40 */ 610, 40, 110, 219, 289, 396, 439, 509, 548, 618, + /* 50 */ 657, 727, 766, 836, 995, 1015, 1015, 1015, 1015, 1015, + /* 60 */ 1015, 1015, 1015, 1015, 1015, 1015, 1015, 1015, 1015, 1015, + /* 70 */ 1015, 1015, 1015, 1035, 1015, 1138, 880, 880, 1577, 1684, + /* 80 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, + /* 90 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, + /* 100 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, 1684, + /* 110 */ 1684, 1684, 1684, 1705, 1684, 1684, 1684, 1684, 1684, 1684, + /* 120 */ 1684, 1684, 1684, 1684, 1684, 1684, 1684, 146, 84, 84, + /* 130 */ 84, 84, 84, 277, 315, 401, 97, 461, 251, 66, + /* 140 */ 66, 51, 1156, 66, 66, 324, 324, 66, 452, 452, + /* 150 */ 452, 452, 133, 114, 114, 4, 11, 2037, 2037, 621, + /* 160 */ 621, 621, 567, 398, 398, 398, 398, 937, 937, 228, + /* 170 */ 251, 331, 1052, 66, 66, 66, 66, 66, 66, 66, + /* 180 */ 66, 66, 66, 66, 66, 66, 66, 66, 66, 66, + /* 190 */ 66, 66, 66, 557, 557, 66, 9, 25, 25, 745, + /* 200 */ 745, 967, 1088, 2037, 2037, 2037, 2037, 2037, 2037, 2037, + /* 210 */ 255, 317, 317, 514, 403, 620, 471, 672, 781, 891, + /* 220 */ 675, 66, 66, 66, 66, 66, 66, 66, 66, 66, + /* 230 */ 66, 508, 66, 66, 66, 66, 66, 66, 66, 66, + /* 240 */ 66, 66, 66, 66, 790, 790, 790, 66, 66, 66, + /* 250 */ 338, 66, 66, 66, 516, 1084, 66, 66, 993, 66, + /* 260 */ 66, 66, 66, 66, 66, 66, 66, 732, 1083, 563, + /* 270 */ 994, 994, 994, 994, 337, 563, 563, 1028, 987, 897, + /* 280 */ 1119, 262, 1214, 1271, 1112, 1214, 1112, 1268, 1239, 262, + /* 290 */ 262, 1239, 262, 1271, 1268, 1302, 1354, 1278, 1168, 1168, + /* 300 */ 1168, 1112, 1303, 1303, 815, 1311, 1264, 1364, 1657, 1657, + /* 310 */ 1595, 1595, 1701, 1701, 1595, 1592, 1596, 1724, 1706, 1730, + /* 320 */ 1730, 1730, 1730, 1595, 1735, 1614, 1596, 1596, 1614, 1724, + /* 330 */ 1706, 1614, 1706, 1614, 1595, 1735, 1621, 1717, 1595, 1735, + /* 340 */ 1758, 1595, 1735, 1595, 1735, 1758, 1679, 1679, 1679, 1734, + /* 350 */ 1781, 1781, 1758, 1679, 1689, 1679, 1734, 1679, 1679, 1645, + /* 360 */ 1791, 1715, 1715, 1758, 1690, 1718, 1690, 1718, 1690, 1718, + /* 370 */ 1690, 1718, 1595, 1751, 1751, 1762, 1762, 1699, 1703, 1826, + /* 380 */ 1595, 1695, 1699, 1707, 1710, 1614, 1847, 1863, 1863, 1871, + /* 390 */ 1871, 1871, 2037, 2037, 2037, 2037, 2037, 2037, 2037, 2037, + /* 400 */ 2037, 2037, 2037, 2037, 2037, 2037, 2037, 193, 837, 1194, + /* 410 */ 1212, 506, 832, 1054, 1390, 925, 1435, 1394, 1102, 1332, + /* 420 */ 1419, 1196, 1420, 1425, 1433, 1447, 1457, 1488, 1443, 1379, + /* 430 */ 1572, 1455, 1503, 1453, 1495, 1515, 1506, 1526, 1460, 1489, + /* 440 */ 1581, 1622, 1534, 667, 1888, 1893, 1875, 1736, 1884, 1885, + /* 450 */ 1877, 1879, 1765, 1754, 1776, 1881, 1881, 1883, 1767, 1889, + /* 460 */ 1768, 1894, 1911, 1772, 1788, 1881, 1789, 1858, 1886, 1881, + /* 470 */ 1770, 1868, 1869, 1872, 1873, 1795, 1812, 1895, 1790, 1927, + /* 480 */ 1926, 1910, 1819, 1774, 1867, 1912, 1870, 1861, 1898, 1800, + /* 490 */ 1827, 1918, 1923, 1925, 1815, 1822, 1928, 1880, 1929, 1930, + /* 500 */ 1931, 1933, 1882, 1897, 1924, 1857, 1932, 1935, 1891, 1922, + /* 510 */ 1938, 1814, 1941, 1942, 1943, 1944, 1939, 1945, 1947, 1874, + /* 520 */ 1830, 1949, 1950, 1859, 1946, 1953, 1834, 1952, 1948, 1951, + /* 530 */ 1954, 1955, 1890, 1904, 1900, 1937, 1908, 1899, 1956, 1961, + /* 540 */ 1965, 1967, 1968, 1969, 1962, 1972, 1952, 1974, 1975, 1976, + /* 550 */ 1977, 1978, 1979, 1982, 1990, 1983, 1984, 1985, 1986, 1988, + /* 560 */ 1989, 1987, 1887, 1876, 1878, 1892, 1896, 1992, 1991, 1998, + /* 570 */ 2006, 2014, +}; +#define YY_REDUCE_COUNT (406) +#define YY_REDUCE_MIN (-272) +#define YY_REDUCE_MAX (1693) +static const short yy_reduce_ofst[] = { + /* 0 */ 109, 113, 272, 760, -178, -176, -192, -183, -180, -134, + /* 10 */ 213, 220, 371, -208, -205, -272, -197, 611, 632, 765, + /* 20 */ 786, 392, 943, 989, 503, 651, 1039, -18, 702, 821, + /* 30 */ 710, 812, -188, 380, -187, 555, 662, 1055, 1063, 1065, + /* 40 */ 1080, -267, -267, -267, -267, -267, -267, -267, -267, -267, + /* 50 */ -267, -267, -267, -267, -267, -267, -267, -267, -267, -267, + /* 60 */ -267, -267, -267, -267, -267, -267, -267, -267, -267, -267, + /* 70 */ -267, -267, -267, -267, -267, -267, -267, -267, 636, 811, + /* 80 */ 917, 936, 1006, 1008, 1017, 1060, 1064, 1069, 1075, 1105, + /* 90 */ 1118, 1123, 1125, 1134, 1140, 1159, 1165, 1169, 1174, 1179, + /* 100 */ 1181, 1184, 1186, 1201, 1246, 1259, 1262, 1281, 1293, 1299, + /* 110 */ 1313, 1327, 1341, 1352, 1356, 1358, 1362, 1366, 1395, 1403, + /* 120 */ 1406, 1411, 1424, 1436, 1439, 1450, 1452, -267, -267, -267, + /* 130 */ -267, -267, -267, -267, -267, 224, -267, 446, -24, 275, + /* 140 */ 546, 518, 573, 560, 53, -181, -111, 485, 606, 671, + /* 150 */ 606, 671, 683, 8, 93, -267, -267, -267, -267, 155, + /* 160 */ 155, 155, 181, 242, 264, 486, 489, -218, 393, 227, + /* 170 */ 604, 347, 347, -171, 431, 650, 715, -166, 562, 609, + /* 180 */ 716, 764, 18, 823, 769, 833, 838, 957, 759, 119, + /* 190 */ 923, 226, 1014, 542, 603, 451, 949, 654, 659, 762, + /* 200 */ 964, -4, 778, 961, 712, 1082, 1100, 1111, 1026, 1117, + /* 210 */ -204, -174, -151, -8, 77, 198, 305, 327, 388, 540, + /* 220 */ 839, 968, 982, 985, 1004, 1023, 1070, 1086, 1097, 1130, + /* 230 */ 1190, 1163, 1199, 1284, 1297, 1300, 1314, 1339, 1353, 1391, + /* 240 */ 1402, 1413, 1416, 1417, 803, 1376, 1400, 1428, 1437, 1446, + /* 250 */ 1378, 1461, 1464, 1465, 1249, 1329, 1466, 1467, 1414, 1468, + /* 260 */ 305, 1469, 1470, 1471, 1472, 1482, 1483, 1389, 1392, 1438, + /* 270 */ 1426, 1427, 1432, 1434, 1378, 1438, 1438, 1440, 1474, 1499, + /* 280 */ 1399, 1421, 1430, 1456, 1441, 1442, 1444, 1415, 1473, 1431, + /* 290 */ 1445, 1476, 1449, 1478, 1418, 1479, 1477, 1485, 1493, 1494, + /* 300 */ 1496, 1458, 1475, 1480, 1459, 1490, 1484, 1518, 1448, 1451, + /* 310 */ 1537, 1538, 1463, 1481, 1541, 1486, 1487, 1491, 1519, 1514, + /* 320 */ 1521, 1523, 1525, 1552, 1556, 1520, 1492, 1498, 1522, 1497, + /* 330 */ 1539, 1528, 1542, 1532, 1571, 1573, 1500, 1504, 1582, 1584, + /* 340 */ 1563, 1586, 1588, 1589, 1597, 1567, 1579, 1585, 1590, 1568, + /* 350 */ 1583, 1587, 1593, 1591, 1598, 1599, 1600, 1602, 1606, 1513, + /* 360 */ 1524, 1548, 1549, 1611, 1574, 1576, 1594, 1603, 1604, 1607, + /* 370 */ 1605, 1608, 1642, 1527, 1529, 1609, 1610, 1601, 1615, 1575, + /* 380 */ 1650, 1578, 1619, 1623, 1625, 1624, 1674, 1685, 1686, 1691, + /* 390 */ 1692, 1693, 1612, 1613, 1617, 1675, 1668, 1673, 1676, 1677, + /* 400 */ 1680, 1666, 1669, 1678, 1681, 1683, 1687, +}; +static const YYACTIONTYPE yy_default[] = { + /* 0 */ 1633, 1633, 1633, 1462, 1230, 1341, 1230, 1230, 1230, 1462, + /* 10 */ 1462, 1462, 1230, 1371, 1371, 1515, 1263, 1230, 1230, 1230, + /* 20 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1461, 1230, 1230, + /* 30 */ 1230, 1230, 1550, 1550, 1230, 1230, 1230, 1230, 1230, 1230, + /* 40 */ 1230, 1230, 1380, 1230, 1387, 1230, 1230, 1230, 1230, 1230, + /* 50 */ 1463, 1464, 1230, 1230, 1230, 1514, 1516, 1479, 1394, 1393, + /* 60 */ 1392, 1391, 1497, 1358, 1385, 1378, 1382, 1457, 1458, 1456, + /* 70 */ 1460, 1464, 1463, 1230, 1381, 1428, 1442, 1427, 1230, 1230, + /* 80 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 90 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 100 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 110 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 120 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1436, 1441, 1447, + /* 130 */ 1440, 1437, 1430, 1429, 1431, 1230, 1432, 1230, 1254, 1230, + /* 140 */ 1230, 1251, 1305, 1230, 1230, 1230, 1230, 1230, 1534, 1533, + /* 150 */ 1230, 1230, 1263, 1422, 1421, 1433, 1434, 1444, 1443, 1522, + /* 160 */ 1586, 1585, 1480, 1230, 1230, 1230, 1230, 1230, 1230, 1550, + /* 170 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 180 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 190 */ 1230, 1230, 1230, 1550, 1550, 1230, 1263, 1550, 1550, 1259, + /* 200 */ 1259, 1365, 1230, 1529, 1332, 1332, 1332, 1332, 1341, 1332, + /* 210 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 220 */ 1230, 1230, 1230, 1230, 1230, 1519, 1517, 1230, 1230, 1230, + /* 230 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 240 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 250 */ 1230, 1230, 1230, 1230, 1337, 1230, 1230, 1230, 1230, 1230, + /* 260 */ 1230, 1230, 1230, 1230, 1230, 1230, 1579, 1230, 1492, 1319, + /* 270 */ 1337, 1337, 1337, 1337, 1339, 1320, 1318, 1331, 1264, 1237, + /* 280 */ 1625, 1397, 1386, 1338, 1360, 1386, 1360, 1622, 1384, 1397, + /* 290 */ 1397, 1384, 1397, 1338, 1622, 1280, 1602, 1275, 1371, 1371, + /* 300 */ 1371, 1360, 1365, 1365, 1459, 1338, 1331, 1230, 1625, 1625, + /* 310 */ 1346, 1346, 1624, 1624, 1346, 1480, 1609, 1406, 1308, 1314, + /* 320 */ 1314, 1314, 1314, 1346, 1248, 1384, 1609, 1609, 1384, 1406, + /* 330 */ 1308, 1384, 1308, 1384, 1346, 1248, 1496, 1619, 1346, 1248, + /* 340 */ 1470, 1346, 1248, 1346, 1248, 1470, 1306, 1306, 1306, 1295, + /* 350 */ 1230, 1230, 1470, 1306, 1280, 1306, 1295, 1306, 1306, 1568, + /* 360 */ 1230, 1474, 1474, 1470, 1364, 1359, 1364, 1359, 1364, 1359, + /* 370 */ 1364, 1359, 1346, 1560, 1560, 1374, 1374, 1379, 1365, 1465, + /* 380 */ 1346, 1230, 1379, 1377, 1375, 1384, 1298, 1582, 1582, 1578, + /* 390 */ 1578, 1578, 1630, 1630, 1529, 1595, 1263, 1263, 1263, 1263, + /* 400 */ 1595, 1282, 1282, 1264, 1264, 1263, 1595, 1230, 1230, 1230, + /* 410 */ 1230, 1230, 1230, 1590, 1230, 1524, 1481, 1350, 1230, 1230, + /* 420 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 430 */ 1230, 1230, 1535, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 440 */ 1230, 1230, 1230, 1411, 1230, 1233, 1526, 1230, 1230, 1230, + /* 450 */ 1230, 1230, 1230, 1230, 1230, 1388, 1389, 1351, 1230, 1230, + /* 460 */ 1230, 1230, 1230, 1230, 1230, 1403, 1230, 1230, 1230, 1398, + /* 470 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1621, 1230, + /* 480 */ 1230, 1230, 1230, 1230, 1230, 1495, 1494, 1230, 1230, 1348, + /* 490 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 500 */ 1230, 1230, 1230, 1278, 1230, 1230, 1230, 1230, 1230, 1230, + /* 510 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 520 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1376, 1230, 1230, + /* 530 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 540 */ 1230, 1230, 1565, 1366, 1230, 1230, 1612, 1230, 1230, 1230, + /* 550 */ 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, 1230, + /* 560 */ 1230, 1606, 1322, 1413, 1230, 1412, 1416, 1252, 1230, 1242, + /* 570 */ 1230, 1230, +}; +/********** End of lemon-generated parsing tables *****************************/ + +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: +** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. +*/ +#ifdef YYFALLBACK +static const YYCODETYPE yyFallback[] = { + 0, /* $ => nothing */ + 0, /* SEMI => nothing */ + 59, /* EXPLAIN => ID */ + 59, /* QUERY => ID */ + 59, /* PLAN => ID */ + 59, /* BEGIN => ID */ + 0, /* TRANSACTION => nothing */ + 59, /* DEFERRED => ID */ + 59, /* IMMEDIATE => ID */ + 59, /* EXCLUSIVE => ID */ + 0, /* COMMIT => nothing */ + 59, /* END => ID */ + 59, /* ROLLBACK => ID */ + 59, /* SAVEPOINT => ID */ + 59, /* RELEASE => ID */ + 0, /* TO => nothing */ + 0, /* TABLE => nothing */ + 0, /* CREATE => nothing */ + 59, /* IF => ID */ + 0, /* NOT => nothing */ + 0, /* EXISTS => nothing */ + 59, /* TEMP => ID */ + 0, /* LP => nothing */ + 0, /* RP => nothing */ + 0, /* AS => nothing */ + 0, /* COMMA => nothing */ + 59, /* WITHOUT => ID */ + 59, /* ABORT => ID */ + 59, /* ACTION => ID */ + 59, /* AFTER => ID */ + 59, /* ANALYZE => ID */ + 59, /* ASC => ID */ + 59, /* ATTACH => ID */ + 59, /* BEFORE => ID */ + 59, /* BY => ID */ + 59, /* CASCADE => ID */ + 59, /* CAST => ID */ + 59, /* CONFLICT => ID */ + 59, /* DATABASE => ID */ + 59, /* DESC => ID */ + 59, /* DETACH => ID */ + 59, /* EACH => ID */ + 59, /* FAIL => ID */ + 0, /* OR => nothing */ + 0, /* AND => nothing */ + 0, /* IS => nothing */ + 59, /* MATCH => ID */ + 59, /* LIKE_KW => ID */ + 0, /* BETWEEN => nothing */ + 0, /* IN => nothing */ + 0, /* ISNULL => nothing */ + 0, /* NOTNULL => nothing */ + 0, /* NE => nothing */ + 0, /* EQ => nothing */ + 0, /* GT => nothing */ + 0, /* LE => nothing */ + 0, /* LT => nothing */ + 0, /* GE => nothing */ + 0, /* ESCAPE => nothing */ + 0, /* ID => nothing */ + 59, /* COLUMNKW => ID */ + 59, /* DO => ID */ + 59, /* FOR => ID */ + 59, /* IGNORE => ID */ + 59, /* INITIALLY => ID */ + 59, /* INSTEAD => ID */ + 59, /* NO => ID */ + 59, /* KEY => ID */ + 59, /* OF => ID */ + 59, /* OFFSET => ID */ + 59, /* PRAGMA => ID */ + 59, /* RAISE => ID */ + 59, /* RECURSIVE => ID */ + 59, /* REPLACE => ID */ + 59, /* RESTRICT => ID */ + 59, /* ROW => ID */ + 59, /* ROWS => ID */ + 59, /* TRIGGER => ID */ + 59, /* VACUUM => ID */ + 59, /* VIEW => ID */ + 59, /* VIRTUAL => ID */ + 59, /* WITH => ID */ + 59, /* NULLS => ID */ + 59, /* FIRST => ID */ + 59, /* LAST => ID */ + 59, /* CURRENT => ID */ + 59, /* FOLLOWING => ID */ + 59, /* PARTITION => ID */ + 59, /* PRECEDING => ID */ + 59, /* RANGE => ID */ + 59, /* UNBOUNDED => ID */ + 59, /* EXCLUDE => ID */ + 59, /* GROUPS => ID */ + 59, /* OTHERS => ID */ + 59, /* TIES => ID */ + 59, /* GENERATED => ID */ + 59, /* ALWAYS => ID */ + 59, /* MATERIALIZED => ID */ + 59, /* REINDEX => ID */ + 59, /* RENAME => ID */ + 59, /* CTIME_KW => ID */ + 0, /* ANY => nothing */ + 0, /* BITAND => nothing */ + 0, /* BITOR => nothing */ + 0, /* LSHIFT => nothing */ + 0, /* RSHIFT => nothing */ + 0, /* PLUS => nothing */ + 0, /* MINUS => nothing */ + 0, /* STAR => nothing */ + 0, /* SLASH => nothing */ + 0, /* REM => nothing */ + 0, /* CONCAT => nothing */ + 0, /* COLLATE => nothing */ + 0, /* BITNOT => nothing */ + 0, /* ON => nothing */ + 0, /* INDEXED => nothing */ + 0, /* STRING => nothing */ + 0, /* JOIN_KW => nothing */ + 0, /* CONSTRAINT => nothing */ + 0, /* DEFAULT => nothing */ + 0, /* NULL => nothing */ + 0, /* PRIMARY => nothing */ + 0, /* UNIQUE => nothing */ + 0, /* CHECK => nothing */ + 0, /* REFERENCES => nothing */ + 0, /* AUTOINCR => nothing */ + 0, /* INSERT => nothing */ + 0, /* DELETE => nothing */ + 0, /* UPDATE => nothing */ + 0, /* SET => nothing */ + 0, /* DEFERRABLE => nothing */ + 0, /* FOREIGN => nothing */ + 0, /* DROP => nothing */ + 0, /* UNION => nothing */ + 0, /* ALL => nothing */ + 0, /* EXCEPT => nothing */ + 0, /* INTERSECT => nothing */ + 0, /* SELECT => nothing */ + 0, /* VALUES => nothing */ + 0, /* DISTINCT => nothing */ + 0, /* DOT => nothing */ + 0, /* FROM => nothing */ + 0, /* JOIN => nothing */ + 0, /* USING => nothing */ + 0, /* ORDER => nothing */ + 0, /* GROUP => nothing */ + 0, /* HAVING => nothing */ + 0, /* LIMIT => nothing */ + 0, /* WHERE => nothing */ + 0, /* RETURNING => nothing */ + 0, /* INTO => nothing */ + 0, /* NOTHING => nothing */ + 0, /* FLOAT => nothing */ + 0, /* BLOB => nothing */ + 0, /* INTEGER => nothing */ + 0, /* VARIABLE => nothing */ + 0, /* CASE => nothing */ + 0, /* WHEN => nothing */ + 0, /* THEN => nothing */ + 0, /* ELSE => nothing */ + 0, /* INDEX => nothing */ + 0, /* ALTER => nothing */ + 0, /* ADD => nothing */ + 0, /* WINDOW => nothing */ + 0, /* OVER => nothing */ + 0, /* FILTER => nothing */ + 0, /* COLUMN => nothing */ + 0, /* AGG_FUNCTION => nothing */ + 0, /* AGG_COLUMN => nothing */ + 0, /* TRUEFALSE => nothing */ + 0, /* ISNOT => nothing */ + 0, /* FUNCTION => nothing */ + 0, /* UMINUS => nothing */ + 0, /* UPLUS => nothing */ + 0, /* TRUTH => nothing */ + 0, /* REGISTER => nothing */ + 0, /* VECTOR => nothing */ + 0, /* SELECT_COLUMN => nothing */ + 0, /* IF_NULL_ROW => nothing */ + 0, /* ASTERISK => nothing */ + 0, /* SPAN => nothing */ + 0, /* ERROR => nothing */ + 0, /* SPACE => nothing */ + 0, /* ILLEGAL => nothing */ +}; +#endif /* YYFALLBACK */ + +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. +*/ +struct yyStackEntry { + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct yyStackEntry yyStackEntry; + +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct yyParser { + yyStackEntry *yytos; /* Pointer to top element of the stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyhwm; /* High-water mark of the stack */ +#endif +#ifndef YYNOERRORRECOVERY + int yyerrcnt; /* Shifts left before out of the error */ +#endif + sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ + sqlite3ParserCTX_SDECL /* A place to hold %extra_context */ +#if YYSTACKDEPTH<=0 + int yystksz; /* Current side of the stack */ + yyStackEntry *yystack; /* The parser's stack */ + yyStackEntry yystk0; /* First stack entry */ +#else + yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ + yyStackEntry *yystackEnd; /* Last entry in the stack */ +#endif +}; +typedef struct yyParser yyParser; + +#ifndef NDEBUG +/* #include */ +/* #include */ +static FILE *yyTraceFILE = 0; +static char *yyTracePrompt = 0; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +**
      +**
    • A FILE* to which trace output should be written. +** If NULL, then tracing is turned off. +**
    • A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +**
    +** +** Outputs: +** None. +*/ +SQLITE_PRIVATE void sqlite3ParserTrace(FILE *TraceFILE, char *zTracePrompt){ + yyTraceFILE = TraceFILE; + yyTracePrompt = zTracePrompt; + if( yyTraceFILE==0 ) yyTracePrompt = 0; + else if( yyTracePrompt==0 ) yyTraceFILE = 0; +} +#endif /* NDEBUG */ + +#if defined(YYCOVERAGE) || !defined(NDEBUG) +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const yyTokenName[] = { + /* 0 */ "$", + /* 1 */ "SEMI", + /* 2 */ "EXPLAIN", + /* 3 */ "QUERY", + /* 4 */ "PLAN", + /* 5 */ "BEGIN", + /* 6 */ "TRANSACTION", + /* 7 */ "DEFERRED", + /* 8 */ "IMMEDIATE", + /* 9 */ "EXCLUSIVE", + /* 10 */ "COMMIT", + /* 11 */ "END", + /* 12 */ "ROLLBACK", + /* 13 */ "SAVEPOINT", + /* 14 */ "RELEASE", + /* 15 */ "TO", + /* 16 */ "TABLE", + /* 17 */ "CREATE", + /* 18 */ "IF", + /* 19 */ "NOT", + /* 20 */ "EXISTS", + /* 21 */ "TEMP", + /* 22 */ "LP", + /* 23 */ "RP", + /* 24 */ "AS", + /* 25 */ "COMMA", + /* 26 */ "WITHOUT", + /* 27 */ "ABORT", + /* 28 */ "ACTION", + /* 29 */ "AFTER", + /* 30 */ "ANALYZE", + /* 31 */ "ASC", + /* 32 */ "ATTACH", + /* 33 */ "BEFORE", + /* 34 */ "BY", + /* 35 */ "CASCADE", + /* 36 */ "CAST", + /* 37 */ "CONFLICT", + /* 38 */ "DATABASE", + /* 39 */ "DESC", + /* 40 */ "DETACH", + /* 41 */ "EACH", + /* 42 */ "FAIL", + /* 43 */ "OR", + /* 44 */ "AND", + /* 45 */ "IS", + /* 46 */ "MATCH", + /* 47 */ "LIKE_KW", + /* 48 */ "BETWEEN", + /* 49 */ "IN", + /* 50 */ "ISNULL", + /* 51 */ "NOTNULL", + /* 52 */ "NE", + /* 53 */ "EQ", + /* 54 */ "GT", + /* 55 */ "LE", + /* 56 */ "LT", + /* 57 */ "GE", + /* 58 */ "ESCAPE", + /* 59 */ "ID", + /* 60 */ "COLUMNKW", + /* 61 */ "DO", + /* 62 */ "FOR", + /* 63 */ "IGNORE", + /* 64 */ "INITIALLY", + /* 65 */ "INSTEAD", + /* 66 */ "NO", + /* 67 */ "KEY", + /* 68 */ "OF", + /* 69 */ "OFFSET", + /* 70 */ "PRAGMA", + /* 71 */ "RAISE", + /* 72 */ "RECURSIVE", + /* 73 */ "REPLACE", + /* 74 */ "RESTRICT", + /* 75 */ "ROW", + /* 76 */ "ROWS", + /* 77 */ "TRIGGER", + /* 78 */ "VACUUM", + /* 79 */ "VIEW", + /* 80 */ "VIRTUAL", + /* 81 */ "WITH", + /* 82 */ "NULLS", + /* 83 */ "FIRST", + /* 84 */ "LAST", + /* 85 */ "CURRENT", + /* 86 */ "FOLLOWING", + /* 87 */ "PARTITION", + /* 88 */ "PRECEDING", + /* 89 */ "RANGE", + /* 90 */ "UNBOUNDED", + /* 91 */ "EXCLUDE", + /* 92 */ "GROUPS", + /* 93 */ "OTHERS", + /* 94 */ "TIES", + /* 95 */ "GENERATED", + /* 96 */ "ALWAYS", + /* 97 */ "MATERIALIZED", + /* 98 */ "REINDEX", + /* 99 */ "RENAME", + /* 100 */ "CTIME_KW", + /* 101 */ "ANY", + /* 102 */ "BITAND", + /* 103 */ "BITOR", + /* 104 */ "LSHIFT", + /* 105 */ "RSHIFT", + /* 106 */ "PLUS", + /* 107 */ "MINUS", + /* 108 */ "STAR", + /* 109 */ "SLASH", + /* 110 */ "REM", + /* 111 */ "CONCAT", + /* 112 */ "COLLATE", + /* 113 */ "BITNOT", + /* 114 */ "ON", + /* 115 */ "INDEXED", + /* 116 */ "STRING", + /* 117 */ "JOIN_KW", + /* 118 */ "CONSTRAINT", + /* 119 */ "DEFAULT", + /* 120 */ "NULL", + /* 121 */ "PRIMARY", + /* 122 */ "UNIQUE", + /* 123 */ "CHECK", + /* 124 */ "REFERENCES", + /* 125 */ "AUTOINCR", + /* 126 */ "INSERT", + /* 127 */ "DELETE", + /* 128 */ "UPDATE", + /* 129 */ "SET", + /* 130 */ "DEFERRABLE", + /* 131 */ "FOREIGN", + /* 132 */ "DROP", + /* 133 */ "UNION", + /* 134 */ "ALL", + /* 135 */ "EXCEPT", + /* 136 */ "INTERSECT", + /* 137 */ "SELECT", + /* 138 */ "VALUES", + /* 139 */ "DISTINCT", + /* 140 */ "DOT", + /* 141 */ "FROM", + /* 142 */ "JOIN", + /* 143 */ "USING", + /* 144 */ "ORDER", + /* 145 */ "GROUP", + /* 146 */ "HAVING", + /* 147 */ "LIMIT", + /* 148 */ "WHERE", + /* 149 */ "RETURNING", + /* 150 */ "INTO", + /* 151 */ "NOTHING", + /* 152 */ "FLOAT", + /* 153 */ "BLOB", + /* 154 */ "INTEGER", + /* 155 */ "VARIABLE", + /* 156 */ "CASE", + /* 157 */ "WHEN", + /* 158 */ "THEN", + /* 159 */ "ELSE", + /* 160 */ "INDEX", + /* 161 */ "ALTER", + /* 162 */ "ADD", + /* 163 */ "WINDOW", + /* 164 */ "OVER", + /* 165 */ "FILTER", + /* 166 */ "COLUMN", + /* 167 */ "AGG_FUNCTION", + /* 168 */ "AGG_COLUMN", + /* 169 */ "TRUEFALSE", + /* 170 */ "ISNOT", + /* 171 */ "FUNCTION", + /* 172 */ "UMINUS", + /* 173 */ "UPLUS", + /* 174 */ "TRUTH", + /* 175 */ "REGISTER", + /* 176 */ "VECTOR", + /* 177 */ "SELECT_COLUMN", + /* 178 */ "IF_NULL_ROW", + /* 179 */ "ASTERISK", + /* 180 */ "SPAN", + /* 181 */ "ERROR", + /* 182 */ "SPACE", + /* 183 */ "ILLEGAL", + /* 184 */ "input", + /* 185 */ "cmdlist", + /* 186 */ "ecmd", + /* 187 */ "cmdx", + /* 188 */ "explain", + /* 189 */ "cmd", + /* 190 */ "transtype", + /* 191 */ "trans_opt", + /* 192 */ "nm", + /* 193 */ "savepoint_opt", + /* 194 */ "create_table", + /* 195 */ "create_table_args", + /* 196 */ "createkw", + /* 197 */ "temp", + /* 198 */ "ifnotexists", + /* 199 */ "dbnm", + /* 200 */ "columnlist", + /* 201 */ "conslist_opt", + /* 202 */ "table_option_set", + /* 203 */ "select", + /* 204 */ "table_option", + /* 205 */ "columnname", + /* 206 */ "carglist", + /* 207 */ "typetoken", + /* 208 */ "typename", + /* 209 */ "signed", + /* 210 */ "plus_num", + /* 211 */ "minus_num", + /* 212 */ "scanpt", + /* 213 */ "scantok", + /* 214 */ "ccons", + /* 215 */ "term", + /* 216 */ "expr", + /* 217 */ "onconf", + /* 218 */ "sortorder", + /* 219 */ "autoinc", + /* 220 */ "eidlist_opt", + /* 221 */ "refargs", + /* 222 */ "defer_subclause", + /* 223 */ "generated", + /* 224 */ "refarg", + /* 225 */ "refact", + /* 226 */ "init_deferred_pred_opt", + /* 227 */ "conslist", + /* 228 */ "tconscomma", + /* 229 */ "tcons", + /* 230 */ "sortlist", + /* 231 */ "eidlist", + /* 232 */ "defer_subclause_opt", + /* 233 */ "orconf", + /* 234 */ "resolvetype", + /* 235 */ "raisetype", + /* 236 */ "ifexists", + /* 237 */ "fullname", + /* 238 */ "selectnowith", + /* 239 */ "oneselect", + /* 240 */ "wqlist", + /* 241 */ "multiselect_op", + /* 242 */ "distinct", + /* 243 */ "selcollist", + /* 244 */ "from", + /* 245 */ "where_opt", + /* 246 */ "groupby_opt", + /* 247 */ "having_opt", + /* 248 */ "orderby_opt", + /* 249 */ "limit_opt", + /* 250 */ "window_clause", + /* 251 */ "values", + /* 252 */ "nexprlist", + /* 253 */ "sclp", + /* 254 */ "as", + /* 255 */ "seltablist", + /* 256 */ "stl_prefix", + /* 257 */ "joinop", + /* 258 */ "indexed_opt", + /* 259 */ "on_opt", + /* 260 */ "using_opt", + /* 261 */ "exprlist", + /* 262 */ "xfullname", + /* 263 */ "idlist", + /* 264 */ "nulls", + /* 265 */ "with", + /* 266 */ "where_opt_ret", + /* 267 */ "setlist", + /* 268 */ "insert_cmd", + /* 269 */ "idlist_opt", + /* 270 */ "upsert", + /* 271 */ "returning", + /* 272 */ "filter_over", + /* 273 */ "likeop", + /* 274 */ "between_op", + /* 275 */ "in_op", + /* 276 */ "paren_exprlist", + /* 277 */ "case_operand", + /* 278 */ "case_exprlist", + /* 279 */ "case_else", + /* 280 */ "uniqueflag", + /* 281 */ "collate", + /* 282 */ "vinto", + /* 283 */ "nmnum", + /* 284 */ "trigger_decl", + /* 285 */ "trigger_cmd_list", + /* 286 */ "trigger_time", + /* 287 */ "trigger_event", + /* 288 */ "foreach_clause", + /* 289 */ "when_clause", + /* 290 */ "trigger_cmd", + /* 291 */ "trnm", + /* 292 */ "tridxby", + /* 293 */ "database_kw_opt", + /* 294 */ "key_opt", + /* 295 */ "add_column_fullname", + /* 296 */ "kwcolumn_opt", + /* 297 */ "create_vtab", + /* 298 */ "vtabarglist", + /* 299 */ "vtabarg", + /* 300 */ "vtabargtoken", + /* 301 */ "lp", + /* 302 */ "anylist", + /* 303 */ "wqitem", + /* 304 */ "wqas", + /* 305 */ "windowdefn_list", + /* 306 */ "windowdefn", + /* 307 */ "window", + /* 308 */ "frame_opt", + /* 309 */ "part_opt", + /* 310 */ "filter_clause", + /* 311 */ "over_clause", + /* 312 */ "range_or_rows", + /* 313 */ "frame_bound", + /* 314 */ "frame_bound_s", + /* 315 */ "frame_bound_e", + /* 316 */ "frame_exclude_opt", + /* 317 */ "frame_exclude", +}; +#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ + +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const yyRuleName[] = { + /* 0 */ "explain ::= EXPLAIN", + /* 1 */ "explain ::= EXPLAIN QUERY PLAN", + /* 2 */ "cmdx ::= cmd", + /* 3 */ "cmd ::= BEGIN transtype trans_opt", + /* 4 */ "transtype ::=", + /* 5 */ "transtype ::= DEFERRED", + /* 6 */ "transtype ::= IMMEDIATE", + /* 7 */ "transtype ::= EXCLUSIVE", + /* 8 */ "cmd ::= COMMIT|END trans_opt", + /* 9 */ "cmd ::= ROLLBACK trans_opt", + /* 10 */ "cmd ::= SAVEPOINT nm", + /* 11 */ "cmd ::= RELEASE savepoint_opt nm", + /* 12 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", + /* 13 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", + /* 14 */ "createkw ::= CREATE", + /* 15 */ "ifnotexists ::=", + /* 16 */ "ifnotexists ::= IF NOT EXISTS", + /* 17 */ "temp ::= TEMP", + /* 18 */ "temp ::=", + /* 19 */ "create_table_args ::= LP columnlist conslist_opt RP table_option_set", + /* 20 */ "create_table_args ::= AS select", + /* 21 */ "table_option_set ::=", + /* 22 */ "table_option_set ::= table_option_set COMMA table_option", + /* 23 */ "table_option ::= WITHOUT nm", + /* 24 */ "table_option ::= nm", + /* 25 */ "columnname ::= nm typetoken", + /* 26 */ "typetoken ::=", + /* 27 */ "typetoken ::= typename LP signed RP", + /* 28 */ "typetoken ::= typename LP signed COMMA signed RP", + /* 29 */ "typename ::= typename ID|STRING", + /* 30 */ "scanpt ::=", + /* 31 */ "scantok ::=", + /* 32 */ "ccons ::= CONSTRAINT nm", + /* 33 */ "ccons ::= DEFAULT scantok term", + /* 34 */ "ccons ::= DEFAULT LP expr RP", + /* 35 */ "ccons ::= DEFAULT PLUS scantok term", + /* 36 */ "ccons ::= DEFAULT MINUS scantok term", + /* 37 */ "ccons ::= DEFAULT scantok ID|INDEXED", + /* 38 */ "ccons ::= NOT NULL onconf", + /* 39 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", + /* 40 */ "ccons ::= UNIQUE onconf", + /* 41 */ "ccons ::= CHECK LP expr RP", + /* 42 */ "ccons ::= REFERENCES nm eidlist_opt refargs", + /* 43 */ "ccons ::= defer_subclause", + /* 44 */ "ccons ::= COLLATE ID|STRING", + /* 45 */ "generated ::= LP expr RP", + /* 46 */ "generated ::= LP expr RP ID", + /* 47 */ "autoinc ::=", + /* 48 */ "autoinc ::= AUTOINCR", + /* 49 */ "refargs ::=", + /* 50 */ "refargs ::= refargs refarg", + /* 51 */ "refarg ::= MATCH nm", + /* 52 */ "refarg ::= ON INSERT refact", + /* 53 */ "refarg ::= ON DELETE refact", + /* 54 */ "refarg ::= ON UPDATE refact", + /* 55 */ "refact ::= SET NULL", + /* 56 */ "refact ::= SET DEFAULT", + /* 57 */ "refact ::= CASCADE", + /* 58 */ "refact ::= RESTRICT", + /* 59 */ "refact ::= NO ACTION", + /* 60 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", + /* 61 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", + /* 62 */ "init_deferred_pred_opt ::=", + /* 63 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", + /* 64 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", + /* 65 */ "conslist_opt ::=", + /* 66 */ "tconscomma ::= COMMA", + /* 67 */ "tcons ::= CONSTRAINT nm", + /* 68 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", + /* 69 */ "tcons ::= UNIQUE LP sortlist RP onconf", + /* 70 */ "tcons ::= CHECK LP expr RP onconf", + /* 71 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", + /* 72 */ "defer_subclause_opt ::=", + /* 73 */ "onconf ::=", + /* 74 */ "onconf ::= ON CONFLICT resolvetype", + /* 75 */ "orconf ::=", + /* 76 */ "orconf ::= OR resolvetype", + /* 77 */ "resolvetype ::= IGNORE", + /* 78 */ "resolvetype ::= REPLACE", + /* 79 */ "cmd ::= DROP TABLE ifexists fullname", + /* 80 */ "ifexists ::= IF EXISTS", + /* 81 */ "ifexists ::=", + /* 82 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", + /* 83 */ "cmd ::= DROP VIEW ifexists fullname", + /* 84 */ "cmd ::= select", + /* 85 */ "select ::= WITH wqlist selectnowith", + /* 86 */ "select ::= WITH RECURSIVE wqlist selectnowith", + /* 87 */ "select ::= selectnowith", + /* 88 */ "selectnowith ::= selectnowith multiselect_op oneselect", + /* 89 */ "multiselect_op ::= UNION", + /* 90 */ "multiselect_op ::= UNION ALL", + /* 91 */ "multiselect_op ::= EXCEPT|INTERSECT", + /* 92 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", + /* 93 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt", + /* 94 */ "values ::= VALUES LP nexprlist RP", + /* 95 */ "values ::= values COMMA LP nexprlist RP", + /* 96 */ "distinct ::= DISTINCT", + /* 97 */ "distinct ::= ALL", + /* 98 */ "distinct ::=", + /* 99 */ "sclp ::=", + /* 100 */ "selcollist ::= sclp scanpt expr scanpt as", + /* 101 */ "selcollist ::= sclp scanpt STAR", + /* 102 */ "selcollist ::= sclp scanpt nm DOT STAR", + /* 103 */ "as ::= AS nm", + /* 104 */ "as ::=", + /* 105 */ "from ::=", + /* 106 */ "from ::= FROM seltablist", + /* 107 */ "stl_prefix ::= seltablist joinop", + /* 108 */ "stl_prefix ::=", + /* 109 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", + /* 110 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", + /* 111 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 112 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 113 */ "dbnm ::=", + /* 114 */ "dbnm ::= DOT nm", + /* 115 */ "fullname ::= nm", + /* 116 */ "fullname ::= nm DOT nm", + /* 117 */ "xfullname ::= nm", + /* 118 */ "xfullname ::= nm DOT nm", + /* 119 */ "xfullname ::= nm DOT nm AS nm", + /* 120 */ "xfullname ::= nm AS nm", + /* 121 */ "joinop ::= COMMA|JOIN", + /* 122 */ "joinop ::= JOIN_KW JOIN", + /* 123 */ "joinop ::= JOIN_KW nm JOIN", + /* 124 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 125 */ "on_opt ::= ON expr", + /* 126 */ "on_opt ::=", + /* 127 */ "indexed_opt ::=", + /* 128 */ "indexed_opt ::= INDEXED BY nm", + /* 129 */ "indexed_opt ::= NOT INDEXED", + /* 130 */ "using_opt ::= USING LP idlist RP", + /* 131 */ "using_opt ::=", + /* 132 */ "orderby_opt ::=", + /* 133 */ "orderby_opt ::= ORDER BY sortlist", + /* 134 */ "sortlist ::= sortlist COMMA expr sortorder nulls", + /* 135 */ "sortlist ::= expr sortorder nulls", + /* 136 */ "sortorder ::= ASC", + /* 137 */ "sortorder ::= DESC", + /* 138 */ "sortorder ::=", + /* 139 */ "nulls ::= NULLS FIRST", + /* 140 */ "nulls ::= NULLS LAST", + /* 141 */ "nulls ::=", + /* 142 */ "groupby_opt ::=", + /* 143 */ "groupby_opt ::= GROUP BY nexprlist", + /* 144 */ "having_opt ::=", + /* 145 */ "having_opt ::= HAVING expr", + /* 146 */ "limit_opt ::=", + /* 147 */ "limit_opt ::= LIMIT expr", + /* 148 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 149 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 150 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret", + /* 151 */ "where_opt ::=", + /* 152 */ "where_opt ::= WHERE expr", + /* 153 */ "where_opt_ret ::=", + /* 154 */ "where_opt_ret ::= WHERE expr", + /* 155 */ "where_opt_ret ::= RETURNING selcollist", + /* 156 */ "where_opt_ret ::= WHERE expr RETURNING selcollist", + /* 157 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret", + /* 158 */ "setlist ::= setlist COMMA nm EQ expr", + /* 159 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", + /* 160 */ "setlist ::= nm EQ expr", + /* 161 */ "setlist ::= LP idlist RP EQ expr", + /* 162 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", + /* 163 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning", + /* 164 */ "upsert ::=", + /* 165 */ "upsert ::= RETURNING selcollist", + /* 166 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert", + /* 167 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert", + /* 168 */ "upsert ::= ON CONFLICT DO NOTHING returning", + /* 169 */ "upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning", + /* 170 */ "returning ::= RETURNING selcollist", + /* 171 */ "insert_cmd ::= INSERT orconf", + /* 172 */ "insert_cmd ::= REPLACE", + /* 173 */ "idlist_opt ::=", + /* 174 */ "idlist_opt ::= LP idlist RP", + /* 175 */ "idlist ::= idlist COMMA nm", + /* 176 */ "idlist ::= nm", + /* 177 */ "expr ::= LP expr RP", + /* 178 */ "expr ::= ID|INDEXED", + /* 179 */ "expr ::= JOIN_KW", + /* 180 */ "expr ::= nm DOT nm", + /* 181 */ "expr ::= nm DOT nm DOT nm", + /* 182 */ "term ::= NULL|FLOAT|BLOB", + /* 183 */ "term ::= STRING", + /* 184 */ "term ::= INTEGER", + /* 185 */ "expr ::= VARIABLE", + /* 186 */ "expr ::= expr COLLATE ID|STRING", + /* 187 */ "expr ::= CAST LP expr AS typetoken RP", + /* 188 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 189 */ "expr ::= ID|INDEXED LP STAR RP", + /* 190 */ "expr ::= ID|INDEXED LP distinct exprlist RP filter_over", + /* 191 */ "expr ::= ID|INDEXED LP STAR RP filter_over", + /* 192 */ "term ::= CTIME_KW", + /* 193 */ "expr ::= LP nexprlist COMMA expr RP", + /* 194 */ "expr ::= expr AND expr", + /* 195 */ "expr ::= expr OR expr", + /* 196 */ "expr ::= expr LT|GT|GE|LE expr", + /* 197 */ "expr ::= expr EQ|NE expr", + /* 198 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 199 */ "expr ::= expr PLUS|MINUS expr", + /* 200 */ "expr ::= expr STAR|SLASH|REM expr", + /* 201 */ "expr ::= expr CONCAT expr", + /* 202 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 203 */ "expr ::= expr likeop expr", + /* 204 */ "expr ::= expr likeop expr ESCAPE expr", + /* 205 */ "expr ::= expr ISNULL|NOTNULL", + /* 206 */ "expr ::= expr NOT NULL", + /* 207 */ "expr ::= expr IS expr", + /* 208 */ "expr ::= expr IS NOT expr", + /* 209 */ "expr ::= NOT expr", + /* 210 */ "expr ::= BITNOT expr", + /* 211 */ "expr ::= PLUS|MINUS expr", + /* 212 */ "between_op ::= BETWEEN", + /* 213 */ "between_op ::= NOT BETWEEN", + /* 214 */ "expr ::= expr between_op expr AND expr", + /* 215 */ "in_op ::= IN", + /* 216 */ "in_op ::= NOT IN", + /* 217 */ "expr ::= expr in_op LP exprlist RP", + /* 218 */ "expr ::= LP select RP", + /* 219 */ "expr ::= expr in_op LP select RP", + /* 220 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 221 */ "expr ::= EXISTS LP select RP", + /* 222 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 223 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 224 */ "case_exprlist ::= WHEN expr THEN expr", + /* 225 */ "case_else ::= ELSE expr", + /* 226 */ "case_else ::=", + /* 227 */ "case_operand ::= expr", + /* 228 */ "case_operand ::=", + /* 229 */ "exprlist ::=", + /* 230 */ "nexprlist ::= nexprlist COMMA expr", + /* 231 */ "nexprlist ::= expr", + /* 232 */ "paren_exprlist ::=", + /* 233 */ "paren_exprlist ::= LP exprlist RP", + /* 234 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 235 */ "uniqueflag ::= UNIQUE", + /* 236 */ "uniqueflag ::=", + /* 237 */ "eidlist_opt ::=", + /* 238 */ "eidlist_opt ::= LP eidlist RP", + /* 239 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 240 */ "eidlist ::= nm collate sortorder", + /* 241 */ "collate ::=", + /* 242 */ "collate ::= COLLATE ID|STRING", + /* 243 */ "cmd ::= DROP INDEX ifexists fullname", + /* 244 */ "cmd ::= VACUUM vinto", + /* 245 */ "cmd ::= VACUUM nm vinto", + /* 246 */ "vinto ::= INTO expr", + /* 247 */ "vinto ::=", + /* 248 */ "cmd ::= PRAGMA nm dbnm", + /* 249 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 250 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 251 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 252 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 253 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 254 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 255 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 256 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 257 */ "trigger_time ::= BEFORE|AFTER", + /* 258 */ "trigger_time ::= INSTEAD OF", + /* 259 */ "trigger_time ::=", + /* 260 */ "trigger_event ::= DELETE|INSERT", + /* 261 */ "trigger_event ::= UPDATE", + /* 262 */ "trigger_event ::= UPDATE OF idlist", + /* 263 */ "when_clause ::=", + /* 264 */ "when_clause ::= WHEN expr", + /* 265 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 266 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 267 */ "trnm ::= nm DOT nm", + /* 268 */ "tridxby ::= INDEXED BY nm", + /* 269 */ "tridxby ::= NOT INDEXED", + /* 270 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", + /* 271 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 272 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 273 */ "trigger_cmd ::= scanpt select scanpt", + /* 274 */ "expr ::= RAISE LP IGNORE RP", + /* 275 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 276 */ "raisetype ::= ROLLBACK", + /* 277 */ "raisetype ::= ABORT", + /* 278 */ "raisetype ::= FAIL", + /* 279 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 280 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 281 */ "cmd ::= DETACH database_kw_opt expr", + /* 282 */ "key_opt ::=", + /* 283 */ "key_opt ::= KEY expr", + /* 284 */ "cmd ::= REINDEX", + /* 285 */ "cmd ::= REINDEX nm dbnm", + /* 286 */ "cmd ::= ANALYZE", + /* 287 */ "cmd ::= ANALYZE nm dbnm", + /* 288 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 289 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 290 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", + /* 291 */ "add_column_fullname ::= fullname", + /* 292 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 293 */ "cmd ::= create_vtab", + /* 294 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 295 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 296 */ "vtabarg ::=", + /* 297 */ "vtabargtoken ::= ANY", + /* 298 */ "vtabargtoken ::= lp anylist RP", + /* 299 */ "lp ::= LP", + /* 300 */ "with ::= WITH wqlist", + /* 301 */ "with ::= WITH RECURSIVE wqlist", + /* 302 */ "wqas ::= AS", + /* 303 */ "wqas ::= AS MATERIALIZED", + /* 304 */ "wqas ::= AS NOT MATERIALIZED", + /* 305 */ "wqitem ::= nm eidlist_opt wqas LP select RP", + /* 306 */ "wqlist ::= wqitem", + /* 307 */ "wqlist ::= wqlist COMMA wqitem", + /* 308 */ "windowdefn_list ::= windowdefn", + /* 309 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 310 */ "windowdefn ::= nm AS LP window RP", + /* 311 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 312 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 313 */ "window ::= ORDER BY sortlist frame_opt", + /* 314 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 315 */ "window ::= frame_opt", + /* 316 */ "window ::= nm frame_opt", + /* 317 */ "frame_opt ::=", + /* 318 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 319 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 320 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 321 */ "frame_bound_s ::= frame_bound", + /* 322 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 323 */ "frame_bound_e ::= frame_bound", + /* 324 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 325 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 326 */ "frame_bound ::= CURRENT ROW", + /* 327 */ "frame_exclude_opt ::=", + /* 328 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 329 */ "frame_exclude ::= NO OTHERS", + /* 330 */ "frame_exclude ::= CURRENT ROW", + /* 331 */ "frame_exclude ::= GROUP|TIES", + /* 332 */ "window_clause ::= WINDOW windowdefn_list", + /* 333 */ "filter_over ::= filter_clause over_clause", + /* 334 */ "filter_over ::= over_clause", + /* 335 */ "filter_over ::= filter_clause", + /* 336 */ "over_clause ::= OVER LP window RP", + /* 337 */ "over_clause ::= OVER nm", + /* 338 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 339 */ "input ::= cmdlist", + /* 340 */ "cmdlist ::= cmdlist ecmd", + /* 341 */ "cmdlist ::= ecmd", + /* 342 */ "ecmd ::= SEMI", + /* 343 */ "ecmd ::= cmdx SEMI", + /* 344 */ "ecmd ::= explain cmdx SEMI", + /* 345 */ "trans_opt ::=", + /* 346 */ "trans_opt ::= TRANSACTION", + /* 347 */ "trans_opt ::= TRANSACTION nm", + /* 348 */ "savepoint_opt ::= SAVEPOINT", + /* 349 */ "savepoint_opt ::=", + /* 350 */ "cmd ::= create_table create_table_args", + /* 351 */ "table_option_set ::= table_option", + /* 352 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 353 */ "columnlist ::= columnname carglist", + /* 354 */ "nm ::= ID|INDEXED", + /* 355 */ "nm ::= STRING", + /* 356 */ "nm ::= JOIN_KW", + /* 357 */ "typetoken ::= typename", + /* 358 */ "typename ::= ID|STRING", + /* 359 */ "signed ::= plus_num", + /* 360 */ "signed ::= minus_num", + /* 361 */ "carglist ::= carglist ccons", + /* 362 */ "carglist ::=", + /* 363 */ "ccons ::= NULL onconf", + /* 364 */ "ccons ::= GENERATED ALWAYS AS generated", + /* 365 */ "ccons ::= AS generated", + /* 366 */ "conslist_opt ::= COMMA conslist", + /* 367 */ "conslist ::= conslist tconscomma tcons", + /* 368 */ "conslist ::= tcons", + /* 369 */ "tconscomma ::=", + /* 370 */ "defer_subclause_opt ::= defer_subclause", + /* 371 */ "resolvetype ::= raisetype", + /* 372 */ "selectnowith ::= oneselect", + /* 373 */ "oneselect ::= values", + /* 374 */ "sclp ::= selcollist COMMA", + /* 375 */ "as ::= ID|STRING", + /* 376 */ "returning ::=", + /* 377 */ "expr ::= term", + /* 378 */ "likeop ::= LIKE_KW|MATCH", + /* 379 */ "exprlist ::= nexprlist", + /* 380 */ "nmnum ::= plus_num", + /* 381 */ "nmnum ::= nm", + /* 382 */ "nmnum ::= ON", + /* 383 */ "nmnum ::= DELETE", + /* 384 */ "nmnum ::= DEFAULT", + /* 385 */ "plus_num ::= INTEGER|FLOAT", + /* 386 */ "foreach_clause ::=", + /* 387 */ "foreach_clause ::= FOR EACH ROW", + /* 388 */ "trnm ::= nm", + /* 389 */ "tridxby ::=", + /* 390 */ "database_kw_opt ::= DATABASE", + /* 391 */ "database_kw_opt ::=", + /* 392 */ "kwcolumn_opt ::=", + /* 393 */ "kwcolumn_opt ::= COLUMNKW", + /* 394 */ "vtabarglist ::= vtabarg", + /* 395 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 396 */ "vtabarg ::= vtabarg vtabargtoken", + /* 397 */ "anylist ::=", + /* 398 */ "anylist ::= anylist LP anylist RP", + /* 399 */ "anylist ::= anylist ANY", + /* 400 */ "with ::=", +}; +#endif /* NDEBUG */ + + +#if YYSTACKDEPTH<=0 +/* +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. +*/ +static int yyGrowStack(yyParser *p){ + int newSize; + int idx; + yyStackEntry *pNew; + + newSize = p->yystksz*2 + 100; + idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; + if( p->yystack==&p->yystk0 ){ + pNew = malloc(newSize*sizeof(pNew[0])); + if( pNew ) pNew[0] = p->yystk0; + }else{ + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + } + if( pNew ){ + p->yystack = pNew; + p->yytos = &p->yystack[idx]; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, p->yystksz, newSize); + } +#endif + p->yystksz = newSize; + } + return pNew==0; +} +#endif + +/* Datatype of the argument to the memory allocated passed as the +** second argument to sqlite3ParserAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + +/* Initialize a new parser that has already been allocated. +*/ +SQLITE_PRIVATE void sqlite3ParserInit(void *yypRawParser sqlite3ParserCTX_PDECL){ + yyParser *yypParser = (yyParser*)yypRawParser; + sqlite3ParserCTX_STORE +#ifdef YYTRACKMAXSTACKDEPTH + yypParser->yyhwm = 0; +#endif +#if YYSTACKDEPTH<=0 + yypParser->yytos = NULL; + yypParser->yystack = NULL; + yypParser->yystksz = 0; + if( yyGrowStack(yypParser) ){ + yypParser->yystack = &yypParser->yystk0; + yypParser->yystksz = 1; + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yypParser->yytos = yypParser->yystack; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; +#if YYSTACKDEPTH>0 + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; +#endif +} + +#ifndef sqlite3Parser_ENGINEALWAYSONSTACK +/* +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to sqlite3Parser and sqlite3ParserFree. +*/ +SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) sqlite3ParserCTX_PDECL){ + yyParser *yypParser; + yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( yypParser ){ + sqlite3ParserCTX_STORE + sqlite3ParserInit(yypParser sqlite3ParserCTX_PARAM); + } + return (void*)yypParser; +} +#endif /* sqlite3Parser_ENGINEALWAYSONSTACK */ + + +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. +*/ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH + switch( yymajor ){ + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are *not* used + ** inside the C code. + */ +/********* Begin destructor definitions ***************************************/ + case 203: /* select */ + case 238: /* selectnowith */ + case 239: /* oneselect */ + case 251: /* values */ +{ +sqlite3SelectDelete(pParse->db, (yypminor->yy303)); +} + break; + case 215: /* term */ + case 216: /* expr */ + case 245: /* where_opt */ + case 247: /* having_opt */ + case 259: /* on_opt */ + case 266: /* where_opt_ret */ + case 277: /* case_operand */ + case 279: /* case_else */ + case 282: /* vinto */ + case 289: /* when_clause */ + case 294: /* key_opt */ + case 310: /* filter_clause */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy626)); +} + break; + case 220: /* eidlist_opt */ + case 230: /* sortlist */ + case 231: /* eidlist */ + case 243: /* selcollist */ + case 246: /* groupby_opt */ + case 248: /* orderby_opt */ + case 252: /* nexprlist */ + case 253: /* sclp */ + case 261: /* exprlist */ + case 267: /* setlist */ + case 276: /* paren_exprlist */ + case 278: /* case_exprlist */ + case 309: /* part_opt */ +{ +sqlite3ExprListDelete(pParse->db, (yypminor->yy562)); +} + break; + case 237: /* fullname */ + case 244: /* from */ + case 255: /* seltablist */ + case 256: /* stl_prefix */ + case 262: /* xfullname */ +{ +sqlite3SrcListDelete(pParse->db, (yypminor->yy607)); +} + break; + case 240: /* wqlist */ +{ +sqlite3WithDelete(pParse->db, (yypminor->yy43)); +} + break; + case 250: /* window_clause */ + case 305: /* windowdefn_list */ +{ +sqlite3WindowListDelete(pParse->db, (yypminor->yy375)); +} + break; + case 260: /* using_opt */ + case 263: /* idlist */ + case 269: /* idlist_opt */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy240)); +} + break; + case 272: /* filter_over */ + case 306: /* windowdefn */ + case 307: /* window */ + case 308: /* frame_opt */ + case 311: /* over_clause */ +{ +sqlite3WindowDelete(pParse->db, (yypminor->yy375)); +} + break; + case 285: /* trigger_cmd_list */ + case 290: /* trigger_cmd */ +{ +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy95)); +} + break; + case 287: /* trigger_event */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy570).b); +} + break; + case 313: /* frame_bound */ + case 314: /* frame_bound_s */ + case 315: /* frame_bound_e */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy81).pExpr); +} + break; +/********* End destructor definitions *****************************************/ + default: break; /* If no destructor action specified: do nothing */ + } +} + +/* +** Pop the parser's stack once. +** +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. +*/ +static void yy_pop_parser_stack(yyParser *pParser){ + yyStackEntry *yytos; + assert( pParser->yytos!=0 ); + assert( pParser->yytos > pParser->yystack ); + yytos = pParser->yytos--; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + yy_destructor(pParser, yytos->major, &yytos->minor); +} + +/* +** Clear all secondary memory allocations from the parser +*/ +SQLITE_PRIVATE void sqlite3ParserFinalize(void *p){ + yyParser *pParser = (yyParser*)p; + while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); +#endif +} + +#ifndef sqlite3Parser_ENGINEALWAYSONSTACK +/* +** Deallocate and destroy a parser. Destructors are called for +** all stack elements before shutting the parser down. +** +** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. +*/ +SQLITE_PRIVATE void sqlite3ParserFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void*) /* Function used to reclaim memory */ +){ +#ifndef YYPARSEFREENEVERNULL + if( p==0 ) return; +#endif + sqlite3ParserFinalize(p); + (*freeProc)(p); +} +#endif /* sqlite3Parser_ENGINEALWAYSONSTACK */ + +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef YYTRACKMAXSTACKDEPTH +SQLITE_PRIVATE int sqlite3ParserStackPeak(void *p){ + yyParser *pParser = (yyParser*)p; + return pParser->yyhwm; +} +#endif + +/* This array of booleans keeps track of the parser statement +** coverage. The element yycoverage[X][Y] is set when the parser +** is in state X and has a lookahead token Y. In a well-tested +** systems, every element of this matrix should end up being set. +*/ +#if defined(YYCOVERAGE) +static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; +#endif + +/* +** Write into out a description of every state/lookahead combination that +** +** (1) has not been used by the parser, and +** (2) is not a syntax error. +** +** Return the number of missed state/lookahead combinations. +*/ +#if defined(YYCOVERAGE) +SQLITE_PRIVATE int sqlite3ParserCoverage(FILE *out){ + int stateno, iLookAhead, i; + int nMissed = 0; + for(stateno=0; statenoYY_MAX_SHIFT ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); +#if defined(YYCOVERAGE) + yycoverage[stateno][iLookAhead] = 1; +#endif + do{ + i = yy_shift_ofst[stateno]; + assert( i>=0 ); + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); + assert( iLookAhead!=YYNOCODE ); + assert( iLookAhead < YYNTOKEN ); + i += iLookAhead; + assert( i<(int)YY_NLOOKAHEAD ); + if( yy_lookahead[i]!=iLookAhead ){ +#ifdef YYFALLBACK + YYCODETYPE iFallback; /* Fallback token */ + assert( iLookAhead %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } +#endif +#ifdef YYWILDCARD + { + int j = i - iLookAhead + YYWILDCARD; + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); + } +#endif /* NDEBUG */ + return yy_action[j]; + } + } +#endif /* YYWILDCARD */ + return yy_default[stateno]; + }else{ + assert( i>=0 && i<(int)(sizeof(yy_action)/sizeof(yy_action[0])) ); + return yy_action[i]; + } + }while(1); +} + +/* +** Find the appropriate action for a parser given the non-terminal +** look-ahead token iLookAhead. +*/ +static YYACTIONTYPE yy_find_reduce_action( + YYACTIONTYPE stateno, /* Current state number */ + YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; +#ifdef YYERRORSYMBOL + if( stateno>YY_REDUCE_COUNT ){ + return yy_default[stateno]; + } +#else + assert( stateno<=YY_REDUCE_COUNT ); +#endif + i = yy_reduce_ofst[stateno]; + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; +#ifdef YYERRORSYMBOL + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + return yy_default[stateno]; + } +#else + assert( i>=0 && iyytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ + + sqlite3ErrorMsg(pParse, "parser stack overflow"); +/******** End %stack_overflow code ********************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument var */ + sqlite3ParserCTX_STORE +} + +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ + if( yyTraceFILE ){ + if( yyNewStateyytos->major], + yyNewState); + }else{ + fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], + yyNewState - YY_MIN_REDUCE); + } + } +} +#else +# define yyTraceShift(X,Y,Z) +#endif + +/* +** Perform a shift action. +*/ +static void yy_shift( + yyParser *yypParser, /* The parser to be shifted */ + YYACTIONTYPE yyNewState, /* The new state to shift in */ + YYCODETYPE yyMajor, /* The major token to shift in */ + sqlite3ParserTOKENTYPE yyMinor /* The minor token to shift in */ +){ + yyStackEntry *yytos; + yypParser->yytos++; +#ifdef YYTRACKMAXSTACKDEPTH + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>yypParser->yystackEnd ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + if( yyGrowStack(yypParser) ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } + } +#endif + if( yyNewState > YY_MAX_SHIFT ){ + yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + } + yytos = yypParser->yytos; + yytos->stateno = yyNewState; + yytos->major = yyMajor; + yytos->minor.yy0 = yyMinor; + yyTraceShift(yypParser, yyNewState, "Shift"); +} + +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { + 188, /* (0) explain ::= EXPLAIN */ + 188, /* (1) explain ::= EXPLAIN QUERY PLAN */ + 187, /* (2) cmdx ::= cmd */ + 189, /* (3) cmd ::= BEGIN transtype trans_opt */ + 190, /* (4) transtype ::= */ + 190, /* (5) transtype ::= DEFERRED */ + 190, /* (6) transtype ::= IMMEDIATE */ + 190, /* (7) transtype ::= EXCLUSIVE */ + 189, /* (8) cmd ::= COMMIT|END trans_opt */ + 189, /* (9) cmd ::= ROLLBACK trans_opt */ + 189, /* (10) cmd ::= SAVEPOINT nm */ + 189, /* (11) cmd ::= RELEASE savepoint_opt nm */ + 189, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + 194, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + 196, /* (14) createkw ::= CREATE */ + 198, /* (15) ifnotexists ::= */ + 198, /* (16) ifnotexists ::= IF NOT EXISTS */ + 197, /* (17) temp ::= TEMP */ + 197, /* (18) temp ::= */ + 195, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ + 195, /* (20) create_table_args ::= AS select */ + 202, /* (21) table_option_set ::= */ + 202, /* (22) table_option_set ::= table_option_set COMMA table_option */ + 204, /* (23) table_option ::= WITHOUT nm */ + 204, /* (24) table_option ::= nm */ + 205, /* (25) columnname ::= nm typetoken */ + 207, /* (26) typetoken ::= */ + 207, /* (27) typetoken ::= typename LP signed RP */ + 207, /* (28) typetoken ::= typename LP signed COMMA signed RP */ + 208, /* (29) typename ::= typename ID|STRING */ + 212, /* (30) scanpt ::= */ + 213, /* (31) scantok ::= */ + 214, /* (32) ccons ::= CONSTRAINT nm */ + 214, /* (33) ccons ::= DEFAULT scantok term */ + 214, /* (34) ccons ::= DEFAULT LP expr RP */ + 214, /* (35) ccons ::= DEFAULT PLUS scantok term */ + 214, /* (36) ccons ::= DEFAULT MINUS scantok term */ + 214, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ + 214, /* (38) ccons ::= NOT NULL onconf */ + 214, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + 214, /* (40) ccons ::= UNIQUE onconf */ + 214, /* (41) ccons ::= CHECK LP expr RP */ + 214, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ + 214, /* (43) ccons ::= defer_subclause */ + 214, /* (44) ccons ::= COLLATE ID|STRING */ + 223, /* (45) generated ::= LP expr RP */ + 223, /* (46) generated ::= LP expr RP ID */ + 219, /* (47) autoinc ::= */ + 219, /* (48) autoinc ::= AUTOINCR */ + 221, /* (49) refargs ::= */ + 221, /* (50) refargs ::= refargs refarg */ + 224, /* (51) refarg ::= MATCH nm */ + 224, /* (52) refarg ::= ON INSERT refact */ + 224, /* (53) refarg ::= ON DELETE refact */ + 224, /* (54) refarg ::= ON UPDATE refact */ + 225, /* (55) refact ::= SET NULL */ + 225, /* (56) refact ::= SET DEFAULT */ + 225, /* (57) refact ::= CASCADE */ + 225, /* (58) refact ::= RESTRICT */ + 225, /* (59) refact ::= NO ACTION */ + 222, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + 222, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 226, /* (62) init_deferred_pred_opt ::= */ + 226, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + 226, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 201, /* (65) conslist_opt ::= */ + 228, /* (66) tconscomma ::= COMMA */ + 229, /* (67) tcons ::= CONSTRAINT nm */ + 229, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + 229, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ + 229, /* (70) tcons ::= CHECK LP expr RP onconf */ + 229, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 232, /* (72) defer_subclause_opt ::= */ + 217, /* (73) onconf ::= */ + 217, /* (74) onconf ::= ON CONFLICT resolvetype */ + 233, /* (75) orconf ::= */ + 233, /* (76) orconf ::= OR resolvetype */ + 234, /* (77) resolvetype ::= IGNORE */ + 234, /* (78) resolvetype ::= REPLACE */ + 189, /* (79) cmd ::= DROP TABLE ifexists fullname */ + 236, /* (80) ifexists ::= IF EXISTS */ + 236, /* (81) ifexists ::= */ + 189, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + 189, /* (83) cmd ::= DROP VIEW ifexists fullname */ + 189, /* (84) cmd ::= select */ + 203, /* (85) select ::= WITH wqlist selectnowith */ + 203, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ + 203, /* (87) select ::= selectnowith */ + 238, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ + 241, /* (89) multiselect_op ::= UNION */ + 241, /* (90) multiselect_op ::= UNION ALL */ + 241, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ + 239, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + 239, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + 251, /* (94) values ::= VALUES LP nexprlist RP */ + 251, /* (95) values ::= values COMMA LP nexprlist RP */ + 242, /* (96) distinct ::= DISTINCT */ + 242, /* (97) distinct ::= ALL */ + 242, /* (98) distinct ::= */ + 253, /* (99) sclp ::= */ + 243, /* (100) selcollist ::= sclp scanpt expr scanpt as */ + 243, /* (101) selcollist ::= sclp scanpt STAR */ + 243, /* (102) selcollist ::= sclp scanpt nm DOT STAR */ + 254, /* (103) as ::= AS nm */ + 254, /* (104) as ::= */ + 244, /* (105) from ::= */ + 244, /* (106) from ::= FROM seltablist */ + 256, /* (107) stl_prefix ::= seltablist joinop */ + 256, /* (108) stl_prefix ::= */ + 255, /* (109) seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + 255, /* (110) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ + 255, /* (111) seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + 255, /* (112) seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + 199, /* (113) dbnm ::= */ + 199, /* (114) dbnm ::= DOT nm */ + 237, /* (115) fullname ::= nm */ + 237, /* (116) fullname ::= nm DOT nm */ + 262, /* (117) xfullname ::= nm */ + 262, /* (118) xfullname ::= nm DOT nm */ + 262, /* (119) xfullname ::= nm DOT nm AS nm */ + 262, /* (120) xfullname ::= nm AS nm */ + 257, /* (121) joinop ::= COMMA|JOIN */ + 257, /* (122) joinop ::= JOIN_KW JOIN */ + 257, /* (123) joinop ::= JOIN_KW nm JOIN */ + 257, /* (124) joinop ::= JOIN_KW nm nm JOIN */ + 259, /* (125) on_opt ::= ON expr */ + 259, /* (126) on_opt ::= */ + 258, /* (127) indexed_opt ::= */ + 258, /* (128) indexed_opt ::= INDEXED BY nm */ + 258, /* (129) indexed_opt ::= NOT INDEXED */ + 260, /* (130) using_opt ::= USING LP idlist RP */ + 260, /* (131) using_opt ::= */ + 248, /* (132) orderby_opt ::= */ + 248, /* (133) orderby_opt ::= ORDER BY sortlist */ + 230, /* (134) sortlist ::= sortlist COMMA expr sortorder nulls */ + 230, /* (135) sortlist ::= expr sortorder nulls */ + 218, /* (136) sortorder ::= ASC */ + 218, /* (137) sortorder ::= DESC */ + 218, /* (138) sortorder ::= */ + 264, /* (139) nulls ::= NULLS FIRST */ + 264, /* (140) nulls ::= NULLS LAST */ + 264, /* (141) nulls ::= */ + 246, /* (142) groupby_opt ::= */ + 246, /* (143) groupby_opt ::= GROUP BY nexprlist */ + 247, /* (144) having_opt ::= */ + 247, /* (145) having_opt ::= HAVING expr */ + 249, /* (146) limit_opt ::= */ + 249, /* (147) limit_opt ::= LIMIT expr */ + 249, /* (148) limit_opt ::= LIMIT expr OFFSET expr */ + 249, /* (149) limit_opt ::= LIMIT expr COMMA expr */ + 189, /* (150) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 245, /* (151) where_opt ::= */ + 245, /* (152) where_opt ::= WHERE expr */ + 266, /* (153) where_opt_ret ::= */ + 266, /* (154) where_opt_ret ::= WHERE expr */ + 266, /* (155) where_opt_ret ::= RETURNING selcollist */ + 266, /* (156) where_opt_ret ::= WHERE expr RETURNING selcollist */ + 189, /* (157) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + 267, /* (158) setlist ::= setlist COMMA nm EQ expr */ + 267, /* (159) setlist ::= setlist COMMA LP idlist RP EQ expr */ + 267, /* (160) setlist ::= nm EQ expr */ + 267, /* (161) setlist ::= LP idlist RP EQ expr */ + 189, /* (162) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + 189, /* (163) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 270, /* (164) upsert ::= */ + 270, /* (165) upsert ::= RETURNING selcollist */ + 270, /* (166) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + 270, /* (167) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + 270, /* (168) upsert ::= ON CONFLICT DO NOTHING returning */ + 270, /* (169) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + 271, /* (170) returning ::= RETURNING selcollist */ + 268, /* (171) insert_cmd ::= INSERT orconf */ + 268, /* (172) insert_cmd ::= REPLACE */ + 269, /* (173) idlist_opt ::= */ + 269, /* (174) idlist_opt ::= LP idlist RP */ + 263, /* (175) idlist ::= idlist COMMA nm */ + 263, /* (176) idlist ::= nm */ + 216, /* (177) expr ::= LP expr RP */ + 216, /* (178) expr ::= ID|INDEXED */ + 216, /* (179) expr ::= JOIN_KW */ + 216, /* (180) expr ::= nm DOT nm */ + 216, /* (181) expr ::= nm DOT nm DOT nm */ + 215, /* (182) term ::= NULL|FLOAT|BLOB */ + 215, /* (183) term ::= STRING */ + 215, /* (184) term ::= INTEGER */ + 216, /* (185) expr ::= VARIABLE */ + 216, /* (186) expr ::= expr COLLATE ID|STRING */ + 216, /* (187) expr ::= CAST LP expr AS typetoken RP */ + 216, /* (188) expr ::= ID|INDEXED LP distinct exprlist RP */ + 216, /* (189) expr ::= ID|INDEXED LP STAR RP */ + 216, /* (190) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ + 216, /* (191) expr ::= ID|INDEXED LP STAR RP filter_over */ + 215, /* (192) term ::= CTIME_KW */ + 216, /* (193) expr ::= LP nexprlist COMMA expr RP */ + 216, /* (194) expr ::= expr AND expr */ + 216, /* (195) expr ::= expr OR expr */ + 216, /* (196) expr ::= expr LT|GT|GE|LE expr */ + 216, /* (197) expr ::= expr EQ|NE expr */ + 216, /* (198) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + 216, /* (199) expr ::= expr PLUS|MINUS expr */ + 216, /* (200) expr ::= expr STAR|SLASH|REM expr */ + 216, /* (201) expr ::= expr CONCAT expr */ + 273, /* (202) likeop ::= NOT LIKE_KW|MATCH */ + 216, /* (203) expr ::= expr likeop expr */ + 216, /* (204) expr ::= expr likeop expr ESCAPE expr */ + 216, /* (205) expr ::= expr ISNULL|NOTNULL */ + 216, /* (206) expr ::= expr NOT NULL */ + 216, /* (207) expr ::= expr IS expr */ + 216, /* (208) expr ::= expr IS NOT expr */ + 216, /* (209) expr ::= NOT expr */ + 216, /* (210) expr ::= BITNOT expr */ + 216, /* (211) expr ::= PLUS|MINUS expr */ + 274, /* (212) between_op ::= BETWEEN */ + 274, /* (213) between_op ::= NOT BETWEEN */ + 216, /* (214) expr ::= expr between_op expr AND expr */ + 275, /* (215) in_op ::= IN */ + 275, /* (216) in_op ::= NOT IN */ + 216, /* (217) expr ::= expr in_op LP exprlist RP */ + 216, /* (218) expr ::= LP select RP */ + 216, /* (219) expr ::= expr in_op LP select RP */ + 216, /* (220) expr ::= expr in_op nm dbnm paren_exprlist */ + 216, /* (221) expr ::= EXISTS LP select RP */ + 216, /* (222) expr ::= CASE case_operand case_exprlist case_else END */ + 278, /* (223) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + 278, /* (224) case_exprlist ::= WHEN expr THEN expr */ + 279, /* (225) case_else ::= ELSE expr */ + 279, /* (226) case_else ::= */ + 277, /* (227) case_operand ::= expr */ + 277, /* (228) case_operand ::= */ + 261, /* (229) exprlist ::= */ + 252, /* (230) nexprlist ::= nexprlist COMMA expr */ + 252, /* (231) nexprlist ::= expr */ + 276, /* (232) paren_exprlist ::= */ + 276, /* (233) paren_exprlist ::= LP exprlist RP */ + 189, /* (234) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + 280, /* (235) uniqueflag ::= UNIQUE */ + 280, /* (236) uniqueflag ::= */ + 220, /* (237) eidlist_opt ::= */ + 220, /* (238) eidlist_opt ::= LP eidlist RP */ + 231, /* (239) eidlist ::= eidlist COMMA nm collate sortorder */ + 231, /* (240) eidlist ::= nm collate sortorder */ + 281, /* (241) collate ::= */ + 281, /* (242) collate ::= COLLATE ID|STRING */ + 189, /* (243) cmd ::= DROP INDEX ifexists fullname */ + 189, /* (244) cmd ::= VACUUM vinto */ + 189, /* (245) cmd ::= VACUUM nm vinto */ + 282, /* (246) vinto ::= INTO expr */ + 282, /* (247) vinto ::= */ + 189, /* (248) cmd ::= PRAGMA nm dbnm */ + 189, /* (249) cmd ::= PRAGMA nm dbnm EQ nmnum */ + 189, /* (250) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + 189, /* (251) cmd ::= PRAGMA nm dbnm EQ minus_num */ + 189, /* (252) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + 210, /* (253) plus_num ::= PLUS INTEGER|FLOAT */ + 211, /* (254) minus_num ::= MINUS INTEGER|FLOAT */ + 189, /* (255) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + 284, /* (256) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + 286, /* (257) trigger_time ::= BEFORE|AFTER */ + 286, /* (258) trigger_time ::= INSTEAD OF */ + 286, /* (259) trigger_time ::= */ + 287, /* (260) trigger_event ::= DELETE|INSERT */ + 287, /* (261) trigger_event ::= UPDATE */ + 287, /* (262) trigger_event ::= UPDATE OF idlist */ + 289, /* (263) when_clause ::= */ + 289, /* (264) when_clause ::= WHEN expr */ + 285, /* (265) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + 285, /* (266) trigger_cmd_list ::= trigger_cmd SEMI */ + 291, /* (267) trnm ::= nm DOT nm */ + 292, /* (268) tridxby ::= INDEXED BY nm */ + 292, /* (269) tridxby ::= NOT INDEXED */ + 290, /* (270) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + 290, /* (271) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + 290, /* (272) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + 290, /* (273) trigger_cmd ::= scanpt select scanpt */ + 216, /* (274) expr ::= RAISE LP IGNORE RP */ + 216, /* (275) expr ::= RAISE LP raisetype COMMA nm RP */ + 235, /* (276) raisetype ::= ROLLBACK */ + 235, /* (277) raisetype ::= ABORT */ + 235, /* (278) raisetype ::= FAIL */ + 189, /* (279) cmd ::= DROP TRIGGER ifexists fullname */ + 189, /* (280) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 189, /* (281) cmd ::= DETACH database_kw_opt expr */ + 294, /* (282) key_opt ::= */ + 294, /* (283) key_opt ::= KEY expr */ + 189, /* (284) cmd ::= REINDEX */ + 189, /* (285) cmd ::= REINDEX nm dbnm */ + 189, /* (286) cmd ::= ANALYZE */ + 189, /* (287) cmd ::= ANALYZE nm dbnm */ + 189, /* (288) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 189, /* (289) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 189, /* (290) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + 295, /* (291) add_column_fullname ::= fullname */ + 189, /* (292) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 189, /* (293) cmd ::= create_vtab */ + 189, /* (294) cmd ::= create_vtab LP vtabarglist RP */ + 297, /* (295) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 299, /* (296) vtabarg ::= */ + 300, /* (297) vtabargtoken ::= ANY */ + 300, /* (298) vtabargtoken ::= lp anylist RP */ + 301, /* (299) lp ::= LP */ + 265, /* (300) with ::= WITH wqlist */ + 265, /* (301) with ::= WITH RECURSIVE wqlist */ + 304, /* (302) wqas ::= AS */ + 304, /* (303) wqas ::= AS MATERIALIZED */ + 304, /* (304) wqas ::= AS NOT MATERIALIZED */ + 303, /* (305) wqitem ::= nm eidlist_opt wqas LP select RP */ + 240, /* (306) wqlist ::= wqitem */ + 240, /* (307) wqlist ::= wqlist COMMA wqitem */ + 305, /* (308) windowdefn_list ::= windowdefn */ + 305, /* (309) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 306, /* (310) windowdefn ::= nm AS LP window RP */ + 307, /* (311) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 307, /* (312) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 307, /* (313) window ::= ORDER BY sortlist frame_opt */ + 307, /* (314) window ::= nm ORDER BY sortlist frame_opt */ + 307, /* (315) window ::= frame_opt */ + 307, /* (316) window ::= nm frame_opt */ + 308, /* (317) frame_opt ::= */ + 308, /* (318) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + 308, /* (319) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + 312, /* (320) range_or_rows ::= RANGE|ROWS|GROUPS */ + 314, /* (321) frame_bound_s ::= frame_bound */ + 314, /* (322) frame_bound_s ::= UNBOUNDED PRECEDING */ + 315, /* (323) frame_bound_e ::= frame_bound */ + 315, /* (324) frame_bound_e ::= UNBOUNDED FOLLOWING */ + 313, /* (325) frame_bound ::= expr PRECEDING|FOLLOWING */ + 313, /* (326) frame_bound ::= CURRENT ROW */ + 316, /* (327) frame_exclude_opt ::= */ + 316, /* (328) frame_exclude_opt ::= EXCLUDE frame_exclude */ + 317, /* (329) frame_exclude ::= NO OTHERS */ + 317, /* (330) frame_exclude ::= CURRENT ROW */ + 317, /* (331) frame_exclude ::= GROUP|TIES */ + 250, /* (332) window_clause ::= WINDOW windowdefn_list */ + 272, /* (333) filter_over ::= filter_clause over_clause */ + 272, /* (334) filter_over ::= over_clause */ + 272, /* (335) filter_over ::= filter_clause */ + 311, /* (336) over_clause ::= OVER LP window RP */ + 311, /* (337) over_clause ::= OVER nm */ + 310, /* (338) filter_clause ::= FILTER LP WHERE expr RP */ + 184, /* (339) input ::= cmdlist */ + 185, /* (340) cmdlist ::= cmdlist ecmd */ + 185, /* (341) cmdlist ::= ecmd */ + 186, /* (342) ecmd ::= SEMI */ + 186, /* (343) ecmd ::= cmdx SEMI */ + 186, /* (344) ecmd ::= explain cmdx SEMI */ + 191, /* (345) trans_opt ::= */ + 191, /* (346) trans_opt ::= TRANSACTION */ + 191, /* (347) trans_opt ::= TRANSACTION nm */ + 193, /* (348) savepoint_opt ::= SAVEPOINT */ + 193, /* (349) savepoint_opt ::= */ + 189, /* (350) cmd ::= create_table create_table_args */ + 202, /* (351) table_option_set ::= table_option */ + 200, /* (352) columnlist ::= columnlist COMMA columnname carglist */ + 200, /* (353) columnlist ::= columnname carglist */ + 192, /* (354) nm ::= ID|INDEXED */ + 192, /* (355) nm ::= STRING */ + 192, /* (356) nm ::= JOIN_KW */ + 207, /* (357) typetoken ::= typename */ + 208, /* (358) typename ::= ID|STRING */ + 209, /* (359) signed ::= plus_num */ + 209, /* (360) signed ::= minus_num */ + 206, /* (361) carglist ::= carglist ccons */ + 206, /* (362) carglist ::= */ + 214, /* (363) ccons ::= NULL onconf */ + 214, /* (364) ccons ::= GENERATED ALWAYS AS generated */ + 214, /* (365) ccons ::= AS generated */ + 201, /* (366) conslist_opt ::= COMMA conslist */ + 227, /* (367) conslist ::= conslist tconscomma tcons */ + 227, /* (368) conslist ::= tcons */ + 228, /* (369) tconscomma ::= */ + 232, /* (370) defer_subclause_opt ::= defer_subclause */ + 234, /* (371) resolvetype ::= raisetype */ + 238, /* (372) selectnowith ::= oneselect */ + 239, /* (373) oneselect ::= values */ + 253, /* (374) sclp ::= selcollist COMMA */ + 254, /* (375) as ::= ID|STRING */ + 271, /* (376) returning ::= */ + 216, /* (377) expr ::= term */ + 273, /* (378) likeop ::= LIKE_KW|MATCH */ + 261, /* (379) exprlist ::= nexprlist */ + 283, /* (380) nmnum ::= plus_num */ + 283, /* (381) nmnum ::= nm */ + 283, /* (382) nmnum ::= ON */ + 283, /* (383) nmnum ::= DELETE */ + 283, /* (384) nmnum ::= DEFAULT */ + 210, /* (385) plus_num ::= INTEGER|FLOAT */ + 288, /* (386) foreach_clause ::= */ + 288, /* (387) foreach_clause ::= FOR EACH ROW */ + 291, /* (388) trnm ::= nm */ + 292, /* (389) tridxby ::= */ + 293, /* (390) database_kw_opt ::= DATABASE */ + 293, /* (391) database_kw_opt ::= */ + 296, /* (392) kwcolumn_opt ::= */ + 296, /* (393) kwcolumn_opt ::= COLUMNKW */ + 298, /* (394) vtabarglist ::= vtabarg */ + 298, /* (395) vtabarglist ::= vtabarglist COMMA vtabarg */ + 299, /* (396) vtabarg ::= vtabarg vtabargtoken */ + 302, /* (397) anylist ::= */ + 302, /* (398) anylist ::= anylist LP anylist RP */ + 302, /* (399) anylist ::= anylist ANY */ + 265, /* (400) with ::= */ +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { + -1, /* (0) explain ::= EXPLAIN */ + -3, /* (1) explain ::= EXPLAIN QUERY PLAN */ + -1, /* (2) cmdx ::= cmd */ + -3, /* (3) cmd ::= BEGIN transtype trans_opt */ + 0, /* (4) transtype ::= */ + -1, /* (5) transtype ::= DEFERRED */ + -1, /* (6) transtype ::= IMMEDIATE */ + -1, /* (7) transtype ::= EXCLUSIVE */ + -2, /* (8) cmd ::= COMMIT|END trans_opt */ + -2, /* (9) cmd ::= ROLLBACK trans_opt */ + -2, /* (10) cmd ::= SAVEPOINT nm */ + -3, /* (11) cmd ::= RELEASE savepoint_opt nm */ + -5, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + -6, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + -1, /* (14) createkw ::= CREATE */ + 0, /* (15) ifnotexists ::= */ + -3, /* (16) ifnotexists ::= IF NOT EXISTS */ + -1, /* (17) temp ::= TEMP */ + 0, /* (18) temp ::= */ + -5, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ + -2, /* (20) create_table_args ::= AS select */ + 0, /* (21) table_option_set ::= */ + -3, /* (22) table_option_set ::= table_option_set COMMA table_option */ + -2, /* (23) table_option ::= WITHOUT nm */ + -1, /* (24) table_option ::= nm */ + -2, /* (25) columnname ::= nm typetoken */ + 0, /* (26) typetoken ::= */ + -4, /* (27) typetoken ::= typename LP signed RP */ + -6, /* (28) typetoken ::= typename LP signed COMMA signed RP */ + -2, /* (29) typename ::= typename ID|STRING */ + 0, /* (30) scanpt ::= */ + 0, /* (31) scantok ::= */ + -2, /* (32) ccons ::= CONSTRAINT nm */ + -3, /* (33) ccons ::= DEFAULT scantok term */ + -4, /* (34) ccons ::= DEFAULT LP expr RP */ + -4, /* (35) ccons ::= DEFAULT PLUS scantok term */ + -4, /* (36) ccons ::= DEFAULT MINUS scantok term */ + -3, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ + -3, /* (38) ccons ::= NOT NULL onconf */ + -5, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + -2, /* (40) ccons ::= UNIQUE onconf */ + -4, /* (41) ccons ::= CHECK LP expr RP */ + -4, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ + -1, /* (43) ccons ::= defer_subclause */ + -2, /* (44) ccons ::= COLLATE ID|STRING */ + -3, /* (45) generated ::= LP expr RP */ + -4, /* (46) generated ::= LP expr RP ID */ + 0, /* (47) autoinc ::= */ + -1, /* (48) autoinc ::= AUTOINCR */ + 0, /* (49) refargs ::= */ + -2, /* (50) refargs ::= refargs refarg */ + -2, /* (51) refarg ::= MATCH nm */ + -3, /* (52) refarg ::= ON INSERT refact */ + -3, /* (53) refarg ::= ON DELETE refact */ + -3, /* (54) refarg ::= ON UPDATE refact */ + -2, /* (55) refact ::= SET NULL */ + -2, /* (56) refact ::= SET DEFAULT */ + -1, /* (57) refact ::= CASCADE */ + -1, /* (58) refact ::= RESTRICT */ + -2, /* (59) refact ::= NO ACTION */ + -3, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + -2, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 0, /* (62) init_deferred_pred_opt ::= */ + -2, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + -2, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 0, /* (65) conslist_opt ::= */ + -1, /* (66) tconscomma ::= COMMA */ + -2, /* (67) tcons ::= CONSTRAINT nm */ + -7, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + -5, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ + -5, /* (70) tcons ::= CHECK LP expr RP onconf */ + -10, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 0, /* (72) defer_subclause_opt ::= */ + 0, /* (73) onconf ::= */ + -3, /* (74) onconf ::= ON CONFLICT resolvetype */ + 0, /* (75) orconf ::= */ + -2, /* (76) orconf ::= OR resolvetype */ + -1, /* (77) resolvetype ::= IGNORE */ + -1, /* (78) resolvetype ::= REPLACE */ + -4, /* (79) cmd ::= DROP TABLE ifexists fullname */ + -2, /* (80) ifexists ::= IF EXISTS */ + 0, /* (81) ifexists ::= */ + -9, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + -4, /* (83) cmd ::= DROP VIEW ifexists fullname */ + -1, /* (84) cmd ::= select */ + -3, /* (85) select ::= WITH wqlist selectnowith */ + -4, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ + -1, /* (87) select ::= selectnowith */ + -3, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ + -1, /* (89) multiselect_op ::= UNION */ + -2, /* (90) multiselect_op ::= UNION ALL */ + -1, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ + -9, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + -10, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + -4, /* (94) values ::= VALUES LP nexprlist RP */ + -5, /* (95) values ::= values COMMA LP nexprlist RP */ + -1, /* (96) distinct ::= DISTINCT */ + -1, /* (97) distinct ::= ALL */ + 0, /* (98) distinct ::= */ + 0, /* (99) sclp ::= */ + -5, /* (100) selcollist ::= sclp scanpt expr scanpt as */ + -3, /* (101) selcollist ::= sclp scanpt STAR */ + -5, /* (102) selcollist ::= sclp scanpt nm DOT STAR */ + -2, /* (103) as ::= AS nm */ + 0, /* (104) as ::= */ + 0, /* (105) from ::= */ + -2, /* (106) from ::= FROM seltablist */ + -2, /* (107) stl_prefix ::= seltablist joinop */ + 0, /* (108) stl_prefix ::= */ + -7, /* (109) seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + -9, /* (110) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ + -7, /* (111) seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + -7, /* (112) seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + 0, /* (113) dbnm ::= */ + -2, /* (114) dbnm ::= DOT nm */ + -1, /* (115) fullname ::= nm */ + -3, /* (116) fullname ::= nm DOT nm */ + -1, /* (117) xfullname ::= nm */ + -3, /* (118) xfullname ::= nm DOT nm */ + -5, /* (119) xfullname ::= nm DOT nm AS nm */ + -3, /* (120) xfullname ::= nm AS nm */ + -1, /* (121) joinop ::= COMMA|JOIN */ + -2, /* (122) joinop ::= JOIN_KW JOIN */ + -3, /* (123) joinop ::= JOIN_KW nm JOIN */ + -4, /* (124) joinop ::= JOIN_KW nm nm JOIN */ + -2, /* (125) on_opt ::= ON expr */ + 0, /* (126) on_opt ::= */ + 0, /* (127) indexed_opt ::= */ + -3, /* (128) indexed_opt ::= INDEXED BY nm */ + -2, /* (129) indexed_opt ::= NOT INDEXED */ + -4, /* (130) using_opt ::= USING LP idlist RP */ + 0, /* (131) using_opt ::= */ + 0, /* (132) orderby_opt ::= */ + -3, /* (133) orderby_opt ::= ORDER BY sortlist */ + -5, /* (134) sortlist ::= sortlist COMMA expr sortorder nulls */ + -3, /* (135) sortlist ::= expr sortorder nulls */ + -1, /* (136) sortorder ::= ASC */ + -1, /* (137) sortorder ::= DESC */ + 0, /* (138) sortorder ::= */ + -2, /* (139) nulls ::= NULLS FIRST */ + -2, /* (140) nulls ::= NULLS LAST */ + 0, /* (141) nulls ::= */ + 0, /* (142) groupby_opt ::= */ + -3, /* (143) groupby_opt ::= GROUP BY nexprlist */ + 0, /* (144) having_opt ::= */ + -2, /* (145) having_opt ::= HAVING expr */ + 0, /* (146) limit_opt ::= */ + -2, /* (147) limit_opt ::= LIMIT expr */ + -4, /* (148) limit_opt ::= LIMIT expr OFFSET expr */ + -4, /* (149) limit_opt ::= LIMIT expr COMMA expr */ + -6, /* (150) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 0, /* (151) where_opt ::= */ + -2, /* (152) where_opt ::= WHERE expr */ + 0, /* (153) where_opt_ret ::= */ + -2, /* (154) where_opt_ret ::= WHERE expr */ + -2, /* (155) where_opt_ret ::= RETURNING selcollist */ + -4, /* (156) where_opt_ret ::= WHERE expr RETURNING selcollist */ + -9, /* (157) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + -5, /* (158) setlist ::= setlist COMMA nm EQ expr */ + -7, /* (159) setlist ::= setlist COMMA LP idlist RP EQ expr */ + -3, /* (160) setlist ::= nm EQ expr */ + -5, /* (161) setlist ::= LP idlist RP EQ expr */ + -7, /* (162) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + -8, /* (163) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 0, /* (164) upsert ::= */ + -2, /* (165) upsert ::= RETURNING selcollist */ + -12, /* (166) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + -9, /* (167) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + -5, /* (168) upsert ::= ON CONFLICT DO NOTHING returning */ + -8, /* (169) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + -2, /* (170) returning ::= RETURNING selcollist */ + -2, /* (171) insert_cmd ::= INSERT orconf */ + -1, /* (172) insert_cmd ::= REPLACE */ + 0, /* (173) idlist_opt ::= */ + -3, /* (174) idlist_opt ::= LP idlist RP */ + -3, /* (175) idlist ::= idlist COMMA nm */ + -1, /* (176) idlist ::= nm */ + -3, /* (177) expr ::= LP expr RP */ + -1, /* (178) expr ::= ID|INDEXED */ + -1, /* (179) expr ::= JOIN_KW */ + -3, /* (180) expr ::= nm DOT nm */ + -5, /* (181) expr ::= nm DOT nm DOT nm */ + -1, /* (182) term ::= NULL|FLOAT|BLOB */ + -1, /* (183) term ::= STRING */ + -1, /* (184) term ::= INTEGER */ + -1, /* (185) expr ::= VARIABLE */ + -3, /* (186) expr ::= expr COLLATE ID|STRING */ + -6, /* (187) expr ::= CAST LP expr AS typetoken RP */ + -5, /* (188) expr ::= ID|INDEXED LP distinct exprlist RP */ + -4, /* (189) expr ::= ID|INDEXED LP STAR RP */ + -6, /* (190) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ + -5, /* (191) expr ::= ID|INDEXED LP STAR RP filter_over */ + -1, /* (192) term ::= CTIME_KW */ + -5, /* (193) expr ::= LP nexprlist COMMA expr RP */ + -3, /* (194) expr ::= expr AND expr */ + -3, /* (195) expr ::= expr OR expr */ + -3, /* (196) expr ::= expr LT|GT|GE|LE expr */ + -3, /* (197) expr ::= expr EQ|NE expr */ + -3, /* (198) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + -3, /* (199) expr ::= expr PLUS|MINUS expr */ + -3, /* (200) expr ::= expr STAR|SLASH|REM expr */ + -3, /* (201) expr ::= expr CONCAT expr */ + -2, /* (202) likeop ::= NOT LIKE_KW|MATCH */ + -3, /* (203) expr ::= expr likeop expr */ + -5, /* (204) expr ::= expr likeop expr ESCAPE expr */ + -2, /* (205) expr ::= expr ISNULL|NOTNULL */ + -3, /* (206) expr ::= expr NOT NULL */ + -3, /* (207) expr ::= expr IS expr */ + -4, /* (208) expr ::= expr IS NOT expr */ + -2, /* (209) expr ::= NOT expr */ + -2, /* (210) expr ::= BITNOT expr */ + -2, /* (211) expr ::= PLUS|MINUS expr */ + -1, /* (212) between_op ::= BETWEEN */ + -2, /* (213) between_op ::= NOT BETWEEN */ + -5, /* (214) expr ::= expr between_op expr AND expr */ + -1, /* (215) in_op ::= IN */ + -2, /* (216) in_op ::= NOT IN */ + -5, /* (217) expr ::= expr in_op LP exprlist RP */ + -3, /* (218) expr ::= LP select RP */ + -5, /* (219) expr ::= expr in_op LP select RP */ + -5, /* (220) expr ::= expr in_op nm dbnm paren_exprlist */ + -4, /* (221) expr ::= EXISTS LP select RP */ + -5, /* (222) expr ::= CASE case_operand case_exprlist case_else END */ + -5, /* (223) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + -4, /* (224) case_exprlist ::= WHEN expr THEN expr */ + -2, /* (225) case_else ::= ELSE expr */ + 0, /* (226) case_else ::= */ + -1, /* (227) case_operand ::= expr */ + 0, /* (228) case_operand ::= */ + 0, /* (229) exprlist ::= */ + -3, /* (230) nexprlist ::= nexprlist COMMA expr */ + -1, /* (231) nexprlist ::= expr */ + 0, /* (232) paren_exprlist ::= */ + -3, /* (233) paren_exprlist ::= LP exprlist RP */ + -12, /* (234) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + -1, /* (235) uniqueflag ::= UNIQUE */ + 0, /* (236) uniqueflag ::= */ + 0, /* (237) eidlist_opt ::= */ + -3, /* (238) eidlist_opt ::= LP eidlist RP */ + -5, /* (239) eidlist ::= eidlist COMMA nm collate sortorder */ + -3, /* (240) eidlist ::= nm collate sortorder */ + 0, /* (241) collate ::= */ + -2, /* (242) collate ::= COLLATE ID|STRING */ + -4, /* (243) cmd ::= DROP INDEX ifexists fullname */ + -2, /* (244) cmd ::= VACUUM vinto */ + -3, /* (245) cmd ::= VACUUM nm vinto */ + -2, /* (246) vinto ::= INTO expr */ + 0, /* (247) vinto ::= */ + -3, /* (248) cmd ::= PRAGMA nm dbnm */ + -5, /* (249) cmd ::= PRAGMA nm dbnm EQ nmnum */ + -6, /* (250) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + -5, /* (251) cmd ::= PRAGMA nm dbnm EQ minus_num */ + -6, /* (252) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + -2, /* (253) plus_num ::= PLUS INTEGER|FLOAT */ + -2, /* (254) minus_num ::= MINUS INTEGER|FLOAT */ + -5, /* (255) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + -11, /* (256) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + -1, /* (257) trigger_time ::= BEFORE|AFTER */ + -2, /* (258) trigger_time ::= INSTEAD OF */ + 0, /* (259) trigger_time ::= */ + -1, /* (260) trigger_event ::= DELETE|INSERT */ + -1, /* (261) trigger_event ::= UPDATE */ + -3, /* (262) trigger_event ::= UPDATE OF idlist */ + 0, /* (263) when_clause ::= */ + -2, /* (264) when_clause ::= WHEN expr */ + -3, /* (265) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + -2, /* (266) trigger_cmd_list ::= trigger_cmd SEMI */ + -3, /* (267) trnm ::= nm DOT nm */ + -3, /* (268) tridxby ::= INDEXED BY nm */ + -2, /* (269) tridxby ::= NOT INDEXED */ + -9, /* (270) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + -8, /* (271) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + -6, /* (272) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + -3, /* (273) trigger_cmd ::= scanpt select scanpt */ + -4, /* (274) expr ::= RAISE LP IGNORE RP */ + -6, /* (275) expr ::= RAISE LP raisetype COMMA nm RP */ + -1, /* (276) raisetype ::= ROLLBACK */ + -1, /* (277) raisetype ::= ABORT */ + -1, /* (278) raisetype ::= FAIL */ + -4, /* (279) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (280) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (281) cmd ::= DETACH database_kw_opt expr */ + 0, /* (282) key_opt ::= */ + -2, /* (283) key_opt ::= KEY expr */ + -1, /* (284) cmd ::= REINDEX */ + -3, /* (285) cmd ::= REINDEX nm dbnm */ + -1, /* (286) cmd ::= ANALYZE */ + -3, /* (287) cmd ::= ANALYZE nm dbnm */ + -6, /* (288) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -7, /* (289) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -6, /* (290) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + -1, /* (291) add_column_fullname ::= fullname */ + -8, /* (292) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -1, /* (293) cmd ::= create_vtab */ + -4, /* (294) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (295) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (296) vtabarg ::= */ + -1, /* (297) vtabargtoken ::= ANY */ + -3, /* (298) vtabargtoken ::= lp anylist RP */ + -1, /* (299) lp ::= LP */ + -2, /* (300) with ::= WITH wqlist */ + -3, /* (301) with ::= WITH RECURSIVE wqlist */ + -1, /* (302) wqas ::= AS */ + -2, /* (303) wqas ::= AS MATERIALIZED */ + -3, /* (304) wqas ::= AS NOT MATERIALIZED */ + -6, /* (305) wqitem ::= nm eidlist_opt wqas LP select RP */ + -1, /* (306) wqlist ::= wqitem */ + -3, /* (307) wqlist ::= wqlist COMMA wqitem */ + -1, /* (308) windowdefn_list ::= windowdefn */ + -3, /* (309) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (310) windowdefn ::= nm AS LP window RP */ + -5, /* (311) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (312) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (313) window ::= ORDER BY sortlist frame_opt */ + -5, /* (314) window ::= nm ORDER BY sortlist frame_opt */ + -1, /* (315) window ::= frame_opt */ + -2, /* (316) window ::= nm frame_opt */ + 0, /* (317) frame_opt ::= */ + -3, /* (318) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + -6, /* (319) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + -1, /* (320) range_or_rows ::= RANGE|ROWS|GROUPS */ + -1, /* (321) frame_bound_s ::= frame_bound */ + -2, /* (322) frame_bound_s ::= UNBOUNDED PRECEDING */ + -1, /* (323) frame_bound_e ::= frame_bound */ + -2, /* (324) frame_bound_e ::= UNBOUNDED FOLLOWING */ + -2, /* (325) frame_bound ::= expr PRECEDING|FOLLOWING */ + -2, /* (326) frame_bound ::= CURRENT ROW */ + 0, /* (327) frame_exclude_opt ::= */ + -2, /* (328) frame_exclude_opt ::= EXCLUDE frame_exclude */ + -2, /* (329) frame_exclude ::= NO OTHERS */ + -2, /* (330) frame_exclude ::= CURRENT ROW */ + -1, /* (331) frame_exclude ::= GROUP|TIES */ + -2, /* (332) window_clause ::= WINDOW windowdefn_list */ + -2, /* (333) filter_over ::= filter_clause over_clause */ + -1, /* (334) filter_over ::= over_clause */ + -1, /* (335) filter_over ::= filter_clause */ + -4, /* (336) over_clause ::= OVER LP window RP */ + -2, /* (337) over_clause ::= OVER nm */ + -5, /* (338) filter_clause ::= FILTER LP WHERE expr RP */ + -1, /* (339) input ::= cmdlist */ + -2, /* (340) cmdlist ::= cmdlist ecmd */ + -1, /* (341) cmdlist ::= ecmd */ + -1, /* (342) ecmd ::= SEMI */ + -2, /* (343) ecmd ::= cmdx SEMI */ + -3, /* (344) ecmd ::= explain cmdx SEMI */ + 0, /* (345) trans_opt ::= */ + -1, /* (346) trans_opt ::= TRANSACTION */ + -2, /* (347) trans_opt ::= TRANSACTION nm */ + -1, /* (348) savepoint_opt ::= SAVEPOINT */ + 0, /* (349) savepoint_opt ::= */ + -2, /* (350) cmd ::= create_table create_table_args */ + -1, /* (351) table_option_set ::= table_option */ + -4, /* (352) columnlist ::= columnlist COMMA columnname carglist */ + -2, /* (353) columnlist ::= columnname carglist */ + -1, /* (354) nm ::= ID|INDEXED */ + -1, /* (355) nm ::= STRING */ + -1, /* (356) nm ::= JOIN_KW */ + -1, /* (357) typetoken ::= typename */ + -1, /* (358) typename ::= ID|STRING */ + -1, /* (359) signed ::= plus_num */ + -1, /* (360) signed ::= minus_num */ + -2, /* (361) carglist ::= carglist ccons */ + 0, /* (362) carglist ::= */ + -2, /* (363) ccons ::= NULL onconf */ + -4, /* (364) ccons ::= GENERATED ALWAYS AS generated */ + -2, /* (365) ccons ::= AS generated */ + -2, /* (366) conslist_opt ::= COMMA conslist */ + -3, /* (367) conslist ::= conslist tconscomma tcons */ + -1, /* (368) conslist ::= tcons */ + 0, /* (369) tconscomma ::= */ + -1, /* (370) defer_subclause_opt ::= defer_subclause */ + -1, /* (371) resolvetype ::= raisetype */ + -1, /* (372) selectnowith ::= oneselect */ + -1, /* (373) oneselect ::= values */ + -2, /* (374) sclp ::= selcollist COMMA */ + -1, /* (375) as ::= ID|STRING */ + 0, /* (376) returning ::= */ + -1, /* (377) expr ::= term */ + -1, /* (378) likeop ::= LIKE_KW|MATCH */ + -1, /* (379) exprlist ::= nexprlist */ + -1, /* (380) nmnum ::= plus_num */ + -1, /* (381) nmnum ::= nm */ + -1, /* (382) nmnum ::= ON */ + -1, /* (383) nmnum ::= DELETE */ + -1, /* (384) nmnum ::= DEFAULT */ + -1, /* (385) plus_num ::= INTEGER|FLOAT */ + 0, /* (386) foreach_clause ::= */ + -3, /* (387) foreach_clause ::= FOR EACH ROW */ + -1, /* (388) trnm ::= nm */ + 0, /* (389) tridxby ::= */ + -1, /* (390) database_kw_opt ::= DATABASE */ + 0, /* (391) database_kw_opt ::= */ + 0, /* (392) kwcolumn_opt ::= */ + -1, /* (393) kwcolumn_opt ::= COLUMNKW */ + -1, /* (394) vtabarglist ::= vtabarg */ + -3, /* (395) vtabarglist ::= vtabarglist COMMA vtabarg */ + -2, /* (396) vtabarg ::= vtabarg vtabargtoken */ + 0, /* (397) anylist ::= */ + -4, /* (398) anylist ::= anylist LP anylist RP */ + -2, /* (399) anylist ::= anylist ANY */ + 0, /* (400) with ::= */ +}; + +static void yy_accept(yyParser*); /* Forward Declaration */ + +/* +** Perform a reduce action and the shift that must immediately +** follow the reduce. +** +** The yyLookahead and yyLookaheadToken parameters provide reduce actions +** access to the lookahead token (if any). The yyLookahead will be YYNOCODE +** if the lookahead token has already been consumed. As this procedure is +** only called from one place, optimizing compilers will in-line it, which +** means that the extra parameters have no performance impact. +*/ +static YYACTIONTYPE yy_reduce( + yyParser *yypParser, /* The parser */ + unsigned int yyruleno, /* Number of the rule by which to reduce */ + int yyLookahead, /* Lookahead token, or YYNOCODE if none */ + sqlite3ParserTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + sqlite3ParserCTX_PDECL /* %extra_context */ +){ + int yygoto; /* The next state */ + YYACTIONTYPE yyact; /* The next action */ + yyStackEntry *yymsp; /* The top of the parser's stack */ + int yysize; /* Amount to pop the stack */ + sqlite3ParserARG_FETCH + (void)yyLookahead; + (void)yyLookaheadToken; + yymsp = yypParser->yytos; + + switch( yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line + ** { ... } // User supplied code + ** #line + ** break; + */ +/********** Begin reduce actions **********************************************/ + YYMINORTYPE yylhsminor; + case 0: /* explain ::= EXPLAIN */ +{ pParse->explain = 1; } + break; + case 1: /* explain ::= EXPLAIN QUERY PLAN */ +{ pParse->explain = 2; } + break; + case 2: /* cmdx ::= cmd */ +{ sqlite3FinishCoding(pParse); } + break; + case 3: /* cmd ::= BEGIN transtype trans_opt */ +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy64);} + break; + case 4: /* transtype ::= */ +{yymsp[1].minor.yy64 = TK_DEFERRED;} + break; + case 5: /* transtype ::= DEFERRED */ + case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); + case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); + case 320: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==320); +{yymsp[0].minor.yy64 = yymsp[0].major; /*A-overwrites-X*/} + break; + case 8: /* cmd ::= COMMIT|END trans_opt */ + case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9); +{sqlite3EndTransaction(pParse,yymsp[-1].major);} + break; + case 10: /* cmd ::= SAVEPOINT nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); +} + break; + case 11: /* cmd ::= RELEASE savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); +} + break; + case 12: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); +} + break; + case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ +{ + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy64,0,0,yymsp[-2].minor.yy64); +} + break; + case 14: /* createkw ::= CREATE */ +{disableLookaside(pParse);} + break; + case 15: /* ifnotexists ::= */ + case 18: /* temp ::= */ yytestcase(yyruleno==18); + case 47: /* autoinc ::= */ yytestcase(yyruleno==47); + case 62: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==62); + case 72: /* defer_subclause_opt ::= */ yytestcase(yyruleno==72); + case 81: /* ifexists ::= */ yytestcase(yyruleno==81); + case 98: /* distinct ::= */ yytestcase(yyruleno==98); + case 241: /* collate ::= */ yytestcase(yyruleno==241); +{yymsp[1].minor.yy64 = 0;} + break; + case 16: /* ifnotexists ::= IF NOT EXISTS */ +{yymsp[-2].minor.yy64 = 1;} + break; + case 17: /* temp ::= TEMP */ +{yymsp[0].minor.yy64 = pParse->db->init.busy==0;} + break; + case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_option_set */ +{ + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy51,0); +} + break; + case 20: /* create_table_args ::= AS select */ +{ + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy303); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy303); +} + break; + case 21: /* table_option_set ::= */ +{yymsp[1].minor.yy51 = 0;} + break; + case 22: /* table_option_set ::= table_option_set COMMA table_option */ +{yylhsminor.yy51 = yymsp[-2].minor.yy51|yymsp[0].minor.yy51;} + yymsp[-2].minor.yy51 = yylhsminor.yy51; + break; + case 23: /* table_option ::= WITHOUT nm */ +{ + if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ + yymsp[-1].minor.yy51 = TF_WithoutRowid | TF_NoVisibleRowid; + }else{ + yymsp[-1].minor.yy51 = 0; + sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); + } +} + break; + case 24: /* table_option ::= nm */ +{ + if( yymsp[0].minor.yy0.n==6 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"strict",6)==0 ){ + yylhsminor.yy51 = TF_Strict; + }else{ + yylhsminor.yy51 = 0; + sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); + } +} + yymsp[0].minor.yy51 = yylhsminor.yy51; + break; + case 25: /* columnname ::= nm typetoken */ +{sqlite3AddColumn(pParse,yymsp[-1].minor.yy0,yymsp[0].minor.yy0);} + break; + case 26: /* typetoken ::= */ + case 65: /* conslist_opt ::= */ yytestcase(yyruleno==65); + case 104: /* as ::= */ yytestcase(yyruleno==104); +{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} + break; + case 27: /* typetoken ::= typename LP signed RP */ +{ + yymsp[-3].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); +} + break; + case 28: /* typetoken ::= typename LP signed COMMA signed RP */ +{ + yymsp[-5].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); +} + break; + case 29: /* typename ::= typename ID|STRING */ +{yymsp[-1].minor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} + break; + case 30: /* scanpt ::= */ +{ + assert( yyLookahead!=YYNOCODE ); + yymsp[1].minor.yy600 = yyLookaheadToken.z; +} + break; + case 31: /* scantok ::= */ +{ + assert( yyLookahead!=YYNOCODE ); + yymsp[1].minor.yy0 = yyLookaheadToken; +} + break; + case 32: /* ccons ::= CONSTRAINT nm */ + case 67: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==67); +{pParse->constraintName = yymsp[0].minor.yy0;} + break; + case 33: /* ccons ::= DEFAULT scantok term */ +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy626,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} + break; + case 34: /* ccons ::= DEFAULT LP expr RP */ +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy626,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} + break; + case 35: /* ccons ::= DEFAULT PLUS scantok term */ +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy626,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} + break; + case 36: /* ccons ::= DEFAULT MINUS scantok term */ +{ + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy626, 0); + sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]); +} + break; + case 37: /* ccons ::= DEFAULT scantok ID|INDEXED */ +{ + Expr *p = tokenExpr(pParse, TK_STRING, yymsp[0].minor.yy0); + if( p ){ + sqlite3ExprIdToTrueFalse(p); + testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) ); + } + sqlite3AddDefaultValue(pParse,p,yymsp[0].minor.yy0.z,yymsp[0].minor.yy0.z+yymsp[0].minor.yy0.n); +} + break; + case 38: /* ccons ::= NOT NULL onconf */ +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy64);} + break; + case 39: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy64,yymsp[0].minor.yy64,yymsp[-2].minor.yy64);} + break; + case 40: /* ccons ::= UNIQUE onconf */ +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy64,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} + break; + case 41: /* ccons ::= CHECK LP expr RP */ +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy626,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} + break; + case 42: /* ccons ::= REFERENCES nm eidlist_opt refargs */ +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy562,yymsp[0].minor.yy64);} + break; + case 43: /* ccons ::= defer_subclause */ +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy64);} + break; + case 44: /* ccons ::= COLLATE ID|STRING */ +{sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} + break; + case 45: /* generated ::= LP expr RP */ +{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy626,0);} + break; + case 46: /* generated ::= LP expr RP ID */ +{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy626,&yymsp[0].minor.yy0);} + break; + case 48: /* autoinc ::= AUTOINCR */ +{yymsp[0].minor.yy64 = 1;} + break; + case 49: /* refargs ::= */ +{ yymsp[1].minor.yy64 = OE_None*0x0101; /* EV: R-19803-45884 */} + break; + case 50: /* refargs ::= refargs refarg */ +{ yymsp[-1].minor.yy64 = (yymsp[-1].minor.yy64 & ~yymsp[0].minor.yy83.mask) | yymsp[0].minor.yy83.value; } + break; + case 51: /* refarg ::= MATCH nm */ +{ yymsp[-1].minor.yy83.value = 0; yymsp[-1].minor.yy83.mask = 0x000000; } + break; + case 52: /* refarg ::= ON INSERT refact */ +{ yymsp[-2].minor.yy83.value = 0; yymsp[-2].minor.yy83.mask = 0x000000; } + break; + case 53: /* refarg ::= ON DELETE refact */ +{ yymsp[-2].minor.yy83.value = yymsp[0].minor.yy64; yymsp[-2].minor.yy83.mask = 0x0000ff; } + break; + case 54: /* refarg ::= ON UPDATE refact */ +{ yymsp[-2].minor.yy83.value = yymsp[0].minor.yy64<<8; yymsp[-2].minor.yy83.mask = 0x00ff00; } + break; + case 55: /* refact ::= SET NULL */ +{ yymsp[-1].minor.yy64 = OE_SetNull; /* EV: R-33326-45252 */} + break; + case 56: /* refact ::= SET DEFAULT */ +{ yymsp[-1].minor.yy64 = OE_SetDflt; /* EV: R-33326-45252 */} + break; + case 57: /* refact ::= CASCADE */ +{ yymsp[0].minor.yy64 = OE_Cascade; /* EV: R-33326-45252 */} + break; + case 58: /* refact ::= RESTRICT */ +{ yymsp[0].minor.yy64 = OE_Restrict; /* EV: R-33326-45252 */} + break; + case 59: /* refact ::= NO ACTION */ +{ yymsp[-1].minor.yy64 = OE_None; /* EV: R-33326-45252 */} + break; + case 60: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ +{yymsp[-2].minor.yy64 = 0;} + break; + case 61: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 76: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==76); + case 171: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==171); +{yymsp[-1].minor.yy64 = yymsp[0].minor.yy64;} + break; + case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ + case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80); + case 213: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==213); + case 216: /* in_op ::= NOT IN */ yytestcase(yyruleno==216); + case 242: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==242); +{yymsp[-1].minor.yy64 = 1;} + break; + case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ +{yymsp[-1].minor.yy64 = 0;} + break; + case 66: /* tconscomma ::= COMMA */ +{pParse->constraintName.n = 0;} + break; + case 68: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy562,yymsp[0].minor.yy64,yymsp[-2].minor.yy64,0);} + break; + case 69: /* tcons ::= UNIQUE LP sortlist RP onconf */ +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy562,yymsp[0].minor.yy64,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} + break; + case 70: /* tcons ::= CHECK LP expr RP onconf */ +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy626,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} + break; + case 71: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ +{ + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy562, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy562, yymsp[-1].minor.yy64); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy64); +} + break; + case 73: /* onconf ::= */ + case 75: /* orconf ::= */ yytestcase(yyruleno==75); +{yymsp[1].minor.yy64 = OE_Default;} + break; + case 74: /* onconf ::= ON CONFLICT resolvetype */ +{yymsp[-2].minor.yy64 = yymsp[0].minor.yy64;} + break; + case 77: /* resolvetype ::= IGNORE */ +{yymsp[0].minor.yy64 = OE_Ignore;} + break; + case 78: /* resolvetype ::= REPLACE */ + case 172: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==172); +{yymsp[0].minor.yy64 = OE_Replace;} + break; + case 79: /* cmd ::= DROP TABLE ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy607, 0, yymsp[-1].minor.yy64); +} + break; + case 82: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ +{ + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy562, yymsp[0].minor.yy303, yymsp[-7].minor.yy64, yymsp[-5].minor.yy64); +} + break; + case 83: /* cmd ::= DROP VIEW ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy607, 1, yymsp[-1].minor.yy64); +} + break; + case 84: /* cmd ::= select */ +{ + SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0, 0}; + sqlite3Select(pParse, yymsp[0].minor.yy303, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy303); +} + break; + case 85: /* select ::= WITH wqlist selectnowith */ +{yymsp[-2].minor.yy303 = attachWithToSelect(pParse,yymsp[0].minor.yy303,yymsp[-1].minor.yy43);} + break; + case 86: /* select ::= WITH RECURSIVE wqlist selectnowith */ +{yymsp[-3].minor.yy303 = attachWithToSelect(pParse,yymsp[0].minor.yy303,yymsp[-1].minor.yy43);} + break; + case 87: /* select ::= selectnowith */ +{ + Select *p = yymsp[0].minor.yy303; + if( p ){ + parserDoubleLinkSelect(pParse, p); + } + yymsp[0].minor.yy303 = p; /*A-overwrites-X*/ +} + break; + case 88: /* selectnowith ::= selectnowith multiselect_op oneselect */ +{ + Select *pRhs = yymsp[0].minor.yy303; + Select *pLhs = yymsp[-2].minor.yy303; + if( pRhs && pRhs->pPrior ){ + SrcList *pFrom; + Token x; + x.n = 0; + parserDoubleLinkSelect(pParse, pRhs); + pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); + pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); + } + if( pRhs ){ + pRhs->op = (u8)yymsp[-1].minor.yy64; + pRhs->pPrior = pLhs; + if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; + pRhs->selFlags &= ~SF_MultiValue; + if( yymsp[-1].minor.yy64!=TK_ALL ) pParse->hasCompound = 1; + }else{ + sqlite3SelectDelete(pParse->db, pLhs); + } + yymsp[-2].minor.yy303 = pRhs; +} + break; + case 89: /* multiselect_op ::= UNION */ + case 91: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==91); +{yymsp[0].minor.yy64 = yymsp[0].major; /*A-overwrites-OP*/} + break; + case 90: /* multiselect_op ::= UNION ALL */ +{yymsp[-1].minor.yy64 = TK_ALL;} + break; + case 92: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ +{ + yymsp[-8].minor.yy303 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy562,yymsp[-5].minor.yy607,yymsp[-4].minor.yy626,yymsp[-3].minor.yy562,yymsp[-2].minor.yy626,yymsp[-1].minor.yy562,yymsp[-7].minor.yy64,yymsp[0].minor.yy626); +} + break; + case 93: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ +{ + yymsp[-9].minor.yy303 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy562,yymsp[-6].minor.yy607,yymsp[-5].minor.yy626,yymsp[-4].minor.yy562,yymsp[-3].minor.yy626,yymsp[-1].minor.yy562,yymsp[-8].minor.yy64,yymsp[0].minor.yy626); + if( yymsp[-9].minor.yy303 ){ + yymsp[-9].minor.yy303->pWinDefn = yymsp[-2].minor.yy375; + }else{ + sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy375); + } +} + break; + case 94: /* values ::= VALUES LP nexprlist RP */ +{ + yymsp[-3].minor.yy303 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy562,0,0,0,0,0,SF_Values,0); +} + break; + case 95: /* values ::= values COMMA LP nexprlist RP */ +{ + Select *pRight, *pLeft = yymsp[-4].minor.yy303; + pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy562,0,0,0,0,0,SF_Values|SF_MultiValue,0); + if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; + if( pRight ){ + pRight->op = TK_ALL; + pRight->pPrior = pLeft; + yymsp[-4].minor.yy303 = pRight; + }else{ + yymsp[-4].minor.yy303 = pLeft; + } +} + break; + case 96: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy64 = SF_Distinct;} + break; + case 97: /* distinct ::= ALL */ +{yymsp[0].minor.yy64 = SF_All;} + break; + case 99: /* sclp ::= */ + case 132: /* orderby_opt ::= */ yytestcase(yyruleno==132); + case 142: /* groupby_opt ::= */ yytestcase(yyruleno==142); + case 229: /* exprlist ::= */ yytestcase(yyruleno==229); + case 232: /* paren_exprlist ::= */ yytestcase(yyruleno==232); + case 237: /* eidlist_opt ::= */ yytestcase(yyruleno==237); +{yymsp[1].minor.yy562 = 0;} + break; + case 100: /* selcollist ::= sclp scanpt expr scanpt as */ +{ + yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy562, yymsp[-2].minor.yy626); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy562, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy562,yymsp[-3].minor.yy600,yymsp[-1].minor.yy600); +} + break; + case 101: /* selcollist ::= sclp scanpt STAR */ +{ + Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); + yymsp[-2].minor.yy562 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy562, p); +} + break; + case 102: /* selcollist ::= sclp scanpt nm DOT STAR */ +{ + Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); + Expr *pLeft = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); + Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); + yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy562, pDot); +} + break; + case 103: /* as ::= AS nm */ + case 114: /* dbnm ::= DOT nm */ yytestcase(yyruleno==114); + case 253: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==253); + case 254: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==254); +{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} + break; + case 105: /* from ::= */ + case 108: /* stl_prefix ::= */ yytestcase(yyruleno==108); +{yymsp[1].minor.yy607 = 0;} + break; + case 106: /* from ::= FROM seltablist */ +{ + yymsp[-1].minor.yy607 = yymsp[0].minor.yy607; + sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy607); +} + break; + case 107: /* stl_prefix ::= seltablist joinop */ +{ + if( ALWAYS(yymsp[-1].minor.yy607 && yymsp[-1].minor.yy607->nSrc>0) ) yymsp[-1].minor.yy607->a[yymsp[-1].minor.yy607->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy64; +} + break; + case 109: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ +{ + yymsp[-6].minor.yy607 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy607,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy626,yymsp[0].minor.yy240); + sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy607, &yymsp[-2].minor.yy0); +} + break; + case 110: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ +{ + yymsp[-8].minor.yy607 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy607,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy626,yymsp[0].minor.yy240); + sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy607, yymsp[-4].minor.yy562); +} + break; + case 111: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ +{ + yymsp[-6].minor.yy607 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy607,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy303,yymsp[-1].minor.yy626,yymsp[0].minor.yy240); + } + break; + case 112: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ +{ + if( yymsp[-6].minor.yy607==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy626==0 && yymsp[0].minor.yy240==0 ){ + yymsp[-6].minor.yy607 = yymsp[-4].minor.yy607; + }else if( yymsp[-4].minor.yy607->nSrc==1 ){ + yymsp[-6].minor.yy607 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy607,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy626,yymsp[0].minor.yy240); + if( yymsp[-6].minor.yy607 ){ + SrcItem *pNew = &yymsp[-6].minor.yy607->a[yymsp[-6].minor.yy607->nSrc-1]; + SrcItem *pOld = yymsp[-4].minor.yy607->a; + pNew->zName = pOld->zName; + pNew->zDatabase = pOld->zDatabase; + pNew->pSelect = pOld->pSelect; + if( pOld->fg.isTabFunc ){ + pNew->u1.pFuncArg = pOld->u1.pFuncArg; + pOld->u1.pFuncArg = 0; + pOld->fg.isTabFunc = 0; + pNew->fg.isTabFunc = 1; + } + pOld->zName = pOld->zDatabase = 0; + pOld->pSelect = 0; + } + sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy607); + }else{ + Select *pSubquery; + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy607); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy607,0,0,0,0,SF_NestedFrom,0); + yymsp[-6].minor.yy607 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy607,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy626,yymsp[0].minor.yy240); + } + } + break; + case 113: /* dbnm ::= */ + case 127: /* indexed_opt ::= */ yytestcase(yyruleno==127); +{yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} + break; + case 115: /* fullname ::= nm */ +{ + yylhsminor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); + if( IN_RENAME_OBJECT && yylhsminor.yy607 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy607->a[0].zName, &yymsp[0].minor.yy0); +} + yymsp[0].minor.yy607 = yylhsminor.yy607; + break; + case 116: /* fullname ::= nm DOT nm */ +{ + yylhsminor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + if( IN_RENAME_OBJECT && yylhsminor.yy607 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy607->a[0].zName, &yymsp[0].minor.yy0); +} + yymsp[-2].minor.yy607 = yylhsminor.yy607; + break; + case 117: /* xfullname ::= nm */ +{yymsp[0].minor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} + break; + case 118: /* xfullname ::= nm DOT nm */ +{yymsp[-2].minor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 119: /* xfullname ::= nm DOT nm AS nm */ +{ + yymsp[-4].minor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ + if( yymsp[-4].minor.yy607 ) yymsp[-4].minor.yy607->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); +} + break; + case 120: /* xfullname ::= nm AS nm */ +{ + yymsp[-2].minor.yy607 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ + if( yymsp[-2].minor.yy607 ) yymsp[-2].minor.yy607->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); +} + break; + case 121: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy64 = JT_INNER; } + break; + case 122: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy64 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} + break; + case 123: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy64 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} + break; + case 124: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy64 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} + break; + case 125: /* on_opt ::= ON expr */ + case 145: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==145); + case 152: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==152); + case 154: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==154); + case 225: /* case_else ::= ELSE expr */ yytestcase(yyruleno==225); + case 246: /* vinto ::= INTO expr */ yytestcase(yyruleno==246); +{yymsp[-1].minor.yy626 = yymsp[0].minor.yy626;} + break; + case 126: /* on_opt ::= */ + case 144: /* having_opt ::= */ yytestcase(yyruleno==144); + case 146: /* limit_opt ::= */ yytestcase(yyruleno==146); + case 151: /* where_opt ::= */ yytestcase(yyruleno==151); + case 153: /* where_opt_ret ::= */ yytestcase(yyruleno==153); + case 226: /* case_else ::= */ yytestcase(yyruleno==226); + case 228: /* case_operand ::= */ yytestcase(yyruleno==228); + case 247: /* vinto ::= */ yytestcase(yyruleno==247); +{yymsp[1].minor.yy626 = 0;} + break; + case 128: /* indexed_opt ::= INDEXED BY nm */ +{yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} + break; + case 129: /* indexed_opt ::= NOT INDEXED */ +{yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} + break; + case 130: /* using_opt ::= USING LP idlist RP */ +{yymsp[-3].minor.yy240 = yymsp[-1].minor.yy240;} + break; + case 131: /* using_opt ::= */ + case 173: /* idlist_opt ::= */ yytestcase(yyruleno==173); +{yymsp[1].minor.yy240 = 0;} + break; + case 133: /* orderby_opt ::= ORDER BY sortlist */ + case 143: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==143); +{yymsp[-2].minor.yy562 = yymsp[0].minor.yy562;} + break; + case 134: /* sortlist ::= sortlist COMMA expr sortorder nulls */ +{ + yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy562,yymsp[-2].minor.yy626); + sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy562,yymsp[-1].minor.yy64,yymsp[0].minor.yy64); +} + break; + case 135: /* sortlist ::= expr sortorder nulls */ +{ + yymsp[-2].minor.yy562 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy626); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy562,yymsp[-1].minor.yy64,yymsp[0].minor.yy64); +} + break; + case 136: /* sortorder ::= ASC */ +{yymsp[0].minor.yy64 = SQLITE_SO_ASC;} + break; + case 137: /* sortorder ::= DESC */ +{yymsp[0].minor.yy64 = SQLITE_SO_DESC;} + break; + case 138: /* sortorder ::= */ + case 141: /* nulls ::= */ yytestcase(yyruleno==141); +{yymsp[1].minor.yy64 = SQLITE_SO_UNDEFINED;} + break; + case 139: /* nulls ::= NULLS FIRST */ +{yymsp[-1].minor.yy64 = SQLITE_SO_ASC;} + break; + case 140: /* nulls ::= NULLS LAST */ +{yymsp[-1].minor.yy64 = SQLITE_SO_DESC;} + break; + case 147: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy626 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy626,0);} + break; + case 148: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy626 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy626,yymsp[0].minor.yy626);} + break; + case 149: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy626 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy626,yymsp[-2].minor.yy626);} + break; + case 150: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ +{ + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy607, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy607,yymsp[0].minor.yy626,0,0); +} + break; + case 155: /* where_opt_ret ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy562); yymsp[-1].minor.yy626 = 0;} + break; + case 156: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy562); yymsp[-3].minor.yy626 = yymsp[-2].minor.yy626;} + break; + case 157: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ +{ + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy607, &yymsp[-4].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy562,"set list"); + yymsp[-5].minor.yy607 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy607, yymsp[-1].minor.yy607); + sqlite3Update(pParse,yymsp[-5].minor.yy607,yymsp[-2].minor.yy562,yymsp[0].minor.yy626,yymsp[-6].minor.yy64,0,0,0); +} + break; + case 158: /* setlist ::= setlist COMMA nm EQ expr */ +{ + yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy562, yymsp[0].minor.yy626); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy562, &yymsp[-2].minor.yy0, 1); +} + break; + case 159: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ +{ + yymsp[-6].minor.yy562 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy562, yymsp[-3].minor.yy240, yymsp[0].minor.yy626); +} + break; + case 160: /* setlist ::= nm EQ expr */ +{ + yylhsminor.yy562 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy626); + sqlite3ExprListSetName(pParse, yylhsminor.yy562, &yymsp[-2].minor.yy0, 1); +} + yymsp[-2].minor.yy562 = yylhsminor.yy562; + break; + case 161: /* setlist ::= LP idlist RP EQ expr */ +{ + yymsp[-4].minor.yy562 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy240, yymsp[0].minor.yy626); +} + break; + case 162: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ +{ + sqlite3Insert(pParse, yymsp[-3].minor.yy607, yymsp[-1].minor.yy303, yymsp[-2].minor.yy240, yymsp[-5].minor.yy64, yymsp[0].minor.yy138); +} + break; + case 163: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ +{ + sqlite3Insert(pParse, yymsp[-4].minor.yy607, 0, yymsp[-3].minor.yy240, yymsp[-6].minor.yy64, 0); +} + break; + case 164: /* upsert ::= */ +{ yymsp[1].minor.yy138 = 0; } + break; + case 165: /* upsert ::= RETURNING selcollist */ +{ yymsp[-1].minor.yy138 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy562); } + break; + case 166: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ +{ yymsp[-11].minor.yy138 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy562,yymsp[-6].minor.yy626,yymsp[-2].minor.yy562,yymsp[-1].minor.yy626,yymsp[0].minor.yy138);} + break; + case 167: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ +{ yymsp[-8].minor.yy138 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy562,yymsp[-3].minor.yy626,0,0,yymsp[0].minor.yy138); } + break; + case 168: /* upsert ::= ON CONFLICT DO NOTHING returning */ +{ yymsp[-4].minor.yy138 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } + break; + case 169: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ +{ yymsp[-7].minor.yy138 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy562,yymsp[-1].minor.yy626,0);} + break; + case 170: /* returning ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy562);} + break; + case 174: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy240 = yymsp[-1].minor.yy240;} + break; + case 175: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy240 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy240,&yymsp[0].minor.yy0);} + break; + case 176: /* idlist ::= nm */ +{yymsp[0].minor.yy240 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} + break; + case 177: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy626 = yymsp[-1].minor.yy626;} + break; + case 178: /* expr ::= ID|INDEXED */ + case 179: /* expr ::= JOIN_KW */ yytestcase(yyruleno==179); +{yymsp[0].minor.yy626=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 180: /* expr ::= nm DOT nm */ +{ + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[0].minor.yy0); + sqlite3RenameTokenMap(pParse, (void*)temp1, &yymsp[-2].minor.yy0); + } + yylhsminor.yy626 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); +} + yymsp[-2].minor.yy626 = yylhsminor.yy626; + break; + case 181: /* expr ::= nm DOT nm DOT nm */ +{ + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-4].minor.yy0, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); + Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); + Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)temp3, &yymsp[0].minor.yy0); + sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[-2].minor.yy0); + } + yylhsminor.yy626 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); +} + yymsp[-4].minor.yy626 = yylhsminor.yy626; + break; + case 182: /* term ::= NULL|FLOAT|BLOB */ + case 183: /* term ::= STRING */ yytestcase(yyruleno==183); +{yymsp[0].minor.yy626=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 184: /* term ::= INTEGER */ +{ + yylhsminor.yy626 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); +} + yymsp[0].minor.yy626 = yylhsminor.yy626; + break; + case 185: /* expr ::= VARIABLE */ +{ + if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ + u32 n = yymsp[0].minor.yy0.n; + yymsp[0].minor.yy626 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy626, n); + }else{ + /* When doing a nested parse, one can include terms in an expression + ** that look like this: #1 #2 ... These terms refer to registers + ** in the virtual machine. #N is the N-th register. */ + Token t = yymsp[0].minor.yy0; /*A-overwrites-X*/ + assert( t.n>=2 ); + if( pParse->nested==0 ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); + yymsp[0].minor.yy626 = 0; + }else{ + yymsp[0].minor.yy626 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); + if( yymsp[0].minor.yy626 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy626->iTable); + } + } +} + break; + case 186: /* expr ::= expr COLLATE ID|STRING */ +{ + yymsp[-2].minor.yy626 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy626, &yymsp[0].minor.yy0, 1); +} + break; + case 187: /* expr ::= CAST LP expr AS typetoken RP */ +{ + yymsp[-5].minor.yy626 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); + sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy626, yymsp[-3].minor.yy626, 0); +} + break; + case 188: /* expr ::= ID|INDEXED LP distinct exprlist RP */ +{ + yylhsminor.yy626 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy562, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy64); +} + yymsp[-4].minor.yy626 = yylhsminor.yy626; + break; + case 189: /* expr ::= ID|INDEXED LP STAR RP */ +{ + yylhsminor.yy626 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); +} + yymsp[-3].minor.yy626 = yylhsminor.yy626; + break; + case 190: /* expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ +{ + yylhsminor.yy626 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy562, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy64); + sqlite3WindowAttach(pParse, yylhsminor.yy626, yymsp[0].minor.yy375); +} + yymsp[-5].minor.yy626 = yylhsminor.yy626; + break; + case 191: /* expr ::= ID|INDEXED LP STAR RP filter_over */ +{ + yylhsminor.yy626 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); + sqlite3WindowAttach(pParse, yylhsminor.yy626, yymsp[0].minor.yy375); +} + yymsp[-4].minor.yy626 = yylhsminor.yy626; + break; + case 192: /* term ::= CTIME_KW */ +{ + yylhsminor.yy626 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); +} + yymsp[0].minor.yy626 = yylhsminor.yy626; + break; + case 193: /* expr ::= LP nexprlist COMMA expr RP */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy562, yymsp[-1].minor.yy626); + yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( yymsp[-4].minor.yy626 ){ + yymsp[-4].minor.yy626->x.pList = pList; + if( ALWAYS(pList->nExpr) ){ + yymsp[-4].minor.yy626->flags |= pList->a[0].pExpr->flags & EP_Propagate; + } + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + break; + case 194: /* expr ::= expr AND expr */ +{yymsp[-2].minor.yy626=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy626,yymsp[0].minor.yy626);} + break; + case 195: /* expr ::= expr OR expr */ + case 196: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==196); + case 197: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==197); + case 198: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==198); + case 199: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==201); +{yymsp[-2].minor.yy626=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy626,yymsp[0].minor.yy626);} + break; + case 202: /* likeop ::= NOT LIKE_KW|MATCH */ +{yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} + break; + case 203: /* expr ::= expr likeop expr */ +{ + ExprList *pList; + int bNot = yymsp[-1].minor.yy0.n & 0x80000000; + yymsp[-1].minor.yy0.n &= 0x7fffffff; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy626); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy626); + yymsp[-2].minor.yy626 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + if( bNot ) yymsp[-2].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy626, 0); + if( yymsp[-2].minor.yy626 ) yymsp[-2].minor.yy626->flags |= EP_InfixFunc; +} + break; + case 204: /* expr ::= expr likeop expr ESCAPE expr */ +{ + ExprList *pList; + int bNot = yymsp[-3].minor.yy0.n & 0x80000000; + yymsp[-3].minor.yy0.n &= 0x7fffffff; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy626); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy626); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy626); + yymsp[-4].minor.yy626 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); + if( bNot ) yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy626, 0); + if( yymsp[-4].minor.yy626 ) yymsp[-4].minor.yy626->flags |= EP_InfixFunc; +} + break; + case 205: /* expr ::= expr ISNULL|NOTNULL */ +{yymsp[-1].minor.yy626 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy626,0);} + break; + case 206: /* expr ::= expr NOT NULL */ +{yymsp[-2].minor.yy626 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy626,0);} + break; + case 207: /* expr ::= expr IS expr */ +{ + yymsp[-2].minor.yy626 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy626,yymsp[0].minor.yy626); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy626, yymsp[-2].minor.yy626, TK_ISNULL); +} + break; + case 208: /* expr ::= expr IS NOT expr */ +{ + yymsp[-3].minor.yy626 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy626,yymsp[0].minor.yy626); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy626, yymsp[-3].minor.yy626, TK_NOTNULL); +} + break; + case 209: /* expr ::= NOT expr */ + case 210: /* expr ::= BITNOT expr */ yytestcase(yyruleno==210); +{yymsp[-1].minor.yy626 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy626, 0);/*A-overwrites-B*/} + break; + case 211: /* expr ::= PLUS|MINUS expr */ +{ + yymsp[-1].minor.yy626 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy626, 0); + /*A-overwrites-B*/ +} + break; + case 212: /* between_op ::= BETWEEN */ + case 215: /* in_op ::= IN */ yytestcase(yyruleno==215); +{yymsp[0].minor.yy64 = 0;} + break; + case 214: /* expr ::= expr between_op expr AND expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy626); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy626); + yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy626, 0); + if( yymsp[-4].minor.yy626 ){ + yymsp[-4].minor.yy626->x.pList = pList; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } + if( yymsp[-3].minor.yy64 ) yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy626, 0); +} + break; + case 217: /* expr ::= expr in_op LP exprlist RP */ +{ + if( yymsp[-1].minor.yy562==0 ){ + /* Expressions of the form + ** + ** expr1 IN () + ** expr1 NOT IN () + ** + ** simplify to constants 0 (false) and 1 (true), respectively, + ** regardless of the value of expr1. + */ + sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy626); + yymsp[-4].minor.yy626 = sqlite3Expr(pParse->db, TK_INTEGER, yymsp[-3].minor.yy64 ? "1" : "0"); + }else{ + Expr *pRHS = yymsp[-1].minor.yy562->a[0].pExpr; + if( yymsp[-1].minor.yy562->nExpr==1 && sqlite3ExprIsConstant(pRHS) && yymsp[-4].minor.yy626->op!=TK_VECTOR ){ + yymsp[-1].minor.yy562->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy562); + pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); + yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy626, pRHS); + }else{ + yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy626, 0); + if( yymsp[-4].minor.yy626==0 ){ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy562); + }else if( yymsp[-4].minor.yy626->pLeft->op==TK_VECTOR ){ + int nExpr = yymsp[-4].minor.yy626->pLeft->x.pList->nExpr; + Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy562); + if( pSelectRHS ){ + parserDoubleLinkSelect(pParse, pSelectRHS); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy626, pSelectRHS); + } + }else{ + yymsp[-4].minor.yy626->x.pList = yymsp[-1].minor.yy562; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy626); + } + } + if( yymsp[-3].minor.yy64 ) yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy626, 0); + } + } + break; + case 218: /* expr ::= LP select RP */ +{ + yymsp[-2].minor.yy626 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy626, yymsp[-1].minor.yy303); + } + break; + case 219: /* expr ::= expr in_op LP select RP */ +{ + yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy626, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy626, yymsp[-1].minor.yy303); + if( yymsp[-3].minor.yy64 ) yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy626, 0); + } + break; + case 220: /* expr ::= expr in_op nm dbnm paren_exprlist */ +{ + SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); + Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); + if( yymsp[0].minor.yy562 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy562); + yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy626, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy626, pSelect); + if( yymsp[-3].minor.yy64 ) yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy626, 0); + } + break; + case 221: /* expr ::= EXISTS LP select RP */ +{ + Expr *p; + p = yymsp[-3].minor.yy626 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy303); + } + break; + case 222: /* expr ::= CASE case_operand case_exprlist case_else END */ +{ + yymsp[-4].minor.yy626 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy626, 0); + if( yymsp[-4].minor.yy626 ){ + yymsp[-4].minor.yy626->x.pList = yymsp[-1].minor.yy626 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy562,yymsp[-1].minor.yy626) : yymsp[-2].minor.yy562; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy626); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy562); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy626); + } +} + break; + case 223: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ +{ + yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy562, yymsp[-2].minor.yy626); + yymsp[-4].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy562, yymsp[0].minor.yy626); +} + break; + case 224: /* case_exprlist ::= WHEN expr THEN expr */ +{ + yymsp[-3].minor.yy562 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy626); + yymsp[-3].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy562, yymsp[0].minor.yy626); +} + break; + case 227: /* case_operand ::= expr */ +{yymsp[0].minor.yy626 = yymsp[0].minor.yy626; /*A-overwrites-X*/} + break; + case 230: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy562 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy562,yymsp[0].minor.yy626);} + break; + case 231: /* nexprlist ::= expr */ +{yymsp[0].minor.yy562 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy626); /*A-overwrites-Y*/} + break; + case 233: /* paren_exprlist ::= LP exprlist RP */ + case 238: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==238); +{yymsp[-2].minor.yy562 = yymsp[-1].minor.yy562;} + break; + case 234: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ +{ + sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, + sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy562, yymsp[-10].minor.yy64, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy626, SQLITE_SO_ASC, yymsp[-8].minor.yy64, SQLITE_IDXTYPE_APPDEF); + if( IN_RENAME_OBJECT && pParse->pNewIndex ){ + sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0); + } +} + break; + case 235: /* uniqueflag ::= UNIQUE */ + case 277: /* raisetype ::= ABORT */ yytestcase(yyruleno==277); +{yymsp[0].minor.yy64 = OE_Abort;} + break; + case 236: /* uniqueflag ::= */ +{yymsp[1].minor.yy64 = OE_None;} + break; + case 239: /* eidlist ::= eidlist COMMA nm collate sortorder */ +{ + yymsp[-4].minor.yy562 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy562, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy64, yymsp[0].minor.yy64); +} + break; + case 240: /* eidlist ::= nm collate sortorder */ +{ + yymsp[-2].minor.yy562 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy64, yymsp[0].minor.yy64); /*A-overwrites-Y*/ +} + break; + case 243: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy607, yymsp[-1].minor.yy64);} + break; + case 244: /* cmd ::= VACUUM vinto */ +{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy626);} + break; + case 245: /* cmd ::= VACUUM nm vinto */ +{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy626);} + break; + case 248: /* cmd ::= PRAGMA nm dbnm */ +{sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} + break; + case 249: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} + break; + case 250: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} + break; + case 251: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} + break; + case 252: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} + break; + case 255: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ +{ + Token all; + all.z = yymsp[-3].minor.yy0.z; + all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy95, &all); +} + break; + case 256: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ +{ + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy64, yymsp[-4].minor.yy570.a, yymsp[-4].minor.yy570.b, yymsp[-2].minor.yy607, yymsp[0].minor.yy626, yymsp[-10].minor.yy64, yymsp[-8].minor.yy64); + yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ +} + break; + case 257: /* trigger_time ::= BEFORE|AFTER */ +{ yymsp[0].minor.yy64 = yymsp[0].major; /*A-overwrites-X*/ } + break; + case 258: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy64 = TK_INSTEAD;} + break; + case 259: /* trigger_time ::= */ +{ yymsp[1].minor.yy64 = TK_BEFORE; } + break; + case 260: /* trigger_event ::= DELETE|INSERT */ + case 261: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==261); +{yymsp[0].minor.yy570.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy570.b = 0;} + break; + case 262: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy570.a = TK_UPDATE; yymsp[-2].minor.yy570.b = yymsp[0].minor.yy240;} + break; + case 263: /* when_clause ::= */ + case 282: /* key_opt ::= */ yytestcase(yyruleno==282); +{ yymsp[1].minor.yy626 = 0; } + break; + case 264: /* when_clause ::= WHEN expr */ + case 283: /* key_opt ::= KEY expr */ yytestcase(yyruleno==283); +{ yymsp[-1].minor.yy626 = yymsp[0].minor.yy626; } + break; + case 265: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ +{ + assert( yymsp[-2].minor.yy95!=0 ); + yymsp[-2].minor.yy95->pLast->pNext = yymsp[-1].minor.yy95; + yymsp[-2].minor.yy95->pLast = yymsp[-1].minor.yy95; +} + break; + case 266: /* trigger_cmd_list ::= trigger_cmd SEMI */ +{ + assert( yymsp[-1].minor.yy95!=0 ); + yymsp[-1].minor.yy95->pLast = yymsp[-1].minor.yy95; +} + break; + case 267: /* trnm ::= nm DOT nm */ +{ + yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; + sqlite3ErrorMsg(pParse, + "qualified table names are not allowed on INSERT, UPDATE, and DELETE " + "statements within triggers"); +} + break; + case 268: /* tridxby ::= INDEXED BY nm */ +{ + sqlite3ErrorMsg(pParse, + "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " + "within triggers"); +} + break; + case 269: /* tridxby ::= NOT INDEXED */ +{ + sqlite3ErrorMsg(pParse, + "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " + "within triggers"); +} + break; + case 270: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ +{yylhsminor.yy95 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy607, yymsp[-3].minor.yy562, yymsp[-1].minor.yy626, yymsp[-7].minor.yy64, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy600);} + yymsp[-8].minor.yy95 = yylhsminor.yy95; + break; + case 271: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ +{ + yylhsminor.yy95 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy240,yymsp[-2].minor.yy303,yymsp[-6].minor.yy64,yymsp[-1].minor.yy138,yymsp[-7].minor.yy600,yymsp[0].minor.yy600);/*yylhsminor.yy95-overwrites-yymsp[-6].minor.yy64*/ +} + yymsp[-7].minor.yy95 = yylhsminor.yy95; + break; + case 272: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ +{yylhsminor.yy95 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy626, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy600);} + yymsp[-5].minor.yy95 = yylhsminor.yy95; + break; + case 273: /* trigger_cmd ::= scanpt select scanpt */ +{yylhsminor.yy95 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy303, yymsp[-2].minor.yy600, yymsp[0].minor.yy600); /*yylhsminor.yy95-overwrites-yymsp[-1].minor.yy303*/} + yymsp[-2].minor.yy95 = yylhsminor.yy95; + break; + case 274: /* expr ::= RAISE LP IGNORE RP */ +{ + yymsp[-3].minor.yy626 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); + if( yymsp[-3].minor.yy626 ){ + yymsp[-3].minor.yy626->affExpr = OE_Ignore; + } +} + break; + case 275: /* expr ::= RAISE LP raisetype COMMA nm RP */ +{ + yymsp[-5].minor.yy626 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); + if( yymsp[-5].minor.yy626 ) { + yymsp[-5].minor.yy626->affExpr = (char)yymsp[-3].minor.yy64; + } +} + break; + case 276: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy64 = OE_Rollback;} + break; + case 278: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy64 = OE_Fail;} + break; + case 279: /* cmd ::= DROP TRIGGER ifexists fullname */ +{ + sqlite3DropTrigger(pParse,yymsp[0].minor.yy607,yymsp[-1].minor.yy64); +} + break; + case 280: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ +{ + sqlite3Attach(pParse, yymsp[-3].minor.yy626, yymsp[-1].minor.yy626, yymsp[0].minor.yy626); +} + break; + case 281: /* cmd ::= DETACH database_kw_opt expr */ +{ + sqlite3Detach(pParse, yymsp[0].minor.yy626); +} + break; + case 284: /* cmd ::= REINDEX */ +{sqlite3Reindex(pParse, 0, 0);} + break; + case 285: /* cmd ::= REINDEX nm dbnm */ +{sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} + break; + case 286: /* cmd ::= ANALYZE */ +{sqlite3Analyze(pParse, 0, 0);} + break; + case 287: /* cmd ::= ANALYZE nm dbnm */ +{sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} + break; + case 288: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ +{ + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy607,&yymsp[0].minor.yy0); +} + break; + case 289: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ +{ + yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; + sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); +} + break; + case 290: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ +{ + sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy607, &yymsp[0].minor.yy0); +} + break; + case 291: /* add_column_fullname ::= fullname */ +{ + disableLookaside(pParse); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy607); +} + break; + case 292: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ +{ + sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy607, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 293: /* cmd ::= create_vtab */ +{sqlite3VtabFinishParse(pParse,0);} + break; + case 294: /* cmd ::= create_vtab LP vtabarglist RP */ +{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} + break; + case 295: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ +{ + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy64); +} + break; + case 296: /* vtabarg ::= */ +{sqlite3VtabArgInit(pParse);} + break; + case 297: /* vtabargtoken ::= ANY */ + case 298: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==298); + case 299: /* lp ::= LP */ yytestcase(yyruleno==299); +{sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} + break; + case 300: /* with ::= WITH wqlist */ + case 301: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==301); +{ sqlite3WithPush(pParse, yymsp[0].minor.yy43, 1); } + break; + case 302: /* wqas ::= AS */ +{yymsp[0].minor.yy534 = M10d_Any;} + break; + case 303: /* wqas ::= AS MATERIALIZED */ +{yymsp[-1].minor.yy534 = M10d_Yes;} + break; + case 304: /* wqas ::= AS NOT MATERIALIZED */ +{yymsp[-2].minor.yy534 = M10d_No;} + break; + case 305: /* wqitem ::= nm eidlist_opt wqas LP select RP */ +{ + yymsp[-5].minor.yy255 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy562, yymsp[-1].minor.yy303, yymsp[-3].minor.yy534); /*A-overwrites-X*/ +} + break; + case 306: /* wqlist ::= wqitem */ +{ + yymsp[0].minor.yy43 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy255); /*A-overwrites-X*/ +} + break; + case 307: /* wqlist ::= wqlist COMMA wqitem */ +{ + yymsp[-2].minor.yy43 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy43, yymsp[0].minor.yy255); +} + break; + case 308: /* windowdefn_list ::= windowdefn */ +{ yylhsminor.yy375 = yymsp[0].minor.yy375; } + yymsp[0].minor.yy375 = yylhsminor.yy375; + break; + case 309: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ +{ + assert( yymsp[0].minor.yy375!=0 ); + sqlite3WindowChain(pParse, yymsp[0].minor.yy375, yymsp[-2].minor.yy375); + yymsp[0].minor.yy375->pNextWin = yymsp[-2].minor.yy375; + yylhsminor.yy375 = yymsp[0].minor.yy375; +} + yymsp[-2].minor.yy375 = yylhsminor.yy375; + break; + case 310: /* windowdefn ::= nm AS LP window RP */ +{ + if( ALWAYS(yymsp[-1].minor.yy375) ){ + yymsp[-1].minor.yy375->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); + } + yylhsminor.yy375 = yymsp[-1].minor.yy375; +} + yymsp[-4].minor.yy375 = yylhsminor.yy375; + break; + case 311: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ +{ + yymsp[-4].minor.yy375 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy375, yymsp[-2].minor.yy562, yymsp[-1].minor.yy562, 0); +} + break; + case 312: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ +{ + yylhsminor.yy375 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy375, yymsp[-2].minor.yy562, yymsp[-1].minor.yy562, &yymsp[-5].minor.yy0); +} + yymsp[-5].minor.yy375 = yylhsminor.yy375; + break; + case 313: /* window ::= ORDER BY sortlist frame_opt */ +{ + yymsp[-3].minor.yy375 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy375, 0, yymsp[-1].minor.yy562, 0); +} + break; + case 314: /* window ::= nm ORDER BY sortlist frame_opt */ +{ + yylhsminor.yy375 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy375, 0, yymsp[-1].minor.yy562, &yymsp[-4].minor.yy0); +} + yymsp[-4].minor.yy375 = yylhsminor.yy375; + break; + case 315: /* window ::= frame_opt */ + case 334: /* filter_over ::= over_clause */ yytestcase(yyruleno==334); +{ + yylhsminor.yy375 = yymsp[0].minor.yy375; +} + yymsp[0].minor.yy375 = yylhsminor.yy375; + break; + case 316: /* window ::= nm frame_opt */ +{ + yylhsminor.yy375 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy375, 0, 0, &yymsp[-1].minor.yy0); +} + yymsp[-1].minor.yy375 = yylhsminor.yy375; + break; + case 317: /* frame_opt ::= */ +{ + yymsp[1].minor.yy375 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); +} + break; + case 318: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ +{ + yylhsminor.yy375 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy64, yymsp[-1].minor.yy81.eType, yymsp[-1].minor.yy81.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy534); +} + yymsp[-2].minor.yy375 = yylhsminor.yy375; + break; + case 319: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ +{ + yylhsminor.yy375 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy64, yymsp[-3].minor.yy81.eType, yymsp[-3].minor.yy81.pExpr, yymsp[-1].minor.yy81.eType, yymsp[-1].minor.yy81.pExpr, yymsp[0].minor.yy534); +} + yymsp[-5].minor.yy375 = yylhsminor.yy375; + break; + case 321: /* frame_bound_s ::= frame_bound */ + case 323: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==323); +{yylhsminor.yy81 = yymsp[0].minor.yy81;} + yymsp[0].minor.yy81 = yylhsminor.yy81; + break; + case 322: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 324: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==324); + case 326: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==326); +{yylhsminor.yy81.eType = yymsp[-1].major; yylhsminor.yy81.pExpr = 0;} + yymsp[-1].minor.yy81 = yylhsminor.yy81; + break; + case 325: /* frame_bound ::= expr PRECEDING|FOLLOWING */ +{yylhsminor.yy81.eType = yymsp[0].major; yylhsminor.yy81.pExpr = yymsp[-1].minor.yy626;} + yymsp[-1].minor.yy81 = yylhsminor.yy81; + break; + case 327: /* frame_exclude_opt ::= */ +{yymsp[1].minor.yy534 = 0;} + break; + case 328: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ +{yymsp[-1].minor.yy534 = yymsp[0].minor.yy534;} + break; + case 329: /* frame_exclude ::= NO OTHERS */ + case 330: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==330); +{yymsp[-1].minor.yy534 = yymsp[-1].major; /*A-overwrites-X*/} + break; + case 331: /* frame_exclude ::= GROUP|TIES */ +{yymsp[0].minor.yy534 = yymsp[0].major; /*A-overwrites-X*/} + break; + case 332: /* window_clause ::= WINDOW windowdefn_list */ +{ yymsp[-1].minor.yy375 = yymsp[0].minor.yy375; } + break; + case 333: /* filter_over ::= filter_clause over_clause */ +{ + if( yymsp[0].minor.yy375 ){ + yymsp[0].minor.yy375->pFilter = yymsp[-1].minor.yy626; + }else{ + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy626); + } + yylhsminor.yy375 = yymsp[0].minor.yy375; +} + yymsp[-1].minor.yy375 = yylhsminor.yy375; + break; + case 335: /* filter_over ::= filter_clause */ +{ + yylhsminor.yy375 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yylhsminor.yy375 ){ + yylhsminor.yy375->eFrmType = TK_FILTER; + yylhsminor.yy375->pFilter = yymsp[0].minor.yy626; + }else{ + sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy626); + } +} + yymsp[0].minor.yy375 = yylhsminor.yy375; + break; + case 336: /* over_clause ::= OVER LP window RP */ +{ + yymsp[-3].minor.yy375 = yymsp[-1].minor.yy375; + assert( yymsp[-3].minor.yy375!=0 ); +} + break; + case 337: /* over_clause ::= OVER nm */ +{ + yymsp[-1].minor.yy375 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yymsp[-1].minor.yy375 ){ + yymsp[-1].minor.yy375->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); + } +} + break; + case 338: /* filter_clause ::= FILTER LP WHERE expr RP */ +{ yymsp[-4].minor.yy626 = yymsp[-1].minor.yy626; } + break; + default: + /* (339) input ::= cmdlist */ yytestcase(yyruleno==339); + /* (340) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==340); + /* (341) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=341); + /* (342) ecmd ::= SEMI */ yytestcase(yyruleno==342); + /* (343) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==343); + /* (344) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=344); + /* (345) trans_opt ::= */ yytestcase(yyruleno==345); + /* (346) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==346); + /* (347) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==347); + /* (348) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==348); + /* (349) savepoint_opt ::= */ yytestcase(yyruleno==349); + /* (350) cmd ::= create_table create_table_args */ yytestcase(yyruleno==350); + /* (351) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=351); + /* (352) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==352); + /* (353) columnlist ::= columnname carglist */ yytestcase(yyruleno==353); + /* (354) nm ::= ID|INDEXED */ yytestcase(yyruleno==354); + /* (355) nm ::= STRING */ yytestcase(yyruleno==355); + /* (356) nm ::= JOIN_KW */ yytestcase(yyruleno==356); + /* (357) typetoken ::= typename */ yytestcase(yyruleno==357); + /* (358) typename ::= ID|STRING */ yytestcase(yyruleno==358); + /* (359) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=359); + /* (360) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=360); + /* (361) carglist ::= carglist ccons */ yytestcase(yyruleno==361); + /* (362) carglist ::= */ yytestcase(yyruleno==362); + /* (363) ccons ::= NULL onconf */ yytestcase(yyruleno==363); + /* (364) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==364); + /* (365) ccons ::= AS generated */ yytestcase(yyruleno==365); + /* (366) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==366); + /* (367) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==367); + /* (368) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=368); + /* (369) tconscomma ::= */ yytestcase(yyruleno==369); + /* (370) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=370); + /* (371) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=371); + /* (372) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=372); + /* (373) oneselect ::= values */ yytestcase(yyruleno==373); + /* (374) sclp ::= selcollist COMMA */ yytestcase(yyruleno==374); + /* (375) as ::= ID|STRING */ yytestcase(yyruleno==375); + /* (376) returning ::= */ yytestcase(yyruleno==376); + /* (377) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=377); + /* (378) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==378); + /* (379) exprlist ::= nexprlist */ yytestcase(yyruleno==379); + /* (380) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=380); + /* (381) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=381); + /* (382) nmnum ::= ON */ yytestcase(yyruleno==382); + /* (383) nmnum ::= DELETE */ yytestcase(yyruleno==383); + /* (384) nmnum ::= DEFAULT */ yytestcase(yyruleno==384); + /* (385) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==385); + /* (386) foreach_clause ::= */ yytestcase(yyruleno==386); + /* (387) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==387); + /* (388) trnm ::= nm */ yytestcase(yyruleno==388); + /* (389) tridxby ::= */ yytestcase(yyruleno==389); + /* (390) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==390); + /* (391) database_kw_opt ::= */ yytestcase(yyruleno==391); + /* (392) kwcolumn_opt ::= */ yytestcase(yyruleno==392); + /* (393) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==393); + /* (394) vtabarglist ::= vtabarg */ yytestcase(yyruleno==394); + /* (395) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==395); + /* (396) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==396); + /* (397) anylist ::= */ yytestcase(yyruleno==397); + /* (398) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==398); + /* (399) anylist ::= anylist ANY */ yytestcase(yyruleno==399); + /* (400) with ::= */ yytestcase(yyruleno==400); + break; +/********** End reduce actions ************************************************/ + }; + assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); + + /* It is not possible for a REDUCE to be followed by an error */ + assert( yyact!=YY_ERROR_ACTION ); + + yymsp += yysize+1; + yypParser->yytos = yymsp; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact, "... then shift"); + return yyact; +} + +/* +** The following code executes when the parse fails +*/ +#ifndef YYNOERRORRECOVERY +static void yy_parse_failed( + yyParser *yypParser /* The parser */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); + } +#endif + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +/************ End %parse_failure code *****************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE +} +#endif /* YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void yy_syntax_error( + yyParser *yypParser, /* The parser */ + int yymajor, /* The major type of the error token */ + sqlite3ParserTOKENTYPE yyminor /* The minor type of the error token */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH +#define TOKEN yyminor +/************ Begin %syntax_error code ****************************************/ + + UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ + if( TOKEN.z[0] ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + }else{ + sqlite3ErrorMsg(pParse, "incomplete input"); + } +/************ End %syntax_error code ******************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE +} + +/* +** The following is executed when the parser accepts +*/ +static void yy_accept( + yyParser *yypParser /* The parser */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + assert( yypParser->yytos==yypParser->yystack ); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ +/*********** End %parse_accept code *******************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "sqlite3ParserAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +**
      +**
    • A pointer to the parser (an opaque structure.) +**
    • The major token number. +**
    • The minor token number. +**
    • An option argument of a grammar-specified type. +**
    +** +** Outputs: +** None. +*/ +SQLITE_PRIVATE void sqlite3Parser( + void *yyp, /* The parser */ + int yymajor, /* The major token code number */ + sqlite3ParserTOKENTYPE yyminor /* The value for the token */ + sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ +){ + YYMINORTYPE yyminorunion; + YYACTIONTYPE yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef YYERRORSYMBOL + int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif + yyParser *yypParser = (yyParser*)yyp; /* The parser */ + sqlite3ParserCTX_FETCH + sqlite3ParserARG_STORE + + assert( yypParser->yytos!=0 ); +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + yyendofinput = (yymajor==0); +#endif + + yyact = yypParser->yytos->stateno; +#ifndef NDEBUG + if( yyTraceFILE ){ + if( yyact < YY_MIN_REDUCE ){ + fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact); + }else{ + fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); + } + } +#endif + + while(1){ /* Exit by "break" */ + assert( yypParser->yytos>=yypParser->yystack ); + assert( yyact==yypParser->yytos->stateno ); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); + if( yyact >= YY_MIN_REDUCE ){ + unsigned int yyruleno = yyact - YY_MIN_REDUCE; /* Reduce by this rule */ + assert( yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ); +#ifndef NDEBUG + if( yyTraceFILE ){ + int yysize = yyRuleInfoNRhs[yyruleno]; + if( yysize ){ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + yyTracePrompt, + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos[yysize].stateno); + }else{ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s.\n", + yyTracePrompt, yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == + (int)(yypParser->yytos - yypParser->yystack)); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>=yypParser->yystackEnd ){ + yyStackOverflow(yypParser); + break; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); + break; + } + } +#endif + } + yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor sqlite3ParserCTX_PARAM); + }else if( yyact <= YY_MAX_SHIFTREDUCE ){ + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt--; +#endif + break; + }else if( yyact==YY_ACCEPT_ACTION ){ + yypParser->yytos--; + yy_accept(yypParser); + return; + }else{ + assert( yyact == YY_ERROR_ACTION ); + yyminorunion.yy0 = yyminor; +#ifdef YYERRORSYMBOL + int yymx; +#endif +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + } +#endif +#ifdef YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if( yypParser->yyerrcnt<0 ){ + yy_syntax_error(yypParser,yymajor,yyminor); + } + yymx = yypParser->yytos->major; + if( yymx==YYERRORSYMBOL || yyerrorhit ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sDiscard input token %s\n", + yyTracePrompt,yyTokenName[yymajor]); + } +#endif + yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); + yymajor = YYNOCODE; + }else{ + while( yypParser->yytos > yypParser->yystack ){ + yyact = yy_find_reduce_action(yypParser->yytos->stateno, + YYERRORSYMBOL); + if( yyact<=YY_MAX_SHIFTREDUCE ) break; + yy_pop_parser_stack(yypParser); + } + if( yypParser->yytos <= yypParser->yystack || yymajor==0 ){ + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yymajor = YYNOCODE; + }else if( yymx!=YYERRORSYMBOL ){ + yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); + } + } + yypParser->yyerrcnt = 3; + yyerrorhit = 1; + if( yymajor==YYNOCODE ) break; + yyact = yypParser->yytos->stateno; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor, yyminor); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + break; +#else /* YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( yypParser->yyerrcnt<=0 ){ + yy_syntax_error(yypParser,yymajor, yyminor); + } + yypParser->yyerrcnt = 3; + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + if( yyendofinput ){ + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + } + break; +#endif + } + } +#ifndef NDEBUG + if( yyTraceFILE ){ + yyStackEntry *i; + char cDiv = '['; + fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); + for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ + fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); + cDiv = ' '; + } + fprintf(yyTraceFILE,"]\n"); + } +#endif + return; +} + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +SQLITE_PRIVATE int sqlite3ParserFallback(int iToken){ +#ifdef YYFALLBACK + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); + return yyFallback[iToken]; +#else + (void)iToken; + return 0; +#endif +} + +/************** End of parse.c ***********************************************/ +/************** Begin file tokenize.c ****************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that splits an SQL input string up into +** individual tokens and sends those tokens one-by-one over to the +** parser for analysis. +*/ +/* #include "sqliteInt.h" */ +/* #include */ + +/* Character classes for tokenizing +** +** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented +** using a lookup table, whereas a switch() directly on c uses a binary search. +** The lookup table is much faster. To maximize speed, and to ensure that +** a lookup table is used, all of the classes need to be small integers and +** all of them need to be used within the switch. +*/ +#define CC_X 0 /* The letter 'x', or start of BLOB literal */ +#define CC_KYWD0 1 /* First letter of a keyword */ +#define CC_KYWD 2 /* Alphabetics or '_'. Usable in a keyword */ +#define CC_DIGIT 3 /* Digits */ +#define CC_DOLLAR 4 /* '$' */ +#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ +#define CC_VARNUM 6 /* '?'. Numeric SQL variables */ +#define CC_SPACE 7 /* Space characters */ +#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ +#define CC_QUOTE2 9 /* '['. [...] style quoted ids */ +#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ +#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ +#define CC_LT 12 /* '<'. Part of < or <= or <> */ +#define CC_GT 13 /* '>'. Part of > or >= */ +#define CC_EQ 14 /* '='. Part of = or == */ +#define CC_BANG 15 /* '!'. Part of != */ +#define CC_SLASH 16 /* '/'. / or c-style comment */ +#define CC_LP 17 /* '(' */ +#define CC_RP 18 /* ')' */ +#define CC_SEMI 19 /* ';' */ +#define CC_PLUS 20 /* '+' */ +#define CC_STAR 21 /* '*' */ +#define CC_PERCENT 22 /* '%' */ +#define CC_COMMA 23 /* ',' */ +#define CC_AND 24 /* '&' */ +#define CC_TILDA 25 /* '~' */ +#define CC_DOT 26 /* '.' */ +#define CC_ID 27 /* unicode characters usable in IDs */ +#define CC_ILLEGAL 28 /* Illegal character */ +#define CC_NUL 29 /* 0x00 */ +#define CC_BOM 30 /* First byte of UTF8 BOM: 0xEF 0xBB 0xBF */ + +static const unsigned char aiClass[] = { +#ifdef SQLITE_ASCII +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 29, 28, 28, 28, 28, 28, 28, 28, 28, 7, 7, 28, 7, 7, 28, 28, +/* 1x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, +/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, +/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, +/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 9, 28, 28, 28, 2, +/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 28, 10, 28, 25, 28, +/* 8x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 9x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* Ax */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* Cx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* Dx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* Ex */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 30, +/* Fx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27 +#endif +#ifdef SQLITE_EBCDIC +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 29, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 7, 7, 28, 28, +/* 1x */ 28, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, +/* 2x */ 28, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, +/* 3x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, +/* 4x */ 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 26, 12, 17, 20, 10, +/* 5x */ 24, 28, 28, 28, 28, 28, 28, 28, 28, 28, 15, 4, 21, 18, 19, 28, +/* 6x */ 11, 16, 28, 28, 28, 28, 28, 28, 28, 28, 28, 23, 22, 2, 13, 6, +/* 7x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 8, 5, 5, 5, 8, 14, 8, +/* 8x */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28, +/* 9x */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28, +/* Ax */ 28, 25, 1, 1, 1, 1, 1, 0, 2, 2, 28, 28, 28, 28, 28, 28, +/* Bx */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 9, 28, 28, 28, 28, 28, +/* Cx */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28, +/* Dx */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28, +/* Ex */ 28, 28, 1, 1, 1, 1, 1, 0, 2, 2, 28, 28, 28, 28, 28, 28, +/* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 28, 28, 28, 28, 28, 28, +#endif +}; + +/* +** The charMap() macro maps alphabetic characters (only) into their +** lower-case ASCII equivalent. On ASCII machines, this is just +** an upper-to-lower case map. On EBCDIC machines we also need +** to adjust the encoding. The mapping is only valid for alphabetics +** which are the only characters for which this feature is used. +** +** Used by keywordhash.h +*/ +#ifdef SQLITE_ASCII +# define charMap(X) sqlite3UpperToLower[(unsigned char)X] +#endif +#ifdef SQLITE_EBCDIC +# define charMap(X) ebcdicToAscii[(unsigned char)X] +const unsigned char ebcdicToAscii[] = { +/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ +}; +#endif + +/* +** The sqlite3KeywordCode function looks up an identifier to determine if +** it is a keyword. If it is a keyword, the token code of that keyword is +** returned. If the input is not a keyword, TK_ID is returned. +** +** The implementation of this routine was generated by a program, +** mkkeywordhash.c, located in the tool subdirectory of the distribution. +** The output of the mkkeywordhash.c program is written into a file +** named keywordhash.h and then included into this source file by +** the #include below. +*/ +/************** Include keywordhash.h in the middle of tokenize.c ************/ +/************** Begin file keywordhash.h *************************************/ +/***** This file contains automatically generated code ****** +** +** The code in this file has been automatically generated by +** +** sqlite/tool/mkkeywordhash.c +** +** The code in this file implements a function that determines whether +** or not a given identifier is really an SQL keyword. The same thing +** might be implemented more directly using a hand-written hash table. +** But by using this automatically generated code, the size of the code +** is substantially reduced. This is important for embedded applications +** on platforms with limited memory. +*/ +/* Hash score: 231 */ +/* zKWText[] encodes 1007 bytes of keyword text in 667 bytes */ +/* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ +/* ABLEFTHENDEFERRABLELSEXCLUDELETEMPORARYISNULLSAVEPOINTERSECT */ +/* IESNOTNULLIKEXCEPTRANSACTIONATURALTERAISEXCLUSIVEXISTS */ +/* CONSTRAINTOFFSETRIGGERANGENERATEDETACHAVINGLOBEGINNEREFERENCES */ +/* UNIQUERYWITHOUTERELEASEATTACHBETWEENOTHINGROUPSCASCADEFAULT */ +/* CASECOLLATECREATECURRENT_DATEIMMEDIATEJOINSERTMATCHPLANALYZE */ +/* PRAGMATERIALIZEDEFERREDISTINCTUPDATEVALUESVIRTUALWAYSWHENWHERE */ +/* CURSIVEABORTAFTERENAMEANDROPARTITIONAUTOINCREMENTCASTCOLUMN */ +/* COMMITCONFLICTCROSSCURRENT_TIMESTAMPRECEDINGFAILASTFILTER */ +/* EPLACEFIRSTFOLLOWINGFROMFULLIMITIFORDERESTRICTOTHERSOVER */ +/* ETURNINGRIGHTROLLBACKROWSUNBOUNDEDUNIONUSINGVACUUMVIEWINDOWBY */ +/* INITIALLYPRIMARY */ +static const char zKWText[666] = { + 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', + 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', + 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', + 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', + 'E','R','R','A','B','L','E','L','S','E','X','C','L','U','D','E','L','E', + 'T','E','M','P','O','R','A','R','Y','I','S','N','U','L','L','S','A','V', + 'E','P','O','I','N','T','E','R','S','E','C','T','I','E','S','N','O','T', + 'N','U','L','L','I','K','E','X','C','E','P','T','R','A','N','S','A','C', + 'T','I','O','N','A','T','U','R','A','L','T','E','R','A','I','S','E','X', + 'C','L','U','S','I','V','E','X','I','S','T','S','C','O','N','S','T','R', + 'A','I','N','T','O','F','F','S','E','T','R','I','G','G','E','R','A','N', + 'G','E','N','E','R','A','T','E','D','E','T','A','C','H','A','V','I','N', + 'G','L','O','B','E','G','I','N','N','E','R','E','F','E','R','E','N','C', + 'E','S','U','N','I','Q','U','E','R','Y','W','I','T','H','O','U','T','E', + 'R','E','L','E','A','S','E','A','T','T','A','C','H','B','E','T','W','E', + 'E','N','O','T','H','I','N','G','R','O','U','P','S','C','A','S','C','A', + 'D','E','F','A','U','L','T','C','A','S','E','C','O','L','L','A','T','E', + 'C','R','E','A','T','E','C','U','R','R','E','N','T','_','D','A','T','E', + 'I','M','M','E','D','I','A','T','E','J','O','I','N','S','E','R','T','M', + 'A','T','C','H','P','L','A','N','A','L','Y','Z','E','P','R','A','G','M', + 'A','T','E','R','I','A','L','I','Z','E','D','E','F','E','R','R','E','D', + 'I','S','T','I','N','C','T','U','P','D','A','T','E','V','A','L','U','E', + 'S','V','I','R','T','U','A','L','W','A','Y','S','W','H','E','N','W','H', + 'E','R','E','C','U','R','S','I','V','E','A','B','O','R','T','A','F','T', + 'E','R','E','N','A','M','E','A','N','D','R','O','P','A','R','T','I','T', + 'I','O','N','A','U','T','O','I','N','C','R','E','M','E','N','T','C','A', + 'S','T','C','O','L','U','M','N','C','O','M','M','I','T','C','O','N','F', + 'L','I','C','T','C','R','O','S','S','C','U','R','R','E','N','T','_','T', + 'I','M','E','S','T','A','M','P','R','E','C','E','D','I','N','G','F','A', + 'I','L','A','S','T','F','I','L','T','E','R','E','P','L','A','C','E','F', + 'I','R','S','T','F','O','L','L','O','W','I','N','G','F','R','O','M','F', + 'U','L','L','I','M','I','T','I','F','O','R','D','E','R','E','S','T','R', + 'I','C','T','O','T','H','E','R','S','O','V','E','R','E','T','U','R','N', + 'I','N','G','R','I','G','H','T','R','O','L','L','B','A','C','K','R','O', + 'W','S','U','N','B','O','U','N','D','E','D','U','N','I','O','N','U','S', + 'I','N','G','V','A','C','U','U','M','V','I','E','W','I','N','D','O','W', + 'B','Y','I','N','I','T','I','A','L','L','Y','P','R','I','M','A','R','Y', +}; +/* aKWHash[i] is the hash value for the i-th keyword */ +static const unsigned char aKWHash[127] = { + 84, 92, 134, 82, 105, 29, 0, 0, 94, 0, 85, 72, 0, + 53, 35, 86, 15, 0, 42, 97, 54, 89, 135, 19, 0, 0, + 140, 0, 40, 129, 0, 22, 107, 0, 9, 0, 0, 123, 80, + 0, 78, 6, 0, 65, 103, 147, 0, 136, 115, 0, 0, 48, + 0, 90, 24, 0, 17, 0, 27, 70, 23, 26, 5, 60, 142, + 110, 122, 0, 73, 91, 71, 145, 61, 120, 74, 0, 49, 0, + 11, 41, 0, 113, 0, 0, 0, 109, 10, 111, 116, 125, 14, + 50, 124, 0, 100, 0, 18, 121, 144, 56, 130, 139, 88, 83, + 37, 30, 126, 0, 0, 108, 51, 131, 128, 0, 34, 0, 0, + 132, 0, 98, 38, 39, 0, 20, 45, 117, 93, +}; +/* aKWNext[] forms the hash collision chain. If aKWHash[i]==0 +** then the i-th keyword has no more hash collisions. Otherwise, +** the next keyword with the same hash is aKWHash[i]-1. */ +static const unsigned char aKWNext[147] = { + 0, 0, 0, 0, 4, 0, 43, 0, 0, 106, 114, 0, 0, + 0, 2, 0, 0, 143, 0, 0, 0, 13, 0, 0, 0, 0, + 141, 0, 0, 119, 52, 0, 0, 137, 12, 0, 0, 62, 0, + 138, 0, 133, 0, 0, 36, 0, 0, 28, 77, 0, 0, 0, + 0, 59, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 69, 0, 0, 0, 0, 0, 146, 3, 0, 58, 0, 1, + 75, 0, 0, 0, 31, 0, 0, 0, 0, 0, 127, 0, 104, + 0, 64, 66, 63, 0, 0, 0, 0, 0, 46, 0, 16, 8, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 101, 0, + 112, 21, 7, 67, 0, 79, 96, 118, 0, 0, 68, 0, 0, + 99, 44, 0, 55, 0, 76, 0, 95, 32, 33, 57, 25, 0, + 102, 0, 0, 87, +}; +/* aKWLen[i] is the length (in bytes) of the i-th keyword */ +static const unsigned char aKWLen[147] = { + 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, + 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 7, + 6, 9, 4, 2, 6, 5, 9, 9, 4, 7, 3, 2, 4, + 4, 6, 11, 6, 2, 7, 5, 5, 9, 6, 10, 4, 6, + 2, 3, 7, 5, 9, 6, 6, 4, 5, 5, 10, 6, 5, + 7, 4, 5, 7, 6, 7, 7, 6, 5, 7, 3, 7, 4, + 7, 6, 12, 9, 4, 6, 5, 4, 7, 6, 12, 8, 8, + 2, 6, 6, 7, 6, 4, 5, 9, 5, 5, 6, 3, 4, + 9, 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, 7, 9, + 4, 4, 6, 7, 5, 9, 4, 4, 5, 2, 5, 8, 6, + 4, 9, 5, 8, 4, 3, 9, 5, 5, 6, 4, 6, 2, + 2, 9, 3, 7, +}; +/* aKWOffset[i] is the index into zKWText[] of the start of +** the text for the i-th keyword. */ +static const unsigned short int aKWOffset[147] = { + 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, + 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, + 86, 90, 90, 94, 99, 101, 105, 111, 119, 123, 123, 123, 126, + 129, 132, 137, 142, 146, 147, 152, 156, 160, 168, 174, 181, 184, + 184, 187, 189, 195, 198, 206, 211, 216, 219, 222, 226, 236, 239, + 244, 244, 248, 252, 259, 265, 271, 277, 277, 283, 284, 288, 295, + 299, 306, 312, 324, 333, 335, 341, 346, 348, 355, 359, 370, 377, + 378, 385, 391, 397, 402, 408, 412, 415, 424, 429, 433, 439, 441, + 444, 453, 455, 457, 466, 470, 476, 482, 490, 495, 495, 495, 511, + 520, 523, 527, 532, 539, 544, 553, 557, 560, 565, 567, 571, 579, + 585, 588, 597, 602, 610, 610, 614, 623, 628, 633, 639, 642, 645, + 648, 650, 655, 659, +}; +/* aKWCode[i] is the parser symbol code for the i-th keyword */ +static const unsigned char aKWCode[147] = { + TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, + TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, + TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, + TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, + TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, + TK_EXCLUDE, TK_DELETE, TK_TEMP, TK_TEMP, TK_OR, + TK_ISNULL, TK_NULLS, TK_SAVEPOINT, TK_INTERSECT, TK_TIES, + TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, TK_LIKE_KW, + TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW, + TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_CONSTRAINT, + TK_INTO, TK_OFFSET, TK_OF, TK_SET, TK_TRIGGER, + TK_RANGE, TK_GENERATED, TK_DETACH, TK_HAVING, TK_LIKE_KW, + TK_BEGIN, TK_JOIN_KW, TK_REFERENCES, TK_UNIQUE, TK_QUERY, + TK_WITHOUT, TK_WITH, TK_JOIN_KW, TK_RELEASE, TK_ATTACH, + TK_BETWEEN, TK_NOTHING, TK_GROUPS, TK_GROUP, TK_CASCADE, + TK_ASC, TK_DEFAULT, TK_CASE, TK_COLLATE, TK_CREATE, + TK_CTIME_KW, TK_IMMEDIATE, TK_JOIN, TK_INSERT, TK_MATCH, + TK_PLAN, TK_ANALYZE, TK_PRAGMA, TK_MATERIALIZED, TK_DEFERRED, + TK_DISTINCT, TK_IS, TK_UPDATE, TK_VALUES, TK_VIRTUAL, + TK_ALWAYS, TK_WHEN, TK_WHERE, TK_RECURSIVE, TK_ABORT, + TK_AFTER, TK_RENAME, TK_AND, TK_DROP, TK_PARTITION, + TK_AUTOINCR, TK_TO, TK_IN, TK_CAST, TK_COLUMNKW, + TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, TK_CTIME_KW, + TK_CURRENT, TK_PRECEDING, TK_FAIL, TK_LAST, TK_FILTER, + TK_REPLACE, TK_FIRST, TK_FOLLOWING, TK_FROM, TK_JOIN_KW, + TK_LIMIT, TK_IF, TK_ORDER, TK_RESTRICT, TK_OTHERS, + TK_OVER, TK_RETURNING, TK_JOIN_KW, TK_ROLLBACK, TK_ROWS, + TK_ROW, TK_UNBOUNDED, TK_UNION, TK_USING, TK_VACUUM, + TK_VIEW, TK_WINDOW, TK_DO, TK_BY, TK_INITIALLY, + TK_ALL, TK_PRIMARY, +}; +/* Hash table decoded: +** 0: INSERT +** 1: IS +** 2: ROLLBACK TRIGGER +** 3: IMMEDIATE +** 4: PARTITION +** 5: TEMP +** 6: +** 7: +** 8: VALUES WITHOUT +** 9: +** 10: MATCH +** 11: NOTHING +** 12: +** 13: OF +** 14: TIES IGNORE +** 15: PLAN +** 16: INSTEAD INDEXED +** 17: +** 18: TRANSACTION RIGHT +** 19: WHEN +** 20: SET HAVING +** 21: MATERIALIZED IF +** 22: ROWS +** 23: SELECT +** 24: +** 25: +** 26: VACUUM SAVEPOINT +** 27: +** 28: LIKE UNION VIRTUAL REFERENCES +** 29: RESTRICT +** 30: +** 31: THEN REGEXP +** 32: TO +** 33: +** 34: BEFORE +** 35: +** 36: +** 37: FOLLOWING COLLATE CASCADE +** 38: CREATE +** 39: +** 40: CASE REINDEX +** 41: EACH +** 42: +** 43: QUERY +** 44: AND ADD +** 45: PRIMARY ANALYZE +** 46: +** 47: ROW ASC DETACH +** 48: CURRENT_TIME CURRENT_DATE +** 49: +** 50: +** 51: EXCLUSIVE TEMPORARY +** 52: +** 53: DEFERRED +** 54: DEFERRABLE +** 55: +** 56: DATABASE +** 57: +** 58: DELETE VIEW GENERATED +** 59: ATTACH +** 60: END +** 61: EXCLUDE +** 62: ESCAPE DESC +** 63: GLOB +** 64: WINDOW ELSE +** 65: COLUMN +** 66: FIRST +** 67: +** 68: GROUPS ALL +** 69: DISTINCT DROP KEY +** 70: BETWEEN +** 71: INITIALLY +** 72: BEGIN +** 73: FILTER CHECK ACTION +** 74: GROUP INDEX +** 75: +** 76: EXISTS DEFAULT +** 77: +** 78: FOR CURRENT_TIMESTAMP +** 79: EXCEPT +** 80: +** 81: CROSS +** 82: +** 83: +** 84: +** 85: CAST +** 86: FOREIGN AUTOINCREMENT +** 87: COMMIT +** 88: CURRENT AFTER ALTER +** 89: FULL FAIL CONFLICT +** 90: EXPLAIN +** 91: CONSTRAINT +** 92: FROM ALWAYS +** 93: +** 94: ABORT +** 95: +** 96: AS DO +** 97: REPLACE WITH RELEASE +** 98: BY RENAME +** 99: RANGE RAISE +** 100: OTHERS +** 101: USING NULLS +** 102: PRAGMA +** 103: JOIN ISNULL OFFSET +** 104: NOT +** 105: OR LAST LEFT +** 106: LIMIT +** 107: +** 108: +** 109: IN +** 110: INTO +** 111: OVER RECURSIVE +** 112: ORDER OUTER +** 113: +** 114: INTERSECT UNBOUNDED +** 115: +** 116: +** 117: RETURNING ON +** 118: +** 119: WHERE +** 120: NO INNER +** 121: NULL +** 122: +** 123: TABLE +** 124: NATURAL NOTNULL +** 125: PRECEDING +** 126: UPDATE UNIQUE +*/ +/* Check to see if z[0..n-1] is a keyword. If it is, write the +** parser symbol code for that keyword into *pType. Always +** return the integer n (the length of the token). */ +static int keywordCode(const char *z, int n, int *pType){ + int i, j; + const char *zKW; + if( n>=2 ){ + i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n*1) % 127; + for(i=((int)aKWHash[i])-1; i>=0; i=((int)aKWNext[i])-1){ + if( aKWLen[i]!=n ) continue; + zKW = &zKWText[aKWOffset[i]]; +#ifdef SQLITE_ASCII + if( (z[0]&~0x20)!=zKW[0] ) continue; + if( (z[1]&~0x20)!=zKW[1] ) continue; + j = 2; + while( j=SQLITE_N_KEYWORD ) return SQLITE_ERROR; + *pzName = zKWText + aKWOffset[i]; + *pnName = aKWLen[i]; + return SQLITE_OK; +} +SQLITE_API int sqlite3_keyword_count(void){ return SQLITE_N_KEYWORD; } +SQLITE_API int sqlite3_keyword_check(const char *zName, int nName){ + return TK_ID!=sqlite3KeywordCode((const u8*)zName, nName); +} + +/************** End of keywordhash.h *****************************************/ +/************** Continuing where we left off in tokenize.c *******************/ + + +/* +** If X is a character that can be used in an identifier then +** IdChar(X) will be true. Otherwise it is false. +** +** For ASCII, any character with the high-order bit set is +** allowed in an identifier. For 7-bit characters, +** sqlite3IsIdChar[X] must be 1. +** +** For EBCDIC, the rules are more complex but have the same +** end result. +** +** Ticket #1066. the SQL standard does not allow '$' in the +** middle of identifiers. But many SQL implementations do. +** SQLite will allow '$' in identifiers for compatibility. +** But the feature is undocumented. +*/ +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */ + 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */ +}; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif + +/* Make the IdChar function accessible from ctime.c and alter.c */ +SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Return the id of the next token in string (*pz). Before returning, set +** (*pz) to point to the byte following the parsed token. +*/ +static int getToken(const unsigned char **pz){ + const unsigned char *z = *pz; + int t; /* Token type to return */ + do { + z += sqlite3GetToken(z, &t); + }while( t==TK_SPACE ); + if( t==TK_ID + || t==TK_STRING + || t==TK_JOIN_KW + || t==TK_WINDOW + || t==TK_OVER + || sqlite3ParserFallback(t)==TK_ID + ){ + t = TK_ID; + } + *pz = z; + return t; +} + +/* +** The following three functions are called immediately after the tokenizer +** reads the keywords WINDOW, OVER and FILTER, respectively, to determine +** whether the token should be treated as a keyword or an SQL identifier. +** This cannot be handled by the usual lemon %fallback method, due to +** the ambiguity in some constructions. e.g. +** +** SELECT sum(x) OVER ... +** +** In the above, "OVER" might be a keyword, or it might be an alias for the +** sum(x) expression. If a "%fallback ID OVER" directive were added to +** grammar, then SQLite would always treat "OVER" as an alias, making it +** impossible to call a window-function without a FILTER clause. +** +** WINDOW is treated as a keyword if: +** +** * the following token is an identifier, or a keyword that can fallback +** to being an identifier, and +** * the token after than one is TK_AS. +** +** OVER is a keyword if: +** +** * the previous token was TK_RP, and +** * the next token is either TK_LP or an identifier. +** +** FILTER is a keyword if: +** +** * the previous token was TK_RP, and +** * the next token is TK_LP. +*/ +static int analyzeWindowKeyword(const unsigned char *z){ + int t; + t = getToken(&z); + if( t!=TK_ID ) return TK_ID; + t = getToken(&z); + if( t!=TK_AS ) return TK_ID; + return TK_WINDOW; +} +static int analyzeOverKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP ){ + int t = getToken(&z); + if( t==TK_LP || t==TK_ID ) return TK_OVER; + } + return TK_ID; +} +static int analyzeFilterKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP && getToken(&z)==TK_LP ){ + return TK_FILTER; + } + return TK_ID; +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** Return the length (in bytes) of the token that begins at z[0]. +** Store the token type in *tokenType before returning. +*/ +SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ + int i, c; + switch( aiClass[*z] ){ /* Switch on the character-class of the first byte + ** of the token. See the comment on the CC_ defines + ** above. */ + case CC_SPACE: { + testcase( z[0]==' ' ); + testcase( z[0]=='\t' ); + testcase( z[0]=='\n' ); + testcase( z[0]=='\f' ); + testcase( z[0]=='\r' ); + for(i=1; sqlite3Isspace(z[i]); i++){} + *tokenType = TK_SPACE; + return i; + } + case CC_MINUS: { + if( z[1]=='-' ){ + for(i=2; (c=z[i])!=0 && c!='\n'; i++){} + *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + return i; + } + *tokenType = TK_MINUS; + return 1; + } + case CC_LP: { + *tokenType = TK_LP; + return 1; + } + case CC_RP: { + *tokenType = TK_RP; + return 1; + } + case CC_SEMI: { + *tokenType = TK_SEMI; + return 1; + } + case CC_PLUS: { + *tokenType = TK_PLUS; + return 1; + } + case CC_STAR: { + *tokenType = TK_STAR; + return 1; + } + case CC_SLASH: { + if( z[1]!='*' || z[2]==0 ){ + *tokenType = TK_SLASH; + return 1; + } + for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} + if( c ) i++; + *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + return i; + } + case CC_PERCENT: { + *tokenType = TK_REM; + return 1; + } + case CC_EQ: { + *tokenType = TK_EQ; + return 1 + (z[1]=='='); + } + case CC_LT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_LE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_NE; + return 2; + }else if( c=='<' ){ + *tokenType = TK_LSHIFT; + return 2; + }else{ + *tokenType = TK_LT; + return 1; + } + } + case CC_GT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_GE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_RSHIFT; + return 2; + }else{ + *tokenType = TK_GT; + return 1; + } + } + case CC_BANG: { + if( z[1]!='=' ){ + *tokenType = TK_ILLEGAL; + return 1; + }else{ + *tokenType = TK_NE; + return 2; + } + } + case CC_PIPE: { + if( z[1]!='|' ){ + *tokenType = TK_BITOR; + return 1; + }else{ + *tokenType = TK_CONCAT; + return 2; + } + } + case CC_COMMA: { + *tokenType = TK_COMMA; + return 1; + } + case CC_AND: { + *tokenType = TK_BITAND; + return 1; + } + case CC_TILDA: { + *tokenType = TK_BITNOT; + return 1; + } + case CC_QUOTE: { + int delim = z[0]; + testcase( delim=='`' ); + testcase( delim=='\'' ); + testcase( delim=='"' ); + for(i=1; (c=z[i])!=0; i++){ + if( c==delim ){ + if( z[i+1]==delim ){ + i++; + }else{ + break; + } + } + } + if( c=='\'' ){ + *tokenType = TK_STRING; + return i+1; + }else if( c!=0 ){ + *tokenType = TK_ID; + return i+1; + }else{ + *tokenType = TK_ILLEGAL; + return i; + } + } + case CC_DOT: { +#ifndef SQLITE_OMIT_FLOATING_POINT + if( !sqlite3Isdigit(z[1]) ) +#endif + { + *tokenType = TK_DOT; + return 1; + } + /* If the next character is a digit, this is a floating point + ** number that begins with ".". Fall thru into the next case */ + /* no break */ deliberate_fall_through + } + case CC_DIGIT: { + testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); + testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); + testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); + testcase( z[0]=='9' ); + *tokenType = TK_INTEGER; +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } +#endif + for(i=0; sqlite3Isdigit(z[i]); i++){} +#ifndef SQLITE_OMIT_FLOATING_POINT + if( z[i]=='.' ){ + i++; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + i += 2; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } +#endif + while( IdChar(z[i]) ){ + *tokenType = TK_ILLEGAL; + i++; + } + return i; + } + case CC_QUOTE2: { + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} + *tokenType = c==']' ? TK_ID : TK_ILLEGAL; + return i; + } + case CC_VARNUM: { + *tokenType = TK_VARIABLE; + for(i=1; sqlite3Isdigit(z[i]); i++){} + return i; + } + case CC_DOLLAR: + case CC_VARALPHA: { + int n = 0; + testcase( z[0]=='$' ); testcase( z[0]=='@' ); + testcase( z[0]==':' ); testcase( z[0]=='#' ); + *tokenType = TK_VARIABLE; + for(i=1; (c=z[i])!=0; i++){ + if( IdChar(c) ){ + n++; +#ifndef SQLITE_OMIT_TCL_VARIABLE + }else if( c=='(' && n>0 ){ + do{ + i++; + }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); + if( c==')' ){ + i++; + }else{ + *tokenType = TK_ILLEGAL; + } + break; + }else if( c==':' && z[i+1]==':' ){ + i++; +#endif + }else{ + break; + } + } + if( n==0 ) *tokenType = TK_ILLEGAL; + return i; + } + case CC_KYWD0: { + for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + if( IdChar(z[i]) ){ + /* This token started out using characters that can appear in keywords, + ** but z[i] is a character not allowed within keywords, so this must + ** be an identifier instead */ + i++; + break; + } + *tokenType = TK_ID; + return keywordCode((char*)z, i, tokenType); + } + case CC_X: { +#ifndef SQLITE_OMIT_BLOB_LITERAL + testcase( z[0]=='x' ); testcase( z[0]=='X' ); + if( z[1]=='\'' ){ + *tokenType = TK_BLOB; + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } + } + if( z[i] ) i++; + return i; + } +#endif + /* If it is not a BLOB literal, then it must be an ID, since no + ** SQL keywords start with the letter 'x'. Fall through */ + /* no break */ deliberate_fall_through + } + case CC_KYWD: + case CC_ID: { + i = 1; + break; + } + case CC_BOM: { + if( z[1]==0xbb && z[2]==0xbf ){ + *tokenType = TK_SPACE; + return 3; + } + i = 1; + break; + } + case CC_NUL: { + *tokenType = TK_ILLEGAL; + return 0; + } + default: { + *tokenType = TK_ILLEGAL; + return 1; + } + } + while( IdChar(z[i]) ){ i++; } + *tokenType = TK_ID; + return i; +} + +/* +** Run the parser on the given SQL string. The parser structure is +** passed in. An SQLITE_ status code is returned. If an error occurs +** then an and attempt is made to write an error message into +** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that +** error message. +*/ +SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ + int nErr = 0; /* Number of errors encountered */ + void *pEngine; /* The LEMON-generated LALR(1) parser */ + int n = 0; /* Length of the next token token */ + int tokenType; /* type of the next token */ + int lastTokenParsed = -1; /* type of the previous token */ + sqlite3 *db = pParse->db; /* The database connection */ + int mxSqlLen; /* Max length of an SQL string */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + yyParser sEngine; /* Space to hold the Lemon-generated Parser object */ +#endif + VVA_ONLY( u8 startedWithOom = db->mallocFailed ); + + assert( zSql!=0 ); + mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + if( db->nVdbeActive==0 ){ + AtomicStore(&db->u1.isInterrupted, 0); + } + pParse->rc = SQLITE_OK; + pParse->zTail = zSql; + assert( pzErrMsg!=0 ); +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_ParserTrace ){ + printf("parser: [[[%s]]]\n", zSql); + sqlite3ParserTrace(stdout, "parser: "); + }else{ + sqlite3ParserTrace(0, 0); + } +#endif +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + pEngine = &sEngine; + sqlite3ParserInit(pEngine, pParse); +#else + pEngine = sqlite3ParserAlloc(sqlite3Malloc, pParse); + if( pEngine==0 ){ + sqlite3OomFault(db); + return SQLITE_NOMEM_BKPT; + } +#endif + assert( pParse->pNewTable==0 ); + assert( pParse->pNewTrigger==0 ); + assert( pParse->nVar==0 ); + assert( pParse->pVList==0 ); + pParse->pParentParse = db->pParse; + db->pParse = pParse; + while( 1 ){ + n = sqlite3GetToken((u8*)zSql, &tokenType); + mxSqlLen -= n; + if( mxSqlLen<0 ){ + pParse->rc = SQLITE_TOOBIG; + break; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( tokenType>=TK_WINDOW ){ + assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER + || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW + ); +#else + if( tokenType>=TK_SPACE ){ + assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); +#endif /* SQLITE_OMIT_WINDOWFUNC */ + if( AtomicLoad(&db->u1.isInterrupted) ){ + pParse->rc = SQLITE_INTERRUPT; + break; + } + if( tokenType==TK_SPACE ){ + zSql += n; + continue; + } + if( zSql[0]==0 ){ + /* Upon reaching the end of input, call the parser two more times + ** with tokens TK_SEMI and 0, in that order. */ + if( lastTokenParsed==TK_SEMI ){ + tokenType = 0; + }else if( lastTokenParsed==0 ){ + break; + }else{ + tokenType = TK_SEMI; + } + n = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + }else if( tokenType==TK_WINDOW ){ + assert( n==6 ); + tokenType = analyzeWindowKeyword((const u8*)&zSql[6]); + }else if( tokenType==TK_OVER ){ + assert( n==4 ); + tokenType = analyzeOverKeyword((const u8*)&zSql[4], lastTokenParsed); + }else if( tokenType==TK_FILTER ){ + assert( n==6 ); + tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); +#endif /* SQLITE_OMIT_WINDOWFUNC */ + }else{ + sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql); + break; + } + } + pParse->sLastToken.z = zSql; + pParse->sLastToken.n = n; + sqlite3Parser(pEngine, tokenType, pParse->sLastToken); + lastTokenParsed = tokenType; + zSql += n; + assert( db->mallocFailed==0 || pParse->rc!=SQLITE_OK || startedWithOom ); + if( pParse->rc!=SQLITE_OK ) break; + } + assert( nErr==0 ); +#ifdef YYTRACKMAXSTACKDEPTH + sqlite3_mutex_enter(sqlite3MallocMutex()); + sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, + sqlite3ParserStackPeak(pEngine) + ); + sqlite3_mutex_leave(sqlite3MallocMutex()); +#endif /* YYDEBUG */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + sqlite3ParserFinalize(pEngine); +#else + sqlite3ParserFree(pEngine, sqlite3_free); +#endif + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM_BKPT; + } + if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ + pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); + } + assert( pzErrMsg!=0 ); + if( pParse->zErrMsg ){ + *pzErrMsg = pParse->zErrMsg; + sqlite3_log(pParse->rc, "%s in \"%s\"", + *pzErrMsg, pParse->zTail); + pParse->zErrMsg = 0; + nErr++; + } + pParse->zTail = zSql; + if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ + sqlite3VdbeDelete(pParse->pVdbe); + pParse->pVdbe = 0; + } +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pParse->nested==0 ){ + sqlite3DbFree(db, pParse->aTableLock); + pParse->aTableLock = 0; + pParse->nTableLock = 0; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_free(pParse->apVtabLock); +#endif + + if( !IN_SPECIAL_PARSE ){ + /* If the pParse->declareVtab flag is set, do not delete any table + ** structure built up in pParse->pNewTable. The calling code (see vtab.c) + ** will take responsibility for freeing the Table structure. + */ + sqlite3DeleteTable(db, pParse->pNewTable); + } + if( !IN_RENAME_OBJECT ){ + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + } + sqlite3DbFree(db, pParse->pVList); + db->pParse = pParse->pParentParse; + pParse->pParentParse = 0; + assert( nErr==0 || pParse->rc!=SQLITE_OK ); + return nErr; +} + + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Insert a single space character into pStr if the current string +** ends with an identifier +*/ +static void addSpaceSeparator(sqlite3_str *pStr){ + if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){ + sqlite3_str_append(pStr, " ", 1); + } +} + +/* +** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return +** the normalization in space obtained from sqlite3DbMalloc(). Or return +** NULL if anything goes wrong or if zSql is NULL. +*/ +SQLITE_PRIVATE char *sqlite3Normalize( + Vdbe *pVdbe, /* VM being reprepared */ + const char *zSql /* The original SQL string */ +){ + sqlite3 *db; /* The database connection */ + int i; /* Next unread byte of zSql[] */ + int n; /* length of current token */ + int tokenType; /* type of current token */ + int prevType = 0; /* Previous non-whitespace token */ + int nParen; /* Number of nested levels of parentheses */ + int iStartIN; /* Start of RHS of IN operator in z[] */ + int nParenAtIN; /* Value of nParent at start of RHS of IN operator */ + u32 j; /* Bytes of normalized SQL generated so far */ + sqlite3_str *pStr; /* The normalized SQL string under construction */ + + db = sqlite3VdbeDb(pVdbe); + tokenType = -1; + nParen = iStartIN = nParenAtIN = 0; + pStr = sqlite3_str_new(db); + assert( pStr!=0 ); /* sqlite3_str_new() never returns NULL */ + for(i=0; zSql[i] && pStr->accError==0; i+=n){ + if( tokenType!=TK_SPACE ){ + prevType = tokenType; + } + n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType); + if( NEVER(n<=0) ) break; + switch( tokenType ){ + case TK_SPACE: { + break; + } + case TK_NULL: { + if( prevType==TK_IS || prevType==TK_NOT ){ + sqlite3_str_append(pStr, " NULL", 5); + break; + } + /* Fall through */ + } + case TK_STRING: + case TK_INTEGER: + case TK_FLOAT: + case TK_VARIABLE: + case TK_BLOB: { + sqlite3_str_append(pStr, "?", 1); + break; + } + case TK_LP: { + nParen++; + if( prevType==TK_IN ){ + iStartIN = pStr->nChar; + nParenAtIN = nParen; + } + sqlite3_str_append(pStr, "(", 1); + break; + } + case TK_RP: { + if( iStartIN>0 && nParen==nParenAtIN ){ + assert( pStr->nChar>=(u32)iStartIN ); + pStr->nChar = iStartIN+1; + sqlite3_str_append(pStr, "?,?,?", 5); + iStartIN = 0; + } + nParen--; + sqlite3_str_append(pStr, ")", 1); + break; + } + case TK_ID: { + iStartIN = 0; + j = pStr->nChar; + if( sqlite3Isquote(zSql[i]) ){ + char *zId = sqlite3DbStrNDup(db, zSql+i, n); + int nId; + int eType = 0; + if( zId==0 ) break; + sqlite3Dequote(zId); + if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){ + sqlite3_str_append(pStr, "?", 1); + sqlite3DbFree(db, zId); + break; + } + nId = sqlite3Strlen30(zId); + if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){ + addSpaceSeparator(pStr); + sqlite3_str_append(pStr, zId, nId); + }else{ + sqlite3_str_appendf(pStr, "\"%w\"", zId); + } + sqlite3DbFree(db, zId); + }else{ + addSpaceSeparator(pStr); + sqlite3_str_append(pStr, zSql+i, n); + } + while( jnChar ){ + pStr->zText[j] = sqlite3Tolower(pStr->zText[j]); + j++; + } + break; + } + case TK_SELECT: { + iStartIN = 0; + /* fall through */ + } + default: { + if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr); + j = pStr->nChar; + sqlite3_str_append(pStr, zSql+i, n); + while( jnChar ){ + pStr->zText[j] = sqlite3Toupper(pStr->zText[j]); + j++; + } + break; + } + } + } + if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1); + return sqlite3_str_finish(pStr); +} +#endif /* SQLITE_ENABLE_NORMALIZE */ + +/************** End of tokenize.c ********************************************/ +/************** Begin file complete.c ****************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that implements the sqlite3_complete() API. +** This code used to be part of the tokenizer.c source file. But by +** separating it out, the code will be automatically omitted from +** static links that do not use it. +*/ +/* #include "sqliteInt.h" */ +#ifndef SQLITE_OMIT_COMPLETE + +/* +** This is defined in tokenize.c. We just have to import the definition. +*/ +#ifndef SQLITE_AMALGAMATION +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif +#endif /* SQLITE_AMALGAMATION */ + + +/* +** Token types used by the sqlite3_complete() routine. See the header +** comments on that procedure for additional information. +*/ +#define tkSEMI 0 +#define tkWS 1 +#define tkOTHER 2 +#ifndef SQLITE_OMIT_TRIGGER +#define tkEXPLAIN 3 +#define tkCREATE 4 +#define tkTEMP 5 +#define tkTRIGGER 6 +#define tkEND 7 +#endif + +/* +** Return TRUE if the given SQL string ends in a semicolon. +** +** Special handling is require for CREATE TRIGGER statements. +** Whenever the CREATE TRIGGER keywords are seen, the statement +** must end with ";END;". +** +** This implementation uses a state machine with 8 states: +** +** (0) INVALID We have not yet seen a non-whitespace character. +** +** (1) START At the beginning or end of an SQL statement. This routine +** returns 1 if it ends in the START state and 0 if it ends +** in any other state. +** +** (2) NORMAL We are in the middle of statement which ends with a single +** semicolon. +** +** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of +** a statement. +** +** (4) CREATE The keyword CREATE has been seen at the beginning of a +** statement, possibly preceded by EXPLAIN and/or followed by +** TEMP or TEMPORARY +** +** (5) TRIGGER We are in the middle of a trigger definition that must be +** ended by a semicolon, the keyword END, and another semicolon. +** +** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at +** the end of a trigger definition. +** +** (7) END We've seen the ";END" of the ";END;" that occurs at the end +** of a trigger definition. +** +** Transitions between states above are determined by tokens extracted +** from the input. The following tokens are significant: +** +** (0) tkSEMI A semicolon. +** (1) tkWS Whitespace. +** (2) tkOTHER Any other SQL token. +** (3) tkEXPLAIN The "explain" keyword. +** (4) tkCREATE The "create" keyword. +** (5) tkTEMP The "temp" or "temporary" keyword. +** (6) tkTRIGGER The "trigger" keyword. +** (7) tkEND The "end" keyword. +** +** Whitespace never causes a state transition and is always ignored. +** This means that a SQL string of all whitespace is invalid. +** +** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed +** to recognize the end of a trigger can be omitted. All we have to do +** is look for a semicolon that is not part of an string or comment. +*/ +SQLITE_API int sqlite3_complete(const char *zSql){ + u8 state = 0; /* Current state, using numbers defined in header comment */ + u8 token; /* Value of the next token */ + +#ifndef SQLITE_OMIT_TRIGGER + /* A complex statement machine used to detect the end of a CREATE TRIGGER + ** statement. This is the normal case. + */ + static const u8 trans[8][8] = { + /* Token: */ + /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ + /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, + /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, + /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, + /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, + /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, + /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, + /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, + }; +#else + /* If triggers are not supported by this compile then the statement machine + ** used to detect the end of a statement is much simpler + */ + static const u8 trans[3][3] = { + /* Token: */ + /* State: ** SEMI WS OTHER */ + /* 0 INVALID: */ { 1, 0, 2, }, + /* 1 START: */ { 1, 1, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, }, + }; +#endif /* SQLITE_OMIT_TRIGGER */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + while( *zSql ){ + switch( *zSql ){ + case ';': { /* A semicolon */ + token = tkSEMI; + break; + } + case ' ': + case '\r': + case '\t': + case '\n': + case '\f': { /* White space is ignored */ + token = tkWS; + break; + } + case '/': { /* C-style comments */ + if( zSql[1]!='*' ){ + token = tkOTHER; + break; + } + zSql += 2; + while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } + if( zSql[0]==0 ) return 0; + zSql++; + token = tkWS; + break; + } + case '-': { /* SQL-style comments from "--" to end of line */ + if( zSql[1]!='-' ){ + token = tkOTHER; + break; + } + while( *zSql && *zSql!='\n' ){ zSql++; } + if( *zSql==0 ) return state==1; + token = tkWS; + break; + } + case '[': { /* Microsoft-style identifiers in [...] */ + zSql++; + while( *zSql && *zSql!=']' ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + case '`': /* Grave-accent quoted symbols used by MySQL */ + case '"': /* single- and double-quoted strings */ + case '\'': { + int c = *zSql; + zSql++; + while( *zSql && *zSql!=c ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + default: { +#ifdef SQLITE_EBCDIC + unsigned char c; +#endif + if( IdChar((u8)*zSql) ){ + /* Keywords and unquoted identifiers */ + int nId; + for(nId=1; IdChar(zSql[nId]); nId++){} +#ifdef SQLITE_OMIT_TRIGGER + token = tkOTHER; +#else + switch( *zSql ){ + case 'c': case 'C': { + if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ + token = tkCREATE; + }else{ + token = tkOTHER; + } + break; + } + case 't': case 'T': { + if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ + token = tkTRIGGER; + }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ + token = tkTEMP; + }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ + token = tkTEMP; + }else{ + token = tkOTHER; + } + break; + } + case 'e': case 'E': { + if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ + token = tkEND; + }else +#ifndef SQLITE_OMIT_EXPLAIN + if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ + token = tkEXPLAIN; + }else +#endif + { + token = tkOTHER; + } + break; + } + default: { + token = tkOTHER; + break; + } + } +#endif /* SQLITE_OMIT_TRIGGER */ + zSql += nId-1; + }else{ + /* Operators and special symbols */ + token = tkOTHER; + } + break; + } + } + state = trans[state][token]; + zSql++; + } + return state==1; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** This routine is the same as the sqlite3_complete() routine described +** above, except that the parameter is required to be UTF-16 encoded, not +** UTF-8. +*/ +SQLITE_API int sqlite3_complete16(const void *zSql){ + sqlite3_value *pVal; + char const *zSql8; + int rc; + +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zSql8 ){ + rc = sqlite3_complete(zSql8); + }else{ + rc = SQLITE_NOMEM_BKPT; + } + sqlite3ValueFree(pVal); + return rc & 0xff; +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_OMIT_COMPLETE */ + +/************** End of complete.c ********************************************/ +/************** Begin file main.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. +*/ +/* #include "sqliteInt.h" */ + +#ifdef SQLITE_ENABLE_FTS3 +/************** Include fts3.h in the middle of main.c ***********************/ +/************** Begin file fts3.h ********************************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. +*/ +/* #include "sqlite3.h" */ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of fts3.h ************************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_RTREE +/************** Include rtree.h in the middle of main.c **********************/ +/************** Begin file rtree.h *******************************************/ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** RTREE library. All it does is declare the sqlite3RtreeInit() interface. +*/ +/* #include "sqlite3.h" */ + +#ifdef SQLITE_OMIT_VIRTUALTABLE +# undef SQLITE_ENABLE_RTREE +#endif + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of rtree.h ***********************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) +/************** Include sqliteicu.h in the middle of main.c ******************/ +/************** Begin file sqliteicu.h ***************************************/ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** ICU extension. All it does is declare the sqlite3IcuInit() interface. +*/ +/* #include "sqlite3.h" */ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of sqliteicu.h *******************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif + +/* +** This is an extension initializer that is a no-op and always +** succeeds, except that it fails if the fault-simulation is set +** to 500. +*/ +static int sqlite3TestExtInit(sqlite3 *db){ + (void)db; + return sqlite3FaultSim(500); +} + + +/* +** Forward declarations of external module initializer functions +** for modules that need them. +*/ +#ifdef SQLITE_ENABLE_FTS1 +SQLITE_PRIVATE int sqlite3Fts1Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_FTS2 +SQLITE_PRIVATE int sqlite3Fts2Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_FTS5 +SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_JSON1 +SQLITE_PRIVATE int sqlite3Json1Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_STMTVTAB +SQLITE_PRIVATE int sqlite3StmtVtabInit(sqlite3*); +#endif + +/* +** An array of pointers to extension initializer functions for +** built-in extensions. +*/ +static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { +#ifdef SQLITE_ENABLE_FTS1 + sqlite3Fts1Init, +#endif +#ifdef SQLITE_ENABLE_FTS2 + sqlite3Fts2Init, +#endif +#ifdef SQLITE_ENABLE_FTS3 + sqlite3Fts3Init, +#endif +#ifdef SQLITE_ENABLE_FTS5 + sqlite3Fts5Init, +#endif +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) + sqlite3IcuInit, +#endif +#ifdef SQLITE_ENABLE_RTREE + sqlite3RtreeInit, +#endif +#ifdef SQLITE_ENABLE_DBPAGE_VTAB + sqlite3DbpageRegister, +#endif +#ifdef SQLITE_ENABLE_DBSTAT_VTAB + sqlite3DbstatRegister, +#endif + sqlite3TestExtInit, +#ifdef SQLITE_ENABLE_JSON1 + sqlite3Json1Init, +#endif +#ifdef SQLITE_ENABLE_STMTVTAB + sqlite3StmtVtabInit, +#endif +#ifdef SQLITE_ENABLE_BYTECODE_VTAB + sqlite3VdbeBytecodeVtabInit, +#endif +}; + +#ifndef SQLITE_AMALGAMATION +/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant +** contains the text of SQLITE_VERSION macro. +*/ +SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; +#endif + +/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns +** a pointer to the to the sqlite3_version[] string constant. +*/ +SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } + +/* IMPLEMENTATION-OF: R-25063-23286 The sqlite3_sourceid() function returns a +** pointer to a string constant whose value is the same as the +** SQLITE_SOURCE_ID C preprocessor macro. Except if SQLite is built using +** an edited copy of the amalgamation, then the last four characters of +** the hash might be different from SQLITE_SOURCE_ID. +*/ +/* SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } */ + +/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function +** returns an integer equal to SQLITE_VERSION_NUMBER. +*/ +SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + +/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled with mutexing code omitted due to +** the SQLITE_THREADSAFE compile-time option being set to 0. +*/ +SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } + +/* +** When compiling the test fixture or with debugging enabled (on Win32), +** this variable being set to non-zero will cause OSTRACE macros to emit +** extra diagnostic information. +*/ +#ifdef SQLITE_HAVE_OS_TRACE +# ifndef SQLITE_DEBUG_OS_TRACE +# define SQLITE_DEBUG_OS_TRACE 0 +# endif + int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +#endif + +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) +/* +** If the following function pointer is not NULL and if +** SQLITE_ENABLE_IOTRACE is enabled, then messages describing +** I/O active are written using this function. These messages +** are intended for debugging activity only. +*/ +SQLITE_API void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...) = 0; +#endif + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** temporary files. +** +** See also the "PRAGMA temp_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_temp_directory = 0; + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** all database files specified with a relative pathname. +** +** See also the "PRAGMA data_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_data_directory = 0; + +/* +** Initialize SQLite. +** +** This routine must be called to initialize the memory allocation, +** VFS, and mutex subsystems prior to doing any serious work with +** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT +** this routine will be called automatically by key routines such as +** sqlite3_open(). +** +** This routine is a no-op except on its very first call for the process, +** or for the first call after a call to sqlite3_shutdown. +** +** The first thread to call this routine runs the initialization to +** completion. If subsequent threads call this routine before the first +** thread has finished the initialization process, then the subsequent +** threads must block until the first thread finishes with the initialization. +** +** The first thread might call this routine recursively. Recursive +** calls to this routine should not block, of course. Otherwise the +** initialization process would never complete. +** +** Let X be the first thread to enter this routine. Let Y be some other +** thread. Then while the initial invocation of this routine by X is +** incomplete, it is required that: +** +** * Calls to this routine from Y must block until the outer-most +** call by X completes. +** +** * Recursive calls to this routine from thread X return immediately +** without blocking. +*/ +SQLITE_API int sqlite3_initialize(void){ + MUTEX_LOGIC( sqlite3_mutex *pMainMtx; ) /* The main static mutex */ + int rc; /* Result code */ +#ifdef SQLITE_EXTRA_INIT + int bRunExtraInit = 0; /* Extra initialization needed */ +#endif + +#ifdef SQLITE_OMIT_WSD + rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + /* If the following assert() fails on some obscure processor/compiler + ** combination, the work-around is to set the correct pointer + ** size at compile-time using -DSQLITE_PTRSIZE=n compile-time option */ + assert( SQLITE_PTRSIZE==sizeof(char*) ); + + /* If SQLite is already completely initialized, then this call + ** to sqlite3_initialize() should be a no-op. But the initialization + ** must be complete. So isInit must not be set until the very end + ** of this routine. + */ + if( sqlite3GlobalConfig.isInit ){ + sqlite3MemoryBarrier(); + return SQLITE_OK; + } + + /* Make sure the mutex subsystem is initialized. If unable to + ** initialize the mutex subsystem, return early with the error. + ** If the system is so sick that we are unable to allocate a mutex, + ** there is not much SQLite is going to be able to do. + ** + ** The mutex subsystem must take care of serializing its own + ** initialization. + */ + rc = sqlite3MutexInit(); + if( rc ) return rc; + + /* Initialize the malloc() system and the recursive pInitMutex mutex. + ** This operation is protected by the STATIC_MAIN mutex. Note that + ** MutexAlloc() is called for a static mutex prior to initializing the + ** malloc subsystem - this implies that the allocation of a static + ** mutex must not require support from the malloc subsystem. + */ + MUTEX_LOGIC( pMainMtx = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) + sqlite3_mutex_enter(pMainMtx); + sqlite3GlobalConfig.isMutexInit = 1; + if( !sqlite3GlobalConfig.isMallocInit ){ + rc = sqlite3MallocInit(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isMallocInit = 1; + if( !sqlite3GlobalConfig.pInitMutex ){ + sqlite3GlobalConfig.pInitMutex = + sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ + rc = SQLITE_NOMEM_BKPT; + } + } + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.nRefInitMutex++; + } + sqlite3_mutex_leave(pMainMtx); + + /* If rc is not SQLITE_OK at this point, then either the malloc + ** subsystem could not be initialized or the system failed to allocate + ** the pInitMutex mutex. Return an error in either case. */ + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Do the rest of the initialization under the recursive mutex so + ** that we will be able to handle recursive calls into + ** sqlite3_initialize(). The recursive calls normally come through + ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other + ** recursive calls might also be possible. + ** + ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls + ** to the xInit method, so the xInit method need not be threadsafe. + ** + ** The following mutex is what serializes access to the appdef pcache xInit + ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the + ** call to sqlite3PcacheInitialize(). + */ + sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); + if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ + sqlite3GlobalConfig.inProgress = 1; +#ifdef SQLITE_ENABLE_SQLLOG + { + extern void sqlite3_init_sqllog(void); + sqlite3_init_sqllog(); + } +#endif + memset(&sqlite3BuiltinFunctions, 0, sizeof(sqlite3BuiltinFunctions)); + sqlite3RegisterBuiltinFunctions(); + if( sqlite3GlobalConfig.isPCacheInit==0 ){ + rc = sqlite3PcacheInitialize(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isPCacheInit = 1; + rc = sqlite3OsInit(); + } +#ifndef SQLITE_OMIT_DESERIALIZE + if( rc==SQLITE_OK ){ + rc = sqlite3MemdbInit(); + } +#endif + if( rc==SQLITE_OK ){ + sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, + sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); + sqlite3MemoryBarrier(); + sqlite3GlobalConfig.isInit = 1; +#ifdef SQLITE_EXTRA_INIT + bRunExtraInit = 1; +#endif + } + sqlite3GlobalConfig.inProgress = 0; + } + sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); + + /* Go back under the static mutex and clean up the recursive + ** mutex to prevent a resource leak. + */ + sqlite3_mutex_enter(pMainMtx); + sqlite3GlobalConfig.nRefInitMutex--; + if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ + assert( sqlite3GlobalConfig.nRefInitMutex==0 ); + sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); + sqlite3GlobalConfig.pInitMutex = 0; + } + sqlite3_mutex_leave(pMainMtx); + + /* The following is just a sanity check to make sure SQLite has + ** been compiled correctly. It is important to run this code, but + ** we don't want to run it too often and soak up CPU cycles for no + ** reason. So we run it once during initialization. + */ +#ifndef NDEBUG +#ifndef SQLITE_OMIT_FLOATING_POINT + /* This section of code's only "output" is via assert() statements. */ + if( rc==SQLITE_OK ){ + u64 x = (((u64)1)<<63)-1; + double y; + assert(sizeof(x)==8); + assert(sizeof(x)==sizeof(y)); + memcpy(&y, &x, 8); + assert( sqlite3IsNaN(y) ); + } +#endif +#endif + + /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT + ** compile-time option. + */ +#ifdef SQLITE_EXTRA_INIT + if( bRunExtraInit ){ + int SQLITE_EXTRA_INIT(const char*); + rc = SQLITE_EXTRA_INIT(0); + } +#endif + + return rc; +} + +/* +** Undo the effects of sqlite3_initialize(). Must not be called while +** there are outstanding database connections or memory allocations or +** while any part of SQLite is otherwise in use in any thread. This +** routine is not threadsafe. But it is safe to invoke this routine +** on when SQLite is already shut down. If SQLite is already shut down +** when this routine is invoked, then this routine is a harmless no-op. +*/ +SQLITE_API int sqlite3_shutdown(void){ +#ifdef SQLITE_OMIT_WSD + int rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + if( sqlite3GlobalConfig.isInit ){ +#ifdef SQLITE_EXTRA_SHUTDOWN + void SQLITE_EXTRA_SHUTDOWN(void); + SQLITE_EXTRA_SHUTDOWN(); +#endif + sqlite3_os_end(); + sqlite3_reset_auto_extension(); + sqlite3GlobalConfig.isInit = 0; + } + if( sqlite3GlobalConfig.isPCacheInit ){ + sqlite3PcacheShutdown(); + sqlite3GlobalConfig.isPCacheInit = 0; + } + if( sqlite3GlobalConfig.isMallocInit ){ + sqlite3MallocEnd(); + sqlite3GlobalConfig.isMallocInit = 0; + +#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES + /* The heap subsystem has now been shutdown and these values are supposed + ** to be NULL or point to memory that was obtained from sqlite3_malloc(), + ** which would rely on that heap subsystem; therefore, make sure these + ** values cannot refer to heap memory that was just invalidated when the + ** heap subsystem was shutdown. This is only done if the current call to + ** this function resulted in the heap subsystem actually being shutdown. + */ + sqlite3_data_directory = 0; + sqlite3_temp_directory = 0; +#endif + } + if( sqlite3GlobalConfig.isMutexInit ){ + sqlite3MutexEnd(); + sqlite3GlobalConfig.isMutexInit = 0; + } + + return SQLITE_OK; +} + +/* +** This API allows applications to modify the global configuration of +** the SQLite library at run-time. +** +** This routine should only be called when there are no outstanding +** database connections or memory allocations. This routine is not +** threadsafe. Failure to heed these warnings can lead to unpredictable +** behavior. +*/ +SQLITE_API int sqlite3_config(int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + + va_start(ap, op); + switch( op ){ + + /* Mutex configuration options are only available in a threadsafe + ** compile. + */ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ + case SQLITE_CONFIG_SINGLETHREAD: { + /* EVIDENCE-OF: R-02748-19096 This option sets the threading mode to + ** Single-thread. */ + sqlite3GlobalConfig.bCoreMutex = 0; /* Disable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ + case SQLITE_CONFIG_MULTITHREAD: { + /* EVIDENCE-OF: R-14374-42468 This option sets the threading mode to + ** Multi-thread. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ + case SQLITE_CONFIG_SERIALIZED: { + /* EVIDENCE-OF: R-41220-51800 This option sets the threading mode to + ** Serialized. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 1; /* Enable mutex on connections */ + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ + case SQLITE_CONFIG_MUTEX: { + /* Specify an alternative mutex implementation */ + sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ + case SQLITE_CONFIG_GETMUTEX: { + /* Retrieve the current mutex implementation */ + *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; + break; + } +#endif + + case SQLITE_CONFIG_MALLOC: { + /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The argument specifies alternative + ** low-level memory allocation routines to be used in place of the memory + ** allocation routines built into SQLite. */ + sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); + break; + } + case SQLITE_CONFIG_GETMALLOC: { + /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is + ** filled with the currently defined memory allocation routines. */ + if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); + *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; + break; + } + case SQLITE_CONFIG_MEMSTATUS: { + /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes + ** single argument of type int, interpreted as a boolean, which enables + ** or disables the collection of memory allocation statistics. */ + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_SMALL_MALLOC: { + sqlite3GlobalConfig.bSmallMalloc = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PAGECACHE: { + /* EVIDENCE-OF: R-18761-36601 There are three arguments to + ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory (pMem), + ** the size of each page cache line (sz), and the number of cache lines + ** (N). */ + sqlite3GlobalConfig.pPage = va_arg(ap, void*); + sqlite3GlobalConfig.szPage = va_arg(ap, int); + sqlite3GlobalConfig.nPage = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PCACHE_HDRSZ: { + /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes + ** a single parameter which is a pointer to an integer and writes into + ** that integer the number of extra bytes per page required for each page + ** in SQLITE_CONFIG_PAGECACHE. */ + *va_arg(ap, int*) = + sqlite3HeaderSizeBtree() + + sqlite3HeaderSizePcache() + + sqlite3HeaderSizePcache1(); + break; + } + + case SQLITE_CONFIG_PCACHE: { + /* no-op */ + break; + } + case SQLITE_CONFIG_GETPCACHE: { + /* now an error */ + rc = SQLITE_ERROR; + break; + } + + case SQLITE_CONFIG_PCACHE2: { + /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. This object specifies the interface to a custom page cache + ** implementation. */ + sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); + break; + } + case SQLITE_CONFIG_GETPCACHE2: { + /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. SQLite copies of the current page cache implementation into + ** that object. */ + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ + sqlite3PCacheSetDefault(); + } + *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; + break; + } + +/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only +** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or +** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */ +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + case SQLITE_CONFIG_HEAP: { + /* EVIDENCE-OF: R-19854-42126 There are three arguments to + ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the + ** number of bytes in the memory buffer, and the minimum allocation size. + */ + sqlite3GlobalConfig.pHeap = va_arg(ap, void*); + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + sqlite3GlobalConfig.mnReq = va_arg(ap, int); + + if( sqlite3GlobalConfig.mnReq<1 ){ + sqlite3GlobalConfig.mnReq = 1; + }else if( sqlite3GlobalConfig.mnReq>(1<<12) ){ + /* cap min request size at 2^12 */ + sqlite3GlobalConfig.mnReq = (1<<12); + } + + if( sqlite3GlobalConfig.pHeap==0 ){ + /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) + ** is NULL, then SQLite reverts to using its default memory allocator + ** (the system malloc() implementation), undoing any prior invocation of + ** SQLITE_CONFIG_MALLOC. + ** + ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to + ** revert to its default implementation when sqlite3_initialize() is run + */ + memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); + }else{ + /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the + ** alternative memory allocator is engaged to handle all of SQLites + ** memory allocation needs. */ +#ifdef SQLITE_ENABLE_MEMSYS3 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); +#endif + } + break; + } +#endif + + case SQLITE_CONFIG_LOOKASIDE: { + sqlite3GlobalConfig.szLookaside = va_arg(ap, int); + sqlite3GlobalConfig.nLookaside = va_arg(ap, int); + break; + } + + /* Record a pointer to the logger function and its first argument. + ** The default is NULL. Logging is disabled if the function pointer is + ** NULL. + */ + case SQLITE_CONFIG_LOG: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); + */ + typedef void(*LOGFUNC_t)(void*,int,const char*); + sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); + sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + break; + } + + /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames + ** can be changed at start-time using the + ** sqlite3_config(SQLITE_CONFIG_URI,1) or + ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. + */ + case SQLITE_CONFIG_URI: { + /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single + ** argument of type int. If non-zero, then URI handling is globally + ** enabled. If the parameter is zero, then URI handling is globally + ** disabled. */ + sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + break; + } + + case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN + ** option takes a single integer argument which is interpreted as a + ** boolean in order to enable or disable the use of covering indices for + ** full table scans in the query optimizer. */ + sqlite3GlobalConfig.bUseCis = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif + + case SQLITE_CONFIG_MMAP_SIZE: { + /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit + ** integer (sqlite3_int64) values that are the default mmap size limit + ** (the default setting for PRAGMA mmap_size) and the maximum allowed + ** mmap size limit. */ + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + /* EVIDENCE-OF: R-53367-43190 If either argument to this option is + ** negative, then that argument is changed to its compile-time default. + ** + ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be + ** silently truncated if necessary so that it does not exceed the + ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE + ** compile-time option. + */ + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.mxMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } + +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ + case SQLITE_CONFIG_WIN32_HEAPSIZE: { + /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit + ** unsigned integer value that specifies the maximum size of the created + ** heap. */ + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + break; + } +#endif + + case SQLITE_CONFIG_PMASZ: { + sqlite3GlobalConfig.szPma = va_arg(ap, unsigned int); + break; + } + + case SQLITE_CONFIG_STMTJRNL_SPILL: { + sqlite3GlobalConfig.nStmtSpill = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + case SQLITE_CONFIG_SORTERREF_SIZE: { + int iVal = va_arg(ap, int); + if( iVal<0 ){ + iVal = SQLITE_DEFAULT_SORTERREF_SIZE; + } + sqlite3GlobalConfig.szSorterRef = (u32)iVal; + break; + } +#endif /* SQLITE_ENABLE_SORTER_REFERENCES */ + +#ifndef SQLITE_OMIT_DESERIALIZE + case SQLITE_CONFIG_MEMDB_MAXSIZE: { + sqlite3GlobalConfig.mxMemdbSize = va_arg(ap, sqlite3_int64); + break; + } +#endif /* SQLITE_OMIT_DESERIALIZE */ + + default: { + rc = SQLITE_ERROR; + break; + } + } + va_end(ap); + return rc; +} + +/* +** Set up the lookaside buffers for a database connection. +** Return SQLITE_OK on success. +** If lookaside is already active, return SQLITE_BUSY. +** +** The sz parameter is the number of bytes in each lookaside slot. +** The cnt parameter is the number of slots. If pStart is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc(). +** If pStart is not NULL then it is sz*cnt bytes of memory to use for +** the lookaside memory. +*/ +static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ +#ifndef SQLITE_OMIT_LOOKASIDE + void *pStart; + sqlite3_int64 szAlloc = sz*(sqlite3_int64)cnt; + int nBig; /* Number of full-size slots */ + int nSm; /* Number smaller LOOKASIDE_SMALL-byte slots */ + + if( sqlite3LookasideUsed(db,0)>0 ){ + return SQLITE_BUSY; + } + /* Free any existing lookaside buffer for this handle before + ** allocating a new one so we don't have to have space for + ** both at the same time. + */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger + ** than a pointer to be useful. + */ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; + if( cnt<0 ) cnt = 0; + if( sz==0 || cnt==0 ){ + sz = 0; + pStart = 0; + }else if( pBuf==0 ){ + sqlite3BeginBenignMalloc(); + pStart = sqlite3Malloc( szAlloc ); /* IMP: R-61949-35727 */ + sqlite3EndBenignMalloc(); + if( pStart ) szAlloc = sqlite3MallocSize(pStart); + }else{ + pStart = pBuf; + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( sz>=LOOKASIDE_SMALL*3 ){ + nBig = szAlloc/(3*LOOKASIDE_SMALL+sz); + nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + }else if( sz>=LOOKASIDE_SMALL*2 ){ + nBig = szAlloc/(LOOKASIDE_SMALL+sz); + nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + }else +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( sz>0 ){ + nBig = szAlloc/sz; + nSm = 0; + }else{ + nBig = nSm = 0; + } + db->lookaside.pStart = pStart; + db->lookaside.pInit = 0; + db->lookaside.pFree = 0; + db->lookaside.sz = (u16)sz; + db->lookaside.szTrue = (u16)sz; + if( pStart ){ + int i; + LookasideSlot *p; + assert( sz > (int)sizeof(LookasideSlot*) ); + p = (LookasideSlot*)pStart; + for(i=0; ipNext = db->lookaside.pInit; + db->lookaside.pInit = p; + p = (LookasideSlot*)&((u8*)p)[sz]; + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + db->lookaside.pSmallInit = 0; + db->lookaside.pSmallFree = 0; + db->lookaside.pMiddle = p; + for(i=0; ipNext = db->lookaside.pSmallInit; + db->lookaside.pSmallInit = p; + p = (LookasideSlot*)&((u8*)p)[LOOKASIDE_SMALL]; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + assert( ((uptr)p)<=szAlloc + (uptr)pStart ); + db->lookaside.pEnd = p; + db->lookaside.bDisable = 0; + db->lookaside.bMalloced = pBuf==0 ?1:0; + db->lookaside.nSlot = nBig+nSm; + }else{ + db->lookaside.pStart = db; +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + db->lookaside.pSmallInit = 0; + db->lookaside.pSmallFree = 0; + db->lookaside.pMiddle = db; +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + db->lookaside.pEnd = db; + db->lookaside.bDisable = 1; + db->lookaside.sz = 0; + db->lookaside.bMalloced = 0; + db->lookaside.nSlot = 0; + } + assert( sqlite3LookasideUsed(db,0)==0 ); +#endif /* SQLITE_OMIT_LOOKASIDE */ + return SQLITE_OK; +} + +/* +** Return the mutex associated with a database connection. +*/ +SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->mutex; +} + +/* +** Free up as much memory as we can from the given database +** connection. +*/ +SQLITE_API int sqlite3_db_release_memory(sqlite3 *db){ + int i; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerShrink(pPager); + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Flush any dirty pages in the pager-cache for any attached database +** to disk. +*/ +SQLITE_API int sqlite3_db_cacheflush(sqlite3 *db){ + int i; + int rc = SQLITE_OK; + int bSeenBusy = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt && sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){ + Pager *pPager = sqlite3BtreePager(pBt); + rc = sqlite3PagerFlush(pPager); + if( rc==SQLITE_BUSY ){ + bSeenBusy = 1; + rc = SQLITE_OK; + } + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return ((rc==SQLITE_OK && bSeenBusy) ? SQLITE_BUSY : rc); +} + +/* +** Configuration settings for an individual database connection +*/ +SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc; + va_start(ap, op); + switch( op ){ + case SQLITE_DBCONFIG_MAINDBNAME: { + /* IMP: R-06824-28531 */ + /* IMP: R-36257-52125 */ + db->aDb[0].zDbSName = va_arg(ap,char*); + rc = SQLITE_OK; + break; + } + case SQLITE_DBCONFIG_LOOKASIDE: { + void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ + int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ + int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ + rc = setupLookaside(db, pBuf, sz, cnt); + break; + } + default: { + static const struct { + int op; /* The opcode */ + u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ + } aFlagOp[] = { + { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, + { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + { SQLITE_DBCONFIG_ENABLE_VIEW, SQLITE_EnableView }, + { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, + { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, + { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, SQLITE_NoCkptOnClose }, + { SQLITE_DBCONFIG_ENABLE_QPSG, SQLITE_EnableQPSG }, + { SQLITE_DBCONFIG_TRIGGER_EQP, SQLITE_TriggerEQP }, + { SQLITE_DBCONFIG_RESET_DATABASE, SQLITE_ResetDatabase }, + { SQLITE_DBCONFIG_DEFENSIVE, SQLITE_Defensive }, + { SQLITE_DBCONFIG_WRITABLE_SCHEMA, SQLITE_WriteSchema| + SQLITE_NoSchemaError }, + { SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, SQLITE_LegacyAlter }, + { SQLITE_DBCONFIG_DQS_DDL, SQLITE_DqsDDL }, + { SQLITE_DBCONFIG_DQS_DML, SQLITE_DqsDML }, + { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, SQLITE_LegacyFileFmt }, + { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, + }; + unsigned int i; + rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ + for(i=0; iflags; + if( onoff>0 ){ + db->flags |= aFlagOp[i].mask; + }else if( onoff==0 ){ + db->flags &= ~(u64)aFlagOp[i].mask; + } + if( oldFlags!=db->flags ){ + sqlite3ExpirePreparedStatements(db, 0); + } + if( pRes ){ + *pRes = (db->flags & aFlagOp[i].mask)!=0; + } + rc = SQLITE_OK; + break; + } + } + break; + } + } + va_end(ap); + return rc; +} + +/* +** This is the default collating function named "BINARY" which is always +** available. +*/ +static int binCollFunc( + void *NotUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int rc, n; + UNUSED_PARAMETER(NotUsed); + n = nKey1xCmp!=binCollFunc || strcmp(p->zName,"BINARY")==0 ); + return p==0 || p->xCmp==binCollFunc; +} + +/* +** Another built-in collating sequence: NOCASE. +** +** This collating sequence is intended to be used for "case independent +** comparison". SQLite's knowledge of upper and lower case equivalents +** extends only to the 26 characters used in the English language. +** +** At the moment there is only a UTF-8 implementation. +*/ +static int nocaseCollatingFunc( + void *NotUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int r = sqlite3StrNICmp( + (const char *)pKey1, (const char *)pKey2, (nKey1lastRowid; +} + +/* +** Set the value returned by the sqlite3_last_insert_rowid() API function. +*/ +SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3 *db, sqlite3_int64 iRowid){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + db->lastRowid = iRowid; + sqlite3_mutex_leave(db->mutex); +} + +/* +** Return the number of changes in the most recent call to sqlite3_exec(). +*/ +SQLITE_API sqlite3_int64 sqlite3_changes64(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->nChange; +} +SQLITE_API int sqlite3_changes(sqlite3 *db){ + return (int)sqlite3_changes64(db); +} + +/* +** Return the number of changes since the database handle was opened. +*/ +SQLITE_API sqlite3_int64 sqlite3_total_changes64(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->nTotalChange; +} +SQLITE_API int sqlite3_total_changes(sqlite3 *db){ + return (int)sqlite3_total_changes64(db); +} + +/* +** Close all open savepoints. This function only manipulates fields of the +** database handle object, it does not close any savepoints that may be open +** at the b-tree/pager level. +*/ +SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ + while( db->pSavepoint ){ + Savepoint *pTmp = db->pSavepoint; + db->pSavepoint = pTmp->pNext; + sqlite3DbFree(db, pTmp); + } + db->nSavepoint = 0; + db->nStatement = 0; + db->isTransactionSavepoint = 0; +} + +/* +** Invoke the destructor function associated with FuncDef p, if any. Except, +** if this is not the last copy of the function, do not invoke it. Multiple +** copies of a single function are created when create_function() is called +** with SQLITE_ANY as the encoding. +*/ +static void functionDestroy(sqlite3 *db, FuncDef *p){ + FuncDestructor *pDestructor = p->u.pDestructor; + if( pDestructor ){ + pDestructor->nRef--; + if( pDestructor->nRef==0 ){ + pDestructor->xDestroy(pDestructor->pUserData); + sqlite3DbFree(db, pDestructor); + } + } +} + +/* +** Disconnect all sqlite3_vtab objects that belong to database connection +** db. This is called when db is being closed. +*/ +static void disconnectAllVtab(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int i; + HashElem *p; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( pSchema ){ + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + Table *pTab = (Table *)sqliteHashData(p); + if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); + } + } + } + for(p=sqliteHashFirst(&db->aModule); p; p=sqliteHashNext(p)){ + Module *pMod = (Module *)sqliteHashData(p); + if( pMod->pEpoTab ){ + sqlite3VtabDisconnect(db, pMod->pEpoTab); + } + } + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); +#else + UNUSED_PARAMETER(db); +#endif +} + +/* +** Return TRUE if database connection db has unfinalized prepared +** statements or unfinished sqlite3_backup objects. +*/ +static int connectionIsBusy(sqlite3 *db){ + int j; + assert( sqlite3_mutex_held(db->mutex) ); + if( db->pVdbe ) return 1; + for(j=0; jnDb; j++){ + Btree *pBt = db->aDb[j].pBt; + if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; + } + return 0; +} + +/* +** Close an existing SQLite database +*/ +static int sqlite3Close(sqlite3 *db, int forceZombie){ + if( !db ){ + /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or + ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ + return SQLITE_OK; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + if( db->mTrace & SQLITE_TRACE_CLOSE ){ + db->trace.xV2(SQLITE_TRACE_CLOSE, db->pTraceArg, db, 0); + } + + /* Force xDisconnect calls on all virtual tables */ + disconnectAllVtab(db); + + /* If a transaction is open, the disconnectAllVtab() call above + ** will not have called the xDisconnect() method on any virtual + ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() + ** call will do so. We need to do this before the check for active + ** SQL statements below, as the v-table implementation may be storing + ** some prepared statements internally. + */ + sqlite3VtabRollback(db); + + /* Legacy behavior (sqlite3_close() behavior) is to return + ** SQLITE_BUSY if the connection can not be closed immediately. + */ + if( !forceZombie && connectionIsBusy(db) ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " + "statements or unfinished backups"); + sqlite3_mutex_leave(db->mutex); + return SQLITE_BUSY; + } + +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + } +#endif + + /* Convert the connection into a zombie and then close it. + */ + db->eOpenState = SQLITE_STATE_ZOMBIE; + sqlite3LeaveMutexAndCloseZombie(db); + return SQLITE_OK; +} + +/* +** Return the transaction state for a single databse, or the maximum +** transaction state over all attached databases if zSchema is null. +*/ +SQLITE_API int sqlite3_txn_state(sqlite3 *db, const char *zSchema){ + int iDb, nDb; + int iTxn = -1; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( zSchema ){ + nDb = iDb = sqlite3FindDbName(db, zSchema); + if( iDb<0 ) nDb--; + }else{ + iDb = 0; + nDb = db->nDb-1; + } + for(; iDb<=nDb; iDb++){ + Btree *pBt = db->aDb[iDb].pBt; + int x = pBt!=0 ? sqlite3BtreeTxnState(pBt) : SQLITE_TXN_NONE; + if( x>iTxn ) iTxn = x; + } + sqlite3_mutex_leave(db->mutex); + return iTxn; +} + +/* +** Two variations on the public interface for closing a database +** connection. The sqlite3_close() version returns SQLITE_BUSY and +** leaves the connection open if there are unfinalized prepared +** statements or unfinished sqlite3_backups. The sqlite3_close_v2() +** version forces the connection to become a zombie if there are +** unclosed resources, and arranges for deallocation when the last +** prepare statement or sqlite3_backup closes. +*/ +SQLITE_API int sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } +SQLITE_API int sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } + + +/* +** Close the mutex on database connection db. +** +** Furthermore, if database connection db is a zombie (meaning that there +** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and +** every sqlite3_stmt has now been finalized and every sqlite3_backup has +** finished, then free all resources. +*/ +SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ + HashElem *i; /* Hash table iterator */ + int j; + + /* If there are outstanding sqlite3_stmt or sqlite3_backup objects + ** or if the connection has not yet been closed by sqlite3_close_v2(), + ** then just leave the mutex and return. + */ + if( db->eOpenState!=SQLITE_STATE_ZOMBIE || connectionIsBusy(db) ){ + sqlite3_mutex_leave(db->mutex); + return; + } + + /* If we reach this point, it means that the database connection has + ** closed all sqlite3_stmt and sqlite3_backup objects and has been + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, + ** go ahead and free all resources. + */ + + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); + + /* Free any outstanding Savepoint structures. */ + sqlite3CloseSavepoints(db); + + /* Close all database connections */ + for(j=0; jnDb; j++){ + struct Db *pDb = &db->aDb[j]; + if( pDb->pBt ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + if( j!=1 ){ + pDb->pSchema = 0; + } + } + } + /* Clear the TEMP schema separately and last */ + if( db->aDb[1].pSchema ){ + sqlite3SchemaClear(db->aDb[1].pSchema); + } + sqlite3VtabUnlockList(db); + + /* Free up the array of auxiliary databases */ + sqlite3CollapseDatabaseArray(db); + assert( db->nDb<=2 ); + assert( db->aDb==db->aDbStatic ); + + /* Tell the code in notify.c that the connection no longer holds any + ** locks and does not require any further unlock-notify callbacks. + */ + sqlite3ConnectionClosed(db); + + for(i=sqliteHashFirst(&db->aFunc); i; i=sqliteHashNext(i)){ + FuncDef *pNext, *p; + p = sqliteHashData(i); + do{ + functionDestroy(db, p); + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + }while( p ); + } + sqlite3HashClear(&db->aFunc); + for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(i); + /* Invoke any destructors registered for collation sequence user data. */ + for(j=0; j<3; j++){ + if( pColl[j].xDel ){ + pColl[j].xDel(pColl[j].pUser); + } + } + sqlite3DbFree(db, pColl); + } + sqlite3HashClear(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ + Module *pMod = (Module *)sqliteHashData(i); + sqlite3VtabEponymousTableClear(db, pMod); + sqlite3VtabModuleUnref(db, pMod); + } + sqlite3HashClear(&db->aModule); +#endif + + sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ + sqlite3ValueFree(db->pErr); + sqlite3CloseExtensions(db); +#if SQLITE_USER_AUTHENTICATION + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); +#endif + + db->eOpenState = SQLITE_STATE_ERROR; + + /* The temp-database schema is allocated differently from the other schema + ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). + ** So it needs to be freed here. Todo: Why not roll the temp schema into + ** the same sqliteMalloc() as the one that allocates the database + ** structure? + */ + sqlite3DbFree(db, db->aDb[1].pSchema); + sqlite3_mutex_leave(db->mutex); + db->eOpenState = SQLITE_STATE_CLOSED; + sqlite3_mutex_free(db->mutex); + assert( sqlite3LookasideUsed(db,0)==0 ); + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + sqlite3_free(db); +} + +/* +** Rollback all database files. If tripCode is not SQLITE_OK, then +** any write cursors are invalidated ("tripped" - as in "tripping a circuit +** breaker") and made to return tripCode if there are any further +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. +*/ +SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ + int i; + int inTrans = 0; + int schemaChange; + assert( sqlite3_mutex_held(db->mutex) ); + sqlite3BeginBenignMalloc(); + + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + schemaChange = (db->mDbFlags & DBFLAG_SchemaChange)!=0 && db->init.busy==0; + + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p ){ + if( sqlite3BtreeTxnState(p)==SQLITE_TXN_WRITE ){ + inTrans = 1; + } + sqlite3BtreeRollback(p, tripCode, !schemaChange); + } + } + sqlite3VtabRollback(db); + sqlite3EndBenignMalloc(); + + if( schemaChange ){ + sqlite3ExpirePreparedStatements(db, 0); + sqlite3ResetAllSchemasOfConnection(db); + } + sqlite3BtreeLeaveAll(db); + + /* Any deferred constraint violations have now been resolved. */ + db->nDeferredCons = 0; + db->nDeferredImmCons = 0; + db->flags &= ~(u64)SQLITE_DeferFKs; + + /* If one has been configured, invoke the rollback-hook callback */ + if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ + db->xRollbackCallback(db->pRollbackArg); + } +} + +/* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +#if defined(SQLITE_NEED_ERR_NAME) +SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_ERROR_SNAPSHOT: zName = "SQLITE_ERROR_SNAPSHOT"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTINIT: zName = "SQLITE_READONLY_CANTINIT"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; + case SQLITE_READONLY_DIRECTORY: zName = "SQLITE_READONLY_DIRECTORY";break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; + case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; + case SQLITE_CANTOPEN_SYMLINK: zName = "SQLITE_CANTOPEN_SYMLINK"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } + } + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} +#endif + +/* +** Return a static string that describes the kind of error specified in the +** argument. +*/ +SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ + static const char* const aMsg[] = { + /* SQLITE_OK */ "not an error", + /* SQLITE_ERROR */ "SQL logic error", + /* SQLITE_INTERNAL */ 0, + /* SQLITE_PERM */ "access permission denied", + /* SQLITE_ABORT */ "query aborted", + /* SQLITE_BUSY */ "database is locked", + /* SQLITE_LOCKED */ "database table is locked", + /* SQLITE_NOMEM */ "out of memory", + /* SQLITE_READONLY */ "attempt to write a readonly database", + /* SQLITE_INTERRUPT */ "interrupted", + /* SQLITE_IOERR */ "disk I/O error", + /* SQLITE_CORRUPT */ "database disk image is malformed", + /* SQLITE_NOTFOUND */ "unknown operation", + /* SQLITE_FULL */ "database or disk is full", + /* SQLITE_CANTOPEN */ "unable to open database file", + /* SQLITE_PROTOCOL */ "locking protocol", + /* SQLITE_EMPTY */ 0, + /* SQLITE_SCHEMA */ "database schema has changed", + /* SQLITE_TOOBIG */ "string or blob too big", + /* SQLITE_CONSTRAINT */ "constraint failed", + /* SQLITE_MISMATCH */ "datatype mismatch", + /* SQLITE_MISUSE */ "bad parameter or other API misuse", +#ifdef SQLITE_DISABLE_LFS + /* SQLITE_NOLFS */ "large file support is disabled", +#else + /* SQLITE_NOLFS */ 0, +#endif + /* SQLITE_AUTH */ "authorization denied", + /* SQLITE_FORMAT */ 0, + /* SQLITE_RANGE */ "column index out of range", + /* SQLITE_NOTADB */ "file is not a database", + /* SQLITE_NOTICE */ "notification message", + /* SQLITE_WARNING */ "warning message", + }; + const char *zErr = "unknown error"; + switch( rc ){ + case SQLITE_ABORT_ROLLBACK: { + zErr = "abort due to ROLLBACK"; + break; + } + case SQLITE_ROW: { + zErr = "another row available"; + break; + } + case SQLITE_DONE: { + zErr = "no more rows available"; + break; + } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rcbusyTimeout; + int delay, prior; + + assert( count>=0 ); + if( count < NDELAY ){ + delay = delays[count]; + prior = totals[count]; + }else{ + delay = delays[NDELAY-1]; + prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); + } + if( prior + delay > tmout ){ + delay = tmout - prior; + if( delay<=0 ) return 0; + } + sqlite3OsSleep(db->pVfs, delay*1000); + return 1; +#else + /* This case for unix systems that lack usleep() support. Sleeping + ** must be done in increments of whole seconds */ + sqlite3 *db = (sqlite3 *)ptr; + int tmout = ((sqlite3 *)ptr)->busyTimeout; + if( (count+1)*1000 > tmout ){ + return 0; + } + sqlite3OsSleep(db->pVfs, 1000000); + return 1; +#endif +} + +/* +** Invoke the given busy handler. +** +** This routine is called when an operation failed to acquire a +** lock on VFS file pFile. +** +** If this routine returns non-zero, the lock is retried. If it +** returns 0, the operation aborts with an SQLITE_BUSY error. +*/ +SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ + int rc; + if( p->xBusyHandler==0 || p->nBusy<0 ) return 0; + rc = p->xBusyHandler(p->pBusyArg, p->nBusy); + if( rc==0 ){ + p->nBusy = -1; + }else{ + p->nBusy++; + } + return rc; +} + +/* +** This routine sets the busy callback for an Sqlite database to the +** given callback function with the given argument. +*/ +SQLITE_API int sqlite3_busy_handler( + sqlite3 *db, + int (*xBusy)(void*,int), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->busyHandler.xBusyHandler = xBusy; + db->busyHandler.pBusyArg = pArg; + db->busyHandler.nBusy = 0; + db->busyTimeout = 0; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** This routine sets the progress callback for an Sqlite database to the +** given callback function with the given argument. The progress callback will +** be invoked every nOps opcodes. +*/ +SQLITE_API void sqlite3_progress_handler( + sqlite3 *db, + int nOps, + int (*xProgress)(void*), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( nOps>0 ){ + db->xProgress = xProgress; + db->nProgressOps = (unsigned)nOps; + db->pProgressArg = pArg; + }else{ + db->xProgress = 0; + db->nProgressOps = 0; + db->pProgressArg = 0; + } + sqlite3_mutex_leave(db->mutex); +} +#endif + + +/* +** This routine installs a default busy handler that waits for the +** specified number of milliseconds before returning 0. +*/ +SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( ms>0 ){ + sqlite3_busy_handler(db, (int(*)(void*,int))sqliteDefaultBusyCallback, + (void*)db); + db->busyTimeout = ms; + }else{ + sqlite3_busy_handler(db, 0, 0); + } + return SQLITE_OK; +} + +/* +** Cause any pending operation to stop at its earliest opportunity. +*/ +SQLITE_API void sqlite3_interrupt(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) && (db==0 || db->eOpenState!=SQLITE_STATE_ZOMBIE) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + AtomicStore(&db->u1.isInterrupted, 1); +} + + +/* +** This function is exactly the same as sqlite3_create_function(), except +** that it is designed to be called by internal code. The difference is +** that if a malloc() fails in sqlite3_create_function(), an error code +** is returned and the mallocFailed flag cleared. +*/ +SQLITE_PRIVATE int sqlite3CreateFunc( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int enc, + void *pUserData, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value **), + FuncDestructor *pDestructor +){ + FuncDef *p; + int nName; + int extraFlags; + + assert( sqlite3_mutex_held(db->mutex) ); + assert( xValue==0 || xSFunc==0 ); + if( zFunctionName==0 /* Must have a valid name */ + || (xSFunc!=0 && xFinal!=0) /* Not both xSFunc and xFinal */ + || ((xFinal==0)!=(xStep==0)) /* Both or neither of xFinal and xStep */ + || ((xValue==0)!=(xInverse==0)) /* Both or neither of xValue, xInverse */ + || (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) + || (255<(nName = sqlite3Strlen30( zFunctionName))) + ){ + return SQLITE_MISUSE_BKPT; + } + + assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); + assert( SQLITE_FUNC_DIRECT==SQLITE_DIRECTONLY ); + extraFlags = enc & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY| + SQLITE_SUBTYPE|SQLITE_INNOCUOUS); + enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); + + /* The SQLITE_INNOCUOUS flag is the same bit as SQLITE_FUNC_UNSAFE. But + ** the meaning is inverted. So flip the bit. */ + assert( SQLITE_FUNC_UNSAFE==SQLITE_INNOCUOUS ); + extraFlags ^= SQLITE_FUNC_UNSAFE; + + +#ifndef SQLITE_OMIT_UTF16 + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + ** + ** If SQLITE_ANY is specified, add three versions of the function + ** to the hash table. + */ + switch( enc ){ + case SQLITE_UTF16: + enc = SQLITE_UTF16NATIVE; + break; + case SQLITE_ANY: { + int rc; + rc = sqlite3CreateFunc(db, zFunctionName, nArg, + (SQLITE_UTF8|extraFlags)^SQLITE_FUNC_UNSAFE, + pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); + if( rc==SQLITE_OK ){ + rc = sqlite3CreateFunc(db, zFunctionName, nArg, + (SQLITE_UTF16LE|extraFlags)^SQLITE_FUNC_UNSAFE, + pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); + } + if( rc!=SQLITE_OK ){ + return rc; + } + enc = SQLITE_UTF16BE; + break; + } + case SQLITE_UTF8: + case SQLITE_UTF16LE: + case SQLITE_UTF16BE: + break; + default: + enc = SQLITE_UTF8; + break; + } +#else + enc = SQLITE_UTF8; +#endif + + /* Check if an existing function is being overridden or deleted. If so, + ** and there are active VMs, then return SQLITE_BUSY. If a function + ** is being overridden/deleted but there are no active VMs, allow the + ** operation to continue but invalidate all precompiled statements. + */ + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 0); + if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==(u32)enc && p->nArg==nArg ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify user-function due to active statements"); + assert( !db->mallocFailed ); + return SQLITE_BUSY; + }else{ + sqlite3ExpirePreparedStatements(db, 0); + } + }else if( xSFunc==0 && xFinal==0 ){ + /* Trying to delete a function that does not exist. This is a no-op. + ** https://sqlite.org/forum/forumpost/726219164b */ + return SQLITE_OK; + } + + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 1); + assert(p || db->mallocFailed); + if( !p ){ + return SQLITE_NOMEM_BKPT; + } + + /* If an older version of the function with a configured destructor is + ** being replaced invoke the destructor function here. */ + functionDestroy(db, p); + + if( pDestructor ){ + pDestructor->nRef++; + } + p->u.pDestructor = pDestructor; + p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; + testcase( p->funcFlags & SQLITE_DETERMINISTIC ); + testcase( p->funcFlags & SQLITE_DIRECTONLY ); + p->xSFunc = xSFunc ? xSFunc : xStep; + p->xFinalize = xFinal; + p->xValue = xValue; + p->xInverse = xInverse; + p->pUserData = pUserData; + p->nArg = (u16)nArg; + return SQLITE_OK; +} + +/* +** Worker function used by utf-8 APIs that create new functions: +** +** sqlite3_create_function() +** sqlite3_create_function_v2() +** sqlite3_create_window_function() +*/ +static int createFunctionApi( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) +){ + int rc = SQLITE_ERROR; + FuncDestructor *pArg = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( xDestroy ){ + pArg = (FuncDestructor *)sqlite3Malloc(sizeof(FuncDestructor)); + if( !pArg ){ + sqlite3OomFault(db); + xDestroy(p); + goto out; + } + pArg->nRef = 0; + pArg->xDestroy = xDestroy; + pArg->pUserData = p; + } + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, + xSFunc, xStep, xFinal, xValue, xInverse, pArg + ); + if( pArg && pArg->nRef==0 ){ + assert( rc!=SQLITE_OK || (xStep==0 && xFinal==0) ); + xDestroy(p); + sqlite3_free(pArg); + } + + out: + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Create new user functions. +*/ +SQLITE_API int sqlite3_create_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, xSFunc, xStep, + xFinal, 0, 0, 0); +} +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, xSFunc, xStep, + xFinal, 0, 0, xDestroy); +} +SQLITE_API int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value **), + void (*xDestroy)(void *) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, 0, xStep, + xFinal, xValue, xInverse, xDestroy); +} + +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API int sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +){ + int rc; + char *zFunc8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0,0,0); + sqlite3DbFree(db, zFunc8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif + + +/* +** The following is the implementation of an SQL function that always +** fails with an error message stating that the function is used in the +** wrong context. The sqlite3_overload_function() API might construct +** SQL function that use this routine so that the functions will exist +** for name resolution but are actually overloaded by the xFindFunction +** method of virtual tables. +*/ +static void sqlite3InvalidFunction( + sqlite3_context *context, /* The function calling context */ + int NotUsed, /* Number of arguments to the function */ + sqlite3_value **NotUsed2 /* Value of each argument */ +){ + const char *zName = (const char*)sqlite3_user_data(context); + char *zErr; + UNUSED_PARAMETER2(NotUsed, NotUsed2); + zErr = sqlite3_mprintf( + "unable to use function %s in the requested context", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); +} + +/* +** Declare that a function has been overloaded by a virtual table. +** +** If the function already exists as a regular global function, then +** this routine is a no-op. If the function does not exist, then create +** a new one that always throws a run-time error. +** +** When virtual tables intend to provide an overloaded function, they +** should call this routine to make sure the global function exists. +** A global function must exist in order for name resolution to work +** properly. +*/ +SQLITE_API int sqlite3_overload_function( + sqlite3 *db, + const char *zName, + int nArg +){ + int rc; + char *zCopy; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + rc = sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)!=0; + sqlite3_mutex_leave(db->mutex); + if( rc ) return SQLITE_OK; + zCopy = sqlite3_mprintf(zName); + if( zCopy==0 ) return SQLITE_NOMEM; + return sqlite3_create_function_v2(db, zName, nArg, SQLITE_UTF8, + zCopy, sqlite3InvalidFunction, 0, 0, sqlite3_free); +} + +#ifndef SQLITE_OMIT_TRACE +/* +** Register a trace function. The pArg from the previously registered trace +** is returned. +** +** A NULL trace function means that no tracing is executes. A non-NULL +** trace is a pointer to a function that is invoked at the start of each +** SQL statement. +*/ +#ifndef SQLITE_OMIT_DEPRECATED +SQLITE_API void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pTraceArg; + db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; + db->trace.xLegacy = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +#endif /* SQLITE_OMIT_DEPRECATED */ + +/* Register a trace callback using the version-2 interface. +*/ +SQLITE_API int sqlite3_trace_v2( + sqlite3 *db, /* Trace this connection */ + unsigned mTrace, /* Mask of events to be traced */ + int(*xTrace)(unsigned,void*,void*,void*), /* Callback to invoke */ + void *pArg /* Context */ +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( mTrace==0 ) xTrace = 0; + if( xTrace==0 ) mTrace = 0; + db->mTrace = mTrace; + db->trace.xV2 = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** Register a profile function. The pArg from the previously registered +** profile function is returned. +** +** A NULL profile function means that no profiling is executes. A non-NULL +** profile is a pointer to a function that is invoked at the conclusion of +** each SQL statement that is run. +*/ +SQLITE_API void *sqlite3_profile( + sqlite3 *db, + void (*xProfile)(void*,const char*,sqlite_uint64), + void *pArg +){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pProfileArg; + db->xProfile = xProfile; + db->pProfileArg = pArg; + db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK; + if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +#endif /* SQLITE_OMIT_DEPRECATED */ +#endif /* SQLITE_OMIT_TRACE */ + +/* +** Register a function to be invoked when a transaction commits. +** If the invoked function returns non-zero, then the commit becomes a +** rollback. +*/ +SQLITE_API void *sqlite3_commit_hook( + sqlite3 *db, /* Attach the hook to this database */ + int (*xCallback)(void*), /* Function to invoke on each commit */ + void *pArg /* Argument to the function */ +){ + void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pCommitArg; + db->xCommitCallback = xCallback; + db->pCommitArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} + +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. +*/ +SQLITE_API void *sqlite3_update_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), + void *pArg /* Argument to the function */ +){ + void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pUpdateArg; + db->xUpdateCallback = xCallback; + db->pUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} + +/* +** Register a callback to be invoked each time a transaction is rolled +** back by this database connection. +*/ +SQLITE_API void *sqlite3_rollback_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*), /* Callback function */ + void *pArg /* Argument to the function */ +){ + void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pRollbackArg; + db->xRollbackCallback = xCallback; + db->pRollbackArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} + +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. +*/ +SQLITE_API void *sqlite3_preupdate_hook( + sqlite3 *db, /* Attach the hook to this database */ + void(*xCallback)( /* Callback function */ + void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64), + void *pArg /* First callback argument */ +){ + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pPreUpdateArg; + db->xPreUpdateCallback = xCallback; + db->pPreUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ + +#ifndef SQLITE_OMIT_WAL +/* +** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). +** Invoke sqlite3_wal_checkpoint if the number of frames in the log file +** is greater than sqlite3.pWalArg cast to an integer (the value configured by +** wal_autocheckpoint()). +*/ +SQLITE_PRIVATE int sqlite3WalDefaultHook( + void *pClientData, /* Argument */ + sqlite3 *db, /* Connection */ + const char *zDb, /* Database */ + int nFrame /* Size of WAL */ +){ + if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ + sqlite3BeginBenignMalloc(); + sqlite3_wal_checkpoint(db, zDb); + sqlite3EndBenignMalloc(); + } + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_WAL */ + +/* +** Configure an sqlite3_wal_hook() callback to automatically checkpoint +** a database after committing a transaction if there are nFrame or +** more frames in the log file. Passing zero or a negative value as the +** nFrame parameter disables automatic checkpoints entirely. +** +** The callback registered by this function replaces any existing callback +** registered using sqlite3_wal_hook(). Likewise, registering a callback +** using sqlite3_wal_hook() disables the automatic checkpoint mechanism +** configured by this function. +*/ +SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ +#ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(nFrame); +#else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( nFrame>0 ){ + sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); + }else{ + sqlite3_wal_hook(db, 0, 0); + } +#endif + return SQLITE_OK; +} + +/* +** Register a callback to be invoked each time a transaction is written +** into the write-ahead-log by this database connection. +*/ +SQLITE_API void *sqlite3_wal_hook( + sqlite3 *db, /* Attach the hook to this db handle */ + int(*xCallback)(void *, sqlite3*, const char*, int), + void *pArg /* First argument passed to xCallback() */ +){ +#ifndef SQLITE_OMIT_WAL + void *pRet; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pWalArg; + db->xWalCallback = xCallback; + db->pWalArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +#else + return 0; +#endif +} + +/* +** Checkpoint database zDb. +*/ +SQLITE_API int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +){ +#ifdef SQLITE_OMIT_WAL + return SQLITE_OK; +#else + int rc; /* Return code */ + int iDb; /* Schema to checkpoint */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + + /* Initialize the output variables to -1 in case an error occurs. */ + if( pnLog ) *pnLog = -1; + if( pnCkpt ) *pnCkpt = -1; + + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint + ** mode: */ + return SQLITE_MISUSE; + } + + sqlite3_mutex_enter(db->mutex); + if( zDb && zDb[0] ){ + iDb = sqlite3FindDbName(db, zDb); + }else{ + iDb = SQLITE_MAX_DB; /* This means process all schemas */ + } + if( iDb<0 ){ + rc = SQLITE_ERROR; + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); + }else{ + db->busyHandler.nBusy = 0; + rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); + sqlite3Error(db, rc); + } + rc = sqlite3ApiExit(db, rc); + + /* If there are no active statements, clear the interrupt flag at this + ** point. */ + if( db->nVdbeActive==0 ){ + AtomicStore(&db->u1.isInterrupted, 0); + } + + sqlite3_mutex_leave(db->mutex); + return rc; +#endif +} + + +/* +** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points +** to contains a zero-length string, all attached databases are +** checkpointed. +*/ +SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ + /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to + ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ + return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); +} + +#ifndef SQLITE_OMIT_WAL +/* +** Run a checkpoint on database iDb. This is a no-op if database iDb is +** not currently open in WAL mode. +** +** If a transaction is open on the database being checkpointed, this +** function returns SQLITE_LOCKED and a checkpoint is not attempted. If +** an error occurs while running the checkpoint, an SQLite error code is +** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** +** The mutex on database handle db should be held by the caller. The mutex +** associated with the specific b-tree being checkpointed is taken by +** this function while the checkpoint is running. +** +** If iDb is passed SQLITE_MAX_DB then all attached databases are +** checkpointed. If an error is encountered it is returned immediately - +** no attempt is made to checkpoint any remaining databases. +** +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL, RESTART +** or TRUNCATE. +*/ +SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Used to iterate through attached dbs */ + int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ + + assert( sqlite3_mutex_held(db->mutex) ); + assert( !pnLog || *pnLog==-1 ); + assert( !pnCkpt || *pnCkpt==-1 ); + testcase( iDb==SQLITE_MAX_ATTACHED ); /* See forum post a006d86f72 */ + testcase( iDb==SQLITE_MAX_DB ); + + for(i=0; inDb && rc==SQLITE_OK; i++){ + if( i==iDb || iDb==SQLITE_MAX_DB ){ + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); + pnLog = 0; + pnCkpt = 0; + if( rc==SQLITE_BUSY ){ + bBusy = 1; + rc = SQLITE_OK; + } + } + } + + return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; +} +#endif /* SQLITE_OMIT_WAL */ + +/* +** This function returns true if main-memory should be used instead of +** a temporary file for transient pager files and statement journals. +** The value returned depends on the value of db->temp_store (runtime +** parameter) and the compile time value of SQLITE_TEMP_STORE. The +** following table describes the relationship between these two values +** and this functions return value. +** +** SQLITE_TEMP_STORE db->temp_store Location of temporary database +** ----------------- -------------- ------------------------------ +** 0 any file (return 0) +** 1 1 file (return 0) +** 1 2 memory (return 1) +** 1 0 file (return 0) +** 2 1 file (return 0) +** 2 2 memory (return 1) +** 2 0 memory (return 1) +** 3 any memory (return 1) +*/ +SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ +#if SQLITE_TEMP_STORE==1 + return ( db->temp_store==2 ); +#endif +#if SQLITE_TEMP_STORE==2 + return ( db->temp_store!=1 ); +#endif +#if SQLITE_TEMP_STORE==3 + UNUSED_PARAMETER(db); + return 1; +#endif +#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 + UNUSED_PARAMETER(db); + return 0; +#endif +} + +/* +** Return UTF-8 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ + const char *z; + if( !db ){ + return sqlite3ErrStr(SQLITE_NOMEM_BKPT); + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return sqlite3ErrStr(SQLITE_MISUSE_BKPT); + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = sqlite3ErrStr(SQLITE_NOMEM_BKPT); + }else{ + testcase( db->pErr==0 ); + z = db->errCode ? (char*)sqlite3_value_text(db->pErr) : 0; + assert( !db->mallocFailed ); + if( z==0 ){ + z = sqlite3ErrStr(db->errCode); + } + } + sqlite3_mutex_leave(db->mutex); + return z; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Return UTF-16 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ + static const u16 outOfMem[] = { + 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 + }; + static const u16 misuse[] = { + 'b', 'a', 'd', ' ', 'p', 'a', 'r', 'a', 'm', 'e', 't', 'e', 'r', ' ', + 'o', 'r', ' ', 'o', 't', 'h', 'e', 'r', ' ', 'A', 'P', 'I', ' ', + 'm', 'i', 's', 'u', 's', 'e', 0 + }; + + const void *z; + if( !db ){ + return (void *)outOfMem; + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return (void *)misuse; + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = (void *)outOfMem; + }else{ + z = sqlite3_value_text16(db->pErr); + if( z==0 ){ + sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); + z = sqlite3_value_text16(db->pErr); + } + /* A malloc() may have failed within the call to sqlite3_value_text16() + ** above. If this is the case, then the db->mallocFailed flag needs to + ** be cleared before returning. Do this directly, instead of via + ** sqlite3ApiExit(), to avoid setting the database handle error message. + */ + sqlite3OomClear(db); + } + sqlite3_mutex_leave(db->mutex); + return z; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Return the most recent error code generated by an SQLite routine. If NULL is +** passed to this function, we assume a malloc() failed during sqlite3_open(). +*/ +SQLITE_API int sqlite3_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM_BKPT; + } + return db->errCode & db->errMask; +} +SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM_BKPT; + } + return db->errCode; +} +SQLITE_API int sqlite3_system_errno(sqlite3 *db){ + return db ? db->iSysErrno : 0; +} + +/* +** Return a string that describes the kind of error specified in the +** argument. For now, this simply calls the internal sqlite3ErrStr() +** function. +*/ +SQLITE_API const char *sqlite3_errstr(int rc){ + return sqlite3ErrStr(rc); +} + +/* +** Create a new collating function for database "db". The name is zName +** and the encoding is enc. +*/ +static int createCollation( + sqlite3* db, + const char *zName, + u8 enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + CollSeq *pColl; + int enc2; + + assert( sqlite3_mutex_held(db->mutex) ); + + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + */ + enc2 = enc; + testcase( enc2==SQLITE_UTF16 ); + testcase( enc2==SQLITE_UTF16_ALIGNED ); + if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ + enc2 = SQLITE_UTF16NATIVE; + } + if( enc2SQLITE_UTF16BE ){ + return SQLITE_MISUSE_BKPT; + } + + /* Check if this call is removing or replacing an existing collation + ** sequence. If so, and there are active VMs, return busy. If there + ** are no active VMs, invalidate any pre-compiled statements. + */ + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); + if( pColl && pColl->xCmp ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify collation sequence due to active statements"); + return SQLITE_BUSY; + } + sqlite3ExpirePreparedStatements(db, 0); + + /* If collation sequence pColl was created directly by a call to + ** sqlite3_create_collation, and not generated by synthCollSeq(), + ** then any copies made by synthCollSeq() need to be invalidated. + ** Also, collation destructor - CollSeq.xDel() - function may need + ** to be called. + */ + if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); + int j; + for(j=0; j<3; j++){ + CollSeq *p = &aColl[j]; + if( p->enc==pColl->enc ){ + if( p->xDel ){ + p->xDel(p->pUser); + } + p->xCmp = 0; + } + } + } + } + + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); + if( pColl==0 ) return SQLITE_NOMEM_BKPT; + pColl->xCmp = xCompare; + pColl->pUser = pCtx; + pColl->xDel = xDel; + pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); + sqlite3Error(db, SQLITE_OK); + return SQLITE_OK; +} + + +/* +** This array defines hard upper bounds on limit values. The +** initializer must be kept in sync with the SQLITE_LIMIT_* +** #defines in sqlite3.h. +*/ +static const int aHardLimit[] = { + SQLITE_MAX_LENGTH, + SQLITE_MAX_SQL_LENGTH, + SQLITE_MAX_COLUMN, + SQLITE_MAX_EXPR_DEPTH, + SQLITE_MAX_COMPOUND_SELECT, + SQLITE_MAX_VDBE_OP, + SQLITE_MAX_FUNCTION_ARG, + SQLITE_MAX_ATTACHED, + SQLITE_MAX_LIKE_PATTERN_LENGTH, + SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ + SQLITE_MAX_TRIGGER_DEPTH, + SQLITE_MAX_WORKER_THREADS, +}; + +/* +** Make sure the hard limits are set to reasonable values +*/ +#if SQLITE_MAX_LENGTH<100 +# error SQLITE_MAX_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH<100 +# error SQLITE_MAX_SQL_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH +# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH +#endif +#if SQLITE_MAX_COMPOUND_SELECT<2 +# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 +#endif +#if SQLITE_MAX_VDBE_OP<40 +# error SQLITE_MAX_VDBE_OP must be at least 40 +#endif +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127 +#endif +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 +# error SQLITE_MAX_ATTACHED must be between 0 and 125 +#endif +#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 +# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 +#endif +#if SQLITE_MAX_COLUMN>32767 +# error SQLITE_MAX_COLUMN must not exceed 32767 +#endif +#if SQLITE_MAX_TRIGGER_DEPTH<1 +# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 +#endif +#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 +# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 +#endif + + +/* +** Change the value of a limit. Report the old value. +** If an invalid limit index is supplied, report -1. +** Make no changes but still report the old value if the +** new limit is negative. +** +** A new lower limit does not shrink existing constructs. +** It merely prevents new constructs that exceed the limit +** from forming. +*/ +SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ + int oldLimit; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + + /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME + ** there is a hard upper bound set at compile-time by a C preprocessor + ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to + ** "_MAX_".) + */ + assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); + assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); + assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); + assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); + assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); + assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== + SQLITE_MAX_LIKE_PATTERN_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); + assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); + + + if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ + return -1; + } + oldLimit = db->aLimit[limitId]; + if( newLimit>=0 ){ /* IMP: R-52476-28732 */ + if( newLimit>aHardLimit[limitId] ){ + newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ + } + db->aLimit[limitId] = newLimit; + } + return oldLimit; /* IMP: R-53341-35419 */ +} + +/* +** This function is used to parse both URIs and non-URI filenames passed by the +** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database +** URIs specified as part of ATTACH statements. +** +** The first argument to this function is the name of the VFS to use (or +** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" +** query parameter. The second argument contains the URI (or non-URI filename) +** itself. When this function is called the *pFlags variable should contain +** the default flags to open the database handle with. The value stored in +** *pFlags may be updated before returning if the URI filename contains +** "cache=xxx" or "mode=xxx" query parameters. +** +** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to +** the VFS that should be used to open the database file. *pzFile is set to +** point to a buffer containing the name of the file to open. The value +** stored in *pzFile is a database name acceptable to sqlite3_uri_parameter() +** and is in the same format as names created using sqlite3_create_filename(). +** The caller must invoke sqlite3_free_filename() (not sqlite3_free()!) on +** the value returned in *pzFile to avoid a memory leak. +** +** If an error occurs, then an SQLite error code is returned and *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to eventually release +** this buffer by calling sqlite3_free(). +*/ +SQLITE_PRIVATE int sqlite3ParseUri( + const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ + const char *zUri, /* Nul-terminated URI to parse */ + unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + char **pzFile, /* OUT: Filename component of URI */ + char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ +){ + int rc = SQLITE_OK; + unsigned int flags = *pFlags; + const char *zVfs = zDefaultVfs; + char *zFile; + char c; + int nUri = sqlite3Strlen30(zUri); + + assert( *pzErrMsg==0 ); + + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ + ){ + char *zOpt; + int eState; /* Parser state when parsing URI */ + int iIn; /* Input character index */ + int iOut = 0; /* Output character index */ + u64 nByte = nUri+8; /* Bytes of space to allocate */ + + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + ** method that there may be extra parameters following the file-name. */ + flags |= SQLITE_OPEN_URI; + + for(iIn=0; iIn=0 && octet<256 ); + if( octet==0 ){ +#ifndef SQLITE_ENABLE_URI_00_ERROR + /* This branch is taken when "%00" appears within the URI. In this + ** case we ignore all text in the remainder of the path, name or + ** value currently being parsed. So ignore the current character + ** and skip to the next "?", "=" or "&", as appropriate. */ + while( (c = zUri[iIn])!=0 && c!='#' + && (eState!=0 || c!='?') + && (eState!=1 || (c!='=' && c!='&')) + && (eState!=2 || c!='&') + ){ + iIn++; + } + continue; +#else + /* If ENABLE_URI_00_ERROR is defined, "%00" in a URI is an error. */ + *pzErrMsg = sqlite3_mprintf("unexpected %%00 in uri"); + rc = SQLITE_ERROR; + goto parse_uri_out; +#endif + } + c = octet; + }else if( eState==1 && (c=='&' || c=='=') ){ + if( zFile[iOut-1]==0 ){ + /* An empty option name. Ignore this option altogether. */ + while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; + continue; + } + if( c=='&' ){ + zFile[iOut++] = '\0'; + }else{ + eState = 2; + } + c = 0; + }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ + c = 0; + eState = 1; + } + zFile[iOut++] = c; + } + if( eState==1 ) zFile[iOut++] = '\0'; + memset(zFile+iOut, 0, 4); /* end-of-options + empty journal filenames */ + + /* Check if there were any options specified that should be interpreted + ** here. Options that are interpreted here include "vfs" and those that + ** correspond to flags that may be passed to the sqlite3_open_v2() + ** method. */ + zOpt = &zFile[sqlite3Strlen30(zFile)+1]; + while( zOpt[0] ){ + int nOpt = sqlite3Strlen30(zOpt); + char *zVal = &zOpt[nOpt+1]; + int nVal = sqlite3Strlen30(zVal); + + if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ + zVfs = zVal; + }else{ + struct OpenMode { + const char *z; + int mode; + } *aMode = 0; + char *zModeType = 0; + int mask = 0; + int limit = 0; + + if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ + static struct OpenMode aCacheMode[] = { + { "shared", SQLITE_OPEN_SHAREDCACHE }, + { "private", SQLITE_OPEN_PRIVATECACHE }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; + aMode = aCacheMode; + limit = mask; + zModeType = "cache"; + } + if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ + static struct OpenMode aOpenMode[] = { + { "ro", SQLITE_OPEN_READONLY }, + { "rw", SQLITE_OPEN_READWRITE }, + { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, + { "memory", SQLITE_OPEN_MEMORY }, + { 0, 0 } + }; + + mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; + aMode = aOpenMode; + limit = mask & flags; + zModeType = "access"; + } + + if( aMode ){ + int i; + int mode = 0; + for(i=0; aMode[i].z; i++){ + const char *z = aMode[i].z; + if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ + mode = aMode[i].mode; + break; + } + } + if( mode==0 ){ + *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); + rc = SQLITE_ERROR; + goto parse_uri_out; + } + if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ + *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", + zModeType, zVal); + rc = SQLITE_PERM; + goto parse_uri_out; + } + flags = (flags & ~mask) | mode; + } + } + + zOpt = &zVal[nVal+1]; + } + + }else{ + zFile = sqlite3_malloc64(nUri+8); + if( !zFile ) return SQLITE_NOMEM_BKPT; + memset(zFile, 0, 4); + zFile += 4; + if( nUri ){ + memcpy(zFile, zUri, nUri); + } + memset(zFile+nUri, 0, 4); + flags &= ~SQLITE_OPEN_URI; + } + + *ppVfs = sqlite3_vfs_find(zVfs); + if( *ppVfs==0 ){ + *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); + rc = SQLITE_ERROR; + } + parse_uri_out: + if( rc!=SQLITE_OK ){ + sqlite3_free_filename(zFile); + zFile = 0; + } + *pFlags = flags; + *pzFile = zFile; + return rc; +} + +/* +** This routine does the core work of extracting URI parameters from a +** database filename for the sqlite3_uri_parameter() interface. +*/ +static const char *uriParameter(const char *zFilename, const char *zParam){ + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + int x = strcmp(zFilename, zParam); + zFilename += sqlite3Strlen30(zFilename) + 1; + if( x==0 ) return zFilename; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return 0; +} + + + +/* +** This routine does the work of opening a database on behalf of +** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" +** is UTF-8 encoded. +*/ +static int openDatabase( + const char *zFilename, /* Database filename UTF-8 encoded */ + sqlite3 **ppDb, /* OUT: Returned database handle */ + unsigned int flags, /* Operational flags */ + const char *zVfs /* Name of the VFS to use */ +){ + sqlite3 *db; /* Store allocated handle here */ + int rc; /* Return code */ + int isThreadsafe; /* True for threadsafe connections */ + char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ + char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ + int i; /* Loop counter */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + + if( sqlite3GlobalConfig.bCoreMutex==0 ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_NOMUTEX ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_FULLMUTEX ){ + isThreadsafe = 1; + }else{ + isThreadsafe = sqlite3GlobalConfig.bFullMutex; + } + + if( flags & SQLITE_OPEN_PRIVATECACHE ){ + flags &= ~SQLITE_OPEN_SHAREDCACHE; + }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ + flags |= SQLITE_OPEN_SHAREDCACHE; + } + + /* Remove harmful bits from the flags parameter + ** + ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were + ** dealt with in the previous code block. Besides these, the only + ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, + ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, + ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask + ** off all other flags. + */ + flags &= ~( SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_MAIN_DB | + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_TRANSIENT_DB | + SQLITE_OPEN_MAIN_JOURNAL | + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_SUBJOURNAL | + SQLITE_OPEN_SUPER_JOURNAL | + SQLITE_OPEN_NOMUTEX | + SQLITE_OPEN_FULLMUTEX | + SQLITE_OPEN_WAL + ); + + /* Allocate the sqlite data structure */ + db = sqlite3MallocZero( sizeof(sqlite3) ); + if( db==0 ) goto opendb_out; + if( isThreadsafe +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS + || sqlite3GlobalConfig.bCoreMutex +#endif + ){ + db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( db->mutex==0 ){ + sqlite3_free(db); + db = 0; + goto opendb_out; + } + if( isThreadsafe==0 ){ + sqlite3MutexWarnOnContention(db->mutex); + } + } + sqlite3_mutex_enter(db->mutex); + db->errMask = 0xff; + db->nDb = 2; + db->eOpenState = SQLITE_STATE_BUSY; + db->aDb = db->aDbStatic; + db->lookaside.bDisable = 1; + db->lookaside.sz = 0; + + assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); + memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; + db->autoCommit = 1; + db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; + db->nextPagesize = 0; + db->init.azInit = sqlite3StdType; /* Any array of string ptrs will do */ +#ifdef SQLITE_ENABLE_SORTER_MMAP + /* Beginning with version 3.37.0, using the VFS xFetch() API to memory-map + ** the temporary files used to do external sorts (see code in vdbesort.c) + ** is disabled. It can still be used either by defining + ** SQLITE_ENABLE_SORTER_MMAP at compile time or by using the + ** SQLITE_TESTCTRL_SORTER_MMAP test-control at runtime. */ + db->nMaxSorterMmap = 0x7FFFFFFF; +#endif + db->flags |= SQLITE_ShortColNames + | SQLITE_EnableTrigger + | SQLITE_EnableView + | SQLITE_CacheSpill +#if !defined(SQLITE_TRUSTED_SCHEMA) || SQLITE_TRUSTED_SCHEMA+0!=0 + | SQLITE_TrustedSchema +#endif +/* The SQLITE_DQS compile-time option determines the default settings +** for SQLITE_DBCONFIG_DQS_DDL and SQLITE_DBCONFIG_DQS_DML. +** +** SQLITE_DQS SQLITE_DBCONFIG_DQS_DDL SQLITE_DBCONFIG_DQS_DML +** ---------- ----------------------- ----------------------- +** undefined on on +** 3 on on +** 2 on off +** 1 off on +** 0 off off +** +** Legacy behavior is 3 (double-quoted string literals are allowed anywhere) +** and so that is the default. But developers are encouranged to use +** -DSQLITE_DQS=0 (best) or -DSQLITE_DQS=1 (second choice) if possible. +*/ +#if !defined(SQLITE_DQS) +# define SQLITE_DQS 3 +#endif +#if (SQLITE_DQS&1)==1 + | SQLITE_DqsDML +#endif +#if (SQLITE_DQS&2)==2 + | SQLITE_DqsDDL +#endif + +#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX + | SQLITE_AutoIndex +#endif +#if SQLITE_DEFAULT_CKPTFULLFSYNC + | SQLITE_CkptFullFSync +#endif +#if SQLITE_DEFAULT_FILE_FORMAT<4 + | SQLITE_LegacyFileFmt +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + | SQLITE_LoadExtension +#endif +#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS + | SQLITE_RecTriggers +#endif +#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS + | SQLITE_ForeignKeys +#endif +#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) + | SQLITE_ReverseOrder +#endif +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + | SQLITE_CellSizeCk +#endif +#if defined(SQLITE_ENABLE_FTS3_TOKENIZER) + | SQLITE_Fts3Tokenizer +#endif +#if defined(SQLITE_ENABLE_QPSG) + | SQLITE_EnableQPSG +#endif +#if defined(SQLITE_DEFAULT_DEFENSIVE) + | SQLITE_Defensive +#endif +#if defined(SQLITE_DEFAULT_LEGACY_ALTER_TABLE) + | SQLITE_LegacyAlter +#endif + ; + sqlite3HashInit(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3HashInit(&db->aModule); +#endif + + /* Add the default collation sequence BINARY. BINARY works for both UTF-8 + ** and UTF-16, so add a version for each to avoid any unnecessary + ** conversions. The only error that can occur here is a malloc() failure. + ** + ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating + ** functions: + */ + createCollation(db, sqlite3StrBINARY, SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, 0, rtrimCollFunc, 0); + if( db->mallocFailed ){ + goto opendb_out; + } + + /* Parse the filename/URI argument + ** + ** Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ + db->openFlags = flags; + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ){ + rc = SQLITE_MISUSE_BKPT; /* IMP: R-18321-05872 */ + }else{ + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + } + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); + sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3_free(zErrMsg); + goto opendb_out; + } + + /* Open the backend database driver */ + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, + flags | SQLITE_OPEN_MAIN_DB); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM_BKPT; + } + sqlite3Error(db, rc); + goto opendb_out; + } + sqlite3BtreeEnter(db->aDb[0].pBt); + db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + if( !db->mallocFailed ){ + sqlite3SetTextEncoding(db, SCHEMA_ENC(db)); + } + sqlite3BtreeLeave(db->aDb[0].pBt); + db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); + + /* The default safety_level for the main database is FULL; for the temp + ** database it is OFF. This matches the pager layer defaults. + */ + db->aDb[0].zDbSName = "main"; + db->aDb[0].safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; + db->aDb[1].zDbSName = "temp"; + db->aDb[1].safety_level = PAGER_SYNCHRONOUS_OFF; + + db->eOpenState = SQLITE_STATE_OPEN; + if( db->mallocFailed ){ + goto opendb_out; + } + + /* Register all built-in functions, but do not attempt to read the + ** database schema yet. This is delayed until the first time the database + ** is accessed. + */ + sqlite3Error(db, SQLITE_OK); + sqlite3RegisterPerConnectionBuiltinFunctions(db); + rc = sqlite3_errcode(db); + + + /* Load compiled-in extensions */ + for(i=0; rc==SQLITE_OK && imDbFlags |= DBFLAG_InternalFunc; +#endif + + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking + ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking + ** mode. Doing nothing at all also makes NORMAL the default. + */ +#ifdef SQLITE_DEFAULT_LOCKING_MODE + db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; + sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), + SQLITE_DEFAULT_LOCKING_MODE); +#endif + + if( rc ) sqlite3Error(db, rc); + + /* Enable the lookaside-malloc subsystem */ + setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside); + + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); + +opendb_out: + if( db ){ + assert( db->mutex!=0 || isThreadsafe==0 + || sqlite3GlobalConfig.bFullMutex==0 ); + sqlite3_mutex_leave(db->mutex); + } + rc = sqlite3_errcode(db); + assert( db!=0 || rc==SQLITE_NOMEM ); + if( rc==SQLITE_NOMEM ){ + sqlite3_close(db); + db = 0; + }else if( rc!=SQLITE_OK ){ + db->eOpenState = SQLITE_STATE_SICK; + } + *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif + sqlite3_free_filename(zOpen); + return rc & 0xff; +} + + +/* +** Open a new database handle. +*/ +SQLITE_API int sqlite3_open( + const char *zFilename, + sqlite3 **ppDb +){ + return openDatabase(zFilename, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); +} +SQLITE_API int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +){ + return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Open a new database handle. +*/ +SQLITE_API int sqlite3_open16( + const void *zFilename, + sqlite3 **ppDb +){ + char const *zFilename8; /* zFilename encoded in UTF-8 instead of UTF-16 */ + sqlite3_value *pVal; + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + if( zFilename==0 ) zFilename = "\000\000"; + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zFilename8 ){ + rc = openDatabase(zFilename8, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); + assert( *ppDb || rc==SQLITE_NOMEM ); + if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ + SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE; + } + }else{ + rc = SQLITE_NOMEM_BKPT; + } + sqlite3ValueFree(pVal); + + return rc & 0xff; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int sqlite3_create_collation( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ + return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); +} + +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int sqlite3_create_collation_v2( + sqlite3* db, + const char *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Register a new collation sequence with the database handle db. +*/ +SQLITE_API int sqlite3_create_collation16( + sqlite3* db, + const void *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ + int rc = SQLITE_OK; + char *zName8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); + if( zName8 ){ + rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); + sqlite3DbFree(db, zName8); + } + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. +*/ +SQLITE_API int sqlite3_collation_needed( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = xCollNeeded; + db->xCollNeeded16 = 0; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. +*/ +SQLITE_API int sqlite3_collation_needed16( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = 0; + db->xCollNeeded16 = xCollNeeded16; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_UTF16 */ + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** This function is now an anachronism. It used to be used to recover from a +** malloc() failure, but SQLite now does this automatically. +*/ +SQLITE_API int sqlite3_global_recover(void){ + return SQLITE_OK; +} +#endif + +/* +** Test to see whether or not the database connection is in autocommit +** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on +** by default. Autocommit is disabled by a BEGIN statement and reenabled +** by the next COMMIT or ROLLBACK. +*/ +SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->autoCommit; +} + +/* +** The following routines are substitutes for constants SQLITE_CORRUPT, +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_NOMEM and possibly other error +** constants. They serve two purposes: +** +** 1. Serve as a convenient place to set a breakpoint in a debugger +** to detect when version error conditions occurs. +** +** 2. Invoke sqlite3_log() to provide the source code location where +** a low-level error is first detected. +*/ +SQLITE_PRIVATE int sqlite3ReportError(int iErr, int lineno, const char *zType){ + sqlite3_log(iErr, "%s at line %d of [%.10s]", + zType, lineno, 20+sqlite3_sourceid()); + return iErr; +} +SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_CORRUPT, lineno, "database corruption"); +} +SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_MISUSE, lineno, "misuse"); +} +SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_CANTOPEN, lineno, "cannot open file"); +} +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO) +SQLITE_PRIVATE int sqlite3CorruptPgnoError(int lineno, Pgno pgno){ + char zMsg[100]; + sqlite3_snprintf(sizeof(zMsg), zMsg, "database corruption page %d", pgno); + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); +} +#endif +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_NOMEM, lineno, "OOM"); +} +SQLITE_PRIVATE int sqlite3IoerrnomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); +} +#endif + +#ifndef SQLITE_OMIT_DEPRECATED +/* +** This is a convenience routine that makes sure that all thread-specific +** data for this thread has been deallocated. +** +** SQLite no longer uses thread-specific data so this routine is now a +** no-op. It is retained for historical compatibility. +*/ +SQLITE_API void sqlite3_thread_cleanup(void){ +} +#endif + +/* +** Return meta information about a specific column of a database table. +** See comment in sqlite3.h (sqlite.h.in) for details. +*/ +SQLITE_API int sqlite3_table_column_metadata( + sqlite3 *db, /* Connection handle */ + const char *zDbName, /* Database name or NULL */ + const char *zTableName, /* Table name */ + const char *zColumnName, /* Column name */ + char const **pzDataType, /* OUTPUT: Declared data type */ + char const **pzCollSeq, /* OUTPUT: Collation sequence name */ + int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ + int *pPrimaryKey, /* OUTPUT: True if column part of PK */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ +){ + int rc; + char *zErrMsg = 0; + Table *pTab = 0; + Column *pCol = 0; + int iCol = 0; + char const *zDataType = 0; + char const *zCollSeq = 0; + int notnull = 0; + int primarykey = 0; + int autoinc = 0; + + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zTableName==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + + /* Ensure the database schema has been loaded */ + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + rc = sqlite3Init(db, &zErrMsg); + if( SQLITE_OK!=rc ){ + goto error_out; + } + + /* Locate the table in question */ + pTab = sqlite3FindTable(db, zTableName, zDbName); + if( !pTab || IsView(pTab) ){ + pTab = 0; + goto error_out; + } + + /* Find the column for which info is requested */ + if( zColumnName==0 ){ + /* Query for existance of table only */ + }else{ + for(iCol=0; iColnCol; iCol++){ + pCol = &pTab->aCol[iCol]; + if( 0==sqlite3StrICmp(pCol->zCnName, zColumnName) ){ + break; + } + } + if( iCol==pTab->nCol ){ + if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ + iCol = pTab->iPKey; + pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; + }else{ + pTab = 0; + goto error_out; + } + } + } + + /* The following block stores the meta information that will be returned + ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey + ** and autoinc. At this point there are two possibilities: + ** + ** 1. The specified column name was rowid", "oid" or "_rowid_" + ** and there is no explicitly declared IPK column. + ** + ** 2. The table is not a view and the column name identified an + ** explicitly declared column. Copy meta information from *pCol. + */ + if( pCol ){ + zDataType = sqlite3ColumnType(pCol,0); + zCollSeq = sqlite3ColumnColl(pCol); + notnull = pCol->notNull!=0; + primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; + autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; + }else{ + zDataType = "INTEGER"; + primarykey = 1; + } + if( !zCollSeq ){ + zCollSeq = sqlite3StrBINARY; + } + +error_out: + sqlite3BtreeLeaveAll(db); + + /* Whether the function call succeeded or failed, set the output parameters + ** to whatever their local counterparts contain. If an error did occur, + ** this has the effect of zeroing all output parameters. + */ + if( pzDataType ) *pzDataType = zDataType; + if( pzCollSeq ) *pzCollSeq = zCollSeq; + if( pNotNull ) *pNotNull = notnull; + if( pPrimaryKey ) *pPrimaryKey = primarykey; + if( pAutoinc ) *pAutoinc = autoinc; + + if( SQLITE_OK==rc && !pTab ){ + sqlite3DbFree(db, zErrMsg); + zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, + zColumnName); + rc = SQLITE_ERROR; + } + sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); + sqlite3DbFree(db, zErrMsg); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Sleep for a little while. Return the amount of time slept. +*/ +SQLITE_API int sqlite3_sleep(int ms){ + sqlite3_vfs *pVfs; + int rc; + pVfs = sqlite3_vfs_find(0); + if( pVfs==0 ) return 0; + + /* This function works in milliseconds, but the underlying OsSleep() + ** API uses microseconds. Hence the 1000's. + */ + rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + return rc; +} + +/* +** Enable or disable the extended result codes. +*/ +SQLITE_API int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->errMask = onoff ? 0xffffffff : 0xff; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Invoke the xFileControl method on a particular database. +*/ +SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ + int rc = SQLITE_ERROR; + Btree *pBtree; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + assert( fd!=0 ); + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_VFS_POINTER ){ + *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ + *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_DATA_VERSION ){ + *(unsigned int*)pArg = sqlite3PagerDataVersion(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_RESERVE_BYTES ){ + int iNew = *(int*)pArg; + *(int*)pArg = sqlite3BtreeGetRequestedReserve(pBtree); + if( iNew>=0 && iNew<=255 ){ + sqlite3BtreeSetPageSize(pBtree, 0, iNew, 0); + } + rc = SQLITE_OK; + }else{ + int nSave = db->busyHandler.nBusy; + rc = sqlite3OsFileControl(fd, op, pArg); + db->busyHandler.nBusy = nSave; + } + sqlite3BtreeLeave(pBtree); + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Interface to the testing logic. +*/ +SQLITE_API int sqlite3_test_control(int op, ...){ + int rc = 0; +#ifdef SQLITE_UNTESTABLE + UNUSED_PARAMETER(op); +#else + va_list ap; + va_start(ap, op); + switch( op ){ + + /* + ** Save the current state of the PRNG. + */ + case SQLITE_TESTCTRL_PRNG_SAVE: { + sqlite3PrngSaveState(); + break; + } + + /* + ** Restore the state of the PRNG to the last state saved using + ** PRNG_SAVE. If PRNG_SAVE has never before been called, then + ** this verb acts like PRNG_RESET. + */ + case SQLITE_TESTCTRL_PRNG_RESTORE: { + sqlite3PrngRestoreState(); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_PRNG_SEED, int x, sqlite3 *db); + ** + ** Control the seed for the pseudo-random number generator (PRNG) that + ** is built into SQLite. Cases: + ** + ** x!=0 && db!=0 Seed the PRNG to the current value of the + ** schema cookie in the main database for db, or + ** x if the schema cookie is zero. This case + ** is convenient to use with database fuzzers + ** as it allows the fuzzer some control over the + ** the PRNG seed. + ** + ** x!=0 && db==0 Seed the PRNG to the value of x. + ** + ** x==0 && db==0 Revert to default behavior of using the + ** xRandomness method on the primary VFS. + ** + ** This test-control also resets the PRNG so that the new seed will + ** be used for the next call to sqlite3_randomness(). + */ +#ifndef SQLITE_OMIT_WSD + case SQLITE_TESTCTRL_PRNG_SEED: { + int x = va_arg(ap, int); + int y; + sqlite3 *db = va_arg(ap, sqlite3*); + assert( db==0 || db->aDb[0].pSchema!=0 ); + if( db && (y = db->aDb[0].pSchema->schema_cookie)!=0 ){ x = y; } + sqlite3Config.iPrngSeed = x; + sqlite3_randomness(0,0); + break; + } +#endif + + /* + ** sqlite3_test_control(BITVEC_TEST, size, program) + ** + ** Run a test against a Bitvec object of size. The program argument + ** is an array of integers that defines the test. Return -1 on a + ** memory allocation error, 0 on success, or non-zero for an error. + ** See the sqlite3BitvecBuiltinTest() for additional information. + */ + case SQLITE_TESTCTRL_BITVEC_TEST: { + int sz = va_arg(ap, int); + int *aProg = va_arg(ap, int*); + rc = sqlite3BitvecBuiltinTest(sz, aProg); + break; + } + + /* + ** sqlite3_test_control(FAULT_INSTALL, xCallback) + ** + ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, + ** if xCallback is not NULL. + ** + ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) + ** is called immediately after installing the new callback and the return + ** value from sqlite3FaultSim(0) becomes the return from + ** sqlite3_test_control(). + */ + case SQLITE_TESTCTRL_FAULT_INSTALL: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); + */ + typedef int(*TESTCALLBACKFUNC_t)(int); + sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); + rc = sqlite3FaultSim(0); + break; + } + + /* + ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) + ** + ** Register hooks to call to indicate which malloc() failures + ** are benign. + */ + case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { + typedef void (*void_function)(void); + void_function xBenignBegin; + void_function xBenignEnd; + xBenignBegin = va_arg(ap, void_function); + xBenignEnd = va_arg(ap, void_function); + sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) + ** + ** Set the PENDING byte to the value in the argument, if X>0. + ** Make no changes if X==0. Return the value of the pending byte + ** as it existing before this routine was called. + ** + ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in + ** an incompatible database file format. Changing the PENDING byte + ** while any database connection is open results in undefined and + ** deleterious behavior. + */ + case SQLITE_TESTCTRL_PENDING_BYTE: { + rc = PENDING_BYTE; +#ifndef SQLITE_OMIT_WSD + { + unsigned int newVal = va_arg(ap, unsigned int); + if( newVal ) sqlite3PendingByte = newVal; + } +#endif + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) + ** + ** This action provides a run-time test to see whether or not + ** assert() was enabled at compile-time. If X is true and assert() + ** is enabled, then the return value is true. If X is true and + ** assert() is disabled, then the return value is zero. If X is + ** false and assert() is enabled, then the assertion fires and the + ** process aborts. If X is false and assert() is disabled, then the + ** return value is zero. + */ + case SQLITE_TESTCTRL_ASSERT: { + volatile int x = 0; + assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 ); + rc = x; + break; + } + + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) + ** + ** This action provides a run-time test to see how the ALWAYS and + ** NEVER macros were defined at compile-time. + ** + ** The return value is ALWAYS(X) if X is true, or 0 if X is false. + ** + ** The recommended test is X==2. If the return value is 2, that means + ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the + ** default setting. If the return value is 1, then ALWAYS() is either + ** hard-coded to true or else it asserts if its argument is false. + ** The first behavior (hard-coded to true) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second + ** behavior (assert if the argument to ALWAYS() is false) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. + ** + ** The run-time test procedure might look something like this: + ** + ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ + ** // ALWAYS() and NEVER() are no-op pass-through macros + ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ + ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. + ** }else{ + ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. + ** } + */ + case SQLITE_TESTCTRL_ALWAYS: { + int x = va_arg(ap,int); + rc = x ? ALWAYS(x) : 0; + break; + } + + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); + ** + ** The integer returned reveals the byte-order of the computer on which + ** SQLite is running: + ** + ** 1 big-endian, determined at run-time + ** 10 little-endian, determined at run-time + ** 432101 big-endian, determined at compile-time + ** 123410 little-endian, determined at compile-time + */ + case SQLITE_TESTCTRL_BYTEORDER: { + rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) + ** + ** Enable or disable various optimizations for testing purposes. The + ** argument N is a bitmask of optimizations to be disabled. For normal + ** operation N should be 0. The idea is that a test program (like the + ** SQL Logic Test or SLT test module) can run the same SQL multiple times + ** with various optimizations disabled to verify that the same answer + ** is obtained in every case. + */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->dbOptFlags = va_arg(ap, u32); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + ** + ** If parameter onoff is non-zero, subsequent calls to localtime() + ** and its variants fail. If onoff is zero, undo this setting. + */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: { + sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, sqlite3*); + ** + ** Toggle the ability to use internal functions on or off for + ** the database connection given in the argument. + */ + case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->mDbFlags ^= DBFLAG_InternalFunc; + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); + ** + ** Set or clear a flag that indicates that the database file is always well- + ** formed and never corrupt. This flag is clear by default, indicating that + ** database files might have arbitrary corruption. Setting the flag during + ** testing causes certain assert() statements in the code to be activated + ** that demonstrat invariants on well-formed database files. + */ + case SQLITE_TESTCTRL_NEVER_CORRUPT: { + sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS, int); + ** + ** Set or clear a flag that causes SQLite to verify that type, name, + ** and tbl_name fields of the sqlite_schema table. This is normally + ** on, but it is sometimes useful to turn it off for testing. + ** + ** 2020-07-22: Disabling EXTRA_SCHEMA_CHECKS also disables the + ** verification of rootpage numbers when parsing the schema. This + ** is useful to make it easier to reach strange internal error states + ** during testing. The EXTRA_SCHEMA_CHECKS setting is always enabled + ** in production. + */ + case SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS: { + sqlite3GlobalConfig.bExtraSchemaChecks = va_arg(ap, int); + break; + } + + /* Set the threshold at which OP_Once counters reset back to zero. + ** By default this is 0x7ffffffe (over 2 billion), but that value is + ** too big to test in a reasonable amount of time, so this control is + ** provided to set a small and easily reachable reset value. + */ + case SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD: { + sqlite3GlobalConfig.iOnceResetThreshold = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); + ** + ** Set the VDBE coverage callback function to xCallback with context + ** pointer ptr. + */ + case SQLITE_TESTCTRL_VDBE_COVERAGE: { +#ifdef SQLITE_VDBE_COVERAGE + typedef void (*branch_callback)(void*,unsigned int, + unsigned char,unsigned char); + sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); + sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); +#endif + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ + case SQLITE_TESTCTRL_SORTER_MMAP: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->nMaxSorterMmap = va_arg(ap, int); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); + ** + ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if + ** not. + */ + case SQLITE_TESTCTRL_ISINIT: { + if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, onOff, tnum); + ** + ** This test control is used to create imposter tables. "db" is a pointer + ** to the database connection. dbName is the database name (ex: "main" or + ** "temp") which will receive the imposter. "onOff" turns imposter mode on + ** or off. "tnum" is the root page of the b-tree to which the imposter + ** table should connect. + ** + ** Enable imposter mode only when the schema has already been parsed. Then + ** run a single CREATE TABLE statement to construct the imposter table in + ** the parsed schema. Then turn imposter mode back off again. + ** + ** If onOff==0 and tnum>0 then reset the schema for all databases, causing + ** the schema to be reparsed the next time it is needed. This has the + ** effect of erasing all imposter tables. + */ + case SQLITE_TESTCTRL_IMPOSTER: { + sqlite3 *db = va_arg(ap, sqlite3*); + sqlite3_mutex_enter(db->mutex); + db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); + db->init.busy = db->init.imposterTable = va_arg(ap,int); + db->init.newTnum = va_arg(ap,int); + if( db->init.busy==0 && db->init.newTnum>0 ){ + sqlite3ResetAllSchemasOfConnection(db); + } + sqlite3_mutex_leave(db->mutex); + break; + } + +#if defined(YYCOVERAGE) + /* sqlite3_test_control(SQLITE_TESTCTRL_PARSER_COVERAGE, FILE *out) + ** + ** This test control (only available when SQLite is compiled with + ** -DYYCOVERAGE) writes a report onto "out" that shows all + ** state/lookahead combinations in the parser state machine + ** which are never exercised. If any state is missed, make the + ** return code SQLITE_ERROR. + */ + case SQLITE_TESTCTRL_PARSER_COVERAGE: { + FILE *out = va_arg(ap, FILE*); + if( sqlite3ParserCoverage(out) ) rc = SQLITE_ERROR; + break; + } +#endif /* defined(YYCOVERAGE) */ + + /* sqlite3_test_control(SQLITE_TESTCTRL_RESULT_INTREAL, sqlite3_context*); + ** + ** This test-control causes the most recent sqlite3_result_int64() value + ** to be interpreted as a MEM_IntReal instead of as an MEM_Int. Normally, + ** MEM_IntReal values only arise during an INSERT operation of integer + ** values into a REAL column, so they can be challenging to test. This + ** test-control enables us to write an intreal() SQL function that can + ** inject an intreal() value at arbitrary places in an SQL statement, + ** for testing purposes. + */ + case SQLITE_TESTCTRL_RESULT_INTREAL: { + sqlite3_context *pCtx = va_arg(ap, sqlite3_context*); + sqlite3ResultIntReal(pCtx); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_SEEK_COUNT, + ** sqlite3 *db, // Database connection + ** u64 *pnSeek // Write seek count here + ** ); + ** + ** This test-control queries the seek-counter on the "main" database + ** file. The seek-counter is written into *pnSeek and is then reset. + ** The seek-count is only available if compiled with SQLITE_DEBUG. + */ + case SQLITE_TESTCTRL_SEEK_COUNT: { + sqlite3 *db = va_arg(ap, sqlite3*); + u64 *pn = va_arg(ap, sqlite3_uint64*); + *pn = sqlite3BtreeSeekCount(db->aDb->pBt); + (void)db; /* Silence harmless unused variable warning */ + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, op, ptr) + ** + ** "ptr" is a pointer to a u32. + ** + ** op==0 Store the current sqlite3SelectTrace in *ptr + ** op==1 Set sqlite3SelectTrace to the value *ptr + ** op==3 Store the current sqlite3WhereTrace in *ptr + ** op==3 Set sqlite3WhereTrace to the value *ptr + */ + case SQLITE_TESTCTRL_TRACEFLAGS: { + int opTrace = va_arg(ap, int); + u32 *ptr = va_arg(ap, u32*); + switch( opTrace ){ + case 0: *ptr = sqlite3SelectTrace; break; + case 1: sqlite3SelectTrace = *ptr; break; + case 2: *ptr = sqlite3WhereTrace; break; + case 3: sqlite3WhereTrace = *ptr; break; + } + break; + } + +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) + /* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue) + ** + ** If "id" is an integer between 1 and SQLITE_NTUNE then set the value + ** of the id-th tuning parameter to *piValue. If "id" is between -1 + ** and -SQLITE_NTUNE, then write the current value of the (-id)-th + ** tuning parameter into *piValue. + ** + ** Tuning parameters are for use during transient development builds, + ** to help find the best values for constants in the query planner. + ** Access tuning parameters using the Tuning(ID) macro. Set the + ** parameters in the CLI using ".testctrl tune ID VALUE". + ** + ** Transient use only. Tuning parameters should not be used in + ** checked-in code. + */ + case SQLITE_TESTCTRL_TUNE: { + int id = va_arg(ap, int); + int *piValue = va_arg(ap, int*); + if( id>0 && id<=SQLITE_NTUNE ){ + Tuning(id) = *piValue; + }else if( id<0 && id>=-SQLITE_NTUNE ){ + *piValue = Tuning(-id); + }else{ + rc = SQLITE_NOTFOUND; + } + break; + } +#endif + } + va_end(ap); +#endif /* SQLITE_UNTESTABLE */ + return rc; +} + +/* +** The Pager stores the Database filename, Journal filename, and WAL filename +** consecutively in memory, in that order. The database filename is prefixed +** by four zero bytes. Locate the start of the database filename by searching +** backwards for the first byte following four consecutive zero bytes. +** +** This only works if the filename passed in was obtained from the Pager. +*/ +static const char *databaseName(const char *zName){ + while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ + zName--; + } + return zName; +} + +/* +** Append text z[] to the end of p[]. Return a pointer to the first +** character after then zero terminator on the new text in p[]. +*/ +static char *appendText(char *p, const char *z){ + size_t n = strlen(z); + memcpy(p, z, n+1); + return p+n+1; +} + +/* +** Allocate memory to hold names for a database, journal file, WAL file, +** and query parameters. The pointer returned is valid for use by +** sqlite3_filename_database() and sqlite3_uri_parameter() and related +** functions. +** +** Memory layout must be compatible with that generated by the pager +** and expected by sqlite3_uri_parameter() and databaseName(). +*/ +SQLITE_API char *sqlite3_create_filename( + const char *zDatabase, + const char *zJournal, + const char *zWal, + int nParam, + const char **azParam +){ + sqlite3_int64 nByte; + int i; + char *pResult, *p; + nByte = strlen(zDatabase) + strlen(zJournal) + strlen(zWal) + 10; + for(i=0; i0 ){ + zFilename += sqlite3Strlen30(zFilename) + 1; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return zFilename[0] ? zFilename : 0; +} + +/* +** Return a boolean value for a query parameter. +*/ +SQLITE_API int sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + bDflt = bDflt!=0; + return z ? sqlite3GetBoolean(z, bDflt) : bDflt; +} + +/* +** Return a 64-bit integer value for a query parameter. +*/ +SQLITE_API sqlite3_int64 sqlite3_uri_int64( + const char *zFilename, /* Filename as passed to xOpen */ + const char *zParam, /* URI parameter sought */ + sqlite3_int64 bDflt /* return if parameter is missing */ +){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + sqlite3_int64 v; + if( z && sqlite3DecOrHexToI64(z, &v)==0 ){ + bDflt = v; + } + return bDflt; +} + +/* +** Translate a filename that was handed to a VFS routine into the corresponding +** database, journal, or WAL file. +** +** It is an error to pass this routine a filename string that was not +** passed into the VFS from the SQLite core. Doing so is similar to +** passing free() a pointer that was not obtained from malloc() - it is +** an error that we cannot easily detect but that will likely cause memory +** corruption. +*/ +SQLITE_API const char *sqlite3_filename_database(const char *zFilename){ + if( zFilename==0 ) return 0; + return databaseName(zFilename); +} +SQLITE_API const char *sqlite3_filename_journal(const char *zFilename){ + if( zFilename==0 ) return 0; + zFilename = databaseName(zFilename); + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + zFilename += sqlite3Strlen30(zFilename) + 1; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return zFilename + 1; +} +SQLITE_API const char *sqlite3_filename_wal(const char *zFilename){ +#ifdef SQLITE_OMIT_WAL + return 0; +#else + zFilename = sqlite3_filename_journal(zFilename); + if( zFilename ) zFilename += sqlite3Strlen30(zFilename) + 1; + return zFilename; +#endif +} + +/* +** Return the Btree pointer identified by zDbName. Return NULL if not found. +*/ +SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ + int iDb = zDbName ? sqlite3FindDbName(db, zDbName) : 0; + return iDb<0 ? 0 : db->aDb[iDb].pBt; +} + +/* +** Return the filename of the database associated with a database +** connection. +*/ +SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeGetFilename(pBt) : 0; +} + +/* +** Return 1 if database is read-only or 0 if read/write. Return -1 if +** no such database exists. +*/ +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ + Btree *pBt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; +} + +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** Obtain a snapshot handle for the snapshot of database zDb currently +** being read by handle db. +*/ +SQLITE_API int sqlite3_snapshot_get( + sqlite3 *db, + const char *zDb, + sqlite3_snapshot **ppSnapshot +){ + int rc = SQLITE_ERROR; +#ifndef SQLITE_OMIT_WAL + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + + if( db->autoCommit==0 ){ + int iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( SQLITE_TXN_WRITE!=sqlite3BtreeTxnState(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); + } + } + } + } + + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + +/* +** Open a read-transaction on the snapshot idendified by pSnapshot. +*/ +SQLITE_API int sqlite3_snapshot_open( + sqlite3 *db, + const char *zDb, + sqlite3_snapshot *pSnapshot +){ + int rc = SQLITE_ERROR; +#ifndef SQLITE_OMIT_WAL + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( db->autoCommit==0 ){ + int iDb; + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE ){ + Pager *pPager = sqlite3BtreePager(pBt); + int bUnlock = 0; + if( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_NONE ){ + if( db->nVdbeActive==0 ){ + rc = sqlite3PagerSnapshotCheck(pPager, pSnapshot); + if( rc==SQLITE_OK ){ + bUnlock = 1; + rc = sqlite3BtreeCommit(pBt); + } + } + }else{ + rc = SQLITE_OK; + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotOpen(pPager, pSnapshot); + } + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + sqlite3PagerSnapshotOpen(pPager, 0); + } + if( bUnlock ){ + sqlite3PagerSnapshotUnlock(pPager); + } + } + } + } + + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + +/* +** Recover as many snapshots as possible from the wal file associated with +** schema zDb of database db. +*/ +SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ + int rc = SQLITE_ERROR; + int iDb; +#ifndef SQLITE_OMIT_WAL + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + + sqlite3_mutex_enter(db->mutex); + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( SQLITE_TXN_NONE==sqlite3BtreeTxnState(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt)); + sqlite3BtreeCommit(pBt); + } + } + } + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + +/* +** Free a snapshot handle obtained from sqlite3_snapshot_get(). +*/ +SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){ + sqlite3_free(pSnapshot); +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +/* +** Given the name of a compile-time option, return true if that option +** was used and false if not. +** +** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix +** is not required for a match. +*/ +SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ + int i, n; + int nOpt; + const char **azCompileOpt; + +#if SQLITE_ENABLE_API_ARMOR + if( zOptName==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + azCompileOpt = sqlite3CompileOptions(&nOpt); + + if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; + n = sqlite3Strlen30(zOptName); + + /* Since nOpt is normally in single digits, a linear search is + ** adequate. No need for a binary search. */ + for(i=0; i=0 && NpNextBlocked){ + int seen = 0; + sqlite3 *p2; + + /* Verify property (1) */ + assert( p->pUnlockConnection || p->pBlockingConnection ); + + /* Verify property (2) */ + for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ + if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; + assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); + assert( db==0 || p->pUnlockConnection!=db ); + assert( db==0 || p->pBlockingConnection!=db ); + } + } +} +#else +# define checkListProperties(x) +#endif + +/* +** Remove connection db from the blocked connections list. If connection +** db is not currently a part of the list, this function is a no-op. +*/ +static void removeFromBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ + if( *pp==db ){ + *pp = (*pp)->pNextBlocked; + break; + } + } +} + +/* +** Add connection db to the blocked connections list. It is assumed +** that it is not already a part of the list. +*/ +static void addToBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for( + pp=&sqlite3BlockedList; + *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; + pp=&(*pp)->pNextBlocked + ); + db->pNextBlocked = *pp; + *pp = db; +} + +/* +** Obtain the STATIC_MAIN mutex. +*/ +static void enterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)); + checkListProperties(0); +} + +/* +** Release the STATIC_MAIN mutex. +*/ +static void leaveMutex(void){ + assertMutexHeld(); + checkListProperties(0); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)); +} + +/* +** Register an unlock-notify callback. +** +** This is called after connection "db" has attempted some operation +** but has received an SQLITE_LOCKED error because another connection +** (call it pOther) in the same process was busy using the same shared +** cache. pOther is found by looking at db->pBlockingConnection. +** +** If there is no blocking connection, the callback is invoked immediately, +** before this routine returns. +** +** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate +** a deadlock. +** +** Otherwise, make arrangements to invoke xNotify when pOther drops +** its locks. +** +** Each call to this routine overrides any prior callbacks registered +** on the same "db". If xNotify==0 then any prior callbacks are immediately +** cancelled. +*/ +SQLITE_API int sqlite3_unlock_notify( + sqlite3 *db, + void (*xNotify)(void **, int), + void *pArg +){ + int rc = SQLITE_OK; + + sqlite3_mutex_enter(db->mutex); + enterMutex(); + + if( xNotify==0 ){ + removeFromBlockedList(db); + db->pBlockingConnection = 0; + db->pUnlockConnection = 0; + db->xUnlockNotify = 0; + db->pUnlockArg = 0; + }else if( 0==db->pBlockingConnection ){ + /* The blocking transaction has been concluded. Or there never was a + ** blocking transaction. In either case, invoke the notify callback + ** immediately. + */ + xNotify(&pArg, 1); + }else{ + sqlite3 *p; + + for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} + if( p ){ + rc = SQLITE_LOCKED; /* Deadlock detected. */ + }else{ + db->pUnlockConnection = db->pBlockingConnection; + db->xUnlockNotify = xNotify; + db->pUnlockArg = pArg; + removeFromBlockedList(db); + addToBlockedList(db); + } + } + + leaveMutex(); + assert( !db->mallocFailed ); + sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** This function is called while stepping or preparing a statement +** associated with connection db. The operation will return SQLITE_LOCKED +** to the user because it requires a lock that will not be available +** until connection pBlocker concludes its current transaction. +*/ +SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ + enterMutex(); + if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ + addToBlockedList(db); + } + db->pBlockingConnection = pBlocker; + leaveMutex(); +} + +/* +** This function is called when +** the transaction opened by database db has just finished. Locks held +** by database connection db have been released. +** +** This function loops through each entry in the blocked connections +** list and does the following: +** +** 1) If the sqlite3.pBlockingConnection member of a list entry is +** set to db, then set pBlockingConnection=0. +** +** 2) If the sqlite3.pUnlockConnection member of a list entry is +** set to db, then invoke the configured unlock-notify callback and +** set pUnlockConnection=0. +** +** 3) If the two steps above mean that pBlockingConnection==0 and +** pUnlockConnection==0, remove the entry from the blocked connections +** list. +*/ +SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ + void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ + int nArg = 0; /* Number of entries in aArg[] */ + sqlite3 **pp; /* Iterator variable */ + void **aArg; /* Arguments to the unlock callback */ + void **aDyn = 0; /* Dynamically allocated space for aArg[] */ + void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ + + aArg = aStatic; + enterMutex(); /* Enter STATIC_MAIN mutex */ + + /* This loop runs once for each entry in the blocked-connections list. */ + for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ + sqlite3 *p = *pp; + + /* Step 1. */ + if( p->pBlockingConnection==db ){ + p->pBlockingConnection = 0; + } + + /* Step 2. */ + if( p->pUnlockConnection==db ){ + assert( p->xUnlockNotify ); + if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + + sqlite3BeginBenignMalloc(); + assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); + assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); + if( (!aDyn && nArg==(int)ArraySize(aStatic)) + || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) + ){ + /* The aArg[] array needs to grow. */ + void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); + if( pNew ){ + memcpy(pNew, aArg, nArg*sizeof(void *)); + sqlite3_free(aDyn); + aDyn = aArg = pNew; + }else{ + /* This occurs when the array of context pointers that need to + ** be passed to the unlock-notify callback is larger than the + ** aStatic[] array allocated on the stack and the attempt to + ** allocate a larger array from the heap has failed. + ** + ** This is a difficult situation to handle. Returning an error + ** code to the caller is insufficient, as even if an error code + ** is returned the transaction on connection db will still be + ** closed and the unlock-notify callbacks on blocked connections + ** will go unissued. This might cause the application to wait + ** indefinitely for an unlock-notify callback that will never + ** arrive. + ** + ** Instead, invoke the unlock-notify callback with the context + ** array already accumulated. We can then clear the array and + ** begin accumulating any further context pointers without + ** requiring any dynamic allocation. This is sub-optimal because + ** it means that instead of one callback with a large array of + ** context pointers the application will receive two or more + ** callbacks with smaller arrays of context pointers, which will + ** reduce the applications ability to prioritize multiple + ** connections. But it is the best that can be done under the + ** circumstances. + */ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + } + sqlite3EndBenignMalloc(); + + aArg[nArg++] = p->pUnlockArg; + xUnlockNotify = p->xUnlockNotify; + p->pUnlockConnection = 0; + p->xUnlockNotify = 0; + p->pUnlockArg = 0; + } + + /* Step 3. */ + if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ + /* Remove connection p from the blocked connections list. */ + *pp = p->pNextBlocked; + p->pNextBlocked = 0; + }else{ + pp = &p->pNextBlocked; + } + } + + if( nArg!=0 ){ + xUnlockNotify(aArg, nArg); + } + sqlite3_free(aDyn); + leaveMutex(); /* Leave STATIC_MAIN mutex */ +} + +/* +** This is called when the database connection passed as an argument is +** being closed. The connection is removed from the blocked list. +*/ +SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ + sqlite3ConnectionUnlocked(db); + enterMutex(); + removeFromBlockedList(db); + checkListProperties(db); + leaveMutex(); +} +#endif + +/************** End of notify.c **********************************************/ +/************** Begin file fts3.c ********************************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ + +/* The full-text index is stored in a series of b+tree (-like) +** structures called segments which map terms to doclists. The +** structures are like b+trees in layout, but are constructed from the +** bottom up in optimal fashion and are not updatable. Since trees +** are built from the bottom up, things will be described from the +** bottom up. +** +** +**** Varints **** +** The basic unit of encoding is a variable-length integer called a +** varint. We encode variable-length integers in little-endian order +** using seven bits * per byte as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** and so on. +** +** This is similar in concept to how sqlite encodes "varints" but +** the encoding is not the same. SQLite varints are big-endian +** are are limited to 9 bytes in length whereas FTS3 varints are +** little-endian and can be up to 10 bytes in length (in theory). +** +** Example encodings: +** +** 1: 0x01 +** 127: 0x7f +** 128: 0x81 0x00 +** +** +**** Document lists **** +** A doclist (document list) holds a docid-sorted list of hits for a +** given term. Doclists hold docids and associated token positions. +** A docid is the unique integer identifier for a single document. +** A position is the index of a word within the document. The first +** word of the document has a position of 0. +** +** FTS3 used to optionally store character offsets using a compile-time +** option. But that functionality is no longer supported. +** +** A doclist is stored like this: +** +** array { +** varint docid; (delta from previous doclist) +** array { (position list for column 0) +** varint position; (2 more than the delta from previous position) +** } +** array { +** varint POS_COLUMN; (marks start of position list for new column) +** varint column; (index of new column) +** array { +** varint position; (2 more than the delta from previous position) +** } +** } +** varint POS_END; (marks end of positions for this document. +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. A "position" is an index of a token in the token stream +** generated by the tokenizer. Note that POS_END and POS_COLUMN occur +** in the same logical place as the position element, and act as sentinals +** ending a position list array. POS_END is 0. POS_COLUMN is 1. +** The positions numbers are not stored literally but rather as two more +** than the difference from the prior position, or the just the position plus +** 2 for the first position. Example: +** +** label: A B C D E F G H I J K +** value: 123 5 9 1 1 14 35 0 234 72 0 +** +** The 123 value is the first docid. For column zero in this document +** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 +** at D signals the start of a new column; the 1 at E indicates that the +** new column is column number 1. There are two positions at 12 and 45 +** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. +** +** A "position-list" is the list of positions for multiple columns for +** a single docid. A "column-list" is the set of positions for a single +** column. Hence, a position-list consists of one or more column-lists, +** a document record consists of a docid followed by a position-list and +** a doclist consists of one or more document records. +** +** A bare doclist omits the position information, becoming an +** array of varint-encoded docids. +** +**** Segment leaf nodes **** +** Segment leaf nodes store terms and doclists, ordered by term. Leaf +** nodes are written using LeafWriter, and read using LeafReader (to +** iterate through a single leaf node's data) and LeavesReader (to +** iterate through a segment's entire leaf layer). Leaf nodes have +** the format: +** +** varint iHeight; (height from leaf level, always 0) +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of prefix shared with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix];(unshared suffix of next term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. +** +** Leaf nodes are broken into blocks which are stored contiguously in +** the %_segments table in sorted order. This means that when the end +** of a node is reached, the next term is in the node with the next +** greater node id. +** +** New data is spilled to a new leaf node when the current node +** exceeds LEAF_MAX bytes (default 2048). New data which itself is +** larger than STANDALONE_MIN (default 1024) is placed in a standalone +** node (a leaf node with a single term and doclist). The goal of +** these settings is to pack together groups of small doclists while +** making it efficient to directly access large doclists. The +** assumption is that large doclists represent terms which are more +** likely to be query targets. +** +** TODO(shess) It may be useful for blocking decisions to be more +** dynamic. For instance, it may make more sense to have a 2.5k leaf +** node rather than splitting into 2k and .5k nodes. My intuition is +** that this might extend through 2x or 4x the pagesize. +** +** +**** Segment interior nodes **** +** Segment interior nodes store blockids for subtree nodes and terms +** to describe what data is stored by the each subtree. Interior +** nodes are written using InteriorWriter, and read using +** InteriorReader. InteriorWriters are created as needed when +** SegmentWriter creates new leaf nodes, or when an interior node +** itself grows too big and must be split. The format of interior +** nodes: +** +** varint iHeight; (height from leaf level, always >0) +** varint iBlockid; (block id of node's leftmost subtree) +** optional { +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of shared prefix with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix]; (unshared suffix of next term) +** } +** } +** +** Here, optional { X } means an optional element, while array { X } +** means zero or more occurrences of X, adjacent in memory. +** +** An interior node encodes n terms separating n+1 subtrees. The +** subtree blocks are contiguous, so only the first subtree's blockid +** is encoded. The subtree at iBlockid will contain all terms less +** than the first term encoded (or all terms if no term is encoded). +** Otherwise, for terms greater than or equal to pTerm[i] but less +** than pTerm[i+1], the subtree for that term will be rooted at +** iBlockid+i. Interior nodes only store enough term data to +** distinguish adjacent children (if the rightmost term of the left +** child is "something", and the leftmost term of the right child is +** "wicked", only "w" is stored). +** +** New data is spilled to a new interior node at the same height when +** the current node exceeds INTERIOR_MAX bytes (default 2048). +** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing +** interior nodes and making the tree too skinny. The interior nodes +** at a given height are naturally tracked by interior nodes at +** height+1, and so on. +** +** +**** Segment directory **** +** The segment directory in table %_segdir stores meta-information for +** merging and deleting segments, and also the root node of the +** segment's tree. +** +** The root node is the top node of the segment's tree after encoding +** the entire segment, restricted to ROOT_MAX bytes (default 1024). +** This could be either a leaf node or an interior node. If the top +** node requires more than ROOT_MAX bytes, it is flushed to %_segments +** and a new root interior node is generated (which should always fit +** within ROOT_MAX because it only needs space for 2 varints, the +** height and the blockid of the previous root). +** +** The meta-information in the segment directory is: +** level - segment level (see below) +** idx - index within level +** - (level,idx uniquely identify a segment) +** start_block - first leaf node +** leaves_end_block - last leaf node +** end_block - last block (including interior nodes) +** root - contents of root node +** +** If the root node is a leaf node, then start_block, +** leaves_end_block, and end_block are all 0. +** +** +**** Segment merging **** +** To amortize update costs, segments are grouped into levels and +** merged in batches. Each increase in level represents exponentially +** more documents. +** +** New documents (actually, document updates) are tokenized and +** written individually (using LeafWriter) to a level 0 segment, with +** incrementing idx. When idx reaches MERGE_COUNT (default 16), all +** level 0 segments are merged into a single level 1 segment. Level 1 +** is populated like level 0, and eventually MERGE_COUNT level 1 +** segments are merged to a single level 2 segment (representing +** MERGE_COUNT^2 updates), and so on. +** +** A segment merge traverses all segments at a given level in +** parallel, performing a straightforward sorted merge. Since segment +** leaf nodes are written in to the %_segments table in order, this +** merge traverses the underlying sqlite disk structures efficiently. +** After the merge, all segment blocks from the merged level are +** deleted. +** +** MERGE_COUNT controls how often we merge segments. 16 seems to be +** somewhat of a sweet spot for insertion performance. 32 and 64 show +** very similar performance numbers to 16 on insertion, though they're +** a tiny bit slower (perhaps due to more overhead in merge-time +** sorting). 8 is about 20% slower than 16, 4 about 50% slower than +** 16, 2 about 66% slower than 16. +** +** At query time, high MERGE_COUNT increases the number of segments +** which need to be scanned and merged. For instance, with 100k docs +** inserted: +** +** MERGE_COUNT segments +** 16 25 +** 8 12 +** 4 10 +** 2 6 +** +** This appears to have only a moderate impact on queries for very +** frequent terms (which are somewhat dominated by segment merge +** costs), and infrequent and non-existent terms still seem to be fast +** even with many segments. +** +** TODO(shess) That said, it would be nice to have a better query-side +** argument for MERGE_COUNT of 16. Also, it is possible/likely that +** optimizations to things like doclist merging will swing the sweet +** spot around. +** +** +** +**** Handling of deletions and updates **** +** Since we're using a segmented structure, with no docid-oriented +** index into the term index, we clearly cannot simply update the term +** index when a document is deleted or updated. For deletions, we +** write an empty doclist (varint(docid) varint(POS_END)), for updates +** we simply write the new doclist. Segment merges overwrite older +** data for a particular docid with newer data, so deletes or updates +** will eventually overtake the earlier data and knock it out. The +** query logic likewise merges doclists so that newer data knocks out +** older data. +*/ + +/************** Include fts3Int.h in the middle of fts3.c ********************/ +/************** Begin file fts3Int.h *****************************************/ +/* +** 2009 Nov 12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +#ifndef _FTSINT_H +#define _FTSINT_H + +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* FTS3/FTS4 require virtual tables */ +#ifdef SQLITE_OMIT_VIRTUALTABLE +# undef SQLITE_ENABLE_FTS3 +# undef SQLITE_ENABLE_FTS4 +#endif + +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +/* # include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT3 +#endif + +/* #include "sqlite3.h" */ +/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ +/************** Begin file fts3_tokenizer.h **********************************/ +/* +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. +*/ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ + +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ +/* #include "sqlite3.h" */ + +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; + +struct sqlite3_tokenizer_module { + + /* + ** Structure version. Should always be set to 0 or 1. + */ + int iVersion; + + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialized by + ** this callback. The caller will do so. + */ + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); + + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); + + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). + */ + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + ); + + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). + */ + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); + + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. + ** + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. + ** + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). + */ + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. + */ + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ + ); + + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ + + /* + ** Configure the language id of a tokenizer cursor. + */ + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); +}; + +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; + +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; + +int fts3_global_term_cnt(int iTerm, int iCol); +int fts3_term_cnt(int iTerm, int iCol); + + +#endif /* _FTS3_TOKENIZER_H_ */ + +/************** End of fts3_tokenizer.h **************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ +/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ +/************** Begin file fts3_hash.h ***************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implementation +** used in SQLite. We've modified it slightly to serve as a standalone +** hash table implementation for the full-text indexing module. +** +*/ +#ifndef _FTS3_HASH_H_ +#define _FTS3_HASH_H_ + +/* Forward declarations of structures. */ +typedef struct Fts3Hash Fts3Hash; +typedef struct Fts3HashElem Fts3HashElem; + +/* A complete hash table is an instance of the following structure. +** The internals of this structure are intended to be opaque -- client +** code should not attempt to access or modify the fields of this structure +** directly. Change this structure only by using the routines below. +** However, many of the "procedures" and "functions" for modifying and +** accessing this structure are really macros, so we can't really make +** this structure opaque. +*/ +struct Fts3Hash { + char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ + char copyKey; /* True if copy of key made on insert */ + int count; /* Number of entries in this table */ + Fts3HashElem *first; /* The first element of the array */ + int htsize; /* Number of buckets in the hash table */ + struct _fts3ht { /* the hash table */ + int count; /* Number of entries with this hash */ + Fts3HashElem *chain; /* Pointer to first entry with this hash */ + } *ht; +}; + +/* Each element in the hash table is an instance of the following +** structure. All elements are stored on a single doubly-linked list. +** +** Again, this structure is intended to be opaque, but it can't really +** be opaque because it is used by macros. +*/ +struct Fts3HashElem { + Fts3HashElem *next, *prev; /* Next and previous elements in the table */ + void *data; /* Data associated with this element */ + void *pKey; int nKey; /* Key associated with this element */ +}; + +/* +** There are 2 different modes of operation for a hash table: +** +** FTS3_HASH_STRING pKey points to a string that is nKey bytes long +** (including the null-terminator, if any). Case +** is respected in comparisons. +** +** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. +** memcmp() is used to compare keys. +** +** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. +*/ +#define FTS3_HASH_STRING 1 +#define FTS3_HASH_BINARY 2 + +/* +** Access routines. To delete, insert a NULL pointer. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); +SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); + +/* +** Shorthand for the functions above +*/ +#define fts3HashInit sqlite3Fts3HashInit +#define fts3HashInsert sqlite3Fts3HashInsert +#define fts3HashFind sqlite3Fts3HashFind +#define fts3HashClear sqlite3Fts3HashClear +#define fts3HashFindElem sqlite3Fts3HashFindElem + +/* +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Fts3Hash h; +** Fts3HashElem *p; +** ... +** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ +** SomeStructure *pData = fts3HashData(p); +** // do something with pData +** } +*/ +#define fts3HashFirst(H) ((H)->first) +#define fts3HashNext(E) ((E)->next) +#define fts3HashData(E) ((E)->data) +#define fts3HashKey(E) ((E)->pKey) +#define fts3HashKeysize(E) ((E)->nKey) + +/* +** Number of entries in a hash table +*/ +#define fts3HashCount(H) ((H)->count) + +#endif /* _FTS3_HASH_H_ */ + +/************** End of fts3_hash.h *******************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ + +/* +** This constant determines the maximum depth of an FTS expression tree +** that the library will create and use. FTS uses recursion to perform +** various operations on the query tree, so the disadvantage of a large +** limit is that it may allow very large queries to use large amounts +** of stack space (perhaps causing a stack overflow). +*/ +#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH +# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 +#endif + + +/* +** This constant controls how often segments are merged. Once there are +** FTS3_MERGE_COUNT segments of level N, they are merged into a single +** segment of level N+1. +*/ +#define FTS3_MERGE_COUNT 16 + +/* +** This is the maximum amount of data (in bytes) to store in the +** Fts3Table.pendingTerms hash table. Normally, the hash table is +** populated as documents are inserted/updated/deleted in a transaction +** and used to create a new segment when the transaction is committed. +** However if this limit is reached midway through a transaction, a new +** segment is created and the hash table cleared immediately. +*/ +#define FTS3_MAX_PENDING_DATA (1*1024*1024) + +/* +** Macro to return the number of elements in an array. SQLite has a +** similar macro called ArraySize(). Use a different name to avoid +** a collision when building an amalgamation with built-in FTS3. +*/ +#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif +#ifndef MAX +# define MAX(x,y) ((x)>(y)?(x):(y)) +#endif + +/* +** Maximum length of a varint encoded integer. The varint format is different +** from that used by SQLite, so the maximum length is 10, not 9. +*/ +#define FTS3_VARINT_MAX 10 + +#define FTS3_BUFFER_PADDING 8 + +/* +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. +** +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. +*/ +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" + +/* +** The testcase() macro is only used by the amalgamation. If undefined, +** make it a no-op. +*/ +#ifndef testcase +# define testcase(X) +#endif + +/* +** Terminator values for position-lists and column-lists. +*/ +#define POS_COLUMN (1) /* Column-list terminator */ +#define POS_END (0) /* Position-list terminator */ + +/* +** The assert_fts3_nc() macro is similar to the assert() macro, except that it +** is used for assert() conditions that are true only if it can be +** guranteed that the database is not corrupt. +*/ +#ifdef SQLITE_DEBUG +SQLITE_API extern int sqlite3_fts3_may_be_corrupt; +# define assert_fts3_nc(x) assert(sqlite3_fts3_may_be_corrupt || (x)) +#else +# define assert_fts3_nc(x) assert(x) +#endif + +/* +** This section provides definitions to allow the +** FTS3 extension to be compiled outside of the +** amalgamation. +*/ +#ifndef SQLITE_AMALGAMATION +/* +** Macros indicating that conditional expressions are always true or +** false. +*/ +#ifdef SQLITE_COVERAGE_TEST +# define ALWAYS(x) (1) +# define NEVER(X) (0) +#elif defined(SQLITE_DEBUG) +# define ALWAYS(x) sqlite3Fts3Always((x)!=0) +# define NEVER(x) sqlite3Fts3Never((x)!=0) +SQLITE_PRIVATE int sqlite3Fts3Always(int b); +SQLITE_PRIVATE int sqlite3Fts3Never(int b); +#else +# define ALWAYS(x) (x) +# define NEVER(x) (x) +#endif + +/* +** Internal types used by SQLite. +*/ +typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ +typedef short int i16; /* 2-byte (or larger) signed integer */ +typedef unsigned int u32; /* 4-byte unsigned integer */ +typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ + +/* +** Macro used to suppress compiler warnings for unused parameters. +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif + +/* +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +#define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) + +#define deliberate_fall_through + +#endif /* SQLITE_AMALGAMATION */ + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); +# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() +#else +# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB +#endif + +typedef struct Fts3Table Fts3Table; +typedef struct Fts3Cursor Fts3Cursor; +typedef struct Fts3Expr Fts3Expr; +typedef struct Fts3Phrase Fts3Phrase; +typedef struct Fts3PhraseToken Fts3PhraseToken; + +typedef struct Fts3Doclist Fts3Doclist; +typedef struct Fts3SegFilter Fts3SegFilter; +typedef struct Fts3DeferredToken Fts3DeferredToken; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; + +typedef struct MatchinfoBuffer MatchinfoBuffer; + +/* +** A connection to a fulltext index is an instance of the following +** structure. The xCreate and xConnect methods create an instance +** of this structure and xDestroy and xDisconnect free that instance. +** All other methods receive a pointer to the structure as one of their +** arguments. +*/ +struct Fts3Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + sqlite3 *db; /* The database connection */ + const char *zDb; /* logical database name */ + const char *zName; /* virtual table name */ + int nColumn; /* number of named columns in virtual table */ + char **azColumn; /* column names. malloced */ + u8 *abNotindexed; /* True for 'notindexed' columns */ + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ + char *zLanguageid; /* languageid=xxx option, or NULL */ + int nAutoincrmerge; /* Value configured by 'automerge' */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ + int bLock; /* Used to prevent recursive content= tbls */ + + /* Precompiled statements used by the implementation. Each of these + ** statements is run and reset within a single virtual table API call. + */ + sqlite3_stmt *aStmt[40]; + sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ + + char *zReadExprlist; + char *zWriteExprlist; + + int nNodeSize; /* Soft limit for node size */ + u8 bFts4; /* True for FTS4, false for FTS3 */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ + u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + + /* + ** The following array of hash tables is used to buffer pending index + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. + */ + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + int iPrevLangid; /* Langid of recently inserted document */ + int bPrevDelete; /* True if last operation was a delete */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + /* State variables used for validating that the transaction control + ** methods of the virtual table are called at appropriate times. These + ** values do not contribute to FTS functionality; they are used for + ** verifying the operation of the SQLite core. + */ + int inTransaction; /* True after xBegin but before xCommit/xRollback */ + int mxSavepoint; /* Largest valid xSavepoint integer */ +#endif + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* True to disable the incremental doclist optimization. This is controled + ** by special insert command 'test-no-incr-doclist'. */ + int bNoIncrDoclist; + + /* Number of segments in a level */ + int nMergeCount; +#endif +}; + +/* Macro to find the number of segments to merge */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) +# define MergeCount(P) ((P)->nMergeCount) +#else +# define MergeCount(P) FTS3_MERGE_COUNT +#endif + +/* +** When the core wants to read from the virtual table, it creates a +** virtual table cursor (an instance of the following structure) using +** the xOpen method. Cursors are destroyed using the xClose method. +*/ +struct Fts3Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + i16 eSearch; /* Search strategy (see below) */ + u8 isEof; /* True if at End Of Results */ + u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + u8 bSeekStmt; /* True if pStmt is a seek */ + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ + Fts3Expr *pExpr; /* Parsed MATCH query string */ + int iLangid; /* Language being queried for */ + int nPhrase; /* Number of matchable phrases in query */ + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ + sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ + char *pNextId; /* Pointer into the body of aDoclist */ + char *aDoclist; /* List of docids for full-text queries */ + int nDoclist; /* Size of buffer at aDoclist */ + u8 bDesc; /* True to sort in descending order */ + int eEvalmode; /* An FTS3_EVAL_XX constant */ + int nRowAvg; /* Average size of database rows, in pages */ + sqlite3_int64 nDoc; /* Documents in table */ + i64 iMinDocid; /* Minimum docid to return */ + i64 iMaxDocid; /* Maximum docid to return */ + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ + MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ +}; + +#define FTS3_EVAL_FILTER 0 +#define FTS3_EVAL_NEXT 1 +#define FTS3_EVAL_MATCHINFO 2 + +/* +** The Fts3Cursor.eSearch member is always set to one of the following. +** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index +** of the column to be searched. For example, in +** +** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); +** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; +** +** Because the LHS of the MATCH operator is 2nd column "b", +** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, +** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" +** indicating that all columns should be searched, +** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. +*/ +#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ +#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ +#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +/* +** The lower 16-bits of the sqlite3_index_info.idxNum value set by +** the xBestIndex() method contains the Fts3Cursor.eSearch value described +** above. The upper 16-bits contain a combination of the following +** bits, used to describe extra constraints on full-text searches. +*/ +#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ +#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ +#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +}; + +/* +** A "phrase" is a sequence of one or more tokens that must match in +** sequence. A single token is the base case and the most common case. +** For a sequence of tokens contained in double-quotes (i.e. "one two three") +** nToken will be the number of tokens in the string. +*/ +struct Fts3PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer z */ + int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ +}; + +struct Fts3Phrase { + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an + ** OR condition. */ + char *pOrPoslist; + i64 iOrDocid; + + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ +}; + +/* +** A tree of these objects forms the RHS of a MATCH operator. +** +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. +** +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. +*/ +struct Fts3Expr { + int eType; /* One of the FTSQUERY_XXX values defined below */ + int nNear; /* Valid if eType==FTSQUERY_NEAR */ + Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ + + /* The following are used by the fts3_snippet.c module. */ + int iPhrase; /* Index of this phrase in matchinfo() results */ + u32 *aMI; /* See above */ +}; + +/* +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: +** +** "a OR b AND c NOT d NEAR e" +** +** is equivalent to: +** +** "a OR (b AND (c NOT (d NEAR e)))" +*/ +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 + + +/* fts3_write.c */ +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, + sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); + +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); +#else +# define sqlite3Fts3FreeDeferredTokens(x) +# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK +# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK +# define sqlite3Fts3FreeDeferredDoclists(x) +# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK +#endif + +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); + +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); + +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, + int, int, int, const char *, int, int, int, Fts3MultiSegReader *); + +/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ +#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 +#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 +#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 +#define FTS3_SEGMENT_PREFIX 0x00000008 +#define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 + +/* Type passed as 4th argument to SegmentReaderIterate() */ +struct Fts3SegFilter { + const char *zTerm; + int nTerm; + int iCol; + int flags; +}; + +struct Fts3MultiSegReader { + /* Used internally by sqlite3Fts3SegReaderXXX() calls */ + Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ + int nSegment; /* Size of apSegment array */ + int nAdvance; /* How many seg-readers to advance */ + Fts3SegFilter *pFilter; /* Pointer to filter object */ + char *aBuffer; /* Buffer to merge doclists in */ + int nBuffer; /* Allocated size of aBuffer[] in bytes */ + + int iColFilter; /* If >=0, filter for this column */ + int bRestart; + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ + + /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ + char *zTerm; /* Pointer to term buffer */ + int nTerm; /* Size of zTerm in bytes */ + char *aDoclist; /* Pointer to doclist buffer */ + int nDoclist; /* Size of aDoclist[] in bytes */ +}; + +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); + +#define fts3GetVarint32(p, piVal) ( \ + (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ +) + +/* fts3.c */ +SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +SQLITE_PRIVATE int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *); +SQLITE_PRIVATE int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*); +SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); +SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); +SQLITE_PRIVATE int sqlite3Fts3ReadInt(const char *z, int *pnOut); + +/* fts3_tokenizer.c */ +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, + sqlite3_tokenizer **, char ** +); +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); + +/* fts3_snippet.c */ +SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); +SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, + const char *, const char *, int, int +); +SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); +SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); + +/* fts3_expr.c */ +SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, + char **, int, int, int, const char *, int, Fts3Expr **, char ** +); +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*); +SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); +#endif + +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, + sqlite3_tokenizer_cursor ** +); + +/* fts3_aux.c */ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); + +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + +/* fts3_tokenize_vtab.c */ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); + +/* fts3_unicode2.c (functions generated by parsing unicode text files) */ +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); +#endif + +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ +#endif /* _FTSINT_H */ + +/************** End of fts3Int.h *********************************************/ +/************** Continuing where we left off in fts3.c ***********************/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) +# define SQLITE_CORE 1 +#endif + +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3.h" */ +#ifndef SQLITE_CORE +/* # include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 +#endif + +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + +#ifndef SQLITE_AMALGAMATION +# if defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; } +SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; } +# endif +#endif + +/* +** This variable is set to false when running tests for which the on disk +** structures should not be corrupt. Otherwise, true. If it is false, extra +** assert() conditions in the fts3 code are activated - conditions that are +** only true if it is guaranteed that the fts3 database is not corrupt. +*/ +#ifdef SQLITE_DEBUG +SQLITE_API int sqlite3_fts3_may_be_corrupt = 1; +#endif + +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. +** The number of bytes written is returned. +*/ +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ + unsigned char *q = (unsigned char *) p; + sqlite_uint64 vu = v; + do{ + *q++ = (unsigned char) ((vu & 0x7f) | 0x80); + vu >>= 7; + }while( vu!=0 ); + q[-1] &= 0x7f; /* turn off high bit in final byte */ + assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); + return (int) (q - (unsigned char *)p); +} + +#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ + v = (v & mask1) | ( (*(const unsigned char*)(ptr++)) << shift ); \ + if( (v & mask2)==0 ){ var = v; return ret; } +#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ + v = (*ptr++); \ + if( (v & mask2)==0 ){ var = v; return ret; } + +SQLITE_PRIVATE int sqlite3Fts3GetVarintU(const char *pBuf, sqlite_uint64 *v){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; + u32 a; + u64 b; + int shift; + + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); + b = (a & 0x0FFFFFFF ); + + for(shift=28; shift<=63; shift+=7){ + u64 c = *p++; + b += (c&0x7F) << shift; + if( (c & 0x80)==0 ) break; + } + *v = b; + return (int)(p - pStart); +} + +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ + return sqlite3Fts3GetVarintU(pBuf, (sqlite3_uint64*)v); +} + +/* +** Read a 64-bit variable-length integer from memory starting at p[0] and +** not extending past pEnd[-1]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +SQLITE_PRIVATE int sqlite3Fts3GetVarintBounded( + const char *pBuf, + const char *pEnd, + sqlite_int64 *v +){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; + const unsigned char *pX = (const unsigned char*)pEnd; + u64 b = 0; + int shift; + for(shift=0; shift<=63; shift+=7){ + u64 c = p=0 ); + return 5; +} + +/* +** Return the number of bytes required to encode v as a varint +*/ +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){ + int i = 0; + do{ + i++; + v >>= 7; + }while( v!=0 ); + return i; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +** +*/ +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ + char quote; /* Quote character (if any ) */ + + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + int iIn = 1; /* Index of next byte to read from input */ + int iOut = 0; /* Index of next byte to write to output */ + + /* If the first byte was a '[', then the close-quote character is a ']' */ + if( quote=='[' ) quote = ']'; + + while( z[iIn] ){ + if( z[iIn]==quote ){ + if( z[iIn+1]!=quote ) break; + z[iOut++] = quote; + iIn += 2; + }else{ + z[iOut++] = z[iIn++]; + } + } + z[iOut] = '\0'; + } +} + +/* +** Read a single varint from the doclist at *pp and advance *pp to point +** to the first byte past the end of the varint. Add the value of the varint +** to *pVal. +*/ +static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + *pVal += iVal; +} + +/* +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and sets *pVal by the varint value. +** +** Argument pStart points to the first byte of the doclist that the +** varint is part of. +*/ +static void fts3GetReverseVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p; + + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; + + sqlite3Fts3GetVarint(p, &iVal); + *pVal = iVal; +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int i; + + assert( p->nPendingData==0 ); + assert( p->pSegments==0 ); + + /* Free any prepared statements held */ + sqlite3_finalize(p->pSeekStmt); + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); + } + sqlite3_free(p->zSegmentsTbl); + sqlite3_free(p->zReadExprlist); + sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); + sqlite3_free(p->zLanguageid); + + /* Invoke the tokenizer destructor to free the tokenizer. */ + p->pTokenizer->pModule->xDestroy(p->pTokenizer); + + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Write an error message into *pzErr +*/ +SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ + va_list ap; + sqlite3_free(*pzErr); + va_start(ap, zFormat); + *pzErr = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Construct one or more SQL statements from the format string given +** and then evaluate those statements. The success code is written +** into *pRc. +** +** If *pRc is initially non-zero then this routine is a no-op. +*/ +static void fts3DbExec( + int *pRc, /* Success code */ + sqlite3 *db, /* Database in which to run SQL */ + const char *zFormat, /* Format string for SQL */ + ... /* Arguments to the format string */ +){ + va_list ap; + char *zSql; + if( *pRc ) return; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + *pRc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } +} + +/* +** The xDestroy() virtual table method. +*/ +static int fts3DestroyMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return code */ + const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ + + /* Drop the shadow tables */ + fts3DbExec(&rc, db, + "DROP TABLE IF EXISTS %Q.'%q_segments';" + "DROP TABLE IF EXISTS %Q.'%q_segdir';" + "DROP TABLE IF EXISTS %Q.'%q_docsize';" + "DROP TABLE IF EXISTS %Q.'%q_stat';" + "%s DROP TABLE IF EXISTS %Q.'%q_content';", + zDb, p->zName, + zDb, p->zName, + zDb, p->zName, + zDb, p->zName, + (p->zContentTbl ? "--" : ""), zDb,p->zName + ); + + /* If everything has worked, invoke fts3DisconnectMethod() to free the + ** memory associated with the Fts3Table structure and return SQLITE_OK. + ** Otherwise, return an SQLite error code. + */ + return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); +} + + +/* +** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table +** passed as the first argument. This is done as part of the xConnect() +** and xCreate() methods. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ + const char *zLanguageid; + + zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); + } + + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", + zCols, p->zName, zLanguageid + ); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(p->db, zSql); + } + + sqlite3_free(zSql); + sqlite3_free(zCols); + *pRc = rc; + } +} + +/* +** Create the %_stat table if it does not already exist. +*/ +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName + ); + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; +} + +/* +** Create the backing store tables (%_content, %_segments and %_segdir) +** required by the FTS3 table passed as the only argument. This is done +** as part of the vtab xCreate() method. +** +** If the p->bHasDocsize boolean is true (indicating that this is an +** FTS4 table, not an FTS3 table) then also create the %_docsize and +** %_stat tables required by FTS4. +*/ +static int fts3CreateTables(Fts3Table *p){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3 *db = p->db; /* The database connection */ + + if( p->zContentTbl==0 ){ + const char *zLanguageid = p->zLanguageid; + char *zContentCols; /* Columns of %_content table */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zLanguageid && zContentCols ){ + zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } + + /* Create other tables */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + p->zDb, p->zName + ); + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segdir'(" + "level INTEGER," + "idx INTEGER," + "start_block INTEGER," + "leaves_end_block INTEGER," + "end_block INTEGER," + "root BLOB," + "PRIMARY KEY(level, idx)" + ");", + p->zDb, p->zName + ); + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + p->zDb, p->zName + ); + } + assert( p->bHasStat==p->bFts4 ); + if( p->bHasStat ){ + sqlite3Fts3CreateStatTable(&rc, p); + } + return rc; +} + +/* +** Store the current database page-size in bytes in p->nPgsz. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. +*/ +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ + + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + p->nPgsz = sqlite3_column_int(pStmt, 0); + rc = sqlite3_finalize(pStmt); + }else if( rc==SQLITE_AUTH ){ + p->nPgsz = 1024; + rc = SQLITE_OK; + } + } + assert( p->nPgsz>0 || rc!=SQLITE_OK ); + sqlite3_free(zSql); + *pRc = rc; + } +} + +/* +** "Special" FTS4 arguments are column specifications of the following form: +** +** = +** +** There may not be whitespace surrounding the "=" character. The +** term may be quoted, but the may not. +*/ +static int fts3IsSpecialColumn( + const char *z, + int *pnKey, + char **pzValue +){ + char *zValue; + const char *zCsr = z; + + while( *zCsr!='=' ){ + if( *zCsr=='\0' ) return 0; + zCsr++; + } + + *pnKey = (int)(zCsr-z); + zValue = sqlite3_mprintf("%s", &zCsr[1]); + if( zValue ){ + sqlite3Fts3Dequote(zValue); + } + *pzValue = zValue; + return 1; +} + +/* +** Append the output of a printf() style formatting to an existing string. +*/ +static void fts3Appendf( + int *pRc, /* IN/OUT: Error code */ + char **pz, /* IN/OUT: Pointer to string buffer */ + const char *zFormat, /* Printf format string to append */ + ... /* Arguments for printf format string */ +){ + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( z && *pz ){ + char *z2 = sqlite3_mprintf("%s%s", *pz, z); + sqlite3_free(z); + z = z2; + } + if( z==0 ) *pRc = SQLITE_NOMEM; + sqlite3_free(*pz); + *pz = z; + } +} + +/* +** Return a copy of input string zInput enclosed in double-quotes (") and +** with all double quote characters escaped. For example: +** +** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" +** +** The pointer returned points to memory obtained from sqlite3_malloc(). It +** is the callers responsibility to call sqlite3_free() to release this +** memory. +*/ +static char *fts3QuoteId(char const *zInput){ + sqlite3_int64 nRet; + char *zRet; + nRet = 2 + (int)strlen(zInput)*2 + 1; + zRet = sqlite3_malloc64(nRet); + if( zRet ){ + int i; + char *z = zRet; + *(z++) = '"'; + for(i=0; zInput[i]; i++){ + if( zInput[i]=='"' ) *(z++) = '"'; + *(z++) = zInput[i]; + } + *(z++) = '"'; + *(z++) = '\0'; + } + return zRet; +} + +/* +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: +** +** SELECT FROM %_content AS x ... +** +** to return the docid, followed by each column of text data in order +** from left to write. If parameter zFunc is not NULL, then instead of +** being returned directly each column of text data is passed to an SQL +** function named zFunc first. For example, if zFunc is "unzip" and the +** table has the three user-defined columns "a", "b", and "c", the following +** string is returned: +** +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } + sqlite3_free(zFree); + }else{ + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + } + } + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); + return zRet; +} + +/* +** Return a list of N comma separated question marks, where N is the number +** of columns in the %_content table (one for the docid plus one for each +** user-defined text column). +** +** If argument zFunc is not NULL, then all but the first question mark +** is preceded by zFunc and an open bracket, and followed by a closed +** bracket. For example, if zFunc is "zip" and the FTS3 table has three +** user-defined text columns, the following string is returned: +** +** "?, zip(?), zip(?), zip(?)" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "?"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", ?"); + } + sqlite3_free(zFree); + return zRet; +} + +/* +** Buffer z contains a positive integer value encoded as utf-8 text. +** Decode this value and store it in *pnOut, returning the number of bytes +** consumed. If an overflow error occurs return a negative value. +*/ +SQLITE_PRIVATE int sqlite3Fts3ReadInt(const char *z, int *pnOut){ + u64 iVal = 0; + int i; + for(i=0; z[i]>='0' && z[i]<='9'; i++){ + iVal = iVal*10 + (z[i] - '0'); + if( iVal>0x7FFFFFFF ) return -1; + } + *pnOut = (int)iVal; + return i; +} + +/* +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. +** +** Only decimal digits ('0'..'9') may be part of an integer value. +** +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. +** +** This function is used when parsing the "prefix=" FTS4 parameter. +*/ +static int fts3GobbleInt(const char **pp, int *pnOut){ + const int MAX_NPREFIX = 10000000; + int nInt = 0; /* Output value */ + int nByte; + nByte = sqlite3Fts3ReadInt(*pp, &nInt); + if( nInt>MAX_NPREFIX ){ + nInt = 0; + } + if( nByte==0 ){ + return SQLITE_ERROR; + } + *pnOut = nInt; + *pp += nByte; + return SQLITE_OK; +} + +/* +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. +** +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. +** +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. +** +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. +*/ +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ +){ + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc64(sizeof(struct Fts3Index) * nIndex); + *apIndex = aIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i=0 ); + if( nPrefix==0 ){ + nIndex--; + i--; + }else{ + aIndex[i].nPrefix = nPrefix; + } + p++; + } + } + + *pnIndex = nIndex; + return SQLITE_OK; +} + +/* +** This function is called when initializing an FTS4 table that uses the +** content=xxx option. It determines the number of and names of the columns +** of the new FTS4 table. +** +** The third argument passed to this function is the value passed to the +** config=xxx option (i.e. "xxx"). This function queries the database for +** a table of that name. If found, the output variables are populated +** as follows: +** +** *pnCol: Set to the number of columns table xxx has, +** +** *pnStr: Set to the total amount of space required to store a copy +** of each columns name, including the nul-terminator. +** +** *pazCol: Set to point to an array of *pnCol strings. Each string is +** the name of the corresponding column in table xxx. The array +** and its contents are allocated using a single allocation. It +** is the responsibility of the caller to free this allocation +** by eventually passing the *pazCol value to sqlite3_free(). +** +** If the table cannot be found, an error code is returned and the output +** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is +** returned (and the output variables are undefined). +*/ +static int fts3ContentColumns( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ + const char *zTbl, /* Name of content table */ + const char ***pazCol, /* OUT: Malloc'd array of column names */ + int *pnCol, /* OUT: Size of array *pazCol */ + int *pnStr, /* OUT: Bytes of string content */ + char **pzErr /* OUT: error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* "SELECT *" statement on zTbl */ + sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); + } + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + const char **azCol; /* Output array */ + sqlite3_int64 nStr = 0; /* Size of all column names (incl. 0x00) */ + int nCol; /* Number of table columns */ + int i; /* Used to iterate through columns */ + + /* Loop through the returned columns. Set nStr to the number of bytes of + ** space required to store a copy of each column name, including the + ** nul-terminator byte. */ + nCol = sqlite3_column_count(pStmt); + for(i=0; i module name ("fts3" or "fts4") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. +*/ +static int fts3InitVtab( + int isCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Table *p = 0; /* Pointer to allocated vtab */ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3_int64 nByte; /* Size of allocation used for *p */ + int iCol; /* Column index */ + int nString = 0; /* Bytes required to hold all column names */ + int nCol = 0; /* Number of columns in the FTS table */ + char *zCsr; /* Space for holding column names */ + int nDb; /* Bytes required to hold database name */ + int nName; /* Bytes required to hold table name */ + int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ + const char **aCol; /* Array of column names */ + sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ + + int nIndex = 0; /* Size of aIndex[] array */ + struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ + + /* The results of parsing supported FTS4 key=value options: */ + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ + char *zContent = 0; /* content=? parameter (or NULL) */ + char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ + char **azNotindexed = 0; /* The set of notindexed= columns */ + int nNotindexed = 0; /* Size of azNotindexed[] array */ + + assert( strlen(argv[0])==4 ); + assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) + || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) + ); + + nDb = (int)strlen(argv[1]) + 1; + nName = (int)strlen(argv[2]) + 1; + + nByte = sizeof(const char *) * (argc-2); + aCol = (const char **)sqlite3_malloc64(nByte); + if( aCol ){ + memset((void*)aCol, 0, nByte); + azNotindexed = (char **)sqlite3_malloc64(nByte); + } + if( azNotindexed ){ + memset(azNotindexed, 0, nByte); + } + if( !aCol || !azNotindexed ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } + + /* Loop through all of the arguments passed by the user to the FTS3/4 + ** module (i.e. all the column names and special arguments). This loop + ** does the following: + ** + ** + Figures out the number of columns the FTSX table will have, and + ** the number of bytes of space that must be allocated to store copies + ** of the column names. + ** + ** + If there is a tokenizer specification included in the arguments, + ** initializes the tokenizer pTokenizer. + */ + for(i=3; rc==SQLITE_OK && i8 + && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3Fts3IsIdChar(z[8]) + ){ + rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); + } + + /* Check if it is an FTS4 special argument. */ + else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + struct Fts4Option { + const char *zOpt; + int nOpt; + } aFts4Opt[] = { + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 }, /* 5 -> CONTENT */ + { "languageid", 10 }, /* 6 -> LANGUAGEID */ + { "notindexed", 10 } /* 7 -> NOTINDEXED */ + }; + + int iOpt; + if( !zVal ){ + rc = SQLITE_NOMEM; + }else{ + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) + ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + + case 5: /* CONTENT */ + sqlite3_free(zContent); + zContent = zVal; + zVal = 0; + break; + + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; + + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; + + default: + assert( iOpt==SizeofArray(aFts4Opt) ); + sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + break; + } + sqlite3_free(zVal); + } + } + + /* Otherwise, the argument is a column name. */ + else { + nString += (int)(strlen(z) + 1); + aCol[nCol++] = z; + } + } + + /* If a content=xxx option was specified, the following: + ** + ** 1. Ignore any compress= and uncompress= options. + ** + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. + */ + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + if( nCol==0 ){ + sqlite3_free((void*)aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); + + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; jdb = db; + p->nColumn = nCol; + p->nPendingData = 0; + p->azColumn = (char **)&p[1]; + p->pTokenizer = pTokenizer; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; + p->bHasDocsize = (isFts4 && bNoDocsize==0); + p->bHasStat = (u8)isFts4; + p->bFts4 = (u8)isFts4; + p->bDescIdx = (u8)bDescIdx; + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->zContentTbl = zContent; + p->zLanguageid = zLanguageid; + zContent = 0; + zLanguageid = 0; + TESTONLY( p->inTransaction = -1 ); + TESTONLY( p->mxSavepoint = -1 ); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } + p->abNotindexed = (u8 *)&p->aIndex[nIndex]; + + /* Fill in the zName and zDb fields of the vtab structure. */ + zCsr = (char *)&p->abNotindexed[nCol]; + p->zName = zCsr; + memcpy(zCsr, argv[2], nName); + zCsr += nName; + p->zDb = zCsr; + memcpy(zCsr, argv[1], nDb); + zCsr += nDb; + + /* Fill in the azColumn array */ + for(iCol=0; iCol0 ){ + memcpy(zCsr, z, n); + } + zCsr[n] = '\0'; + sqlite3Fts3Dequote(zCsr); + p->azColumn[iCol] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); + } + + /* Fill in the abNotindexed array */ + for(iCol=0; iColazColumn[iCol]); + for(i=0; iazColumn[iCol], zNot, n) + ){ + p->abNotindexed[iCol] = 1; + sqlite3_free(zNot); + azNotindexed[i] = 0; + } + } + } + for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); + p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); + if( rc!=SQLITE_OK ) goto fts3_init_out; + + /* If this is an xCreate call, create the underlying tables in the + ** database. TODO: For xConnect(), it could verify that said tables exist. + */ + if( isCreate ){ + rc = fts3CreateTables(p); + } + + /* Check to see if a legacy fts3 table has been "upgraded" by the + ** addition of a %_stat table so that it can use incremental merge. + */ + if( !isFts4 && !isCreate ){ + p->bHasStat = 2; + } + + /* Figure out the page-size for the database. This is required in order to + ** estimate the cost of loading large doclists from the database. */ + fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; + +#if defined(SQLITE_DEBUG)||defined(SQLITE_TEST) + p->nMergeCount = FTS3_MERGE_COUNT; +#endif + + /* Declare the table schema to SQLite. */ + fts3DeclareVtab(&rc, p); + +fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aIndex); + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + sqlite3_free(zContent); + sqlite3_free(zLanguageid); + for(i=0; ipModule->xDestroy(pTokenizer); + } + }else{ + assert( p->pSegments==0 ); + *ppVTab = &p->base; + } + return rc; +} + +/* +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts3InitVtab(). +*/ +static int fts3ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts3CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 + if( sqlite3_libversion_number()>=3008012 ){ + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} + +/* +** Implementation of the xBestIndex method for FTS3 tables. There +** are three possible strategies, in order of preference: +** +** 1. Direct lookup by rowid or docid. +** 2. Full-text search using a MATCH operator on a non-docid column. +** 3. Linear scan of %_content table. +*/ +static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts3Table *p = (Fts3Table *)pVTab; + int i; /* Iterator variable */ + int iCons = -1; /* Index of constraint to use */ + + int iLangidCons = -1; /* Index of langid=x constraint, if present */ + int iDocidGe = -1; /* Index of docid>=x constraint, if present */ + int iDocidLe = -1; /* Index of docid<=x constraint, if present */ + int iIdx; + + if( p->bLock ){ + return SQLITE_ERROR; + } + + /* By default use a full table scan. This is an expensive option, + ** so search through the constraints to see if a more efficient + ** strategy is possible. + */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 5000000; + for(i=0; inConstraint; i++){ + int bDocid; /* True if this constraint is on docid */ + struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; + } + continue; + } + + bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); + + /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ + if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ + pInfo->idxNum = FTS3_DOCID_SEARCH; + pInfo->estimatedCost = 1.0; + iCons = i; + } + + /* A MATCH constraint. Use a full-text search. + ** + ** If there is more than one MATCH constraint available, use the first + ** one encountered. If there is both a MATCH constraint and a direct + ** rowid/docid lookup, prefer the MATCH strategy. This is done even + ** though the rowid/docid lookup is faster than a MATCH query, selecting + ** it would lead to an "unable to use function MATCH in the requested + ** context" error. + */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn + ){ + pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; + pInfo->estimatedCost = 2.0; + iCons = i; + } + + /* Equality constraint on the langid column */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && pCons->iColumn==p->nColumn + 2 + ){ + iLangidCons = i; + } + + if( bDocid ){ + switch( pCons->op ){ + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: + iDocidGe = i; + break; + + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: + iDocidLe = i; + break; + } + } + } + + /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ + if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); + + iIdx = 1; + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; + pInfo->aConstraintUsage[iCons].omit = 1; + } + if( iLangidCons>=0 ){ + pInfo->idxNum |= FTS3_HAVE_LANGID; + pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; + } + if( iDocidGe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_GE; + pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; + } + if( iDocidLe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_LE; + pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; + } + + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. + */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + pInfo->orderByConsumed = 1; + } + } + + assert( p->pSegments==0 ); + return SQLITE_OK; +} + +/* +** Implementation of xOpen method. +*/ +static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ + + UNUSED_PARAMETER(pVTab); + + /* Allocate a buffer large enough for an Fts3Cursor structure. If the + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** if the allocation fails, return SQLITE_NOMEM. + */ + *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3Cursor)); + return SQLITE_OK; +} + +/* +** Finalize the statement handle at pCsr->pStmt. +** +** Or, if that statement handle is one created by fts3CursorSeekStmt(), +** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement +** pointer there instead of finalizing it. +*/ +static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){ + if( pCsr->bSeekStmt ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + if( p->pSeekStmt==0 ){ + p->pSeekStmt = pCsr->pStmt; + sqlite3_reset(pCsr->pStmt); + pCsr->pStmt = 0; + } + pCsr->bSeekStmt = 0; + } + sqlite3_finalize(pCsr->pStmt); +} + +/* +** Free all resources currently held by the cursor passed as the only +** argument. +*/ +static void fts3ClearCursor(Fts3Cursor *pCsr){ + fts3CursorFinalizeStmt(pCsr); + sqlite3Fts3FreeDeferredTokens(pCsr); + sqlite3_free(pCsr->aDoclist); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + sqlite3Fts3ExprFree(pCsr->pExpr); + memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); +} + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + fts3ClearCursor(pCsr); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: +** +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. +*/ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->pStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + if( p->pSeekStmt ){ + pCsr->pStmt = p->pSeekStmt; + p->pSeekStmt = 0; + }else{ + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + p->bLock++; + rc = sqlite3_prepare_v3( + p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); + p->bLock--; + sqlite3_free(zSql); + } + if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; + } + return rc; +} + +/* +** Position the pCsr->pStmt statement so that it is on the row +** of the %_content table that contains the last match. Return +** SQLITE_OK on success. +*/ +static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->isRequireSeek ){ + rc = fts3CursorSeekStmt(pCsr); + if( rc==SQLITE_OK ){ + Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab; + pTab->bLock++; + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + pTab->bLock--; + return SQLITE_OK; + }else{ + pTab->bLock--; + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ + /* If no row was found and no error has occurred, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. */ + rc = FTS_CORRUPT_VTAB; + pCsr->isEof = 1; + } + } + } + } + + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; +} + +/* +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. +** +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. +** +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. +** +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +*/ +static int fts3ScanInteriorNode( + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + i64 nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + u64 iChild; /* Block id of child node to descend to */ + int nBuffer = 0; /* Total term size */ + + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. + ** + ** Even if the data structure on disk is corrupted, this (reading two + ** varints from the buffer) does not risk an overread. If zNode is a + ** root node, then the buffer comes from a SELECT statement. SQLite does + ** not make this guarantee explicitly, but in practice there are always + ** either more than 20 bytes of allocated space following the nNode bytes of + ** contents, or two zero bytes. Or, if the node is read from the %_segments + ** table, then there are always 20 bytes of zeroed padding following the + ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + */ + zCsr += sqlite3Fts3GetVarintU(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarintU(zCsr, &iChild); + if( zCsr>zEnd ){ + return FTS_CORRUPT_VTAB; + } + + while( zCsrnBuffer ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + } + isFirstTerm = 0; + zCsr += fts3GetVarint32(zCsr, &nSuffix); + + assert( nPrefix>=0 && nSuffix>=0 ); + if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr || nSuffix==0 ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + if( (i64)nPrefix+nSuffix>nAlloc ){ + char *zNew; + nAlloc = ((i64)nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc); + if( !zNew ){ + rc = SQLITE_NOMEM; + goto finish_scan; + } + zBuffer = zNew; + } + assert( zBuffer ); + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; + + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = (i64)iChild; + piFirst = 0; + } + + if( piLast && cmp<0 ){ + *piLast = (i64)iChild; + piLast = 0; + } + + iChild++; + }; + + if( piFirst ) *piFirst = (i64)iChild; + if( piLast ) *piLast = (i64)iChild; + + finish_scan: + sqlite3_free(zBuffer); + return rc; +} + + +/* +** The buffer pointed to by argument zNode (size nNode bytes) contains an +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) +** contains a term. This function searches the sub-tree headed by the zNode +** node for the range of leaf nodes that may contain the specified term +** or terms for which the specified term is a prefix. +** +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** left-most leaf node in the tree that may contain the specified term. +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the +** right-most leaf node that may contain a term for which the specified +** term is a prefix. +** +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the +** segment does contain any such terms, they are stored within the identified +** range. Because this function only inspects interior segment nodes (and +** never loads leaf nodes into memory), it is not possible to be sure. +** +** If an error occurs, an error code other than SQLITE_OK is returned. +*/ +static int fts3SelectLeaf( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ +){ + int rc = SQLITE_OK; /* Return code */ + int iHeight; /* Height of this node in tree */ + + assert( piLeaf || piLeaf2 ); + + fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + assert_fts3_nc( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); + + if( rc==SQLITE_OK && iHeight>1 ){ + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob = 0; /* Size of zBlob in bytes */ + + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); + } + sqlite3_free(zBlob); + piLeaf = 0; + zBlob = 0; + } + + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); + } + if( rc==SQLITE_OK ){ + int iNewHeight = 0; + fts3GetVarint32(zBlob, &iNewHeight); + if( iNewHeight>=iHeight ){ + rc = FTS_CORRUPT_VTAB; + }else{ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } + } + sqlite3_free(zBlob); + } + + return rc; +} + +/* +** This function is used to create delta-encoded serialized lists of FTS3 +** varints. Each call to this function appends a single varint to a list. +*/ +static void fts3PutDeltaVarint( + char **pp, /* IN/OUT: Output pointer */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + assert_fts3_nc( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); + *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); + *piPrev = iVal; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a position-list. After it returns, *ppPoslist points to the +** first byte after the position-list. +** +** A position list is list of positions (delta encoded) and columns for +** a single document record of a doclist. So, in other words, this +** routine advances *ppPoslist so that it points to the next docid in +** the doclist, or to the first byte past the end of the doclist. +** +** If pp is not NULL, then the contents of the position list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. +*/ +static void fts3PoslistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* The end of a position list is marked by a zero encoded as an FTS3 + ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by + ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail + ** of some other, multi-byte, value. + ** + ** The following while-loop moves pEnd to point to the first byte that is not + ** immediately preceded by a byte with the 0x80 bit set. Then increments + ** pEnd once more so that it points to the byte immediately following the + ** last byte in the position-list. + */ + while( *pEnd | c ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && (*pEnd)==0 ); + } + pEnd++; /* Advance past the POS_END terminator byte */ + + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a column-list. After it returns, *ppPoslist points to the +** to the terminator (POS_COLUMN or POS_END) byte of the column-list. +** +** A column-list is list of delta-encoded positions for a single column +** within a single document within a doclist. +** +** The column-list is terminated either by a POS_COLUMN varint (1) or +** a POS_END varint (0). This routine leaves *ppPoslist pointing to +** the POS_COLUMN or POS_END that terminates the column-list. +** +** If pp is not NULL, then the contents of the column-list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. The POS_COLUMN or POS_END terminator +** is not copied into *pp. +*/ +static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; + + /* A column-list is terminated by either a 0x01 or 0x00 byte that is + ** not part of a multi-byte varint. + */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && ((*pEnd)&0xfe)==0 ); + } + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} + +/* +** Value used to signify the end of an position-list. This must be +** as large or larger than any value that might appear on the +** position-list, even a position list that has been corrupted. +*/ +#define POSITION_LIST_END LARGEST_INT64 + +/* +** This function is used to help parse position-lists. When this function is +** called, *pp may point to the start of the next varint in the position-list +** being parsed, or it may point to 1 byte past the end of the position-list +** (in which case **pp will be a terminator bytes POS_END (0) or +** (1)). +** +** If *pp points past the end of the current position-list, set *pi to +** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, +** increment the current value of *pi by the value read, and set *pp to +** point to the next value before returning. +** +** Before calling this routine *pi must be initialized to the value of +** the previous position, or zero if we are reading the first position +** in the position-list. Because positions are delta-encoded, the value +** of the previous position is needed in order to compute the value of +** the next position. +*/ +static void fts3ReadNextPos( + char **pp, /* IN/OUT: Pointer into position-list buffer */ + sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ +){ + if( (**pp)&0xFE ){ + int iVal; + *pp += fts3GetVarint32((*pp), &iVal); + *pi += iVal; + *pi -= 2; + }else{ + *pi = POSITION_LIST_END; + } +} + +/* +** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by +** the value of iCol encoded as a varint to *pp. This will start a new +** column list. +** +** Set *pp to point to the byte just after the last byte written before +** returning (do not modify it if iCol==0). Return the total number of bytes +** written (0 if iCol==0). +*/ +static int fts3PutColNumber(char **pp, int iCol){ + int n = 0; /* Number of bytes written */ + if( iCol ){ + char *p = *pp; /* Output pointer */ + n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); + *p = 0x01; + *pp = &p[n]; + } + return n; +} + +/* +** Compute the union of two position lists. The output written +** into *pp contains all positions of both *pp1 and *pp2 in sorted +** order and with any duplicates removed. All pointers are +** updated appropriately. The caller is responsible for insuring +** that there is enough space in *pp to hold the complete output. +*/ +static int fts3PoslistMerge( + char **pp, /* Output buffer */ + char **pp1, /* Left input list */ + char **pp2 /* Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + + while( *p1 || *p2 ){ + int iCol1; /* The current column index in pp1 */ + int iCol2; /* The current column index in pp2 */ + + if( *p1==POS_COLUMN ){ + fts3GetVarint32(&p1[1], &iCol1); + if( iCol1==0 ) return FTS_CORRUPT_VTAB; + } + else if( *p1==POS_END ) iCol1 = 0x7fffffff; + else iCol1 = 0; + + if( *p2==POS_COLUMN ){ + fts3GetVarint32(&p2[1], &iCol2); + if( iCol2==0 ) return FTS_CORRUPT_VTAB; + } + else if( *p2==POS_END ) iCol2 = 0x7fffffff; + else iCol2 = 0; + + if( iCol1==iCol2 ){ + sqlite3_int64 i1 = 0; /* Last position from pp1 */ + sqlite3_int64 i2 = 0; /* Last position from pp2 */ + sqlite3_int64 iPrev = 0; + int n = fts3PutColNumber(&p, iCol1); + p1 += n; + p2 += n; + + /* At this point, both p1 and p2 point to the start of column-lists + ** for the same column (the column with index iCol1 and iCol2). + ** A column-list is a list of non-negative delta-encoded varints, each + ** incremented by 2 before being stored. Each list is terminated by a + ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists + ** and writes the results to buffer p. p is left pointing to the byte + ** after the list written. No terminator (POS_END or POS_COLUMN) is + ** written to the output. + */ + fts3GetDeltaVarint(&p1, &i1); + fts3GetDeltaVarint(&p2, &i2); + if( i1<2 || i2<2 ){ + break; + } + do { + fts3PutDeltaVarint(&p, &iPrev, (i1pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. +** when the *pp1 token appears before the *pp2 token, but not more than nToken +** slots before it. +** +** e.g. nToken==1 searches for adjacent positions. +*/ +static int fts3PoslistPhraseMerge( + char **pp, /* IN/OUT: Preallocated output buffer */ + int nToken, /* Maximum difference in token positions */ + int isSaveLeft, /* Save the left position */ + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + int iCol1 = 0; + int iCol2 = 0; + + /* Never set both isSaveLeft and isExact for the same invocation. */ + assert( isSaveLeft==0 || isExact==0 ); + + assert_fts3_nc( p!=0 && *p1!=0 && *p2!=0 ); + if( *p1==POS_COLUMN ){ + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + } + if( *p2==POS_COLUMN ){ + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } + + while( 1 ){ + if( iCol1==iCol2 ){ + char *pSave = p; + sqlite3_int64 iPrev = 0; + sqlite3_int64 iPos1 = 0; + sqlite3_int64 iPos2 = 0; + + if( iCol1 ){ + *p++ = POS_COLUMN; + p += sqlite3Fts3PutVarint(p, iCol1); + } + + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + if( iPos1<0 || iPos2<0 ) break; + + while( 1 ){ + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + ){ + sqlite3_int64 iSave; + iSave = isSaveLeft ? iPos1 : iPos2; + fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; + pSave = 0; + assert( p ); + } + if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ + if( (*p2&0xFE)==0 ) break; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else{ + if( (*p1&0xFE)==0 ) break; + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + } + } + + if( pSave ){ + assert( pp && p ); + p = pSave; + } + + fts3ColumnlistCopy(0, &p1); + fts3ColumnlistCopy(0, &p2); + assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); + if( 0==*p1 || 0==*p2 ) break; + + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } + + /* Advance pointer p1 or p2 (whichever corresponds to the smaller of + ** iCol1 and iCol2) so that it points to either the 0x00 that marks the + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. + */ + else if( iCol1=pEnd ){ + *pp = 0; + }else{ + u64 iVal; + *pp += sqlite3Fts3GetVarintU(*pp, &iVal); + if( bDescIdx ){ + *pVal = (i64)((u64)*pVal - iVal); + }else{ + *pVal = (i64)((u64)*pVal + iVal); + } + } +} + +/* +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. +** +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. +** +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). +** +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. +*/ +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_uint64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + assert_fts3_nc( *pbFirst==0 || iVal>=*piPrev ); + iWrite = (u64)iVal - (u64)*piPrev; + }else{ + assert_fts3_nc( *piPrev>=iVal ); + iWrite = (u64)*piPrev - (u64)iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert_fts3_nc( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + + +/* +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). +** +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. +*/ +/* #define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i64)((u64)i1-i2)) */ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1>i2?1:((i1==i2)?0:-1))) + +/* +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. +** +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. +** +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. +*/ +static int fts3DoclistOrMerge( + int bDescDoclist, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + int rc = SQLITE_OK; + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + + /* Allocate space for the output. Both the input and output doclists + ** are delta encoded. If they are in ascending order (bDescDoclist==0), + ** then the first docid in each list is simply encoded as a varint. For + ** each subsequent docid, the varint stored is the difference between the + ** current and previous docid (a positive number - since the list is in + ** ascending order). + ** + ** The first docid written to the output is therefore encoded using the + ** same number of bytes as it is in whichever of the input lists it is + ** read from. And each subsequent docid read from the same input list + ** consumes either the same or less bytes as it did in the input (since + ** the difference between it and the previous value in the output must + ** be a positive value less than or equal to the delta value read from + ** the input list). The same argument applies to all but the first docid + ** read from the 'other' list. And to the contents of all position lists + ** that will be copied and merged from the input to the output. + ** + ** However, if the first docid copied to the output is a negative number, + ** then the encoding of the first docid from the 'other' input list may + ** be larger in the output than it was in the input (since the delta value + ** may be a larger positive integer than the actual docid). + ** + ** The space required to store the output is therefore the sum of the + ** sizes of the two inputs, plus enough space for exactly one of the input + ** docids to grow. + ** + ** A symetric argument may be made if the doclists are in descending + ** order. + */ + aOut = sqlite3_malloc64((i64)n1+n2+FTS3_VARINT_MAX-1+FTS3_BUFFER_PADDING); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + rc = fts3PoslistMerge(&p, &p1, &p2); + if( rc ) break; + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + + assert( (p-aOut)<=((p1?(p1-a1):n1)+(p2?(p2-a2):n2)+FTS3_VARINT_MAX-1) ); + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + p = aOut = 0; + }else{ + assert( (p-aOut)<=n1+n2+FTS3_VARINT_MAX-1 ); + memset(&aOut[(p-aOut)], 0, FTS3_BUFFER_PADDING); + } + *paOut = aOut; + *pnOut = (int)(p-aOut); + return rc; +} + +/* +** This function does a "phrase" merge of two doclists. In a phrase merge, +** the output contains a copy of each position from the right-hand input +** doclist for which there is a position in the left-hand input doclist +** exactly nDist tokens before it. +** +** If the docids in the input doclists are sorted in ascending order, +** parameter bDescDoclist should be false. If they are sorted in ascending +** order, it should be passed a non-zero value. +** +** The right-hand input doclist is overwritten by this function. +*/ +static int fts3DoclistPhraseMerge( + int bDescDoclist, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + char *aLeft, int nLeft, /* Left doclist */ + char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *aRight = *paRight; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut; + + assert( nDist>0 ); + if( bDescDoclist ){ + aOut = sqlite3_malloc64((sqlite3_int64)*pnRight + FTS3_VARINT_MAX); + if( aOut==0 ) return SQLITE_NOMEM; + }else{ + aOut = aRight; + } + p = aOut; + + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } + } + + *pnRight = (int)(p - aOut); + if( bDescDoclist ){ + sqlite3_free(aRight); + *paRight = aOut; + } + + return SQLITE_OK; +} + +/* +** Argument pList points to a position list nList bytes in size. This +** function checks to see if the position list contains any entries for +** a token in position 0 (of any column). If so, it writes argument iDelta +** to the output buffer pOut, followed by a position list consisting only +** of the entries from pList at position 0, and terminated by an 0x00 byte. +** The value returned is the number of bytes written to pOut (if any). +*/ +SQLITE_PRIVATE int sqlite3Fts3FirstFilter( + sqlite3_int64 iDelta, /* Varint that may be written to pOut */ + char *pList, /* Position list (no 0x00 term) */ + int nList, /* Size of pList in bytes */ + char *pOut /* Write output here */ +){ + int nOut = 0; + int bWritten = 0; /* True once iDelta has been written */ + char *p = pList; + char *pEnd = &pList[nList]; + + if( *p!=0x01 ){ + if( *p==0x02 ){ + nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); + pOut[nOut++] = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } + + while( paaOutput); i++){ + if( pTS->aaOutput[i] ){ + if( !aOut ){ + aOut = pTS->aaOutput[i]; + nOut = pTS->anOutput[i]; + pTS->aaOutput[i] = 0; + }else{ + int nNew; + char *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + + sqlite3_free(pTS->aaOutput[i]); + sqlite3_free(aOut); + pTS->aaOutput[i] = 0; + aOut = aNew; + nOut = nNew; + } + } + } + + pTS->aaOutput[0] = aOut; + pTS->anOutput[0] = nOut; + return SQLITE_OK; +} + +/* +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed +** as the first argument. The merge is an "OR" merge (see function +** fts3DoclistOrMerge() for details). +** +** This function is called with the doclist for each term that matches +** a queried prefix. It merges all these doclists into one, the doclist +** for the specified prefix. Since there can be a very large number of +** doclists to merge, the merging is done pair-wise using the TermSelect +** object. +** +** This function returns SQLITE_OK if the merge is successful, or an +** SQLite error code (SQLITE_NOMEM) if an error occurs. +*/ +static int fts3TermSelectMerge( + Fts3Table *p, /* FTS table handle */ + TermSelect *pTS, /* TermSelect object to merge into */ + char *aDoclist, /* Pointer to doclist */ + int nDoclist /* Size of aDoclist in bytes */ +){ + if( pTS->aaOutput[0]==0 ){ + /* If this is the first term selected, copy the doclist to the output + ** buffer using memcpy(). + ** + ** Add FTS3_VARINT_MAX bytes of unused space to the end of the + ** allocation. This is so as to ensure that the buffer is big enough + ** to hold the current doclist AND'd with any other doclist. If the + ** doclists are stored in order=ASC order, this padding would not be + ** required (since the size of [doclistA AND doclistB] is always less + ** than or equal to the size of [doclistA] in that case). But this is + ** not true for order=DESC. For example, a doclist containing (1, -1) + ** may be smaller than (-1), as in the first example the -1 may be stored + ** as a single-byte delta, whereas in the second it must be stored as a + ** FTS3_VARINT_MAX byte varint. + ** + ** Similar padding is added in the fts3DoclistOrMerge() function. + */ + pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); + pTS->anOutput[0] = nDoclist; + if( pTS->aaOutput[0] ){ + memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + memset(&pTS->aaOutput[0][nDoclist], 0, FTS3_VARINT_MAX); + }else{ + return SQLITE_NOMEM; + } + }else{ + char *aMerge = aDoclist; + int nMerge = nDoclist; + int iOut; + + for(iOut=0; iOutaaOutput); iOut++){ + if( pTS->aaOutput[iOut]==0 ){ + assert( iOut>0 ); + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + break; + }else{ + char *aNew; + int nNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; + } + + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } + } + } + } + return SQLITE_OK; +} + +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ +static int fts3SegReaderCursorAppend( + Fts3MultiSegReader *pCsr, + Fts3SegReader *pNew +){ + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + sqlite3_int64 nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc64(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + pCsr->apSegment = apNew; + } + pCsr->apSegment[pCsr->nSegment++] = pNew; + return SQLITE_OK; +} + +/* +** Add seg-reader objects to the Fts3MultiSegReader object passed as the +** 8th argument. +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +static int fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc = SQLITE_OK; /* Error code */ + sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ + int rc2; /* Result of sqlite3_reset() */ + + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex && p->iPrevLangid==iLangid ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); + } + + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; + + /* Read the values returned by the SELECT into local variables. */ + sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); + sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); + + /* If zTerm is not NULL, and this segment is not stored entirely on its + ** root node, the range of leaves scanned can be reduced. Do this. */ + if( iStartBlock && zTerm && zRoot ){ + sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); + if( rc!=SQLITE_OK ) goto finished; + if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; + } + + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + (isPrefix==0 && isScan==0), + iStartBlock, iLeavesEndBlock, + iEndBlock, zRoot, nRoot, &pSeg + ); + if( rc!=SQLITE_OK ) goto finished; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } + } + + finished: + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_DONE ) rc = rc2; + + return rc; +} + +/* +** Set up a cursor object for iterating through a full-text index or a +** single level therein. +*/ +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language-id to search */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelbase.pVtab; + + if( isPrefix ){ + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr + ); + pSegcsr->bLookup = 1; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero( + p, pCsr->iLangid, zTerm, nTerm, pSegcsr + ); + } + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); + pSegcsr->bLookup = !isPrefix; + } + } + + *ppSegcsr = pSegcsr; + return rc; +} + +/* +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +*/ +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ + sqlite3Fts3SegReaderFinish(pSegcsr); + sqlite3_free(pSegcsr); +} + +/* +** This function retrieves the doclist for the specified term (or term +** prefix) from the database. +*/ +static int fts3TermSelect( + Fts3Table *p, /* Virtual table handle */ + Fts3PhraseToken *pTok, /* Token to query for */ + int iColumn, /* Column to query (or -ve for all columns) */ + int *pnOut, /* OUT: Size of buffer at *ppOut */ + char **ppOut /* OUT: Malloced result buffer */ +){ + int rc; /* Return code */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Object for pair-wise doclist merging */ + Fts3SegFilter filter; /* Segment term filter configuration */ + + pSegcsr = pTok->pSegcsr; + memset(&tsc, 0, sizeof(TermSelect)); + + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS + | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) + | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); + filter.iCol = iColumn; + filter.zTerm = pTok->z; + filter.nTerm = pTok->n; + + rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); + while( SQLITE_OK==rc + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) + ){ + rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); + } + + if( rc==SQLITE_OK ){ + rc = fts3TermSelectFinishMerge(p, &tsc); + } + if( rc==SQLITE_OK ){ + *ppOut = tsc.aaOutput[0]; + *pnOut = tsc.anOutput[0]; + }else{ + int i; + for(i=0; ipSegcsr = 0; + return rc; +} + +/* +** This function counts the total number of docids in the doclist stored +** in buffer aList[], size nList bytes. +** +** If the isPoslist argument is true, then it is assumed that the doclist +** contains a position-list following each docid. Otherwise, it is assumed +** that the doclist is simply a list of docids stored as delta encoded +** varints. +*/ +static int fts3DoclistCountDocids(char *aList, int nList){ + int nDoc = 0; /* Return value */ + if( aList ){ + char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Cursor */ + while( peSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + Fts3Table *pTab = (Fts3Table*)pCursor->pVtab; + pTab->bLock++; + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + }else{ + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; + } + pTab->bLock--; + }else{ + rc = fts3EvalNext((Fts3Cursor *)pCursor); + } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** If the numeric type of argument pVal is "integer", then return it +** converted to a 64-bit signed integer. Otherwise, return a copy of +** the second parameter, iDefault. +*/ +static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + +/* +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. +** +** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against +** the %_content table. +** +** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry +** in the %_content table. +** +** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The +** column on the left-hand side of the MATCH operator is column +** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand +** side of the MATCH operator. +*/ +static int fts3FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_OK; + char *zSql; /* SQL statement used to access %_content */ + int eSearch; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + + sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ + sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ + sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ + sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ + int iIdx; + + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + if( p->bLock ){ + return SQLITE_ERROR; + } + + eSearch = (idxNum & 0x0000FFFF); + assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); + assert( p->pSegments==0 ); + + /* Collect arguments into local variables */ + iIdx = 0; + if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; + assert( iIdx==nVal ); + + /* In case the cursor has been used before, clear it now. */ + fts3ClearCursor(pCsr); + + /* Set the lower and upper bounds on docids to return */ + pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); + pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); + + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } + pCsr->eSearch = (i16)eSearch; + + if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ + int iCol = eSearch-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(pCons); + + if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + + pCsr->iLangid = 0; + if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); + + assert( p->base.zErrMsg==0 ); + rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + &p->base.zErrMsg + ); + if( rc!=SQLITE_OK ){ + return rc; + } + + rc = fts3EvalStart(pCsr); + sqlite3Fts3SegmentsClose(p); + if( rc!=SQLITE_OK ) return rc; + pCsr->pNextId = pCsr->aDoclist; + pCsr->iPrevId = 0; + } + + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. + */ + if( eSearch==FTS3_FULLSCAN_SEARCH ){ + if( pDocidGe || pDocidLe ){ + zSql = sqlite3_mprintf( + "SELECT %s WHERE rowid BETWEEN %lld AND %lld ORDER BY rowid %s", + p->zReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid, + (pCsr->bDesc ? "DESC" : "ASC") + ); + }else{ + zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); + } + if( zSql ){ + p->bLock++; + rc = sqlite3_prepare_v3( + p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); + p->bLock--; + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + }else if( eSearch==FTS3_DOCID_SEARCH ){ + rc = fts3CursorSeekStmt(pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); + } + } + if( rc!=SQLITE_OK ) return rc; + + return fts3NextMethod(pCursor); +} + +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor*)pCursor; + if( pCsr->isEof ){ + fts3ClearCursor(pCsr); + pCsr->isEof = 1; + } + return pCsr->isEof; +} + +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts3 +** exposes %_content.docid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + *pRowid = pCsr->iPrevId; + return SQLITE_OK; +} + +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. +** +** If: +** +** (iCol < p->nColumn) -> The value of the iCol'th user column. +** (iCol == p->nColumn) -> Magic column with the same name as the table. +** (iCol == p->nColumn+1) -> Docid column +** (iCol == p->nColumn+2) -> Langid column +*/ +static int fts3ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + + /* The column value supplied by SQLite must be in range. */ + assert( iCol>=0 && iCol<=p->nColumn+2 ); + + switch( iCol-p->nColumn ){ + case 0: + /* The special 'table-name' column */ + sqlite3_result_pointer(pCtx, pCsr, "fts3cursor", 0); + break; + + case 1: + /* The docid column */ + sqlite3_result_int64(pCtx, pCsr->iPrevId); + break; + + case 2: + if( pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); + break; + }else if( p->zLanguageid==0 ){ + sqlite3_result_int(pCtx, 0); + break; + }else{ + iCol = p->nColumn; + /* no break */ deliberate_fall_through + } + + default: + /* A user column. Or, if this is a full-table scan, possibly the + ** language-id column. Seek the cursor. */ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)-1>iCol ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } + break; + } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} + +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +*/ +static int fts3UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); +} + +/* +** Implementation of xSync() method. Flush the contents of the pending-terms +** hash-table to the database. +*/ +static int fts3SyncMethod(sqlite3_vtab *pVtab){ + + /* Following an incremental-merge operation, assuming that the input + ** segments are not completely consumed (the usual case), they are updated + ** in place to remove the entries that have already been merged. This + ** involves updating the leaf block that contains the smallest unmerged + ** entry and each block (if any) between the leaf and the root node. So + ** if the height of the input segment b-trees is N, and input segments + ** are merged eight at a time, updating the input segments at the end + ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually + ** small - often between 0 and 2. So the overhead of the incremental + ** merge is somewhere between 8 and 24 blocks. To avoid this overhead + ** dwarfing the actual productive work accomplished, the incremental merge + ** is only attempted if it will write at least 64 leaf blocks. Hence + ** nMinMerge. + ** + ** Of course, updating the input segments also involves deleting a bunch + ** of blocks from the segments table. But this is not considered overhead + ** as it would also be required by a crisis-merge that used the same input + ** segments. + */ + const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + + Fts3Table *p = (Fts3Table*)pVtab; + int rc; + i64 iLastRowid = sqlite3_last_insert_rowid(p->db); + + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * mxLevel; + A += (A/2); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); + } + sqlite3Fts3SegmentsClose(p); + sqlite3_set_last_insert_rowid(p->db, iLastRowid); + return rc; +} + +/* +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + char *zTbl = sqlite3_mprintf("%s_stat", p->zName); + if( zTbl ){ + int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0); + sqlite3_free(zTbl); + p->bHasStat = (res==SQLITE_OK); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* +** Implementation of xBegin() method. +*/ +static int fts3BeginMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + int rc; + UNUSED_PARAMETER(pVtab); + assert( p->pSegments==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=1 ); + p->nLeafAdd = 0; + rc = fts3SetHasStat(p); +#ifdef SQLITE_DEBUG + if( rc==SQLITE_OK ){ + p->inTransaction = 1; + p->mxSavepoint = -1; + } +#endif + return rc; +} + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts3SyncMethod(). +*/ +static int fts3CommitMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(pVtab); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. +*/ +static int fts3RollbackMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + sqlite3Fts3PendingTermsClear(p); + assert( p->inTransaction!=0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; +} + +/* +** When called, *ppPoslist must point to the byte immediately following the +** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function +** moves *ppPoslist so that it instead points to the first byte of the +** same position list. +*/ +static void fts3ReversePoslist(char *pStart, char **ppPoslist){ + char *p = &(*ppPoslist)[-2]; + char c = 0; + + /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ + while( p>pStart && (c=*p--)==0 ); + + /* Search backwards for a varint with value zero (the end of the previous + ** poslist). This is an 0x00 byte preceded by some byte that does not + ** have the 0x80 bit set. */ + while( p>pStart && (*p & 0x80) | c ){ + c = *p--; + } + assert( p==pStart || c==0 ); + + /* At this point p points to that preceding byte without the 0x80 bit + ** set. So to find the start of the poslist, skip forward 2 bytes then + ** over a varint. + ** + ** Normally. The other case is that p==pStart and the poslist to return + ** is the first in the doclist. In this case do not skip forward 2 bytes. + ** The second part of the if condition (c==0 && *ppPoslist>&p[2]) + ** is required for cases where the first byte of a doclist and the + ** doclist is empty. For example, if the first docid is 10, a doclist + ** that begins with: + ** + ** 0x0A 0x00 + */ + if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; } + while( *p++&0x80 ); + *ppPoslist = p; +} + +/* +** Helper function used by the implementation of the overloaded snippet(), +** offsets() and optimize() SQL functions. +** +** If the value passed as the third argument is a blob of size +** sizeof(Fts3Cursor*), then the blob contents are copied to the +** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error +** message is written to context pContext and SQLITE_ERROR returned. The +** string passed via zFunc is used as part of the error message. +*/ +static int fts3FunctionArg( + sqlite3_context *pContext, /* SQL function call context */ + const char *zFunc, /* Function name */ + sqlite3_value *pVal, /* argv[0] passed to function */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ +){ + int rc; + *ppCsr = (Fts3Cursor*)sqlite3_value_pointer(pVal, "fts3cursor"); + if( (*ppCsr)!=0 ){ + rc = SQLITE_OK; + }else{ + char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** Implementation of the snippet() function for FTS3 +*/ +static void fts3SnippetFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of apVal[] array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + const char *zStart = ""; + const char *zEnd = ""; + const char *zEllipsis = "..."; + int iCol = -1; + int nToken = 15; /* Default number of tokens in snippet */ + + /* There must be at least one argument passed to this function (otherwise + ** the non-overloaded version would have been called instead of this one). + */ + assert( nVal>=1 ); + + if( nVal>6 ){ + sqlite3_result_error(pContext, + "wrong number of arguments to function snippet()", -1); + return; + } + if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; + + switch( nVal ){ + case 6: nToken = sqlite3_value_int(apVal[5]); + /* no break */ deliberate_fall_through + case 5: iCol = sqlite3_value_int(apVal[4]); + /* no break */ deliberate_fall_through + case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); + /* no break */ deliberate_fall_through + case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); + /* no break */ deliberate_fall_through + case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); + } + if( !zEllipsis || !zEnd || !zStart ){ + sqlite3_result_error_nomem(pContext); + }else if( nToken==0 ){ + sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); + }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); + } +} + +/* +** Implementation of the offsets() function for FTS3 +*/ +static void fts3OffsetsFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; + assert( pCsr ); + if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Offsets(pContext, pCsr); + } +} + +/* +** Implementation of the special optimize() function for FTS3. This +** function merges all segments in the database to a single segment. +** Example usage is: +** +** SELECT optimize(t) FROM t LIMIT 1; +** +** where 't' is the name of an FTS3 table. +*/ +static void fts3OptimizeFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + int rc; /* Return code */ + Fts3Table *p; /* Virtual table handle */ + Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; + p = (Fts3Table *)pCursor->base.pVtab; + assert( p ); + + rc = sqlite3Fts3Optimize(p); + + switch( rc ){ + case SQLITE_OK: + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); + break; + case SQLITE_DONE: + sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); + break; + default: + sqlite3_result_error_code(pContext, rc); + break; + } +} + +/* +** Implementation of the matchinfo() function for FTS3 +*/ +static void fts3MatchinfoFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + assert( nVal==1 || nVal==2 ); + if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ + const char *zArg = 0; + if( nVal>1 ){ + zArg = (const char *)sqlite3_value_text(apVal[1]); + } + sqlite3Fts3Matchinfo(pContext, pCsr, zArg); + } +} + +/* +** This routine implements the xFindFunction method for the FTS3 +** virtual table. +*/ +static int fts3FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* Unused */ +){ + struct Overloaded { + const char *zName; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aOverload[] = { + { "snippet", fts3SnippetFunc }, + { "offsets", fts3OffsetsFunc }, + { "optimize", fts3OptimizeFunc }, + { "matchinfo", fts3MatchinfoFunc }, + }; + int i; /* Iterator variable */ + + UNUSED_PARAMETER(pVtab); + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(ppArg); + + for(i=0; idb; /* Database connection */ + int rc; /* Return Code */ + + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + + /* As it happens, the pending terms table is always empty here. This is + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method + ** flushes the pending terms table. But leave the (no-op) call to + ** PendingTermsFlush() in in case that changes. + */ + assert( p->nPendingData==0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } + + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); + } + + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", + p->zDb, p->zName, zName + ); + } + if( p->bHasStat ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", + p->zDb, p->zName, zName + ); + } + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", + p->zDb, p->zName, zName + ); + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", + p->zDb, p->zName, zName + ); + return rc; +} + +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ +static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(iSavepoint); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); + if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ + rc = fts3SyncMethod(pVtab); + } + return rc; +} + +/* +** The xRelease() method. +** +** This is a no-op. +*/ +static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(iSavepoint); + UNUSED_PARAMETER(pVtab); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint-1 ); + return SQLITE_OK; +} + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ +static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(iSavepoint); + assert( p->inTransaction ); + TESTONLY( p->mxSavepoint = iSavepoint ); + sqlite3Fts3PendingTermsClear(p); + return SQLITE_OK; +} + +/* +** Return true if zName is the extension on one of the shadow tables used +** by this module. +*/ +static int fts3ShadowName(const char *zName){ + static const char *azName[] = { + "content", "docsize", "segdir", "segments", "stat", + }; + unsigned int i; + for(i=0; ieType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; + }else{ + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); + } + } +} + +/* +** Arguments pList/nList contain the doclist for token iToken of phrase p. +** It is merged into the main doclist stored in p->doclist.aAll/nAll. +** +** This function assumes that pList points to a buffer allocated using +** sqlite3_malloc(). This function takes responsibility for eventually +** freeing the buffer. +** +** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. +*/ +static int fts3EvalPhraseMergeToken( + Fts3Table *pTab, /* FTS Table pointer */ + Fts3Phrase *p, /* Phrase to merge pList/nList into */ + int iToken, /* Token pList/nList corresponds to */ + char *pList, /* Pointer to doclist */ + int nList /* Number of bytes in pList */ +){ + int rc = SQLITE_OK; + assert( iToken!=p->iDoclistToken ); + + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; + } + + else if( p->iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; + } + + else if( p->doclist.aAll==0 ){ + sqlite3_free(pList); + } + + else { + char *pLeft; + char *pRight; + int nLeft; + int nRight; + int nDiff; + + if( p->iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; + } + + rc = fts3DoclistPhraseMerge( + pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight + ); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; + } + + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; + return rc; +} + +/* +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist +** does not take deferred tokens into account. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p /* Phrase object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); + + if( pToken->pSegcsr ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + } + } + assert( pToken->pSegcsr==0 ); + } + + return rc; +} + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +/* +** This function is called on each phrase after the position lists for +** any deferred tokens have been loaded into memory. It updates the phrases +** current position list to include only those positions that are really +** instances of the phrase (after considering deferred tokens). If this +** means that the phrase does not appear in the current row, doclist.pList +** and doclist.nList are both zeroed. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + int iToken; /* Used to iterate through phrase tokens */ + char *aPoslist = 0; /* Position list for deferred tokens */ + int nPoslist = 0; /* Number of bytes in aPoslist */ + int iPrev = -1; /* Token number of previous deferred token */ + + assert( pPhrase->doclist.bFreeList==0 ); + + for(iToken=0; iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; + + }else{ + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + assert( iPrev>=0 ); + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = (int)(aOut - aPoslist); + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + } + } + iPrev = iToken; + } + } + + if( iPrev>=0 ){ + int nMaxUndeferred = pPhrase->iDoclistToken; + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; + pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + sqlite3_free(aPoslist); + } + } + + return SQLITE_OK; +} +#endif /* SQLITE_DISABLE_FTS4_DEFERRED */ + +/* +** Maximum number of tokens a phrase may have to be considered for the +** incremental doclists strategy. +*/ +#define MAX_INCR_PHRASE_TOKENS 4 + +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. +** +** If parameter bOptOk is true, then the phrase may (or may not) use the +** incremental loading strategy. Otherwise, the entire doclist is loaded into +** memory within this call. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; /* Error code */ + int i; + + /* Determine if doclists may be loaded from disk incrementally. This is + ** possible if the bOptOk argument is true, the FTS doclists will be + ** scanned in forward order, and the phrase consists of + ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" + ** tokens or prefix tokens that cannot use a prefix-index. */ + int bHaveIncr = 0; + int bIncrOk = (bOptOk + && pCsr->bDesc==pTab->bDescIdx + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + && pTab->bNoIncrDoclist==0 +#endif + ); + for(i=0; bIncrOk==1 && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ + bIncrOk = 0; + } + if( pToken->pSegcsr ) bHaveIncr = 1; + } + + if( bIncrOk && bHaveIncr ){ + /* Use the incremental approach. */ + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + for(i=0; rc==SQLITE_OK && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; + if( pSegcsr ){ + rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); + } + } + p->bIncr = 1; + }else{ + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; + } + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; +} + +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. It is used by this module to iterate through phrase +** doclists in reverse and by the fts3_write.c module to iterate through +** pending-terms lists when writing to databases with "order=desc". +** +** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** descending (parameter bDescIdx==1) order of docid. Regardless, this +** function iterates from the end of the doclist to the beginning. +*/ +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; + + while( pDocid0 ); + assert( *pbEof==0 ); + assert_fts3_nc( p || *piDocid==0 ); + assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); + + if( p==0 ){ + p = aDoclist; + p += sqlite3Fts3GetVarint(p, piDocid); + }else{ + fts3PoslistCopy(0, &p); + while( p<&aDoclist[nDoclist] && *p==0 ) p++; + if( p>=&aDoclist[nDoclist] ){ + *pbEof = 1; + }else{ + sqlite3_int64 iVar; + p += sqlite3Fts3GetVarint(p, &iVar); + *piDocid += ((bDescIdx ? -1 : 1) * iVar); + } + } + + *ppIter = p; +} + +/* +** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof +** to true if EOF is reached. +*/ +static void fts3EvalDlPhraseNext( + Fts3Table *pTab, + Fts3Doclist *pDL, + u8 *pbEof +){ + char *pIter; /* Used to iterate through aAll */ + char *pEnd; /* 1 byte past end of aAll */ + + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + assert( pDL->aAll!=0 || pIter==0 ); + }else{ + pIter = pDL->aAll; + } + + if( pIter==0 || pIter>=(pEnd = pDL->aAll + pDL->nAll) ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (int)(pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } +} + +/* +** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). +*/ +typedef struct TokenDoclist TokenDoclist; +struct TokenDoclist { + int bIgnore; + sqlite3_int64 iDocid; + char *pList; + int nList; +}; + +/* +** Token pToken is an incrementally loaded token that is part of a +** multi-token phrase. Advance it to the next matching document in the +** database and populate output variable *p with the details of the new +** entry. Or, if the iterator has reached EOF, set *pbEof to true. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int incrPhraseTokenNext( + Fts3Table *pTab, /* Virtual table handle */ + Fts3Phrase *pPhrase, /* Phrase to advance token of */ + int iToken, /* Specific token to advance */ + TokenDoclist *p, /* OUT: Docid and doclist for new entry */ + u8 *pbEof /* OUT: True if iterator is at EOF */ +){ + int rc = SQLITE_OK; + + if( pPhrase->iDoclistToken==iToken ){ + assert( p->bIgnore==0 ); + assert( pPhrase->aToken[iToken].pSegcsr==0 ); + fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); + p->pList = pPhrase->doclist.pList; + p->nList = pPhrase->doclist.nList; + p->iDocid = pPhrase->doclist.iDocid; + }else{ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + assert( pToken->pDeferred==0 ); + assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); + if( pToken->pSegcsr ){ + assert( p->bIgnore==0 ); + rc = sqlite3Fts3MsrIncrNext( + pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList + ); + if( p->pList==0 ) *pbEof = 1; + }else{ + p->bIgnore = 1; + } + } + + return rc; +} + + +/* +** The phrase iterator passed as the second argument: +** +** * features at least one token that uses an incremental doclist, and +** +** * does not contain any deferred tokens. +** +** Advance it to the next matching documnent in the database and populate +** the Fts3Doclist.pList and nList fields. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +*/ +static int fts3EvalIncrPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + u8 bEof = 0; + + /* This is only called if it is guaranteed that the phrase has at least + ** one incremental token. In which case the bIncr flag is set. */ + assert( p->bIncr==1 ); + + if( p->nToken==1 ){ + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( pDL->pList==0 ) bEof = 1; + }else{ + int bDescDoclist = pCsr->bDesc; + struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; + + memset(a, 0, sizeof(a)); + assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); + assert( p->iDoclistTokennToken && bEof==0; i++){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ + iMax = a[i].iDocid; + bMaxSet = 1; + } + } + assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); + assert( rc!=SQLITE_OK || bMaxSet ); + + /* Keep advancing iterators until they all point to the same document */ + for(i=0; inToken; i++){ + while( rc==SQLITE_OK && bEof==0 + && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 + ){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ + iMax = a[i].iDocid; + i = 0; + } + } + } + + /* Check if the current entries really are a phrase match */ + if( bEof==0 ){ + int nList = 0; + int nByte = a[p->nToken-1].nList; + char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING); + if( !aDoclist ) return SQLITE_NOMEM; + memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); + memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING); + + for(i=0; i<(p->nToken-1); i++){ + if( a[i].bIgnore==0 ){ + char *pL = a[i].pList; + char *pR = aDoclist; + char *pOut = aDoclist; + int nDist = p->nToken-1-i; + int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); + if( res==0 ) break; + nList = (int)(pOut - aDoclist); + } + } + if( i==(p->nToken-1) ){ + pDL->iDocid = iMax; + pDL->pList = aDoclist; + pDL->nList = nList; + pDL->bFreeList = 1; + break; + } + sqlite3_free(aDoclist); + } + } + } + + *pbEof = bEof; + return rc; +} + +/* +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +*/ +static int fts3EvalPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + + if( p->bIncr ){ + rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; + }else{ + fts3EvalDlPhraseNext(pTab, pDL, pbEof); + } + + return rc; +} + +/* +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the +** expression. Also the Fts3Expr.bDeferred variable is set to true for any +** expressions for which all descendent tokens are deferred. +** +** If parameter bOptOk is zero, then it is guaranteed that the +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for +** each phrase in the expression (subject to deferred token processing). +** Or, if bOptOk is non-zero, then one or more tokens within the expression +** may be loaded incrementally, meaning doclist.aAll/nAll is not available. +** +** If an error occurs within this function, *pRc is set to an SQLite error +** code before returning. +*/ +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expression to initialize phrases in */ + int *pRc /* IN/OUT: Error code */ +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int nToken = pExpr->pPhrase->nToken; + if( nToken ){ + int i; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); + } + *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); + }else{ + fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); + } + } +} + +/* +** An array of the following structures is assembled as part of the process +** of selecting tokens to defer before the query starts executing (as part +** of the xFilter() method). There is one element in the array for each +** token in the FTS expression. +** +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong +** to phrases that are connected only by AND and NEAR operators (not OR or +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered +** separately. The root of a tokens AND/NEAR cluster is stored in +** Fts3TokenAndCost.pRoot. +*/ +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ + int nOvfl; /* Number of overflow pages to load doclist */ + int iCol; /* The column the token must match */ +}; + +/* +** This function is used to populate an allocated Fts3TokenAndCost array. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if an error occurs during execution, *pRc is set to an +** SQLite error code. +*/ +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ + Fts3Expr *pExpr, /* Expression to consider */ + Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ + Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; + pTC->pRoot = pRoot; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType!=FTSQUERY_NOT ){ + assert( pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND + || pExpr->eType==FTSQUERY_NEAR + ); + assert( pExpr->pLeft && pExpr->pRight ); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pLeft; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); + } + } +} + +/* +** Determine the average document (row) size in pages. If successful, +** write this value to *pnPage and return SQLITE_OK. Otherwise, return +** an SQLite error code. +** +** The average document size in pages is calculated by first calculating +** determining the average size in bytes, B. If B is less than the amount +** of data that will fit on a single leaf page of an intkey table in +** this database, then the average docsize is 1. Otherwise, it is 1 plus +** the number of overflow pages consumed by a record B bytes in size. +*/ +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + int rc = SQLITE_OK; + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + testcase( a==0 ); /* If %_stat.value set to X'' */ + if( a ){ + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc); + while( anDoc = nDoc; + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + } + + *pnPage = pCsr->nRowAvg; + return rc; +} + +/* +** This function is called to select the tokens (if any) that will be +** deferred. The array aTC[] has already been populated when this is +** called. +** +** This function is called once for each AND/NEAR cluster in the +** expression. Each invocation determines which tokens to defer within +** the cluster with root node pRoot. See comments above the definition +** of struct Fts3TokenAndCost for more details. +** +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() +** called on each token to defer. Otherwise, an SQLite error code is +** returned. +*/ +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Consider tokens with this root node */ + Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ + int nTC /* Number of entries in aTC[] */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int nDocSize = 0; /* Number of pages per doc loaded */ + int rc = SQLITE_OK; /* Return code */ + int ii; /* Iterator variable for various purposes */ + int nOvfl = 0; /* Total overflow pages used by doclists */ + int nToken = 0; /* Total number of tokens in cluster */ + + int nMinEst = 0; /* The minimum count for any phrase so far. */ + int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + + /* Tokens are never deferred for FTS tables created using the content=xxx + ** option. The reason being that it is not guaranteed that the content + ** table actually contains the same data as the index. To prevent this from + ** causing any problems, the deferred token optimization is completely + ** disabled for content=xxx tables. */ + if( pTab->zContentTbl ){ + return SQLITE_OK; + } + + /* Count the tokens in this AND/NEAR cluster. If none of the doclists + ** associated with the tokens spill onto overflow pages, or if there is + ** only 1 token, exit early. No tokens to defer in this case. */ + for(ii=0; ii0 ); + + + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer + ** to retrieve the entire doclist for the token from the full-text index. + ** Load the doclists for tokens that are either: + ** + ** a. The cheapest token in the entire query (i.e. the one visited by the + ** first iteration of this loop), or + ** + ** b. Part of a multi-token phrase. + ** + ** After each token doclist is loaded, merge it with the others from the + ** same phrase and count the number of documents that the merged doclist + ** contains. Set variable "nMinEst" to the smallest number of documents in + ** any phrase doclist for which 1 or more token doclists have been loaded. + ** Let nOther be the number of other phrases for which it is certain that + ** one or more tokens will not be deferred. + ** + ** Then, for each token, defer it if loading the doclist would result in + ** loading N or more overflow pages into memory, where N is computed as: + ** + ** (nMinEst + 4^nOther - 1) / (4^nOther) + */ + for(ii=0; iinOvfl) + ){ + pTC = &aTC[iTC]; + } + } + assert( pTC ); + + if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ + /* The number of overflow pages to load for this (and therefore all + ** subsequent) tokens is greater than the estimated number of pages + ** that will be loaded if all subsequent tokens are deferred. + */ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + }else{ + /* Set nLoad4 to the value of (4^nOther) for the next iteration of the + ** for-loop. Except, limit the value to 2^24 to prevent it from + ** overflowing the 32-bit integer it is stored in. */ + if( ii<12 ) nLoad4 = nLoad4*4; + + if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken( + pTab, pTC->pPhrase, pTC->iToken,pList,nList + ); + } + if( rc==SQLITE_OK ){ + int nCount; + nCount = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); + if( ii==0 || nCountpToken = 0; + } + + return rc; +} + +/* +** This function is called from within the xFilter method. It initializes +** the full-text query currently stored in pCsr->pExpr. To iterate through +** the results of a query, the caller does: +** +** fts3EvalStart(pCsr); +** while( 1 ){ +** fts3EvalNext(pCsr); +** if( pCsr->bEof ) break; +** ... return row pCsr->iPrevId to the caller ... +** } +*/ +static int fts3EvalStart(Fts3Cursor *pCsr){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + int nOr = 0; + + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); + + /* Determine which, if any, tokens in the expression should be deferred. */ +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ + Fts3TokenAndCost *aTC; + aTC = (Fts3TokenAndCost *)sqlite3_malloc64( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr * 2 + ); + + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + Fts3Expr **apOr = (Fts3Expr **)&aTC[nToken]; + int ii; + Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; + + fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); + nToken = (int)(pTC-aTC); + nOr = (int)(ppOr-apOr); + + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iipExpr, &rc); + return rc; +} + +/* +** Invalidate the current position list for phrase pPhrase. +*/ +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ + if( pPhrase->doclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; +} + +/* +** This function is called to edit the position list associated with +** the phrase object passed as the fifth argument according to a NEAR +** condition. For example: +** +** abc NEAR/5 "def ghi" +** +** Parameter nNear is passed the NEAR distance of the expression (5 in +** the example above). When this function is called, *paPoslist points to +** the position list, and *pnToken is the number of phrase tokens in the +** phrase on the other side of the NEAR operator to pPhrase. For example, +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to +** the position list associated with phrase "abc". +** +** All positions in the pPhrase position list that are not sufficiently +** close to a position in the *paPoslist position list are removed. If this +** leaves 0 positions, zero is returned. Otherwise, non-zero. +** +** Before returning, *paPoslist is set to point to the position lsit +** associated with pPhrase. And *pnToken is set to the number of tokens in +** pPhrase. +*/ +static int fts3EvalNearTrim( + int nNear, /* NEAR distance. As in "NEAR/nNear". */ + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + int nNew; + char *p2; + char *pOut; + int res; + + assert( pPhrase->doclist.pList ); + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + if( res ){ + nNew = (int)(pOut - pPhrase->doclist.pList) - 1; + assert_fts3_nc( nNew<=pPhrase->doclist.nList && nNew>0 ); + if( nNew>=0 && nNew<=pPhrase->doclist.nList ){ + assert( pPhrase->doclist.pList[nNew]=='\0' ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + } + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + } + + return res; +} + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is called. +** Otherwise, it advances the expression passed as the second argument to +** point to the next matching row in the database. Expressions iterate through +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, +** or descending if it is non-zero. +** +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if +** successful, the following variables in pExpr are set: +** +** Fts3Expr.bEof (non-zero if EOF - there is no next row) +** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) +** +** If the expression is of type FTSQUERY_PHRASE, and the expression is not +** at EOF, then the following variables are populated with the position list +** for the phrase for the visited row: +** +** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) +** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** +** It says above that this function advances the expression to the next +** matching row. This is usually true, but there are the following exceptions: +** +** 1. Deferred tokens are not taken into account. If a phrase consists +** entirely of deferred tokens, it is assumed to match every row in +** the db. In this case the position-list is not populated at all. +** +** Or, if a phrase contains one or more deferred tokens and one or +** more non-deferred tokens, then the expression is advanced to the +** next possible match, considering only non-deferred tokens. In other +** words, if the phrase is "A B C", and "B" is deferred, the expression +** is advanced to the next row that contains an instance of "A * C", +** where "*" may match any single token. The position list in this case +** is populated as for "A * C" before returning. +** +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** advanced to point to the next row that matches "x AND y". +** +** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is +** really a match, taking into account deferred tokens and NEAR operators. +*/ +static void fts3EvalNextRow( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expr. to advance to next matching row */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + + if( pLeft->bDeferred ){ + /* LHS is entirely deferred. So we assume it matches every row. + ** Advance the RHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + /* RHS is entirely deferred. So we assume it matches every row. + ** Advance the LHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + /* Neither the RHS or LHS are deferred. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else{ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ + assert( pRight->eType==FTSQUERY_PHRASE ); + if( pRight->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pRight->pPhrase->doclist; + while( *pRc==SQLITE_OK && pRight->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pLeft->pPhrase->doclist; + while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pLeft, pRc); + } + } + pRight->bEof = pLeft->bEof = 1; + } + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else if( pLeft->bEof || iCmp>0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNextRow(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalInvalidatePoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +/* +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR +** cluster, then this function returns 1 immediately. +** +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** +** If the current row is a match, the position list associated with each +** phrase in the NEAR expression is edited in place to contain only those +** phrase instances sufficiently close to their peers to satisfy all NEAR +** constraints. In this case it returns 1. If the NEAR expression does not +** match the current row, 0 is returned. The position lists may or may not +** be edited if 0 is returned. +*/ +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this - it's the way the parser in fts3_expr.c works. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + sqlite3_int64 nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + assert( p->pRight->pPhrase->doclist.nList>0 ); + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + aTmp = sqlite3_malloc64(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + } + + sqlite3_free(aTmp); + } + + return res; +} + +/* +** This function is a helper function for sqlite3Fts3EvalTestDeferred(). +** Assuming no error occurs or has occurred, It returns non-zero if the +** expression passed as the second argument matches the row that pCsr +** currently points to, or zero if it does not. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is +** undefined. +*/ +static int fts3EvalTestExpr( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ + int *pRc /* IN/OUT: Error code */ +){ + int bHit = 1; /* Return value */ + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) + ); + + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pPhrase); + } + } + + break; + + case FTSQUERY_OR: { + int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; + break; + } + + case FTSQUERY_NOT: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + ); + break; + + default: { +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalInvalidatePoslist(pPhrase); + } + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else +#endif + { + bHit = ( + pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId + && pExpr->pPhrase->doclist.nList>0 + ); + } + break; + } + } + } + return bHit; +} + +/* +** This function is called as the second part of each xNext operation when +** iterating through the results of a full-text query. At this point the +** cursor points to a row that matches the query expression, with the +** following caveats: +** +** * Up until this point, "NEAR" operators in the expression have been +** treated as "AND". +** +** * Deferred tokens have not yet been considered. +** +** If *pRc is not SQLITE_OK when this function is called, it immediately +** returns 0. Otherwise, it tests whether or not after considering NEAR +** operators and deferred tokens the current row is still a match for the +** expression. It returns 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does *not* match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK ){ + + /* If there are one or more deferred tokens, load the current row into + ** memory and scan it to determine the position list for each deferred + ** token. Then, see if this row is really a match, considering deferred + ** tokens and NEAR operators (neither of which were taken into account + ** earlier, by fts3EvalNextRow()). + */ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + } + bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + + /* Free the position-lists accumulated for each deferred token above. */ + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; + } + return (rc==SQLITE_OK && bMiss); +} + +/* +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. +*/ +static int fts3EvalNext(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; + assert( pCsr->isEof==0 ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + fts3EvalNextRow(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); + } + + /* Check if the cursor is past the end of the docid range specified + ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ + if( rc==SQLITE_OK && ( + (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) + || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) + )){ + pCsr->isEof = 1; + } + + return rc; +} + +/* +** Restart interation for expression pExpr so that the next call to +** fts3EvalNext() visits the first row. Do not allow incremental +** loading or merging of phrase doclists for this iteration. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op. If an error occurs within this function, *pRc is set to an +** SQLite error code before returning. +*/ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + + if( pPhrase ){ + fts3EvalInvalidatePoslist(pPhrase); + if( pPhrase->bIncr ){ + int i; + for(i=0; inToken; i++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[i]; + assert( pToken->pDeferred==0 ); + if( pToken->pSegcsr ){ + sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); + } + } + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); + } + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + pPhrase->pOrPoslist = 0; + } + + pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; + + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); + } +} + +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. +*/ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; + + do{ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += fts3GetVarint32(p, &iCol); + }while( iColpLeft, nCol); + fts3EvalUpdateCounts(pExpr->pRight, nCol); + } +} + +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. +*/ +static int fts3EvalGatherStats( + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ + + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + assert( pRoot->bStart ); + + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } + + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNextRow(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && sqlite3Fts3EvalTestDeferred(pCsr, &rc) + ); + + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot, pTab->nColumn); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidbEof==0 ); + if( pRoot->bEof ) rc = FTS_CORRUPT_VTAB; + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); + } + } + return rc; +} + +/* +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; + + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; + aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; + } + }else{ + rc = fts3EvalGatherStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; + } + } + } + + return rc; +} + +/* +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurrence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. +*/ +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + int iCol, /* Column to return position list for */ + char **ppOut /* OUT: Pointer to position list */ +){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter; + int iThis; + sqlite3_int64 iDocid; + + /* If this phrase is applies specifically to some column other than + ** column iCol, return a NULL pointer. */ + *ppOut = 0; + assert( iCol>=0 && iColnColumn ); + if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ + return SQLITE_OK; + } + + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; + if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ + int rc = SQLITE_OK; + int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ + int bOr = 0; + u8 bTreeEof = 0; + Fts3Expr *p; /* Used to iterate from pExpr to root */ + Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + int bMatch; + + /* Check if this phrase descends from an OR expression node. If not, + ** return NULL. Otherwise, the entry that corresponds to docid + ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the + ** tree that the node is part of has been marked as EOF, but the node + ** itself is not EOF, then it may point to an earlier entry. */ + pNear = pExpr; + for(p=pExpr->pParent; p; p=p->pParent){ + if( p->eType==FTSQUERY_OR ) bOr = 1; + if( p->eType==FTSQUERY_NEAR ) pNear = p; + if( p->bEof ) bTreeEof = 1; + } + if( bOr==0 ) return SQLITE_OK; + + /* This is the descendent of an OR node. In this case we cannot use + ** an incremental phrase. Load the entire doclist for the phrase + ** into memory in this case. */ + if( pPhrase->bIncr ){ + int bEofSave = pNear->bEof; + fts3EvalRestart(pCsr, pNear, &rc); + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + if( bEofSave==0 && pNear->iDocid==iDocid ) break; + } + assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + } + if( bTreeEof ){ + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + } + } + if( rc!=SQLITE_OK ) return rc; + + bMatch = 1; + for(p=pNear; p; p=p->pLeft){ + u8 bEof = 0; + Fts3Expr *pTest = p; + Fts3Phrase *pPh; + assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE ); + if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight; + assert( pTest->eType==FTSQUERY_PHRASE ); + pPh = pTest->pPhrase; + + pIter = pPh->pOrPoslist; + iDocid = pPh->iOrDocid; + if( pCsr->bDesc==bDescDoclist ){ + bEof = !pPh->doclist.nAll || + (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll)); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ + sqlite3Fts3DoclistNext( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &bEof + ); + } + }else{ + bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ + int dummy; + sqlite3Fts3DoclistPrev( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &dummy, &bEof + ); + } + } + pPh->pOrPoslist = pIter; + pPh->iOrDocid = iDocid; + if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0; + } + + if( bMatch ){ + pIter = pPhrase->pOrPoslist; + }else{ + pIter = 0; + } + } + if( pIter==0 ) return SQLITE_OK; + + if( *pIter==0x01 ){ + pIter++; + pIter += fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. +*/ +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalInvalidatePoslist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } + } +} + + +/* +** Return SQLITE_CORRUPT_VTAB. +*/ +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ + return SQLITE_CORRUPT_VTAB; +} +#endif + +#if !SQLITE_CORE +/* +** Initialize API pointer table, if required. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_fts3_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts3Init(db); +} +#endif + +#endif + +/************** End of fts3.c ************************************************/ +/************** Begin file fts3_aux.c ****************************************/ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +typedef struct Fts3auxTable Fts3auxTable; +typedef struct Fts3auxCursor Fts3auxCursor; + +struct Fts3auxTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts3Table *pFts3Tab; +}; + +struct Fts3auxCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts3MultiSegReader csr; /* Must be right after "base" */ + Fts3SegFilter filter; + char *zStop; + int nStop; /* Byte-length of string zStop */ + int iLangid; /* Language id to query */ + int isEof; /* True if cursor is at EOF */ + sqlite3_int64 iRowid; /* Current rowid */ + + int iCol; /* Current value of 'col' column */ + int nStat; /* Size of aStat[] array */ + struct Fts3auxColstats { + sqlite3_int64 nDoc; /* 'documents' values for current csr row */ + sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ + } *aStat; +}; + +/* +** Schema of the terms table. +*/ +#define FTS3_AUX_SCHEMA \ + "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" + +/* +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. +*/ +static int fts3auxConnectMethod( + sqlite3 *db, /* Database connection */ + void *pUnused, /* Unused */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + sqlite3_int64 nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3auxTable *p; /* Virtual table object to return */ + + UNUSED_PARAMETER(pUnused); + + /* The user should invoke this in one of two forms: + ** + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); + */ + if( argc!=4 && argc!=5 ) goto bad_args; + + zDb = argv[1]; + nDb = (int)strlen(zDb); + if( argc==5 ){ + if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ + zDb = argv[3]; + nDb = (int)strlen(zDb); + zFts3 = argv[4]; + }else{ + goto bad_args; + } + }else{ + zFts3 = argv[3]; + } + nFts3 = (int)strlen(zFts3); + + rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); + if( rc!=SQLITE_OK ) return rc; + + nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; + p = (Fts3auxTable *)sqlite3_malloc64(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, nByte); + + p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; + + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; + + bad_args: + sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); + return SQLITE_ERROR; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3auxTable *p = (Fts3auxTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); + } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(p); + return SQLITE_OK; +} + +#define FTS4AUX_EQ_CONSTRAINT 1 +#define FTS4AUX_GE_CONSTRAINT 2 +#define FTS4AUX_LE_CONSTRAINT 4 + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3auxBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + int iEq = -1; + int iGe = -1; + int iLe = -1; + int iLangid = -1; + int iNext = 1; /* Next free argvIndex value */ + + UNUSED_PARAMETER(pVTab); + + /* This vtab delivers always results in "ORDER BY term ASC" order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + + /* Search for equality and range constraints on the "term" column. + ** And equality constraints on the hidden "languageid" column. */ + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable ){ + int op = pInfo->aConstraint[i].op; + int iCol = pInfo->aConstraint[i].iColumn; + + if( iCol==0 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; + if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; + } + if( iCol==4 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; + } + } + } + + if( iEq>=0 ){ + pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; + pInfo->aConstraintUsage[iEq].argvIndex = iNext++; + pInfo->estimatedCost = 5; + }else{ + pInfo->idxNum = 0; + pInfo->estimatedCost = 20000; + if( iGe>=0 ){ + pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; + pInfo->aConstraintUsage[iGe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + if( iLe>=0 ){ + pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; + pInfo->aConstraintUsage[iLe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + } + if( iLangid>=0 ){ + pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; + pInfo->estimatedCost--; + } + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ + + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3auxCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->zStop); + sqlite3_free(pCsr->aStat); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ + if( nSize>pCsr->nStat ){ + struct Fts3auxColstats *aNew; + aNew = (struct Fts3auxColstats *)sqlite3_realloc64(pCsr->aStat, + sizeof(struct Fts3auxColstats) * nSize + ); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCsr->nStat], 0, + sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) + ); + pCsr->aStat = aNew; + pCsr->nStat = nSize; + } + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + + /* Increment our pretend rowid value. */ + pCsr->iRowid++; + + for(pCsr->iCol++; pCsr->iColnStat; pCsr->iCol++){ + if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; + } + + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc==SQLITE_ROW ){ + int i = 0; + int nDoclist = pCsr->csr.nDoclist; + char *aDoclist = pCsr->csr.aDoclist; + int iCol; + + int eState = 0; + + if( pCsr->zStop ){ + int n = (pCsr->nStopcsr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; + int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); + if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ + pCsr->isEof = 1; + return SQLITE_OK; + } + } + + if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; + memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); + iCol = 0; + + while( iaStat[0].nDoc++; + eState = 1; + iCol = 0; + break; + + /* State 1. In this state we are expecting either a 1, indicating + ** that the following integer will be a column number, or the + ** start of a position list for column 0. + ** + ** The only difference between state 1 and state 2 is that if the + ** integer encountered in state 1 is not 0 or 1, then we need to + ** increment the column 0 "nDoc" count for this term. + */ + case 1: + assert( iCol==0 ); + if( v>1 ){ + pCsr->aStat[1].nDoc++; + } + eState = 2; + /* fall through */ + + case 2: + if( v==0 ){ /* 0x00. Next integer will be a docid. */ + eState = 0; + }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ + eState = 3; + }else{ /* 2 or greater. A position. */ + pCsr->aStat[iCol+1].nOcc++; + pCsr->aStat[0].nOcc++; + } + break; + + /* State 3. The integer just read is a column number. */ + default: assert( eState==3 ); + iCol = (int)v; + if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; + pCsr->aStat[iCol+1].nDoc++; + eState = 2; + break; + } + } + + pCsr->iCol = 0; + rc = SQLITE_OK; + }else{ + pCsr->isEof = 1; + } + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3auxFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + int isScan = 0; + int iLangVal = 0; /* Language id to query */ + + int iEq = -1; /* Index of term=? value in apVal */ + int iGe = -1; /* Index of term>=? value in apVal */ + int iLe = -1; /* Index of term<=? value in apVal */ + int iLangid = -1; /* Index of languageid=? value in apVal */ + int iNext = 0; + + UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxStr); + + assert( idxStr==0 ); + assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 + || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT + || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) + ); + + if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ + iEq = iNext++; + }else{ + isScan = 1; + if( idxNum & FTS4AUX_GE_CONSTRAINT ){ + iGe = iNext++; + } + if( idxNum & FTS4AUX_LE_CONSTRAINT ){ + iLe = iNext++; + } + } + if( iNextfilter.zTerm); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->aStat); + sqlite3_free(pCsr->zStop); + memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); + + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; + + if( iEq>=0 || iGe>=0 ){ + const unsigned char *zStr = sqlite3_value_text(apVal[0]); + assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); + if( zStr ){ + pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); + if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; + pCsr->filter.nTerm = (int)strlen(pCsr->filter.zTerm); + } + } + + if( iLe>=0 ){ + pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); + if( pCsr->zStop==0 ) return SQLITE_NOMEM; + pCsr->nStop = (int)strlen(pCsr->zStop); + } + + if( iLangid>=0 ){ + iLangVal = sqlite3_value_int(apVal[iLangid]); + + /* If the user specified a negative value for the languageid, use zero + ** instead. This works, as the "languageid=?" constraint will also + ** be tested by the VDBE layer. The test will always be false (since + ** this module will not return a row with a negative languageid), and + ** so the overall query will return zero rows. */ + if( iLangVal<0 ) iLangVal = 0; + } + pCsr->iLangid = iLangVal; + + rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, + pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + } + + if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); + return rc; +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3auxColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3auxCursor *p = (Fts3auxCursor *)pCursor; + + assert( p->isEof==0 ); + switch( iCol ){ + case 0: /* term */ + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; + + case 1: /* col */ + if( p->iCol ){ + sqlite3_result_int(pCtx, p->iCol-1); + }else{ + sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); + } + break; + + case 2: /* documents */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); + break; + + case 3: /* occurrences */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); + break; + + default: /* languageid */ + assert( iCol==4 ); + sqlite3_result_int(pCtx, p->iLangid); + break; + } + + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3auxRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3aux module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ + static const sqlite3_module fts3aux_module = { + 0, /* iVersion */ + fts3auxConnectMethod, /* xCreate */ + fts3auxConnectMethod, /* xConnect */ + fts3auxBestIndexMethod, /* xBestIndex */ + fts3auxDisconnectMethod, /* xDisconnect */ + fts3auxDisconnectMethod, /* xDestroy */ + fts3auxOpenMethod, /* xOpen */ + fts3auxCloseMethod, /* xClose */ + fts3auxFilterMethod, /* xFilter */ + fts3auxNextMethod, /* xNext */ + fts3auxEofMethod, /* xEof */ + fts3auxColumnMethod, /* xColumn */ + fts3auxRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_aux.c ********************************************/ +/************** Begin file fts3_expr.c ***************************************/ +/* +** 2008 Nov 28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This module contains code that implements a parser for fts3 query strings +** (the right-hand argument to the MATCH operator). Because the supported +** syntax is relatively simple, the whole tokenizer/parser system is +** hand-coded. +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* +** By default, this module parses the legacy syntax that has been +** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined, then it uses the new syntax. The differences between +** the new and the old syntaxes are: +** +** a) The new syntax supports parenthesis. The old does not. +** +** b) The new syntax supports the AND and NOT operators. The old does not. +** +** c) The old syntax supports the "-" token qualifier. This is not +** supported by the new syntax (it is replaced by the NOT operator). +** +** d) When using the old syntax, the OR operator has a greater precedence +** than an implicit AND. When using the new, both implicity and explicit +** AND operators have a higher precedence than OR. +** +** If compiled with SQLITE_TEST defined, then this module exports the +** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable +** to zero causes the module to use the old syntax. If it is set to +** non-zero the new syntax is activated. This is so both syntaxes can +** be tested using a single build of testfixture. +** +** The following describes the syntax supported by the fts3 MATCH +** operator in a similar format to that used by the lemon parser +** generator. This module does not use actually lemon, it uses a +** custom parser. +** +** query ::= andexpr (OR andexpr)*. +** +** andexpr ::= notexpr (AND? notexpr)*. +** +** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. +** notexpr ::= LP query RP. +** +** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** +** distance_opt ::= . +** distance_opt ::= / INTEGER. +** +** phrase ::= TOKEN. +** phrase ::= COLUMN:TOKEN. +** phrase ::= "TOKEN TOKEN TOKEN...". +*/ + +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_fts3_enable_parentheses = 0; +#else +# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS +# define sqlite3_fts3_enable_parentheses 1 +# else +# define sqlite3_fts3_enable_parentheses 0 +# endif +#endif + +/* +** Default span for NEAR operators. +*/ +#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 + +/* #include */ +/* #include */ + +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ +typedef struct ParseContext ParseContext; +struct ParseContext { + sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ + int iLangid; /* Language id used with tokenizer */ + const char **azCol; /* Array of column names for fts3 table */ + int bFts4; /* True to allow FTS4-only syntax */ + int nCol; /* Number of entries in azCol[] */ + int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ + sqlite3_context *pCtx; /* Write error message here */ + int nNest; /* Number of nested brackets */ +}; + +/* +** This function is equivalent to the standard isspace() function. +** +** The standard isspace() can be awkward to use safely, because although it +** is defined to accept an argument of type int, its behavior when passed +** an integer that falls outside of the range of the unsigned char type +** is undefined (and sometimes, "undefined" means segfault). This wrapper +** is defined to accept an argument of type char, and always returns 0 for +** any values that fall outside of the range of the unsigned char type (i.e. +** negative values). +*/ +static int fts3isspace(char c){ + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; +} + +/* +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, +** zero the memory before returning a pointer to it. If unsuccessful, +** return NULL. +*/ +static void *fts3MallocZero(sqlite3_int64 nByte){ + void *pRet = sqlite3_malloc64(nByte); + if( pRet ) memset(pRet, 0, nByte); + return pRet; +} + +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer( + sqlite3_tokenizer *pTokenizer, + int iLangid, + const char *z, + int n, + sqlite3_tokenizer_cursor **ppCsr +){ + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr = 0; + int rc; + + rc = pModule->xOpen(pTokenizer, z, n, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( rc==SQLITE_OK ){ + pCsr->pTokenizer = pTokenizer; + if( pModule->iVersion>=1 ){ + rc = pModule->xLanguageid(pCsr, iLangid); + if( rc!=SQLITE_OK ){ + pModule->xClose(pCsr); + pCsr = 0; + } + } + } + *ppCsr = pCsr; + return rc; +} + +/* +** Function getNextNode(), which is called by fts3ExprParse(), may itself +** call fts3ExprParse(). So this forward declaration is required. +*/ +static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); + +/* +** Extract the next token from buffer z (length n) using the tokenizer +** and other information (column names etc.) in pParse. Create an Fts3Expr +** structure of type FTSQUERY_PHRASE containing a phrase consisting of this +** single token and set *ppExpr to point to it. If the end of the buffer is +** reached before a token is found, set *ppExpr to zero. It is the +** responsibility of the caller to eventually deallocate the allocated +** Fts3Expr structure (if any) by passing it to sqlite3_free(). +** +** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation +** fails. +*/ +static int getNextToken( + ParseContext *pParse, /* fts3 query parse context */ + int iCol, /* Value for Fts3Phrase.iColumn */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + sqlite3_tokenizer_cursor *pCursor; + Fts3Expr *pRet = 0; + int i = 0; + + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, i, &pCursor); + if( rc==SQLITE_OK ){ + const char *zToken; + int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; + sqlite3_int64 nByte; /* total space to allocate */ + + rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + pRet = (Fts3Expr *)fts3MallocZero(nByte); + if( !pRet ){ + rc = SQLITE_NOMEM; + }else{ + pRet->eType = FTSQUERY_PHRASE; + pRet->pPhrase = (Fts3Phrase *)&pRet[1]; + pRet->pPhrase->nToken = 1; + pRet->pPhrase->iColumn = iCol; + pRet->pPhrase->aToken[0].n = nToken; + pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); + + if( iEndpPhrase->aToken[0].isPrefix = 1; + iEnd++; + } + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } + } + + } + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + + pModule->xClose(pCursor); + } + + *ppExpr = pRet; + return rc; +} + + +/* +** Enlarge a memory allocation. If an out-of-memory allocation occurs, +** then free the old allocation. +*/ +static void *fts3ReallocOrFree(void *pOrig, sqlite3_int64 nNew){ + void *pRet = sqlite3_realloc64(pOrig, nNew); + if( !pRet ){ + sqlite3_free(pOrig); + } + return pRet; +} + +/* +** Buffer zInput, length nInput, contains the contents of a quoted string +** that appeared as part of an fts3 query expression. Neither quote character +** is included in the buffer. This function attempts to tokenize the entire +** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE +** containing the results. +** +** If successful, SQLITE_OK is returned and *ppExpr set to point at the +** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory +** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set +** to 0. +*/ +static int getNextString( + ParseContext *pParse, /* fts3 query parse context */ + const char *zInput, int nInput, /* Input string */ + Fts3Expr **ppExpr /* OUT: expression */ +){ + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + Fts3Expr *p = 0; + sqlite3_tokenizer_cursor *pCursor = 0; + char *zTemp = 0; + int nTemp = 0; + + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ + rc = sqlite3Fts3OpenTokenizer( + pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); + if( rc==SQLITE_OK ){ + int ii; + for(ii=0; rc==SQLITE_OK; ii++){ + const char *zByte; + int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); + if( rc==SQLITE_OK ){ + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); + nToken = ii+1; + } + } + + pModule->xClose(pCursor); + pCursor = 0; + } + + if( rc==SQLITE_DONE ){ + int jj; + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); + p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; + p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + if( zTemp ){ + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + }else{ + assert( nTemp==0 ); + } + + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; + } + rc = SQLITE_OK; + } + + *ppExpr = p; + return rc; +no_mem: + + if( pCursor ){ + pModule->xClose(pCursor); + } + sqlite3_free(zTemp); + sqlite3_free(p); + *ppExpr = 0; + return SQLITE_NOMEM; +} + +/* +** The output variable *ppExpr is populated with an allocated Fts3Expr +** structure, or set to 0 if the end of the input buffer is reached. +** +** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM +** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. +** If SQLITE_ERROR is returned, pContext is populated with an error message. +*/ +static int getNextNode( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + static const struct Fts3Keyword { + char *z; /* Keyword text */ + unsigned char n; /* Length of the keyword */ + unsigned char parenOnly; /* Only valid in paren mode */ + unsigned char eType; /* Keyword code */ + } aKeyword[] = { + { "OR" , 2, 0, FTSQUERY_OR }, + { "AND", 3, 1, FTSQUERY_AND }, + { "NOT", 3, 1, FTSQUERY_NOT }, + { "NEAR", 4, 0, FTSQUERY_NEAR } + }; + int ii; + int iCol; + int iColLen; + int rc; + Fts3Expr *pRet = 0; + + const char *zInput = z; + int nInput = n; + + pParse->isNot = 0; + + /* Skip over any whitespace before checking for a keyword, an open or + ** close bracket, or a quoted string. + */ + while( nInput>0 && fts3isspace(*zInput) ){ + nInput--; + zInput++; + } + if( nInput==0 ){ + return SQLITE_DONE; + } + + /* See if we are dealing with a keyword. */ + for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ + const struct Fts3Keyword *pKey = &aKeyword[ii]; + + if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ + continue; + } + + if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ + int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; + int nKey = pKey->n; + char cNext; + + /* If this is a "NEAR" keyword, check for an explicit nearness. */ + if( pKey->eType==FTSQUERY_NEAR ){ + assert( nKey==4 ); + if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ + nKey += 1+sqlite3Fts3ReadInt(&zInput[nKey+1], &nNear); + } + } + + /* At this point this is probably a keyword. But for that to be true, + ** the next byte must contain either whitespace, an open or close + ** parenthesis, a quote character, or EOF. + */ + cNext = zInput[nKey]; + if( fts3isspace(cNext) + || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 + ){ + pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); + if( !pRet ){ + return SQLITE_NOMEM; + } + pRet->eType = pKey->eType; + pRet->nNear = nNear; + *ppExpr = pRet; + *pnConsumed = (int)((zInput - z) + nKey); + return SQLITE_OK; + } + + /* Turns out that wasn't a keyword after all. This happens if the + ** user has supplied a token such as "ORacle". Continue. + */ + } + } + + /* See if we are dealing with a quoted phrase. If this is the case, then + ** search for the closing quote and pass the whole string to getNextString() + ** for processing. This is easy to do, as fts3 has no syntax for escaping + ** a quote character embedded in a string. + */ + if( *zInput=='"' ){ + for(ii=1; iinNest++; +#if !defined(SQLITE_MAX_EXPR_DEPTH) + if( pParse->nNest>1000 ) return SQLITE_ERROR; +#elif SQLITE_MAX_EXPR_DEPTH>0 + if( pParse->nNest>SQLITE_MAX_EXPR_DEPTH ) return SQLITE_ERROR; +#endif + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (int)((zInput - z) + 1); + *ppExpr = 0; + return SQLITE_DONE; + } + } + + /* If control flows to this point, this must be a regular token, or + ** the end of the input. Read a regular token using the sqlite3_tokenizer + ** interface. Before doing so, figure out if there is an explicit + ** column specifier for the token. + ** + ** TODO: Strangely, it is not possible to associate a column specifier + ** with a quoted phrase, only with a single token. Not sure if this was + ** an implementation artifact or an intentional decision when fts3 was + ** first implemented. Whichever it was, this module duplicates the + ** limitation. + */ + iCol = pParse->iDefaultCol; + iColLen = 0; + for(ii=0; iinCol; ii++){ + const char *zStr = pParse->azCol[ii]; + int nStr = (int)strlen(zStr); + if( nInput>nStr && zInput[nStr]==':' + && sqlite3_strnicmp(zStr, zInput, nStr)==0 + ){ + iCol = ii; + iColLen = (int)((zInput - z) + nStr + 1); + break; + } + } + rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); + *pnConsumed += iColLen; + return rc; +} + +/* +** The argument is an Fts3Expr structure for a binary operator (any type +** except an FTSQUERY_PHRASE). Return an integer value representing the +** precedence of the operator. Lower values have a higher precedence (i.e. +** group more tightly). For example, in the C language, the == operator +** groups more tightly than ||, and would therefore have a higher precedence. +** +** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined), the order of the operators in precedence from highest to +** lowest is: +** +** NEAR +** NOT +** AND (including implicit ANDs) +** OR +** +** Note that when using the old query syntax, the OR operator has a higher +** precedence than the AND operator. +*/ +static int opPrecedence(Fts3Expr *p){ + assert( p->eType!=FTSQUERY_PHRASE ); + if( sqlite3_fts3_enable_parentheses ){ + return p->eType; + }else if( p->eType==FTSQUERY_NEAR ){ + return 1; + }else if( p->eType==FTSQUERY_OR ){ + return 2; + } + assert( p->eType==FTSQUERY_AND ); + return 3; +} + +/* +** Argument ppHead contains a pointer to the current head of a query +** expression tree being parsed. pPrev is the expression node most recently +** inserted into the tree. This function adds pNew, which is always a binary +** operator node, into the expression tree based on the relative precedence +** of pNew and the existing nodes of the tree. This may result in the head +** of the tree changing, in which case *ppHead is set to the new root node. +*/ +static void insertBinaryOperator( + Fts3Expr **ppHead, /* Pointer to the root node of a tree */ + Fts3Expr *pPrev, /* Node most recently inserted into the tree */ + Fts3Expr *pNew /* New binary node to insert into expression tree */ +){ + Fts3Expr *pSplit = pPrev; + while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ + pSplit = pSplit->pParent; + } + + if( pSplit->pParent ){ + assert( pSplit->pParent->pRight==pSplit ); + pSplit->pParent->pRight = pNew; + pNew->pParent = pSplit->pParent; + }else{ + *ppHead = pNew; + } + pNew->pLeft = pSplit; + pSplit->pParent = pNew; +} + +/* +** Parse the fts3 query expression found in buffer z, length n. This function +** returns either when the end of the buffer is reached or an unmatched +** closing bracket - ')' - is encountered. +** +** If successful, SQLITE_OK is returned, *ppExpr is set to point to the +** parsed form of the expression and *pnConsumed is set to the number of +** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM +** (out of memory error) or SQLITE_ERROR (parse error) is returned. +*/ +static int fts3ExprParse( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + Fts3Expr *pRet = 0; + Fts3Expr *pPrev = 0; + Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ + int nIn = n; + const char *zIn = z; + int rc = SQLITE_OK; + int isRequirePhrase = 1; + + while( rc==SQLITE_OK ){ + Fts3Expr *p = 0; + int nByte = 0; + + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); + if( rc==SQLITE_OK ){ + if( p ){ + int isPhrase; + + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; + }else{ + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); + + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; + } + + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( + (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) + || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } + + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } + }else{ + insertBinaryOperator(&pRet, pPrev, p); + } + isRequirePhrase = !isPhrase; + } + pPrev = p; + } + assert( nByte>0 ); + } + assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); + nIn -= nByte; + zIn += nByte; + } + + if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ + if( !pRet ){ + rc = SQLITE_ERROR; + }else{ + Fts3Expr *pIter = pNotBranch; + while( pIter->pLeft ){ + pIter = pIter->pLeft; + } + pIter->pLeft = pRet; + pRet->pParent = pIter; + pRet = pNotBranch; + } + } + } + *pnConsumed = n - nIn; + +exprparse_out: + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRet); + sqlite3Fts3ExprFree(pNotBranch); + pRet = 0; + } + *ppExpr = pRet; + return rc; +} + +/* +** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** as the only argument is more than nMaxDepth. +*/ +static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ + int rc = SQLITE_OK; + if( p ){ + if( nMaxDepth<0 ){ + rc = SQLITE_TOOBIG; + }else{ + rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); + } + } + } + return rc; +} + +/* +** This function attempts to transform the expression tree at (*pp) to +** an equivalent but more balanced form. The tree is modified in place. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. +** +** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** +** Otherwise, if an error occurs, an SQLite error code is returned and +** expression (*pp) freed. +*/ +static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ + int rc = SQLITE_OK; /* Return code */ + Fts3Expr *pRoot = *pp; /* Initial root node */ + Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ + int eType = pRoot->eType; /* Type of node in this tree */ + + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc64(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + } + + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + apLeaf[iLvl] = 0; + } + } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } + } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + }else if( eType==FTSQUERY_NOT ){ + Fts3Expr *pLeft = pRoot->pLeft; + Fts3Expr *pRight = pRoot->pRight; + + pRoot->pLeft = 0; + pRoot->pRight = 0; + pLeft->pParent = 0; + pRight->pParent = 0; + + rc = fts3ExprBalance(&pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprBalance(&pRight, nMaxDepth-1); + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRight); + sqlite3Fts3ExprFree(pLeft); + }else{ + assert( pLeft && pRight ); + pRoot->pLeft = pLeft; + pLeft->pParent = pRoot; + pRoot->pRight = pRight; + pRight->pParent = pRoot; + } + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRoot); + pRoot = 0; + } + *pp = pRoot; + return rc; +} + +/* +** This function is similar to sqlite3Fts3ExprParse(), with the following +** differences: +** +** 1. It does not do expression rebalancing. +** 2. It does not check that the expression does not exceed the +** maximum allowable depth. +** 3. Even if it fails, *ppExpr may still be set to point to an +** expression tree. It should be deleted using sqlite3Fts3ExprFree() +** in this case. +*/ +static int fts3ExprParseUnbalanced( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + int nParsed; + int rc; + ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); + sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.bFts4 = bFts4; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = (int)strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + assert( rc==SQLITE_OK || *ppExpr==0 ); + + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Parameters z and n contain a pointer to and length of a buffer containing +** an fts3 query expression, respectively. This function attempts to parse the +** query expression and create a tree of Fts3Expr structures representing the +** parsed expression. If successful, *ppExpr is set to point to the head +** of the parsed expression tree and SQLITE_OK is returned. If an error +** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse +** error) is returned and *ppExpr is set to 0. +** +** If parameter n is a negative number, then z is assumed to point to a +** nul-terminated string and the length is determined using strlen(). +** +** The first parameter, pTokenizer, is passed the fts3 tokenizer module to +** use to normalize query tokens while parsing the expression. The azCol[] +** array, which is assumed to contain nCol entries, should contain the names +** of each column in the target fts3 table, in order from left to right. +** Column names must be nul-terminated strings. +** +** The iDefaultCol parameter should be passed the index of the table column +** that appears on the left-hand-side of the MATCH operator (the default +** column to match against for tokens for which a column name is not explicitly +** specified as part of the query string), or -1 if tokens may by default +** match any table column. +*/ +SQLITE_PRIVATE int sqlite3Fts3ExprParse( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + char **pzErr /* OUT: Error message (sqlite3_malloc) */ +){ + int rc = fts3ExprParseUnbalanced( + pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr + ); + + /* Rebalance the expression. And check that its depth does not exceed + ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ + if( rc==SQLITE_OK && *ppExpr ){ + rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(*ppExpr); + *ppExpr = 0; + if( rc==SQLITE_TOOBIG ){ + sqlite3Fts3ErrMsg(pzErr, + "FTS expression tree is too large (maximum depth %d)", + SQLITE_FTS3_MAX_EXPR_DEPTH + ); + rc = SQLITE_ERROR; + }else if( rc==SQLITE_ERROR ){ + sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); + } + } + + return rc; +} + +/* +** Free a single node of an expression tree. +*/ +static void fts3FreeExprNode(Fts3Expr *p){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); + sqlite3_free(p); +} + +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** +** This function would be simpler if it recursively called itself. But +** that would mean passing a sufficiently large expression to ExprParse() +** could cause a stack overflow. +*/ +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ + Fts3Expr *p; + assert( pDel==0 || pDel->pParent==0 ); + for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ + assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); + } + while( p ){ + Fts3Expr *pParent = p->pParent; + fts3FreeExprNode(p); + if( pParent && p==pParent->pLeft && pParent->pRight ){ + p = pParent->pRight; + while( p && (p->pLeft || p->pRight) ){ + assert( p==p->pParent->pRight || p==p->pParent->pLeft ); + p = (p->pLeft ? p->pLeft : p->pRight); + } + }else{ + p = pParent; + } + } +} + +/**************************************************************************** +***************************************************************************** +** Everything after this point is just test code. +*/ + +#ifdef SQLITE_TEST + +/* #include */ + +/* +** Return a pointer to a buffer containing a text representation of the +** expression passed as the first argument. The buffer is obtained from +** sqlite3_malloc(). It is the responsibility of the caller to use +** sqlite3_free() to release the memory. If an OOM condition is encountered, +** NULL is returned. +** +** If the second argument is not NULL, then its contents are prepended to +** the returned expression text and then freed using sqlite3_free(). +*/ +static char *exprToString(Fts3Expr *pExpr, char *zBuf){ + if( pExpr==0 ){ + return sqlite3_mprintf(""); + } + switch( pExpr->eType ){ + case FTSQUERY_PHRASE: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + zBuf = sqlite3_mprintf( + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); + for(i=0; zBuf && inToken; i++){ + zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, + pPhrase->aToken[i].n, pPhrase->aToken[i].z, + (pPhrase->aToken[i].isPrefix?"+":"") + ); + } + return zBuf; + } + + case FTSQUERY_NEAR: + zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); + break; + case FTSQUERY_NOT: + zBuf = sqlite3_mprintf("%zNOT ", zBuf); + break; + case FTSQUERY_AND: + zBuf = sqlite3_mprintf("%zAND ", zBuf); + break; + case FTSQUERY_OR: + zBuf = sqlite3_mprintf("%zOR ", zBuf); + break; + } + + if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); + if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); + + if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); + + return zBuf; +} + +/* +** This is the implementation of a scalar SQL function used to test the +** expression parser. It should be called as follows: +** +** fts3_exprtest(, , , ...); +** +** The first argument, , is the name of the fts3 tokenizer used +** to parse the query expression (see README.tokenizers). The second argument +** is the query expression to parse. Each subsequent argument is the name +** of a column of the fts3 table that the query expression may refer to. +** For example: +** +** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); +*/ +static void fts3ExprTestCommon( + int bRebalance, + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_tokenizer *pTokenizer = 0; + int rc; + char **azCol = 0; + const char *zExpr; + int nExpr; + int nCol; + int ii; + Fts3Expr *pExpr; + char *zBuf = 0; + Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context); + const char *zTokenizer = 0; + char *zErr = 0; + + if( argc<3 ){ + sqlite3_result_error(context, + "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 + ); + return; + } + + zTokenizer = (const char*)sqlite3_value_text(argv[0]); + rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_error(context, zErr, -1); + } + sqlite3_free(zErr); + return; + } + + zExpr = (const char *)sqlite3_value_text(argv[1]); + nExpr = sqlite3_value_bytes(argv[1]); + nCol = argc-2; + azCol = (char **)sqlite3_malloc64(nCol*sizeof(char *)); + if( !azCol ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + for(ii=0; iipModule->xDestroy(pTokenizer); + } + sqlite3_free(azCol); +} + +static void fts3ExprTest( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fts3ExprTestCommon(0, context, argc, argv); +} +static void fts3ExprTestRebalance( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fts3ExprTestCommon(1, context, argc, argv); +} + +/* +** Register the query expression parser test function fts3_exprtest() +** with database connection db. +*/ +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){ + int rc = sqlite3_create_function( + db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0 + ); + } + return rc; +} + +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_expr.c *******************************************/ +/************** Begin file fts3_hash.c ***************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the implementation of generic hash-tables used in SQLite. +** We've modified it slightly to serve as a standalone hash table +** implementation for the full-text indexing module. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_hash.h" */ + +/* +** Malloc and Free functions +*/ +static void *fts3HashMalloc(sqlite3_int64 n){ + void *p = sqlite3_malloc64(n); + if( p ){ + memset(p, 0, n); + } + return p; +} +static void fts3HashFree(void *p){ + sqlite3_free(p); +} + +/* Turn bulk memory into a hash table object by initializing the +** fields of the Hash structure. +** +** "pNew" is a pointer to the hash table that is to be initialized. +** keyClass is one of the constants +** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass +** determines what kind of key the hash table will use. "copyKey" is +** true if the hash table should make its own private copy of keys and +** false if it should just use the supplied pointer. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ + assert( pNew!=0 ); + assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); + pNew->keyClass = keyClass; + pNew->copyKey = copyKey; + pNew->first = 0; + pNew->count = 0; + pNew->htsize = 0; + pNew->ht = 0; +} + +/* Remove all entries from a hash table. Reclaim all memory. +** Call this routine to delete a hash table or to reset a hash table +** to the empty state. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){ + Fts3HashElem *elem; /* For looping over all elements of the table */ + + assert( pH!=0 ); + elem = pH->first; + pH->first = 0; + fts3HashFree(pH->ht); + pH->ht = 0; + pH->htsize = 0; + while( elem ){ + Fts3HashElem *next_elem = elem->next; + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree(elem); + elem = next_elem; + } + pH->count = 0; +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_STRING +*/ +static int fts3StrHash(const void *pKey, int nKey){ + const char *z = (const char *)pKey; + unsigned h = 0; + if( nKey<=0 ) nKey = (int) strlen(z); + while( nKey > 0 ){ + h = (h<<3) ^ h ^ *z++; + nKey--; + } + return (int)(h & 0x7fffffff); +} +static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return strncmp((const char*)pKey1,(const char*)pKey2,n1); +} + +/* +** Hash and comparison functions when the mode is FTS3_HASH_BINARY +*/ +static int fts3BinHash(const void *pKey, int nKey){ + int h = 0; + const char *z = (const char *)pKey; + while( nKey-- > 0 ){ + h = (h<<3) ^ h ^ *(z++); + } + return h & 0x7fffffff; +} +static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return memcmp(pKey1,pKey2,n1); +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** The C syntax in this function definition may be unfamilar to some +** programmers, so we provide the following additional explanation: +** +** The name of the function is "ftsHashFunction". The function takes a +** single parameter "keyClass". The return value of ftsHashFunction() +** is a pointer to another function. Specifically, the return value +** of ftsHashFunction() is a pointer to a function that takes two parameters +** with types "const void*" and "int" and returns an "int". +*/ +static int (*ftsHashFunction(int keyClass))(const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrHash; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinHash; + } +} + +/* +** Return a pointer to the appropriate hash function given the key class. +** +** For help in interpreted the obscure C code in the function definition, +** see the header comment on the previous function. +*/ +static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrCompare; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinCompare; + } +} + +/* Link an element into the hash table +*/ +static void fts3HashInsertElement( + Fts3Hash *pH, /* The complete hash table */ + struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ + Fts3HashElem *pNew /* The element to be inserted */ +){ + Fts3HashElem *pHead; /* First element already in pEntry */ + pHead = pEntry->chain; + if( pHead ){ + pNew->next = pHead; + pNew->prev = pHead->prev; + if( pHead->prev ){ pHead->prev->next = pNew; } + else { pH->first = pNew; } + pHead->prev = pNew; + }else{ + pNew->next = pH->first; + if( pH->first ){ pH->first->prev = pNew; } + pNew->prev = 0; + pH->first = pNew; + } + pEntry->count++; + pEntry->chain = pNew; +} + + +/* Resize the hash table so that it cantains "new_size" buckets. +** "new_size" must be a power of 2. The hash table might fail +** to resize if sqliteMalloc() fails. +** +** Return non-zero if a memory allocation error occurs. +*/ +static int fts3Rehash(Fts3Hash *pH, int new_size){ + struct _fts3ht *new_ht; /* The new hash table */ + Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( (new_size & (new_size-1))==0 ); + new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); + if( new_ht==0 ) return 1; + fts3HashFree(pH->ht); + pH->ht = new_ht; + pH->htsize = new_size; + xHash = ftsHashFunction(pH->keyClass); + for(elem=pH->first, pH->first=0; elem; elem = next_elem){ + int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); + next_elem = elem->next; + fts3HashInsertElement(pH, &new_ht[h], elem); + } + return 0; +} + +/* This function (for internal use only) locates an element in an +** hash table that matches the given key. The hash for this key has +** already been computed and is passed as the 4th parameter. +*/ +static Fts3HashElem *fts3FindElementByHash( + const Fts3Hash *pH, /* The pH to be searched */ + const void *pKey, /* The key we are searching for */ + int nKey, + int h /* The hash for this key. */ +){ + Fts3HashElem *elem; /* Used to loop thru the element list */ + int count; /* Number of elements left to test */ + int (*xCompare)(const void*,int,const void*,int); /* comparison function */ + + if( pH->ht ){ + struct _fts3ht *pEntry = &pH->ht[h]; + elem = pEntry->chain; + count = pEntry->count; + xCompare = ftsCompareFunction(pH->keyClass); + while( count-- && elem ){ + if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ + return elem; + } + elem = elem->next; + } + } + return 0; +} + +/* Remove a single entry from the hash table given a pointer to that +** element and a hash on the element's key. +*/ +static void fts3RemoveElementByHash( + Fts3Hash *pH, /* The pH containing "elem" */ + Fts3HashElem* elem, /* The element to be removed from the pH */ + int h /* Hash value for the element */ +){ + struct _fts3ht *pEntry; + if( elem->prev ){ + elem->prev->next = elem->next; + }else{ + pH->first = elem->next; + } + if( elem->next ){ + elem->next->prev = elem->prev; + } + pEntry = &pH->ht[h]; + if( pEntry->chain==elem ){ + pEntry->chain = elem->next; + } + pEntry->count--; + if( pEntry->count<=0 ){ + pEntry->chain = 0; + } + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); + } + fts3HashFree( elem ); + pH->count--; + if( pH->count<=0 ){ + assert( pH->first==0 ); + assert( pH->count==0 ); + fts3HashClear(pH); + } +} + +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem( + const Fts3Hash *pH, + const void *pKey, + int nKey +){ + int h; /* A hash on key */ + int (*xHash)(const void*,int); /* The hash function */ + + if( pH==0 || pH->ht==0 ) return 0; + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + h = (*xHash)(pKey,nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); +} + +/* +** Attempt to locate an element of the hash table pH with a key +** that matches pKey,nKey. Return the data for this element if it is +** found, or NULL if there is no match. +*/ +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ + Fts3HashElem *pElem; /* The element that matches key (if any) */ + + pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); + return pElem ? pElem->data : 0; +} + +/* Insert an element into the hash table pH. The key is pKey,nKey +** and the data is "data". +** +** If no element exists with a matching key, then a new +** element is created. A copy of the key is made if the copyKey +** flag is set. NULL is returned. +** +** If another element already exists with the same key, then the +** new data replaces the old data and the old data is returned. +** The key is not copied in this instance. If a malloc fails, then +** the new data is returned and the hash table is unchanged. +** +** If the "data" parameter to this function is NULL, then the +** element corresponding to "key" is removed from the hash table. +*/ +SQLITE_PRIVATE void *sqlite3Fts3HashInsert( + Fts3Hash *pH, /* The hash table to insert into */ + const void *pKey, /* The key */ + int nKey, /* Number of bytes in the key */ + void *data /* The data */ +){ + int hraw; /* Raw hash value of the key */ + int h; /* the hash of the key modulo hash table size */ + Fts3HashElem *elem; /* Used to loop thru the element list */ + Fts3HashElem *new_elem; /* New element added to the pH */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( pH!=0 ); + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + hraw = (*xHash)(pKey, nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + elem = fts3FindElementByHash(pH,pKey,nKey,h); + if( elem ){ + void *old_data = elem->data; + if( data==0 ){ + fts3RemoveElementByHash(pH,elem,h); + }else{ + elem->data = data; + } + return old_data; + } + if( data==0 ) return 0; + if( (pH->htsize==0 && fts3Rehash(pH,8)) + || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) + ){ + pH->count = 0; + return data; + } + assert( pH->htsize>0 ); + new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); + if( new_elem==0 ) return data; + if( pH->copyKey && pKey!=0 ){ + new_elem->pKey = fts3HashMalloc( nKey ); + if( new_elem->pKey==0 ){ + fts3HashFree(new_elem); + return data; + } + memcpy((void*)new_elem->pKey, pKey, nKey); + }else{ + new_elem->pKey = (void*)pKey; + } + new_elem->nKey = nKey; + pH->count++; + assert( pH->htsize>0 ); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + fts3HashInsertElement(pH, &pH->ht[h], new_elem); + new_elem->data = data; + return 0; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_hash.c *******************************************/ +/************** Begin file fts3_porter.c *************************************/ +/* +** 2006 September 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Implementation of the full-text-search tokenizer that implements +** a Porter stemmer. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_tokenizer.h" */ + +/* +** Class derived from sqlite3_tokenizer +*/ +typedef struct porter_tokenizer { + sqlite3_tokenizer base; /* Base class */ +} porter_tokenizer; + +/* +** Class derived from sqlite3_tokenizer_cursor +*/ +typedef struct porter_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *zInput; /* input we are tokenizing */ + int nInput; /* size of the input */ + int iOffset; /* current position in zInput */ + int iToken; /* index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAllocated; /* space allocated to zToken buffer */ +} porter_tokenizer_cursor; + + +/* +** Create a new tokenizer instance. +*/ +static int porterCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + porter_tokenizer *t; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int porterDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is zInput[0..nInput-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int porterOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, int nInput, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + porter_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->zInput = zInput; + if( zInput==0 ){ + c->nInput = 0; + }else if( nInput<0 ){ + c->nInput = (int)strlen(zInput); + }else{ + c->nInput = nInput; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->zToken = NULL; /* no space allocated, yet. */ + c->nAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** porterOpen() above. +*/ +static int porterClose(sqlite3_tokenizer_cursor *pCursor){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + sqlite3_free(c->zToken); + sqlite3_free(c); + return SQLITE_OK; +} +/* +** Vowel or consonant +*/ +static const char cType[] = { + 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 2, 1 +}; + +/* +** isConsonant() and isVowel() determine if their first character in +** the string they point to is a consonant or a vowel, according +** to Porter ruls. +** +** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. +** 'Y' is a consonant unless it follows another consonant, +** in which case it is a vowel. +** +** In these routine, the letters are in reverse order. So the 'y' rule +** is that 'y' is a consonant unless it is followed by another +** consonent. +*/ +static int isVowel(const char*); +static int isConsonant(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return j; + return z[1]==0 || isVowel(z + 1); +} +static int isVowel(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return 1-j; + return isConsonant(z + 1); +} + +/* +** Let any sequence of one or more vowels be represented by V and let +** C be sequence of one or more consonants. Then every word can be +** represented as: +** +** [C] (VC){m} [V] +** +** In prose: A word is an optional consonant followed by zero or +** vowel-consonant pairs followed by an optional vowel. "m" is the +** number of vowel consonant pairs. This routine computes the value +** of m for the first i bytes of a word. +** +** Return true if the m-value for z is 1 or more. In other words, +** return true if z contains at least one vowel that is followed +** by a consonant. +** +** In this routine z[] is in reverse order. So we are really looking +** for an instance of a consonant followed by a vowel. +*/ +static int m_gt_0(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* Like mgt0 above except we are looking for a value of m which is +** exactly 1 +*/ +static int m_eq_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 1; + while( isConsonant(z) ){ z++; } + return *z==0; +} + +/* Like mgt0 above except we are looking for a value of m>1 instead +** or m>0 +*/ +static int m_gt_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if there is a vowel anywhere within z[0..n-1] +*/ +static int hasVowel(const char *z){ + while( isConsonant(z) ){ z++; } + return *z!=0; +} + +/* +** Return TRUE if the word ends in a double consonant. +** +** The text is reversed here. So we are really looking at +** the first two characters of z[]. +*/ +static int doubleConsonant(const char *z){ + return isConsonant(z) && z[0]==z[1]; +} + +/* +** Return TRUE if the word ends with three letters which +** are consonant-vowel-consonent and where the final consonant +** is not 'w', 'x', or 'y'. +** +** The word is reversed here. So we are really checking the +** first three letters and the first one cannot be in [wxy]. +*/ +static int star_oh(const char *z){ + return + isConsonant(z) && + z[0]!='w' && z[0]!='x' && z[0]!='y' && + isVowel(z+1) && + isConsonant(z+2); +} + +/* +** If the word ends with zFrom and xCond() is true for the stem +** of the word that preceeds the zFrom ending, then change the +** ending to zTo. +** +** The input word *pz and zFrom are both in reverse order. zTo +** is in normal order. +** +** Return TRUE if zFrom matches. Return FALSE if zFrom does not +** match. Not that TRUE is returned even if xCond() fails and +** no substitution occurs. +*/ +static int stem( + char **pz, /* The word being stemmed (Reversed) */ + const char *zFrom, /* If the ending matches this... (Reversed) */ + const char *zTo, /* ... change the ending to this (not reversed) */ + int (*xCond)(const char*) /* Condition that must be true */ +){ + char *z = *pz; + while( *zFrom && *zFrom==*z ){ z++; zFrom++; } + if( *zFrom!=0 ) return 0; + if( xCond && !xCond(z) ) return 1; + while( *zTo ){ + *(--z) = *(zTo++); + } + *pz = z; + return 1; +} + +/* +** This is the fallback stemmer used when the porter stemmer is +** inappropriate. The input word is copied into the output with +** US-ASCII case folding. If the input word is too long (more +** than 20 bytes if it contains no digits or more than 6 bytes if +** it contains digits) then word is truncated to 20 or 6 bytes +** by taking 10 or 3 bytes from the beginning and end. +*/ +static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ + int i, mx, j; + int hasDigit = 0; + for(i=0; i='A' && c<='Z' ){ + zOut[i] = c - 'A' + 'a'; + }else{ + if( c>='0' && c<='9' ) hasDigit = 1; + zOut[i] = c; + } + } + mx = hasDigit ? 3 : 10; + if( nIn>mx*2 ){ + for(j=mx, i=nIn-mx; i=(int)sizeof(zReverse)-7 ){ + /* The word is too big or too small for the porter stemmer. + ** Fallback to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + for(i=0, j=sizeof(zReverse)-6; i='A' && c<='Z' ){ + zReverse[j] = c + 'a' - 'A'; + }else if( c>='a' && c<='z' ){ + zReverse[j] = c; + }else{ + /* The use of a character not in [a-zA-Z] means that we fallback + ** to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + } + memset(&zReverse[sizeof(zReverse)-5], 0, 5); + z = &zReverse[j+1]; + + + /* Step 1a */ + if( z[0]=='s' ){ + if( + !stem(&z, "sess", "ss", 0) && + !stem(&z, "sei", "i", 0) && + !stem(&z, "ss", "ss", 0) + ){ + z++; + } + } + + /* Step 1b */ + z2 = z; + if( stem(&z, "dee", "ee", m_gt_0) ){ + /* Do nothing. The work was all in the test */ + }else if( + (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) + && z!=z2 + ){ + if( stem(&z, "ta", "ate", 0) || + stem(&z, "lb", "ble", 0) || + stem(&z, "zi", "ize", 0) ){ + /* Do nothing. The work was all in the test */ + }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ + z++; + }else if( m_eq_1(z) && star_oh(z) ){ + *(--z) = 'e'; + } + } + + /* Step 1c */ + if( z[0]=='y' && hasVowel(z+1) ){ + z[0] = 'i'; + } + + /* Step 2 */ + switch( z[1] ){ + case 'a': + if( !stem(&z, "lanoita", "ate", m_gt_0) ){ + stem(&z, "lanoit", "tion", m_gt_0); + } + break; + case 'c': + if( !stem(&z, "icne", "ence", m_gt_0) ){ + stem(&z, "icna", "ance", m_gt_0); + } + break; + case 'e': + stem(&z, "rezi", "ize", m_gt_0); + break; + case 'g': + stem(&z, "igol", "log", m_gt_0); + break; + case 'l': + if( !stem(&z, "ilb", "ble", m_gt_0) + && !stem(&z, "illa", "al", m_gt_0) + && !stem(&z, "iltne", "ent", m_gt_0) + && !stem(&z, "ile", "e", m_gt_0) + ){ + stem(&z, "ilsuo", "ous", m_gt_0); + } + break; + case 'o': + if( !stem(&z, "noitazi", "ize", m_gt_0) + && !stem(&z, "noita", "ate", m_gt_0) + ){ + stem(&z, "rota", "ate", m_gt_0); + } + break; + case 's': + if( !stem(&z, "msila", "al", m_gt_0) + && !stem(&z, "ssenevi", "ive", m_gt_0) + && !stem(&z, "ssenluf", "ful", m_gt_0) + ){ + stem(&z, "ssensuo", "ous", m_gt_0); + } + break; + case 't': + if( !stem(&z, "itila", "al", m_gt_0) + && !stem(&z, "itivi", "ive", m_gt_0) + ){ + stem(&z, "itilib", "ble", m_gt_0); + } + break; + } + + /* Step 3 */ + switch( z[0] ){ + case 'e': + if( !stem(&z, "etaci", "ic", m_gt_0) + && !stem(&z, "evita", "", m_gt_0) + ){ + stem(&z, "ezila", "al", m_gt_0); + } + break; + case 'i': + stem(&z, "itici", "ic", m_gt_0); + break; + case 'l': + if( !stem(&z, "laci", "ic", m_gt_0) ){ + stem(&z, "luf", "", m_gt_0); + } + break; + case 's': + stem(&z, "ssen", "", m_gt_0); + break; + } + + /* Step 4 */ + switch( z[1] ){ + case 'a': + if( z[0]=='l' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'c': + if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'e': + if( z[0]=='r' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'i': + if( z[0]=='c' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'l': + if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'n': + if( z[0]=='t' ){ + if( z[2]=='a' ){ + if( m_gt_1(z+3) ){ + z += 3; + } + }else if( z[2]=='e' ){ + if( !stem(&z, "tneme", "", m_gt_1) + && !stem(&z, "tnem", "", m_gt_1) + ){ + stem(&z, "tne", "", m_gt_1); + } + } + } + break; + case 'o': + if( z[0]=='u' ){ + if( m_gt_1(z+2) ){ + z += 2; + } + }else if( z[3]=='s' || z[3]=='t' ){ + stem(&z, "noi", "", m_gt_1); + } + break; + case 's': + if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 't': + if( !stem(&z, "eta", "", m_gt_1) ){ + stem(&z, "iti", "", m_gt_1); + } + break; + case 'u': + if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 'v': + case 'z': + if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + } + + /* Step 5a */ + if( z[0]=='e' ){ + if( m_gt_1(z+1) ){ + z++; + }else if( m_eq_1(z+1) && !star_oh(z+1) ){ + z++; + } + } + + /* Step 5b */ + if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ + z++; + } + + /* z[] is now the stemmed word in reverse order. Flip it back + ** around into forward order and return. + */ + *pnOut = i = (int)strlen(z); + zOut[i] = 0; + while( *z ){ + zOut[--i] = *(z++); + } +} + +/* +** Characters that can be part of a token. We assume any character +** whose value is greater than 0x80 (any UTF character) can be +** part of a token. In other words, delimiters all must have +** values of 0x7f or lower. +*/ +static const char porterIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ +}; +#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30])) + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to porterOpen(). +*/ +static int porterNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ + const char **pzToken, /* OUT: *pzToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + const char *z = c->zInput; + + while( c->iOffsetnInput ){ + int iStartOffset, ch; + + /* Scan past delimiter characters */ + while( c->iOffsetnInput && isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnInput && !isDelim(z[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int n = c->iOffset-iStartOffset; + if( n>c->nAllocated ){ + char *pNew; + c->nAllocated = n+20; + pNew = sqlite3_realloc(c->zToken, c->nAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->zToken = pNew; + } + porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); + *pzToken = c->zToken; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the porter-stemmer tokenizer +*/ +static const sqlite3_tokenizer_module porterTokenizerModule = { + 0, + porterCreate, + porterDestroy, + porterOpen, + porterClose, + porterNext, + 0 +}; + +/* +** Allocate a new porter tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &porterTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_porter.c *****************************************/ +/************** Begin file fts3_tokenizer.c **********************************/ +/* +** 2007 June 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is part of an SQLite module implementing full-text search. +** This particular file implements the generic tokenizer interface. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +/* +** Return true if the two-argument version of fts3_tokenizer() +** has been activated via a prior call to sqlite3_db_config(db, +** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); +*/ +static int fts3TokenizerEnabled(sqlite3_context *context){ + sqlite3 *db = sqlite3_context_db_handle(context); + int isEnabled = 0; + sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled); + return isEnabled; +} + +/* +** Implementation of the SQL scalar function for accessing the underlying +** hash table. This function may be called as follows: +** +** SELECT (); +** SELECT (, ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). +** +** If the argument is specified, it must be a blob value +** containing a pointer to be stored as the hash data corresponding +** to the string . If is not specified, then +** the string must already exist in the has table. Otherwise, +** an error is returned. +** +** Whether or not the argument is specified, the value returned +** is a blob containing the pointer stored as the hash data corresponding +** to string (after the hash-table is updated, if applicable). +*/ +static void fts3TokenizerFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + void *pPtr = 0; + const unsigned char *zName; + int nName; + + assert( argc==1 || argc==2 ); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + + zName = sqlite3_value_text(argv[0]); + nName = sqlite3_value_bytes(argv[0])+1; + + if( argc==2 ){ + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[1]) ){ + void *pOld; + int n = sqlite3_value_bytes(argv[1]); + if( zName==0 || n!=sizeof(pPtr) ){ + sqlite3_result_error(context, "argument type mismatch", -1); + return; + } + pPtr = *(void **)sqlite3_value_blob(argv[1]); + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); + if( pOld==pPtr ){ + sqlite3_result_error(context, "out of memory", -1); + } + }else{ + sqlite3_result_error(context, "fts3tokenize disabled", -1); + return; + } + }else{ + if( zName ){ + pPtr = sqlite3Fts3HashFind(pHash, zName, nName); + } + if( !pPtr ){ + char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + } + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){ + sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); + } +} + +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ + static const char isFtsIdChar[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ + }; + return (c&0x80 || isFtsIdChar[(int)(c)]); +} + +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ + const char *z1; + const char *z2 = 0; + + /* Find the start of the next token. */ + z1 = zStr; + while( z2==0 ){ + char c = *z1; + switch( c ){ + case '\0': return 0; /* No more tokens here */ + case '\'': + case '"': + case '`': { + z2 = z1; + while( *++z2 && (*z2!=c || *++z2==c) ); + break; + } + case '[': + z2 = &z1[1]; + while( *z2 && z2[0]!=']' ) z2++; + if( *z2 ) z2++; + break; + + default: + if( sqlite3Fts3IsIdChar(*z1) ){ + z2 = &z1[1]; + while( sqlite3Fts3IsIdChar(*z2) ) z2++; + }else{ + z1++; + } + } + } + + *pn = (int)(z2-z1); + return z1; +} + +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( + Fts3Hash *pHash, /* Tokenizer hash table */ + const char *zArg, /* Tokenizer name */ + sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ + char **pzErr /* OUT: Set to malloced error message */ +){ + int rc; + char *z = (char *)zArg; + int n = 0; + char *zCopy; + char *zEnd; /* Pointer to nul-term of zCopy */ + sqlite3_tokenizer_module *m; + + zCopy = sqlite3_mprintf("%s", zArg); + if( !zCopy ) return SQLITE_NOMEM; + zEnd = &zCopy[strlen(zCopy)]; + + z = (char *)sqlite3Fts3NextToken(zCopy, &n); + if( z==0 ){ + assert( n==0 ); + z = zCopy; + } + z[n] = '\0'; + sqlite3Fts3Dequote(z); + + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); + if( !m ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z); + rc = SQLITE_ERROR; + }else{ + char const **aArg = 0; + int iArg = 0; + z = &z[n+1]; + while( zxCreate(iArg, aArg, ppTok); + assert( rc!=SQLITE_OK || *ppTok ); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer"); + }else{ + (*ppTok)->pModule = m; + } + sqlite3_free((void *)aArg); + } + + sqlite3_free(zCopy); + return rc; +} + + +#ifdef SQLITE_TEST + +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +#endif +/* #include */ + +/* +** Implementation of a special SQL scalar function for testing tokenizers +** designed to be used in concert with the Tcl testing framework. This +** function must be called with two or more arguments: +** +** SELECT (, ..., ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') +** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). +** +** The return value is a string that may be interpreted as a Tcl +** list. For each token in the , three elements are +** added to the returned list. The first is the token position, the +** second is the token text (folded, stemmed, etc.) and the third is the +** substring of associated with the token. For example, +** using the built-in "simple" tokenizer: +** +** SELECT fts_tokenizer_test('simple', 'I don't see how'); +** +** will return the string: +** +** "{0 i I 1 dont don't 2 see see 3 how how}" +** +*/ +static void testFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + sqlite3_tokenizer_module *p; + sqlite3_tokenizer *pTokenizer = 0; + sqlite3_tokenizer_cursor *pCsr = 0; + + const char *zErr = 0; + + const char *zName; + int nName; + const char *zInput; + int nInput; + + const char *azArg[64]; + + const char *zToken; + int nToken = 0; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int i; + + Tcl_Obj *pRet; + + if( argc<2 ){ + sqlite3_result_error(context, "insufficient arguments", -1); + return; + } + + nName = sqlite3_value_bytes(argv[0]); + zName = (const char *)sqlite3_value_text(argv[0]); + nInput = sqlite3_value_bytes(argv[argc-1]); + zInput = (const char *)sqlite3_value_text(argv[argc-1]); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + + if( !p ){ + char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr2, -1); + sqlite3_free(zErr2); + return; + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + + for(i=1; ixCreate(argc-2, azArg, &pTokenizer) ){ + zErr = "error in xCreate()"; + goto finish; + } + pTokenizer->pModule = p; + if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ + zErr = "error in xOpen()"; + goto finish; + } + + while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ + Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + zToken = &zInput[iStart]; + nToken = iEnd-iStart; + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + } + + if( SQLITE_OK!=p->xClose(pCsr) ){ + zErr = "error in xClose()"; + goto finish; + } + if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ + zErr = "error in xDestroy()"; + goto finish; + } + +finish: + if( zErr ){ + sqlite3_result_error(context, zErr, -1); + }else{ + sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); + } + Tcl_DecrRefCount(pRet); +} + +static +int registerTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module *p +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; + + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); + sqlite3_step(pStmt); + + return sqlite3_finalize(pStmt); +} + + +static +int queryTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB + && sqlite3_column_bytes(pStmt, 0)==sizeof(*pp) + ){ + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); + +/* +** Implementation of the scalar function fts3_tokenizer_internal_test(). +** This function is used for testing only, it is not included in the +** build unless SQLITE_TEST is defined. +** +** The purpose of this is to test that the fts3_tokenizer() function +** can be used as designed by the C-code in the queryTokenizer and +** registerTokenizer() functions above. These two functions are repeated +** in the README.tokenizer file as an example, so it is important to +** test them. +** +** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar +** function with no arguments. An assert() will fail if a problem is +** detected. i.e.: +** +** SELECT fts3_tokenizer_internal_test(); +** +*/ +static void intTestFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int rc; + const sqlite3_tokenizer_module *p1; + const sqlite3_tokenizer_module *p2; + sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + + /* Test the query function */ + sqlite3Fts3SimpleTokenizerModule(&p1); + rc = queryTokenizer(db, "simple", &p2); + assert( rc==SQLITE_OK ); + assert( p1==p2 ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_ERROR ); + assert( p2==0 ); + assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); + + /* Test the storage function */ + if( fts3TokenizerEnabled(context) ){ + rc = registerTokenizer(db, "nosuchtokenizer", p1); + assert( rc==SQLITE_OK ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_OK ); + assert( p2==p1 ); + } + + sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); +} + +#endif + +/* +** Set up SQL objects in database db used to access the contents of +** the hash table pointed to by argument pHash. The hash table must +** been initialized to use string keys, and to take a private copy +** of the key when a value is inserted. i.e. by a call similar to: +** +** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); +** +** This function adds a scalar function (see header comment above +** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is +** defined at compilation time, a temporary virtual table (see header +** comment above struct HashTableVtab) to the database schema. Both +** provide read/write access to the contents of *pHash. +** +** The third argument to this function, zName, is used as the name +** of both the scalar and, if created, the virtual table. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitHashTable( + sqlite3 *db, + Fts3Hash *pHash, + const char *zName +){ + int rc = SQLITE_OK; + void *p = (void *)pHash; + const int any = SQLITE_UTF8|SQLITE_DIRECTONLY; + +#ifdef SQLITE_TEST + char *zTest = 0; + char *zTest2 = 0; + void *pdb = (void *)db; + zTest = sqlite3_mprintf("%s_test", zName); + zTest2 = sqlite3_mprintf("%s_internal_test", zName); + if( !zTest || !zTest2 ){ + rc = SQLITE_NOMEM; + } +#endif + + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0); + } +#ifdef SQLITE_TEST + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); + } +#endif + +#ifdef SQLITE_TEST + sqlite3_free(zTest); + sqlite3_free(zTest2); +#endif + + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenizer.c **************************************/ +/************** Begin file fts3_tokenizer1.c *********************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "simple" full-text-search tokenizer. +*/ + +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_tokenizer.h" */ + +typedef struct simple_tokenizer { + sqlite3_tokenizer base; + char delim[128]; /* flag ASCII delimiters */ +} simple_tokenizer; + +typedef struct simple_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *pInput; /* input we are tokenizing */ + int nBytes; /* size of the input */ + int iOffset; /* current position in pInput */ + int iToken; /* index of next token to be returned */ + char *pToken; /* storage for current token */ + int nTokenAllocated; /* space allocated to zToken buffer */ +} simple_tokenizer_cursor; + + +static int simpleDelim(simple_tokenizer *t, unsigned char c){ + return c<0x80 && t->delim[c]; +} +static int fts3_isalnum(int x){ + return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); +} + +/* +** Create a new tokenizer instance. +*/ +static int simpleCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + simple_tokenizer *t; + + t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + + /* TODO(shess) Delimiters need to remain the same from run to run, + ** else we need to reindex. One solution would be a meta-table to + ** track such information in the database, then we'd only want this + ** information on the initial create. + */ + if( argc>1 ){ + int i, n = (int)strlen(argv[1]); + for(i=0; i=0x80 ){ + sqlite3_free(t); + return SQLITE_ERROR; + } + t->delim[ch] = 1; + } + } else { + /* Mark non-alphanumeric ASCII characters as delimiters */ + int i; + for(i=1; i<0x80; i++){ + t->delim[i] = !fts3_isalnum(i) ? -1 : 0; + } + } + + *ppTokenizer = &t->base; + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int simpleOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *pInput, int nBytes, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + simple_tokenizer_cursor *c; + + UNUSED_PARAMETER(pTokenizer); + + c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->pInput = pInput; + if( pInput==0 ){ + c->nBytes = 0; + }else if( nBytes<0 ){ + c->nBytes = (int)strlen(pInput); + }else{ + c->nBytes = nBytes; + } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->pToken = NULL; /* no space allocated, yet. */ + c->nTokenAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + sqlite3_free(c->pToken); + sqlite3_free(c); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int simpleNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; + unsigned char *p = (unsigned char *)c->pInput; + + while( c->iOffsetnBytes ){ + int iStartOffset; + + /* Scan past delimiter characters */ + while( c->iOffsetnBytes && simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnBytes && !simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int i, n = c->iOffset-iStartOffset; + if( n>c->nTokenAllocated ){ + char *pNew; + c->nTokenAllocated = n+20; + pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->pToken = pNew; + } + for(i=0; ipToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); + } + *ppToken = c->pToken; + *pnBytes = n; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + + return SQLITE_OK; + } + } + return SQLITE_DONE; +} + +/* +** The set of routines that implement the simple tokenizer +*/ +static const sqlite3_tokenizer_module simpleTokenizerModule = { + 0, + simpleCreate, + simpleDestroy, + simpleOpen, + simpleClose, + simpleNext, + 0, +}; + +/* +** Allocate a new simple tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &simpleTokenizerModule; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenizer1.c *************************************/ +/************** Begin file fts3_tokenize_vtab.c ******************************/ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts3tokenize" virtual table module. +** An fts3tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts3tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +typedef struct Fts3tokTable Fts3tokTable; +typedef struct Fts3tokCursor Fts3tokCursor; + +/* +** Virtual table structure. +*/ +struct Fts3tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + const sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer *pTok; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts3tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + char *zInput; /* Input string */ + sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ + int iRowid; /* Current 'rowid' value */ + const char *zToken; /* Current 'token' value */ + int nToken; /* Size of zToken in bytes */ + int iStart; /* Current 'start' value */ + int iEnd; /* Current 'end' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Query FTS for the tokenizer implementation named zName. +*/ +static int fts3tokQueryTokenizer( + Fts3Hash *pHash, + const char *zName, + const sqlite3_tokenizer_module **pp, + char **pzErr +){ + sqlite3_tokenizer_module *p; + int nName = (int)strlen(zName); + + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + if( !p ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName); + return SQLITE_ERROR; + } + + *pp = p; + return SQLITE_OK; +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts3tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte = 0; + char **azDequote; + + for(i=0; i1 ) azArg = (const char * const *)&azDequote[1]; + rc = pMod->xCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); + } + + if( rc==SQLITE_OK ){ + pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(Fts3tokTable)); + pTab->pMod = pMod; + pTab->pTok = pTok; + *ppVtab = &pTab->base; + }else{ + if( pTok ){ + pMod->xDestroy(pTok); + } + } + + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3tokTable *pTab = (Fts3tokTable *)pVtab; + + pTab->pMod->xDestroy(pTab->pTok); + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + UNUSED_PARAMETER(pVTab); + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3tokCursor *pCsr; + UNUSED_PARAMETER(pVTab); + + pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts3tokOpenMethod(). +*/ +static void fts3tokResetCursor(Fts3tokCursor *pCsr){ + if( pCsr->pCsr ){ + Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); + pTab->pMod->xClose(pCsr->pCsr); + pCsr->pCsr = 0; + } + sqlite3_free(pCsr->zInput); + pCsr->zInput = 0; + pCsr->zToken = 0; + pCsr->nToken = 0; + pCsr->iStart = 0; + pCsr->iEnd = 0; + pCsr->iPos = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + fts3tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + int rc; /* Return code */ + + pCsr->iRowid++; + rc = pTab->pMod->xNext(pCsr->pCsr, + &pCsr->zToken, &pCsr->nToken, + &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos + ); + + if( rc!=SQLITE_OK ){ + fts3tokResetCursor(pCsr); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + fts3tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc64(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); + if( rc==SQLITE_OK ){ + pCsr->pCsr->pTokenizer = pTab->pTok; + } + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts3tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + return (pCsr->zToken==0); +} + +/* +** xColumn - Return a column value. +*/ +static int fts3tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pCsr->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pCsr->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pCsr->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ + static const sqlite3_module fts3tok_module = { + 0, /* iVersion */ + fts3tokConnectMethod, /* xCreate */ + fts3tokConnectMethod, /* xConnect */ + fts3tokBestIndexMethod, /* xBestIndex */ + fts3tokDisconnectMethod, /* xDisconnect */ + fts3tokDisconnectMethod, /* xDestroy */ + fts3tokOpenMethod, /* xOpen */ + fts3tokCloseMethod, /* xClose */ + fts3tokFilterMethod, /* xFilter */ + fts3tokNextMethod, /* xNext */ + fts3tokEofMethod, /* xEof */ + fts3tokColumnMethod, /* xColumn */ + fts3tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenize_vtab.c **********************************/ +/************** Begin file fts3_write.c **************************************/ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file is part of the SQLite FTS3 extension module. Specifically, +** this file contains code to insert, update and delete rows from FTS3 +** tables. It also contains code to merge FTS3 b-tree segments. Some +** of the sub-routines used to merge segments are also used by the query +** code in fts3.c. +*/ + +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +#define FTS_MAX_APPENDABLE_HEIGHT 16 + +/* +** When full-text index nodes are loaded from disk, the buffer that they +** are loaded into has the following number of bytes of padding at the end +** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer +** of 920 bytes is allocated for it. +** +** This means that if we have a pointer into a buffer containing node data, +** it is always safe to read up to two varints from it without risking an +** overread, even if the node data is corrupted. +*/ +#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) + +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. +*/ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif + +/* +** The values that may be meaningfully bound to the :1 parameter in +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. +*/ +#define FTS_STAT_DOCTOTAL 0 +#define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 + +/* +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic +** and incremental merge operation that takes place. This is used for +** debugging FTS only, it should not usually be turned on in production +** systems. +*/ +#ifdef FTS3_LOG_MERGES +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ + sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); +} +#else +#define fts3LogMerge(x, y) +#endif + + +typedef struct PendingList PendingList; +typedef struct SegmentNode SegmentNode; +typedef struct SegmentWriter SegmentWriter; + +/* +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. +*/ +struct PendingList { + int nData; + char *aData; + int nSpace; + sqlite3_int64 iLastDocid; + sqlite3_int64 iLastCol; + sqlite3_int64 iLastPos; +}; + + +/* +** Each cursor has a (possibly empty) linked list of the following objects. +*/ +struct Fts3DeferredToken { + Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ + int iCol; /* Column token must occur in */ + Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ + PendingList *pList; /* Doclist is assembled here */ +}; + +/* +** An instance of this structure is used to iterate through the terms on +** a contiguous set of segment b-tree leaf nodes. Although the details of +** this structure are only manipulated by code in this file, opaque handles +** of type Fts3SegReader* are also used by code in fts3.c to iterate through +** terms when querying the full-text index. See functions: +** +** sqlite3Fts3SegReaderNew() +** sqlite3Fts3SegReaderFree() +** sqlite3Fts3SegReaderIterate() +** +** Methods used to manipulate Fts3SegReader structures: +** +** fts3SegReaderNext() +** fts3SegReaderFirstDocid() +** fts3SegReaderNextDocid() +*/ +struct Fts3SegReader { + int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ + u8 bLookup; /* True for a lookup only */ + u8 rootOnly; /* True for a root-only reader */ + + sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ + sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ + sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ + sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ + + char *aNode; /* Pointer to node data (or NULL) */ + int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + + Fts3HashElem **ppNextElem; + + /* Variables set by fts3SegReaderNext(). These may be read directly + ** by the caller. They are valid from the time SegmentReaderNew() returns + ** until SegmentReaderNext() returns something other than SQLITE_OK + ** (i.e. SQLITE_DONE). + */ + int nTerm; /* Number of bytes in current term */ + char *zTerm; /* Pointer to current term */ + int nTermAlloc; /* Allocated size of zTerm buffer */ + char *aDoclist; /* Pointer to doclist of current entry */ + int nDoclist; /* Size of doclist in current entry */ + + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ + char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ + sqlite3_int64 iDocid; +}; + +#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) +#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0) + +/* +** An instance of this structure is used to create a segment b-tree in the +** database. The internal details of this type are only accessed by the +** following functions: +** +** fts3SegWriterAdd() +** fts3SegWriterFlush() +** fts3SegWriterFree() +*/ +struct SegmentWriter { + SegmentNode *pTree; /* Pointer to interior tree structure */ + sqlite3_int64 iFirst; /* First slot in %_segments written */ + sqlite3_int64 iFree; /* Next free slot in %_segments */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nSize; /* Size of allocation at aData */ + int nData; /* Bytes of data in aData */ + char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ +}; + +/* +** Type SegmentNode is used by the following three functions to create +** the interior part of the segment b+-tree structures (everything except +** the leaf nodes). These functions and type are only ever used by code +** within the fts3SegWriterXXX() family of functions described above. +** +** fts3NodeAddTerm() +** fts3NodeWrite() +** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. +*/ +struct SegmentNode { + SegmentNode *pParent; /* Parent node (or NULL for root node) */ + SegmentNode *pRight; /* Pointer to right-sibling */ + SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ + int nEntry; /* Number of terms written to node so far */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nData; /* Bytes of valid data so far */ + char *aData; /* Node data */ +}; + +/* +** Valid values for the second argument to fts3SqlStmt(). +*/ +#define SQL_DELETE_CONTENT 0 +#define SQL_IS_EMPTY 1 +#define SQL_DELETE_ALL_CONTENT 2 +#define SQL_DELETE_ALL_SEGMENTS 3 +#define SQL_DELETE_ALL_SEGDIR 4 +#define SQL_DELETE_ALL_DOCSIZE 5 +#define SQL_DELETE_ALL_STAT 6 +#define SQL_SELECT_CONTENT_BY_ROWID 7 +#define SQL_NEXT_SEGMENT_INDEX 8 +#define SQL_INSERT_SEGMENTS 9 +#define SQL_NEXT_SEGMENTS_ID 10 +#define SQL_INSERT_SEGDIR 11 +#define SQL_SELECT_LEVEL 12 +#define SQL_SELECT_LEVEL_RANGE 13 +#define SQL_SELECT_LEVEL_COUNT 14 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 +#define SQL_DELETE_SEGDIR_LEVEL 16 +#define SQL_DELETE_SEGMENTS_RANGE 17 +#define SQL_CONTENT_INSERT 18 +#define SQL_DELETE_DOCSIZE 19 +#define SQL_REPLACE_DOCSIZE 20 +#define SQL_SELECT_DOCSIZE 21 +#define SQL_SELECT_STAT 22 +#define SQL_REPLACE_STAT 23 + +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 +#define SQL_DELETE_SEGDIR_RANGE 26 +#define SQL_SELECT_ALL_LANGID 27 +#define SQL_FIND_MERGE_LEVEL 28 +#define SQL_MAX_LEAF_NODE_ESTIMATE 29 +#define SQL_DELETE_SEGDIR_ENTRY 30 +#define SQL_SHIFT_SEGDIR_ENTRY 31 +#define SQL_SELECT_SEGDIR 32 +#define SQL_CHOMP_SEGDIR 33 +#define SQL_SEGMENT_IS_APPENDABLE 34 +#define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 + +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + +/* +** This function is used to obtain an SQLite prepared statement handle +** for the statement identified by the second argument. If successful, +** *pp is set to the requested statement handle and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned and *pp is set to 0. +** +** If argument apVal is not NULL, then it must point to an array with +** at least as many entries as the requested statement has bound +** parameters. The values are bound to the statements parameters before +** returning. +*/ +static int fts3SqlStmt( + Fts3Table *p, /* Virtual table handle */ + int eStmt, /* One of the SQL_XXX constants above */ + sqlite3_stmt **pp, /* OUT: Statement handle */ + sqlite3_value **apVal /* Values to bind to statement */ +){ + const char *azSql[] = { +/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", +/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", +/* 2 */ "DELETE FROM %Q.'%q_content'", +/* 3 */ "DELETE FROM %Q.'%q_segments'", +/* 4 */ "DELETE FROM %Q.'%q_segdir'", +/* 5 */ "DELETE FROM %Q.'%q_docsize'", +/* 6 */ "DELETE FROM %Q.'%q_stat'", +/* 7 */ "SELECT %s WHERE rowid=?", +/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", +/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", +/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", + + /* Return segments in order from oldest to newest.*/ +/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", +/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", + +/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + +/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", +/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", +/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", +/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", +/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", +/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", +/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", + +/* This statement is used to determine which level to read the input from +** when performing an incremental merge. It returns the absolute level number +** of the oldest level in the db that contains at least ? segments. Or, +** if no level in the FTS index contains more than ? segments, the statement +** returns zero rows. */ +/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " + " GROUP BY level HAVING cnt>=?" + " ORDER BY (level %% 1024) ASC, 2 DESC LIMIT 1", + +/* Estimate the upper limit on the number of leaf nodes in a new segment +** created by merging the oldest :2 segments from absolute level :1. See +** function sqlite3Fts3Incrmerge() for details. */ +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " + " FROM (SELECT * FROM %Q.'%q_segdir' " + " WHERE level = ? ORDER BY idx ASC LIMIT ?" + " )", + +/* SQL_DELETE_SEGDIR_ENTRY +** Delete the %_segdir entry on absolute level :1 with index :2. */ +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_SHIFT_SEGDIR_ENTRY +** Modify the idx value for the segment with idx=:3 on absolute level :2 +** to :1. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", + +/* SQL_SELECT_SEGDIR +** Read a single entry from the %_segdir table. The entry from absolute +** level :1 with index value :2. */ +/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_CHOMP_SEGDIR +** Update the start_block (:1) and root (:2) fields of the %_segdir +** entry located on absolute level :3 with index :4. */ +/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" + "WHERE level = ? AND idx = ?", + +/* SQL_SEGMENT_IS_APPENDABLE +** Return a single row if the segment with end_block=? is appendable. Or +** no rows otherwise. */ +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + +/* SQL_SELECT_INDEXES +** Return the list of valid segment indexes for absolute level ? */ +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + + }; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt; + + assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); + assert( eStmt=0 ); + + pStmt = p->aStmt[eStmt]; + if( !pStmt ){ + int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB; + char *zSql; + if( eStmt==SQL_CONTENT_INSERT ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); + }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ + f &= ~SQLITE_PREPARE_NO_VTAB; + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); + }else{ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); + } + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + p->aStmt[eStmt] = pStmt; + } + } + if( apVal ){ + int i; + int nParam = sqlite3_bind_parameter_count(pStmt); + for(i=0; rc==SQLITE_OK && inPendingData==0 ){ + sqlite3_stmt *pStmt; + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } + } + + return rc; +} + +/* +** FTS maintains a separate indexes for each language-id (a 32-bit integer). +** Within each language id, a separate index is maintained to store the +** document terms, and each configured prefix size (configured the FTS +** "prefix=" option). And each index consists of multiple levels ("relative +** levels"). +** +** All three of these values (the language id, the specific index and the +** level within the index) are encoded in 64-bit integer values stored +** in the %_segdir table on disk. This function is used to convert three +** separate component values into the single 64-bit integer value that +** can be used to query the %_segdir table. +** +** Specifically, each language-id/index combination is allocated 1024 +** 64-bit integer level values ("absolute levels"). The main terms index +** for language-id 0 is allocate values 0-1023. The first prefix index +** (if any) for language-id 0 is allocated values 1024-2047. And so on. +** Language 1 indexes are allocated immediately following language 0. +** +** So, for a system with nPrefix prefix indexes configured, the block of +** absolute levels that corresponds to language-id iLangid and index +** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). +*/ +static sqlite3_int64 getAbsoluteLevel( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index in p->aIndex[] */ + int iLevel /* Level of segments */ +){ + sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ + assert_fts3_nc( iLangid>=0 ); + assert( p->nIndex>0 ); + assert( iIndex>=0 && iIndexnIndex ); + + iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; + return iBase + iLevel; +} + +/* +** Set *ppStmt to a statement handle that may be used to iterate through +** all rows in the %_segdir table, from oldest to newest. If successful, +** return SQLITE_OK. If an error occurs while preparing the statement, +** return an SQLite error code. +** +** There is only ever one instance of this SQL statement compiled for +** each FTS3 table. +** +** The statement returns the following columns from the %_segdir table: +** +** 0: idx +** 1: start_block +** 2: leaves_end_block +** 3: end_block +** 4: root +*/ +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iLangid, /* Language being queried */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select (relative level) */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ + int rc; + sqlite3_stmt *pStmt = 0; + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + + if( iLevel<0 ){ + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); + } + } + *ppStmt = pStmt; + return rc; +} + + +/* +** Append a single varint to a PendingList buffer. SQLITE_OK is returned +** if successful, or an SQLite error code otherwise. +** +** This function also serves to allocate the PendingList structure itself. +** For example, to create a new PendingList structure containing two +** varints: +** +** PendingList *p = 0; +** fts3PendingListAppendVarint(&p, 1); +** fts3PendingListAppendVarint(&p, 2); +*/ +static int fts3PendingListAppendVarint( + PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ + sqlite3_int64 i /* Value to append to data */ +){ + PendingList *p = *pp; + + /* Allocate or grow the PendingList as required. */ + if( !p ){ + p = sqlite3_malloc(sizeof(*p) + 100); + if( !p ){ + return SQLITE_NOMEM; + } + p->nSpace = 100; + p->aData = (char *)&p[1]; + p->nData = 0; + } + else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ + int nNew = p->nSpace * 2; + p = sqlite3_realloc(p, sizeof(*p) + nNew); + if( !p ){ + sqlite3_free(*pp); + *pp = 0; + return SQLITE_NOMEM; + } + p->nSpace = nNew; + p->aData = (char *)&p[1]; + } + + /* Append the new serialized varint to the end of the list. */ + p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); + p->aData[p->nData] = '\0'; + *pp = p; + return SQLITE_OK; +} + +/* +** Add a docid/column/position entry to a PendingList structure. Non-zero +** is returned if the structure is sqlite3_realloced as part of adding +** the entry. Otherwise, zero. +** +** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. +** Zero is always returned in this case. Otherwise, if no OOM error occurs, +** it is set to SQLITE_OK. +*/ +static int fts3PendingListAppend( + PendingList **pp, /* IN/OUT: PendingList structure */ + sqlite3_int64 iDocid, /* Docid for entry to add */ + sqlite3_int64 iCol, /* Column for entry to add */ + sqlite3_int64 iPos, /* Position of term for entry to add */ + int *pRc /* OUT: Return code */ +){ + PendingList *p = *pp; + int rc = SQLITE_OK; + + assert( !p || p->iLastDocid<=iDocid ); + + if( !p || p->iLastDocid!=iDocid ){ + u64 iDelta = (u64)iDocid - (u64)(p ? p->iLastDocid : 0); + if( p ){ + assert( p->nDatanSpace ); + assert( p->aData[p->nData]==0 ); + p->nData++; + } + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ + goto pendinglistappend_out; + } + p->iLastCol = -1; + p->iLastPos = 0; + p->iLastDocid = iDocid; + } + if( iCol>0 && p->iLastCol!=iCol ){ + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) + || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) + ){ + goto pendinglistappend_out; + } + p->iLastCol = iCol; + p->iLastPos = 0; + } + if( iCol>=0 ){ + assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); + rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); + if( rc==SQLITE_OK ){ + p->iLastPos = iPos; + } + } + + pendinglistappend_out: + *pRc = rc; + if( p!=*pp ){ + *pp = p; + return 1; + } + return 0; +} + +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + +/* +** Tokenize the nul-terminated string zText and add all tokens to the +** pending-terms hash-table. The docid used is that currently stored in +** p->iPrevDocid, and the column is specified by argument iCol. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3PendingTermsAdd( + Fts3Table *p, /* Table into which text will be inserted */ + int iLangid, /* Language id to use */ + const char *zText, /* Text of document to be inserted */ + int iCol, /* Column into which text is being inserted */ + u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ +){ + int rc; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int nWord = 0; + + char const *zToken; + int nToken = 0; + + sqlite3_tokenizer *pTokenizer = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr; + int (*xNext)(sqlite3_tokenizer_cursor *pCursor, + const char**,int*,int*,int*,int*); + + assert( pTokenizer && pModule ); + + /* If the user has inserted a NULL value, this function may be called with + ** zText==0. In this case, add zero token entries to the hash table and + ** return early. */ + if( zText==0 ){ + *pnWord = 0; + return SQLITE_OK; + } + + rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } + + xNext = pModule->xNext; + while( SQLITE_OK==rc + && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) + ){ + int i; + if( iPos>=nWord ) nWord = iPos+1; + + /* Positions cannot be negative; we use -1 as a terminator internally. + ** Tokens must have a non-zero length. + */ + if( iPos<0 || !zToken || nToken<=0 ){ + rc = SQLITE_ERROR; + break; + } + + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); + } + } + + pModule->xClose(pCsr); + *pnWord += nWord; + return (rc==SQLITE_DONE ? SQLITE_OK : rc); +} + +/* +** Calling this function indicates that subsequent calls to +** fts3PendingTermsAdd() are to add term/position-list pairs for the +** contents of the document with docid iDocid. +*/ +static int fts3PendingTermsDocid( + Fts3Table *p, /* Full-text table handle */ + int bDelete, /* True if this op is a delete */ + int iLangid, /* Language id of row being written */ + sqlite_int64 iDocid /* Docid of row being written */ +){ + assert( iLangid>=0 ); + assert( bDelete==1 || bDelete==0 ); + + /* TODO(shess) Explore whether partially flushing the buffer on + ** forced-flush would provide better performance. I suspect that if + ** we ordered the doclists by size and flushed the largest until the + ** buffer was half empty, that would let the less frequent terms + ** generate longer doclists. + */ + if( iDocidiPrevDocid + || (iDocid==p->iPrevDocid && p->bPrevDelete==0) + || p->iPrevLangid!=iLangid + || p->nPendingData>p->nMaxPendingData + ){ + int rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ) return rc; + } + p->iPrevDocid = iDocid; + p->iPrevLangid = iLangid; + p->bPrevDelete = bDelete; + return SQLITE_OK; +} + +/* +** Discard the contents of the pending-terms hash tables. +*/ +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); + } + fts3HashClear(pHash); + } + p->nPendingData = 0; +} + +/* +** This function is called by the xUpdate() method as part of an INSERT +** operation. It adds entries for each term in the new record to the +** pendingTerms hash table. +** +** Argument apVal is the same as the similarly named argument passed to +** fts3InsertData(). Parameter iDocid is the docid of the new row. +*/ +static int fts3InsertTerms( + Fts3Table *p, + int iLangid, + sqlite3_value **apVal, + u32 *aSz +){ + int i; /* Iterator variable */ + for(i=2; inColumn+2; i++){ + int iCol = i-2; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); + if( rc!=SQLITE_OK ){ + return rc; + } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); + } + } + return SQLITE_OK; +} + +/* +** This function is called by the xUpdate() method for an INSERT operation. +** The apVal parameter is passed a copy of the apVal argument passed by +** SQLite to the xUpdate() method. i.e: +** +** apVal[0] Not used for INSERT. +** apVal[1] rowid +** apVal[2] Left-most user-defined column +** ... +** apVal[p->nColumn+1] Right-most user-defined column +** apVal[p->nColumn+2] Hidden column with same name as table +** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +** apVal[p->nColumn+4] Hidden languageid column +*/ +static int fts3InsertData( + Fts3Table *p, /* Full-text table */ + sqlite3_value **apVal, /* Array of values to insert */ + sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ +){ + int rc; /* Return code */ + sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ + + if( p->zContentTbl ){ + sqlite3_value *pRowid = apVal[p->nColumn+3]; + if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ + pRowid = apVal[1]; + } + if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ + return SQLITE_CONSTRAINT; + } + *piDocid = sqlite3_value_int64(pRowid); + return SQLITE_OK; + } + + /* Locate the statement handle used to insert data into the %_content + ** table. The SQL for this statement is: + ** + ** INSERT INTO %_content VALUES(?, ?, ?, ...) + ** + ** The statement features N '?' variables, where N is the number of user + ** defined columns in the FTS3 table, plus one for the docid field. + */ + rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); + if( rc==SQLITE_OK && p->zLanguageid ){ + rc = sqlite3_bind_int( + pContentInsert, p->nColumn+2, + sqlite3_value_int(apVal[p->nColumn+4]) + ); + } + if( rc!=SQLITE_OK ) return rc; + + /* There is a quirk here. The users INSERT statement may have specified + ** a value for the "rowid" field, for the "docid" field, or for both. + ** Which is a problem, since "rowid" and "docid" are aliases for the + ** same value. For example: + ** + ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); + ** + ** In FTS3, this is an error. It is an error to specify non-NULL values + ** for both docid and some other rowid alias. + */ + if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ + if( SQLITE_NULL==sqlite3_value_type(apVal[0]) + && SQLITE_NULL!=sqlite3_value_type(apVal[1]) + ){ + /* A rowid/docid conflict. */ + return SQLITE_ERROR; + } + rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); + if( rc!=SQLITE_OK ) return rc; + } + + /* Execute the statement to insert the record. Set *piDocid to the + ** new docid value. + */ + sqlite3_step(pContentInsert); + rc = sqlite3_reset(pContentInsert); + + *piDocid = sqlite3_last_insert_rowid(p->db); + return rc; +} + + + +/* +** Remove all data from the FTS3 table. Clear the hash table containing +** pending terms. +*/ +static int fts3DeleteAll(Fts3Table *p, int bContent){ + int rc = SQLITE_OK; /* Return code */ + + /* Discard the contents of the pending-terms hash table. */ + sqlite3Fts3PendingTermsClear(p); + + /* Delete everything from the shadow tables. Except, leave %_content as + ** is if bContent is false. */ + assert( p->zContentTbl==0 || bContent==0 ); + if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); + } + if( p->bHasStat ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); + } + return rc; +} + +/* +** +*/ +static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ + int iLangid = 0; + if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); + return iLangid; +} + +/* +** The first element in the apVal[] array is assumed to contain the docid +** (an integer) of a row about to be deleted. Remove all terms from the +** full-text index. +*/ +static void fts3DeleteTerms( + int *pRC, /* Result code */ + Fts3Table *p, /* The FTS table to delete from */ + sqlite3_value *pRowid, /* The docid to be deleted */ + u32 *aSz, /* Sizes of deleted document written here */ + int *pbFound /* OUT: Set to true if row really does exist */ +){ + int rc; + sqlite3_stmt *pSelect; + + assert( *pbFound==0 ); + if( *pRC ) return; + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + int i; + int iLangid = langidFromSelect(p, pSelect); + i64 iDocid = sqlite3_column_int64(pSelect, 0); + rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); + for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ + int iCol = i-1; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); + } + } + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + *pRC = rc; + return; + } + *pbFound = 1; + } + rc = sqlite3_reset(pSelect); + }else{ + sqlite3_reset(pSelect); + } + *pRC = rc; +} + +/* +** Forward declaration to account for the circular dependency between +** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). +*/ +static int fts3SegmentMerge(Fts3Table *, int, int, int); + +/* +** This function allocates a new level iLevel index in the segdir table. +** Usually, indexes are allocated within a level sequentially starting +** with 0, so the allocated index is one greater than the value returned +** by: +** +** SELECT max(idx) FROM %_segdir WHERE level = :iLevel +** +** However, if there are already FTS3_MERGE_COUNT indexes at the requested +** level, they are merged into a single level (iLevel+1) segment and the +** allocated index is 0. +** +** If successful, *piIdx is set to the allocated index slot and SQLITE_OK +** returned. Otherwise, an SQLite error code is returned. +*/ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ + int rc; /* Return Code */ + sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ + int iNext = 0; /* Result of query pNextIdx */ + + assert( iLangid>=0 ); + assert( p->nIndex>=1 ); + + /* Set variable iNext to the next available segdir index at level iLevel. */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ + iNext = sqlite3_column_int(pNextIdx, 0); + } + rc = sqlite3_reset(pNextIdx); + } + + if( rc==SQLITE_OK ){ + /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already + ** full, merge all segments in level iLevel into a single iLevel+1 + ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, + ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. + */ + if( iNext>=MergeCount(p) ){ + fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); + rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); + *piIdx = 0; + }else{ + *piIdx = iNext; + } + } + + return rc; +} + +/* +** The %_segments table is declared as follows: +** +** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) +** +** This function reads data from a single row of the %_segments table. The +** specific row is identified by the iBlockid parameter. If paBlob is not +** NULL, then a buffer is allocated using sqlite3_malloc() and populated +** with the contents of the blob stored in the "block" column of the +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set +** to the size of the blob in bytes before returning. +** +** If an error occurs, or the table does not contain the specified row, +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If +** paBlob is non-NULL, then it is the responsibility of the caller to +** eventually free the returned buffer. +** +** This function may leave an open sqlite3_blob* handle in the +** Fts3Table.pSegments variable. This handle is reused by subsequent calls +** to this function. The handle may be closed by calling the +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy +** performance improvement, but the blob handle should always be closed +** before control is returned to the user (to prevent a lock being held +** on the database file for longer than necessary). Thus, any virtual table +** method (xFilter etc.) that may directly or indirectly call this function +** must call sqlite3Fts3SegmentsClose() before returning. +*/ +SQLITE_PRIVATE int sqlite3Fts3ReadBlock( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ + char **paBlob, /* OUT: Blob data in malloc'd buffer */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ +){ + int rc; /* Return code */ + + /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ + assert( pnBlob ); + + if( p->pSegments ){ + rc = sqlite3_blob_reopen(p->pSegments, iBlockid); + }else{ + if( 0==p->zSegmentsTbl ){ + p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); + if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; + } + rc = sqlite3_blob_open( + p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments + ); + } + + if( rc==SQLITE_OK ){ + int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; + if( paBlob ){ + char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + if( !aByte ){ + rc = SQLITE_NOMEM; + }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } + rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); + memset(&aByte[nByte], 0, FTS3_NODE_PADDING); + if( rc!=SQLITE_OK ){ + sqlite3_free(aByte); + aByte = 0; + } + } + *paBlob = aByte; + } + }else if( rc==SQLITE_ERROR ){ + rc = FTS_CORRUPT_VTAB; + } + + return rc; +} + +/* +** Close the blob handle at p->pSegments, if it is open. See comments above +** the sqlite3Fts3ReadBlock() function for details. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ + sqlite3_blob_close(p->pSegments); + p->pSegments = 0; +} + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} + +/* +** Set an Fts3SegReader cursor to point at EOF. +*/ +static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ + if( !fts3SegReaderIsRootOnly(pSeg) ){ + sqlite3_free(pSeg->aNode); + sqlite3_blob_close(pSeg->pBlob); + pSeg->pBlob = 0; + } + pSeg->aNode = 0; +} + +/* +** Move the iterator passed as the first argument to the next term in the +** segment. If successful, SQLITE_OK is returned. If there is no next term, +** SQLITE_DONE. Otherwise, an SQLite error code. +*/ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ + char *pNext; /* Cursor variable */ + int nPrefix; /* Number of bytes in term prefix */ + int nSuffix; /* Number of bytes in term suffix */ + + if( !pReader->aDoclist ){ + pNext = pReader->aNode; + }else{ + pNext = &pReader->aDoclist[pReader->nDoclist]; + } + + if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ + + if( fts3SegReaderIsPending(pReader) ){ + Fts3HashElem *pElem = *(pReader->ppNextElem); + sqlite3_free(pReader->aNode); + pReader->aNode = 0; + if( pElem ){ + char *aCopy; + PendingList *pList = (PendingList *)fts3HashData(pElem); + int nCopy = pList->nData+1; + pReader->zTerm = (char *)fts3HashKey(pElem); + pReader->nTerm = fts3HashKeysize(pElem); + aCopy = (char*)sqlite3_malloc(nCopy); + if( !aCopy ) return SQLITE_NOMEM; + memcpy(aCopy, pList->aData, nCopy); + pReader->nNode = pReader->nDoclist = nCopy; + pReader->aNode = pReader->aDoclist = aCopy; + pReader->ppNextElem++; + assert( pReader->aNode ); + } + return SQLITE_OK; + } + + fts3SegReaderSetEof(pReader); + + /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf + ** blocks have already been traversed. */ +#ifdef CORRUPT_DB + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock || CORRUPT_DB ); +#endif + if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ + return SQLITE_OK; + } + + rc = sqlite3Fts3ReadBlock( + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) + ); + if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } + pNext = pReader->aNode; + } + + assert( !fts3SegReaderIsPending(pReader) ); + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; + + /* Because of the FTS3_NODE_PADDING bytes of padding, the following is + ** safe (no risk of overread) even if the node data is corrupted. */ + pNext += fts3GetVarint32(pNext, &nPrefix); + pNext += fts3GetVarint32(pNext, &nSuffix); + if( nSuffix<=0 + || (&pReader->aNode[pReader->nNode] - pNext)pReader->nTerm + ){ + return FTS_CORRUPT_VTAB; + } + + /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are + ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer + ** overflow - hence the (i64) casts. */ + if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){ + i64 nNew = ((i64)nPrefix+nSuffix)*2; + char *zNew = sqlite3_realloc64(pReader->zTerm, nNew); + if( !zNew ){ + return SQLITE_NOMEM; + } + pReader->zTerm = zNew; + pReader->nTermAlloc = nNew; + } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); + pReader->nTerm = nPrefix+nSuffix; + pNext += nSuffix; + pNext += fts3GetVarint32(pNext, &pReader->nDoclist); + pReader->aDoclist = pNext; + pReader->pOffsetList = 0; + + /* Check that the doclist does not appear to extend past the end of the + ** b-tree node. And that the final byte of the doclist is 0x00. If either + ** of these statements is untrue, then the data structure is corrupt. + */ + if( pReader->nDoclist > pReader->nNode-(pReader->aDoclist-pReader->aNode) + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) + || pReader->nDoclist==0 + ){ + return FTS_CORRUPT_VTAB; + } + return SQLITE_OK; +} + +/* +** Set the SegReader to point to the first docid in the doclist associated +** with the current term. +*/ +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; + assert( pReader->aDoclist ); + assert( !pReader->pOffsetList ); + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + } + return rc; +} + +/* +** Advance the SegReader to point to the next docid in the doclist +** associated with the current term. +** +** If arguments ppOffsetList and pnOffsetList are not NULL, then +** *ppOffsetList is set to point to the first column-offset list +** in the doclist entry (i.e. immediately past the docid varint). +** *pnOffsetList is set to the length of the set of column-offset +** lists, not including the nul-terminator byte. For example: +*/ +static int fts3SegReaderNextDocid( + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ +){ + int rc = SQLITE_OK; + char *p = pReader->pOffsetList; + char c = 0; + + assert( p ); + + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } + }else{ + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte past + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + /* List may have been edited in place by fts3EvalNearTrim() */ + while( p=pEnd ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + u64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarintU(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid = (i64)((u64)pReader->iDocid - iDelta); + }else{ + pReader->iDocid = (i64)((u64)pReader->iDocid + iDelta); + } + } + } + } + + return rc; +} + + +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl +){ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; + + assert( p->bFts4 ); + assert( pgsz>0 ); + + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + sqlite3_int64 jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } + } + } + } + *pnOvfl = nOvfl; + return rc; +} + +/* +** Free all allocations associated with the iterator passed as the +** second argument. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ + if( pReader ){ + if( !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + } + if( !fts3SegReaderIsRootOnly(pReader) ){ + sqlite3_free(pReader->aNode); + } + sqlite3_blob_close(pReader->pBlob); + } + sqlite3_free(pReader); +} + +/* +** Allocate a new SegReader object. +*/ +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( + int iAge, /* Segment "age". */ + int bLookup, /* True for a lookup only */ + sqlite3_int64 iStartLeaf, /* First leaf to traverse */ + sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ + sqlite3_int64 iEndBlock, /* Final block of segment */ + const char *zRoot, /* Buffer containing root node */ + int nRoot, /* Size of buffer containing root node */ + Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +){ + Fts3SegReader *pReader; /* Newly allocated SegReader object */ + int nExtra = 0; /* Bytes to allocate segment root node */ + + assert( zRoot!=0 || nRoot==0 ); +#ifdef CORRUPT_DB + assert( zRoot!=0 || CORRUPT_DB ); +#endif + + if( iStartLeaf==0 ){ + if( iEndLeaf!=0 ) return FTS_CORRUPT_VTAB; + nExtra = nRoot + FTS3_NODE_PADDING; + } + + pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + if( !pReader ){ + return SQLITE_NOMEM; + } + memset(pReader, 0, sizeof(Fts3SegReader)); + pReader->iIdx = iAge; + pReader->bLookup = bLookup!=0; + pReader->iStartBlock = iStartLeaf; + pReader->iLeafEndBlock = iEndLeaf; + pReader->iEndBlock = iEndBlock; + + if( nExtra ){ + /* The entire segment is stored in the root node. */ + pReader->aNode = (char *)&pReader[1]; + pReader->rootOnly = 1; + pReader->nNode = nRoot; + if( nRoot ) memcpy(pReader->aNode, zRoot, nRoot); + memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); + }else{ + pReader->iCurrentBlock = iStartLeaf-1; + } + *ppReader = pReader; + return SQLITE_OK; +} + +/* +** This is a comparison function used as a qsort() callback when sorting +** an array of pending terms by term. This occurs as part of flushing +** the contents of the pending-terms hash table to the database. +*/ +static int SQLITE_CDECL fts3CompareElemByTerm( + const void *lhs, + const void *rhs +){ + char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); + char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); + int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); + int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); + + int n = (n1aIndex */ + const char *zTerm, /* Term to search for */ + int nTerm, /* Size of buffer zTerm */ + int bPrefix, /* True for a prefix iterator */ + Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ +){ + Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ + Fts3HashElem *pE; /* Iterator variable */ + Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ + int nElem = 0; /* Size of array at aElem */ + int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; + + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ + int nAlloc = 0; /* Size of allocated array at aElem */ + + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ + char *zKey = (char *)fts3HashKey(pE); + int nKey = fts3HashKeysize(pE); + if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ + if( nElem==nAlloc ){ + Fts3HashElem **aElem2; + nAlloc += 16; + aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem, nAlloc*sizeof(Fts3HashElem *) + ); + if( !aElem2 ){ + rc = SQLITE_NOMEM; + nElem = 0; + break; + } + aElem = aElem2; + } + + aElem[nElem++] = pE; + } + } + + /* If more than one term matches the prefix, sort the Fts3HashElem + ** objects in term order using qsort(). This uses the same comparison + ** callback as is used when flushing terms to disk. + */ + if( nElem>1 ){ + qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); + } + + }else{ + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. + ** + ** Because the stack address of pE may be accessed via the aElem pointer + ** below, the "Fts3HashElem *pE" must be declared so that it is valid + ** within this entire function, not just this "else{...}" block. + */ + pE = fts3HashFindElem(pHash, zTerm, nTerm); + if( pE ){ + aElem = &pE; + nElem = 1; + } + } + + if( nElem>0 ){ + sqlite3_int64 nByte; + nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); + pReader = (Fts3SegReader *)sqlite3_malloc64(nByte); + if( !pReader ){ + rc = SQLITE_NOMEM; + }else{ + memset(pReader, 0, nByte); + pReader->iIdx = 0x7FFFFFFF; + pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; + memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); + } + } + + if( bPrefix ){ + sqlite3_free(aElem); + } + *ppReader = pReader; + return rc; +} + +/* +** Compare the entries pointed to by two Fts3SegReader structures. +** Comparison is as follows: +** +** 1) EOF is greater than not EOF. +** +** 2) The current terms (if any) are compared using memcmp(). If one +** term is a prefix of another, the longer term is considered the +** larger. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc; + if( pLhs->aNode && pRhs->aNode ){ + int rc2 = pLhs->nTerm - pRhs->nTerm; + if( rc2<0 ){ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); + }else{ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); + } + if( rc==0 ){ + rc = rc2; + } + }else{ + rc = (pLhs->aNode==0) - (pRhs->aNode==0); + } + if( rc==0 ){ + rc = pRhs->iIdx - pLhs->iIdx; + } + assert_fts3_nc( rc!=0 ); + return rc; +} + +/* +** A different comparison function for SegReader structures. In this +** version, it is assumed that each SegReader points to an entry in +** a doclist for identical terms. Comparison is made as follows: +** +** 1) EOF (end of doclist in this case) is greater than not EOF. +** +** 2) By current docid. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} + +/* +** Compare the term that the Fts3SegReader object passed as the first argument +** points to with the term specified by arguments zTerm and nTerm. +** +** If the pSeg iterator is already at EOF, return 0. Otherwise, return +** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are +** equal, or +ve if the pSeg term is greater than zTerm/nTerm. +*/ +static int fts3SegReaderTermCmp( + Fts3SegReader *pSeg, /* Segment reader object */ + const char *zTerm, /* Term to compare to */ + int nTerm /* Size of term zTerm in bytes */ +){ + int res = 0; + if( pSeg->aNode ){ + if( pSeg->nTerm>nTerm ){ + res = memcmp(pSeg->zTerm, zTerm, nTerm); + }else{ + res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); + } + if( res==0 ){ + res = pSeg->nTerm-nTerm; + } + } + return res; +} + +/* +** Argument apSegment is an array of nSegment elements. It is known that +** the final (nSegment-nSuspect) members are already in sorted order +** (according to the comparison function provided). This function shuffles +** the array around until all entries are in sorted order. +*/ +static void fts3SegReaderSort( + Fts3SegReader **apSegment, /* Array to sort entries of */ + int nSegment, /* Size of apSegment array */ + int nSuspect, /* Unsorted entry count */ + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ +){ + int i; /* Iterator variable */ + + assert( nSuspect<=nSegment ); + + if( nSuspect==nSegment ) nSuspect--; + for(i=nSuspect-1; i>=0; i--){ + int j; + for(j=i; j<(nSegment-1); j++){ + Fts3SegReader *pTmp; + if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; + pTmp = apSegment[j+1]; + apSegment[j+1] = apSegment[j]; + apSegment[j] = pTmp; + } + } + +#ifndef NDEBUG + /* Check that the list really is sorted now. */ + for(i=0; i<(nSuspect-1); i++){ + assert( xCmp(apSegment[i], apSegment[i+1])<0 ); + } +#endif +} + +/* +** Insert a record into the %_segments table. +*/ +static int fts3WriteSegment( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iBlock, /* Block id for new block */ + char *z, /* Pointer to buffer containing block data */ + int n /* Size of buffer z in bytes */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iBlock); + sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); + } + return rc; +} + +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} + +/* +** Insert a record into the %_segdir table. +*/ +static int fts3WriteSegdir( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ + int iIdx, /* Value for "idx" field */ + sqlite3_int64 iStartBlock, /* Value for "start_block" field */ + sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ + sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ + char *zRoot, /* Blob value for "root" field */ + int nRoot /* Number of bytes in buffer zRoot */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iLevel); + sqlite3_bind_int(pStmt, 2, iIdx); + sqlite3_bind_int64(pStmt, 3, iStartBlock); + sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } + sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 6); + } + return rc; +} + +/* +** Return the size of the common prefix (if any) shared by zPrev and +** zNext, in bytes. For example, +** +** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 +** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 +** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 +*/ +static int fts3PrefixCompress( + const char *zPrev, /* Buffer containing previous term */ + int nPrev, /* Size of buffer zPrev in bytes */ + const char *zNext, /* Buffer containing next term */ + int nNext /* Size of buffer zNext in bytes */ +){ + int n; + for(n=0; nnData; /* Current size of node in bytes */ + int nReq = nData; /* Required space after adding zTerm */ + int nPrefix; /* Number of bytes of prefix compression */ + int nSuffix; /* Suffix length */ + + nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of + ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when + ** compared with BINARY collation. This indicates corruption. */ + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + + nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; + if( nReq<=p->nNodeSize || !pTree->zTerm ){ + + if( nReq>p->nNodeSize ){ + /* An unusual case: this is the first term to be added to the node + ** and the static node buffer (p->nNodeSize bytes) is not large + ** enough. Use a separately malloced buffer instead This wastes + ** p->nNodeSize bytes, but since this scenario only comes about when + ** the database contain two terms that share a prefix of almost 2KB, + ** this is not expected to be a serious problem. + */ + assert( pTree->aData==(char *)&pTree[1] ); + pTree->aData = (char *)sqlite3_malloc(nReq); + if( !pTree->aData ){ + return SQLITE_NOMEM; + } + } + + if( pTree->zTerm ){ + /* There is no prefix-length field for first term in a node */ + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); + } + + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); + memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); + pTree->nData = nData + nSuffix; + pTree->nEntry++; + + if( isCopyTerm ){ + if( pTree->nMalloczMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pTree->nMalloc = nTerm*2; + pTree->zMalloc = zNew; + } + pTree->zTerm = pTree->zMalloc; + memcpy(pTree->zTerm, zTerm, nTerm); + pTree->nTerm = nTerm; + }else{ + pTree->zTerm = (char *)zTerm; + pTree->nTerm = nTerm; + } + return SQLITE_OK; + } + } + + /* If control flows to here, it was not possible to append zTerm to the + ** current node. Create a new node (a right-sibling of the current node). + ** If this is the first node in the tree, the term is added to it. + ** + ** Otherwise, the term is not added to the new node, it is left empty for + ** now. Instead, the term is inserted into the parent of pTree. If pTree + ** has no parent, one is created here. + */ + pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + if( !pNew ){ + return SQLITE_NOMEM; + } + memset(pNew, 0, sizeof(SegmentNode)); + pNew->nData = 1 + FTS3_VARINT_MAX; + pNew->aData = (char *)&pNew[1]; + + if( pTree ){ + SegmentNode *pParent = pTree->pParent; + rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); + if( pTree->pParent==0 ){ + pTree->pParent = pParent; + } + pTree->pRight = pNew; + pNew->pLeftmost = pTree->pLeftmost; + pNew->pParent = pParent; + pNew->zMalloc = pTree->zMalloc; + pNew->nMalloc = pTree->nMalloc; + pTree->zMalloc = 0; + }else{ + pNew->pLeftmost = pNew; + rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); + } + + *ppTree = pNew; + return rc; +} + +/* +** Helper function for fts3NodeWrite(). +*/ +static int fts3TreeFinishNode( + SegmentNode *pTree, + int iHeight, + sqlite3_int64 iLeftChild +){ + int nStart; + assert( iHeight>=1 && iHeight<128 ); + nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); + pTree->aData[nStart] = (char)iHeight; + sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); + return nStart; +} + +/* +** Write the buffer for the segment node pTree and all of its peers to the +** database. Then call this function recursively to write the parent of +** pTree and its peers to the database. +** +** Except, if pTree is a root node, do not write it to the database. Instead, +** set output variables *paRoot and *pnRoot to contain the root node. +** +** If successful, SQLITE_OK is returned and output variable *piLast is +** set to the largest blockid written to the database (or zero if no +** blocks were written to the db). Otherwise, an SQLite error code is +** returned. +*/ +static int fts3NodeWrite( + Fts3Table *p, /* Virtual table handle */ + SegmentNode *pTree, /* SegmentNode handle */ + int iHeight, /* Height of this node in tree */ + sqlite3_int64 iLeaf, /* Block id of first leaf node */ + sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ + sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ + char **paRoot, /* OUT: Data for root node */ + int *pnRoot /* OUT: Size of root node in bytes */ +){ + int rc = SQLITE_OK; + + if( !pTree->pParent ){ + /* Root node of the tree. */ + int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); + *piLast = iFree-1; + *pnRoot = pTree->nData - nStart; + *paRoot = &pTree->aData[nStart]; + }else{ + SegmentNode *pIter; + sqlite3_int64 iNextFree = iFree; + sqlite3_int64 iNextLeaf = iLeaf; + for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ + int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); + int nWrite = pIter->nData - nStart; + + rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); + iNextFree++; + iNextLeaf += (pIter->nEntry+1); + } + if( rc==SQLITE_OK ){ + assert( iNextLeaf==iFree ); + rc = fts3NodeWrite( + p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot + ); + } + } + + return rc; +} + +/* +** Free all memory allocations associated with the tree pTree. +*/ +static void fts3NodeFree(SegmentNode *pTree){ + if( pTree ){ + SegmentNode *p = pTree->pLeftmost; + fts3NodeFree(p->pParent); + while( p ){ + SegmentNode *pRight = p->pRight; + if( p->aData!=(char *)&p[1] ){ + sqlite3_free(p->aData); + } + assert( pRight==0 || p->zMalloc==0 ); + sqlite3_free(p->zMalloc); + sqlite3_free(p); + p = pRight; + } + } +} + +/* +** Add a term to the segment being constructed by the SegmentWriter object +** *ppWriter. When adding the first term to a segment, *ppWriter should +** be passed NULL. This function will allocate a new SegmentWriter object +** and return it via the input/output variable *ppWriter in this case. +** +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterAdd( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ + int isCopyTerm, /* True if buffer zTerm must be copied */ + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of term in bytes */ + const char *aDoclist, /* Pointer to buffer containing doclist */ + int nDoclist /* Size of doclist in bytes */ +){ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + int nReq; /* Number of bytes required on leaf page */ + int nData; + SegmentWriter *pWriter = *ppWriter; + + if( !pWriter ){ + int rc; + sqlite3_stmt *pStmt; + + /* Allocate the SegmentWriter structure */ + pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + if( !pWriter ) return SQLITE_NOMEM; + memset(pWriter, 0, sizeof(SegmentWriter)); + *ppWriter = pWriter; + + /* Allocate a buffer in which to accumulate data */ + pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + if( !pWriter->aData ) return SQLITE_NOMEM; + pWriter->nSize = p->nNodeSize; + + /* Find the next free blockid in the %_segments table */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + pWriter->iFree = sqlite3_column_int64(pStmt, 0); + pWriter->iFirst = pWriter->iFree; + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + nData = pWriter->nData; + + nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; + + /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of + ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when + ** compared with BINARY collation. This indicates corruption. */ + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + + /* Figure out how many bytes are required by this new entry */ + nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ + nSuffix + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + + if( nData>0 && nData+nReq>p->nNodeSize ){ + int rc; + + /* The current leaf node is full. Write it out to the database. */ + if( pWriter->iFree==LARGEST_INT64 ) return FTS_CORRUPT_VTAB; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); + if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; + + /* Add the current term to the interior node tree. The term added to + ** the interior tree must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pWriter->zTerm), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + assert( nPrefixpTree, isCopyTerm, zTerm, nPrefix+1); + if( rc!=SQLITE_OK ) return rc; + + nData = 0; + pWriter->nTerm = 0; + + nPrefix = 0; + nSuffix = nTerm; + nReq = 1 + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ + nTerm + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + } + + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; + + /* If the buffer currently allocated is too small for this entry, realloc + ** the buffer to make it large enough. + */ + if( nReq>pWriter->nSize ){ + char *aNew = sqlite3_realloc(pWriter->aData, nReq); + if( !aNew ) return SQLITE_NOMEM; + pWriter->aData = aNew; + pWriter->nSize = nReq; + } + assert( nData+nReq<=pWriter->nSize ); + + /* Append the prefix-compressed term and doclist to the buffer. */ + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); + assert( nSuffix>0 ); + memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); + nData += nSuffix; + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); + assert( nDoclist>0 ); + memcpy(&pWriter->aData[nData], aDoclist, nDoclist); + pWriter->nData = nData + nDoclist; + + /* Save the current term so that it can be used to prefix-compress the next. + ** If the isCopyTerm parameter is true, then the buffer pointed to by + ** zTerm is transient, so take a copy of the term data. Otherwise, just + ** store a copy of the pointer. + */ + if( isCopyTerm ){ + if( nTerm>pWriter->nMalloc ){ + char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pWriter->nMalloc = nTerm*2; + pWriter->zMalloc = zNew; + pWriter->zTerm = zNew; + } + assert( pWriter->zTerm==pWriter->zMalloc ); + assert( nTerm>0 ); + memcpy(pWriter->zTerm, zTerm, nTerm); + }else{ + pWriter->zTerm = (char *)zTerm; + } + pWriter->nTerm = nTerm; + + return SQLITE_OK; +} + +/* +** Flush all data associated with the SegmentWriter object pWriter to the +** database. This function must be called after all terms have been added +** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is +** returned. Otherwise, an SQLite error code. +*/ +static int fts3SegWriterFlush( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ + sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ + int iIdx /* Value for 'idx' column of %_segdir */ +){ + int rc; /* Return code */ + if( pWriter->pTree ){ + sqlite3_int64 iLast = 0; /* Largest block id written to database */ + sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ + char *zRoot = NULL; /* Pointer to buffer containing root node */ + int nRoot = 0; /* Size of buffer zRoot */ + + iLastLeaf = pWriter->iFree; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); + if( rc==SQLITE_OK ){ + rc = fts3NodeWrite(p, pWriter->pTree, 1, + pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); + } + }else{ + /* The entire tree fits on the root node. Write it to the segdir table. */ + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); + } + p->nLeafAdd++; + return rc; +} + +/* +** Release all memory held by the SegmentWriter object passed as the +** first argument. +*/ +static void fts3SegWriterFree(SegmentWriter *pWriter){ + if( pWriter ){ + sqlite3_free(pWriter->aData); + sqlite3_free(pWriter->zMalloc); + fts3NodeFree(pWriter->pTree); + sqlite3_free(pWriter); + } +} + +/* +** The first value in the apVal[] array is assumed to contain an integer. +** This function tests if there exist any documents with docid values that +** are different from that integer. i.e. if deleting the document with docid +** pRowid would mean the FTS3 table were empty. +** +** If successful, *pisEmpty is set to true if the table is empty except for +** document pRowid, or false otherwise, and SQLITE_OK is returned. If an +** error occurs, an SQLite error code is returned. +*/ +static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ + sqlite3_stmt *pStmt; + int rc; + if( p->zContentTbl ){ + /* If using the content=xxx option, assume the table is never empty */ + *pisEmpty = 0; + rc = SQLITE_OK; + }else{ + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Set *pnMax to the largest segment level in the database for the index +** iIndex. +** +** Segment levels are stored in the 'level' column of the %_segdir table. +** +** Return SQLITE_OK if successful, or an SQLite error code if not. +*/ +static int fts3SegmentMaxLevel( + Fts3Table *p, + int iLangid, + int iIndex, + sqlite3_int64 *pnMax +){ + sqlite3_stmt *pStmt; + int rc; + assert( iIndex>=0 && iIndexnIndex ); + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnMax = sqlite3_column_int64(pStmt, 0); + } + return sqlite3_reset(pStmt); +} + +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + (((u64)iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} + +/* +** Delete all entries in the %_segments table associated with the segment +** opened with seg-reader pSeg. This function does not affect the contents +** of the %_segdir table. +*/ +static int fts3DeleteSegment( + Fts3Table *p, /* FTS table handle */ + Fts3SegReader *pSeg /* Segment to delete */ +){ + int rc = SQLITE_OK; /* Return code */ + if( pSeg->iStartBlock ){ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + } + return rc; +} + +/* +** This function is used after merging multiple segments into a single large +** segment to delete the old, now redundant, segment b-trees. Specifically, +** it: +** +** 1) Deletes all %_segments entries for the segments associated with +** each of the SegReader objects in the array passed as the third +** argument, and +** +** 2) deletes all %_segdir entries with level iLevel, or all %_segdir +** entries regardless of level if (iLevel<0). +** +** SQLITE_OK is returned if successful, otherwise an SQLite error code. +*/ +static int fts3DeleteSegdir( + Fts3Table *p, /* Virtual table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, /* Level of %_segdir entries to delete */ + Fts3SegReader **apSegment, /* Array of SegReader objects */ + int nReader /* Size of array apSegment */ +){ + int rc = SQLITE_OK; /* Return Code */ + int i; /* Iterator variable */ + sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ + + for(i=0; rc==SQLITE_OK && i=0 || iLevel==FTS3_SEGCURSOR_ALL ); + if( iLevel==FTS3_SEGCURSOR_ALL ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pDelete, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** When this function is called, buffer *ppList (size *pnList bytes) contains +** a position list that may (or may not) feature multiple columns. This +** function adjusts the pointer *ppList and the length *pnList so that they +** identify the subset of the position list that corresponds to column iCol. +** +** If there are no entries in the input position list for column iCol, then +** *pnList is set to zero before returning. +** +** If parameter bZero is non-zero, then any part of the input list following +** the end of the output list is zeroed before returning. +*/ +static void fts3ColumnFilter( + int iCol, /* Column to filter on */ + int bZero, /* Zero out anything following *ppList */ + char **ppList, /* IN/OUT: Pointer to position list */ + int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ +){ + char *pList = *ppList; + int nList = *pnList; + char *pEnd = &pList[nList]; + int iCurrent = 0; + char *p = pList; + + assert( iCol>=0 ); + while( 1 ){ + char c = 0; + while( p0){ + memset(&pList[nList], 0, pEnd - &pList[nList]); + } + *ppList = pList; + *pnList = nList; +} + +/* +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any +** existing data). Grow the buffer if required. +** +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered +** trying to resize the buffer, return SQLITE_NOMEM. +*/ +static int fts3MsrBufferData( + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + char *pList, + int nList +){ + if( nList>pMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; + } + + assert( nList>0 ); + memcpy(pMsr->aBuffer, pList, nList); + return SQLITE_OK; +} + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; + } + + while( 1 ){ + Fts3SegReader *pSeg; + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + int rc; + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( rc==SQLITE_OK + && jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + if( rc!=SQLITE_OK ) return rc; + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); + + if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + pList = pMsr->aBuffer; + } + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); + } + + if( nList>0 ){ + *paPoslist = pList; + *piDocid = iDocid; + *pnPoslist = nList; + break; + } + } + } + + return SQLITE_OK; +} + +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ +){ + int i; + int nSeg = pCsr->nSegment; + + /* If the Fts3SegFilter defines a specific term (or term prefix) to search + ** for, then advance each segment iterator until it points to a term of + ** equal or greater value than the specified term. This prevents many + ** unnecessary merge/sort operations for the case where single segment + ** b-tree leaf nodes contain more than one term. + */ + for(i=0; pCsr->bRestart==0 && inSegment; i++){ + int res = 0; + Fts3SegReader *pSeg = pCsr->apSegment[i]; + do { + int rc = fts3SegReaderNext(p, pSeg, 0); + if( rc!=SQLITE_OK ) return rc; + }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 ); + + if( pSeg->bLookup && res!=0 ){ + fts3SegReaderSetEof(pSeg); + } + } + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); + + return SQLITE_OK; +} + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); +} + +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} + +/* +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: +** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() +** +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. +*/ +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ + + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); + + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + + return SQLITE_OK; +} + +static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, int nReq){ + if( nReq>pCsr->nBuffer ){ + char *aNew; + pCsr->nBuffer = nReq*2; + aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); + if( !aNew ){ + return SQLITE_NOMEM; + } + pCsr->aBuffer = aNew; + } + return SQLITE_OK; +} + + +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + int rc = SQLITE_OK; + + int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); + int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); + int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); + int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); + int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); + + Fts3SegReader **apSegment = pCsr->apSegment; + int nSegment = pCsr->nSegment; + Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( pCsr->nSegment==0 ) return SQLITE_OK; + + do { + int nMerge; + int i; + + /* Advance the first pCsr->nAdvance entries in the apSegment[] array + ** forward. Then sort the list in order of current term again. + */ + for(i=0; inAdvance; i++){ + Fts3SegReader *pSeg = apSegment[i]; + if( pSeg->bLookup ){ + fts3SegReaderSetEof(pSeg); + }else{ + rc = fts3SegReaderNext(p, pSeg, 0); + } + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); + pCsr->nAdvance = 0; + + /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ + assert( rc==SQLITE_OK ); + if( apSegment[0]->aNode==0 ) break; + + pCsr->nTerm = apSegment[0]->nTerm; + pCsr->zTerm = apSegment[0]->zTerm; + + /* If this is a prefix-search, and if the term that apSegment[0] points + ** to does not share a suffix with pFilter->zTerm/nTerm, then all + ** required callbacks have been made. In this case exit early. + ** + ** Similarly, if this is a search for an exact match, and the first term + ** of segment apSegment[0] is not a match, exit early. + */ + if( pFilter->zTerm && !isScan ){ + if( pCsr->nTermnTerm + || (!isPrefix && pCsr->nTerm>pFilter->nTerm) + || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) + ){ + break; + } + } + + nMerge = 1; + while( nMergeaNode + && apSegment[nMerge]->nTerm==pCsr->nTerm + && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) + ){ + nMerge++; + } + + assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); + if( nMerge==1 + && !isIgnoreEmpty + && !isFirst + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ + pCsr->nDoclist = apSegment[0]->nDoclist; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; + }else{ + int nDoclist = 0; /* Size of doclist */ + sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ + + /* The current term of the first nMerge entries in the array + ** of Fts3SegReader objects is the same. The doclists must be merged + ** and a single term returned with the merged doclist. + */ + for(i=0; ipOffsetList ){ + int j; /* Number of segments that share a docid */ + char *pList = 0; + int nList = 0; + int nByte; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + + if( isColFilter ){ + fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); + } + + if( !isIgnoreEmpty || nList>0 ){ + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + if( iPrev<=iDocid ) return FTS_CORRUPT_VTAB; + iDelta = (i64)((u64)iPrev - (u64)iDocid); + }else{ + if( nDoclist>0 && iPrev>=iDocid ) return FTS_CORRUPT_VTAB; + iDelta = (i64)((u64)iDocid - (u64)iPrev); + } + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); + + rc = fts3GrowSegReaderBuffer(pCsr, nByte+nDoclist+FTS3_NODE_PADDING); + if( rc ) return rc; + + if( isFirst ){ + char *a = &pCsr->aBuffer[nDoclist]; + int nWrite; + + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); + if( nWrite ){ + iPrev = iDocid; + nDoclist += nWrite; + } + }else{ + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); + nDoclist += nList; + pCsr->aBuffer[nDoclist++] = '\0'; + } + } + } + + fts3SegReaderSort(apSegment, nMerge, j, xCmp); + } + if( nDoclist>0 ){ + rc = fts3GrowSegReaderBuffer(pCsr, nDoclist+FTS3_NODE_PADDING); + if( rc ) return rc; + memset(&pCsr->aBuffer[nDoclist], 0, FTS3_NODE_PADDING); + pCsr->aDoclist = pCsr->aBuffer; + pCsr->nDoclist = nDoclist; + rc = SQLITE_ROW; + } + } + pCsr->nAdvance = nMerge; + }while( rc==SQLITE_OK ); + + return rc; +} + + +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + if( pCsr ){ + int i; + for(i=0; inSegment; i++){ + sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); + } + sqlite3_free(pCsr->apSegment); + sqlite3_free(pCsr->aBuffer); + + pCsr->nSegment = 0; + pCsr->apSegment = 0; + pCsr->aBuffer = 0; + } +} + +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + u64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = (i64)iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = ((i64)iVal * (i64)iMul); + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 0; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize = 0, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1 = 0; + sqlite3_stmt *pUpdate2 = 0; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} + +/* +** Merge all level iLevel segments in the database into a single +** iLevel+1 segment. Or, if iLevel<0, merge all segments into a +** single segment with a level equal to the numerically largest level +** currently present in the database. +** +** If this function is called with iLevel<0, but there is only one +** segment in the database, SQLITE_DONE is returned immediately. +** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, +** an SQLite error code is returned. +*/ +static int fts3SegmentMerge( + Fts3Table *p, + int iLangid, /* Language id to merge */ + int iIndex, /* Index in p->aIndex[] to merge */ + int iLevel /* Level to merge */ +){ + int rc; /* Return code */ + int iIdx = 0; /* Index of new segment */ + sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ + SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ + Fts3SegFilter filter; /* Segment term filter condition */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ + + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); + + rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); + if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } + + if( iLevel==FTS3_SEGCURSOR_ALL ){ + /* This call is to merge all segments in the database to a single + ** segment. The level of the new segment is equal to the numerically + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ + if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){ + rc = SQLITE_DONE; + goto finished; + } + iNewLevel = iMaxLevel; + bIgnoreEmpty = 1; + + }else{ + /* This call is to merge all segments at level iLevel. find the next + ** available segment index at level iLevel+1. The call to + ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to + ** a single iLevel+2 segment if necessary. */ + assert( FTS3_SEGCURSOR_PENDING==-1 ); + iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); + } + if( rc!=SQLITE_OK ) goto finished; + + assert( csr.nSegment>0 ); + assert_fts3_nc( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert_fts3_nc( + iNewLevelnLeafData); + } + } + } + + finished: + fts3SegWriterFree(pWriter); + sqlite3Fts3SegReaderFinish(&csr); + return rc; +} + + +/* +** Flush the contents of pendingTerms to level 0 segments. +*/ +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + sqlite3Fts3PendingTermsClear(p); + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; + } + rc = sqlite3_reset(pStmt); + } + } + return rc; +} + +/* +** Encode N integers as varints into a blob. +*/ +static void fts3EncodeIntArray( + int N, /* The number of integers to encode */ + u32 *a, /* The integer values */ + char *zBuf, /* Write the BLOB here */ + int *pNBuf /* Write number of bytes if zBuf[] used here */ +){ + int i, j; + for(i=j=0; iiPrevDocid. The sizes are encoded as +** a blob of varints. +*/ +static void fts3InsertDocsize( + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column, in tokens */ +){ + char *pBlob; /* The BLOB encoding of the document size */ + int nBlob; /* Number of bytes in the BLOB */ + sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ + int rc; /* Result code from subfunctions */ + + if( *pRC ) return; + pBlob = sqlite3_malloc64( 10*(sqlite3_int64)p->nColumn ); + if( pBlob==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); + if( rc ){ + sqlite3_free(pBlob); + *pRC = rc; + return; + } + sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); +} + +/* +** Record 0 of the %_stat table contains a blob consisting of N varints, +** where N is the number of user defined columns in the fts3 table plus +** two. If nCol is the number of user defined columns, then values of the +** varints are set as follows: +** +** Varint 0: Total number of rows in the table. +** +** Varint 1..nCol: For each column, the total number of tokens stored in +** the column for all rows of the table. +** +** Varint 1+nCol: The total size, in bytes, of all text values in all +** columns of all rows of the table. +** +*/ +static void fts3UpdateDocTotals( + int *pRC, /* The result code */ + Fts3Table *p, /* Table being updated */ + u32 *aSzIns, /* Size increases */ + u32 *aSzDel, /* Size decreases */ + int nChng /* Change in the number of documents */ +){ + char *pBlob; /* Storage for BLOB written into %_stat */ + int nBlob; /* Size of BLOB written into %_stat */ + u32 *a; /* Array of integers that becomes the BLOB */ + sqlite3_stmt *pStmt; /* Statement for reading and writing */ + int i; /* Loop counter */ + int rc; /* Result code from subfunctions */ + + const int nStat = p->nColumn+2; + + if( *pRC ) return; + a = sqlite3_malloc64( (sizeof(u32)+10)*(sqlite3_int64)nStat ); + if( a==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + pBlob = (char*)&a[nStat]; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + fts3DecodeIntArray(nStat, a, + sqlite3_column_blob(pStmt, 0), + sqlite3_column_bytes(pStmt, 0)); + }else{ + memset(a, 0, sizeof(u32)*(nStat) ); + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + sqlite3_free(a); + *pRC = rc; + return; + } + if( nChng<0 && a[0]<(u32)(-nChng) ){ + a[0] = 0; + }else{ + a[0] += nChng; + } + for(i=0; inColumn+1; i++){ + u32 x = a[i+1]; + if( x+aSzIns[i] < aSzDel[i] ){ + x = 0; + }else{ + x = x + aSzIns[i] - aSzDel[i]; + } + a[i+1] = x; + } + fts3EncodeIntArray(nStat, a, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); + sqlite3_free(a); +} + +/* +** Merge the entire database so that there is one segment for each +** iIndex/iLangid combination. +*/ +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int bSeenDone = 0; + int rc; + sqlite3_stmt *pAllLangid = 0; + + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + } + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int i; + int iLangid = sqlite3_column_int(pAllLangid, 0); + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + sqlite3Fts3SegmentsClose(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +} + +/* +** This function is called when the user executes the following statement: +** +** INSERT INTO () VALUES('rebuild'); +** +** The entire FTS index is discarded and rebuilt. If the table is one +** created using the content=xxx option, then the new index is based on +** the current contents of the xxx table. Otherwise, it is rebuilt based +** on the contents of the %_content table. +*/ +static int fts3DoRebuild(Fts3Table *p){ + int rc; /* Return Code */ + + rc = fts3DeleteAll(p, 0); + if( rc==SQLITE_OK ){ + u32 *aSz = 0; + u32 *aSzIns = 0; + u32 *aSzDel = 0; + sqlite3_stmt *pStmt = 0; + int nEntry = 0; + + /* Compose and prepare an SQL statement to loop through the content table */ + char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + sqlite3_int64 nByte = sizeof(u32) * ((sqlite3_int64)p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc64(nByte); + if( aSz==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(aSz, 0, nByte); + aSzIns = &aSz[p->nColumn+1]; + aSzDel = &aSzIns[p->nColumn+1]; + } + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int iCol; + int iLangid = langidFromSelect(p, pStmt); + rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); + memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSz); + } + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + }else{ + nEntry++; + for(iCol=0; iCol<=p->nColumn; iCol++){ + aSzIns[iCol] += aSz[iCol]; + } + } + } + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); + } + sqlite3_free(aSz); + + if( pStmt ){ + int rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + return rc; +} + + +/* +** This function opens a cursor used to read the input data for an +** incremental merge operation. Specifically, it opens a cursor to scan +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** level iAbsLevel. +*/ +static int fts3IncrmergeCsr( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to open */ + int nSeg, /* Number of segments to merge */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ +){ + int rc; /* Return Code */ + sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ + sqlite3_int64 nByte; /* Bytes allocated at pCsr->apSegment[] */ + + /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ + memset(pCsr, 0, sizeof(*pCsr)); + nByte = sizeof(Fts3SegReader *) * nSeg; + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc64(nByte); + + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->apSegment, 0, nByte); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + } + if( rc==SQLITE_OK ){ + int i; + int rc2; + sqlite3_bind_int64(pStmt, 1, iAbsLevel); + assert( pCsr->nSegment==0 ); + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] + ); + pCsr->nSegment++; + } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +typedef struct IncrmergeWriter IncrmergeWriter; +typedef struct NodeWriter NodeWriter; +typedef struct Blob Blob; +typedef struct NodeReader NodeReader; + +/* +** An instance of the following structure is used as a dynamic buffer +** to build up nodes or other blobs of data in. +** +** The function blobGrowBuffer() is used to extend the allocation. +*/ +struct Blob { + char *a; /* Pointer to allocation */ + int n; /* Number of valid bytes of data in a[] */ + int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ +}; + +/* +** This structure is used to build up buffers containing segment b-tree +** nodes (blocks). +*/ +struct NodeWriter { + sqlite3_int64 iBlock; /* Current block id */ + Blob key; /* Last key written to the current block */ + Blob block; /* Current block image */ +}; + +/* +** An object of this type contains the state required to create or append +** to an appendable b-tree segment. +*/ +struct IncrmergeWriter { + int nLeafEst; /* Space allocated for leaf blocks */ + int nWork; /* Number of leaf pages flushed */ + sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ + int iIdx; /* Index of *output* segment in iAbsLevel+1 */ + sqlite3_int64 iStart; /* Block number of first allocated block */ + sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ + NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; +}; + +/* +** An object of the following type is used to read data from a single +** FTS segment node. See the following functions: +** +** nodeReaderInit() +** nodeReaderNext() +** nodeReaderRelease() +*/ +struct NodeReader { + const char *aNode; + int nNode; + int iOff; /* Current offset within aNode[] */ + + /* Output variables. Containing the current node entry. */ + sqlite3_int64 iChild; /* Pointer to child node */ + Blob term; /* Current term */ + const char *aDoclist; /* Pointer to doclist */ + int nDoclist; /* Size of doclist in bytes */ +}; + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if the allocation at pBlob->a is not already at least nMin +** bytes in size, extend (realloc) it to be so. +** +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc +** to reflect the new size of the pBlob->a[] buffer. +*/ +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ + if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ + int nAlloc = nMin; + char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + if( a ){ + pBlob->nAlloc = nAlloc; + pBlob->a = a; + }else{ + *pRc = SQLITE_NOMEM; + } + } +} + +/* +** Attempt to advance the node-reader object passed as the first argument to +** the next entry on the node. +** +** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Otherwise return SQLITE_OK. If there is no next entry on the node +** (e.g. because the current entry is the last) set NodeReader->aNode to +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** variables for the new entry. +*/ +static int nodeReaderNext(NodeReader *p){ + int bFirst = (p->term.n==0); /* True for first term on the node */ + int nPrefix = 0; /* Bytes to copy from previous term */ + int nSuffix = 0; /* Bytes to append to the prefix */ + int rc = SQLITE_OK; /* Return code */ + + assert( p->aNode ); + if( p->iChild && bFirst==0 ) p->iChild++; + if( p->iOff>=p->nNode ){ + /* EOF */ + p->aNode = 0; + }else{ + if( bFirst==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + } + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + + if( nPrefix>p->term.n || nSuffix>p->nNode-p->iOff || nSuffix==0 ){ + return FTS_CORRUPT_VTAB; + } + blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); + if( rc==SQLITE_OK ){ + memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); + p->term.n = nPrefix+nSuffix; + p->iOff += nSuffix; + if( p->iChild==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + if( (p->nNode-p->iOff)nDoclist ){ + return FTS_CORRUPT_VTAB; + } + p->aDoclist = &p->aNode[p->iOff]; + p->iOff += p->nDoclist; + } + } + } + + assert_fts3_nc( p->iOff<=p->nNode ); + return rc; +} + +/* +** Release all dynamic resources held by node-reader object *p. +*/ +static void nodeReaderRelease(NodeReader *p){ + sqlite3_free(p->term.a); +} + +/* +** Initialize a node-reader object to read the node in buffer aNode/nNode. +** +** If successful, SQLITE_OK is returned and the NodeReader object set to +** point to the first entry on the node (if any). Otherwise, an SQLite +** error code is returned. +*/ +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ + memset(p, 0, sizeof(NodeReader)); + p->aNode = aNode; + p->nNode = nNode; + + /* Figure out if this is a leaf or an internal node. */ + if( aNode && aNode[0] ){ + /* An internal node. */ + p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); + }else{ + p->iOff = 1; + } + + return aNode ? nodeReaderNext(p) : SQLITE_OK; +} + +/* +** This function is called while writing an FTS segment each time a leaf o +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed +** to be greater than the largest key on the node just written, but smaller +** than or equal to the first key that will be written to the next leaf +** node. +** +** The block id of the leaf node just written to disk may be found in +** (pWriter->aNodeWriter[0].iBlock) when this function is called. +*/ +static int fts3IncrmergePush( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + const char *zTerm, /* Term to write to internal node */ + int nTerm /* Bytes at zTerm */ +){ + sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; + int iLayer; + + assert( nTerm>0 ); + for(iLayer=1; ALWAYS(iLayeraNodeWriter[iLayer]; + int rc = SQLITE_OK; + int nPrefix; + int nSuffix; + int nSpace; + + /* Figure out how much space the key will consume if it is written to + ** the current node of layer iLayer. Due to the prefix compression, + ** the space required changes depending on which node the key is to + ** be added to. */ + nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + if(nSuffix<=0 ) return FTS_CORRUPT_VTAB; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ + /* If the current node of layer iLayer contains zero keys, or if adding + ** the key to it will not cause it to grow to larger than nNodeSize + ** bytes in size, write the key here. */ + + Blob *pBlk = &pNode->block; + if( pBlk->n==0 ){ + blobGrowBuffer(pBlk, p->nNodeSize, &rc); + if( rc==SQLITE_OK ){ + pBlk->a[0] = (char)iLayer; + pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); + } + } + blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); + blobGrowBuffer(&pNode->key, nTerm, &rc); + + if( rc==SQLITE_OK ){ + if( pNode->key.n ){ + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); + } + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); + pBlk->n += nSuffix; + + memcpy(pNode->key.a, zTerm, nTerm); + pNode->key.n = nTerm; + } + }else{ + /* Otherwise, flush the current node of layer iLayer to disk. + ** Then allocate a new, empty sibling node. The key will be written + ** into the parent of this node. */ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + + assert( pNode->block.nAlloc>=p->nNodeSize ); + pNode->block.a[0] = (char)iLayer; + pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); + + iNextPtr = pNode->iBlock; + pNode->iBlock++; + pNode->key.n = 0; + } + + if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; + iPtr = iNextPtr; + } + + assert( 0 ); + return 0; +} + +/* +** Append a term and (optionally) doclist to the FTS segment node currently +** stored in blob *pNode. The node need not contain any terms, but the +** header must be written before this function is called. +** +** A node header is a single 0x00 byte for a leaf node, or a height varint +** followed by the left-hand-child varint for an internal node. +** +** The term to be appended is passed via arguments zTerm/nTerm. For a +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal +** node, both aDoclist and nDoclist must be passed 0. +** +** If the size of the value in blob pPrev is zero, then this is the first +** term written to the node. Otherwise, pPrev contains a copy of the +** previous term. Before this function returns, it is updated to contain a +** copy of zTerm/nTerm. +** +** It is assumed that the buffer associated with pNode is already large +** enough to accommodate the new entry. The buffer associated with pPrev +** is extended by this function if requrired. +** +** If an error (i.e. OOM condition) occurs, an SQLite error code is +** returned. Otherwise, SQLITE_OK. +*/ +static int fts3AppendToNode( + Blob *pNode, /* Current node image to append to */ + Blob *pPrev, /* Buffer containing previous term written */ + const char *zTerm, /* New term to write */ + int nTerm, /* Size of zTerm in bytes */ + const char *aDoclist, /* Doclist (or NULL) to write */ + int nDoclist /* Size of aDoclist in bytes */ +){ + int rc = SQLITE_OK; /* Return code */ + int bFirst = (pPrev->n==0); /* True if this is the first term written */ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + + /* Node must have already been started. There must be a doclist for a + ** leaf node, and there must not be a doclist for an internal node. */ + assert( pNode->n>0 ); + assert_fts3_nc( (pNode->a[0]=='\0')==(aDoclist!=0) ); + + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); + } + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; + + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; + } + + assert( pNode->n<=pNode->nAlloc ); + + return SQLITE_OK; +} + +/* +** Append the current term and doclist pointed to by cursor pCsr to the +** appendable b-tree segment opened for writing by pWriter. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int fts3IncrmergeAppend( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +){ + const char *zTerm = pCsr->zTerm; + int nTerm = pCsr->nTerm; + const char *aDoclist = pCsr->aDoclist; + int nDoclist = pCsr->nDoclist; + int rc = SQLITE_OK; /* Return code */ + int nSpace; /* Total space in bytes required on leaf */ + int nPrefix; /* Size of prefix shared with previous term */ + int nSuffix; /* Size of suffix (nTerm - nPrefix) */ + NodeWriter *pLeaf; /* Object used to write leaf nodes */ + + pLeaf = &pWriter->aNodeWriter[0]; + nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + + /* If the current block is not empty, and if adding this term/doclist + ** to the current block would make it larger than Fts3Table.nNodeSize + ** bytes, write this block out to the database. */ + if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); + pWriter->nWork++; + + /* Add the current term to the parent node. The term added to the + ** parent must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pLeaf->key), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); + } + + /* Advance to the next output block */ + pLeaf->iBlock++; + pLeaf->key.n = 0; + pLeaf->block.n = 0; + + nSuffix = nTerm; + nSpace = 1; + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + } + + pWriter->nLeafData += nSpace; + blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); + if( rc==SQLITE_OK ){ + if( pLeaf->block.n==0 ){ + pLeaf->block.n = 1; + pLeaf->block.a[0] = '\0'; + } + rc = fts3AppendToNode( + &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist + ); + } + + return rc; +} + +/* +** This function is called to release all dynamic resources held by the +** merge-writer object pWriter, and if no error has occurred, to flush +** all outstanding node buffers held by pWriter to disk. +** +** If *pRc is not SQLITE_OK when this function is called, then no attempt +** is made to write any data to disk. Instead, this function serves only +** to release outstanding resources. +** +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while +** flushing buffers to disk, *pRc is set to an SQLite error code before +** returning. +*/ +static void fts3IncrmergeRelease( + Fts3Table *p, /* FTS3 table handle */ + IncrmergeWriter *pWriter, /* Merge-writer object */ + int *pRc /* IN/OUT: Error code */ +){ + int i; /* Used to iterate through non-root layers */ + int iRoot; /* Index of root in pWriter->aNodeWriter */ + NodeWriter *pRoot; /* NodeWriter for root node */ + int rc = *pRc; /* Error code */ + + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + ** root node. If the segment fits entirely on a single leaf node, iRoot + ** will be set to 0. If the root node is the parent of the leaves, iRoot + ** will be 1. And so on. */ + for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ + NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; + if( pNode->block.n>0 ) break; + assert( *pRc || pNode->block.nAlloc==0 ); + assert( *pRc || pNode->key.nAlloc==0 ); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Empty output segment. This is a no-op. */ + if( iRoot<0 ) return; + + /* The entire output segment fits on a single node. Normally, this means + ** the node would be stored as a blob in the "root" column of the %_segdir + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle + ** segments that fit entirely on the root node with start_block!=0. + ** + ** Instead, create a synthetic root node that contains nothing but a + ** pointer to the single content node. So that the segment consists of a + ** single leaf and a single interior (root) node. + ** + ** Todo: Better might be to defer allocating space in the %_segments + ** table until we are sure it is needed. + */ + if( iRoot==0 ){ + Blob *pBlock = &pWriter->aNodeWriter[1].block; + blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); + if( rc==SQLITE_OK ){ + pBlock->a[0] = 0x01; + pBlock->n = 1 + sqlite3Fts3PutVarint( + &pBlock->a[1], pWriter->aNodeWriter[0].iBlock + ); + } + iRoot = 1; + } + pRoot = &pWriter->aNodeWriter[iRoot]; + + /* Flush all currently outstanding nodes to disk. */ + for(i=0; iaNodeWriter[i]; + if( pNode->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + } + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); + } + + /* Write the %_segdir record. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, + pWriter->iAbsLevel+1, /* level */ + pWriter->iIdx, /* idx */ + pWriter->iStart, /* start_block */ + pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ + pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ + pRoot->block.a, pRoot->block.n /* root */ + ); + } + sqlite3_free(pRoot->block.a); + sqlite3_free(pRoot->key.a); + + *pRc = rc; +} + +/* +** Compare the term in buffer zLhs (size in bytes nLhs) with that in +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of +** the other, it is considered to be smaller than the other. +** +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve +** if it is greater. +*/ +static int fts3TermCmp( + const char *zLhs, int nLhs, /* LHS of comparison */ + const char *zRhs, int nRhs /* RHS of comparison */ +){ + int nCmp = MIN(nLhs, nRhs); + int res; + + res = (nCmp ? memcmp(zLhs, zRhs, nCmp) : 0); + if( res==0 ) res = nLhs - nRhs; + + return res; +} + + +/* +** Query to see if the entry in the %_segments table with blockid iEnd is +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before +** returning. Otherwise, set *pbRes to 0. +** +** Or, if an error occurs while querying the database, return an SQLite +** error code. The final value of *pbRes is undefined in this case. +** +** This is used to test if a segment is an "appendable" segment. If it +** is, then a NULL entry has been inserted into the %_segments table +** with blockid %_segdir.end_block. +*/ +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ + int bRes = 0; /* Result to set *pbRes to */ + sqlite3_stmt *pCheck = 0; /* Statement to query database with */ + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); + } + + *pbRes = bRes; + return rc; +} + +/* +** This function is called when initializing an incremental-merge operation. +** It checks if the existing segment with index value iIdx at absolute level +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the +** merge-writer object *pWriter is initialized to write to it. +** +** An existing segment can be appended to by an incremental merge if: +** +** * It was initially created as an appendable segment (with all required +** space pre-allocated), and +** +** * The first key read from the input (arguments zKey and nKey) is +** greater than the largest key currently stored in the potential +** output segment. +*/ +static int fts3IncrmergeLoad( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of candidate output segment */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return code */ + sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); + if( rc==SQLITE_OK ){ + sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ + sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ + sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ + const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ + int nRoot = 0; /* Size of aRoot[] in bytes */ + int rc2; /* Return code from sqlite3_reset() */ + int bAppendable = 0; /* Set to true if segment is appendable */ + + /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ + sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); + sqlite3_bind_int(pSelect, 2, iIdx); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + iStart = sqlite3_column_int64(pSelect, 1); + iLeafEnd = sqlite3_column_int64(pSelect, 2); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + pWriter->bNoLeafData = (pWriter->nLeafData==0); + nRoot = sqlite3_column_bytes(pSelect, 4); + aRoot = sqlite3_column_blob(pSelect, 4); + if( aRoot==0 ){ + sqlite3_reset(pSelect); + return nRoot ? SQLITE_NOMEM : FTS_CORRUPT_VTAB; + } + }else{ + return sqlite3_reset(pSelect); + } + + /* Check for the zero-length marker in the %_segments table */ + rc = fts3IsAppendable(p, iEnd, &bAppendable); + + /* Check that zKey/nKey is larger than the largest key the candidate */ + if( rc==SQLITE_OK && bAppendable ){ + char *aLeaf = 0; + int nLeaf = 0; + + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); + if( rc==SQLITE_OK ){ + NodeReader reader; + for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + assert( reader.aNode ); + } + if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ + bAppendable = 0; + } + nodeReaderRelease(&reader); + } + sqlite3_free(aLeaf); + } + + if( rc==SQLITE_OK && bAppendable ){ + /* It is possible to append to this segment. Set up the IncrmergeWriter + ** object to do so. */ + int i; + int nHeight = (int)aRoot[0]; + NodeWriter *pNode; + if( nHeight<1 || nHeight>=FTS_MAX_APPENDABLE_HEIGHT ){ + sqlite3_reset(pSelect); + return FTS_CORRUPT_VTAB; + } + + pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; + pWriter->iStart = iStart; + pWriter->iEnd = iEnd; + pWriter->iAbsLevel = iAbsLevel; + pWriter->iIdx = iIdx; + + for(i=nHeight+1; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + + pNode = &pWriter->aNodeWriter[nHeight]; + pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; + blobGrowBuffer(&pNode->block, + MAX(nRoot, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aRoot, nRoot); + pNode->block.n = nRoot; + memset(&pNode->block.a[nRoot], 0, FTS3_NODE_PADDING); + } + + for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ + NodeReader reader; + pNode = &pWriter->aNodeWriter[i]; + + if( pNode->block.a){ + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + assert_fts3_nc( reader.term.n>0 || reader.aNode==0 ); + if( reader.term.n>0 ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + } + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock,0); + blobGrowBuffer(&pNode->block, + MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; + memset(&pNode->block.a[nBlock], 0, FTS3_NODE_PADDING); + } + sqlite3_free(aBlock); + } + } + } + nodeReaderRelease(&reader); + } + } + + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** Determine the largest segment index value that exists within absolute +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus +** one before returning SQLITE_OK. Or, if there are no segments at all +** within level iAbsLevel, set *piIdx to zero. +** +** If an error occurs, return an SQLite error code. The final value of +** *piIdx is undefined in this case. +*/ +static int fts3IncrmergeOutputIdx( + Fts3Table *p, /* FTS Table handle */ + sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ + int *piIdx /* OUT: Next free index at iAbsLevel+1 */ +){ + int rc; + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ + + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + *piIdx = sqlite3_column_int(pOutputIdx, 0); + rc = sqlite3_reset(pOutputIdx); + } + + return rc; +} + +/* +** Allocate an appendable output segment on absolute level iAbsLevel+1 +** with idx value iIdx. +** +** In the %_segdir table, a segment is defined by the values in three +** columns: +** +** start_block +** leaves_end_block +** end_block +** +** When an appendable segment is allocated, it is estimated that the +** maximum number of leaf blocks that may be required is the sum of the +** number of leaf blocks consumed by the input segments, plus the number +** of input segments, multiplied by two. This value is stored in stack +** variable nLeafEst. +** +** A total of 16*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous +** array of leaf nodes starts at the first block allocated. The array +** of interior nodes that are parents of the leaf nodes start at block +** (start_block + (1 + end_block - start_block) / 16). And so on. +** +** In the actual code below, the value "16" is replaced with the +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. +*/ +static int fts3IncrmergeWriter( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of new output segment */ + Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ + IncrmergeWriter *pWriter /* Populate this object */ +){ + int rc; /* Return Code */ + int i; /* Iterator variable */ + int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ + sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ + + /* Calculate nLeafEst. */ + rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); + if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ + nLeafEst = sqlite3_column_int(pLeafEst, 0); + } + rc = sqlite3_reset(pLeafEst); + } + if( rc!=SQLITE_OK ) return rc; + + /* Calculate the first block to use in the output segment */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ + pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); + pWriter->iEnd = pWriter->iStart - 1; + pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; + } + rc = sqlite3_reset(pFirstBlock); + } + if( rc!=SQLITE_OK ) return rc; + + /* Insert the marker in the %_segments table to make sure nobody tries + ** to steal the space just allocated. This is also used to identify + ** appendable segments. */ + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; + + /* Set up the array of NodeWriter objects */ + for(i=0; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } + return SQLITE_OK; +} + +/* +** Remove an entry from the %_segdir table. This involves running the +** following two statements: +** +** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx +** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx +** +** The DELETE statement removes the specific %_segdir level. The UPDATE +** statement ensures that the remaining segments have contiguously allocated +** idx values. +*/ +static int fts3RemoveSegdirEntry( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ + int iIdx /* Index of %_segdir entry to delete */ +){ + int rc; /* Return code */ + sqlite3_stmt *pDelete = 0; /* DELETE statement */ + + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, iAbsLevel); + sqlite3_bind_int(pDelete, 2, iIdx); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); + } + + return rc; +} + +/* +** One or more segments have just been removed from absolute level iAbsLevel. +** Update the 'idx' values of the remaining segments in the level so that +** the idx values are a contiguous sequence starting from 0. +*/ +static int fts3RepackSegdirLevel( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel /* Absolute level to repack */ +){ + int rc; /* Return code */ + int *aIdx = 0; /* Array of remaining idx values */ + int nIdx = 0; /* Valid entries in aIdx[] */ + int nAlloc = 0; /* Allocated size of aIdx[] */ + int i; /* Iterator variable */ + sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ + sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ + + rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int64(pSelect, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pSelect) ){ + if( nIdx>=nAlloc ){ + int *aNew; + nAlloc += 16; + aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + if( !aNew ){ + rc = SQLITE_NOMEM; + break; + } + aIdx = aNew; + } + aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 2, iAbsLevel); + } + + assert( p->bIgnoreSavepoint==0 ); + p->bIgnoreSavepoint = 1; + for(i=0; rc==SQLITE_OK && ibIgnoreSavepoint = 0; + + sqlite3_free(aIdx); + return rc; +} + +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ + pNode->a[0] = (char)iHeight; + if( iChild ){ + assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); + pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); + }else{ + assert( pNode->nAlloc>=1 ); + pNode->n = 1; + } +} + +/* +** The first two arguments are a pointer to and the size of a segment b-tree +** node. The node may be a leaf or an internal node. +** +** This function creates a new node image in blob object *pNew by copying +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. +*/ +static int fts3TruncateNode( + const char *aNode, /* Current node image */ + int nNode, /* Size of aNode in bytes */ + Blob *pNew, /* OUT: Write new node image here */ + const char *zTerm, /* Omit all terms smaller than this */ + int nTerm, /* Size of zTerm in bytes */ + sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +){ + NodeReader reader; /* Reader object */ + Blob prev = {0, 0, 0}; /* Previous term written to new node */ + int rc = SQLITE_OK; /* Return code */ + int bLeaf; /* True for a leaf node */ + + if( nNode<1 ) return FTS_CORRUPT_VTAB; + bLeaf = aNode[0]=='\0'; + + /* Allocate required output space */ + blobGrowBuffer(pNew, nNode, &rc); + if( rc!=SQLITE_OK ) return rc; + pNew->n = 0; + + /* Populate new node buffer */ + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + if( pNew->n==0 ){ + int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); + if( res<0 || (bLeaf==0 && res==0) ) continue; + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + rc = fts3AppendToNode( + pNew, &prev, reader.term.a, reader.term.n, + reader.aDoclist, reader.nDoclist + ); + if( rc!=SQLITE_OK ) break; + } + if( pNew->n==0 ){ + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; + } + assert( pNew->n<=pNew->nAlloc ); + + nodeReaderRelease(&reader); + sqlite3_free(prev.a); + return rc; +} + +/* +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** level iAbsLevel. This may involve deleting entries from the %_segments +** table, and modifying existing entries in both the %_segments and %_segdir +** tables. +** +** SQLITE_OK is returned if the segment is updated successfully. Or an +** SQLite error code otherwise. +*/ +static int fts3TruncateSegment( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ + int iIdx, /* Index within level of segment to modify */ + const char *zTerm, /* Remove terms smaller than this */ + int nTerm /* Number of bytes in buffer zTerm */ +){ + int rc = SQLITE_OK; /* Return code */ + Blob root = {0,0,0}; /* New root page image */ + Blob block = {0,0,0}; /* Buffer used for any other block */ + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ + sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ + sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); + if( rc==SQLITE_OK ){ + int rc2; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + } + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; + } + + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; + + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); + if( rc==SQLITE_OK ){ + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); + } + sqlite3_free(aBlock); + } + + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); + sqlite3_bind_null(pChomp, 2); + } + } + + sqlite3_free(root.a); + sqlite3_free(block.a); + return rc; +} + +/* +** This function is called after an incrmental-merge operation has run to +** merge (or partially merge) two or more segments from absolute level +** iAbsLevel. +** +** Each input segment is either removed from the db completely (if all of +** its data was copied to the output segment by the incrmerge operation) +** or modified in place so that it no longer contains those entries that +** have been duplicated in the output segment. +*/ +static int fts3IncrmergeChomp( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ + Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ + int *pnRem /* Number of segments not deleted */ +){ + int i; + int nRem = 0; + int rc = SQLITE_OK; + + for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ + Fts3SegReader *pSeg = 0; + int j; + + /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding + ** somewhere in the pCsr->apSegment[] array. */ + for(j=0; ALWAYS(jnSegment); j++){ + pSeg = pCsr->apSegment[j]; + if( pSeg->iIdx==i ) break; + } + assert( jnSegment && pSeg->iIdx==i ); + + if( pSeg->aNode==0 ){ + /* Seg-reader is at EOF. Remove the entire input segment. */ + rc = fts3DeleteSegment(p, pSeg); + if( rc==SQLITE_OK ){ + rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + } + *pnRem = 0; + }else{ + /* The incremental merge did not copy all the data from this + ** segment to the upper level. The segment is modified in place + ** so that it contains no keys smaller than zTerm/nTerm. */ + const char *zTerm = pSeg->zTerm; + int nTerm = pSeg->nTerm; + rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + nRem++; + } + } + + if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ + rc = fts3RepackSegdirLevel(p, iAbsLevel); + } + + *pnRem = nRem; + return rc; +} + +/* +** Store an incr-merge hint in the database. +*/ +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pReplace = 0; + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); + } + + return rc; +} + +/* +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. +** +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. +*/ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pSelect = 0; + int rc; + + pHint->n = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *aHint = sqlite3_column_blob(pSelect, 0); + int nHint = sqlite3_column_bytes(pSelect, 0); + if( aHint ){ + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } + } + } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } + + return rc; +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. +*/ +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ +){ + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); + } +} + +/* +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. +** +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. +*/ +static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ + const int nHint = pHint->n; + int i; + + i = pHint->n-1; + if( (pHint->a[i] & 0x80) ) return FTS_CORRUPT_VTAB; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + if( i==0 ) return FTS_CORRUPT_VTAB; + i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += fts3GetVarint32(&pHint->a[i], pnInput); + assert( i<=nHint ); + if( i!=nHint ) return FTS_CORRUPT_VTAB; + + return SQLITE_OK; +} + + +/* +** Attempt an incremental merge that writes nMerge leaf blocks. +** +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. +*/ +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ + int rc; /* Return code */ + int nRem = nMerge; /* Number of leaf pages yet to be written */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ + int nSeg = 0; /* Number of input segments */ + sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ + + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; + + rc = fts3IncrmergeHintLoad(p, &hint); + while( rc==SQLITE_OK && nRem>0 ){ + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ + + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. + */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + nSeg = sqlite3_column_int(pFindLevel, 1); + assert( nSeg>=2 ); + }else{ + nSeg = -1; + } + rc = sqlite3_reset(pFindLevel); + + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + /* Based on the scan in the block above, it is known that there + ** are no levels with a relative level smaller than that of + ** iAbsLevel with more than nSeg segments, or if nSeg is -1, + ** no levels with more than nMin segments. Use this to limit the + ** value of nHintSeg to avoid a large memory allocation in case the + ** merge-hint is corrupt*/ + iAbsLevel = iHintAbsLevel; + nSeg = MIN(MAX(nMin,nSeg), nHintSeg); + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } + } + + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<=0 ) break; + + assert( nMod<=0x7FFFFFFF ); + if( iAbsLevel<0 || iAbsLevel>(nMod<<32) ){ + rc = FTS_CORRUPT_VTAB; + break; + } + + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using + ** the 'hint' parameters, it is possible that there are less than nSeg + ** segments available in level iAbsLevel. In this case, no work is + ** done on iAbsLevel - fall through to the next iteration of the loop + ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); + pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore = 0; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } + } + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + } + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + ){ + int bEmpty = 0; + rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( rc==SQLITE_OK ){ + bEmpty = 1; + }else if( rc!=SQLITE_ROW ){ + sqlite3Fts3SegReaderFinish(pCsr); + break; + } + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); + } + + if( rc==SQLITE_OK && pWriter->nLeafEst ){ + fts3LogMerge(nSeg, iAbsLevel); + if( bEmpty==0 ){ + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + } + + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + nRem -= (1 + pWriter->nWork); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); + } + } + } + + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } + } + + sqlite3Fts3SegReaderFinish(pCsr); + } + + /* Write the hint values into the %_stat table for the next incr-merger */ + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); + } + + sqlite3_free(pWriter); + sqlite3_free(hint.a); + return rc; +} + +/* +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +** +** This function used for parameters to merge= and incrmerge= +** commands. +*/ +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('merge=A,B'); +** +** A and B are integers that decode to be the number of leaf pages +** written for the merge, and the minimum number of segments on a level +** before it will be selected for a merge, respectively. +*/ +static int fts3DoIncrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing "A,B" */ +){ + int rc; + int nMin = (MergeCount(p) / 2); + int nMerge = 0; + const char *z = zParam; + + /* Read the first integer value */ + nMerge = fts3Getint(&z); + + /* If the first integer value is followed by a ',', read the second + ** integer value. */ + if( z[0]==',' && z[1]!='\0' ){ + z++; + nMin = fts3Getint(&z); + } + + if( z[0]!='\0' || nMin<2 ){ + rc = SQLITE_ERROR; + }else{ + rc = SQLITE_OK; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); + } + sqlite3Fts3SegmentsClose(p); + } + return rc; +} + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. +*/ +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>MergeCount(p) ){ + p->nAutoincrmerge = 8; + } + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + if( rc ) return rc; + } + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; +} + +/* +** Return a 64-bit checksum for the FTS index entry specified by the +** arguments to this function. +*/ +static u64 fts3ChecksumEntry( + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of zTerm in bytes */ + int iLangid, /* Language id for current row */ + int iIndex, /* Index (0..Fts3Table.nIndex-1) */ + i64 iDocid, /* Docid for current row. */ + int iCol, /* Column number */ + int iPos /* Position */ +){ + int i; + u64 ret = (u64)iDocid; + + ret += (ret<<3) + iLangid; + ret += (ret<<3) + iIndex; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + for(i=0; inIndex-1) */ + int *pRc /* OUT: Return code */ +){ + Fts3SegFilter filter; + Fts3MultiSegReader csr; + int rc; + u64 cksum = 0; + + assert( *pRc==SQLITE_OK ); + + memset(&filter, 0, sizeof(filter)); + memset(&csr, 0, sizeof(csr)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + filter.flags |= FTS3_SEGMENT_SCAN; + + rc = sqlite3Fts3SegReaderCursor( + p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + } + + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; + + i64 iDocid = 0; + i64 iCol = 0; + u64 iPos = 0; + + pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); + while( pCsrbDescIdx ){ + iDocid = (i64)((u64)iDocid - iVal); + }else{ + iDocid = (i64)((u64)iDocid + iVal); + } + } + }else{ + iPos += (iVal - 2); + cksum = cksum ^ fts3ChecksumEntry( + csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid, + (int)iCol, (int)iPos + ); + } + } + } + } + } + sqlite3Fts3SegReaderFinish(&csr); + + *pRc = rc; + return cksum; +} + +/* +** Check if the contents of the FTS index match the current contents of the +** content table. If no error occurs and the contents do match, set *pbOk +** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk +** to false before returning. +** +** If an error occurs (e.g. an OOM or IO error), return an SQLite error +** code. The final value of *pbOk is undefined in this case. +*/ +static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ + int rc = SQLITE_OK; /* Return code */ + u64 cksum1 = 0; /* Checksum based on FTS index contents */ + u64 cksum2 = 0; /* Checksum based on %_content contents */ + sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ + + /* This block calculates the checksum according to the FTS index. */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int iLangid = sqlite3_column_int(pAllLangid, 0); + int i; + for(i=0; inIndex; i++){ + cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + } + } + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* This block calculates the checksum according to the %_content table */ + if( rc==SQLITE_OK ){ + sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; + sqlite3_stmt *pStmt = 0; + char *zSql; + + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iDocid = sqlite3_column_int64(pStmt, 0); + int iLang = langidFromSelect(p, pStmt); + int iCol; + + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, -1, &pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } + } + } + } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + } + + sqlite3_finalize(pStmt); + } + + *pbOk = (cksum1==cksum2); + return rc; +} + +/* +** Run the integrity-check. If no error occurs and the current contents of +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. +** +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** error code. +** +** The integrity-check works as follows. For each token and indexed token +** prefix in the document set, a 64-bit checksum is calculated (by code +** in fts3ChecksumEntry()) based on the following: +** +** + The index number (0 for the main index, 1 for the first prefix +** index etc.), +** + The token (or token prefix) text itself, +** + The language-id of the row it appears in, +** + The docid of the row it appears in, +** + The column it appears in, and +** + The tokens position within that column. +** +** The checksums for all entries in the index are XORed together to create +** a single checksum for the entire index. +** +** The integrity-check code calculates the same checksum in two ways: +** +** 1. By scanning the contents of the FTS index, and +** 2. By scanning and tokenizing the content table. +** +** If the two checksums are identical, the integrity-check is deemed to have +** passed. +*/ +static int fts3DoIntegrityCheck( + Fts3Table *p /* FTS3 table handle */ +){ + int rc; + int bOk = 0; + rc = fts3IntegrityCheck(p, &bOk); + if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; + return rc; +} + +/* +** Handle a 'special' INSERT of the form: +** +** "INSERT INTO tbl(tbl) VALUES()" +** +** Argument pVal contains the result of . Currently the only +** meaningful value to insert is the text 'optimize'. +*/ +static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ + int rc = SQLITE_ERROR; /* Return Code */ + const char *zVal = (const char *)sqlite3_value_text(pVal); + int nVal = sqlite3_value_bytes(pVal); + + if( !zVal ){ + return SQLITE_NOMEM; + }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ + rc = fts3DoOptimize(p, 0); + }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ + rc = fts3DoRebuild(p); + }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ + rc = fts3DoIntegrityCheck(p); + }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ + rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + }else{ + int v; + if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ + v = atoi(&zVal[9]); + if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v; + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + v = atoi(&zVal[11]); + if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v; + rc = SQLITE_OK; + }else if( nVal>21 && 0==sqlite3_strnicmp(zVal,"test-no-incr-doclist=",21) ){ + p->bNoIncrDoclist = atoi(&zVal[21]); + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal,"mergecount=",11) ){ + v = atoi(&zVal[11]); + if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v; + rc = SQLITE_OK; + } +#endif + } + return rc; +} + +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +/* +** Delete all cached deferred doclists. Deferred doclists are cached +** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ + fts3PendingListDelete(pDef->pList); + pDef->pList = 0; + } +} + +/* +** Free all entries in the pCsr->pDeffered list. Entries are added to +** this list using sqlite3Fts3DeferToken(). +*/ +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + Fts3DeferredToken *pNext; + for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ + pNext = pDef->pNext; + fts3PendingListDelete(pDef->pList); + sqlite3_free(pDef); + } + pCsr->pDeferred = 0; +} + +/* +** Generate deferred-doclists for all tokens in the pCsr->pDeferred list +** based on the row that pCsr currently points to. +** +** A deferred-doclist is like any other doclist with position information +** included, except that it only contains entries for a single row of the +** table, not for all rows. +*/ +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return code */ + if( pCsr->pDeferred ){ + int i; /* Used to iterate through table columns */ + sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ + Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ + + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer *pT = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pT->pModule; + + assert( pCsr->isRequireSeek==0 ); + iDocid = sqlite3_column_int64(pCsr->pStmt, 0); + + for(i=0; inColumn && rc==SQLITE_OK; i++){ + if( p->abNotindexed[i]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; + + rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + } + } + } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } + + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + if( pDef->pList ){ + rc = fts3PendingListAppendVarint(&pDef->pList, 0); + } + } + } + + return rc; +} + +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ + return SQLITE_OK; + } + + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} + +/* +** Add an entry for token pToken to the pCsr->pDeferred list. +*/ +SQLITE_PRIVATE int sqlite3Fts3DeferToken( + Fts3Cursor *pCsr, /* Fts3 table cursor */ + Fts3PhraseToken *pToken, /* Token to defer */ + int iCol /* Column that token must appear in (or -1) */ +){ + Fts3DeferredToken *pDeferred; + pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + if( !pDeferred ){ + return SQLITE_NOMEM; + } + memset(pDeferred, 0, sizeof(*pDeferred)); + pDeferred->pToken = pToken; + pDeferred->pNext = pCsr->pDeferred; + pDeferred->iCol = iCol; + pCsr->pDeferred = pDeferred; + + assert( pToken->pDeferred==0 ); + pToken->pDeferred = pDeferred; + + return SQLITE_OK; +} +#endif + +/* +** SQLite value pRowid contains the rowid of a row that may or may not be +** present in the FTS3 table. If it is, delete it and adjust the contents +** of subsiduary data structures accordingly. +*/ +static int fts3DeleteByRowid( + Fts3Table *p, + sqlite3_value *pRowid, + int *pnChng, /* IN/OUT: Decrement if row is deleted */ + u32 *aSzDel +){ + int rc = SQLITE_OK; /* Return code */ + int bFound = 0; /* True if *pRowid really is in the table */ + + fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); + if( bFound && rc==SQLITE_OK ){ + int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ + rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p, 1); + *pnChng = 0; + memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); + }else{ + *pnChng = *pnChng - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + } + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } + } + } + } + + return rc; +} + +/* +** This function does the work for the xUpdate method of FTS3 virtual +** tables. The schema of the virtual table being: +** +** CREATE TABLE
    ( +** , +**
    HIDDEN, +** docid HIDDEN, +** HIDDEN +** ); +** +** +*/ +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( + sqlite3_vtab *pVtab, /* FTS3 vtab object */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return Code */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ + u32 *aSzDel = 0; /* Sizes of deleted documents */ + int nChng = 0; /* Net change in number of documents */ + int bInsertDone = 0; + + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); + + assert( p->pSegments==0 ); + assert( + nArg==1 /* DELETE operations */ + || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ + ); + + /* Check for a "special" INSERT operation. One of the form: + ** + ** INSERT INTO xyz(xyz) VALUES('command'); + */ + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + ){ + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; + } + + if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ + rc = SQLITE_CONSTRAINT; + goto update_out; + } + + /* Allocate space to hold the change in document sizes */ + aSzDel = sqlite3_malloc64(sizeof(aSzDel[0])*((sqlite3_int64)p->nColumn+1)*2); + if( aSzDel==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } + aSzIns = &aSzDel[p->nColumn+1]; + memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); + + rc = fts3Writelock(p); + if( rc!=SQLITE_OK ) goto update_out; + + /* If this is an INSERT operation, or an UPDATE that modifies the rowid + ** value, then this operation requires constraint handling. + ** + ** If the on-conflict mode is REPLACE, this means that the existing row + ** should be deleted from the database before inserting the new row. Or, + ** if the on-conflict mode is other than REPLACE, then this method must + ** detect the conflict and return SQLITE_CONSTRAINT before beginning to + ** modify the database file. + */ + if( nArg>1 && p->zContentTbl==0 ){ + /* Find the value object that holds the new rowid value. */ + sqlite3_value *pNewRowid = apVal[3+p->nColumn]; + if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ + pNewRowid = apVal[1]; + } + + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + sqlite3_value_type(apVal[0])==SQLITE_NULL + || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) + )){ + /* The new rowid is not NULL (in this case the rowid will be + ** automatically assigned and there is no chance of a conflict), and + ** the statement is either an INSERT or an UPDATE that modifies the + ** rowid column. So if the conflict mode is REPLACE, then delete any + ** existing row with rowid=pNewRowid. + ** + ** Or, if the conflict mode is not REPLACE, insert the new record into + ** the %_content table. If we hit the duplicate rowid constraint (or any + ** other error) while doing so, return immediately. + ** + ** This branch may also run if pNewRowid contains a value that cannot + ** be losslessly converted to an integer. In this case, the eventual + ** call to fts3InsertData() (either just below or further on in this + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** invoked, it will delete zero rows (since no row will have + ** docid=$pNewRowid if $pNewRowid is not an integer value). + */ + if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ + rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); + }else{ + rc = fts3InsertData(p, apVal, pRowid); + bInsertDone = 1; + } + } + } + if( rc!=SQLITE_OK ){ + goto update_out; + } + + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); + rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); + } + + /* If this is an INSERT or UPDATE operation, insert the new record. */ + if( nArg>1 && rc==SQLITE_OK ){ + int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); + if( bInsertDone==0 ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } + } + if( rc==SQLITE_OK ){ + rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); + } + if( rc==SQLITE_OK ){ + assert( p->iPrevDocid==*pRowid ); + rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSzIns); + } + nChng++; + } + + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); + } + + update_out: + sqlite3_free(aSzDel); + sqlite3Fts3SegmentsClose(p); + return rc; +} + +/* +** Flush any data in the pending-terms hash table to disk. If successful, +** merge all segments in the database (including the new segment, if +** there was any data to flush) into a single segment. +*/ +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ + int rc; + rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; + }else{ + sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); + sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + } + } + sqlite3Fts3SegmentsClose(p); + return rc; +} + +#endif + +/************** End of fts3_write.c ******************************************/ +/************** Begin file fts3_snippet.c ************************************/ +/* +** 2009 Oct 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ + +#ifndef SQLITE_AMALGAMATION +typedef sqlite3_int64 i64; +#endif + +/* +** Characters that may appear in the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ +#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ +#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ +#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ +#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ +#define FTS3_MATCHINFO_LCS 's' /* nCol values */ +#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ + +/* +** The default value for the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_DEFAULT "pcx" + + +/* +** Used as an fts3ExprIterate() context when loading phrase doclists to +** Fts3Expr.aDoclist[]/nDoclist. +*/ +typedef struct LoadDoclistCtx LoadDoclistCtx; +struct LoadDoclistCtx { + Fts3Cursor *pCsr; /* FTS3 Cursor */ + int nPhrase; /* Number of phrases seen so far */ + int nToken; /* Number of tokens seen so far */ +}; + +/* +** The following types are used as part of the implementation of the +** fts3BestSnippet() routine. +*/ +typedef struct SnippetIter SnippetIter; +typedef struct SnippetPhrase SnippetPhrase; +typedef struct SnippetFragment SnippetFragment; + +struct SnippetIter { + Fts3Cursor *pCsr; /* Cursor snippet is being generated from */ + int iCol; /* Extract snippet from this column */ + int nSnippet; /* Requested snippet length (in tokens) */ + int nPhrase; /* Number of phrases in query */ + SnippetPhrase *aPhrase; /* Array of size nPhrase */ + int iCurrent; /* First token of current snippet */ +}; + +struct SnippetPhrase { + int nToken; /* Number of tokens in phrase */ + char *pList; /* Pointer to start of phrase position list */ + i64 iHead; /* Next value in position list */ + char *pHead; /* Position list data following iHead */ + i64 iTail; /* Next value in trailing position list */ + char *pTail; /* Position list data following iTail */ +}; + +struct SnippetFragment { + int iCol; /* Column snippet is extracted from */ + int iPos; /* Index of first token in snippet */ + u64 covered; /* Mask of query phrases covered */ + u64 hlmask; /* Mask of snippet terms to highlight */ +}; + +/* +** This type is used as an fts3ExprIterate() context object while +** accumulating the data returned by the matchinfo() function. +*/ +typedef struct MatchInfo MatchInfo; +struct MatchInfo { + Fts3Cursor *pCursor; /* FTS3 Cursor */ + int nCol; /* Number of columns in table */ + int nPhrase; /* Number of matchable phrases in query */ + sqlite3_int64 nDoc; /* Number of docs in database */ + char flag; + u32 *aMatchinfo; /* Pre-allocated buffer */ +}; + +/* +** An instance of this structure is used to manage a pair of buffers, each +** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below +** for details. +*/ +struct MatchinfoBuffer { + u8 aRef[3]; + int nElem; + int bGlobal; /* Set if global data is loaded */ + char *zMatchinfo; + u32 aMatchinfo[1]; +}; + + +/* +** The snippet() and offsets() functions both return text values. An instance +** of the following structure is used to accumulate those values while the +** functions are running. See fts3StringAppend() for details. +*/ +typedef struct StrBuffer StrBuffer; +struct StrBuffer { + char *z; /* Pointer to buffer containing string */ + int n; /* Length of z in bytes (excl. nul-term) */ + int nAlloc; /* Allocated size of buffer z in bytes */ +}; + + +/************************************************************************* +** Start of MatchinfoBuffer code. +*/ + +/* +** Allocate a two-slot MatchinfoBuffer object. +*/ +static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ + MatchinfoBuffer *pRet; + sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1) + + sizeof(MatchinfoBuffer); + sqlite3_int64 nStr = strlen(zMatchinfo); + + pRet = sqlite3_malloc64(nByte + nStr+1); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + + sizeof(u32)*((int)nElem+1); + pRet->nElem = (int)nElem; + pRet->zMatchinfo = ((char*)pRet) + nByte; + memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); + pRet->aRef[0] = 1; + } + + return pRet; +} + +static void fts3MIBufferFree(void *p){ + MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); + + assert( (u32*)p==&pBuf->aMatchinfo[1] + || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + ); + if( (u32*)p==&pBuf->aMatchinfo[1] ){ + pBuf->aRef[1] = 0; + }else{ + pBuf->aRef[2] = 0; + } + + if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ + sqlite3_free(pBuf); + } +} + +static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ + void (*xRet)(void*) = 0; + u32 *aOut = 0; + + if( p->aRef[1]==0 ){ + p->aRef[1] = 1; + aOut = &p->aMatchinfo[1]; + xRet = fts3MIBufferFree; + } + else if( p->aRef[2]==0 ){ + p->aRef[2] = 1; + aOut = &p->aMatchinfo[p->nElem+2]; + xRet = fts3MIBufferFree; + }else{ + aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32)); + if( aOut ){ + xRet = sqlite3_free; + if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); + } + } + + *paOut = aOut; + return xRet; +} + +static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ + p->bGlobal = 1; + memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); +} + +/* +** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() +*/ +SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ + if( p ){ + assert( p->aRef[0]==1 ); + p->aRef[0] = 0; + if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ + sqlite3_free(p); + } + } +} + +/* +** End of MatchinfoBuffer code. +*************************************************************************/ + + +/* +** This function is used to help iterate through a position-list. A position +** list is a list of unique integers, sorted from smallest to largest. Each +** element of the list is represented by an FTS3 varint that takes the value +** of the difference between the current element and the previous one plus +** two. For example, to store the position-list: +** +** 4 9 113 +** +** the three varints: +** +** 6 7 106 +** +** are encoded. +** +** When this function is called, *pp points to the start of an element of +** the list. *piPos contains the value of the previous entry in the list. +** After it returns, *piPos contains the value of the next element of the +** list and *pp is advanced to the following varint. +*/ +static void fts3GetDeltaPosition(char **pp, i64 *piPos){ + int iVal; + *pp += fts3GetVarint32(*pp, &iVal); + *piPos += (iVal-2); +} + +/* +** Helper function for fts3ExprIterate() (see below). +*/ +static int fts3ExprIterate2( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int *piPhrase, /* Pointer to phrase counter */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int rc; /* Return code */ + int eType = pExpr->eType; /* Type of expression node pExpr */ + + if( eType!=FTSQUERY_PHRASE ){ + assert( pExpr->pLeft && pExpr->pRight ); + rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); + if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ + rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); + } + }else{ + rc = x(pExpr, *piPhrase, pCtx); + (*piPhrase)++; + } + return rc; +} + +/* +** Iterate through all phrase nodes in an FTS3 query, except those that +** are part of a sub-tree that is the right-hand-side of a NOT operator. +** For each phrase node found, the supplied callback function is invoked. +** +** If the callback function returns anything other than SQLITE_OK, +** the iteration is abandoned and the error code returned immediately. +** Otherwise, SQLITE_OK is returned after a callback has been made for +** all eligible phrase nodes. +*/ +static int fts3ExprIterate( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int iPhrase = 0; /* Variable used as the phrase counter */ + return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); +} + + +/* +** This is an fts3ExprIterate() callback used while loading the doclists +** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** fts3ExprLoadDoclists(). +*/ +static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; + LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; + + UNUSED_PARAMETER(iPhrase); + + p->nPhrase++; + p->nToken += pPhrase->nToken; + + return rc; +} + +/* +** Load the doclists for each phrase in the query associated with FTS3 cursor +** pCsr. +** +** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable +** phrases in the expression (all phrases except those directly or +** indirectly descended from the right-hand-side of a NOT operator). If +** pnToken is not NULL, then it is set to the number of tokens in all +** matchable phrases of the expression. +*/ +static int fts3ExprLoadDoclists( + Fts3Cursor *pCsr, /* Fts3 cursor for current query */ + int *pnPhrase, /* OUT: Number of phrases in query */ + int *pnToken /* OUT: Number of tokens in query */ +){ + int rc; /* Return Code */ + LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ + sCtx.pCsr = pCsr; + rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); + if( pnPhrase ) *pnPhrase = sCtx.nPhrase; + if( pnToken ) *pnToken = sCtx.nToken; + return rc; +} + +static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + (*(int *)ctx)++; + pExpr->iPhrase = iPhrase; + return SQLITE_OK; +} +static int fts3ExprPhraseCount(Fts3Expr *pExpr){ + int nPhrase = 0; + (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + return nPhrase; +} + +/* +** Advance the position list iterator specified by the first two +** arguments so that it points to the first element with a value greater +** than or equal to parameter iNext. +*/ +static void fts3SnippetAdvance(char **ppIter, i64 *piIter, int iNext){ + char *pIter = *ppIter; + if( pIter ){ + i64 iIter = *piIter; + + while( iIteriCurrent<0 ){ + /* The SnippetIter object has just been initialized. The first snippet + ** candidate always starts at offset 0 (even if this candidate has a + ** score of 0.0). + */ + pIter->iCurrent = 0; + + /* Advance the 'head' iterator of each phrase to the first offset that + ** is greater than or equal to (iNext+nSnippet). + */ + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet); + } + }else{ + int iStart; + int iEnd = 0x7FFFFFFF; + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pHead && pPhrase->iHeadiHead; + } + } + if( iEnd==0x7FFFFFFF ){ + return 1; + } + + pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1); + fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart); + } + } + + return 0; +} + +/* +** Retrieve information about the current candidate snippet of snippet +** iterator pIter. +*/ +static void fts3SnippetDetails( + SnippetIter *pIter, /* Snippet iterator */ + u64 mCovered, /* Bitmask of phrases already covered */ + int *piToken, /* OUT: First token of proposed snippet */ + int *piScore, /* OUT: "Score" for this snippet */ + u64 *pmCover, /* OUT: Bitmask of phrases covered */ + u64 *pmHighlight /* OUT: Bitmask of terms to highlight */ +){ + int iStart = pIter->iCurrent; /* First token of snippet */ + int iScore = 0; /* Score of this snippet */ + int i; /* Loop counter */ + u64 mCover = 0; /* Mask of phrases covered by this snippet */ + u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */ + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pTail ){ + char *pCsr = pPhrase->pTail; + i64 iCsr = pPhrase->iTail; + + while( iCsr<(iStart+pIter->nSnippet) && iCsr>=iStart ){ + int j; + u64 mPhrase = (u64)1 << (i%64); + u64 mPos = (u64)1 << (iCsr - iStart); + assert( iCsr>=iStart && (iCsr - iStart)<=64 ); + assert( i>=0 ); + if( (mCover|mCovered)&mPhrase ){ + iScore++; + }else{ + iScore += 1000; + } + mCover |= mPhrase; + + for(j=0; jnToken; j++){ + mHighlight |= (mPos>>j); + } + + if( 0==(*pCsr & 0x0FE) ) break; + fts3GetDeltaPosition(&pCsr, &iCsr); + } + } + } + + /* Set the output variables before returning. */ + *piToken = iStart; + *piScore = iScore; + *pmCover = mCover; + *pmHighlight = mHighlight; +} + +/* +** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). +** Each invocation populates an element of the SnippetIter.aPhrase[] array. +*/ +static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ + SnippetIter *p = (SnippetIter *)ctx; + SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; + char *pCsr; + int rc; + + pPhrase->nToken = pExpr->pPhrase->nToken; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( pCsr ){ + i64 iFirst = 0; + pPhrase->pList = pCsr; + fts3GetDeltaPosition(&pCsr, &iFirst); + if( iFirst<0 ){ + rc = FTS_CORRUPT_VTAB; + }else{ + pPhrase->pHead = pCsr; + pPhrase->pTail = pCsr; + pPhrase->iHead = iFirst; + pPhrase->iTail = iFirst; + } + }else{ + assert( rc!=SQLITE_OK || ( + pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 + )); + } + + return rc; +} + +/* +** Select the fragment of text consisting of nFragment contiguous tokens +** from column iCol that represent the "best" snippet. The best snippet +** is the snippet with the highest score, where scores are calculated +** by adding: +** +** (a) +1 point for each occurrence of a matchable phrase in the snippet. +** +** (b) +1000 points for the first occurrence of each matchable phrase in +** the snippet for which the corresponding mCovered bit is not set. +** +** The selected snippet parameters are stored in structure *pFragment before +** returning. The score of the selected snippet is stored in *piScore +** before returning. +*/ +static int fts3BestSnippet( + int nSnippet, /* Desired snippet length */ + Fts3Cursor *pCsr, /* Cursor to create snippet for */ + int iCol, /* Index of column to create snippet from */ + u64 mCovered, /* Mask of phrases already covered */ + u64 *pmSeen, /* IN/OUT: Mask of phrases seen */ + SnippetFragment *pFragment, /* OUT: Best snippet found */ + int *piScore /* OUT: Score of snippet pFragment */ +){ + int rc; /* Return Code */ + int nList; /* Number of phrases in expression */ + SnippetIter sIter; /* Iterates through snippet candidates */ + sqlite3_int64 nByte; /* Number of bytes of space to allocate */ + int iBestScore = -1; /* Best snippet score found so far */ + int i; /* Loop counter */ + + memset(&sIter, 0, sizeof(sIter)); + + /* Iterate through the phrases in the expression to count them. The same + ** callback makes sure the doclists are loaded for each phrase. + */ + rc = fts3ExprLoadDoclists(pCsr, &nList, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Now that it is known how many phrases there are, allocate and zero + ** the required space using malloc(). + */ + nByte = sizeof(SnippetPhrase) * nList; + sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc64(nByte); + if( !sIter.aPhrase ){ + return SQLITE_NOMEM; + } + memset(sIter.aPhrase, 0, nByte); + + /* Initialize the contents of the SnippetIter object. Then iterate through + ** the set of phrases in the expression to populate the aPhrase[] array. + */ + sIter.pCsr = pCsr; + sIter.iCol = iCol; + sIter.nSnippet = nSnippet; + sIter.nPhrase = nList; + sIter.iCurrent = -1; + rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); + if( rc==SQLITE_OK ){ + + /* Set the *pmSeen output variable. */ + for(i=0; iiCol = iCol; + while( !fts3SnippetNextCandidate(&sIter) ){ + int iPos; + int iScore; + u64 mCover; + u64 mHighlite; + fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite); + assert( iScore>=0 ); + if( iScore>iBestScore ){ + pFragment->iPos = iPos; + pFragment->hlmask = mHighlite; + pFragment->covered = mCover; + iBestScore = iScore; + } + } + + *piScore = iBestScore; + } + sqlite3_free(sIter.aPhrase); + return rc; +} + + +/* +** Append a string to the string-buffer passed as the first argument. +** +** If nAppend is negative, then the length of the string zAppend is +** determined using strlen(). +*/ +static int fts3StringAppend( + StrBuffer *pStr, /* Buffer to append to */ + const char *zAppend, /* Pointer to data to append to buffer */ + int nAppend /* Size of zAppend in bytes (or -1) */ +){ + if( nAppend<0 ){ + nAppend = (int)strlen(zAppend); + } + + /* If there is insufficient space allocated at StrBuffer.z, use realloc() + ** to grow the buffer until so that it is big enough to accomadate the + ** appended data. + */ + if( pStr->n+nAppend+1>=pStr->nAlloc ){ + sqlite3_int64 nAlloc = pStr->nAlloc+(sqlite3_int64)nAppend+100; + char *zNew = sqlite3_realloc64(pStr->z, nAlloc); + if( !zNew ){ + return SQLITE_NOMEM; + } + pStr->z = zNew; + pStr->nAlloc = nAlloc; + } + assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); + + /* Append the data to the string buffer. */ + memcpy(&pStr->z[pStr->n], zAppend, nAppend); + pStr->n += nAppend; + pStr->z[pStr->n] = '\0'; + + return SQLITE_OK; +} + +/* +** The fts3BestSnippet() function often selects snippets that end with a +** query term. That is, the final term of the snippet is always a term +** that requires highlighting. For example, if 'X' is a highlighted term +** and '.' is a non-highlighted term, BestSnippet() may select: +** +** ........X.....X +** +** This function "shifts" the beginning of the snippet forward in the +** document so that there are approximately the same number of +** non-highlighted terms to the right of the final highlighted term as there +** are to the left of the first highlighted term. For example, to this: +** +** ....X.....X.... +** +** This is done as part of extracting the snippet text, not when selecting +** the snippet. Snippet selection is done based on doclists only, so there +** is no way for fts3BestSnippet() to know whether or not the document +** actually contains terms that follow the final highlighted term. +*/ +static int fts3SnippetShift( + Fts3Table *pTab, /* FTS3 table snippet comes from */ + int iLangid, /* Language id to use in tokenizing */ + int nSnippet, /* Number of tokens desired for snippet */ + const char *zDoc, /* Document text to extract snippet from */ + int nDoc, /* Size of buffer zDoc in bytes */ + int *piPos, /* IN/OUT: First token of snippet */ + u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */ +){ + u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */ + + if( hlmask ){ + int nLeft; /* Tokens to the left of first highlight */ + int nRight; /* Tokens to the right of last highlight */ + int nDesired; /* Ideal number of tokens to shift forward */ + + for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); + for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); + assert( (nSnippet-1-nRight)<=63 && (nSnippet-1-nRight)>=0 ); + nDesired = (nLeft-nRight)/2; + + /* Ideally, the start of the snippet should be pushed forward in the + ** document nDesired tokens. This block checks if there are actually + ** nDesired tokens to the right of the snippet. If so, *piPos and + ** *pHlMask are updated to shift the snippet nDesired tokens to the + ** right. Otherwise, the snippet is shifted by the number of tokens + ** available. + */ + if( nDesired>0 ){ + int nShift; /* Number of tokens to shift snippet by */ + int iCurrent = 0; /* Token counter */ + int rc; /* Return Code */ + sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer_cursor *pC; + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + + /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) + ** or more tokens in zDoc/nDoc. + */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); + if( rc!=SQLITE_OK ){ + return rc; + } + while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ + const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); + } + pMod->xClose(pC); + if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } + + nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; + assert( nShift<=nDesired ); + if( nShift>0 ){ + *piPos += nShift; + *pHlmask = hlmask >> nShift; + } + } + } + return SQLITE_OK; +} + +/* +** Extract the snippet text for fragment pFragment from cursor pCsr and +** append it to string buffer pOut. +*/ +static int fts3SnippetText( + Fts3Cursor *pCsr, /* FTS3 Cursor */ + SnippetFragment *pFragment, /* Snippet to extract */ + int iFragment, /* Fragment number */ + int isLast, /* True for final fragment in snippet */ + int nSnippet, /* Number of tokens in extracted snippet */ + const char *zOpen, /* String inserted before highlighted term */ + const char *zClose, /* String inserted after highlighted term */ + const char *zEllipsis, /* String inserted between snippets */ + StrBuffer *pOut /* Write output here */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc; /* Return code */ + const char *zDoc; /* Document text to extract snippet from */ + int nDoc; /* Size of zDoc in bytes */ + int iCurrent = 0; /* Current token number of document */ + int iEnd = 0; /* Byte offset of end of current token */ + int isShiftDone = 0; /* True after snippet is shifted */ + int iPos = pFragment->iPos; /* First token of snippet */ + u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */ + int iCol = pFragment->iCol+1; /* Query column to extract text from */ + sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ + + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } + return SQLITE_OK; + } + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol); + + /* Open a token cursor on the document. */ + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); + if( rc!=SQLITE_OK ){ + return rc; + } + + while( rc==SQLITE_OK ){ + const char *ZDUMMY; /* Dummy argument used with tokenizer */ + int DUMMY1 = -1; /* Dummy argument used with tokenizer */ + int iBegin = 0; /* Offset in zDoc of start of token */ + int iFin = 0; /* Offset in zDoc of end of token */ + int isHighlight = 0; /* True for highlighted terms */ + + /* Variable DUMMY1 is initialized to a negative value above. Elsewhere + ** in the FTS code the variable that the third argument to xNext points to + ** is initialized to zero before the first (*but not necessarily + ** subsequent*) call to xNext(). This is done for a particular application + ** that needs to know whether or not the tokenizer is being used for + ** snippet generation or for some other purpose. + ** + ** Extreme care is required when writing code to depend on this + ** initialization. It is not a documented part of the tokenizer interface. + ** If a tokenizer is used directly by any code outside of FTS, this + ** convention might not be respected. */ + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + /* Special case - the last token of the snippet is also the last token + ** of the column. Append any punctuation that occurred between the end + ** of the previous token and the end of the document to the output. + ** Then break out of the loop. */ + rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); + } + break; + } + if( iCurrentiLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask + ); + isShiftDone = 1; + + /* Now that the shift has been done, check if the initial "..." are + ** required. They are required if (a) this is not the first fragment, + ** or (b) this fragment does not begin at position 0 of its column. + */ + if( rc==SQLITE_OK ){ + if( iPos>0 || iFragment>0 ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + }else if( iBegin ){ + rc = fts3StringAppend(pOut, zDoc, iBegin); + } + } + if( rc!=SQLITE_OK || iCurrent=(iPos+nSnippet) ){ + if( isLast ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + } + break; + } + + /* Set isHighlight to true if this term should be highlighted. */ + isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; + + if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); + if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); + + iEnd = iFin; + } + + pMod->xClose(pC); + return rc; +} + + +/* +** This function is used to count the entries in a column-list (a +** delta-encoded list of term offsets within a single column of a single +** row). When this function is called, *ppCollist should point to the +** beginning of the first varint in the column-list (the varint that +** contains the position of the first matching term in the column data). +** Before returning, *ppCollist is set to point to the first byte after +** the last varint in the column-list (either the 0x00 signifying the end +** of the position-list, or the 0x01 that precedes the column number of +** the next column in the position-list). +** +** The number of elements in the column-list is returned. +*/ +static int fts3ColumnlistCount(char **ppCollist){ + char *pEnd = *ppCollist; + char c = 0; + int nEntry = 0; + + /* A column-list is terminated by either a 0x01 or 0x00. */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + if( !c ) nEntry++; + } + + *ppCollist = pEnd; + return nEntry; +} + +/* +** This function gathers 'y' or 'b' data for a single phrase. +*/ +static int fts3ExprLHits( + Fts3Expr *pExpr, /* Phrase expression node */ + MatchInfo *p /* Matchinfo context */ +){ + Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; + int iStart; + Fts3Phrase *pPhrase = pExpr->pPhrase; + char *pIter = pPhrase->doclist.pList; + int iCol = 0; + + assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); + if( p->flag==FTS3_MATCHINFO_LHITS ){ + iStart = pExpr->iPhrase * p->nCol; + }else{ + iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); + } + + if( pIter ) while( 1 ){ + int nHit = fts3ColumnlistCount(&pIter); + if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ + if( p->flag==FTS3_MATCHINFO_LHITS ){ + p->aMatchinfo[iStart + iCol] = (u32)nHit; + }else if( nHit ){ + p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); + } + } + assert( *pIter==0x00 || *pIter==0x01 ); + if( *pIter!=0x01 ) break; + pIter++; + pIter += fts3GetVarint32(pIter, &iCol); + if( iCol>=p->nCol ) return FTS_CORRUPT_VTAB; + } + return SQLITE_OK; +} + +/* +** Gather the results for matchinfo directives 'y' and 'b'. +*/ +static int fts3ExprLHitGather( + Fts3Expr *pExpr, + MatchInfo *p +){ + int rc = SQLITE_OK; + assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); + if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ + if( pExpr->pLeft ){ + rc = fts3ExprLHitGather(pExpr->pLeft, p); + if( rc==SQLITE_OK ) rc = fts3ExprLHitGather(pExpr->pRight, p); + }else{ + rc = fts3ExprLHits(pExpr, p); + } + } + return rc; +} + +/* +** fts3ExprIterate() callback used to collect the "global" matchinfo stats +** for a single query. +** +** fts3ExprIterate() callback to load the 'global' elements of a +** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements +** of the matchinfo array that are constant for all rows returned by the +** current query. +** +** Argument pCtx is actually a pointer to a struct of type MatchInfo. This +** function populates Matchinfo.aMatchinfo[] as follows: +** +** for(iCol=0; iColpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); +} + +/* +** fts3ExprIterate() callback used to collect the "local" part of the +** FTS3_MATCHINFO_HITS array. The local stats are those elements of the +** array that are different for each row returned by the query. +*/ +static int fts3ExprLocalHitsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + int rc = SQLITE_OK; + MatchInfo *p = (MatchInfo *)pCtx; + int iStart = iPhrase * p->nCol * 3; + int i; + + for(i=0; inCol && rc==SQLITE_OK; i++){ + char *pCsr; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); + if( pCsr ){ + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; + } + } + + return rc; +} + +static int fts3MatchinfoCheck( + Fts3Table *pTab, + char cArg, + char **pzErr +){ + if( (cArg==FTS3_MATCHINFO_NPHRASE) + || (cArg==FTS3_MATCHINFO_NCOL) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) + || (cArg==FTS3_MATCHINFO_LCS) + || (cArg==FTS3_MATCHINFO_HITS) + || (cArg==FTS3_MATCHINFO_LHITS) + || (cArg==FTS3_MATCHINFO_LHITS_BM) + ){ + return SQLITE_OK; + } + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); + return SQLITE_ERROR; +} + +static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + size_t nVal; /* Number of integers output by cArg */ + + switch( cArg ){ + case FTS3_MATCHINFO_NDOC: + case FTS3_MATCHINFO_NPHRASE: + case FTS3_MATCHINFO_NCOL: + nVal = 1; + break; + + case FTS3_MATCHINFO_AVGLENGTH: + case FTS3_MATCHINFO_LENGTH: + case FTS3_MATCHINFO_LCS: + nVal = pInfo->nCol; + break; + + case FTS3_MATCHINFO_LHITS: + nVal = pInfo->nCol * pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_LHITS_BM: + nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + break; + + default: + assert( cArg==FTS3_MATCHINFO_HITS ); + nVal = pInfo->nCol * pInfo->nPhrase * 3; + break; + } + + return nVal; +} + +static int fts3MatchinfoSelectDoctotal( + Fts3Table *pTab, + sqlite3_stmt **ppStmt, + sqlite3_int64 *pnDoc, + const char **paLen, + const char **ppEnd +){ + sqlite3_stmt *pStmt; + const char *a; + const char *pEnd; + sqlite3_int64 nDoc; + int n; + + + if( !*ppStmt ){ + int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); + if( rc!=SQLITE_OK ) return rc; + } + pStmt = *ppStmt; + assert( sqlite3_data_count(pStmt)==1 ); + + n = sqlite3_column_bytes(pStmt, 0); + a = sqlite3_column_blob(pStmt, 0); + if( a==0 ){ + return FTS_CORRUPT_VTAB; + } + pEnd = a + n; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc); + if( nDoc<=0 || a>pEnd ){ + return FTS_CORRUPT_VTAB; + } + *pnDoc = nDoc; + + if( paLen ) *paLen = a; + if( ppEnd ) *ppEnd = pEnd; + return SQLITE_OK; +} + +/* +** An instance of the following structure is used to store state while +** iterating through a multi-column position-list corresponding to the +** hits for a single phrase on a single row in order to calculate the +** values for a matchinfo() FTS3_MATCHINFO_LCS request. +*/ +typedef struct LcsIterator LcsIterator; +struct LcsIterator { + Fts3Expr *pExpr; /* Pointer to phrase expression */ + int iPosOffset; /* Tokens count up to end of this phrase */ + char *pRead; /* Cursor used to iterate through aDoclist */ + int iPos; /* Current position */ +}; + +/* +** If LcsIterator.iCol is set to the following value, the iterator has +** finished iterating through all offsets for all columns. +*/ +#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; + +static int fts3MatchinfoLcsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number (numbered from zero) */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + LcsIterator *aIter = (LcsIterator *)pCtx; + aIter[iPhrase].pExpr = pExpr; + return SQLITE_OK; +} + +/* +** Advance the iterator passed as an argument to the next position. Return +** 1 if the iterator is at EOF or if it now points to the start of the +** position list for the next column. +*/ +static int fts3LcsIteratorAdvance(LcsIterator *pIter){ + char *pRead = pIter->pRead; + sqlite3_int64 iRead; + int rc = 0; + + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + if( iRead==0 || iRead==1 ){ + pRead = 0; + rc = 1; + }else{ + pIter->iPos += (int)(iRead-2); + } + + pIter->pRead = pRead; + return rc; +} + +/* +** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. +** +** If the call is successful, the longest-common-substring lengths for each +** column are written into the first nCol elements of the pInfo->aMatchinfo[] +** array before returning. SQLITE_OK is returned in this case. +** +** Otherwise, if an error occurs, an SQLite error code is returned and the +** data written to the first nCol elements of pInfo->aMatchinfo[] is +** undefined. +*/ +static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ + LcsIterator *aIter; + int i; + int iCol; + int nToken = 0; + int rc = SQLITE_OK; + + /* Allocate and populate the array of LcsIterator objects. The array + ** contains one element for each matchable phrase in the query. + **/ + aIter = sqlite3_malloc64(sizeof(LcsIterator) * pCsr->nPhrase); + if( !aIter ) return SQLITE_NOMEM; + memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); + (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + nToken -= pIter->pExpr->pPhrase->nToken; + pIter->iPosOffset = nToken; + } + + for(iCol=0; iColnCol; iCol++){ + int nLcs = 0; /* LCS value for this column */ + int nLive = 0; /* Number of iterators in aIter not at EOF */ + + for(i=0; inPhrase; i++){ + LcsIterator *pIt = &aIter[i]; + rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); + if( rc!=SQLITE_OK ) goto matchinfo_lcs_out; + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(pIt); + if( pIt->pRead==0 ){ + rc = FTS_CORRUPT_VTAB; + goto matchinfo_lcs_out; + } + nLive++; + } + } + + while( nLive>0 ){ + LcsIterator *pAdv = 0; /* The iterator to advance by one position */ + int nThisLcs = 0; /* LCS for the current iterator positions */ + + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + if( pIter->pRead==0 ){ + /* This iterator is already at EOF for this column. */ + nThisLcs = 0; + }else{ + if( pAdv==0 || pIter->iPosiPos ){ + pAdv = pIter; + } + if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ + nThisLcs++; + }else{ + nThisLcs = 1; + } + if( nThisLcs>nLcs ) nLcs = nThisLcs; + } + } + if( fts3LcsIteratorAdvance(pAdv) ) nLive--; + } + + pInfo->aMatchinfo[iCol] = nLcs; + } + + matchinfo_lcs_out: + sqlite3_free(aIter); + return rc; +} + +/* +** Populate the buffer pInfo->aMatchinfo[] with an array of integers to +** be returned by the matchinfo() function. Argument zArg contains the +** format string passed as the second argument to matchinfo (or the +** default value "pcx" if no second argument was specified). The format +** string has already been validated and the pInfo->aMatchinfo[] array +** is guaranteed to be large enough for the output. +** +** If bGlobal is true, then populate all fields of the matchinfo() output. +** If it is false, then assume that those fields that do not change between +** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) +** have already been populated. +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. If a value other than SQLITE_OK is returned, the state the +** pInfo->aMatchinfo[] buffer is left in is undefined. +*/ +static int fts3MatchinfoValues( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + int bGlobal, /* True to grab the global stats */ + MatchInfo *pInfo, /* Matchinfo context object */ + const char *zArg /* Matchinfo format string */ +){ + int rc = SQLITE_OK; + int i; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_stmt *pSelect = 0; + + for(i=0; rc==SQLITE_OK && zArg[i]; i++){ + pInfo->flag = zArg[i]; + switch( zArg[i] ){ + case FTS3_MATCHINFO_NPHRASE: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; + break; + + case FTS3_MATCHINFO_NCOL: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; + break; + + case FTS3_MATCHINFO_NDOC: + if( bGlobal ){ + sqlite3_int64 nDoc = 0; + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0, 0); + pInfo->aMatchinfo[0] = (u32)nDoc; + } + break; + + case FTS3_MATCHINFO_AVGLENGTH: + if( bGlobal ){ + sqlite3_int64 nDoc; /* Number of rows in table */ + const char *a; /* Aggregate column length array */ + const char *pEnd; /* First byte past end of length array */ + + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a, &pEnd); + if( rc==SQLITE_OK ){ + int iCol; + for(iCol=0; iColnCol; iCol++){ + u32 iVal; + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + if( a>pEnd ){ + rc = SQLITE_CORRUPT_VTAB; + break; + } + iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + pInfo->aMatchinfo[iCol] = iVal; + } + } + } + break; + + case FTS3_MATCHINFO_LENGTH: { + sqlite3_stmt *pSelectDocsize = 0; + rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); + if( rc==SQLITE_OK ){ + int iCol; + const char *a = sqlite3_column_blob(pSelectDocsize, 0); + const char *pEnd = a + sqlite3_column_bytes(pSelectDocsize, 0); + for(iCol=0; iColnCol; iCol++){ + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nToken); + if( a>pEnd ){ + rc = SQLITE_CORRUPT_VTAB; + break; + } + pInfo->aMatchinfo[iCol] = (u32)nToken; + } + } + sqlite3_reset(pSelectDocsize); + break; + } + + case FTS3_MATCHINFO_LCS: + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3MatchinfoLcs(pCsr, pInfo); + } + break; + + case FTS3_MATCHINFO_LHITS_BM: + case FTS3_MATCHINFO_LHITS: { + size_t nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); + memset(pInfo->aMatchinfo, 0, nZero); + rc = fts3ExprLHitGather(pCsr->pExpr, pInfo); + break; + } + + default: { + Fts3Expr *pExpr; + assert( zArg[i]==FTS3_MATCHINFO_HITS ); + pExpr = pCsr->pExpr; + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc!=SQLITE_OK ) break; + if( bGlobal ){ + if( pCsr->pDeferred ){ + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0); + if( rc!=SQLITE_OK ) break; + } + rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + sqlite3Fts3EvalTestDeferred(pCsr, &rc); + if( rc!=SQLITE_OK ) break; + } + (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + break; + } + } + + pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); + } + + sqlite3_reset(pSelect); + return rc; +} + + +/* +** Populate pCsr->aMatchinfo[] with data for the current row. The +** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). +*/ +static void fts3GetMatchinfo( + sqlite3_context *pCtx, /* Return results here */ + Fts3Cursor *pCsr, /* FTS3 Cursor object */ + const char *zArg /* Second argument to matchinfo() function */ +){ + MatchInfo sInfo; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int bGlobal = 0; /* Collect 'global' stats as well as local */ + + u32 *aOut = 0; + void (*xDestroyOut)(void*) = 0; + + memset(&sInfo, 0, sizeof(MatchInfo)); + sInfo.pCursor = pCsr; + sInfo.nCol = pTab->nColumn; + + /* If there is cached matchinfo() data, but the format string for the + ** cache does not match the format string for this request, discard + ** the cached data. */ + if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + pCsr->pMIBuffer = 0; + } + + /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the + ** matchinfo function has been called for this query. In this case + ** allocate the array used to accumulate the matchinfo data and + ** initialize those elements that are constant for every row. + */ + if( pCsr->pMIBuffer==0 ){ + size_t nMatchinfo = 0; /* Number of u32 elements in match-info */ + int i; /* Used to iterate through zArg */ + + /* Determine the number of phrases in the query */ + pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); + sInfo.nPhrase = pCsr->nPhrase; + + /* Determine the number of integers in the buffer returned by this call. */ + for(i=0; zArg[i]; i++){ + char *zErr = 0; + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } + nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); + } + + /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ + pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); + if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; + + pCsr->isMatchinfoNeeded = 1; + bGlobal = 1; + } + + if( rc==SQLITE_OK ){ + xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); + if( xDestroyOut==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + sInfo.aMatchinfo = aOut; + sInfo.nPhrase = pCsr->nPhrase; + rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); + if( bGlobal ){ + fts3MIBufferSetGlobal(pCsr->pMIBuffer); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + if( xDestroyOut ) xDestroyOut(aOut); + }else{ + int n = pCsr->pMIBuffer->nElem * sizeof(u32); + sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); + } +} + +/* +** Implementation of snippet() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3Snippet( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr, /* Cursor object */ + const char *zStart, /* Snippet start text - "" */ + const char *zEnd, /* Snippet end text - "" */ + const char *zEllipsis, /* Snippet ellipsis text - "..." */ + int iCol, /* Extract snippet from this column */ + int nToken /* Approximate number of tokens in snippet */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int i; + StrBuffer res = {0, 0, 0}; + + /* The returned text includes up to four fragments of text extracted from + ** the data in the current row. The first iteration of the for(...) loop + ** below attempts to locate a single fragment of text nToken tokens in + ** size that contains at least one instance of all phrases in the query + ** expression that appear in the current row. If such a fragment of text + ** cannot be found, the second iteration of the loop attempts to locate + ** a pair of fragments, and so on. + */ + int nSnippet = 0; /* Number of fragments in this snippet */ + SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */ + int nFToken = -1; /* Number of tokens in each fragment */ + + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + + /* Limit the snippet length to 64 tokens. */ + if( nToken<-64 ) nToken = -64; + if( nToken>+64 ) nToken = +64; + + for(nSnippet=1; 1; nSnippet++){ + + int iSnip; /* Loop counter 0..nSnippet-1 */ + u64 mCovered = 0; /* Bitmask of phrases covered by snippet */ + u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */ + + if( nToken>=0 ){ + nFToken = (nToken+nSnippet-1) / nSnippet; + }else{ + nFToken = -1 * nToken; + } + + for(iSnip=0; iSnipnColumn; iRead++){ + SnippetFragment sF = {0, 0, 0, 0}; + int iS = 0; + if( iCol>=0 && iRead!=iCol ) continue; + + /* Find the best snippet of nFToken tokens in column iRead. */ + rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); + if( rc!=SQLITE_OK ){ + goto snippet_out; + } + if( iS>iBestScore ){ + *pFragment = sF; + iBestScore = iS; + } + } + + mCovered |= pFragment->covered; + } + + /* If all query phrases seen by fts3BestSnippet() are present in at least + ** one of the nSnippet snippet fragments, break out of the loop. + */ + assert( (mCovered&mSeen)==mCovered ); + if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break; + } + + assert( nFToken>0 ); + + for(i=0; ipCsr, pExpr, p->iCol, &pList); + nTerm = pExpr->pPhrase->nToken; + if( pList ){ + fts3GetDeltaPosition(&pList, &iPos); + assert_fts3_nc( iPos>=0 ); + } + + for(iTerm=0; iTermaTerm[p->iTerm++]; + pT->iOff = nTerm-iTerm-1; + pT->pList = pList; + pT->iPos = iPos; + } + + return rc; +} + +/* +** Implementation of offsets() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3Offsets( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr /* Cursor object */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; + int rc; /* Return Code */ + int nToken; /* Number of tokens in query */ + int iCol; /* Column currently being processed */ + StrBuffer res = {0, 0, 0}; /* Result string */ + TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */ + + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + + memset(&sCtx, 0, sizeof(sCtx)); + assert( pCsr->isRequireSeek==0 ); + + /* Count the number of terms in the query */ + rc = fts3ExprLoadDoclists(pCsr, 0, &nToken); + if( rc!=SQLITE_OK ) goto offsets_out; + + /* Allocate the array of TermOffset iterators. */ + sCtx.aTerm = (TermOffset *)sqlite3_malloc64(sizeof(TermOffset)*nToken); + if( 0==sCtx.aTerm ){ + rc = SQLITE_NOMEM; + goto offsets_out; + } + sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; + + /* Loop through the table columns, appending offset information to + ** string-buffer res for each column. + */ + for(iCol=0; iColnColumn; iCol++){ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ + const char *ZDUMMY; /* Dummy argument used with xNext() */ + int NDUMMY = 0; /* Dummy argument used with xNext() */ + int iStart = 0; + int iEnd = 0; + int iCurrent = 0; + const char *zDoc; + int nDoc; + + /* Initialize the contents of sCtx.aTerm[] for column iCol. There is + ** no way that this operation can fail, so the return code from + ** fts3ExprIterate() can be discarded. + */ + sCtx.iCol = iCol; + sCtx.iTerm = 0; + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); + + /* Retreive the text stored in column iCol. If an SQL NULL is stored + ** in column iCol, jump immediately to the next iteration of the loop. + ** If an OOM occurs while retrieving the data (this can happen if SQLite + ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM + ** to the caller. + */ + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){ + continue; + } + rc = SQLITE_NOMEM; + goto offsets_out; + } + + /* Initialize a tokenizer iterator to iterate through column iCol. */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, + zDoc, nDoc, &pC + ); + if( rc!=SQLITE_OK ) goto offsets_out; + + rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + while( rc==SQLITE_OK ){ + int i; /* Used to loop through terms */ + int iMinPos = 0x7FFFFFFF; /* Position of next token */ + TermOffset *pTerm = 0; /* TermOffset associated with next token */ + + for(i=0; ipList && (pT->iPos-pT->iOff)iPos-pT->iOff; + pTerm = pT; + } + } + + if( !pTerm ){ + /* All offsets for this column have been gathered. */ + rc = SQLITE_DONE; + }else{ + assert_fts3_nc( iCurrent<=iMinPos ); + if( 0==(0xFE&*pTerm->pList) ){ + pTerm->pList = 0; + }else{ + fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos); + } + while( rc==SQLITE_OK && iCurrentxNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + } + if( rc==SQLITE_OK ){ + char aBuffer[64]; + sqlite3_snprintf(sizeof(aBuffer), aBuffer, + "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart + ); + rc = fts3StringAppend(&res, aBuffer, -1); + }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } + } + } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + + pMod->xClose(pC); + if( rc!=SQLITE_OK ) goto offsets_out; + } + + offsets_out: + sqlite3_free(sCtx.aTerm); + assert( rc!=SQLITE_DONE ); + sqlite3Fts3SegmentsClose(pTab); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(res.z); + }else{ + sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); + } + return; +} + +/* +** Implementation of matchinfo() function. +*/ +SQLITE_PRIVATE void sqlite3Fts3Matchinfo( + sqlite3_context *pContext, /* Function call context */ + Fts3Cursor *pCsr, /* FTS3 table cursor */ + const char *zArg /* Second arg to matchinfo() function */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + const char *zFormat; + + if( zArg ){ + zFormat = zArg; + }else{ + zFormat = FTS3_MATCHINFO_DEFAULT; + } + + if( !pCsr->pExpr ){ + sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); + return; + }else{ + /* Retrieve matchinfo() data. */ + fts3GetMatchinfo(pContext, pCsr, zFormat); + sqlite3Fts3SegmentsClose(pTab); + } +} + +#endif + +/************** End of fts3_snippet.c ****************************************/ +/************** Begin file fts3_unicode.c ************************************/ +/* +** 2012 May 24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "unicode" full-text-search tokenizer. +*/ + +#ifndef SQLITE_DISABLE_FTS3_UNICODE + +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_tokenizer.h" */ + +/* +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied +** from the sqlite3 source file utf.c. If this file is compiled as part +** of the amalgamation, they are not required. +*/ +#ifndef SQLITE_AMALGAMATION + +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } + +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (u8)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ +} + +#endif /* ifndef SQLITE_AMALGAMATION */ + +typedef struct unicode_tokenizer unicode_tokenizer; +typedef struct unicode_cursor unicode_cursor; + +struct unicode_tokenizer { + sqlite3_tokenizer base; + int eRemoveDiacritic; + int nException; + int *aiException; +}; + +struct unicode_cursor { + sqlite3_tokenizer_cursor base; + const unsigned char *aInput; /* Input text being tokenized */ + int nInput; /* Size of aInput[] in bytes */ + int iOff; /* Current offset within aInput[] */ + int iToken; /* Index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAlloc; /* space allocated at zToken */ +}; + + +/* +** Destroy a tokenizer allocated by unicodeCreate(). +*/ +static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ + if( pTokenizer ){ + unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer; + sqlite3_free(p->aiException); + sqlite3_free(p); + } + return SQLITE_OK; +} + +/* +** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE +** statement has specified that the tokenizer for this table shall consider +** all characters in string zIn/nIn to be separators (if bAlnum==0) or +** token characters (if bAlnum==1). +** +** For each codepoint in the zIn/nIn string, this function checks if the +** sqlite3FtsUnicodeIsalnum() function already returns the desired result. +** If so, no action is taken. Otherwise, the codepoint is added to the +** unicode_tokenizer.aiException[] array. For the purposes of tokenization, +** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all +** codepoints in the aiException[] array. +** +** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() +** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. +** It is not possible to change the behavior of the tokenizer with respect +** to these codepoints. +*/ +static int unicodeAddExceptions( + unicode_tokenizer *p, /* Tokenizer to add exceptions to */ + int bAlnum, /* Replace Isalnum() return value with this */ + const char *zIn, /* Array of characters to make exceptions */ + int nIn /* Length of z in bytes */ +){ + const unsigned char *z = (const unsigned char *)zIn; + const unsigned char *zTerm = &z[nIn]; + unsigned int iCode; + int nEntry = 0; + + assert( bAlnum==0 || bAlnum==1 ); + + while( zaiException,(p->nException+nEntry)*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + nNew = p->nException; + + z = (const unsigned char *)zIn; + while( zi; j--) aNew[j] = aNew[j-1]; + aNew[i] = (int)iCode; + nNew++; + } + } + p->aiException = aNew; + p->nException = nNew; + } + + return SQLITE_OK; +} + +/* +** Return true if the p->aiException[] array contains the value iCode. +*/ +static int unicodeIsException(unicode_tokenizer *p, int iCode){ + if( p->nException>0 ){ + int *a = p->aiException; + int iLo = 0; + int iHi = p->nException-1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( iCode==a[iTest] ){ + return 1; + }else if( iCode>a[iTest] ){ + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + } + + return 0; +} + +/* +** Return true if, for the purposes of tokenization, codepoint iCode is +** considered a token character (not a separator). +*/ +static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){ + assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); + return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode); +} + +/* +** Create a new tokenizer instance. +*/ +static int unicodeCreate( + int nArg, /* Size of array argv[] */ + const char * const *azArg, /* Tokenizer creation arguments */ + sqlite3_tokenizer **pp /* OUT: New tokenizer handle */ +){ + unicode_tokenizer *pNew; /* New tokenizer object */ + int i; + int rc = SQLITE_OK; + + pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); + if( pNew==NULL ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(unicode_tokenizer)); + pNew->eRemoveDiacritic = 1; + + for(i=0; rc==SQLITE_OK && ieRemoveDiacritic = 1; + } + else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ + pNew->eRemoveDiacritic = 0; + } + else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){ + pNew->eRemoveDiacritic = 2; + } + else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); + } + else if( n>=11 && memcmp("separators=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 0, &z[11], n-11); + } + else{ + /* Unrecognized argument */ + rc = SQLITE_ERROR; + } + } + + if( rc!=SQLITE_OK ){ + unicodeDestroy((sqlite3_tokenizer *)pNew); + pNew = 0; + } + *pp = (sqlite3_tokenizer *)pNew; + return rc; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int unicodeOpen( + sqlite3_tokenizer *p, /* The tokenizer */ + const char *aInput, /* Input string */ + int nInput, /* Size of string aInput in bytes */ + sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */ +){ + unicode_cursor *pCsr; + + pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(unicode_cursor)); + + pCsr->aInput = (const unsigned char *)aInput; + if( aInput==0 ){ + pCsr->nInput = 0; + pCsr->aInput = (const unsigned char*)""; + }else if( nInput<0 ){ + pCsr->nInput = (int)strlen(aInput); + }else{ + pCsr->nInput = nInput; + } + + *pp = &pCsr->base; + UNUSED_PARAMETER(p); + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){ + unicode_cursor *pCsr = (unicode_cursor *) pCursor; + sqlite3_free(pCsr->zToken); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int unicodeNext( + sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */ + const char **paToken, /* OUT: Token text */ + int *pnToken, /* OUT: Number of bytes at *paToken */ + int *piStart, /* OUT: Starting offset of token */ + int *piEnd, /* OUT: Ending offset of token */ + int *piPos /* OUT: Position integer of token */ +){ + unicode_cursor *pCsr = (unicode_cursor *)pC; + unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); + unsigned int iCode = 0; + char *zOut; + const unsigned char *z = &pCsr->aInput[pCsr->iOff]; + const unsigned char *zStart = z; + const unsigned char *zEnd; + const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; + + /* Scan past any delimiter characters before the start of the next token. + ** Return SQLITE_DONE early if this takes us all the way to the end of + ** the input. */ + while( z=zTerm ) return SQLITE_DONE; + + zOut = pCsr->zToken; + do { + int iOut; + + /* Grow the output buffer if required. */ + if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ + char *zNew = sqlite3_realloc64(pCsr->zToken, pCsr->nAlloc+64); + if( !zNew ) return SQLITE_NOMEM; + zOut = &zNew[zOut - pCsr->zToken]; + pCsr->zToken = zNew; + pCsr->nAlloc += 64; + } + + /* Write the folded case of the last character read to the output */ + zEnd = z; + iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic); + if( iOut ){ + WRITE_UTF8(zOut, iOut); + } + + /* If the cursor is not at EOF, read the next character */ + if( z>=zTerm ) break; + READ_UTF8(z, zTerm, iCode); + }while( unicodeIsAlnum(p, (int)iCode) + || sqlite3FtsUnicodeIsdiacritic((int)iCode) + ); + + /* Set the output variables and return. */ + pCsr->iOff = (int)(z - pCsr->aInput); + *paToken = pCsr->zToken; + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); + *piPos = pCsr->iToken++; + return SQLITE_OK; +} + +/* +** Set *ppModule to a pointer to the sqlite3_tokenizer_module +** structure for the unicode tokenizer. +*/ +SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ + static const sqlite3_tokenizer_module module = { + 0, + unicodeCreate, + unicodeDestroy, + unicodeOpen, + unicodeClose, + unicodeNext, + 0, + }; + *ppModule = &module; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ + +/************** End of fts3_unicode.c ****************************************/ +/************** Begin file fts3_unicode2.c ***********************************/ +/* +** 2012-05-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +*/ + +/* +** DO NOT EDIT THIS MACHINE GENERATED FILE. +*/ + +#ifndef SQLITE_DISABLE_FTS3_UNICODE +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) + +/* #include */ + +/* +** Return true if the argument corresponds to a unicode codepoint +** classified as either a letter or a number. Otherwise false. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ + /* Each unsigned integer in the following array corresponds to a contiguous + ** range of unicode codepoints that are not either letters or numbers (i.e. + ** codepoints for which this function should return 0). + ** + ** The most significant 22 bits in each 32-bit value contain the first + ** codepoint in the range. The least significant 10 bits are used to store + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints + ** using this format. + */ + static const unsigned int aEntry[] = { + 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, + 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, + 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, + 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, + 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, + 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, + 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, + 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, + 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, + 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, + 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, + 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, + 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, + 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, + 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, + 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, + 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, + 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, + 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, + 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, + 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, + 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, + 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, + 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, + 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, + 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, + 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, + 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, + 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, + 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, + 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, + 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, + 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, + 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, + 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, + 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, + 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, + 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, + 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, + 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, + 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, + 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, + 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, + 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, + 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, + 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, + 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, + 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, + 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, + 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, + 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, + 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, + 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, + 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, + 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, + 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, + 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, + 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, + 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, + 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, + 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, + 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, + 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, + 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, + 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, + 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, + 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, + 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, + 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, + 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, + 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, + 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, + 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, + 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, + 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, + 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, + 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, + 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, + 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, + 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, + 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, + 0x380400F0, + }; + static const unsigned int aAscii[4] = { + 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, + }; + + if( (unsigned int)c<128 ){ + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); + }else if( (unsigned int)c<(1<<22) ){ + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; + int iRes = 0; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aEntry[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( aEntry[0]=aEntry[iRes] ); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); + } + return 1; +} + + +/* +** If the argument is a codepoint corresponding to a lowercase letter +** in the ASCII range with a diacritic added, return the codepoint +** of the ASCII letter only. For example, if passed 235 - "LATIN +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER +** E"). The resuls of passing a codepoint that corresponds to an +** uppercase letter are undefined. +*/ +static int remove_diacritic(int c, int bComplex){ + unsigned short aDia[] = { + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, + }; +#define HIBIT ((unsigned char)0x80) + unsigned char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', + 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', + 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', + 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'u'|HIBIT, 'a'|HIBIT, 'g', 'k', 'o', + 'o'|HIBIT, 'j', 'g', 'n', 'a'|HIBIT, 'a', + 'e', 'i', 'o', 'r', 'u', 's', + 't', 'h', 'a', 'e', 'o'|HIBIT, 'o', + 'o'|HIBIT, 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', + 'c'|HIBIT, 'd', 'd', 'e'|HIBIT, 'e', 'e'|HIBIT, + 'f', 'g', 'h', 'h', 'i', 'i'|HIBIT, + 'k', 'l', 'l'|HIBIT, 'l', 'm', 'n', + 'o'|HIBIT, 'p', 'r', 'r'|HIBIT, 'r', 's', + 's'|HIBIT, 't', 'u', 'u'|HIBIT, 'v', 'w', + 'w', 'x', 'y', 'z', 'h', 't', + 'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, + 'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT, + 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', + }; + + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( key>=aDia[iRes] ); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); +} + + +/* +** Return true if the argument interpreted as a unicode codepoint +** is a diacritical modifier character. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){ + unsigned int mask0 = 0x08029FDF; + unsigned int mask1 = 0x000361F8; + if( c<768 || c>817 ) return 0; + return (c < 768+32) ? + (mask0 & ((unsigned int)1 << (c-768))) : + (mask1 & ((unsigned int)1 << (c-768-32))); +} + + +/* +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. + ** + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. + */ + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; + + int ret = c; + + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); + + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + const struct TableEntry *p; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + assert( c>aEntry[0].iCode ); + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); + } + + if( eRemoveDiacritic ){ + ret = remove_diacritic(ret, eRemoveDiacritic==2); + } + } + + else if( c>=66560 && c<66600 ){ + ret = c + 40; + } + + return ret; +} +#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ + +/************** End of fts3_unicode2.c ***************************************/ +/************** Begin file json1.c *******************************************/ +/* +** 2015-08-12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements JSON functions. The interface is +** modeled after MySQL JSON functions: +** +** https://dev.mysql.com/doc/refman/5.7/en/json.html +** +** For the time being, all JSON is stored as pure text. (We might add +** a JSONB type in the future which stores a binary encoding of JSON in +** a BLOB, but there is no support for JSONB in the current implementation. +** This implementation parses JSON text at 250 MB/s, so it is hard to see +** how JSONB might improve on that.) +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) +#if !defined(SQLITEINT_H) +/* #include "sqlite3ext.h" */ +#endif +SQLITE_EXTENSION_INIT1 +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAM +# define UNUSED_PARAM(X) (void)(X) +#endif + +#ifndef LARGEST_INT64 +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +#ifndef deliberate_fall_through +# define deliberate_fall_through +#endif + +/* +** Versions of isspace(), isalnum() and isdigit() to which it is safe +** to pass signed char values. +*/ +#ifdef sqlite3Isdigit + /* Use the SQLite core versions if this routine is part of the + ** SQLite amalgamation */ +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isxdigit(x) sqlite3Isxdigit(x) +#else + /* Use the standard library for separate compilation */ +#include /* amalgamator: keep */ +# define safe_isdigit(x) isdigit((unsigned char)(x)) +# define safe_isalnum(x) isalnum((unsigned char)(x)) +# define safe_isxdigit(x) isxdigit((unsigned char)(x)) +#endif + +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function, resulting in a 7% overall performance +** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +*/ +static const char jsonIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define safe_isspace(x) (jsonIsSpace[(unsigned char)x]) + +#ifndef SQLITE_AMALGAMATION + /* Unsigned integer types. These are already defined in the sqliteInt.h, + ** but the definitions need to be repeated for separate compilation. */ + typedef sqlite3_uint64 u64; + typedef unsigned int u32; + typedef unsigned short int u16; + typedef unsigned char u8; +#endif + +/* Objects */ +typedef struct JsonString JsonString; +typedef struct JsonNode JsonNode; +typedef struct JsonParse JsonParse; + +/* An instance of this object represents a JSON string +** under construction. Really, this is a generic string accumulator +** that can be and is used to create strings other than JSON. +*/ +struct JsonString { + sqlite3_context *pCtx; /* Function context - put error messages here */ + char *zBuf; /* Append JSON content here */ + u64 nAlloc; /* Bytes of storage available in zBuf[] */ + u64 nUsed; /* Bytes of zBuf[] currently used */ + u8 bStatic; /* True if zBuf is static space */ + u8 bErr; /* True if an error has been encountered */ + char zSpace[100]; /* Initial static space */ +}; + +/* JSON type values +*/ +#define JSON_NULL 0 +#define JSON_TRUE 1 +#define JSON_FALSE 2 +#define JSON_INT 3 +#define JSON_REAL 4 +#define JSON_STRING 5 +#define JSON_ARRAY 6 +#define JSON_OBJECT 7 + +/* The "subtype" set for JSON values */ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ + +/* +** Names of the various JSON types: +*/ +static const char * const jsonType[] = { + "null", "true", "false", "integer", "real", "text", "array", "object" +}; + +/* Bit values for the JsonNode.jnFlag field +*/ +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ +#define JNODE_REMOVE 0x04 /* Do not output */ +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ +#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ +#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_LABEL 0x40 /* Is a label of an object */ + + +/* A single node of parsed JSON +*/ +struct JsonNode { + u8 eType; /* One of the JSON_ type values */ + u8 jnFlags; /* JNODE flags */ + u32 n; /* Bytes of content, or number of sub-nodes */ + union { + const char *zJContent; /* Content for INT, REAL, and STRING */ + u32 iAppend; /* More terms for ARRAY and OBJECT */ + u32 iKey; /* Key for ARRAY objects in json_tree() */ + u32 iReplace; /* Replacement content for JNODE_REPLACE */ + JsonNode *pPatch; /* Node chain of patch for JNODE_PATCH */ + } u; +}; + +/* A completely parsed JSON string +*/ +struct JsonParse { + u32 nNode; /* Number of slots of aNode[] used */ + u32 nAlloc; /* Number of slots of aNode[] allocated */ + JsonNode *aNode; /* Array of nodes containing the parse */ + const char *zJson; /* Original JSON string */ + u32 *aUp; /* Index of parent of each node */ + u8 oom; /* Set to true if out of memory */ + u8 nErr; /* Number of errors seen */ + u16 iDepth; /* Nesting depth */ + int nJson; /* Length of the zJson string in bytes */ + u32 iHold; /* Replace cache line with the lowest iHold value */ +}; + +/* +** Maximum nesting depth of JSON for this implementation. +** +** This limit is needed to avoid a stack overflow in the recursive +** descent parser. A depth of 2000 is far deeper than any sane JSON +** should go. +*/ +#define JSON_MAX_DEPTH 2000 + +/************************************************************************** +** Utility routines for dealing with JsonString objects +**************************************************************************/ + +/* Set the JsonString object to an empty string +*/ +static void jsonZero(JsonString *p){ + p->zBuf = p->zSpace; + p->nAlloc = sizeof(p->zSpace); + p->nUsed = 0; + p->bStatic = 1; +} + +/* Initialize the JsonString object +*/ +static void jsonInit(JsonString *p, sqlite3_context *pCtx){ + p->pCtx = pCtx; + p->bErr = 0; + jsonZero(p); +} + + +/* Free all allocated memory and reset the JsonString object back to its +** initial state. +*/ +static void jsonReset(JsonString *p){ + if( !p->bStatic ) sqlite3_free(p->zBuf); + jsonZero(p); +} + + +/* Report an out-of-memory (OOM) condition +*/ +static void jsonOom(JsonString *p){ + p->bErr = 1; + sqlite3_result_error_nomem(p->pCtx); + jsonReset(p); +} + +/* Enlarge pJson->zBuf so that it can hold at least N more bytes. +** Return zero on success. Return non-zero on an OOM error +*/ +static int jsonGrow(JsonString *p, u32 N){ + u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; + char *zNew; + if( p->bStatic ){ + if( p->bErr ) return 1; + zNew = sqlite3_malloc64(nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + memcpy(zNew, p->zBuf, (size_t)p->nUsed); + p->zBuf = zNew; + p->bStatic = 0; + }else{ + zNew = sqlite3_realloc64(p->zBuf, nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + p->zBuf = zNew; + } + p->nAlloc = nTotal; + return SQLITE_OK; +} + +/* Append N bytes from zIn onto the end of the JsonString string. +*/ +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ + if( N==0 ) return; + if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; +} + +/* Append formatted text (not to exceed N bytes) to the JsonString. +*/ +static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ + va_list ap; + if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + va_start(ap, zFormat); + sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); + va_end(ap); + p->nUsed += (int)strlen(p->zBuf+p->nUsed); +} + +/* Append a single character +*/ +static void jsonAppendChar(JsonString *p, char c){ + if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; + p->zBuf[p->nUsed++] = c; +} + +/* Append a comma separator to the output buffer, if the previous +** character is not '[' or '{'. +*/ +static void jsonAppendSeparator(JsonString *p){ + char c; + if( p->nUsed==0 ) return; + c = p->zBuf[p->nUsed-1]; + if( c!='[' && c!='{' ) jsonAppendChar(p, ','); +} + +/* Append the N-byte string in zIn to the end of the JsonString string +** under construction. Enclose the string in "..." and escape +** any double-quotes or backslash characters contained within the +** string. +*/ +static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ + u32 i; + if( zIn==0 || ((N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0) ) return; + p->zBuf[p->nUsed++] = '"'; + for(i=0; inUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + }else if( c<=0x1f ){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + if( aSpecial[c] ){ + c = aSpecial[c]; + goto json_simple_escape; + } + if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + p->zBuf[p->nUsed++] = 'u'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0' + (c>>4); + c = "0123456789abcdef"[c&0xf]; + } + p->zBuf[p->nUsed++] = c; + } + p->zBuf[p->nUsed++] = '"'; + assert( p->nUsednAlloc ); +} + +/* +** Append a function parameter value to the JSON string under +** construction. +*/ +static void jsonAppendValue( + JsonString *p, /* Append to this JSON string */ + sqlite3_value *pValue /* Value to append */ +){ + switch( sqlite3_value_type(pValue) ){ + case SQLITE_NULL: { + jsonAppendRaw(p, "null", 4); + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + jsonAppendRaw(p, z, n); + break; + } + case SQLITE_TEXT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){ + jsonAppendRaw(p, z, n); + }else{ + jsonAppendString(p, z, n); + } + break; + } + default: { + if( p->bErr==0 ){ + sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); + p->bErr = 2; + jsonReset(p); + } + break; + } + } +} + + +/* Make the JSON in p the result of the SQL function. +*/ +static void jsonResult(JsonString *p){ + if( p->bErr==0 ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, + SQLITE_UTF8); + jsonZero(p); + } + assert( p->bStatic ); +} + +/************************************************************************** +** Utility routines for dealing with JsonNode and JsonParse objects +**************************************************************************/ + +/* +** Return the number of consecutive JsonNode slots need to represent +** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and +** OBJECT types, the number might be larger. +** +** Appended elements are not counted. The value returned is the number +** by which the JsonNode counter should increment in order to go to the +** next peer value. +*/ +static u32 jsonNodeSize(JsonNode *pNode){ + return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; +} + +/* +** Reclaim all memory allocated by a JsonParse object. But do not +** delete the JsonParse object itself. +*/ +static void jsonParseReset(JsonParse *pParse){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + pParse->nNode = 0; + pParse->nAlloc = 0; + sqlite3_free(pParse->aUp); + pParse->aUp = 0; +} + +/* +** Free a JsonParse object that was obtained from sqlite3_malloc(). +*/ +static void jsonParseFree(JsonParse *pParse){ + jsonParseReset(pParse); + sqlite3_free(pParse); +} + +/* +** Convert the JsonNode pNode into a pure JSON string and +** append to pOut. Subsubstructure is also included. Return +** the number of JsonNode objects that are encoded. +*/ +static void jsonRenderNode( + JsonNode *pNode, /* The node to render */ + JsonString *pOut, /* Write JSON here */ + sqlite3_value **aReplace /* Replacement values */ +){ + if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ + if( pNode->jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); + return; + } + pNode = pNode->u.pPatch; + } + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + jsonAppendRaw(pOut, "null", 4); + break; + } + case JSON_TRUE: { + jsonAppendRaw(pOut, "true", 4); + break; + } + case JSON_FALSE: { + jsonAppendRaw(pOut, "false", 5); + break; + } + case JSON_STRING: { + if( pNode->jnFlags & JNODE_RAW ){ + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); + break; + } + /* no break */ deliberate_fall_through + } + case JSON_REAL: + case JSON_INT: { + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + break; + } + case JSON_ARRAY: { + u32 j = 1; + jsonAppendChar(pOut, '['); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + } + j += jsonNodeSize(&pNode[j]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; + } + jsonAppendChar(pOut, ']'); + break; + } + case JSON_OBJECT: { + u32 j = 1; + jsonAppendChar(pOut, '{'); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + jsonAppendChar(pOut, ':'); + jsonRenderNode(&pNode[j+1], pOut, aReplace); + } + j += 1 + jsonNodeSize(&pNode[j+1]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; + } + jsonAppendChar(pOut, '}'); + break; + } + } +} + +/* +** Return a JsonNode and all its descendents as a JSON string. +*/ +static void jsonReturnJson( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ +){ + JsonString s; + jsonInit(&s, pCtx); + jsonRenderNode(pNode, &s, aReplace); + jsonResult(&s); + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); +} + +/* +** Translate a single byte of Hex into an integer. +** This routine only works if h really is a valid hexadecimal +** character: 0..9a..fA..F +*/ +static u8 jsonHexToInt(int h){ + assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); +#ifdef SQLITE_EBCDIC + h += 9*(1&~(h>>4)); +#else + h += 9*(1&(h>>6)); +#endif + return (u8)(h & 0xf); +} + +/* +** Convert a 4-byte hex string into an integer +*/ +static u32 jsonHexToInt4(const char *z){ + u32 v; + assert( safe_isxdigit(z[0]) ); + assert( safe_isxdigit(z[1]) ); + assert( safe_isxdigit(z[2]) ); + assert( safe_isxdigit(z[3]) ); + v = (jsonHexToInt(z[0])<<12) + + (jsonHexToInt(z[1])<<8) + + (jsonHexToInt(z[2])<<4) + + jsonHexToInt(z[3]); + return v; +} + +/* +** Make the JsonNode the return value of the function. +*/ +static void jsonReturn( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ +){ + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + sqlite3_result_null(pCtx); + break; + } + case JSON_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSON_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + case JSON_INT: { + sqlite3_int64 i = 0; + const char *z = pNode->u.zJContent; + if( z[0]=='-' ){ z++; } + while( z[0]>='0' && z[0]<='9' ){ + unsigned v = *(z++) - '0'; + if( i>=LARGEST_INT64/10 ){ + if( i>LARGEST_INT64/10 ) goto int_as_real; + if( z[0]>='0' && z[0]<='9' ) goto int_as_real; + if( v==9 ) goto int_as_real; + if( v==8 ){ + if( pNode->u.zJContent[0]=='-' ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + goto int_done; + }else{ + goto int_as_real; + } + } + } + i = i*10 + v; + } + if( pNode->u.zJContent[0]=='-' ){ i = -i; } + sqlite3_result_int64(pCtx, i); + int_done: + break; + int_as_real: ; /* no break */ deliberate_fall_through + } + case JSON_REAL: { + double r; +#ifdef SQLITE_AMALGAMATION + const char *z = pNode->u.zJContent; + sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); +#else + r = strtod(pNode->u.zJContent, 0); +#endif + sqlite3_result_double(pCtx, r); + break; + } + case JSON_STRING: { +#if 0 /* Never happens because JNODE_RAW is only set by json_set(), + ** json_insert() and json_replace() and those routines do not + ** call jsonReturn() */ + if( pNode->jnFlags & JNODE_RAW ){ + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, + SQLITE_TRANSIENT); + }else +#endif + assert( (pNode->jnFlags & JNODE_RAW)==0 ); + if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ + /* JSON formatted without any backslash-escapes */ + sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, + SQLITE_TRANSIENT); + }else{ + /* Translate JSON formatted string into raw text */ + u32 i; + u32 n = pNode->n; + const char *z = pNode->u.zJContent; + char *zOut; + u32 j; + zOut = sqlite3_malloc( n+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(i=1, j=0; i>6)); + zOut[j++] = 0x80 | (v&0x3f); + }else{ + u32 vlo; + if( (v&0xfc00)==0xd800 + && i>18); + zOut[j++] = 0x80 | ((v>>12)&0x3f); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + }else{ + zOut[j++] = 0xe0 | (v>>12); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + } + } + }else{ + if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + } + zOut[j++] = c; + } + } + } + zOut[j] = 0; + sqlite3_result_text(pCtx, zOut, j, sqlite3_free); + } + break; + } + case JSON_ARRAY: + case JSON_OBJECT: { + jsonReturnJson(pNode, pCtx, aReplace); + break; + } + } +} + +/* Forward reference */ +static int jsonParseAddNode(JsonParse*,u32,u32,const char*); + +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define JSON_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define JSON_NOINLINE __declspec(noinline) +#else +# define JSON_NOINLINE +#endif + + +static JSON_NOINLINE int jsonParseAddNodeExpand( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + u32 nNew; + JsonNode *pNew; + assert( pParse->nNode>=pParse->nAlloc ); + if( pParse->oom ) return -1; + nNew = pParse->nAlloc*2 + 10; + pNew = sqlite3_realloc64(pParse->aNode, sizeof(JsonNode)*nNew); + if( pNew==0 ){ + pParse->oom = 1; + return -1; + } + pParse->nAlloc = nNew; + pParse->aNode = pNew; + assert( pParse->nNodenAlloc ); + return jsonParseAddNode(pParse, eType, n, zContent); +} + +/* +** Create a new JsonNode instance based on the arguments and append that +** instance to the JsonParse. Return the index in pParse->aNode[] of the +** new node, or -1 if a memory allocation fails. +*/ +static int jsonParseAddNode( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + JsonNode *p; + if( pParse->nNode>=pParse->nAlloc ){ + return jsonParseAddNodeExpand(pParse, eType, n, zContent); + } + p = &pParse->aNode[pParse->nNode]; + p->eType = (u8)eType; + p->jnFlags = 0; + p->n = n; + p->u.zJContent = zContent; + return pParse->nNode++; +} + +/* +** Return true if z[] begins with 4 (or more) hexadecimal digits +*/ +static int jsonIs4Hex(const char *z){ + int i; + for(i=0; i<4; i++) if( !safe_isxdigit(z[i]) ) return 0; + return 1; +} + +/* +** Parse a single JSON value which begins at pParse->zJson[i]. Return the +** index of the first character past the end of the value parsed. +** +** Return negative for a syntax error. Special cases: return -2 if the +** first non-whitespace character is '}' and return -3 if the first +** non-whitespace character is ']'. +*/ +static int jsonParseValue(JsonParse *pParse, u32 i){ + char c; + u32 j; + int iThis; + int x; + JsonNode *pNode; + const char *z = pParse->zJson; + while( safe_isspace(z[i]) ){ i++; } + if( (c = z[i])=='{' ){ + /* Parse object */ + iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( safe_isspace(z[j]) ){ j++; } + if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + x = jsonParseValue(pParse, j); + if( x<0 ){ + pParse->iDepth--; + if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; + return -1; + } + if( pParse->oom ) return -1; + pNode = &pParse->aNode[pParse->nNode-1]; + if( pNode->eType!=JSON_STRING ) return -1; + pNode->jnFlags |= JNODE_LABEL; + j = x; + while( safe_isspace(z[j]) ){ j++; } + if( z[j]!=':' ) return -1; + j++; + x = jsonParseValue(pParse, j); + pParse->iDepth--; + if( x<0 ) return -1; + j = x; + while( safe_isspace(z[j]) ){ j++; } + c = z[j]; + if( c==',' ) continue; + if( c!='}' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + return j+1; + }else if( c=='[' ){ + /* Parse array */ + iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( safe_isspace(z[j]) ){ j++; } + if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + x = jsonParseValue(pParse, j); + pParse->iDepth--; + if( x<0 ){ + if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; + return -1; + } + j = x; + while( safe_isspace(z[j]) ){ j++; } + c = z[j]; + if( c==',' ) continue; + if( c!=']' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + return j+1; + }else if( c=='"' ){ + /* Parse string */ + u8 jnFlags = 0; + j = i+1; + for(;;){ + c = z[j]; + if( (c & ~0x1f)==0 ){ + /* Control characters are not allowed in strings */ + return -1; + } + if( c=='\\' ){ + c = z[++j]; + if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' + || c=='n' || c=='r' || c=='t' + || (c=='u' && jsonIs4Hex(z+j+1)) ){ + jnFlags = JNODE_ESCAPE; + }else{ + return -1; + } + }else if( c=='"' ){ + break; + } + j++; + } + jsonParseAddNode(pParse, JSON_STRING, j+1-i, &z[i]); + if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; + return j+1; + }else if( c=='n' + && strncmp(z+i,"null",4)==0 + && !safe_isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return i+4; + }else if( c=='t' + && strncmp(z+i,"true",4)==0 + && !safe_isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + return i+4; + }else if( c=='f' + && strncmp(z+i,"false",5)==0 + && !safe_isalnum(z[i+5]) ){ + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + return i+5; + }else if( c=='-' || (c>='0' && c<='9') ){ + /* Parse number */ + u8 seenDP = 0; + u8 seenE = 0; + assert( '-' < '0' ); + if( c<='0' ){ + j = c=='-' ? i+1 : i; + if( z[j]=='0' && z[j+1]>='0' && z[j+1]<='9' ) return -1; + } + j = i+1; + for(;; j++){ + c = z[j]; + if( c>='0' && c<='9' ) continue; + if( c=='.' ){ + if( z[j-1]=='-' ) return -1; + if( seenDP ) return -1; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ) return -1; + if( seenE ) return -1; + seenDP = seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ) return -1; + continue; + } + break; + } + if( z[j-1]<'0' ) return -1; + jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, + j - i, &z[i]); + return j; + }else if( c=='}' ){ + return -2; /* End of {...} */ + }else if( c==']' ){ + return -3; /* End of [...] */ + }else if( c==0 ){ + return 0; /* End of file */ + }else{ + return -1; /* Syntax error */ + } +} + +/* +** Parse a complete JSON string. Return 0 on success or non-zero if there +** are any errors. If an error occurs, free all memory associated with +** pParse. +** +** pParse is uninitialized when this routine is called. +*/ +static int jsonParse( + JsonParse *pParse, /* Initialize and fill this JsonParse object */ + sqlite3_context *pCtx, /* Report errors here */ + const char *zJson /* Input JSON text to be parsed */ +){ + int i; + memset(pParse, 0, sizeof(*pParse)); + if( zJson==0 ) return 1; + pParse->zJson = zJson; + i = jsonParseValue(pParse, 0); + if( pParse->oom ) i = -1; + if( i>0 ){ + assert( pParse->iDepth==0 ); + while( safe_isspace(zJson[i]) ) i++; + if( zJson[i] ) i = -1; + } + if( i<=0 ){ + if( pCtx!=0 ){ + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + }else{ + sqlite3_result_error(pCtx, "malformed JSON", -1); + } + } + jsonParseReset(pParse); + return 1; + } + return 0; +} + +/* Mark node i of pParse as being a child of iParent. Call recursively +** to fill in all the descendants of node i. +*/ +static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ + JsonNode *pNode = &pParse->aNode[i]; + u32 j; + pParse->aUp[i] = iParent; + switch( pNode->eType ){ + case JSON_ARRAY: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ + jsonParseFillInParentage(pParse, i+j, i); + } + break; + } + case JSON_OBJECT: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ + pParse->aUp[i+j] = i; + jsonParseFillInParentage(pParse, i+j+1, i); + } + break; + } + default: { + break; + } + } +} + +/* +** Compute the parentage of all nodes in a completed parse. +*/ +static int jsonParseFindParents(JsonParse *pParse){ + u32 *aUp; + assert( pParse->aUp==0 ); + aUp = pParse->aUp = sqlite3_malloc64( sizeof(u32)*pParse->nNode ); + if( aUp==0 ){ + pParse->oom = 1; + return SQLITE_NOMEM; + } + jsonParseFillInParentage(pParse, 0, 0); + return SQLITE_OK; +} + +/* +** Magic number used for the JSON parse cache in sqlite3_get_auxdata() +*/ +#define JSON_CACHE_ID (-429938) /* First cache entry */ +#define JSON_CACHE_SZ 4 /* Max number of cache entries */ + +/* +** Obtain a complete parse of the JSON found in the first argument +** of the argv array. Use the sqlite3_get_auxdata() cache for this +** parse if it is available. If the cache is not available or if it +** is no longer valid, parse the JSON again and return the new parse, +** and also register the new parse so that it will be available for +** future sqlite3_get_auxdata() calls. +*/ +static JsonParse *jsonParseCached( + sqlite3_context *pCtx, + sqlite3_value **argv, + sqlite3_context *pErrCtx +){ + const char *zJson = (const char*)sqlite3_value_text(argv[0]); + int nJson = sqlite3_value_bytes(argv[0]); + JsonParse *p; + JsonParse *pMatch = 0; + int iKey; + int iMinKey = 0; + u32 iMinHold = 0xffffffff; + u32 iMaxHold = 0; + if( zJson==0 ) return 0; + for(iKey=0; iKeynJson==nJson + && memcmp(p->zJson,zJson,nJson)==0 + ){ + p->nErr = 0; + pMatch = p; + }else if( p->iHoldiHold; + iMinKey = iKey; + } + if( p->iHold>iMaxHold ){ + iMaxHold = p->iHold; + } + } + if( pMatch ){ + pMatch->nErr = 0; + pMatch->iHold = iMaxHold+1; + return pMatch; + } + p = sqlite3_malloc64( sizeof(*p) + nJson + 1 ); + if( p==0 ){ + sqlite3_result_error_nomem(pCtx); + return 0; + } + memset(p, 0, sizeof(*p)); + p->zJson = (char*)&p[1]; + memcpy((char*)p->zJson, zJson, nJson+1); + if( jsonParse(p, pErrCtx, p->zJson) ){ + sqlite3_free(p); + return 0; + } + p->nJson = nJson; + p->iHold = iMaxHold+1; + sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, + (void(*)(void*))jsonParseFree); + return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); +} + +/* +** Compare the OBJECT label at pNode against zKey,nKey. Return true on +** a match. +*/ +static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ + if( pNode->jnFlags & JNODE_RAW ){ + if( pNode->n!=nKey ) return 0; + return strncmp(pNode->u.zJContent, zKey, nKey)==0; + }else{ + if( pNode->n!=nKey+2 ) return 0; + return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; + } +} + +/* forward declaration */ +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); + +/* +** Search along zPath to find the node specified. Return a pointer +** to that node, or NULL if zPath is malformed or if there is no such +** node. +** +** If pApnd!=0, then try to append new nodes to complete zPath if it is +** possible to do so and if no existing node corresponds to zPath. If +** new nodes are appended *pApnd is set to 1. +*/ +static JsonNode *jsonLookupStep( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this node */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + const char **pzErr /* Make *pzErr point to any syntax error in zPath */ +){ + u32 i, j, nKey; + const char *zKey; + JsonNode *pRoot = &pParse->aNode[iRoot]; + if( zPath[0]==0 ) return pRoot; + if( pRoot->jnFlags & JNODE_REPLACE ) return 0; + if( zPath[0]=='.' ){ + if( pRoot->eType!=JSON_OBJECT ) return 0; + zPath++; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ){ + i++; + }else{ + *pzErr = zPath; + return 0; + } + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + } + if( nKey==0 ){ + *pzErr = zPath; + return 0; + } + j = 1; + for(;;){ + while( j<=pRoot->n ){ + if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ + return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); + } + j++; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( pApnd ){ + u32 iStart, iLabel; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); + zPath += i; + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + pParse->aNode[iLabel].jnFlags |= JNODE_RAW; + } + return pNode; + } + }else if( zPath[0]=='[' ){ + i = 0; + j = 1; + while( safe_isdigit(zPath[j]) ){ + i = i*10 + zPath[j] - '0'; + j++; + } + if( j<2 || zPath[j]!=']' ){ + if( zPath[1]=='#' ){ + JsonNode *pBase = pRoot; + int iBase = iRoot; + if( pRoot->eType!=JSON_ARRAY ) return 0; + for(;;){ + while( j<=pBase->n ){ + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++; + j += jsonNodeSize(&pBase[j]); + } + if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; + iBase += pBase->u.iAppend; + pBase = &pParse->aNode[iBase]; + j = 1; + } + j = 2; + if( zPath[2]=='-' && safe_isdigit(zPath[3]) ){ + unsigned int x = 0; + j = 3; + do{ + x = x*10 + zPath[j] - '0'; + j++; + }while( safe_isdigit(zPath[j]) ); + if( x>i ) return 0; + i -= x; + } + if( zPath[j]!=']' ){ + *pzErr = zPath; + return 0; + } + }else{ + *pzErr = zPath; + return 0; + } + } + if( pRoot->eType!=JSON_ARRAY ) return 0; + zPath += j + 1; + j = 1; + for(;;){ + while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( j<=pRoot->n ){ + return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); + } + if( i==0 && pApnd ){ + u32 iStart; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + } + return pNode; + } + }else{ + *pzErr = zPath; + } + return 0; +} + +/* +** Append content to pParse that will complete zPath. Return a pointer +** to the inserted node, or return NULL if the append fails. +*/ +static JsonNode *jsonLookupAppend( + JsonParse *pParse, /* Append content to the JSON parse */ + const char *zPath, /* Description of content to append */ + int *pApnd, /* Set this flag to 1 */ + const char **pzErr /* Make this point to any syntax error */ +){ + *pApnd = 1; + if( zPath[0]==0 ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; + } + if( zPath[0]=='.' ){ + jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + }else if( strncmp(zPath,"[0]",3)==0 ){ + jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + }else{ + return 0; + } + if( pParse->oom ) return 0; + return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); +} + +/* +** Return the text of a syntax error message on a JSON path. Space is +** obtained from sqlite3_malloc(). +*/ +static char *jsonPathSyntaxError(const char *zErr){ + return sqlite3_mprintf("JSON path error near '%q'", zErr); +} + +/* +** Do a node lookup using zPath. Return a pointer to the node on success. +** Return NULL if not found or if there is an error. +** +** On an error, write an error message into pCtx and increment the +** pParse->nErr counter. +** +** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if +** nodes are appended. +*/ +static JsonNode *jsonLookup( + JsonParse *pParse, /* The JSON to search */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + sqlite3_context *pCtx /* Report errors here, if not NULL */ +){ + const char *zErr = 0; + JsonNode *pNode = 0; + char *zMsg; + + if( zPath==0 ) return 0; + if( zPath[0]!='$' ){ + zErr = zPath; + goto lookup_err; + } + zPath++; + pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); + if( zErr==0 ) return pNode; + +lookup_err: + pParse->nErr++; + assert( zErr!=0 && pCtx!=0 ); + zMsg = jsonPathSyntaxError(zErr); + if( zMsg ){ + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); + }else{ + sqlite3_result_error_nomem(pCtx); + } + return 0; +} + + +/* +** Report the wrong number of arguments for json_insert(), json_replace() +** or json_set(). +*/ +static void jsonWrongNumArgs( + sqlite3_context *pCtx, + const char *zFuncName +){ + char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", + zFuncName); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); +} + +/* +** Mark all NULL entries in the Object passed in as JNODE_REMOVE. +*/ +static void jsonRemoveAllNulls(JsonNode *pNode){ + int i, n; + assert( pNode->eType==JSON_OBJECT ); + n = pNode->n; + for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ + switch( pNode[i].eType ){ + case JSON_NULL: + pNode[i].jnFlags |= JNODE_REMOVE; + break; + case JSON_OBJECT: + jsonRemoveAllNulls(&pNode[i]); + break; + } + } +} + + +/**************************************************************************** +** SQL functions used for testing and debugging +****************************************************************************/ + +#ifdef SQLITE_DEBUG +/* +** The json_parse(JSON) function returns a string which describes +** a parse of the JSON provided. Or it returns NULL if JSON is not +** well-formed. +*/ +static void jsonParseFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString s; /* Output string - not real JSON */ + JsonParse x; /* The parse */ + u32 i; + + assert( argc==1 ); + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonParseFindParents(&x); + jsonInit(&s, ctx); + for(i=0; inNode ); + if( argc==2 ){ + const char *zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(p, zPath, 0, ctx); + }else{ + pNode = p->aNode; + } + if( pNode==0 ){ + return; + } + if( pNode->eType==JSON_ARRAY ){ + assert( (pNode->jnFlags & JNODE_APPEND)==0 ); + for(i=1; i<=pNode->n; n++){ + i += jsonNodeSize(&pNode[i]); + } + } + sqlite3_result_int64(ctx, n); +} + +/* +** json_extract(JSON, PATH, ...) +** +** Return the element described by PATH. Return NULL if there is no +** PATH element. If there are multiple PATHs, then return a JSON array +** with the result from each path. Throw an error if the JSON or any PATH +** is malformed. +*/ +static void jsonExtractFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + JsonNode *pNode; + const char *zPath; + JsonString jx; + int i; + + if( argc<2 ) return; + p = jsonParseCached(ctx, argv, ctx); + if( p==0 ) return; + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '['); + for(i=1; inErr ) break; + if( argc>2 ){ + jsonAppendSeparator(&jx); + if( pNode ){ + jsonRenderNode(pNode, &jx, 0); + }else{ + jsonAppendRaw(&jx, "null", 4); + } + }else if( pNode ){ + jsonReturn(pNode, ctx, 0); + } + } + if( argc>2 && i==argc ){ + jsonAppendChar(&jx, ']'); + jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } + jsonReset(&jx); +} + +/* This is the RFC 7396 MergePatch algorithm. +*/ +static JsonNode *jsonMergePatch( + JsonParse *pParse, /* The JSON parser that contains the TARGET */ + u32 iTarget, /* Node of the TARGET in pParse */ + JsonNode *pPatch /* The PATCH */ +){ + u32 i, j; + u32 iRoot; + JsonNode *pTarget; + if( pPatch->eType!=JSON_OBJECT ){ + return pPatch; + } + assert( iTarget>=0 && iTargetnNode ); + pTarget = &pParse->aNode[iTarget]; + assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); + if( pTarget->eType!=JSON_OBJECT ){ + jsonRemoveAllNulls(pPatch); + return pPatch; + } + iRoot = iTarget; + for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ + u32 nKey; + const char *zKey; + assert( pPatch[i].eType==JSON_STRING ); + assert( pPatch[i].jnFlags & JNODE_LABEL ); + nKey = pPatch[i].n; + zKey = pPatch[i].u.zJContent; + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ + assert( pTarget[j].eType==JSON_STRING ); + assert( pTarget[j].jnFlags & JNODE_LABEL ); + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; + if( pPatch[i+1].eType==JSON_NULL ){ + pTarget[j+1].jnFlags |= JNODE_REMOVE; + }else{ + JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); + if( pNew==0 ) return 0; + pTarget = &pParse->aNode[iTarget]; + if( pNew!=&pTarget[j+1] ){ + pTarget[j+1].u.pPatch = pNew; + pTarget[j+1].jnFlags |= JNODE_PATCH; + } + } + break; + } + } + if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ + int iStart, iPatch; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); + iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + if( pParse->oom ) return 0; + jsonRemoveAllNulls(pPatch); + pTarget = &pParse->aNode[iTarget]; + pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; + pParse->aNode[iRoot].u.iAppend = iStart - iRoot; + iRoot = iStart; + pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; + pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; + } + } + return pTarget; +} + +/* +** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON +** object that is the result of running the RFC 7396 MergePatch() algorithm +** on the two arguments. +*/ +static void jsonPatchFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The JSON that is being patched */ + JsonParse y; /* The patch */ + JsonNode *pResult; /* The result of the merge */ + + UNUSED_PARAM(argc); + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ + jsonParseReset(&x); + return; + } + pResult = jsonMergePatch(&x, 0, y.aNode); + assert( pResult!=0 || x.oom ); + if( pResult ){ + jsonReturnJson(pResult, ctx, 0); + }else{ + sqlite3_result_error_nomem(ctx); + } + jsonParseReset(&x); + jsonParseReset(&y); +} + + +/* +** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON +** object that contains all name/value given in arguments. Or if any name +** is not a string or if any value is a BLOB, throw an error. +*/ +static void jsonObjectFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + int i; + JsonString jx; + const char *z; + u32 n; + + if( argc&1 ){ + sqlite3_result_error(ctx, "json_object() requires an even number " + "of arguments", -1); + return; + } + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '{'); + for(i=0; ijnFlags |= JNODE_REMOVE; + } + if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ + jsonReturnJson(x.aNode, ctx, 0); + } +remove_done: + jsonParseReset(&x); +} + +/* +** json_replace(JSON, PATH, VALUE, ...) +** +** Replace the value at PATH with VALUE. If PATH does not already exist, +** this routine is a no-op. If JSON or PATH is malformed, throw an error. +*/ +static void jsonReplaceFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, "replace"); + return; + } + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i+=2){ + zPath = (const char*)sqlite3_value_text(argv[i]); + pNode = jsonLookup(&x, zPath, 0, ctx); + if( x.nErr ) goto replace_err; + if( pNode ){ + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->u.iReplace = i + 1; + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); + }else{ + jsonReturnJson(x.aNode, ctx, argv); + } +replace_err: + jsonParseReset(&x); +} + +/* +** json_set(JSON, PATH, VALUE, ...) +** +** Set the value at PATH to VALUE. Create the PATH if it does not already +** exist. Overwrite existing values that do exist. +** If JSON or PATH is malformed, throw an error. +** +** json_insert(JSON, PATH, VALUE, ...) +** +** Create PATH and initialize it to VALUE. If PATH already exists, this +** routine is a no-op. If JSON or PATH is malformed, throw an error. +*/ +static void jsonSetFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + int bApnd; + int bIsSet = *(int*)sqlite3_user_data(ctx); + + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); + return; + } + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i+=2){ + zPath = (const char*)sqlite3_value_text(argv[i]); + bApnd = 0; + pNode = jsonLookup(&x, zPath, &bApnd, ctx); + if( x.oom ){ + sqlite3_result_error_nomem(ctx); + goto jsonSetDone; + }else if( x.nErr ){ + goto jsonSetDone; + }else if( pNode && (bApnd || bIsSet) ){ + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->u.iReplace = i + 1; + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); + }else{ + jsonReturnJson(x.aNode, ctx, argv); + } +jsonSetDone: + jsonParseReset(&x); +} + +/* +** json_type(JSON) +** json_type(JSON, PATH) +** +** Return the top-level "type" of a JSON string. Throw an error if +** either the JSON or PATH inputs are not well-formed. +*/ +static void jsonTypeFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + const char *zPath; + JsonNode *pNode; + + p = jsonParseCached(ctx, argv, ctx); + if( p==0 ) return; + if( argc==2 ){ + zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(p, zPath, 0, ctx); + }else{ + pNode = p->aNode; + } + if( pNode ){ + sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); + } +} + +/* +** json_valid(JSON) +** +** Return 1 if JSON is a well-formed JSON string according to RFC-7159. +** Return 0 otherwise. +*/ +static void jsonValidFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + UNUSED_PARAM(argc); + p = jsonParseCached(ctx, argv, 0); + sqlite3_result_int(ctx, p!=0); +} + + +/**************************************************************************** +** Aggregate SQL function implementations +****************************************************************************/ +/* +** json_group_array(VALUE) +** +** Return a JSON array composed of all values in the aggregate. +*/ +static void jsonArrayStep( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString *pStr; + UNUSED_PARAM(argc); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); + if( pStr ){ + if( pStr->zBuf==0 ){ + jsonInit(pStr, ctx); + jsonAppendChar(pStr, '['); + }else if( pStr->nUsed>1 ){ + jsonAppendChar(pStr, ','); + } + pStr->pCtx = ctx; + jsonAppendValue(pStr, argv[0]); + } +} +static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + pStr->pCtx = ctx; + jsonAppendChar(pStr, ']'); + if( pStr->bErr ){ + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); + assert( pStr->bStatic ); + }else if( isFinal ){ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic = 1; + }else{ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); + pStr->nUsed--; + } + }else{ + sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); + } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} +static void jsonArrayValue(sqlite3_context *ctx){ + jsonArrayCompute(ctx, 0); +} +static void jsonArrayFinal(sqlite3_context *ctx){ + jsonArrayCompute(ctx, 1); +} + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** This method works for both json_group_array() and json_group_object(). +** It works by removing the first element of the group by searching forward +** to the first comma (",") that is not within a string and deleting all +** text through that comma. +*/ +static void jsonGroupInverse( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + unsigned int i; + int inStr = 0; + int nNest = 0; + char *z; + char c; + JsonString *pStr; + UNUSED_PARAM(argc); + UNUSED_PARAM(argv); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); +#ifdef NEVER + /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will + ** always have been called to initalize it */ + if( NEVER(!pStr) ) return; +#endif + z = pStr->zBuf; + for(i=1; inUsed && ((c = z[i])!=',' || inStr || nNest); i++){ + if( c=='"' ){ + inStr = !inStr; + }else if( c=='\\' ){ + i++; + }else if( !inStr ){ + if( c=='{' || c=='[' ) nNest++; + if( c=='}' || c==']' ) nNest--; + } + } + if( inUsed ){ + pStr->nUsed -= i; + memmove(&z[1], &z[i+1], (size_t)pStr->nUsed-1); + z[pStr->nUsed] = 0; + }else{ + pStr->nUsed = 1; + } +} +#else +# define jsonGroupInverse 0 +#endif + + +/* +** json_group_obj(NAME,VALUE) +** +** Return a JSON object composed of all names and values in the aggregate. +*/ +static void jsonObjectStep( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString *pStr; + const char *z; + u32 n; + UNUSED_PARAM(argc); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); + if( pStr ){ + if( pStr->zBuf==0 ){ + jsonInit(pStr, ctx); + jsonAppendChar(pStr, '{'); + }else if( pStr->nUsed>1 ){ + jsonAppendChar(pStr, ','); + } + pStr->pCtx = ctx; + z = (const char*)sqlite3_value_text(argv[0]); + n = (u32)sqlite3_value_bytes(argv[0]); + jsonAppendString(pStr, z, n); + jsonAppendChar(pStr, ':'); + jsonAppendValue(pStr, argv[1]); + } +} +static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + jsonAppendChar(pStr, '}'); + if( pStr->bErr ){ + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); + assert( pStr->bStatic ); + }else if( isFinal ){ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic = 1; + }else{ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); + pStr->nUsed--; + } + }else{ + sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); + } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} +static void jsonObjectValue(sqlite3_context *ctx){ + jsonObjectCompute(ctx, 0); +} +static void jsonObjectFinal(sqlite3_context *ctx){ + jsonObjectCompute(ctx, 1); +} + + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/**************************************************************************** +** The json_each virtual table +****************************************************************************/ +typedef struct JsonEachCursor JsonEachCursor; +struct JsonEachCursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + u32 iRowid; /* The rowid */ + u32 iBegin; /* The first node of the scan */ + u32 i; /* Index in sParse.aNode[] of current row */ + u32 iEnd; /* EOF when i equals or exceeds this value */ + u8 eType; /* Type of top-level element */ + u8 bRecursive; /* True for json_tree(). False for json_each() */ + char *zJson; /* Input JSON */ + char *zRoot; /* Path by which to filter zJson */ + JsonParse sParse; /* Parse of the input JSON */ +}; + +/* Constructor for the json_each virtual table */ +static int jsonEachConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define JEACH_KEY 0 +#define JEACH_VALUE 1 +#define JEACH_TYPE 2 +#define JEACH_ATOM 3 +#define JEACH_ID 4 +#define JEACH_PARENT 5 +#define JEACH_FULLKEY 6 +#define JEACH_PATH 7 +/* The xBestIndex method assumes that the JSON and ROOT columns are +** the last two columns in the table. Should this ever changes, be +** sure to update the xBestIndex method. */ +#define JEACH_JSON 8 +#define JEACH_ROOT 9 + + UNUSED_PARAM(pzErr); + UNUSED_PARAM(argv); + UNUSED_PARAM(argc); + UNUSED_PARAM(pAux); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," + "json HIDDEN,root HIDDEN)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + return rc; +} + +/* destructor for json_each virtual table */ +static int jsonEachDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object for json_each(). */ +static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + JsonEachCursor *pCur; + + UNUSED_PARAM(p); + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object for json_tree(). */ +static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + int rc = jsonEachOpenEach(p, ppCursor); + if( rc==SQLITE_OK ){ + JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; + pCur->bRecursive = 1; + } + return rc; +} + +/* Reset a JsonEachCursor back to its original state. Free any memory +** held. */ +static void jsonEachCursorReset(JsonEachCursor *p){ + sqlite3_free(p->zJson); + sqlite3_free(p->zRoot); + jsonParseReset(&p->sParse); + p->iRowid = 0; + p->i = 0; + p->iEnd = 0; + p->eType = 0; + p->zJson = 0; + p->zRoot = 0; +} + +/* Destructor for a jsonEachCursor object */ +static int jsonEachClose(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + jsonEachCursorReset(p); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* Return TRUE if the jsonEachCursor object has been advanced off the end +** of the JSON object */ +static int jsonEachEof(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + return p->i >= p->iEnd; +} + +/* Advance the cursor to the next element for json_tree() */ +static int jsonEachNext(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + if( p->bRecursive ){ + if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; + p->i++; + p->iRowid++; + if( p->iiEnd ){ + u32 iUp = p->sParse.aUp[p->i]; + JsonNode *pUp = &p->sParse.aNode[iUp]; + p->eType = pUp->eType; + if( pUp->eType==JSON_ARRAY ){ + if( iUp==p->i-1 ){ + pUp->u.iKey = 0; + }else{ + pUp->u.iKey++; + } + } + } + }else{ + switch( p->eType ){ + case JSON_ARRAY: { + p->i += jsonNodeSize(&p->sParse.aNode[p->i]); + p->iRowid++; + break; + } + case JSON_OBJECT: { + p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); + p->iRowid++; + break; + } + default: { + p->i = p->iEnd; + break; + } + } + } + return SQLITE_OK; +} + +/* Append the name of the path for element i to pStr +*/ +static void jsonEachComputePath( + JsonEachCursor *p, /* The cursor */ + JsonString *pStr, /* Write the path here */ + u32 i /* Path to this element */ +){ + JsonNode *pNode, *pUp; + u32 iUp; + if( i==0 ){ + jsonAppendChar(pStr, '$'); + return; + } + iUp = p->sParse.aUp[i]; + jsonEachComputePath(p, pStr, iUp); + pNode = &p->sParse.aNode[i]; + pUp = &p->sParse.aNode[iUp]; + if( pUp->eType==JSON_ARRAY ){ + jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); + }else{ + assert( pUp->eType==JSON_OBJECT ); + if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; + assert( pNode->eType==JSON_STRING ); + assert( pNode->jnFlags & JNODE_LABEL ); + jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); + } +} + +/* Return the value of a column */ +static int jsonEachColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + JsonNode *pThis = &p->sParse.aNode[p->i]; + switch( i ){ + case JEACH_KEY: { + if( p->i==0 ) break; + if( p->eType==JSON_OBJECT ){ + jsonReturn(pThis, ctx, 0); + }else if( p->eType==JSON_ARRAY ){ + u32 iKey; + if( p->bRecursive ){ + if( p->iRowid==0 ) break; + iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; + }else{ + iKey = p->iRowid; + } + sqlite3_result_int64(ctx, (sqlite3_int64)iKey); + } + break; + } + case JEACH_VALUE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_TYPE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); + break; + } + case JEACH_ATOM: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + if( pThis->eType>=JSON_ARRAY ) break; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_ID: { + sqlite3_result_int64(ctx, + (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); + break; + } + case JEACH_PARENT: { + if( p->i>p->iBegin && p->bRecursive ){ + sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); + } + break; + } + case JEACH_FULLKEY: { + JsonString x; + jsonInit(&x, ctx); + if( p->bRecursive ){ + jsonEachComputePath(p, &x, p->i); + }else{ + if( p->zRoot ){ + jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); + }else{ + jsonAppendChar(&x, '$'); + } + if( p->eType==JSON_ARRAY ){ + jsonPrintf(30, &x, "[%d]", p->iRowid); + }else if( p->eType==JSON_OBJECT ){ + jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1); + } + } + jsonResult(&x); + break; + } + case JEACH_PATH: { + if( p->bRecursive ){ + JsonString x; + jsonInit(&x, ctx); + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + jsonResult(&x); + break; + } + /* For json_each() path and root are the same so fall through + ** into the root case */ + /* no break */ deliberate_fall_through + } + default: { + const char *zRoot = p->zRoot; + if( zRoot==0 ) zRoot = "$"; + sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); + break; + } + case JEACH_JSON: { + assert( i==JEACH_JSON ); + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +/* Return the current rowid value */ +static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + JsonEachCursor *p = (JsonEachCursor*)cur; + *pRowid = p->iRowid; + return SQLITE_OK; +} + +/* The query strategy is to look for an equality constraint on the json +** column. Without such a constraint, the table cannot operate. idxNum is +** 1 if the constraint is found, 3 if the constraint and zRoot are found, +** and 0 otherwise. +*/ +static int jsonEachBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop counter or computed array index */ + int aIdx[2]; /* Index of constraints for JSON and ROOT */ + int unusableMask = 0; /* Mask of unusable JSON and ROOT constraints */ + int idxMask = 0; /* Mask of usable == constraints JSON and ROOT */ + const struct sqlite3_index_constraint *pConstraint; + + /* This implementation assumes that JSON and ROOT are the last two + ** columns in the table */ + assert( JEACH_ROOT == JEACH_JSON+1 ); + UNUSED_PARAM(tab); + aIdx[0] = aIdx[1] = -1; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + int iCol; + int iMask; + if( pConstraint->iColumn < JEACH_JSON ) continue; + iCol = pConstraint->iColumn - JEACH_JSON; + assert( iCol==0 || iCol==1 ); + iMask = 1 << iCol; + if( pConstraint->usable==0 ){ + unusableMask |= iMask; + }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + aIdx[iCol] = i; + idxMask |= iMask; + } + } + if( (unusableMask & ~idxMask)!=0 ){ + /* If there are any unusable constraints on JSON or ROOT, then reject + ** this entire plan */ + return SQLITE_CONSTRAINT; + } + if( aIdx[0]<0 ){ + /* No JSON input. Leave estimatedCost at the huge value that it was + ** initialized to to discourage the query planner from selecting this + ** plan. */ + pIdxInfo->idxNum = 0; + }else{ + pIdxInfo->estimatedCost = 1.0; + i = aIdx[0]; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + if( aIdx[1]<0 ){ + pIdxInfo->idxNum = 1; /* Only JSON supplied. Plan 1 */ + }else{ + i = aIdx[1]; + pIdxInfo->aConstraintUsage[i].argvIndex = 2; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->idxNum = 3; /* Both JSON and ROOT are supplied. Plan 3 */ + } + } + return SQLITE_OK; +} + +/* Start a search on a new JSON string */ +static int jsonEachFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + const char *z; + const char *zRoot = 0; + sqlite3_int64 n; + + UNUSED_PARAM(idxStr); + UNUSED_PARAM(argc); + jsonEachCursorReset(p); + if( idxNum==0 ) return SQLITE_OK; + z = (const char*)sqlite3_value_text(argv[0]); + if( z==0 ) return SQLITE_OK; + n = sqlite3_value_bytes(argv[0]); + p->zJson = sqlite3_malloc64( n+1 ); + if( p->zJson==0 ) return SQLITE_NOMEM; + memcpy(p->zJson, z, (size_t)n+1); + if( jsonParse(&p->sParse, 0, p->zJson) ){ + int rc = SQLITE_NOMEM; + if( p->sParse.oom==0 ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); + if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; + } + jsonEachCursorReset(p); + return rc; + }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ + jsonEachCursorReset(p); + return SQLITE_NOMEM; + }else{ + JsonNode *pNode = 0; + if( idxNum==3 ){ + const char *zErr = 0; + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + n = sqlite3_value_bytes(argv[1]); + p->zRoot = sqlite3_malloc64( n+1 ); + if( p->zRoot==0 ) return SQLITE_NOMEM; + memcpy(p->zRoot, zRoot, (size_t)n+1); + if( zRoot[0]!='$' ){ + zErr = zRoot; + }else{ + pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); + } + if( zErr ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + }else if( pNode==0 ){ + return SQLITE_OK; + } + }else{ + pNode = p->sParse.aNode; + } + p->iBegin = p->i = (int)(pNode - p->sParse.aNode); + p->eType = pNode->eType; + if( p->eType>=JSON_ARRAY ){ + pNode->u.iKey = 0; + p->iEnd = p->i + pNode->n + 1; + if( p->bRecursive ){ + p->eType = p->sParse.aNode[p->sParse.aUp[p->i]].eType; + if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ + p->i--; + } + }else{ + p->i++; + } + }else{ + p->iEnd = p->i+1; + } + } + return SQLITE_OK; +} + +/* The methods of the json_each virtual table */ +static sqlite3_module jsonEachModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenEach, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ +}; + +/* The methods of the json_tree virtual table. */ +static sqlite3_module jsonTreeModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenTree, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ +}; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/**************************************************************************** +** The following routines are the only publically visible identifiers in this +** file. Call the following routines in order to register the various SQL +** functions and the virtual table implemented by this file. +****************************************************************************/ + +SQLITE_PRIVATE int sqlite3Json1Init(sqlite3 *db){ + int rc = SQLITE_OK; + unsigned int i; + static const struct { + const char *zName; + int nArg; + int flag; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "json", 1, 0, jsonRemoveFunc }, + { "json_array", -1, 0, jsonArrayFunc }, + { "json_array_length", 1, 0, jsonArrayLengthFunc }, + { "json_array_length", 2, 0, jsonArrayLengthFunc }, + { "json_extract", -1, 0, jsonExtractFunc }, + { "json_insert", -1, 0, jsonSetFunc }, + { "json_object", -1, 0, jsonObjectFunc }, + { "json_patch", 2, 0, jsonPatchFunc }, + { "json_quote", 1, 0, jsonQuoteFunc }, + { "json_remove", -1, 0, jsonRemoveFunc }, + { "json_replace", -1, 0, jsonReplaceFunc }, + { "json_set", -1, 1, jsonSetFunc }, + { "json_type", 1, 0, jsonTypeFunc }, + { "json_type", 2, 0, jsonTypeFunc }, + { "json_valid", 1, 0, jsonValidFunc }, + +#if SQLITE_DEBUG + /* DEBUG and TESTING functions */ + { "json_parse", 1, 0, jsonParseFunc }, + { "json_test1", 1, 0, jsonTest1Func }, +#endif + }; + static const struct { + const char *zName; + int nArg; + void (*xStep)(sqlite3_context*,int,sqlite3_value**); + void (*xFinal)(sqlite3_context*); + void (*xValue)(sqlite3_context*); + } aAgg[] = { + { "json_group_array", 1, + jsonArrayStep, jsonArrayFinal, jsonArrayValue }, + { "json_group_object", 2, + jsonObjectStep, jsonObjectFinal, jsonObjectValue }, + }; +#ifndef SQLITE_OMIT_VIRTUALTABLE + static const struct { + const char *zName; + sqlite3_module *pModule; + } aMod[] = { + { "json_each", &jsonEachModule }, + { "json_tree", &jsonTreeModule }, + }; +#endif + static const int enc = + SQLITE_UTF8 | + SQLITE_DETERMINISTIC | + SQLITE_INNOCUOUS; + for(i=0; i */ +/* #include */ +/* #include */ +/* #include */ + +/* The following macro is used to suppress compiler warnings. +*/ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(x) (void)(x) +#endif + +typedef struct Rtree Rtree; +typedef struct RtreeCursor RtreeCursor; +typedef struct RtreeNode RtreeNode; +typedef struct RtreeCell RtreeCell; +typedef struct RtreeConstraint RtreeConstraint; +typedef struct RtreeMatchArg RtreeMatchArg; +typedef struct RtreeGeomCallback RtreeGeomCallback; +typedef union RtreeCoord RtreeCoord; +typedef struct RtreeSearchPoint RtreeSearchPoint; + +/* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ +#define RTREE_MAX_DIMENSIONS 5 + +/* Maximum number of auxiliary columns */ +#define RTREE_MAX_AUX_COLUMN 100 + +/* Size of hash table Rtree.aHash. This hash table is not expected to +** ever contain very many entries, so a fixed number of buckets is +** used. +*/ +#define HASHSIZE 97 + +/* The xBestIndex method of this virtual table requires an estimate of +** the number of rows in the virtual table to calculate the costs of +** various strategies. If possible, this estimate is loaded from the +** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum). +** Otherwise, if no sqlite_stat1 entry is available, use +** RTREE_DEFAULT_ROWEST. +*/ +#define RTREE_DEFAULT_ROWEST 1048576 +#define RTREE_MIN_ROWEST 100 + +/* +** An rtree virtual-table object. +*/ +struct Rtree { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* Host database connection */ + int iNodeSize; /* Size in bytes of each node in the node table */ + u8 nDim; /* Number of dimensions */ + u8 nDim2; /* Twice the number of dimensions */ + u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ + u8 nBytesPerCell; /* Bytes consumed per cell */ + u8 inWrTrans; /* True if inside write transaction */ + u8 nAux; /* # of auxiliary columns in %_rowid */ +#ifdef SQLITE_ENABLE_GEOPOLY + u8 nAuxNotNull; /* Number of initial not-null aux columns */ +#endif +#ifdef SQLITE_DEBUG + u8 bCorrupt; /* Shadow table corruption detected */ +#endif + int iDepth; /* Current depth of the r-tree structure */ + char *zDb; /* Name of database containing r-tree table */ + char *zName; /* Name of r-tree table */ + u32 nBusy; /* Current number of users of this structure */ + i64 nRowEst; /* Estimated number of rows in this table */ + u32 nCursor; /* Number of open cursors */ + u32 nNodeRef; /* Number RtreeNodes with positive nRef */ + char *zReadAuxSql; /* SQL for statement to read aux data */ + + /* List of nodes removed during a CondenseTree operation. List is + ** linked together via the pointer normally used for hash chains - + ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree + ** headed by the node (leaf nodes have RtreeNode.iNode==0). + */ + RtreeNode *pDeleted; + int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ + + /* Blob I/O on xxx_node */ + sqlite3_blob *pNodeBlob; + + /* Statements to read/write/delete a record from xxx_node */ + sqlite3_stmt *pWriteNode; + sqlite3_stmt *pDeleteNode; + + /* Statements to read/write/delete a record from xxx_rowid */ + sqlite3_stmt *pReadRowid; + sqlite3_stmt *pWriteRowid; + sqlite3_stmt *pDeleteRowid; + + /* Statements to read/write/delete a record from xxx_parent */ + sqlite3_stmt *pReadParent; + sqlite3_stmt *pWriteParent; + sqlite3_stmt *pDeleteParent; + + /* Statement for writing to the "aux:" fields, if there are any */ + sqlite3_stmt *pWriteAux; + + RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ +}; + +/* Possible values for Rtree.eCoordType: */ +#define RTREE_COORD_REAL32 0 +#define RTREE_COORD_INT32 1 + +/* +** If SQLITE_RTREE_INT_ONLY is defined, then this virtual table will +** only deal with integer coordinates. No floating point operations +** will be done. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ + typedef int RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0 +#else + typedef double RtreeDValue; /* High accuracy coordinate */ + typedef float RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0.0 +#endif + +/* +** Set the Rtree.bCorrupt flag +*/ +#ifdef SQLITE_DEBUG +# define RTREE_IS_CORRUPT(X) ((X)->bCorrupt = 1) +#else +# define RTREE_IS_CORRUPT(X) +#endif + +/* +** When doing a search of an r-tree, instances of the following structure +** record intermediate results from the tree walk. +** +** The id is always a node-id. For iLevel>=1 the id is the node-id of +** the node that the RtreeSearchPoint represents. When iLevel==0, however, +** the id is of the parent node and the cell that RtreeSearchPoint +** represents is the iCell-th entry in the parent node. +*/ +struct RtreeSearchPoint { + RtreeDValue rScore; /* The score for this node. Smallest goes first. */ + sqlite3_int64 id; /* Node ID */ + u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ + u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ + u8 iCell; /* Cell index within the node */ +}; + +/* +** The minimum number of cells allowed for a node is a third of the +** maximum. In Gutman's notation: +** +** m = M/3 +** +** If an R*-tree "Reinsert" operation is required, the same number of +** cells are removed from the overfull node and reinserted into the tree. +*/ +#define RTREE_MINCELLS(p) ((((p)->iNodeSize-4)/(p)->nBytesPerCell)/3) +#define RTREE_REINSERT(p) RTREE_MINCELLS(p) +#define RTREE_MAXCELLS 51 + +/* +** The smallest possible node-size is (512-64)==448 bytes. And the largest +** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). +** Therefore all non-root nodes must contain at least 3 entries. Since +** 3^40 is greater than 2^64, an r-tree structure always has a depth of +** 40 or less. +*/ +#define RTREE_MAX_DEPTH 40 + + +/* +** Number of entries in the cursor RtreeNode cache. The first entry is +** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining +** entries cache the RtreeNode for the first elements of the priority queue. +*/ +#define RTREE_CACHE_SZ 5 + +/* +** An rtree cursor object. +*/ +struct RtreeCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + u8 atEOF; /* True if at end of search */ + u8 bPoint; /* True if sPoint is valid */ + u8 bAuxValid; /* True if pReadAux is valid */ + int iStrategy; /* Copy of idxNum search parameter */ + int nConstraint; /* Number of entries in aConstraint */ + RtreeConstraint *aConstraint; /* Search constraints. */ + int nPointAlloc; /* Number of slots allocated for aPoint[] */ + int nPoint; /* Number of slots used in aPoint[] */ + int mxLevel; /* iLevel value for root of the tree */ + RtreeSearchPoint *aPoint; /* Priority queue for search points */ + sqlite3_stmt *pReadAux; /* Statement to read aux-data */ + RtreeSearchPoint sPoint; /* Cached next search point */ + RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ + u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ +}; + +/* Return the Rtree of a RtreeCursor */ +#define RTREE_OF_CURSOR(X) ((Rtree*)((X)->base.pVtab)) + +/* +** A coordinate can be either a floating point number or a integer. All +** coordinates within a single R-Tree are always of the same time. +*/ +union RtreeCoord { + RtreeValue f; /* Floating point value */ + int i; /* Integer value */ + u32 u; /* Unsigned for byte-order conversions */ +}; + +/* +** The argument is an RtreeCoord. Return the value stored within the RtreeCoord +** formatted as a RtreeDValue (double or int64). This macro assumes that local +** variable pRtree points to the Rtree structure associated with the +** RtreeCoord. +*/ +#ifdef SQLITE_RTREE_INT_ONLY +# define DCOORD(coord) ((RtreeDValue)coord.i) +#else +# define DCOORD(coord) ( \ + (pRtree->eCoordType==RTREE_COORD_REAL32) ? \ + ((double)coord.f) : \ + ((double)coord.i) \ + ) +#endif + +/* +** A search constraint. +*/ +struct RtreeConstraint { + int iCoord; /* Index of constrained coordinate */ + int op; /* Constraining operation */ + union { + RtreeDValue rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + } u; + sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ +}; + +/* Possible values for RtreeConstraint.op */ +#define RTREE_EQ 0x41 /* A */ +#define RTREE_LE 0x42 /* B */ +#define RTREE_LT 0x43 /* C */ +#define RTREE_GE 0x44 /* D */ +#define RTREE_GT 0x45 /* E */ +#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ +#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ + +/* Special operators available only on cursors. Needs to be consecutive +** with the normal values above, but must be less than RTREE_MATCH. These +** are used in the cursor for contraints such as x=NULL (RTREE_FALSE) or +** x<'xyz' (RTREE_TRUE) */ +#define RTREE_TRUE 0x3f /* ? */ +#define RTREE_FALSE 0x40 /* @ */ + +/* +** An rtree structure node. +*/ +struct RtreeNode { + RtreeNode *pParent; /* Parent node */ + i64 iNode; /* The node number */ + int nRef; /* Number of references to this node */ + int isDirty; /* True if the node needs to be written to disk */ + u8 *zData; /* Content of the node, as should be on disk */ + RtreeNode *pNext; /* Next node in this hash collision chain */ +}; + +/* Return the number of cells in a node */ +#define NCELL(pNode) readInt16(&(pNode)->zData[2]) + +/* +** A single cell from a node, deserialized +*/ +struct RtreeCell { + i64 iRowid; /* Node or entry ID */ + RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; /* Bounding box coordinates */ +}; + + +/* +** This object becomes the sqlite3_user_data() for the SQL functions +** that are created by sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() and which appear on the right of MATCH +** operators in order to constrain a search. +** +** xGeom and xQueryFunc are the callback functions. Exactly one of +** xGeom and xQueryFunc fields is non-NULL, depending on whether the +** SQL function was created using sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback(). +** +** This object is deleted automatically by the destructor mechanism in +** sqlite3_create_function_v2(). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + void (*xDestructor)(void*); + void *pContext; +}; + +/* +** An instance of this structure (in the form of a BLOB) is returned by +** the SQL functions that sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() create, and is read as the right-hand +** operand to the MATCH operator of an R-Tree. +*/ +struct RtreeMatchArg { + u32 iSize; /* Size of this object */ + RtreeGeomCallback cb; /* Info about the callback functions */ + int nParam; /* Number of parameters to the SQL function */ + sqlite3_value **apSqlParam; /* Original SQL parameter values */ + RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ +}; + +#ifndef MAX +# define MAX(x,y) ((x) < (y) ? (y) : (x)) +#endif +#ifndef MIN +# define MIN(x,y) ((x) > (y) ? (y) : (x)) +#endif + +/* What version of GCC is being used. 0 means GCC is not being used . +** Note that the GCC_VERSION macro will also be set correctly when using +** clang, since clang works hard to be gcc compatible. So the gcc +** optimizations will also work when compiling with clang. +*/ +#ifndef GCC_VERSION +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#endif + +/* The testcase() macro should already be defined in the amalgamation. If +** it is not, make it a no-op. +*/ +#ifndef SQLITE_AMALGAMATION +# define testcase(X) +#endif + +/* +** Make sure that the compiler intrinsics we desire are enabled when +** compiling with an appropriate version of MSVC unless prevented by +** the SQLITE_DISABLE_INTRINSIC define. +*/ +#if !defined(SQLITE_DISABLE_INTRINSIC) +# if defined(_MSC_VER) && _MSC_VER>=1400 +# if !defined(_WIN32_WCE) +/* # include */ +# pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_byteswap_uint64) +# else +/* # include */ +# endif +# endif +#endif + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SQLITE_BYTEORDER +#if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__) +# define SQLITE_BYTEORDER 1234 +#elif defined(sparc) || defined(__ppc__) +# define SQLITE_BYTEORDER 4321 +#else +# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ +#endif +#endif + + +/* What version of MSVC is being used. 0 means MSVC is not being used */ +#ifndef MSVC_VERSION +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif +#endif + +/* +** Functions to deserialize a 16 bit integer, 32 bit real number and +** 64 bit integer. The deserialized value is returned. +*/ +static int readInt16(u8 *p){ + return (p[0]<<8) + p[1]; +} +static void readCoord(u8 *p, RtreeCoord *pCoord){ + assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + pCoord->u = _byteswap_ulong(*(u32*)p); +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + pCoord->u = __builtin_bswap32(*(u32*)p); +#elif SQLITE_BYTEORDER==4321 + pCoord->u = *(u32*)p; +#else + pCoord->u = ( + (((u32)p[0]) << 24) + + (((u32)p[1]) << 16) + + (((u32)p[2]) << 8) + + (((u32)p[3]) << 0) + ); +#endif +} +static i64 readInt64(u8 *p){ +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + u64 x; + memcpy(&x, p, 8); + return (i64)_byteswap_uint64(x); +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + u64 x; + memcpy(&x, p, 8); + return (i64)__builtin_bswap64(x); +#elif SQLITE_BYTEORDER==4321 + i64 x; + memcpy(&x, p, 8); + return x; +#else + return (i64)( + (((u64)p[0]) << 56) + + (((u64)p[1]) << 48) + + (((u64)p[2]) << 40) + + (((u64)p[3]) << 32) + + (((u64)p[4]) << 24) + + (((u64)p[5]) << 16) + + (((u64)p[6]) << 8) + + (((u64)p[7]) << 0) + ); +#endif +} + +/* +** Functions to serialize a 16 bit integer, 32 bit real number and +** 64 bit integer. The value returned is the number of bytes written +** to the argument buffer (always 2, 4 and 8 respectively). +*/ +static void writeInt16(u8 *p, int i){ + p[0] = (i>> 8)&0xFF; + p[1] = (i>> 0)&0xFF; +} +static int writeCoord(u8 *p, RtreeCoord *pCoord){ + u32 i; + assert( ((((char*)p) - (char*)0)&3)==0 ); /* p is always 4-byte aligned */ + assert( sizeof(RtreeCoord)==4 ); + assert( sizeof(u32)==4 ); +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + i = __builtin_bswap32(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = _byteswap_ulong(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==4321 + i = pCoord->u; + memcpy(p, &i, 4); +#else + i = pCoord->u; + p[0] = (i>>24)&0xFF; + p[1] = (i>>16)&0xFF; + p[2] = (i>> 8)&0xFF; + p[3] = (i>> 0)&0xFF; +#endif + return 4; +} +static int writeInt64(u8 *p, i64 i){ +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + i = (i64)__builtin_bswap64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = (i64)_byteswap_uint64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==4321 + memcpy(p, &i, 8); +#else + p[0] = (i>>56)&0xFF; + p[1] = (i>>48)&0xFF; + p[2] = (i>>40)&0xFF; + p[3] = (i>>32)&0xFF; + p[4] = (i>>24)&0xFF; + p[5] = (i>>16)&0xFF; + p[6] = (i>> 8)&0xFF; + p[7] = (i>> 0)&0xFF; +#endif + return 8; +} + +/* +** Increment the reference count of node p. +*/ +static void nodeReference(RtreeNode *p){ + if( p ){ + assert( p->nRef>0 ); + p->nRef++; + } +} + +/* +** Clear the content of node p (set all bytes to 0x00). +*/ +static void nodeZero(Rtree *pRtree, RtreeNode *p){ + memset(&p->zData[2], 0, pRtree->iNodeSize-2); + p->isDirty = 1; +} + +/* +** Given a node number iNode, return the corresponding key to use +** in the Rtree.aHash table. +*/ +static unsigned int nodeHash(i64 iNode){ + return ((unsigned)iNode) % HASHSIZE; +} + +/* +** Search the node hash table for node iNode. If found, return a pointer +** to it. Otherwise, return 0. +*/ +static RtreeNode *nodeHashLookup(Rtree *pRtree, i64 iNode){ + RtreeNode *p; + for(p=pRtree->aHash[nodeHash(iNode)]; p && p->iNode!=iNode; p=p->pNext); + return p; +} + +/* +** Add node pNode to the node hash table. +*/ +static void nodeHashInsert(Rtree *pRtree, RtreeNode *pNode){ + int iHash; + assert( pNode->pNext==0 ); + iHash = nodeHash(pNode->iNode); + pNode->pNext = pRtree->aHash[iHash]; + pRtree->aHash[iHash] = pNode; +} + +/* +** Remove node pNode from the node hash table. +*/ +static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){ + RtreeNode **pp; + if( pNode->iNode!=0 ){ + pp = &pRtree->aHash[nodeHash(pNode->iNode)]; + for( ; (*pp)!=pNode; pp = &(*pp)->pNext){ assert(*pp); } + *pp = pNode->pNext; + pNode->pNext = 0; + } +} + +/* +** Allocate and return new r-tree node. Initially, (RtreeNode.iNode==0), +** indicating that node has not yet been assigned a node number. It is +** assigned a node number when nodeWrite() is called to write the +** node contents out to the database. +*/ +static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ + RtreeNode *pNode; + pNode = (RtreeNode *)sqlite3_malloc64(sizeof(RtreeNode) + pRtree->iNodeSize); + if( pNode ){ + memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pRtree->nNodeRef++; + pNode->pParent = pParent; + pNode->isDirty = 1; + nodeReference(pParent); + } + return pNode; +} + +/* +** Clear the Rtree.pNodeBlob object +*/ +static void nodeBlobReset(Rtree *pRtree){ + if( pRtree->pNodeBlob && pRtree->inWrTrans==0 && pRtree->nCursor==0 ){ + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + sqlite3_blob_close(pBlob); + } +} + +/* +** Obtain a reference to an r-tree node. +*/ +static int nodeAcquire( + Rtree *pRtree, /* R-tree structure */ + i64 iNode, /* Node number to load */ + RtreeNode *pParent, /* Either the parent node or NULL */ + RtreeNode **ppNode /* OUT: Acquired node */ +){ + int rc = SQLITE_OK; + RtreeNode *pNode = 0; + + /* Check if the requested node is already in the hash table. If so, + ** increase its reference count and return it. + */ + if( (pNode = nodeHashLookup(pRtree, iNode))!=0 ){ + if( pParent && pParent!=pNode->pParent ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + pNode->nRef++; + *ppNode = pNode; + return SQLITE_OK; + } + + if( pRtree->pNodeBlob ){ + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + rc = sqlite3_blob_reopen(pBlob, iNode); + pRtree->pNodeBlob = pBlob; + if( rc ){ + nodeBlobReset(pRtree); + if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM; + } + } + if( pRtree->pNodeBlob==0 ){ + char *zTab = sqlite3_mprintf("%s_node", pRtree->zName); + if( zTab==0 ) return SQLITE_NOMEM; + rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, zTab, "data", iNode, 0, + &pRtree->pNodeBlob); + sqlite3_free(zTab); + } + if( rc ){ + nodeBlobReset(pRtree); + *ppNode = 0; + /* If unable to open an sqlite3_blob on the desired row, that can only + ** be because the shadow tables hold erroneous data. */ + if( rc==SQLITE_ERROR ){ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + } + }else if( pRtree->iNodeSize==sqlite3_blob_bytes(pRtree->pNodeBlob) ){ + pNode = (RtreeNode *)sqlite3_malloc64(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pRtree->nNodeRef++; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + rc = sqlite3_blob_read(pRtree->pNodeBlob, pNode->zData, + pRtree->iNodeSize, 0); + } + } + + /* If the root node was just loaded, set pRtree->iDepth to the height + ** of the r-tree structure. A height of zero means all data is stored on + ** the root node. A height of one means the children of the root node + ** are the leaves, and so on. If the depth as specified on the root node + ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt. + */ + if( rc==SQLITE_OK && pNode && iNode==1 ){ + pRtree->iDepth = readInt16(pNode->zData); + if( pRtree->iDepth>RTREE_MAX_DEPTH ){ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + } + } + + /* If no error has occurred so far, check if the "number of entries" + ** field on the node is too large. If so, set the return code to + ** SQLITE_CORRUPT_VTAB. + */ + if( pNode && rc==SQLITE_OK ){ + if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + } + } + + if( rc==SQLITE_OK ){ + if( pNode!=0 ){ + nodeReference(pParent); + nodeHashInsert(pRtree, pNode); + }else{ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + } + *ppNode = pNode; + }else{ + if( pNode ){ + pRtree->nNodeRef--; + sqlite3_free(pNode); + } + *ppNode = 0; + } + + return rc; +} + +/* +** Overwrite cell iCell of node pNode with the contents of pCell. +*/ +static void nodeOverwriteCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node into which the cell is to be written */ + RtreeCell *pCell, /* The cell to write */ + int iCell /* Index into pNode into which pCell is written */ +){ + int ii; + u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; + p += writeInt64(p, pCell->iRowid); + for(ii=0; iinDim2; ii++){ + p += writeCoord(p, &pCell->aCoord[ii]); + } + pNode->isDirty = 1; +} + +/* +** Remove the cell with index iCell from node pNode. +*/ +static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ + u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; + u8 *pSrc = &pDst[pRtree->nBytesPerCell]; + int nByte = (NCELL(pNode) - iCell - 1) * pRtree->nBytesPerCell; + memmove(pDst, pSrc, nByte); + writeInt16(&pNode->zData[2], NCELL(pNode)-1); + pNode->isDirty = 1; +} + +/* +** Insert the contents of cell pCell into node pNode. If the insert +** is successful, return SQLITE_OK. +** +** If there is not enough free space in pNode, return SQLITE_FULL. +*/ +static int nodeInsertCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* Write new cell into this node */ + RtreeCell *pCell /* The cell to be inserted */ +){ + int nCell; /* Current number of cells in pNode */ + int nMaxCell; /* Maximum number of cells for pNode */ + + nMaxCell = (pRtree->iNodeSize-4)/pRtree->nBytesPerCell; + nCell = NCELL(pNode); + + assert( nCell<=nMaxCell ); + if( nCellzData[2], nCell+1); + pNode->isDirty = 1; + } + + return (nCell==nMaxCell); +} + +/* +** If the node is dirty, write it out to the database. +*/ +static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ + int rc = SQLITE_OK; + if( pNode->isDirty ){ + sqlite3_stmt *p = pRtree->pWriteNode; + if( pNode->iNode ){ + sqlite3_bind_int64(p, 1, pNode->iNode); + }else{ + sqlite3_bind_null(p, 1); + } + sqlite3_bind_blob(p, 2, pNode->zData, pRtree->iNodeSize, SQLITE_STATIC); + sqlite3_step(p); + pNode->isDirty = 0; + rc = sqlite3_reset(p); + sqlite3_bind_null(p, 2); + if( pNode->iNode==0 && rc==SQLITE_OK ){ + pNode->iNode = sqlite3_last_insert_rowid(pRtree->db); + nodeHashInsert(pRtree, pNode); + } + } + return rc; +} + +/* +** Release a reference to a node. If the node is dirty and the reference +** count drops to zero, the node data is written to the database. +*/ +static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ + int rc = SQLITE_OK; + if( pNode ){ + assert( pNode->nRef>0 ); + assert( pRtree->nNodeRef>0 ); + pNode->nRef--; + if( pNode->nRef==0 ){ + pRtree->nNodeRef--; + if( pNode->iNode==1 ){ + pRtree->iDepth = -1; + } + if( pNode->pParent ){ + rc = nodeRelease(pRtree, pNode->pParent); + } + if( rc==SQLITE_OK ){ + rc = nodeWrite(pRtree, pNode); + } + nodeHashDelete(pRtree, pNode); + sqlite3_free(pNode); + } + } + return rc; +} + +/* +** Return the 64-bit integer value associated with cell iCell of +** node pNode. If pNode is a leaf node, this is a rowid. If it is +** an internal node, then the 64-bit integer is a child page number. +*/ +static i64 nodeGetRowid( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract the ID */ + int iCell /* The cell index from which to extract the ID */ +){ + assert( iCellzData[4 + pRtree->nBytesPerCell*iCell]); +} + +/* +** Return coordinate iCoord from cell iCell in node pNode. +*/ +static void nodeGetCoord( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract a coordinate */ + int iCell, /* The index of the cell within the node */ + int iCoord, /* Which coordinate to extract */ + RtreeCoord *pCoord /* OUT: Space to write result to */ +){ + readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); +} + +/* +** Deserialize cell iCell of node pNode. Populate the structure pointed +** to by pCell with the results. +*/ +static void nodeGetCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node containing the cell to be read */ + int iCell, /* Index of the cell within the node */ + RtreeCell *pCell /* OUT: Write the cell contents here */ +){ + u8 *pData; + RtreeCoord *pCoord; + int ii = 0; + pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); + pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); + pCoord = pCell->aCoord; + do{ + readCoord(pData, &pCoord[ii]); + readCoord(pData+4, &pCoord[ii+1]); + pData += 8; + ii += 2; + }while( iinDim2 ); +} + + +/* Forward declaration for the function that does the work of +** the virtual table module xCreate() and xConnect() methods. +*/ +static int rtreeInit( + sqlite3 *, void *, int, const char *const*, sqlite3_vtab **, char **, int +); + +/* +** Rtree virtual table module xCreate method. +*/ +static int rtreeCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 1); +} + +/* +** Rtree virtual table module xConnect method. +*/ +static int rtreeConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 0); +} + +/* +** Increment the r-tree reference count. +*/ +static void rtreeReference(Rtree *pRtree){ + pRtree->nBusy++; +} + +/* +** Decrement the r-tree reference count. When the reference count reaches +** zero the structure is deleted. +*/ +static void rtreeRelease(Rtree *pRtree){ + pRtree->nBusy--; + if( pRtree->nBusy==0 ){ + pRtree->inWrTrans = 0; + assert( pRtree->nCursor==0 ); + nodeBlobReset(pRtree); + assert( pRtree->nNodeRef==0 || pRtree->bCorrupt ); + sqlite3_finalize(pRtree->pWriteNode); + sqlite3_finalize(pRtree->pDeleteNode); + sqlite3_finalize(pRtree->pReadRowid); + sqlite3_finalize(pRtree->pWriteRowid); + sqlite3_finalize(pRtree->pDeleteRowid); + sqlite3_finalize(pRtree->pReadParent); + sqlite3_finalize(pRtree->pWriteParent); + sqlite3_finalize(pRtree->pDeleteParent); + sqlite3_finalize(pRtree->pWriteAux); + sqlite3_free(pRtree->zReadAuxSql); + sqlite3_free(pRtree); + } +} + +/* +** Rtree virtual table module xDisconnect method. +*/ +static int rtreeDisconnect(sqlite3_vtab *pVtab){ + rtreeRelease((Rtree *)pVtab); + return SQLITE_OK; +} + +/* +** Rtree virtual table module xDestroy method. +*/ +static int rtreeDestroy(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + int rc; + char *zCreate = sqlite3_mprintf( + "DROP TABLE '%q'.'%q_node';" + "DROP TABLE '%q'.'%q_rowid';" + "DROP TABLE '%q'.'%q_parent';", + pRtree->zDb, pRtree->zName, + pRtree->zDb, pRtree->zName, + pRtree->zDb, pRtree->zName + ); + if( !zCreate ){ + rc = SQLITE_NOMEM; + }else{ + nodeBlobReset(pRtree); + rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + } + if( rc==SQLITE_OK ){ + rtreeRelease(pRtree); + } + + return rc; +} + +/* +** Rtree virtual table module xOpen method. +*/ +static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + int rc = SQLITE_NOMEM; + Rtree *pRtree = (Rtree *)pVTab; + RtreeCursor *pCsr; + + pCsr = (RtreeCursor *)sqlite3_malloc64(sizeof(RtreeCursor)); + if( pCsr ){ + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = pVTab; + rc = SQLITE_OK; + pRtree->nCursor++; + } + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + + return rc; +} + + +/* +** Reset a cursor back to its initial state. +*/ +static void resetCursor(RtreeCursor *pCsr){ + Rtree *pRtree = (Rtree *)(pCsr->base.pVtab); + int ii; + sqlite3_stmt *pStmt; + if( pCsr->aConstraint ){ + int i; /* Used to iterate through constraint array */ + for(i=0; inConstraint; i++){ + sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; + if( pInfo ){ + if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); + sqlite3_free(pInfo); + } + } + sqlite3_free(pCsr->aConstraint); + pCsr->aConstraint = 0; + } + for(ii=0; iiaNode[ii]); + sqlite3_free(pCsr->aPoint); + pStmt = pCsr->pReadAux; + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + pCsr->pReadAux = pStmt; + +} + +/* +** Rtree virtual table module xClose method. +*/ +static int rtreeClose(sqlite3_vtab_cursor *cur){ + Rtree *pRtree = (Rtree *)(cur->pVtab); + RtreeCursor *pCsr = (RtreeCursor *)cur; + assert( pRtree->nCursor>0 ); + resetCursor(pCsr); + sqlite3_finalize(pCsr->pReadAux); + sqlite3_free(pCsr); + pRtree->nCursor--; + nodeBlobReset(pRtree); + return SQLITE_OK; +} + +/* +** Rtree virtual table module xEof method. +** +** Return non-zero if the cursor does not currently point to a valid +** record (i.e if the scan has finished), or zero otherwise. +*/ +static int rtreeEof(sqlite3_vtab_cursor *cur){ + RtreeCursor *pCsr = (RtreeCursor *)cur; + return pCsr->atEOF; +} + +/* +** Convert raw bits from the on-disk RTree record into a coordinate value. +** The on-disk format is big-endian and needs to be converted for little- +** endian platforms. The on-disk record stores integer coordinates if +** eInt is true and it stores 32-bit floating point records if eInt is +** false. a[] is the four bytes of the on-disk record to be decoded. +** Store the results in "r". +** +** There are five versions of this macro. The last one is generic. The +** other four are various architectures-specific optimizations. +*/ +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = _byteswap_ulong(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = __builtin_bswap32(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ + ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==4321 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#else +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ + +((u32)a[2]<<8) + a[3]; \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#endif + +/* +** Check the RTree node or entry given by pCellData and p against the MATCH +** constraint pConstraint. +*/ +static int rtreeCallbackConstraint( + RtreeConstraint *pConstraint, /* The constraint to test */ + int eInt, /* True if RTree holding integer coordinates */ + u8 *pCellData, /* Raw cell content */ + RtreeSearchPoint *pSearch, /* Container of this cell */ + sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ + int *peWithin /* OUT: visibility of the cell */ +){ + sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ + int nCoord = pInfo->nCoord; /* No. of coordinates */ + int rc; /* Callback return code */ + RtreeCoord c; /* Translator union */ + sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ + + assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); + assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); + + if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ + pInfo->iRowid = readInt64(pCellData); + } + pCellData += 8; +#ifndef SQLITE_RTREE_INT_ONLY + if( eInt==0 ){ + switch( nCoord ){ + case 10: readCoord(pCellData+36, &c); aCoord[9] = c.f; + readCoord(pCellData+32, &c); aCoord[8] = c.f; + case 8: readCoord(pCellData+28, &c); aCoord[7] = c.f; + readCoord(pCellData+24, &c); aCoord[6] = c.f; + case 6: readCoord(pCellData+20, &c); aCoord[5] = c.f; + readCoord(pCellData+16, &c); aCoord[4] = c.f; + case 4: readCoord(pCellData+12, &c); aCoord[3] = c.f; + readCoord(pCellData+8, &c); aCoord[2] = c.f; + default: readCoord(pCellData+4, &c); aCoord[1] = c.f; + readCoord(pCellData, &c); aCoord[0] = c.f; + } + }else +#endif + { + switch( nCoord ){ + case 10: readCoord(pCellData+36, &c); aCoord[9] = c.i; + readCoord(pCellData+32, &c); aCoord[8] = c.i; + case 8: readCoord(pCellData+28, &c); aCoord[7] = c.i; + readCoord(pCellData+24, &c); aCoord[6] = c.i; + case 6: readCoord(pCellData+20, &c); aCoord[5] = c.i; + readCoord(pCellData+16, &c); aCoord[4] = c.i; + case 4: readCoord(pCellData+12, &c); aCoord[3] = c.i; + readCoord(pCellData+8, &c); aCoord[2] = c.i; + default: readCoord(pCellData+4, &c); aCoord[1] = c.i; + readCoord(pCellData, &c); aCoord[0] = c.i; + } + } + if( pConstraint->op==RTREE_MATCH ){ + int eWithin = 0; + rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, + nCoord, aCoord, &eWithin); + if( eWithin==0 ) *peWithin = NOT_WITHIN; + *prScore = RTREE_ZERO; + }else{ + pInfo->aCoord = aCoord; + pInfo->iLevel = pSearch->iLevel - 1; + pInfo->rScore = pInfo->rParentScore = pSearch->rScore; + pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; + rc = pConstraint->u.xQueryFunc(pInfo); + if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; + if( pInfo->rScore<*prScore || *prScorerScore; + } + } + return rc; +} + +/* +** Check the internal RTree node given by pCellData against constraint p. +** If this constraint cannot be satisfied by any child within the node, +** set *peWithin to NOT_WITHIN. +*/ +static void rtreeNonleafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ + + /* p->iCoord might point to either a lower or upper bound coordinate + ** in a coordinate pair. But make pCellData point to the lower bound. + */ + pCellData += 8 + 4*(p->iCoord&0xfe); + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE + || p->op==RTREE_FALSE ); + assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ + switch( p->op ){ + case RTREE_TRUE: return; /* Always satisfied */ + case RTREE_FALSE: break; /* Never satisfied */ + case RTREE_LE: + case RTREE_LT: + case RTREE_EQ: + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the lower bound of the coordinate pair */ + if( p->u.rValue>=val ) return; + if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ + /* Fall through for the RTREE_EQ case */ + + default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ + pCellData += 4; + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the upper bound of the coordinate pair */ + if( p->u.rValue<=val ) return; + } + *peWithin = NOT_WITHIN; +} + +/* +** Check the leaf RTree cell given by pCellData against constraint p. +** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. +** If the constraint is satisfied, leave *peWithin unchanged. +** +** The constraint is of the form: xN op $val +** +** The op is given by p->op. The xN is p->iCoord-th coordinate in +** pCellData. $val is given by p->u.rValue. +*/ +static void rtreeLeafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + RtreeDValue xN; /* Coordinate value converted to a double */ + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE + || p->op==RTREE_FALSE ); + pCellData += 8 + p->iCoord*4; + assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ + RTREE_DECODE_COORD(eInt, pCellData, xN); + switch( p->op ){ + case RTREE_TRUE: return; /* Always satisfied */ + case RTREE_FALSE: break; /* Never satisfied */ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; + } + *peWithin = NOT_WITHIN; +} + +/* +** One of the cells in node pNode is guaranteed to have a 64-bit +** integer value equal to iRowid. Return the index of this cell. +*/ +static int nodeRowidIndex( + Rtree *pRtree, + RtreeNode *pNode, + i64 iRowid, + int *piIndex +){ + int ii; + int nCell = NCELL(pNode); + assert( nCell<200 ); + for(ii=0; iipParent; + if( ALWAYS(pParent) ){ + return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex); + }else{ + *piIndex = -1; + return SQLITE_OK; + } +} + +/* +** Compare two search points. Return negative, zero, or positive if the first +** is less than, equal to, or greater than the second. +** +** The rScore is the primary key. Smaller rScore values come first. +** If the rScore is a tie, then use iLevel as the tie breaker with smaller +** iLevel values coming first. In this way, if rScore is the same for all +** SearchPoints, then iLevel becomes the deciding factor and the result +** is a depth-first search, which is the desired default behavior. +*/ +static int rtreeSearchPointCompare( + const RtreeSearchPoint *pA, + const RtreeSearchPoint *pB +){ + if( pA->rScorerScore ) return -1; + if( pA->rScore>pB->rScore ) return +1; + if( pA->iLeveliLevel ) return -1; + if( pA->iLevel>pB->iLevel ) return +1; + return 0; +} + +/* +** Interchange two search points in a cursor. +*/ +static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ + RtreeSearchPoint t = p->aPoint[i]; + assert( iaPoint[i] = p->aPoint[j]; + p->aPoint[j] = t; + i++; j++; + if( i=RTREE_CACHE_SZ ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + }else{ + RtreeNode *pTemp = p->aNode[i]; + p->aNode[i] = p->aNode[j]; + p->aNode[j] = pTemp; + } + } +} + +/* +** Return the search point with the lowest current score. +*/ +static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ + return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; +} + +/* +** Get the RtreeNode for the search point with the lowest score. +*/ +static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ + sqlite3_int64 id; + int ii = 1 - pCur->bPoint; + assert( ii==0 || ii==1 ); + assert( pCur->bPoint || pCur->nPoint ); + if( pCur->aNode[ii]==0 ){ + assert( pRC!=0 ); + id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; + *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); + } + return pCur->aNode[ii]; +} + +/* +** Push a new element onto the priority queue +*/ +static RtreeSearchPoint *rtreeEnqueue( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + int i, j; + RtreeSearchPoint *pNew; + if( pCur->nPoint>=pCur->nPointAlloc ){ + int nNew = pCur->nPointAlloc*2 + 8; + pNew = sqlite3_realloc64(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + if( pNew==0 ) return 0; + pCur->aPoint = pNew; + pCur->nPointAlloc = nNew; + } + i = pCur->nPoint++; + pNew = pCur->aPoint + i; + pNew->rScore = rScore; + pNew->iLevel = iLevel; + assert( iLevel<=RTREE_MAX_DEPTH ); + while( i>0 ){ + RtreeSearchPoint *pParent; + j = (i-1)/2; + pParent = pCur->aPoint + j; + if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; + rtreeSearchPointSwap(pCur, j, i); + i = j; + pNew = pParent; + } + return pNew; +} + +/* +** Allocate a new RtreeSearchPoint and return a pointer to it. Return +** NULL if malloc fails. +*/ +static RtreeSearchPoint *rtreeSearchPointNew( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + RtreeSearchPoint *pNew, *pFirst; + pFirst = rtreeSearchPointFirst(pCur); + pCur->anQueue[iLevel]++; + if( pFirst==0 + || pFirst->rScore>rScore + || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) + ){ + if( pCur->bPoint ){ + int ii; + pNew = rtreeEnqueue(pCur, rScore, iLevel); + if( pNew==0 ) return 0; + ii = (int)(pNew - pCur->aPoint) + 1; + assert( ii==1 ); + if( ALWAYS(iiaNode[ii]==0 ); + pCur->aNode[ii] = pCur->aNode[0]; + }else{ + nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); + } + pCur->aNode[0] = 0; + *pNew = pCur->sPoint; + } + pCur->sPoint.rScore = rScore; + pCur->sPoint.iLevel = iLevel; + pCur->bPoint = 1; + return &pCur->sPoint; + }else{ + return rtreeEnqueue(pCur, rScore, iLevel); + } +} + +#if 0 +/* Tracing routines for the RtreeSearchPoint queue */ +static void tracePoint(RtreeSearchPoint *p, int idx, RtreeCursor *pCur){ + if( idx<0 ){ printf(" s"); }else{ printf("%2d", idx); } + printf(" %d.%05lld.%02d %g %d", + p->iLevel, p->id, p->iCell, p->rScore, p->eWithin + ); + idx++; + if( idxaNode[idx]); + }else{ + printf("\n"); + } +} +static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ + int ii; + printf("=== %9s ", zPrefix); + if( pCur->bPoint ){ + tracePoint(&pCur->sPoint, -1, pCur); + } + for(ii=0; iinPoint; ii++){ + if( ii>0 || pCur->bPoint ) printf(" "); + tracePoint(&pCur->aPoint[ii], ii, pCur); + } +} +# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) +#else +# define RTREE_QUEUE_TRACE(A,B) /* no-op */ +#endif + +/* Remove the search point with the lowest current score. +*/ +static void rtreeSearchPointPop(RtreeCursor *p){ + int i, j, k, n; + i = 1 - p->bPoint; + assert( i==0 || i==1 ); + if( p->aNode[i] ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + } + if( p->bPoint ){ + p->anQueue[p->sPoint.iLevel]--; + p->bPoint = 0; + }else if( ALWAYS(p->nPoint) ){ + p->anQueue[p->aPoint[0].iLevel]--; + n = --p->nPoint; + p->aPoint[0] = p->aPoint[n]; + if( naNode[1] = p->aNode[n+1]; + p->aNode[n+1] = 0; + } + i = 0; + while( (j = i*2+1)aPoint[k], &p->aPoint[j])<0 ){ + if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, k); + i = k; + }else{ + break; + } + }else{ + if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, j); + i = j; + }else{ + break; + } + } + } + } +} + + +/* +** Continue the search on cursor pCur until the front of the queue +** contains an entry suitable for returning as a result-set row, +** or until the RtreeSearchPoint queue is empty, indicating that the +** query has completed. +*/ +static int rtreeStepToLeaf(RtreeCursor *pCur){ + RtreeSearchPoint *p; + Rtree *pRtree = RTREE_OF_CURSOR(pCur); + RtreeNode *pNode; + int eWithin; + int rc = SQLITE_OK; + int nCell; + int nConstraint = pCur->nConstraint; + int ii; + int eInt; + RtreeSearchPoint x; + + eInt = pRtree->eCoordType==RTREE_COORD_INT32; + while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + u8 *pCellData; + pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); + if( rc ) return rc; + nCell = NCELL(pNode); + assert( nCell<200 ); + pCellData = pNode->zData + (4+pRtree->nBytesPerCell*p->iCell); + while( p->iCellaConstraint + ii; + if( pConstraint->op>=RTREE_MATCH ){ + rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, + &rScore, &eWithin); + if( rc ) return rc; + }else if( p->iLevel==1 ){ + rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); + }else{ + rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); + } + if( eWithin==NOT_WITHIN ){ + p->iCell++; + pCellData += pRtree->nBytesPerCell; + break; + } + } + if( eWithin==NOT_WITHIN ) continue; + p->iCell++; + x.iLevel = p->iLevel - 1; + if( x.iLevel ){ + x.id = readInt64(pCellData); + for(ii=0; iinPoint; ii++){ + if( pCur->aPoint[ii].id==x.id ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + } + x.iCell = 0; + }else{ + x.id = p->id; + x.iCell = p->iCell - 1; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-S:"); + rtreeSearchPointPop(pCur); + } + if( rScoreeWithin = (u8)eWithin; + p->id = x.id; + p->iCell = x.iCell; + RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); + break; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-Se:"); + rtreeSearchPointPop(pCur); + } + } + pCur->atEOF = p==0; + return SQLITE_OK; +} + +/* +** Rtree virtual table module xNext method. +*/ +static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + int rc = SQLITE_OK; + + /* Move to the next entry that matches the configured constraints. */ + RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); + if( pCsr->bAuxValid ){ + pCsr->bAuxValid = 0; + sqlite3_reset(pCsr->pReadAux); + } + rtreeSearchPointPop(pCsr); + rc = rtreeStepToLeaf(pCsr); + return rc; +} + +/* +** Rtree virtual table module xRowid method. +*/ +static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc==SQLITE_OK && ALWAYS(p) ){ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } + return rc; +} + +/* +** Rtree virtual table module xColumn method. +*/ +static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + Rtree *pRtree = (Rtree *)cur->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + RtreeCoord c; + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + + if( rc ) return rc; + if( NEVER(p==0) ) return SQLITE_OK; + if( i==0 ){ + sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); + }else if( i<=pRtree->nDim2 ){ + nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + sqlite3_result_double(ctx, c.f); + }else +#endif + { + assert( pRtree->eCoordType==RTREE_COORD_INT32 ); + sqlite3_result_int(ctx, c.i); + } + }else{ + if( !pCsr->bAuxValid ){ + if( pCsr->pReadAux==0 ){ + rc = sqlite3_prepare_v3(pRtree->db, pRtree->zReadAuxSql, -1, 0, + &pCsr->pReadAux, 0); + if( rc ) return rc; + } + sqlite3_bind_int64(pCsr->pReadAux, 1, + nodeGetRowid(pRtree, pNode, p->iCell)); + rc = sqlite3_step(pCsr->pReadAux); + if( rc==SQLITE_ROW ){ + pCsr->bAuxValid = 1; + }else{ + sqlite3_reset(pCsr->pReadAux); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; + } + } + sqlite3_result_value(ctx, + sqlite3_column_value(pCsr->pReadAux, i - pRtree->nDim2 + 1)); + } + return SQLITE_OK; +} + +/* +** Use nodeAcquire() to obtain the leaf node containing the record with +** rowid iRowid. If successful, set *ppLeaf to point to the node and +** return SQLITE_OK. If there is no such record in the table, set +** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf +** to zero and return an SQLite error code. +*/ +static int findLeafNode( + Rtree *pRtree, /* RTree to search */ + i64 iRowid, /* The rowid searching for */ + RtreeNode **ppLeaf, /* Write the node here */ + sqlite3_int64 *piNode /* Write the node-id here */ +){ + int rc; + *ppLeaf = 0; + sqlite3_bind_int64(pRtree->pReadRowid, 1, iRowid); + if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ + i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); + if( piNode ) *piNode = iNode; + rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); + sqlite3_reset(pRtree->pReadRowid); + }else{ + rc = sqlite3_reset(pRtree->pReadRowid); + } + return rc; +} + +/* +** This function is called to configure the RtreeConstraint object passed +** as the second argument for a MATCH constraint. The value passed as the +** first argument to this function is the right-hand operand to the MATCH +** operator. +*/ +static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ + RtreeMatchArg *pBlob, *pSrc; /* BLOB returned by geometry function */ + sqlite3_rtree_query_info *pInfo; /* Callback information */ + + pSrc = sqlite3_value_pointer(pValue, "RtreeMatchArg"); + if( pSrc==0 ) return SQLITE_ERROR; + pInfo = (sqlite3_rtree_query_info*) + sqlite3_malloc64( sizeof(*pInfo)+pSrc->iSize ); + if( !pInfo ) return SQLITE_NOMEM; + memset(pInfo, 0, sizeof(*pInfo)); + pBlob = (RtreeMatchArg*)&pInfo[1]; + memcpy(pBlob, pSrc, pSrc->iSize); + pInfo->pContext = pBlob->cb.pContext; + pInfo->nParam = pBlob->nParam; + pInfo->aParam = pBlob->aParam; + pInfo->apSqlParam = pBlob->apSqlParam; + + if( pBlob->cb.xGeom ){ + pCons->u.xGeom = pBlob->cb.xGeom; + }else{ + pCons->op = RTREE_QUERY; + pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; + } + pCons->pInfo = pInfo; + return SQLITE_OK; +} + +/* +** Rtree virtual table module xFilter method. +*/ +static int rtreeFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeNode *pRoot = 0; + int ii; + int rc = SQLITE_OK; + int iCell = 0; + + rtreeReference(pRtree); + + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ + resetCursor(pCsr); + + pCsr->iStrategy = idxNum; + if( idxNum==1 ){ + /* Special case - lookup by rowid. */ + RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the leaf */ + i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iNode = 0; + int eType = sqlite3_value_numeric_type(argv[0]); + if( eType==SQLITE_INTEGER + || (eType==SQLITE_FLOAT && sqlite3_value_double(argv[0])==iRowid) + ){ + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + }else{ + rc = SQLITE_OK; + pLeaf = 0; + } + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = (u8)iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; + } + }else{ + /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array + ** with the configured constraints. + */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && argc>0 ){ + pCsr->aConstraint = sqlite3_malloc64(sizeof(RtreeConstraint)*argc); + pCsr->nConstraint = argc; + if( !pCsr->aConstraint ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); + assert( (idxStr==0 && argc==0) + || (idxStr && (int)strlen(idxStr)==argc*2) ); + for(ii=0; iiaConstraint[ii]; + int eType = sqlite3_value_numeric_type(argv[ii]); + p->op = idxStr[ii*2]; + p->iCoord = idxStr[ii*2+1]-'0'; + if( p->op>=RTREE_MATCH ){ + /* A MATCH operator. The right-hand-side must be a blob that + ** can be cast into an RtreeMatchArg object. One created using + ** an sqlite3_rtree_geometry_callback() SQL user function. + */ + rc = deserializeGeometry(argv[ii], p); + if( rc!=SQLITE_OK ){ + break; + } + p->pInfo->nCoord = pRtree->nDim2; + p->pInfo->anQueue = pCsr->anQueue; + p->pInfo->mxLevel = pRtree->iDepth + 1; + }else if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ +#ifdef SQLITE_RTREE_INT_ONLY + p->u.rValue = sqlite3_value_int64(argv[ii]); +#else + p->u.rValue = sqlite3_value_double(argv[ii]); +#endif + }else{ + p->u.rValue = RTREE_ZERO; + if( eType==SQLITE_NULL ){ + p->op = RTREE_FALSE; + }else if( p->op==RTREE_LT || p->op==RTREE_LE ){ + p->op = RTREE_TRUE; + }else{ + p->op = RTREE_FALSE; + } + } + } + } + } + if( rc==SQLITE_OK ){ + RtreeSearchPoint *pNew; + assert( pCsr->bPoint==0 ); /* Due to the resetCursor() call above */ + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); + if( NEVER(pNew==0) ){ /* Because pCsr->bPoint was FALSE */ + return SQLITE_NOMEM; + } + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); + } + } + + nodeRelease(pRtree, pRoot); + rtreeRelease(pRtree); + return rc; +} + +/* +** Rtree virtual table module xBestIndex method. There are three +** table scan strategies to choose from (in order from most to +** least desirable): +** +** idxNum idxStr Strategy +** ------------------------------------------------ +** 1 Unused Direct lookup by rowid. +** 2 See below R-tree query or full-table scan. +** ------------------------------------------------ +** +** If strategy 1 is used, then idxStr is not meaningful. If strategy +** 2 is used, idxStr is formatted to contain 2 bytes for each +** constraint used. The first two bytes of idxStr correspond to +** the constraint in sqlite3_index_info.aConstraintUsage[] with +** (argvIndex==1) etc. +** +** The first of each pair of bytes in idxStr identifies the constraint +** operator as follows: +** +** Operator Byte Value +** ---------------------- +** = 0x41 ('A') +** <= 0x42 ('B') +** < 0x43 ('C') +** >= 0x44 ('D') +** > 0x45 ('E') +** MATCH 0x46 ('F') +** ---------------------- +** +** The second of each pair of bytes identifies the coordinate column +** to which the constraint applies. The leftmost coordinate column +** is 'a', the second from the left 'b' etc. +*/ +static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + Rtree *pRtree = (Rtree*)tab; + int rc = SQLITE_OK; + int ii; + int bMatch = 0; /* True if there exists a MATCH constraint */ + i64 nRow; /* Estimated rows returned by this scan */ + + int iIdx = 0; + char zIdxStr[RTREE_MAX_DIMENSIONS*8+1]; + memset(zIdxStr, 0, sizeof(zIdxStr)); + + /* Check if there exists a MATCH constraint - even an unusable one. If there + ** is, do not consider the lookup-by-rowid plan as using such a plan would + ** require the VDBE to evaluate the MATCH constraint, which is not currently + ** possible. */ + for(ii=0; iinConstraint; ii++){ + if( pIdxInfo->aConstraint[ii].op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + bMatch = 1; + } + } + + assert( pIdxInfo->idxStr==0 ); + for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; + + if( bMatch==0 && p->usable + && p->iColumn<=0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + /* We have an equality constraint on the rowid. Use strategy 1. */ + int jj; + for(jj=0; jjaConstraintUsage[jj].argvIndex = 0; + pIdxInfo->aConstraintUsage[jj].omit = 0; + } + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[ii].argvIndex = 1; + pIdxInfo->aConstraintUsage[jj].omit = 1; + + /* This strategy involves a two rowid lookups on an B-Tree structures + ** and then a linear search of an R-Tree node. This should be + ** considered almost as quick as a direct rowid lookup (for which + ** sqlite uses an internal cost of 0.0). It is expected to return + ** a single row. + */ + pIdxInfo->estimatedCost = 30.0; + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + return SQLITE_OK; + } + + if( p->usable + && ((p->iColumn>0 && p->iColumn<=pRtree->nDim2) + || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) + ){ + u8 op; + switch( p->op ){ + case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; + case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; + case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; + case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; + case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; + case SQLITE_INDEX_CONSTRAINT_MATCH: op = RTREE_MATCH; break; + default: op = 0; break; + } + if( op ){ + zIdxStr[iIdx++] = op; + zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0'); + pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); + pIdxInfo->aConstraintUsage[ii].omit = 1; + } + } + } + + pIdxInfo->idxNum = 2; + pIdxInfo->needToFreeIdxStr = 1; + if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ + return SQLITE_NOMEM; + } + + nRow = pRtree->nRowEst >> (iIdx/2); + pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; + pIdxInfo->estimatedRows = nRow; + + return rc; +} + +/* +** Return the N-dimensional volumn of the cell stored in *p. +*/ +static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ + RtreeDValue area = (RtreeDValue)1; + assert( pRtree->nDim>=1 && pRtree->nDim<=5 ); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + switch( pRtree->nDim ){ + case 5: area = p->aCoord[9].f - p->aCoord[8].f; + case 4: area *= p->aCoord[7].f - p->aCoord[6].f; + case 3: area *= p->aCoord[5].f - p->aCoord[4].f; + case 2: area *= p->aCoord[3].f - p->aCoord[2].f; + default: area *= p->aCoord[1].f - p->aCoord[0].f; + } + }else +#endif + { + switch( pRtree->nDim ){ + case 5: area = (i64)p->aCoord[9].i - (i64)p->aCoord[8].i; + case 4: area *= (i64)p->aCoord[7].i - (i64)p->aCoord[6].i; + case 3: area *= (i64)p->aCoord[5].i - (i64)p->aCoord[4].i; + case 2: area *= (i64)p->aCoord[3].i - (i64)p->aCoord[2].i; + default: area *= (i64)p->aCoord[1].i - (i64)p->aCoord[0].i; + } + } + return area; +} + +/* +** Return the margin length of cell p. The margin length is the sum +** of the objects size in each dimension. +*/ +static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ + RtreeDValue margin = 0; + int ii = pRtree->nDim2 - 2; + do{ + margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); + ii -= 2; + }while( ii>=0 ); + return margin; +} + +/* +** Store the union of cells p1 and p2 in p1. +*/ +static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ + int ii = 0; + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + do{ + p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f); + p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f); + ii += 2; + }while( iinDim2 ); + }else{ + do{ + p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i); + p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i); + ii += 2; + }while( iinDim2 ); + } +} + +/* +** Return true if the area covered by p2 is a subset of the area covered +** by p1. False otherwise. +*/ +static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ + int ii; + int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); + for(ii=0; iinDim2; ii+=2){ + RtreeCoord *a1 = &p1->aCoord[ii]; + RtreeCoord *a2 = &p2->aCoord[ii]; + if( (!isInt && (a2[0].fa1[1].f)) + || ( isInt && (a2[0].ia1[1].i)) + ){ + return 0; + } + } + return 1; +} + +/* +** Return the amount cell p would grow by if it were unioned with pCell. +*/ +static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ + RtreeDValue area; + RtreeCell cell; + memcpy(&cell, p, sizeof(RtreeCell)); + area = cellArea(pRtree, &cell); + cellUnion(pRtree, &cell, pCell); + return (cellArea(pRtree, &cell)-area); +} + +static RtreeDValue cellOverlap( + Rtree *pRtree, + RtreeCell *p, + RtreeCell *aCell, + int nCell +){ + int ii; + RtreeDValue overlap = RTREE_ZERO; + for(ii=0; iinDim2; jj+=2){ + RtreeDValue x1, x2; + x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); + x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); + if( x2iDepth-iHeight); ii++){ + int iCell; + sqlite3_int64 iBest = 0; + + RtreeDValue fMinGrowth = RTREE_ZERO; + RtreeDValue fMinArea = RTREE_ZERO; + + int nCell = NCELL(pNode); + RtreeCell cell; + RtreeNode *pChild; + + RtreeCell *aCell = 0; + + /* Select the child node which will be enlarged the least if pCell + ** is inserted into it. Resolve ties by choosing the entry with + ** the smallest area. + */ + for(iCell=0; iCellpParent ){ + RtreeNode *pParent = p->pParent; + RtreeCell cell; + int iCell; + + cnt++; + if( NEVER(cnt>100) ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + rc = nodeParentIndex(pRtree, p, &iCell); + if( NEVER(rc!=SQLITE_OK) ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + + nodeGetCell(pRtree, pParent, iCell, &cell); + if( !cellContains(pRtree, &cell, pCell) ){ + cellUnion(pRtree, &cell, pCell); + nodeOverwriteCell(pRtree, pParent, &cell, iCell); + } + + p = pParent; + } + return SQLITE_OK; +} + +/* +** Write mapping (iRowid->iNode) to the _rowid table. +*/ +static int rowidWrite(Rtree *pRtree, sqlite3_int64 iRowid, sqlite3_int64 iNode){ + sqlite3_bind_int64(pRtree->pWriteRowid, 1, iRowid); + sqlite3_bind_int64(pRtree->pWriteRowid, 2, iNode); + sqlite3_step(pRtree->pWriteRowid); + return sqlite3_reset(pRtree->pWriteRowid); +} + +/* +** Write mapping (iNode->iPar) to the _parent table. +*/ +static int parentWrite(Rtree *pRtree, sqlite3_int64 iNode, sqlite3_int64 iPar){ + sqlite3_bind_int64(pRtree->pWriteParent, 1, iNode); + sqlite3_bind_int64(pRtree->pWriteParent, 2, iPar); + sqlite3_step(pRtree->pWriteParent); + return sqlite3_reset(pRtree->pWriteParent); +} + +static int rtreeInsertCell(Rtree *, RtreeNode *, RtreeCell *, int); + + +/* +** Arguments aIdx, aDistance and aSpare all point to arrays of size +** nIdx. The aIdx array contains the set of integers from 0 to +** (nIdx-1) in no particular order. This function sorts the values +** in aIdx according to the indexed values in aDistance. For +** example, assuming the inputs: +** +** aIdx = { 0, 1, 2, 3 } +** aDistance = { 5.0, 2.0, 7.0, 6.0 } +** +** this function sets the aIdx array to contain: +** +** aIdx = { 0, 1, 2, 3 } +** +** The aSpare array is used as temporary working space by the +** sorting algorithm. +*/ +static void SortByDistance( + int *aIdx, + int nIdx, + RtreeDValue *aDistance, + int *aSpare +){ + if( nIdx>1 ){ + int iLeft = 0; + int iRight = 0; + + int nLeft = nIdx/2; + int nRight = nIdx-nLeft; + int *aLeft = aIdx; + int *aRight = &aIdx[nLeft]; + + SortByDistance(aLeft, nLeft, aDistance, aSpare); + SortByDistance(aRight, nRight, aDistance, aSpare); + + memcpy(aSpare, aLeft, sizeof(int)*nLeft); + aLeft = aSpare; + + while( iLeft1 ){ + + int iLeft = 0; + int iRight = 0; + + int nLeft = nIdx/2; + int nRight = nIdx-nLeft; + int *aLeft = aIdx; + int *aRight = &aIdx[nLeft]; + + SortByDimension(pRtree, aLeft, nLeft, iDim, aCell, aSpare); + SortByDimension(pRtree, aRight, nRight, iDim, aCell, aSpare); + + memcpy(aSpare, aLeft, sizeof(int)*nLeft); + aLeft = aSpare; + while( iLeftnDim+1)*(sizeof(int*)+nCell*sizeof(int)); + + aaSorted = (int **)sqlite3_malloc64(nByte); + if( !aaSorted ){ + return SQLITE_NOMEM; + } + + aSpare = &((int *)&aaSorted[pRtree->nDim])[pRtree->nDim*nCell]; + memset(aaSorted, 0, nByte); + for(ii=0; iinDim; ii++){ + int jj; + aaSorted[ii] = &((int *)&aaSorted[pRtree->nDim])[ii*nCell]; + for(jj=0; jjnDim; ii++){ + RtreeDValue margin = RTREE_ZERO; + RtreeDValue fBestOverlap = RTREE_ZERO; + RtreeDValue fBestArea = RTREE_ZERO; + int iBestLeft = 0; + int nLeft; + + for( + nLeft=RTREE_MINCELLS(pRtree); + nLeft<=(nCell-RTREE_MINCELLS(pRtree)); + nLeft++ + ){ + RtreeCell left; + RtreeCell right; + int kk; + RtreeDValue overlap; + RtreeDValue area; + + memcpy(&left, &aCell[aaSorted[ii][0]], sizeof(RtreeCell)); + memcpy(&right, &aCell[aaSorted[ii][nCell-1]], sizeof(RtreeCell)); + for(kk=1; kk<(nCell-1); kk++){ + if( kk0 ){ + RtreeNode *pChild = nodeHashLookup(pRtree, iRowid); + RtreeNode *p; + for(p=pNode; p; p=p->pParent){ + if( p==pChild ) return SQLITE_CORRUPT_VTAB; + } + if( pChild ){ + nodeRelease(pRtree, pChild->pParent); + nodeReference(pNode); + pChild->pParent = pNode; + } + } + return xSetMapping(pRtree, iRowid, pNode->iNode); +} + +static int SplitNode( + Rtree *pRtree, + RtreeNode *pNode, + RtreeCell *pCell, + int iHeight +){ + int i; + int newCellIsRight = 0; + + int rc = SQLITE_OK; + int nCell = NCELL(pNode); + RtreeCell *aCell; + int *aiUsed; + + RtreeNode *pLeft = 0; + RtreeNode *pRight = 0; + + RtreeCell leftbbox; + RtreeCell rightbbox; + + /* Allocate an array and populate it with a copy of pCell and + ** all cells from node pLeft. Then zero the original node. + */ + aCell = sqlite3_malloc64((sizeof(RtreeCell)+sizeof(int))*(nCell+1)); + if( !aCell ){ + rc = SQLITE_NOMEM; + goto splitnode_out; + } + aiUsed = (int *)&aCell[nCell+1]; + memset(aiUsed, 0, sizeof(int)*(nCell+1)); + for(i=0; iiNode==1 ){ + pRight = nodeNew(pRtree, pNode); + pLeft = nodeNew(pRtree, pNode); + pRtree->iDepth++; + pNode->isDirty = 1; + writeInt16(pNode->zData, pRtree->iDepth); + }else{ + pLeft = pNode; + pRight = nodeNew(pRtree, pLeft->pParent); + pLeft->nRef++; + } + + if( !pLeft || !pRight ){ + rc = SQLITE_NOMEM; + goto splitnode_out; + } + + memset(pLeft->zData, 0, pRtree->iNodeSize); + memset(pRight->zData, 0, pRtree->iNodeSize); + + rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, + &leftbbox, &rightbbox); + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + + /* Ensure both child nodes have node numbers assigned to them by calling + ** nodeWrite(). Node pRight always needs a node number, as it was created + ** by nodeNew() above. But node pLeft sometimes already has a node number. + ** In this case avoid the all to nodeWrite(). + */ + if( SQLITE_OK!=(rc = nodeWrite(pRtree, pRight)) + || (0==pLeft->iNode && SQLITE_OK!=(rc = nodeWrite(pRtree, pLeft))) + ){ + goto splitnode_out; + } + + rightbbox.iRowid = pRight->iNode; + leftbbox.iRowid = pLeft->iNode; + + if( pNode->iNode==1 ){ + rc = rtreeInsertCell(pRtree, pLeft->pParent, &leftbbox, iHeight+1); + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + }else{ + RtreeNode *pParent = pLeft->pParent; + int iCell; + rc = nodeParentIndex(pRtree, pLeft, &iCell); + if( ALWAYS(rc==SQLITE_OK) ){ + nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); + rc = AdjustTree(pRtree, pParent, &leftbbox); + assert( rc==SQLITE_OK ); + } + if( NEVER(rc!=SQLITE_OK) ){ + goto splitnode_out; + } + } + if( (rc = rtreeInsertCell(pRtree, pRight->pParent, &rightbbox, iHeight+1)) ){ + goto splitnode_out; + } + + for(i=0; iiRowid ){ + newCellIsRight = 1; + } + if( rc!=SQLITE_OK ){ + goto splitnode_out; + } + } + if( pNode->iNode==1 ){ + for(i=0; iiRowid, pLeft, iHeight); + } + + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRight); + pRight = 0; + } + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pLeft); + pLeft = 0; + } + +splitnode_out: + nodeRelease(pRtree, pRight); + nodeRelease(pRtree, pLeft); + sqlite3_free(aCell); + return rc; +} + +/* +** If node pLeaf is not the root of the r-tree and its pParent pointer is +** still NULL, load all ancestor nodes of pLeaf into memory and populate +** the pLeaf->pParent chain all the way up to the root node. +** +** This operation is required when a row is deleted (or updated - an update +** is implemented as a delete followed by an insert). SQLite provides the +** rowid of the row to delete, which can be used to find the leaf on which +** the entry resides (argument pLeaf). Once the leaf is located, this +** function is called to determine its ancestry. +*/ +static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ + int rc = SQLITE_OK; + RtreeNode *pChild = pLeaf; + while( rc==SQLITE_OK && pChild->iNode!=1 && pChild->pParent==0 ){ + int rc2 = SQLITE_OK; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pRtree->pReadParent, 1, pChild->iNode); + rc = sqlite3_step(pRtree->pReadParent); + if( rc==SQLITE_ROW ){ + RtreeNode *pTest; /* Used to test for reference loops */ + i64 iNode; /* Node number of parent node */ + + /* Before setting pChild->pParent, test that we are not creating a + ** loop of references (as we would if, say, pChild==pParent). We don't + ** want to do this as it leads to a memory leak when trying to delete + ** the referenced counted node structures. + */ + iNode = sqlite3_column_int64(pRtree->pReadParent, 0); + for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent); + if( pTest==0 ){ + rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent); + } + } + rc = sqlite3_reset(pRtree->pReadParent); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && !pChild->pParent ){ + RTREE_IS_CORRUPT(pRtree); + rc = SQLITE_CORRUPT_VTAB; + } + pChild = pChild->pParent; + } + return rc; +} + +static int deleteCell(Rtree *, RtreeNode *, int, int); + +static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ + int rc; + int rc2; + RtreeNode *pParent = 0; + int iCell; + + assert( pNode->nRef==1 ); + + /* Remove the entry in the parent cell. */ + rc = nodeParentIndex(pRtree, pNode, &iCell); + if( rc==SQLITE_OK ){ + pParent = pNode->pParent; + pNode->pParent = 0; + rc = deleteCell(pRtree, pParent, iCell, iHeight+1); + testcase( rc!=SQLITE_OK ); + } + rc2 = nodeRelease(pRtree, pParent); + if( rc==SQLITE_OK ){ + rc = rc2; + } + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Remove the xxx_node entry. */ + sqlite3_bind_int64(pRtree->pDeleteNode, 1, pNode->iNode); + sqlite3_step(pRtree->pDeleteNode); + if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteNode)) ){ + return rc; + } + + /* Remove the xxx_parent entry. */ + sqlite3_bind_int64(pRtree->pDeleteParent, 1, pNode->iNode); + sqlite3_step(pRtree->pDeleteParent); + if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteParent)) ){ + return rc; + } + + /* Remove the node from the in-memory hash table and link it into + ** the Rtree.pDeleted list. Its contents will be re-inserted later on. + */ + nodeHashDelete(pRtree, pNode); + pNode->iNode = iHeight; + pNode->pNext = pRtree->pDeleted; + pNode->nRef++; + pRtree->pDeleted = pNode; + + return SQLITE_OK; +} + +static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ + RtreeNode *pParent = pNode->pParent; + int rc = SQLITE_OK; + if( pParent ){ + int ii; + int nCell = NCELL(pNode); + RtreeCell box; /* Bounding box for pNode */ + nodeGetCell(pRtree, pNode, 0, &box); + for(ii=1; iiiNode; + rc = nodeParentIndex(pRtree, pNode, &ii); + if( rc==SQLITE_OK ){ + nodeOverwriteCell(pRtree, pParent, &box, ii); + rc = fixBoundingBox(pRtree, pParent); + } + } + return rc; +} + +/* +** Delete the cell at index iCell of node pNode. After removing the +** cell, adjust the r-tree data structure if required. +*/ +static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ + RtreeNode *pParent; + int rc; + + if( SQLITE_OK!=(rc = fixLeafParent(pRtree, pNode)) ){ + return rc; + } + + /* Remove the cell from the node. This call just moves bytes around + ** the in-memory node image, so it cannot fail. + */ + nodeDeleteCell(pRtree, pNode, iCell); + + /* If the node is not the tree root and now has less than the minimum + ** number of cells, remove it from the tree. Otherwise, update the + ** cell in the parent node so that it tightly contains the updated + ** node. + */ + pParent = pNode->pParent; + assert( pParent || pNode->iNode==1 ); + if( pParent ){ + if( NCELL(pNode)nDim; iDim++){ + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); + aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); + } + } + for(iDim=0; iDimnDim; iDim++){ + aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); + } + + for(ii=0; iinDim; iDim++){ + RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - + DCOORD(aCell[ii].aCoord[iDim*2])); + aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); + } + } + + SortByDistance(aOrder, nCell, aDistance, aSpare); + nodeZero(pRtree, pNode); + + for(ii=0; rc==SQLITE_OK && ii<(nCell-(RTREE_MINCELLS(pRtree)+1)); ii++){ + RtreeCell *p = &aCell[aOrder[ii]]; + nodeInsertCell(pRtree, pNode, p); + if( p->iRowid==pCell->iRowid ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, p->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, p->iRowid, pNode->iNode); + } + } + } + if( rc==SQLITE_OK ){ + rc = fixBoundingBox(pRtree, pNode); + } + for(; rc==SQLITE_OK && iiiNode currently contains + ** the height of the sub-tree headed by the cell. + */ + RtreeNode *pInsert; + RtreeCell *p = &aCell[aOrder[ii]]; + rc = ChooseLeaf(pRtree, p, iHeight, &pInsert); + if( rc==SQLITE_OK ){ + int rc2; + rc = rtreeInsertCell(pRtree, pInsert, p, iHeight); + rc2 = nodeRelease(pRtree, pInsert); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + sqlite3_free(aCell); + return rc; +} + +/* +** Insert cell pCell into node pNode. Node pNode is the head of a +** subtree iHeight high (leaf nodes have iHeight==0). +*/ +static int rtreeInsertCell( + Rtree *pRtree, + RtreeNode *pNode, + RtreeCell *pCell, + int iHeight +){ + int rc = SQLITE_OK; + if( iHeight>0 ){ + RtreeNode *pChild = nodeHashLookup(pRtree, pCell->iRowid); + if( pChild ){ + nodeRelease(pRtree, pChild->pParent); + nodeReference(pNode); + pChild->pParent = pNode; + } + } + if( nodeInsertCell(pRtree, pNode, pCell) ){ + if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ + rc = SplitNode(pRtree, pNode, pCell, iHeight); + }else{ + pRtree->iReinsertHeight = iHeight; + rc = Reinsert(pRtree, pNode, pCell, iHeight); + } + }else{ + rc = AdjustTree(pRtree, pNode, pCell); + if( ALWAYS(rc==SQLITE_OK) ){ + if( iHeight==0 ){ + rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); + }else{ + rc = parentWrite(pRtree, pCell->iRowid, pNode->iNode); + } + } + } + return rc; +} + +static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ + int ii; + int rc = SQLITE_OK; + int nCell = NCELL(pNode); + + for(ii=0; rc==SQLITE_OK && iiiNode currently contains + ** the height of the sub-tree headed by the cell. + */ + rc = ChooseLeaf(pRtree, &cell, (int)pNode->iNode, &pInsert); + if( rc==SQLITE_OK ){ + int rc2; + rc = rtreeInsertCell(pRtree, pInsert, &cell, (int)pNode->iNode); + rc2 = nodeRelease(pRtree, pInsert); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + return rc; +} + +/* +** Select a currently unused rowid for a new r-tree record. +*/ +static int rtreeNewRowid(Rtree *pRtree, i64 *piRowid){ + int rc; + sqlite3_bind_null(pRtree->pWriteRowid, 1); + sqlite3_bind_null(pRtree->pWriteRowid, 2); + sqlite3_step(pRtree->pWriteRowid); + rc = sqlite3_reset(pRtree->pWriteRowid); + *piRowid = sqlite3_last_insert_rowid(pRtree->db); + return rc; +} + +/* +** Remove the entry with rowid=iDelete from the r-tree structure. +*/ +static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ + int rc; /* Return code */ + RtreeNode *pLeaf = 0; /* Leaf node containing record iDelete */ + int iCell; /* Index of iDelete cell in pLeaf */ + RtreeNode *pRoot = 0; /* Root node of rtree structure */ + + + /* Obtain a reference to the root node to initialize Rtree.iDepth */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + + /* Obtain a reference to the leaf node that contains the entry + ** about to be deleted. + */ + if( rc==SQLITE_OK ){ + rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); + } + +#ifdef CORRUPT_DB + assert( pLeaf!=0 || rc!=SQLITE_OK || CORRUPT_DB ); +#endif + + /* Delete the cell in question from the leaf node. */ + if( rc==SQLITE_OK && pLeaf ){ + int rc2; + rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); + if( rc==SQLITE_OK ){ + rc = deleteCell(pRtree, pLeaf, iCell, 0); + } + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + + /* Delete the corresponding entry in the _rowid table. */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pRtree->pDeleteRowid, 1, iDelete); + sqlite3_step(pRtree->pDeleteRowid); + rc = sqlite3_reset(pRtree->pDeleteRowid); + } + + /* Check if the root node now has exactly one child. If so, remove + ** it, schedule the contents of the child for reinsertion and + ** reduce the tree height by one. + ** + ** This is equivalent to copying the contents of the child into + ** the root node (the operation that Gutman's paper says to perform + ** in this scenario). + */ + if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ + int rc2; + RtreeNode *pChild = 0; + i64 iChild = nodeGetRowid(pRtree, pRoot, 0); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); /* tag-20210916a */ + if( rc==SQLITE_OK ){ + rc = removeNode(pRtree, pChild, pRtree->iDepth-1); + } + rc2 = nodeRelease(pRtree, pChild); + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ){ + pRtree->iDepth--; + writeInt16(pRoot->zData, pRtree->iDepth); + pRoot->isDirty = 1; + } + } + + /* Re-insert the contents of any underfull nodes removed from the tree. */ + for(pLeaf=pRtree->pDeleted; pLeaf; pLeaf=pRtree->pDeleted){ + if( rc==SQLITE_OK ){ + rc = reinsertNodeContent(pRtree, pLeaf); + } + pRtree->pDeleted = pLeaf->pNext; + pRtree->nNodeRef--; + sqlite3_free(pLeaf); + } + + /* Release the reference to the root node. */ + if( rc==SQLITE_OK ){ + rc = nodeRelease(pRtree, pRoot); + }else{ + nodeRelease(pRtree, pRoot); + } + + return rc; +} + +/* +** Rounding constants for float->double conversion. +*/ +#define RNDTOWARDS (1.0 - 1.0/8388608.0) /* Round towards zero */ +#define RNDAWAY (1.0 + 1.0/8388608.0) /* Round away from zero */ + +#if !defined(SQLITE_RTREE_INT_ONLY) +/* +** Convert an sqlite3_value into an RtreeValue (presumably a float) +** while taking care to round toward negative or positive, respectively. +*/ +static RtreeValue rtreeValueDown(sqlite3_value *v){ + double d = sqlite3_value_double(v); + float f = (float)d; + if( f>d ){ + f = (float)(d*(d<0 ? RNDAWAY : RNDTOWARDS)); + } + return f; +} +static RtreeValue rtreeValueUp(sqlite3_value *v){ + double d = sqlite3_value_double(v); + float f = (float)d; + if( fbase.zErrMsg) to an appropriate value and returns +** SQLITE_CONSTRAINT. +** +** Parameter iCol is the index of the leftmost column involved in the +** constraint failure. If it is 0, then the constraint that failed is +** the unique constraint on the id column. Otherwise, it is the rtree +** (c1<=c2) constraint on columns iCol and iCol+1 that has failed. +** +** If an OOM occurs, SQLITE_NOMEM is returned instead of SQLITE_CONSTRAINT. +*/ +static int rtreeConstraintError(Rtree *pRtree, int iCol){ + sqlite3_stmt *pStmt = 0; + char *zSql; + int rc; + + assert( iCol==0 || iCol%2 ); + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", pRtree->zDb, pRtree->zName); + if( zSql ){ + rc = sqlite3_prepare_v2(pRtree->db, zSql, -1, &pStmt, 0); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + if( iCol==0 ){ + const char *zCol = sqlite3_column_name(pStmt, 0); + pRtree->base.zErrMsg = sqlite3_mprintf( + "UNIQUE constraint failed: %s.%s", pRtree->zName, zCol + ); + }else{ + const char *zCol1 = sqlite3_column_name(pStmt, iCol); + const char *zCol2 = sqlite3_column_name(pStmt, iCol+1); + pRtree->base.zErrMsg = sqlite3_mprintf( + "rtree constraint failed: %s.(%s<=%s)", pRtree->zName, zCol1, zCol2 + ); + } + } + + sqlite3_finalize(pStmt); + return (rc==SQLITE_OK ? SQLITE_CONSTRAINT : rc); +} + + + +/* +** The xUpdate method for rtree module virtual tables. +*/ +static int rtreeUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **aData, + sqlite_int64 *pRowid +){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ + + if( pRtree->nNodeRef ){ + /* Unable to write to the btree while another cursor is reading from it, + ** since the write might do a rebalance which would disrupt the read + ** cursor. */ + return SQLITE_LOCKED_VTAB; + } + rtreeReference(pRtree); + assert(nData>=1); + + cell.iRowid = 0; /* Used only to suppress a compiler warning */ + + /* Constraint handling. A write operation on an r-tree table may return + ** SQLITE_CONSTRAINT for two reasons: + ** + ** 1. A duplicate rowid value, or + ** 2. The supplied data violates the "x2>=x1" constraint. + ** + ** In the first case, if the conflict-handling mode is REPLACE, then + ** the conflicting row can be removed before proceeding. In the second + ** case, SQLITE_CONSTRAINT must be returned regardless of the + ** conflict-handling mode specified by the user. + */ + if( nData>1 ){ + int ii; + int nn = nData - 4; + + if( nn > pRtree->nDim2 ) nn = pRtree->nDim2; + /* Populate the cell.aCoord[] array. The first coordinate is aData[3]. + ** + ** NB: nData can only be less than nDim*2+3 if the rtree is mis-declared + ** with "column" that are interpreted as table constraints. + ** Example: CREATE VIRTUAL TABLE bad USING rtree(x,y,CHECK(y>5)); + ** This problem was discovered after years of use, so we silently ignore + ** these kinds of misdeclared tables to avoid breaking any legacy. + */ + +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + for(ii=0; iicell.aCoord[ii+1].f ){ + rc = rtreeConstraintError(pRtree, ii+1); + goto constraint; + } + } + }else +#endif + { + for(ii=0; iicell.aCoord[ii+1].i ){ + rc = rtreeConstraintError(pRtree, ii+1); + goto constraint; + } + } + } + + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( sqlite3_value_type(aData[2])!=SQLITE_NULL ){ + cell.iRowid = sqlite3_value_int64(aData[2]); + if( sqlite3_value_type(aData[0])==SQLITE_NULL + || sqlite3_value_int64(aData[0])!=cell.iRowid + ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = rtreeConstraintError(pRtree, 0); + goto constraint; + } + } + } + bHaveRowid = 1; + } + } + + /* If aData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( sqlite3_value_type(aData[0])!=SQLITE_NULL ){ + rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(aData[0])); + } + + /* If the aData[] array contains more than one element, elements + ** (aData[2]..aData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf = 0; + + /* Figure out the rowid of the new row. */ + if( bHaveRowid==0 ){ + rc = rtreeNewRowid(pRtree, &cell.iRowid); + } + *pRowid = cell.iRowid; + + if( rc==SQLITE_OK ){ + rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); + } + if( rc==SQLITE_OK ){ + int rc2; + pRtree->iReinsertHeight = -1; + rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + if( rc==SQLITE_OK && pRtree->nAux ){ + sqlite3_stmt *pUp = pRtree->pWriteAux; + int jj; + sqlite3_bind_int64(pUp, 1, *pRowid); + for(jj=0; jjnAux; jj++){ + sqlite3_bind_value(pUp, jj+2, aData[pRtree->nDim2+3+jj]); + } + sqlite3_step(pUp); + rc = sqlite3_reset(pUp); + } + } + +constraint: + rtreeRelease(pRtree); + return rc; +} + +/* +** Called when a transaction starts. +*/ +static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + assert( pRtree->inWrTrans==0 ); + pRtree->inWrTrans++; + return SQLITE_OK; +} + +/* +** Called when a transaction completes (either by COMMIT or ROLLBACK). +** The sqlite3_blob object should be released at this point. +*/ +static int rtreeEndTransaction(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + pRtree->inWrTrans = 0; + nodeBlobReset(pRtree); + return SQLITE_OK; +} + +/* +** The xRename method for rtree module virtual tables. +*/ +static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_NOMEM; + char *zSql = sqlite3_mprintf( + "ALTER TABLE %Q.'%q_node' RENAME TO \"%w_node\";" + "ALTER TABLE %Q.'%q_parent' RENAME TO \"%w_parent\";" + "ALTER TABLE %Q.'%q_rowid' RENAME TO \"%w_rowid\";" + , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName + ); + if( zSql ){ + nodeBlobReset(pRtree); + rc = sqlite3_exec(pRtree->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } + return rc; +} + +/* +** The xSavepoint method. +** +** This module does not need to do anything to support savepoints. However, +** it uses this hook to close any open blob handle. This is done because a +** DROP TABLE command - which fortunately always opens a savepoint - cannot +** succeed if there are any open blob handles. i.e. if the blob handle were +** not closed here, the following would fail: +** +** BEGIN; +** INSERT INTO rtree... +** DROP TABLE ; -- Would fail with SQLITE_LOCKED +** COMMIT; +*/ +static int rtreeSavepoint(sqlite3_vtab *pVtab, int iSavepoint){ + Rtree *pRtree = (Rtree *)pVtab; + u8 iwt = pRtree->inWrTrans; + UNUSED_PARAMETER(iSavepoint); + pRtree->inWrTrans = 0; + nodeBlobReset(pRtree); + pRtree->inWrTrans = iwt; + return SQLITE_OK; +} + +/* +** This function populates the pRtree->nRowEst variable with an estimate +** of the number of rows in the virtual table. If possible, this is based +** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. +*/ +static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ + const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; + char *zSql; + sqlite3_stmt *p; + int rc; + i64 nRow = RTREE_MIN_ROWEST; + + rc = sqlite3_table_column_metadata( + db, pRtree->zDb, "sqlite_stat1",0,0,0,0,0,0 + ); + if( rc!=SQLITE_OK ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + return rc==SQLITE_ERROR ? SQLITE_OK : rc; + } + zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); + if( rc==SQLITE_OK ){ + if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); + rc = sqlite3_finalize(p); + } + sqlite3_free(zSql); + } + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + return rc; +} + + +/* +** Return true if zName is the extension on one of the shadow tables used +** by this module. +*/ +static int rtreeShadowName(const char *zName){ + static const char *azName[] = { + "node", "parent", "rowid" + }; + unsigned int i; + for(i=0; idb = db; + + if( isCreate ){ + char *zCreate; + sqlite3_str *p = sqlite3_str_new(db); + int ii; + sqlite3_str_appendf(p, + "CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY,nodeno", + zDb, zPrefix); + for(ii=0; iinAux; ii++){ + sqlite3_str_appendf(p,",a%d",ii); + } + sqlite3_str_appendf(p, + ");CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY,data);", + zDb, zPrefix); + sqlite3_str_appendf(p, + "CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY,parentnode);", + zDb, zPrefix); + sqlite3_str_appendf(p, + "INSERT INTO \"%w\".\"%w_node\"VALUES(1,zeroblob(%d))", + zDb, zPrefix, pRtree->iNodeSize); + zCreate = sqlite3_str_finish(p); + if( !zCreate ){ + return SQLITE_NOMEM; + } + rc = sqlite3_exec(db, zCreate, 0, 0, 0); + sqlite3_free(zCreate); + if( rc!=SQLITE_OK ){ + return rc; + } + } + + appStmt[0] = &pRtree->pWriteNode; + appStmt[1] = &pRtree->pDeleteNode; + appStmt[2] = &pRtree->pReadRowid; + appStmt[3] = &pRtree->pWriteRowid; + appStmt[4] = &pRtree->pDeleteRowid; + appStmt[5] = &pRtree->pReadParent; + appStmt[6] = &pRtree->pWriteParent; + appStmt[7] = &pRtree->pDeleteParent; + + rc = rtreeQueryStat1(db, pRtree); + for(i=0; inAux==0 ){ + zFormat = azSql[i]; + }else { + /* An UPSERT is very slightly slower than REPLACE, but it is needed + ** if there are auxiliary columns */ + zFormat = "INSERT INTO\"%w\".\"%w_rowid\"(rowid,nodeno)VALUES(?1,?2)" + "ON CONFLICT(rowid)DO UPDATE SET nodeno=excluded.nodeno"; + } + zSql = sqlite3_mprintf(zFormat, zDb, zPrefix); + if( zSql ){ + rc = sqlite3_prepare_v3(db, zSql, -1, f, appStmt[i], 0); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3_free(zSql); + } + if( pRtree->nAux ){ + pRtree->zReadAuxSql = sqlite3_mprintf( + "SELECT * FROM \"%w\".\"%w_rowid\" WHERE rowid=?1", + zDb, zPrefix); + if( pRtree->zReadAuxSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_str *p = sqlite3_str_new(db); + int ii; + char *zSql; + sqlite3_str_appendf(p, "UPDATE \"%w\".\"%w_rowid\"SET ", zDb, zPrefix); + for(ii=0; iinAux; ii++){ + if( ii ) sqlite3_str_append(p, ",", 1); +#ifdef SQLITE_ENABLE_GEOPOLY + if( iinAuxNotNull ){ + sqlite3_str_appendf(p,"a%d=coalesce(?%d,a%d)",ii,ii+2,ii); + }else +#endif + { + sqlite3_str_appendf(p,"a%d=?%d",ii,ii+2); + } + } + sqlite3_str_appendf(p, " WHERE rowid=?1"); + zSql = sqlite3_str_finish(p); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v3(db, zSql, -1, f, &pRtree->pWriteAux, 0); + sqlite3_free(zSql); + } + } + } + + return rc; +} + +/* +** The second argument to this function contains the text of an SQL statement +** that returns a single integer value. The statement is compiled and executed +** using database connection db. If successful, the integer value returned +** is written to *piVal and SQLITE_OK returned. Otherwise, an SQLite error +** code is returned and the value of *piVal after returning is not defined. +*/ +static int getIntFromStmt(sqlite3 *db, const char *zSql, int *piVal){ + int rc = SQLITE_NOMEM; + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *piVal = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_finalize(pStmt); + } + } + return rc; +} + +/* +** This function is called from within the xConnect() or xCreate() method to +** determine the node-size used by the rtree table being created or connected +** to. If successful, pRtree->iNodeSize is populated and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned. +** +** If this function is being called as part of an xConnect(), then the rtree +** table already exists. In this case the node-size is determined by inspecting +** the root node of the tree. +** +** Otherwise, for an xCreate(), use 64 bytes less than the database page-size. +** This ensures that each node is stored on a single database page. If the +** database page-size is so large that more than RTREE_MAXCELLS entries +** would fit in a single node, use a smaller node-size. +*/ +static int getNodeSize( + sqlite3 *db, /* Database handle */ + Rtree *pRtree, /* Rtree handle */ + int isCreate, /* True for xCreate, false for xConnect */ + char **pzErr /* OUT: Error message, if any */ +){ + int rc; + char *zSql; + if( isCreate ){ + int iPageSize = 0; + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", pRtree->zDb); + rc = getIntFromStmt(db, zSql, &iPageSize); + if( rc==SQLITE_OK ){ + pRtree->iNodeSize = iPageSize-64; + if( (4+pRtree->nBytesPerCell*RTREE_MAXCELLS)iNodeSize ){ + pRtree->iNodeSize = 4+pRtree->nBytesPerCell*RTREE_MAXCELLS; + } + }else{ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + }else{ + zSql = sqlite3_mprintf( + "SELECT length(data) FROM '%q'.'%q_node' WHERE nodeno = 1", + pRtree->zDb, pRtree->zName + ); + rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + }else if( pRtree->iNodeSize<(512-64) ){ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + *pzErr = sqlite3_mprintf("undersize RTree blobs in \"%q_node\"", + pRtree->zName); + } + } + + sqlite3_free(zSql); + return rc; +} + +/* +** Return the length of a token +*/ +static int rtreeTokenLength(const char *z){ + int dummy = 0; + return sqlite3GetToken((const unsigned char*)z,&dummy); +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the r-tree virtual table. +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... +*/ +static int rtreeInit( + sqlite3 *db, /* Database connection */ + void *pAux, /* One of the RTREE_COORD_* constants */ + int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ + sqlite3_vtab **ppVtab, /* OUT: New virtual table */ + char **pzErr, /* OUT: Error message, if any */ + int isCreate /* True for xCreate, false for xConnect */ +){ + int rc = SQLITE_OK; + Rtree *pRtree; + int nDb; /* Length of string argv[1] */ + int nName; /* Length of string argv[2] */ + int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32); + sqlite3_str *pSql; + char *zSql; + int ii = 4; + int iErr; + + const char *aErrMsg[] = { + 0, /* 0 */ + "Wrong number of columns for an rtree table", /* 1 */ + "Too few columns for an rtree table", /* 2 */ + "Too many columns for an rtree table", /* 3 */ + "Auxiliary rtree columns must be last" /* 4 */ + }; + + assert( RTREE_MAX_AUX_COLUMN<256 ); /* Aux columns counted by a u8 */ + if( argc<6 || argc>RTREE_MAX_AUX_COLUMN+3 ){ + *pzErr = sqlite3_mprintf("%s", aErrMsg[2 + (argc>=6)]); + return SQLITE_ERROR; + } + + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Allocate the sqlite3_vtab structure */ + nDb = (int)strlen(argv[1]); + nName = (int)strlen(argv[2]); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); + if( !pRtree ){ + return SQLITE_NOMEM; + } + memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + pRtree->nBusy = 1; + pRtree->base.pModule = &rtreeModule; + pRtree->zDb = (char *)&pRtree[1]; + pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->eCoordType = (u8)eCoordType; + memcpy(pRtree->zDb, argv[1], nDb); + memcpy(pRtree->zName, argv[2], nName); + + + /* Create/Connect to the underlying relational database schema. If + ** that is successful, call sqlite3_declare_vtab() to configure + ** the r-tree table schema. + */ + pSql = sqlite3_str_new(db); + sqlite3_str_appendf(pSql, "CREATE TABLE x(%.*s INT", + rtreeTokenLength(argv[3]), argv[3]); + for(ii=4; iinAux++; + sqlite3_str_appendf(pSql, ",%.*s", rtreeTokenLength(zArg+1), zArg+1); + }else if( pRtree->nAux>0 ){ + break; + }else{ + static const char *azFormat[] = {",%.*s REAL", ",%.*s INT"}; + pRtree->nDim2++; + sqlite3_str_appendf(pSql, azFormat[eCoordType], + rtreeTokenLength(zArg), zArg); + } + } + sqlite3_str_appendf(pSql, ");"); + zSql = sqlite3_str_finish(pSql); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else if( iinDim = pRtree->nDim2/2; + if( pRtree->nDim<1 ){ + iErr = 2; + }else if( pRtree->nDim2>RTREE_MAX_DIMENSIONS*2 ){ + iErr = 3; + }else if( pRtree->nDim2 % 2 ){ + iErr = 1; + }else{ + iErr = 0; + } + if( iErr ){ + *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]); + goto rtreeInit_fail; + } + pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; + + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate, pzErr); + if( rc ) goto rtreeInit_fail; + rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate); + if( rc ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + goto rtreeInit_fail; + } + + *ppVtab = (sqlite3_vtab *)pRtree; + return SQLITE_OK; + +rtreeInit_fail: + if( rc==SQLITE_OK ) rc = SQLITE_ERROR; + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); + rtreeRelease(pRtree); + return rc; +} + + +/* +** Implementation of a scalar function that decodes r-tree nodes to +** human readable strings. This can be used for debugging and analysis. +** +** The scalar function takes two arguments: (1) the number of dimensions +** to the rtree (between 1 and 5, inclusive) and (2) a blob of data containing +** an r-tree node. For a two-dimensional r-tree structure called "rt", to +** deserialize all nodes, a statement like: +** +** SELECT rtreenode(2, data) FROM rt_node; +** +** The human readable string takes the form of a Tcl list with one +** entry for each cell in the r-tree node. Each entry is itself a +** list, containing the 8-byte rowid/pageno followed by the +** *2 coordinates. +*/ +static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ + RtreeNode node; + Rtree tree; + int ii; + int nData; + int errCode; + sqlite3_str *pOut; + + UNUSED_PARAMETER(nArg); + memset(&node, 0, sizeof(RtreeNode)); + memset(&tree, 0, sizeof(Rtree)); + tree.nDim = (u8)sqlite3_value_int(apArg[0]); + if( tree.nDim<1 || tree.nDim>5 ) return; + tree.nDim2 = tree.nDim*2; + tree.nBytesPerCell = 8 + 8 * tree.nDim; + node.zData = (u8 *)sqlite3_value_blob(apArg[1]); + nData = sqlite3_value_bytes(apArg[1]); + if( nData<4 ) return; + if( nData0 ) sqlite3_str_append(pOut, " ", 1); + sqlite3_str_appendf(pOut, "{%lld", cell.iRowid); + for(jj=0; jjrc==SQLITE_OK ) pCheck->rc = rc; +} + +/* +** The second and subsequent arguments to this function are a format string +** and printf style arguments. This function formats the string and attempts +** to compile it as an SQL statement. +** +** If successful, a pointer to the new SQL statement is returned. Otherwise, +** NULL is returned and an error code left in RtreeCheck.rc. +*/ +static sqlite3_stmt *rtreeCheckPrepare( + RtreeCheck *pCheck, /* RtreeCheck object */ + const char *zFmt, ... /* Format string and trailing args */ +){ + va_list ap; + char *z; + sqlite3_stmt *pRet = 0; + + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + + if( pCheck->rc==SQLITE_OK ){ + if( z==0 ){ + pCheck->rc = SQLITE_NOMEM; + }else{ + pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0); + } + } + + sqlite3_free(z); + va_end(ap); + return pRet; +} + +/* +** The second and subsequent arguments to this function are a printf() +** style format string and arguments. This function formats the string and +** appends it to the report being accumuated in pCheck. +*/ +static void rtreeCheckAppendMsg(RtreeCheck *pCheck, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + if( pCheck->rc==SQLITE_OK && pCheck->nErrrc = SQLITE_NOMEM; + }else{ + pCheck->zReport = sqlite3_mprintf("%z%s%z", + pCheck->zReport, (pCheck->zReport ? "\n" : ""), z + ); + if( pCheck->zReport==0 ){ + pCheck->rc = SQLITE_NOMEM; + } + } + pCheck->nErr++; + } + va_end(ap); +} + +/* +** This function is a no-op if there is already an error code stored +** in the RtreeCheck object indicated by the first argument. NULL is +** returned in this case. +** +** Otherwise, the contents of rtree table node iNode are loaded from +** the database and copied into a buffer obtained from sqlite3_malloc(). +** If no error occurs, a pointer to the buffer is returned and (*pnNode) +** is set to the size of the buffer in bytes. +** +** Or, if an error does occur, NULL is returned and an error code left +** in the RtreeCheck object. The final value of *pnNode is undefined in +** this case. +*/ +static u8 *rtreeCheckGetNode(RtreeCheck *pCheck, i64 iNode, int *pnNode){ + u8 *pRet = 0; /* Return value */ + + if( pCheck->rc==SQLITE_OK && pCheck->pGetNode==0 ){ + pCheck->pGetNode = rtreeCheckPrepare(pCheck, + "SELECT data FROM %Q.'%q_node' WHERE nodeno=?", + pCheck->zDb, pCheck->zTab + ); + } + + if( pCheck->rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck->pGetNode, 1, iNode); + if( sqlite3_step(pCheck->pGetNode)==SQLITE_ROW ){ + int nNode = sqlite3_column_bytes(pCheck->pGetNode, 0); + const u8 *pNode = (const u8*)sqlite3_column_blob(pCheck->pGetNode, 0); + pRet = sqlite3_malloc64(nNode); + if( pRet==0 ){ + pCheck->rc = SQLITE_NOMEM; + }else{ + memcpy(pRet, pNode, nNode); + *pnNode = nNode; + } + } + rtreeCheckReset(pCheck, pCheck->pGetNode); + if( pCheck->rc==SQLITE_OK && pRet==0 ){ + rtreeCheckAppendMsg(pCheck, "Node %lld missing from database", iNode); + } + } + + return pRet; +} + +/* +** This function is used to check that the %_parent (if bLeaf==0) or %_rowid +** (if bLeaf==1) table contains a specified entry. The schemas of the +** two tables are: +** +** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) +** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER, ...) +** +** In both cases, this function checks that there exists an entry with +** IPK value iKey and the second column set to iVal. +** +*/ +static void rtreeCheckMapping( + RtreeCheck *pCheck, /* RtreeCheck object */ + int bLeaf, /* True for a leaf cell, false for interior */ + i64 iKey, /* Key for mapping */ + i64 iVal /* Expected value for mapping */ +){ + int rc; + sqlite3_stmt *pStmt; + const char *azSql[2] = { + "SELECT parentnode FROM %Q.'%q_parent' WHERE nodeno=?1", + "SELECT nodeno FROM %Q.'%q_rowid' WHERE rowid=?1" + }; + + assert( bLeaf==0 || bLeaf==1 ); + if( pCheck->aCheckMapping[bLeaf]==0 ){ + pCheck->aCheckMapping[bLeaf] = rtreeCheckPrepare(pCheck, + azSql[bLeaf], pCheck->zDb, pCheck->zTab + ); + } + if( pCheck->rc!=SQLITE_OK ) return; + + pStmt = pCheck->aCheckMapping[bLeaf]; + sqlite3_bind_int64(pStmt, 1, iKey); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_DONE ){ + rtreeCheckAppendMsg(pCheck, "Mapping (%lld -> %lld) missing from %s table", + iKey, iVal, (bLeaf ? "%_rowid" : "%_parent") + ); + }else if( rc==SQLITE_ROW ){ + i64 ii = sqlite3_column_int64(pStmt, 0); + if( ii!=iVal ){ + rtreeCheckAppendMsg(pCheck, + "Found (%lld -> %lld) in %s table, expected (%lld -> %lld)", + iKey, ii, (bLeaf ? "%_rowid" : "%_parent"), iKey, iVal + ); + } + } + rtreeCheckReset(pCheck, pStmt); +} + +/* +** Argument pCell points to an array of coordinates stored on an rtree page. +** This function checks that the coordinates are internally consistent (no +** x1>x2 conditions) and adds an error message to the RtreeCheck object +** if they are not. +** +** Additionally, if pParent is not NULL, then it is assumed to point to +** the array of coordinates on the parent page that bound the page +** containing pCell. In this case it is also verified that the two +** sets of coordinates are mutually consistent and an error message added +** to the RtreeCheck object if they are not. +*/ +static void rtreeCheckCellCoord( + RtreeCheck *pCheck, + i64 iNode, /* Node id to use in error messages */ + int iCell, /* Cell number to use in error messages */ + u8 *pCell, /* Pointer to cell coordinates */ + u8 *pParent /* Pointer to parent coordinates */ +){ + RtreeCoord c1, c2; + RtreeCoord p1, p2; + int i; + + for(i=0; inDim; i++){ + readCoord(&pCell[4*2*i], &c1); + readCoord(&pCell[4*(2*i + 1)], &c2); + + /* printf("%e, %e\n", c1.u.f, c2.u.f); */ + if( pCheck->bInt ? c1.i>c2.i : c1.f>c2.f ){ + rtreeCheckAppendMsg(pCheck, + "Dimension %d of cell %d on node %lld is corrupt", i, iCell, iNode + ); + } + + if( pParent ){ + readCoord(&pParent[4*2*i], &p1); + readCoord(&pParent[4*(2*i + 1)], &p2); + + if( (pCheck->bInt ? c1.ibInt ? c2.i>p2.i : c2.f>p2.f) + ){ + rtreeCheckAppendMsg(pCheck, + "Dimension %d of cell %d on node %lld is corrupt relative to parent" + , i, iCell, iNode + ); + } + } + } +} + +/* +** Run rtreecheck() checks on node iNode, which is at depth iDepth within +** the r-tree structure. Argument aParent points to the array of coordinates +** that bound node iNode on the parent node. +** +** If any problems are discovered, an error message is appended to the +** report accumulated in the RtreeCheck object. +*/ +static void rtreeCheckNode( + RtreeCheck *pCheck, + int iDepth, /* Depth of iNode (0==leaf) */ + u8 *aParent, /* Buffer containing parent coords */ + i64 iNode /* Node to check */ +){ + u8 *aNode = 0; + int nNode = 0; + + assert( iNode==1 || aParent!=0 ); + assert( pCheck->nDim>0 ); + + aNode = rtreeCheckGetNode(pCheck, iNode, &nNode); + if( aNode ){ + if( nNode<4 ){ + rtreeCheckAppendMsg(pCheck, + "Node %lld is too small (%d bytes)", iNode, nNode + ); + }else{ + int nCell; /* Number of cells on page */ + int i; /* Used to iterate through cells */ + if( aParent==0 ){ + iDepth = readInt16(aNode); + if( iDepth>RTREE_MAX_DEPTH ){ + rtreeCheckAppendMsg(pCheck, "Rtree depth out of range (%d)", iDepth); + sqlite3_free(aNode); + return; + } + } + nCell = readInt16(&aNode[2]); + if( (4 + nCell*(8 + pCheck->nDim*2*4))>nNode ){ + rtreeCheckAppendMsg(pCheck, + "Node %lld is too small for cell count of %d (%d bytes)", + iNode, nCell, nNode + ); + }else{ + for(i=0; inDim*2*4)]; + i64 iVal = readInt64(pCell); + rtreeCheckCellCoord(pCheck, iNode, i, &pCell[8], aParent); + + if( iDepth>0 ){ + rtreeCheckMapping(pCheck, 0, iVal, iNode); + rtreeCheckNode(pCheck, iDepth-1, &pCell[8], iVal); + pCheck->nNonLeaf++; + }else{ + rtreeCheckMapping(pCheck, 1, iVal, iNode); + pCheck->nLeaf++; + } + } + } + } + sqlite3_free(aNode); + } +} + +/* +** The second argument to this function must be either "_rowid" or +** "_parent". This function checks that the number of entries in the +** %_rowid or %_parent table is exactly nExpect. If not, it adds +** an error message to the report in the RtreeCheck object indicated +** by the first argument. +*/ +static void rtreeCheckCount(RtreeCheck *pCheck, const char *zTbl, i64 nExpect){ + if( pCheck->rc==SQLITE_OK ){ + sqlite3_stmt *pCount; + pCount = rtreeCheckPrepare(pCheck, "SELECT count(*) FROM %Q.'%q%s'", + pCheck->zDb, pCheck->zTab, zTbl + ); + if( pCount ){ + if( sqlite3_step(pCount)==SQLITE_ROW ){ + i64 nActual = sqlite3_column_int64(pCount, 0); + if( nActual!=nExpect ){ + rtreeCheckAppendMsg(pCheck, "Wrong number of entries in %%%s table" + " - expected %lld, actual %lld" , zTbl, nExpect, nActual + ); + } + } + pCheck->rc = sqlite3_finalize(pCount); + } + } +} + +/* +** This function does the bulk of the work for the rtree integrity-check. +** It is called by rtreecheck(), which is the SQL function implementation. +*/ +static int rtreeCheckTable( + sqlite3 *db, /* Database handle to access db through */ + const char *zDb, /* Name of db ("main", "temp" etc.) */ + const char *zTab, /* Name of rtree table to check */ + char **pzReport /* OUT: sqlite3_malloc'd report text */ +){ + RtreeCheck check; /* Common context for various routines */ + sqlite3_stmt *pStmt = 0; /* Used to find column count of rtree table */ + int bEnd = 0; /* True if transaction should be closed */ + int nAux = 0; /* Number of extra columns. */ + + /* Initialize the context object */ + memset(&check, 0, sizeof(check)); + check.db = db; + check.zDb = zDb; + check.zTab = zTab; + + /* If there is not already an open transaction, open one now. This is + ** to ensure that the queries run as part of this integrity-check operate + ** on a consistent snapshot. */ + if( sqlite3_get_autocommit(db) ){ + check.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + bEnd = 1; + } + + /* Find the number of auxiliary columns */ + if( check.rc==SQLITE_OK ){ + pStmt = rtreeCheckPrepare(&check, "SELECT * FROM %Q.'%q_rowid'", zDb, zTab); + if( pStmt ){ + nAux = sqlite3_column_count(pStmt) - 2; + sqlite3_finalize(pStmt); + }else + if( check.rc!=SQLITE_NOMEM ){ + check.rc = SQLITE_OK; + } + } + + /* Find number of dimensions in the rtree table. */ + pStmt = rtreeCheckPrepare(&check, "SELECT * FROM %Q.%Q", zDb, zTab); + if( pStmt ){ + int rc; + check.nDim = (sqlite3_column_count(pStmt) - 1 - nAux) / 2; + if( check.nDim<1 ){ + rtreeCheckAppendMsg(&check, "Schema corrupt or not an rtree"); + }else if( SQLITE_ROW==sqlite3_step(pStmt) ){ + check.bInt = (sqlite3_column_type(pStmt, 1)==SQLITE_INTEGER); + } + rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_CORRUPT ) check.rc = rc; + } + + /* Do the actual integrity-check */ + if( check.nDim>=1 ){ + if( check.rc==SQLITE_OK ){ + rtreeCheckNode(&check, 0, 0, 1); + } + rtreeCheckCount(&check, "_rowid", check.nLeaf); + rtreeCheckCount(&check, "_parent", check.nNonLeaf); + } + + /* Finalize SQL statements used by the integrity-check */ + sqlite3_finalize(check.pGetNode); + sqlite3_finalize(check.aCheckMapping[0]); + sqlite3_finalize(check.aCheckMapping[1]); + + /* If one was opened, close the transaction */ + if( bEnd ){ + int rc = sqlite3_exec(db, "END", 0, 0, 0); + if( check.rc==SQLITE_OK ) check.rc = rc; + } + *pzReport = check.zReport; + return check.rc; +} + +/* +** Usage: +** +** rtreecheck(); +** rtreecheck(, ); +** +** Invoking this SQL function runs an integrity-check on the named rtree +** table. The integrity-check verifies the following: +** +** 1. For each cell in the r-tree structure (%_node table), that: +** +** a) for each dimension, (coord1 <= coord2). +** +** b) unless the cell is on the root node, that the cell is bounded +** by the parent cell on the parent node. +** +** c) for leaf nodes, that there is an entry in the %_rowid +** table corresponding to the cell's rowid value that +** points to the correct node. +** +** d) for cells on non-leaf nodes, that there is an entry in the +** %_parent table mapping from the cell's child node to the +** node that it resides on. +** +** 2. That there are the same number of entries in the %_rowid table +** as there are leaf cells in the r-tree structure, and that there +** is a leaf cell that corresponds to each entry in the %_rowid table. +** +** 3. That there are the same number of entries in the %_parent table +** as there are non-leaf cells in the r-tree structure, and that +** there is a non-leaf cell that corresponds to each entry in the +** %_parent table. +*/ +static void rtreecheck( + sqlite3_context *ctx, + int nArg, + sqlite3_value **apArg +){ + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error(ctx, + "wrong number of arguments to function rtreecheck()", -1 + ); + }else{ + int rc; + char *zReport = 0; + const char *zDb = (const char*)sqlite3_value_text(apArg[0]); + const char *zTab; + if( nArg==1 ){ + zTab = zDb; + zDb = "main"; + }else{ + zTab = (const char*)sqlite3_value_text(apArg[1]); + } + rc = rtreeCheckTable(sqlite3_context_db_handle(ctx), zDb, zTab, &zReport); + if( rc==SQLITE_OK ){ + sqlite3_result_text(ctx, zReport ? zReport : "ok", -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(ctx, rc); + } + sqlite3_free(zReport); + } +} + +/* Conditionally include the geopoly code */ +#ifdef SQLITE_ENABLE_GEOPOLY +/************** Include geopoly.c in the middle of rtree.c *******************/ +/************** Begin file geopoly.c *****************************************/ +/* +** 2018-05-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements an alternative R-Tree virtual table that +** uses polygons to express the boundaries of 2-dimensional objects. +** +** This file is #include-ed onto the end of "rtree.c" so that it has +** access to all of the R-Tree internals. +*/ +/* #include */ + +/* Enable -DGEOPOLY_ENABLE_DEBUG for debugging facilities */ +#ifdef GEOPOLY_ENABLE_DEBUG + static int geo_debug = 0; +# define GEODEBUG(X) if(geo_debug)printf X +#else +# define GEODEBUG(X) +#endif + +#ifndef JSON_NULL /* The following stuff repeats things found in json1 */ +/* +** Versions of isspace(), isalnum() and isdigit() to which it is safe +** to pass signed char values. +*/ +#ifdef sqlite3Isdigit + /* Use the SQLite core versions if this routine is part of the + ** SQLite amalgamation */ +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isxdigit(x) sqlite3Isxdigit(x) +#else + /* Use the standard library for separate compilation */ +#include /* amalgamator: keep */ +# define safe_isdigit(x) isdigit((unsigned char)(x)) +# define safe_isalnum(x) isalnum((unsigned char)(x)) +# define safe_isxdigit(x) isxdigit((unsigned char)(x)) +#endif + +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function. +*/ +static const char geopolyIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define safe_isspace(x) (geopolyIsSpace[(unsigned char)x]) +#endif /* JSON NULL - back to original code */ + +/* Compiler and version */ +#ifndef GCC_VERSION +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#endif +#ifndef MSVC_VERSION +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif +#endif + +/* Datatype for coordinates +*/ +typedef float GeoCoord; + +/* +** Internal representation of a polygon. +** +** The polygon consists of a sequence of vertexes. There is a line +** segment between each pair of vertexes, and one final segment from +** the last vertex back to the first. (This differs from the GeoJSON +** standard in which the final vertex is a repeat of the first.) +** +** The polygon follows the right-hand rule. The area to the right of +** each segment is "outside" and the area to the left is "inside". +** +** The on-disk representation consists of a 4-byte header followed by +** the values. The 4-byte header is: +** +** encoding (1 byte) 0=big-endian, 1=little-endian +** nvertex (3 bytes) Number of vertexes as a big-endian integer +** +** Enough space is allocated for 4 coordinates, to work around over-zealous +** warnings coming from some compiler (notably, clang). In reality, the size +** of each GeoPoly memory allocate is adjusted as necessary so that the +** GeoPoly.a[] array at the end is the appropriate size. +*/ +typedef struct GeoPoly GeoPoly; +struct GeoPoly { + int nVertex; /* Number of vertexes */ + unsigned char hdr[4]; /* Header for on-disk representation */ + GeoCoord a[8]; /* 2*nVertex values. X (longitude) first, then Y */ +}; + +/* The size of a memory allocation needed for a GeoPoly object sufficient +** to hold N coordinate pairs. +*/ +#define GEOPOLY_SZ(N) (sizeof(GeoPoly) + sizeof(GeoCoord)*2*((N)-4)) + +/* Macros to access coordinates of a GeoPoly. +** We have to use these macros, rather than just say p->a[i] in order +** to silence (incorrect) UBSAN warnings if the array index is too large. +*/ +#define GeoX(P,I) (((GeoCoord*)(P)->a)[(I)*2]) +#define GeoY(P,I) (((GeoCoord*)(P)->a)[(I)*2+1]) + + +/* +** State of a parse of a GeoJSON input. +*/ +typedef struct GeoParse GeoParse; +struct GeoParse { + const unsigned char *z; /* Unparsed input */ + int nVertex; /* Number of vertexes in a[] */ + int nAlloc; /* Space allocated to a[] */ + int nErr; /* Number of errors encountered */ + GeoCoord *a; /* Array of vertexes. From sqlite3_malloc64() */ +}; + +/* Do a 4-byte byte swap */ +static void geopolySwab32(unsigned char *a){ + unsigned char t = a[0]; + a[0] = a[3]; + a[3] = t; + t = a[1]; + a[1] = a[2]; + a[2] = t; +} + +/* Skip whitespace. Return the next non-whitespace character. */ +static char geopolySkipSpace(GeoParse *p){ + while( safe_isspace(p->z[0]) ) p->z++; + return p->z[0]; +} + +/* Parse out a number. Write the value into *pVal if pVal!=0. +** return non-zero on success and zero if the next token is not a number. +*/ +static int geopolyParseNumber(GeoParse *p, GeoCoord *pVal){ + char c = geopolySkipSpace(p); + const unsigned char *z = p->z; + int j = 0; + int seenDP = 0; + int seenE = 0; + if( c=='-' ){ + j = 1; + c = z[j]; + } + if( c=='0' && z[j+1]>='0' && z[j+1]<='9' ) return 0; + for(;; j++){ + c = z[j]; + if( safe_isdigit(c) ) continue; + if( c=='.' ){ + if( z[j-1]=='-' ) return 0; + if( seenDP ) return 0; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ) return 0; + if( seenE ) return -1; + seenDP = seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ) return 0; + continue; + } + break; + } + if( z[j-1]<'0' ) return 0; + if( pVal ){ +#ifdef SQLITE_AMALGAMATION + /* The sqlite3AtoF() routine is much much faster than atof(), if it + ** is available */ + double r; + (void)sqlite3AtoF((const char*)p->z, &r, j, SQLITE_UTF8); + *pVal = r; +#else + *pVal = (GeoCoord)atof((const char*)p->z); +#endif + } + p->z += j; + return 1; +} + +/* +** If the input is a well-formed JSON array of coordinates with at least +** four coordinates and where each coordinate is itself a two-value array, +** then convert the JSON into a GeoPoly object and return a pointer to +** that object. +** +** If any error occurs, return NULL. +*/ +static GeoPoly *geopolyParseJson(const unsigned char *z, int *pRc){ + GeoParse s; + int rc = SQLITE_OK; + memset(&s, 0, sizeof(s)); + s.z = z; + if( geopolySkipSpace(&s)=='[' ){ + s.z++; + while( geopolySkipSpace(&s)=='[' ){ + int ii = 0; + char c; + s.z++; + if( s.nVertex>=s.nAlloc ){ + GeoCoord *aNew; + s.nAlloc = s.nAlloc*2 + 16; + aNew = sqlite3_realloc64(s.a, s.nAlloc*sizeof(GeoCoord)*2 ); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + s.nErr++; + break; + } + s.a = aNew; + } + while( geopolyParseNumber(&s, ii<=1 ? &s.a[s.nVertex*2+ii] : 0) ){ + ii++; + if( ii==2 ) s.nVertex++; + c = geopolySkipSpace(&s); + s.z++; + if( c==',' ) continue; + if( c==']' && ii>=2 ) break; + s.nErr++; + rc = SQLITE_ERROR; + goto parse_json_err; + } + if( geopolySkipSpace(&s)==',' ){ + s.z++; + continue; + } + break; + } + if( geopolySkipSpace(&s)==']' + && s.nVertex>=4 + && s.a[0]==s.a[s.nVertex*2-2] + && s.a[1]==s.a[s.nVertex*2-1] + && (s.z++, geopolySkipSpace(&s)==0) + ){ + GeoPoly *pOut; + int x = 1; + s.nVertex--; /* Remove the redundant vertex at the end */ + pOut = sqlite3_malloc64( GEOPOLY_SZ((sqlite3_int64)s.nVertex) ); + x = 1; + if( pOut==0 ) goto parse_json_err; + pOut->nVertex = s.nVertex; + memcpy(pOut->a, s.a, s.nVertex*2*sizeof(GeoCoord)); + pOut->hdr[0] = *(unsigned char*)&x; + pOut->hdr[1] = (s.nVertex>>16)&0xff; + pOut->hdr[2] = (s.nVertex>>8)&0xff; + pOut->hdr[3] = s.nVertex&0xff; + sqlite3_free(s.a); + if( pRc ) *pRc = SQLITE_OK; + return pOut; + }else{ + s.nErr++; + rc = SQLITE_ERROR; + } + } +parse_json_err: + if( pRc ) *pRc = rc; + sqlite3_free(s.a); + return 0; +} + +/* +** Given a function parameter, try to interpret it as a polygon, either +** in the binary format or JSON text. Compute a GeoPoly object and +** return a pointer to that object. Or if the input is not a well-formed +** polygon, put an error message in sqlite3_context and return NULL. +*/ +static GeoPoly *geopolyFuncParam( + sqlite3_context *pCtx, /* Context for error messages */ + sqlite3_value *pVal, /* The value to decode */ + int *pRc /* Write error here */ +){ + GeoPoly *p = 0; + int nByte; + if( sqlite3_value_type(pVal)==SQLITE_BLOB + && (nByte = sqlite3_value_bytes(pVal))>=(4+6*sizeof(GeoCoord)) + ){ + const unsigned char *a = sqlite3_value_blob(pVal); + int nVertex; + if( a==0 ){ + sqlite3_result_error_nomem(pCtx); + return 0; + } + nVertex = (a[1]<<16) + (a[2]<<8) + a[3]; + if( (a[0]==0 || a[0]==1) + && (nVertex*2*sizeof(GeoCoord) + 4)==(unsigned int)nByte + ){ + p = sqlite3_malloc64( sizeof(*p) + (nVertex-1)*2*sizeof(GeoCoord) ); + if( p==0 ){ + if( pRc ) *pRc = SQLITE_NOMEM; + if( pCtx ) sqlite3_result_error_nomem(pCtx); + }else{ + int x = 1; + p->nVertex = nVertex; + memcpy(p->hdr, a, nByte); + if( a[0] != *(unsigned char*)&x ){ + int ii; + for(ii=0; iihdr[0] ^= 1; + } + } + } + if( pRc ) *pRc = SQLITE_OK; + return p; + }else if( sqlite3_value_type(pVal)==SQLITE_TEXT ){ + const unsigned char *zJson = sqlite3_value_text(pVal); + if( zJson==0 ){ + if( pRc ) *pRc = SQLITE_NOMEM; + return 0; + } + return geopolyParseJson(zJson, pRc); + }else{ + if( pRc ) *pRc = SQLITE_ERROR; + return 0; + } +} + +/* +** Implementation of the geopoly_blob(X) function. +** +** If the input is a well-formed Geopoly BLOB or JSON string +** then return the BLOB representation of the polygon. Otherwise +** return NULL. +*/ +static void geopolyBlobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** SQL function: geopoly_json(X) +** +** Interpret X as a polygon and render it as a JSON array +** of coordinates. Or, if X is not a valid polygon, return NULL. +*/ +static void geopolyJsonFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_str *x = sqlite3_str_new(db); + int i; + sqlite3_str_append(x, "[", 1); + for(i=0; inVertex; i++){ + sqlite3_str_appendf(x, "[%!g,%!g],", GeoX(p,i), GeoY(p,i)); + } + sqlite3_str_appendf(x, "[%!g,%!g]]", GeoX(p,0), GeoY(p,0)); + sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_free(p); + } +} + +/* +** SQL function: geopoly_svg(X, ....) +** +** Interpret X as a polygon and render it as a SVG . +** Additional arguments are added as attributes to the . +*/ +static void geopolySvgFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p; + if( argc<1 ) return; + p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_str *x = sqlite3_str_new(db); + int i; + char cSep = '\''; + sqlite3_str_appendf(x, ""); + sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_free(p); + } +} + +/* +** SQL Function: geopoly_xform(poly, A, B, C, D, E, F) +** +** Transform and/or translate a polygon as follows: +** +** x1 = A*x0 + B*y0 + E +** y1 = C*x0 + D*y0 + F +** +** For a translation: +** +** geopoly_xform(poly, 1, 0, 0, 1, x-offset, y-offset) +** +** Rotate by R around the point (0,0): +** +** geopoly_xform(poly, cos(R), sin(R), -sin(R), cos(R), 0, 0) +*/ +static void geopolyXformFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + double A = sqlite3_value_double(argv[1]); + double B = sqlite3_value_double(argv[2]); + double C = sqlite3_value_double(argv[3]); + double D = sqlite3_value_double(argv[4]); + double E = sqlite3_value_double(argv[5]); + double F = sqlite3_value_double(argv[6]); + GeoCoord x1, y1, x0, y0; + int ii; + if( p ){ + for(ii=0; iinVertex; ii++){ + x0 = GeoX(p,ii); + y0 = GeoY(p,ii); + x1 = (GeoCoord)(A*x0 + B*y0 + E); + y1 = (GeoCoord)(C*x0 + D*y0 + F); + GeoX(p,ii) = x1; + GeoY(p,ii) = y1; + } + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** Compute the area enclosed by the polygon. +** +** This routine can also be used to detect polygons that rotate in +** the wrong direction. Polygons are suppose to be counter-clockwise (CCW). +** This routine returns a negative value for clockwise (CW) polygons. +*/ +static double geopolyArea(GeoPoly *p){ + double rArea = 0.0; + int ii; + for(ii=0; iinVertex-1; ii++){ + rArea += (GeoX(p,ii) - GeoX(p,ii+1)) /* (x0 - x1) */ + * (GeoY(p,ii) + GeoY(p,ii+1)) /* (y0 + y1) */ + * 0.5; + } + rArea += (GeoX(p,ii) - GeoX(p,0)) /* (xN - x0) */ + * (GeoY(p,ii) + GeoY(p,0)) /* (yN + y0) */ + * 0.5; + return rArea; +} + +/* +** Implementation of the geopoly_area(X) function. +** +** If the input is a well-formed Geopoly BLOB then return the area +** enclosed by the polygon. If the polygon circulates clockwise instead +** of counterclockwise (as it should) then return the negative of the +** enclosed area. Otherwise return NULL. +*/ +static void geopolyAreaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3_result_double(context, geopolyArea(p)); + sqlite3_free(p); + } +} + +/* +** Implementation of the geopoly_ccw(X) function. +** +** If the rotation of polygon X is clockwise (incorrect) instead of +** counter-clockwise (the correct winding order according to RFC7946) +** then reverse the order of the vertexes in polygon X. +** +** In other words, this routine returns a CCW polygon regardless of the +** winding order of its input. +** +** Use this routine to sanitize historical inputs that that sometimes +** contain polygons that wind in the wrong direction. +*/ +static void geopolyCcwFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + if( geopolyArea(p)<0.0 ){ + int ii, jj; + for(ii=1, jj=p->nVertex-1; iihdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +#define GEOPOLY_PI 3.1415926535897932385 + +/* Fast approximation for sine(X) for X between -0.5*pi and 2*pi +*/ +static double geopolySine(double r){ + assert( r>=-0.5*GEOPOLY_PI && r<=2.0*GEOPOLY_PI ); + if( r>=1.5*GEOPOLY_PI ){ + r -= 2.0*GEOPOLY_PI; + } + if( r>=0.5*GEOPOLY_PI ){ + return -geopolySine(r-GEOPOLY_PI); + }else{ + double r2 = r*r; + double r3 = r2*r; + double r5 = r3*r2; + return 0.9996949*r - 0.1656700*r3 + 0.0075134*r5; + } +} + +/* +** Function: geopoly_regular(X,Y,R,N) +** +** Construct a simple, convex, regular polygon centered at X, Y +** with circumradius R and with N sides. +*/ +static void geopolyRegularFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double x = sqlite3_value_double(argv[0]); + double y = sqlite3_value_double(argv[1]); + double r = sqlite3_value_double(argv[2]); + int n = sqlite3_value_int(argv[3]); + int i; + GeoPoly *p; + + if( n<3 || r<=0.0 ) return; + if( n>1000 ) n = 1000; + p = sqlite3_malloc64( sizeof(*p) + (n-1)*2*sizeof(GeoCoord) ); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + i = 1; + p->hdr[0] = *(unsigned char*)&i; + p->hdr[1] = 0; + p->hdr[2] = (n>>8)&0xff; + p->hdr[3] = n&0xff; + for(i=0; ihdr, 4+8*n, SQLITE_TRANSIENT); + sqlite3_free(p); +} + +/* +** If pPoly is a polygon, compute its bounding box. Then: +** +** (1) if aCoord!=0 store the bounding box in aCoord, returning NULL +** (2) otherwise, compute a GeoPoly for the bounding box and return the +** new GeoPoly +** +** If pPoly is NULL but aCoord is not NULL, then compute a new GeoPoly from +** the bounding box in aCoord and return a pointer to that GeoPoly. +*/ +static GeoPoly *geopolyBBox( + sqlite3_context *context, /* For recording the error */ + sqlite3_value *pPoly, /* The polygon */ + RtreeCoord *aCoord, /* Results here */ + int *pRc /* Error code here */ +){ + GeoPoly *pOut = 0; + GeoPoly *p; + float mnX, mxX, mnY, mxY; + if( pPoly==0 && aCoord!=0 ){ + p = 0; + mnX = aCoord[0].f; + mxX = aCoord[1].f; + mnY = aCoord[2].f; + mxY = aCoord[3].f; + goto geopolyBboxFill; + }else{ + p = geopolyFuncParam(context, pPoly, pRc); + } + if( p ){ + int ii; + mnX = mxX = GeoX(p,0); + mnY = mxY = GeoY(p,0); + for(ii=1; iinVertex; ii++){ + double r = GeoX(p,ii); + if( rmxX ) mxX = (float)r; + r = GeoY(p,ii); + if( rmxY ) mxY = (float)r; + } + if( pRc ) *pRc = SQLITE_OK; + if( aCoord==0 ){ + geopolyBboxFill: + pOut = sqlite3_realloc64(p, GEOPOLY_SZ(4)); + if( pOut==0 ){ + sqlite3_free(p); + if( context ) sqlite3_result_error_nomem(context); + if( pRc ) *pRc = SQLITE_NOMEM; + return 0; + } + pOut->nVertex = 4; + ii = 1; + pOut->hdr[0] = *(unsigned char*)ⅈ + pOut->hdr[1] = 0; + pOut->hdr[2] = 0; + pOut->hdr[3] = 4; + GeoX(pOut,0) = mnX; + GeoY(pOut,0) = mnY; + GeoX(pOut,1) = mxX; + GeoY(pOut,1) = mnY; + GeoX(pOut,2) = mxX; + GeoY(pOut,2) = mxY; + GeoX(pOut,3) = mnX; + GeoY(pOut,3) = mxY; + }else{ + sqlite3_free(p); + aCoord[0].f = mnX; + aCoord[1].f = mxX; + aCoord[2].f = mnY; + aCoord[3].f = mxY; + } + }else if( aCoord ){ + memset(aCoord, 0, sizeof(RtreeCoord)*4); + } + return pOut; +} + +/* +** Implementation of the geopoly_bbox(X) SQL function. +*/ +static void geopolyBBoxFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyBBox(context, argv[0], 0, 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** State vector for the geopoly_group_bbox() aggregate function. +*/ +typedef struct GeoBBox GeoBBox; +struct GeoBBox { + int isInit; + RtreeCoord a[4]; +}; + + +/* +** Implementation of the geopoly_group_bbox(X) aggregate SQL function. +*/ +static void geopolyBBoxStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + RtreeCoord a[4]; + int rc = SQLITE_OK; + (void)geopolyBBox(context, argv[0], a, &rc); + if( rc==SQLITE_OK ){ + GeoBBox *pBBox; + pBBox = (GeoBBox*)sqlite3_aggregate_context(context, sizeof(*pBBox)); + if( pBBox==0 ) return; + if( pBBox->isInit==0 ){ + pBBox->isInit = 1; + memcpy(pBBox->a, a, sizeof(RtreeCoord)*4); + }else{ + if( a[0].f < pBBox->a[0].f ) pBBox->a[0] = a[0]; + if( a[1].f > pBBox->a[1].f ) pBBox->a[1] = a[1]; + if( a[2].f < pBBox->a[2].f ) pBBox->a[2] = a[2]; + if( a[3].f > pBBox->a[3].f ) pBBox->a[3] = a[3]; + } + } +} +static void geopolyBBoxFinal( + sqlite3_context *context +){ + GeoPoly *p; + GeoBBox *pBBox; + pBBox = (GeoBBox*)sqlite3_aggregate_context(context, 0); + if( pBBox==0 ) return; + p = geopolyBBox(context, 0, pBBox->a, 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + + +/* +** Determine if point (x0,y0) is beneath line segment (x1,y1)->(x2,y2). +** Returns: +** +** +2 x0,y0 is on the line segement +** +** +1 x0,y0 is beneath line segment +** +** 0 x0,y0 is not on or beneath the line segment or the line segment +** is vertical and x0,y0 is not on the line segment +** +** The left-most coordinate min(x1,x2) is not considered to be part of +** the line segment for the purposes of this analysis. +*/ +static int pointBeneathLine( + double x0, double y0, + double x1, double y1, + double x2, double y2 +){ + double y; + if( x0==x1 && y0==y1 ) return 2; + if( x1x2 ) return 0; + }else if( x1>x2 ){ + if( x0<=x2 || x0>x1 ) return 0; + }else{ + /* Vertical line segment */ + if( x0!=x1 ) return 0; + if( y0y1 && y0>y2 ) return 0; + return 2; + } + y = y1 + (y2-y1)*(x0-x1)/(x2-x1); + if( y0==y ) return 2; + if( y0nVertex-1; ii++){ + v = pointBeneathLine(x0,y0,GeoX(p1,ii), GeoY(p1,ii), + GeoX(p1,ii+1),GeoY(p1,ii+1)); + if( v==2 ) break; + cnt += v; + } + if( v!=2 ){ + v = pointBeneathLine(x0,y0,GeoX(p1,ii), GeoY(p1,ii), + GeoX(p1,0), GeoY(p1,0)); + } + if( v==2 ){ + sqlite3_result_int(context, 1); + }else if( ((v+cnt)&1)==0 ){ + sqlite3_result_int(context, 0); + }else{ + sqlite3_result_int(context, 2); + } + sqlite3_free(p1); +} + +/* Forward declaration */ +static int geopolyOverlap(GeoPoly *p1, GeoPoly *p2); + +/* +** SQL function: geopoly_within(P1,P2) +** +** Return +2 if P1 and P2 are the same polygon +** Return +1 if P2 is contained within P1 +** Return 0 if any part of P2 is on the outside of P1 +** +*/ +static void geopolyWithinFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); + GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + if( p1 && p2 ){ + int x = geopolyOverlap(p1, p2); + if( x<0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_int(context, x==2 ? 1 : x==4 ? 2 : 0); + } + } + sqlite3_free(p1); + sqlite3_free(p2); +} + +/* Objects used by the overlap algorihm. */ +typedef struct GeoEvent GeoEvent; +typedef struct GeoSegment GeoSegment; +typedef struct GeoOverlap GeoOverlap; +struct GeoEvent { + double x; /* X coordinate at which event occurs */ + int eType; /* 0 for ADD, 1 for REMOVE */ + GeoSegment *pSeg; /* The segment to be added or removed */ + GeoEvent *pNext; /* Next event in the sorted list */ +}; +struct GeoSegment { + double C, B; /* y = C*x + B */ + double y; /* Current y value */ + float y0; /* Initial y value */ + unsigned char side; /* 1 for p1, 2 for p2 */ + unsigned int idx; /* Which segment within the side */ + GeoSegment *pNext; /* Next segment in a list sorted by y */ +}; +struct GeoOverlap { + GeoEvent *aEvent; /* Array of all events */ + GeoSegment *aSegment; /* Array of all segments */ + int nEvent; /* Number of events */ + int nSegment; /* Number of segments */ +}; + +/* +** Add a single segment and its associated events. +*/ +static void geopolyAddOneSegment( + GeoOverlap *p, + GeoCoord x0, + GeoCoord y0, + GeoCoord x1, + GeoCoord y1, + unsigned char side, + unsigned int idx +){ + GeoSegment *pSeg; + GeoEvent *pEvent; + if( x0==x1 ) return; /* Ignore vertical segments */ + if( x0>x1 ){ + GeoCoord t = x0; + x0 = x1; + x1 = t; + t = y0; + y0 = y1; + y1 = t; + } + pSeg = p->aSegment + p->nSegment; + p->nSegment++; + pSeg->C = (y1-y0)/(x1-x0); + pSeg->B = y1 - x1*pSeg->C; + pSeg->y0 = y0; + pSeg->side = side; + pSeg->idx = idx; + pEvent = p->aEvent + p->nEvent; + p->nEvent++; + pEvent->x = x0; + pEvent->eType = 0; + pEvent->pSeg = pSeg; + pEvent = p->aEvent + p->nEvent; + p->nEvent++; + pEvent->x = x1; + pEvent->eType = 1; + pEvent->pSeg = pSeg; +} + + + +/* +** Insert all segments and events for polygon pPoly. +*/ +static void geopolyAddSegments( + GeoOverlap *p, /* Add segments to this Overlap object */ + GeoPoly *pPoly, /* Take all segments from this polygon */ + unsigned char side /* The side of pPoly */ +){ + unsigned int i; + GeoCoord *x; + for(i=0; i<(unsigned)pPoly->nVertex-1; i++){ + x = &GeoX(pPoly,i); + geopolyAddOneSegment(p, x[0], x[1], x[2], x[3], side, i); + } + x = &GeoX(pPoly,i); + geopolyAddOneSegment(p, x[0], x[1], pPoly->a[0], pPoly->a[1], side, i); +} + +/* +** Merge two lists of sorted events by X coordinate +*/ +static GeoEvent *geopolyEventMerge(GeoEvent *pLeft, GeoEvent *pRight){ + GeoEvent head, *pLast; + head.pNext = 0; + pLast = &head; + while( pRight && pLeft ){ + if( pRight->x <= pLeft->x ){ + pLast->pNext = pRight; + pLast = pRight; + pRight = pRight->pNext; + }else{ + pLast->pNext = pLeft; + pLast = pLeft; + pLeft = pLeft->pNext; + } + } + pLast->pNext = pRight ? pRight : pLeft; + return head.pNext; +} + +/* +** Sort an array of nEvent event objects into a list. +*/ +static GeoEvent *geopolySortEventsByX(GeoEvent *aEvent, int nEvent){ + int mx = 0; + int i, j; + GeoEvent *p; + GeoEvent *a[50]; + for(i=0; ipNext = 0; + for(j=0; j=mx ) mx = j+1; + } + p = 0; + for(i=0; iy - pLeft->y; + if( r==0.0 ) r = pRight->C - pLeft->C; + if( r<0.0 ){ + pLast->pNext = pRight; + pLast = pRight; + pRight = pRight->pNext; + }else{ + pLast->pNext = pLeft; + pLast = pLeft; + pLeft = pLeft->pNext; + } + } + pLast->pNext = pRight ? pRight : pLeft; + return head.pNext; +} + +/* +** Sort a list of GeoSegments in order of increasing Y and in the event of +** a tie, increasing C (slope). +*/ +static GeoSegment *geopolySortSegmentsByYAndC(GeoSegment *pList){ + int mx = 0; + int i; + GeoSegment *p; + GeoSegment *a[50]; + while( pList ){ + p = pList; + pList = pList->pNext; + p->pNext = 0; + for(i=0; i=mx ) mx = i+1; + } + p = 0; + for(i=0; inVertex + p2->nVertex + 2; + GeoOverlap *p; + sqlite3_int64 nByte; + GeoEvent *pThisEvent; + double rX; + int rc = 0; + int needSort = 0; + GeoSegment *pActive = 0; + GeoSegment *pSeg; + unsigned char aOverlap[4]; + + nByte = sizeof(GeoEvent)*nVertex*2 + + sizeof(GeoSegment)*nVertex + + sizeof(GeoOverlap); + p = sqlite3_malloc64( nByte ); + if( p==0 ) return -1; + p->aEvent = (GeoEvent*)&p[1]; + p->aSegment = (GeoSegment*)&p->aEvent[nVertex*2]; + p->nEvent = p->nSegment = 0; + geopolyAddSegments(p, p1, 1); + geopolyAddSegments(p, p2, 2); + pThisEvent = geopolySortEventsByX(p->aEvent, p->nEvent); + rX = pThisEvent && pThisEvent->x==0.0 ? -1.0 : 0.0; + memset(aOverlap, 0, sizeof(aOverlap)); + while( pThisEvent ){ + if( pThisEvent->x!=rX ){ + GeoSegment *pPrev = 0; + int iMask = 0; + GEODEBUG(("Distinct X: %g\n", pThisEvent->x)); + rX = pThisEvent->x; + if( needSort ){ + GEODEBUG(("SORT\n")); + pActive = geopolySortSegmentsByYAndC(pActive); + needSort = 0; + } + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + if( pPrev ){ + if( pPrev->y!=pSeg->y ){ + GEODEBUG(("MASK: %d\n", iMask)); + aOverlap[iMask] = 1; + } + } + iMask ^= pSeg->side; + pPrev = pSeg; + } + pPrev = 0; + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + double y = pSeg->C*rX + pSeg->B; + GEODEBUG(("Segment %d.%d %g->%g\n", pSeg->side, pSeg->idx, pSeg->y, y)); + pSeg->y = y; + if( pPrev ){ + if( pPrev->y>pSeg->y && pPrev->side!=pSeg->side ){ + rc = 1; + GEODEBUG(("Crossing: %d.%d and %d.%d\n", + pPrev->side, pPrev->idx, + pSeg->side, pSeg->idx)); + goto geopolyOverlapDone; + }else if( pPrev->y!=pSeg->y ){ + GEODEBUG(("MASK: %d\n", iMask)); + aOverlap[iMask] = 1; + } + } + iMask ^= pSeg->side; + pPrev = pSeg; + } + } + GEODEBUG(("%s %d.%d C=%g B=%g\n", + pThisEvent->eType ? "RM " : "ADD", + pThisEvent->pSeg->side, pThisEvent->pSeg->idx, + pThisEvent->pSeg->C, + pThisEvent->pSeg->B)); + if( pThisEvent->eType==0 ){ + /* Add a segment */ + pSeg = pThisEvent->pSeg; + pSeg->y = pSeg->y0; + pSeg->pNext = pActive; + pActive = pSeg; + needSort = 1; + }else{ + /* Remove a segment */ + if( pActive==pThisEvent->pSeg ){ + pActive = pActive->pNext; + }else{ + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + if( pSeg->pNext==pThisEvent->pSeg ){ + pSeg->pNext = pSeg->pNext->pNext; + break; + } + } + } + } + pThisEvent = pThisEvent->pNext; + } + if( aOverlap[3]==0 ){ + rc = 0; + }else if( aOverlap[1]!=0 && aOverlap[2]==0 ){ + rc = 3; + }else if( aOverlap[1]==0 && aOverlap[2]!=0 ){ + rc = 2; + }else if( aOverlap[1]==0 && aOverlap[2]==0 ){ + rc = 4; + }else{ + rc = 1; + } + +geopolyOverlapDone: + sqlite3_free(p); + return rc; +} + +/* +** SQL function: geopoly_overlap(P1,P2) +** +** Determine whether or not P1 and P2 overlap. Return value: +** +** 0 The two polygons are disjoint +** 1 They overlap +** 2 P1 is completely contained within P2 +** 3 P2 is completely contained within P1 +** 4 P1 and P2 are the same polygon +** NULL Either P1 or P2 or both are not valid polygons +*/ +static void geopolyOverlapFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); + GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + if( p1 && p2 ){ + int x = geopolyOverlap(p1, p2); + if( x<0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_int(context, x); + } + } + sqlite3_free(p1); + sqlite3_free(p2); +} + +/* +** Enable or disable debugging output +*/ +static void geopolyDebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ +#ifdef GEOPOLY_ENABLE_DEBUG + geo_debug = sqlite3_value_int(argv[0]); +#endif +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the geopoly virtual table. +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... +*/ +static int geopolyInit( + sqlite3 *db, /* Database connection */ + void *pAux, /* One of the RTREE_COORD_* constants */ + int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ + sqlite3_vtab **ppVtab, /* OUT: New virtual table */ + char **pzErr, /* OUT: Error message, if any */ + int isCreate /* True for xCreate, false for xConnect */ +){ + int rc = SQLITE_OK; + Rtree *pRtree; + sqlite3_int64 nDb; /* Length of string argv[1] */ + sqlite3_int64 nName; /* Length of string argv[2] */ + sqlite3_str *pSql; + char *zSql; + int ii; + + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Allocate the sqlite3_vtab structure */ + nDb = strlen(argv[1]); + nName = strlen(argv[2]); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); + if( !pRtree ){ + return SQLITE_NOMEM; + } + memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + pRtree->nBusy = 1; + pRtree->base.pModule = &rtreeModule; + pRtree->zDb = (char *)&pRtree[1]; + pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->eCoordType = RTREE_COORD_REAL32; + pRtree->nDim = 2; + pRtree->nDim2 = 4; + memcpy(pRtree->zDb, argv[1], nDb); + memcpy(pRtree->zName, argv[2], nName); + + + /* Create/Connect to the underlying relational database schema. If + ** that is successful, call sqlite3_declare_vtab() to configure + ** the r-tree table schema. + */ + pSql = sqlite3_str_new(db); + sqlite3_str_appendf(pSql, "CREATE TABLE x(_shape"); + pRtree->nAux = 1; /* Add one for _shape */ + pRtree->nAuxNotNull = 1; /* The _shape column is always not-null */ + for(ii=3; iinAux++; + sqlite3_str_appendf(pSql, ",%s", argv[ii]); + } + sqlite3_str_appendf(pSql, ");"); + zSql = sqlite3_str_finish(pSql); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else if( SQLITE_OK!=(rc = sqlite3_declare_vtab(db, zSql)) ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + sqlite3_free(zSql); + if( rc ) goto geopolyInit_fail; + pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; + + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate, pzErr); + if( rc ) goto geopolyInit_fail; + rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate); + if( rc ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + goto geopolyInit_fail; + } + + *ppVtab = (sqlite3_vtab *)pRtree; + return SQLITE_OK; + +geopolyInit_fail: + if( rc==SQLITE_OK ) rc = SQLITE_ERROR; + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); + rtreeRelease(pRtree); + return rc; +} + + +/* +** GEOPOLY virtual table module xCreate method. +*/ +static int geopolyCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 1); +} + +/* +** GEOPOLY virtual table module xConnect method. +*/ +static int geopolyConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 0); +} + + +/* +** GEOPOLY virtual table module xFilter method. +** +** Query plans: +** +** 1 rowid lookup +** 2 search for objects overlapping the same bounding box +** that contains polygon argv[0] +** 3 search for objects overlapping the same bounding box +** that contains polygon argv[0] +** 4 full table scan +*/ +static int geopolyFilter( + sqlite3_vtab_cursor *pVtabCursor, /* The cursor to initialize */ + int idxNum, /* Query plan */ + const char *idxStr, /* Not Used */ + int argc, sqlite3_value **argv /* Parameters to the query plan */ +){ + Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeNode *pRoot = 0; + int rc = SQLITE_OK; + int iCell = 0; + + rtreeReference(pRtree); + + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ + resetCursor(pCsr); + + pCsr->iStrategy = idxNum; + if( idxNum==1 ){ + /* Special case - lookup by rowid. */ + RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the leaf */ + i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = (u8)iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; + } + }else{ + /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array + ** with the configured constraints. + */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && idxNum<=3 ){ + RtreeCoord bbox[4]; + RtreeConstraint *p; + assert( argc==1 ); + geopolyBBox(0, argv[0], bbox, &rc); + if( rc ){ + goto geopoly_filter_end; + } + pCsr->aConstraint = p = sqlite3_malloc(sizeof(RtreeConstraint)*4); + pCsr->nConstraint = 4; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*4); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); + if( idxNum==2 ){ + /* Overlap query */ + p->op = 'B'; + p->iCoord = 0; + p->u.rValue = bbox[1].f; + p++; + p->op = 'D'; + p->iCoord = 1; + p->u.rValue = bbox[0].f; + p++; + p->op = 'B'; + p->iCoord = 2; + p->u.rValue = bbox[3].f; + p++; + p->op = 'D'; + p->iCoord = 3; + p->u.rValue = bbox[2].f; + }else{ + /* Within query */ + p->op = 'D'; + p->iCoord = 0; + p->u.rValue = bbox[0].f; + p++; + p->op = 'B'; + p->iCoord = 1; + p->u.rValue = bbox[1].f; + p++; + p->op = 'D'; + p->iCoord = 2; + p->u.rValue = bbox[2].f; + p++; + p->op = 'B'; + p->iCoord = 3; + p->u.rValue = bbox[3].f; + } + } + } + if( rc==SQLITE_OK ){ + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + goto geopoly_filter_end; + } + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); + } + } + +geopoly_filter_end: + nodeRelease(pRtree, pRoot); + rtreeRelease(pRtree); + return rc; +} + +/* +** Rtree virtual table module xBestIndex method. There are three +** table scan strategies to choose from (in order from most to +** least desirable): +** +** idxNum idxStr Strategy +** ------------------------------------------------ +** 1 "rowid" Direct lookup by rowid. +** 2 "rtree" R-tree overlap query using geopoly_overlap() +** 3 "rtree" R-tree within query using geopoly_within() +** 4 "fullscan" full-table scan. +** ------------------------------------------------ +*/ +static int geopolyBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int ii; + int iRowidTerm = -1; + int iFuncTerm = -1; + int idxNum = 0; + + for(ii=0; iinConstraint; ii++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; + if( !p->usable ) continue; + if( p->iColumn<0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + iRowidTerm = ii; + break; + } + if( p->iColumn==0 && p->op>=SQLITE_INDEX_CONSTRAINT_FUNCTION ){ + /* p->op==SQLITE_INDEX_CONSTRAINT_FUNCTION for geopoly_overlap() + ** p->op==(SQLITE_INDEX_CONTRAINT_FUNCTION+1) for geopoly_within(). + ** See geopolyFindFunction() */ + iFuncTerm = ii; + idxNum = p->op - SQLITE_INDEX_CONSTRAINT_FUNCTION + 2; + } + } + + if( iRowidTerm>=0 ){ + pIdxInfo->idxNum = 1; + pIdxInfo->idxStr = "rowid"; + pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; + pIdxInfo->estimatedCost = 30.0; + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + return SQLITE_OK; + } + if( iFuncTerm>=0 ){ + pIdxInfo->idxNum = idxNum; + pIdxInfo->idxStr = "rtree"; + pIdxInfo->aConstraintUsage[iFuncTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iFuncTerm].omit = 0; + pIdxInfo->estimatedCost = 300.0; + pIdxInfo->estimatedRows = 10; + return SQLITE_OK; + } + pIdxInfo->idxNum = 4; + pIdxInfo->idxStr = "fullscan"; + pIdxInfo->estimatedCost = 3000000.0; + pIdxInfo->estimatedRows = 100000; + return SQLITE_OK; +} + + +/* +** GEOPOLY virtual table module xColumn method. +*/ +static int geopolyColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + Rtree *pRtree = (Rtree *)cur->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; + if( i==0 && sqlite3_vtab_nochange(ctx) ) return SQLITE_OK; + if( i<=pRtree->nAux ){ + if( !pCsr->bAuxValid ){ + if( pCsr->pReadAux==0 ){ + rc = sqlite3_prepare_v3(pRtree->db, pRtree->zReadAuxSql, -1, 0, + &pCsr->pReadAux, 0); + if( rc ) return rc; + } + sqlite3_bind_int64(pCsr->pReadAux, 1, + nodeGetRowid(pRtree, pNode, p->iCell)); + rc = sqlite3_step(pCsr->pReadAux); + if( rc==SQLITE_ROW ){ + pCsr->bAuxValid = 1; + }else{ + sqlite3_reset(pCsr->pReadAux); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; + } + } + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pReadAux, i+2)); + } + return SQLITE_OK; +} + + +/* +** The xUpdate method for GEOPOLY module virtual tables. +** +** For DELETE: +** +** argv[0] = the rowid to be deleted +** +** For INSERT: +** +** argv[0] = SQL NULL +** argv[1] = rowid to insert, or an SQL NULL to select automatically +** argv[2] = _shape column +** argv[3] = first application-defined column.... +** +** For UPDATE: +** +** argv[0] = rowid to modify. Never NULL +** argv[1] = rowid after the change. Never NULL +** argv[2] = new value for _shape +** argv[3] = new value for first application-defined column.... +*/ +static int geopolyUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **aData, + sqlite_int64 *pRowid +){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + i64 oldRowid; /* The old rowid */ + int oldRowidValid; /* True if oldRowid is valid */ + i64 newRowid; /* The new rowid */ + int newRowidValid; /* True if newRowid is valid */ + int coordChange = 0; /* Change in coordinates */ + + if( pRtree->nNodeRef ){ + /* Unable to write to the btree while another cursor is reading from it, + ** since the write might do a rebalance which would disrupt the read + ** cursor. */ + return SQLITE_LOCKED_VTAB; + } + rtreeReference(pRtree); + assert(nData>=1); + + oldRowidValid = sqlite3_value_type(aData[0])!=SQLITE_NULL;; + oldRowid = oldRowidValid ? sqlite3_value_int64(aData[0]) : 0; + newRowidValid = nData>1 && sqlite3_value_type(aData[1])!=SQLITE_NULL; + newRowid = newRowidValid ? sqlite3_value_int64(aData[1]) : 0; + cell.iRowid = newRowid; + + if( nData>1 /* not a DELETE */ + && (!oldRowidValid /* INSERT */ + || !sqlite3_value_nochange(aData[2]) /* UPDATE _shape */ + || oldRowid!=newRowid) /* Rowid change */ + ){ + geopolyBBox(0, aData[2], cell.aCoord, &rc); + if( rc ){ + if( rc==SQLITE_ERROR ){ + pVtab->zErrMsg = + sqlite3_mprintf("_shape does not contain a valid polygon"); + } + goto geopoly_update_end; + } + coordChange = 1; + + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( newRowidValid && (!oldRowidValid || oldRowid!=newRowid) ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = rtreeConstraintError(pRtree, 0); + } + } + } + } + + /* If aData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( rc==SQLITE_OK && (nData==1 || (coordChange && oldRowidValid)) ){ + rc = rtreeDeleteRowid(pRtree, oldRowid); + } + + /* If the aData[] array contains more than one element, elements + ** (aData[2]..aData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 && coordChange ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf = 0; + if( !newRowidValid ){ + rc = rtreeNewRowid(pRtree, &cell.iRowid); + } + *pRowid = cell.iRowid; + if( rc==SQLITE_OK ){ + rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); + } + if( rc==SQLITE_OK ){ + int rc2; + pRtree->iReinsertHeight = -1; + rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + /* Change the data */ + if( rc==SQLITE_OK && nData>1 ){ + sqlite3_stmt *pUp = pRtree->pWriteAux; + int jj; + int nChange = 0; + sqlite3_bind_int64(pUp, 1, cell.iRowid); + assert( pRtree->nAux>=1 ); + if( sqlite3_value_nochange(aData[2]) ){ + sqlite3_bind_null(pUp, 2); + }else{ + GeoPoly *p = 0; + if( sqlite3_value_type(aData[2])==SQLITE_TEXT + && (p = geopolyFuncParam(0, aData[2], &rc))!=0 + && rc==SQLITE_OK + ){ + sqlite3_bind_blob(pUp, 2, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT); + }else{ + sqlite3_bind_value(pUp, 2, aData[2]); + } + sqlite3_free(p); + nChange = 1; + } + for(jj=1; jjnAux; jj++){ + nChange++; + sqlite3_bind_value(pUp, jj+2, aData[jj+2]); + } + if( nChange ){ + sqlite3_step(pUp); + rc = sqlite3_reset(pUp); + } + } + +geopoly_update_end: + rtreeRelease(pRtree); + return rc; +} + +/* +** Report that geopoly_overlap() is an overloaded function suitable +** for use in xBestIndex. +*/ +static int geopolyFindFunction( + sqlite3_vtab *pVtab, + int nArg, + const char *zName, + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), + void **ppArg +){ + if( sqlite3_stricmp(zName, "geopoly_overlap")==0 ){ + *pxFunc = geopolyOverlapFunc; + *ppArg = 0; + return SQLITE_INDEX_CONSTRAINT_FUNCTION; + } + if( sqlite3_stricmp(zName, "geopoly_within")==0 ){ + *pxFunc = geopolyWithinFunc; + *ppArg = 0; + return SQLITE_INDEX_CONSTRAINT_FUNCTION+1; + } + return 0; +} + + +static sqlite3_module geopolyModule = { + 3, /* iVersion */ + geopolyCreate, /* xCreate - create a table */ + geopolyConnect, /* xConnect - connect to an existing table */ + geopolyBestIndex, /* xBestIndex - Determine search strategy */ + rtreeDisconnect, /* xDisconnect - Disconnect from a table */ + rtreeDestroy, /* xDestroy - Drop a table */ + rtreeOpen, /* xOpen - open a cursor */ + rtreeClose, /* xClose - close a cursor */ + geopolyFilter, /* xFilter - configure scan constraints */ + rtreeNext, /* xNext - advance a cursor */ + rtreeEof, /* xEof */ + geopolyColumn, /* xColumn - read data */ + rtreeRowid, /* xRowid - read data */ + geopolyUpdate, /* xUpdate - write data */ + rtreeBeginTransaction, /* xBegin - begin transaction */ + rtreeEndTransaction, /* xSync - sync transaction */ + rtreeEndTransaction, /* xCommit - commit transaction */ + rtreeEndTransaction, /* xRollback - rollback transaction */ + geopolyFindFunction, /* xFindFunction - function overloading */ + rtreeRename, /* xRename - rename the table */ + rtreeSavepoint, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + rtreeShadowName /* xShadowName */ +}; + +static int sqlite3_geopoly_init(sqlite3 *db){ + int rc = SQLITE_OK; + static const struct { + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + signed char nArg; + unsigned char bPure; + const char *zName; + } aFunc[] = { + { geopolyAreaFunc, 1, 1, "geopoly_area" }, + { geopolyBlobFunc, 1, 1, "geopoly_blob" }, + { geopolyJsonFunc, 1, 1, "geopoly_json" }, + { geopolySvgFunc, -1, 1, "geopoly_svg" }, + { geopolyWithinFunc, 2, 1, "geopoly_within" }, + { geopolyContainsPointFunc, 3, 1, "geopoly_contains_point" }, + { geopolyOverlapFunc, 2, 1, "geopoly_overlap" }, + { geopolyDebugFunc, 1, 0, "geopoly_debug" }, + { geopolyBBoxFunc, 1, 1, "geopoly_bbox" }, + { geopolyXformFunc, 7, 1, "geopoly_xform" }, + { geopolyRegularFunc, 4, 1, "geopoly_regular" }, + { geopolyCcwFunc, 1, 1, "geopoly_ccw" }, + }; + static const struct { + void (*xStep)(sqlite3_context*,int,sqlite3_value**); + void (*xFinal)(sqlite3_context*); + const char *zName; + } aAgg[] = { + { geopolyBBoxStep, geopolyBBoxFinal, "geopoly_group_bbox" }, + }; + int i; + for(i=0; ixDestructor ) pInfo->xDestructor(pInfo->pContext); + sqlite3_free(p); +} + +/* +** This routine frees the BLOB that is returned by geomCallback(). +*/ +static void rtreeMatchArgFree(void *pArg){ + int i; + RtreeMatchArg *p = (RtreeMatchArg*)pArg; + for(i=0; inParam; i++){ + sqlite3_value_free(p->apSqlParam[i]); + } + sqlite3_free(p); +} + +/* +** Each call to sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback() creates an ordinary SQLite +** scalar function that is implemented by this routine. +** +** All this function does is construct an RtreeMatchArg object that +** contains the geometry-checking callback routines and a list of +** parameters to this function, then return that RtreeMatchArg object +** as a BLOB. +** +** The R-Tree MATCH operator will read the returned BLOB, deserialize +** the RtreeMatchArg object, and use the RtreeMatchArg object to figure +** out which elements of the R-Tree should be returned by the query. +*/ +static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ + RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); + RtreeMatchArg *pBlob; + sqlite3_int64 nBlob; + int memErr = 0; + + nBlob = sizeof(RtreeMatchArg) + (nArg-1)*sizeof(RtreeDValue) + + nArg*sizeof(sqlite3_value*); + pBlob = (RtreeMatchArg *)sqlite3_malloc64(nBlob); + if( !pBlob ){ + sqlite3_result_error_nomem(ctx); + }else{ + int i; + pBlob->iSize = nBlob; + pBlob->cb = pGeomCtx[0]; + pBlob->apSqlParam = (sqlite3_value**)&pBlob->aParam[nArg]; + pBlob->nParam = nArg; + for(i=0; iapSqlParam[i] = sqlite3_value_dup(aArg[i]); + if( pBlob->apSqlParam[i]==0 ) memErr = 1; +#ifdef SQLITE_RTREE_INT_ONLY + pBlob->aParam[i] = sqlite3_value_int64(aArg[i]); +#else + pBlob->aParam[i] = sqlite3_value_double(aArg[i]); +#endif + } + if( memErr ){ + sqlite3_result_error_nomem(ctx); + rtreeMatchArgFree(pBlob); + }else{ + sqlite3_result_pointer(ctx, pBlob, "RtreeMatchArg", rtreeMatchArgFree); + } + } +} + +/* +** Register a new geometry function for use with the r-tree MATCH operator. +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zGeom, /* Name of the new SQL function */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ + void *pContext /* Extra data associated with the callback */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = xGeom; + pGeomCtx->xQueryFunc = 0; + pGeomCtx->xDestructor = 0; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); +} + +/* +** Register a new 2nd-generation geometry function for use with the +** r-tree MATCH operator. +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zQueryFunc, /* Name of new SQL function */ + int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ + void *pContext, /* Extra data passed into the callback */ + void (*xDestructor)(void*) /* Destructor for the extra data */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ){ + if( xDestructor ) xDestructor(pContext); + return SQLITE_NOMEM; + } + pGeomCtx->xGeom = 0; + pGeomCtx->xQueryFunc = xQueryFunc; + pGeomCtx->xDestructor = xDestructor; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); +} + +#if !SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_rtree_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3RtreeInit(db); +} +#endif + +#endif + +/************** End of rtree.c ***********************************************/ +/************** Begin file icu.c *********************************************/ +/* +** 2007 May 6 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ +** +** This file implements an integration between the ICU library +** ("International Components for Unicode", an open-source library +** for handling unicode data) and SQLite. The integration uses +** ICU to provide the following to SQLite: +** +** * An implementation of the SQL regexp() function (and hence REGEXP +** operator) using the ICU uregex_XX() APIs. +** +** * Implementations of the SQL scalar upper() and lower() functions +** for case mapping. +** +** * Integration of ICU and SQLite collation sequences. +** +** * An implementation of the LIKE operator that uses ICU to +** provide case-independent matching. +*/ + +#if !defined(SQLITE_CORE) \ + || defined(SQLITE_ENABLE_ICU) \ + || defined(SQLITE_ENABLE_ICU_COLLATIONS) + +/* Include ICU headers */ +#include +#include +#include +#include + +/* #include */ + +#ifndef SQLITE_CORE +/* #include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 +#else +/* #include "sqlite3.h" */ +#endif + +/* +** This function is called when an ICU function called from within +** the implementation of an SQL scalar function returns an error. +** +** The scalar function context passed as the first argument is +** loaded with an error message based on the following two args. +*/ +static void icuFunctionError( + sqlite3_context *pCtx, /* SQLite scalar function context */ + const char *zName, /* Name of ICU function that failed */ + UErrorCode e /* Error code returned by ICU function */ +){ + char zBuf[128]; + sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); + zBuf[127] = '\0'; + sqlite3_result_error(pCtx, zBuf, -1); +} + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + +/* +** Maximum length (in bytes) of the pattern in a LIKE or GLOB +** operator. +*/ +#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +#endif + +/* +** Version of sqlite3_free() that is always a function, never a macro. +*/ +static void xFree(void *p){ + sqlite3_free(p); +} + +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. It is copied here from SQLite source +** code file utf8.c. +*/ +static const unsigned char icuUtf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define SQLITE_ICU_READ_UTF8(zIn, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = icuUtf8Trans1[c-0xc0]; \ + while( (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + } + +#define SQLITE_ICU_SKIP_UTF8(zIn) \ + assert( *zIn ); \ + if( *(zIn++)>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){zIn++;} \ + } + + +/* +** Compare two UTF-8 strings for equality where the first string is +** a "LIKE" expression. Return true (1) if they are the same and +** false (0) if they are different. +*/ +static int icuLikeCompare( + const uint8_t *zPattern, /* LIKE pattern */ + const uint8_t *zString, /* The UTF-8 string to compare against */ + const UChar32 uEsc /* The escape character */ +){ + static const uint32_t MATCH_ONE = (uint32_t)'_'; + static const uint32_t MATCH_ALL = (uint32_t)'%'; + + int prevEscape = 0; /* True if the previous character was uEsc */ + + while( 1 ){ + + /* Read (and consume) the next character from the input pattern. */ + uint32_t uPattern; + SQLITE_ICU_READ_UTF8(zPattern, uPattern); + if( uPattern==0 ) break; + + /* There are now 4 possibilities: + ** + ** 1. uPattern is an unescaped match-all character "%", + ** 2. uPattern is an unescaped match-one character "_", + ** 3. uPattern is an unescaped escape character, or + ** 4. uPattern is to be handled as an ordinary character + */ + if( uPattern==MATCH_ALL && !prevEscape && uPattern!=(uint32_t)uEsc ){ + /* Case 1. */ + uint8_t c; + + /* Skip any MATCH_ALL or MATCH_ONE characters that follow a + ** MATCH_ALL. For each MATCH_ONE, skip one character in the + ** test string. + */ + while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ + if( c==MATCH_ONE ){ + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); + } + zPattern++; + } + + if( *zPattern==0 ) return 1; + + while( *zString ){ + if( icuLikeCompare(zPattern, zString, uEsc) ){ + return 1; + } + SQLITE_ICU_SKIP_UTF8(zString); + } + return 0; + + }else if( uPattern==MATCH_ONE && !prevEscape && uPattern!=(uint32_t)uEsc ){ + /* Case 2. */ + if( *zString==0 ) return 0; + SQLITE_ICU_SKIP_UTF8(zString); + + }else if( uPattern==(uint32_t)uEsc && !prevEscape ){ + /* Case 3. */ + prevEscape = 1; + + }else{ + /* Case 4. */ + uint32_t uString; + SQLITE_ICU_READ_UTF8(zString, uString); + uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT); + uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT); + if( uString!=uPattern ){ + return 0; + } + prevEscape = 0; + } + } + + return *zString==0; +} + +/* +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A LIKE B +** +** is implemented as like(B, A). If there is an escape character E, +** +** A LIKE B ESCAPE E +** +** is mapped to like(B, A, E). +*/ +static void icuLikeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *zA = sqlite3_value_text(argv[0]); + const unsigned char *zB = sqlite3_value_text(argv[1]); + UChar32 uEsc = 0; + + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + if( sqlite3_value_bytes(argv[0])>SQLITE_MAX_LIKE_PATTERN_LENGTH ){ + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; + } + + + if( argc==3 ){ + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + int nE= sqlite3_value_bytes(argv[2]); + const unsigned char *zE = sqlite3_value_text(argv[2]); + int i = 0; + if( zE==0 ) return; + U8_NEXT(zE, i, nE, uEsc); + if( i!=nE){ + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } + } + + if( zA && zB ){ + sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); + } +} + +/* +** Function to delete compiled regexp objects. Registered as +** a destructor function with sqlite3_set_auxdata(). +*/ +static void icuRegexpDelete(void *p){ + URegularExpression *pExpr = (URegularExpression *)p; + uregex_close(pExpr); +} + +/* +** Implementation of SQLite REGEXP operator. This scalar function takes +** two arguments. The first is a regular expression pattern to compile +** the second is a string to match against that pattern. If either +** argument is an SQL NULL, then NULL Is returned. Otherwise, the result +** is 1 if the string matches the pattern, or 0 otherwise. +** +** SQLite maps the regexp() function to the regexp() operator such +** that the following two are equivalent: +** +** zString REGEXP zPattern +** regexp(zPattern, zString) +** +** Uses the following ICU regexp APIs: +** +** uregex_open() +** uregex_matches() +** uregex_close() +*/ +static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + UErrorCode status = U_ZERO_ERROR; + URegularExpression *pExpr; + UBool res; + const UChar *zString = sqlite3_value_text16(apArg[1]); + + (void)nArg; /* Unused parameter */ + + /* If the left hand side of the regexp operator is NULL, + ** then the result is also NULL. + */ + if( !zString ){ + return; + } + + pExpr = sqlite3_get_auxdata(p, 0); + if( !pExpr ){ + const UChar *zPattern = sqlite3_value_text16(apArg[0]); + if( !zPattern ){ + return; + } + pExpr = uregex_open(zPattern, -1, 0, 0, &status); + + if( U_SUCCESS(status) ){ + sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); + }else{ + assert(!pExpr); + icuFunctionError(p, "uregex_open", status); + return; + } + } + + /* Configure the text that the regular expression operates on. */ + uregex_setText(pExpr, zString, -1, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_setText", status); + return; + } + + /* Attempt the match */ + res = uregex_matches(pExpr, 0, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "uregex_matches", status); + return; + } + + /* Set the text that the regular expression operates on to a NULL + ** pointer. This is not really necessary, but it is tidier than + ** leaving the regular expression object configured with an invalid + ** pointer after this function returns. + */ + uregex_setText(pExpr, 0, 0, &status); + + /* Return 1 or 0. */ + sqlite3_result_int(p, res ? 1 : 0); +} + +/* +** Implementations of scalar functions for case mapping - upper() and +** lower(). Function upper() converts its input to upper-case (ABC). +** Function lower() converts to lower-case (abc). +** +** ICU provides two types of case mapping, "general" case mapping and +** "language specific". Refer to ICU documentation for the differences +** between the two. +** +** To utilise "general" case mapping, the upper() or lower() scalar +** functions are invoked with one argument: +** +** upper('ABC') -> 'abc' +** lower('abc') -> 'ABC' +** +** To access ICU "language specific" case mapping, upper() or lower() +** should be invoked with two arguments. The second argument is the name +** of the locale to use. Passing an empty string ("") or SQL NULL value +** as the second argument is the same as invoking the 1 argument version +** of upper() or lower(). +** +** lower('I', 'en_us') -> 'i' +** lower('I', 'tr_tr') -> '\u131' (small dotless i) +** +** http://www.icu-project.org/userguide/posix.html#case_mappings +*/ +static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ + const UChar *zInput; /* Pointer to input string */ + UChar *zOutput = 0; /* Pointer to output buffer */ + int nInput; /* Size of utf-16 input string in bytes */ + int nOut; /* Size of output buffer in bytes */ + int cnt; + int bToUpper; /* True for toupper(), false for tolower() */ + UErrorCode status; + const char *zLocale = 0; + + assert(nArg==1 || nArg==2); + bToUpper = (sqlite3_user_data(p)!=0); + if( nArg==2 ){ + zLocale = (const char *)sqlite3_value_text(apArg[1]); + } + + zInput = sqlite3_value_text16(apArg[0]); + if( !zInput ){ + return; + } + nOut = nInput = sqlite3_value_bytes16(apArg[0]); + if( nOut==0 ){ + sqlite3_result_text16(p, "", 0, SQLITE_STATIC); + return; + } + + for(cnt=0; cnt<2; cnt++){ + UChar *zNew = sqlite3_realloc(zOutput, nOut); + if( zNew==0 ){ + sqlite3_free(zOutput); + sqlite3_result_error_nomem(p); + return; + } + zOutput = zNew; + status = U_ZERO_ERROR; + if( bToUpper ){ + nOut = 2*u_strToUpper(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + }else{ + nOut = 2*u_strToLower(zOutput,nOut/2,zInput,nInput/2,zLocale,&status); + } + + if( U_SUCCESS(status) ){ + sqlite3_result_text16(p, zOutput, nOut, xFree); + }else if( status==U_BUFFER_OVERFLOW_ERROR ){ + assert( cnt==0 ); + continue; + }else{ + icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); + } + return; + } + assert( 0 ); /* Unreachable */ +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ + +/* +** Collation sequence destructor function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static void icuCollationDel(void *pCtx){ + UCollator *p = (UCollator *)pCtx; + ucol_close(p); +} + +/* +** Collation sequence comparison function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static int icuCollationColl( + void *pCtx, + int nLeft, + const void *zLeft, + int nRight, + const void *zRight +){ + UCollationResult res; + UCollator *p = (UCollator *)pCtx; + res = ucol_strcoll(p, (UChar *)zLeft, nLeft/2, (UChar *)zRight, nRight/2); + switch( res ){ + case UCOL_LESS: return -1; + case UCOL_GREATER: return +1; + case UCOL_EQUAL: return 0; + } + assert(!"Unexpected return value from ucol_strcoll()"); + return 0; +} + +/* +** Implementation of the scalar function icu_load_collation(). +** +** This scalar function is used to add ICU collation based collation +** types to an SQLite database connection. It is intended to be called +** as follows: +** +** SELECT icu_load_collation(, ); +** +** Where is a string containing an ICU locale identifier (i.e. +** "en_AU", "tr_TR" etc.) and is the name of the +** collation sequence to create. +*/ +static void icuLoadCollation( + sqlite3_context *p, + int nArg, + sqlite3_value **apArg +){ + sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); + UErrorCode status = U_ZERO_ERROR; + const char *zLocale; /* Locale identifier - (eg. "jp_JP") */ + const char *zName; /* SQL Collation sequence name (eg. "japanese") */ + UCollator *pUCollator; /* ICU library collation object */ + int rc; /* Return code from sqlite3_create_collation_x() */ + + assert(nArg==2); + (void)nArg; /* Unused parameter */ + zLocale = (const char *)sqlite3_value_text(apArg[0]); + zName = (const char *)sqlite3_value_text(apArg[1]); + + if( !zLocale || !zName ){ + return; + } + + pUCollator = ucol_open(zLocale, &status); + if( !U_SUCCESS(status) ){ + icuFunctionError(p, "ucol_open", status); + return; + } + assert(p); + + rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator, + icuCollationColl, icuCollationDel + ); + if( rc!=SQLITE_OK ){ + ucol_close(pUCollator); + sqlite3_result_error(p, "Error registering collation function", -1); + } +} + +/* +** Register the ICU extension functions with database db. +*/ +SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db){ +# define SQLITEICU_EXTRAFLAGS (SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS) + static const struct IcuScalar { + const char *zName; /* Function name */ + unsigned char nArg; /* Number of arguments */ + unsigned int enc; /* Optimal text encoding */ + unsigned char iContext; /* sqlite3_user_data() context */ + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } scalars[] = { + {"icu_load_collation",2,SQLITE_UTF8|SQLITE_DIRECTONLY,1, icuLoadCollation}, +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + {"regexp", 2, SQLITE_ANY|SQLITEICU_EXTRAFLAGS, 0, icuRegexpFunc}, + {"lower", 1, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF16|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"lower", 1, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"like", 2, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuLikeFunc}, + {"like", 3, SQLITE_UTF8|SQLITEICU_EXTRAFLAGS, 0, icuLikeFunc}, +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ + }; + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && i<(int)(sizeof(scalars)/sizeof(scalars[0])); i++){ + const struct IcuScalar *p = &scalars[i]; + rc = sqlite3_create_function( + db, p->zName, p->nArg, p->enc, + p->iContext ? (void*)db : (void*)0, + p->xFunc, 0, 0 + ); + } + + return rc; +} + +#if !SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_icu_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3IcuInit(db); +} +#endif + +#endif + +/************** End of icu.c *************************************************/ +/************** Begin file fts3_icu.c ****************************************/ +/* +** 2007 June 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements a tokenizer for fts3 based on the ICU library. +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +#ifdef SQLITE_ENABLE_ICU + +/* #include */ +/* #include */ +/* #include "fts3_tokenizer.h" */ + +#include +/* #include */ +/* #include */ +#include + +typedef struct IcuTokenizer IcuTokenizer; +typedef struct IcuCursor IcuCursor; + +struct IcuTokenizer { + sqlite3_tokenizer base; + char *zLocale; +}; + +struct IcuCursor { + sqlite3_tokenizer_cursor base; + + UBreakIterator *pIter; /* ICU break-iterator object */ + int nChar; /* Number of UChar elements in pInput */ + UChar *aChar; /* Copy of input using utf-16 encoding */ + int *aOffset; /* Offsets of each character in utf-8 input */ + + int nBuffer; + char *zBuffer; + + int iToken; +}; + +/* +** Create a new tokenizer instance. +*/ +static int icuCreate( + int argc, /* Number of entries in argv[] */ + const char * const *argv, /* Tokenizer creation arguments */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ +){ + IcuTokenizer *p; + int n = 0; + + if( argc>0 ){ + n = strlen(argv[0])+1; + } + p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n); + if( !p ){ + return SQLITE_NOMEM; + } + memset(p, 0, sizeof(IcuTokenizer)); + + if( n ){ + p->zLocale = (char *)&p[1]; + memcpy(p->zLocale, argv[0], n); + } + + *ppTokenizer = (sqlite3_tokenizer *)p; + + return SQLITE_OK; +} + +/* +** Destroy a tokenizer +*/ +static int icuDestroy(sqlite3_tokenizer *pTokenizer){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int icuOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, /* Input string */ + int nInput, /* Length of zInput in bytes */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +){ + IcuTokenizer *p = (IcuTokenizer *)pTokenizer; + IcuCursor *pCsr; + + const int32_t opt = U_FOLD_CASE_DEFAULT; + UErrorCode status = U_ZERO_ERROR; + int nChar; + + UChar32 c; + int iInput = 0; + int iOut = 0; + + *ppCursor = 0; + + if( zInput==0 ){ + nInput = 0; + zInput = ""; + }else if( nInput<0 ){ + nInput = strlen(zInput); + } + nChar = nInput+1; + pCsr = (IcuCursor *)sqlite3_malloc64( + sizeof(IcuCursor) + /* IcuCursor */ + ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ + (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ + ); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(IcuCursor)); + pCsr->aChar = (UChar *)&pCsr[1]; + pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; + + pCsr->aOffset[iOut] = iInput; + U8_NEXT(zInput, iInput, nInput, c); + while( c>0 ){ + int isError = 0; + c = u_foldCase(c, opt); + U16_APPEND(pCsr->aChar, iOut, nChar, c, isError); + if( isError ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->aOffset[iOut] = iInput; + + if( iInputpIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status); + if( !U_SUCCESS(status) ){ + sqlite3_free(pCsr); + return SQLITE_ERROR; + } + pCsr->nChar = iOut; + + ubrk_first(pCsr->pIter); + *ppCursor = (sqlite3_tokenizer_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a tokenization cursor previously opened by a call to icuOpen(). +*/ +static int icuClose(sqlite3_tokenizer_cursor *pCursor){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + ubrk_close(pCsr->pIter); + sqlite3_free(pCsr->zBuffer); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Extract the next token from a tokenization cursor. +*/ +static int icuNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + IcuCursor *pCsr = (IcuCursor *)pCursor; + + int iStart = 0; + int iEnd = 0; + int nByte = 0; + + while( iStart==iEnd ){ + UChar32 c; + + iStart = ubrk_current(pCsr->pIter); + iEnd = ubrk_next(pCsr->pIter); + if( iEnd==UBRK_DONE ){ + return SQLITE_DONE; + } + + while( iStartaChar, iWhite, pCsr->nChar, c); + if( u_isspace(c) ){ + iStart = iWhite; + }else{ + break; + } + } + assert(iStart<=iEnd); + } + + do { + UErrorCode status = U_ZERO_ERROR; + if( nByte ){ + char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte); + if( !zNew ){ + return SQLITE_NOMEM; + } + pCsr->zBuffer = zNew; + pCsr->nBuffer = nByte; + } + + u_strToUTF8( + pCsr->zBuffer, pCsr->nBuffer, &nByte, /* Output vars */ + &pCsr->aChar[iStart], iEnd-iStart, /* Input vars */ + &status /* Output success/failure */ + ); + } while( nByte>pCsr->nBuffer ); + + *ppToken = pCsr->zBuffer; + *pnBytes = nByte; + *piStartOffset = pCsr->aOffset[iStart]; + *piEndOffset = pCsr->aOffset[iEnd]; + *piPosition = pCsr->iToken++; + + return SQLITE_OK; +} + +/* +** The set of routines that implement the simple tokenizer +*/ +static const sqlite3_tokenizer_module icuTokenizerModule = { + 0, /* iVersion */ + icuCreate, /* xCreate */ + icuDestroy, /* xCreate */ + icuOpen, /* xOpen */ + icuClose, /* xClose */ + icuNext, /* xNext */ + 0, /* xLanguageid */ +}; + +/* +** Set *ppModule to point at the implementation of the ICU tokenizer. +*/ +SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &icuTokenizerModule; +} + +#endif /* defined(SQLITE_ENABLE_ICU) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_icu.c ********************************************/ +/************** Begin file sqlite3rbu.c **************************************/ +/* +** 2014 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** +** OVERVIEW +** +** The RBU extension requires that the RBU update be packaged as an +** SQLite database. The tables it expects to find are described in +** sqlite3rbu.h. Essentially, for each table xyz in the target database +** that the user wishes to write to, a corresponding data_xyz table is +** created in the RBU database and populated with one row for each row to +** update, insert or delete from the target table. +** +** The update proceeds in three stages: +** +** 1) The database is updated. The modified database pages are written +** to a *-oal file. A *-oal file is just like a *-wal file, except +** that it is named "-oal" instead of "-wal". +** Because regular SQLite clients do not look for file named +** "-oal", they go on using the original database in +** rollback mode while the *-oal file is being generated. +** +** During this stage RBU does not update the database by writing +** directly to the target tables. Instead it creates "imposter" +** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses +** to update each b-tree individually. All updates required by each +** b-tree are completed before moving on to the next, and all +** updates are done in sorted key order. +** +** 2) The "-oal" file is moved to the equivalent "-wal" +** location using a call to rename(2). Before doing this the RBU +** module takes an EXCLUSIVE lock on the database file, ensuring +** that there are no other active readers. +** +** Once the EXCLUSIVE lock is released, any other database readers +** detect the new *-wal file and read the database in wal mode. At +** this point they see the new version of the database - including +** the updates made as part of the RBU update. +** +** 3) The new *-wal file is checkpointed. This proceeds in the same way +** as a regular database checkpoint, except that a single frame is +** checkpointed each time sqlite3rbu_step() is called. If the RBU +** handle is closed before the entire *-wal file is checkpointed, +** the checkpoint progress is saved in the RBU database and the +** checkpoint can be resumed by another RBU client at some point in +** the future. +** +** POTENTIAL PROBLEMS +** +** The rename() call might not be portable. And RBU is not currently +** syncing the directory after renaming the file. +** +** When state is saved, any commit to the *-oal file and the commit to +** the RBU update database are not atomic. So if the power fails at the +** wrong moment they might get out of sync. As the main database will be +** committed before the RBU update database this will likely either just +** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE +** constraint violations). +** +** If some client does modify the target database mid RBU update, or some +** other error occurs, the RBU extension will keep throwing errors. It's +** not really clear how to get out of this state. The system could just +** by delete the RBU update database and *-oal file and have the device +** download the update again and start over. +** +** At present, for an UPDATE, both the new.* and old.* records are +** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all +** fields are collected. This means we're probably writing a lot more +** data to disk when saving the state of an ongoing update to the RBU +** update database than is strictly necessary. +** +*/ + +/* #include */ +/* #include */ +/* #include */ + +/* #include "sqlite3.h" */ + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) +/************** Include sqlite3rbu.h in the middle of sqlite3rbu.c ***********/ +/************** Begin file sqlite3rbu.h **************************************/ +/* +** 2014 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the public interface for the RBU extension. +*/ + +/* +** SUMMARY +** +** Writing a transaction containing a large number of operations on +** b-tree indexes that are collectively larger than the available cache +** memory can be very inefficient. +** +** The problem is that in order to update a b-tree, the leaf page (at least) +** containing the entry being inserted or deleted must be modified. If the +** working set of leaves is larger than the available cache memory, then a +** single leaf that is modified more than once as part of the transaction +** may be loaded from or written to the persistent media multiple times. +** Additionally, because the index updates are likely to be applied in +** random order, access to pages within the database is also likely to be in +** random order, which is itself quite inefficient. +** +** One way to improve the situation is to sort the operations on each index +** by index key before applying them to the b-tree. This leads to an IO +** pattern that resembles a single linear scan through the index b-tree, +** and all but guarantees each modified leaf page is loaded and stored +** exactly once. SQLite uses this trick to improve the performance of +** CREATE INDEX commands. This extension allows it to be used to improve +** the performance of large transactions on existing databases. +** +** Additionally, this extension allows the work involved in writing the +** large transaction to be broken down into sub-transactions performed +** sequentially by separate processes. This is useful if the system cannot +** guarantee that a single update process will run for long enough to apply +** the entire update, for example because the update is being applied on a +** mobile device that is frequently rebooted. Even after the writer process +** has committed one or more sub-transactions, other database clients continue +** to read from the original database snapshot. In other words, partially +** applied transactions are not visible to other clients. +** +** "RBU" stands for "Resumable Bulk Update". As in a large database update +** transmitted via a wireless network to a mobile device. A transaction +** applied using this extension is hence refered to as an "RBU update". +** +** +** LIMITATIONS +** +** An "RBU update" transaction is subject to the following limitations: +** +** * The transaction must consist of INSERT, UPDATE and DELETE operations +** only. +** +** * INSERT statements may not use any default values. +** +** * UPDATE and DELETE statements must identify their target rows by +** non-NULL PRIMARY KEY values. Rows with NULL values stored in PRIMARY +** KEY fields may not be updated or deleted. If the table being written +** has no PRIMARY KEY, affected rows must be identified by rowid. +** +** * UPDATE statements may not modify PRIMARY KEY columns. +** +** * No triggers will be fired. +** +** * No foreign key violations are detected or reported. +** +** * CHECK constraints are not enforced. +** +** * No constraint handling mode except for "OR ROLLBACK" is supported. +** +** +** PREPARATION +** +** An "RBU update" is stored as a separate SQLite database. A database +** containing an RBU update is an "RBU database". For each table in the +** target database to be updated, the RBU database should contain a table +** named "data_" containing the same set of columns as the +** target table, and one more - "rbu_control". The data_% table should +** have no PRIMARY KEY or UNIQUE constraints, but each column should have +** the same type as the corresponding column in the target database. +** The "rbu_control" column should have no type at all. For example, if +** the target database contains: +** +** CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c UNIQUE); +** +** Then the RBU database should contain: +** +** CREATE TABLE data_t1(a INTEGER, b TEXT, c, rbu_control); +** +** The order of the columns in the data_% table does not matter. +** +** Instead of a regular table, the RBU database may also contain virtual +** tables or view named using the data_ naming scheme. +** +** Instead of the plain data_ naming scheme, RBU database tables +** may also be named data_, where is any sequence +** of zero or more numeric characters (0-9). This can be significant because +** tables within the RBU database are always processed in order sorted by +** name. By judicious selection of the portion of the names +** of the RBU tables the user can therefore control the order in which they +** are processed. This can be useful, for example, to ensure that "external +** content" FTS4 tables are updated before their underlying content tables. +** +** If the target database table is a virtual table or a table that has no +** PRIMARY KEY declaration, the data_% table must also contain a column +** named "rbu_rowid". This column is mapped to the tables implicit primary +** key column - "rowid". Virtual tables for which the "rowid" column does +** not function like a primary key value cannot be updated using RBU. For +** example, if the target db contains either of the following: +** +** CREATE VIRTUAL TABLE x1 USING fts3(a, b); +** CREATE TABLE x1(a, b) +** +** then the RBU database should contain: +** +** CREATE TABLE data_x1(a, b, rbu_rowid, rbu_control); +** +** All non-hidden columns (i.e. all columns matched by "SELECT *") of the +** target table must be present in the input table. For virtual tables, +** hidden columns are optional - they are updated by RBU if present in +** the input table, or not otherwise. For example, to write to an fts4 +** table with a hidden languageid column such as: +** +** CREATE VIRTUAL TABLE ft1 USING fts4(a, b, languageid='langid'); +** +** Either of the following input table schemas may be used: +** +** CREATE TABLE data_ft1(a, b, langid, rbu_rowid, rbu_control); +** CREATE TABLE data_ft1(a, b, rbu_rowid, rbu_control); +** +** For each row to INSERT into the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain integer value 0. The +** other columns should be set to the values that make up the new record +** to insert. +** +** If the target database table has an INTEGER PRIMARY KEY, it is not +** possible to insert a NULL value into the IPK column. Attempting to +** do so results in an SQLITE_MISMATCH error. +** +** For each row to DELETE from the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain integer value 1. The +** real primary key values of the row to delete should be stored in the +** corresponding columns of the data_% table. The values stored in the +** other columns are not used. +** +** For each row to UPDATE from the target database as part of the RBU +** update, the corresponding data_% table should contain a single record +** with the "rbu_control" column set to contain a value of type text. +** The real primary key values identifying the row to update should be +** stored in the corresponding columns of the data_% table row, as should +** the new values of all columns being update. The text value in the +** "rbu_control" column must contain the same number of characters as +** there are columns in the target database table, and must consist entirely +** of 'x' and '.' characters (or in some special cases 'd' - see below). For +** each column that is being updated, the corresponding character is set to +** 'x'. For those that remain as they are, the corresponding character of the +** rbu_control value should be set to '.'. For example, given the tables +** above, the update statement: +** +** UPDATE t1 SET c = 'usa' WHERE a = 4; +** +** is represented by the data_t1 row created by: +** +** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..x'); +** +** Instead of an 'x' character, characters of the rbu_control value specified +** for UPDATEs may also be set to 'd'. In this case, instead of updating the +** target table with the value stored in the corresponding data_% column, the +** user-defined SQL function "rbu_delta()" is invoked and the result stored in +** the target table column. rbu_delta() is invoked with two arguments - the +** original value currently stored in the target table column and the +** value specified in the data_xxx table. +** +** For example, this row: +** +** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..d'); +** +** is similar to an UPDATE statement such as: +** +** UPDATE t1 SET c = rbu_delta(c, 'usa') WHERE a = 4; +** +** Finally, if an 'f' character appears in place of a 'd' or 's' in an +** ota_control string, the contents of the data_xxx table column is assumed +** to be a "fossil delta" - a patch to be applied to a blob value in the +** format used by the fossil source-code management system. In this case +** the existing value within the target database table must be of type BLOB. +** It is replaced by the result of applying the specified fossil delta to +** itself. +** +** If the target database table is a virtual table or a table with no PRIMARY +** KEY, the rbu_control value should not include a character corresponding +** to the rbu_rowid value. For example, this: +** +** INSERT INTO data_ft1(a, b, rbu_rowid, rbu_control) +** VALUES(NULL, 'usa', 12, '.x'); +** +** causes a result similar to: +** +** UPDATE ft1 SET b = 'usa' WHERE rowid = 12; +** +** The data_xxx tables themselves should have no PRIMARY KEY declarations. +** However, RBU is more efficient if reading the rows in from each data_xxx +** table in "rowid" order is roughly the same as reading them sorted by +** the PRIMARY KEY of the corresponding target database table. In other +** words, rows should be sorted using the destination table PRIMARY KEY +** fields before they are inserted into the data_xxx tables. +** +** USAGE +** +** The API declared below allows an application to apply an RBU update +** stored on disk to an existing target database. Essentially, the +** application: +** +** 1) Opens an RBU handle using the sqlite3rbu_open() function. +** +** 2) Registers any required virtual table modules with the database +** handle returned by sqlite3rbu_db(). Also, if required, register +** the rbu_delta() implementation. +** +** 3) Calls the sqlite3rbu_step() function one or more times on +** the new handle. Each call to sqlite3rbu_step() performs a single +** b-tree operation, so thousands of calls may be required to apply +** a complete update. +** +** 4) Calls sqlite3rbu_close() to close the RBU update handle. If +** sqlite3rbu_step() has been called enough times to completely +** apply the update to the target database, then the RBU database +** is marked as fully applied. Otherwise, the state of the RBU +** update application is saved in the RBU database for later +** resumption. +** +** See comments below for more detail on APIs. +** +** If an update is only partially applied to the target database by the +** time sqlite3rbu_close() is called, various state information is saved +** within the RBU database. This allows subsequent processes to automatically +** resume the RBU update from where it left off. +** +** To remove all RBU extension state information, returning an RBU database +** to its original contents, it is sufficient to drop all tables that begin +** with the prefix "rbu_" +** +** DATABASE LOCKING +** +** An RBU update may not be applied to a database in WAL mode. Attempting +** to do so is an error (SQLITE_ERROR). +** +** While an RBU handle is open, a SHARED lock may be held on the target +** database file. This means it is possible for other clients to read the +** database, but not to write it. +** +** If an RBU update is started and then suspended before it is completed, +** then an external client writes to the database, then attempting to resume +** the suspended RBU update is also an error (SQLITE_BUSY). +*/ + +#ifndef _SQLITE3RBU_H +#define _SQLITE3RBU_H + +/* #include "sqlite3.h" ** Required for error code definitions ** */ + +#if 0 +extern "C" { +#endif + +typedef struct sqlite3rbu sqlite3rbu; + +/* +** Open an RBU handle. +** +** Argument zTarget is the path to the target database. Argument zRbu is +** the path to the RBU database. Each call to this function must be matched +** by a call to sqlite3rbu_close(). When opening the databases, RBU passes +** the SQLITE_CONFIG_URI flag to sqlite3_open_v2(). So if either zTarget +** or zRbu begin with "file:", it will be interpreted as an SQLite +** database URI, not a regular file name. +** +** If the zState argument is passed a NULL value, the RBU extension stores +** the current state of the update (how many rows have been updated, which +** indexes are yet to be updated etc.) within the RBU database itself. This +** can be convenient, as it means that the RBU application does not need to +** organize removing a separate state file after the update is concluded. +** Or, if zState is non-NULL, it must be a path to a database file in which +** the RBU extension can store the state of the update. +** +** When resuming an RBU update, the zState argument must be passed the same +** value as when the RBU update was started. +** +** Once the RBU update is finished, the RBU extension does not +** automatically remove any zState database file, even if it created it. +** +** By default, RBU uses the default VFS to access the files on disk. To +** use a VFS other than the default, an SQLite "file:" URI containing a +** "vfs=..." option may be passed as the zTarget option. +** +** IMPORTANT NOTE FOR ZIPVFS USERS: The RBU extension works with all of +** SQLite's built-in VFSs, including the multiplexor VFS. However it does +** not work out of the box with zipvfs. Refer to the comment describing +** the zipvfs_create_vfs() API below for details on using RBU with zipvfs. +*/ +SQLITE_API sqlite3rbu *sqlite3rbu_open( + const char *zTarget, + const char *zRbu, + const char *zState +); + +/* +** Open an RBU handle to perform an RBU vacuum on database file zTarget. +** An RBU vacuum is similar to SQLite's built-in VACUUM command, except +** that it can be suspended and resumed like an RBU update. +** +** The second argument to this function identifies a database in which +** to store the state of the RBU vacuum operation if it is suspended. The +** first time sqlite3rbu_vacuum() is called, to start an RBU vacuum +** operation, the state database should either not exist or be empty +** (contain no tables). If an RBU vacuum is suspended by calling +** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has +** returned SQLITE_DONE, the vacuum state is stored in the state database. +** The vacuum can be resumed by calling this function to open a new RBU +** handle specifying the same target and state databases. +** +** If the second argument passed to this function is NULL, then the +** name of the state database is "-vacuum", where +** is the name of the target database file. In this case, on UNIX, if the +** state database is not already present in the file-system, it is created +** with the same permissions as the target db is made. +** +** With an RBU vacuum, it is an SQLITE_MISUSE error if the name of the +** state database ends with "-vactmp". This name is reserved for internal +** use. +** +** This function does not delete the state database after an RBU vacuum +** is completed, even if it created it. However, if the call to +** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents +** of the state tables within the state database are zeroed. This way, +** the next call to sqlite3rbu_vacuum() opens a handle that starts a +** new RBU vacuum operation. +** +** As with sqlite3rbu_open(), Zipvfs users should rever to the comment +** describing the sqlite3rbu_create_vfs() API function below for +** a description of the complications associated with using RBU with +** zipvfs databases. +*/ +SQLITE_API sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +); + +/* +** Configure a limit for the amount of temp space that may be used by +** the RBU handle passed as the first argument. The new limit is specified +** in bytes by the second parameter. If it is positive, the limit is updated. +** If the second parameter to this function is passed zero, then the limit +** is removed entirely. If the second parameter is negative, the limit is +** not modified (this is useful for querying the current limit). +** +** In all cases the returned value is the current limit in bytes (zero +** indicates unlimited). +** +** If the temp space limit is exceeded during operation, an SQLITE_FULL +** error is returned. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu*, sqlite3_int64); + +/* +** Return the current amount of temp file space, in bytes, currently used by +** the RBU handle passed as the only argument. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu*); + +/* +** Internally, each RBU connection uses a separate SQLite database +** connection to access the target and rbu update databases. This +** API allows the application direct access to these database handles. +** +** The first argument passed to this function must be a valid, open, RBU +** handle. The second argument should be passed zero to access the target +** database handle, or non-zero to access the rbu update database handle. +** Accessing the underlying database handles may be useful in the +** following scenarios: +** +** * If any target tables are virtual tables, it may be necessary to +** call sqlite3_create_module() on the target database handle to +** register the required virtual table implementations. +** +** * If the data_xxx tables in the RBU source database are virtual +** tables, the application may need to call sqlite3_create_module() on +** the rbu update db handle to any required virtual table +** implementations. +** +** * If the application uses the "rbu_delta()" feature described above, +** it must use sqlite3_create_function() or similar to register the +** rbu_delta() implementation with the target database handle. +** +** If an error has occurred, either while opening or stepping the RBU object, +** this function may return NULL. The error code and message may be collected +** when sqlite3rbu_close() is called. +** +** Database handles returned by this function remain valid until the next +** call to any sqlite3rbu_xxx() function other than sqlite3rbu_db(). +*/ +SQLITE_API sqlite3 *sqlite3rbu_db(sqlite3rbu*, int bRbu); + +/* +** Do some work towards applying the RBU update to the target db. +** +** Return SQLITE_DONE if the update has been completely applied, or +** SQLITE_OK if no error occurs but there remains work to do to apply +** the RBU update. If an error does occur, some other error code is +** returned. +** +** Once a call to sqlite3rbu_step() has returned a value other than +** SQLITE_OK, all subsequent calls on the same RBU handle are no-ops +** that immediately return the same value. +*/ +SQLITE_API int sqlite3rbu_step(sqlite3rbu *pRbu); + +/* +** Force RBU to save its state to disk. +** +** If a power failure or application crash occurs during an update, following +** system recovery RBU may resume the update from the point at which the state +** was last saved. In other words, from the most recent successful call to +** sqlite3rbu_close() or this function. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *pRbu); + +/* +** Close an RBU handle. +** +** If the RBU update has been completely applied, mark the RBU database +** as fully applied. Otherwise, assuming no error has occurred, save the +** current state of the RBU update appliation to the RBU database. +** +** If an error has already occurred as part of an sqlite3rbu_step() +** or sqlite3rbu_open() call, or if one occurs within this function, an +** SQLite error code is returned. Additionally, if pzErrmsg is not NULL, +** *pzErrmsg may be set to point to a buffer containing a utf-8 formatted +** English language error message. It is the responsibility of the caller to +** eventually free any such buffer using sqlite3_free(). +** +** Otherwise, if no error occurs, this function returns SQLITE_OK if the +** update has been partially applied, or SQLITE_DONE if it has been +** completely applied. +*/ +SQLITE_API int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); + +/* +** Return the total number of key-value operations (inserts, deletes or +** updates) that have been performed on the target database since the +** current RBU update was started. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); + +/* +** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) +** progress indications for the two stages of an RBU update. This API may +** be useful for driving GUI progress indicators and similar. +** +** An RBU update is divided into two stages: +** +** * Stage 1, in which changes are accumulated in an oal/wal file, and +** * Stage 2, in which the contents of the wal file are copied into the +** main database. +** +** The update is visible to non-RBU clients during stage 2. During stage 1 +** non-RBU reader clients may see the original database. +** +** If this API is called during stage 2 of the update, output variable +** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) +** to a value between 0 and 10000 to indicate the permyriadage progress of +** stage 2. A value of 5000 indicates that stage 2 is half finished, +** 9000 indicates that it is 90% finished, and so on. +** +** If this API is called during stage 1 of the update, output variable +** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The +** value to which (*pnOne) is set depends on whether or not the RBU +** database contains an "rbu_count" table. The rbu_count table, if it +** exists, must contain the same columns as the following: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There must be one row in the table for each source (data_xxx) table within +** the RBU database. The 'tbl' column should contain the name of the source +** table. The 'cnt' column should contain the number of rows within the +** source table. +** +** If the rbu_count table is present and populated correctly and this +** API is called during stage 1, the *pnOne output variable is set to the +** permyriadage progress of the same stage. If the rbu_count table does +** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count +** table exists but is not correctly populated, the value of the *pnOne +** output variable during stage 1 is undefined. +*/ +SQLITE_API void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int*pnTwo); + +/* +** Obtain an indication as to the current stage of an RBU update or vacuum. +** This function always returns one of the SQLITE_RBU_STATE_XXX constants +** defined in this file. Return values should be interpreted as follows: +** +** SQLITE_RBU_STATE_OAL: +** RBU is currently building a *-oal file. The next call to sqlite3rbu_step() +** may either add further data to the *-oal file, or compute data that will +** be added by a subsequent call. +** +** SQLITE_RBU_STATE_MOVE: +** RBU has finished building the *-oal file. The next call to sqlite3rbu_step() +** will move the *-oal file to the equivalent *-wal path. If the current +** operation is an RBU update, then the updated version of the database +** file will become visible to ordinary SQLite clients following the next +** call to sqlite3rbu_step(). +** +** SQLITE_RBU_STATE_CHECKPOINT: +** RBU is currently performing an incremental checkpoint. The next call to +** sqlite3rbu_step() will copy a page of data from the *-wal file into +** the target database file. +** +** SQLITE_RBU_STATE_DONE: +** The RBU operation has finished. Any subsequent calls to sqlite3rbu_step() +** will immediately return SQLITE_DONE. +** +** SQLITE_RBU_STATE_ERROR: +** An error has occurred. Any subsequent calls to sqlite3rbu_step() will +** immediately return the SQLite error code associated with the error. +*/ +#define SQLITE_RBU_STATE_OAL 1 +#define SQLITE_RBU_STATE_MOVE 2 +#define SQLITE_RBU_STATE_CHECKPOINT 3 +#define SQLITE_RBU_STATE_DONE 4 +#define SQLITE_RBU_STATE_ERROR 5 + +SQLITE_API int sqlite3rbu_state(sqlite3rbu *pRbu); + +/* +** Create an RBU VFS named zName that accesses the underlying file-system +** via existing VFS zParent. Or, if the zParent parameter is passed NULL, +** then the new RBU VFS uses the default system VFS to access the file-system. +** The new object is registered as a non-default VFS with SQLite before +** returning. +** +** Part of the RBU implementation uses a custom VFS object. Usually, this +** object is created and deleted automatically by RBU. +** +** The exception is for applications that also use zipvfs. In this case, +** the custom VFS must be explicitly created by the user before the RBU +** handle is opened. The RBU VFS should be installed so that the zipvfs +** VFS uses the RBU VFS, which in turn uses any other VFS layers in use +** (for example multiplexor) to access the file-system. For example, +** to assemble an RBU enabled VFS stack that uses both zipvfs and +** multiplexor (error checking omitted): +** +** // Create a VFS named "multiplex" (not the default). +** sqlite3_multiplex_initialize(0, 0); +** +** // Create an rbu VFS named "rbu" that uses multiplexor. If the +** // second argument were replaced with NULL, the "rbu" VFS would +** // access the file-system via the system default VFS, bypassing the +** // multiplexor. +** sqlite3rbu_create_vfs("rbu", "multiplex"); +** +** // Create a zipvfs VFS named "zipvfs" that uses rbu. +** zipvfs_create_vfs_v3("zipvfs", "rbu", 0, xCompressorAlgorithmDetector); +** +** // Make zipvfs the default VFS. +** sqlite3_vfs_register(sqlite3_vfs_find("zipvfs"), 1); +** +** Because the default VFS created above includes a RBU functionality, it +** may be used by RBU clients. Attempting to use RBU with a zipvfs VFS stack +** that does not include the RBU layer results in an error. +** +** The overhead of adding the "rbu" VFS to the system is negligible for +** non-RBU users. There is no harm in an application accessing the +** file-system via "rbu" all the time, even if it only uses RBU functionality +** occasionally. +*/ +SQLITE_API int sqlite3rbu_create_vfs(const char *zName, const char *zParent); + +/* +** Deregister and destroy an RBU vfs created by an earlier call to +** sqlite3rbu_create_vfs(). +** +** VFS objects are not reference counted. If a VFS object is destroyed +** before all database handles that use it have been closed, the results +** are undefined. +*/ +SQLITE_API void sqlite3rbu_destroy_vfs(const char *zName); + +#if 0 +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _SQLITE3RBU_H */ + +/************** End of sqlite3rbu.h ******************************************/ +/************** Continuing where we left off in sqlite3rbu.c *****************/ + +#if defined(_WIN32_WCE) +/* #include "windows.h" */ +#endif + +/* Maximum number of prepared UPDATE statements held by this module */ +#define SQLITE_RBU_UPDATE_CACHESIZE 16 + +/* Delta checksums disabled by default. Compile with -DRBU_ENABLE_DELTA_CKSUM +** to enable checksum verification. +*/ +#ifndef RBU_ENABLE_DELTA_CKSUM +# define RBU_ENABLE_DELTA_CKSUM 0 +#endif + +/* +** Swap two objects of type TYPE. +*/ +#if !defined(SQLITE_AMALGAMATION) +# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} +#endif + +/* +** The rbu_state table is used to save the state of a partially applied +** update so that it can be resumed later. The table consists of integer +** keys mapped to values as follows: +** +** RBU_STATE_STAGE: +** May be set to integer values 1, 2, 4 or 5. As follows: +** 1: the *-rbu file is currently under construction. +** 2: the *-rbu file has been constructed, but not yet moved +** to the *-wal path. +** 4: the checkpoint is underway. +** 5: the rbu update has been checkpointed. +** +** RBU_STATE_TBL: +** Only valid if STAGE==1. The target database name of the table +** currently being written. +** +** RBU_STATE_IDX: +** Only valid if STAGE==1. The target database name of the index +** currently being written, or NULL if the main table is currently being +** updated. +** +** RBU_STATE_ROW: +** Only valid if STAGE==1. Number of rows already processed for the current +** table/index. +** +** RBU_STATE_PROGRESS: +** Trbul number of sqlite3rbu_step() calls made so far as part of this +** rbu update. +** +** RBU_STATE_CKPT: +** Valid if STAGE==4. The 64-bit checksum associated with the wal-index +** header created by recovering the *-wal file. This is used to detect +** cases when another client appends frames to the *-wal file in the +** middle of an incremental checkpoint (an incremental checkpoint cannot +** be continued if this happens). +** +** RBU_STATE_COOKIE: +** Valid if STAGE==1. The current change-counter cookie value in the +** target db file. +** +** RBU_STATE_OALSZ: +** Valid if STAGE==1. The size in bytes of the *-oal file. +** +** RBU_STATE_DATATBL: +** Only valid if STAGE==1. The RBU database name of the table +** currently being read. +*/ +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 +#define RBU_STATE_PHASEONESTEP 9 +#define RBU_STATE_DATATBL 10 + +#define RBU_STAGE_OAL 1 +#define RBU_STAGE_MOVE 2 +#define RBU_STAGE_CAPTURE 3 +#define RBU_STAGE_CKPT 4 +#define RBU_STAGE_DONE 5 + + +#define RBU_CREATE_STATE \ + "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)" + +typedef struct RbuFrame RbuFrame; +typedef struct RbuObjIter RbuObjIter; +typedef struct RbuState RbuState; +typedef struct RbuSpan RbuSpan; +typedef struct rbu_vfs rbu_vfs; +typedef struct rbu_file rbu_file; +typedef struct RbuUpdateStmt RbuUpdateStmt; + +#if !defined(SQLITE_AMALGAMATION) +typedef unsigned int u32; +typedef unsigned short u16; +typedef unsigned char u8; +typedef sqlite3_int64 i64; +#endif + +/* +** These values must match the values defined in wal.c for the equivalent +** locks. These are not magic numbers as they are part of the SQLite file +** format. +*/ +#define WAL_LOCK_WRITE 0 +#define WAL_LOCK_CKPT 1 +#define WAL_LOCK_READ0 3 + +#define SQLITE_FCNTL_RBUCNT 5149216 + +/* +** A structure to store values read from the rbu_state table in memory. +*/ +struct RbuState { + int eStage; + char *zTbl; + char *zDataTbl; + char *zIdx; + i64 iWalCksum; + int nRow; + i64 nProgress; + u32 iCookie; + i64 iOalSz; + i64 nPhaseOneStep; +}; + +struct RbuUpdateStmt { + char *zMask; /* Copy of update mask used with pUpdate */ + sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ + RbuUpdateStmt *pNext; +}; + +struct RbuSpan { + const char *zSpan; + int nSpan; +}; + +/* +** An iterator of this type is used to iterate through all objects in +** the target database that require updating. For each such table, the +** iterator visits, in order: +** +** * the table itself, +** * each index of the table (zero or more points to visit), and +** * a special "cleanup table" state. +** +** abIndexed: +** If the table has no indexes on it, abIndexed is set to NULL. Otherwise, +** it points to an array of flags nTblCol elements in size. The flag is +** set for each column that is either a part of the PK or a part of an +** index. Or clear otherwise. +** +** If there are one or more partial indexes on the table, all fields of +** this array set set to 1. This is because in that case, the module has +** no way to tell which fields will be required to add and remove entries +** from the partial indexes. +** +*/ +struct RbuObjIter { + sqlite3_stmt *pTblIter; /* Iterate through tables */ + sqlite3_stmt *pIdxIter; /* Index iterator */ + int nTblCol; /* Size of azTblCol[] array */ + char **azTblCol; /* Array of unquoted target column names */ + char **azTblType; /* Array of target column types */ + int *aiSrcOrder; /* src table col -> target table col */ + u8 *abTblPk; /* Array of flags, set on target PK columns */ + u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ + u8 *abIndexed; /* Array of flags, set on indexed & PK cols */ + int eType; /* Table type - an RBU_PK_XXX value */ + + /* Output variables. zTbl==0 implies EOF. */ + int bCleanup; /* True in "cleanup" state */ + const char *zTbl; /* Name of target db table */ + const char *zDataTbl; /* Name of rbu db table (or null) */ + const char *zIdx; /* Name of target db index (or null) */ + int iTnum; /* Root page of current object */ + int iPkTnum; /* If eType==EXTERNAL, root of PK index */ + int bUnique; /* Current index is unique */ + int nIndex; /* Number of aux. indexes on table zTbl */ + + /* Statements created by rbuObjIterPrepareAll() */ + int nCol; /* Number of columns in current object */ + sqlite3_stmt *pSelect; /* Source data */ + sqlite3_stmt *pInsert; /* Statement for INSERT operations */ + sqlite3_stmt *pDelete; /* Statement for DELETE ops */ + sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ + int nIdxCol; + RbuSpan *aIdxCol; + char *zIdxSql; + + /* Last UPDATE used (for PK b-tree updates only), or NULL. */ + RbuUpdateStmt *pRbuUpdate; +}; + +/* +** Values for RbuObjIter.eType +** +** 0: Table does not exist (error) +** 1: Table has an implicit rowid. +** 2: Table has an explicit IPK column. +** 3: Table has an external PK index. +** 4: Table is WITHOUT ROWID. +** 5: Table is a virtual table. +*/ +#define RBU_PK_NOTABLE 0 +#define RBU_PK_NONE 1 +#define RBU_PK_IPK 2 +#define RBU_PK_EXTERNAL 3 +#define RBU_PK_WITHOUT_ROWID 4 +#define RBU_PK_VTAB 5 + + +/* +** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs +** one of the following operations. +*/ +#define RBU_INSERT 1 /* Insert on a main table b-tree */ +#define RBU_DELETE 2 /* Delete a row from a main table b-tree */ +#define RBU_REPLACE 3 /* Delete and then insert a row */ +#define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */ +#define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */ + +#define RBU_UPDATE 6 /* Update a row in a main table b-tree */ + +/* +** A single step of an incremental checkpoint - frame iWalFrame of the wal +** file should be copied to page iDbPage of the database file. +*/ +struct RbuFrame { + u32 iDbPage; + u32 iWalFrame; +}; + +/* +** RBU handle. +** +** nPhaseOneStep: +** If the RBU database contains an rbu_count table, this value is set to +** a running estimate of the number of b-tree operations required to +** finish populating the *-oal file. This allows the sqlite3_bp_progress() +** API to calculate the permyriadage progress of populating the *-oal file +** using the formula: +** +** permyriadage = (10000 * nProgress) / nPhaseOneStep +** +** nPhaseOneStep is initialized to the sum of: +** +** nRow * (nIndex + 1) +** +** for all source tables in the RBU database, where nRow is the number +** of rows in the source table and nIndex the number of indexes on the +** corresponding target database table. +** +** This estimate is accurate if the RBU update consists entirely of +** INSERT operations. However, it is inaccurate if: +** +** * the RBU update contains any UPDATE operations. If the PK specified +** for an UPDATE operation does not exist in the target table, then +** no b-tree operations are required on index b-trees. Or if the +** specified PK does exist, then (nIndex*2) such operations are +** required (one delete and one insert on each index b-tree). +** +** * the RBU update contains any DELETE operations for which the specified +** PK does not exist. In this case no operations are required on index +** b-trees. +** +** * the RBU update contains REPLACE operations. These are similar to +** UPDATE operations. +** +** nPhaseOneStep is updated to account for the conditions above during the +** first pass of each source table. The updated nPhaseOneStep value is +** stored in the rbu_state table if the RBU update is suspended. +*/ +struct sqlite3rbu { + int eStage; /* Value of RBU_STATE_STAGE field */ + sqlite3 *dbMain; /* target database handle */ + sqlite3 *dbRbu; /* rbu database handle */ + char *zTarget; /* Path to target db */ + char *zRbu; /* Path to rbu db */ + char *zState; /* Path to state db (or NULL if zRbu) */ + char zStateDb[5]; /* Db name for state ("stat" or "main") */ + int rc; /* Value returned by last rbu_step() call */ + char *zErrmsg; /* Error message if rc!=SQLITE_OK */ + int nStep; /* Rows processed for current object */ + int nProgress; /* Rows processed for all objects */ + RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ + const char *zVfsName; /* Name of automatically created rbu vfs */ + rbu_file *pTargetFd; /* File handle open on target db */ + int nPagePerSector; /* Pages per sector for pTargetFd */ + i64 iOalSz; + i64 nPhaseOneStep; + + /* The following state variables are used as part of the incremental + ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding + ** function rbuSetupCheckpoint() for details. */ + u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ + u32 mLock; + int nFrame; /* Entries in aFrame[] array */ + int nFrameAlloc; /* Allocated size of aFrame[] array */ + RbuFrame *aFrame; + int pgsz; + u8 *aBuf; + i64 iWalCksum; + i64 szTemp; /* Current size of all temp files in use */ + i64 szTempLimit; /* Total size limit for temp files */ + + /* Used in RBU vacuum mode only */ + int nRbu; /* Number of RBU VFS in the stack */ + rbu_file *pRbuFd; /* Fd for main db of dbRbu */ +}; + +/* +** An rbu VFS is implemented using an instance of this structure. +** +** Variable pRbu is only non-NULL for automatically created RBU VFS objects. +** It is NULL for RBU VFS objects created explicitly using +** sqlite3rbu_create_vfs(). It is used to track the total amount of temp +** space used by the RBU handle. +*/ +struct rbu_vfs { + sqlite3_vfs base; /* rbu VFS shim methods */ + sqlite3_vfs *pRealVfs; /* Underlying VFS */ + sqlite3_mutex *mutex; /* Mutex to protect pMain */ + sqlite3rbu *pRbu; /* Owner RBU object */ + rbu_file *pMain; /* List of main db files */ + rbu_file *pMainRbu; /* List of main db files with pRbu!=0 */ +}; + +/* +** Each file opened by an rbu VFS is represented by an instance of +** the following structure. +** +** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable +** "sz" is set to the current size of the database file. +*/ +struct rbu_file { + sqlite3_file base; /* sqlite3_file methods */ + sqlite3_file *pReal; /* Underlying file handle */ + rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ + sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ + i64 sz; /* Size of file in bytes (temp only) */ + + int openFlags; /* Flags this file was opened with */ + u32 iCookie; /* Cookie value for main db files */ + u8 iWriteVer; /* "write-version" value for main db files */ + u8 bNolock; /* True to fail EXCLUSIVE locks */ + + int nShm; /* Number of entries in apShm[] array */ + char **apShm; /* Array of mmap'd *-shm regions */ + char *zDel; /* Delete this when closing file */ + + const char *zWal; /* Wal filename for this main db file */ + rbu_file *pWalFd; /* Wal file descriptor for this main db */ + rbu_file *pMainNext; /* Next MAIN_DB file */ + rbu_file *pMainRbuNext; /* Next MAIN_DB file with pRbu!=0 */ +}; + +/* +** True for an RBU vacuum handle, or false otherwise. +*/ +#define rbuIsVacuum(p) ((p)->zTarget==0) + + +/************************************************************************* +** The following three functions, found below: +** +** rbuDeltaGetInt() +** rbuDeltaChecksum() +** rbuDeltaApply() +** +** are lifted from the fossil source code (http://fossil-scm.org). They +** are used to implement the scalar SQL function rbu_fossil_delta(). +*/ + +/* +** Read bytes from *pz and convert them into a positive integer. When +** finished, leave *pz pointing to the first character past the end of +** the integer. The *pLen parameter holds the length of the string +** in *pz and is decremented once for each character in the integer. +*/ +static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ + static const signed char zValue[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, + }; + unsigned int v = 0; + int c; + unsigned char *z = (unsigned char*)*pz; + unsigned char *zStart = z; + while( (c = zValue[0x7f&*(z++)])>=0 ){ + v = (v<<6) + c; + } + z--; + *pLen -= z - zStart; + *pz = (char*)z; + return v; +} + +#if RBU_ENABLE_DELTA_CKSUM +/* +** Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; + } + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; + } + return sum3; +} +#endif + +/* +** Apply a delta. +** +** The output buffer should be big enough to hold the whole output +** file and a NUL terminator at the end. The delta_output_size() +** routine will determine this size for you. +** +** The delta string should be null-terminated. But the delta string +** may contain embedded NUL characters (if the input and output are +** binary files) so we also have to pass in the length of the delta in +** the lenDelta parameter. +** +** This function returns the size of the output file in bytes (excluding +** the final NUL terminator character). Except, if the delta string is +** malformed or intended for use with a source file other than zSrc, +** then this routine returns -1. +** +** Refer to the delta_create() documentation above for a description +** of the delta file format. +*/ +static int rbuDeltaApply( + const char *zSrc, /* The source or pattern file */ + int lenSrc, /* Length of the source file */ + const char *zDelta, /* Delta to apply to the pattern */ + int lenDelta, /* Length of the delta */ + char *zOut /* Write the output into this preallocated buffer */ +){ + unsigned int limit; + unsigned int total = 0; +#if RBU_ENABLE_DELTA_CKSUM + char *zOrigOut = zOut; +#endif + + limit = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + zDelta++; lenDelta--; + while( *zDelta && lenDelta>0 ){ + unsigned int cnt, ofst; + cnt = rbuDeltaGetInt(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = rbuDeltaGetInt(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + return -1; + } + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: copy exceeds output file size */ + return -1; + } + if( (int)(ofst+cnt) > lenSrc ){ + /* ERROR: copy extends past end of input */ + return -1; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; + } + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: insert command gives an output larger than predicted */ + return -1; + } + if( (int)cnt>lenDelta ){ + /* ERROR: insert count exceeds size of delta */ + return -1; + } + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; + } + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#if RBU_ENABLE_DELTA_CKSUM + if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ + /* ERROR: bad checksum */ + return -1; + } +#endif + if( total!=limit ){ + /* ERROR: generated size does not match predicted size */ + return -1; + } + return total; + } + default: { + /* ERROR: unknown delta operator */ + return -1; + } + } + } + /* ERROR: unterminated delta */ + return -1; +} + +static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){ + int size; + size = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + return size; +} + +/* +** End of code taken from fossil. +*************************************************************************/ + +/* +** Implementation of SQL scalar function rbu_fossil_delta(). +** +** This function applies a fossil delta patch to a blob. Exactly two +** arguments must be passed to this function. The first is the blob to +** patch and the second the patch to apply. If no error occurs, this +** function returns the patched blob. +*/ +static void rbuFossilDeltaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aDelta; + int nDelta; + const char *aOrig; + int nOrig; + + int nOut; + int nOut2; + char *aOut; + + assert( argc==2 ); + + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nDelta = sqlite3_value_bytes(argv[1]); + aDelta = (const char*)sqlite3_value_blob(argv[1]); + + /* Figure out the size of the output */ + nOut = rbuDeltaOutputSize(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; + } + + aOut = sqlite3_malloc(nOut+1); + if( aOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut); + if( nOut2!=nOut ){ + sqlite3_free(aOut); + sqlite3_result_error(context, "corrupt fossil delta", -1); + }else{ + sqlite3_result_blob(context, aOut, nOut, sqlite3_free); + } + } +} + + +/* +** Prepare the SQL statement in buffer zSql against database handle db. +** If successful, set *ppStmt to point to the new statement and return +** SQLITE_OK. +** +** Otherwise, if an error does occur, set *ppStmt to NULL and return +** an SQLite error code. Additionally, set output variable *pzErrmsg to +** point to a buffer containing an error message. It is the responsibility +** of the caller to (eventually) free this buffer using sqlite3_free(). +*/ +static int prepareAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + const char *zSql +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + *ppStmt = 0; + } + return rc; +} + +/* +** Reset the SQL statement passed as the first argument. Return a copy +** of the value returned by sqlite3_reset(). +** +** If an error has occurred, then set *pzErrmsg to point to a buffer +** containing an error message. It is the responsibility of the caller +** to eventually free this buffer using sqlite3_free(). +*/ +static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); + } + return rc; +} + +/* +** Unless it is NULL, argument zSql points to a buffer allocated using +** sqlite3_malloc containing an SQL statement. This function prepares the SQL +** statement against database db and frees the buffer. If statement +** compilation is successful, *ppStmt is set to point to the new statement +** handle and SQLITE_OK is returned. +** +** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code +** returned. In this case, *pzErrmsg may also be set to point to an error +** message. It is the responsibility of the caller to free this error message +** buffer using sqlite3_free(). +** +** If argument zSql is NULL, this function assumes that an OOM has occurred. +** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. +*/ +static int prepareFreeAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + char *zSql +){ + int rc; + assert( *pzErrmsg==0 ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + *ppStmt = 0; + }else{ + rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql); + sqlite3_free(zSql); + } + return rc; +} + +/* +** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated +** by an earlier call to rbuObjIterCacheTableInfo(). +*/ +static void rbuObjIterFreeCols(RbuObjIter *pIter){ + int i; + for(i=0; inTblCol; i++){ + sqlite3_free(pIter->azTblCol[i]); + sqlite3_free(pIter->azTblType[i]); + } + sqlite3_free(pIter->azTblCol); + pIter->azTblCol = 0; + pIter->azTblType = 0; + pIter->aiSrcOrder = 0; + pIter->abTblPk = 0; + pIter->abNotNull = 0; + pIter->nTblCol = 0; + pIter->eType = 0; /* Invalid value */ +} + +/* +** Finalize all statements and free all allocations that are specific to +** the current object (table/index pair). +*/ +static void rbuObjIterClearStatements(RbuObjIter *pIter){ + RbuUpdateStmt *pUp; + + sqlite3_finalize(pIter->pSelect); + sqlite3_finalize(pIter->pInsert); + sqlite3_finalize(pIter->pDelete); + sqlite3_finalize(pIter->pTmpInsert); + pUp = pIter->pRbuUpdate; + while( pUp ){ + RbuUpdateStmt *pTmp = pUp->pNext; + sqlite3_finalize(pUp->pUpdate); + sqlite3_free(pUp); + pUp = pTmp; + } + sqlite3_free(pIter->aIdxCol); + sqlite3_free(pIter->zIdxSql); + + pIter->pSelect = 0; + pIter->pInsert = 0; + pIter->pDelete = 0; + pIter->pRbuUpdate = 0; + pIter->pTmpInsert = 0; + pIter->nCol = 0; + pIter->nIdxCol = 0; + pIter->aIdxCol = 0; + pIter->zIdxSql = 0; +} + +/* +** Clean up any resources allocated as part of the iterator object passed +** as the only argument. +*/ +static void rbuObjIterFinalize(RbuObjIter *pIter){ + rbuObjIterClearStatements(pIter); + sqlite3_finalize(pIter->pTblIter); + sqlite3_finalize(pIter->pIdxIter); + rbuObjIterFreeCols(pIter); + memset(pIter, 0, sizeof(RbuObjIter)); +} + +/* +** Advance the iterator to the next position. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the next entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ + int rc = p->rc; + if( rc==SQLITE_OK ){ + + /* Free any SQLite statements used while processing the previous object */ + rbuObjIterClearStatements(pIter); + if( pIter->zIdx==0 ){ + rc = sqlite3_exec(p->dbMain, + "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;" + , 0, 0, &p->zErrmsg + ); + } + + if( rc==SQLITE_OK ){ + if( pIter->bCleanup ){ + rbuObjIterFreeCols(pIter); + pIter->bCleanup = 0; + rc = sqlite3_step(pIter->pTblIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); + pIter->zTbl = 0; + }else{ + pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); + pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); + rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; + } + }else{ + if( pIter->zIdx==0 ){ + sqlite3_stmt *pIdx = pIter->pIdxIter; + rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pIter->pIdxIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg); + pIter->bCleanup = 1; + pIter->zIdx = 0; + }else{ + pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); + pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1); + pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); + rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM; + } + } + } + } + } + + if( rc!=SQLITE_OK ){ + rbuObjIterFinalize(pIter); + p->rc = rc; + } + return rc; +} + + +/* +** The implementation of the rbu_target_name() SQL function. This function +** accepts one or two arguments. The first argument is the name of a table - +** the name of a table in the RBU database. The second, if it is present, is 1 +** for a view or 0 for a table. +** +** For a non-vacuum RBU handle, if the table name matches the pattern: +** +** data[0-9]_ +** +** where is any sequence of 1 or more characters, is returned. +** Otherwise, if the only argument does not match the above pattern, an SQL +** NULL is returned. +** +** "data_t1" -> "t1" +** "data0123_t2" -> "t2" +** "dataAB_t3" -> NULL +** +** For an rbu vacuum handle, a copy of the first argument is returned if +** the second argument is either missing or 0 (not a view). +*/ +static void rbuTargetNameFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + sqlite3rbu *p = sqlite3_user_data(pCtx); + const char *zIn; + assert( argc==1 || argc==2 ); + + zIn = (const char*)sqlite3_value_text(argv[0]); + if( zIn ){ + if( rbuIsVacuum(p) ){ + assert( argc==2 || argc==1 ); + if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ + sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); + } + }else{ + if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); + } + } + } + } +} + +/* +** Initialize the iterator structure passed as the second argument. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the first entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ + int rc; + memset(pIter, 0, sizeof(RbuObjIter)); + + rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, + sqlite3_mprintf( + "SELECT rbu_target_name(name, type='view') AS target, name " + "FROM sqlite_schema " + "WHERE type IN ('table', 'view') AND target IS NOT NULL " + " %s " + "ORDER BY name" + , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : "")); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, + "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " + " FROM main.sqlite_schema " + " WHERE type='index' AND tbl_name = ?" + ); + } + + pIter->bCleanup = 1; + p->rc = rc; + return rbuObjIterNext(p, pIter); +} + +/* +** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs, +** an error code is stored in the RBU handle passed as the first argument. +** +** If an error has already occurred (p->rc is already set to something other +** than SQLITE_OK), then this function returns NULL without modifying the +** stored error code. In this case it still calls sqlite3_free() on any +** printf() parameters associated with %z conversions. +*/ +static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){ + char *zSql = 0; + va_list ap; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ) p->rc = SQLITE_NOMEM; + }else{ + sqlite3_free(zSql); + zSql = 0; + } + va_end(ap); + return zSql; +} + +/* +** Argument zFmt is a sqlite3_mprintf() style format string. The trailing +** arguments are the usual subsitution values. This function performs +** the printf() style substitutions and executes the result as an SQL +** statement on the RBU handles database. +** +** If an error occurs, an error code and error message is stored in the +** RBU handle. If an error has already occurred when this function is +** called, it is a no-op. +*/ +static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){ + va_list ap; + char *zSql; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg); + } + } + sqlite3_free(zSql); + va_end(ap); + return p->rc; +} + +/* +** Attempt to allocate and return a pointer to a zeroed block of nByte +** bytes. +** +** If an error (i.e. an OOM condition) occurs, return NULL and leave an +** error code in the rbu handle passed as the first argument. Or, if an +** error has already occurred when this function is called, return NULL +** immediately without attempting the allocation or modifying the stored +** error code. +*/ +static void *rbuMalloc(sqlite3rbu *p, sqlite3_int64 nByte){ + void *pRet = 0; + if( p->rc==SQLITE_OK ){ + assert( nByte>0 ); + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + + +/* +** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that +** there is room for at least nCol elements. If an OOM occurs, store an +** error code in the RBU handle passed as the first argument. +*/ +static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ + sqlite3_int64 nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; + char **azNew; + + azNew = (char**)rbuMalloc(p, nByte); + if( azNew ){ + pIter->azTblCol = azNew; + pIter->azTblType = &azNew[nCol]; + pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; + pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol]; + pIter->abNotNull = (u8*)&pIter->abTblPk[nCol]; + pIter->abIndexed = (u8*)&pIter->abNotNull[nCol]; + } +} + +/* +** The first argument must be a nul-terminated string. This function +** returns a copy of the string in memory obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free this memory +** using sqlite3_free(). +** +** If an OOM condition is encountered when attempting to allocate memory, +** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise, +** if the allocation succeeds, (*pRc) is left unchanged. +*/ +static char *rbuStrndup(const char *zStr, int *pRc){ + char *zRet = 0; + + if( *pRc==SQLITE_OK ){ + if( zStr ){ + size_t nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc64(nCopy); + if( zRet ){ + memcpy(zRet, zStr, nCopy); + }else{ + *pRc = SQLITE_NOMEM; + } + } + } + + return zRet; +} + +/* +** Finalize the statement passed as the second argument. +** +** If the sqlite3_finalize() call indicates that an error occurs, and the +** rbu handle error code is not already set, set the error code and error +** message accordingly. +*/ +static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + p->rc = rc; + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } +} + +/* Determine the type of a table. +** +** peType is of type (int*), a pointer to an output parameter of type +** (int). This call sets the output parameter as follows, depending +** on the type of the table specified by parameters dbName and zTbl. +** +** RBU_PK_NOTABLE: No such table. +** RBU_PK_NONE: Table has an implicit rowid. +** RBU_PK_IPK: Table has an explicit IPK column. +** RBU_PK_EXTERNAL: Table has an external PK index. +** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID. +** RBU_PK_VTAB: Table is a virtual table. +** +** Argument *piPk is also of type (int*), and also points to an output +** parameter. Unless the table has an external primary key index +** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, +** if the table does have an external primary key index, then *piPk +** is set to the root page number of the primary key index before +** returning. +** +** ALGORITHM: +** +** if( no entry exists in sqlite_schema ){ +** return RBU_PK_NOTABLE +** }else if( sql for the entry starts with "CREATE VIRTUAL" ){ +** return RBU_PK_VTAB +** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){ +** if( the index that is the pk exists in sqlite_schema ){ +** *piPK = rootpage of that index. +** return RBU_PK_EXTERNAL +** }else{ +** return RBU_PK_WITHOUT_ROWID +** } +** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){ +** return RBU_PK_IPK +** }else{ +** return RBU_PK_NONE +** } +*/ +static void rbuTableType( + sqlite3rbu *p, + const char *zTab, + int *peType, + int *piTnum, + int *piPk +){ + /* + ** 0) SELECT count(*) FROM sqlite_schema where name=%Q AND IsVirtual(%Q) + ** 1) PRAGMA index_list = ? + ** 2) SELECT count(*) FROM sqlite_schema where name=%Q + ** 3) PRAGMA table_info = ? + */ + sqlite3_stmt *aStmt[4] = {0, 0, 0, 0}; + + *peType = RBU_PK_NOTABLE; + *piPk = 0; + + assert( p->rc==SQLITE_OK ); + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg, + sqlite3_mprintf( + "SELECT " + " (sql COLLATE nocase BETWEEN 'CREATE VIRTUAL' AND 'CREATE VIRTUAM')," + " rootpage" + " FROM sqlite_schema" + " WHERE name=%Q", zTab + )); + if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){ + /* Either an error, or no such table. */ + goto rbuTableType_end; + } + if( sqlite3_column_int(aStmt[0], 0) ){ + *peType = RBU_PK_VTAB; /* virtual table */ + goto rbuTableType_end; + } + *piTnum = sqlite3_column_int(aStmt[0], 1); + + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg, + sqlite3_mprintf("PRAGMA index_list=%Q",zTab) + ); + if( p->rc ) goto rbuTableType_end; + while( sqlite3_step(aStmt[1])==SQLITE_ROW ){ + const u8 *zOrig = sqlite3_column_text(aStmt[1], 3); + const u8 *zIdx = sqlite3_column_text(aStmt[1], 1); + if( zOrig && zIdx && zOrig[0]=='p' ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg, + sqlite3_mprintf( + "SELECT rootpage FROM sqlite_schema WHERE name = %Q", zIdx + )); + if( p->rc==SQLITE_OK ){ + if( sqlite3_step(aStmt[2])==SQLITE_ROW ){ + *piPk = sqlite3_column_int(aStmt[2], 0); + *peType = RBU_PK_EXTERNAL; + }else{ + *peType = RBU_PK_WITHOUT_ROWID; + } + } + goto rbuTableType_end; + } + } + + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info=%Q",zTab) + ); + if( p->rc==SQLITE_OK ){ + while( sqlite3_step(aStmt[3])==SQLITE_ROW ){ + if( sqlite3_column_int(aStmt[3],5)>0 ){ + *peType = RBU_PK_IPK; /* explicit IPK column */ + goto rbuTableType_end; + } + } + *peType = RBU_PK_NONE; + } + +rbuTableType_end: { + unsigned int i; + for(i=0; iabIndexed[] array. +*/ +static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ + sqlite3_stmt *pList = 0; + int bIndex = 0; + + if( p->rc==SQLITE_OK ){ + memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol); + p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); + } + + pIter->nIndex = 0; + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ + const char *zIdx = (const char*)sqlite3_column_text(pList, 1); + int bPartial = sqlite3_column_int(pList, 4); + sqlite3_stmt *pXInfo = 0; + if( zIdx==0 ) break; + if( bPartial ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + if( iCid>=0 ) pIter->abIndexed[iCid] = 1; + if( iCid==-2 ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } + } + rbuFinalize(p, pXInfo); + bIndex = 1; + pIter->nIndex++; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + /* "PRAGMA index_list" includes the main PK b-tree */ + pIter->nIndex--; + } + + rbuFinalize(p, pList); + if( bIndex==0 ) pIter->abIndexed = 0; +} + + +/* +** If they are not already populated, populate the pIter->azTblCol[], +** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to +** the table (not index) that the iterator currently points to. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. If +** an error does occur, an error code and error message are also left in +** the RBU handle. +*/ +static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ + if( pIter->azTblCol==0 ){ + sqlite3_stmt *pStmt = 0; + int nCol = 0; + int i; /* for() loop iterator variable */ + int bRbuRowid = 0; /* If input table has column "rbu_rowid" */ + int iOrder = 0; + int iTnum = 0; + + /* Figure out the type of table this step will deal with. */ + assert( pIter->eType==0 ); + rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum); + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl); + } + if( p->rc ) return p->rc; + if( pIter->zIdx==0 ) pIter->iTnum = iTnum; + + assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK + || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID + || pIter->eType==RBU_PK_VTAB + ); + + /* Populate the azTblCol[] and nTblCol variables based on the columns + ** of the input table. Ignore any input table columns that begin with + ** "rbu_". */ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) + ); + if( p->rc==SQLITE_OK ){ + nCol = sqlite3_column_count(pStmt); + rbuAllocateIterArrays(p, pIter, nCol); + } + for(i=0; p->rc==SQLITE_OK && irc); + pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol; + pIter->azTblCol[pIter->nTblCol++] = zCopy; + } + else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){ + bRbuRowid = 1; + } + } + sqlite3_finalize(pStmt); + pStmt = 0; + + if( p->rc==SQLITE_OK + && rbuIsVacuum(p)==0 + && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) + ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf( + "table %q %s rbu_rowid column", pIter->zDataTbl, + (bRbuRowid ? "may not have" : "requires") + ); + } + + /* Check that all non-HIDDEN columns in the destination table are also + ** present in the input table. Populate the abTblPk[], azTblType[] and + ** aiTblOrder[] arrays at the same time. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl) + ); + } + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */ + for(i=iOrder; inTblCol; i++){ + if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; + } + if( i==pIter->nTblCol ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("column missing from %q: %s", + pIter->zDataTbl, zName + ); + }else{ + int iPk = sqlite3_column_int(pStmt, 5); + int bNotNull = sqlite3_column_int(pStmt, 3); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + + if( i!=iOrder ){ + SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]); + SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]); + } + + pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc); + assert( iPk>=0 ); + pIter->abTblPk[iOrder] = (u8)iPk; + pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0); + iOrder++; + } + } + + rbuFinalize(p, pStmt); + rbuObjIterCacheIndexedCols(p, pIter); + assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); + assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 ); + } + + return p->rc; +} + +/* +** This function constructs and returns a pointer to a nul-terminated +** string containing some SQL clause or list based on one or more of the +** column names currently stored in the pIter->azTblCol[] array. +*/ +static char *rbuObjIterGetCollist( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter /* Object iterator for column names */ +){ + char *zList = 0; + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + const char *z = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z); + zSep = ", "; + } + return zList; +} + +/* +** Return a comma separated list of the quoted PRIMARY KEY column names, +** in order, for the current table. Before each column name, add the text +** zPre. After each column name, add the zPost text. Use zSeparator as +** the separator text (usually ", "). +*/ +static char *rbuObjIterGetPkList( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter, /* Object iterator for column names */ + const char *zPre, /* Before each quoted column name */ + const char *zSeparator, /* Separator to use between columns */ + const char *zPost /* After each quoted column name */ +){ + int iPk = 1; + char *zRet = 0; + const char *zSep = ""; + while( 1 ){ + int i; + for(i=0; inTblCol; i++){ + if( (int)pIter->abTblPk[i]==iPk ){ + const char *zCol = pIter->azTblCol[i]; + zRet = rbuMPrintf(p, "%z%s%s\"%w\"%s", zRet, zSep, zPre, zCol, zPost); + zSep = zSeparator; + break; + } + } + if( i==pIter->nTblCol ) break; + iPk++; + } + return zRet; +} + +/* +** This function is called as part of restarting an RBU vacuum within +** stage 1 of the process (while the *-oal file is being built) while +** updating a table (not an index). The table may be a rowid table or +** a WITHOUT ROWID table. It queries the target database to find the +** largest key that has already been written to the target table and +** constructs a WHERE clause that can be used to extract the remaining +** rows from the source table. For a rowid table, the WHERE clause +** is of the form: +** +** "WHERE _rowid_ > ?" +** +** and for WITHOUT ROWID tables: +** +** "WHERE (key1, key2) > (?, ?)" +** +** Instead of "?" placeholders, the actual WHERE clauses created by +** this function contain literal SQL values. +*/ +static char *rbuVacuumTableStart( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter, /* RBU iterator object */ + int bRowid, /* True for a rowid table */ + const char *zWrite /* Target table name prefix */ +){ + sqlite3_stmt *pMax = 0; + char *zRet = 0; + if( bRowid ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, + sqlite3_mprintf( + "SELECT max(_rowid_) FROM \"%s%w\"", zWrite, pIter->zTbl + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + sqlite3_int64 iMax = sqlite3_column_int64(pMax, 0); + zRet = rbuMPrintf(p, " WHERE _rowid_ > %lld ", iMax); + } + rbuFinalize(p, pMax); + }else{ + char *zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", " DESC"); + char *zSelect = rbuObjIterGetPkList(p, pIter, "quote(", "||','||", ")"); + char *zList = rbuObjIterGetPkList(p, pIter, "", ", ", ""); + + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, + sqlite3_mprintf( + "SELECT %s FROM \"%s%w\" ORDER BY %s LIMIT 1", + zSelect, zWrite, pIter->zTbl, zOrder + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + const char *zVal = (const char*)sqlite3_column_text(pMax, 0); + zRet = rbuMPrintf(p, " WHERE (%s) > (%s) ", zList, zVal); + } + rbuFinalize(p, pMax); + } + + sqlite3_free(zOrder); + sqlite3_free(zSelect); + sqlite3_free(zList); + } + return zRet; +} + +/* +** This function is called as part of restating an RBU vacuum when the +** current operation is writing content to an index. If possible, it +** queries the target index b-tree for the largest key already written to +** it, then composes and returns an expression that can be used in a WHERE +** clause to select the remaining required rows from the source table. +** It is only possible to return such an expression if: +** +** * The index contains no DESC columns, and +** * The last key written to the index before the operation was +** suspended does not contain any NULL values. +** +** The expression is of the form: +** +** (index-field1, index-field2, ...) > (?, ?, ...) +** +** except that the "?" placeholders are replaced with literal values. +** +** If the expression cannot be created, NULL is returned. In this case, +** the caller has to use an OFFSET clause to extract only the required +** rows from the sourct table, just as it does for an RBU update operation. +*/ +char *rbuVacuumIndexStart( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter /* RBU iterator object */ +){ + char *zOrder = 0; + char *zLhs = 0; + char *zSelect = 0; + char *zVector = 0; + char *zRet = 0; + int bFailed = 0; + const char *zSep = ""; + int iCol = 0; + sqlite3_stmt *pXInfo = 0; + + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + const char *zCol; + if( sqlite3_column_int(pXInfo, 3) ){ + bFailed = 1; + break; + } + + if( iCid<0 ){ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else{ + zCol = "_rowid_"; + } + }else{ + zCol = pIter->azTblCol[iCid]; + } + + zLhs = rbuMPrintf(p, "%z%s \"%w\" COLLATE %Q", + zLhs, zSep, zCol, zCollate + ); + zOrder = rbuMPrintf(p, "%z%s \"rbu_imp_%d%w\" COLLATE %Q DESC", + zOrder, zSep, iCol, zCol, zCollate + ); + zSelect = rbuMPrintf(p, "%z%s quote(\"rbu_imp_%d%w\")", + zSelect, zSep, iCol, zCol + ); + zSep = ", "; + iCol++; + } + rbuFinalize(p, pXInfo); + if( bFailed ) goto index_start_out; + + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pSel = 0; + + p->rc = prepareFreeAndCollectError(p->dbMain, &pSel, &p->zErrmsg, + sqlite3_mprintf("SELECT %s FROM \"rbu_imp_%w\" ORDER BY %s LIMIT 1", + zSelect, pIter->zTbl, zOrder + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSel) ){ + zSep = ""; + for(iCol=0; iColnCol; iCol++){ + const char *zQuoted = (const char*)sqlite3_column_text(pSel, iCol); + if( zQuoted==0 ){ + p->rc = SQLITE_NOMEM; + }else if( zQuoted[0]=='N' ){ + bFailed = 1; + break; + } + zVector = rbuMPrintf(p, "%z%s%s", zVector, zSep, zQuoted); + zSep = ", "; + } + + if( !bFailed ){ + zRet = rbuMPrintf(p, "(%s) > (%s)", zLhs, zVector); + } + } + rbuFinalize(p, pSel); + } + + index_start_out: + sqlite3_free(zOrder); + sqlite3_free(zSelect); + sqlite3_free(zVector); + sqlite3_free(zLhs); + return zRet; +} + +/* +** This function is used to create a SELECT list (the list of SQL +** expressions that follows a SELECT keyword) for a SELECT statement +** used to read from an data_xxx or rbu_tmp_xxx table while updating the +** index object currently indicated by the iterator object passed as the +** second argument. A "PRAGMA index_xinfo = " statement is used +** to obtain the required information. +** +** If the index is of the following form: +** +** CREATE INDEX i1 ON t1(c, b COLLATE nocase); +** +** and "t1" is a table with an explicit INTEGER PRIMARY KEY column +** "ipk", the returned string is: +** +** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'" +** +** As well as the returned string, three other malloc'd strings are +** returned via output parameters. As follows: +** +** pzImposterCols: ... +** pzImposterPk: ... +** pzWhere: ... +*/ +static char *rbuObjIterGetIndexCols( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter, /* Object iterator for column names */ + char **pzImposterCols, /* OUT: Columns for imposter table */ + char **pzImposterPk, /* OUT: Imposter PK clause */ + char **pzWhere, /* OUT: WHERE clause */ + int *pnBind /* OUT: Trbul number of columns */ +){ + int rc = p->rc; /* Error code */ + int rc2; /* sqlite3_finalize() return code */ + char *zRet = 0; /* String to return */ + char *zImpCols = 0; /* String to return via *pzImposterCols */ + char *zImpPK = 0; /* String to return via *pzImposterPK */ + char *zWhere = 0; /* String to return via *pzWhere */ + int nBind = 0; /* Value to return via *pnBind */ + const char *zCom = ""; /* Set to ", " later on */ + const char *zAnd = ""; /* Set to " AND " later on */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */ + + if( rc==SQLITE_OK ){ + assert( p->zErrmsg==0 ); + rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) + ); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + const char *zCol = 0; + const char *zType; + + if( iCid==-2 ){ + int iSeq = sqlite3_column_int(pXInfo, 0); + zRet = sqlite3_mprintf("%z%s(%.*s) COLLATE %Q", zRet, zCom, + pIter->aIdxCol[iSeq].nSpan, pIter->aIdxCol[iSeq].zSpan, zCollate + ); + zType = ""; + }else { + if( iCid<0 ){ + /* An integer primary key. If the table has an explicit IPK, use + ** its name. Otherwise, use "rbu_rowid". */ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else if( rbuIsVacuum(p) ){ + zCol = "_rowid_"; + }else{ + zCol = "rbu_rowid"; + } + zType = "INTEGER"; + }else{ + zCol = pIter->azTblCol[iCid]; + zType = pIter->azTblType[iCid]; + } + zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom,zCol,zCollate); + } + + if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ + const char *zOrder = (bDesc ? " DESC" : ""); + zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s", + zImpPK, zCom, nBind, zCol, zOrder + ); + } + zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q", + zImpCols, zCom, nBind, zCol, zType, zCollate + ); + zWhere = sqlite3_mprintf( + "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol + ); + if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM; + zCom = ", "; + zAnd = " AND "; + nBind++; + } + + rc2 = sqlite3_finalize(pXInfo); + if( rc==SQLITE_OK ) rc = rc2; + + if( rc!=SQLITE_OK ){ + sqlite3_free(zRet); + sqlite3_free(zImpCols); + sqlite3_free(zImpPK); + sqlite3_free(zWhere); + zRet = 0; + zImpCols = 0; + zImpPK = 0; + zWhere = 0; + p->rc = rc; + } + + *pzImposterCols = zImpCols; + *pzImposterPk = zImpPK; + *pzWhere = zWhere; + *pnBind = nBind; + return zRet; +} + +/* +** Assuming the current table columns are "a", "b" and "c", and the zObj +** paramter is passed "old", return a string of the form: +** +** "old.a, old.b, old.b" +** +** With the column names escaped. +** +** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append +** the text ", old._rowid_" to the returned value. +*/ +static char *rbuObjIterGetOldlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zObj +){ + char *zList = 0; + if( p->rc==SQLITE_OK && pIter->abIndexed ){ + const char *zS = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abIndexed[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol); + }else{ + zList = sqlite3_mprintf("%z%sNULL", zList, zS); + } + zS = ", "; + if( zList==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + } + + /* For a table with implicit rowids, append "old._rowid_" to the list. */ + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj); + } + } + return zList; +} + +/* +** Return an expression that can be used in a WHERE clause to match the +** primary key of the current table. For example, if the table is: +** +** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)); +** +** Return the string: +** +** "b = ?1 AND c = ?2" +*/ +static char *rbuObjIterGetWhere( + sqlite3rbu *p, + RbuObjIter *pIter +){ + char *zList = 0; + if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1); + }else if( pIter->eType==RBU_PK_EXTERNAL ){ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1); + zSep = " AND "; + } + } + zList = rbuMPrintf(p, + "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList + ); + + }else{ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1); + zSep = " AND "; + } + } + } + return zList; +} + +/* +** The SELECT statement iterating through the keys for the current object +** (p->objiter.pSelect) currently points to a valid row. However, there +** is something wrong with the rbu_control value in the rbu_control value +** stored in the (p->nCol+1)'th column. Set the error code and error message +** of the RBU handle to something reflecting this. +*/ +static void rbuBadControlError(sqlite3rbu *p){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid rbu_control value"); +} + + +/* +** Return a nul-terminated string containing the comma separated list of +** assignments that should be included following the "SET" keyword of +** an UPDATE statement used to update the table object that the iterator +** passed as the second argument currently points to if the rbu_control +** column of the data_xxx table entry is set to zMask. +** +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). +** +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. +*/ +static char *rbuObjIterGetSetlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zMask +){ + char *zList = 0; + if( p->rc==SQLITE_OK ){ + int i; + + if( (int)strlen(zMask)!=pIter->nTblCol ){ + rbuBadControlError(p); + }else{ + const char *zSep = ""; + for(i=0; inTblCol; i++){ + char c = zMask[pIter->aiSrcOrder[i]]; + if( c=='x' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", + zList, zSep, pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='d' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='f' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + } + } + } + return zList; +} + +/* +** Return a nul-terminated string consisting of nByte comma separated +** "?" expressions. For example, if nByte is 3, return a pointer to +** a buffer containing the string "?,?,?". +** +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). +** +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. +*/ +static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ + char *zRet = 0; + sqlite3_int64 nByte = 2*(sqlite3_int64)nBind + 1; + + zRet = (char*)rbuMalloc(p, nByte); + if( zRet ){ + int i; + for(i=0; izIdx==0 ); + if( p->rc==SQLITE_OK ){ + const char *zSep = "PRIMARY KEY("; + sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = */ + + p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){ + const char *zOrig = (const char*)sqlite3_column_text(pXList,3); + if( zOrig && strcmp(zOrig, "pk")==0 ){ + const char *zIdx = (const char*)sqlite3_column_text(pXList,1); + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + break; + } + } + rbuFinalize(p, pXList); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + if( sqlite3_column_int(pXInfo, 5) ){ + /* int iCid = sqlite3_column_int(pXInfo, 0); */ + const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2); + const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : ""; + z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc); + zSep = ", "; + } + } + z = rbuMPrintf(p, "%z)", z); + rbuFinalize(p, pXInfo); + } + return z; +} + +/* +** This function creates the second imposter table used when writing to +** a table b-tree where the table has an external primary key. If the +** iterator passed as the second argument does not currently point to +** a table (not index) with an external primary key, this function is a +** no-op. +** +** Assuming the iterator does point to a table with an external PK, this +** function creates a WITHOUT ROWID imposter table named "rbu_imposter2" +** used to access that PK index. For example, if the target table is +** declared as follows: +** +** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c)); +** +** then the imposter table schema is: +** +** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID; +** +*/ +static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){ + int tnum = pIter->iPkTnum; /* Root page of PK index */ + sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */ + const char *zIdx = 0; /* Name of PK index */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */ + const char *zComma = ""; + char *zCols = 0; /* Used to build up list of table cols */ + char *zPk = 0; /* Used to build up table PK declaration */ + + /* Figure out the name of the primary key index for the current table. + ** This is needed for the argument to "PRAGMA index_xinfo". Set + ** zIdx to point to a nul-terminated string containing this name. */ + p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg, + "SELECT name FROM sqlite_schema WHERE rootpage = ?" + ); + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(pQuery, 1, tnum); + if( SQLITE_ROW==sqlite3_step(pQuery) ){ + zIdx = (const char*)sqlite3_column_text(pQuery, 0); + } + } + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + rbuFinalize(p, pQuery); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int bKey = sqlite3_column_int(pXInfo, 5); + if( bKey ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %Q", zCols, zComma, + iCid, pIter->azTblType[iCid], zCollate + ); + zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":""); + zComma = ", "; + } + } + zCols = rbuMPrintf(p, "%z, id INTEGER", zCols); + rbuFinalize(p, pXInfo); + + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID", + zCols, zPk + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + } +} + +/* +** If an error has already occurred when this function is called, it +** immediately returns zero (without doing any work). Or, if an error +** occurs during the execution of this function, it sets the error code +** in the sqlite3rbu object indicated by the first argument and returns +** zero. +** +** The iterator passed as the second argument is guaranteed to point to +** a table (not an index) when this function is called. This function +** attempts to create any imposter table required to write to the main +** table b-tree of the table before returning. Non-zero is returned if +** an imposter table are created, or zero otherwise. +** +** An imposter table is required in all cases except RBU_PK_VTAB. Only +** virtual tables are written to directly. The imposter table has the +** same schema as the actual target table (less any UNIQUE constraints). +** More precisely, the "same schema" means the same columns, types, +** collation sequences. For tables that do not have an external PRIMARY +** KEY, it also means the same PRIMARY KEY declaration. +*/ +static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){ + int tnum = pIter->iTnum; + const char *zComma = ""; + char *zSql = 0; + int iCol; + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + + for(iCol=0; p->rc==SQLITE_OK && iColnTblCol; iCol++){ + const char *zPk = ""; + const char *zCol = pIter->azTblCol[iCol]; + const char *zColl = 0; + + p->rc = sqlite3_table_column_metadata( + p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0 + ); + + if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){ + /* If the target table column is an "INTEGER PRIMARY KEY", add + ** "PRIMARY KEY" to the imposter table column declaration. */ + zPk = "PRIMARY KEY "; + } + zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %Q%s", + zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl, + (pIter->abNotNull[iCol] ? " NOT NULL" : "") + ); + zComma = ", "; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + char *zPk = rbuWithoutRowidPK(p, pIter); + if( zPk ){ + zSql = rbuMPrintf(p, "%z, %z", zSql, zPk); + } + } + + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s", + pIter->zTbl, zSql, + (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + } +} + +/* +** Prepare a statement used to insert rows into the "rbu_tmp_xxx" table. +** Specifically a statement of the form: +** +** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...); +** +** The number of bound variables is equal to the number of columns in +** the target table, plus one (for the rbu_control column), plus one more +** (for the rbu_rowid column) if the target table is an implicit IPK or +** virtual table. +*/ +static void rbuObjIterPrepareTmpInsert( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zCollist, + const char *zRbuRowid +){ + int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE); + char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid); + if( zBind ){ + assert( pIter->pTmpInsert==0 ); + p->rc = prepareFreeAndCollectError( + p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( + "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", + p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind + )); + } +} + +static void rbuTmpInsertFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = sqlite3_user_data(pCtx); + int rc = SQLITE_OK; + int i; + + assert( sqlite3_value_int(apVal[0])!=0 + || p->objiter.eType==RBU_PK_EXTERNAL + || p->objiter.eType==RBU_PK_NONE + ); + if( sqlite3_value_int(apVal[0])!=0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + } + + for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); + } + if( rc==SQLITE_OK ){ + sqlite3_step(p->objiter.pTmpInsert); + rc = sqlite3_reset(p->objiter.pTmpInsert); + } + + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + } +} + +static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ + sqlite3_stmt *pStmt = 0; + int rc = p->rc; + char *zRet = 0; + + assert( pIter->zIdxSql==0 && pIter->nIdxCol==0 && pIter->aIdxCol==0 ); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + "SELECT trim(sql) FROM sqlite_schema WHERE type='index' AND name=?" + ); + } + if( rc==SQLITE_OK ){ + int rc2; + rc = sqlite3_bind_text(pStmt, 1, pIter->zIdx, -1, SQLITE_STATIC); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + char *zSql = (char*)sqlite3_column_text(pStmt, 0); + if( zSql ){ + pIter->zIdxSql = zSql = rbuStrndup(zSql, &rc); + } + if( zSql ){ + int nParen = 0; /* Number of open parenthesis */ + int i; + int iIdxCol = 0; + int nIdxAlloc = 0; + for(i=0; zSql[i]; i++){ + char c = zSql[i]; + + /* If necessary, grow the pIter->aIdxCol[] array */ + if( iIdxCol==nIdxAlloc ){ + RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc( + pIter->aIdxCol, (nIdxAlloc+16)*sizeof(RbuSpan) + ); + if( aIdxCol==0 ){ + rc = SQLITE_NOMEM; + break; + } + pIter->aIdxCol = aIdxCol; + nIdxAlloc += 16; + } + + if( c=='(' ){ + if( nParen==0 ){ + assert( iIdxCol==0 ); + pIter->aIdxCol[0].zSpan = &zSql[i+1]; + } + nParen++; + } + else if( c==')' ){ + nParen--; + if( nParen==0 ){ + int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + pIter->aIdxCol[iIdxCol++].nSpan = nSpan; + i++; + break; + } + }else if( c==',' && nParen==1 ){ + int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + pIter->aIdxCol[iIdxCol++].nSpan = nSpan; + pIter->aIdxCol[iIdxCol].zSpan = &zSql[i+1]; + }else if( c=='"' || c=='\'' || c=='`' ){ + for(i++; 1; i++){ + if( zSql[i]==c ){ + if( zSql[i+1]!=c ) break; + i++; + } + } + }else if( c=='[' ){ + for(i++; 1; i++){ + if( zSql[i]==']' ) break; + } + }else if( c=='-' && zSql[i+1]=='-' ){ + for(i=i+2; zSql[i] && zSql[i]!='\n'; i++); + if( zSql[i]=='\0' ) break; + }else if( c=='/' && zSql[i+1]=='*' ){ + for(i=i+2; zSql[i] && (zSql[i]!='*' || zSql[i+1]!='/'); i++); + if( zSql[i]=='\0' ) break; + i++; + } + } + if( zSql[i] ){ + zRet = rbuStrndup(&zSql[i], &rc); + } + pIter->nIdxCol = iIdxCol; + } + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + p->rc = rc; + return zRet; +} + +/* +** Ensure that the SQLite statement handles required to update the +** target database object currently indicated by the iterator passed +** as the second argument are available. +*/ +static int rbuObjIterPrepareAll( + sqlite3rbu *p, + RbuObjIter *pIter, + int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */ +){ + assert( pIter->bCleanup==0 ); + if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){ + const int tnum = pIter->iTnum; + char *zCollist = 0; /* List of indexed columns */ + char **pz = &p->zErrmsg; + const char *zIdx = pIter->zIdx; + char *zLimit = 0; + + if( nOffset ){ + zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset); + if( !zLimit ) p->rc = SQLITE_NOMEM; + } + + if( zIdx ){ + const char *zTbl = pIter->zTbl; + char *zImposterCols = 0; /* Columns for imposter table */ + char *zImposterPK = 0; /* Primary key declaration for imposter */ + char *zWhere = 0; /* WHERE clause on PK columns */ + char *zBind = 0; + char *zPart = 0; + int nBind = 0; + + assert( pIter->eType!=RBU_PK_VTAB ); + zPart = rbuObjIterGetIndexWhere(p, pIter); + zCollist = rbuObjIterGetIndexCols( + p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind + ); + zBind = rbuObjIterGetBindlist(p, nBind); + + /* Create the imposter table used to write to this index. */ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID", + zTbl, zImposterCols, zImposterPK + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + + /* Create the statement to insert index entries */ + pIter->nCol = nBind; + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pInsert, &p->zErrmsg, + sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind) + ); + } + + /* And to delete index entries */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pDelete, &p->zErrmsg, + sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) + ); + } + + /* Create the SELECT statement to read keys in sorted order */ + if( p->rc==SQLITE_OK ){ + char *zSql; + if( rbuIsVacuum(p) ){ + char *zStart = 0; + if( nOffset ){ + zStart = rbuVacuumIndexStart(p, pIter); + if( zStart ){ + sqlite3_free(zLimit); + zLimit = 0; + } + } + + zSql = sqlite3_mprintf( + "SELECT %s, 0 AS rbu_control FROM '%q' %s %s %s ORDER BY %s%s", + zCollist, + pIter->zDataTbl, + zPart, + (zStart ? (zPart ? "AND" : "WHERE") : ""), zStart, + zCollist, zLimit + ); + sqlite3_free(zStart); + }else + + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s ORDER BY %s%s", + zCollist, p->zStateDb, pIter->zDataTbl, + zPart, zCollist, zLimit + ); + }else{ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s " + "UNION ALL " + "SELECT %s, rbu_control FROM '%q' " + "%s %s typeof(rbu_control)='integer' AND rbu_control!=1 " + "ORDER BY %s%s", + zCollist, p->zStateDb, pIter->zDataTbl, zPart, + zCollist, pIter->zDataTbl, + zPart, + (zPart ? "AND" : "WHERE"), + zCollist, zLimit + ); + } + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu,&pIter->pSelect,pz,zSql); + }else{ + sqlite3_free(zSql); + } + } + + sqlite3_free(zImposterCols); + sqlite3_free(zImposterPK); + sqlite3_free(zWhere); + sqlite3_free(zBind); + sqlite3_free(zPart); + }else{ + int bRbuRowid = (pIter->eType==RBU_PK_VTAB) + ||(pIter->eType==RBU_PK_NONE) + ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); + const char *zTbl = pIter->zTbl; /* Table this step applies to */ + const char *zWrite; /* Imposter table name */ + + char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid); + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old"); + char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new"); + + zCollist = rbuObjIterGetCollist(p, pIter); + pIter->nCol = pIter->nTblCol; + + /* Create the imposter table or tables (if required). */ + rbuCreateImposterTable(p, pIter); + rbuCreateImposterTable2(p, pIter); + zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_"); + + /* Create the INSERT statement to write to the target PK b-tree */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz, + sqlite3_mprintf( + "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", + zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings + ) + ); + } + + /* Create the DELETE statement to write to the target PK b-tree. + ** Because it only performs INSERT operations, this is not required for + ** an rbu vacuum handle. */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, + sqlite3_mprintf( + "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere + ) + ); + } + + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ + const char *zRbuRowid = ""; + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zRbuRowid = ", rbu_rowid"; + } + + /* Create the rbu_tmp_xxx table and the triggers to populate it. */ + rbuMPrintfExec(p, p->dbRbu, + "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " + "SELECT *%s FROM '%q' WHERE 0;" + , p->zStateDb, pIter->zDataTbl + , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") + , pIter->zDataTbl + ); + + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(3, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(3, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(4, %s);" + "END;", + zWrite, zTbl, zOldlist, + zWrite, zTbl, zOldlist, + zWrite, zTbl, zNewlist + ); + + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(0, %s);" + "END;", + zWrite, zTbl, zNewlist + ); + } + + rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid); + } + + /* Create the SELECT statement to read keys from data_xxx */ + if( p->rc==SQLITE_OK ){ + const char *zRbuRowid = ""; + char *zStart = 0; + char *zOrder = 0; + if( bRbuRowid ){ + zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; + } + + if( rbuIsVacuum(p) ){ + if( nOffset ){ + zStart = rbuVacuumTableStart(p, pIter, bRbuRowid, zWrite); + if( zStart ){ + sqlite3_free(zLimit); + zLimit = 0; + } + } + if( bRbuRowid ){ + zOrder = rbuMPrintf(p, "_rowid_"); + }else{ + zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", ""); + } + } + + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s,%s rbu_control%s FROM '%q'%s %s %s %s", + zCollist, + (rbuIsVacuum(p) ? "0 AS " : ""), + zRbuRowid, + pIter->zDataTbl, (zStart ? zStart : ""), + (zOrder ? "ORDER BY" : ""), zOrder, + zLimit + ) + ); + } + sqlite3_free(zStart); + sqlite3_free(zOrder); + } + + sqlite3_free(zWhere); + sqlite3_free(zOldlist); + sqlite3_free(zNewlist); + sqlite3_free(zBindings); + } + sqlite3_free(zCollist); + sqlite3_free(zLimit); + } + + return p->rc; +} + +/* +** Set output variable *ppStmt to point to an UPDATE statement that may +** be used to update the imposter table for the main table b-tree of the +** table object that pIter currently points to, assuming that the +** rbu_control column of the data_xyz table contains zMask. +** +** If the zMask string does not specify any columns to update, then this +** is not an error. Output variable *ppStmt is set to NULL in this case. +*/ +static int rbuGetUpdateStmt( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter, /* Object iterator */ + const char *zMask, /* rbu_control value ('x.x.') */ + sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */ +){ + RbuUpdateStmt **pp; + RbuUpdateStmt *pUp = 0; + int nUp = 0; + + /* In case an error occurs */ + *ppStmt = 0; + + /* Search for an existing statement. If one is found, shift it to the front + ** of the LRU queue and return immediately. Otherwise, leave nUp pointing + ** to the number of statements currently in the cache and pUp to the + ** last object in the list. */ + for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){ + pUp = *pp; + if( strcmp(pUp->zMask, zMask)==0 ){ + *pp = pUp->pNext; + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; + *ppStmt = pUp->pUpdate; + return SQLITE_OK; + } + nUp++; + } + assert( pUp==0 || pUp->pNext==0 ); + + if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){ + for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext)); + *pp = 0; + sqlite3_finalize(pUp->pUpdate); + pUp->pUpdate = 0; + }else{ + pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1); + } + + if( pUp ){ + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zSet = rbuObjIterGetSetlist(p, pIter, zMask); + char *zUpdate = 0; + + pUp->zMask = (char*)&pUp[1]; + memcpy(pUp->zMask, zMask, pIter->nTblCol); + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; + + if( zSet ){ + const char *zPrefix = ""; + + if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_"; + zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s", + zPrefix, pIter->zTbl, zSet, zWhere + ); + p->rc = prepareFreeAndCollectError( + p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate + ); + *ppStmt = pUp->pUpdate; + } + sqlite3_free(zWhere); + sqlite3_free(zSet); + } + + return p->rc; +} + +static sqlite3 *rbuOpenDbhandle( + sqlite3rbu *p, + const char *zName, + int bUseVfs +){ + sqlite3 *db = 0; + if( p->rc==SQLITE_OK ){ + const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; + p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0); + if( p->rc ){ + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + sqlite3_close(db); + db = 0; + } + } + return db; +} + +/* +** Free an RbuState object allocated by rbuLoadState(). +*/ +static void rbuFreeState(RbuState *p){ + if( p ){ + sqlite3_free(p->zTbl); + sqlite3_free(p->zDataTbl); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} + +/* +** Allocate an RbuState object and load the contents of the rbu_state +** table into it. Return a pointer to the new object. It is the +** responsibility of the caller to eventually free the object using +** sqlite3_free(). +** +** If an error occurs, leave an error code and message in the rbu handle +** and return NULL. +*/ +static RbuState *rbuLoadState(sqlite3rbu *p){ + RbuState *pRet = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int rc2; + + pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); + if( pRet==0 ) return 0; + + rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + switch( sqlite3_column_int(pStmt, 0) ){ + case RBU_STATE_STAGE: + pRet->eStage = sqlite3_column_int(pStmt, 1); + if( pRet->eStage!=RBU_STAGE_OAL + && pRet->eStage!=RBU_STAGE_MOVE + && pRet->eStage!=RBU_STAGE_CKPT + ){ + p->rc = SQLITE_CORRUPT; + } + break; + + case RBU_STATE_TBL: + pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_IDX: + pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_ROW: + pRet->nRow = sqlite3_column_int(pStmt, 1); + break; + + case RBU_STATE_PROGRESS: + pRet->nProgress = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_CKPT: + pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_COOKIE: + pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_OALSZ: + pRet->iOalSz = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_DATATBL: + pRet->zDataTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + default: + rc = SQLITE_CORRUPT; + break; + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + + p->rc = rc; + return pRet; +} + + +/* +** Open the database handle and attach the RBU database as "rbu". If an +** error occurs, leave an error code and message in the RBU handle. +*/ +static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){ + assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); + assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); + + /* Open the RBU database */ + p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( p->zState==0 ){ + const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); + p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile); + } + } + + /* If using separate RBU and state databases, attach the state database to + ** the RBU db handle now. */ + if( p->zState ){ + rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState); + memcpy(p->zStateDb, "stat", 4); + }else{ + memcpy(p->zStateDb, "main", 4); + } + +#if 0 + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0); + } +#endif + + /* If it has not already been created, create the rbu_state table */ + rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); + +#if 0 + if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK ){ + int rc2; + int bOk = 0; + sqlite3_stmt *pCnt = 0; + p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, + "SELECT count(*) FROM stat.sqlite_schema" + ); + if( p->rc==SQLITE_OK + && sqlite3_step(pCnt)==SQLITE_ROW + && 1==sqlite3_column_int(pCnt, 0) + ){ + bOk = 1; + } + rc2 = sqlite3_finalize(pCnt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + if( p->rc==SQLITE_OK && bOk==0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid state database"); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + } + } + } +#endif + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + int bOpen = 0; + int rc; + p->nRbu = 0; + p->pRbuFd = 0; + rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( rc!=SQLITE_NOTFOUND ) p->rc = rc; + if( p->eStage>=RBU_STAGE_MOVE ){ + bOpen = 1; + }else{ + RbuState *pState = rbuLoadState(p); + if( pState ){ + bOpen = (pState->eStage>=RBU_STAGE_MOVE); + rbuFreeState(pState); + } + } + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); + } + + p->eStage = 0; + if( p->rc==SQLITE_OK && p->dbMain==0 ){ + if( !rbuIsVacuum(p) ){ + p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); + }else if( p->pRbuFd->pWalFd ){ + if( pbRetry ){ + p->pRbuFd->bNolock = 0; + sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); + p->dbMain = 0; + p->dbRbu = 0; + *pbRetry = 1; + return; + } + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); + }else{ + char *zTarget; + char *zExtra = 0; + if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){ + zExtra = &p->zRbu[5]; + while( *zExtra ){ + if( *zExtra++=='?' ) break; + } + if( *zExtra=='\0' ) zExtra = 0; + } + + zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s", + sqlite3_db_filename(p->dbRbu, "main"), + (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) + ); + + if( zTarget==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); + sqlite3_free(zTarget); + } + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + } + rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_schema"); + + /* Mark the database file just opened as an RBU target database. If + ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use. + ** This is an error. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + } + + if( p->rc==SQLITE_NOTFOUND ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("rbu vfs not found"); + } +} + +/* +** This routine is a copy of the sqlite3FileSuffix3() routine from the core. +** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. +** +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than +** three characters, then shorten the suffix on z[] to be the last three +** characters of the original suffix. +** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. +** +** Examples: +** +** test.db-journal => test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa +*/ +static void rbuFileSuffix3(const char *zBase, char *z){ +#ifdef SQLITE_ENABLE_8_3_NAMES +#if SQLITE_ENABLE_8_3_NAMES<2 + if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) +#endif + { + int i, sz; + sz = (int)strlen(z)&0xffffff; + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4); + } +#endif +} + +/* +** Return the current wal-index header checksum for the target database +** as a 64-bit integer. +** +** The checksum is store in the first page of xShmMap memory as an 8-byte +** blob starting at byte offset 40. +*/ +static i64 rbuShmChecksum(sqlite3rbu *p){ + i64 iRet = 0; + if( p->rc==SQLITE_OK ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + u32 volatile *ptr; + p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); + if( p->rc==SQLITE_OK ){ + iRet = ((i64)ptr[10] << 32) + ptr[11]; + } + } + return iRet; +} + +/* +** This function is called as part of initializing or reinitializing an +** incremental checkpoint. +** +** It populates the sqlite3rbu.aFrame[] array with the set of +** (wal frame -> db page) copy operations required to checkpoint the +** current wal file, and obtains the set of shm locks required to safely +** perform the copy operations directly on the file-system. +** +** If argument pState is not NULL, then the incremental checkpoint is +** being resumed. In this case, if the checksum of the wal-index-header +** following recovery is not the same as the checksum saved in the RbuState +** object, then the rbu handle is set to DONE state. This occurs if some +** other client appends a transaction to the wal file in the middle of +** an incremental checkpoint. +*/ +static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ + + /* If pState is NULL, then the wal file may not have been opened and + ** recovered. Running a read-statement here to ensure that doing so + ** does not interfere with the "capture" process below. */ + if( pState==0 ){ + p->eStage = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_schema", 0, 0, 0); + } + } + + /* Assuming no error has occurred, run a "restart" checkpoint with the + ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following + ** special behaviour in the rbu VFS: + ** + ** * If the exclusive shm WRITER or READ0 lock cannot be obtained, + ** the checkpoint fails with SQLITE_BUSY (normally SQLite would + ** proceed with running a passive checkpoint instead of failing). + ** + ** * Attempts to read from the *-wal file or write to the database file + ** do not perform any IO. Instead, the frame/page combinations that + ** would be read/written are recorded in the sqlite3rbu.aFrame[] + ** array. + ** + ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, + ** READ0 and CHECKPOINT locks taken as part of the checkpoint are + ** no-ops. These locks will not be released until the connection + ** is closed. + ** + ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL + ** error. + ** + ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the + ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] + ** array populated with a set of (frame -> page) mappings. Because the + ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy + ** data from the wal file into the database file according to the + ** contents of aFrame[]. + */ + if( p->rc==SQLITE_OK ){ + int rc2; + p->eStage = RBU_STAGE_CAPTURE; + rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); + if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; + } + + if( p->rc==SQLITE_OK && p->nFrame>0 ){ + p->eStage = RBU_STAGE_CKPT; + p->nStep = (pState ? pState->nRow : 0); + p->aBuf = rbuMalloc(p, p->pgsz); + p->iWalCksum = rbuShmChecksum(p); + } + + if( p->rc==SQLITE_OK ){ + if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){ + p->rc = SQLITE_DONE; + p->eStage = RBU_STAGE_DONE; + }else{ + int nSectorSize; + sqlite3_file *pDb = p->pTargetFd->pReal; + sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal; + assert( p->nPagePerSector==0 ); + nSectorSize = pDb->pMethods->xSectorSize(pDb); + if( nSectorSize>p->pgsz ){ + p->nPagePerSector = nSectorSize / p->pgsz; + }else{ + p->nPagePerSector = 1; + } + + /* Call xSync() on the wal file. This causes SQLite to sync the + ** directory in which the target database and the wal file reside, in + ** case it has not been synced since the rename() call in + ** rbuMoveOalFile(). */ + p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL); + } + } +} + +/* +** Called when iAmt bytes are read from offset iOff of the wal file while +** the rbu object is in capture mode. Record the frame number of the frame +** being read in the aFrame[] array. +*/ +static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ + const u32 mReq = (1<mLock!=mReq ){ + pRbu->rc = SQLITE_BUSY; + return SQLITE_INTERNAL; + } + + pRbu->pgsz = iAmt; + if( pRbu->nFrame==pRbu->nFrameAlloc ){ + int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; + RbuFrame *aNew; + aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame)); + if( aNew==0 ) return SQLITE_NOMEM; + pRbu->aFrame = aNew; + pRbu->nFrameAlloc = nNew; + } + + iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1; + if( pRbu->iMaxFrame